<commit_before>package flickr

import (
	"io"
	"os"
	"fmt"
	"net"
	"http"
	"sort"
	"bufio"
	"bytes"
	"io/ioutil"
	"crypto/md5"
)

const (
	endpoint        = "http://api.flickr.com/services/rest/?"
	uploadEndpoint  = "http://api.flickr.com/services/upload/"
	replaceEndpoint = "http://api.flickr.com/services/replace/"
	apiHost         = "api.flickr.com"
)

type Request struct {
	ApiKey string
	Method string
	Args   map[string]string
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() os.Error { return nil }

type Error string

func (e Error) String() string {
	return string(e)
}

func (request *Request) Sign(secret string) {
	args := request.Args

	sorted_keys := make([]string, len(args)+2)

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	// Sort array keys
	i := 0
	for k, _ := range args {
		sorted_keys[i] = k
		i++
	}
	sort.SortStrings(sorted_keys)

	// Build out ordered key-value string prefixed by secret
	s := secret
	for _, key := range sorted_keys {
		if args[key] != "" {
			s += fmt.Sprintf("%s%s", key, args[key])
		}
	}

	// Since we're only adding two keys, it's easier
	// and more space-efficient to just delete them
	// them copy the whole map
	args["api_key"] = "", false
	args["method"] = "", false

	// Have the full string, now hash
	hash := md5.New()
	hash.Write([]byte(s))

	// Add api_sig as one of the args
	args["api_sig"] = fmt.Sprintf("%x", hash.Sum())
}

func (request *Request) Execute() (response string, ret os.Error) {
	args := request.Args

	if request.ApiKey == "" || request.Method == "" {
		return "", Error("Need both API key and method")
	}

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	s := endpoint + encodeQuery(args)

	res, _, err := http.Get(s)
	defer res.Body.Close()
	if err != nil {
		return "", err
	}

	body, _ := ioutil.ReadAll(res.Body)
	return string(body), nil
}

func encodeQuery(args map[string]string) string {
	i := 0
	s := bytes.NewBuffer(nil)
	for k, v := range args {
		if i != 0 {
			s.WriteString("&")
		}
		i++
		s.WriteString(k + http.URLEscape(v))
	}
	return s.String()
}

func (request *Request) buildPost(url string, filename string, filetype string) (*http.Request, os.Error) {
	f, err := os.Open(filename, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}

	request.Args["api_key"] = request.ApiKey

	boundary := "----###---###--flickr-go-rules"
	end := "\r\n"

	body := bytes.NewBuffer(nil)
	for k, v := range request.Args {
		body.WriteString("--" + boundary + end)
		body.WriteString("Content-Disposition: form-data; name=\"" + k + "\"" + end + end)
		body.WriteString(v + end)
	}

	body.WriteString("--" + boundary + end)
	body.WriteString("Content-Disposition: form-data; name=\"photo\"; filename=\"photo.jpg\"" + end)
	body.WriteString("Content-Type: " + filetype + end + end)

	// Write file
	_, err = io.Copy(body, f)
	if err != nil {
		return nil, err
	}

	body.WriteString(end)
	body.WriteString("--" + boundary + "--" + end)

	postRequest := new(http.Request)
	postRequest.Method = "POST"
	postRequest.RawURL = url
	postRequest.Host = apiHost
	postRequest.Header = map[string]string{
		"Content-Type": "multipart/form-data; boundary=" + boundary + end,
	}

	postRequest.Body = nopCloser{body}
	postRequest.ContentLength = int64(body.Len())
	return postRequest, nil
}

func (request *Request) Upload(filename string, filetype string) (response string, err os.Error) {
	postRequest, err := request.buildPost(uploadEndpoint, filename, filetype)
	if err != nil {
		return "", err
	}

	return sendPost(postRequest)
}

func (request *Request) Replace(filename string, filetype string) (response string, err os.Error) {
	postRequest, err := request.buildPost(replaceEndpoint, filename, filetype)
	if err != nil {
		return "", err
	}
	return sendPost(postRequest)
}

func sendPost(postRequest *http.Request) (body string, err os.Error) {
	// Create and use TCP connection (lifted mostly wholesale from http.send)
	conn, err := net.Dial("tcp", "", "api.flickr.com:80")
	defer conn.Close()

	if err != nil {
		return "", err
	}
	postRequest.Write(conn)

	reader := bufio.NewReader(conn)
	resp, err := http.ReadResponse(reader, postRequest.Method)
	if err != nil {
		return "", err
	}
	rawBody, _ := ioutil.ReadAll(resp.Body)

	return string(rawBody), nil
}
<commit_msg>Helpful to close files you open<commit_after>package flickr

import (
	"io"
	"os"
	"fmt"
	"net"
	"http"
	"sort"
	"bufio"
	"bytes"
	"io/ioutil"
	"crypto/md5"
)

const (
	endpoint        = "http://api.flickr.com/services/rest/?"
	uploadEndpoint  = "http://api.flickr.com/services/upload/"
	replaceEndpoint = "http://api.flickr.com/services/replace/"
	apiHost         = "api.flickr.com"
)

type Request struct {
	ApiKey string
	Method string
	Args   map[string]string
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() os.Error { return nil }

type Error string

func (e Error) String() string {
	return string(e)
}

func (request *Request) Sign(secret string) {
	args := request.Args

	sorted_keys := make([]string, len(args)+2)

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	// Sort array keys
	i := 0
	for k, _ := range args {
		sorted_keys[i] = k
		i++
	}
	sort.SortStrings(sorted_keys)

	// Build out ordered key-value string prefixed by secret
	s := secret
	for _, key := range sorted_keys {
		if args[key] != "" {
			s += fmt.Sprintf("%s%s", key, args[key])
		}
	}

	// Since we're only adding two keys, it's easier
	// and more space-efficient to just delete them
	// them copy the whole map
	args["api_key"] = "", false
	args["method"] = "", false

	// Have the full string, now hash
	hash := md5.New()
	hash.Write([]byte(s))

	// Add api_sig as one of the args
	args["api_sig"] = fmt.Sprintf("%x", hash.Sum())
}

func (request *Request) Execute() (response string, ret os.Error) {
	args := request.Args

	if request.ApiKey == "" || request.Method == "" {
		return "", Error("Need both API key and method")
	}

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	s := endpoint + encodeQuery(args)

	res, _, err := http.Get(s)
	defer res.Body.Close()
	if err != nil {
		return "", err
	}

	body, _ := ioutil.ReadAll(res.Body)
	return string(body), nil
}

func encodeQuery(args map[string]string) string {
	i := 0
	s := bytes.NewBuffer(nil)
	for k, v := range args {
		if i != 0 {
			s.WriteString("&")
		}
		i++
		s.WriteString(k + http.URLEscape(v))
	}
	return s.String()
}

func (request *Request) buildPost(url string, filename string, filetype string) (*http.Request, os.Error) {
	f, err := os.Open(filename, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	request.Args["api_key"] = request.ApiKey

	boundary := "----###---###--flickr-go-rules"
	end := "\r\n"

	body := bytes.NewBuffer(nil)
	for k, v := range request.Args {
		body.WriteString("--" + boundary + end)
		body.WriteString("Content-Disposition: form-data; name=\"" + k + "\"" + end + end)
		body.WriteString(v + end)
	}

	body.WriteString("--" + boundary + end)
	body.WriteString("Content-Disposition: form-data; name=\"photo\"; filename=\"photo.jpg\"" + end)
	body.WriteString("Content-Type: " + filetype + end + end)

	// Write file
	_, err = io.Copy(body, f)
	if err != nil {
		return nil, err
	}

	body.WriteString(end)
	body.WriteString("--" + boundary + "--" + end)

	postRequest := new(http.Request)
	postRequest.Method = "POST"
	postRequest.RawURL = url
	postRequest.Host = apiHost
	postRequest.Header = map[string]string{
		"Content-Type": "multipart/form-data; boundary=" + boundary + end,
	}

	postRequest.Body = nopCloser{body}
	postRequest.ContentLength = int64(body.Len())
	return postRequest, nil
}

func (request *Request) Upload(filename string, filetype string) (response string, err os.Error) {
	postRequest, err := request.buildPost(uploadEndpoint, filename, filetype)
	if err != nil {
		return "", err
	}

	return sendPost(postRequest)
}

func (request *Request) Replace(filename string, filetype string) (response string, err os.Error) {
	postRequest, err := request.buildPost(replaceEndpoint, filename, filetype)
	if err != nil {
		return "", err
	}
	return sendPost(postRequest)
}

func sendPost(postRequest *http.Request) (body string, err os.Error) {
	// Create and use TCP connection (lifted mostly wholesale from http.send)
	conn, err := net.Dial("tcp", "", "api.flickr.com:80")
	defer conn.Close()

	if err != nil {
		return "", err
	}
	postRequest.Write(conn)

	reader := bufio.NewReader(conn)
	resp, err := http.ReadResponse(reader, postRequest.Method)
	if err != nil {
		return "", err
	}
	rawBody, _ := ioutil.ReadAll(resp.Body)

	return string(rawBody), nil
}
<|endoftext|>


<commit_before>// Copyright 2013 The Gonum Authors. All rights reserved.
// Use of this code is governed by a BSD-style
// license that can be found in the LICENSE file

package floats

import (
	"errors"
	"math"
)

// Add returns the element-wise sum of all the slices with the
// results stored in the first slice.
// For computational efficiency, it is assumed that all of
// the variadic arguments have the same length. If this is
// in doubt, EqLen can be used.
func Add(dst []float64, slices ...[]float64) []float64 {
	if len(slices) == 0 {
		return nil
	}
	if len(dst) != len(slices[0]) {
		panic("floats: length of destination does not match length of the slices")
	}
	for _, slice := range slices {
		for j, val := range slice {
			dst[j] += val
		}
	}
	return dst
}

// AddConst adds the value c to all of the values in s
func AddConst(c float64, s []float64) {
	for i := range s {
		s[i] += c
	}
}

// ApplyFunc applies a function f (math.Exp, math.Sin, etc.) to every element
// of the slice s
func Apply(f func(float64) float64, s []float64) {
	for i, val := range s {
		s[i] = f(val)
	}
}

// Count applies the function f to every element of s and returns the number
// of times the function returned true
func Count(f func(float64) bool, s []float64) int {
	var n int
	for _, val := range s {
		if f(val) {
			n++
		}
	}
	return n
}

// Cumprod finds the cumulative product of the first i elements in
// s and puts them in place into the ith element of the
// destination. Panic will occur if lengths of do not match
func CumProd(dst, s []float64) []float64 {
	if len(dst) != len(s) {
		panic("floats: length of destination does not match length of the source")
	}
	dst[0] = s[0]
	for i := 1; i < len(s); i++ {
		dst[i] = dst[i-1] * s[i]
	}
	return dst
}

// Cumsum finds the cumulative sum of the first i elements in
// s and puts them in place into the ith element of the
// destination. Panic will occur if lengths of arguments do not match
func CumSum(dst, s []float64) []float64 {
	if len(dst) != len(s) {
		panic("floats: length of destination does not match length of the source")
	}
	dst[0] = s[0]
	for i := 1; i < len(s); i++ {
		dst[i] = dst[i-1] + s[i]
	}
	return dst
}

// Dot computes the dot product of s1 and s2, i.e.
// sum_{i = 1}^N s1[n]*s2[n]
// Panic will occur if lengths of arguments do not match
func Dot(s1, s2 []float64) float64 {
	if len(s1) != len(s2) {
		panic("floats: lengths of the slices do not match")
	}
	var sum float64
	for i, val := range s1 {
		sum += val * s2[i]
	}
	return sum
}

// Eq returns false if the slices have different lengths
// or if |s1[i] - s2[i]| > tol for any i.
func Eq(s1, s2 []float64, tol float64) bool {
	if len(s1) != len(s2) {
		return false
	}
	for i, val := range s1 {
		if math.Abs(s2[i]-val) > tol {
			return false
		}
	}
	return true
}

// Eqlen returns true if all of the slices have equal length,
// and false otherwise. Returns true if there are no input slices
func EqLen(slices ...[]float64) bool {
	// This length check is needed: http://play.golang.org/p/sdty6YiLhM
	if len(slices) == 0 {
		return true
	}
	l := len(slices[0])
	for i := 1; i < len(slices); i++ {
		if len(slices[i]) != l {
			return false
		}
	}
	return true
}

// Find applies f to every element of s and returns the first
// k elements for which the f returns true, or all such elements
// if k < 0.
// Find will reslice inds to have 0 length, and will append
// found indices to inds.
// If k > 0 and there are fewer than k elements in s satisfying f,
// all of the found elements will be returned along with an error
func Find(inds []int, f func(float64) bool, s []float64, k int) ([]int, error) {

	// inds is also returned to allow for calling with nil

	// Reslice inds to have zero length
	inds = inds[:0]

	// If zero elements requested, can just return
	if k == 0 {
		return inds, nil
	}

	// If k < 0, return all of the found indices
	if k < 0 {
		for i, val := range s {
			if f(val) {
				inds = append(inds, i)
			}
		}
		return inds, nil
	}

	// Otherwise, find the first k elements
	nFound := 0
	for i, val := range s {
		if f(val) {
			inds = append(inds, i)
			nFound++
			if nFound == k {
				return inds, nil
			}
		}
	}
	// Finished iterating over the loop, which means k elements were not found
	return inds, errors.New("floats: insufficient elements found")
}

// LogSpan returns a set of n equally spaced points in log space between,
// l and u where N is equal to len(dst). The first element of the
// resulting dst will be l and the final element of dst will be u.
// Panics if len(dst) < 2
// Note that this call will return NaNs if either l or u are negative, and
// will return all zeros if l or u is zero.
func LogSpan(dst []float64, l, u float64) []float64 {
	Span(dst, math.Log(l), math.Log(u))
	Apply(math.Exp, dst)
	return dst
}

// Logsumexp returns the log of the sum of the exponentials of the values in s
func LogSumExp(s []float64) (lse float64) {
	// Want to do this in a numerically stable way which avoids
	// overflow and underflow
	// First, find the maximum value in the slice.
	maxval, _ := Max(s)
	if math.IsInf(maxval, 0) {
		// If it's infinity either way, the logsumexp will be infinity as well
		// returning now avoids NaNs
		return maxval
	}
	// Compute the sumexp part
	for _, val := range s {
		lse += math.Exp(val - maxval)
	}
	// Take the log and add back on the constant taken out
	lse = math.Log(lse) + maxval
	return lse
}

// Max returns the maximum value in the slice and the location of
// the maximum value. If the input slice is empty, the code will panic
func Max(s []float64) (max float64, ind int) {
	max = s[0]
	ind = 0
	for i, val := range s {
		if val > max {
			max = val
			ind = i
		}
	}
	return max, ind
}

// Min returns the minimum value in the slice and the index of
// the minimum value. If the input slice is empty, zero is returned
// as the minimum value and -1 is returned as the index.
func Min(s []float64) (min float64, ind int) {
	min = s[0]
	ind = 0
	for i, val := range s {
		if val < min {
			min = val
			ind = i
		}
	}
	return min, ind
}

// Nearest returns the index of the element in s
// whose value is nearest to v.  If several such
// elements exist, the lowest index is returned
func Nearest(s []float64, v float64) (ind int) {
	dist := math.Abs(v - s[0])
	ind = 0
	for i, val := range s {
		newDist := math.Abs(v - val)
		if newDist < dist {
			dist = newDist
			ind = i
		}
	}
	return
}

// NearestInSpan return the index of a hypothetical vector created
// by Span with length n and bounds l and u whose value is closest
// to v. Assumes u > l. If the value is greater than u or less than
// l, the function will panic.
func NearestWithinSpan(n int, l, u float64, v float64) int {
	if v < l || v > u {
		panic("floats: value outside span bounds")
	}

	// Can't guarantee anything about exactly halfway between
	// because of floating point weirdness
	return int((float64(n)-1)/(u-l)*(v-l) + 0.5)
}

// Norm returns the L norm of the slice S, defined as
// (sum_{i=1}^N s[i]^N)^{1/N}
// Special cases:
// L = math.Inf(1) gives the maximum value
// Does not correctly compute the zero norm (use Count)
func Norm(s []float64, L float64) (norm float64) {
	// Should this complain if L is not positive?
	// Should this be done in log space for better numerical stability?
	//	would be more cost
	//	maybe only if L is high?
	if L == 2 {
		for _, val := range s {
			norm += val * val
		}
		return math.Pow(norm, 0.5)
	}
	if L == 1 {
		for _, val := range s {
			norm += math.Abs(val)
		}
		return norm
	}
	if math.IsInf(L, 1) {
		norm, _ = Max(s)
		return norm
	}
	for _, val := range s {
		norm += math.Pow(math.Abs(val), L)
	}
	return math.Pow(norm, 1/L)
}

// Prod returns the product of the elements of the slice
// Returns 1 if len(s) = 0
func Prod(s []float64) (prod float64) {
	prod = 1
	for _, val := range s {
		prod *= val
	}
	return prod
}

// Scale multiplies every element in s by c
func Scale(c float64, s []float64) {
	for i := range s {
		s[i] *= c
	}
}

// Span returns a set of N equally spaced points between l and u, where N
// is equal to the length of the destination. The first element of the destination
// is l, the final element of the destination is u.
// Panics if len(dst) < 2
func Span(dst []float64, l, u float64) []float64 {
	n := len(dst)
	if n < 2 {
		panic("floats: destination must have length >1")
	}
	step := (u - l) / float64(n-1)
	for i := range dst {
		dst[i] = l + step*float64(i)
	}
	return dst
}

// Sub subtracts, element-wise, the first argument from the second. Assumes
// the lengths of s and t match (can be tested with EqLen)
func Sub(s, t []float64) {
	if len(s) != len(t) {
		panic("floats: length of the slices do not match")
	}
	for i, val := range t {
		s[i] -= val
	}
}

// SubTo subtracts, element-wise, the first argument from the second and
// stores the result in dest. Panics if the lengths of s and t do not match
func SubTo(dst, s, t []float64) []float64 {
	if len(s) != len(t) {
		panic("floats: length of subtractor and subtractee do not match")
	}
	if len(dst) != len(s) {
		if dst == nil {
			dst = make([]float64, len(s))
		} else {
			panic("floats: length of destination does not match length of subtractor")
		}
	}
	for i, val := range t {
		dst[i] = s[i] - val
	}
	return dst
}

// Sum returns the sum of the elements of the slice
func Sum(s []float64) (sum float64) {
	for _, val := range s {
		sum += val
	}
	return
}
<commit_msg>Fixed error in dot comment<commit_after>// Copyright 2013 The Gonum Authors. All rights reserved.
// Use of this code is governed by a BSD-style
// license that can be found in the LICENSE file

package floats

import (
	"errors"
	"math"
)

// Add returns the element-wise sum of all the slices with the
// results stored in the first slice.
// For computational efficiency, it is assumed that all of
// the variadic arguments have the same length. If this is
// in doubt, EqLen can be used.
func Add(dst []float64, slices ...[]float64) []float64 {
	if len(slices) == 0 {
		return nil
	}
	if len(dst) != len(slices[0]) {
		panic("floats: length of destination does not match length of the slices")
	}
	for _, slice := range slices {
		for j, val := range slice {
			dst[j] += val
		}
	}
	return dst
}

// AddConst adds the value c to all of the values in s
func AddConst(c float64, s []float64) {
	for i := range s {
		s[i] += c
	}
}

// ApplyFunc applies a function f (math.Exp, math.Sin, etc.) to every element
// of the slice s
func Apply(f func(float64) float64, s []float64) {
	for i, val := range s {
		s[i] = f(val)
	}
}

// Count applies the function f to every element of s and returns the number
// of times the function returned true
func Count(f func(float64) bool, s []float64) int {
	var n int
	for _, val := range s {
		if f(val) {
			n++
		}
	}
	return n
}

// Cumprod finds the cumulative product of the first i elements in
// s and puts them in place into the ith element of the
// destination. Panic will occur if lengths of do not match
func CumProd(dst, s []float64) []float64 {
	if len(dst) != len(s) {
		panic("floats: length of destination does not match length of the source")
	}
	dst[0] = s[0]
	for i := 1; i < len(s); i++ {
		dst[i] = dst[i-1] * s[i]
	}
	return dst
}

// Cumsum finds the cumulative sum of the first i elements in
// s and puts them in place into the ith element of the
// destination. Panic will occur if lengths of arguments do not match
func CumSum(dst, s []float64) []float64 {
	if len(dst) != len(s) {
		panic("floats: length of destination does not match length of the source")
	}
	dst[0] = s[0]
	for i := 1; i < len(s); i++ {
		dst[i] = dst[i-1] + s[i]
	}
	return dst
}

// Dot computes the dot product of s1 and s2, i.e.
// sum_{i = 1}^N s1[i]*s2[i]
// Panic will occur if lengths of arguments do not match
func Dot(s1, s2 []float64) float64 {
	if len(s1) != len(s2) {
		panic("floats: lengths of the slices do not match")
	}
	var sum float64
	for i, val := range s1 {
		sum += val * s2[i]
	}
	return sum
}

// Eq returns false if the slices have different lengths
// or if |s1[i] - s2[i]| > tol for any i.
func Eq(s1, s2 []float64, tol float64) bool {
	if len(s1) != len(s2) {
		return false
	}
	for i, val := range s1 {
		if math.Abs(s2[i]-val) > tol {
			return false
		}
	}
	return true
}

// Eqlen returns true if all of the slices have equal length,
// and false otherwise. Returns true if there are no input slices
func EqLen(slices ...[]float64) bool {
	// This length check is needed: http://play.golang.org/p/sdty6YiLhM
	if len(slices) == 0 {
		return true
	}
	l := len(slices[0])
	for i := 1; i < len(slices); i++ {
		if len(slices[i]) != l {
			return false
		}
	}
	return true
}

// Find applies f to every element of s and returns the first
// k elements for which the f returns true, or all such elements
// if k < 0.
// Find will reslice inds to have 0 length, and will append
// found indices to inds.
// If k > 0 and there are fewer than k elements in s satisfying f,
// all of the found elements will be returned along with an error
func Find(inds []int, f func(float64) bool, s []float64, k int) ([]int, error) {

	// inds is also returned to allow for calling with nil

	// Reslice inds to have zero length
	inds = inds[:0]

	// If zero elements requested, can just return
	if k == 0 {
		return inds, nil
	}

	// If k < 0, return all of the found indices
	if k < 0 {
		for i, val := range s {
			if f(val) {
				inds = append(inds, i)
			}
		}
		return inds, nil
	}

	// Otherwise, find the first k elements
	nFound := 0
	for i, val := range s {
		if f(val) {
			inds = append(inds, i)
			nFound++
			if nFound == k {
				return inds, nil
			}
		}
	}
	// Finished iterating over the loop, which means k elements were not found
	return inds, errors.New("floats: insufficient elements found")
}

// LogSpan returns a set of n equally spaced points in log space between,
// l and u where N is equal to len(dst). The first element of the
// resulting dst will be l and the final element of dst will be u.
// Panics if len(dst) < 2
// Note that this call will return NaNs if either l or u are negative, and
// will return all zeros if l or u is zero.
func LogSpan(dst []float64, l, u float64) []float64 {
	Span(dst, math.Log(l), math.Log(u))
	Apply(math.Exp, dst)
	return dst
}

// Logsumexp returns the log of the sum of the exponentials of the values in s
func LogSumExp(s []float64) (lse float64) {
	// Want to do this in a numerically stable way which avoids
	// overflow and underflow
	// First, find the maximum value in the slice.
	maxval, _ := Max(s)
	if math.IsInf(maxval, 0) {
		// If it's infinity either way, the logsumexp will be infinity as well
		// returning now avoids NaNs
		return maxval
	}
	// Compute the sumexp part
	for _, val := range s {
		lse += math.Exp(val - maxval)
	}
	// Take the log and add back on the constant taken out
	lse = math.Log(lse) + maxval
	return lse
}

// Max returns the maximum value in the slice and the location of
// the maximum value. If the input slice is empty, the code will panic
func Max(s []float64) (max float64, ind int) {
	max = s[0]
	ind = 0
	for i, val := range s {
		if val > max {
			max = val
			ind = i
		}
	}
	return max, ind
}

// Min returns the minimum value in the slice and the index of
// the minimum value. If the input slice is empty, zero is returned
// as the minimum value and -1 is returned as the index.
func Min(s []float64) (min float64, ind int) {
	min = s[0]
	ind = 0
	for i, val := range s {
		if val < min {
			min = val
			ind = i
		}
	}
	return min, ind
}

// Nearest returns the index of the element in s
// whose value is nearest to v.  If several such
// elements exist, the lowest index is returned
func Nearest(s []float64, v float64) (ind int) {
	dist := math.Abs(v - s[0])
	ind = 0
	for i, val := range s {
		newDist := math.Abs(v - val)
		if newDist < dist {
			dist = newDist
			ind = i
		}
	}
	return
}

// NearestInSpan return the index of a hypothetical vector created
// by Span with length n and bounds l and u whose value is closest
// to v. Assumes u > l. If the value is greater than u or less than
// l, the function will panic.
func NearestWithinSpan(n int, l, u float64, v float64) int {
	if v < l || v > u {
		panic("floats: value outside span bounds")
	}

	// Can't guarantee anything about exactly halfway between
	// because of floating point weirdness
	return int((float64(n)-1)/(u-l)*(v-l) + 0.5)
}

// Norm returns the L norm of the slice S, defined as
// (sum_{i=1}^N s[i]^N)^{1/N}
// Special cases:
// L = math.Inf(1) gives the maximum value
// Does not correctly compute the zero norm (use Count)
func Norm(s []float64, L float64) (norm float64) {
	// Should this complain if L is not positive?
	// Should this be done in log space for better numerical stability?
	//	would be more cost
	//	maybe only if L is high?
	if L == 2 {
		for _, val := range s {
			norm += val * val
		}
		return math.Pow(norm, 0.5)
	}
	if L == 1 {
		for _, val := range s {
			norm += math.Abs(val)
		}
		return norm
	}
	if math.IsInf(L, 1) {
		norm, _ = Max(s)
		return norm
	}
	for _, val := range s {
		norm += math.Pow(math.Abs(val), L)
	}
	return math.Pow(norm, 1/L)
}

// Prod returns the product of the elements of the slice
// Returns 1 if len(s) = 0
func Prod(s []float64) (prod float64) {
	prod = 1
	for _, val := range s {
		prod *= val
	}
	return prod
}

// Scale multiplies every element in s by c
func Scale(c float64, s []float64) {
	for i := range s {
		s[i] *= c
	}
}

// Span returns a set of N equally spaced points between l and u, where N
// is equal to the length of the destination. The first element of the destination
// is l, the final element of the destination is u.
// Panics if len(dst) < 2
func Span(dst []float64, l, u float64) []float64 {
	n := len(dst)
	if n < 2 {
		panic("floats: destination must have length >1")
	}
	step := (u - l) / float64(n-1)
	for i := range dst {
		dst[i] = l + step*float64(i)
	}
	return dst
}

// Sub subtracts, element-wise, the first argument from the second. Assumes
// the lengths of s and t match (can be tested with EqLen)
func Sub(s, t []float64) {
	if len(s) != len(t) {
		panic("floats: length of the slices do not match")
	}
	for i, val := range t {
		s[i] -= val
	}
}

// SubTo subtracts, element-wise, the first argument from the second and
// stores the result in dest. Panics if the lengths of s and t do not match
func SubTo(dst, s, t []float64) []float64 {
	if len(s) != len(t) {
		panic("floats: length of subtractor and subtractee do not match")
	}
	if len(dst) != len(s) {
		if dst == nil {
			dst = make([]float64, len(s))
		} else {
			panic("floats: length of destination does not match length of subtractor")
		}
	}
	for i, val := range t {
		dst[i] = s[i] - val
	}
	return dst
}

// Sum returns the sum of the elements of the slice
func Sum(s []float64) (sum float64) {
	for _, val := range s {
		sum += val
	}
	return
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package handler
<commit_msg>[test-handler] Write mock handler component for testing<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package handler

import (
	//	"os"
	//	"reflect"
	"testing"
	//	"time"
	//
	. "github.com/TheThingsNetwork/ttn/core"
	//	"github.com/TheThingsNetwork/ttn/utils/errors"
	. "github.com/TheThingsNetwork/ttn/utils/errors/checks"
	//	"github.com/TheThingsNetwork/ttn/utils/pointer"
	. "github.com/TheThingsNetwork/ttn/utils/testing"
	"github.com/brocaar/lorawan"
)

func TestRegister(t *testing.T) {
	{
		Desc(t, "Registration valid HRegistration")

		devStorage := newMockDevStorage()
		pktStorage := newMockPktStorage()
		handler := New(devStorage, pktStorage, GetLogger(t, "Handler"))
		an := newMockAckNacker()
		r := newTestRegistration(
			[8]byte{1, 1, 1, 1, 1, 1, 1, 1},
			[8]byte{2, 2, 2, 2, 2, 2, 2, 2},
		)

		err := handler.Register(r, an)
		CheckErrors(t, nil, err)
	}
}

// ----- TYPE utilities

//
// MOCK DEV STORAGE
//
type mockDevStorage struct {
	Failures     map[string]error
	LookupEntry  devEntry
	Personalized HRegistration
	Activated    HRegistration
}

func newMockDevStorage(failures ...string) *mockDevStorage {
	return &mockDevStorage{
		Failures: make(map[string]error),
	}
}

func (s mockDevStorage) Lookup(appEUI lorawan.EUI64, devEUI lorawan.EUI64) (devEntry, error) {
	if s.Failures["Lookup"] != nil {
		return devEntry{}, s.Failures["Lookup"]
	}
	return s.LookupEntry, nil
}

func (s *mockDevStorage) StorePersonalized(r HRegistration) error {
	s.Personalized = r
	return s.Failures["StorePersonalized"]
}

func (s *mockDevStorage) StoreActivated(r HRegistration) error {
	s.Activated = r
	return s.Failures["StoreActivated"]
}

func (s mockDevStorage) Close() error {
	return s.Failures["Close"]
}

//
// MOCK PKT STORAGE
//
type mockPktStorage struct {
	Failures  map[string]error
	PullEntry APacket
	Pushed    APacket
}

func newMockPktStorage() *mockPktStorage {
	return &mockPktStorage{
		Failures: make(map[string]error),
	}
}

func (s *mockPktStorage) Push(p APacket) error {
	s.Pushed = p
	return s.Failures["Push"]
}

func (s *mockPktStorage) Pull(appEUI lorawan.EUI64, devEUI lorawan.EUI64) (APacket, error) {
	if s.Failures["Pull"] != nil {
		return nil, s.Failures["Pull"]
	}
	return s.PullEntry, nil
}

func (s *mockPktStorage) Close() error {
	return s.Failures["Close"]
}

//
// MOCK ACK/NACKER
//
type mockAckNacker struct{}

func newMockAckNacker() *mockAckNacker {
	return &mockAckNacker{}
}

func (an mockAckNacker) Ack(p Packet) error {
	return nil
}

func (an mockAckNacker) Nack() error {
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.

package indexer

import (
	"bytes"
	"errors"
	"github.com/couchbase/cbauth/metakv"
	"github.com/couchbase/indexing/secondary/common"
	"io/ioutil"
	"net/http"
	"runtime"
	"time"
)

const (
	indexerMetaDir          = "/indexer/"
	indexerSettingsMetaPath = indexerMetaDir + "settings"
	indexCompactonMetaPath  = indexerMetaDir + "triggerCompaction"
)

// Implements dynamic settings management for indexer
type settingsManager struct {
	supvCmdch       MsgChannel
	supvMsgch       MsgChannel
	config          common.Config
	cancelCh        chan struct{}
	compactionToken []byte
}

func NewSettingsManager(supvCmdch MsgChannel,
	supvMsgch MsgChannel, config common.Config) (settingsManager, common.Config, Message) {
	s := settingsManager{
		supvCmdch: supvCmdch,
		supvMsgch: supvMsgch,
		config:    config,
		cancelCh:  make(chan struct{}),
	}

	value, _, err := metakv.Get(indexerSettingsMetaPath)
	if err != nil {
		return s, nil, &MsgError{
			err: Error{
				category: INDEXER,
				cause:    err,
				severity: FATAL,
			}}
	}

	if len(value) > 0 {
		config.Update(value)
	}
	http.HandleFunc("/settings", s.handleSettingsReq)
	http.HandleFunc("/triggerCompaction", s.handleCompactionTrigger)
	go func() {
		for {
			err := metakv.RunObserveChildren("/", s.metaKVCallback, s.cancelCh)
			if err == nil {
				return
			} else {
				common.Errorf("IndexerSettingsManager: metakv notifier failed (%v)..Restarting", err)
			}
		}
	}()

	return s, config, &MsgSuccess{}
}

func (s *settingsManager) writeOk(w http.ResponseWriter) {
	w.WriteHeader(200)
	w.Write([]byte("OK\n"))
}

func (s *settingsManager) writeError(w http.ResponseWriter, err error) {
	w.WriteHeader(400)
	w.Write([]byte(err.Error() + "\n"))
}

func (s *settingsManager) writeJson(w http.ResponseWriter, json []byte) {
	header := w.Header()
	header["Content-Type"] = []string{"application/json"}
	w.WriteHeader(200)
	w.Write(json)
	w.Write([]byte("\n"))
}

func (s *settingsManager) handleSettingsReq(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		bytes, _ := ioutil.ReadAll(r.Body)

		config := s.config.Clone()
		current, rev, err := metakv.Get(indexerSettingsMetaPath)
		if err == nil {
			if len(current) > 0 {
				config.Update(current)
			}
			err = config.Update(bytes)
		}

		if err != nil {
			s.writeError(w, err)
			return
		}

		settingsConfig := config.SectionConfig("settings.", false)
		newSettingsBytes := settingsConfig.Json()
		if err = metakv.Set(indexerSettingsMetaPath, newSettingsBytes, rev); err != nil {
			s.writeError(w, err)
			return
		}
		s.writeOk(w)
	} else if r.Method == "GET" {
		settingsConfig, err := getSettingsConfig(s.config)
		if err != nil {
			s.writeError(w, err)
			return
		}
		s.writeJson(w, settingsConfig.Json())
	} else {
		s.writeError(w, errors.New("Unsupported method"))
		return
	}
}

func (s *settingsManager) handleCompactionTrigger(w http.ResponseWriter, r *http.Request) {
	_, rev, err := metakv.Get(indexCompactonMetaPath)
	if err != nil {
		s.writeError(w, err)
		return
	}

	newToken := time.Now().String()
	if err = metakv.Set(indexCompactonMetaPath, []byte(newToken), rev); err != nil {
		s.writeError(w, err)
		return
	}

	s.writeOk(w)
}

func (s *settingsManager) run() {
loop:
	for {
		select {
		case cmd, ok := <-s.supvCmdch:
			if ok {
				if cmd.GetMsgType() == STORAGE_MGR_SHUTDOWN {
					common.Infof("SettingsManager::run Shutting Down")
					close(s.cancelCh)
					s.supvCmdch <- &MsgSuccess{}
					break loop
				}
			} else {
				break loop
			}
		}
	}
}

func (s *settingsManager) metaKVCallback(path string, value []byte, rev interface{}) error {
	if path == indexerSettingsMetaPath {
		common.Infof("New settings received: \n%s", string(value))
		config := s.config.Clone()
		config.Update(value)
		s.config = config
		s.supvMsgch <- &MsgConfigUpdate{
			cfg: s.config,
		}
	} else if path == indexCompactonMetaPath {
		currentToken := s.compactionToken
		s.compactionToken = value
		if currentToken == nil || bytes.Equal(currentToken, value) {
			return nil
		}

		common.Infof("Manual compaction trigger requested")
		replych := make(chan []IndexStorageStats)
		statReq := &MsgIndexStorageStats{respch: replych}
		s.supvMsgch <- statReq
		stats := <-replych
		// XXX: minFile size check can be applied
		go func() {
			for _, is := range stats {
				errch := make(chan error)
				compactReq := &MsgIndexCompact{
					instId: is.InstId,
					errch:  errch,
				}
				common.Infof("ManualCompaction: Compacting index instance:%v", is.InstId)
				s.supvMsgch <- compactReq
				err := <-errch
				if err == nil {
					common.Infof("ManualCompaction: Finished compacting index instance:%v", is.InstId)
				} else {
					common.Errorf("ManualCompaction: Index instance:%v Compaction failed with reason - %v", is.InstId, err)
				}
			}
		}()
	}

	return nil
}

func getSettingsConfig(cfg common.Config) (common.Config, error) {
	settingsConfig := cfg.SectionConfig("settings.", false)
	current, _, err := metakv.Get(indexerSettingsMetaPath)
	if err == nil {
		if len(current) > 0 {
			settingsConfig.Update(current)
		}
	}
	return settingsConfig, err
}
<commit_msg>indexer: Add dynamic gomaxprocs setup based on settings<commit_after>// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.

package indexer

import (
	"bytes"
	"errors"
	"github.com/couchbase/cbauth/metakv"
	"github.com/couchbase/indexing/secondary/common"
	"io/ioutil"
	"net/http"
	"runtime"
	"time"
)

const (
	indexerMetaDir          = "/indexer/"
	indexerSettingsMetaPath = indexerMetaDir + "settings"
	indexCompactonMetaPath  = indexerMetaDir + "triggerCompaction"
)

// Implements dynamic settings management for indexer
type settingsManager struct {
	supvCmdch       MsgChannel
	supvMsgch       MsgChannel
	config          common.Config
	cancelCh        chan struct{}
	compactionToken []byte
}

func NewSettingsManager(supvCmdch MsgChannel,
	supvMsgch MsgChannel, config common.Config) (settingsManager, common.Config, Message) {
	s := settingsManager{
		supvCmdch: supvCmdch,
		supvMsgch: supvMsgch,
		config:    config,
		cancelCh:  make(chan struct{}),
	}

	value, _, err := metakv.Get(indexerSettingsMetaPath)
	if err != nil {
		return s, nil, &MsgError{
			err: Error{
				category: INDEXER,
				cause:    err,
				severity: FATAL,
			}}
	}

	if len(value) > 0 {
		config.Update(value)
	}

	setNumCPUs(config)

	http.HandleFunc("/settings", s.handleSettingsReq)
	http.HandleFunc("/triggerCompaction", s.handleCompactionTrigger)
	go func() {
		for {
			err := metakv.RunObserveChildren("/", s.metaKVCallback, s.cancelCh)
			if err == nil {
				return
			} else {
				common.Errorf("IndexerSettingsManager: metakv notifier failed (%v)..Restarting", err)
			}
		}
	}()

	return s, config, &MsgSuccess{}
}

func (s *settingsManager) writeOk(w http.ResponseWriter) {
	w.WriteHeader(200)
	w.Write([]byte("OK\n"))
}

func (s *settingsManager) writeError(w http.ResponseWriter, err error) {
	w.WriteHeader(400)
	w.Write([]byte(err.Error() + "\n"))
}

func (s *settingsManager) writeJson(w http.ResponseWriter, json []byte) {
	header := w.Header()
	header["Content-Type"] = []string{"application/json"}
	w.WriteHeader(200)
	w.Write(json)
	w.Write([]byte("\n"))
}

func (s *settingsManager) handleSettingsReq(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		bytes, _ := ioutil.ReadAll(r.Body)

		config := s.config.Clone()
		current, rev, err := metakv.Get(indexerSettingsMetaPath)
		if err == nil {
			if len(current) > 0 {
				config.Update(current)
			}
			err = config.Update(bytes)
		}

		if err != nil {
			s.writeError(w, err)
			return
		}

		settingsConfig := config.SectionConfig("settings.", false)
		newSettingsBytes := settingsConfig.Json()
		if err = metakv.Set(indexerSettingsMetaPath, newSettingsBytes, rev); err != nil {
			s.writeError(w, err)
			return
		}
		s.writeOk(w)
	} else if r.Method == "GET" {
		settingsConfig, err := getSettingsConfig(s.config)
		if err != nil {
			s.writeError(w, err)
			return
		}
		s.writeJson(w, settingsConfig.Json())
	} else {
		s.writeError(w, errors.New("Unsupported method"))
		return
	}
}

func (s *settingsManager) handleCompactionTrigger(w http.ResponseWriter, r *http.Request) {
	_, rev, err := metakv.Get(indexCompactonMetaPath)
	if err != nil {
		s.writeError(w, err)
		return
	}

	newToken := time.Now().String()
	if err = metakv.Set(indexCompactonMetaPath, []byte(newToken), rev); err != nil {
		s.writeError(w, err)
		return
	}

	s.writeOk(w)
}

func (s *settingsManager) run() {
loop:
	for {
		select {
		case cmd, ok := <-s.supvCmdch:
			if ok {
				if cmd.GetMsgType() == STORAGE_MGR_SHUTDOWN {
					common.Infof("SettingsManager::run Shutting Down")
					close(s.cancelCh)
					s.supvCmdch <- &MsgSuccess{}
					break loop
				}
			} else {
				break loop
			}
		}
	}
}

func (s *settingsManager) metaKVCallback(path string, value []byte, rev interface{}) error {
	if path == indexerSettingsMetaPath {
		common.Infof("New settings received: \n%s", string(value))
		config := s.config.Clone()
		config.Update(value)
		s.config = config
		setNumCPUs(config)

		s.supvMsgch <- &MsgConfigUpdate{
			cfg: s.config,
		}
	} else if path == indexCompactonMetaPath {
		currentToken := s.compactionToken
		s.compactionToken = value
		if currentToken == nil || bytes.Equal(currentToken, value) {
			return nil
		}

		common.Infof("Manual compaction trigger requested")
		replych := make(chan []IndexStorageStats)
		statReq := &MsgIndexStorageStats{respch: replych}
		s.supvMsgch <- statReq
		stats := <-replych
		// XXX: minFile size check can be applied
		go func() {
			for _, is := range stats {
				errch := make(chan error)
				compactReq := &MsgIndexCompact{
					instId: is.InstId,
					errch:  errch,
				}
				common.Infof("ManualCompaction: Compacting index instance:%v", is.InstId)
				s.supvMsgch <- compactReq
				err := <-errch
				if err == nil {
					common.Infof("ManualCompaction: Finished compacting index instance:%v", is.InstId)
				} else {
					common.Errorf("ManualCompaction: Index instance:%v Compaction failed with reason - %v", is.InstId, err)
				}
			}
		}()
	}

	return nil
}

func getSettingsConfig(cfg common.Config) (common.Config, error) {
	settingsConfig := cfg.SectionConfig("settings.", false)
	current, _, err := metakv.Get(indexerSettingsMetaPath)
	if err == nil {
		if len(current) > 0 {
			settingsConfig.Update(current)
		}
	}
	return settingsConfig, err
}

func setNumCPUs(config common.Config) {
	ncpu := config["settings.max_cpu_procs"].Int()
	if ncpu == 0 {
		ncpu = runtime.NumCPU()
	}

	common.Infof("Setting maxcpus = %d", ncpu)
	runtime.GOMAXPROCS(ncpu)
}
<|endoftext|>


<commit_before>package networks

import (
	"fmt"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/pagination"
)

// AdminState gives users a solid type to work with for create and update
// operations. It is recommended that users use the `Up` and `Down` enums.
type AdminState *bool

// Convenience vars for AdminStateUp values.
var (
	iTrue  = true
	iFalse = false

	Up   AdminState = &iTrue
	Down AdminState = &iFalse
)

type networkOpts struct {
	AdminStateUp *bool
	Name         string
	Shared       *bool
	TenantID     string
}

// ListOptsBuilder allows extensions to add additional parameters to the
// List request.
type ListOptsBuilder interface {
	ToNetworkListQuery() (string, error)
}

// ListOpts allows the filtering and sorting of paginated collections through
// the API. Filtering is achieved by passing in struct field values that map to
// the network attributes you want to see returned. SortKey allows you to sort
// by a particular network attribute. SortDir sets the direction, and is either
// `asc' or `desc'. Marker and Limit are used for pagination.
type ListOpts struct {
	Status       string `q:"status"`
	Name         string `q:"name"`
	AdminStateUp *bool  `q:"admin_state_up"`
	TenantID     string `q:"tenant_id"`
	Shared       *bool  `q:"shared"`
	ID           string `q:"id"`
	Marker       string `q:"marker"`
	Limit        int    `q:"limit"`
	SortKey      string `q:"sort_key"`
	SortDir      string `q:"sort_dir"`
}

// ToNetworkListQuery formats a ListOpts into a query string.
func (opts ListOpts) ToNetworkListQuery() (string, error) {
	q, err := gophercloud.BuildQueryString(opts)
	if err != nil {
		return "", err
	}
	return q.String(), nil
}

// List returns a Pager which allows you to iterate over a collection of
// networks. It accepts a ListOpts struct, which allows you to filter and sort
// the returned collection for greater efficiency.
func List(c *gophercloud.ServiceClient, opts ListOptsBuilder) pagination.Pager {
	url := listURL(c)
	if opts != nil {
		query, err := opts.ToNetworkListQuery()
		if err != nil {
			return pagination.Pager{Err: err}
		}
		url += query
	}

	return pagination.NewPager(c, url, func(r pagination.PageResult) pagination.Page {
		return NetworkPage{pagination.LinkedPageBase{PageResult: r}}
	})
}

// Get retrieves a specific network based on its unique ID.
func Get(c *gophercloud.ServiceClient, id string) GetResult {
	var res GetResult
	_, res.Err = c.Get(getURL(c, id), &res.Body, nil)
	return res
}

// CreateOptsBuilder is the interface options structs have to satisfy in order
// to be used in the main Create operation in this package. Since many
// extensions decorate or modify the common logic, it is useful for them to
// satisfy a basic interface in order for them to be used.
type CreateOptsBuilder interface {
	ToNetworkCreateMap() (map[string]interface{}, error)
}

// CreateOpts is the common options struct used in this package's Create
// operation.
type CreateOpts networkOpts

// ToNetworkCreateMap casts a CreateOpts struct to a map.
func (opts CreateOpts) ToNetworkCreateMap() (map[string]interface{}, error) {
	n := make(map[string]interface{})

	if opts.AdminStateUp != nil {
		n["admin_state_up"] = &opts.AdminStateUp
	}
	if opts.Name != "" {
		n["name"] = opts.Name
	}
	if opts.Shared != nil {
		n["shared"] = &opts.Shared
	}
	if opts.TenantID != "" {
		n["tenant_id"] = opts.TenantID
	}

	return map[string]interface{}{"network": n}, nil
}

// Create accepts a CreateOpts struct and creates a new network using the values
// provided. This operation does not actually require a request body, i.e. the
// CreateOpts struct argument can be empty.
//
// The tenant ID that is contained in the URI is the tenant that creates the
// network. An admin user, however, has the option of specifying another tenant
// ID in the CreateOpts struct.
func Create(c *gophercloud.ServiceClient, opts CreateOptsBuilder) CreateResult {
	var res CreateResult

	reqBody, err := opts.ToNetworkCreateMap()
	if err != nil {
		res.Err = err
		return res
	}

	_, res.Err = c.Post(createURL(c), reqBody, &res.Body, nil)
	return res
}

// UpdateOptsBuilder is the interface options structs have to satisfy in order
// to be used in the main Update operation in this package. Since many
// extensions decorate or modify the common logic, it is useful for them to
// satisfy a basic interface in order for them to be used.
type UpdateOptsBuilder interface {
	ToNetworkUpdateMap() (map[string]interface{}, error)
}

// UpdateOpts is the common options struct used in this package's Update
// operation.
type UpdateOpts networkOpts

// ToNetworkUpdateMap casts a UpdateOpts struct to a map.
func (opts UpdateOpts) ToNetworkUpdateMap() (map[string]interface{}, error) {
	n := make(map[string]interface{})

	if opts.AdminStateUp != nil {
		n["admin_state_up"] = &opts.AdminStateUp
	}
	if opts.Name != "" {
		n["name"] = opts.Name
	}
	if opts.Shared != nil {
		n["shared"] = &opts.Shared
	}

	return map[string]interface{}{"network": n}, nil
}

// Update accepts a UpdateOpts struct and updates an existing network using the
// values provided. For more information, see the Create function.
func Update(c *gophercloud.ServiceClient, networkID string, opts UpdateOptsBuilder) UpdateResult {
	var res UpdateResult

	reqBody, err := opts.ToNetworkUpdateMap()
	if err != nil {
		res.Err = err
		return res
	}

	// Send request to API
	_, res.Err = c.Put(updateURL(c, networkID), reqBody, &res.Body, &gophercloud.RequestOpts{
		OkCodes: []int{200, 201},
	})

	return res
}

// Delete accepts a unique ID and deletes the network associated with it.
func Delete(c *gophercloud.ServiceClient, networkID string) DeleteResult {
	var res DeleteResult
	_, res.Err = c.Delete(deleteURL(c, networkID), nil)
	return res
}

// IDFromName is a convienience function that returns a network's ID given its name.
func IDFromName(client *gophercloud.ServiceClient, name string) (string, error) {
	networkCount := 0
	networkID := ""
	if name == "" {
		return "", fmt.Errorf("A network name must be provided.")
	}
	pager := List(client, nil)
	pager.EachPage(func(page pagination.Page) (bool, error) {
		networkList, err := ExtractNetworks(page)
		if err != nil {
			return false, err
		}

		for _, n := range networkList {
			if n.Name == name {
				networkCount++
				networkID = n.ID
			}
		}
		return true, nil
	})

	switch networkCount {
	case 0:
		return "", fmt.Errorf("Unable to find network: %s", name)
	case 1:
		return networkID, nil
	default:
		return "", fmt.Errorf("Found %d networks matching %s", networkCount, name)
	}
}
<commit_msg>fix typo in comment<commit_after>package networks

import (
	"fmt"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/pagination"
)

// AdminState gives users a solid type to work with for create and update
// operations. It is recommended that users use the `Up` and `Down` enums.
type AdminState *bool

// Convenience vars for AdminStateUp values.
var (
	iTrue  = true
	iFalse = false

	Up   AdminState = &iTrue
	Down AdminState = &iFalse
)

type networkOpts struct {
	AdminStateUp *bool
	Name         string
	Shared       *bool
	TenantID     string
}

// ListOptsBuilder allows extensions to add additional parameters to the
// List request.
type ListOptsBuilder interface {
	ToNetworkListQuery() (string, error)
}

// ListOpts allows the filtering and sorting of paginated collections through
// the API. Filtering is achieved by passing in struct field values that map to
// the network attributes you want to see returned. SortKey allows you to sort
// by a particular network attribute. SortDir sets the direction, and is either
// `asc' or `desc'. Marker and Limit are used for pagination.
type ListOpts struct {
	Status       string `q:"status"`
	Name         string `q:"name"`
	AdminStateUp *bool  `q:"admin_state_up"`
	TenantID     string `q:"tenant_id"`
	Shared       *bool  `q:"shared"`
	ID           string `q:"id"`
	Marker       string `q:"marker"`
	Limit        int    `q:"limit"`
	SortKey      string `q:"sort_key"`
	SortDir      string `q:"sort_dir"`
}

// ToNetworkListQuery formats a ListOpts into a query string.
func (opts ListOpts) ToNetworkListQuery() (string, error) {
	q, err := gophercloud.BuildQueryString(opts)
	if err != nil {
		return "", err
	}
	return q.String(), nil
}

// List returns a Pager which allows you to iterate over a collection of
// networks. It accepts a ListOpts struct, which allows you to filter and sort
// the returned collection for greater efficiency.
func List(c *gophercloud.ServiceClient, opts ListOptsBuilder) pagination.Pager {
	url := listURL(c)
	if opts != nil {
		query, err := opts.ToNetworkListQuery()
		if err != nil {
			return pagination.Pager{Err: err}
		}
		url += query
	}

	return pagination.NewPager(c, url, func(r pagination.PageResult) pagination.Page {
		return NetworkPage{pagination.LinkedPageBase{PageResult: r}}
	})
}

// Get retrieves a specific network based on its unique ID.
func Get(c *gophercloud.ServiceClient, id string) GetResult {
	var res GetResult
	_, res.Err = c.Get(getURL(c, id), &res.Body, nil)
	return res
}

// CreateOptsBuilder is the interface options structs have to satisfy in order
// to be used in the main Create operation in this package. Since many
// extensions decorate or modify the common logic, it is useful for them to
// satisfy a basic interface in order for them to be used.
type CreateOptsBuilder interface {
	ToNetworkCreateMap() (map[string]interface{}, error)
}

// CreateOpts is the common options struct used in this package's Create
// operation.
type CreateOpts networkOpts

// ToNetworkCreateMap casts a CreateOpts struct to a map.
func (opts CreateOpts) ToNetworkCreateMap() (map[string]interface{}, error) {
	n := make(map[string]interface{})

	if opts.AdminStateUp != nil {
		n["admin_state_up"] = &opts.AdminStateUp
	}
	if opts.Name != "" {
		n["name"] = opts.Name
	}
	if opts.Shared != nil {
		n["shared"] = &opts.Shared
	}
	if opts.TenantID != "" {
		n["tenant_id"] = opts.TenantID
	}

	return map[string]interface{}{"network": n}, nil
}

// Create accepts a CreateOpts struct and creates a new network using the values
// provided. This operation does not actually require a request body, i.e. the
// CreateOpts struct argument can be empty.
//
// The tenant ID that is contained in the URI is the tenant that creates the
// network. An admin user, however, has the option of specifying another tenant
// ID in the CreateOpts struct.
func Create(c *gophercloud.ServiceClient, opts CreateOptsBuilder) CreateResult {
	var res CreateResult

	reqBody, err := opts.ToNetworkCreateMap()
	if err != nil {
		res.Err = err
		return res
	}

	_, res.Err = c.Post(createURL(c), reqBody, &res.Body, nil)
	return res
}

// UpdateOptsBuilder is the interface options structs have to satisfy in order
// to be used in the main Update operation in this package. Since many
// extensions decorate or modify the common logic, it is useful for them to
// satisfy a basic interface in order for them to be used.
type UpdateOptsBuilder interface {
	ToNetworkUpdateMap() (map[string]interface{}, error)
}

// UpdateOpts is the common options struct used in this package's Update
// operation.
type UpdateOpts networkOpts

// ToNetworkUpdateMap casts a UpdateOpts struct to a map.
func (opts UpdateOpts) ToNetworkUpdateMap() (map[string]interface{}, error) {
	n := make(map[string]interface{})

	if opts.AdminStateUp != nil {
		n["admin_state_up"] = &opts.AdminStateUp
	}
	if opts.Name != "" {
		n["name"] = opts.Name
	}
	if opts.Shared != nil {
		n["shared"] = &opts.Shared
	}

	return map[string]interface{}{"network": n}, nil
}

// Update accepts a UpdateOpts struct and updates an existing network using the
// values provided. For more information, see the Create function.
func Update(c *gophercloud.ServiceClient, networkID string, opts UpdateOptsBuilder) UpdateResult {
	var res UpdateResult

	reqBody, err := opts.ToNetworkUpdateMap()
	if err != nil {
		res.Err = err
		return res
	}

	// Send request to API
	_, res.Err = c.Put(updateURL(c, networkID), reqBody, &res.Body, &gophercloud.RequestOpts{
		OkCodes: []int{200, 201},
	})

	return res
}

// Delete accepts a unique ID and deletes the network associated with it.
func Delete(c *gophercloud.ServiceClient, networkID string) DeleteResult {
	var res DeleteResult
	_, res.Err = c.Delete(deleteURL(c, networkID), nil)
	return res
}

// IDFromName is a convenience function that returns a network's ID given its name.
func IDFromName(client *gophercloud.ServiceClient, name string) (string, error) {
	networkCount := 0
	networkID := ""
	if name == "" {
		return "", fmt.Errorf("A network name must be provided.")
	}
	pager := List(client, nil)
	pager.EachPage(func(page pagination.Page) (bool, error) {
		networkList, err := ExtractNetworks(page)
		if err != nil {
			return false, err
		}

		for _, n := range networkList {
			if n.Name == name {
				networkCount++
				networkID = n.ID
			}
		}
		return true, nil
	})

	switch networkCount {
	case 0:
		return "", fmt.Errorf("Unable to find network: %s", name)
	case 1:
		return networkID, nil
	default:
		return "", fmt.Errorf("Found %d networks matching %s", networkCount, name)
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2019 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cachebench

import (
	"errors"
	"math/rand"
	"strconv"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/allegro/bigcache"
	"github.com/cespare/xxhash"
	"github.com/coocood/freecache"
	"github.com/dgraph-io/dgraph/ristretto"
	"github.com/golang/groupcache/lru"
	"github.com/pingcap/go-ycsb/pkg/generator"
)

const (
	// based on 21million dataset, we observed a maximum key length of 77,
	// with minimum length being 6 and average length being 25. We also
	// observed that 99% of keys had length <64 bytes.
	maxKeyLength = 128
	// workloadSize is the size of array storing sequence of keys that we
	// have in our workload. In the benchmark, we iterate over this array b.N
	// number of times in circular fashion starting at a random position.
	workloadSize = 2 << 20
)

var (
	errKeyNotFound  = errors.New("key not found")
	errInvalidValue = errors.New("invalid value")
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

//========================================================================
//                           Possible workloads
//========================================================================

func zipfKeyList() [][]byte {
	// To ensure repetition of keys in the array,
	// we are generating keys in the range from 0 to workloadSize/3.
	maxKey := int64(workloadSize) / 3

	// scrambled zipfian to ensure same keys are not together
	z := generator.NewScrambledZipfian(0, maxKey, generator.ZipfianConstant)
	r := rand.New(rand.NewSource(time.Now().UnixNano()))

	keys := make([][]byte, workloadSize)
	for i := 0; i < workloadSize; i++ {
		keys[i] = []byte(strconv.Itoa(int(z.Next(r))))
	}

	return keys
}

func oneKeyList() [][]byte {
	v := rand.Int() % (workloadSize / 3)
	s := []byte(strconv.Itoa(v))

	keys := make([][]byte, workloadSize)
	for i := 0; i < workloadSize; i++ {
		keys[i] = s
	}

	return keys
}

//========================================================================
//                               BigCache
//========================================================================

type BigCache struct {
	c *bigcache.BigCache
}

func (b *BigCache) Get(key []byte) ([]byte, error) {
	return b.c.Get(string(key))
}

func (b *BigCache) Set(key, value []byte) error {
	return b.c.Set(string(key), value)
}

func newBigCache(keysInWindow int) *BigCache {
	cache, err := bigcache.NewBigCache(bigcache.Config{
		Shards:             256,
		LifeWindow:         0,
		MaxEntriesInWindow: keysInWindow,
		MaxEntrySize:       maxKeyLength,
		Verbose:            false,
	})
	if err != nil {
		panic(err)
	}

	// Enforce full initialization of internal structures. This is taken
	// from GetPutBenchmark.java from java caffeine. It is required in
	// caffeine given that it keeps buffering the keys for applying the
	// necessary changes later. This is probably unnecessary here.
	for i := 0; i < 2*workloadSize; i++ {
		_ = cache.Set(strconv.Itoa(i), []byte("data"))
	}
	_ = cache.Reset()

	return &BigCache{cache}
}

//========================================================================
//                            FreeCache
//========================================================================

type FreeCache struct {
	c *freecache.Cache
}

func (f *FreeCache) Get(key []byte) ([]byte, error) {
	return f.c.Get(key)
}

func (f *FreeCache) Set(key, value []byte) error {
	return f.c.Set(key, value, 0)
}

func newFreeCache(keysInWindow int) *FreeCache {
	cache := freecache.NewCache(keysInWindow * maxKeyLength)

	// Enforce full initialization of internal structures
	// (probably not required, see above in bigcache)
	for i := 0; i < 2*workloadSize; i++ {
		_ = cache.Set([]byte(strconv.Itoa(i)), []byte("data"), 0)
	}
	cache.Clear()

	return &FreeCache{cache}
}

//========================================================================
//                            GroupCache
//========================================================================

const (
	segmentAndOpVal = 255
)

type GroupCache struct {
	shards [256]*lru.Cache
	locks  [256]sync.Mutex
}

func (g *GroupCache) Get(key []byte) ([]byte, error) {
	hashVal := xxhash.Sum64(key)
	shardNum := hashVal & segmentAndOpVal

	g.locks[shardNum].Lock()
	v, ok := g.shards[shardNum].Get(string(key))
	g.locks[shardNum].Unlock()

	if ok {
		return v.([]byte), nil
	}
	return nil, errKeyNotFound
}

func (g *GroupCache) Set(key, value []byte) error {
	hashVal := xxhash.Sum64(key)
	shardNum := hashVal & segmentAndOpVal

	g.locks[shardNum].Lock()
	g.shards[shardNum].Add(string(key), value)
	g.locks[shardNum].Unlock()

	return nil
}

func newGroupCache(keysInWindow int) *GroupCache {
	gc := &GroupCache{}
	for i := 0; i < 256; i++ {
		gc.shards[i] = lru.New(keysInWindow / 256)
	}

	// Enforce full initialization of internal structures
	for j := 0; j < 2*workloadSize; j++ {
		_ = gc.Set([]byte(strconv.Itoa(j)), []byte("data"))
	}
	for i := 0; i < 256; i++ {
		gc.shards[i].Clear()
	}

	return gc
}

//========================================================================
//                              sync.Map
//========================================================================

type SyncMap struct {
	c *sync.Map
}

func (m *SyncMap) Get(key []byte) ([]byte, error) {
	v, ok := m.c.Load(string(key))
	if !ok {
		return nil, errKeyNotFound
	}

	tv, ok := v.([]byte)
	if !ok {
		return nil, errInvalidValue
	}

	return tv, nil
}

func (m *SyncMap) Set(key, value []byte) error {
	// We are not performing any initialization here unlike other caches
	// given that there is no function available to reset the map.
	m.c.Store(string(key), value)
	return nil
}

func newSyncMap() *SyncMap {
	return &SyncMap{new(sync.Map)}
}

//========================================================================
//                         Benchmark Code
//========================================================================

func runCacheBenchmark(b *testing.B, cache ristretto.Cache, keys [][]byte, pctWrites uint64) {
	b.ReportAllocs()

	size := len(keys)
	mask := size - 1
	rc := uint64(0)

	// initialize cache
	for i := 0; i < size; i++ {
		_ = cache.Set(keys[i], []byte("data"))
	}

	b.ResetTimer()
	b.RunParallel(func(pb *testing.PB) {
		index := rand.Int() & mask
		mc := atomic.AddUint64(&rc, 1)

		if pctWrites*mc/100 != pctWrites*(mc-1)/100 {
			for pb.Next() {
				_ = cache.Set(keys[index&mask], []byte("data"))
				index = index + 1
			}
		} else {
			for pb.Next() {
				_, _ = cache.Get(keys[index&mask])
				index = index + 1
			}
		}
	})
}

func BenchmarkCaches(b *testing.B) {
	zipfList := zipfKeyList()
	oneList := oneKeyList()

	// two datasets (zipf, onekey)
	// 3 caches (bigcache, freecache, sync.Map)
	// 3 types of benchmark (read, write, mixed)
	benchmarks := []struct {
		name      string
		cache     ristretto.Cache
		keys      [][]byte
		pctWrites uint64
	}{
		{"BigCacheZipfRead", newBigCache(b.N), zipfList, 0},
		{"FreeCacheZipfRead", newFreeCache(b.N), zipfList, 0},
		{"GroupCacheZipfRead", newGroupCache(b.N), zipfList, 0},
		{"SyncMapZipfRead", newSyncMap(), zipfList, 0},

		{"BigCacheOneKeyRead", newBigCache(b.N), oneList, 0},
		{"FreeCacheOneKeyRead", newFreeCache(b.N), oneList, 0},
		{"GroupCacheOneKeyRead", newGroupCache(b.N), oneList, 0},
		{"SyncMapOneKeyRead", newSyncMap(), oneList, 0},

		{"BigCacheZipfWrite", newBigCache(b.N), zipfList, 100},
		{"FreeCacheZipfWrite", newFreeCache(b.N), zipfList, 100},
		{"GroupCacheZipfWrite", newGroupCache(b.N), zipfList, 100},
		{"SyncMapZipfWrite", newSyncMap(), zipfList, 100},

		{"BigCacheOneKeyWrite", newBigCache(b.N), oneList, 100},
		{"FreeCacheOneKeyWrite", newFreeCache(b.N), oneList, 100},
		{"GroupCacheOneKeyWrite", newGroupCache(b.N), oneList, 100},
		{"SyncMapOneKeyWrite", newSyncMap(), oneList, 100},

		{"BigCacheZipfMixed", newBigCache(b.N), zipfList, 25},
		{"FreeCacheZipfMixed", newFreeCache(b.N), zipfList, 25},
		{"GroupCacheZipfMixed", newGroupCache(b.N), zipfList, 25},
		{"SyncMapZipfMixed", newSyncMap(), zipfList, 25},

		{"BigCacheOneKeyMixed", newBigCache(b.N), oneList, 25},
		{"FreeCacheOneKeyMixed", newFreeCache(b.N), oneList, 25},
		{"GroupCacheOneKeyMixed", newGroupCache(b.N), oneList, 25},
		{"SyncMapOneKeyMixed", newSyncMap(), oneList, 25},
	}

	for _, bm := range benchmarks {
		b.Run(bm.name, func(b *testing.B) {
			runCacheBenchmark(b, bm.cache, bm.keys, bm.pctWrites)
		})
	}
}
<commit_msg>Fix ristretto import<commit_after>/*
 * Copyright 2019 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cachebench

import (
	"errors"
	"math/rand"
	"strconv"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/allegro/bigcache"
	"github.com/cespare/xxhash"
	"github.com/coocood/freecache"
	"github.com/dgraph-io/badger/ristretto"
	"github.com/golang/groupcache/lru"
	"github.com/pingcap/go-ycsb/pkg/generator"
)

const (
	// based on 21million dataset, we observed a maximum key length of 77,
	// with minimum length being 6 and average length being 25. We also
	// observed that 99% of keys had length <64 bytes.
	maxKeyLength = 128
	// workloadSize is the size of array storing sequence of keys that we
	// have in our workload. In the benchmark, we iterate over this array b.N
	// number of times in circular fashion starting at a random position.
	workloadSize = 2 << 20
)

var (
	errKeyNotFound  = errors.New("key not found")
	errInvalidValue = errors.New("invalid value")
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

//========================================================================
//                           Possible workloads
//========================================================================

func zipfKeyList() [][]byte {
	// To ensure repetition of keys in the array,
	// we are generating keys in the range from 0 to workloadSize/3.
	maxKey := int64(workloadSize) / 3

	// scrambled zipfian to ensure same keys are not together
	z := generator.NewScrambledZipfian(0, maxKey, generator.ZipfianConstant)
	r := rand.New(rand.NewSource(time.Now().UnixNano()))

	keys := make([][]byte, workloadSize)
	for i := 0; i < workloadSize; i++ {
		keys[i] = []byte(strconv.Itoa(int(z.Next(r))))
	}

	return keys
}

func oneKeyList() [][]byte {
	v := rand.Int() % (workloadSize / 3)
	s := []byte(strconv.Itoa(v))

	keys := make([][]byte, workloadSize)
	for i := 0; i < workloadSize; i++ {
		keys[i] = s
	}

	return keys
}

//========================================================================
//                               BigCache
//========================================================================

type BigCache struct {
	c *bigcache.BigCache
}

func (b *BigCache) Get(key []byte) ([]byte, error) {
	return b.c.Get(string(key))
}

func (b *BigCache) Set(key, value []byte) error {
	return b.c.Set(string(key), value)
}

func newBigCache(keysInWindow int) *BigCache {
	cache, err := bigcache.NewBigCache(bigcache.Config{
		Shards:             256,
		LifeWindow:         0,
		MaxEntriesInWindow: keysInWindow,
		MaxEntrySize:       maxKeyLength,
		Verbose:            false,
	})
	if err != nil {
		panic(err)
	}

	// Enforce full initialization of internal structures. This is taken
	// from GetPutBenchmark.java from java caffeine. It is required in
	// caffeine given that it keeps buffering the keys for applying the
	// necessary changes later. This is probably unnecessary here.
	for i := 0; i < 2*workloadSize; i++ {
		_ = cache.Set(strconv.Itoa(i), []byte("data"))
	}
	_ = cache.Reset()

	return &BigCache{cache}
}

//========================================================================
//                            FreeCache
//========================================================================

type FreeCache struct {
	c *freecache.Cache
}

func (f *FreeCache) Get(key []byte) ([]byte, error) {
	return f.c.Get(key)
}

func (f *FreeCache) Set(key, value []byte) error {
	return f.c.Set(key, value, 0)
}

func newFreeCache(keysInWindow int) *FreeCache {
	cache := freecache.NewCache(keysInWindow * maxKeyLength)

	// Enforce full initialization of internal structures
	// (probably not required, see above in bigcache)
	for i := 0; i < 2*workloadSize; i++ {
		_ = cache.Set([]byte(strconv.Itoa(i)), []byte("data"), 0)
	}
	cache.Clear()

	return &FreeCache{cache}
}

//========================================================================
//                            GroupCache
//========================================================================

const (
	segmentAndOpVal = 255
)

type GroupCache struct {
	shards [256]*lru.Cache
	locks  [256]sync.Mutex
}

func (g *GroupCache) Get(key []byte) ([]byte, error) {
	hashVal := xxhash.Sum64(key)
	shardNum := hashVal & segmentAndOpVal

	g.locks[shardNum].Lock()
	v, ok := g.shards[shardNum].Get(string(key))
	g.locks[shardNum].Unlock()

	if ok {
		return v.([]byte), nil
	}
	return nil, errKeyNotFound
}

func (g *GroupCache) Set(key, value []byte) error {
	hashVal := xxhash.Sum64(key)
	shardNum := hashVal & segmentAndOpVal

	g.locks[shardNum].Lock()
	g.shards[shardNum].Add(string(key), value)
	g.locks[shardNum].Unlock()

	return nil
}

func newGroupCache(keysInWindow int) *GroupCache {
	gc := &GroupCache{}
	for i := 0; i < 256; i++ {
		gc.shards[i] = lru.New(keysInWindow / 256)
	}

	// Enforce full initialization of internal structures
	for j := 0; j < 2*workloadSize; j++ {
		_ = gc.Set([]byte(strconv.Itoa(j)), []byte("data"))
	}
	for i := 0; i < 256; i++ {
		gc.shards[i].Clear()
	}

	return gc
}

//========================================================================
//                              sync.Map
//========================================================================

type SyncMap struct {
	c *sync.Map
}

func (m *SyncMap) Get(key []byte) ([]byte, error) {
	v, ok := m.c.Load(string(key))
	if !ok {
		return nil, errKeyNotFound
	}

	tv, ok := v.([]byte)
	if !ok {
		return nil, errInvalidValue
	}

	return tv, nil
}

func (m *SyncMap) Set(key, value []byte) error {
	// We are not performing any initialization here unlike other caches
	// given that there is no function available to reset the map.
	m.c.Store(string(key), value)
	return nil
}

func newSyncMap() *SyncMap {
	return &SyncMap{new(sync.Map)}
}

//========================================================================
//                         Benchmark Code
//========================================================================

func runCacheBenchmark(b *testing.B, cache ristretto.Cache, keys [][]byte, pctWrites uint64) {
	b.ReportAllocs()

	size := len(keys)
	mask := size - 1
	rc := uint64(0)

	// initialize cache
	for i := 0; i < size; i++ {
		_ = cache.Set(keys[i], []byte("data"))
	}

	b.ResetTimer()
	b.RunParallel(func(pb *testing.PB) {
		index := rand.Int() & mask
		mc := atomic.AddUint64(&rc, 1)

		if pctWrites*mc/100 != pctWrites*(mc-1)/100 {
			for pb.Next() {
				_ = cache.Set(keys[index&mask], []byte("data"))
				index = index + 1
			}
		} else {
			for pb.Next() {
				_, _ = cache.Get(keys[index&mask])
				index = index + 1
			}
		}
	})
}

func BenchmarkCaches(b *testing.B) {
	zipfList := zipfKeyList()
	oneList := oneKeyList()

	// two datasets (zipf, onekey)
	// 3 caches (bigcache, freecache, sync.Map)
	// 3 types of benchmark (read, write, mixed)
	benchmarks := []struct {
		name      string
		cache     ristretto.Cache
		keys      [][]byte
		pctWrites uint64
	}{
		{"BigCacheZipfRead", newBigCache(b.N), zipfList, 0},
		{"FreeCacheZipfRead", newFreeCache(b.N), zipfList, 0},
		{"GroupCacheZipfRead", newGroupCache(b.N), zipfList, 0},
		{"SyncMapZipfRead", newSyncMap(), zipfList, 0},

		{"BigCacheOneKeyRead", newBigCache(b.N), oneList, 0},
		{"FreeCacheOneKeyRead", newFreeCache(b.N), oneList, 0},
		{"GroupCacheOneKeyRead", newGroupCache(b.N), oneList, 0},
		{"SyncMapOneKeyRead", newSyncMap(), oneList, 0},

		{"BigCacheZipfWrite", newBigCache(b.N), zipfList, 100},
		{"FreeCacheZipfWrite", newFreeCache(b.N), zipfList, 100},
		{"GroupCacheZipfWrite", newGroupCache(b.N), zipfList, 100},
		{"SyncMapZipfWrite", newSyncMap(), zipfList, 100},

		{"BigCacheOneKeyWrite", newBigCache(b.N), oneList, 100},
		{"FreeCacheOneKeyWrite", newFreeCache(b.N), oneList, 100},
		{"GroupCacheOneKeyWrite", newGroupCache(b.N), oneList, 100},
		{"SyncMapOneKeyWrite", newSyncMap(), oneList, 100},

		{"BigCacheZipfMixed", newBigCache(b.N), zipfList, 25},
		{"FreeCacheZipfMixed", newFreeCache(b.N), zipfList, 25},
		{"GroupCacheZipfMixed", newGroupCache(b.N), zipfList, 25},
		{"SyncMapZipfMixed", newSyncMap(), zipfList, 25},

		{"BigCacheOneKeyMixed", newBigCache(b.N), oneList, 25},
		{"FreeCacheOneKeyMixed", newFreeCache(b.N), oneList, 25},
		{"GroupCacheOneKeyMixed", newGroupCache(b.N), oneList, 25},
		{"SyncMapOneKeyMixed", newSyncMap(), oneList, 25},
	}

	for _, bm := range benchmarks {
		b.Run(bm.name, func(b *testing.B) {
			runCacheBenchmark(b, bm.cache, bm.keys, bm.pctWrites)
		})
	}
}
<|endoftext|>


<commit_before>package model

import (
	rdef "go.polydawn.net/repeatr/api/def"
)

type Project struct {
	Names      map[string]ReleaseRecord   // map name->{ware,backstory}
	RunRecords map[string]*rdef.RunRecord // map rrhid->rr
	Memos      map[string]string          // index frmhid->rrhid
}

type ReleaseRecord struct {
	Ware         rdef.Ware
	RunRecordHID string // blank if a name was manual
}

func (p *Project) Init() {
	p.Names = make(map[string]ReleaseRecord)
	p.RunRecords = make(map[string]*rdef.RunRecord)
	p.Memos = make(map[string]string)
}

func (p *Project) PutManualName(name string, ware rdef.Ware) {
	_, hadPrev := p.Names[name]
	p.Names[name] = ReleaseRecord{ware, ""}
	if hadPrev {
		p.retainFilter()
	}
}

func (p *Project) PutEval(rr *rdef.RunRecord) {
	var savedAny bool
	for name, value := range rr.Results {
		if (name[0] >= 'a' && name[0] <= 'z') ||
			(name[0] >= 'A' && name[0] <= 'Z') {
			savedAny = true
		}
		p.Names[name] = ReleaseRecord{value.Ware, rr.HID}

	}
	// If no interesting results were saved, bail;
	//  we don't need to save this record, nor potentially evict old ones.
	// REVIEW: this means we won't memoize evals that didn't have named results.
	if savedAny == false {
		return
	}
	p.RunRecords[rr.HID] = rr
	p.Memos[rr.FormulaHID] = rr.HID
	p.retainFilter()
}

func (p *Project) retainFilter() {

}
<commit_msg>Implement retain filter and index rebuild.<commit_after>package model

import (
	rdef "go.polydawn.net/repeatr/api/def"
)

type Project struct {
	Names      map[string]ReleaseRecord   // map name->{ware,backstory}
	RunRecords map[string]*rdef.RunRecord // map rrhid->rr
	Memos      map[string]string          // index frmhid->rrhid
}

type ReleaseRecord struct {
	Ware         rdef.Ware
	RunRecordHID string // blank if a name was manual
}

func (p *Project) Init() {
	p.Names = make(map[string]ReleaseRecord)
	p.RunRecords = make(map[string]*rdef.RunRecord)
	p.Memos = make(map[string]string)
}

func (p *Project) PutManualName(name string, ware rdef.Ware) {
	_, hadPrev := p.Names[name]
	p.Names[name] = ReleaseRecord{ware, ""}
	if hadPrev {
		p.retainFilter()
	}
}

func (p *Project) DeleteName(name string) {
	_, hadPrev := p.Names[name]
	if hadPrev {
		delete(p.Names, name)
		p.retainFilter()
	}
}

func (p *Project) PutEval(rr *rdef.RunRecord) {
	var savedAny bool
	for name, value := range rr.Results {
		if (name[0] >= 'a' && name[0] <= 'z') ||
			(name[0] >= 'A' && name[0] <= 'Z') {
			savedAny = true
		}
		p.Names[name] = ReleaseRecord{value.Ware, rr.HID}

	}
	// If no interesting results were saved, bail;
	//  we don't need to save this record, nor potentially evict old ones.
	// REVIEW: this means we won't memoize evals that didn't have named results.
	if savedAny == false {
		return
	}
	p.RunRecords[rr.HID] = rr
	p.Memos[rr.FormulaHID] = rr.HID
	p.retainFilter()
}

func (p *Project) retainFilter() {
	// "Sweep".  (The `Names` map is the marks.)
	oldRunRecords := p.RunRecords
	oldMemos := p.Memos
	p.RunRecords = make(map[string]*rdef.RunRecord)
	p.Memos = make(map[string]string)
	// Rebuild `RunRecords` by whitelisting prev values still ref'd by `Names`.
	for _, release := range p.Names {
		p.RunRecords[release.RunRecordHID] = oldRunRecords[release.RunRecordHID]
	}
	// Rebuild `Memos` index from `RunRecords`.
	for _, runRecord := range p.RunRecords {
		p.Memos[runRecord.FormulaHID] = runRecord.HID
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package iterator

import (
	"regexp"

	"github.com/cayleygraph/cayley/graph"
	"github.com/cayleygraph/quad"
)

func newRegex(qs graph.Namer, sub graph.Iterator, re *regexp.Regexp, refs bool) graph.Iterator {
	return NewValueFilter(qs, sub, func(v quad.Value) (bool, error) {
		switch v := v.(type) {
		case quad.String:
			return re.MatchString(string(v)), nil
		case quad.TypedString:
			return re.MatchString(string(v.Value)), nil
		default:
			if refs {
				switch v := v.(type) {
				case quad.BNode:
					return re.MatchString(string(v)), nil
				case quad.IRI:
					return re.MatchString(string(v)), nil
				}
			}
		}
		return false, nil
	})
}

// NewRegex returns an unary operator -- a filter across the values in the relevant
// subiterator. It works similarly to gremlin's filter{it.matches('exp')},
// reducing the iterator set to values whose string representation passes a
// regular expression test.
func NewRegex(sub graph.Iterator, re *regexp.Regexp, qs graph.Namer) graph.Iterator {
	return newRegex(qs, sub, re, false)
}

// NewRegexWithRefs is like NewRegex but allows regexp iterator to match IRIs and BNodes.
//
// Consider using it carefully. In most cases it's better to reconsider
// your graph structure instead of relying on slow unoptimizable regexp.
//
// An example of incorrect usage is to match IRIs:
// 	<http://example.org/page>
// 	<http://example.org/page/foo>
// Via regexp like:
//	http://example.org/page.*
//
// The right way is to explicitly link graph nodes and query them by this relation:
// 	<http://example.org/page/foo> <type> <http://example.org/page>
func NewRegexWithRefs(sub graph.Iterator, re *regexp.Regexp, qs graph.Namer) graph.Iterator {
	return newRegex(qs, sub, re, true)
}
<commit_msg>Apply regex filter to lang strings as well (#834)<commit_after>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package iterator

import (
	"regexp"

	"github.com/cayleygraph/cayley/graph"
	"github.com/cayleygraph/quad"
)

func newRegex(qs graph.Namer, sub graph.Iterator, re *regexp.Regexp, refs bool) graph.Iterator {
	return NewValueFilter(qs, sub, func(v quad.Value) (bool, error) {
		switch v := v.(type) {
		case quad.String:
			return re.MatchString(string(v)), nil
		case quad.LangString:
			return re.MatchString(string(v.Value)), nil
		case quad.TypedString:
			return re.MatchString(string(v.Value)), nil
		default:
			if refs {
				switch v := v.(type) {
				case quad.BNode:
					return re.MatchString(string(v)), nil
				case quad.IRI:
					return re.MatchString(string(v)), nil
				}
			}
		}
		return false, nil
	})
}

// NewRegex returns an unary operator -- a filter across the values in the relevant
// subiterator. It works similarly to gremlin's filter{it.matches('exp')},
// reducing the iterator set to values whose string representation passes a
// regular expression test.
func NewRegex(sub graph.Iterator, re *regexp.Regexp, qs graph.Namer) graph.Iterator {
	return newRegex(qs, sub, re, false)
}

// NewRegexWithRefs is like NewRegex but allows regexp iterator to match IRIs and BNodes.
//
// Consider using it carefully. In most cases it's better to reconsider
// your graph structure instead of relying on slow unoptimizable regexp.
//
// An example of incorrect usage is to match IRIs:
// 	<http://example.org/page>
// 	<http://example.org/page/foo>
// Via regexp like:
//	http://example.org/page.*
//
// The right way is to explicitly link graph nodes and query them by this relation:
// 	<http://example.org/page/foo> <type> <http://example.org/page>
func NewRegexWithRefs(sub graph.Iterator, re *regexp.Regexp, qs graph.Namer) graph.Iterator {
	return newRegex(qs, sub, re, true)
}
<|endoftext|>


<commit_before>package integration_test

import (
	"path/filepath"
	"testing"
	"time"

	"github.com/cloudfoundry/libbuildpack/cutlass"
	"github.com/sclevine/spec"

	. "github.com/onsi/gomega"
)

func testMultipleProjects(t *testing.T, context spec.G, it spec.S) {
	var (
		Expect = NewWithT(t).Expect
		app    *cutlass.App
	)

	it.Before(func() {
		app = cutlass.New(filepath.Join(settings.FixturesPath, "source_apps", "multiple_projects_msbuild"))
	})

	it.After(func() {
		app = DestroyApp(t, app)
	})

	it("compiles both apps", func() {
		PushAppAndConfirm(t, app)
		Expect(app.GetBody("/")).To(ContainSubstring("Hello, I'm a string!"))
		Eventually(app.Stdout.String, 10*time.Second).Should(ContainSubstring("Hello from a secondary project!"))
	})

	context("Deploying a self-contained solution with multiple projects", func() {
		it.Before(func() {
			app = cutlass.New(filepath.Join(settings.FixturesPath, "self_contained_apps", "self_contained_solution_2.2"))
		})
		it("can run the app", func() {
			PushAppAndConfirm(t, app)
			Expect(app.GetBody("/")).To(ContainSubstring("Hello World!"))
		})
	})
}
<commit_msg>add eventually to multiple_projects_test<commit_after>package integration_test

import (
	"path/filepath"
	"testing"
	"time"

	"github.com/cloudfoundry/libbuildpack/cutlass"
	"github.com/sclevine/spec"

	. "github.com/onsi/gomega"
)

func testMultipleProjects(t *testing.T, context spec.G, it spec.S) {
	var (
		Expect     = NewWithT(t).Expect
		Eventually = NewWithT(t).Eventually
		app        *cutlass.App
	)

	it.Before(func() {
		app = cutlass.New(filepath.Join(settings.FixturesPath, "source_apps", "multiple_projects_msbuild"))
	})

	it.After(func() {
		app = DestroyApp(t, app)
	})

	it("compiles both apps", func() {
		PushAppAndConfirm(t, app)
		Expect(app.GetBody("/")).To(ContainSubstring("Hello, I'm a string!"))
		Eventually(app.Stdout.String, 10*time.Second).Should(ContainSubstring("Hello from a secondary project!"))
	})

	context("Deploying a self-contained solution with multiple projects", func() {
		it.Before(func() {
			app = cutlass.New(filepath.Join(settings.FixturesPath, "self_contained_apps", "self_contained_solution_2.2"))
		})
		it("can run the app", func() {
			PushAppAndConfirm(t, app)
			Expect(app.GetBody("/")).To(ContainSubstring("Hello World!"))
		})
	})
}
<|endoftext|>