<commit_before>package imagick

import (
	"errors"
	"fmt"
	"io/ioutil"
	"strings"

	"github.com/gographics/imagick/imagick"
	"github.com/pressly/imgry"
)

var (
	ErrEngineReleased = errors.New("imagick: engine has been released.")
	ErrEngineFailure  = errors.New("imagick: unable to request a MagickWand")
)

type Engine struct{}

func (ng Engine) Version() string {
	v, _ := imagick.GetVersion()
	return fmt.Sprintf("%s", v)
}

func (ng Engine) LoadFile(filename string, srcFormat ...string) (imgry.Image, error) {
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return ng.LoadBlob(b, srcFormat...)
}

func (ng Engine) LoadBlob(b []byte, srcFormat ...string) (imgry.Image, error) {
	if len(b) == 0 {
		return nil, imgry.ErrInvalidImageData
	}

	mw := imagick.NewMagickWand()
	if !mw.IsVerified() {
		return nil, ErrEngineFailure
	}

	// Offer a hint to the decoder of the file format
	if len(srcFormat) > 0 {
		f := srcFormat[0]
		if f != "" {
			mw.SetFormat(f)
		}
	}

	err := mw.ReadImageBlob(b)
	if err != nil {
		mw.Destroy()
		return nil, imgry.ErrInvalidImageData
	}

	// TODO: perhaps we pass the engine instance like Image{engine: i}
	im := &Image{mw: mw, data: b}
	if err := im.sync(); err != nil {
		return nil, err
	}

	return im, nil
}

func (ng Engine) GetImageInfo(b []byte, srcFormat ...string) (*imgry.ImageInfo, error) {
	if len(b) == 0 {
		return nil, imgry.ErrInvalidImageData
	}

	mw := imagick.NewMagickWand()
	if !mw.IsVerified() {
		return nil, ErrEngineFailure
	}
	defer mw.Destroy()

	err := mw.PingImageBlob(b)
	if err != nil {
		return nil, imgry.ErrInvalidImageData
	}

	w, h := int(mw.GetImageWidth()), int(mw.GetImageHeight())
	ar := float64(int(float64(w)/float64(h)*10000)) / 10000

	format := strings.ToLower(mw.GetImageFormat())
	if format == "jpeg" {
		format = "jpg"
	}

	imfo := &imgry.ImageInfo{
		Format: format, Width: w, Height: h,
		AspectRatio: ar, ContentLength: len(b),
	}

	return imfo, nil
}

type Image struct {
	mw *imagick.MagickWand

	data   []byte
	width  int
	height int
	format string
}

func (i *Image) Data() []byte {
	return i.data
}

func (i *Image) Width() int {
	return i.width
}

func (i *Image) Height() int {
	return i.height
}

func (i *Image) Format() string {
	return i.format
}

func (i *Image) SetFormat(format string) error {
	if i.Released() {
		return ErrEngineReleased
	}
	if err := i.mw.SetImageFormat(format); err != nil {
		return err
	}
	if err := i.sync(); err != nil {
		return err
	}
	return nil
}

func (i *Image) Released() bool {
	return i.mw == nil
}

func (i *Image) Release() {
	if i.mw != nil {
		i.mw.Destroy()
		i.mw = nil
	}
}

func (i *Image) Clone() imgry.Image {
	i2 := &Image{}
	i2.data = i.data
	i2.width = i.width
	i2.height = i.height
	i2.format = i.format
	if i.mw != nil && i.mw.IsVerified() {
		i2.mw = i.mw.Clone()
	}
	return i2
}

func (i *Image) SizeIt(sz *imgry.Sizing) error {
	if i.Released() {
		return ErrEngineReleased
	}

	if err := i.sizeFrames(sz); err != nil {
		return err
	}

	if sz.Format != "" {
		if err := i.mw.SetFormat(sz.Format); err != nil {
			return err
		}
	}

	if sz.Quality > 0 {
		if err := i.mw.SetImageCompressionQuality(uint(sz.Quality)); err != nil {
			return err
		}
	}

	if err := i.sync(sz.Flatten); err != nil {
		return err
	}

	return nil
}

func (i *Image) sizeFrames(sz *imgry.Sizing) error {
	// Shortcut if there is nothing to size
	if sz.Size.Equal(imgry.ZeroRect) && sz.CropBox.Equal(imgry.ZeroFloatingRect) {
		return nil
	}

	// TODO: instead of Coalesce, better to change the offsets.
	// This is required for animated image resizing
	if !sz.Flatten && i.mw.GetNumberImages() > 1 {
		i.mw = i.mw.CoalesceImages()
	}

	i.mw.SetFirstIterator()
	for n := true; n; n = i.mw.NextImage() {

		pw, ph := int(i.mw.GetImageWidth()), int(i.mw.GetImageHeight())
		srcSize := imgry.NewRect(pw, ph)

		// Initial crop of the source image
		cropBox, cropOrigin, err := sz.CalcCropBox(srcSize)
		if err != nil {
			return err
		}
		if cropBox != nil && cropOrigin != nil && !cropBox.Equal(imgry.ZeroRect) {
			err := i.mw.CropImage(uint(cropBox.Width), uint(cropBox.Height), cropOrigin.X, cropOrigin.Y)
			if err != nil {
				return err
			}
			srcSize = cropBox
			i.mw.ResetImagePage("")
		}

		// Resize the image
		resizeRect, cropBox, cropOrigin := sz.CalcResizeRect(srcSize)
		if resizeRect != nil && !resizeRect.Equal(imgry.ZeroRect) {
			err := i.mw.ResizeImage(uint(resizeRect.Width), uint(resizeRect.Height), imagick.FILTER_LANCZOS, 1.0)
			if err != nil {
				return err
			}
			i.mw.ResetImagePage("")
		}

		// Perform any final crops from an operation
		if cropBox != nil && cropOrigin != nil && !cropBox.Equal(imgry.ZeroRect) {
			err := i.mw.CropImage(uint(cropBox.Width), uint(cropBox.Height), cropOrigin.X, cropOrigin.Y)
			if err != nil {
				return err
			}
			i.mw.ResetImagePage("")
		}

		if sz.Flatten {
			break
		}
	}

	return nil
}

func (i *Image) WriteToFile(fn string) error {
	err := ioutil.WriteFile(fn, i.Data(), 0664)
	return err
}

func (i *Image) sync(optFlatten ...bool) error {
	if i.Released() {
		return ErrEngineReleased
	}

	var flatten bool
	if len(optFlatten) > 0 {
		flatten = optFlatten[0]
	}

	if flatten {
		i.data = i.mw.GetImageBlob()
	} else {
		i.data = i.mw.GetImagesBlob()
	}

	i.width = int(i.mw.GetImageWidth())
	i.height = int(i.mw.GetImageHeight())

	i.format = strings.ToLower(i.mw.GetImageFormat())
	if i.format == "jpeg" {
		i.format = "jpg"
	}

	return nil
}
<commit_msg>Destroy MagickWand on error<commit_after>package imagick

import (
	"errors"
	"fmt"
	"io/ioutil"
	"strings"

	"github.com/gographics/imagick/imagick"
	"github.com/pressly/imgry"
)

var (
	ErrEngineReleased = errors.New("imagick: engine has been released.")
	ErrEngineFailure  = errors.New("imagick: unable to request a MagickWand")
)

type Engine struct{}

func (ng Engine) Version() string {
	v, _ := imagick.GetVersion()
	return fmt.Sprintf("%s", v)
}

func (ng Engine) LoadFile(filename string, srcFormat ...string) (imgry.Image, error) {
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return ng.LoadBlob(b, srcFormat...)
}

func (ng Engine) LoadBlob(b []byte, srcFormat ...string) (imgry.Image, error) {
	if len(b) == 0 {
		return nil, imgry.ErrInvalidImageData
	}

	mw := imagick.NewMagickWand()
	if !mw.IsVerified() {
		return nil, ErrEngineFailure
	}

	// Offer a hint to the decoder of the file format
	if len(srcFormat) > 0 {
		f := srcFormat[0]
		if f != "" {
			mw.SetFormat(f)
		}
	}

	err := mw.ReadImageBlob(b)
	if err != nil {
		mw.Destroy()
		return nil, imgry.ErrInvalidImageData
	}

	// TODO: perhaps we pass the engine instance like Image{engine: i}
	im := &Image{mw: mw, data: b}
	if err := im.sync(); err != nil {
		mw.Destroy()
		return nil, err
	}

	return im, nil
}

func (ng Engine) GetImageInfo(b []byte, srcFormat ...string) (*imgry.ImageInfo, error) {
	if len(b) == 0 {
		return nil, imgry.ErrInvalidImageData
	}

	mw := imagick.NewMagickWand()
	if !mw.IsVerified() {
		return nil, ErrEngineFailure
	}
	defer mw.Destroy()

	err := mw.PingImageBlob(b)
	if err != nil {
		return nil, imgry.ErrInvalidImageData
	}

	w, h := int(mw.GetImageWidth()), int(mw.GetImageHeight())
	ar := float64(int(float64(w)/float64(h)*10000)) / 10000

	format := strings.ToLower(mw.GetImageFormat())
	if format == "jpeg" {
		format = "jpg"
	}

	imfo := &imgry.ImageInfo{
		Format: format, Width: w, Height: h,
		AspectRatio: ar, ContentLength: len(b),
	}

	return imfo, nil
}

type Image struct {
	mw *imagick.MagickWand

	data   []byte
	width  int
	height int
	format string
}

func (i *Image) Data() []byte {
	return i.data
}

func (i *Image) Width() int {
	return i.width
}

func (i *Image) Height() int {
	return i.height
}

func (i *Image) Format() string {
	return i.format
}

func (i *Image) SetFormat(format string) error {
	if i.Released() {
		return ErrEngineReleased
	}
	if err := i.mw.SetImageFormat(format); err != nil {
		return err
	}
	if err := i.sync(); err != nil {
		return err
	}
	return nil
}

func (i *Image) Released() bool {
	return i.mw == nil
}

func (i *Image) Release() {
	if i.mw != nil {
		i.mw.Destroy()
		i.mw = nil
	}
}

func (i *Image) Clone() imgry.Image {
	i2 := &Image{}
	i2.data = i.data
	i2.width = i.width
	i2.height = i.height
	i2.format = i.format
	if i.mw != nil && i.mw.IsVerified() {
		i2.mw = i.mw.Clone()
	}
	return i2
}

func (i *Image) SizeIt(sz *imgry.Sizing) error {
	if i.Released() {
		return ErrEngineReleased
	}

	if err := i.sizeFrames(sz); err != nil {
		return err
	}

	if sz.Format != "" {
		if err := i.mw.SetFormat(sz.Format); err != nil {
			return err
		}
	}

	if sz.Quality > 0 {
		if err := i.mw.SetImageCompressionQuality(uint(sz.Quality)); err != nil {
			return err
		}
	}

	if err := i.sync(sz.Flatten); err != nil {
		return err
	}

	return nil
}

func (i *Image) sizeFrames(sz *imgry.Sizing) error {
	// Shortcut if there is nothing to size
	if sz.Size.Equal(imgry.ZeroRect) && sz.CropBox.Equal(imgry.ZeroFloatingRect) {
		return nil
	}

	// TODO: instead of Coalesce, better to change the offsets.
	// This is required for animated image resizing
	if !sz.Flatten && i.mw.GetNumberImages() > 1 {
		i.mw = i.mw.CoalesceImages()
	}

	i.mw.SetFirstIterator()
	for n := true; n; n = i.mw.NextImage() {

		pw, ph := int(i.mw.GetImageWidth()), int(i.mw.GetImageHeight())
		srcSize := imgry.NewRect(pw, ph)

		// Initial crop of the source image
		cropBox, cropOrigin, err := sz.CalcCropBox(srcSize)
		if err != nil {
			return err
		}
		if cropBox != nil && cropOrigin != nil && !cropBox.Equal(imgry.ZeroRect) {
			err := i.mw.CropImage(uint(cropBox.Width), uint(cropBox.Height), cropOrigin.X, cropOrigin.Y)
			if err != nil {
				return err
			}
			srcSize = cropBox
			i.mw.ResetImagePage("")
		}

		// Resize the image
		resizeRect, cropBox, cropOrigin := sz.CalcResizeRect(srcSize)
		if resizeRect != nil && !resizeRect.Equal(imgry.ZeroRect) {
			err := i.mw.ResizeImage(uint(resizeRect.Width), uint(resizeRect.Height), imagick.FILTER_LANCZOS, 1.0)
			if err != nil {
				return err
			}
			i.mw.ResetImagePage("")
		}

		// Perform any final crops from an operation
		if cropBox != nil && cropOrigin != nil && !cropBox.Equal(imgry.ZeroRect) {
			err := i.mw.CropImage(uint(cropBox.Width), uint(cropBox.Height), cropOrigin.X, cropOrigin.Y)
			if err != nil {
				return err
			}
			i.mw.ResetImagePage("")
		}

		if sz.Flatten {
			break
		}
	}

	return nil
}

func (i *Image) WriteToFile(fn string) error {
	err := ioutil.WriteFile(fn, i.Data(), 0664)
	return err
}

func (i *Image) sync(optFlatten ...bool) error {
	if i.Released() {
		return ErrEngineReleased
	}

	var flatten bool
	if len(optFlatten) > 0 {
		flatten = optFlatten[0]
	}

	if flatten {
		i.data = i.mw.GetImageBlob()
	} else {
		i.data = i.mw.GetImagesBlob()
	}

	i.width = int(i.mw.GetImageWidth())
	i.height = int(i.mw.GetImageHeight())

	i.format = strings.ToLower(i.mw.GetImageFormat())
	if i.format == "jpeg" {
		i.format = "jpg"
	}

	return nil
}
<|endoftext|>


<commit_before>package etcd

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"
)

var (
	curlChan chan string
)

// SetCurlChan sets a channel to which cURL commands which can be used to
// re-produce requests are sent.  This is useful for debugging.
func SetCurlChan(c chan string) {
	curlChan = c
}

func ClearCurlChan() {
	curlChan = nil
}

// get issues a GET request
func (c *Client) get(key string, options options) (*RawResponse, error) {
	logger.Debugf("get %s [%s]", key, c.cluster.Leader)

	p := path.Join("keys", key)
	// If consistency level is set to STRONG, append
	// the `consistent` query string.
	if c.config.Consistency == STRONG_CONSISTENCY {
		options["consistent"] = true
	}

	str, err := options.toParameters(VALID_GET_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	resp, err := c.sendRequest("GET", p, nil)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// put issues a PUT request
func (c *Client) put(key string, value string, ttl uint64,
	options options) (*RawResponse, error) {

	logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader)
	p := path.Join("keys", key)

	str, err := options.toParameters(VALID_PUT_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	resp, err := c.sendRequest("PUT", p, buildValues(value, ttl))

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// post issues a POST request
func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error) {
	logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader)
	p := path.Join("keys", key)

	resp, err := c.sendRequest("POST", p, buildValues(value, ttl))

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// delete issues a DELETE request
func (c *Client) delete(key string, options options) (*RawResponse, error) {
	logger.Debugf("delete %s [%s]", key, c.cluster.Leader)

	p := path.Join("keys", key)

	str, err := options.toParameters(VALID_DELETE_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	resp, err := c.sendRequest("DELETE", p, nil)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// sendRequest sends a HTTP request and returns a Response as defined by etcd
func (c *Client) sendRequest(method string, relativePath string,
	values url.Values) (*RawResponse, error) {

	var req *http.Request
	var resp *http.Response
	var httpPath string
	var err error
	var b []byte

	trial := 0

	// if we connect to a follower, we will retry until we found a leader
	for {
		trial++
		logger.Debug("begin trail ", trial)
		if trial > 2*len(c.cluster.Machines) {
			return nil, fmt.Errorf("Cannot reach servers after %v time", trial)
		}

		if method == "GET" && c.config.Consistency == WEAK_CONSISTENCY {
			// If it's a GET and consistency level is set to WEAK,
			// then use a random machine.
			httpPath = c.getHttpPath(true, relativePath)
		} else {
			// Else use the leader.
			httpPath = c.getHttpPath(false, relativePath)
		}

		// Return a cURL command if curlChan is set
		if curlChan != nil {
			command := fmt.Sprintf("curl -X %s %s", method, httpPath)
			for key, value := range values {
				command += fmt.Sprintf(" -d %s=%s", key, value[0])
			}
			curlChan <- command
		}

		logger.Debug("send.request.to ", httpPath, " | method ", method)

		if values == nil {
			req, _ = http.NewRequest(method, httpPath, nil)
		} else {
			req, _ = http.NewRequest(method, httpPath,
				strings.NewReader(values.Encode()))

			req.Header.Set("Content-Type",
				"application/x-www-form-urlencoded; param=value")
		}

		// network error, change a machine!
		if resp, err = c.httpClient.Do(req); err != nil {
			c.switchLeader(trial % len(c.cluster.Machines))
			time.Sleep(time.Millisecond * 200)
			continue
		}

		if resp != nil {
			logger.Debug("recv.response.from ", httpPath)

			var ok bool
			ok, b = c.handleResp(resp)

			if !ok {
				continue
			}

			logger.Debug("recv.success.", httpPath)
			break
		}

		// should not reach here
		// err and resp should not be nil at the same time
		logger.Debug("error.from ", httpPath)
		return nil, err
	}

	r := &RawResponse{
		StatusCode: resp.StatusCode,
		Body:       b,
		Header:     resp.Header,
	}

	return r, nil
}

// handle HTTP response
// If status code is OK, read the http body.
// If status code is TemporaryRedirect, update leader.
// If status coid is InternalServerError, sleep for 200ms.
func (c *Client) handleResp(resp *http.Response) (bool, []byte) {
	defer resp.Body.Close()

	code := resp.StatusCode

	if code == http.StatusTemporaryRedirect {
		u, err := resp.Location()

		if err != nil {
			logger.Warning(err)
		} else {
			c.updateLeader(u)
		}

		return false, nil

	} else if code == http.StatusInternalServerError {
		time.Sleep(time.Millisecond * 200)

	} else if code == http.StatusOK ||
		code == http.StatusCreated ||
		code == http.StatusBadRequest {
		b, err := ioutil.ReadAll(resp.Body)

		if err != nil {
			return false, nil
		}

		return true, b
	}

	logger.Warning("bad status code ", resp.StatusCode)
	return false, nil
}

func (c *Client) getHttpPath(random bool, s ...string) string {
	var machine string
	if random {
		machine = c.cluster.Machines[rand.Intn(len(c.cluster.Machines))]
	} else {
		machine = c.cluster.Leader
	}

	fullPath := machine + "/" + version
	for _, seg := range s {
		fullPath = fullPath + "/" + seg
	}

	return fullPath
}

// buildValues builds a url.Values map according to the given value and ttl
func buildValues(value string, ttl uint64) url.Values {
	v := url.Values{}

	if value != "" {
		v.Set("value", value)
	}

	if ttl > 0 {
		v.Set("ttl", fmt.Sprintf("%v", ttl))
	}

	return v
}
<commit_msg>refactor typo<commit_after>package etcd

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"
)

var (
	curlChan chan string
)

// SetCurlChan sets a channel to which cURL commands which can be used to
// re-produce requests are sent.  This is useful for debugging.
func SetCurlChan(c chan string) {
	curlChan = c
}

func ClearCurlChan() {
	curlChan = nil
}

// get issues a GET request
func (c *Client) get(key string, options options) (*RawResponse, error) {
	logger.Debugf("get %s [%s]", key, c.cluster.Leader)

	p := path.Join("keys", key)
	// If consistency level is set to STRONG, append
	// the `consistent` query string.
	if c.config.Consistency == STRONG_CONSISTENCY {
		options["consistent"] = true
	}

	str, err := options.toParameters(VALID_GET_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	resp, err := c.sendRequest("GET", p, nil)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// put issues a PUT request
func (c *Client) put(key string, value string, ttl uint64,
	options options) (*RawResponse, error) {

	logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader)
	p := path.Join("keys", key)

	str, err := options.toParameters(VALID_PUT_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	resp, err := c.sendRequest("PUT", p, buildValues(value, ttl))

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// post issues a POST request
func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error) {
	logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader)
	p := path.Join("keys", key)

	resp, err := c.sendRequest("POST", p, buildValues(value, ttl))

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// delete issues a DELETE request
func (c *Client) delete(key string, options options) (*RawResponse, error) {
	logger.Debugf("delete %s [%s]", key, c.cluster.Leader)

	p := path.Join("keys", key)

	str, err := options.toParameters(VALID_DELETE_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	resp, err := c.sendRequest("DELETE", p, nil)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// sendRequest sends a HTTP request and returns a Response as defined by etcd
func (c *Client) sendRequest(method string, relativePath string,
	values url.Values) (*RawResponse, error) {

	var req *http.Request
	var resp *http.Response
	var httpPath string
	var err error
	var b []byte

	trial := 0

	// if we connect to a follower, we will retry until we found a leader
	for {
		trial++
		logger.Debug("begin trail ", trial)
		if trial > 2*len(c.cluster.Machines) {
			return nil, fmt.Errorf("Cannot reach servers after %v time", trial)
		}

		if method == "GET" && c.config.Consistency == WEAK_CONSISTENCY {
			// If it's a GET and consistency level is set to WEAK,
			// then use a random machine.
			httpPath = c.getHttpPath(true, relativePath)
		} else {
			// Else use the leader.
			httpPath = c.getHttpPath(false, relativePath)
		}

		// Return a cURL command if curlChan is set
		if curlChan != nil {
			command := fmt.Sprintf("curl -X %s %s", method, httpPath)
			for key, value := range values {
				command += fmt.Sprintf(" -d %s=%s", key, value[0])
			}
			curlChan <- command
		}

		logger.Debug("send.request.to ", httpPath, " | method ", method)

		if values == nil {
			req, _ = http.NewRequest(method, httpPath, nil)
		} else {
			req, _ = http.NewRequest(method, httpPath,
				strings.NewReader(values.Encode()))

			req.Header.Set("Content-Type",
				"application/x-www-form-urlencoded; param=value")
		}

		// network error, change a machine!
		if resp, err = c.httpClient.Do(req); err != nil {
			c.switchLeader(trial % len(c.cluster.Machines))
			time.Sleep(time.Millisecond * 200)
			continue
		}

		if resp != nil {
			logger.Debug("recv.response.from ", httpPath)

			var ok bool
			ok, b = c.handleResp(resp)

			if !ok {
				continue
			}

			logger.Debug("recv.success.", httpPath)
			break
		}

		// should not reach here
		// err and resp should not be nil at the same time
		logger.Debug("error.from ", httpPath)
		return nil, err
	}

	r := &RawResponse{
		StatusCode: resp.StatusCode,
		Body:       b,
		Header:     resp.Header,
	}

	return r, nil
}

// handle HTTP response
// If status code is OK, read the http body.
// If status code is TemporaryRedirect, update leader.
// If status code is InternalServerError, sleep for 200ms.
func (c *Client) handleResp(resp *http.Response) (bool, []byte) {
	defer resp.Body.Close()

	code := resp.StatusCode

	if code == http.StatusTemporaryRedirect {
		u, err := resp.Location()

		if err != nil {
			logger.Warning(err)
		} else {
			c.updateLeader(u)
		}

		return false, nil

	} else if code == http.StatusInternalServerError {
		time.Sleep(time.Millisecond * 200)

	} else if code == http.StatusOK ||
		code == http.StatusCreated ||
		code == http.StatusBadRequest {
		b, err := ioutil.ReadAll(resp.Body)

		if err != nil {
			return false, nil
		}

		return true, b
	}

	logger.Warning("bad status code ", resp.StatusCode)
	return false, nil
}

func (c *Client) getHttpPath(random bool, s ...string) string {
	var machine string
	if random {
		machine = c.cluster.Machines[rand.Intn(len(c.cluster.Machines))]
	} else {
		machine = c.cluster.Leader
	}

	fullPath := machine + "/" + version
	for _, seg := range s {
		fullPath = fullPath + "/" + seg
	}

	return fullPath
}

// buildValues builds a url.Values map according to the given value and ttl
func buildValues(value string, ttl uint64) url.Values {
	v := url.Values{}

	if value != "" {
		v.Set("value", value)
	}

	if ttl > 0 {
		v.Set("ttl", fmt.Sprintf("%v", ttl))
	}

	return v
}
<|endoftext|>


<commit_before>package htlcswitch

import (
	"sync"
	"sync/atomic"

	"github.com/lightningnetwork/lnd/lnwire"
)

// packetQueue is a goroutine-safe queue of htlc packets which over flow the
// current commitment transaction. An HTLC will overflow the current commitment
// transaction if one attempts to add a new HTLC to the state machine which
// already has the max number of pending HTLC's present on the commitment
// transaction.  Packets are removed from the queue by the channelLink itself
// as additional slots become available on the commitment transaction itself.
// In order to synchronize properly we use a semaphore to allow the channelLink
// to signal the number of slots available, and a condition variable to allow
// the packetQueue to know when new items have been added to the queue.
type packetQueue struct {
	// totalHtlcAmt is the sum of the value of all pending HTLC's currently
	// residing within the overflow queue. This value should only read or
	// modified *atomically*.
	totalHtlcAmt int64

	// queueLen is an internal counter that reflects the size of the queue
	// at any given instance. This value is intended to be use atomically
	// as this value is used by internal methods to obtain the length of
	// the queue w/o grabbing the main lock. This allows callers to avoid a
	// deadlock situation where the main goroutine is attempting a send
	// with the lock held.
	queueLen int32

	queue []*htlcPacket

	wg sync.WaitGroup

	// freeSlots serves as a semaphore who's current value signals the
	// number of available slots on the commitment transaction.
	freeSlots chan struct{}

	queueCond *sync.Cond
	queueMtx  sync.Mutex

	// outgoingPkts is a channel that the channelLink will receive on in
	// order to drain the packetQueue as new slots become available on the
	// commitment transaction.
	outgoingPkts chan *htlcPacket

	quit chan struct{}
}

// newPacketQueue returns a new instance of the packetQueue. The maxFreeSlots
// value should reflect the max number of HTLC's that we're allowed to have
// outstanding within the commitment transaction.
func newPacketQueue(maxFreeSlots int) *packetQueue {
	p := &packetQueue{
		outgoingPkts: make(chan *htlcPacket),
		freeSlots:    make(chan struct{}, maxFreeSlots),
		quit:         make(chan struct{}),
	}
	p.queueCond = sync.NewCond(&p.queueMtx)

	return p
}

// Start starts all goroutines that packetQueue needs to perform its normal
// duties.
func (p *packetQueue) Start() {
	p.wg.Add(1)
	go p.packetCoordinator()
}

// Stop signals the packetQueue for a graceful shutdown, and waits for all
// goroutines to exit.
func (p *packetQueue) Stop() {
	close(p.quit)

	p.queueCond.Signal()
}

// packetCoordinator is a goroutine that handles the packet overflow queue.
// Using a synchronized queue, outside callers are able to append to the end of
// the queue, waking up the coordinator when the queue transitions from empty
// to non-empty. The packetCoordinator will then aggressively try to empty out
// the queue, passing new htlcPackets to the channelLink as slots within the
// commitment transaction become available.
//
// Future iterations of the packetCoordinator will implement congestion
// avoidance logic in the face of persistent htlcPacket back-pressure.
//
// TODO(roasbeef): later will need to add back pressure handling heuristics
// like reg congestion avoidance:
//   * random dropping, RED, etc
func (p *packetQueue) packetCoordinator() {
	defer p.wg.Done()

	for {
		// First, we'll check our condition. If the queue of packets is
		// empty, then we'll wait until a new item is added.
		p.queueCond.L.Lock()
		for len(p.queue) == 0 {
			p.queueCond.Wait()

			// If we were woke up in order to exit, then we'll do
			// so. Otherwise, we'll check the message queue for any
			// new items.
			select {
			case <-p.quit:
				p.queueCond.L.Unlock()
				return
			default:
			}
		}

		nextPkt := p.queue[0]

		p.queueCond.L.Unlock()

		// If there aren't any further messages to sent (or the link
		// didn't immediately read our message), then we'll block and
		// wait for a new message to be sent into the overflow queue,
		// or for the link's htlcForwarder to wake up.
		select {
		case <-p.freeSlots:

			select {
			case p.outgoingPkts <- nextPkt:
				// Pop the item off the front of the queue and
				// slide down the reference one to re-position
				// the head pointer. This will set us up for
				// the next iteration.  If the queue is empty
				// at this point, then we'll block at the top.
				p.queueCond.L.Lock()
				p.queue[0] = nil
				p.queue = p.queue[1:]
				atomic.AddInt32(&p.queueLen, -1)
				atomic.AddInt64(&p.totalHtlcAmt, int64(-nextPkt.amount))
				p.queueCond.L.Unlock()
			case <-p.quit:
				return
			}

		case <-p.quit:
			return

		default:
		}
	}
}

// AddPkt adds the referenced packet to the overflow queue, preserving ordering
// of the existing items.
func (p *packetQueue) AddPkt(pkt *htlcPacket) {
	// First, we'll lock the condition, and add the message to the end of
	// the message queue, and increment the internal atomic for tracking
	// the queue's length.
	p.queueCond.L.Lock()
	p.queue = append(p.queue, pkt)
	atomic.AddInt32(&p.queueLen, 1)
	atomic.AddInt64(&p.totalHtlcAmt, int64(pkt.amount))
	p.queueCond.L.Unlock()

	// With the message added, we signal to the msgConsumer that there are
	// additional messages to consume.
	p.queueCond.Signal()
}

// SignalFreeSlot signals to the queue that a new slot has opened up within the
// commitment transaction. The max amount of free slots has been defined when
// initially creating the packetQueue itself. This method, combined with AddPkt
// creates the following abstraction: a synchronized queue of infinite length
// which can be added to at will, which flows onto a commitment of fixed
// capacity.
func (p *packetQueue) SignalFreeSlot() {
	// We'll only send over a free slot signal if the queue *is not* empty.
	// Otherwise, it's possible that we attempt to overfill the free slots
	// semaphore and block indefinitely below.
	if atomic.LoadInt32(&p.queueLen) == 0 {
		return
	}

	select {
	case p.freeSlots <- struct{}{}:
	case <-p.quit:
		return
	}
}

// Length returns the number of pending htlc packets present within the over
// flow queue.
func (p *packetQueue) Length() int32 {
	return atomic.LoadInt32(&p.queueLen)
}

// TotalHtlcAmount is the total amount (in mSAT) of all HTLC's currently
// residing within the overflow queue.
func (p *packetQueue) TotalHtlcAmount() lnwire.MilliSatoshi {
	// TODO(roasbeef): also factor in fee rate?
	return lnwire.MilliSatoshi(atomic.LoadInt64(&p.totalHtlcAmt))
}
<commit_msg>htlcswitch: ensure the packet queue exits on stop<commit_after>package htlcswitch

import (
	"sync"
	"sync/atomic"
	"time"

	"github.com/lightningnetwork/lnd/lnwire"
)

// packetQueue is a goroutine-safe queue of htlc packets which over flow the
// current commitment transaction. An HTLC will overflow the current commitment
// transaction if one attempts to add a new HTLC to the state machine which
// already has the max number of pending HTLC's present on the commitment
// transaction.  Packets are removed from the queue by the channelLink itself
// as additional slots become available on the commitment transaction itself.
// In order to synchronize properly we use a semaphore to allow the channelLink
// to signal the number of slots available, and a condition variable to allow
// the packetQueue to know when new items have been added to the queue.
type packetQueue struct {
	// totalHtlcAmt is the sum of the value of all pending HTLC's currently
	// residing within the overflow queue. This value should only read or
	// modified *atomically*.
	totalHtlcAmt int64 // To be used atomically.

	// queueLen is an internal counter that reflects the size of the queue
	// at any given instance. This value is intended to be use atomically
	// as this value is used by internal methods to obtain the length of
	// the queue w/o grabbing the main lock. This allows callers to avoid a
	// deadlock situation where the main goroutine is attempting a send
	// with the lock held.
	queueLen int32 // To be used atomically.

	streamShutdown int32 // To be used atomically.

	queue []*htlcPacket

	wg sync.WaitGroup

	// freeSlots serves as a semaphore who's current value signals the
	// number of available slots on the commitment transaction.
	freeSlots chan struct{}

	queueCond *sync.Cond
	queueMtx  sync.Mutex

	// outgoingPkts is a channel that the channelLink will receive on in
	// order to drain the packetQueue as new slots become available on the
	// commitment transaction.
	outgoingPkts chan *htlcPacket

	quit chan struct{}
}

// newPacketQueue returns a new instance of the packetQueue. The maxFreeSlots
// value should reflect the max number of HTLC's that we're allowed to have
// outstanding within the commitment transaction.
func newPacketQueue(maxFreeSlots int) *packetQueue {
	p := &packetQueue{
		outgoingPkts: make(chan *htlcPacket),
		freeSlots:    make(chan struct{}, maxFreeSlots),
		quit:         make(chan struct{}),
	}
	p.queueCond = sync.NewCond(&p.queueMtx)

	return p
}

// Start starts all goroutines that packetQueue needs to perform its normal
// duties.
func (p *packetQueue) Start() {
	p.wg.Add(1)
	go p.packetCoordinator()
}

// Stop signals the packetQueue for a graceful shutdown, and waits for all
// goroutines to exit.
func (p *packetQueue) Stop() {
	close(p.quit)

	// Now that we've closed the channel, we'll repeatedly signal the msg
	// consumer until we've detected that it has exited.
	for atomic.LoadInt32(&p.streamShutdown) == 0 {
		p.queueCond.Signal()
		time.Sleep(time.Millisecond * 100)
	}
}

// packetCoordinator is a goroutine that handles the packet overflow queue.
// Using a synchronized queue, outside callers are able to append to the end of
// the queue, waking up the coordinator when the queue transitions from empty
// to non-empty. The packetCoordinator will then aggressively try to empty out
// the queue, passing new htlcPackets to the channelLink as slots within the
// commitment transaction become available.
//
// Future iterations of the packetCoordinator will implement congestion
// avoidance logic in the face of persistent htlcPacket back-pressure.
//
// TODO(roasbeef): later will need to add back pressure handling heuristics
// like reg congestion avoidance:
//   * random dropping, RED, etc
func (p *packetQueue) packetCoordinator() {
	defer atomic.StoreInt32(&p.streamShutdown, 1)

	for {
		// First, we'll check our condition. If the queue of packets is
		// empty, then we'll wait until a new item is added.
		p.queueCond.L.Lock()
		for len(p.queue) == 0 {
			p.queueCond.Wait()

			// If we were woke up in order to exit, then we'll do
			// so. Otherwise, we'll check the message queue for any
			// new items.
			select {
			case <-p.quit:
				p.queueCond.L.Unlock()
				return
			default:
			}
		}

		nextPkt := p.queue[0]

		p.queueCond.L.Unlock()

		// If there aren't any further messages to sent (or the link
		// didn't immediately read our message), then we'll block and
		// wait for a new message to be sent into the overflow queue,
		// or for the link's htlcForwarder to wake up.
		select {
		case <-p.freeSlots:

			select {
			case p.outgoingPkts <- nextPkt:
				// Pop the item off the front of the queue and
				// slide down the reference one to re-position
				// the head pointer. This will set us up for
				// the next iteration.  If the queue is empty
				// at this point, then we'll block at the top.
				p.queueCond.L.Lock()
				p.queue[0] = nil
				p.queue = p.queue[1:]
				atomic.AddInt32(&p.queueLen, -1)
				atomic.AddInt64(&p.totalHtlcAmt, int64(-nextPkt.amount))
				p.queueCond.L.Unlock()
			case <-p.quit:
				return
			}

		case <-p.quit:
			return

		default:
		}
	}
}

// AddPkt adds the referenced packet to the overflow queue, preserving ordering
// of the existing items.
func (p *packetQueue) AddPkt(pkt *htlcPacket) {
	// First, we'll lock the condition, and add the message to the end of
	// the message queue, and increment the internal atomic for tracking
	// the queue's length.
	p.queueCond.L.Lock()
	p.queue = append(p.queue, pkt)
	atomic.AddInt32(&p.queueLen, 1)
	atomic.AddInt64(&p.totalHtlcAmt, int64(pkt.amount))
	p.queueCond.L.Unlock()

	// With the message added, we signal to the msgConsumer that there are
	// additional messages to consume.
	p.queueCond.Signal()
}

// SignalFreeSlot signals to the queue that a new slot has opened up within the
// commitment transaction. The max amount of free slots has been defined when
// initially creating the packetQueue itself. This method, combined with AddPkt
// creates the following abstraction: a synchronized queue of infinite length
// which can be added to at will, which flows onto a commitment of fixed
// capacity.
func (p *packetQueue) SignalFreeSlot() {
	// We'll only send over a free slot signal if the queue *is not* empty.
	// Otherwise, it's possible that we attempt to overfill the free slots
	// semaphore and block indefinitely below.
	if atomic.LoadInt32(&p.queueLen) == 0 {
		return
	}

	select {
	case p.freeSlots <- struct{}{}:
	case <-p.quit:
		return
	}
}

// Length returns the number of pending htlc packets present within the over
// flow queue.
func (p *packetQueue) Length() int32 {
	return atomic.LoadInt32(&p.queueLen)
}

// TotalHtlcAmount is the total amount (in mSAT) of all HTLC's currently
// residing within the overflow queue.
func (p *packetQueue) TotalHtlcAmount() lnwire.MilliSatoshi {
	// TODO(roasbeef): also factor in fee rate?
	return lnwire.MilliSatoshi(atomic.LoadInt64(&p.totalHtlcAmt))
}
<|endoftext|>


<commit_before>package ec2

import (
	"fmt"
	"launchpad.net/goamz/ec2"
	"launchpad.net/goamz/s3"
	"launchpad.net/juju/go/environs"
	"launchpad.net/juju/go/state"
	"sync"
)

const zkPortSuffix = ":2181"

func init() {
	environs.RegisterProvider("ec2", environProvider{})
}

type environProvider struct{}

var _ environs.EnvironProvider = environProvider{}

type environ struct {
	name             string
	config           *providerConfig
	ec2              *ec2.EC2
	s3               *s3.S3
	checkBucket      sync.Once
	checkBucketError error
}

var _ environs.Environ = (*environ)(nil)

type instance struct {
	*ec2.Instance
}

func (inst *instance) String() string {
	return inst.Id()
}

var _ environs.Instance = (*instance)(nil)

func (inst *instance) Id() string {
	return inst.InstanceId
}

func (inst *instance) DNSName() string {
	return inst.Instance.DNSName
}

func (environProvider) Open(name string, config interface{}) (e environs.Environ, err error) {
	cfg := config.(*providerConfig)
	if Regions[cfg.region].EC2Endpoint == "" {
		return nil, fmt.Errorf("no ec2 endpoint found for region %q, opening %q", cfg.region, name)
	}
	return &environ{
		name:   name,
		config: cfg,
		ec2:    ec2.New(cfg.auth, Regions[cfg.region]),
		s3:     s3.New(cfg.auth, Regions[cfg.region]),
	}, nil
}

func (e *environ) Bootstrap() error {
	_, err := e.loadState()
	if err == nil {
		return fmt.Errorf("environment is already bootstrapped")
	}
	if s3err, _ := err.(*s3.Error); s3err != nil && s3err.StatusCode != 404 {
		return err
	}
	inst, err := e.startInstance(0, nil, true)
	if err != nil {
		return fmt.Errorf("cannot start bootstrap instance: %v", err)
	}
	err = e.saveState(&bootstrapState{
		ZookeeperInstances: []string{inst.Id()},
	})
	if err != nil {
		// ignore error on StopInstance because the previous error is
		// more important.
		e.StopInstances([]environs.Instance{inst})
		return err
	}
	// TODO make safe in the case of racing Bootstraps
	// If two Bootstraps are called concurrently, there's
	// no way to use S3 to make sure that only one succeeds.
	// Perhaps consider using SimpleDB for state storage
	// which would enable that possibility.

	return nil
}

func (e *environ) StateInfo() (*state.Info, error) {
	st, err := e.loadState()
	if err != nil {
		return nil, err
	}
	resp, err := e.ec2.Instances(st.ZookeeperInstances, nil)
	if err != nil {
		return nil, fmt.Errorf("cannot list instances: %v", err)
	}
	var insts []environs.Instance
	for i := range resp.Reservations {
		r := &resp.Reservations[i]
		for j := range r.Instances {
			insts = append(insts, &instance{&r.Instances[j]})
		}
	}

	addrs := make([]string, len(insts))
	for i, inst := range insts {
		addr := inst.DNSName()
		if addr == "" {
			return nil, fmt.Errorf("zookeeper instance %q does not yet have a DNS address", inst.Id())
		}
		addrs[i] = addr + zkPortSuffix
	}
	return &state.Info{Addrs: addrs}, nil
}

func (e *environ) StartInstance(machineId int, info *state.Info) (environs.Instance, error) {
	return e.startInstance(machineId, info, false)
}

// startInstance is the internal version of StartInstance, used by Bootstrap
// as well as via StartInstance itself. If master is true, a bootstrap
// instance will be started.
func (e *environ) startInstance(machineId int, info *state.Info, master bool) (environs.Instance, error) {
	image, err := FindImageSpec(DefaultImageConstraint)
	if err != nil {
		return nil, fmt.Errorf("cannot find image: %v", err)
	}
	groups, err := e.setUpGroups(machineId)
	if err != nil {
		return nil, fmt.Errorf("cannot set up groups: %v", err)
	}
	instances, err := e.ec2.RunInstances(&ec2.RunInstances{
		ImageId:        image.ImageId,
		MinCount:       1,
		MaxCount:       1,
		UserData:       nil,
		InstanceType:   "m1.small",
		SecurityGroups: groups,
	})
	if err != nil {
		return nil, fmt.Errorf("cannot run instances: %v", err)
	}
	if len(instances.Instances) != 1 {
		return nil, fmt.Errorf("expected 1 started instance, got %d", len(instances.Instances))
	}
	return &instance{&instances.Instances[0]}, nil
}

func (e *environ) StopInstances(insts []environs.Instance) error {
	if len(insts) == 0 {
		return nil
	}
	names := make([]string, len(insts))
	for i, inst := range insts {
		names[i] = inst.(*instance).InstanceId
	}
	_, err := e.ec2.TerminateInstances(names)
	return err
}

func (e *environ) Instances() ([]environs.Instance, error) {
	filter := ec2.NewFilter()
	filter.Add("instance-state-name", "pending", "running")
	filter.Add("group-name", e.groupName())

	resp, err := e.ec2.Instances(nil, filter)
	if err != nil {
		return nil, err
	}
	var insts []environs.Instance
	for i := range resp.Reservations {
		r := &resp.Reservations[i]
		for j := range r.Instances {
			insts = append(insts, &instance{&r.Instances[j]})
		}
	}
	return insts, nil
}

func (e *environ) Destroy() error {
	insts, err := e.Instances()
	if err != nil {
		return err
	}
	err = e.StopInstances(insts)
	if err != nil {
		return err
	}
	return e.deleteState()
}

func (e *environ) machineGroupName(machineId int) string {
	return fmt.Sprintf("%s-%d", e.groupName(), machineId)
}

func (e *environ) groupName() string {
	return "juju-" + e.name
}

// setUpGroups creates the security groups for the new machine, and
// returns them.
//
// Instances are tagged with a group so they can be distinguished from
// other instances that might be running on the same EC2 account.  In
// addition, a specific machine security group is created for each
// machine, so that its firewall rules can be configured per machine.
func (e *environ) setUpGroups(machineId int) ([]ec2.SecurityGroup, error) {
	jujuGroup := ec2.SecurityGroup{Name: e.groupName()}
	jujuMachineGroup := ec2.SecurityGroup{Name: e.machineGroupName(machineId)}

	f := ec2.NewFilter()
	f.Add("group-name", jujuGroup.Name, jujuMachineGroup.Name)
	groups, err := e.ec2.SecurityGroups(nil, f)
	if err != nil {
		return nil, fmt.Errorf("cannot get security groups: %v", err)
	}

	for _, g := range groups.Groups {
		switch g.Name {
		case jujuGroup.Name:
			jujuGroup = g.SecurityGroup
		case jujuMachineGroup.Name:
			jujuMachineGroup = g.SecurityGroup
		}
	}

	// Create the provider group if doesn't exist.
	if jujuGroup.Id == "" {
		r, err := e.ec2.CreateSecurityGroup(jujuGroup.Name, "juju group for "+e.name)
		if err != nil {
			return nil, fmt.Errorf("cannot create juju security group: %v", err)
		}
		jujuGroup = r.SecurityGroup
	}

	// Create the machine-specific group, but first see if there's
	// one already existing from a previous machine launch;
	// if so, delete it, since it can have the wrong firewall setup
	if jujuMachineGroup.Id != "" {
		_, err := e.ec2.DeleteSecurityGroup(jujuMachineGroup)
		if err != nil {
			return nil, fmt.Errorf("cannot delete old security group %q: %v", jujuMachineGroup.Name, err)
		}
	}
	descr := fmt.Sprintf("juju group for %s machine %d", e.name, machineId)
	r, err := e.ec2.CreateSecurityGroup(jujuMachineGroup.Name, descr)
	if err != nil {
		return nil, fmt.Errorf("cannot create machine group %q: %v", jujuMachineGroup.Name, err)
	}
	return []ec2.SecurityGroup{jujuGroup, r.SecurityGroup}, nil
}
<commit_msg>add TODO<commit_after>package ec2

import (
	"fmt"
	"launchpad.net/goamz/ec2"
	"launchpad.net/goamz/s3"
	"launchpad.net/juju/go/environs"
	"launchpad.net/juju/go/state"
	"sync"
)

const zkPortSuffix = ":2181"

func init() {
	environs.RegisterProvider("ec2", environProvider{})
}

type environProvider struct{}

var _ environs.EnvironProvider = environProvider{}

type environ struct {
	name             string
	config           *providerConfig
	ec2              *ec2.EC2
	s3               *s3.S3
	checkBucket      sync.Once
	checkBucketError error
}

var _ environs.Environ = (*environ)(nil)

type instance struct {
	*ec2.Instance
}

func (inst *instance) String() string {
	return inst.Id()
}

var _ environs.Instance = (*instance)(nil)

func (inst *instance) Id() string {
	return inst.InstanceId
}

func (inst *instance) DNSName() string {
	return inst.Instance.DNSName
}

func (environProvider) Open(name string, config interface{}) (e environs.Environ, err error) {
	cfg := config.(*providerConfig)
	if Regions[cfg.region].EC2Endpoint == "" {
		return nil, fmt.Errorf("no ec2 endpoint found for region %q, opening %q", cfg.region, name)
	}
	return &environ{
		name:   name,
		config: cfg,
		ec2:    ec2.New(cfg.auth, Regions[cfg.region]),
		s3:     s3.New(cfg.auth, Regions[cfg.region]),
	}, nil
}

func (e *environ) Bootstrap() error {
	_, err := e.loadState()
	if err == nil {
		return fmt.Errorf("environment is already bootstrapped")
	}
	if s3err, _ := err.(*s3.Error); s3err != nil && s3err.StatusCode != 404 {
		return err
	}
	inst, err := e.startInstance(0, nil, true)
	if err != nil {
		return fmt.Errorf("cannot start bootstrap instance: %v", err)
	}
	err = e.saveState(&bootstrapState{
		ZookeeperInstances: []string{inst.Id()},
	})
	if err != nil {
		// ignore error on StopInstance because the previous error is
		// more important.
		e.StopInstances([]environs.Instance{inst})
		return err
	}
	// TODO make safe in the case of racing Bootstraps
	// If two Bootstraps are called concurrently, there's
	// no way to use S3 to make sure that only one succeeds.
	// Perhaps consider using SimpleDB for state storage
	// which would enable that possibility.

	return nil
}

func (e *environ) StateInfo() (*state.Info, error) {
	st, err := e.loadState()
	if err != nil {
		return nil, err
	}
	resp, err := e.ec2.Instances(st.ZookeeperInstances, nil)
	if err != nil {
		return nil, fmt.Errorf("cannot list instances: %v", err)
	}
	var insts []environs.Instance
	for i := range resp.Reservations {
		r := &resp.Reservations[i]
		for j := range r.Instances {
			insts = append(insts, &instance{&r.Instances[j]})
		}
	}

	addrs := make([]string, len(insts))
	for i, inst := range insts {
		// TODO make this poll until the DNSName becomes available.
		addr := inst.DNSName()
		if addr == "" {
			return nil, fmt.Errorf("zookeeper instance %q does not yet have a DNS address", inst.Id())
		}
		addrs[i] = addr + zkPortSuffix
	}
	return &state.Info{Addrs: addrs}, nil
}

func (e *environ) StartInstance(machineId int, info *state.Info) (environs.Instance, error) {
	return e.startInstance(machineId, info, false)
}

// startInstance is the internal version of StartInstance, used by Bootstrap
// as well as via StartInstance itself. If master is true, a bootstrap
// instance will be started.
func (e *environ) startInstance(machineId int, info *state.Info, master bool) (environs.Instance, error) {
	image, err := FindImageSpec(DefaultImageConstraint)
	if err != nil {
		return nil, fmt.Errorf("cannot find image: %v", err)
	}
	groups, err := e.setUpGroups(machineId)
	if err != nil {
		return nil, fmt.Errorf("cannot set up groups: %v", err)
	}
	instances, err := e.ec2.RunInstances(&ec2.RunInstances{
		ImageId:        image.ImageId,
		MinCount:       1,
		MaxCount:       1,
		UserData:       nil,
		InstanceType:   "m1.small",
		SecurityGroups: groups,
	})
	if err != nil {
		return nil, fmt.Errorf("cannot run instances: %v", err)
	}
	if len(instances.Instances) != 1 {
		return nil, fmt.Errorf("expected 1 started instance, got %d", len(instances.Instances))
	}
	return &instance{&instances.Instances[0]}, nil
}

func (e *environ) StopInstances(insts []environs.Instance) error {
	if len(insts) == 0 {
		return nil
	}
	names := make([]string, len(insts))
	for i, inst := range insts {
		names[i] = inst.(*instance).InstanceId
	}
	_, err := e.ec2.TerminateInstances(names)
	return err
}

func (e *environ) Instances() ([]environs.Instance, error) {
	filter := ec2.NewFilter()
	filter.Add("instance-state-name", "pending", "running")
	filter.Add("group-name", e.groupName())

	resp, err := e.ec2.Instances(nil, filter)
	if err != nil {
		return nil, err
	}
	var insts []environs.Instance
	for i := range resp.Reservations {
		r := &resp.Reservations[i]
		for j := range r.Instances {
			insts = append(insts, &instance{&r.Instances[j]})
		}
	}
	return insts, nil
}

func (e *environ) Destroy() error {
	insts, err := e.Instances()
	if err != nil {
		return err
	}
	err = e.StopInstances(insts)
	if err != nil {
		return err
	}
	return e.deleteState()
}

func (e *environ) machineGroupName(machineId int) string {
	return fmt.Sprintf("%s-%d", e.groupName(), machineId)
}

func (e *environ) groupName() string {
	return "juju-" + e.name
}

// setUpGroups creates the security groups for the new machine, and
// returns them.
//
// Instances are tagged with a group so they can be distinguished from
// other instances that might be running on the same EC2 account.  In
// addition, a specific machine security group is created for each
// machine, so that its firewall rules can be configured per machine.
func (e *environ) setUpGroups(machineId int) ([]ec2.SecurityGroup, error) {
	jujuGroup := ec2.SecurityGroup{Name: e.groupName()}
	jujuMachineGroup := ec2.SecurityGroup{Name: e.machineGroupName(machineId)}

	f := ec2.NewFilter()
	f.Add("group-name", jujuGroup.Name, jujuMachineGroup.Name)
	groups, err := e.ec2.SecurityGroups(nil, f)
	if err != nil {
		return nil, fmt.Errorf("cannot get security groups: %v", err)
	}

	for _, g := range groups.Groups {
		switch g.Name {
		case jujuGroup.Name:
			jujuGroup = g.SecurityGroup
		case jujuMachineGroup.Name:
			jujuMachineGroup = g.SecurityGroup
		}
	}

	// Create the provider group if doesn't exist.
	if jujuGroup.Id == "" {
		r, err := e.ec2.CreateSecurityGroup(jujuGroup.Name, "juju group for "+e.name)
		if err != nil {
			return nil, fmt.Errorf("cannot create juju security group: %v", err)
		}
		jujuGroup = r.SecurityGroup
	}

	// Create the machine-specific group, but first see if there's
	// one already existing from a previous machine launch;
	// if so, delete it, since it can have the wrong firewall setup
	if jujuMachineGroup.Id != "" {
		_, err := e.ec2.DeleteSecurityGroup(jujuMachineGroup)
		if err != nil {
			return nil, fmt.Errorf("cannot delete old security group %q: %v", jujuMachineGroup.Name, err)
		}
	}
	descr := fmt.Sprintf("juju group for %s machine %d", e.name, machineId)
	r, err := e.ec2.CreateSecurityGroup(jujuMachineGroup.Name, descr)
	if err != nil {
		return nil, fmt.Errorf("cannot create machine group %q: %v", jujuMachineGroup.Name, err)
	}
	return []ec2.SecurityGroup{jujuGroup, r.SecurityGroup}, nil
}
<|endoftext|>


<commit_before>/*
	Copyright (c) 2014-2015, Percona LLC and/or its affiliates. All rights reserved.

	This program is free software: you can redistribute it and/or modify
	it under the terms of the GNU Affero General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU Affero General Public License for more details.

	You should have received a copy of the GNU Affero General Public License
	along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package event

import (
	"time"

	"github.com/percona/go-mysql/log"
)

// A Result contains a global class and per-ID classes with finalized metric
// statistics. The classes are keyed on class ID.
type Result struct {
	Global    *Class            // all classes
	Class     map[string]*Class // keyed on class ID
	RateLimit uint
	Error     string
}

// An Aggregator groups events by class ID. When there are no more events,
// a call to Finalize computes all metric statistics and returns a Result.
type Aggregator struct {
	samples     bool
	utcOffset   time.Duration
	outlierTime float64
	// --
	global    *Class
	classes   map[string]*Class
	rateLimit uint
}

// NewAggregator returns a new Aggregator.
func NewAggregator(samples bool, utcOffset time.Duration, outlierTime float64) *Aggregator {
	a := &Aggregator{
		samples:     samples,
		utcOffset:   utcOffset,
		outlierTime: outlierTime,
		// --
		global:  NewClass("", "", false),
		classes: make(map[string]*Class),
	}
	return a
}

// AddEvent adds the event to the aggregator, automatically creating new classes
// as needed.
func (a *Aggregator) AddEvent(event *log.Event, id, fingerprint string) {
	if a.rateLimit != event.RateLimit {
		a.rateLimit = event.RateLimit
	}

	outlier := false
	if a.outlierTime > 0 && event.TimeMetrics["Query_time"] > a.outlierTime {
		outlier = true
	}

	a.global.AddEvent(event, outlier)

	class, ok := a.classes[id]
	if !ok {
		class = NewClass(id, fingerprint, a.samples)
		a.classes[id] = class
	}
	class.AddEvent(event, outlier)
}

// Finalize calculates all metric statistics and returns a Result.
// Call this function when done adding events to the aggregator.
func (a *Aggregator) Finalize() Result {
	a.global.Finalize(a.rateLimit)
	a.global.UniqueQueries = uint(len(a.classes))
	for _, class := range a.classes {
		class.Finalize(a.rateLimit)
		class.UniqueQueries = 1
		if class.Example != nil && class.Example.Ts != "" {
			if t, err := time.Parse("060102 15:04:05", class.Example.Ts); err != nil {
				class.Example.Ts = ""
			} else {
				class.Example.Ts = t.Add(a.utcOffset).Format("2006-01-02 15:04:05")
			}
		}
	}
	return Result{
		Global:    a.global,
		Class:     a.classes,
		RateLimit: a.rateLimit,
	}
}
<commit_msg>Just a comment.<commit_after>/*
	Copyright (c) 2014-2015, Percona LLC and/or its affiliates. All rights reserved.

	This program is free software: you can redistribute it and/or modify
	it under the terms of the GNU Affero General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU Affero General Public License for more details.

	You should have received a copy of the GNU Affero General Public License
	along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package event

import (
	"time"

	"github.com/percona/go-mysql/log"
)

// A Result contains a global class and per-ID classes with finalized metric
// statistics. The classes are keyed on class ID.
type Result struct {
	Global    *Class            // all classes
	Class     map[string]*Class // keyed on class ID
	RateLimit uint
	Error     string
}

// An Aggregator groups events by class ID. When there are no more events,
// a call to Finalize computes all metric statistics and returns a Result.
type Aggregator struct {
	samples     bool
	utcOffset   time.Duration
	outlierTime float64
	// --
	global    *Class
	classes   map[string]*Class
	rateLimit uint
}

// NewAggregator returns a new Aggregator.
// outlierTime is https://www.percona.com/doc/percona-server/5.5/diagnostics/slow_extended_55.html#slow_query_log_always_write_time
func NewAggregator(samples bool, utcOffset time.Duration, outlierTime float64) *Aggregator {
	a := &Aggregator{
		samples:     samples,
		utcOffset:   utcOffset,
		outlierTime: outlierTime,
		// --
		global:  NewClass("", "", false),
		classes: make(map[string]*Class),
	}
	return a
}

// AddEvent adds the event to the aggregator, automatically creating new classes
// as needed.
func (a *Aggregator) AddEvent(event *log.Event, id, fingerprint string) {
	if a.rateLimit != event.RateLimit {
		a.rateLimit = event.RateLimit
	}

	outlier := false
	if a.outlierTime > 0 && event.TimeMetrics["Query_time"] > a.outlierTime {
		outlier = true
	}

	a.global.AddEvent(event, outlier)

	class, ok := a.classes[id]
	if !ok {
		class = NewClass(id, fingerprint, a.samples)
		a.classes[id] = class
	}
	class.AddEvent(event, outlier)
}

// Finalize calculates all metric statistics and returns a Result.
// Call this function when done adding events to the aggregator.
func (a *Aggregator) Finalize() Result {
	a.global.Finalize(a.rateLimit)
	a.global.UniqueQueries = uint(len(a.classes))
	for _, class := range a.classes {
		class.Finalize(a.rateLimit)
		class.UniqueQueries = 1
		if class.Example != nil && class.Example.Ts != "" {
			if t, err := time.Parse("060102 15:04:05", class.Example.Ts); err != nil {
				class.Example.Ts = ""
			} else {
				class.Example.Ts = t.Add(a.utcOffset).Format("2006-01-02 15:04:05")
			}
		}
	}
	return Result{
		Global:    a.global,
		Class:     a.classes,
		RateLimit: a.rateLimit,
	}
}
<|endoftext|>


<commit_before>//
// Written by Michael Mattioli
//

package main

import (
    "fmt"
    "strconv"
    "bytes"
)

// LargestPalindrome returns the largest integer that is a palindrome and has two factors both of
// string length n.
func LargestPalindrome(n int) int {

    isPalindrome := func(n int) bool {
        ten := 1
        for t := n; t != 0; t /= 10 {
            ten *= 10
        }
        ten /= 10
        for n != 0 {
            if (n / ten) != (n % 10) {
                return false
            }
            n %= ten
            n /= 10
            ten /= 100
        }
        return true
    }

    startingFactor := func(n int) int {
        var b bytes.Buffer
        for i := 0; i < n; i++ {
            b.WriteString("9")
        }
        f, _ := strconv.Atoi(b.String())
        return f
    }

    sf := startingFactor(n)

    var f1, f2 int
    for i := sf; len(strconv.Itoa(i)) == n; i-- {
        for j := sf; len(strconv.Itoa(j)) == n; j-- {
            if isPalindrome(i * j) && (i * j) > (f1 * f2) {
                f1, f2 = i, j
            }
        }
    }

    return f1 * f2

}

func main() {
    fmt.Println(LargestPalindrome(1))
    fmt.Println(LargestPalindrome(2))
    fmt.Println(LargestPalindrome(3))
    fmt.Println(LargestPalindrome(4))
    fmt.Println(LargestPalindrome(5))
    fmt.Println(LargestPalindrome(6))
}
<commit_msg>Optimize digit counting in challenge 322<commit_after>//
// Written by Michael Mattioli
//

package main

import (
    "fmt"
    "strconv"
    "bytes"
)

// LargestPalindrome returns the largest integer that is a palindrome and has two factors both of
// string length n.
func LargestPalindrome(n int) int {

    // Determine if an integer is a palindrome.
    isPalindrome := func(n int) bool {
        ten := 1
        for t := n; t != 0; t /= 10 {
            ten *= 10
        }
        ten /= 10
        for n != 0 {
            if (n / ten) != (n % 10) {
                return false
            }
            n %= ten
            n /= 10
            ten /= 100
        }
        return true
    }

    // Since we're looking for the largest number, we'll start with the largest possible factor(s).
    startingFactor := func(n int) int {
        var b bytes.Buffer
        for i := 0; i < n; i++ {
            b.WriteString("9")
        }
        f, _ := strconv.Atoi(b.String())
        return f
    }

    // Count the number of digits in an integer.
    digits := func(n int) int {
        var d int
        for n != 0 {
            n /= 10
            d++
        }
        return d
    }

    sf := startingFactor(n)
    var f1, f2 int
    for i := sf; digits(i) == n; i-- {
        for j := sf; digits(j) == n; j-- {
            if isPalindrome(i * j) && (i * j) > (f1 * f2) {
                f1, f2 = i, j
            }
        }
    }

    return f1 * f2

}

func main() {
    fmt.Println(LargestPalindrome(2))
    fmt.Println(LargestPalindrome(3))
    fmt.Println(LargestPalindrome(4))
    fmt.Println(LargestPalindrome(5))
    fmt.Println(LargestPalindrome(6))
}
<|endoftext|>


<commit_before>package s3api

import (
	"context"
	"encoding/xml"
	"fmt"
	"math"
	"net/http"
	"time"

	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)

type ListAllMyBucketsResult struct {
	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult"`
	Owner   *s3.Owner
	Buckets []*s3.Bucket `xml:"Buckets>Bucket"`
}

func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {

	var response ListAllMyBucketsResult

	entries, _, err := s3a.list(s3a.option.BucketsPath, "", "", false, math.MaxInt32)

	if err != nil {
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	identityId := r.Header.Get(xhttp.AmzIdentityId)

	var buckets []*s3.Bucket
	for _, entry := range entries {
		if entry.IsDirectory {
			if id, ok := entry.Extended[xhttp.AmzIdentityId]; ok {
				if identityId != string(id) {
					continue
				}
			}
			buckets = append(buckets, &s3.Bucket{
				Name:         aws.String(entry.Name),
				CreationDate: aws.Time(time.Unix(entry.Attributes.Crtime, 0).UTC()),
			})
		}
	}

	response = ListAllMyBucketsResult{
		Owner: &s3.Owner{
			ID:          aws.String(identityId),
			DisplayName: aws.String(identityId),
		},
		Buckets: buckets,
	}

	writeSuccessResponseXML(w, encodeResponse(response))
}

func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) {

	bucket, _ := getBucketAndObject(r)

	// avoid duplicated buckets
	errCode := s3err.ErrNone
	if err := s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
		if resp, err := client.CollectionList(context.Background(), &filer_pb.CollectionListRequest{
			IncludeEcVolumes:     true,
			IncludeNormalVolumes: true,
		}); err != nil {
			glog.Errorf("list collection: %v", err)
			return fmt.Errorf("list collections: %v", err)
		} else {
			for _, c := range resp.Collections {
				if bucket == c.Name {
					errCode = s3err.ErrBucketAlreadyExists
					break
				}
			}
		}
		return nil
	}); err != nil {
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}
	if exist, err := s3a.exists(s3a.option.BucketsPath, bucket, true); err == nil && exist {
		errCode = s3err.ErrBucketAlreadyExists
	}
	if errCode != s3err.ErrNone {
		writeErrorResponse(w, errCode, r.URL)
		return
	}

	fn := func(entry *filer_pb.Entry) {
		if identityId := r.Header.Get(xhttp.AmzIdentityId); identityId != "" {
			if entry.Extended == nil {
				entry.Extended = make(map[string][]byte)
			}
			entry.Extended[xhttp.AmzIdentityId] = []byte(identityId)
		}
	}

	// create the folder for bucket, but lazily create actual collection
	if err := s3a.mkdir(s3a.option.BucketsPath, bucket, fn); err != nil {
		glog.Errorf("PutBucketHandler mkdir: %v", err)
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	writeSuccessResponseEmpty(w)
}

func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {

	bucket, _ := getBucketAndObject(r)

	if entry, err := s3a.get(s3a.option.BucketsPath, bucket); entry != nil && err == nil {
		if id, ok := entry.Extended[xhttp.AmzIdentityId]; ok {
			if string(id) != r.Header.Get(xhttp.AmzIdentityId) {
				writeErrorResponse(w, s3err.ErrAccessDenied, r.URL)
				return
			}
		}
	}

	err := s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		// delete collection
		deleteCollectionRequest := &filer_pb.DeleteCollectionRequest{
			Collection: bucket,
		}

		glog.V(1).Infof("delete collection: %v", deleteCollectionRequest)
		if _, err := client.DeleteCollection(context.Background(), deleteCollectionRequest); err != nil {
			return fmt.Errorf("delete collection %s: %v", bucket, err)
		}

		return nil
	})

	err = s3a.rm(s3a.option.BucketsPath, bucket, false, true)

	if err != nil {
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	writeResponse(w, http.StatusNoContent, nil, mimeNone)
}

func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {

	bucket, _ := getBucketAndObject(r)

	entry, err := s3a.get(s3a.option.BucketsPath, bucket)
	if entry == nil || err != nil {
		writeErrorResponse(w, s3err.ErrNoSuchBucket, r.URL)
		return
	}
	if id, ok := entry.Extended[xhttp.AmzIdentityId]; ok {
		if string(id) != r.Header.Get(xhttp.AmzIdentityId) {
			writeErrorResponse(w, s3err.ErrAccessDenied, r.URL)
			return
		}
	}

	writeSuccessResponseEmpty(w)
}
<commit_msg>return NoSuchBucket instead of InternalError delete non-existed bucket.<commit_after>package s3api

import (
	"context"
	"encoding/xml"
	"fmt"
	"math"
	"net/http"
	"time"

	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)

type ListAllMyBucketsResult struct {
	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult"`
	Owner   *s3.Owner
	Buckets []*s3.Bucket `xml:"Buckets>Bucket"`
}

func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {

	var response ListAllMyBucketsResult

	entries, _, err := s3a.list(s3a.option.BucketsPath, "", "", false, math.MaxInt32)

	if err != nil {
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	identityId := r.Header.Get(xhttp.AmzIdentityId)

	var buckets []*s3.Bucket
	for _, entry := range entries {
		if entry.IsDirectory {
			if id, ok := entry.Extended[xhttp.AmzIdentityId]; ok {
				if identityId != string(id) {
					continue
				}
			}
			buckets = append(buckets, &s3.Bucket{
				Name:         aws.String(entry.Name),
				CreationDate: aws.Time(time.Unix(entry.Attributes.Crtime, 0).UTC()),
			})
		}
	}

	response = ListAllMyBucketsResult{
		Owner: &s3.Owner{
			ID:          aws.String(identityId),
			DisplayName: aws.String(identityId),
		},
		Buckets: buckets,
	}

	writeSuccessResponseXML(w, encodeResponse(response))
}

func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) {

	bucket, _ := getBucketAndObject(r)

	// avoid duplicated buckets
	errCode := s3err.ErrNone
	if err := s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
		if resp, err := client.CollectionList(context.Background(), &filer_pb.CollectionListRequest{
			IncludeEcVolumes:     true,
			IncludeNormalVolumes: true,
		}); err != nil {
			glog.Errorf("list collection: %v", err)
			return fmt.Errorf("list collections: %v", err)
		} else {
			for _, c := range resp.Collections {
				if bucket == c.Name {
					errCode = s3err.ErrBucketAlreadyExists
					break
				}
			}
		}
		return nil
	}); err != nil {
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}
	if exist, err := s3a.exists(s3a.option.BucketsPath, bucket, true); err == nil && exist {
		errCode = s3err.ErrBucketAlreadyExists
	}
	if errCode != s3err.ErrNone {
		writeErrorResponse(w, errCode, r.URL)
		return
	}

	fn := func(entry *filer_pb.Entry) {
		if identityId := r.Header.Get(xhttp.AmzIdentityId); identityId != "" {
			if entry.Extended == nil {
				entry.Extended = make(map[string][]byte)
			}
			entry.Extended[xhttp.AmzIdentityId] = []byte(identityId)
		}
	}

	// create the folder for bucket, but lazily create actual collection
	if err := s3a.mkdir(s3a.option.BucketsPath, bucket, fn); err != nil {
		glog.Errorf("PutBucketHandler mkdir: %v", err)
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	writeSuccessResponseEmpty(w)
}

func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {

	bucket, _ := getBucketAndObject(r)

	entry, err := s3a.get(s3a.option.BucketsPath, bucket)
	if entry == nil || err == filer_pb.ErrNotFound {
		writeErrorResponse(w, s3err.ErrNoSuchBucket, r.URL)
		return
	}
	if id, ok := entry.Extended[xhttp.AmzIdentityId]; ok {
		if string(id) != r.Header.Get(xhttp.AmzIdentityId) {
			writeErrorResponse(w, s3err.ErrAccessDenied, r.URL)
			return
		}
	}

	err = s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		// delete collection
		deleteCollectionRequest := &filer_pb.DeleteCollectionRequest{
			Collection: bucket,
		}

		glog.V(1).Infof("delete collection: %v", deleteCollectionRequest)
		if _, err := client.DeleteCollection(context.Background(), deleteCollectionRequest); err != nil {
			return fmt.Errorf("delete collection %s: %v", bucket, err)
		}

		return nil
	})

	err = s3a.rm(s3a.option.BucketsPath, bucket, false, true)

	if err != nil {
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	writeResponse(w, http.StatusNoContent, nil, mimeNone)
}

func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {

	bucket, _ := getBucketAndObject(r)

	entry, err := s3a.get(s3a.option.BucketsPath, bucket)
	if entry == nil || err != nil {
		writeErrorResponse(w, s3err.ErrNoSuchBucket, r.URL)
		return
	}
	if id, ok := entry.Extended[xhttp.AmzIdentityId]; ok {
		if string(id) != r.Header.Get(xhttp.AmzIdentityId) {
			writeErrorResponse(w, s3err.ErrAccessDenied, r.URL)
			return
		}
	}

	writeSuccessResponseEmpty(w)
}
<|endoftext|>


<commit_before>package money

import (
	"fmt"
	"math/big"

	"github.com/juju/errors"
)

// Money represents a monetary value
type Money struct {
	rat *big.Rat
}

// New creates a new instance with a zero value.
func New() *Money {
	return &Money{
		rat: big.NewRat(0, 1),
	}
}

// Parse a string to create a new money value.
func Parse(s string) (*Money, error) {
	rat := new(big.Rat)
	if _, err := fmt.Sscan(s, rat); err != nil {
		return nil, errors.Trace(err)
	}

	return &Money{rat}, nil
}

// Format the money value with a specific deciaml precision.
func (money *Money) Format(prec int) string {
	return money.rat.FloatString(prec)
}

// Mul multiplies the money value n times and returns the result.
func (money *Money) Mul(n int64) *Money {
	b := big.NewRat(n, 1)
	result := New()
	result.rat.Mul(money.rat, b)
	return result
}

// Add two money values together and returns the result.
func (money *Money) Add(other *Money) *Money {
	result := New()
	result.rat.Add(money.rat, other.rat)
	return result
}

// Sub subtracts two money values and returns the result.
func (money *Money) Sub(other *Money) *Money {
	result := New()
	result.rat.Sub(money.rat, other.rat)
	return result
}

// Div divides two money values and returns the result.
func (money *Money) Div(other *Money) *Money {
	result := New()
	result.rat.Quo(money.rat, other.rat)
	return result
}

// LessThan returns true if a money value is less than the other.
func (money *Money) LessThan(other *Money) bool {
	return money.rat.Cmp(other.rat) == -1
}
<commit_msg>Parse formats with pauses instead of dots. Function to calc the cents of a money value.<commit_after>package money

import (
	"fmt"
	"math/big"
	"strings"

	"github.com/juju/errors"
)

// Money represents a monetary value
type Money struct {
	rat *big.Rat
}

// New creates a new instance with a zero value.
func New() *Money {
	return &Money{
		rat: big.NewRat(0, 1),
	}
}

// Parse a string to create a new money value. It can read `XX.YY` and `XX,YY`.
func Parse(s string) (*Money, error) {
	s = strings.Replace(s, ",", ".", -1)

	rat := new(big.Rat)
	if _, err := fmt.Sscan(s, rat); err != nil {
		return nil, errors.Trace(err)
	}

	return &Money{rat}, nil
}

// Cents returns the value with cents precision (2 decimal places) as a number.
func (money *Money) Cents() int64 {
	cents := big.NewInt(100)

	v := money.rat.Num()
	v.Mul(v, cents)
	v.Quo(v, money.rat.Denom())

	return v.Int64()
}

// Format the money value with a specific decimal precision.
func (money *Money) Format(prec int) string {
	return money.rat.FloatString(prec)
}

// Mul multiplies the money value n times and returns the result.
func (money *Money) Mul(n int64) *Money {
	b := big.NewRat(n, 1)
	result := New()
	result.rat.Mul(money.rat, b)
	return result
}

// Add two money values together and returns the result.
func (money *Money) Add(other *Money) *Money {
	result := New()
	result.rat.Add(money.rat, other.rat)
	return result
}

// Sub subtracts two money values and returns the result.
func (money *Money) Sub(other *Money) *Money {
	result := New()
	result.rat.Sub(money.rat, other.rat)
	return result
}

// Div divides two money values and returns the result.
func (money *Money) Div(other *Money) *Money {
	result := New()
	result.rat.Quo(money.rat, other.rat)
	return result
}

// LessThan returns true if a money value is less than the other.
func (money *Money) LessThan(other *Money) bool {
	return money.rat.Cmp(other.rat) == -1
}
<|endoftext|>


<commit_before>package rest

import (
	"encoding/json"
	"net/url"
)

type Model interface {
	Validate() error
	MakeKey(uint64) []byte
	Filter(url.Values) bool
	Merge(Model)
	Clone() Model
}

type Marshalable interface {
	GetModel() Model
	SetModel(Model)
	Marshal() ([]byte, error)
	Unmarshal([]byte) error
	MarshalJSON() ([]byte, error)
	UnmarshalJSON([]byte) error
	Clone() Marshalable
}

type MarshalableModel struct {
	m Marshalable
}

func (m *MarshalableModel) Validate() error {
	return m.m.GetModel().Validate()
}

func (m *MarshalableModel) MakeKey(n uint64) []byte {
	return m.m.GetModel().MakeKey(n)
}

func (m *MarshalableModel) Filter(v url.Values) bool {
	return m.m.GetModel().Filter(v)
}

func (m *MarshalableModel) Merge(other MarshalableModel) {
	m.m.GetModel().Merge(other.m.GetModel())
}

func (m *MarshalableModel) Marshal() ([]byte, error) {
	return m.m.Marshal()
}

func (m *MarshalableModel) Unmarshal(data []byte) error {
	return m.m.Unmarshal(data)
}

func (m *MarshalableModel) MarshalJSON() ([]byte, error) {
	return m.m.MarshalJSON()
}

func (m *MarshalableModel) UnmarshalJSON(data []byte) error {
	return m.m.UnmarshalJSON(data)
}

func (m *MarshalableModel) Clone() MarshalableModel {
	return MarshalableModel{m: m.m.Clone()}
}

type JSONMarshalable struct {
	model Model
}

func NewJSONModel(model Model) MarshalableModel {
	return MarshalableModel{m: &JSONMarshalable{model: model}}
}

func (j *JSONMarshalable) GetModel() Model {
	return j.model
}

func (j *JSONMarshalable) SetModel(m Model) {
	j.model = m
}

func (j *JSONMarshalable) Marshal() ([]byte, error) {
	return json.Marshal(&j.model)
}

func (j *JSONMarshalable) Unmarshal(data []byte) error {
	return json.Unmarshal(data, &j.model)
}

func (j *JSONMarshalable) MarshalJSON() ([]byte, error) {
	return json.Marshal(&j.model)
}

func (j *JSONMarshalable) UnmarshalJSON(data []byte) error {
	return json.Unmarshal(data, &j.model)
}

func (j *JSONMarshalable) Clone() Marshalable {
	return &JSONMarshalable{model: j.model.Clone()}
}
<commit_msg>Add docstrings to rest/model<commit_after>package rest

import (
	"encoding/json"
	"net/url"
)

// Model defines the interface required for REST handling
type Model interface {
	Validate() error
	MakeKey(uint64) []byte
	Filter(url.Values) bool
	Merge(Model)
	Clone() Model
}

// Marshalable defines the interface required for Model marshalling
type Marshalable interface {
	GetModel() Model
	SetModel(Model)
	Marshal() ([]byte, error)
	Unmarshal([]byte) error
	MarshalJSON() ([]byte, error)
	UnmarshalJSON([]byte) error
	Clone() Marshalable
}

// MarshalableModel is the integrated type for Model and Marshalable
type MarshalableModel struct {
	m Marshalable
}

// Validate delegates the call to Marshalable.Model.Validate
func (m *MarshalableModel) Validate() error {
	return m.m.GetModel().Validate()
}

// MakeKey delegates the call to Marshalable.Model.MakeKey
func (m *MarshalableModel) MakeKey(n uint64) []byte {
	return m.m.GetModel().MakeKey(n)
}

// Filter delegates the call to Marshalable.Model.Filter
func (m *MarshalableModel) Filter(v url.Values) bool {
	return m.m.GetModel().Filter(v)
}

// Merge delegates the call to Marshalable.Model.Merge
func (m *MarshalableModel) Merge(other MarshalableModel) {
	m.m.GetModel().Merge(other.m.GetModel())
}

// Marshal delegates the call to Marshalable.Marshal
func (m *MarshalableModel) Marshal() ([]byte, error) {
	return m.m.Marshal()
}

// Unmarshal delegates the call to Marshalable.Unmarshal
func (m *MarshalableModel) Unmarshal(data []byte) error {
	return m.m.Unmarshal(data)
}

// MarshalJSON delegates the call to Marshalable.MarshalJSON
func (m *MarshalableModel) MarshalJSON() ([]byte, error) {
	return m.m.MarshalJSON()
}

// UnmarshalJSON delegates the call to Marshalable.UnmarshalJSON
func (m *MarshalableModel) UnmarshalJSON(data []byte) error {
	return m.m.UnmarshalJSON(data)
}

// Clonel delegates the call to Marshalable.Clone
func (m *MarshalableModel) Clone() MarshalableModel {
	return MarshalableModel{m: m.m.Clone()}
}

// JSONMarshalable implements Marshalable for JSON marshalling
type JSONMarshalable struct {
	model Model
}

// NewJSONModel returns a new JSONMarshalable MarshalableModel
func NewJSONModel(model Model) MarshalableModel {
	return MarshalableModel{m: &JSONMarshalable{model: model}}
}

 // GetModel returns the model in JSONMarshalable
func (j *JSONMarshalable) GetModel() Model {
	return j.model
}

// SetModel sets the model in JSONMarshalable
func (j *JSONMarshalable) SetModel(m Model) {
	j.model = m
}

// Marshal marshalls the model to JSON
func (j *JSONMarshalable) Marshal() ([]byte, error) {
	return json.Marshal(&j.model)
}

// Unmarshal unmarshalls the model from JSON
func (j *JSONMarshalable) Unmarshal(data []byte) error {
	return json.Unmarshal(data, &j.model)
}

// MarshalJSON marshalls the model to JSON
func (j *JSONMarshalable) MarshalJSON() ([]byte, error) {
	return json.Marshal(&j.model)
}

// UnmarshalJSON unmarshalls the model from JSON
func (j *JSONMarshalable) UnmarshalJSON(data []byte) error {
	return json.Unmarshal(data, &j.model)
}

// Clone will clone the JSONMarshalable instance
func (j *JSONMarshalable) Clone() Marshalable {
	return &JSONMarshalable{model: j.model.Clone()}
}
<|endoftext|>


<commit_before>package agent

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
)

type AgentConf struct {
	Key     string
	Secret  string
	Region  string
	Address string
	Port    int
	Loop    int
}

var (
	VERSION string
)

// Prepare a new agent configuration
//
// The default parameters are
//  * REGION: eu-west-1
// 	* ADDRESS: 127.0.0.1
//  * PORT: 1234
//  * LOOP: 60
func NewConf() *AgentConf {
	daemonConf := new(AgentConf)
	daemonConf.Key = os.Getenv("AWS_ACCESS_KEY_ID")

	daemonConf.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY")
	if daemonConf.Secret == "" {
		daemonConf.Secret = os.Getenv("AWS_SECRET_KEY")
	}

	daemonConf.Region = "eu-west-1"
	daemonConf.Address = "127.0.0.1"
	daemonConf.Port = 1234
	daemonConf.Loop = 60

	return daemonConf
}

// Merge default configuration with a JSON configuration
func (d *AgentConf) MergeWithFileAtPath(path string) error {
	src, err := os.Open(path)
	if err == nil {
		L.Info(fmt.Sprintf("Merge default configuration with file at path: %s", path))
		err = d.MergeWithReader(src)
		if err != nil {
			L.Warning(fmt.Sprintf("Unable to merge file %s, wrong JSON format probably", path))
		}
	} else {
		L.Warning(fmt.Sprintf("Missing file at path %s", path))
	}

	return err
}

// Merge default configuration with a JSON configuration
func (d *AgentConf) MergeWithReader(src io.Reader) error {
	file, err := ioutil.ReadAll(src)
	if err != nil {
		return err
	}

	err = json.Unmarshal(file, d)

	return err
}
<commit_msg>Minor on comments<commit_after>package agent

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
)

type AgentConf struct {
	Key     string
	Secret  string
	Region  string
	Address string
	Port    int
	Loop    int
}

var (
	VERSION string
)

// Prepare a new agent configuration
//
// The default parameters are
//  * REGION: eu-west-1
//  * ADDRESS: 127.0.0.1
//  * PORT: 1234
//  * LOOP: 60
func NewConf() *AgentConf {
	daemonConf := new(AgentConf)
	daemonConf.Key = os.Getenv("AWS_ACCESS_KEY_ID")

	daemonConf.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY")
	if daemonConf.Secret == "" {
		daemonConf.Secret = os.Getenv("AWS_SECRET_KEY")
	}

	daemonConf.Region = "eu-west-1"
	daemonConf.Address = "127.0.0.1"
	daemonConf.Port = 1234
	daemonConf.Loop = 60

	return daemonConf
}

// Merge default configuration with a JSON configuration
func (d *AgentConf) MergeWithFileAtPath(path string) error {
	src, err := os.Open(path)
	if err == nil {
		L.Info(fmt.Sprintf("Merge default configuration with file at path: %s", path))
		err = d.MergeWithReader(src)
		if err != nil {
			L.Warning(fmt.Sprintf("Unable to merge file %s, wrong JSON format probably", path))
		}
	} else {
		L.Warning(fmt.Sprintf("Missing file at path %s", path))
	}

	return err
}

// Merge default configuration with a JSON configuration
func (d *AgentConf) MergeWithReader(src io.Reader) error {
	file, err := ioutil.ReadAll(src)
	if err != nil {
		return err
	}

	err = json.Unmarshal(file, d)

	return err
}
<|endoftext|>


<commit_before>package input

import (
	"fmt"
	"testing"
	"time"

	"github.com/grafana/metrictank/cluster"
	"github.com/grafana/metrictank/conf"
	"github.com/grafana/metrictank/idx"
	"github.com/grafana/metrictank/idx/memory"
	"github.com/grafana/metrictank/mdata"
	"github.com/grafana/metrictank/mdata/cache"
	backendStore "github.com/grafana/metrictank/store"
	"github.com/raintank/schema"
)

func TestIngestValidAndInvalidTagsAndValuesWithAndWithoutRejection(t *testing.T) {
	type testCase struct {
		name                    string
		rejectInvalidTags       bool
		tags                    []string
		expectedInvalidMdInc    uint32
		expectedInvalidTagMdInc uint32
		expectedIndexSizeInc    uint32
	}

	testCases := []testCase{
		{
			name:                    "valid_with_rejection",
			rejectInvalidTags:       true,
			tags:                    []string{"valid=tag"},
			expectedInvalidMdInc:    0,
			expectedInvalidTagMdInc: 0,
			expectedIndexSizeInc:    1,
		}, {
			name:                    "valid_without_rejection",
			rejectInvalidTags:       false,
			tags:                    []string{"valid=tag"},
			expectedInvalidMdInc:    0,
			expectedInvalidTagMdInc: 0,
			expectedIndexSizeInc:    1,
		}, {
			name:                    "invalid_tags_with_rejection",
			rejectInvalidTags:       true,
			tags:                    generateInvalidTags(t),
			expectedInvalidMdInc:    1,
			expectedInvalidTagMdInc: 1,
			expectedIndexSizeInc:    0,
		}, {
			name:                    "invalid_tags_without_rejection",
			rejectInvalidTags:       false,
			tags:                    generateInvalidTags(t),
			expectedInvalidMdInc:    1,
			expectedInvalidTagMdInc: 1,
			expectedIndexSizeInc:    1,
		}, {
			name:                    "invalid_tag_values_with_rejection",
			rejectInvalidTags:       true,
			tags:                    generateInvalidTagValues(t),
			expectedInvalidMdInc:    1,
			expectedInvalidTagMdInc: 1,
			expectedIndexSizeInc:    0,
		}, {
			name:                    "invalid_tag_values_without_rejection",
			rejectInvalidTags:       false,
			tags:                    generateInvalidTagValues(t),
			expectedInvalidMdInc:    1,
			expectedInvalidTagMdInc: 1,
			expectedIndexSizeInc:    1,
		},
	}

	for _, tc := range testCases {
		handler, index, reset := getDefaultHandler(t)
		rejectInvalidTags = tc.rejectInvalidTags
		for i, tag := range tc.tags {
			data := getTestMetricData()
			data.Tags = []string{tag}
			testIngestMetricData(
				t,
				fmt.Sprintf("%s_%d", tc.name, i),
				data,
				handler,
				index,
				tc.expectedInvalidMdInc,
				tc.expectedInvalidTagMdInc,
				tc.expectedIndexSizeInc,
			)
		}
		reset()
	}
}

func generateInvalidTags(t *testing.T) []string {
	t.Helper()

	invalidChars := ";!^"
	validChar := "a"

	tagKeys := generateInvalidStrings(t, invalidChars, validChar)
	res := make([]string, 0, len(tagKeys))
	for _, tagKey := range tagKeys {
		res = append(res, fmt.Sprintf("%s=value", tagKey))
	}
	return res
}

func generateInvalidTagValues(t *testing.T) []string {
	t.Helper()

	invalidChars := ";"
	validChar := "a"

	tagValues := generateInvalidStrings(t, invalidChars, validChar)
	res := make([]string, 0, len(tagValues))
	for _, tagValue := range tagValues {
		res = append(res, fmt.Sprintf("tag=%s", tagValue))
	}

	res = append(res, fmt.Sprintf("tag=~"))
	res = append(res, fmt.Sprintf("tag=~a"))
	res = append(res, fmt.Sprintf("tag=~aa"))
	return res
}

func generateInvalidStrings(t *testing.T, invalidChars, validChar string) []string {
	t.Helper()

	// * 4 because we generate
	// - one tag which is only the invalid char we currently process
	// - one 3 letter tag which starts with the invalid char
	// - one 3 letter tag which ends with the invalid char
	// - one 3 letter tag where the invalid char is in the middle
	res := make([]string, len(invalidChars)*4)
	for i, invalidChar := range invalidChars {
		invalidCharStr := string(invalidChar)
		res[i*4] = invalidCharStr
		res[i*4+1] = invalidCharStr + validChar + validChar
		res[i*4+2] = validChar + validChar + invalidCharStr
		res[i*4+3] = validChar + invalidCharStr + validChar
	}
	return res
}

func getTestMetricData() schema.MetricData {
	return schema.MetricData{
		Id:       "1.12345678901234567890123456789012",
		OrgId:    1,
		Name:     "abc",
		Interval: 1,
		Value:    2,
		Time:     3,
		Mtype:    "gauge",
	}
}

func testIngestMetricData(t *testing.T, tc string, data schema.MetricData, handler DefaultHandler, index idx.MetricIndex, expectedInvalidMdInc, expectedInvalidTagMdInc, expectedIndexSizeInc uint32) {
	originalInvalidCnt := handler.invalidMD.Peek()
	originalInvalidTagCnt := handler.invalidTagMD.Peek()
	originalIndexSize := uint32(len(index.List(1)))
	data.SetId()
	handler.ProcessMetricData(&data, 0)

	invalidCnt := handler.invalidMD.Peek()
	if invalidCnt != originalInvalidCnt+expectedInvalidMdInc {
		t.Fatalf("TC %s: Invalid metric counter has not been updated correctly, expected %d, got %d", tc, originalInvalidCnt+expectedInvalidMdInc, invalidCnt)
	}
	invalidTagCnt := handler.invalidTagMD.Peek()

	if invalidTagCnt != originalInvalidTagCnt+expectedInvalidTagMdInc {
		t.Fatalf("TC %s: Invalid tag counter has not been updated correctly, expected %d, got %d", tc, originalInvalidTagCnt+expectedInvalidTagMdInc, invalidTagCnt)
	}

	indexSize := uint32(len(index.List(1)))
	if indexSize != originalIndexSize+expectedIndexSizeInc {
		t.Fatalf("TC %s: Index size has not been updated correctly, expected %d, got %d", tc, originalIndexSize+expectedIndexSizeInc, indexSize)
	}
}

func getDefaultHandler(t *testing.T) (DefaultHandler, idx.MetricIndex, func()) {
	t.Helper()

	oldRejectInvalidTags := rejectInvalidTags
	oldSchemas := mdata.Schemas
	oldTagSupport := memory.TagSupport
	memory.TagSupport = true
	index := memory.New()

	reset := func() {
		rejectInvalidTags = oldRejectInvalidTags
		mdata.Schemas = oldSchemas
		memory.TagSupport = oldTagSupport
		index.Stop()
	}

	mdata.Schemas = conf.NewSchemas(nil)
	metrics := mdata.NewAggMetrics(nil, nil, false, 3600, 7200, 3600)
	return NewDefaultHandler(metrics, index, "test"), index, reset
}

func BenchmarkProcessMetricDataUniqueMetrics(b *testing.B) {
	cluster.Init("default", "test", time.Now(), "http", 6060)

	store := backendStore.NewDevnullStore()

	mdata.SetSingleSchema(conf.NewRetentionMT(10, 10000, 600, 10, 0))
	mdata.SetSingleAgg(conf.Avg, conf.Min, conf.Max)

	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, false, 800, 8000, 0)
	metricIndex := memory.New()
	metricIndex.Init()
	defer metricIndex.Stop()

	in := NewDefaultHandler(aggmetrics, metricIndex, "BenchmarkProcess")

	// timestamps start at 10 and go up from there. (we can't use 0, see AggMetric.Add())
	datas := make([]*schema.MetricData, b.N)
	for i := 0; i < b.N; i++ {
		name := fmt.Sprintf("fake.metric.%d", i)
		metric := &schema.MetricData{
			Id:       "1.12345678901234567890123456789012",
			OrgId:    500,
			Name:     name,
			Interval: 10,
			Value:    1234.567,
			Unit:     "ms",
			Time:     int64((i + 1) * 10),
			Mtype:    "gauge",
			Tags:     []string{"some=tag", "ok=yes"},
		}
		datas[i] = metric
	}

	b.ResetTimer()
	b.ReportAllocs()
	for i := 0; i < b.N; i++ {
		in.ProcessMetricData(datas[i], 1)
	}
}

func BenchmarkProcessMetricDataSameMetric(b *testing.B) {
	cluster.Init("default", "test", time.Now(), "http", 6060)

	store := backendStore.NewDevnullStore()

	mdata.SetSingleSchema(conf.NewRetentionMT(10, 10000, 600, 10, 0))
	mdata.SetSingleAgg(conf.Avg, conf.Min, conf.Max)

	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, false, 800, 8000, 0)
	metricIndex := memory.New()
	metricIndex.Init()
	defer metricIndex.Stop()
	in := NewDefaultHandler(aggmetrics, metricIndex, "BenchmarkProcess")

	// timestamps start at 10 and go up from there. (we can't use 0, see AggMetric.Add())
	datas := make([]*schema.MetricData, b.N)
	for i := 0; i < b.N; i++ {
		name := "fake.metric.same"
		metric := &schema.MetricData{
			Id:       "1.12345678901234567890123456789012",
			OrgId:    500,
			Name:     name,
			Interval: 10,
			Value:    1234.567,
			Unit:     "ms",
			Time:     int64((i + 1) * 10),
			Mtype:    "gauge",
			Tags:     []string{"some=tag", "ok=yes"},
		}
		datas[i] = metric
	}

	b.ResetTimer()
	b.ReportAllocs()
	for i := 0; i < b.N; i++ {
		in.ProcessMetricData(datas[i], 1)
	}
}
<commit_msg>benchmark for ingest metric points<commit_after>package input

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/grafana/metrictank/cluster"
	"github.com/grafana/metrictank/conf"
	"github.com/grafana/metrictank/idx"
	"github.com/grafana/metrictank/idx/memory"
	"github.com/grafana/metrictank/mdata"
	"github.com/grafana/metrictank/mdata/cache"
	backendStore "github.com/grafana/metrictank/store"
	"github.com/raintank/schema"
	"github.com/raintank/schema/msg"
	"golang.org/x/sync/errgroup"
)

func TestIngestValidAndInvalidTagsAndValuesWithAndWithoutRejection(t *testing.T) {
	type testCase struct {
		name                    string
		rejectInvalidTags       bool
		tags                    []string
		expectedInvalidMdInc    uint32
		expectedInvalidTagMdInc uint32
		expectedIndexSizeInc    uint32
	}

	testCases := []testCase{
		{
			name:                    "valid_with_rejection",
			rejectInvalidTags:       true,
			tags:                    []string{"valid=tag"},
			expectedInvalidMdInc:    0,
			expectedInvalidTagMdInc: 0,
			expectedIndexSizeInc:    1,
		}, {
			name:                    "valid_without_rejection",
			rejectInvalidTags:       false,
			tags:                    []string{"valid=tag"},
			expectedInvalidMdInc:    0,
			expectedInvalidTagMdInc: 0,
			expectedIndexSizeInc:    1,
		}, {
			name:                    "invalid_tags_with_rejection",
			rejectInvalidTags:       true,
			tags:                    generateInvalidTags(t),
			expectedInvalidMdInc:    1,
			expectedInvalidTagMdInc: 1,
			expectedIndexSizeInc:    0,
		}, {
			name:                    "invalid_tags_without_rejection",
			rejectInvalidTags:       false,
			tags:                    generateInvalidTags(t),
			expectedInvalidMdInc:    1,
			expectedInvalidTagMdInc: 1,
			expectedIndexSizeInc:    1,
		}, {
			name:                    "invalid_tag_values_with_rejection",
			rejectInvalidTags:       true,
			tags:                    generateInvalidTagValues(t),
			expectedInvalidMdInc:    1,
			expectedInvalidTagMdInc: 1,
			expectedIndexSizeInc:    0,
		}, {
			name:                    "invalid_tag_values_without_rejection",
			rejectInvalidTags:       false,
			tags:                    generateInvalidTagValues(t),
			expectedInvalidMdInc:    1,
			expectedInvalidTagMdInc: 1,
			expectedIndexSizeInc:    1,
		},
	}

	for _, tc := range testCases {
		handler, index, reset := getDefaultHandler(t)
		rejectInvalidTags = tc.rejectInvalidTags
		for i, tag := range tc.tags {
			data := getTestMetricData()
			data.Tags = []string{tag}
			testIngestMetricData(
				t,
				fmt.Sprintf("%s_%d", tc.name, i),
				data,
				handler,
				index,
				tc.expectedInvalidMdInc,
				tc.expectedInvalidTagMdInc,
				tc.expectedIndexSizeInc,
			)
		}
		reset()
	}
}

func generateInvalidTags(t *testing.T) []string {
	t.Helper()

	invalidChars := ";!^"
	validChar := "a"

	tagKeys := generateInvalidStrings(t, invalidChars, validChar)
	res := make([]string, 0, len(tagKeys))
	for _, tagKey := range tagKeys {
		res = append(res, fmt.Sprintf("%s=value", tagKey))
	}
	return res
}

func generateInvalidTagValues(t *testing.T) []string {
	t.Helper()

	invalidChars := ";"
	validChar := "a"

	tagValues := generateInvalidStrings(t, invalidChars, validChar)
	res := make([]string, 0, len(tagValues))
	for _, tagValue := range tagValues {
		res = append(res, fmt.Sprintf("tag=%s", tagValue))
	}

	res = append(res, fmt.Sprintf("tag=~"))
	res = append(res, fmt.Sprintf("tag=~a"))
	res = append(res, fmt.Sprintf("tag=~aa"))
	return res
}

func generateInvalidStrings(t *testing.T, invalidChars, validChar string) []string {
	t.Helper()

	// * 4 because we generate
	// - one tag which is only the invalid char we currently process
	// - one 3 letter tag which starts with the invalid char
	// - one 3 letter tag which ends with the invalid char
	// - one 3 letter tag where the invalid char is in the middle
	res := make([]string, len(invalidChars)*4)
	for i, invalidChar := range invalidChars {
		invalidCharStr := string(invalidChar)
		res[i*4] = invalidCharStr
		res[i*4+1] = invalidCharStr + validChar + validChar
		res[i*4+2] = validChar + validChar + invalidCharStr
		res[i*4+3] = validChar + invalidCharStr + validChar
	}
	return res
}

func getTestMetricData() schema.MetricData {
	return schema.MetricData{
		Id:       "1.12345678901234567890123456789012",
		OrgId:    1,
		Name:     "abc",
		Interval: 1,
		Value:    2,
		Time:     3,
		Mtype:    "gauge",
	}
}

func testIngestMetricData(t *testing.T, tc string, data schema.MetricData, handler DefaultHandler, index idx.MetricIndex, expectedInvalidMdInc, expectedInvalidTagMdInc, expectedIndexSizeInc uint32) {
	originalInvalidCnt := handler.invalidMD.Peek()
	originalInvalidTagCnt := handler.invalidTagMD.Peek()
	originalIndexSize := uint32(len(index.List(1)))
	data.SetId()
	handler.ProcessMetricData(&data, 0)

	invalidCnt := handler.invalidMD.Peek()
	if invalidCnt != originalInvalidCnt+expectedInvalidMdInc {
		t.Fatalf("TC %s: Invalid metric counter has not been updated correctly, expected %d, got %d", tc, originalInvalidCnt+expectedInvalidMdInc, invalidCnt)
	}
	invalidTagCnt := handler.invalidTagMD.Peek()

	if invalidTagCnt != originalInvalidTagCnt+expectedInvalidTagMdInc {
		t.Fatalf("TC %s: Invalid tag counter has not been updated correctly, expected %d, got %d", tc, originalInvalidTagCnt+expectedInvalidTagMdInc, invalidTagCnt)
	}

	indexSize := uint32(len(index.List(1)))
	if indexSize != originalIndexSize+expectedIndexSizeInc {
		t.Fatalf("TC %s: Index size has not been updated correctly, expected %d, got %d", tc, originalIndexSize+expectedIndexSizeInc, indexSize)
	}
}

func getDefaultHandler(t *testing.T) (DefaultHandler, idx.MetricIndex, func()) {
	t.Helper()

	oldRejectInvalidTags := rejectInvalidTags
	oldSchemas := mdata.Schemas
	oldTagSupport := memory.TagSupport
	memory.TagSupport = true
	index := memory.New()

	reset := func() {
		rejectInvalidTags = oldRejectInvalidTags
		mdata.Schemas = oldSchemas
		memory.TagSupport = oldTagSupport
		index.Stop()
	}

	mdata.Schemas = conf.NewSchemas(nil)
	metrics := mdata.NewAggMetrics(nil, nil, false, 3600, 7200, 3600)
	return NewDefaultHandler(metrics, index, "test"), index, reset
}

func BenchmarkProcessMetricDataUniqueMetrics(b *testing.B) {
	cluster.Init("default", "test", time.Now(), "http", 6060)

	store := backendStore.NewDevnullStore()

	mdata.SetSingleSchema(conf.NewRetentionMT(10, 10000, 600, 10, 0))
	mdata.SetSingleAgg(conf.Avg, conf.Min, conf.Max)

	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, false, 800, 8000, 0)
	metricIndex := memory.New()
	metricIndex.Init()
	defer metricIndex.Stop()

	in := NewDefaultHandler(aggmetrics, metricIndex, "BenchmarkProcess")

	// timestamps start at 10 and go up from there. (we can't use 0, see AggMetric.Add())
	datas := make([]*schema.MetricData, b.N)
	for i := 0; i < b.N; i++ {
		name := fmt.Sprintf("fake.metric.%d", i)
		metric := &schema.MetricData{
			Id:       "1.12345678901234567890123456789012",
			OrgId:    500,
			Name:     name,
			Interval: 10,
			Value:    1234.567,
			Unit:     "ms",
			Time:     int64((i + 1) * 10),
			Mtype:    "gauge",
			Tags:     []string{"some=tag", "ok=yes"},
		}
		datas[i] = metric
	}

	b.ResetTimer()
	b.ReportAllocs()
	for i := 0; i < b.N; i++ {
		in.ProcessMetricData(datas[i], 1)
	}
}

func BenchmarkProcessMetricDataSameMetric(b *testing.B) {
	cluster.Init("default", "test", time.Now(), "http", 6060)

	store := backendStore.NewDevnullStore()

	mdata.SetSingleSchema(conf.NewRetentionMT(10, 10000, 600, 10, 0))
	mdata.SetSingleAgg(conf.Avg, conf.Min, conf.Max)

	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, false, 800, 8000, 0)
	metricIndex := memory.New()
	metricIndex.Init()
	defer metricIndex.Stop()
	in := NewDefaultHandler(aggmetrics, metricIndex, "BenchmarkProcess")

	// timestamps start at 10 and go up from there. (we can't use 0, see AggMetric.Add())
	datas := make([]*schema.MetricData, b.N)
	for i := 0; i < b.N; i++ {
		name := "fake.metric.same"
		metric := &schema.MetricData{
			Id:       "1.12345678901234567890123456789012",
			OrgId:    500,
			Name:     name,
			Interval: 10,
			Value:    1234.567,
			Unit:     "ms",
			Time:     int64((i + 1) * 10),
			Mtype:    "gauge",
			Tags:     []string{"some=tag", "ok=yes"},
		}
		datas[i] = metric
	}

	b.ResetTimer()
	b.ReportAllocs()
	for i := 0; i < b.N; i++ {
		in.ProcessMetricData(datas[i], 1)
	}
}

func BenchmarkProcessMetricPoint(b *testing.B) {
	metricCount := 100
	cluster.Init("default", "test", time.Now(), "http", 6060)

	store := backendStore.NewDevnullStore()

	mdata.SetSingleSchema(conf.NewRetentionMT(10, 10000, 600, 10, 0))
	mdata.SetSingleAgg(conf.Avg, conf.Min, conf.Max)

	aggmetrics := mdata.NewAggMetrics(store, &cache.MockCache{}, false, 800, 8000, 0)
	metricIndex := memory.New()
	metricIndex.Init()
	defer metricIndex.Stop()

	in := NewDefaultHandler(aggmetrics, metricIndex, "BenchmarkProcess")

	datas := make([]schema.MetricData, metricCount)
	mkeys := make([]schema.MKey, metricCount)
	var err error
	for i := 0; i < metricCount; i++ {
		name := fmt.Sprintf("fake.metric.%d", i)
		metric := schema.MetricData{
			Id:       "1.12345678901234567890123456789012",
			OrgId:    500,
			Name:     name,
			Interval: 10,
			Value:    1234.567,
			Unit:     "ms",
			Time:     int64((i + 1) * 10),
			Mtype:    "gauge",
			Tags:     []string{"some=tag", "ok=yes"},
		}
		datas[i] = metric
		datas[i].SetId()
		in.ProcessMetricData(&datas[i], 1)
		mkeys[i], err = schema.MKeyFromString(datas[i].Id)
		if err != nil {
			b.Fatalf("Error getting MKey from MetricData: %s", err)
		}
	}

	metricPoints := make(chan schema.MetricPoint, 100)
	rootCtx := context.Background()
	group, _ := errgroup.WithContext(rootCtx)
	for i := 0; i < 10; i++ {
		group.Go(func() error {
			for point := range metricPoints {
				in.ProcessMetricPoint(point, msg.FormatMetricPoint, 1)
			}
			return nil
		})
	}

	b.ResetTimer()
	b.ReportAllocs()

	go func() {
		for i := 0; i < b.N; i++ {
			for j := 0; j < metricCount; j++ {
				metricPoints <- schema.MetricPoint{
					MKey:  mkeys[j],
					Value: float64(i + 1),
					Time:  uint32(i + 1),
				}
			}
		}

		close(metricPoints)
	}()

	group.Wait()
}
<|endoftext|>