<commit_before>package timeline

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/pkg/errors"

	"golang.org/x/net/websocket"
)

var rtmStartURL = "https://slack.com/api/rtm.start"

var slackAPIEndpoint = "https://slack.com/api/"

var origin = "http://localhost"

type rtmStartResponse struct {
	OK    bool   `json:"ok"`
	URL   string `json:"url"`
	Error string `json:"error"`
}

type slackMessage struct {
	Type      string `json:"type"`
	UserID    string `json:"user"`
	Text      string `json:"text"`
	ChannelID string `json:"channel"`
}

type userListResponse struct {
	OK    bool   `json:"ok"`
	User  user   `json:"user"`
	Error string `json:"error"`
}

type user struct {
	ID      string  `json:"id"`
	Name    string  `json:"name"`
	Profile profile `json:"profile"`
}

type profile struct {
	ImageURL string `json:"image_48"`
}

type slackClient struct {
	Token string
}

func (cli *slackClient) connectToRTM() (*websocket.Conn, error) {
	v := url.Values{
		"token": {cli.Token},
	}
	resp, e := http.Get(rtmStartURL + "?" + v.Encode())
	if e != nil {
		return nil, e
	}
	defer resp.Body.Close()
	byteArray, e := ioutil.ReadAll(resp.Body)
	if e != nil {
		return nil, e
	}
	res := rtmStartResponse{}
	e = json.Unmarshal(byteArray, &res)
	if e != nil {
		return nil, e
	}
	if !res.OK {
		return nil, fmt.Errorf(res.Error)
	}
	ws, e := websocket.Dial(res.URL, "", origin)
	if e != nil {
		return nil, e
	}
	return ws, nil
}

func (cli *slackClient) polling(messageChan chan *slackMessage, errorChan chan error) {
	ws, e := cli.connectToRTM()
	if e != nil {
		errorChan <- e
		return
	}
	defer ws.Close()
	for {
		var msg = make([]byte, 1024)
		n, e := ws.Read(msg)
		if e != nil {
			errorChan <- e
		} else {
			message := slackMessage{}
			err := json.Unmarshal(msg[:n], &message)
			if err != nil {
				errorChan <- errors.Wrap(err, fmt.Sprintf("failed to unmarshal. json: '%s'", string(msg[:n])))
			} else {
				messageChan <- &message
			}
		}
	}
}

func (cli *slackClient) postMessage(channelID, text, userName, iconURL string) ([]byte, error) {
	res, e := http.PostForm(slackAPIEndpoint+"chat.postMessage", url.Values{
		"token":      {cli.Token},
		"channel":    {channelID},
		"text":       {text},
		"username":   {userName},
		"as_user":    {"false"},
		"icon_url":   {iconURL},
		"link_names": {"0"},
	})
	if e != nil {
		return nil, e
	}
	defer res.Body.Close()
	byteArray, e := ioutil.ReadAll(res.Body)
	if e != nil {
		return nil, e
	}
	return byteArray, nil
}

func (cli *slackClient) getUser(userID string) (*user, error) {
	res, e := http.PostForm(slackAPIEndpoint+"users.info", url.Values{
		"token": {cli.Token},
		"user":  {userID},
	})
	if e != nil {
		return nil, e
	}
	defer res.Body.Close()
	b, e := ioutil.ReadAll(res.Body)
	if e != nil {
		return nil, e
	}
	r := userListResponse{}
	e = json.Unmarshal(b, &r)
	if e != nil {
		return nil, e
	}
	if !r.OK {
		return nil, fmt.Errorf(r.Error)
	}
	u := r.User
	return &u, nil
}
<commit_msg>:imp: let unmarshal error logging<commit_after>package timeline

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/pkg/errors"

	"golang.org/x/net/websocket"
)

var rtmStartURL = "https://slack.com/api/rtm.start"

var slackAPIEndpoint = "https://slack.com/api/"

var origin = "http://localhost"

type rtmStartResponse struct {
	OK    bool   `json:"ok"`
	URL   string `json:"url"`
	Error string `json:"error"`
}

type slackMessage struct {
	Type      string `json:"type"`
	UserID    string `json:"user"`
	Text      string `json:"text"`
	ChannelID string `json:"channel"`
}

type userListResponse struct {
	OK    bool   `json:"ok"`
	User  user   `json:"user"`
	Error string `json:"error"`
}

type user struct {
	ID      string  `json:"id"`
	Name    string  `json:"name"`
	Profile profile `json:"profile"`
}

type profile struct {
	ImageURL string `json:"image_48"`
}

type slackClient struct {
	Token string
}

func (cli *slackClient) connectToRTM() (*websocket.Conn, error) {
	v := url.Values{
		"token": {cli.Token},
	}
	resp, e := http.Get(rtmStartURL + "?" + v.Encode())
	if e != nil {
		return nil, e
	}
	defer resp.Body.Close()
	byteArray, e := ioutil.ReadAll(resp.Body)
	if e != nil {
		return nil, e
	}
	res := rtmStartResponse{}
	e = json.Unmarshal(byteArray, &res)
	if e != nil {
		return nil, e
	}
	if !res.OK {
		return nil, fmt.Errorf(res.Error)
	}
	ws, e := websocket.Dial(res.URL, "", origin)
	if e != nil {
		return nil, e
	}
	return ws, nil
}

func (cli *slackClient) polling(messageChan chan *slackMessage, errorChan chan error) {
	ws, e := cli.connectToRTM()
	if e != nil {
		errorChan <- e
		return
	}
	defer ws.Close()
	for {
		var msg = make([]byte, 1024)
		n, e := ws.Read(msg)
		if e != nil {
			errorChan <- e
		} else {
			message := slackMessage{}
			err := json.Unmarshal(msg[:n], &message)
			if err == nil {
				fmt.Printf("%+v\n", errors.Wrap(err, fmt.Sprintf("failed to unmarshal. json: '%s'", string(msg[:n]))))
			} else {
				messageChan <- &message
			}
		}
	}
}

func (cli *slackClient) postMessage(channelID, text, userName, iconURL string) ([]byte, error) {
	res, e := http.PostForm(slackAPIEndpoint+"chat.postMessage", url.Values{
		"token":      {cli.Token},
		"channel":    {channelID},
		"text":       {text},
		"username":   {userName},
		"as_user":    {"false"},
		"icon_url":   {iconURL},
		"link_names": {"0"},
	})
	if e != nil {
		return nil, e
	}
	defer res.Body.Close()
	byteArray, e := ioutil.ReadAll(res.Body)
	if e != nil {
		return nil, e
	}
	return byteArray, nil
}

func (cli *slackClient) getUser(userID string) (*user, error) {
	res, e := http.PostForm(slackAPIEndpoint+"users.info", url.Values{
		"token": {cli.Token},
		"user":  {userID},
	})
	if e != nil {
		return nil, e
	}
	defer res.Body.Close()
	b, e := ioutil.ReadAll(res.Body)
	if e != nil {
		return nil, e
	}
	r := userListResponse{}
	e = json.Unmarshal(b, &r)
	if e != nil {
		return nil, e
	}
	if !r.OK {
		return nil, fmt.Errorf(r.Error)
	}
	u := r.User
	return &u, nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Google, Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

package layers

import (
	"github.com/google/gopacket"
)

// EAPOL defines an EAP over LAN (802.1x) layer.
type EAPOL struct {
	BaseLayer
	Version uint8
	Type    EAPOLType
    Length  uint16
}

// LayerType returns LayerTypeEAPOL.
func (e *EAPOL) LayerType() gopacket.LayerType { return LayerTypeEAPOL }

// DecodeFromBytes decodes the given bytes into this layer.
func (e *EAPOL) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error {
	e.Version = data[0]
	e.Type = EAPOLType(data[1])
    e.Length = binary.BigEndian.Uint16(data[2:4])
	e.BaseLayer = BaseLayer{data[:4], data[4:]}
	return nil
}

// SerializeTo writes the serialized form of this layer into the
// SerializationBuffer, implementing gopacket.SerializableLayer
func (e *EAPOL) SerializeTo(b gopacket.SerializeBuffer, opts gopacket.SerializeOptions) error {
    bytes, _ := b.PrependBytes(4)
    bytes[0] = e.Version
    bytes[1] = byte(e.Type)
    binary.BigEndian.PutUint16(bytes[2:], e.Length)
    return nil
}

// CanDecode returns the set of layer types that this DecodingLayer can decode.
func (e *EAPOL) CanDecode() gopacket.LayerClass {
	return LayerTypeEAPOL
}

// NextLayerType returns the layer type contained by this DecodingLayer.
func (e *EAPOL) NextLayerType() gopacket.LayerType {
	return e.Type.LayerType()
}

func decodeEAPOL(data []byte, p gopacket.PacketBuilder) error {
	e := &EAPOL{}
	return decodingLayerDecoder(e, data, p)
}
<commit_msg>import binary library<commit_after>// Copyright 2012 Google, Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

package layers

import (
	"encoding/binary"
	"github.com/google/gopacket"
)

// EAPOL defines an EAP over LAN (802.1x) layer.
type EAPOL struct {
	BaseLayer
	Version uint8
	Type    EAPOLType
    Length  uint16
}

// LayerType returns LayerTypeEAPOL.
func (e *EAPOL) LayerType() gopacket.LayerType { return LayerTypeEAPOL }

// DecodeFromBytes decodes the given bytes into this layer.
func (e *EAPOL) DecodeFromBytes(data []byte, df gopacket.DecodeFeedback) error {
	e.Version = data[0]
	e.Type = EAPOLType(data[1])
    e.Length = binary.BigEndian.Uint16(data[2:4])
	e.BaseLayer = BaseLayer{data[:4], data[4:]}
	return nil
}

// SerializeTo writes the serialized form of this layer into the
// SerializationBuffer, implementing gopacket.SerializableLayer
func (e *EAPOL) SerializeTo(b gopacket.SerializeBuffer, opts gopacket.SerializeOptions) error {
    bytes, _ := b.PrependBytes(4)
    bytes[0] = e.Version
    bytes[1] = byte(e.Type)
    binary.BigEndian.PutUint16(bytes[2:], e.Length)
    return nil
}

// CanDecode returns the set of layer types that this DecodingLayer can decode.
func (e *EAPOL) CanDecode() gopacket.LayerClass {
	return LayerTypeEAPOL
}

// NextLayerType returns the layer type contained by this DecodingLayer.
func (e *EAPOL) NextLayerType() gopacket.LayerType {
	return e.Type.LayerType()
}

func decodeEAPOL(data []byte, p gopacket.PacketBuilder) error {
	e := &EAPOL{}
	return decodingLayerDecoder(e, data, p)
}
<|endoftext|>


<commit_before>package api

import (
	"math"
	"reflect"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/api/response"
	"github.com/grafana/metrictank/conf"
	"github.com/grafana/metrictank/mdata"
	"github.com/grafana/metrictank/stats"
	"github.com/grafana/metrictank/util"
)

var (
	// metric api.request.render.chosen_archive is the archive chosen for the request.
	// 0 means original data, 1 means first agg level, 2 means 2nd
	reqRenderChosenArchive = stats.NewMeter32("api.request.render.chosen_archive", false)
	// metric api.request.render.points_fetched is the number of points that need to be fetched for a /render request.
	reqRenderPointsFetched = stats.NewMeter32("api.request.render.points_fetched", false)
	// metric api.request.render.points_returned is the number of points the request will return
	// best effort: not aware of summarize(), aggregation functions, runtime normalization. but does account for runtime consolidation
	reqRenderPointsReturned = stats.NewMeter32("api.request.render.points_returned", false)

	errUnSatisfiable   = response.NewError(404, "request cannot be satisfied due to lack of available retentions")
	errMaxPointsPerReq = response.NewError(413, "request exceeds max-points-per-req-hard limit. Reduce the time range or number of targets or ask your admin to increase the limit.")
)

func getRetentions(req models.Req) []conf.Retention {
	return mdata.Schemas.Get(req.SchemaId).Retentions.Rets
}

// planRequests updates the requests with all details for fetching.
// Notes:
// [1] MDP-optimization may reduce amount of points down to MDP/2, but not lower. TODO: how about reduce to MDP exactly if possible, and a bit lower otherwise
//     Typically MDP matches number of pixels, which is very dense. So MDP/2 is still quite dense, and for our purposes we consider MDP/2 points to contain the same amount of "information".
// [2] MDP-optimizable requests (when considered by themselves) incur no significant information loss. See [1]
//     Though consider this case:
//     series A 10s:7d,5min:70d
//     series B 10s:7d,4min:30d
//     Let's say a request comes in for 2 days worth of data with MDP=800. Using the high-res data would be 17280 points and require runtime consolidation
//     Both series can be MDP optimized: pick archive=1 and get 576 and 720 points respectively. Neither lost information.
//     However, if it then turns out that both series need to be combined in an aggregation function, they need to be reduced to 20 min resolution, which results in coarse points.
//     Thus MDP-optimized series can still possibly result in some information loss, though this seems quite rare.
//     If you want to aggregate different data together, just give it compatible intervals. For our purposes we will consider MDP-optimizing safe.
// [3] Requests in the same PNGroup will need to be normalized together anyway.
//     Because the consolidation function for normalization is always set taking into account the rollups that we have (see executePlan()) we can better read from a coarser archive.
//     Any request in a PNGroup has already been vetted to be worthy of pre-normalization, thus there is absolutely no loss of information.
//
// planRequests follows these steps:
// 1) Initial parameters.
//    select the highest resolution possible within TTL for all requests. there's 4 cases:
//    * requests in the same PNGroup,    and MDP-optimizable: reduce aggressively: to longest common interval such that points >=MDP/2
//    * requests in the same PNGroup but not MDP-optimizable: reduce conservatively: to shortest common interval that still meets TTL
//    * MDP optimizable singles     : longest interval such that points >= MDP/2
//    * non-MDP-optimizable singles : shortest interval that still meets TTL
//
// 2) apply max-points-per-req-soft (meaning: pick coarser data as needed)
//    The optimizations in the previous step should increase the odds of meeting this limit.
//    If we still breach this limit, we could
//    a) reduce the already MDP-optimized ones further but that would definitely result in loss of accuracy
//    b) reduce non-MDP-optimizable series.
//    For "fairness" across series, and because we used to simply reduce any series without regard for how it would be used, we pick the latter. better would be both
// 3) subject to max-points-per-req-hard: reject the query if it can't be met
//
// note: it is assumed that all requests have the same from & to.
// also takes a "now" value which we compare the TTL against
func planRequests(now, from, to uint32, reqs *ReqMap, planMDP uint32, mpprSoft, mpprHard int) (*ReqsPlan, error) {

	ok, rp := false, NewReqsPlan(*reqs)

	for group, split := range rp.pngroups {
		if len(split.mdpno) > 0 {
			split.mdpno, ok = planHighestResMulti(now, from, to, split.mdpno)
			if !ok {
				return nil, errUnSatisfiable
			}
		}
		if len(split.mdpyes) > 0 {
			split.mdpyes, ok = planLowestResForMDPMulti(now, from, to, split.mdpyes)
			if !ok {
				return nil, errUnSatisfiable
			}
			rp.pngroups[group] = split
		}
	}
	for i, req := range rp.single.mdpno {
		rp.single.mdpno[i], ok = planHighestResSingle(now, from, to, req)
		if !ok {
			return nil, errUnSatisfiable
		}
	}
	for i, req := range rp.single.mdpyes {
		rp.single.mdpyes[i], ok = planLowestResForMDPSingle(now, from, to, req)
		if !ok {
			return nil, errUnSatisfiable
		}
	}

	if mpprSoft > 0 {
		// at this point, all MDP-optimizable series have already been optimized
		// we can try to reduce the resolution of non-MDP-optimizable series
		// if metrictank is already handling all, or most of your queries, then we have been able to determine
		// MDP-optimizability very well. If the request came from Graphite, we have to assume it may run GR-functions.
		// thus in the former case, we pretty much know that this is going to have an adverse effect on your queries,
		// and you should probably not use this option, or we should even get rid of it.
		// in the latter case though, it's quite likely we were too cautious and categorized many series as non-MDP
		// optimizable whereas in reality they should be, so in that case this option is a welcome way to reduce the
		// impact of big queries
		// we could do two approaches: gradually reduce the interval of all series/groups being read, or just aggressively
		// adjust one group at a time. The latter seems simpler, so for now we do just that.
		if rp.PointsFetch() > uint32(mpprSoft) {
			for group, split := range rp.pngroups {
				if len(split.mdpno) > 0 {
					split.mdpno, ok = planLowestResForMDPMulti(now, from, to, split.mdpno)
					if !ok {
						return nil, errUnSatisfiable
					}
					rp.pngroups[group] = split
					if rp.PointsFetch() <= uint32(mpprSoft) {
						goto HonoredSoft
					}
				}
			}
			for i, req := range rp.single.mdpno {
				rp.single.mdpno[i], ok = planLowestResForMDPSingle(now, from, to, req)
				if !ok {
					return nil, errUnSatisfiable
				}
				// for every 10 requests we adjusted, check if we honor soft now.
				// note that there may be thousands of requests
				if i%10 == 9 {
					if rp.PointsFetch() <= uint32(mpprSoft) {
						goto HonoredSoft
					}
				}
			}
		}
	}
HonoredSoft:

	if mpprHard > 0 && int(rp.PointsFetch()) > mpprHard {
		return nil, errMaxPointsPerReq

	}

	// send out some metrics and we're done!
	for _, r := range rp.single.mdpyes {
		reqRenderChosenArchive.ValueUint32(uint32(r.Archive))
	}
	for _, r := range rp.single.mdpno {
		reqRenderChosenArchive.ValueUint32(uint32(r.Archive))
	}
	for _, split := range rp.pngroups {
		for _, r := range split.mdpyes {
			reqRenderChosenArchive.ValueUint32(uint32(r.Archive))
		}
		for _, r := range split.mdpno {
			reqRenderChosenArchive.ValueUint32(uint32(r.Archive))
		}
	}
	reqRenderPointsFetched.ValueUint32(rp.PointsFetch())
	reqRenderPointsReturned.ValueUint32(rp.PointsReturn(planMDP))

	return &rp, nil
}

func planHighestResSingle(now, from, to uint32, req models.Req) (models.Req, bool) {
	rets := getRetentions(req)
	minTTL := now - from
	var ok bool
	for i, ret := range rets {
		// skip non-ready option.
		if ret.Ready > from {
			continue
		}
		ok = true
		req.Plan(i, ret)

		if req.TTL >= minTTL {
			break
		}
	}
	return req, ok
}

func planLowestResForMDPSingle(now, from, to uint32, req models.Req) (models.Req, bool) {
	rets := getRetentions(req)
	var ok bool
	for i := len(rets) - 1; i >= 0; i-- {
		// skip non-ready option.
		if rets[i].Ready > from {
			continue
		}
		ok = true
		req.Plan(i, rets[i])
		if req.PointsFetch() >= req.MaxPoints/2 {
			break
		}
	}
	return req, ok
}
func planHighestResMulti(now, from, to uint32, reqs []models.Req) ([]models.Req, bool) {
	minTTL := now - from

	var listIntervals []uint32
	var seenIntervals = make(map[uint32]struct{})

	for i := range reqs {
		req := &reqs[i]
		var ok bool
		rets := getRetentions(*req)
		for i, ret := range rets {
			// skip non-ready option.
			if ret.Ready > from {
				continue
			}
			ok = true
			req.Plan(i, ret)

			if req.TTL >= minTTL {
				break
			}
		}
		if !ok {
			return nil, ok
		}
		if _, ok := seenIntervals[req.ArchInterval]; !ok {
			listIntervals = append(listIntervals, req.ArchInterval)
			seenIntervals[req.ArchInterval] = struct{}{}
		}
	}
	interval := util.Lcm(listIntervals)

	// plan all our requests so that they result in the common output interval.
	for i := range reqs {
		req := &reqs[i]
		req.AdjustTo(interval, from, getRetentions(*req))
	}

	return reqs, true
}

// note: we can assume all reqs have the same MDP.
func planLowestResForMDPMulti(now, from, to uint32, reqs []models.Req) ([]models.Req, bool) {
	var ok bool
	minTTL := now - from

	// if we were to set each req to their coarsest interval that results in >= MDP/2 points,
	// we'd still have to align them to their LCM interval, which may push them in to
	// "too coarse" territory.
	// instead, we pick the coarsest allowable artificial interval...
	maxInterval := (2 * (to - from)) / reqs[0].MaxPoints
	// ...and then we look for the combination of intervals that scores highest.
	// the bigger the interval the better (load less points), adjusted for number of reqs that
	// have that interval. but their combined LCM may not exceed maxInterval.

	var validIntervalss [][]uint32

	for i := range reqs {
		req := &reqs[i]
		var ok bool
		rets := getRetentions(*req)
		var validIntervals []uint32
		for _, ret := range rets {
			if ret.Ready <= from && req.TTL >= minTTL {
				ok = true
				validIntervals = append(validIntervals, uint32(ret.SecondsPerPoint))
			}
		}
		if !ok {
			return nil, ok
		}
		// add our sequence of valid intervals to the list, unless it's there already
		var found bool
		for _, v := range validIntervalss {
			if reflect.DeepEqual(v, validIntervals) {
				found = true
				break
			}
		}
		if !found {
			validIntervalss = append(validIntervalss, validIntervals)
		}
	}

	combos := util.AllCombinationsUint32(validIntervalss)
	var maxScore int

	lowestInterval := uint32(math.MaxUint32) // lowest interval we find
	var candidateInterval uint32             // the candidate MDP-optimized interval
	var interval uint32                      // will be set to either of the two above
	for _, combo := range combos {
		candidateInterval = util.Lcm(combo)
		if candidateInterval <= maxInterval {
			var score int
			for _, req := range reqs {
				rets := getRetentions(req)
				// we know that every request must have a ready retention with an interval that fits into the candidate LCM
				// only a matter of finding the best (largest) one
				for i := len(rets); i >= 0; i-- {
					ret := rets[i]
					if uint32(ret.SecondsPerPoint) <= candidateInterval && candidateInterval%uint32(ret.SecondsPerPoint) == 0 && ret.Ready <= from && req.TTL >= minTTL {
						score += ret.SecondsPerPoint
					}
				}
			}
			if score > maxScore {
				maxScore = score
				interval = candidateInterval
			}
			if candidateInterval < lowestInterval {
				lowestInterval = candidateInterval
			}
		}
	}
	// if we didn't find a suitable MDP-optimized one, just pick the lowest one we've seen.
	if interval == 0 {
		interval = lowestInterval
	}
	// now we finally found our optimal interval that we want to use.
	// plan all our requests so that they result in the common output interval.
	for i := range reqs {
		req := &reqs[i]
		rets := getRetentions(*req)
		for i := len(rets); i >= 0; i-- {
			ret := rets[i]
			if ret.Ready <= from && req.TTL >= minTTL {
				if uint32(ret.SecondsPerPoint) == interval {
					req.Plan(i, ret)
					break
				}
				if interval%uint32(ret.SecondsPerPoint) == 0 {
					req.Plan(i, ret)
					req.PlanNormalization(interval)
					break
				}
			}
		}

	}
	return reqs, ok
}
<commit_msg>refer to http errors by their name as per contribution docs<commit_after>package api

import (
	"math"
	"net/http"
	"reflect"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/api/response"
	"github.com/grafana/metrictank/conf"
	"github.com/grafana/metrictank/mdata"
	"github.com/grafana/metrictank/stats"
	"github.com/grafana/metrictank/util"
)

var (
	// metric api.request.render.chosen_archive is the archive chosen for the request.
	// 0 means original data, 1 means first agg level, 2 means 2nd
	reqRenderChosenArchive = stats.NewMeter32("api.request.render.chosen_archive", false)
	// metric api.request.render.points_fetched is the number of points that need to be fetched for a /render request.
	reqRenderPointsFetched = stats.NewMeter32("api.request.render.points_fetched", false)
	// metric api.request.render.points_returned is the number of points the request will return
	// best effort: not aware of summarize(), aggregation functions, runtime normalization. but does account for runtime consolidation
	reqRenderPointsReturned = stats.NewMeter32("api.request.render.points_returned", false)

	errUnSatisfiable   = response.NewError(http.StatusNotFound, "request cannot be satisfied due to lack of available retentions")
	errMaxPointsPerReq = response.NewError(http.StatusRequestEntityTooLarge, "request exceeds max-points-per-req-hard limit. Reduce the time range or number of targets or ask your admin to increase the limit.")
)

func getRetentions(req models.Req) []conf.Retention {
	return mdata.Schemas.Get(req.SchemaId).Retentions.Rets
}

// planRequests updates the requests with all details for fetching.
// Notes:
// [1] MDP-optimization may reduce amount of points down to MDP/2, but not lower. TODO: how about reduce to MDP exactly if possible, and a bit lower otherwise
//     Typically MDP matches number of pixels, which is very dense. So MDP/2 is still quite dense, and for our purposes we consider MDP/2 points to contain the same amount of "information".
// [2] MDP-optimizable requests (when considered by themselves) incur no significant information loss. See [1]
//     Though consider this case:
//     series A 10s:7d,5min:70d
//     series B 10s:7d,4min:30d
//     Let's say a request comes in for 2 days worth of data with MDP=800. Using the high-res data would be 17280 points and require runtime consolidation
//     Both series can be MDP optimized: pick archive=1 and get 576 and 720 points respectively. Neither lost information.
//     However, if it then turns out that both series need to be combined in an aggregation function, they need to be reduced to 20 min resolution, which results in coarse points.
//     Thus MDP-optimized series can still possibly result in some information loss, though this seems quite rare.
//     If you want to aggregate different data together, just give it compatible intervals. For our purposes we will consider MDP-optimizing safe.
// [3] Requests in the same PNGroup will need to be normalized together anyway.
//     Because the consolidation function for normalization is always set taking into account the rollups that we have (see executePlan()) we can better read from a coarser archive.
//     Any request in a PNGroup has already been vetted to be worthy of pre-normalization, thus there is absolutely no loss of information.
//
// planRequests follows these steps:
// 1) Initial parameters.
//    select the highest resolution possible within TTL for all requests. there's 4 cases:
//    * requests in the same PNGroup,    and MDP-optimizable: reduce aggressively: to longest common interval such that points >=MDP/2
//    * requests in the same PNGroup but not MDP-optimizable: reduce conservatively: to shortest common interval that still meets TTL
//    * MDP optimizable singles     : longest interval such that points >= MDP/2
//    * non-MDP-optimizable singles : shortest interval that still meets TTL
//
// 2) apply max-points-per-req-soft (meaning: pick coarser data as needed)
//    The optimizations in the previous step should increase the odds of meeting this limit.
//    If we still breach this limit, we could
//    a) reduce the already MDP-optimized ones further but that would definitely result in loss of accuracy
//    b) reduce non-MDP-optimizable series.
//    For "fairness" across series, and because we used to simply reduce any series without regard for how it would be used, we pick the latter. better would be both
// 3) subject to max-points-per-req-hard: reject the query if it can't be met
//
// note: it is assumed that all requests have the same from & to.
// also takes a "now" value which we compare the TTL against
func planRequests(now, from, to uint32, reqs *ReqMap, planMDP uint32, mpprSoft, mpprHard int) (*ReqsPlan, error) {

	ok, rp := false, NewReqsPlan(*reqs)

	for group, split := range rp.pngroups {
		if len(split.mdpno) > 0 {
			split.mdpno, ok = planHighestResMulti(now, from, to, split.mdpno)
			if !ok {
				return nil, errUnSatisfiable
			}
		}
		if len(split.mdpyes) > 0 {
			split.mdpyes, ok = planLowestResForMDPMulti(now, from, to, split.mdpyes)
			if !ok {
				return nil, errUnSatisfiable
			}
			rp.pngroups[group] = split
		}
	}
	for i, req := range rp.single.mdpno {
		rp.single.mdpno[i], ok = planHighestResSingle(now, from, to, req)
		if !ok {
			return nil, errUnSatisfiable
		}
	}
	for i, req := range rp.single.mdpyes {
		rp.single.mdpyes[i], ok = planLowestResForMDPSingle(now, from, to, req)
		if !ok {
			return nil, errUnSatisfiable
		}
	}

	if mpprSoft > 0 {
		// at this point, all MDP-optimizable series have already been optimized
		// we can try to reduce the resolution of non-MDP-optimizable series
		// if metrictank is already handling all, or most of your queries, then we have been able to determine
		// MDP-optimizability very well. If the request came from Graphite, we have to assume it may run GR-functions.
		// thus in the former case, we pretty much know that this is going to have an adverse effect on your queries,
		// and you should probably not use this option, or we should even get rid of it.
		// in the latter case though, it's quite likely we were too cautious and categorized many series as non-MDP
		// optimizable whereas in reality they should be, so in that case this option is a welcome way to reduce the
		// impact of big queries
		// we could do two approaches: gradually reduce the interval of all series/groups being read, or just aggressively
		// adjust one group at a time. The latter seems simpler, so for now we do just that.
		if rp.PointsFetch() > uint32(mpprSoft) {
			for group, split := range rp.pngroups {
				if len(split.mdpno) > 0 {
					split.mdpno, ok = planLowestResForMDPMulti(now, from, to, split.mdpno)
					if !ok {
						return nil, errUnSatisfiable
					}
					rp.pngroups[group] = split
					if rp.PointsFetch() <= uint32(mpprSoft) {
						goto HonoredSoft
					}
				}
			}
			for i, req := range rp.single.mdpno {
				rp.single.mdpno[i], ok = planLowestResForMDPSingle(now, from, to, req)
				if !ok {
					return nil, errUnSatisfiable
				}
				// for every 10 requests we adjusted, check if we honor soft now.
				// note that there may be thousands of requests
				if i%10 == 9 {
					if rp.PointsFetch() <= uint32(mpprSoft) {
						goto HonoredSoft
					}
				}
			}
		}
	}
HonoredSoft:

	if mpprHard > 0 && int(rp.PointsFetch()) > mpprHard {
		return nil, errMaxPointsPerReq

	}

	// send out some metrics and we're done!
	for _, r := range rp.single.mdpyes {
		reqRenderChosenArchive.ValueUint32(uint32(r.Archive))
	}
	for _, r := range rp.single.mdpno {
		reqRenderChosenArchive.ValueUint32(uint32(r.Archive))
	}
	for _, split := range rp.pngroups {
		for _, r := range split.mdpyes {
			reqRenderChosenArchive.ValueUint32(uint32(r.Archive))
		}
		for _, r := range split.mdpno {
			reqRenderChosenArchive.ValueUint32(uint32(r.Archive))
		}
	}
	reqRenderPointsFetched.ValueUint32(rp.PointsFetch())
	reqRenderPointsReturned.ValueUint32(rp.PointsReturn(planMDP))

	return &rp, nil
}

func planHighestResSingle(now, from, to uint32, req models.Req) (models.Req, bool) {
	rets := getRetentions(req)
	minTTL := now - from
	var ok bool
	for i, ret := range rets {
		// skip non-ready option.
		if ret.Ready > from {
			continue
		}
		ok = true
		req.Plan(i, ret)

		if req.TTL >= minTTL {
			break
		}
	}
	return req, ok
}

func planLowestResForMDPSingle(now, from, to uint32, req models.Req) (models.Req, bool) {
	rets := getRetentions(req)
	var ok bool
	for i := len(rets) - 1; i >= 0; i-- {
		// skip non-ready option.
		if rets[i].Ready > from {
			continue
		}
		ok = true
		req.Plan(i, rets[i])
		if req.PointsFetch() >= req.MaxPoints/2 {
			break
		}
	}
	return req, ok
}
func planHighestResMulti(now, from, to uint32, reqs []models.Req) ([]models.Req, bool) {
	minTTL := now - from

	var listIntervals []uint32
	var seenIntervals = make(map[uint32]struct{})

	for i := range reqs {
		req := &reqs[i]
		var ok bool
		rets := getRetentions(*req)
		for i, ret := range rets {
			// skip non-ready option.
			if ret.Ready > from {
				continue
			}
			ok = true
			req.Plan(i, ret)

			if req.TTL >= minTTL {
				break
			}
		}
		if !ok {
			return nil, ok
		}
		if _, ok := seenIntervals[req.ArchInterval]; !ok {
			listIntervals = append(listIntervals, req.ArchInterval)
			seenIntervals[req.ArchInterval] = struct{}{}
		}
	}
	interval := util.Lcm(listIntervals)

	// plan all our requests so that they result in the common output interval.
	for i := range reqs {
		req := &reqs[i]
		req.AdjustTo(interval, from, getRetentions(*req))
	}

	return reqs, true
}

// note: we can assume all reqs have the same MDP.
func planLowestResForMDPMulti(now, from, to uint32, reqs []models.Req) ([]models.Req, bool) {
	var ok bool
	minTTL := now - from

	// if we were to set each req to their coarsest interval that results in >= MDP/2 points,
	// we'd still have to align them to their LCM interval, which may push them in to
	// "too coarse" territory.
	// instead, we pick the coarsest allowable artificial interval...
	maxInterval := (2 * (to - from)) / reqs[0].MaxPoints
	// ...and then we look for the combination of intervals that scores highest.
	// the bigger the interval the better (load less points), adjusted for number of reqs that
	// have that interval. but their combined LCM may not exceed maxInterval.

	var validIntervalss [][]uint32

	for i := range reqs {
		req := &reqs[i]
		var ok bool
		rets := getRetentions(*req)
		var validIntervals []uint32
		for _, ret := range rets {
			if ret.Ready <= from && req.TTL >= minTTL {
				ok = true
				validIntervals = append(validIntervals, uint32(ret.SecondsPerPoint))
			}
		}
		if !ok {
			return nil, ok
		}
		// add our sequence of valid intervals to the list, unless it's there already
		var found bool
		for _, v := range validIntervalss {
			if reflect.DeepEqual(v, validIntervals) {
				found = true
				break
			}
		}
		if !found {
			validIntervalss = append(validIntervalss, validIntervals)
		}
	}

	combos := util.AllCombinationsUint32(validIntervalss)
	var maxScore int

	lowestInterval := uint32(math.MaxUint32) // lowest interval we find
	var candidateInterval uint32             // the candidate MDP-optimized interval
	var interval uint32                      // will be set to either of the two above
	for _, combo := range combos {
		candidateInterval = util.Lcm(combo)
		if candidateInterval <= maxInterval {
			var score int
			for _, req := range reqs {
				rets := getRetentions(req)
				// we know that every request must have a ready retention with an interval that fits into the candidate LCM
				// only a matter of finding the best (largest) one
				for i := len(rets); i >= 0; i-- {
					ret := rets[i]
					if uint32(ret.SecondsPerPoint) <= candidateInterval && candidateInterval%uint32(ret.SecondsPerPoint) == 0 && ret.Ready <= from && req.TTL >= minTTL {
						score += ret.SecondsPerPoint
					}
				}
			}
			if score > maxScore {
				maxScore = score
				interval = candidateInterval
			}
			if candidateInterval < lowestInterval {
				lowestInterval = candidateInterval
			}
		}
	}
	// if we didn't find a suitable MDP-optimized one, just pick the lowest one we've seen.
	if interval == 0 {
		interval = lowestInterval
	}
	// now we finally found our optimal interval that we want to use.
	// plan all our requests so that they result in the common output interval.
	for i := range reqs {
		req := &reqs[i]
		rets := getRetentions(*req)
		for i := len(rets); i >= 0; i-- {
			ret := rets[i]
			if ret.Ready <= from && req.TTL >= minTTL {
				if uint32(ret.SecondsPerPoint) == interval {
					req.Plan(i, ret)
					break
				}
				if interval%uint32(ret.SecondsPerPoint) == 0 {
					req.Plan(i, ret)
					req.PlanNormalization(interval)
					break
				}
			}
		}

	}
	return reqs, ok
}
<|endoftext|>


<commit_before>// Copyright 2013 Travis Keep. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or
// at http://opensource.org/licenses/BSD-3-Clause.

// Package tasks handles tasks that can be started and stopped
package tasks

import (
  "github.com/keep94/gofunctional3/functional"
  "github.com/keep94/tasks/recurring"
  "sync"
  "time"
)

// Task represents any task
type Task interface {

  // Do performs the task. execution is the specific execution of this task.
  Do(execution *Execution)
}

// TaskFunc wraps a simple function to implement Task.
type TaskFunc func(execution *Execution)

func (f TaskFunc) Do(execution *Execution) {
  f(execution)
}

// Clock represents the system clock.
type Clock interface {

  // Now returns the current time
  Now() time.Time

  // After waits for given duration to elapse and then sends current time on
  // the returned channel.
  After(d time.Duration) <-chan time.Time
}

// Execution represents a particular execution of some task.
// Execution instances are safe to use with multiple goroutines.
type Execution struct {
  Clock
  ended chan struct{}
  done chan struct{}
  bEnded bool
  err error
  lock sync.Mutex
}

// Run executes a task in the current goroutine and exits when the task
// finishes.
func Run(task Task) error {
  return RunForTesting(task, systemClock{})
}

// RunForTesting work just like Run except it allows caller to specify
// an implementation of the Clock interface for testing.
func RunForTesting(task Task, clock Clock) (err error) {
  execution := &Execution{
      Clock: clock, done: make(chan struct{}), ended: make(chan struct{})}
  task.Do(execution)
  execution.End()
  close(execution.done)
  return execution.Error()
}

// Start starts a task in a separate goroutine and returns immediately.
// Start returns that particular execution of the task.
func Start(task Task) *Execution {
  execution := &Execution{
      Clock: systemClock{},
      done: make(chan struct{}),
      ended: make(chan struct{})}
  go func() {
    task.Do(execution)
    execution.End()
    close(execution.done)
  }()
  return execution
}

// Error returns error from this execution.
func (e *Execution) Error() error {
  e.lock.Lock()
  defer e.lock.Unlock()
  return e.err
}

// End signals that execution should end.
func (e *Execution) End() {
  if e.markEnded() {
    close(e.ended)
  }
}

// Ended returns a channel that gets closed when this execution is signaled
// to end.
func (e *Execution) Ended() <-chan struct{} {
  return e.ended
}

// Done returns a channel that gets closed when this execution is done.
func (e *Execution) Done() <-chan struct{} {
  return e.done
}

// IsDone returns true if this execution is done or false if it is still
// in progress.
func (e *Execution) IsDone() bool {
  select {
    case <-e.done:
      return true
    default:
      return false
  }
  return false
}

// IsEnded returns true if this execution has been signaled to end.
func (e *Execution) IsEnded() bool {
  select {
    case <-e.ended:
      return true
    default:
      return false
  }
  return false
}

// Sleep sleeps for the specified duration or until this execution should
// end, whichever comes first. Sleep returns true if it slept the entire
// duration or false if it returned early because this execution should end.
func (e *Execution) Sleep(d time.Duration) bool {
  select {
    case <-e.ended:
      return false
    case <-e.After(d):
      return true
  }
  return false
}

// SetError lets a task report an error.
func (e *Execution) SetError(err error) {
  if err == nil {
    return
  }
  e.lock.Lock()
  defer e.lock.Unlock()
  e.err = err
}

func (e *Execution) markEnded() bool {
  e.lock.Lock()
  defer e.lock.Unlock()
  result := !e.bEnded
  e.bEnded = true
  return result
}

// RecurringTask returns a task that does t at each time that r specifies.
// The returned task ends when there are no more times from r or if some
// error happens while executing one of the tasks.
func RecurringTask(t Task, r recurring.R) Task {
  return &recurringTask{t, r}
}

// ParallelTasks returns a task that performs all the passed in tasks in
// parallel.
func ParallelTasks(tasks ...Task) Task {
  return parallelTasks(tasks)
}

// SeriesTasks returns a task that performas all the passed in tasks in
// series. If one of the tasks reports an error, the others following it
// don't get executed.
func SeriesTasks(tasks ...Task) Task {
  return seriesTasks(tasks)
}

// RepeatingTask returns a task that performs the pased in task n times.
func RepeatingTask(t Task, n int) Task {
  return &repeatingTask{t, n}
}

// ClockForTesting is a test implementation of Clock.
// Current time advances only when After() is called.
type ClockForTesting struct {

  // The current time
  Current time.Time
}

func (c *ClockForTesting) Now() time.Time {
  return c.Current
}

// After immediately advances current time by d and send that currnet time
// on the returned channel.
func (c *ClockForTesting) After(d time.Duration) <-chan time.Time {
  c.Current = c.Current.Add(d)
  result := make(chan time.Time, 1)
  result <- c.Current
  close(result)
  return result
}

// SingleExecutor executes tasks one at a time. SingleExecutor instances are
// safe to use with multiple goroutines.
type SingleExecutor struct {
  me *MultiExecutor
}

// NewSingleExecutor returns a new SingleExecutor.
func NewSingleExecutor() *SingleExecutor {
  return &SingleExecutor{NewMultiExecutor(&singleTaskCollection{})}
}

// Start starts task t and returns its Execution. Start blocks until this
// instance actually starts t. Start interrupts any currently running task
// before starting t.
func (se *SingleExecutor) Start(t Task) *Execution {
  return se.me.Start(t)
}

// Current returns the current running task and its execution. If no task
// is running, Current may return nil, nil or it may return the last run
// task along with its execution.
func (se *SingleExecutor) Current() (Task, *Execution) {
  return se.me.Tasks().(*singleTaskCollection).Current()
}

// Close frees the resources of this instance and always returns nil. Close
// interrupts any currently running task.
func (se *SingleExecutor) Close() error {
  return se.me.Close()
}

// Interface TaskCollection represents a collection of running tasks.
type TaskCollection interface {
  // Add adds a task and execution of that task to this collection.
  Add(t Task, e *Execution)

  // Remove removes task t from this collection.
  Remove(t Task)

  // Conflicts returns the execution of all tasks that conflict with t.
  // If t is nil it means return the executions of all tasks in this
  // collection.
  Conflicts(t Task) []*Execution
}

// MultiExecutor executes multiple tasks at one time while ensuring that no
// conflicting tasks execute in parallel.
// MultiExecutor is safe to use with multiple goroutines.
type MultiExecutor struct {
  tc TaskCollection
  taskCh chan Task
  taskRetCh chan *Execution
}

// NewMultiExecutor returns a new MultiExecutor. tc is the TaskCollection that
// will hold running tasks. tc shall be safe to use with multiple goroutines
// and each MultiExecutor shall have its own TaskCollection instance.
func NewMultiExecutor(tc TaskCollection) *MultiExecutor {
  result := &MultiExecutor{
      tc: tc,
      taskCh: make(chan Task),
      taskRetCh: make(chan *Execution)}
  go result.loop()
  return result
}

// Start starts task t and returns its Execution. Start blocks until this
// instance actually starts t. Start interrupts any currently running
// conflicting tasks before starting t.
func (me *MultiExecutor) Start(t Task) *Execution {
  if t == nil {
    panic("Got a nil task.")
  }
  me.taskCh <- t
  return <-me.taskRetCh
}

// Tasks returns the running tasks.
func (me *MultiExecutor) Tasks() TaskCollection {
  return me.tc
}

// Close frees the resources of this instance and always returns nil. Close
// interrupts any currently running tasks.
func (me *MultiExecutor) Close() error {
  close(me.taskCh)
  for _, e := range me.tc.Conflicts(nil) {
    e.End()
    <-e.Done()
  }
  return nil
}

func (me *MultiExecutor) loop() {
  for {
    // Get the next task from the Start method.
    t := <-me.taskCh
    if t == nil {  // Our taskCh has been closed.
      close(me.taskRetCh)
      return
    }

    // Interrupt the conflicting tasks and wait for them to end.
    for _, e := range me.tc.Conflicts(t) {
      e.End()
      <-e.Done()
    }

    // Start executing our task taking care to remove it from the collection
    // of running tasks when it completes.
    exec := Start(TaskFunc(func(e *Execution) {
      t.Do(e)
      me.tc.Remove(t)
    }))

    // Add our newly running task to the collection of running tasks.
    me.tc.Add(t, exec)

    // Tell Start method that we have started
    me.taskRetCh <- exec
  }
}

type singleTaskCollection struct {
  mutex sync.Mutex
  t Task
  e *Execution
}

func (stc *singleTaskCollection) Add(t Task, e *Execution) {
  stc.mutex.Lock()
  defer stc.mutex.Unlock()
  if stc.t != nil || stc.e != nil {
    panic("Trying to add a task to a full singleTaskCollection.")
  }
  stc.t = t
  stc.e = e
}

func (stc *singleTaskCollection) Remove(t Task) {
  stc.mutex.Lock()
  defer stc.mutex.Unlock()
  if stc.t == t {
    stc.t = nil
    stc.e = nil
  }
}

func (stc *singleTaskCollection) Conflicts(t Task) []*Execution {
  stc.mutex.Lock()
  defer stc.mutex.Unlock()
  if stc.e == nil {
    return nil
  }
  return []*Execution{stc.e}
}

func (stc *singleTaskCollection) Current() (Task, *Execution) {
  stc.mutex.Lock()
  defer stc.mutex.Unlock()
  return stc.t, stc.e
}

type recurringTask struct {
  t Task
  r recurring.R
}

func (rt *recurringTask) Do(e *Execution) {
  s := rt.r.ForTime(e.Now())
  defer s.Close()
  var t time.Time
  var err error
  for err = s.Next(&t); err == nil; err = s.Next(&t) {
    dur := t.Sub(e.Now())
    if dur <= 0 {
      continue
    }
    if !e.Sleep(dur) {
      return
    }
    rt.t.Do(e)
    if e.Error() != nil {
      return
    }
  }
  if err != functional.Done {
    e.SetError(err)
  }
}

type parallelTasks []Task

func (p parallelTasks) Do(e *Execution) {
  var wg sync.WaitGroup
  wg.Add(len(p))
  for _, task := range p {
    go func(t Task) {
      t.Do(e)
      wg.Done()
    }(task)
  }
  wg.Wait()
}

type seriesTasks []Task

func (s seriesTasks) Do(e *Execution) {
  for _, task := range s {
    task.Do(e)
    if e.IsEnded() || e.Error() != nil {
      return
    }
  }
}

type repeatingTask struct {
  t Task
  n int
}

func (r *repeatingTask) Do(e *Execution) {
  for i := 0; i < r.n; i++ {
    r.t.Do(e)
    if e.IsEnded() || e.Error() != nil {
      return
    }
  }
}

type systemClock struct {
}

func (s systemClock) Now() time.Time {
  return time.Now()
}

func (s systemClock) After(d time.Duration) <-chan time.Time {
  return time.After(d)
}
<commit_msg>Fixed up documentation for TaskCollection.<commit_after>// Copyright 2013 Travis Keep. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or
// at http://opensource.org/licenses/BSD-3-Clause.

// Package tasks handles tasks that can be started and stopped
package tasks

import (
  "github.com/keep94/gofunctional3/functional"
  "github.com/keep94/tasks/recurring"
  "sync"
  "time"
)

// Task represents any task
type Task interface {

  // Do performs the task. execution is the specific execution of this task.
  Do(execution *Execution)
}

// TaskFunc wraps a simple function to implement Task.
type TaskFunc func(execution *Execution)

func (f TaskFunc) Do(execution *Execution) {
  f(execution)
}

// Clock represents the system clock.
type Clock interface {

  // Now returns the current time
  Now() time.Time

  // After waits for given duration to elapse and then sends current time on
  // the returned channel.
  After(d time.Duration) <-chan time.Time
}

// Execution represents a particular execution of some task.
// Execution instances are safe to use with multiple goroutines.
type Execution struct {
  Clock
  ended chan struct{}
  done chan struct{}
  bEnded bool
  err error
  lock sync.Mutex
}

// Run executes a task in the current goroutine and exits when the task
// finishes.
func Run(task Task) error {
  return RunForTesting(task, systemClock{})
}

// RunForTesting work just like Run except it allows caller to specify
// an implementation of the Clock interface for testing.
func RunForTesting(task Task, clock Clock) (err error) {
  execution := &Execution{
      Clock: clock, done: make(chan struct{}), ended: make(chan struct{})}
  task.Do(execution)
  execution.End()
  close(execution.done)
  return execution.Error()
}

// Start starts a task in a separate goroutine and returns immediately.
// Start returns that particular execution of the task.
func Start(task Task) *Execution {
  execution := &Execution{
      Clock: systemClock{},
      done: make(chan struct{}),
      ended: make(chan struct{})}
  go func() {
    task.Do(execution)
    execution.End()
    close(execution.done)
  }()
  return execution
}

// Error returns error from this execution.
func (e *Execution) Error() error {
  e.lock.Lock()
  defer e.lock.Unlock()
  return e.err
}

// End signals that execution should end.
func (e *Execution) End() {
  if e.markEnded() {
    close(e.ended)
  }
}

// Ended returns a channel that gets closed when this execution is signaled
// to end.
func (e *Execution) Ended() <-chan struct{} {
  return e.ended
}

// Done returns a channel that gets closed when this execution is done.
func (e *Execution) Done() <-chan struct{} {
  return e.done
}

// IsDone returns true if this execution is done or false if it is still
// in progress.
func (e *Execution) IsDone() bool {
  select {
    case <-e.done:
      return true
    default:
      return false
  }
  return false
}

// IsEnded returns true if this execution has been signaled to end.
func (e *Execution) IsEnded() bool {
  select {
    case <-e.ended:
      return true
    default:
      return false
  }
  return false
}

// Sleep sleeps for the specified duration or until this execution should
// end, whichever comes first. Sleep returns true if it slept the entire
// duration or false if it returned early because this execution should end.
func (e *Execution) Sleep(d time.Duration) bool {
  select {
    case <-e.ended:
      return false
    case <-e.After(d):
      return true
  }
  return false
}

// SetError lets a task report an error.
func (e *Execution) SetError(err error) {
  if err == nil {
    return
  }
  e.lock.Lock()
  defer e.lock.Unlock()
  e.err = err
}

func (e *Execution) markEnded() bool {
  e.lock.Lock()
  defer e.lock.Unlock()
  result := !e.bEnded
  e.bEnded = true
  return result
}

// RecurringTask returns a task that does t at each time that r specifies.
// The returned task ends when there are no more times from r or if some
// error happens while executing one of the tasks.
func RecurringTask(t Task, r recurring.R) Task {
  return &recurringTask{t, r}
}

// ParallelTasks returns a task that performs all the passed in tasks in
// parallel.
func ParallelTasks(tasks ...Task) Task {
  return parallelTasks(tasks)
}

// SeriesTasks returns a task that performas all the passed in tasks in
// series. If one of the tasks reports an error, the others following it
// don't get executed.
func SeriesTasks(tasks ...Task) Task {
  return seriesTasks(tasks)
}

// RepeatingTask returns a task that performs the pased in task n times.
func RepeatingTask(t Task, n int) Task {
  return &repeatingTask{t, n}
}

// ClockForTesting is a test implementation of Clock.
// Current time advances only when After() is called.
type ClockForTesting struct {

  // The current time
  Current time.Time
}

func (c *ClockForTesting) Now() time.Time {
  return c.Current
}

// After immediately advances current time by d and send that currnet time
// on the returned channel.
func (c *ClockForTesting) After(d time.Duration) <-chan time.Time {
  c.Current = c.Current.Add(d)
  result := make(chan time.Time, 1)
  result <- c.Current
  close(result)
  return result
}

// SingleExecutor executes tasks one at a time. SingleExecutor instances are
// safe to use with multiple goroutines.
type SingleExecutor struct {
  me *MultiExecutor
}

// NewSingleExecutor returns a new SingleExecutor.
func NewSingleExecutor() *SingleExecutor {
  return &SingleExecutor{NewMultiExecutor(&singleTaskCollection{})}
}

// Start starts task t and returns its Execution. Start blocks until this
// instance actually starts t. Start interrupts any currently running task
// before starting t.
func (se *SingleExecutor) Start(t Task) *Execution {
  return se.me.Start(t)
}

// Current returns the current running task and its execution. If no task
// is running, Current may return nil, nil or it may return the last run
// task along with its execution.
func (se *SingleExecutor) Current() (Task, *Execution) {
  return se.me.Tasks().(*singleTaskCollection).Current()
}

// Close frees the resources of this instance and always returns nil. Close
// interrupts any currently running task.
func (se *SingleExecutor) Close() error {
  return se.me.Close()
}

// Interface TaskCollection represents a collection of running tasks.
// The methods in this interface are for MultiExecutor only. Clients must not
// call these methods directly. Implementations of this interface can provide
// methods giving clients a read-only view of running tasks and executions.
type TaskCollection interface {
  // Add adds a task and execution of that task to this collection.
  Add(t Task, e *Execution)

  // Remove removes task t from this collection.
  Remove(t Task)

  // Conflicts returns the execution of all tasks that conflict with t.
  // If t is nil it means return the executions of all tasks in this
  // collection.
  Conflicts(t Task) []*Execution
}

// MultiExecutor executes multiple tasks at one time while ensuring that no
// conflicting tasks execute in parallel.
// MultiExecutor is safe to use with multiple goroutines.
type MultiExecutor struct {
  tc TaskCollection
  taskCh chan Task
  taskRetCh chan *Execution
}

// NewMultiExecutor returns a new MultiExecutor. tc is the TaskCollection that
// will hold running tasks. tc shall be safe to use with multiple goroutines
// and each MultiExecutor shall have its own TaskCollection instance.
func NewMultiExecutor(tc TaskCollection) *MultiExecutor {
  result := &MultiExecutor{
      tc: tc,
      taskCh: make(chan Task),
      taskRetCh: make(chan *Execution)}
  go result.loop()
  return result
}

// Start starts task t and returns its Execution. Start blocks until this
// instance actually starts t. Start interrupts any currently running
// conflicting tasks before starting t.
func (me *MultiExecutor) Start(t Task) *Execution {
  if t == nil {
    panic("Got a nil task.")
  }
  me.taskCh <- t
  return <-me.taskRetCh
}

// Tasks returns the running tasks.
func (me *MultiExecutor) Tasks() TaskCollection {
  return me.tc
}

// Close frees the resources of this instance and always returns nil. Close
// interrupts any currently running tasks.
func (me *MultiExecutor) Close() error {
  close(me.taskCh)
  for _, e := range me.tc.Conflicts(nil) {
    e.End()
    <-e.Done()
  }
  return nil
}

func (me *MultiExecutor) loop() {
  for {
    // Get the next task from the Start method.
    t := <-me.taskCh
    if t == nil {  // Our taskCh has been closed.
      close(me.taskRetCh)
      return
    }

    // Interrupt the conflicting tasks and wait for them to end.
    for _, e := range me.tc.Conflicts(t) {
      e.End()
      <-e.Done()
    }

    // Start executing our task taking care to remove it from the collection
    // of running tasks when it completes.
    exec := Start(TaskFunc(func(e *Execution) {
      t.Do(e)
      me.tc.Remove(t)
    }))

    // Add our newly running task to the collection of running tasks.
    me.tc.Add(t, exec)

    // Tell Start method that we have started
    me.taskRetCh <- exec
  }
}

type singleTaskCollection struct {
  mutex sync.Mutex
  t Task
  e *Execution
}

func (stc *singleTaskCollection) Add(t Task, e *Execution) {
  stc.mutex.Lock()
  defer stc.mutex.Unlock()
  if stc.t != nil || stc.e != nil {
    panic("Trying to add a task to a full singleTaskCollection.")
  }
  stc.t = t
  stc.e = e
}

func (stc *singleTaskCollection) Remove(t Task) {
  stc.mutex.Lock()
  defer stc.mutex.Unlock()
  if stc.t == t {
    stc.t = nil
    stc.e = nil
  }
}

func (stc *singleTaskCollection) Conflicts(t Task) []*Execution {
  stc.mutex.Lock()
  defer stc.mutex.Unlock()
  if stc.e == nil {
    return nil
  }
  return []*Execution{stc.e}
}

func (stc *singleTaskCollection) Current() (Task, *Execution) {
  stc.mutex.Lock()
  defer stc.mutex.Unlock()
  return stc.t, stc.e
}

type recurringTask struct {
  t Task
  r recurring.R
}

func (rt *recurringTask) Do(e *Execution) {
  s := rt.r.ForTime(e.Now())
  defer s.Close()
  var t time.Time
  var err error
  for err = s.Next(&t); err == nil; err = s.Next(&t) {
    dur := t.Sub(e.Now())
    if dur <= 0 {
      continue
    }
    if !e.Sleep(dur) {
      return
    }
    rt.t.Do(e)
    if e.Error() != nil {
      return
    }
  }
  if err != functional.Done {
    e.SetError(err)
  }
}

type parallelTasks []Task

func (p parallelTasks) Do(e *Execution) {
  var wg sync.WaitGroup
  wg.Add(len(p))
  for _, task := range p {
    go func(t Task) {
      t.Do(e)
      wg.Done()
    }(task)
  }
  wg.Wait()
}

type seriesTasks []Task

func (s seriesTasks) Do(e *Execution) {
  for _, task := range s {
    task.Do(e)
    if e.IsEnded() || e.Error() != nil {
      return
    }
  }
}

type repeatingTask struct {
  t Task
  n int
}

func (r *repeatingTask) Do(e *Execution) {
  for i := 0; i < r.n; i++ {
    r.t.Do(e)
    if e.IsEnded() || e.Error() != nil {
      return
    }
  }
}

type systemClock struct {
}

func (s systemClock) Now() time.Time {
  return time.Now()
}

func (s systemClock) After(d time.Duration) <-chan time.Time {
  return time.After(d)
}
<|endoftext|>


<commit_before>package transmission

type (
	// Torrent represents a transmission torrent
	Torrent struct {
		ID            int           `json:"id"`
		Name          string        `json:"name"`
		Status        int           `json:"status"`
		Added         int           `json:"addedDate"`
		LeftUntilDone int           `json:"leftUntilDone"`
		Eta           int           `json:"eta"`
		UploadRatio   float64       `json:"uploadRatio"`
		RateDownload  int           `json:"rateDownload"`
		RateUpload    int           `json:"rateUpload"`
		DownloadDir   string        `json:"downloadDir"`
		IsFinished    bool          `json:"isFinished"`
		PercentDone   float64       `json:"percentDone"`
		SeedRatioMode int           `json:"seedRatioMode"`
		HashString    string        `json:"hashString"`
		Error         int           `json:"error"`
		ErrorString   string        `json:"errorString"`
		Files         []File        `json:"files"`
		FilesStats    []FileStat    `json:"fileStats"`
		TrackerStats  []TrackerStat `json:"trackerStats"`
		Peers         []Peer        `json:"peers"`
	}

	// ByID implements the sort Interface to sort by ID
	ByID []Torrent
	// ByName implements the sort Interface to sort by Name
	ByName []Torrent
	// ByDate implements the sort Interface to sort by Date
	ByDate []Torrent
	// ByRatio implements the sort Interface to sort by Ratio
	ByRatio []Torrent

	// File is a file contained inside a torrent
	File struct {
		BytesCompleted int    `json:"bytesCompleted"`
		Length         int    `json:"length"`
		Name           string `json:"name"`
	}

	// FileStat describe a file's priority & if it's wanted
	FileStat struct {
		BytesCompleted int  `json:"bytesCompleted"`
		Priority       int  `json:"priority"`
		Wanted         bool `json:"wanted"`
	}

	// TrackerStat has stats about the torrent's tracker
	TrackerStat struct {
		Announce              string `json:"announce"`
		AnnounceState         int    `json:"announceState"`
		DownloadCount         int    `json:"downloadCount"`
		HasAnnounced          bool   `json:"hasAnnounced"`
		HasScraped            bool   `json:"hasScraped"`
		Host                  string `json:"host"`
		ID                    int    `json:"id"`
		IsBackup              bool   `json:"isBackup"`
		LastAnnouncePeerCount int    `json:"lastAnnouncePeerCount"`
		LastAnnounceResult    string `json:"lastAnnounceResult"`
		LastAnnounceStartTime int    `json:"lastAnnounceStartTime"`
		LastAnnounceSucceeded bool   `json:"lastAnnounceSucceeded"`
		LastAnnounceTime      int    `json:"lastAnnounceTime"`
		LastAnnounceTimedOut  bool   `json:"lastAnnounceTimedOut"`
		LastScrapeResult      string `json:"lastScrapeResult"`
		LastScrapeStartTime   int    `json:"lastScrapeStartTime"`
		LastScrapeSucceeded   bool   `json:"lastScrapeSucceeded"`
		LastScrapeTime        int    `json:"lastScrapeTime"`
		LastScrapeTimedOut    int    `json:"lastScrapeTimedOut"`
		LeecherCount          int    `json:"leecherCount"`
		NextAnnounceTime      int    `json:"nextAnnounceTime"`
		NextScrapeTime        int    `json:"nextScrapeTime"`
		Scrape                string `json:"scrape"`
		ScrapeState           int    `json:"scrapeState"`
		SeederCount           int    `json:"seederCount"`
		Tier                  int    `json:"tier"`
	}

	// Peer of a torrent
	Peer struct {
		Address            string `json:"address"`
		ClientIsChoked     bool   `json:"clientIsChoked"`
		ClientIsInterested bool   `json:"clientIsInterested"`
		ClientName         string `json:"clientName"`
		FlagStr            string `json:"flagStr"`
		IsDownloadingFrom  bool   `json:"isDownloadingFrom"`
		IsEncrypted        bool   `json:"isEncrypted"`
		IsIncoming         bool   `json:"isIncoming"`
		IsUTP              bool   `json:"isUTP"`
		IsUploadingTo      bool   `json:"isUploadingTo"`
		PeerIsChoked       bool   `json:"peerIsChoked"`
		PeerIsInterested   bool   `json:"peerIsInterested"`
		Port               int    `json:"port"`
		Progress           int    `json:"progress"`
		RateToClient       int    `json:"rateToClient"`
		RateToPeer         int    `json:"rateToPeer"`
	}
)

func (t ByID) Len() int           { return len(t) }
func (t ByID) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t ByID) Less(i, j int) bool { return t[i].ID < t[j].ID }

func (t ByName) Len() int           { return len(t) }
func (t ByName) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t ByName) Less(i, j int) bool { return t[i].Name < t[j].Name }

func (t ByDate) Len() int           { return len(t) }
func (t ByDate) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t ByDate) Less(i, j int) bool { return t[i].Added < t[j].Added }

func (t ByRatio) Len() int           { return len(t) }
func (t ByRatio) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t ByRatio) Less(i, j int) bool { return t[i].UploadRatio < t[j].UploadRatio }
<commit_msg>A Torrent's Peer Progress is a float64 not int<commit_after>package transmission

type (
	// Torrent represents a transmission torrent
	Torrent struct {
		ID            int           `json:"id"`
		Name          string        `json:"name"`
		Status        int           `json:"status"`
		Added         int           `json:"addedDate"`
		LeftUntilDone int           `json:"leftUntilDone"`
		Eta           int           `json:"eta"`
		UploadRatio   float64       `json:"uploadRatio"`
		RateDownload  int           `json:"rateDownload"`
		RateUpload    int           `json:"rateUpload"`
		DownloadDir   string        `json:"downloadDir"`
		IsFinished    bool          `json:"isFinished"`
		PercentDone   float64       `json:"percentDone"`
		SeedRatioMode int           `json:"seedRatioMode"`
		HashString    string        `json:"hashString"`
		Error         int           `json:"error"`
		ErrorString   string        `json:"errorString"`
		Files         []File        `json:"files"`
		FilesStats    []FileStat    `json:"fileStats"`
		TrackerStats  []TrackerStat `json:"trackerStats"`
		Peers         []Peer        `json:"peers"`
	}

	// ByID implements the sort Interface to sort by ID
	ByID []Torrent
	// ByName implements the sort Interface to sort by Name
	ByName []Torrent
	// ByDate implements the sort Interface to sort by Date
	ByDate []Torrent
	// ByRatio implements the sort Interface to sort by Ratio
	ByRatio []Torrent

	// File is a file contained inside a torrent
	File struct {
		BytesCompleted int    `json:"bytesCompleted"`
		Length         int    `json:"length"`
		Name           string `json:"name"`
	}

	// FileStat describe a file's priority & if it's wanted
	FileStat struct {
		BytesCompleted int  `json:"bytesCompleted"`
		Priority       int  `json:"priority"`
		Wanted         bool `json:"wanted"`
	}

	// TrackerStat has stats about the torrent's tracker
	TrackerStat struct {
		Announce              string `json:"announce"`
		AnnounceState         int    `json:"announceState"`
		DownloadCount         int    `json:"downloadCount"`
		HasAnnounced          bool   `json:"hasAnnounced"`
		HasScraped            bool   `json:"hasScraped"`
		Host                  string `json:"host"`
		ID                    int    `json:"id"`
		IsBackup              bool   `json:"isBackup"`
		LastAnnouncePeerCount int    `json:"lastAnnouncePeerCount"`
		LastAnnounceResult    string `json:"lastAnnounceResult"`
		LastAnnounceStartTime int    `json:"lastAnnounceStartTime"`
		LastAnnounceSucceeded bool   `json:"lastAnnounceSucceeded"`
		LastAnnounceTime      int    `json:"lastAnnounceTime"`
		LastAnnounceTimedOut  bool   `json:"lastAnnounceTimedOut"`
		LastScrapeResult      string `json:"lastScrapeResult"`
		LastScrapeStartTime   int    `json:"lastScrapeStartTime"`
		LastScrapeSucceeded   bool   `json:"lastScrapeSucceeded"`
		LastScrapeTime        int    `json:"lastScrapeTime"`
		LastScrapeTimedOut    int    `json:"lastScrapeTimedOut"`
		LeecherCount          int    `json:"leecherCount"`
		NextAnnounceTime      int    `json:"nextAnnounceTime"`
		NextScrapeTime        int    `json:"nextScrapeTime"`
		Scrape                string `json:"scrape"`
		ScrapeState           int    `json:"scrapeState"`
		SeederCount           int    `json:"seederCount"`
		Tier                  int    `json:"tier"`
	}

	// Peer of a torrent
	Peer struct {
		Address            string  `json:"address"`
		ClientIsChoked     bool    `json:"clientIsChoked"`
		ClientIsInterested bool    `json:"clientIsInterested"`
		ClientName         string  `json:"clientName"`
		FlagStr            string  `json:"flagStr"`
		IsDownloadingFrom  bool    `json:"isDownloadingFrom"`
		IsEncrypted        bool    `json:"isEncrypted"`
		IsIncoming         bool    `json:"isIncoming"`
		IsUTP              bool    `json:"isUTP"`
		IsUploadingTo      bool    `json:"isUploadingTo"`
		PeerIsChoked       bool    `json:"peerIsChoked"`
		PeerIsInterested   bool    `json:"peerIsInterested"`
		Port               int     `json:"port"`
		Progress           float64 `json:"progress"`
		RateToClient       int     `json:"rateToClient"`
		RateToPeer         int     `json:"rateToPeer"`
	}
)

func (t ByID) Len() int           { return len(t) }
func (t ByID) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t ByID) Less(i, j int) bool { return t[i].ID < t[j].ID }

func (t ByName) Len() int           { return len(t) }
func (t ByName) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t ByName) Less(i, j int) bool { return t[i].Name < t[j].Name }

func (t ByDate) Len() int           { return len(t) }
func (t ByDate) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t ByDate) Less(i, j int) bool { return t[i].Added < t[j].Added }

func (t ByRatio) Len() int           { return len(t) }
func (t ByRatio) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t ByRatio) Less(i, j int) bool { return t[i].UploadRatio < t[j].UploadRatio }
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"net/http"
)

const (
	scBear = "Bearer"
)

func responseAccountInfo(w http.ResponseWriter, info map[string]interface{}) error {
	buff, err := json.Marshal(info)
	if err != nil {
		return erro.Wrap(err)
	}

	if _, err := w.Write(buff); err != nil {
		err = erro.Wrap(err)
		log.Err(erro.Unwrap(err))
		log.Debug(err)
	}
	return nil
}

func accountInfoApi(w http.ResponseWriter, r *http.Request, sys *system) error {
	req := newAccountInfoRequest(r)

	if req.scheme() != scBear {
		return responseError(w, http.StatusBadRequest, errInvReq, "authorization scheme "+req.scheme()+" is not supported")
	}

	log.Debug("Authrization scheme " + req.scheme() + " is OK")

	tokId := req.token()
	if tokId == "" {
		return responseError(w, http.StatusBadRequest, errInvReq, "no token")
	}

	log.Debug("Token " + mosaic(tokId) + " is declared")

	tok, err := sys.tokCont.get(tokId)
	if err != nil {
		return erro.Wrap(err)
	} else if tok == nil {
		return responseError(w, http.StatusBadRequest, errInvTok, "token "+mosaic(tokId)+" is not exist")
	} else if !tok.valid() {
		return responseError(w, http.StatusBadRequest, errInvTok, "token "+mosaic(tokId)+" is invalid")
	}

	log.Debug("Token " + mosaic(tokId) + " is exist")

	t, err := sys.taCont.get(tok.taId())
	if err != nil {
		return erro.Wrap(err)
	} else if t == nil {
		return responseError(w, http.StatusBadRequest, errInvTok, "token "+mosaic(tokId)+" is linked to invalid TA "+tok.taId())
	}

	log.Debug("Token TA " + t.id() + " is exist")

	acc, err := sys.accCont.get(tok.accountId())
	if err != nil {
		return erro.Wrap(err)
	} else if acc == nil {
		return responseError(w, http.StatusBadRequest, errInvTok, "token "+mosaic(tokId)+" is linked to invalid account "+tok.accountId())
	}

	log.Debug("Token account " + acc.id() + " is exist")

	clms := scopesToClaims(tok.scopes())
	for clm := range tok.claims() {
		clms[clm] = true
	}

	log.Debug("Token claims ", clms, " will be returned")

	info := map[string]interface{}{}
	for clmName := range clms {
		clm := acc.attribute(clmName)
		if clm == nil || clm == "" {
			continue
		}
		info[clmName] = clm
	}

	return responseAccountInfo(w, info)
	// panic("not yet implemented")
}
<commit_msg>コメントアウトされてた要らないコードを削除<commit_after>package main

import (
	"encoding/json"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"net/http"
)

const (
	scBear = "Bearer"
)

func responseAccountInfo(w http.ResponseWriter, info map[string]interface{}) error {
	buff, err := json.Marshal(info)
	if err != nil {
		return erro.Wrap(err)
	}

	if _, err := w.Write(buff); err != nil {
		err = erro.Wrap(err)
		log.Err(erro.Unwrap(err))
		log.Debug(err)
	}
	return nil
}

func accountInfoApi(w http.ResponseWriter, r *http.Request, sys *system) error {
	req := newAccountInfoRequest(r)

	if req.scheme() != scBear {
		return responseError(w, http.StatusBadRequest, errInvReq, "authorization scheme "+req.scheme()+" is not supported")
	}

	log.Debug("Authrization scheme " + req.scheme() + " is OK")

	tokId := req.token()
	if tokId == "" {
		return responseError(w, http.StatusBadRequest, errInvReq, "no token")
	}

	log.Debug("Token " + mosaic(tokId) + " is declared")

	tok, err := sys.tokCont.get(tokId)
	if err != nil {
		return erro.Wrap(err)
	} else if tok == nil {
		return responseError(w, http.StatusBadRequest, errInvTok, "token "+mosaic(tokId)+" is not exist")
	} else if !tok.valid() {
		return responseError(w, http.StatusBadRequest, errInvTok, "token "+mosaic(tokId)+" is invalid")
	}

	log.Debug("Token " + mosaic(tokId) + " is exist")

	t, err := sys.taCont.get(tok.taId())
	if err != nil {
		return erro.Wrap(err)
	} else if t == nil {
		return responseError(w, http.StatusBadRequest, errInvTok, "token "+mosaic(tokId)+" is linked to invalid TA "+tok.taId())
	}

	log.Debug("Token TA " + t.id() + " is exist")

	acc, err := sys.accCont.get(tok.accountId())
	if err != nil {
		return erro.Wrap(err)
	} else if acc == nil {
		return responseError(w, http.StatusBadRequest, errInvTok, "token "+mosaic(tokId)+" is linked to invalid account "+tok.accountId())
	}

	log.Debug("Token account " + acc.id() + " is exist")

	clms := scopesToClaims(tok.scopes())
	for clm := range tok.claims() {
		clms[clm] = true
	}

	log.Debug("Token claims ", clms, " will be returned")

	info := map[string]interface{}{}
	for clmName := range clms {
		clm := acc.attribute(clmName)
		if clm == nil || clm == "" {
			continue
		}
		info[clmName] = clm
	}

	return responseAccountInfo(w, info)
}
<|endoftext|>


<commit_before>package testutil

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"strings"

	"v.io/tools/lib/runutil"
	"v.io/tools/lib/util"
)

var (
	// cleanGo is used to control whether the initTest function removes
	// all stale Go object files and binaries. It is use to prevent the
	// test of this package from interfering with other concurrently
	// running tests that might be sharing the same object files.
	cleanGo = true
)

const (
	// Number of lines to be included in the error messsage of an xUnit report.
	numLinesToOutput = 15
)

// binDirPath returns the path to the directory for storing temporary
// binaries.
func binDirPath() string {
	return filepath.Join(os.Getenv("TMPDIR"), "bin")
}

// initTest carries out the initial actions for the given test.
func initTest(ctx *util.Context, testName string, profiles []string) (func() error, error) {
	// Output the hostname.
	hostname, err := os.Hostname()
	if err != nil {
		return nil, fmt.Errorf("Hostname() failed: %v", err)
	}
	fmt.Fprintf(ctx.Stdout(), "hostname = %q\n", hostname)

	// Create a working test directory under $HOME/tmp and set the
	// TMPDIR environment variable to it.
	rootDir := filepath.Join(os.Getenv("HOME"), "tmp", testName)
	if err := ctx.Run().MkdirAll(rootDir, os.FileMode(0755)); err != nil {
		return nil, err
	}
	workDir, err := ctx.Run().TempDir(rootDir, "")
	if err != nil {
		return nil, fmt.Errorf("TempDir() failed: %v", err)
	}
	if err := os.Setenv("TMPDIR", workDir); err != nil {
		return nil, err
	}
	fmt.Fprintf(ctx.Stdout(), "workdir = %q\n", workDir)

	// Create a temporary directory for storing binaries.
	if err := ctx.Run().MkdirAll(binDirPath(), os.FileMode(0755)); err != nil {
		return nil, err
	}

	// Setup profiles.
	for _, profile := range profiles {
		if err := ctx.Run().Command("v23", "profile", "setup", profile); err != nil {
			return nil, err
		}
	}

	// Descend into the working directory (unless doing a "dry
	// run" in which case the working directory does not exist).
	cwd, err := os.Getwd()
	if err != nil {
		return nil, err
	}
	if !ctx.DryRun() {
		if err := ctx.Run().Chdir(workDir); err != nil {
			return nil, err
		}
	}

	// Remove all stale Go object files and binaries.
	if cleanGo {
		if err := ctx.Run().Command("v23", "goext", "distclean"); err != nil {
			return nil, err
		}
	}

	// Remove xUnit test report file.
	if err := ctx.Run().RemoveAll(XUnitReportPath(testName)); err != nil {
		return nil, err
	}

	return func() error {
		return ctx.Run().Chdir(cwd)
	}, nil
}

// genXUnitReportOnCmdError generates an xUnit test report if the given command
// function returns an error.
func genXUnitReportOnCmdError(ctx *util.Context, testName, testCaseName, failureSummary string, commandFunc func(runutil.Opts) error) (*TestResult, error) {
	var out bytes.Buffer
	opts := ctx.Run().Opts()
	opts.Stdout = io.MultiWriter(&out, opts.Stdout)
	opts.Stderr = io.MultiWriter(&out, opts.Stderr)
	if err := commandFunc(opts); err != nil {
		xUnitFilePath := XUnitReportPath(testName)

		// Create a test suite to wrap up the error.
		// Include last <numLinesToOutput> lines of the output in the error message.
		lines := strings.Split(out.String(), "\n")
		startLine := int(math.Max(0, float64(len(lines)-numLinesToOutput)))
		errMsg := "......\n" + strings.Join(lines[startLine:], "\n")
		s := createTestSuiteWithFailure(testName, testCaseName, failureSummary, errMsg, 0)
		suites := []testSuite{*s}

		// xUnit file exists, append existing suites.
		if _, err := os.Stat(xUnitFilePath); err == nil {
			bytes, err := ioutil.ReadFile(xUnitFilePath)
			if err != nil {
				return nil, fmt.Errorf("ReadFile(%s) failed: %v", xUnitFilePath, err)
			}
			var existingSuites testSuites
			if err := xml.Unmarshal(bytes, &existingSuites); err != nil {
				return nil, fmt.Errorf("Unmarshal() failed: %v\n%v", err, string(bytes))
			}
			suites = append(suites, existingSuites.Suites...)
		}

		// Create xUnit report with suites.
		if err := createXUnitReport(ctx, testName, suites); err != nil {
			return nil, err
		}

		// Return test result.
		if err == runutil.CommandTimedOutErr {
			return &TestResult{Status: TestTimedOut}, nil
		}
		return &TestResult{Status: TestFailed}, nil
	}
	return nil, nil
}

func Pass(ctx *util.Context, format string, a ...interface{}) {
	strOK := "ok"
	if ctx.Color() {
		strOK = util.ColorString("ok", util.Green)
	}
	fmt.Fprintf(ctx.Stdout(), "%s   ", strOK)
	fmt.Fprintf(ctx.Stdout(), format, a...)
}

func Fail(ctx *util.Context, format string, a ...interface{}) {
	strFail := "fail"
	if ctx.Color() {
		strFail = util.ColorString("fail", util.Red)
	}
	fmt.Fprintf(ctx.Stderr(), "%s ", strFail)
	fmt.Fprintf(ctx.Stderr(), format, a...)
}
<commit_msg>TBR: tools/lib/testutil: don't fail on empty xUnit file.<commit_after>package testutil

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"strings"

	"v.io/tools/lib/runutil"
	"v.io/tools/lib/util"
)

var (
	// cleanGo is used to control whether the initTest function removes
	// all stale Go object files and binaries. It is use to prevent the
	// test of this package from interfering with other concurrently
	// running tests that might be sharing the same object files.
	cleanGo = true
)

const (
	// Number of lines to be included in the error messsage of an xUnit report.
	numLinesToOutput = 15
)

// binDirPath returns the path to the directory for storing temporary
// binaries.
func binDirPath() string {
	return filepath.Join(os.Getenv("TMPDIR"), "bin")
}

// initTest carries out the initial actions for the given test.
func initTest(ctx *util.Context, testName string, profiles []string) (func() error, error) {
	// Output the hostname.
	hostname, err := os.Hostname()
	if err != nil {
		return nil, fmt.Errorf("Hostname() failed: %v", err)
	}
	fmt.Fprintf(ctx.Stdout(), "hostname = %q\n", hostname)

	// Create a working test directory under $HOME/tmp and set the
	// TMPDIR environment variable to it.
	rootDir := filepath.Join(os.Getenv("HOME"), "tmp", testName)
	if err := ctx.Run().MkdirAll(rootDir, os.FileMode(0755)); err != nil {
		return nil, err
	}
	workDir, err := ctx.Run().TempDir(rootDir, "")
	if err != nil {
		return nil, fmt.Errorf("TempDir() failed: %v", err)
	}
	if err := os.Setenv("TMPDIR", workDir); err != nil {
		return nil, err
	}
	fmt.Fprintf(ctx.Stdout(), "workdir = %q\n", workDir)

	// Create a temporary directory for storing binaries.
	if err := ctx.Run().MkdirAll(binDirPath(), os.FileMode(0755)); err != nil {
		return nil, err
	}

	// Setup profiles.
	for _, profile := range profiles {
		if err := ctx.Run().Command("v23", "profile", "setup", profile); err != nil {
			return nil, err
		}
	}

	// Descend into the working directory (unless doing a "dry
	// run" in which case the working directory does not exist).
	cwd, err := os.Getwd()
	if err != nil {
		return nil, err
	}
	if !ctx.DryRun() {
		if err := ctx.Run().Chdir(workDir); err != nil {
			return nil, err
		}
	}

	// Remove all stale Go object files and binaries.
	if cleanGo {
		if err := ctx.Run().Command("v23", "goext", "distclean"); err != nil {
			return nil, err
		}
	}

	// Remove xUnit test report file.
	if err := ctx.Run().RemoveAll(XUnitReportPath(testName)); err != nil {
		return nil, err
	}

	return func() error {
		return ctx.Run().Chdir(cwd)
	}, nil
}

// genXUnitReportOnCmdError generates an xUnit test report if the given command
// function returns an error.
func genXUnitReportOnCmdError(ctx *util.Context, testName, testCaseName, failureSummary string, commandFunc func(runutil.Opts) error) (*TestResult, error) {
	var out bytes.Buffer
	opts := ctx.Run().Opts()
	opts.Stdout = io.MultiWriter(&out, opts.Stdout)
	opts.Stderr = io.MultiWriter(&out, opts.Stderr)
	if err := commandFunc(opts); err != nil {
		xUnitFilePath := XUnitReportPath(testName)

		// Create a test suite to wrap up the error.
		// Include last <numLinesToOutput> lines of the output in the error message.
		lines := strings.Split(out.String(), "\n")
		startLine := int(math.Max(0, float64(len(lines)-numLinesToOutput)))
		errMsg := "......\n" + strings.Join(lines[startLine:], "\n")
		s := createTestSuiteWithFailure(testName, testCaseName, failureSummary, errMsg, 0)
		suites := []testSuite{*s}

		// xUnit file exists, append existing suites.
		if _, err := os.Stat(xUnitFilePath); err == nil {
			bytes, err := ioutil.ReadFile(xUnitFilePath)
			if err != nil {
				return nil, fmt.Errorf("ReadFile(%s) failed: %v", xUnitFilePath, err)
			}
			var existingSuites testSuites
			if err := xml.Unmarshal(bytes, &existingSuites); err != nil {
				fmt.Fprintf(ctx.Stderr(), "Unmarshal() failed: %v\n%v", err, string(bytes))
			} else {
				suites = append(suites, existingSuites.Suites...)
			}
		}

		// Create xUnit report with suites.
		if err := createXUnitReport(ctx, testName, suites); err != nil {
			return nil, err
		}

		// Return test result.
		if err == runutil.CommandTimedOutErr {
			return &TestResult{Status: TestTimedOut}, nil
		}
		return &TestResult{Status: TestFailed}, nil
	}
	return nil, nil
}

func Pass(ctx *util.Context, format string, a ...interface{}) {
	strOK := "ok"
	if ctx.Color() {
		strOK = util.ColorString("ok", util.Green)
	}
	fmt.Fprintf(ctx.Stdout(), "%s   ", strOK)
	fmt.Fprintf(ctx.Stdout(), format, a...)
}

func Fail(ctx *util.Context, format string, a ...interface{}) {
	strFail := "fail"
	if ctx.Color() {
		strFail = util.ColorString("fail", util.Red)
	}
	fmt.Fprintf(ctx.Stderr(), "%s ", strFail)
	fmt.Fprintf(ctx.Stderr(), format, a...)
}
<|endoftext|>


<commit_before>package network

import(
	"net"
	"fmt"
	"log"
	)

type ID string
type Message struct {
    Address ID
    Data    []byte
}
var client_port int = ;
var master_port int = ;

func getSenderID() {

}
func broadcast(){

}
func listen() {
	//local
	//socket

}
func bind(port int) *net.UDPconn{
	//local, err
	//Socket, err
}
func clientWorker() {
	//Create socket
	// bind
	// listen
	//broadcast
	//close socket
}
func masterWorker( from_client, to_clients chan Message) {
	//Create socket
	// need to bind
	// broadcast
	//listen
	// close socket

}
<commit_msg>Delete Network.go<commit_after><|endoftext|>


<commit_before>package configstack

import (
	"github.com/gruntwork-io/terragrunt/config"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/gruntwork-io/terragrunt/util"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

func TestFindStackInSubfolders(t *testing.T) {
	t.Parallel()

	filePaths := []string{
		"/stage/data-stores/redis/" + config.DefaultTerragruntConfigPath,
		"/stage/data-stores/postgres/" + config.DefaultTerragruntConfigPath,
		"/stage/ecs-cluster/" + config.DefaultTerragruntConfigPath,
		"/stage/kms-master-key/" + config.DefaultTerragruntConfigPath,
		"/stage/vpc/" + config.DefaultTerragruntConfigPath,
	}

	tempFolder := createTempFolder(t)
	writeDummyTerragruntConfigs(t, tempFolder, filePaths)

	envFolder := filepath.ToSlash(util.JoinPath(tempFolder + "/stage"))
	terragruntOptions := options.NewTerragruntOptions(envFolder)
	terragruntOptions.WorkingDir = envFolder

	stack, err := FindStackInSubfolders(terragruntOptions)
	if err != nil {
		t.Fatalf("Failed when calling method under test: %s\n", err.Error())
	}

	var modulePaths []string

	for _, module := range stack.Modules {
		relPath := strings.Replace(module.Path, tempFolder, "", 1)
		relPath = filepath.ToSlash(util.JoinPath(relPath, config.DefaultTerragruntConfigPath))

		modulePaths = append(modulePaths, relPath)
	}

	for _, filePath := range filePaths {
		filePathFound := util.ListContainsElement(modulePaths, filePath)
		assert.True(t, filePathFound, "The filePath %s was not found by Terragrunt.\n", filePath)
	}

}

func createTempFolder(t *testing.T) string {
	tmpFolder, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("Failed to create temp directory: %s\n", err.Error())
	}

	return filepath.ToSlash(tmpFolder)
}

// Create a dummy Terragrunt config file at each of the given paths
func writeDummyTerragruntConfigs(t *testing.T, tmpFolder string, paths []string) {
	contents := []byte("terragrunt = {}")
	for _, path := range paths {
		absPath := util.JoinPath(tmpFolder, path)

		containingDir := filepath.Dir(absPath)
		createDirIfNotExist(t, containingDir)

		err := ioutil.WriteFile(absPath, contents, os.ModePerm)
		if err != nil {
			t.Fatalf("Failed to write file at path %s: %s\n", path, err.Error())
		}
	}
}

func createDirIfNotExist(t *testing.T, path string) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		err = os.MkdirAll(path, os.ModePerm)
		if err != nil {
			t.Fatalf("Failed to create directory: %s\n", err.Error())
		}
	}
}
<commit_msg>Give the dummy .tfvars files a nested terraform { source = "..." } so they won't be excluded by the new logic<commit_after>package configstack

import (
	"github.com/gruntwork-io/terragrunt/config"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/gruntwork-io/terragrunt/util"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

func TestFindStackInSubfolders(t *testing.T) {
	t.Parallel()

	filePaths := []string{
		"/stage/data-stores/redis/" + config.DefaultTerragruntConfigPath,
		"/stage/data-stores/postgres/" + config.DefaultTerragruntConfigPath,
		"/stage/ecs-cluster/" + config.DefaultTerragruntConfigPath,
		"/stage/kms-master-key/" + config.DefaultTerragruntConfigPath,
		"/stage/vpc/" + config.DefaultTerragruntConfigPath,
	}

	tempFolder := createTempFolder(t)
	writeDummyTerragruntConfigs(t, tempFolder, filePaths)

	envFolder := filepath.ToSlash(util.JoinPath(tempFolder + "/stage"))
	terragruntOptions := options.NewTerragruntOptions(envFolder)
	terragruntOptions.WorkingDir = envFolder

	stack, err := FindStackInSubfolders(terragruntOptions)
	if err != nil {
		t.Fatalf("Failed when calling method under test: %s\n", err.Error())
	}

	var modulePaths []string

	for _, module := range stack.Modules {
		relPath := strings.Replace(module.Path, tempFolder, "", 1)
		relPath = filepath.ToSlash(util.JoinPath(relPath, config.DefaultTerragruntConfigPath))

		modulePaths = append(modulePaths, relPath)
	}

	for _, filePath := range filePaths {
		filePathFound := util.ListContainsElement(modulePaths, filePath)
		assert.True(t, filePathFound, "The filePath %s was not found by Terragrunt.\n", filePath)
	}

}

func createTempFolder(t *testing.T) string {
	tmpFolder, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("Failed to create temp directory: %s\n", err.Error())
	}

	return filepath.ToSlash(tmpFolder)
}

// Create a dummy Terragrunt config file at each of the given paths
func writeDummyTerragruntConfigs(t *testing.T, tmpFolder string, paths []string) {
	contents := []byte("terragrunt = {\nterraform {\nsource = \"test\"\n}\n}")
	for _, path := range paths {
		absPath := util.JoinPath(tmpFolder, path)

		containingDir := filepath.Dir(absPath)
		createDirIfNotExist(t, containingDir)

		err := ioutil.WriteFile(absPath, contents, os.ModePerm)
		if err != nil {
			t.Fatalf("Failed to write file at path %s: %s\n", path, err.Error())
		}
	}
}

func createDirIfNotExist(t *testing.T, path string) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		err = os.MkdirAll(path, os.ModePerm)
		if err != nil {
			t.Fatalf("Failed to create directory: %s\n", err.Error())
		}
	}
}
<|endoftext|>


<commit_before>package utils

import (
	"encoding/binary"
	"encoding/json"
	"html"
	"io"
	"net"
	"os"
	"os/signal"
	"syscall"
)

func CatchExit(callback func()) {
	sig := make(chan os.Signal)
	signal.Notify(sig, os.Kill, os.Interrupt, syscall.SIGTERM)

	go func() {
		for {
			switch <-sig {
			case os.Kill, os.Interrupt, syscall.SIGTERM:
				callback()
				os.Exit(1)
			}
		}
	}()
}

func CopyFile(src, dst string) (int64, error) {
	if src == dst {
		return 0, nil
	}
	sf, err := os.Open(src)
	if err != nil {
		return 0, err
	}
	if _, err := os.Lstat(dst); err != nil && !os.IsNotExist(err) {
		return 0, err
	}
	df, err := os.Create(dst)
	if err != nil {
		return 0, err
	}
	defer df.Close()
	return io.Copy(df, sf)
}

func JSONUnmarshalFile(filename string, v interface{}) (err error) {
	f, err := os.Open(filename)
	if err != nil {
		return
	}
	defer f.Close()
	return json.NewDecoder(f).Decode(v)
}

func JSONMarshalFile(filename string, v interface{}) (err error) {
	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return
	}
	defer f.Close()
	return json.NewEncoder(f).Encode(v)
}

func HtmlToText(s string, limit int) string {
	var i int
	var tagStart bool
	var sigStart bool
	var sig []rune
	text := make([]rune, len(s))
	appendr := func(r rune) {
		text[i] = r
		i++
	}
	for _, r := range []rune(s) {
		if limit > -1 && i >= limit {
			break
		}
		switch r {
		case '<':
			tagStart = true
		case '>':
			tagStart = false
		case '&':
			if tagStart {
				break
			}
			sigStart = true
			sig = nil
		case ';':
			if tagStart {
				break
			}
			if sigStart {
				if tr := []rune(html.UnescapeString(string(append([]rune{'&'}, append(sig, ';')...)))); len(tr) > 0 {
					appendr(tr[0])
				}
			}
			sigStart = false
			sig = nil
		default:
			if tagStart {
				break
			}
			if sigStart {
				if r >= 'A' && r <= 'Z' {
					r += 32 // ToLower
				}
				sig = append(sig, r)
				break
			}
			appendr(r)
		}
	}
	return string(text[:i])
}

func SplitToLines(s string, chars string) (lines []string) {
	for i, j, l := 0, 0, len(s); i < l; i++ {
		switch s[i] {
		case '\r', '\n':
			if i > j {
				lines = append(lines, s[j:i])
			}
			j = i + 1
		default:
			if i == l-1 && j < l {
				lines = append(lines, s[j:])
			}
		}
	}
	return
}

func SplitByFirstByte(s string, c byte) (string, string) {
	for i, l := 0, len(s); i < l; i++ {
		if s[i] == c {
			return s[:i], s[i+1:]
		}
	}
	return s, ""
}

func SplitByLastByte(s string, c byte) (string, string) {
	for i := len(s) - 1; i >= 0; i-- {
		if s[i] == c {
			return s[:i], s[i+1:]
		}
	}
	return s, ""
}

func FileExt(filename string) (ext string) {
	for i := len(filename) - 1; i > 0; i-- {
		if filename[i] == '.' {
			ext = filename[i+1:]
			break
		}
	}
	return
}

// PathClean has the same function with path.Clean(strings.ToLower(strings.Replace(strings.TrimSpace(s), "\\", "/", -1))),
// but it's faster!
func PathClean(path string, toLower bool) string {
	pl := len(path)
	if pl == 0 {
		return "."
	}
	var n int
	var c byte
	var root bool
	var newpath = make([]byte, pl)
	for i := 0; i < pl; i++ {
		switch c = path[i]; c {
		case ' ':
			if n == 0 {
				continue
			}
			newpath[n] = ' '
			n++
		case '/', '\\':
			if n > 0 {
				if newpath[n-1] == '/' {
					continue
				} else if newpath[n-1] == '.' && n > 1 && newpath[n-2] == '/' {
					n--
					continue
				}
			}
			if n == 0 {
				root = true
			}
			newpath[n] = '/'
			n++
		case '.':
			if n > 1 && newpath[n-1] == '.' && newpath[n-2] == '/' {
				if n = n - 2; n > 0 {
					for n = n - 1; n > 0; n-- {
						if newpath[n] == '/' {
							break
						}
					}
				}
				continue
			}
			newpath[n] = '.'
			n++
		default:
			if toLower && c >= 'A' && c <= 'Z' {
				c += 32 // ToLower
			}
			newpath[n] = c
			n++
		}
	}
	// trim right spaces
	if n > 0 && newpath[n-1] == ' ' {
		for n > 0 && newpath[n-1] == ' ' {
			n--
		}
	}
	if n > 1 && newpath[n-1] == '.' && newpath[n-2] == '/' {
		n--
	}
	if n > 0 && newpath[n-1] == '/' && (!root || n > 1) {
		n--
	}
	if n == 0 {
		return "."
	}
	return string(newpath[:n])
}

func Ipv4ToLong(ipStr string) uint32 {
	ip := net.ParseIP(ipStr)
	if ip == nil {
		return 0
	}
	ip = ip.To4()
	return binary.BigEndian.Uint32(ip)
}

func LongToIpv4(ipLong uint32) string {
	ipByte := make([]byte, 4)
	binary.BigEndian.PutUint32(ipByte, ipLong)
	ip := net.IP(ipByte)
	return ip.String()
}
<commit_msg>add Contains method of utils package<commit_after>package utils

import (
	"encoding/binary"
	"encoding/json"
	"html"
	"io"
	"net"
	"os"
	"os/signal"
	"syscall"
)

func Contains(p interface{}, c interface{}) bool {
	switch a := p.(type) {
	case []string:
		if len(a) == 0 {
			return false
		}
		s, ok := c.(string)
		if !ok {
			return false
		}
		for _, i := range a {
			if i == s {
				return true
			}
		}
		return false
	case []interface{}:
		if len(a) == 0 {
			return false
		}
		for _, i := range a {
			if i == c {
				return true
			}
		}
		return false
	default:
		return false
	}
}

func CatchExit(callback func()) {
	sig := make(chan os.Signal)
	signal.Notify(sig, os.Kill, os.Interrupt, syscall.SIGTERM)

	go func() {
		for {
			switch <-sig {
			case os.Kill, os.Interrupt, syscall.SIGTERM:
				callback()
				os.Exit(1)
			}
		}
	}()
}

func CopyFile(src, dst string) (int64, error) {
	if src == dst {
		return 0, nil
	}
	sf, err := os.Open(src)
	if err != nil {
		return 0, err
	}
	if _, err := os.Lstat(dst); err != nil && !os.IsNotExist(err) {
		return 0, err
	}
	df, err := os.Create(dst)
	if err != nil {
		return 0, err
	}
	defer df.Close()
	return io.Copy(df, sf)
}

func JSONUnmarshalFile(filename string, v interface{}) (err error) {
	f, err := os.Open(filename)
	if err != nil {
		return
	}
	defer f.Close()
	return json.NewDecoder(f).Decode(v)
}

func JSONMarshalFile(filename string, v interface{}) (err error) {
	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return
	}
	defer f.Close()
	return json.NewEncoder(f).Encode(v)
}

func HtmlToText(s string, limit int) string {
	var i int
	var tagStart bool
	var sigStart bool
	var sig []rune
	text := make([]rune, len(s))
	appendr := func(r rune) {
		text[i] = r
		i++
	}
	for _, r := range []rune(s) {
		if limit > -1 && i >= limit {
			break
		}
		switch r {
		case '<':
			tagStart = true
		case '>':
			tagStart = false
		case '&':
			if tagStart {
				break
			}
			sigStart = true
			sig = nil
		case ';':
			if tagStart {
				break
			}
			if sigStart {
				if tr := []rune(html.UnescapeString(string(append([]rune{'&'}, append(sig, ';')...)))); len(tr) > 0 {
					appendr(tr[0])
				}
			}
			sigStart = false
			sig = nil
		default:
			if tagStart {
				break
			}
			if sigStart {
				if r >= 'A' && r <= 'Z' {
					r += 32 // ToLower
				}
				sig = append(sig, r)
				break
			}
			appendr(r)
		}
	}
	return string(text[:i])
}

func SplitToLines(s string, chars string) (lines []string) {
	for i, j, l := 0, 0, len(s); i < l; i++ {
		switch s[i] {
		case '\r', '\n':
			if i > j {
				lines = append(lines, s[j:i])
			}
			j = i + 1
		default:
			if i == l-1 && j < l {
				lines = append(lines, s[j:])
			}
		}
	}
	return
}

func SplitByFirstByte(s string, c byte) (string, string) {
	for i, l := 0, len(s); i < l; i++ {
		if s[i] == c {
			return s[:i], s[i+1:]
		}
	}
	return s, ""
}

func SplitByLastByte(s string, c byte) (string, string) {
	for i := len(s) - 1; i >= 0; i-- {
		if s[i] == c {
			return s[:i], s[i+1:]
		}
	}
	return s, ""
}

func FileExt(filename string) (ext string) {
	for i := len(filename) - 1; i > 0; i-- {
		if filename[i] == '.' {
			ext = filename[i+1:]
			break
		}
	}
	return
}

// PathClean has the same function with path.Clean(strings.ToLower(strings.Replace(strings.TrimSpace(s), "\\", "/", -1))),
// but it's faster!
func PathClean(path string, toLower bool) string {
	pl := len(path)
	if pl == 0 {
		return "."
	}
	var n int
	var c byte
	var root bool
	var newpath = make([]byte, pl)
	for i := 0; i < pl; i++ {
		switch c = path[i]; c {
		case ' ':
			if n == 0 {
				continue
			}
			newpath[n] = ' '
			n++
		case '/', '\\':
			if n > 0 {
				if newpath[n-1] == '/' {
					continue
				} else if newpath[n-1] == '.' && n > 1 && newpath[n-2] == '/' {
					n--
					continue
				}
			}
			if n == 0 {
				root = true
			}
			newpath[n] = '/'
			n++
		case '.':
			if n > 1 && newpath[n-1] == '.' && newpath[n-2] == '/' {
				if n = n - 2; n > 0 {
					for n = n - 1; n > 0; n-- {
						if newpath[n] == '/' {
							break
						}
					}
				}
				continue
			}
			newpath[n] = '.'
			n++
		default:
			if toLower && c >= 'A' && c <= 'Z' {
				c += 32 // ToLower
			}
			newpath[n] = c
			n++
		}
	}
	// trim right spaces
	if n > 0 && newpath[n-1] == ' ' {
		for n > 0 && newpath[n-1] == ' ' {
			n--
		}
	}
	if n > 1 && newpath[n-1] == '.' && newpath[n-2] == '/' {
		n--
	}
	if n > 0 && newpath[n-1] == '/' && (!root || n > 1) {
		n--
	}
	if n == 0 {
		return "."
	}
	return string(newpath[:n])
}

func Ipv4ToLong(ipStr string) uint32 {
	ip := net.ParseIP(ipStr)
	if ip == nil {
		return 0
	}
	ip = ip.To4()
	return binary.BigEndian.Uint32(ip)
}

func LongToIpv4(ipLong uint32) string {
	ipByte := make([]byte, 4)
	binary.BigEndian.PutUint32(ipByte, ipLong)
	ip := net.IP(ipByte)
	return ip.String()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"image"
	"image/color"
	"image/draw"
	"image/jpeg"
	"log"
	"net/http"
	"os"
	"strconv"
)

var root = flag.String("root", "app", "file system path")

func main() {
	http.HandleFunc("/black/", blackHandler)
	http.HandleFunc("/green/", greenHandler)
	http.HandleFunc("/blue/", blueHandler)
	http.Handle("/", http.FileServer(http.Dir(*root)))
	log.Println("Listening on " + os.Getenv("PORT"))
	err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}

func blackHandler(w http.ResponseWriter, r *http.Request) {
	m := image.NewRGBA(image.Rect(0, 0, 240, 240))
	black := color.RGBA{0, 0, 0, 255}
	draw.Draw(m, m.Bounds(), &image.Uniform{black}, image.ZP, draw.Src)

	var img image.Image = m
	writeImage(w, &img)
}

func greenHandler(w http.ResponseWriter, r *http.Request) {
	// prepare
	m := image.NewRGBA(image.Rect(0, 0, 240, 240))
	green := color.RGBA{0, 128, 0, 255}
	draw.Draw(m, m.Bounds(), &image.Uniform{green}, image.ZP, draw.Src)

	var img image.Image = m
	writeImage(w, &img)
}

func blueHandler(w http.ResponseWriter, r *http.Request) {
	m := image.NewRGBA(image.Rect(0, 0, 240, 240))
	blue := color.RGBA{0, 0, 255, 255}
	draw.Draw(m, m.Bounds(), &image.Uniform{blue}, image.ZP, draw.Src)

	var img image.Image = m
	writeImage(w, &img)
}

func writeImage(w http.ResponseWriter, img *image.Image) {

	buffer := new(bytes.Buffer)
	if err := jpeg.Encode(buffer, *img, nil); err != nil {
		log.Fatalln("unable to encode image")
	}

	w.Header().Set("Content-Type", "image/jpeg")
	w.Header().Set("Content-Length", strconv.Itoa(len(buffer.Bytes())))
	if _, err := w.Write(buffer.Bytes()); err != nil {
		log.Fatalln("unable to write image")
	}
}
<commit_msg>move image handles into separate file<commit_after>package main

import (
	"flag"
	"log"
	"net/http"
	"os"
)

var root = flag.String("root", "app", "file system path")

func main() {
	http.HandleFunc("/black/", blackHandler)
	http.HandleFunc("/green/", greenHandler)
	http.HandleFunc("/blue/", blueHandler)
	http.Handle("/", http.FileServer(http.Dir(*root)))
	log.Println("Listening on " + os.Getenv("PORT"))
	err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}
<|endoftext|>