<commit_before>package somatree

import "github.com/satori/go.uuid"

func (teg *SomaTreeElemGroup) updateCheckInstances() {
	// object has no checks
	if len(teg.Checks) == 0 {
		return
	}

	// process checks
checksloop:
	for i, _ := range teg.Checks {
		if teg.Checks[i].Inherited == false && teg.Checks[i].ChildrenOnly == true {
			continue checksloop
		}
		if teg.Checks[i].View == "local" {
			continue checksloop
		}
		hasBrokenConstraint := false
		hasServiceConstraint := false
		hasAttributeConstraint := false
		view := teg.Checks[i].View

		attributes := []CheckConstraint{}
		oncallC := ""                                  // Id
		systemC := map[string]string{}                 // Id->Value
		nativeC := map[string]string{}                 // Property->Value
		serviceC := map[string]string{}                // Id->Value
		customC := map[string]string{}                 // Id->Value
		attributeC := map[string]map[string][]string{} // svcId->attr->[ value, ... ]

		// these constaint types must always match for the instance to
		// be valid. defer service and attribute
	constraintcheck:
		for _, c := range teg.Checks[i].Constraints {
			switch c.Type {
			case "native":
				if teg.evalNativeProp(c.Key, c.Value) {
					nativeC[c.Key] = c.Value
				} else {
					hasBrokenConstraint = true
					break constraintcheck
				}
			case "system":
				if id, hit := teg.evalSystemProp(c.Key, c.Value, view); hit {
					systemC[id] = c.Value
				} else {
					hasBrokenConstraint = true
					break constraintcheck
				}
			case "oncall":
				if id, hit := teg.evalOncallProp(c.Key, c.Value, view); hit {
					oncallC = id
				} else {
					hasBrokenConstraint = true
					break constraintcheck
				}
			case "custom":
				if id, hit := teg.evalCustomProp(c.Key, c.Value, view); hit {
					customC[id] = c.Value
				} else {
					hasBrokenConstraint = true
					break constraintcheck
				}
			case "service":
				hasServiceConstraint = true
				if id, hit := teg.evalServiceProp(c.Key, c.Value, view); hit {
					serviceC[id] = c.Value
				} else {
					hasBrokenConstraint = true
					break constraintcheck
				}
			case "attribute":
				hasAttributeConstraint = true
				attributes = append(attributes, c)
			}
		}
		if hasBrokenConstraint {
			continue checksloop
		}

		if hasServiceConstraint && hasAttributeConstraint {
		svcattrloop:
			for id, _ := range serviceC {
				for _, attr := range attributes {
					hit := teg.evalAttributeOfService(id, view, attr.Key, attr.Value)
					if hit {
						attributeC[id][attr.Key] = append(attributeC[id][attr.Key], attr.Value)
					} else {
						hasBrokenConstraint = true
						break svcattrloop
					}
				}
			}
		} else if hasAttributeConstraint {
			attrCount := len(attributes)
			for _, attr := range attributes {
				hit, svcIdMap := teg.evalAttributeProp(view, attr.Key, attr.Value)
				if hit {
					for id, _ := range svcIdMap {
						serviceC[id] = svcIdMap[id]
						attributeC[id][attr.Key] = append(attributeC[id][attr.Key], attr.Value)
					}
				}
			}
			// delete all services that did not match all attributes
			for id, _ := range attributeC {
				if len(attributeC[id]) != attrCount {
					delete(serviceC, id)
					delete(attributeC, id)
				}
			}
			if len(serviceC) > 0 {
				hasServiceConstraint = true
			}
		}
		if hasBrokenConstraint {
			continue checksloop
		}
		// check triggered, create instances

		if !hasServiceConstraint {
			// TODO create 1x
			continue checksloop
		}

		for svcId, _ := range serviceC {
			svcCfg := teg.getServiceMap(svcId)

			// calculate how many instances this service spawns
			combinations := 1
			for attr, _ := range svcCfg {
				combinations = combinations * len(svcCfg[attr])
			}

			// build all attribute combinations
			results := make([]map[string]string, 0, combinations)
			for attr, _ := range svcCfg {
				if len(results) == 0 {
					for i, _ := range svcCfg[attr] {
						res := map[string]string{}
						res[attr] = svcCfg[attr][i]
						results = append(results, res)
					}
					continue
				}
				ires := make([]map[string]string, 0, combinations)
				for r, _ := range results {
					for j, _ := range svcCfg[attr] {
						res := map[string]string{}
						for k, v := range results[r] {
							res[k] = v
						}
						res[attr] = svcCfg[attr][j]
						ires = append(ires, res)
					}
				}
				results = ires
			}
			// build a CheckInstance for every result
			for _, y := range results {
				// ensure we have a full copy and not a header copy
				cfg := map[string]string{}
				for k, v := range y {
					cfg[k] = v
				}
				inst := CheckInstance{
					CheckId: func(id string) uuid.UUID {
						f, _ := uuid.FromString(id)
						return f
					}(i),
					ConstraintOncall:    oncallC,
					ConstraintService:   serviceC,
					ConstraintSystem:    systemC,
					ConstraintCustom:    customC,
					ConstraintNative:    nativeC,
					ConstraintAttribute: attributeC,
					InstanceService: func(id string) uuid.UUID {
						f, _ := uuid.FromString(id)
						return f
					}(svcId),
					InstanceServiceConfig: cfg,
				}
				inst.calcConstraintHash()
				inst.calcConstraintValHash()
				inst.calcInstanceSvcCfgHash()
				// TODO lookup existing instance ids for check in teg.CheckInstances
				for _, exInstId := range teg.CheckInstances[i] {
					exInst := teg.Instances[exInstId]
					// TODO check existing for same ConstraintHash
					if exInst.ConstraintHash == inst.ConstraintHash {
					}
					// TODO ... same ConstraintValHash
					// TODO ... ... same InstanceSvcCfgHash --> instance update
					// TODO ... same InstanceSvcCfgHash     --> instance update (new constraints)
					// TODO add new instances
					// TODO remove old instances
				}
				inst.Version = 0
				inst.CheckId, _ = uuid.FromString(i)
				inst.InstanceId = uuid.NewV4()
				teg.Instances[inst.InstanceId.String()] = inst
				teg.CheckInstances[i] = append(teg.CheckInstances[i], inst.InstanceId.String())
			}
		}
	}
}

func (teg *SomaTreeElemGroup) evalNativeProp(
	prop string, val string) bool {
	switch prop {
	case "environment":
		env := teg.Parent.(Bucketeer).GetEnvironment()
		if val == env {
			return true
		}
	case "object_type":
		if val == "node" {
			return true
		}
	case "object_state":
		if val == teg.State {
			return true
		}
	}
	return false
}

func (teg *SomaTreeElemGroup) evalSystemProp(
	prop string, val string, view string) (string, bool) {
	// TODO
	return "", false
}

func (teg *SomaTreeElemGroup) evalOncallProp(
	prop string, val string, view string) (string, bool) {
	// TODO
	return "", false
}

func (teg *SomaTreeElemGroup) evalCustomProp(
	prop string, val string, view string) (string, bool) {
	// TODO
	return "", false
}

func (teg *SomaTreeElemGroup) evalServiceProp(
	prop string, val string, view string) (string, bool) {
	// TODO
	return "", false
}

func (teg *SomaTreeElemGroup) evalAttributeOfService(
	svcName string, view string, attribute string, value string) bool {
	// TODO
	return false
}

func (teg *SomaTreeElemGroup) evalAttributeProp(
	view string, attr string, value string) (bool, map[string]string) {
	f := map[string]string{}
	// TODO
	return false, f
}

func (teg *SomaTreeElemGroup) getServiceMap(serviceId string) map[string][]string {
	svc := new(PropertyService)
	svc = teg.PropertyService[serviceId].(*PropertyService)

	res := map[string][]string{}
	for _, v := range svc.Attributes {
		res[v.Attribute] = append(res[v.Attribute], v.Value)
	}
	return res
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<commit_msg>WIP tree_element_group__configurator<commit_after>package somatree

import "github.com/satori/go.uuid"

func (teg *SomaTreeElemGroup) updateCheckInstances() {
	// object has no checks
	if len(teg.Checks) == 0 {
		return
	}

	// process checks
checksloop:
	for i, _ := range teg.Checks {
		if teg.Checks[i].Inherited == false && teg.Checks[i].ChildrenOnly == true {
			continue checksloop
		}
		if teg.Checks[i].View == "local" {
			continue checksloop
		}
		hasBrokenConstraint := false
		hasServiceConstraint := false
		hasAttributeConstraint := false
		view := teg.Checks[i].View

		attributes := []CheckConstraint{}
		oncallC := ""                                  // Id
		systemC := map[string]string{}                 // Id->Value
		nativeC := map[string]string{}                 // Property->Value
		serviceC := map[string]string{}                // Id->Value
		customC := map[string]string{}                 // Id->Value
		attributeC := map[string]map[string][]string{} // svcId->attr->[ value, ... ]

		newInstances := map[string]CheckInstance{}
		newCheckInstances := []string{}

		// these constaint types must always match for the instance to
		// be valid. defer service and attribute
	constraintcheck:
		for _, c := range teg.Checks[i].Constraints {
			switch c.Type {
			case "native":
				if teg.evalNativeProp(c.Key, c.Value) {
					nativeC[c.Key] = c.Value
				} else {
					hasBrokenConstraint = true
					break constraintcheck
				}
			case "system":
				if id, hit := teg.evalSystemProp(c.Key, c.Value, view); hit {
					systemC[id] = c.Value
				} else {
					hasBrokenConstraint = true
					break constraintcheck
				}
			case "oncall":
				if id, hit := teg.evalOncallProp(c.Key, c.Value, view); hit {
					oncallC = id
				} else {
					hasBrokenConstraint = true
					break constraintcheck
				}
			case "custom":
				if id, hit := teg.evalCustomProp(c.Key, c.Value, view); hit {
					customC[id] = c.Value
				} else {
					hasBrokenConstraint = true
					break constraintcheck
				}
			case "service":
				hasServiceConstraint = true
				if id, hit := teg.evalServiceProp(c.Key, c.Value, view); hit {
					serviceC[id] = c.Value
				} else {
					hasBrokenConstraint = true
					break constraintcheck
				}
			case "attribute":
				hasAttributeConstraint = true
				attributes = append(attributes, c)
			}
		}
		if hasBrokenConstraint {
			continue checksloop
		}

		/* if the check has both service and attribute constraints,
		* then for the check to hit, the tree element needs to have
		* all the services, and each of them needs to match all
		* attribute constraints
		 */
		if hasServiceConstraint && hasAttributeConstraint {
		svcattrloop:
			for id, _ := range serviceC {
				for _, attr := range attributes {
					hit := teg.evalAttributeOfService(id, view, attr.Key, attr.Value)
					if hit {
						attributeC[id][attr.Key] = append(attributeC[id][attr.Key], attr.Value)
					} else {
						hasBrokenConstraint = true
						break svcattrloop
					}
				}
			}
			/* if the check has only attribute constraints and no
			* service constraint, then we pull in every service that
			* matches all attribute constraints and generate a check
			* instance for it
			 */
		} else if hasAttributeConstraint {
			attrCount := len(attributes)
			for _, attr := range attributes {
				hit, svcIdMap := teg.evalAttributeProp(view, attr.Key, attr.Value)
				if hit {
					for id, _ := range svcIdMap {
						serviceC[id] = svcIdMap[id]
						attributeC[id][attr.Key] = append(attributeC[id][attr.Key], attr.Value)
					}
				}
			}
			// delete all services that did not match all attributes
			for id, _ := range attributeC {
				if len(attributeC[id]) != attrCount {
					delete(serviceC, id)
					delete(attributeC, id)
				}
			}
			// declare service constraints in effect if we found a
			// service that bound all attribute constraints
			if len(serviceC) > 0 {
				hasServiceConstraint = true
			}
		}
		if hasBrokenConstraint {
			continue checksloop
		}
		// check triggered, create instances

		/* if there are no service constraints, one check instance is
		* created for this check
		 */
		if !hasServiceConstraint {
			inst := CheckInstance{
				InstanceId: uuid.UUID{},
				CheckId: func(id string) uuid.UUID {
					f, _ := uuid.FromString(id)
					return f
				}(i),
				ConstraintOncall:      oncallC,
				ConstraintService:     serviceC,
				ConstraintSystem:      systemC,
				ConstraintCustom:      customC,
				ConstraintNative:      nativeC,
				ConstraintAttribute:   attributeC,
				InstanceService:       uuid.Nil,
				InstanceServiceConfig: nil,
				InstanceSvcCfgHash:    "",
			}
			inst.calcConstraintHash()
			inst.calcConstraintValHash()

		nosvcinstanceloop:
			for _, exInstId := range teg.CheckInstances[i] {
				exInst := teg.Instances[exInstId]
				// ignore instances with service constraints
				if exInst.InstanceSvcCfgHash != "" {
					continue nosvcinstanceloop
				}
				// check if an instance exists bound against the same
				// constraints
				if exInst.ConstraintHash == inst.ConstraintHash {
					inst.InstanceId, _ = uuid.FromString(exInst.InstanceId.String())
					inst.Version = exInst.Version + 1
					break nosvcinstanceloop
				}
			}
			if uuid.Equal(uuid.Nil, inst.InstanceId) {
				// no match was found during nosvcinstanceloop, this
				// is a new instance
				inst.Version = 0
				inst.InstanceId = uuid.NewV4()
			}
			newInstances[inst.InstanceId.String()] = inst
			newCheckInstances = append(newCheckInstances, inst.InstanceId.String())
		}

		/* if service constraints are in effect, then we generate
		* instances for every service that bound.
		* Since service attributes can be specified more than once,
		* but the semantics are unclear what the expected behaviour of
		* for example a file age check is that is specified against
		* more than one file path; all possible attribute value
		* permutations for each service are built and then one check
		* instance is built for each of these service config
		* permutations.
		 */
	serviceconstraintloop:
		for svcId, _ := range serviceC {
			if !hasServiceConstraint {
				break serviceconstraintloop
			}

			svcCfg := teg.getServiceMap(svcId)

			// calculate how many instances this service spawns
			combinations := 1
			for attr, _ := range svcCfg {
				combinations = combinations * len(svcCfg[attr])
			}

			// build all attribute combinations
			results := make([]map[string]string, 0, combinations)
			for attr, _ := range svcCfg {
				if len(results) == 0 {
					for i, _ := range svcCfg[attr] {
						res := map[string]string{}
						res[attr] = svcCfg[attr][i]
						results = append(results, res)
					}
					continue
				}
				ires := make([]map[string]string, 0, combinations)
				for r, _ := range results {
					for j, _ := range svcCfg[attr] {
						res := map[string]string{}
						for k, v := range results[r] {
							res[k] = v
						}
						res[attr] = svcCfg[attr][j]
						ires = append(ires, res)
					}
				}
				results = ires
			}
			// build a CheckInstance for every result
			for _, y := range results {
				// ensure we have a full copy and not a header copy
				cfg := map[string]string{}
				for k, v := range y {
					cfg[k] = v
				}
				inst := CheckInstance{
					InstanceId: uuid.UUID{},
					CheckId: func(id string) uuid.UUID {
						f, _ := uuid.FromString(id)
						return f
					}(i),
					ConstraintOncall:    oncallC,
					ConstraintService:   serviceC,
					ConstraintSystem:    systemC,
					ConstraintCustom:    customC,
					ConstraintNative:    nativeC,
					ConstraintAttribute: attributeC,
					InstanceService: func(id string) uuid.UUID {
						f, _ := uuid.FromString(id)
						return f
					}(svcId),
					InstanceServiceConfig: cfg,
				}
				inst.calcConstraintHash()
				inst.calcConstraintValHash()
				inst.calcInstanceSvcCfgHash()

				// lookup existing instance ids for check in teg.CheckInstances
				// to determine if this is an update
			instanceloop:
				for _, exInstId := range teg.CheckInstances[i] {
					exInst := teg.Instances[exInstId]
					// this existing instance is for the same service
					// configuration -> this is an update
					if exInst.InstanceSvcCfgHash == inst.InstanceSvcCfgHash {
						inst.InstanceId, _ = uuid.FromString(exInst.InstanceId.String())
						inst.Version = exInst.Version + 1
						break instanceloop
					}
				}
				if uuid.Equal(uuid.Nil, inst.InstanceId) {
					// no match was found during instanceloop, this is
					// a new instance
					inst.Version = 0
					inst.InstanceId = uuid.NewV4()
				}
				newInstances[inst.InstanceId.String()] = inst
				newCheckInstances = append(newCheckInstances, inst.InstanceId.String())
			}
		} // LOOPEND: range serviceC
		// all new check instances have been built, check which
		// existing instances did not get an update and need to be
		// deleted
		for _, oldInstanceId := range teg.CheckInstances[i] {
			if _, ok := newInstances[oldInstanceId]; !ok {
				// there is no new version for this instance id
				// TODO action/instance_delete
				delete(teg.Instances, oldInstanceId)
				continue
			}
			// TODO action/instance_update
			delete(teg.Instances, oldInstanceId)
			teg.Instances[oldInstanceId] = newInstances[oldInstanceId]
		}
		for _, newInstanceId := range newCheckInstances {
			if _, ok := teg.Instances[newInstanceId]; !ok {
				// this instance is new, not an update
				// TODO action/instance_create
				teg.Instances[newInstanceId] = newInstances[newInstanceId]
			}
		}
		delete(teg.CheckInstances, i)
		teg.CheckInstances[i] = newCheckInstances
	} // LOOPEND: range teg.Checks
}

func (teg *SomaTreeElemGroup) evalNativeProp(
	prop string, val string) bool {
	switch prop {
	case "environment":
		env := teg.Parent.(Bucketeer).GetEnvironment()
		if val == env {
			return true
		}
	case "object_type":
		if val == "node" {
			return true
		}
	case "object_state":
		if val == teg.State {
			return true
		}
	}
	return false
}

func (teg *SomaTreeElemGroup) evalSystemProp(
	prop string, val string, view string) (string, bool) {
	// TODO
	return "", false
}

func (teg *SomaTreeElemGroup) evalOncallProp(
	prop string, val string, view string) (string, bool) {
	// TODO
	return "", false
}

func (teg *SomaTreeElemGroup) evalCustomProp(
	prop string, val string, view string) (string, bool) {
	// TODO
	return "", false
}

func (teg *SomaTreeElemGroup) evalServiceProp(
	prop string, val string, view string) (string, bool) {
	// TODO
	return "", false
}

func (teg *SomaTreeElemGroup) evalAttributeOfService(
	svcName string, view string, attribute string, value string) bool {
	// TODO
	return false
}

func (teg *SomaTreeElemGroup) evalAttributeProp(
	view string, attr string, value string) (bool, map[string]string) {
	f := map[string]string{}
	// TODO
	return false, f
}

func (teg *SomaTreeElemGroup) getServiceMap(serviceId string) map[string][]string {
	svc := new(PropertyService)
	svc = teg.PropertyService[serviceId].(*PropertyService)

	res := map[string][]string{}
	for _, v := range svc.Attributes {
		res[v.Attribute] = append(res[v.Attribute], v.Value)
	}
	return res
}

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<|endoftext|>


<commit_before>package http

import (
	"fmt"
	"github.com/buildkite/agent/buildkite/logger"
	stdhttp "net/http"
	// stdhttputil "net/http/httputil"
	"bytes"
	"io"
	"net"
	"strings"
	"time"
)

type body interface {
	ToBody() (*bytes.Buffer, error)
	ContentType() string
}

type Request struct {
	Session       *Session // A session to inherit defaults from
	Endpoint      string   // Endpoint can include a path, i.e. https://agent.buildkite.com/v2
	Path          string
	Method        string
	Headers       []Header
	Body          body
	ContentType   string
	Accept        string
	UserAgent     string
	Timeout       time.Duration
	Retries       int
	RetryCallback func(*Response) bool
}

func (r *Request) String() string {
	return fmt.Sprintf("http.Request{Method: %s, URL: %s}", r.Method, r.URL())
}

func (r *Request) Copy() Request {
	return Request{
		Session:       r.Session,
		Endpoint:      r.Endpoint,
		Path:          r.Path,
		Method:        r.Method,
		Headers:       r.Headers,
		Body:          r.Body,
		ContentType:   r.ContentType,
		Accept:        r.Accept,
		UserAgent:     r.UserAgent,
		Timeout:       r.Timeout,
		Retries:       r.Retries,
		RetryCallback: r.RetryCallback,
	}
}

func (r *Request) AddHeader(name string, value string) {
	if r.Headers == nil {
		r.Headers = []Header{}
	}

	r.Headers = append(r.Headers, Header{Name: name, Value: value})
}

func (r *Request) URL() string {
	if r.Session != nil && r.Session.Endpoint != "" {
		return r.join(r.Session.Endpoint, r.Path)
	} else if r.Endpoint != "" {
		return r.join(r.Endpoint, r.Path)
	} else {
		return r.Path
	}
}

func (r *Request) Do() (*Response, error) {
	var response *Response
	var err error

	seconds := 5 * time.Second
	ticker := time.NewTicker(seconds)
	retries := 1

	// The retires value can't be less than 0
	max := r.Retries
	if max <= 0 {
		max = 1
	}

	for {
		// Only show the retries in the log if we're on our second+
		// attempt
		if retries > 1 {
			logger.Debug("%s %s (Attempt %d/%d)", r.Method, r.URL(), retries, max)
		} else {
			logger.Debug("%s %s", r.Method, r.URL())
		}

		response, err = r.send()

		if err == nil {
			break
		}

		if retries >= max {
			logger.Warn("%s %s (%d/%d) (%T: %v)", r.Method, r.URL(), retries, max, err, err)
			break
		} else {
			if r.RetryCallback != nil {
				// If the RetryCallback returns false, don't
				// bother retrying
				if r.RetryCallback(response) == false {
					break
				}
			}

			logger.Warn("%s %s (%d/%d) (%T: %v) Trying again in %s", r.Method, r.URL(), retries, max, err, err, seconds)
		}

		// We don't return this response, so we should make sure we close it's body
		if response != nil && response.Body != nil {
			response.Body.Close()
		}

		retries++
		<-ticker.C
	}

	return response, err
}

func (r *Request) send() (*Response, error) {
	var body io.Reader
	var err error

	if r.Body != nil {
		body, err = r.Body.ToBody()
		if err != nil {
			return nil, err
		}
	}

	req, err := stdhttp.NewRequest(r.Method, r.URL(), body)
	if err != nil {
		return nil, err
	}

	// Add the content type of the body if there is one
	if r.Body != nil {
		req.Header.Set("Content-Type", r.Body.ContentType())
	}

	// Add in the headers
	for _, header := range r.Headers {
		req.Header.Set(header.Name, header.Value)
	}

	// Add in the sessions headers (if we have a session)
	if r.Session != nil {
		for _, header := range r.Session.Headers {
			req.Header.Set(header.Name, header.Value)
		}
	}

	// debug, _ := stdhttputil.DumpRequest(req, true)
	// logger.Debug("%s", debug)

	// Construct a new dialer and bump the default timeout
	dialer := &net.Dialer{
		Timeout: 60 * time.Second,
	}

	// Set the custom timeout
	if r.Timeout > 0 {
		dialer.Timeout = r.Timeout
	}

	// New transport and bump the TLSHandshakeTimeout
	transport := &stdhttp.Transport{
		Dial:                dialer.Dial,
		Proxy:               stdhttp.ProxyFromEnvironment,
		TLSHandshakeTimeout: 60 * time.Second,
	}

	client := &stdhttp.Client{Transport: transport}

	// Perform the stdhttp request
	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	response := &Response{
		StatusCode: res.StatusCode,
		Body:       &Body{reader: res.Body},
	}

	// Was the request not a: 200, 201, 202, etc
	if res.StatusCode/100 != 2 {
		return response, Error{Status: res.Status}
	}

	return response, nil
}

func (r *Request) join(endpoint string, path string) string {
	return strings.TrimRight(endpoint, "/") + "/" + strings.TrimLeft(path, "/")
}
<commit_msg>Make sure to set the request header.<commit_after>package http

import (
	"fmt"
	"github.com/buildkite/agent/buildkite/logger"
	stdhttp "net/http"
	// stdhttputil "net/http/httputil"
	"bytes"
	"io"
	"net"
	"strings"
	"time"
)

type body interface {
	ToBody() (*bytes.Buffer, error)
	ContentType() string
}

type Request struct {
	Session       *Session // A session to inherit defaults from
	Endpoint      string   // Endpoint can include a path, i.e. https://agent.buildkite.com/v2
	Path          string
	Method        string
	Headers       []Header
	Body          body
	ContentType   string
	Accept        string
	UserAgent     string
	Timeout       time.Duration
	Retries       int
	RetryCallback func(*Response) bool
}

func (r *Request) String() string {
	return fmt.Sprintf("http.Request{Method: %s, URL: %s}", r.Method, r.URL())
}

func (r *Request) Copy() Request {
	return Request{
		Session:       r.Session,
		Endpoint:      r.Endpoint,
		Path:          r.Path,
		Method:        r.Method,
		Headers:       r.Headers,
		Body:          r.Body,
		ContentType:   r.ContentType,
		Accept:        r.Accept,
		UserAgent:     r.UserAgent,
		Timeout:       r.Timeout,
		Retries:       r.Retries,
		RetryCallback: r.RetryCallback,
	}
}

func (r *Request) AddHeader(name string, value string) {
	if r.Headers == nil {
		r.Headers = []Header{}
	}

	r.Headers = append(r.Headers, Header{Name: name, Value: value})
}

func (r *Request) URL() string {
	if r.Session != nil && r.Session.Endpoint != "" {
		return r.join(r.Session.Endpoint, r.Path)
	} else if r.Endpoint != "" {
		return r.join(r.Endpoint, r.Path)
	} else {
		return r.Path
	}
}

func (r *Request) Do() (*Response, error) {
	var response *Response
	var err error

	seconds := 5 * time.Second
	ticker := time.NewTicker(seconds)
	retries := 1

	// The retires value can't be less than 0
	max := r.Retries
	if max <= 0 {
		max = 1
	}

	for {
		// Only show the retries in the log if we're on our second+
		// attempt
		if retries > 1 {
			logger.Debug("%s %s (Attempt %d/%d)", r.Method, r.URL(), retries, max)
		} else {
			logger.Debug("%s %s", r.Method, r.URL())
		}

		response, err = r.send()

		if err == nil {
			break
		}

		if retries >= max {
			logger.Warn("%s %s (%d/%d) (%T: %v)", r.Method, r.URL(), retries, max, err, err)
			break
		} else {
			if r.RetryCallback != nil {
				// If the RetryCallback returns false, don't
				// bother retrying
				if r.RetryCallback(response) == false {
					break
				}
			}

			logger.Warn("%s %s (%d/%d) (%T: %v) Trying again in %s", r.Method, r.URL(), retries, max, err, err, seconds)
		}

		// We don't return this response, so we should make sure we close it's body
		if response != nil && response.Body != nil {
			response.Body.Close()
		}

		retries++
		<-ticker.C
	}

	return response, err
}

func (r *Request) send() (*Response, error) {
	var body io.Reader
	var err error

	if r.Body != nil {
		body, err = r.Body.ToBody()
		if err != nil {
			return nil, err
		}
	}

	req, err := stdhttp.NewRequest(r.Method, r.URL(), body)
	if err != nil {
		return nil, err
	}

	// Add the content type of the body if there is one
	if r.Body != nil {
		req.Header.Set("Content-Type", r.Body.ContentType())
	}

	// Add in the headers
	for _, header := range r.Headers {
		req.Header.Set(header.Name, header.Value)
	}

	// Add the user agent
	if r.UserAgent != "" {
		req.Header.Set("User-Agent", r.UserAgent)
	}

	// Add in the sessions headers (if we have a session)
	if r.Session != nil {
		for _, header := range r.Session.Headers {
			req.Header.Set(header.Name, header.Value)
		}
	}

	// debug, _ := stdhttputil.DumpRequest(req, true)
	// logger.Debug("%s", debug)

	// Construct a new dialer and bump the default timeout
	dialer := &net.Dialer{
		Timeout: 60 * time.Second,
	}

	// Set the custom timeout
	if r.Timeout > 0 {
		dialer.Timeout = r.Timeout
	}

	// New transport and bump the TLSHandshakeTimeout
	transport := &stdhttp.Transport{
		Dial:                dialer.Dial,
		Proxy:               stdhttp.ProxyFromEnvironment,
		TLSHandshakeTimeout: 60 * time.Second,
	}

	client := &stdhttp.Client{Transport: transport}

	// Perform the stdhttp request
	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	response := &Response{
		StatusCode: res.StatusCode,
		Body:       &Body{reader: res.Body},
	}

	// Was the request not a: 200, 201, 202, etc
	if res.StatusCode/100 != 2 {
		return response, Error{Status: res.Status}
	}

	return response, nil
}

func (r *Request) join(endpoint string, path string) string {
	return strings.TrimRight(endpoint, "/") + "/" + strings.TrimLeft(path, "/")
}
<|endoftext|>


<commit_before>//
// Copyright © 2019 Guy M. Allard
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package main

import (
	"log"
	"net"
	"os"
	"strconv"
	"time"

	"github.com/gmallard/stompngo"
	"github.com/gmallard/stompngo/senv"
	"github.com/gmallard/stompngo_examples/sngecomm"
)

var (
	exampid = "putget: "
	ll      = log.New(os.Stdout, "PUGT ", log.Ldate|log.Lmicroseconds|log.Lshortfile)
	tag     = "putgemm"
	conn    *stompngo.Connection
	n       net.Conn
	e       error
	u2      = false
	wns     int64
	wd      time.Duration
)

func main() {
	st := time.Now()
	// Environment variable controls
	if os.Getenv("STOMP_2CONN") != "" {
		u2 = true
	}
	wt := os.Getenv("STOMP_WTIME")
	if wt != "" {
		_, err := strconv.ParseInt(wt, 10, 64)
		if err != nil {
			ll.Fatalf("%stag:%s connsess:%s conversion error:%v",
				exampid, tag, sngecomm.Lcs,
				e.Error()) // Handle this ......
		} else {
			d, err := time.ParseDuration(wt + "ms")
			if err != nil {
				ll.Fatalf("%stag:%s connsess:%s parse error:%v",
					exampid, tag, sngecomm.Lcs,
					e.Error()) // Handle this ......
			}
			wd = d
			wns = wd.Nanoseconds()
		}
	}
	// Standard example connect sequence
	n, conn, e = sngecomm.CommonConnect(exampid, tag, ll)
	if e != nil {
		ll.Fatalf("%stag:%s connsess:%s main_on_connect error:%v",
			exampid, tag, sngecomm.Lcs,
			e.Error()) // Handle this ......
	}

	ll.Printf("%stag:%s connsess:%s START nmsgs:%d\n",
		exampid, tag, conn.Session(), senv.Nmsgs())
	// Put
	qname := sngecomm.Dest()
	sh := stompngo.Headers{"destination", qname}
	amsg := "A message from putget"
	for i := 0; i < senv.Nmsgs(); i++ {
		e = conn.Send(sh, amsg)
		if e != nil {
			ll.Fatalln("Send error:", e)
		}
	}
	// Possible 2nd connection
	if u2 {
		// Standard example disconnect sequence
		e = sngecomm.CommonDisconnect(n, conn, exampid, tag, ll)
		if e != nil {
			ll.Fatalf("%stag:%s connsess:%s main_on_disconnect1 error:%v",
				exampid, tag, conn.Session(),
				e.Error()) // Handle this ......
		}
		if wns > 0 {
			time.Sleep(wd)
		}
		//
		n, conn, e = sngecomm.CommonConnect(exampid, tag, ll)
		if e != nil {
			ll.Fatalf("%stag:%s connsess:%s main_on_connect1 error:%v",
				exampid, tag, sngecomm.Lcs,
				e.Error()) // Handle this ......
		}
	} else if wns > 0 {
		time.Sleep(wd)
	}
	// Get
	id := "putget-subid1"
	sc := sngecomm.HandleSubscribe(conn, qname, id, sngecomm.AckMode())
	ll.Printf("%stag:%s connsess:%s subscribe_complete id:%v dest:%v\n",
		exampid, tag, conn.Session(),
		id, qname)
	//
	for i := 0; i < senv.Nmsgs(); i++ {
		md := <-sc
		if md.Error != nil {
			ll.Fatalf("%stag:%s connsess:%s recv_error dest:%s error:%v",
				exampid, tag, conn.Session(),
				qname, md.Error) // Handle this ......
		}
		ll.Printf("%stag:%s connsess:%s message is:%s\n",
			exampid, tag, conn.Session(),
			md.Message.BodyString())
	}
	// Standard example disconnect sequence
	e = sngecomm.CommonDisconnect(n, conn, exampid, tag, ll)
	if e != nil {
		ll.Fatalf("%stag:%s connsess:%s main_on_disconnect error:%v",
			exampid, tag, conn.Session(),
			e.Error()) // Handle this ......
	}
	ll.Printf("%stag:%s connsess:%s main_elapsed:%v\n",
		exampid, tag, conn.Session(),
		time.Now().Sub(st))
}
<commit_msg>Add status messages.<commit_after>//
// Copyright © 2019 Guy M. Allard
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package main

import (
	"log"
	"net"
	"os"
	"strconv"
	"time"

	"github.com/gmallard/stompngo"
	"github.com/gmallard/stompngo/senv"
	"github.com/gmallard/stompngo_examples/sngecomm"
)

var (
	exampid = "putget: "
	ll      = log.New(os.Stdout, "PUGT ", log.Ldate|log.Lmicroseconds|log.Lshortfile)
	tag     = "putgemm"
	conn    *stompngo.Connection
	n       net.Conn
	e       error
	u2      = false
	wns     int64
	wd      time.Duration
)

func main() {
	st := time.Now()
	// Environment variable controls
	if os.Getenv("STOMP_2CONN") != "" {
		u2 = true
	}
	wt := os.Getenv("STOMP_WTIME")
	if wt != "" {
		_, err := strconv.ParseInt(wt, 10, 64)
		if err != nil {
			ll.Fatalf("%stag:%s connsess:%s conversion error:%v",
				exampid, tag, sngecomm.Lcs,
				e.Error()) // Handle this ......
		} else {
			d, err := time.ParseDuration(wt + "ms")
			if err != nil {
				ll.Fatalf("%stag:%s connsess:%s parse error:%v",
					exampid, tag, sngecomm.Lcs,
					e.Error()) // Handle this ......
			}
			wd = d
			wns = wd.Nanoseconds()
		}
	}
	// Standard example connect sequence
	n, conn, e = sngecomm.CommonConnect(exampid, tag, ll)
	if e != nil {
		ll.Fatalf("%stag:%s connsess:%s main_on_connect error:%v",
			exampid, tag, sngecomm.Lcs,
			e.Error()) // Handle this ......
	}

	ll.Printf("%stag:%s connsess:%s START nmsgs:%d\n",
		exampid, tag, conn.Session(), senv.Nmsgs())
	// Put
	qname := sngecomm.Dest()
	sh := stompngo.Headers{"destination", qname}
	amsg := "A message from putget"
	// Status message
	ll.Printf("%stag:%s connsess:%s will PUT: %s, count: %d\n",
		exampid, tag, conn.Session(), amsg, senv.Nmsgs())
	for i := 0; i < senv.Nmsgs(); i++ {
		e = conn.Send(sh, amsg)
		if e != nil {
			ll.Fatalln("Send error:", e)
		}
	}
	// Possible 2nd connection
	if u2 {
		// Standard example disconnect sequence
		e = sngecomm.CommonDisconnect(n, conn, exampid, tag, ll)
		if e != nil {
			ll.Fatalf("%stag:%s connsess:%s main_on_disconnect1 error:%v",
				exampid, tag, conn.Session(),
				e.Error()) // Handle this ......
		}
		if wns > 0 {
			time.Sleep(wd)
		}
		//
		n, conn, e = sngecomm.CommonConnect(exampid, tag, ll)
		if e != nil {
			ll.Fatalf("%stag:%s connsess:%s main_on_connect1 error:%v",
				exampid, tag, sngecomm.Lcs,
				e.Error()) // Handle this ......
		}
	} else if wns > 0 {
		time.Sleep(wd)
	}
	// Status message
	ll.Printf("%stag:%s connsess:%s PUT is done\n",
		exampid, tag, conn.Session())
	// Get
	id := "putget-subid1"
	sc := sngecomm.HandleSubscribe(conn, qname, id, sngecomm.AckMode())
	ll.Printf("%stag:%s connsess:%s subscribe_complete id:%v dest:%v\n",
		exampid, tag, conn.Session(),
		id, qname)
	//
	for i := 0; i < senv.Nmsgs(); i++ {
		md := <-sc
		if md.Error != nil {
			ll.Fatalf("%stag:%s connsess:%s recv_error dest:%s error:%v",
				exampid, tag, conn.Session(),
				qname, md.Error) // Handle this ......
		}
		ll.Printf("%stag:%s connsess:%s message is:%s\n",
			exampid, tag, conn.Session(),
			md.Message.BodyString())
	}
	// Status message
	ll.Printf("%stag:%s connsess:%s GET is done\n",
		exampid, tag, conn.Session())
	// Standard example disconnect sequence
	e = sngecomm.CommonDisconnect(n, conn, exampid, tag, ll)
	if e != nil {
		ll.Fatalf("%stag:%s connsess:%s main_on_disconnect error:%v",
			exampid, tag, conn.Session(),
			e.Error()) // Handle this ......
	}
	ll.Printf("%stag:%s connsess:%s main_elapsed:%v\n",
		exampid, tag, conn.Session(),
		time.Now().Sub(st))
}
<|endoftext|>


<commit_before>package key

// Mapping holds a set of unique keys corresponding
// to Hasher values.
type Mapping struct {
	keys map[uint64] *entry
}

// Hasher represents a value that can be used as a map key.
type Hasher interface {
	Hashcode() uint64
	Equals(m Hasher) bool
}

type entry struct {
	mkey Hasher
	key Key
	next *entry
}

// Key represents a comparable value.
type Key interface{}

type customKey struct {
	key Hasher
}

// NewMapping creates a new Mapping object
func NewMapping() *Mapping {
	return &Mapping{make(map[uint64] *entry)}
}

// Key returns a comparable Key value corresponding
// to mkey. If the same mkey is passed twice to
// the same Mapping, the same Key will be returned.
func (m *Mapping) Key(mkey Hasher) Key {
	h := mkey.Hashcode()
	if e, ok := m.keys[h]; ok {
		for ; e != nil; e = e.next {
			if e.mkey.Equals(mkey) {
				return e.key
			}
		}
	}
	k := &customKey{mkey}
	m.keys[h] = &entry{mkey, k, m.keys[h]}
	return k
}

// Value returns the value for a given Key,
// if it was created with the Key function;
// otherwise it returns nil.
func (m *Mapping) Value(k Key) Hasher {
	if k, ok := k.(*customKey); ok {
		return k.key
	}
	return nil
}
<commit_msg>rename Value to Original<commit_after>package key

// Mapping holds a set of unique keys corresponding
// to Hasher values.
type Mapping struct {
	keys map[uint64] *entry
}

// Hasher represents a value that can be used as a map key.
type Hasher interface {
	Hashcode() uint64
	Equals(m Hasher) bool
}

type entry struct {
	mkey Hasher
	key Key
	next *entry
}

// Key represents a comparable value.
type Key interface{}

type customKey struct {
	key Hasher
}

// NewMapping creates a new Mapping object
func NewMapping() *Mapping {
	return &Mapping{make(map[uint64] *entry)}
}

// Key returns a comparable Key value corresponding
// to mkey. If the same mkey is passed twice to
// the same Mapping, the same Key will be returned.
func (m *Mapping) Key(mkey Hasher) Key {
	h := mkey.Hashcode()
	if e, ok := m.keys[h]; ok {
		for ; e != nil; e = e.next {
			if e.mkey.Equals(mkey) {
				return e.key
			}
		}
	}
	k := &customKey{mkey}
	m.keys[h] = &entry{mkey, k, m.keys[h]}
	return k
}

// Value returns the original key value for a given Key,
// if it was created with the Key function;
// otherwise it returns nil.
func (m *Mapping) Original(k Key) Hasher {
	if k, ok := k.(*customKey); ok {
		return k.key
	}
	return nil
}
<|endoftext|>


<commit_before>package graphtest

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"sort"

	graphdriver "github.com/containers/storage/drivers"
	"github.com/containers/storage/pkg/archive"
	"github.com/containers/storage/pkg/stringid"
)

func randomContent(size int, seed int64) []byte {
	s := rand.NewSource(seed)
	content := make([]byte, size)

	for i := 0; i < len(content); i += 7 {
		val := s.Int63()
		for j := 0; i+j < len(content) && j < 7; j++ {
			content[i+j] = byte(val)
			val >>= 8
		}
	}

	return content
}

func addFiles(drv graphdriver.Driver, layer string, seed int64) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	if err := ioutil.WriteFile(path.Join(root, "file-a"), randomContent(64, seed), 0755); err != nil {
		return err
	}
	if err := os.MkdirAll(path.Join(root, "dir-b"), 0755); err != nil {
		return err
	}
	if err := ioutil.WriteFile(path.Join(root, "dir-b", "file-b"), randomContent(128, seed+1), 0755); err != nil {
		return err
	}

	return ioutil.WriteFile(path.Join(root, "file-c"), randomContent(128*128, seed+2), 0755)
}

func checkFile(drv graphdriver.Driver, layer, filename string, content []byte) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	fileContent, err := ioutil.ReadFile(path.Join(root, filename))
	if err != nil {
		return err
	}

	if bytes.Compare(fileContent, content) != 0 {
		return fmt.Errorf("mismatched file content %v, expecting %v", fileContent, content)
	}

	return nil
}

func addFile(drv graphdriver.Driver, layer, filename string, content []byte) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	return ioutil.WriteFile(path.Join(root, filename), content, 0755)
}

func addDirectory(drv graphdriver.Driver, layer, dir string) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	return os.MkdirAll(path.Join(root, dir), 0755)
}

func removeAll(drv graphdriver.Driver, layer string, names ...string) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	for _, filename := range names {
		if err := os.RemoveAll(path.Join(root, filename)); err != nil {
			return err
		}
	}
	return nil
}

func checkFileRemoved(drv graphdriver.Driver, layer, filename string) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	if _, err := os.Stat(path.Join(root, filename)); err == nil {
		return fmt.Errorf("file still exists: %s", path.Join(root, filename))
	} else if !os.IsNotExist(err) {
		return err
	}

	return nil
}

func addManyFiles(drv graphdriver.Driver, layer string, count int, seed int64) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	for i := 0; i < count; i += 100 {
		dir := path.Join(root, fmt.Sprintf("directory-%d", i))
		if err := os.MkdirAll(dir, 0755); err != nil {
			return err
		}
		for j := 0; i+j < count && j < 100; j++ {
			file := path.Join(dir, fmt.Sprintf("file-%d", i+j))
			if err := ioutil.WriteFile(file, randomContent(64, seed+int64(i+j)), 0755); err != nil {
				return err
			}
		}
	}

	return nil
}

func changeManyFiles(drv graphdriver.Driver, layer string, count int, seed int64) ([]archive.Change, error) {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return nil, err
	}
	defer drv.Put(layer)

	changes := []archive.Change{}
	for i := 0; i < count; i += 100 {
		archiveRoot := fmt.Sprintf("/directory-%d", i)
		if err := os.MkdirAll(path.Join(root, archiveRoot), 0755); err != nil {
			return nil, err
		}
		for j := 0; i+j < count && j < 100; j++ {
			if j == 0 {
				changes = append(changes, archive.Change{
					Path: archiveRoot,
					Kind: archive.ChangeModify,
				})
			}
			var change archive.Change
			switch j % 3 {
			// Update file
			case 0:
				var originalFileInfo, updatedFileInfo os.FileInfo
				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d", i+j))
				change.Kind = archive.ChangeModify
				if originalFileInfo, err = os.Stat(path.Join(root, change.Path)); err != nil {
					return nil, err
				}
				for updatedFileInfo == nil || updatedFileInfo.ModTime().Equal(originalFileInfo.ModTime()) {
					if err := ioutil.WriteFile(path.Join(root, change.Path), randomContent(64, seed+int64(i+j)), 0755); err != nil {
						return nil, err
					}
					if updatedFileInfo, err = os.Stat(path.Join(root, change.Path)); err != nil {
						return nil, err
					}
				}
			// Add file
			case 1:
				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d-%d", seed, i+j))
				change.Kind = archive.ChangeAdd
				if err := ioutil.WriteFile(path.Join(root, change.Path), randomContent(64, seed+int64(i+j)), 0755); err != nil {
					return nil, err
				}
			// Remove file
			case 2:
				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d", i+j))
				change.Kind = archive.ChangeDelete
				if err := os.Remove(path.Join(root, change.Path)); err != nil {
					return nil, err
				}
			}
			changes = append(changes, change)
		}
	}

	return changes, nil
}

func checkManyFiles(drv graphdriver.Driver, layer string, count int, seed int64) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	for i := 0; i < count; i += 100 {
		dir := path.Join(root, fmt.Sprintf("directory-%d", i))
		for j := 0; i+j < count && j < 100; j++ {
			file := path.Join(dir, fmt.Sprintf("file-%d", i+j))
			fileContent, err := ioutil.ReadFile(file)
			if err != nil {
				return err
			}

			content := randomContent(64, seed+int64(i+j))

			if bytes.Compare(fileContent, content) != 0 {
				return fmt.Errorf("mismatched file content %v, expecting %v", fileContent, content)
			}
		}
	}

	return nil
}

type changeList []archive.Change

func (c changeList) Less(i, j int) bool {
	if c[i].Path == c[j].Path {
		return c[i].Kind < c[j].Kind
	}
	return c[i].Path < c[j].Path
}
func (c changeList) Len() int      { return len(c) }
func (c changeList) Swap(i, j int) { c[j], c[i] = c[i], c[j] }

func checkChanges(expected, actual []archive.Change) error {
	if len(expected) != len(actual) {
		return fmt.Errorf("unexpected number of changes, expected %d, got %d", len(expected), len(actual))
	}
	sort.Sort(changeList(expected))
	sort.Sort(changeList(actual))

	for i := range expected {
		if expected[i] != actual[i] {
			return fmt.Errorf("unexpected change, expecting %v, got %v", expected[i], actual[i])
		}
	}

	return nil
}

func addLayerFiles(drv graphdriver.Driver, layer, parent string, i int) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	if err := ioutil.WriteFile(path.Join(root, "top-id"), []byte(layer), 0755); err != nil {
		return err
	}
	layerDir := path.Join(root, fmt.Sprintf("layer-%d", i))
	if err := os.MkdirAll(layerDir, 0755); err != nil {
		return err
	}
	if err := ioutil.WriteFile(path.Join(layerDir, "layer-id"), []byte(layer), 0755); err != nil {
		return err
	}
	if err := ioutil.WriteFile(path.Join(layerDir, "parent-id"), []byte(parent), 0755); err != nil {
		return err
	}

	return nil
}

func addManyLayers(drv graphdriver.Driver, baseLayer string, count int) (string, error) {
	lastLayer := baseLayer
	for i := 1; i <= count; i++ {
		nextLayer := stringid.GenerateRandomID()
		if err := drv.Create(nextLayer, lastLayer, nil); err != nil {
			return "", err
		}
		if err := addLayerFiles(drv, nextLayer, lastLayer, i); err != nil {
			return "", err
		}

		lastLayer = nextLayer

	}
	return lastLayer, nil
}

func checkManyLayers(drv graphdriver.Driver, layer string, count int) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	layerIDBytes, err := ioutil.ReadFile(path.Join(root, "top-id"))
	if err != nil {
		return err
	}

	if bytes.Compare(layerIDBytes, []byte(layer)) != 0 {
		return fmt.Errorf("mismatched file content %v, expecting %v", layerIDBytes, []byte(layer))
	}

	for i := count; i > 0; i-- {
		layerDir := path.Join(root, fmt.Sprintf("layer-%d", i))

		thisLayerIDBytes, err := ioutil.ReadFile(path.Join(layerDir, "layer-id"))
		if err != nil {
			return err
		}
		if bytes.Compare(thisLayerIDBytes, layerIDBytes) != 0 {
			return fmt.Errorf("mismatched file content %v, expecting %v", thisLayerIDBytes, layerIDBytes)
		}
		layerIDBytes, err = ioutil.ReadFile(path.Join(layerDir, "parent-id"))
		if err != nil {
			return err
		}
	}
	return nil
}

// readDir reads a directory just like ioutil.ReadDir()
// then hides specific files (currently "lost+found")
// so the tests don't "see" it
func readDir(dir string) ([]os.FileInfo, error) {
	a, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}

	b := a[:0]
	for _, x := range a {
		if x.Name() != "lost+found" { // ext4 always have this dir
			b = append(b, x)
		}
	}

	return b, nil
}
<commit_msg>Use math/rand.Rand.Read instead of a loop with Int63<commit_after>package graphtest

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"sort"

	graphdriver "github.com/containers/storage/drivers"
	"github.com/containers/storage/pkg/archive"
	"github.com/containers/storage/pkg/stringid"
)

func randomContent(size int, seed int64) []byte {
	content := make([]byte, size)

	rng := rand.New(rand.NewSource(seed))
	read, err := rng.Read(content)
	if err != nil || read != size {
		panic("Unexpected failure of math/rand.Rand.Read")
	}

	return content
}

func addFiles(drv graphdriver.Driver, layer string, seed int64) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	if err := ioutil.WriteFile(path.Join(root, "file-a"), randomContent(64, seed), 0755); err != nil {
		return err
	}
	if err := os.MkdirAll(path.Join(root, "dir-b"), 0755); err != nil {
		return err
	}
	if err := ioutil.WriteFile(path.Join(root, "dir-b", "file-b"), randomContent(128, seed+1), 0755); err != nil {
		return err
	}

	return ioutil.WriteFile(path.Join(root, "file-c"), randomContent(128*128, seed+2), 0755)
}

func checkFile(drv graphdriver.Driver, layer, filename string, content []byte) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	fileContent, err := ioutil.ReadFile(path.Join(root, filename))
	if err != nil {
		return err
	}

	if bytes.Compare(fileContent, content) != 0 {
		return fmt.Errorf("mismatched file content %v, expecting %v", fileContent, content)
	}

	return nil
}

func addFile(drv graphdriver.Driver, layer, filename string, content []byte) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	return ioutil.WriteFile(path.Join(root, filename), content, 0755)
}

func addDirectory(drv graphdriver.Driver, layer, dir string) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	return os.MkdirAll(path.Join(root, dir), 0755)
}

func removeAll(drv graphdriver.Driver, layer string, names ...string) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	for _, filename := range names {
		if err := os.RemoveAll(path.Join(root, filename)); err != nil {
			return err
		}
	}
	return nil
}

func checkFileRemoved(drv graphdriver.Driver, layer, filename string) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	if _, err := os.Stat(path.Join(root, filename)); err == nil {
		return fmt.Errorf("file still exists: %s", path.Join(root, filename))
	} else if !os.IsNotExist(err) {
		return err
	}

	return nil
}

func addManyFiles(drv graphdriver.Driver, layer string, count int, seed int64) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	for i := 0; i < count; i += 100 {
		dir := path.Join(root, fmt.Sprintf("directory-%d", i))
		if err := os.MkdirAll(dir, 0755); err != nil {
			return err
		}
		for j := 0; i+j < count && j < 100; j++ {
			file := path.Join(dir, fmt.Sprintf("file-%d", i+j))
			if err := ioutil.WriteFile(file, randomContent(64, seed+int64(i+j)), 0755); err != nil {
				return err
			}
		}
	}

	return nil
}

func changeManyFiles(drv graphdriver.Driver, layer string, count int, seed int64) ([]archive.Change, error) {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return nil, err
	}
	defer drv.Put(layer)

	changes := []archive.Change{}
	for i := 0; i < count; i += 100 {
		archiveRoot := fmt.Sprintf("/directory-%d", i)
		if err := os.MkdirAll(path.Join(root, archiveRoot), 0755); err != nil {
			return nil, err
		}
		for j := 0; i+j < count && j < 100; j++ {
			if j == 0 {
				changes = append(changes, archive.Change{
					Path: archiveRoot,
					Kind: archive.ChangeModify,
				})
			}
			var change archive.Change
			switch j % 3 {
			// Update file
			case 0:
				var originalFileInfo, updatedFileInfo os.FileInfo
				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d", i+j))
				change.Kind = archive.ChangeModify
				if originalFileInfo, err = os.Stat(path.Join(root, change.Path)); err != nil {
					return nil, err
				}
				for updatedFileInfo == nil || updatedFileInfo.ModTime().Equal(originalFileInfo.ModTime()) {
					if err := ioutil.WriteFile(path.Join(root, change.Path), randomContent(64, seed+int64(i+j)), 0755); err != nil {
						return nil, err
					}
					if updatedFileInfo, err = os.Stat(path.Join(root, change.Path)); err != nil {
						return nil, err
					}
				}
			// Add file
			case 1:
				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d-%d", seed, i+j))
				change.Kind = archive.ChangeAdd
				if err := ioutil.WriteFile(path.Join(root, change.Path), randomContent(64, seed+int64(i+j)), 0755); err != nil {
					return nil, err
				}
			// Remove file
			case 2:
				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d", i+j))
				change.Kind = archive.ChangeDelete
				if err := os.Remove(path.Join(root, change.Path)); err != nil {
					return nil, err
				}
			}
			changes = append(changes, change)
		}
	}

	return changes, nil
}

func checkManyFiles(drv graphdriver.Driver, layer string, count int, seed int64) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	for i := 0; i < count; i += 100 {
		dir := path.Join(root, fmt.Sprintf("directory-%d", i))
		for j := 0; i+j < count && j < 100; j++ {
			file := path.Join(dir, fmt.Sprintf("file-%d", i+j))
			fileContent, err := ioutil.ReadFile(file)
			if err != nil {
				return err
			}

			content := randomContent(64, seed+int64(i+j))

			if bytes.Compare(fileContent, content) != 0 {
				return fmt.Errorf("mismatched file content %v, expecting %v", fileContent, content)
			}
		}
	}

	return nil
}

type changeList []archive.Change

func (c changeList) Less(i, j int) bool {
	if c[i].Path == c[j].Path {
		return c[i].Kind < c[j].Kind
	}
	return c[i].Path < c[j].Path
}
func (c changeList) Len() int      { return len(c) }
func (c changeList) Swap(i, j int) { c[j], c[i] = c[i], c[j] }

func checkChanges(expected, actual []archive.Change) error {
	if len(expected) != len(actual) {
		return fmt.Errorf("unexpected number of changes, expected %d, got %d", len(expected), len(actual))
	}
	sort.Sort(changeList(expected))
	sort.Sort(changeList(actual))

	for i := range expected {
		if expected[i] != actual[i] {
			return fmt.Errorf("unexpected change, expecting %v, got %v", expected[i], actual[i])
		}
	}

	return nil
}

func addLayerFiles(drv graphdriver.Driver, layer, parent string, i int) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	if err := ioutil.WriteFile(path.Join(root, "top-id"), []byte(layer), 0755); err != nil {
		return err
	}
	layerDir := path.Join(root, fmt.Sprintf("layer-%d", i))
	if err := os.MkdirAll(layerDir, 0755); err != nil {
		return err
	}
	if err := ioutil.WriteFile(path.Join(layerDir, "layer-id"), []byte(layer), 0755); err != nil {
		return err
	}
	if err := ioutil.WriteFile(path.Join(layerDir, "parent-id"), []byte(parent), 0755); err != nil {
		return err
	}

	return nil
}

func addManyLayers(drv graphdriver.Driver, baseLayer string, count int) (string, error) {
	lastLayer := baseLayer
	for i := 1; i <= count; i++ {
		nextLayer := stringid.GenerateRandomID()
		if err := drv.Create(nextLayer, lastLayer, nil); err != nil {
			return "", err
		}
		if err := addLayerFiles(drv, nextLayer, lastLayer, i); err != nil {
			return "", err
		}

		lastLayer = nextLayer

	}
	return lastLayer, nil
}

func checkManyLayers(drv graphdriver.Driver, layer string, count int) error {
	root, err := drv.Get(layer, graphdriver.MountOpts{})
	if err != nil {
		return err
	}
	defer drv.Put(layer)

	layerIDBytes, err := ioutil.ReadFile(path.Join(root, "top-id"))
	if err != nil {
		return err
	}

	if bytes.Compare(layerIDBytes, []byte(layer)) != 0 {
		return fmt.Errorf("mismatched file content %v, expecting %v", layerIDBytes, []byte(layer))
	}

	for i := count; i > 0; i-- {
		layerDir := path.Join(root, fmt.Sprintf("layer-%d", i))

		thisLayerIDBytes, err := ioutil.ReadFile(path.Join(layerDir, "layer-id"))
		if err != nil {
			return err
		}
		if bytes.Compare(thisLayerIDBytes, layerIDBytes) != 0 {
			return fmt.Errorf("mismatched file content %v, expecting %v", thisLayerIDBytes, layerIDBytes)
		}
		layerIDBytes, err = ioutil.ReadFile(path.Join(layerDir, "parent-id"))
		if err != nil {
			return err
		}
	}
	return nil
}

// readDir reads a directory just like ioutil.ReadDir()
// then hides specific files (currently "lost+found")
// so the tests don't "see" it
func readDir(dir string) ([]os.FileInfo, error) {
	a, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}

	b := a[:0]
	for _, x := range a {
		if x.Name() != "lost+found" { // ext4 always have this dir
			b = append(b, x)
		}
	}

	return b, nil
}
<|endoftext|>


<commit_before>package kinesis

import (
	"bytes"
	"errors"
	"log"
	"os"
	"text/template"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/kinesis"
	"github.com/gliderlabs/logspout/router"
)

func init() {
	router.AdapterFactories.Register(NewKinesisAdapter, "kinesis")
}

type KinesisAdapter struct {
	Client   *kinesis.Kinesis
	Drainers map[string]*Drainer
}

func NewKinesisAdapter(route *router.Route) (router.LogAdapter, error) {
	drainers := make(map[string]*Drainer)
	client := kinesis.New(&aws.Config{})

	return &KinesisAdapter{
		Client:   client,
		Drainers: drainers,
	}, nil
}

func (a *KinesisAdapter) Stream(logstream chan *router.Message) {
	for {
		m, open := <-logstream
		d, err := a.findDrainer(m)
		if err != nil {
			logErr(err)
			break
		}

		if !open {
			logErr(d.Buffer.Flush())
			break
		}

		logErr(d.Buffer.Add(m))
	}
}

func (a *KinesisAdapter) findDrainer(m *router.Message) (*Drainer, error) {
	var d *Drainer
	var ok bool

	streamName, err := streamName(m)
	if err != nil {
		return nil, err
	}

	if d, ok = a.Drainers[streamName]; !ok {
		d = newDrainer(a.Client, streamName)

		a.Drainers[streamName] = d
	}

	return d, nil
}

func streamName(m *router.Message) (string, error) {
	streamTmplString := os.Getenv("KINESIS_STREAM_TEMPLATE")
	if streamTmplString == "" {
		return "", errors.New("The stream name template is missing. Please set the KINESIS_STREAM_TEMPLATE env variable")
	}

	streamTmpl, err := template.New("kinesisStream").Parse(streamTmplString)
	if err != nil {
		return "", err
	}

	var streamName bytes.Buffer
	err = streamTmpl.Execute(&streamName, m)
	if err != nil {
		return "", err
	}

	return streamName.String(), nil

}

func logErr(err error) {
	if err != nil {
		log.Println("kinesis: ", err.Error())
	}
}
<commit_msg>If the channel is closed, flush all the buffers.<commit_after>package kinesis

import (
	"bytes"
	"errors"
	"log"
	"os"
	"text/template"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/kinesis"
	"github.com/gliderlabs/logspout/router"
)

func init() {
	router.AdapterFactories.Register(NewKinesisAdapter, "kinesis")
}

type KinesisAdapter struct {
	Client   *kinesis.Kinesis
	Drainers map[string]*Drainer
}

func NewKinesisAdapter(route *router.Route) (router.LogAdapter, error) {
	drainers := make(map[string]*Drainer)
	client := kinesis.New(&aws.Config{})

	return &KinesisAdapter{
		Client:   client,
		Drainers: drainers,
	}, nil
}

func (a *KinesisAdapter) Stream(logstream chan *router.Message) {
	for {
		m, open := <-logstream
		if !open {
			log.Println("kinesis: Channel is closed, flushing all the buffers")
			logErrs(a.FlushAll())
			break
		}

		d, err := a.findDrainer(m)
		if err != nil {
			logErr(err)
			break
		}

		logErr(d.Buffer.Add(m))
	}
}

func (a *KinesisAdapter) findDrainer(m *router.Message) (*Drainer, error) {
	var d *Drainer
	var ok bool

	streamName, err := streamName(m)
	if err != nil {
		return nil, err
	}

	if d, ok = a.Drainers[streamName]; !ok {
		d = newDrainer(a.Client, streamName)

		a.Drainers[streamName] = d
	}

	return d, nil
}

func (a *KinesisAdapter) FlushAll() []error {
	var err []error

	for _, d := range a.Drainers {
		err = append(err, d.Buffer.Flush())
	}

	return err
}

func streamName(m *router.Message) (string, error) {
	streamTmplString := os.Getenv("KINESIS_STREAM_TEMPLATE")
	if streamTmplString == "" {
		return "", errors.New("The stream name template is missing. Please set the KINESIS_STREAM_TEMPLATE env variable")
	}

	streamTmpl, err := template.New("kinesisStream").Parse(streamTmplString)
	if err != nil {
		return "", err
	}

	var streamName bytes.Buffer
	err = streamTmpl.Execute(&streamName, m)
	if err != nil {
		return "", err
	}

	return streamName.String(), nil

}

func logErr(err error) {
	if err != nil {
		log.Println("kinesis: ", err.Error())
	}
}

func logErrs(err []error) {
	if err != nil {
		for _, e := range err {
			log.Println("kinesis: ", e.Error())
		}
	}
}
<|endoftext|>


<commit_before>package datastore

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/jeffpierce/cassabon/config"
	"github.com/jeffpierce/cassabon/logging"
)

// IndexResponse defines the individual elements returned as an array by "GET /paths".
type IndexResponse struct {
	Path   string `json:"path"`
	Depth  int    `json:"depth"`
	Tenant string `json:"tenant"`
	Leaf   bool   `json:"leaf"`
}

// ElasticResponse is the struct we unmarshal the response from an ElasticSearch query to.
type ElasticResponse struct {
	Took     int      `json:"took"`
	TimedOut bool     `json:"timed_out"`
	Shards   ERShards `json:"_shards"`
	Hits     ERHits   `json:"hits"`
}

type ERShards struct {
	Total      int `json:"total"`
	Successful int `json:"successful"`
	Failed     int `json:"failed"`
}

type ERHits struct {
	Total    int           `json:"total"`
	MaxScore float32       `json:"max_score"`
	Hits     []ERSearchHit `json:"hits"`
}

type ERSearchHit struct {
	Index  string        `json:"_index"`
	Type   string        `json:"_type"`
	ID     string        `json:"_id"`
	Score  float32       `json:"_score"`
	Source IndexResponse `json:"_source"`
}

type ERQuery struct {
	Sort  []map[string]map[string]string                            `json:"sort"`
	Query map[string]map[string][]map[string]map[string]interface{} `json:"query"`
}

type IndexManager struct {
	wg *sync.WaitGroup
}

func (im *IndexManager) Init(bootstrap bool) {
	// If bootstrap is true, initialize mapping in ElasticSearch
	if bootstrap {
		im.initMapping()
	}
}

func (im *IndexManager) Start(wg *sync.WaitGroup) {
	im.wg = wg
	im.wg.Add(1)
	go im.run()
}

func (im *IndexManager) run() {

	defer config.G.OnPanic()

	// Wait for entries to arrive, and process them.
	for {
		select {
		case <-config.G.OnReload2:
			config.G.Log.System.LogDebug("IndexManager::run received QUIT message")
			im.wg.Done()
			return
		case metric := <-config.G.Channels.IndexStore:
			go im.index(metric.Path)
		case query := <-config.G.Channels.IndexRequest:
			go im.query(query)
		}
	}
}

// initMapping initializes ElasticSearch for cassabon.
func (im *IndexManager) initMapping() {
	mapping := map[string]map[string]map[string]map[string]map[string]string{
		"mappings": map[string]map[string]map[string]map[string]string{
			"path": map[string]map[string]map[string]string{
				"properties": map[string]map[string]string{
					"path": map[string]string{
						"type":  "string",
						"index": "not_analyzed",
					},
					"depth": map[string]string{
						"type": "long",
					},
					"tenant": map[string]string{
						"type": "string",
					},
					"leaf": map[string]string{
						"type": "bool",
					},
				},
			},
		},
	}

	jsonMap, _ := json.Marshal(mapping)
	config.G.Log.System.LogDebug("%s", string(jsonMap))

	putreq, _ := http.NewRequest("PUT", config.G.ElasticSearch.PutURL, bytes.NewBuffer(jsonMap))
	r := im.httpRequest(putreq)

	if r == nil {
		config.G.Log.System.LogFatal("Could not initialize mapping for ElasticSearch.")
	}
}

// getAllLeafNodes queries ElasticSearch for all leaf nodes. Used for populating metric manager's stat paths on reboot.
func (im *IndexManager) getAllLeafNodes() []string {
	sort := []map[string]map[string]string{
		{
			"path": map[string]string{
				"order": "asc",
			},
		},
	}
	query := map[string]map[string][]map[string]map[string]interface{}{
		"bool": map[string][]map[string]map[string]interface{}{
			"must": []map[string]map[string]interface{}{
				{
					"match": map[string]interface{}{
						"leaf": true,
					},
				},
			},
		},
	}

	fullQuery := ERQuery{sort, query}
	getreq := im.prepRequest(fullQuery)
	r := im.httpRequest(getreq)

	var esResp ElasticResponse
	var pathList []string

	if r != nil {
		_ = json.Unmarshal(r, &esResp)

		config.G.Log.System.LogDebug("esResp: %v", esResp)

		for _, hit := range esResp.Hits.Hits {
			pathList = append(pathList, hit.Source.Path)
		}
	} else {
		logging.Statsd.Client.Inc("indexmgr.es.err.get", 1, 1.0)
		config.G.Log.System.LogError("Error querying ES.")
	}

	return pathList
}

func (im *IndexManager) prepRequest(fullQuery ERQuery) *http.Request {
	jsonQuery, _ := json.Marshal(fullQuery)
	config.G.Log.System.LogDebug("%s", string(jsonQuery))

	// Get the count so that we capture all of the possible paths.
	countreq, _ := http.NewRequest("GET", config.G.ElasticSearch.CountURL, strings.NewReader(string(jsonQuery)))
	size := "size=" + im.getCount(countreq)

	searchURL := strings.Join([]string{config.G.ElasticSearch.SearchURL, size}, "?")
	getreq, _ := http.NewRequest("GET", searchURL, strings.NewReader(string(jsonQuery)))

	return getreq
}

// IndexMetricPath takes a metric path string and sends it off to be processed by processMetricPath().
func (im *IndexManager) index(path string) {
	it := time.Now()
	config.G.Log.System.LogDebug("IndexManager::index path=%s", path)
	splitPath := strings.Split(path, ".")
	im.processMetricPath(splitPath, len(splitPath), true)
	logging.Statsd.Client.TimingDuration("indexmgr.index", time.Since(it), 1.0)
}

func (im *IndexManager) httpRequest(req *http.Request) []byte {
	client := &http.Client{Timeout: time.Duration(15 * time.Second)}
	resp, err := client.Do(req)

	if err != nil {
		logging.Statsd.Client.Inc("indexmgr.es.err.httpreq", 1, 1.0)
		config.G.Log.System.LogError("Received error from ElasticSearch: %v, request: %v", err.Error(), req)
		return nil
	}

	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)
	return body
}

// processMetricPath recursively indexes the metric path via the ElasticSearch REST API.
func (im *IndexManager) processMetricPath(splitPath []string, pathLen int, isLeaf bool) {
	// Process the metric path one node at a time.  We store metrics in ElasticSearch.
	retries := 0
	for pathLen > 0 {

		// Construct the metric string
		metricPath := strings.Join(splitPath, ".")

		// Strip % off the end to avoid invalid escape errors.
		if string(metricPath[len(metricPath)-1]) == "%" {
			metricPath = metricPath[:len(metricPath)-1]
		}
		config.G.Log.System.LogDebug("IndexManager indexing \"%s\"", metricPath)

		pathToIndex := IndexResponse{
			metricPath,
			pathLen,
			"",
			isLeaf,
		}

		urlToPut := strings.Join([]string{config.G.ElasticSearch.PutURL, metricPath}, "/")

		// Marshal the struct into JSON
		jsonPath, err := json.Marshal(pathToIndex)

		if err != nil {
			logging.Statsd.Client.Inc("indexmgr.es.err.json", 1, 1.0)
			config.G.Log.System.LogError("Unable to marshal pathToIndex of %v, error is %v", pathToIndex, err.Error())
			return // Let's not fill up ES with junk if we can't marshal our path struct.
		}

		putreq, err := http.NewRequest("PUT", urlToPut, bytes.NewBuffer(jsonPath))

		if err != nil {
			logging.Statsd.Client.Inc("indexmgr.es.err.put", 1, 1.0)
			config.G.Log.System.LogError("Error when attempting to index %v: %v", metricPath, err.Error())
		}

		r := im.httpRequest(putreq)
		if r != nil {
			// Pop the last node of the metric off, set isLeaf to false, and resume loop.
			_, splitPath = splitPath[len(splitPath)-1], splitPath[:len(splitPath)-1]
			isLeaf = false
			pathLen = len(splitPath)
			retries = 0
		} else {
			logging.Statsd.Client.Inc("indexmgr.es.err.pmr.req", 1, 1.0)
			retries++
			config.G.Log.System.LogWarn("processMetricPath's httprequest to ES came back as nil, sending to retry in %d seconds.", retries)
			time.Sleep(time.Duration(retries) * time.Second)
		}
	}
}

func (im *IndexManager) getCount(req *http.Request) string {
	var resp ElasticResponse
	r := im.httpRequest(req)
	if r != nil {
		_ = json.Unmarshal(r, &resp)
		config.G.Log.System.LogDebug("total: %v", resp.Hits.Total)
		return strconv.Itoa(resp.Hits.Total)
	} else {
		return "0"
	}
}

// query returns the data matched by the supplied query.
func (im *IndexManager) query(q config.IndexQuery) {
	switch strings.ToLower(q.Method) {
	case "delete":
		// TODO
	default:
		im.queryGET(q)
	}
}

// query returns the data matched by the supplied query.
func (im *IndexManager) queryGET(q config.IndexQuery) {

	config.G.Log.System.LogDebug("IndexManager::query %v", q.Query)

	// Query particulars are mandatory.
	if q.Query == "" {
		q.Channel <- config.APIQueryResponse{config.AQS_BADREQUEST, "no query specified", []byte{}}
		return
	}
	// Convert query to form suitable for Elasticsearch regexp search.
	regexpQuery := strings.Replace(q.Query, ".", "\\.", -1)
	regexpQuery = strings.Replace(regexpQuery, "*", ".*", -1)

	// Get number of nodes in the path for the ElasticSearch Query
	pathDepth := len(strings.Split(q.Query, "."))

	var esResp ElasticResponse
	var respList []IndexResponse
	var resp config.APIQueryResponse

	// It's turtles all the way down!  This is totally Vijay's fault.
	// http://github.com/vijaykramesh -- JP
	sort := []map[string]map[string]string{
		{
			"path": map[string]string{
				"order": "asc",
			},
		},
	}
	query := map[string]map[string][]map[string]map[string]interface{}{
		"bool": map[string][]map[string]map[string]interface{}{
			"must": []map[string]map[string]interface{}{
				{
					"regexp": map[string]interface{}{
						"path": regexpQuery,
					},
				},
				{
					"match": map[string]interface{}{
						"depth": pathDepth,
					},
				},
			},
		},
	}

	fullQuery := ERQuery{sort, query}
	getreq := im.prepRequest(fullQuery)
	r := im.httpRequest(getreq)

	if r != nil {
		_ = json.Unmarshal(r, &esResp)

		config.G.Log.System.LogDebug("esResp: %v", esResp)

		for _, hit := range esResp.Hits.Hits {
			respList = append(respList, hit.Source)
		}

		jsonResp, _ := json.Marshal(respList)

		resp = config.APIQueryResponse{config.AQS_OK, "", jsonResp}
	} else {
		logging.Statsd.Client.Inc("indexmgr.es.err.get", 1, 1.0)
		config.G.Log.System.LogError("Error querying ES.")
		resp = config.APIQueryResponse{config.AQS_ERROR, "Error querying ES", []byte{}}
	}

	// If the API gave up on us because we took too long, writing to the channel
	// will cause first a data race, and then a panic (write on closed channel).
	// We check, but if we lose a race we will need to recover.
	defer func() {
		_ = recover()
	}()

	// Check whether the channel is closed before attempting a write.
	select {
	case <-q.Channel:
		// Immediate return means channel is closed (we know there is no data in it).
	default:
		// If the channel would have blocked, it is open, we can write to it.
		q.Channel <- resp
	}
}
<commit_msg>Added queuing system to index storage.<commit_after>package datastore

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/jeffpierce/cassabon/config"
	"github.com/jeffpierce/cassabon/logging"

	"github.com/otium/queue"
)

// IndexResponse defines the individual elements returned as an array by "GET /paths".
type IndexResponse struct {
	Path   string `json:"path"`
	Depth  int    `json:"depth"`
	Tenant string `json:"tenant"`
	Leaf   bool   `json:"leaf"`
}

// ElasticResponse is the struct we unmarshal the response from an ElasticSearch query to.
type ElasticResponse struct {
	Took     int      `json:"took"`
	TimedOut bool     `json:"timed_out"`
	Shards   ERShards `json:"_shards"`
	Hits     ERHits   `json:"hits"`
}

type ERShards struct {
	Total      int `json:"total"`
	Successful int `json:"successful"`
	Failed     int `json:"failed"`
}

type ERHits struct {
	Total    int           `json:"total"`
	MaxScore float32       `json:"max_score"`
	Hits     []ERSearchHit `json:"hits"`
}

type ERSearchHit struct {
	Index  string        `json:"_index"`
	Type   string        `json:"_type"`
	ID     string        `json:"_id"`
	Score  float32       `json:"_score"`
	Source IndexResponse `json:"_source"`
}

type ERQuery struct {
	Sort  []map[string]map[string]string                            `json:"sort"`
	Query map[string]map[string][]map[string]map[string]interface{} `json:"query"`
}

type IndexManager struct {
	wg         *sync.WaitGroup
	IndexQueue *queue.Queue
}

func (im *IndexManager) Init(bootstrap bool) {
	// If bootstrap is true, initialize mapping in ElasticSearch
	if bootstrap {
		im.initMapping()
	}

	// Initialize index worker queue.
	im.IndexQueue = queue.NewQueue(func(metricPath interface{}) {
		path, _ := metricPath.(string)
		im.index(path)
	}, 100)
}

func (im *IndexManager) Start(wg *sync.WaitGroup) {
	im.wg = wg
	im.wg.Add(1)
	go im.run()
}

func (im *IndexManager) run() {

	defer config.G.OnPanic()

	// Wait for entries to arrive, and process them.
	for {
		select {
		case <-config.G.OnReload2:
			config.G.Log.System.LogDebug("IndexManager::run received QUIT message")
			im.wg.Done()
			return
		case metric := <-config.G.Channels.IndexStore:
			im.IndexQueue.Push(metric.Path)
		case query := <-config.G.Channels.IndexRequest:
			go im.query(query)
		}
	}
}

// initMapping initializes ElasticSearch for cassabon.
func (im *IndexManager) initMapping() {
	mapping := map[string]map[string]map[string]map[string]map[string]string{
		"mappings": map[string]map[string]map[string]map[string]string{
			"path": map[string]map[string]map[string]string{
				"properties": map[string]map[string]string{
					"path": map[string]string{
						"type":  "string",
						"index": "not_analyzed",
					},
					"depth": map[string]string{
						"type": "long",
					},
					"tenant": map[string]string{
						"type": "string",
					},
					"leaf": map[string]string{
						"type": "boolean",
					},
				},
			},
		},
	}

	jsonMap, _ := json.Marshal(mapping)
	config.G.Log.System.LogDebug("%s", string(jsonMap))

	putreq, _ := http.NewRequest("PUT", config.G.ElasticSearch.MapURL, bytes.NewBuffer(jsonMap))
	r := im.httpRequest(putreq)

	config.G.Log.System.LogDebug("%v", string(r))

	if r == nil {
		config.G.Log.System.LogFatal("Could not initialize mapping for ElasticSearch.")
	}
}

// getAllLeafNodes queries ElasticSearch for all leaf nodes. Used for populating metric manager's stat paths on reboot.
func (im *IndexManager) getAllLeafNodes() []string {
	sort := []map[string]map[string]string{
		{
			"path": map[string]string{
				"order": "asc",
			},
		},
	}
	query := map[string]map[string][]map[string]map[string]interface{}{
		"bool": map[string][]map[string]map[string]interface{}{
			"must": []map[string]map[string]interface{}{
				{
					"match": map[string]interface{}{
						"leaf": true,
					},
				},
			},
		},
	}

	fullQuery := ERQuery{sort, query}
	getreq := im.prepRequest(fullQuery)
	r := im.httpRequest(getreq)

	var esResp ElasticResponse
	var pathList []string

	if r != nil {
		_ = json.Unmarshal(r, &esResp)

		config.G.Log.System.LogDebug("esResp: %v", esResp)

		for _, hit := range esResp.Hits.Hits {
			pathList = append(pathList, hit.Source.Path)
		}
	} else {
		logging.Statsd.Client.Inc("indexmgr.es.err.get", 1, 1.0)
		config.G.Log.System.LogError("Error querying ES.")
	}

	config.G.Log.System.LogDebug("Retrieved %v stat paths.", len(pathList))
	return pathList
}

func (im *IndexManager) prepRequest(fullQuery ERQuery) *http.Request {
	jsonQuery, _ := json.Marshal(fullQuery)
	config.G.Log.System.LogDebug("%s", string(jsonQuery))

	// Get the count so that we capture all of the possible paths.
	countreq, _ := http.NewRequest("GET", config.G.ElasticSearch.CountURL, strings.NewReader(string(jsonQuery)))
	size := "size=" + im.getCount(countreq)

	searchURL := strings.Join([]string{config.G.ElasticSearch.SearchURL, size}, "?")
	getreq, _ := http.NewRequest("GET", searchURL, strings.NewReader(string(jsonQuery)))

	return getreq
}

// IndexMetricPath takes a metric path string and sends it off to be processed by processMetricPath().
func (im *IndexManager) index(path string) {
	it := time.Now()
	config.G.Log.System.LogDebug("IndexManager::index path=%s", path)
	splitPath := strings.Split(path, ".")
	im.processMetricPath(splitPath, len(splitPath), true)
	logging.Statsd.Client.TimingDuration("indexmgr.index", time.Since(it), 1.0)
}

func (im *IndexManager) httpRequest(req *http.Request) []byte {
	client := &http.Client{Timeout: time.Duration(15 * time.Second)}
	resp, err := client.Do(req)

	if err != nil {
		logging.Statsd.Client.Inc("indexmgr.es.err.httpreq", 1, 1.0)
		config.G.Log.System.LogError("Received error from ElasticSearch: %v, request: %v", err.Error(), req)
		return nil
	}

	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)
	return body
}

// processMetricPath recursively indexes the metric path via the ElasticSearch REST API.
func (im *IndexManager) processMetricPath(splitPath []string, pathLen int, isLeaf bool) {
	// Process the metric path one node at a time.  We store metrics in ElasticSearch.
	retries := 0
	for pathLen > 0 {

		// Construct the metric string
		metricPath := strings.Join(splitPath, ".")

		// Strip % off the end to avoid invalid escape errors.
		if string(metricPath[len(metricPath)-1]) == "%" {
			metricPath = metricPath[:len(metricPath)-1]
		}
		config.G.Log.System.LogDebug("IndexManager indexing \"%s\"", metricPath)

		pathToIndex := IndexResponse{
			metricPath,
			pathLen,
			"",
			isLeaf,
		}

		urlToPut := strings.Join([]string{config.G.ElasticSearch.PutURL, metricPath}, "/")

		// Marshal the struct into JSON
		jsonPath, err := json.Marshal(pathToIndex)

		if err != nil {
			logging.Statsd.Client.Inc("indexmgr.es.err.json", 1, 1.0)
			config.G.Log.System.LogError("Unable to marshal pathToIndex of %v, error is %v", pathToIndex, err.Error())
			return // Let's not fill up ES with junk if we can't marshal our path struct.
		}

		putreq, err := http.NewRequest("PUT", urlToPut, bytes.NewBuffer(jsonPath))

		if err != nil {
			logging.Statsd.Client.Inc("indexmgr.es.err.put", 1, 1.0)
			config.G.Log.System.LogError("Error when attempting to index %v: %v", metricPath, err.Error())
		}

		r := im.httpRequest(putreq)
		if r != nil {
			// Pop the last node of the metric off, set isLeaf to false, and resume loop.
			_, splitPath = splitPath[len(splitPath)-1], splitPath[:len(splitPath)-1]
			isLeaf = false
			pathLen = len(splitPath)
			retries = 0
		} else {
			logging.Statsd.Client.Inc("indexmgr.es.err.pmr.req", 1, 1.0)
			retries++
			config.G.Log.System.LogWarn("processMetricPath's httprequest to ES came back as nil, sending to retry in %d seconds.", retries)
			time.Sleep(time.Duration(retries) * time.Second)
		}
	}
}

func (im *IndexManager) getCount(req *http.Request) string {
	var resp ElasticResponse
	r := im.httpRequest(req)
	if r != nil {
		_ = json.Unmarshal(r, &resp)
		config.G.Log.System.LogDebug("total: %v", resp.Hits.Total)
		return strconv.Itoa(resp.Hits.Total)
	} else {
		return "0"
	}
}

// query returns the data matched by the supplied query.
func (im *IndexManager) query(q config.IndexQuery) {
	switch strings.ToLower(q.Method) {
	case "delete":
		// TODO
	default:
		im.queryGET(q)
	}
}

// query returns the data matched by the supplied query.
func (im *IndexManager) queryGET(q config.IndexQuery) {

	config.G.Log.System.LogDebug("IndexManager::query %v", q.Query)

	// Query particulars are mandatory.
	if q.Query == "" {
		q.Channel <- config.APIQueryResponse{config.AQS_BADREQUEST, "no query specified", []byte{}}
		return
	}
	// Convert query to form suitable for Elasticsearch regexp search.
	regexpQuery := strings.Replace(q.Query, ".", "\\.", -1)
	regexpQuery = strings.Replace(regexpQuery, "*", ".*", -1)

	// Get number of nodes in the path for the ElasticSearch Query
	pathDepth := len(strings.Split(q.Query, "."))

	var esResp ElasticResponse
	var respList []IndexResponse
	var resp config.APIQueryResponse

	// It's turtles all the way down!  This is totally Vijay's fault.
	// http://github.com/vijaykramesh -- JP
	sort := []map[string]map[string]string{
		{
			"path": map[string]string{
				"order": "asc",
			},
		},
	}
	query := map[string]map[string][]map[string]map[string]interface{}{
		"bool": map[string][]map[string]map[string]interface{}{
			"must": []map[string]map[string]interface{}{
				{
					"regexp": map[string]interface{}{
						"path": regexpQuery,
					},
				},
				{
					"match": map[string]interface{}{
						"depth": pathDepth,
					},
				},
			},
		},
	}

	fullQuery := ERQuery{sort, query}
	getreq := im.prepRequest(fullQuery)
	r := im.httpRequest(getreq)

	if r != nil {
		_ = json.Unmarshal(r, &esResp)

		config.G.Log.System.LogDebug("esResp: %v", esResp)

		for _, hit := range esResp.Hits.Hits {
			respList = append(respList, hit.Source)
		}

		jsonResp, _ := json.Marshal(respList)

		resp = config.APIQueryResponse{config.AQS_OK, "", jsonResp}
	} else {
		logging.Statsd.Client.Inc("indexmgr.es.err.get", 1, 1.0)
		config.G.Log.System.LogError("Error querying ES.")
		resp = config.APIQueryResponse{config.AQS_ERROR, "Error querying ES", []byte{}}
	}

	// If the API gave up on us because we took too long, writing to the channel
	// will cause first a data race, and then a panic (write on closed channel).
	// We check, but if we lose a race we will need to recover.
	defer func() {
		_ = recover()
	}()

	// Check whether the channel is closed before attempting a write.
	select {
	case <-q.Channel:
		// Immediate return means channel is closed (we know there is no data in it).
	default:
		// If the channel would have blocked, it is open, we can write to it.
		q.Channel <- resp
	}
}
<|endoftext|>


<commit_before>package controllers

import (
	"adexchange/lib"
	m "adexchange/models"
	"github.com/astaxie/beego"
	"time"
)

type TrackingController struct {
	beego.Controller
}

func (this *RequestController) TrackImp() {
	adRequest := m.AdRequest{}
	adResponse := new(m.TrackResponse)
	beego.Debug("Enter Tracking imp")

	if err := this.ParseForm(&adRequest); err != nil {
		adResponse.StatusCode = lib.ERROR_PARSE_REQUEST
	} else {
		adResponse.StatusCode = lib.STATUS_SUCCESS
		clientIp := GetClientIP(this.Ctx.Input)
		beego.Debug("Clk Client IP:" + clientIp)
		adRequest.Ip = clientIp
		adRequest.RequestTime = time.Now().Unix()
		go SendLog(adRequest, 2)
	}

	this.Data["json"] = &adResponse
	this.ServeJson()

}

func (this *RequestController) TrackClk() {
	adRequest := m.AdRequest{}
	adResponse := new(m.TrackResponse)
	beego.Debug("Enter Tracking clk")

	if err := this.ParseForm(&adRequest); err != nil {
		adResponse.StatusCode = lib.ERROR_PARSE_REQUEST
	} else {
		adResponse.StatusCode = lib.STATUS_SUCCESS
		clientIp := GetClientIP(this.Ctx.Input)
		beego.Debug("Imp Client IP:" + clientIp)
		adRequest.Ip = clientIp
		adRequest.RequestTime = time.Now().Unix()
		go SendLog(adRequest, 3)
	}

	cacheKey := lib.GetMd5String(adRequest.Bid)

	originalTrackingUrl := GetCachedClkUrl(cacheKey)

	if len(originalTrackingUrl) > 0 {
		this.Ctx.Redirect(302, originalTrackingUrl)
	} else {
		adResponse.StatusCode = lib.ERROR_AD_EXPIRED
		this.Data["json"] = &adResponse
		this.ServeJson()
	}
}
<commit_msg>refine tracking response<commit_after>package controllers

import (
	"adexchange/lib"
	m "adexchange/models"
	"github.com/astaxie/beego"
	"time"
)

type TrackingController struct {
	beego.Controller
}

func (this *RequestController) TrackImp() {
	adRequest := m.AdRequest{}
	adResponse := new(m.TrackResponse)
	beego.Debug("Enter Tracking imp")

	if err := this.ParseForm(&adRequest); err != nil {
		adResponse.StatusCode = lib.ERROR_PARSE_REQUEST
	} else {
		adResponse.StatusCode = lib.STATUS_SUCCESS
		clientIp := GetClientIP(this.Ctx.Input)
		beego.Debug("Clk Client IP:" + clientIp)
		adRequest.Ip = clientIp
		adRequest.RequestTime = time.Now().Unix()
		go SendLog(adRequest, 2)
	}

	//this.Data["json"] = &adResponse
	//this.ServeJson()

}

func (this *RequestController) TrackClk() {
	adRequest := m.AdRequest{}
	adResponse := new(m.TrackResponse)
	beego.Debug("Enter Tracking clk")

	if err := this.ParseForm(&adRequest); err != nil {
		adResponse.StatusCode = lib.ERROR_PARSE_REQUEST
	} else {
		adResponse.StatusCode = lib.STATUS_SUCCESS
		clientIp := GetClientIP(this.Ctx.Input)
		beego.Debug("Imp Client IP:" + clientIp)
		adRequest.Ip = clientIp
		adRequest.RequestTime = time.Now().Unix()
		go SendLog(adRequest, 3)
	}

	cacheKey := lib.GetMd5String(adRequest.Bid)

	originalTrackingUrl := GetCachedClkUrl(cacheKey)

	if len(originalTrackingUrl) > 0 {
		this.Ctx.Redirect(302, originalTrackingUrl)
	}
	//else {
	//	adResponse.StatusCode = lib.ERROR_AD_EXPIRED
	//	this.Data["json"] = &adResponse
	//	this.ServeJson()
	//}
}
<|endoftext|>


<commit_before>package kcp

import (
	"net"
	"sync"
	"time"

	"github.com/v2ray/v2ray-core/common/alloc"
	"github.com/v2ray/v2ray-core/common/log"
	v2net "github.com/v2ray/v2ray-core/common/net"
	"github.com/v2ray/v2ray-core/common/serial"
	"github.com/v2ray/v2ray-core/transport/internet"
	"github.com/v2ray/v2ray-core/transport/internet/udp"
)

// Listener defines a server listening for connections
type Listener struct {
	sync.Mutex
	running       bool
	block         Authenticator
	sessions      map[string]*Connection
	awaitingConns chan *Connection
	hub           *udp.UDPHub
	localAddr     *net.UDPAddr
}

func NewListener(address v2net.Address, port v2net.Port) (*Listener, error) {
	l := &Listener{
		block:         NewSimpleAuthenticator(),
		sessions:      make(map[string]*Connection),
		awaitingConns: make(chan *Connection, 64),
		localAddr: &net.UDPAddr{
			IP:   address.IP(),
			Port: int(port),
		},
		running: true,
	}
	hub, err := udp.ListenUDP(address, port, l.OnReceive)
	if err != nil {
		return nil, err
	}
	l.hub = hub
	log.Info("KCP|Listener: listening on ", address, ":", port)
	return l, nil
}

func (this *Listener) OnReceive(payload *alloc.Buffer, src v2net.Destination) {
	defer payload.Release()

	if valid := this.block.Open(payload); !valid {
		log.Info("KCP|Listener: discarding invalid payload from ", src)
		return
	}
	if !this.running {
		return
	}
	this.Lock()
	defer this.Unlock()
	if !this.running {
		return
	}
	srcAddrStr := src.NetAddr()
	conn, found := this.sessions[srcAddrStr]
	if !found {
		conv := serial.BytesToUint16(payload.Value)
		writer := &Writer{
			hub:      this.hub,
			dest:     src,
			listener: this,
		}
		srcAddr := &net.UDPAddr{
			IP:   src.Address().IP(),
			Port: int(src.Port()),
		}
		conn = NewConnection(conv, writer, this.localAddr, srcAddr, this.block)
		select {
		case this.awaitingConns <- conn:
		case <-time.After(time.Second * 5):
			conn.Close()
			return
		}
		this.sessions[srcAddrStr] = conn
	}
	conn.Input(payload.Value)
}

func (this *Listener) Remove(dest string) {
	if !this.running {
		return
	}
	this.Lock()
	defer this.Unlock()
	if !this.running {
		return
	}
	delete(this.sessions, dest)
}

// Accept implements the Accept method in the Listener interface; it waits for the next call and returns a generic Conn.
func (this *Listener) Accept() (internet.Connection, error) {
	for {
		if !this.running {
			return nil, errClosedListener
		}
		select {
		case conn := <-this.awaitingConns:
			return conn, nil
		case <-time.After(time.Second):

		}
	}
}

// Close stops listening on the UDP address. Already Accepted connections are not closed.
func (this *Listener) Close() error {
	if !this.running {
		return errClosedListener
	}
	this.Lock()
	defer this.Unlock()

	this.running = false
	close(this.awaitingConns)
	for _, conn := range this.sessions {
		go conn.Terminate()
	}
	this.hub.Close()

	return nil
}

// Addr returns the listener's network address, The Addr returned is shared by all invocations of Addr, so do not modify it.
func (this *Listener) Addr() net.Addr {
	return this.localAddr
}

type Writer struct {
	dest     v2net.Destination
	hub      *udp.UDPHub
	listener *Listener
}

func (this *Writer) Write(payload []byte) (int, error) {
	return this.hub.WriteTo(payload, this.dest)
}

func (this *Writer) Close() error {
	this.listener.Remove(this.dest.NetAddr())
	return nil
}

func ListenKCP(address v2net.Address, port v2net.Port) (internet.Listener, error) {
	return NewListener(address, port)
}

func init() {
	internet.KCPListenFunc = ListenKCP
}
<commit_msg>more accurate source id<commit_after>package kcp

import (
	"net"
	"sync"
	"time"

	"github.com/v2ray/v2ray-core/common/alloc"
	"github.com/v2ray/v2ray-core/common/log"
	v2net "github.com/v2ray/v2ray-core/common/net"
	"github.com/v2ray/v2ray-core/common/serial"
	"github.com/v2ray/v2ray-core/transport/internet"
	"github.com/v2ray/v2ray-core/transport/internet/udp"
)

// Listener defines a server listening for connections
type Listener struct {
	sync.Mutex
	running       bool
	block         Authenticator
	sessions      map[string]*Connection
	awaitingConns chan *Connection
	hub           *udp.UDPHub
	localAddr     *net.UDPAddr
}

func NewListener(address v2net.Address, port v2net.Port) (*Listener, error) {
	l := &Listener{
		block:         NewSimpleAuthenticator(),
		sessions:      make(map[string]*Connection),
		awaitingConns: make(chan *Connection, 64),
		localAddr: &net.UDPAddr{
			IP:   address.IP(),
			Port: int(port),
		},
		running: true,
	}
	hub, err := udp.ListenUDP(address, port, l.OnReceive)
	if err != nil {
		return nil, err
	}
	l.hub = hub
	log.Info("KCP|Listener: listening on ", address, ":", port)
	return l, nil
}

func (this *Listener) OnReceive(payload *alloc.Buffer, src v2net.Destination) {
	defer payload.Release()

	if valid := this.block.Open(payload); !valid {
		log.Info("KCP|Listener: discarding invalid payload from ", src)
		return
	}
	if !this.running {
		return
	}
	this.Lock()
	defer this.Unlock()
	if !this.running {
		return
	}
	conv := serial.BytesToUint16(payload.Value)
	sourceId := src.NetAddr() + "|" + serial.Uint16ToString(conv)
	conn, found := this.sessions[sourceId]
	if !found {
		writer := &Writer{
			hub:      this.hub,
			dest:     src,
			listener: this,
		}
		srcAddr := &net.UDPAddr{
			IP:   src.Address().IP(),
			Port: int(src.Port()),
		}
		conn = NewConnection(conv, writer, this.localAddr, srcAddr, this.block)
		select {
		case this.awaitingConns <- conn:
		case <-time.After(time.Second * 5):
			conn.Close()
			return
		}
		this.sessions[sourceId] = conn
	}
	conn.Input(payload.Value)
}

func (this *Listener) Remove(dest string) {
	if !this.running {
		return
	}
	this.Lock()
	defer this.Unlock()
	if !this.running {
		return
	}
	delete(this.sessions, dest)
}

// Accept implements the Accept method in the Listener interface; it waits for the next call and returns a generic Conn.
func (this *Listener) Accept() (internet.Connection, error) {
	for {
		if !this.running {
			return nil, errClosedListener
		}
		select {
		case conn := <-this.awaitingConns:
			return conn, nil
		case <-time.After(time.Second):

		}
	}
}

// Close stops listening on the UDP address. Already Accepted connections are not closed.
func (this *Listener) Close() error {
	if !this.running {
		return errClosedListener
	}
	this.Lock()
	defer this.Unlock()

	this.running = false
	close(this.awaitingConns)
	for _, conn := range this.sessions {
		go conn.Terminate()
	}
	this.hub.Close()

	return nil
}

// Addr returns the listener's network address, The Addr returned is shared by all invocations of Addr, so do not modify it.
func (this *Listener) Addr() net.Addr {
	return this.localAddr
}

type Writer struct {
	dest     v2net.Destination
	hub      *udp.UDPHub
	listener *Listener
}

func (this *Writer) Write(payload []byte) (int, error) {
	return this.hub.WriteTo(payload, this.dest)
}

func (this *Writer) Close() error {
	this.listener.Remove(this.dest.NetAddr())
	return nil
}

func ListenKCP(address v2net.Address, port v2net.Port) (internet.Listener, error) {
	return NewListener(address, port)
}

func init() {
	internet.KCPListenFunc = ListenKCP
}
<|endoftext|>


<commit_before>package http

import (
	"encoding/json"
	"fmt"
	"net/http"
)

func (s *TriggerServer) approvalsHandler(resp http.ResponseWriter, req *http.Request) {
	// unknown lists all
	approvals, err := s.approvalsManager.List()
	if err != nil {
		fmt.Fprintf(resp, "%s", err)
		resp.WriteHeader(http.StatusInternalServerError)
		return
	}

	bts, err := json.Marshal(&approvals)
	if err != nil {
		fmt.Fprintf(resp, "%s", err)
		resp.WriteHeader(http.StatusInternalServerError)
		return
	}

	resp.Write(bts)
}
<commit_msg>empty list<commit_after>package http

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/rusenask/keel/types"
)

func (s *TriggerServer) approvalsHandler(resp http.ResponseWriter, req *http.Request) {
	// unknown lists all
	approvals, err := s.approvalsManager.List()
	if err != nil {
		fmt.Fprintf(resp, "%s", err)
		resp.WriteHeader(http.StatusInternalServerError)
		return
	}

	if len(approvals) == 0 {
		approvals = make([]*types.Approval, 0)
	}

	bts, err := json.Marshal(&approvals)
	if err != nil {
		fmt.Fprintf(resp, "%s", err)
		resp.WriteHeader(http.StatusInternalServerError)
		return
	}

	resp.Write(bts)
}
<|endoftext|>


<commit_before>/*
Package latency provides an easy way to create latency histograms and generate graphs.

Example that creates a histogram with 10 buckets and records an event that took 16ms.

b := make(Buckets, 10)

b.Record(16 * time.Millisecond)

b.Plot("test.svg"))

10 buckets is good for tracking latency from 0ms to 1s.
Even if too few buckets are specified, new buckets will be added as needed.

Minimum resolution: 1 millisecond.
Scale: powers of 2.
*/
package latency

import (
	"fmt"
	"math"
	"os"
	"time"

	"code.google.com/p/plotinum/plot"
	"code.google.com/p/plotinum/plotter"
)

type Buckets []int

func (b *Buckets) Record(d time.Duration) {
	ms := float64(d.Nanoseconds() / 1e6)
	bucket := int(math.Ceil(math.Log2(ms)))
	if len(*b) < bucket+1 {
		// Extend the slice.
		*b = append(*b, make(Buckets, bucket+1-len(*b))...)
	}
	(*b)[bucket]++
}

// Plot saves an image of the latency histogram to filePath. The extension of filePath defines
// the format to be used - png, svg, etc.
func (b *Buckets) Plot(filePath string) error {
	count := len(*b)

	xys := make(plotter.XYs, count)

	for bucket, freq := range *b {
		xys[bucket].X = float64(bucket)
		xys[bucket].Y = float64(freq)
	}

	p, err := plot.New()
	if err != nil {
		return fmt.Errorf("error generating plot: %v", err)
	}
	p.Title.Text = "Latency histogram"
	p.X.Label.Text = "Latency (ms)"
	p.Y.Label.Text = "Frequency"

	h, err := plotter.NewHistogram(xys, count)
	if err != nil {
		return fmt.Errorf("error generating histogram: %v", err)

	}
	p.Add(h)

	// Save the plot to a file. Units in inches (one inch == 72 points).
	fmt.Fprintf(os.Stderr, "Saving latency histogram to", filePath)
	return p.Save(8, 6, filePath)
}
<commit_msg>Configurable resolution. Dont crash for low latency<commit_after>/*
Package latency provides an easy way to create latency histograms and generate graphs.

Example that creates a histogram with 10 buckets and records an event that took 16ms (16000us).

b := make(Buckets, 20)

b.Record(16 * time.Millisecond)

b.Plot("test.svg"))

20 buckets is good for tracking latency from 0us to 1s. Buckets are created in a scale of powers of
2. Even if too few buckets are specified, new buckets will be added as needed.

*/
package latency

import (
	"fmt"
	"math"
	"os"
	"time"

	"code.google.com/p/plotinum/plot"
	"code.google.com/p/plotinum/plotter"
)

// Resolution can be changed if the latency tracking should happen at lower or right resolution than
// the default of 1 Millisecond.
// TODO(nictuku): Merge with Buckets.
var Resolution = time.Millisecond

type Buckets []int

func (b *Buckets) Record(d time.Duration) {
	var bucket int
	if d < Resolution {
		// Avoid getting to log2(0) = -Inf.
		bucket = 0
	} else {
		f := float64(d / Resolution)
		bucket = int(math.Ceil(math.Log2(f)))
		if len(*b) < bucket+1 {
			// Extend the slice.
			*b = append(*b, make(Buckets, bucket+1-len(*b))...)
		}
	}
	(*b)[bucket]++
}

// Plot saves an image of the latency histogram to filePath. The extension of filePath defines
// the format to be used - png, svg, etc.
func (b *Buckets) Plot(filePath string) error {
	count := len(*b)

	xys := make(plotter.XYs, count)

	for bucket, freq := range *b {
		xys[bucket].X = float64(bucket)
		xys[bucket].Y = float64(freq)
	}

	p, err := plot.New()
	if err != nil {
		return fmt.Errorf("error generating plot: %v", err)
	}
	p.Title.Text = "Latency histogram"
	p.X.Label.Text = fmt.Sprintf("Latency (%v resolution)", Resolution)
	p.Y.Label.Text = "Frequency"

	h, err := plotter.NewHistogram(xys, count)
	if err != nil {
		return fmt.Errorf("error generating histogram: %v", err)

	}
	p.Add(h)

	// Save the plot to a file. Units in inches (one inch == 72 points).
	fmt.Fprintf(os.Stderr, "Saving latency histogram to %v\n", filePath)
	return p.Save(8, 6, filePath)
}
<|endoftext|>