<commit_before>package gateway

import (
	"net/http"
	"strconv"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/sla"
	"github.com/funkygao/golib/hack"
	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

// GET /v1/msgs/:appid/:topic/:ver?group=xx&batch=10&reset=<newest|oldest>&ack=1&q=<dead|retry>
func (this *subServer) subHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	var (
		topic      string
		ver        string
		myAppid    string
		hisAppid   string
		reset      string
		group      string
		shadow     string
		rawTopic   string
		partition  string
		partitionN int = -1
		offset     string
		offsetN    int64    = -1
		limit      int      // max messages to include in the message set
		delayedAck bool     // explicit application level acknowledgement
		tagFilters []MsgTag = nil
		err        error
	)

	if Options.EnableClientStats {
		this.gw.clientStates.RegisterSubClient(r)
	}

	query := r.URL.Query()
	group = query.Get("group")
	reset = query.Get("reset")
	if !manager.Default.ValidateGroupName(r.Header, group) {
		writeBadRequest(w, "illegal group")
		return
	}

	limit, err = getHttpQueryInt(&query, "batch", 1)
	if err != nil {
		writeBadRequest(w, "illegal limit")
		return
	}
	if limit > Options.MaxSubBatchSize && Options.MaxSubBatchSize > 0 {
		limit = Options.MaxSubBatchSize
	}

	ver = params.ByName(UrlParamVersion)
	topic = params.ByName(UrlParamTopic)
	hisAppid = params.ByName(UrlParamAppid)
	myAppid = r.Header.Get(HttpHeaderAppid)
	realIp := getHttpRemoteIp(r)

	// auth
	if err = manager.Default.AuthSub(myAppid, r.Header.Get(HttpHeaderSubkey),
		hisAppid, topic, group); err != nil {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"), err)

		writeAuthFailure(w, err)
		return
	}

	// fetch the client ack partition and offset
	delayedAck = query.Get("ack") == "1"
	if delayedAck {
		// consumers use explicit acknowledges in order to signal a message as processed successfully
		// if consumers fail to ACK, the message hangs and server will refuse to move ahead

		// get the partitionN and offsetN from client header
		// client will ack with partition=-1, offset=-1:
		// 1. handshake phase
		// 2. when 204 No Content
		partition = r.Header.Get(HttpHeaderPartition)
		offset = r.Header.Get(HttpHeaderOffset)
		if partition != "" && offset != "" {
			// convert partition and offset to int
			offsetN, err = strconv.ParseInt(offset, 10, 64)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} offset:%s",
					myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
					group, r.Header.Get("User-Agent"), offset)

				writeBadRequest(w, "ack with bad offset")
				return
			}
			partitionN, err = strconv.Atoi(partition)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} partition:%s",
					myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
					group, r.Header.Get("User-Agent"), partition)

				writeBadRequest(w, "ack with bad partition")
				return
			}
		} else if len(partition+offset) != 0 {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s partition:%s offset:%s UA:%s} partial ack",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, partition, offset, r.Header.Get("User-Agent"))

			writeBadRequest(w, "partial ack not allowed")
			return
		}
	}

	shadow = query.Get("q")

	log.Debug("sub[%s] %s(%s): {app:%s q:%s topic:%s ver:%s group:%s batch:%d ack:%s partition:%s offset:%s UA:%s}",
		myAppid, r.RemoteAddr, realIp, hisAppid, shadow, topic, ver,
		group, limit, query.Get("ack"), partition, offset, r.Header.Get("User-Agent"))

	// calculate raw topic according to shadow
	if shadow != "" {
		if !sla.ValidateShadowName(shadow) {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s q:%s UA:%s} invalid shadow name",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "invalid shadow name")
			return
		}

		if !manager.Default.IsShadowedTopic(hisAppid, topic, ver, myAppid, group) {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s q:%s UA:%s} not a shadowed topic",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "register shadow first")
			return
		}

		rawTopic = manager.Default.ShadowTopic(shadow, myAppid, hisAppid, topic, ver, group)
	} else {
		rawTopic = manager.Default.KafkaTopic(hisAppid, topic, ver)
	}

	cluster, found := manager.Default.LookupCluster(hisAppid)
	if !found {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} cluster not found",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"))

		writeBadRequest(w, "invalid appid")
		return
	}

	fetcher, err := store.DefaultSubStore.Fetch(cluster, rawTopic,
		myAppid+"."+group, r.RemoteAddr, reset, Options.PermitStandbySub)
	if err != nil {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"), err)

		writeBadRequest(w, err.Error())
		return
	}

	// commit the acked offset
	if delayedAck && partitionN >= 0 && offsetN >= 0 {
		if err = fetcher.CommitUpto(&sarama.ConsumerMessage{
			Topic:     rawTopic,
			Partition: int32(partitionN),
			Offset:    offsetN,
		}); err != nil {
			// during rebalance, this might happen, but with no bad effects
			log.Trace("sub land[%s] %s(%s): {app:%s topic:%s/%s ver:%s group:%s ack:1 partition:%s offset:%s UA:%s} %v",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, partition, ver,
				group, offset, r.Header.Get("User-Agent"), err)
		} else {
			log.Debug("sub land %s(%s): {G:%s, T:%s/%s, O:%s}",
				r.RemoteAddr, realIp, group, rawTopic, partition, offset)
		}
	}

	tag := r.Header.Get(HttpHeaderMsgTag)
	if tag != "" {
		tagFilters = parseMessageTag(tag)
	}

	err = this.pumpMessages(w, r, fetcher, limit, myAppid, hisAppid, topic, ver, group, delayedAck, tagFilters)
	if err != nil {
		// e,g. broken pipe, io timeout, client gone
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s ack:%s partition:%s offset:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, query.Get("ack"), partition, offset, r.Header.Get("User-Agent"), err)

		writeServerError(w, err.Error())

		if err = fetcher.Close(); err != nil {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s} %v",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver, group, err)
		}
	}
}

func (this *subServer) pumpMessages(w http.ResponseWriter, r *http.Request,
	fetcher store.Fetcher, limit int, myAppid, hisAppid, topic, ver,
	group string, delayedAck bool, tagFilters []MsgTag) error {
	clientGoneCh := w.(http.CloseNotifier).CloseNotify()

	var (
		metaBuf     []byte = nil
		n                  = 0
		idleTimeout        = Options.SubTimeout
		realIp             = getHttpRemoteIp(r)
		chunkedEver        = false
	)
	for {
		select {
		case <-clientGoneCh:
			// FIXME access log will not be able to record this behavior
			return ErrClientGone

		case <-this.gw.shutdownCh:
			// don't call me again
			w.Header().Set("Connection", "close")

			if !chunkedEver {
				w.WriteHeader(http.StatusNoContent)
				w.Write([]byte{})
			}

			return nil

		case err := <-fetcher.Errors():
			// e,g. consume a non-existent topic
			// e,g. conn with broker is broken
			return err

		case <-this.gw.timer.After(idleTimeout):
			if chunkedEver {
				// response already sent in chunk
				log.Debug("chunked sub idle timeout %s {A:%s/G:%s->A:%s T:%s V:%s}",
					idleTimeout, myAppid, group, hisAppid, topic, ver)
				return nil
			}

			w.WriteHeader(http.StatusNoContent)
			w.Write([]byte{}) // without this, client cant get response
			return nil

		case msg := <-fetcher.Messages():
			partition := strconv.FormatInt(int64(msg.Partition), 10)

			if limit == 1 {
				w.Header().Set(HttpHeaderMsgKey, string(msg.Key))
				w.Header().Set(HttpHeaderPartition, partition)
				w.Header().Set(HttpHeaderOffset, strconv.FormatInt(msg.Offset, 10))
			}

			var (
				tags    []MsgTag
				bodyIdx int
				err     error
			)
			if IsTaggedMessage(msg.Value) {
				// TagMarkStart + tag + TagMarkEnd + body
				tags, bodyIdx, err = ExtractMessageTag(msg.Value)
				if limit == 1 && err == nil {
					// needn't check 'index out of range' here
					w.Header().Set(HttpHeaderMsgTag, hack.String(msg.Value[1:bodyIdx-1]))
				} else {
					// not a valid tagged message, treat it as non-tagged message
				}
			}

			if len(tags) > 0 {
				// TODO compare with tagFilters
			}

			if limit == 1 {
				// non-batch mode, just the message itself without meta
				if _, err = w.Write(msg.Value[bodyIdx:]); err != nil {
					// when remote close silently, the write still ok
					return err
				}
			} else {
				// batch mode, write MessageSet
				// MessageSet => [Partition(int32) Offset(int64) MessageSize(int32) Message] BigEndian
				if metaBuf == nil {
					// initialize the reuseable buffer
					metaBuf = make([]byte, 8)

					// remove the middleware added header
					w.Header().Del("Content-Type")
				}

				if err = writeI32(w, metaBuf, msg.Partition); err != nil {
					return err
				}
				if err = writeI64(w, metaBuf, msg.Offset); err != nil {
					return err
				}
				if err = writeI32(w, metaBuf, int32(len(msg.Value[bodyIdx:]))); err != nil {
					return err
				}
				// TODO add tag?
				if _, err = w.Write(msg.Value[bodyIdx:]); err != nil {
					return err
				}
			}

			if !delayedAck {
				log.Debug("sub auto commit offset %s(%s): {G:%s, T:%s/%d, O:%d}",
					r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset)

				// ignore the offset commit err on purpose:
				// during rebalance, offset commit often encounter errors because fetcher
				// underlying partition offset tracker has changed
				// e,g.
				// topic has partition: 0, 1
				// 1. got msg(p=0) from fetcher
				// 2. rebalanced, then start consuming p=1
				// 3. commit the msg offset, still msg(p=0) => error
				// BUT, it has no fatal effects.
				// The worst case is between 1-3, kateway shutdown, sub client
				// will get 1 duplicated msg.
				fetcher.CommitUpto(msg)
			} else {
				log.Debug("sub take off %s(%s): {G:%s, T:%s/%d, O:%d}",
					r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset)
			}

			this.subMetrics.ConsumeOk(myAppid, topic, ver)
			this.subMetrics.ConsumedOk(hisAppid, topic, ver)

			n++
			if n >= limit {
				return nil
			}

			// http chunked: len in hex
			// curl CURLOPT_HTTP_TRANSFER_DECODING will auto unchunk
			w.(http.Flusher).Flush()

			chunkedEver = true

			if n == 1 {
				log.Debug("sub idle timeout %s->1s %s(%s): {G:%s, T:%s/%d, O:%d B:%d}",
					idleTimeout, r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset, limit)
				idleTimeout = time.Second
			}

		}
	}
}
<commit_msg>fix printf fmt err<commit_after>package gateway

import (
	"net/http"
	"strconv"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/sla"
	"github.com/funkygao/golib/hack"
	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

// GET /v1/msgs/:appid/:topic/:ver?group=xx&batch=10&reset=<newest|oldest>&ack=1&q=<dead|retry>
func (this *subServer) subHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	var (
		topic      string
		ver        string
		myAppid    string
		hisAppid   string
		reset      string
		group      string
		shadow     string
		rawTopic   string
		partition  string
		partitionN int = -1
		offset     string
		offsetN    int64    = -1
		limit      int      // max messages to include in the message set
		delayedAck bool     // explicit application level acknowledgement
		tagFilters []MsgTag = nil
		err        error
	)

	if Options.EnableClientStats {
		this.gw.clientStates.RegisterSubClient(r)
	}

	query := r.URL.Query()
	group = query.Get("group")
	reset = query.Get("reset")
	if !manager.Default.ValidateGroupName(r.Header, group) {
		writeBadRequest(w, "illegal group")
		return
	}

	limit, err = getHttpQueryInt(&query, "batch", 1)
	if err != nil {
		writeBadRequest(w, "illegal limit")
		return
	}
	if limit > Options.MaxSubBatchSize && Options.MaxSubBatchSize > 0 {
		limit = Options.MaxSubBatchSize
	}

	ver = params.ByName(UrlParamVersion)
	topic = params.ByName(UrlParamTopic)
	hisAppid = params.ByName(UrlParamAppid)
	myAppid = r.Header.Get(HttpHeaderAppid)
	realIp := getHttpRemoteIp(r)

	// auth
	if err = manager.Default.AuthSub(myAppid, r.Header.Get(HttpHeaderSubkey),
		hisAppid, topic, group); err != nil {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"), err)

		writeAuthFailure(w, err)
		return
	}

	// fetch the client ack partition and offset
	delayedAck = query.Get("ack") == "1"
	if delayedAck {
		// consumers use explicit acknowledges in order to signal a message as processed successfully
		// if consumers fail to ACK, the message hangs and server will refuse to move ahead

		// get the partitionN and offsetN from client header
		// client will ack with partition=-1, offset=-1:
		// 1. handshake phase
		// 2. when 204 No Content
		partition = r.Header.Get(HttpHeaderPartition)
		offset = r.Header.Get(HttpHeaderOffset)
		if partition != "" && offset != "" {
			// convert partition and offset to int
			offsetN, err = strconv.ParseInt(offset, 10, 64)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} offset:%s",
					myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
					group, r.Header.Get("User-Agent"), offset)

				writeBadRequest(w, "ack with bad offset")
				return
			}
			partitionN, err = strconv.Atoi(partition)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} partition:%s",
					myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
					group, r.Header.Get("User-Agent"), partition)

				writeBadRequest(w, "ack with bad partition")
				return
			}
		} else if len(partition+offset) != 0 {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s partition:%s offset:%s UA:%s} partial ack",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, partition, offset, r.Header.Get("User-Agent"))

			writeBadRequest(w, "partial ack not allowed")
			return
		}
	}

	shadow = query.Get("q")

	log.Debug("sub[%s] %s(%s): {app:%s q:%s topic:%s ver:%s group:%s batch:%d ack:%s partition:%s offset:%s UA:%s}",
		myAppid, r.RemoteAddr, realIp, hisAppid, shadow, topic, ver,
		group, limit, query.Get("ack"), partition, offset, r.Header.Get("User-Agent"))

	// calculate raw topic according to shadow
	if shadow != "" {
		if !sla.ValidateShadowName(shadow) {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s q:%s UA:%s} invalid shadow name",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "invalid shadow name")
			return
		}

		if !manager.Default.IsShadowedTopic(hisAppid, topic, ver, myAppid, group) {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s q:%s UA:%s} not a shadowed topic",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "register shadow first")
			return
		}

		rawTopic = manager.Default.ShadowTopic(shadow, myAppid, hisAppid, topic, ver, group)
	} else {
		rawTopic = manager.Default.KafkaTopic(hisAppid, topic, ver)
	}

	cluster, found := manager.Default.LookupCluster(hisAppid)
	if !found {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} cluster not found",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"))

		writeBadRequest(w, "invalid appid")
		return
	}

	fetcher, err := store.DefaultSubStore.Fetch(cluster, rawTopic,
		myAppid+"."+group, r.RemoteAddr, reset, Options.PermitStandbySub)
	if err != nil {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"), err)

		writeBadRequest(w, err.Error())
		return
	}

	// commit the acked offset
	if delayedAck && partitionN >= 0 && offsetN >= 0 {
		if err = fetcher.CommitUpto(&sarama.ConsumerMessage{
			Topic:     rawTopic,
			Partition: int32(partitionN),
			Offset:    offsetN,
		}); err != nil {
			// during rebalance, this might happen, but with no bad effects
			log.Trace("sub land[%s] %s(%s): {app:%s topic:%s/%s ver:%s group:%s ack:1 offset:%s UA:%s} %v",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, partition, ver,
				group, offset, r.Header.Get("User-Agent"), err)
		} else {
			log.Debug("sub land %s(%s): {G:%s, T:%s/%s, O:%s}",
				r.RemoteAddr, realIp, group, rawTopic, partition, offset)
		}
	}

	tag := r.Header.Get(HttpHeaderMsgTag)
	if tag != "" {
		tagFilters = parseMessageTag(tag)
	}

	err = this.pumpMessages(w, r, fetcher, limit, myAppid, hisAppid, topic, ver, group, delayedAck, tagFilters)
	if err != nil {
		// e,g. broken pipe, io timeout, client gone
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s ack:%s partition:%s offset:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, query.Get("ack"), partition, offset, r.Header.Get("User-Agent"), err)

		writeServerError(w, err.Error())

		if err = fetcher.Close(); err != nil {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s} %v",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver, group, err)
		}
	}
}

func (this *subServer) pumpMessages(w http.ResponseWriter, r *http.Request,
	fetcher store.Fetcher, limit int, myAppid, hisAppid, topic, ver,
	group string, delayedAck bool, tagFilters []MsgTag) error {
	clientGoneCh := w.(http.CloseNotifier).CloseNotify()

	var (
		metaBuf     []byte = nil
		n                  = 0
		idleTimeout        = Options.SubTimeout
		realIp             = getHttpRemoteIp(r)
		chunkedEver        = false
	)
	for {
		select {
		case <-clientGoneCh:
			// FIXME access log will not be able to record this behavior
			return ErrClientGone

		case <-this.gw.shutdownCh:
			// don't call me again
			w.Header().Set("Connection", "close")

			if !chunkedEver {
				w.WriteHeader(http.StatusNoContent)
				w.Write([]byte{})
			}

			return nil

		case err := <-fetcher.Errors():
			// e,g. consume a non-existent topic
			// e,g. conn with broker is broken
			return err

		case <-this.gw.timer.After(idleTimeout):
			if chunkedEver {
				// response already sent in chunk
				log.Debug("chunked sub idle timeout %s {A:%s/G:%s->A:%s T:%s V:%s}",
					idleTimeout, myAppid, group, hisAppid, topic, ver)
				return nil
			}

			w.WriteHeader(http.StatusNoContent)
			w.Write([]byte{}) // without this, client cant get response
			return nil

		case msg := <-fetcher.Messages():
			partition := strconv.FormatInt(int64(msg.Partition), 10)

			if limit == 1 {
				w.Header().Set(HttpHeaderMsgKey, string(msg.Key))
				w.Header().Set(HttpHeaderPartition, partition)
				w.Header().Set(HttpHeaderOffset, strconv.FormatInt(msg.Offset, 10))
			}

			var (
				tags    []MsgTag
				bodyIdx int
				err     error
			)
			if IsTaggedMessage(msg.Value) {
				// TagMarkStart + tag + TagMarkEnd + body
				tags, bodyIdx, err = ExtractMessageTag(msg.Value)
				if limit == 1 && err == nil {
					// needn't check 'index out of range' here
					w.Header().Set(HttpHeaderMsgTag, hack.String(msg.Value[1:bodyIdx-1]))
				} else {
					// not a valid tagged message, treat it as non-tagged message
				}
			}

			if len(tags) > 0 {
				// TODO compare with tagFilters
			}

			if limit == 1 {
				// non-batch mode, just the message itself without meta
				if _, err = w.Write(msg.Value[bodyIdx:]); err != nil {
					// when remote close silently, the write still ok
					return err
				}
			} else {
				// batch mode, write MessageSet
				// MessageSet => [Partition(int32) Offset(int64) MessageSize(int32) Message] BigEndian
				if metaBuf == nil {
					// initialize the reuseable buffer
					metaBuf = make([]byte, 8)

					// remove the middleware added header
					w.Header().Del("Content-Type")
				}

				if err = writeI32(w, metaBuf, msg.Partition); err != nil {
					return err
				}
				if err = writeI64(w, metaBuf, msg.Offset); err != nil {
					return err
				}
				if err = writeI32(w, metaBuf, int32(len(msg.Value[bodyIdx:]))); err != nil {
					return err
				}
				// TODO add tag?
				if _, err = w.Write(msg.Value[bodyIdx:]); err != nil {
					return err
				}
			}

			if !delayedAck {
				log.Debug("sub auto commit offset %s(%s): {G:%s, T:%s/%d, O:%d}",
					r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset)

				// ignore the offset commit err on purpose:
				// during rebalance, offset commit often encounter errors because fetcher
				// underlying partition offset tracker has changed
				// e,g.
				// topic has partition: 0, 1
				// 1. got msg(p=0) from fetcher
				// 2. rebalanced, then start consuming p=1
				// 3. commit the msg offset, still msg(p=0) => error
				// BUT, it has no fatal effects.
				// The worst case is between 1-3, kateway shutdown, sub client
				// will get 1 duplicated msg.
				fetcher.CommitUpto(msg)
			} else {
				log.Debug("sub take off %s(%s): {G:%s, T:%s/%d, O:%d}",
					r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset)
			}

			this.subMetrics.ConsumeOk(myAppid, topic, ver)
			this.subMetrics.ConsumedOk(hisAppid, topic, ver)

			n++
			if n >= limit {
				return nil
			}

			// http chunked: len in hex
			// curl CURLOPT_HTTP_TRANSFER_DECODING will auto unchunk
			w.(http.Flusher).Flush()

			chunkedEver = true

			if n == 1 {
				log.Debug("sub idle timeout %s->1s %s(%s): {G:%s, T:%s/%d, O:%d B:%d}",
					idleTimeout, r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset, limit)
				idleTimeout = time.Second
			}

		}
	}
}
<|endoftext|>


<commit_before>package gateway

import (
	"net/http"
	"strconv"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/sla"
	"github.com/funkygao/golib/hack"
	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

// GET /v1/msgs/:appid/:topic/:ver?group=xx&batch=10&reset=<newest|oldest>&ack=1&q=<dead|retry>
func (this *subServer) subHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	var (
		topic      string
		ver        string
		myAppid    string
		hisAppid   string
		reset      string
		group      string
		shadow     string
		rawTopic   string
		partition  string
		partitionN int = -1
		offset     string
		offsetN    int64    = -1
		limit      int      // max messages to include in the message set
		delayedAck bool     // explicit application level acknowledgement
		tagFilters []MsgTag = nil
		err        error
	)

	if Options.EnableClientStats {
		this.gw.clientStates.RegisterSubClient(r)
	}

	query := r.URL.Query()
	group = query.Get("group")
	reset = query.Get("reset")
	if !manager.Default.ValidateGroupName(r.Header, group) {
		writeBadRequest(w, "illegal group")
		return
	}

	limit, err = getHttpQueryInt(&query, "batch", 1)
	if err != nil {
		writeBadRequest(w, "illegal limit")
		return
	}
	if limit > Options.MaxSubBatchSize && Options.MaxSubBatchSize > 0 {
		limit = Options.MaxSubBatchSize
	}

	ver = params.ByName(UrlParamVersion)
	topic = params.ByName(UrlParamTopic)
	hisAppid = params.ByName(UrlParamAppid)
	myAppid = r.Header.Get(HttpHeaderAppid)
	realIp := getHttpRemoteIp(r)

	// auth
	if err = manager.Default.AuthSub(myAppid, r.Header.Get(HttpHeaderSubkey),
		hisAppid, topic, group); err != nil {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"), err)

		writeAuthFailure(w, err)
		return
	}

	// fetch the client ack partition and offset
	delayedAck = query.Get("ack") == "1"
	if delayedAck {
		// consumers use explicit acknowledges in order to signal a message as processed successfully
		// if consumers fail to ACK, the message hangs and server will refuse to move ahead

		// get the partitionN and offsetN from client header
		// client will ack with partition=-1, offset=-1:
		// 1. handshake phase
		// 2. when 204 No Content
		partition = r.Header.Get(HttpHeaderPartition)
		offset = r.Header.Get(HttpHeaderOffset)
		if partition != "" && offset != "" {
			// convert partition and offset to int
			offsetN, err = strconv.ParseInt(offset, 10, 64)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} offset:%s",
					myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
					group, r.Header.Get("User-Agent"), offset)

				writeBadRequest(w, "ack with bad offset")
				return
			}
			partitionN, err = strconv.Atoi(partition)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} partition:%s",
					myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
					group, r.Header.Get("User-Agent"), partition)

				writeBadRequest(w, "ack with bad partition")
				return
			}
		} else if len(partition+offset) != 0 {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s partition:%s offset:%s UA:%s} partial ack",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, partition, offset, r.Header.Get("User-Agent"))

			writeBadRequest(w, "partial ack not allowed")
			return
		}
	}

	shadow = query.Get("q")

	log.Debug("sub[%s] %s(%s): {app:%s q:%s topic:%s ver:%s group:%s batch:%d ack:%s partition:%s offset:%s UA:%s}",
		myAppid, r.RemoteAddr, realIp, hisAppid, shadow, topic, ver,
		group, limit, query.Get("ack"), partition, offset, r.Header.Get("User-Agent"))

	// calculate raw topic according to shadow
	if shadow != "" {
		if !sla.ValidateShadowName(shadow) {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s q:%s UA:%s} invalid shadow name",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "invalid shadow name")
			return
		}

		if !manager.Default.IsShadowedTopic(hisAppid, topic, ver, myAppid, group) {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s q:%s UA:%s} not a shadowed topic",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "register shadow first")
			return
		}

		rawTopic = manager.Default.ShadowTopic(shadow, myAppid, hisAppid, topic, ver, group)
	} else {
		rawTopic = manager.Default.KafkaTopic(hisAppid, topic, ver)
	}

	cluster, found := manager.Default.LookupCluster(hisAppid)
	if !found {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} cluster not found",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"))

		writeBadRequest(w, "invalid appid")
		return
	}

	fetcher, err := store.DefaultSubStore.Fetch(cluster, rawTopic,
		myAppid+"."+group, r.RemoteAddr, reset, Options.PermitStandbySub)
	if err != nil {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"), err)

		writeBadRequest(w, err.Error())
		return
	}

	// commit the acked offset
	if delayedAck && partitionN >= 0 && offsetN >= 0 {
		// what if shutdown kateway now?
		// the commit will be ok, and when pumpMessages, the conn will get http.StatusNoContent
		if err = fetcher.CommitUpto(&sarama.ConsumerMessage{
			Topic:     rawTopic,
			Partition: int32(partitionN),
			Offset:    offsetN,
		}); err != nil {
			log.Error("sub commit[%s] %s(%s): {app:%s topic:%s ver:%s group:%s ack:1 partition:%s offset:%s UA:%s} %v",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, partition, offset, r.Header.Get("User-Agent"), err)

			writeBadRequest(w, err.Error())
			return
		} else {
			log.Debug("sub land %s(%s): {G:%s, T:%s, P:%s, O:%s}",
				r.RemoteAddr, realIp, group, rawTopic, partition, offset)
		}
	}

	tag := r.Header.Get(HttpHeaderMsgTag)
	if tag != "" {
		tagFilters = parseMessageTag(tag)
	}

	err = this.pumpMessages(w, r, fetcher, limit, myAppid, hisAppid, topic, ver, group, delayedAck, tagFilters)
	if err != nil {
		// e,g. broken pipe, io timeout, client gone
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s ack:%s partition:%s offset:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, query.Get("ack"), partition, offset, r.Header.Get("User-Agent"), err)

		writeServerError(w, err.Error())

		if err = fetcher.Close(); err != nil {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s} %v",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver, group, err)
		}
	}
}

func (this *subServer) pumpMessages(w http.ResponseWriter, r *http.Request,
	fetcher store.Fetcher, limit int, myAppid, hisAppid, topic, ver,
	group string, delayedAck bool, tagFilters []MsgTag) error {
	clientGoneCh := w.(http.CloseNotifier).CloseNotify()

	var (
		metaBuf     []byte = nil
		n                  = 0
		idleTimeout        = Options.SubTimeout
		realIp             = getHttpRemoteIp(r)
		chunkedEver        = false
	)
	for {
		select {
		case <-clientGoneCh:
			// FIXME access log will not be able to record this behavior
			return ErrClientGone

		case <-this.gw.shutdownCh:
			// don't call me again
			w.Header().Set("Connection", "close")

			if !chunkedEver {
				w.WriteHeader(http.StatusNoContent)
				w.Write([]byte{})
			}

			return nil

		case err := <-fetcher.Errors():
			// e,g. consume a non-existent topic
			// e,g. conn with broker is broken
			return err

		case <-this.gw.timer.After(idleTimeout):
			if chunkedEver {
				// response already sent in chunk
				log.Debug("chunked sub idle timeout %s {A:%s/G:%s->A:%s T:%s V:%s}",
					idleTimeout, myAppid, group, hisAppid, topic, ver)
				return nil
			}

			w.WriteHeader(http.StatusNoContent)
			w.Write([]byte{}) // without this, client cant get response
			return nil

		case msg := <-fetcher.Messages():
			partition := strconv.FormatInt(int64(msg.Partition), 10)

			if limit == 1 {
				w.Header().Set(HttpHeaderMsgKey, string(msg.Key))
				w.Header().Set(HttpHeaderPartition, partition)
				w.Header().Set(HttpHeaderOffset, strconv.FormatInt(msg.Offset, 10))
			}

			var (
				tags    []MsgTag
				bodyIdx int
				err     error
			)
			if IsTaggedMessage(msg.Value) {
				// TagMarkStart + tag + TagMarkEnd + body
				tags, bodyIdx, err = ExtractMessageTag(msg.Value)
				if limit == 1 && err == nil {
					// needn't check 'index out of range' here
					w.Header().Set(HttpHeaderMsgTag, hack.String(msg.Value[1:bodyIdx-1]))
				} else {
					// not a valid tagged message, treat it as non-tagged message
				}
			}

			if len(tags) > 0 {
				// TODO compare with tagFilters
			}

			if limit == 1 {
				// non-batch mode, just the message itself without meta
				if _, err = w.Write(msg.Value[bodyIdx:]); err != nil {
					// when remote close silently, the write still ok
					return err
				}
			} else {
				// batch mode, write MessageSet
				// MessageSet => [Partition(int32) Offset(int64) MessageSize(int32) Message] BigEndian
				if metaBuf == nil {
					// initialize the reuseable buffer
					metaBuf = make([]byte, 8)

					// remove the middleware added header
					w.Header().Del("Content-Type")
				}

				if err = writeI32(w, metaBuf, msg.Partition); err != nil {
					return err
				}
				if err = writeI64(w, metaBuf, msg.Offset); err != nil {
					return err
				}
				if err = writeI32(w, metaBuf, int32(len(msg.Value[bodyIdx:]))); err != nil {
					return err
				}
				// TODO add tag?
				if _, err = w.Write(msg.Value[bodyIdx:]); err != nil {
					return err
				}
			}

			if !delayedAck {
				log.Debug("sub auto commit offset %s(%s): {G:%s, T:%s, P:%d, O:%d}",
					r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset)

				if err = fetcher.CommitUpto(msg); err != nil {
					return err
				}
			} else {
				log.Debug("sub take off %s(%s): {G:%s, T:%s, P:%d, O:%d}",
					r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset)
			}

			this.subMetrics.ConsumeOk(myAppid, topic, ver)
			this.subMetrics.ConsumedOk(hisAppid, topic, ver)

			n++
			if n >= limit {
				return nil
			}

			// http chunked: len in hex
			// curl CURLOPT_HTTP_TRANSFER_DECODING will auto unchunk
			w.(http.Flusher).Flush()

			chunkedEver = true

			if n == 1 {
				log.Debug("sub idle timeout %s->1s %s(%s): {G:%s, T:%s, P:%d, O:%d B:%d}",
					idleTimeout, r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset, limit)
				idleTimeout = time.Second
			}

		}
	}
}
<commit_msg>when commit offset errors, return server error<commit_after>package gateway

import (
	"net/http"
	"strconv"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/sla"
	"github.com/funkygao/golib/hack"
	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

// GET /v1/msgs/:appid/:topic/:ver?group=xx&batch=10&reset=<newest|oldest>&ack=1&q=<dead|retry>
func (this *subServer) subHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	var (
		topic      string
		ver        string
		myAppid    string
		hisAppid   string
		reset      string
		group      string
		shadow     string
		rawTopic   string
		partition  string
		partitionN int = -1
		offset     string
		offsetN    int64    = -1
		limit      int      // max messages to include in the message set
		delayedAck bool     // explicit application level acknowledgement
		tagFilters []MsgTag = nil
		err        error
	)

	if Options.EnableClientStats {
		this.gw.clientStates.RegisterSubClient(r)
	}

	query := r.URL.Query()
	group = query.Get("group")
	reset = query.Get("reset")
	if !manager.Default.ValidateGroupName(r.Header, group) {
		writeBadRequest(w, "illegal group")
		return
	}

	limit, err = getHttpQueryInt(&query, "batch", 1)
	if err != nil {
		writeBadRequest(w, "illegal limit")
		return
	}
	if limit > Options.MaxSubBatchSize && Options.MaxSubBatchSize > 0 {
		limit = Options.MaxSubBatchSize
	}

	ver = params.ByName(UrlParamVersion)
	topic = params.ByName(UrlParamTopic)
	hisAppid = params.ByName(UrlParamAppid)
	myAppid = r.Header.Get(HttpHeaderAppid)
	realIp := getHttpRemoteIp(r)

	// auth
	if err = manager.Default.AuthSub(myAppid, r.Header.Get(HttpHeaderSubkey),
		hisAppid, topic, group); err != nil {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"), err)

		writeAuthFailure(w, err)
		return
	}

	// fetch the client ack partition and offset
	delayedAck = query.Get("ack") == "1"
	if delayedAck {
		// consumers use explicit acknowledges in order to signal a message as processed successfully
		// if consumers fail to ACK, the message hangs and server will refuse to move ahead

		// get the partitionN and offsetN from client header
		// client will ack with partition=-1, offset=-1:
		// 1. handshake phase
		// 2. when 204 No Content
		partition = r.Header.Get(HttpHeaderPartition)
		offset = r.Header.Get(HttpHeaderOffset)
		if partition != "" && offset != "" {
			// convert partition and offset to int
			offsetN, err = strconv.ParseInt(offset, 10, 64)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} offset:%s",
					myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
					group, r.Header.Get("User-Agent"), offset)

				writeBadRequest(w, "ack with bad offset")
				return
			}
			partitionN, err = strconv.Atoi(partition)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} partition:%s",
					myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
					group, r.Header.Get("User-Agent"), partition)

				writeBadRequest(w, "ack with bad partition")
				return
			}
		} else if len(partition+offset) != 0 {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s partition:%s offset:%s UA:%s} partial ack",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, partition, offset, r.Header.Get("User-Agent"))

			writeBadRequest(w, "partial ack not allowed")
			return
		}
	}

	shadow = query.Get("q")

	log.Debug("sub[%s] %s(%s): {app:%s q:%s topic:%s ver:%s group:%s batch:%d ack:%s partition:%s offset:%s UA:%s}",
		myAppid, r.RemoteAddr, realIp, hisAppid, shadow, topic, ver,
		group, limit, query.Get("ack"), partition, offset, r.Header.Get("User-Agent"))

	// calculate raw topic according to shadow
	if shadow != "" {
		if !sla.ValidateShadowName(shadow) {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s q:%s UA:%s} invalid shadow name",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "invalid shadow name")
			return
		}

		if !manager.Default.IsShadowedTopic(hisAppid, topic, ver, myAppid, group) {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s q:%s UA:%s} not a shadowed topic",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "register shadow first")
			return
		}

		rawTopic = manager.Default.ShadowTopic(shadow, myAppid, hisAppid, topic, ver, group)
	} else {
		rawTopic = manager.Default.KafkaTopic(hisAppid, topic, ver)
	}

	cluster, found := manager.Default.LookupCluster(hisAppid)
	if !found {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} cluster not found",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"))

		writeBadRequest(w, "invalid appid")
		return
	}

	fetcher, err := store.DefaultSubStore.Fetch(cluster, rawTopic,
		myAppid+"."+group, r.RemoteAddr, reset, Options.PermitStandbySub)
	if err != nil {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"), err)

		writeBadRequest(w, err.Error())
		return
	}

	// commit the acked offset
	if delayedAck && partitionN >= 0 && offsetN >= 0 {
		// what if shutdown kateway now?
		// the commit will be ok, and when pumpMessages, the conn will get http.StatusNoContent
		if err = fetcher.CommitUpto(&sarama.ConsumerMessage{
			Topic:     rawTopic,
			Partition: int32(partitionN),
			Offset:    offsetN,
		}); err != nil {
			log.Error("sub commit[%s] %s(%s): {app:%s topic:%s ver:%s group:%s ack:1 partition:%s offset:%s UA:%s} %v",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, partition, offset, r.Header.Get("User-Agent"), err)

			// when consumer group rebalances, this err might happen
			// when client retry, it get resolved
			writeServerError(w, err.Error())
			return
		} else {
			log.Debug("sub land %s(%s): {G:%s, T:%s, P:%s, O:%s}",
				r.RemoteAddr, realIp, group, rawTopic, partition, offset)
		}
	}

	tag := r.Header.Get(HttpHeaderMsgTag)
	if tag != "" {
		tagFilters = parseMessageTag(tag)
	}

	err = this.pumpMessages(w, r, fetcher, limit, myAppid, hisAppid, topic, ver, group, delayedAck, tagFilters)
	if err != nil {
		// e,g. broken pipe, io timeout, client gone
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s ack:%s partition:%s offset:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, query.Get("ack"), partition, offset, r.Header.Get("User-Agent"), err)

		writeServerError(w, err.Error())

		if err = fetcher.Close(); err != nil {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s} %v",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver, group, err)
		}
	}
}

func (this *subServer) pumpMessages(w http.ResponseWriter, r *http.Request,
	fetcher store.Fetcher, limit int, myAppid, hisAppid, topic, ver,
	group string, delayedAck bool, tagFilters []MsgTag) error {
	clientGoneCh := w.(http.CloseNotifier).CloseNotify()

	var (
		metaBuf     []byte = nil
		n                  = 0
		idleTimeout        = Options.SubTimeout
		realIp             = getHttpRemoteIp(r)
		chunkedEver        = false
	)
	for {
		select {
		case <-clientGoneCh:
			// FIXME access log will not be able to record this behavior
			return ErrClientGone

		case <-this.gw.shutdownCh:
			// don't call me again
			w.Header().Set("Connection", "close")

			if !chunkedEver {
				w.WriteHeader(http.StatusNoContent)
				w.Write([]byte{})
			}

			return nil

		case err := <-fetcher.Errors():
			// e,g. consume a non-existent topic
			// e,g. conn with broker is broken
			return err

		case <-this.gw.timer.After(idleTimeout):
			if chunkedEver {
				// response already sent in chunk
				log.Debug("chunked sub idle timeout %s {A:%s/G:%s->A:%s T:%s V:%s}",
					idleTimeout, myAppid, group, hisAppid, topic, ver)
				return nil
			}

			w.WriteHeader(http.StatusNoContent)
			w.Write([]byte{}) // without this, client cant get response
			return nil

		case msg := <-fetcher.Messages():
			partition := strconv.FormatInt(int64(msg.Partition), 10)

			if limit == 1 {
				w.Header().Set(HttpHeaderMsgKey, string(msg.Key))
				w.Header().Set(HttpHeaderPartition, partition)
				w.Header().Set(HttpHeaderOffset, strconv.FormatInt(msg.Offset, 10))
			}

			var (
				tags    []MsgTag
				bodyIdx int
				err     error
			)
			if IsTaggedMessage(msg.Value) {
				// TagMarkStart + tag + TagMarkEnd + body
				tags, bodyIdx, err = ExtractMessageTag(msg.Value)
				if limit == 1 && err == nil {
					// needn't check 'index out of range' here
					w.Header().Set(HttpHeaderMsgTag, hack.String(msg.Value[1:bodyIdx-1]))
				} else {
					// not a valid tagged message, treat it as non-tagged message
				}
			}

			if len(tags) > 0 {
				// TODO compare with tagFilters
			}

			if limit == 1 {
				// non-batch mode, just the message itself without meta
				if _, err = w.Write(msg.Value[bodyIdx:]); err != nil {
					// when remote close silently, the write still ok
					return err
				}
			} else {
				// batch mode, write MessageSet
				// MessageSet => [Partition(int32) Offset(int64) MessageSize(int32) Message] BigEndian
				if metaBuf == nil {
					// initialize the reuseable buffer
					metaBuf = make([]byte, 8)

					// remove the middleware added header
					w.Header().Del("Content-Type")
				}

				if err = writeI32(w, metaBuf, msg.Partition); err != nil {
					return err
				}
				if err = writeI64(w, metaBuf, msg.Offset); err != nil {
					return err
				}
				if err = writeI32(w, metaBuf, int32(len(msg.Value[bodyIdx:]))); err != nil {
					return err
				}
				// TODO add tag?
				if _, err = w.Write(msg.Value[bodyIdx:]); err != nil {
					return err
				}
			}

			if !delayedAck {
				log.Debug("sub auto commit offset %s(%s): {G:%s, T:%s, P:%d, O:%d}",
					r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset)

				if err = fetcher.CommitUpto(msg); err != nil {
					return err
				}
			} else {
				log.Debug("sub take off %s(%s): {G:%s, T:%s, P:%d, O:%d}",
					r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset)
			}

			this.subMetrics.ConsumeOk(myAppid, topic, ver)
			this.subMetrics.ConsumedOk(hisAppid, topic, ver)

			n++
			if n >= limit {
				return nil
			}

			// http chunked: len in hex
			// curl CURLOPT_HTTP_TRANSFER_DECODING will auto unchunk
			w.(http.Flusher).Flush()

			chunkedEver = true

			if n == 1 {
				log.Debug("sub idle timeout %s->1s %s(%s): {G:%s, T:%s, P:%d, O:%d B:%d}",
					idleTimeout, r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset, limit)
				idleTimeout = time.Second
			}

		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"strings"
	"time"

	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
)

var conn dbox.IConnection
var count int
var ratioTableName string

var (
	sourcetablename = "salespls-summary"
	calctablename   = "salespls-summary"
	desttablename   = "salespls-summary"
	t0              time.Time
	masters         = toolkit.M{}
	reforsrc        = "ref"
	trxsrc          = "COGSMATERIALADJUST"
	basefield       = "PL8A"
	allocfield      = "PL9"
)

type plalloc struct {
	ID                     string `bson:"_id" json:"_id"`
	Key                    string
	Txt1, Txt2, Txt3       string
	Ref1                   float64
	Current                float64
	Expect                 float64
	Absorbed               float64
	Ratio1, Ratio2, Ratio3 float64
}

type allocmap map[string]*plalloc

var (
	yrtotals = allocmap{}
	ratios   = allocmap{}
	totals   = map[string]float64{
		"2014-2015": -50041697769.24013,
		"2015-2016": -53588372484.758606,
	}
	miles = map[string][]*plalloc{
		"2014-2015": []*plalloc{},
		"2015-2016": []*plalloc{},
	}
)

func main() {
	setinitialconnection()
	conn.NewQuery().From(desttablename).
		Where(dbox.Eq("key.trxsrc", "nakulrd")).
		Delete().
		Exec(nil)

	prepmastercalc()
	buildratio()
	for _, v := range []string{"2015-2016", "2014-2015"} {
		processTable(v)
	}
}

func buildratio() {
	connratio, _ := modules.GetDboxIConnection("db_godrej")
	defer connratio.Close()

	/*
		    ftrx := &dbox.Filter{}
			if reforsrc == "ref" {
				ftrx = dbox.Eq("key.ref", trxsrc)
			} else {
				dbox.Eq("key.trxsrc", trxsrc)
			}
	*/

	csp, _ := connratio.NewQuery().From(calctablename).
		Where(dbox.Eq("key.customer_channelid", "I3"),
		dbox.Ne("key.customer_customergroup", "")).
		Select().Cursor(nil)
	defer csp.Close()

	i := 0
	count := csp.Count()
	t0 := time.Now()
	mstone := 0

	for {
		mr := toolkit.M{}
		if ef := csp.Fetch(&mr, 1, false); ef != nil {
			break
		}
		i++
		makeProgressLog("Disc, SPG & Promo by KA Ratio", i, count, 5, &mstone, t0)
		key := mr.Get("key", toolkit.M{}).(toolkit.M)
		fiscal := key.GetString("fiscal")
		ka := key.GetString("customer_customergroup")
		name := key.GetString("customer_customergroupname")
		if ka != "" {
			basevalue := mr.GetFloat64(basefield)
			keyratio := fiscal + "_" + ka + "_" + name
			adjustAllocs(&ratios, keyratio, 0, 0, 0, basevalue)
			adjustAllocs(&yrtotals, fiscal, 0, 0, 0, basevalue)
		}
	}

	for k, v := range ratios {
		kts := strings.Split(k, "_")
		fiscal := kts[0]
		v.Txt1 = kts[1]
		v.Txt2 = kts[2]
		v.Ratio1 = toolkit.Div(v.Ref1, yrtotals[kts[0]].Ref1)
		v.Expect = v.Ratio1 * totals[fiscal]

		miles[fiscal] = append(miles[fiscal], v)
	}
}

func processTable(fiscal string) {
	connsave, _ := modules.GetDboxIConnection("db_godrej")
	defer connsave.Close()
	qsave := connsave.NewQuery().SetConfig("multiexec", true).From(desttablename).Save()

	connselect, _ := modules.GetDboxIConnection("db_godrej")
	defer connselect.Close()

	cursor, _ := connselect.NewQuery().
		From(calctablename).
		Where(dbox.Eq("key.date_fiscal", fiscal),
		dbox.Eq("key.ref", trxsrc),
		dbox.Eq("key.customer_channelid", "I3")).
		Select().Cursor(nil)
	defer cursor.Close()

	allocs := miles[fiscal]
	allocidx := 0

	absorbed := float64(0)
	group := allocs[0].Txt1
	groupname := allocs[1].Txt2

	i := 0
	count := cursor.Count()
	mstone := 0
	t0 = time.Now()
	for {
		mr := toolkit.M{}
		e := cursor.Fetch(&mr, 1, false)
		if e != nil || i >= count {
			break
		}
		i++
		makeProgressLog("Processing", i, count, 5, &mstone, t0)

		key := mr.Get("key", toolkit.M{}).(toolkit.M)
		key.Set("customer_customergroup", group)
		key.Set("customer_customergroupname", groupname)
		mr.Set("key", key)

		gdrj.CalcSum(mr, masters)
		esave := qsave.Exec(toolkit.M{}.Set("data", mr))
		if esave != nil {
			toolkit.Printfn("Erorr: %s", esave.Error())
			os.Exit(100)
		}

		allocv := mr.GetFloat64(allocfield)
		absorbed += allocv

		if absorbed <= allocs[allocidx].Expect {
			allocidx++
			if allocidx >= len(allocs) {
				allocidx = 0
			}
			absorbed = float64(0)
			group = allocs[0].Txt1
			groupname = allocs[1].Txt2
		}
	}
}

func adjustAllocs(allocsmap *allocmap, key string, current, expect, absorbed, ref1 float64) {
	allocs := *allocsmap
	alloc := allocs[key]
	if alloc == nil {
		alloc = new(plalloc)
		alloc.Key = key
		alloc.ID = key
	}
	alloc.Current += current
	alloc.Expect += expect
	alloc.Ref1 += ref1
	alloc.Absorbed += absorbed
	allocs[key] = alloc
	*allocsmap = allocs
}

func makeProgressLog(reference string, i, count, step int, current *int, tstart time.Time) int {
	perstep := count * step / 100
	icurrent := *current
	if icurrent == 0 {
		icurrent = perstep
	}
	pct := i * 100 / count
	if i >= icurrent {
		toolkit.Printfn("%s, %d of %d [%d pct] in %s",
			reference, i, count, pct, time.Since(tstart).String())
		icurrent += perstep
	}
	*current = icurrent
	return icurrent
}

func isPL(id string) bool {
	if strings.HasPrefix(id, "PL7A") ||
		//strings.HasPrefix(id, "PL28") ||
		strings.HasPrefix(id, "PL29A") ||
		strings.HasPrefix(id, "PL31") {
		return true
	}
	return false
}

func buildmap(holder interface{},
	fnModel func() orm.IModel,
	filter *dbox.Filter,
	fnIter func(holder interface{}, obj interface{})) interface{} {
	crx, ecrx := gdrj.Find(fnModel(), filter, nil)
	if ecrx != nil {
		toolkit.Printfn("Cursor Error: %s", ecrx.Error())
		os.Exit(100)
	}
	defer crx.Close()
	for {
		s := fnModel()
		e := crx.Fetch(s, 1, false)
		if e != nil {
			break
		}
		fnIter(holder, s)
	}
	return holder
}

func prepmastercalc() {
	toolkit.Println("--> PL MODEL")
	masters.Set("plmodel", buildmap(map[string]*gdrj.PLModel{},
		func() orm.IModel {
			return new(gdrj.PLModel)
		},
		nil,
		func(holder, obj interface{}) {
			h := holder.(map[string]*gdrj.PLModel)
			o := obj.(*gdrj.PLModel)
			h[o.ID] = o
		}).(map[string]*gdrj.PLModel))
}

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}
<commit_msg>asdads<commit_after>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"strings"
	"time"

	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
)

var conn dbox.IConnection
var count int
var ratioTableName string

var (
	sourcetablename = "salespls-summary"
	calctablename   = "salespls-summary"
	desttablename   = "salespls-summary"
	t0              time.Time
	masters         = toolkit.M{}
	reforsrc        = "ref"
	trxsrc          = "COGSMATERIALADJUST"
	basefield       = "PL8A"
	allocfield      = "PL9"
)

type plalloc struct {
	ID                     string `bson:"_id" json:"_id"`
	Key                    string
	Txt1, Txt2, Txt3       string
	Ref1                   float64
	Current                float64
	Expect                 float64
	Absorbed               float64
	Ratio1, Ratio2, Ratio3 float64
}

type allocmap map[string]*plalloc

var (
	yrtotals = allocmap{}
	ratios   = allocmap{}
	totals   = map[string]float64{
		"2014-2015": -50041697769.24013,
		"2015-2016": -53588372484.758606,
	}
	miles = map[string][]*plalloc{
		"2014-2015": []*plalloc{},
		"2015-2016": []*plalloc{},
	}
)

func main() {
	setinitialconnection()
	conn.NewQuery().From(desttablename).
		Where(dbox.Eq("key.trxsrc", "nakulrd")).
		Delete().
		Exec(nil)

	prepmastercalc()
	buildratio()
	for _, v := range []string{"2015-2016", "2014-2015"} {
		processTable(v)
	}
}

func buildratio() {
	connratio, _ := modules.GetDboxIConnection("db_godrej")
	defer connratio.Close()

	/*
		    ftrx := &dbox.Filter{}
			if reforsrc == "ref" {
				ftrx = dbox.Eq("key.ref", trxsrc)
			} else {
				dbox.Eq("key.trxsrc", trxsrc)
			}
	*/

	csp, _ := connratio.NewQuery().From(calctablename).
		Where(dbox.Eq("key.customer_channelid", "I3"),
		dbox.Ne("key.customer_customergroup", "")).
		Select().Cursor(nil)
	defer csp.Close()

	i := 0
	count := csp.Count()
	t0 := time.Now()
	mstone := 0

	for {
		mr := toolkit.M{}
		if ef := csp.Fetch(&mr, 1, false); ef != nil {
			break
		}
		i++
		makeProgressLog("Disc, SPG & Promo by KA Ratio", i, count, 5, &mstone, t0)
		key := mr.Get("key", toolkit.M{}).(toolkit.M)
		fiscal := key.GetString("fiscal")
		ka := key.GetString("customer_customergroup")
		name := key.GetString("customer_customergroupname")
		if ka != "" {
			basevalue := mr.GetFloat64(basefield)
			keyratio := fiscal + "_" + ka + "_" + name
			adjustAllocs(&ratios, keyratio, 0, 0, 0, basevalue)
			adjustAllocs(&yrtotals, fiscal, 0, 0, 0, basevalue)
		}
	}

	for k, v := range ratios {
		kts := strings.Split(k, "_")
		fiscal := kts[0]
		v.Txt1 = kts[1]
		v.Txt2 = kts[2]
		v.Ratio1 = toolkit.Div(v.Ref1, yrtotals[kts[0]].Ref1)
		v.Expect = v.Ratio1 * totals[fiscal]

		miles[fiscal] = append(miles[fiscal], v)
	}
	toolkit.Printfn("Ratio: %s", toolkit.JsonString(miles))
}

func processTable(fiscal string) {
	connsave, _ := modules.GetDboxIConnection("db_godrej")
	defer connsave.Close()
	qsave := connsave.NewQuery().SetConfig("multiexec", true).From(desttablename).Save()

	connselect, _ := modules.GetDboxIConnection("db_godrej")
	defer connselect.Close()

	cursor, _ := connselect.NewQuery().
		From(calctablename).
		Where(dbox.Eq("key.date_fiscal", fiscal),
		dbox.Eq("key.ref", trxsrc),
		dbox.Eq("key.customer_channelid", "I3")).
		Select().Cursor(nil)
	defer cursor.Close()

	allocs := miles[fiscal]
	allocidx := 0

	absorbed := float64(0)
	group := allocs[0].Txt1
	groupname := allocs[1].Txt2

	i := 0
	count := cursor.Count()
	mstone := 0
	t0 = time.Now()
	for {
		mr := toolkit.M{}
		e := cursor.Fetch(&mr, 1, false)
		if e != nil || i >= count {
			break
		}
		i++
		makeProgressLog("Processing", i, count, 5, &mstone, t0)

		key := mr.Get("key", toolkit.M{}).(toolkit.M)
		key.Set("customer_customergroup", group)
		key.Set("customer_customergroupname", groupname)
		mr.Set("key", key)

		gdrj.CalcSum(mr, masters)
		esave := qsave.Exec(toolkit.M{}.Set("data", mr))
		if esave != nil {
			toolkit.Printfn("Erorr: %s", esave.Error())
			os.Exit(100)
		}

		allocv := mr.GetFloat64(allocfield)
		absorbed += allocv

		if absorbed <= allocs[allocidx].Expect {
			allocidx++
			if allocidx >= len(allocs) {
				allocidx = 0
			}
			absorbed = float64(0)
			group = allocs[0].Txt1
			groupname = allocs[1].Txt2
		}
	}
}

func adjustAllocs(allocsmap *allocmap, key string, current, expect, absorbed, ref1 float64) {
	allocs := *allocsmap
	alloc := allocs[key]
	if alloc == nil {
		alloc = new(plalloc)
		alloc.Key = key
		alloc.ID = key
	}
	alloc.Current += current
	alloc.Expect += expect
	alloc.Ref1 += ref1
	alloc.Absorbed += absorbed
	allocs[key] = alloc
	*allocsmap = allocs
}

func makeProgressLog(reference string, i, count, step int, current *int, tstart time.Time) int {
	perstep := count * step / 100
	icurrent := *current
	if icurrent == 0 {
		icurrent = perstep
	}
	pct := i * 100 / count
	if i >= icurrent {
		toolkit.Printfn("%s, %d of %d [%d pct] in %s",
			reference, i, count, pct, time.Since(tstart).String())
		icurrent += perstep
	}
	*current = icurrent
	return icurrent
}

func isPL(id string) bool {
	if strings.HasPrefix(id, "PL7A") ||
		//strings.HasPrefix(id, "PL28") ||
		strings.HasPrefix(id, "PL29A") ||
		strings.HasPrefix(id, "PL31") {
		return true
	}
	return false
}

func buildmap(holder interface{},
	fnModel func() orm.IModel,
	filter *dbox.Filter,
	fnIter func(holder interface{}, obj interface{})) interface{} {
	crx, ecrx := gdrj.Find(fnModel(), filter, nil)
	if ecrx != nil {
		toolkit.Printfn("Cursor Error: %s", ecrx.Error())
		os.Exit(100)
	}
	defer crx.Close()
	for {
		s := fnModel()
		e := crx.Fetch(s, 1, false)
		if e != nil {
			break
		}
		fnIter(holder, s)
	}
	return holder
}

func prepmastercalc() {
	toolkit.Println("--> PL MODEL")
	masters.Set("plmodel", buildmap(map[string]*gdrj.PLModel{},
		func() orm.IModel {
			return new(gdrj.PLModel)
		},
		nil,
		func(holder, obj interface{}) {
			h := holder.(map[string]*gdrj.PLModel)
			o := obj.(*gdrj.PLModel)
			h[o.ID] = o
		}).(map[string]*gdrj.PLModel))
}

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package goon

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"reflect"
	"strconv"
	"strings"
)

const (
	f_Ignore = "ignore"
)

// Type factory interface for instantiating types while unmarshaling.
type TypeFactory interface {
	New(typename, pkgname string) interface{}
}

// Unmarshal error message list.
// Each line is a single error message generated during unmarshaling.
type Errors struct {
	Msgs []string
}

type deserialiser struct {
	fileset     *token.FileSet
	typefactory TypeFactory
	errors      []string
	pkgname     string
}

// Used to keep track of sequence element token positions.
type seqelement struct {
	item interface{}
	pos  token.Pos
}

// Used to keep track of map element token positions.
type mapelement struct {
	key  interface{}
	kpos token.Pos
	val  interface{}
	vpos token.Pos
}

func (d *deserialiser) deserialiseStruct(typename, pkgname string, elts []ast.Expr) interface{} {
	newstruct := d.typefactory.New(typename, pkgname)

	if newstruct == nil {
		return nil
	}

	rval := reflect.ValueOf(newstruct).Elem() // It's always a pointer to our data.
	rtype := rval.Type()
	kind := rtype.Kind()

	// Make sure we're actually deserialising a struct.
	if kind != reflect.Struct || rtype.Name() != typename {
		return nil
	}

	for _, expr := range elts {
		// Make sure it's a key value pair as intended for struct elements.
		kvexpr, ok := expr.(*ast.KeyValueExpr)
		if !ok {
			continue
		}
		ident, ok := kvexpr.Key.(*ast.Ident)
		if !ok {
			continue
		}

		// Look for our field
		fval := rval.FieldByName(ident.Name)

		if !fval.IsValid() || !fval.CanSet() || !ok {
			continue
		}

		if val, pos := d.deserialise(kvexpr.Value); val != nil {
			if ftype, ok := rtype.FieldByName(ident.Name); ok {
				props := strings.Split(ftype.Tag.Get("goon"), ",")

				ignore := false
				for _, prop := range props {
					switch prop {
					case f_Ignore:
						ignore = true
					}
				}

				if !ignore {
					d.assignValue(val, fval, ftype.Type, pos)
				}
			}
		}
	}

	return newstruct
}

// Handle sequence types.
func (d *deserialiser) deserialiseSeq(elts []ast.Expr) []interface{} {
	if len(elts) == 0 {
		return nil
	}

	seqvals := make([]interface{}, 0, len(elts))
	for _, elt := range elts {
		if val, pos := d.deserialise(elt); val != nil {
			seqvals = append(seqvals, &seqelement{val, pos})
		}
	}
	return seqvals
}

func (d *deserialiser) deserialiseMap(elts []ast.Expr) []interface{} {
	if len(elts) == 0 {
		return nil
	}

	mappairs := make([]interface{}, 0, len(elts))
	for _, elt := range elts {
		kvexpr, ok := elt.(*ast.KeyValueExpr)
		if !ok {
			continue
		}

		var key interface{}
		var kpos token.Pos

		// Make sure we're dealing with
		if keyident, ok := kvexpr.Key.(*ast.Ident); ok {
			key, kpos = keyident.Name, keyident.NamePos
		} else {
			key, kpos = d.deserialise(kvexpr.Key)
		}

		if key != nil {
			if val, vpos := d.deserialise(kvexpr.Value); val != nil {
				mappairs = append(mappairs, &mapelement{key, kpos, val, vpos})
			}
		}
	}
	return mappairs
}

// A composite is either a sequence, map or struct.
func (d *deserialiser) deserialiseComposite(c *ast.CompositeLit) interface{} {
	switch t := c.Type.(type) {
	// Standard type.
	case *ast.Ident:
		if d.typefactory != nil {
			return d.deserialiseStruct(t.Name, d.pkgname, c.Elts)
		} else {
			return d.deserialiseMap(c.Elts)
		}
	// Type in package.
	case *ast.SelectorExpr:
		if pkg, ok := t.X.(*ast.Ident); ok {
			if d.typefactory != nil {
				return d.deserialiseStruct(t.Sel.Name, pkg.Name, c.Elts)
			} else {
				return d.deserialiseMap(c.Elts)
			}
		}
	case *ast.ArrayType:
		return d.deserialiseSeq(c.Elts)
	case *ast.MapType:
		return d.deserialiseMap(c.Elts)
	}

	return nil
}

// Used to set values to a reflection value object.
func (d *deserialiser) assignValue(inval interface{}, outputval reflect.Value, outtype reflect.Type, pos token.Pos) (success bool) {
	inputval := reflect.ValueOf(inval)

	fkind := outputval.Kind()
	setkind := inputval.Kind()

	// Instantiate our type if it's a pointer.
	if fkind == reflect.Ptr && fkind != setkind && outtype != nil {
		newptr := reflect.New(outtype.Elem())
		outputval.Set(newptr)
		outputval = newptr.Elem()
		fkind = outputval.Kind()
	}

	// Recover from type setting failures.
	defer func() {
		if r := recover(); r != nil {
			success = false
			d.errors = append(d.errors,
				fmt.Sprintf("goon: Unable to assign %v value to %v (%v)",
					setkind, outputval.Type().Name(), d.fileset.Position(pos)))
		}
	}()

	// Try to assign the standard types.
	switch fkind {
	case reflect.Float32, reflect.Float64:
		if inputval.Kind() == reflect.Int64 {
			outputval.SetFloat(float64(inputval.Int()))
		} else {
			outputval.SetFloat(inputval.Float())
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		outputval.SetUint(uint64(inputval.Int()))
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		outputval.SetInt(inputval.Int())
	case reflect.Bool:
		outputval.SetBool(inputval.Bool())
	case reflect.Struct:
		outputval.Set(inputval.Elem())
	case reflect.Slice:
		slicelen := inputval.Len()
		newslice := reflect.MakeSlice(outtype, 0, slicelen)
		for i := 0; i < slicelen; i++ {
			if _inval, ok := inputval.Index(i).Interface().(*seqelement); ok {
				_outval := reflect.New(outtype.Elem()).Elem()
				if d.assignValue(_inval.item, _outval, _outval.Type(), _inval.pos) && _outval.IsValid() {
					newslice = reflect.Append(newslice, _outval)
				}
			}
		}
		outputval.Set(newslice)
	case reflect.Map:
		slicelen := inputval.Len()
		newmap := reflect.MakeMap(outtype)
		for i := 0; i < slicelen; i++ {
			if _inval, ok := inputval.Index(i).Interface().(*mapelement); ok {
				_outkey := reflect.New(outtype.Key()).Elem()
				_outval := reflect.New(outtype.Elem()).Elem()
				if d.assignValue(_inval.key, _outkey, _outkey.Type(), _inval.kpos) && _outkey.IsValid() {
					if d.assignValue(_inval.val, _outval, _outval.Type(), _inval.kpos) && _outval.IsValid() {
						newmap.SetMapIndex(_outkey, _outval)
					}
				}
			}
		}
		outputval.Set(newmap)
	default:
		outputval.Set(inputval)
	}

	success = true
	return
}

// Main recursive deserialise function.
// Handles string, float, int types and recurses into complex types if needed.
func (d *deserialiser) deserialise(astval interface{}) (retval interface{}, pos token.Pos) {
	if actual, ok := astval.(*ast.UnaryExpr); ok {
		// Reassign to the actual type.
		astval = actual.X
	}

	switch t := astval.(type) {
	case *ast.CompositeLit:
		pos = t.Lbrace
		retval = d.deserialiseComposite(t)
	case *ast.BasicLit:
		pos = t.ValuePos
		switch t.Kind {
		case token.STRING:
			retval = strings.Trim(t.Value, "\"")
		case token.INT:
			if i, err := strconv.ParseInt(t.Value, 10, 64); err == nil {
				retval = i
			}
		case token.FLOAT:
			if f, err := strconv.ParseFloat(t.Value, 64); err == nil {
				retval = f
			}
		}
	// Booleans are treated this way.
	case *ast.Ident:
		if b, err := strconv.ParseBool(t.Name); err == nil {
			retval = b
		}
	}

	return
}

// Unmarshal reads goon data and returns a map of all named variables and their corresponding deserialised type.
//
// If data != nil, Unmarshal parses the source from data and the filename is only used when recording position information.
// The type of the argument for the 'data' parameter must be string, []byte, or io.Reader.
// If 'data' == nil, ParseFile parses the file specified by filename.
func Unmarshal(filename string, data interface{}) (map[string]interface{}, *Errors) {
	return UnmarshalTyped(filename, data, nil)
}

// UnmarshalTyped does the same thing as Unmarshal but provides a way to instantiate custom types by string names through
// a type factory, tf.
//
// If tf is unspecified, UnmarshalTyped will attempt to instantiate a map instead.
func UnmarshalTyped(filename string, data interface{}, tf TypeFactory) (deserialised map[string]interface{}, errs *Errors) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, filename, data, 0)
	if err != nil {
		errs = &Errors{[]string{fmt.Sprintf("%v", err)}}
		return
	}

	ds := &deserialiser{
		fileset:     fset,
		typefactory: tf,
		errors:      make([]string, 0, 8),
		pkgname:     f.Name.Name,
	}

	deserialised = make(map[string]interface{})

	for _, decl := range f.Decls {
		d, ok := decl.(*ast.GenDecl)
		if !ok || d.Tok != token.VAR || len(d.Specs) == 0 {
			continue
		}

		valspec, ok := d.Specs[0].(*ast.ValueSpec)
		if !ok {
			continue
		}

		//ast.Print(fset, f)
		name := valspec.Names[0].Name
		val, _ := ds.deserialise(valspec.Values[0])
		deserialised[name] = val
	}

	if len(ds.errors) > 0 {
		errs = &Errors{ds.errors}
	}

	return
}
<commit_msg>Add method to allow unmarshalling of a single value.<commit_after>package goon

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"reflect"
	"strconv"
	"strings"
)

const (
	f_Ignore = "ignore"
)

// Type factory interface for instantiating types while unmarshaling.
type TypeFactory interface {
	New(typename, pkgname string) interface{}
}

// Unmarshal error message list.
// Each line is a single error message generated during unmarshaling.
type Errors struct {
	Msgs []string
}

type deserialiser struct {
	fileset     *token.FileSet
	typefactory TypeFactory
	errors      []string
	pkgname     string
}

// Used to keep track of sequence element token positions.
type seqelement struct {
	item interface{}
	pos  token.Pos
}

// Used to keep track of map element token positions.
type mapelement struct {
	key  interface{}
	kpos token.Pos
	val  interface{}
	vpos token.Pos
}

func (d *deserialiser) deserialiseStruct(typename, pkgname string, elts []ast.Expr) interface{} {
	newstruct := d.typefactory.New(typename, pkgname)

	if newstruct == nil {
		return nil
	}

	rval := reflect.ValueOf(newstruct).Elem() // It's always a pointer to our data.
	rtype := rval.Type()
	kind := rtype.Kind()

	// Make sure we're actually deserialising a struct.
	if kind != reflect.Struct || rtype.Name() != typename {
		return nil
	}

	for _, expr := range elts {
		// Make sure it's a key value pair as intended for struct elements.
		kvexpr, ok := expr.(*ast.KeyValueExpr)
		if !ok {
			continue
		}
		ident, ok := kvexpr.Key.(*ast.Ident)
		if !ok {
			continue
		}

		// Look for our field
		fval := rval.FieldByName(ident.Name)

		if !fval.IsValid() || !fval.CanSet() || !ok {
			continue
		}

		if val, pos := d.deserialise(kvexpr.Value); val != nil {
			if ftype, ok := rtype.FieldByName(ident.Name); ok {
				props := strings.Split(ftype.Tag.Get("goon"), ",")

				ignore := false
				for _, prop := range props {
					switch prop {
					case f_Ignore:
						ignore = true
					}
				}

				if !ignore {
					d.assignValue(val, fval, ftype.Type, pos)
				}
			}
		}
	}

	return newstruct
}

// Handle sequence types.
func (d *deserialiser) deserialiseSeq(elts []ast.Expr) []interface{} {
	if len(elts) == 0 {
		return nil
	}

	seqvals := make([]interface{}, 0, len(elts))
	for _, elt := range elts {
		if val, pos := d.deserialise(elt); val != nil {
			seqvals = append(seqvals, &seqelement{val, pos})
		}
	}
	return seqvals
}

func (d *deserialiser) deserialiseMap(elts []ast.Expr) []interface{} {
	if len(elts) == 0 {
		return nil
	}

	mappairs := make([]interface{}, 0, len(elts))
	for _, elt := range elts {
		kvexpr, ok := elt.(*ast.KeyValueExpr)
		if !ok {
			continue
		}

		var key interface{}
		var kpos token.Pos

		// Make sure we're dealing with
		if keyident, ok := kvexpr.Key.(*ast.Ident); ok {
			key, kpos = keyident.Name, keyident.NamePos
		} else {
			key, kpos = d.deserialise(kvexpr.Key)
		}

		if key != nil {
			if val, vpos := d.deserialise(kvexpr.Value); val != nil {
				mappairs = append(mappairs, &mapelement{key, kpos, val, vpos})
			}
		}
	}
	return mappairs
}

// A composite is either a sequence, map or struct.
func (d *deserialiser) deserialiseComposite(c *ast.CompositeLit) interface{} {
	switch t := c.Type.(type) {
	// Standard type.
	case *ast.Ident:
		if d.typefactory != nil {
			return d.deserialiseStruct(t.Name, d.pkgname, c.Elts)
		} else {
			return d.deserialiseMap(c.Elts)
		}
	// Type in package.
	case *ast.SelectorExpr:
		if pkg, ok := t.X.(*ast.Ident); ok {
			if d.typefactory != nil {
				return d.deserialiseStruct(t.Sel.Name, pkg.Name, c.Elts)
			} else {
				return d.deserialiseMap(c.Elts)
			}
		}
	case *ast.ArrayType:
		return d.deserialiseSeq(c.Elts)
	case *ast.MapType:
		return d.deserialiseMap(c.Elts)
	}

	return nil
}

// Used to set values to a reflection value object.
func (d *deserialiser) assignValue(inval interface{}, outputval reflect.Value, outtype reflect.Type, pos token.Pos) (success bool) {
	inputval := reflect.ValueOf(inval)

	fkind := outputval.Kind()
	setkind := inputval.Kind()

	// Instantiate our type if it's a pointer.
	if fkind == reflect.Ptr && fkind != setkind && outtype != nil {
		newptr := reflect.New(outtype.Elem())
		outputval.Set(newptr)
		outputval = newptr.Elem()
		fkind = outputval.Kind()
	}

	// Recover from type setting failures.
	defer func() {
		if r := recover(); r != nil {
			success = false
			d.errors = append(d.errors,
				fmt.Sprintf("goon: Unable to assign %v value to %v (%v)",
					setkind, outputval.Type().Name(), d.fileset.Position(pos)))
		}
	}()

	// Try to assign the standard types.
	switch fkind {
	case reflect.Float32, reflect.Float64:
		if inputval.Kind() == reflect.Int64 {
			outputval.SetFloat(float64(inputval.Int()))
		} else {
			outputval.SetFloat(inputval.Float())
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		outputval.SetUint(uint64(inputval.Int()))
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		outputval.SetInt(inputval.Int())
	case reflect.Bool:
		outputval.SetBool(inputval.Bool())
	case reflect.Struct:
		outputval.Set(inputval.Elem())
	case reflect.Slice:
		slicelen := inputval.Len()
		newslice := reflect.MakeSlice(outtype, 0, slicelen)
		for i := 0; i < slicelen; i++ {
			if _inval, ok := inputval.Index(i).Interface().(*seqelement); ok {
				_outval := reflect.New(outtype.Elem()).Elem()
				if d.assignValue(_inval.item, _outval, _outval.Type(), _inval.pos) && _outval.IsValid() {
					newslice = reflect.Append(newslice, _outval)
				}
			}
		}
		outputval.Set(newslice)
	case reflect.Map:
		slicelen := inputval.Len()
		newmap := reflect.MakeMap(outtype)
		for i := 0; i < slicelen; i++ {
			if _inval, ok := inputval.Index(i).Interface().(*mapelement); ok {
				_outkey := reflect.New(outtype.Key()).Elem()
				_outval := reflect.New(outtype.Elem()).Elem()
				if d.assignValue(_inval.key, _outkey, _outkey.Type(), _inval.kpos) && _outkey.IsValid() {
					if d.assignValue(_inval.val, _outval, _outval.Type(), _inval.kpos) && _outval.IsValid() {
						newmap.SetMapIndex(_outkey, _outval)
					}
				}
			}
		}
		outputval.Set(newmap)
	default:
		outputval.Set(inputval)
	}

	success = true
	return
}

// Main recursive deserialise function.
// Handles string, float, int types and recurses into complex types if needed.
func (d *deserialiser) deserialise(astval interface{}) (retval interface{}, pos token.Pos) {
	if actual, ok := astval.(*ast.UnaryExpr); ok {
		// Reassign to the actual type.
		astval = actual.X
	}

	switch t := astval.(type) {
	case *ast.CompositeLit:
		pos = t.Lbrace
		retval = d.deserialiseComposite(t)
	case *ast.BasicLit:
		pos = t.ValuePos
		switch t.Kind {
		case token.STRING:
			retval = strings.Trim(t.Value, "\"")
		case token.INT:
			if i, err := strconv.ParseInt(t.Value, 10, 64); err == nil {
				retval = i
			}
		case token.FLOAT:
			if f, err := strconv.ParseFloat(t.Value, 64); err == nil {
				retval = f
			}
		}
	// Booleans are treated this way.
	case *ast.Ident:
		if b, err := strconv.ParseBool(t.Name); err == nil {
			retval = b
		}
	}

	return
}

// Unmarshal reads goon data and returns a map of all named variables and their corresponding deserialised type.
//
// If data != nil, Unmarshal parses the source from data and the filename is only used when recording position information.
// The type of the argument for the 'data' parameter must be string, []byte, or io.Reader.
// If 'data' == nil, ParseFile parses the file specified by filename.
func Unmarshal(filename string, data interface{}) (map[string]interface{}, *Errors) {
	return UnmarshalTyped(filename, data, nil)
}

// UnmarshalTyped does the same thing as Unmarshal but provides a way to instantiate custom types by string names through
// a type factory, tf.
//
// If tf is unspecified, UnmarshalTyped will attempt to instantiate a map instead.
func UnmarshalTyped(filename string, data interface{}, tf TypeFactory) (deserialised map[string]interface{}, errs *Errors) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, filename, data, 0)
	if err != nil {
		errs = &Errors{[]string{fmt.Sprintf("%v", err)}}
		return
	}

	ds := &deserialiser{
		fileset:     fset,
		typefactory: tf,
		errors:      make([]string, 0, 8),
		pkgname:     f.Name.Name,
	}

	deserialised = make(map[string]interface{})

	for _, decl := range f.Decls {
		d, ok := decl.(*ast.GenDecl)
		if !ok || d.Tok != token.VAR || len(d.Specs) == 0 {
			continue
		}

		valspec, ok := d.Specs[0].(*ast.ValueSpec)
		if !ok {
			continue
		}

		//ast.Print(fset, f)
		name := valspec.Names[0].Name
		val, _ := ds.deserialise(valspec.Values[0])
		deserialised[name] = val
	}

	if len(ds.errors) > 0 {
		errs = &Errors{ds.errors}
	}

	return
}

// Returns a single value from UnmarshalTyped.
//
// TODO: Possibly cache these values so files won't be read more than once.
func UnmarshalTypedOne(varname, filename string, data interface{}, tf TypeFactory) (interface{}, *Errors) {
	varmap, errs := UnmarshalTyped(filename, data, tf)

	if errs != nil {
		return nil, errs
	}

	if val, ok := varmap[varname]; ok {
		return val, nil
	}
	return nil, &Errors{[]string{fmt.Sprintf("Variable %v does not exist in %v.", varname, filename)}}
}
<|endoftext|>


<commit_before>package dht

import (
	"bytes"
	"net"
	"sync"
	"time"

	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peer"

	"github.com/google/gopacket/routing"
	netroute "github.com/libp2p/go-netroute"

	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
)

// QueryFilterFunc is a filter applied when considering peers to dial when querying
type QueryFilterFunc func(dht *IpfsDHT, ai peer.AddrInfo) bool

// RouteTableFilterFunc is a filter applied when considering connections to keep in
// the local route table.
type RouteTableFilterFunc func(dht *IpfsDHT, conns []network.Conn) bool

// PublicQueryFilter returns true if the peer is suspected of being publicly accessible
func PublicQueryFilter(_ *IpfsDHT, ai peer.AddrInfo) bool {
	if len(ai.Addrs) == 0 {
		return false
	}

	var hasPublicAddr bool
	for _, a := range ai.Addrs {
		if !isRelayAddr(a) && manet.IsPublicAddr(a) {
			hasPublicAddr = true
		}
	}
	return hasPublicAddr
}

var _ QueryFilterFunc = PublicQueryFilter

// PublicRoutingTableFilter allows a peer to be added to the routing table if the connections to that peer indicate
// that it is on a public network
func PublicRoutingTableFilter(dht *IpfsDHT, conns []network.Conn) bool {
	if len(conns) == 0 {
		return false
	}

	// Do we have a public address for this peer?
	id := conns[0].RemotePeer()
	known := dht.peerstore.PeerInfo(id)
	for _, a := range known.Addrs {
		if !isRelayAddr(a) && manet.IsPublicAddr(a) {
			return true
		}
	}

	return false
}

var _ RouteTableFilterFunc = PublicRoutingTableFilter

// PrivateQueryFilter doens't currently restrict which peers we are willing to query from the local DHT.
func PrivateQueryFilter(dht *IpfsDHT, ai peer.AddrInfo) bool {
	return len(ai.Addrs) > 0
}

var _ QueryFilterFunc = PrivateQueryFilter

// We call this very frequently but routes can technically change at runtime.
// Cache it for two minutes.
const routerCacheTime = 2 * time.Minute

var routerCache struct {
	sync.RWMutex
	router  routing.Router
	expires time.Time
}

func getCachedRouter() routing.Router {
	routerCache.RLock()
	router := routerCache.router
	expires := routerCache.expires
	routerCache.RUnlock()

	if time.Now().Before(expires) {
		return router
	}

	routerCache.Lock()
	defer routerCache.Unlock()

	now := time.Now()
	if now.Before(routerCache.expires) {
		return router
	}
	routerCache.router, _ = netroute.New()
	routerCache.expires = now.Add(routerCacheTime)
	return router
}

// PrivateRoutingTableFilter allows a peer to be added to the routing table if the connections to that peer indicate
// that it is on a private network
func PrivateRoutingTableFilter(dht *IpfsDHT, conns []network.Conn) bool {
	router := getCachedRouter()
	myAdvertisedIPs := make([]net.IP, 0)
	for _, a := range dht.Host().Addrs() {
		if manet.IsPublicAddr(a) && !isRelayAddr(a) {
			ip, err := manet.ToIP(a)
			if err != nil {
				continue
			}
			myAdvertisedIPs = append(myAdvertisedIPs, ip)
		}
	}

	for _, c := range conns {
		ra := c.RemoteMultiaddr()
		if manet.IsPrivateAddr(ra) && !isRelayAddr(ra) {
			return true
		}

		if manet.IsPublicAddr(ra) {
			ip, err := manet.ToIP(ra)
			if err != nil {
				continue
			}

			// if the ip is the same as one of the local host's public advertised IPs - then consider it local
			for _, i := range myAdvertisedIPs {
				if i.Equal(ip) {
					return true
				}
				if ip.To4() == nil {
					if i.To4() == nil && isEUI(ip) && sameV6Net(i, ip) {
						return true
					}
				}
			}

			// if there's no gateway - a direct host in the OS routing table - then consider it local
			// This is relevant in particular to ipv6 networks where the addresses may all be public,
			// but the nodes are aware of direct links between each other.
			if router != nil {
				_, gw, _, err := router.Route(ip)
				if gw == nil && err == nil {
					return true
				}
			}
		}
	}

	return false
}

var _ RouteTableFilterFunc = PrivateRoutingTableFilter

func isEUI(ip net.IP) bool {
	// per rfc 2373
	return len(ip) == net.IPv6len && ip[11] == 0xff && ip[12] == 0xfe
}

func sameV6Net(a, b net.IP) bool {
	return len(a) == net.IPv6len && len(b) == net.IPv6len && bytes.Equal(a[0:8], b[0:8]) //nolint
}

func isRelayAddr(a ma.Multiaddr) bool {
	found := false
	ma.ForEach(a, func(c ma.Component) bool {
		found = c.Protocol().Code == ma.P_CIRCUIT
		return !found
	})
	return found
}
<commit_msg>stricter definition of public for DHT<commit_after>package dht

import (
	"bytes"
	"net"
	"sync"
	"time"

	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peer"

	"github.com/google/gopacket/routing"
	netroute "github.com/libp2p/go-netroute"

	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
)

// QueryFilterFunc is a filter applied when considering peers to dial when querying
type QueryFilterFunc func(dht *IpfsDHT, ai peer.AddrInfo) bool

// RouteTableFilterFunc is a filter applied when considering connections to keep in
// the local route table.
type RouteTableFilterFunc func(dht *IpfsDHT, conns []network.Conn) bool

var publicCIDR6 = "2000::/3"
var public6 *net.IPNet

func init() {
	_, public6, _ = net.ParseCIDR(publicCIDR6)
}

// isPublicAddr follows the logic of manet.IsPublicAddr, except it uses
// a stricter definition of "public" for ipv6: namely "is it in 2000::/3"?
func isPublicAddr(a ma.Multiaddr) bool {
	ip, err := manet.ToIP(a)
	if err != nil {
		return false
	}
	if ip.To4() != nil {
		return !inAddrRange(ip, manet.Private4) && !inAddrRange(ip, manet.Unroutable4)
	}

	return public6.Contains(ip)
}

// isPrivateAddr follows the logic of manet.IsPrivateAddr, except that
// it uses a stricter definition of "public" for ipv6
func isPrivateAddr(a ma.Multiaddr) bool {
	ip, err := manet.ToIP(a)
	if err != nil {
		return false
	}
	if ip.To4() != nil {
		return inAddrRange(ip, manet.Private4)
	}

	return !public6.Contains(ip) && !inAddrRange(ip, manet.Unroutable6)
}

// PublicQueryFilter returns true if the peer is suspected of being publicly accessible
func PublicQueryFilter(_ *IpfsDHT, ai peer.AddrInfo) bool {
	if len(ai.Addrs) == 0 {
		return false
	}

	var hasPublicAddr bool
	for _, a := range ai.Addrs {
		if !isRelayAddr(a) && isPublicAddr(a) {
			hasPublicAddr = true
		}
	}
	return hasPublicAddr
}

var _ QueryFilterFunc = PublicQueryFilter

// PublicRoutingTableFilter allows a peer to be added to the routing table if the connections to that peer indicate
// that it is on a public network
func PublicRoutingTableFilter(dht *IpfsDHT, conns []network.Conn) bool {
	if len(conns) == 0 {
		return false
	}

	// Do we have a public address for this peer?
	id := conns[0].RemotePeer()
	known := dht.peerstore.PeerInfo(id)
	for _, a := range known.Addrs {
		if !isRelayAddr(a) && isPublicAddr(a) {
			return true
		}
	}

	return false
}

var _ RouteTableFilterFunc = PublicRoutingTableFilter

// PrivateQueryFilter doens't currently restrict which peers we are willing to query from the local DHT.
func PrivateQueryFilter(dht *IpfsDHT, ai peer.AddrInfo) bool {
	return len(ai.Addrs) > 0
}

var _ QueryFilterFunc = PrivateQueryFilter

// We call this very frequently but routes can technically change at runtime.
// Cache it for two minutes.
const routerCacheTime = 2 * time.Minute

var routerCache struct {
	sync.RWMutex
	router  routing.Router
	expires time.Time
}

func getCachedRouter() routing.Router {
	routerCache.RLock()
	router := routerCache.router
	expires := routerCache.expires
	routerCache.RUnlock()

	if time.Now().Before(expires) {
		return router
	}

	routerCache.Lock()
	defer routerCache.Unlock()

	now := time.Now()
	if now.Before(routerCache.expires) {
		return router
	}
	routerCache.router, _ = netroute.New()
	routerCache.expires = now.Add(routerCacheTime)
	return router
}

// PrivateRoutingTableFilter allows a peer to be added to the routing table if the connections to that peer indicate
// that it is on a private network
func PrivateRoutingTableFilter(dht *IpfsDHT, conns []network.Conn) bool {
	router := getCachedRouter()
	myAdvertisedIPs := make([]net.IP, 0)
	for _, a := range dht.Host().Addrs() {
		if isPublicAddr(a) && !isRelayAddr(a) {
			ip, err := manet.ToIP(a)
			if err != nil {
				continue
			}
			myAdvertisedIPs = append(myAdvertisedIPs, ip)
		}
	}

	for _, c := range conns {
		ra := c.RemoteMultiaddr()
		if isPrivateAddr(ra) && !isRelayAddr(ra) {
			return true
		}

		if isPublicAddr(ra) {
			ip, err := manet.ToIP(ra)
			if err != nil {
				continue
			}

			// if the ip is the same as one of the local host's public advertised IPs - then consider it local
			for _, i := range myAdvertisedIPs {
				if i.Equal(ip) {
					return true
				}
				if ip.To4() == nil {
					if i.To4() == nil && isEUI(ip) && sameV6Net(i, ip) {
						return true
					}
				}
			}

			// if there's no gateway - a direct host in the OS routing table - then consider it local
			// This is relevant in particular to ipv6 networks where the addresses may all be public,
			// but the nodes are aware of direct links between each other.
			if router != nil {
				_, gw, _, err := router.Route(ip)
				if gw == nil && err == nil {
					return true
				}
			}
		}
	}

	return false
}

var _ RouteTableFilterFunc = PrivateRoutingTableFilter

func isEUI(ip net.IP) bool {
	// per rfc 2373
	return len(ip) == net.IPv6len && ip[11] == 0xff && ip[12] == 0xfe
}

func sameV6Net(a, b net.IP) bool {
	return len(a) == net.IPv6len && len(b) == net.IPv6len && bytes.Equal(a[0:8], b[0:8]) //nolint
}

func isRelayAddr(a ma.Multiaddr) bool {
	found := false
	ma.ForEach(a, func(c ma.Component) bool {
		found = c.Protocol().Code == ma.P_CIRCUIT
		return !found
	})
	return found
}

func inAddrRange(ip net.IP, ipnets []*net.IPNet) bool {
	for _, ipnet := range ipnets {
		if ipnet.Contains(ip) {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>package data

import (
	"fmt"
	. "github.com/smartystreets/goconvey/convey"
	"math"
	"reflect"
	"testing"
)

var testCases = []struct {
	input Value
}{
	// Blob
	{Blob("")},
	{Blob("hoge")},
	{Blob("日本語")},
	// Bool
	{Bool(false)},
	{Bool(true)},
	// Float
	{Float(2.34)},
	{Float(math.NaN())}, // NaN == NaN is false
	{Float(math.Inf(-1))},
	{Float(2.000000000000000000000000000000000000000000000000000000000001)},
	{Float(2.0)},
	{Float(0.0)},
	{Float(-0.0)},
	// Int
	{Int(2)}, // same as Float(2.0)
	{Int(-6)},
	// Null
	{Null{}},
	// String
	{String("")},
	{String("hoge")},
	{String("日本語")},
	// Timestamp
	{Timestamp{}},
	// Array
	{Array{}},
	{Array{String("hoge")}},
	{Array{Int(2), Float(3.0)}},
	{Array{Float(2.0), Int(3)}},
	{Array{Int(-6), Array{String("hoge")}}},
	// Map
	{Map{}},
	{Map{"hoge": String("hoge")}},
	{Map{"a": Int(2), "b": Float(3.0)}},
	{Map{"b": Int(3), "a": Float(2.0)}},
	{Map{"i": Int(-6), "xy": Map{"h": String("hoge")}}},
}

func TestEquality(t *testing.T) {
	for i, tc1 := range testCases {
		for j, tc2 := range testCases {
			left := tc1.input
			right := tc2.input
			Convey(fmt.Sprintf("When comparing %#v and %#v", left, right), t, func() {
				de := reflect.DeepEqual(left, right)
				he := Equal(left, right)

				Convey("Then the output should be the same", func() {
					if // int vs float
					((i == 8 || i == 9) && (j == 12)) ||
						((j == 8 || j == 9) && (i == 12)) ||
						// array
						((i == 21 && j == 22) || (j == 21 && i == 22)) ||
						// map
						((i == 26 && j == 27) || (j == 26 && i == 27)) {
						So(de, ShouldBeFalse)
						So(he, ShouldBeTrue)
					} else {
						if de != he {
							fmt.Printf("%v vs %v: %t/%t\n", left, right, de, he)
						}
						So(de, ShouldEqual, he)
					}
				})
			})
		}
	}
}

func BenchmarkDeepEqual(b *testing.B) {
	for n := 0; n < b.N; n++ {
		for _, tc1 := range testCases {
			for _, tc2 := range testCases {
				reflect.DeepEqual(tc1.input, tc2.input)
			}
		}
	}
}

func BenchmarkEqual(b *testing.B) {
	for n := 0; n < b.N; n++ {
		for _, tc1 := range testCases {
			for _, tc2 := range testCases {
				Equal(tc1.input, tc2.input)
			}
		}
	}
}

func BenchmarkHash(b *testing.B) {
	var h HashValue
	for n := 0; n < b.N; n++ {
		for _, tc := range testCases {
			h += Hash(tc.input)
		}
	}
}
<commit_msg>Add test for Hash<commit_after>package data

import (
	"fmt"
	. "github.com/smartystreets/goconvey/convey"
	"math"
	"reflect"
	"testing"
	"time"
)

var testCases = []struct {
	input Value
}{
	// Blob
	{Blob("")},
	{Blob("hoge")},
	{Blob("日本語")},
	// Bool
	{Bool(false)},
	{Bool(true)},
	// Float
	{Float(2.34)},
	{Float(math.NaN())}, // NaN == NaN is false
	{Float(math.Inf(-1))},
	{Float(2.000000000000000000000000000000000000000000000000000000000001)},
	{Float(2.0)},
	{Float(0.0)},
	{Float(-0.0)},
	// Int
	{Int(2)}, // same as Float(2.0)
	{Int(-6)},
	// Null
	{Null{}},
	// String
	{String("")},
	{String("hoge")},
	{String("日本語")},
	// Timestamp
	{Timestamp{}},
	// Array
	{Array{}},
	{Array{String("hoge")}},
	{Array{Int(2), Float(3.0)}},
	{Array{Float(2.0), Int(3)}},
	{Array{Int(-6), Array{String("hoge")}}},
	// Map
	{Map{}},
	{Map{"hoge": String("hoge")}},
	{Map{"a": Int(2), "b": Float(3.0)}},
	{Map{"b": Int(3), "a": Float(2.0)}},
	{Map{"b": Int(3), "a": Float(2.5)}},
	{Map{"i": Int(-6), "xy": Map{"h": String("hoge")}}},
}

func TestHash(t *testing.T) {
	now := time.Now()
	for now.Nanosecond()%1000 == 999 {
		now = time.Now()
	}

	Convey("Given a map containing all types", t, func() {
		m := Map{
			"null":      Null{},
			"true":      True,
			"false":     False,
			"int":       Int(10),
			"float":     Float(2.5),
			"string":    String("hoge"),
			"blob":      Blob("hoge"),
			"timestamp": Timestamp(now),
			"array": Array{Null{}, True, False, Int(10), Float(2.5), String("hoge"),
				Blob("hoge"), Timestamp(now), Array{Null{}, True, False, Int(10), Float(2.5), String("hoge")},
				Map{
					"null":   Null{},
					"true":   True,
					"false":  False,
					"int":    Int(10),
					"float":  Float(2.5),
					"string": String("hoge"),
				},
			},
			"map": Map{
				"null":      Null{},
				"true":      True,
				"false":     False,
				"int":       Int(10),
				"float":     Float(2.5),
				"string":    String("hoge"),
				"blob":      Blob("hoge"),
				"timestamp": Timestamp(now),
				"array": Array{Null{}, True, False, Int(10), Float(2.5), String("hoge"),
					Blob("hoge"), Timestamp(now), Array{Null{}, True, False, Int(10), Float(2.5), String("hoge")},
					Map{
						"null":   Null{},
						"true":   True,
						"false":  False,
						"int":    Int(10),
						"float":  Float(2.5),
						"string": String("hoge"),
					},
				},
			},
		}

		Convey("When a map doesn't contain something invalid", func() {
			Convey("Then Hash should always return the same value", func() {
				So(Hash(m), ShouldEqual, Hash(m))
			})
		})

		Convey("When comparing a float having an integer value to int and vice vasa", func() {
			c := m.Copy()
			c["int"] = Float(10.0)
			m["float"] = Float(3.0)
			c["float"] = Int(3)
			So(c.Set("array[3]", Float(10.0)), ShouldBeNil)
			So(m.Set("map.int", Float(10.0)), ShouldBeNil)

			Convey("Then Hash should return the same value", func() {
				So(Hash(m), ShouldEqual, Hash(m))
			})
		})

		Convey("When comparing timestamps whose differences are less than 1us", func() {
			t := Timestamp(now.Add(1))
			c := m.Copy()
			Convey("Then Hash should return the same value", func() {
				c["timestamp"] = t
				So(Hash(c), ShouldEqual, Hash(m))
			})

			Convey("Then Hash should behave samely when they're in an array", func() {
				So(m.Set("array[7]", t), ShouldBeNil)
				So(Hash(c), ShouldEqual, Hash(m))
			})

			Convey("Then Hash should behave samely when they're in a map", func() {
				So(m.Set("map.timestamp", t), ShouldBeNil)
				So(Hash(m), ShouldEqual, Hash(m))
			})
		})

		Convey("When a map contains NaN", func() {
			Convey("Then Hash should alwasy return different values", func() {
				m["nan"] = Float(math.NaN())
				So(Hash(m), ShouldNotEqual, Hash(m))
			})

			Convey("Then NaN in an array should behave samely", func() {
				So(m.Set("array[0]", Float(math.NaN())), ShouldBeNil)
				So(Hash(m), ShouldNotEqual, Hash(m))
			})

			Convey("Then NaN in a map should behave samely", func() {
				So(m.Set("map.float", Float(math.NaN())), ShouldBeNil)
				So(Hash(m), ShouldNotEqual, Hash(m))
			})
		})
	})
}

func TestEquality(t *testing.T) {
	for i, tc1 := range testCases {
		for j, tc2 := range testCases {
			left := tc1.input
			right := tc2.input
			Convey(fmt.Sprintf("When comparing %#v and %#v", left, right), t, func() {
				de := reflect.DeepEqual(left, right)
				he := Equal(left, right)

				Convey("Then the output should be the same", func() {
					if // int vs float
					((i == 8 || i == 9) && (j == 12)) ||
						((j == 8 || j == 9) && (i == 12)) ||
						// array
						((i == 21 && j == 22) || (j == 21 && i == 22)) ||
						// map
						((i == 26 && j == 27) || (j == 26 && i == 27)) {
						So(de, ShouldBeFalse)
						So(he, ShouldBeTrue)
					} else {
						if de != he {
							fmt.Printf("%v vs %v: %t/%t\n", left, right, de, he)
						}
						So(de, ShouldEqual, he)
					}
				})
			})
		}
	}
}

func BenchmarkDeepEqual(b *testing.B) {
	for n := 0; n < b.N; n++ {
		for _, tc1 := range testCases {
			for _, tc2 := range testCases {
				reflect.DeepEqual(tc1.input, tc2.input)
			}
		}
	}
}

func BenchmarkEqual(b *testing.B) {
	for n := 0; n < b.N; n++ {
		for _, tc1 := range testCases {
			for _, tc2 := range testCases {
				Equal(tc1.input, tc2.input)
			}
		}
	}
}

func BenchmarkHash(b *testing.B) {
	var h HashValue
	for n := 0; n < b.N; n++ {
		for _, tc := range testCases {
			h += Hash(tc.input)
		}
	}
}
<|endoftext|>


<commit_before>package build

import (
	"fmt"

	"github.com/flant/dapp/pkg/dapp"
	"github.com/flant/dapp/pkg/image"
)

func NewPrepareImagesPhase() *PrepareImagesPhase {
	return &PrepareImagesPhase{}
}

type PrepareImagesPhase struct{}

func (p *PrepareImagesPhase) Run(c *Conveyor) error {
	if debug() {
		fmt.Printf("PrepareImagesPhase.Run\n")
	}

	for _, dimg := range c.DimgsInOrder {
		var prevImage, prevBuiltImage image.Image

		err := dimg.PrepareBaseImage(c)
		if err != nil {
			return fmt.Errorf("error preparing base image %s of dimg %s: %s", dimg.GetBaseImage().Name(), dimg.GetName(), err)
		}

		prevImage = dimg.baseImage
		for _, s := range dimg.GetStages() {
			if prevImage.IsExists() {
				prevBuiltImage = prevImage
			}

			img := s.GetImage()
			if !img.IsExists() {
				if debug() {
					fmt.Printf("  %s\n", s.Name())
				}

				imageServiceCommitChangeOptions := img.Container().ServiceCommitChangeOptions()
				imageServiceCommitChangeOptions.AddLabel(map[string]string{
					"dapp-version":       dapp.Version,
					"dapp-cache-version": BuildCacheVersion,
					"dapp-dimg":          "false",
					"dapp-dev-mode":      "false",
				})

				if c.SshAuthSock != "" {
					imageRunOptions := img.Container().RunOptions()
					imageRunOptions.AddVolume(fmt.Sprintf("%s:/tmp/dapp-ssh-agent", c.SshAuthSock))
					imageRunOptions.AddEnv(map[string]string{"SSH_AUTH_SOCK": "/tmp/dapp-ssh-agent"})
				}

				err := s.PrepareImage(c, prevBuiltImage, img)
				if err != nil {
					return fmt.Errorf("error preparing stage %s: %s", s.Name(), err)
				}
			}

			prevImage = img
		}
	}

	return nil
}
<commit_msg>[go build] add dapp label<commit_after>package build

import (
	"fmt"

	"github.com/flant/dapp/pkg/dapp"
	"github.com/flant/dapp/pkg/image"
)

func NewPrepareImagesPhase() *PrepareImagesPhase {
	return &PrepareImagesPhase{}
}

type PrepareImagesPhase struct{}

func (p *PrepareImagesPhase) Run(c *Conveyor) error {
	if debug() {
		fmt.Printf("PrepareImagesPhase.Run\n")
	}

	for _, dimg := range c.DimgsInOrder {
		var prevImage, prevBuiltImage image.Image

		err := dimg.PrepareBaseImage(c)
		if err != nil {
			return fmt.Errorf("error preparing base image %s of dimg %s: %s", dimg.GetBaseImage().Name(), dimg.GetName(), err)
		}

		prevImage = dimg.baseImage
		for _, s := range dimg.GetStages() {
			if prevImage.IsExists() {
				prevBuiltImage = prevImage
			}

			img := s.GetImage()
			if !img.IsExists() {
				if debug() {
					fmt.Printf("  %s\n", s.Name())
				}

				imageServiceCommitChangeOptions := img.Container().ServiceCommitChangeOptions()
				imageServiceCommitChangeOptions.AddLabel(map[string]string{
					"dapp":               c.ProjectName,
					"dapp-version":       dapp.Version,
					"dapp-cache-version": BuildCacheVersion,
					"dapp-dimg":          "false",
					"dapp-dev-mode":      "false",
				})

				if c.SshAuthSock != "" {
					imageRunOptions := img.Container().RunOptions()
					imageRunOptions.AddVolume(fmt.Sprintf("%s:/tmp/dapp-ssh-agent", c.SshAuthSock))
					imageRunOptions.AddEnv(map[string]string{"SSH_AUTH_SOCK": "/tmp/dapp-ssh-agent"})
				}

				err := s.PrepareImage(c, prevBuiltImage, img)
				if err != nil {
					return fmt.Errorf("error preparing stage %s: %s", s.Name(), err)
				}
			}

			prevImage = img
		}
	}

	return nil
}
<|endoftext|>