<commit_before>package memory

import (
	"github.com/xephonhq/xephon-k/pkg/common"
)

// Index is a map of inverted index with tag name as key and tag value as term for the inverted index
type Index struct {
	tagKeyIndex     map[string]map[string]bool // map[string]bool is used as set
	invertedIndexes map[string]*InvertedIndex
}

// InvertedIndex use Term for tag value postings for a list of sorted series ID
// TODO: Series ID should use locality sensitive hashing https://en.wikipedia.org/wiki/Locality-sensitive_hashing
type InvertedIndex struct {
	Term     string
	Postings []common.SeriesID
}

var initialPostingSize = 10

func NewIndex(capacity int) *Index {
	return &Index{
		tagKeyIndex:     make(map[string]map[string]bool, capacity),
		invertedIndexes: make(map[string]*InvertedIndex, capacity),
	}
}

func newInvertedIndex(term string) *InvertedIndex {
	return &InvertedIndex{
		Term:     term,
		Postings: make([]common.SeriesID, 0, initialPostingSize),
	}
}

func (idx *Index) Filter(f *common.Filter) []common.SeriesID {
	// TODO: we need locking to ensure the correctness
	// can we have multiple read lock on a same object?
	empty := []common.SeriesID{}
	switch f.Type {
	case "tag_match":
		// TODO: what if the tag user provided is invalid, the should be checked at API
		// and we assume everything is right here?
		return idx.Get(f.Key, f.Value)
	case "and":
		return Intersect(idx.Filter(f.LeftOperand), idx.Filter(f.RightOperand))
	case "or":
		return Union(idx.Filter(f.LeftOperand), idx.Filter(f.RightOperand))
	case "in":
		// TODO: in is just multiple and?
		log.Warn("in is not implemented")
		return empty
	default:
		// TODO: this should be checked in upper level
		log.Warn("%s is unsupported", f.Type)
		return empty
	}
}

func (idx *Index) Get(tagKey string, tagValue string) []common.SeriesID {
	term := Term(tagKey, tagValue)
	iidx, ok := idx.invertedIndexes[term]
	if ok {
		return iidx.Postings
	} else {
		return []common.SeriesID{}
	}
}

func (idx *Index) Add(id common.SeriesID, tagKey string, tagValue string) {
	// update tagKeyIndex
	_, ok := idx.tagKeyIndex[tagKey]
	if !ok {
		idx.tagKeyIndex[tagKey] = make(map[string]bool)
	}
	idx.tagKeyIndex[tagKey][tagValue] = true

	// TODO: should add separator, in Prometheus `db.go` it's `const sep = '\xff'`
	term := Term(tagKey, tagValue)
	// create the inverted index if not exists
	_, ok = idx.invertedIndexes[term]
	if !ok {
		idx.invertedIndexes[term] = newInvertedIndex(term)
	}
	idx.invertedIndexes[term].Add(id)
}

// TODO: actually we can have a fixed size map to cache the hot series, so there is no need to lookup if the id is already in there
func (iidx *InvertedIndex) Add(id common.SeriesID) {
	// binary search and insert the value if not found
	low, high := 0, len(iidx.Postings)
	for low < high {
		// TODO: use custom compare function or compare it directly using <
		mid := low + (high-low)/2 // avoid overflow, copied from `src/sort/search.go` sort.Search
		if iidx.Postings[mid] >= id {
			high = mid
		} else {
			low = mid + 1
		}
	}

	// not found
	if low == len(iidx.Postings) {
		iidx.Postings = append(iidx.Postings, id)
		return
	} else if iidx.Postings[low] != id {
		// insert it to the slice https://github.com/golang/go/wiki/SliceTricks#insert
		iidx.Postings = append(iidx.Postings, id) // we append id here, but any value is ok, it will be overwritten by following copy
		copy(iidx.Postings[low+1:], iidx.Postings[low:])
		iidx.Postings[low] = id

	}

	// found
	// TODO: should have some sort of cache
	return
}

// Intersect is used for AND, i.e. app=nginx AND os=ubuntu
// - sort lists by length
// - loop through the element in the shortest list,
// 	 - use exponential search to find if the element exists in other lists, only add it to result if it appears in all lists
//   - if any list reaches its end, the outer loop breaks
// NOTE:
// - we didn't use the algorithm in the VLDB paper, just a naive one with some similar ideas
// - in fact, this is just the `join` operation in RDBMS
// TODO:
// - it is also possible to sort by value range
// Ref
// - https://www.quora.com/Which-is-the-best-algorithm-to-merge-k-ordered-lists
// 	 - 'adaptive list intersection'
// - Improving performance of List intersection http://www.vldb.org/pvldb/2/vldb09-pvldb37.pdf
// 	 - Dynamic probe
// - Exponential (galloping) search https://en.wikipedia.org/wiki/Exponential_search
func Intersect(postings ...[]common.SeriesID) []common.SeriesID {
	// posting is a sorted list, see InvertedIndex
	// sort by list length using selection sort, assume the number of list is small
	listCount := len(postings)
	allLength := make([]int, listCount)
	// NOTE: probeStart is not used by sorting lists, we just use the loop to initialize all element to 1,
	// because exponential search can't start from 0, 0 * 2 = 0
	probeStart := make([]int, listCount)
	for i := 0; i < listCount; i++ {
		shortestIndex := i
		shortestLength := len(postings[i])
		for j := i + 1; j < listCount; j++ {
			curLength := len(postings[j])
			if curLength < shortestLength {
				shortestIndex = j
				shortestLength = curLength
			}
		}
		// swap if needed
		if i != shortestIndex {
			postings[i], postings[shortestIndex] = postings[shortestIndex], postings[i]
		}
		allLength[i] = shortestLength
		probeStart[i] = 1
	}

	// walk all the elements in the shortest list
	// assume the intersection is same length as the shortest list, allocate the space
	intersection := make([]common.SeriesID, 0, allLength[0])
OUTER:
	for i := 0; i < allLength[0]; i++ {
		cur := postings[0][i]
		// probe all the other lists, if one of them don't met, go to next loop
		for k := 1; k < listCount; k++ {
			// exponential search, use a smaller range for following binary search
			bound := probeStart[k]
			size := allLength[k]
			for bound < size && postings[k][bound] < cur {
				bound *= 2
			}

			// binary search
			low := bound / 2
			// NOTE: Go does not have `(a < b)? a : b` http://stackoverflow.com/questions/19979178/what-is-the-idiomatic-go-equivalent-of-cs-ternary-operator
			high := min(bound, size)
			for low < high {
				mid := low + (high-low)/2
				if postings[k][mid] >= cur {
					high = mid
				} else {
					low = mid + 1
				}
			}
			// this list reaches end, no need to continue the outer loop
			if low == size {
				// http://relistan.com/continue-statement-with-labels-in-go/
				//log.Infof("break outer in %d th list", k)
				break OUTER
			}
			probeStart[k] = low + 1
			// got the nearest one, but not the same one, no need to check other lists, continue the outer loo
			if postings[k][low] != cur {
				continue OUTER
			}
		}
		// if you made it here, then you are in all the lists
		intersection = append(intersection, cur)
	}
	return intersection
}

// Union is used for OR, i.e. app=nginx OR app=apache
// - sort all the lists by length? or just pick the smallest one?
// - get first len(smallest) elements of each array into an array and sort it? this is nk * log(k)
// NOTE
// - Linear search merge duplicate compare
// - Divide and Conquer merge requires extra space
// - Heap merge requires using Heap (e... such a brainless note, a.k.a I don't know how to write heap)
// - need to exclude lists when they reaches the end, might use a map
// Ref
// - https://en.wikipedia.org/wiki/K-Way_Merge_Algorithms
// - https://github.com/prometheus/tsdb/issues/50
// - k-way merging and k-ary sorts http://cs.uno.edu/people/faculty/bill/k-way-merge-n-sort-ACM-SE-Regl-1993.pdf
// - https://www.cs.cmu.edu/~adamchik/15-121/lectures/Binary%20Heaps/heaps.html
func Union(postings ...[]common.SeriesID) []common.SeriesID {
	listCount := len(postings)
	remainLists := make(map[int]bool, listCount)
	posList := make([]int, listCount)
	allLength := make([]int, listCount)
	// we assume there are many duplication between lists, so we use the longest list's length as initial capacity
	maxLength := len(postings[0])
	for i := 0; i < listCount; i++ {
		remainLists[i] = true
		posList[i] = 0
		allLength[i] = len(postings[i])
		if maxLength < allLength[i] {
			maxLength = allLength[i]
		}
	}

	// FIXME: this is linear search merge, the slowest one, nk, but when k is small, this is fine
	// TODO: it seems there is not need for sorting
	union := make([]common.SeriesID, 0, maxLength)
	lastVal := common.SeriesID(0)
	for len(remainLists) > 0 {
		// pick any one as the initial value http://stackoverflow.com/questions/23482786/get-an-arbitrary-key-item-from-a-map
		var first int
		for i := range remainLists {
			first = i
			break
		}
		smallestVal := postings[first][posList[first]]
		smallestIndex := first
		for i := range remainLists {
			curVal := postings[i][posList[i]]
			// deal with duplication
			if curVal == lastVal {
				//log.Infof("dup %s", curVal)
				posList[i]++
				// check the next element if there is any
				if posList[i] == allLength[i] {
					delete(remainLists, i)
				} else {
					// NOTE: if we don't do this, [1], [1, 2], [9] would end up be [1, 9, 2],
					// sometimes it would be the correct [1, 2, 9] due to the randomness of first
					curVal = postings[i][posList[i]]
				}
			}

			if curVal < smallestVal {
				// smaller value
				smallestVal = curVal
				smallestIndex = i
			}
		}
		//log.Infof("%s %s %d", lastVal, smallestVal, smallestIndex)

		// the random picked list's first unmerged element is same as last value
		// its index must have been updated in above loop, so we skip following logic
		if lastVal == smallestVal {
			continue
		}
		posList[smallestIndex]++
		if posList[smallestIndex] == allLength[smallestIndex] {
			delete(remainLists, smallestIndex)
		}
		lastVal = smallestVal
		union = append(union, smallestVal)
	}
	return union
}

func min(a int, b int) int {
	if a < b {
		return a
	}
	return b
}

// TODO: should add separator, in Prometheus `db.go` it's `const sep = '\xff'`
func Term(tagKey string, tagValue string) string {
	return tagKey + tagValue
}

func max(a int, b int) int {
	if a > b {
		return a
	}
	return b
}
<commit_msg>[mem] Add mutex for in memory index as well<commit_after>package memory

import (
	"github.com/xephonhq/xephon-k/pkg/common"
	"sync"
)

// Index is a map of inverted index with tag name as key and tag value as term for the inverted index
type Index struct {
	mu              sync.RWMutex
	tagKeyIndex     map[string]map[string]bool // map[string]bool is used as set
	invertedIndexes map[string]*InvertedIndex
}

// InvertedIndex use Term for tag value postings for a list of sorted series ID
// TODO: Series ID should use locality sensitive hashing https://en.wikipedia.org/wiki/Locality-sensitive_hashing
type InvertedIndex struct {
	mu       sync.RWMutex
	Term     string
	Postings []common.SeriesID
}

var initialPostingSize = 10

func NewIndex(capacity int) *Index {
	return &Index{
		tagKeyIndex:     make(map[string]map[string]bool, capacity),
		invertedIndexes: make(map[string]*InvertedIndex, capacity),
	}
}

func newInvertedIndex(term string) *InvertedIndex {
	return &InvertedIndex{
		Term:     term,
		Postings: make([]common.SeriesID, 0, initialPostingSize),
	}
}

func (idx *Index) Filter(f *common.Filter) []common.SeriesID {
	// TODO: we need locking to ensure the correctness
	// can we have multiple read lock on a same object?
	empty := []common.SeriesID{}
	switch f.Type {
	case "tag_match":
		// TODO: what if the tag user provided is invalid, the should be checked at API
		// and we assume everything is right here?
		return idx.Get(f.Key, f.Value)
	case "and":
		return Intersect(idx.Filter(f.LeftOperand), idx.Filter(f.RightOperand))
	case "or":
		return Union(idx.Filter(f.LeftOperand), idx.Filter(f.RightOperand))
	case "in":
		// TODO: in is just multiple and?
		log.Warn("in is not implemented")
		return empty
	default:
		// TODO: this should be checked in upper level
		log.Warn("%s is unsupported", f.Type)
		return empty
	}
}

func (idx *Index) Get(tagKey string, tagValue string) []common.SeriesID {
	term := Term(tagKey, tagValue)
	iidx, ok := idx.invertedIndexes[term]
	if ok {
		return iidx.Postings
	} else {
		return []common.SeriesID{}
	}
}

func (idx *Index) Add(id common.SeriesID, tagKey string, tagValue string) {
	idx.mu.Lock()
	// update tagKeyIndex
	_, ok := idx.tagKeyIndex[tagKey]
	if !ok {
		idx.tagKeyIndex[tagKey] = make(map[string]bool)
	}
	idx.tagKeyIndex[tagKey][tagValue] = true

	// TODO: should add separator, in Prometheus `db.go` it's `const sep = '\xff'`
	term := Term(tagKey, tagValue)
	// create the inverted index if not exists
	_, ok = idx.invertedIndexes[term]
	if !ok {
		idx.invertedIndexes[term] = newInvertedIndex(term)
	}
	// NOTE: we unlock here because each inverted index also have its own lock
	idx.mu.Unlock()
	idx.invertedIndexes[term].Add(id)
}

// TODO: actually we can have a fixed size map to cache the hot series, so there is no need to lookup if the id is already in there
func (iidx *InvertedIndex) Add(id common.SeriesID) {
	iidx.mu.Lock()
	defer iidx.mu.Unlock()
	// binary search and insert the value if not found
	low, high := 0, len(iidx.Postings)
	for low < high {
		// TODO: use custom compare function or compare it directly using <
		mid := low + (high-low)/2 // avoid overflow, copied from `src/sort/search.go` sort.Search
		if iidx.Postings[mid] >= id {
			high = mid
		} else {
			low = mid + 1
		}
	}

	// not found
	if low == len(iidx.Postings) {
		iidx.Postings = append(iidx.Postings, id)
		return
	} else if iidx.Postings[low] != id {
		// insert it to the slice https://github.com/golang/go/wiki/SliceTricks#insert
		iidx.Postings = append(iidx.Postings, id) // we append id here, but any value is ok, it will be overwritten by following copy
		copy(iidx.Postings[low+1:], iidx.Postings[low:])
		iidx.Postings[low] = id

	}

	// found
	// TODO: should have some sort of cache
	return
}

// Intersect is used for AND, i.e. app=nginx AND os=ubuntu
// - sort lists by length
// - loop through the element in the shortest list,
// 	 - use exponential search to find if the element exists in other lists, only add it to result if it appears in all lists
//   - if any list reaches its end, the outer loop breaks
// NOTE:
// - we didn't use the algorithm in the VLDB paper, just a naive one with some similar ideas
// - in fact, this is just the `join` operation in RDBMS
// TODO:
// - it is also possible to sort by value range
// Ref
// - https://www.quora.com/Which-is-the-best-algorithm-to-merge-k-ordered-lists
// 	 - 'adaptive list intersection'
// - Improving performance of List intersection http://www.vldb.org/pvldb/2/vldb09-pvldb37.pdf
// 	 - Dynamic probe
// - Exponential (galloping) search https://en.wikipedia.org/wiki/Exponential_search
func Intersect(postings ...[]common.SeriesID) []common.SeriesID {
	// posting is a sorted list, see InvertedIndex
	// sort by list length using selection sort, assume the number of list is small
	listCount := len(postings)
	allLength := make([]int, listCount)
	// NOTE: probeStart is not used by sorting lists, we just use the loop to initialize all element to 1,
	// because exponential search can't start from 0, 0 * 2 = 0
	probeStart := make([]int, listCount)
	for i := 0; i < listCount; i++ {
		shortestIndex := i
		shortestLength := len(postings[i])
		for j := i + 1; j < listCount; j++ {
			curLength := len(postings[j])
			if curLength < shortestLength {
				shortestIndex = j
				shortestLength = curLength
			}
		}
		// swap if needed
		if i != shortestIndex {
			postings[i], postings[shortestIndex] = postings[shortestIndex], postings[i]
		}
		allLength[i] = shortestLength
		probeStart[i] = 1
	}

	// walk all the elements in the shortest list
	// assume the intersection is same length as the shortest list, allocate the space
	intersection := make([]common.SeriesID, 0, allLength[0])
OUTER:
	for i := 0; i < allLength[0]; i++ {
		cur := postings[0][i]
		// probe all the other lists, if one of them don't met, go to next loop
		for k := 1; k < listCount; k++ {
			// exponential search, use a smaller range for following binary search
			bound := probeStart[k]
			size := allLength[k]
			for bound < size && postings[k][bound] < cur {
				bound *= 2
			}

			// binary search
			low := bound / 2
			// NOTE: Go does not have `(a < b)? a : b` http://stackoverflow.com/questions/19979178/what-is-the-idiomatic-go-equivalent-of-cs-ternary-operator
			high := min(bound, size)
			for low < high {
				mid := low + (high-low)/2
				if postings[k][mid] >= cur {
					high = mid
				} else {
					low = mid + 1
				}
			}
			// this list reaches end, no need to continue the outer loop
			if low == size {
				// http://relistan.com/continue-statement-with-labels-in-go/
				//log.Infof("break outer in %d th list", k)
				break OUTER
			}
			probeStart[k] = low + 1
			// got the nearest one, but not the same one, no need to check other lists, continue the outer loo
			if postings[k][low] != cur {
				continue OUTER
			}
		}
		// if you made it here, then you are in all the lists
		intersection = append(intersection, cur)
	}
	return intersection
}

// Union is used for OR, i.e. app=nginx OR app=apache
// - sort all the lists by length? or just pick the smallest one?
// - get first len(smallest) elements of each array into an array and sort it? this is nk * log(k)
// NOTE
// - Linear search merge duplicate compare
// - Divide and Conquer merge requires extra space
// - Heap merge requires using Heap (e... such a brainless note, a.k.a I don't know how to write heap)
// - need to exclude lists when they reaches the end, might use a map
// Ref
// - https://en.wikipedia.org/wiki/K-Way_Merge_Algorithms
// - https://github.com/prometheus/tsdb/issues/50
// - k-way merging and k-ary sorts http://cs.uno.edu/people/faculty/bill/k-way-merge-n-sort-ACM-SE-Regl-1993.pdf
// - https://www.cs.cmu.edu/~adamchik/15-121/lectures/Binary%20Heaps/heaps.html
func Union(postings ...[]common.SeriesID) []common.SeriesID {
	listCount := len(postings)
	remainLists := make(map[int]bool, listCount)
	posList := make([]int, listCount)
	allLength := make([]int, listCount)
	// we assume there are many duplication between lists, so we use the longest list's length as initial capacity
	maxLength := len(postings[0])
	for i := 0; i < listCount; i++ {
		remainLists[i] = true
		posList[i] = 0
		allLength[i] = len(postings[i])
		if maxLength < allLength[i] {
			maxLength = allLength[i]
		}
	}

	// FIXME: this is linear search merge, the slowest one, nk, but when k is small, this is fine
	// TODO: it seems there is not need for sorting
	union := make([]common.SeriesID, 0, maxLength)
	lastVal := common.SeriesID(0)
	for len(remainLists) > 0 {
		// pick any one as the initial value http://stackoverflow.com/questions/23482786/get-an-arbitrary-key-item-from-a-map
		var first int
		for i := range remainLists {
			first = i
			break
		}
		smallestVal := postings[first][posList[first]]
		smallestIndex := first
		for i := range remainLists {
			curVal := postings[i][posList[i]]
			// deal with duplication
			if curVal == lastVal {
				//log.Infof("dup %s", curVal)
				posList[i]++
				// check the next element if there is any
				if posList[i] == allLength[i] {
					delete(remainLists, i)
				} else {
					// NOTE: if we don't do this, [1], [1, 2], [9] would end up be [1, 9, 2],
					// sometimes it would be the correct [1, 2, 9] due to the randomness of first
					curVal = postings[i][posList[i]]
				}
			}

			if curVal < smallestVal {
				// smaller value
				smallestVal = curVal
				smallestIndex = i
			}
		}
		//log.Infof("%s %s %d", lastVal, smallestVal, smallestIndex)

		// the random picked list's first unmerged element is same as last value
		// its index must have been updated in above loop, so we skip following logic
		if lastVal == smallestVal {
			continue
		}
		posList[smallestIndex]++
		if posList[smallestIndex] == allLength[smallestIndex] {
			delete(remainLists, smallestIndex)
		}
		lastVal = smallestVal
		union = append(union, smallestVal)
	}
	return union
}

func min(a int, b int) int {
	if a < b {
		return a
	}
	return b
}

// TODO: should add separator, in Prometheus `db.go` it's `const sep = '\xff'`
func Term(tagKey string, tagValue string) string {
	return tagKey + tagValue
}

func max(a int, b int) int {
	if a > b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package web

import (
	"context"
	"io/ioutil"
	"net/http"
	"sync/atomic"
	"time"

	"github.com/atlassian/gostatsd"
	"github.com/atlassian/gostatsd/pb"

	"github.com/atlassian/gostatsd/pkg/stats"
	"github.com/golang/protobuf/proto"
	"github.com/sirupsen/logrus"
)

type rawHttpHandlerV2 struct {
	requestSuccess           uint64 // atomic
	requestFailureRead       uint64 // atomic
	requestFailureDecompress uint64 // atomic
	requestFailureEncoding   uint64 // atomic
	requestFailureUnmarshal  uint64 // atomic
	metricsProcessed         uint64 // atomic
	eventsProcessed          uint64 // atomic

	logger     logrus.FieldLogger
	handler    gostatsd.PipelineHandler
	serverName string
}

func newRawHttpHandlerV2(logger logrus.FieldLogger, serverName string, handler gostatsd.PipelineHandler) *rawHttpHandlerV2 {
	return &rawHttpHandlerV2{
		logger:     logger,
		handler:    handler,
		serverName: serverName,
	}
}

func (rhh *rawHttpHandlerV2) RunMetrics(ctx context.Context) {
	statser := stats.FromContext(ctx).WithTags([]string{"server-name:" + rhh.serverName})

	notify, cancel := statser.RegisterFlush()
	defer cancel()

	for {
		select {
		case <-notify:
			rhh.emitMetrics(statser)
		case <-ctx.Done():
			return
		}
	}
}

func (rhh *rawHttpHandlerV2) emitMetrics(statser stats.Statser) {
	requestSuccess := atomic.SwapUint64(&rhh.requestSuccess, 0)
	requestFailureRead := atomic.SwapUint64(&rhh.requestFailureRead, 0)
	requestFailureDecompress := atomic.SwapUint64(&rhh.requestFailureDecompress, 0)
	requestFailureEncoding := atomic.SwapUint64(&rhh.requestFailureEncoding, 0)
	requestFailureUnmarshal := atomic.SwapUint64(&rhh.requestFailureUnmarshal, 0)
	metricsProcessed := atomic.SwapUint64(&rhh.metricsProcessed, 0)
	eventsProcessed := atomic.SwapUint64(&rhh.eventsProcessed, 0)

	statser.Count("http.incoming", float64(requestSuccess), []string{"result:success"})
	statser.Count("http.incoming", float64(requestFailureRead), []string{"result:failure", "failure:read"})
	statser.Count("http.incoming", float64(requestFailureDecompress), []string{"result:failure", "failure:decompress"})
	statser.Count("http.incoming", float64(requestFailureEncoding), []string{"result:failure", "failure:encoding"})
	statser.Count("http.incoming", float64(requestFailureUnmarshal), []string{"result:failure", "failure:unmarshal"})
	statser.Count("http.incoming.metrics", float64(metricsProcessed), nil)
	statser.Count("http.incoming.events", float64(eventsProcessed), nil)
}

func (rhh *rawHttpHandlerV2) readBody(req *http.Request) ([]byte, int) {
	b, err := ioutil.ReadAll(req.Body)
	if err != nil {
		atomic.AddUint64(&rhh.requestFailureRead, 1)
		rhh.logger.WithError(err).Info("failed reading body")
		return nil, http.StatusInternalServerError
	}
	req.Body.Close()

	encoding := req.Header.Get("Content-Encoding")
	switch encoding {
	case "deflate":
		b, err = decompress(b)
		if err != nil {
			atomic.AddUint64(&rhh.requestFailureDecompress, 1)
			rhh.logger.WithError(err).Info("failed decompressing body")
			return nil, http.StatusBadRequest
		}
	case "identity", "":
		// no action
	default:
		atomic.AddUint64(&rhh.requestFailureEncoding, 1)
		if len(encoding) > 64 {
			encoding = encoding[0:64]
		}
		rhh.logger.WithField("encoding", encoding).Info("invalid encoding")
		return nil, http.StatusBadRequest
	}

	return b, 0
}

func (rhh *rawHttpHandlerV2) MetricHandler(w http.ResponseWriter, req *http.Request) {
	b, errCode := rhh.readBody(req)

	if errCode != 0 {
		w.WriteHeader(errCode)
		return
	}

	var msg pb.RawMessageV2
	err := proto.Unmarshal(b, &msg)
	if err != nil {
		atomic.AddUint64(&rhh.requestFailureUnmarshal, 1)
		rhh.logger.WithError(err).Error("failed to unmarshal")
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	mm := translateFromProtobufV2(&msg)
	mm.DispatchMetrics(req.Context(), rhh.handler)

	atomic.AddUint64(&rhh.requestSuccess, 1)
	w.WriteHeader(http.StatusAccepted)
}

func (rhh *rawHttpHandlerV2) EventHandler(w http.ResponseWriter, req *http.Request) {
	b, errCode := rhh.readBody(req)

	if errCode != 0 {
		w.WriteHeader(errCode)
		return
	}

	var msg pb.EventV2
	err := proto.Unmarshal(b, &msg)
	if err != nil {
		atomic.AddUint64(&rhh.requestFailureUnmarshal, 1)
		rhh.logger.WithError(err).Error("failed to unmarshal")
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	event := &gostatsd.Event{
		Title:          msg.Title,
		Text:           msg.Text,
		DateHappened:   msg.DateHappened,
		Hostname:       msg.Hostname,
		AggregationKey: msg.AggregationKey,
		SourceTypeName: msg.SourceTypeName,
		Tags:           msg.Tags,
		SourceIP:       gostatsd.IP(msg.SourceIP),
	}

	switch msg.Priority {
	case pb.EventV2_Normal:
		event.Priority = gostatsd.PriNormal
	case pb.EventV2_Low:
		event.Priority = gostatsd.PriLow
	default:
		event.Priority = gostatsd.PriNormal
	}

	switch msg.Type {
	case pb.EventV2_Info:
		event.AlertType = gostatsd.AlertInfo
	case pb.EventV2_Warning:
		event.AlertType = gostatsd.AlertWarning
	case pb.EventV2_Error:
		event.AlertType = gostatsd.AlertError
	case pb.EventV2_Success:
		event.AlertType = gostatsd.AlertSuccess
	default:
		event.AlertType = gostatsd.AlertInfo
	}

	rhh.handler.DispatchEvent(req.Context(), event)

	atomic.AddUint64(&rhh.eventsProcessed, 1)
	atomic.AddUint64(&rhh.requestSuccess, 1)
	w.WriteHeader(http.StatusAccepted)
}

func translateFromProtobufV2(pbMetricMap *pb.RawMessageV2) *gostatsd.MetricMap {
	now := gostatsd.Nanotime(time.Now().UnixNano())
	mm := gostatsd.NewMetricMap()

	for metricName, tagMap := range pbMetricMap.Gauges {
		mm.Gauges[metricName] = map[string]gostatsd.Gauge{}
		for tagsKey, gauge := range tagMap.TagMap {
			mm.Gauges[metricName][tagsKey] = gostatsd.Gauge{
				Value:     gauge.Value,
				Timestamp: now,
				Hostname:  gauge.Hostname,
				Tags:      gauge.Tags,
			}
		}
	}

	for metricName, tagMap := range pbMetricMap.Counters {
		mm.Counters[metricName] = map[string]gostatsd.Counter{}
		for tagsKey, counter := range tagMap.TagMap {
			mm.Counters[metricName][tagsKey] = gostatsd.Counter{
				Value:     counter.Value,
				Timestamp: now,
				Tags:      counter.Tags,
				Hostname:  counter.Hostname,
			}
		}
	}

	for metricName, tagMap := range pbMetricMap.Timers {
		mm.Timers[metricName] = map[string]gostatsd.Timer{}
		for tagsKey, timer := range tagMap.TagMap {
			mm.Timers[metricName][tagsKey] = gostatsd.Timer{
				Values:       timer.Values,
				Timestamp:    now,
				Tags:         timer.Tags,
				Hostname:     timer.Hostname,
				SampledCount: timer.SampleCount,
			}
		}
	}

	for metricName, tagMap := range pbMetricMap.Sets {
		mm.Sets[metricName] = map[string]gostatsd.Set{}
		for tagsKey, set := range tagMap.TagMap {
			mm.Sets[metricName][tagsKey] = gostatsd.Set{
				Values:    map[string]struct{}{},
				Timestamp: now,
				Tags:      set.Tags,
				Hostname:  set.Hostname,
			}
			for _, value := range set.Values {
				mm.Sets[metricName][tagsKey].Values[value] = struct{}{}
			}
		}
	}

	return mm
}
<commit_msg>Dispatch MetricMap from rawHttpHandlerV2<commit_after>package web

import (
	"context"
	"io/ioutil"
	"net/http"
	"sync/atomic"
	"time"

	"github.com/atlassian/gostatsd"
	"github.com/atlassian/gostatsd/pb"

	"github.com/atlassian/gostatsd/pkg/stats"
	"github.com/golang/protobuf/proto"
	"github.com/sirupsen/logrus"
)

type rawHttpHandlerV2 struct {
	requestSuccess           uint64 // atomic
	requestFailureRead       uint64 // atomic
	requestFailureDecompress uint64 // atomic
	requestFailureEncoding   uint64 // atomic
	requestFailureUnmarshal  uint64 // atomic
	metricsProcessed         uint64 // atomic
	eventsProcessed          uint64 // atomic

	logger     logrus.FieldLogger
	handler    gostatsd.PipelineHandler
	serverName string
}

func newRawHttpHandlerV2(logger logrus.FieldLogger, serverName string, handler gostatsd.PipelineHandler) *rawHttpHandlerV2 {
	return &rawHttpHandlerV2{
		logger:     logger,
		handler:    handler,
		serverName: serverName,
	}
}

func (rhh *rawHttpHandlerV2) RunMetrics(ctx context.Context) {
	statser := stats.FromContext(ctx).WithTags([]string{"server-name:" + rhh.serverName})

	notify, cancel := statser.RegisterFlush()
	defer cancel()

	for {
		select {
		case <-notify:
			rhh.emitMetrics(statser)
		case <-ctx.Done():
			return
		}
	}
}

func (rhh *rawHttpHandlerV2) emitMetrics(statser stats.Statser) {
	requestSuccess := atomic.SwapUint64(&rhh.requestSuccess, 0)
	requestFailureRead := atomic.SwapUint64(&rhh.requestFailureRead, 0)
	requestFailureDecompress := atomic.SwapUint64(&rhh.requestFailureDecompress, 0)
	requestFailureEncoding := atomic.SwapUint64(&rhh.requestFailureEncoding, 0)
	requestFailureUnmarshal := atomic.SwapUint64(&rhh.requestFailureUnmarshal, 0)
	metricsProcessed := atomic.SwapUint64(&rhh.metricsProcessed, 0)
	eventsProcessed := atomic.SwapUint64(&rhh.eventsProcessed, 0)

	statser.Count("http.incoming", float64(requestSuccess), []string{"result:success"})
	statser.Count("http.incoming", float64(requestFailureRead), []string{"result:failure", "failure:read"})
	statser.Count("http.incoming", float64(requestFailureDecompress), []string{"result:failure", "failure:decompress"})
	statser.Count("http.incoming", float64(requestFailureEncoding), []string{"result:failure", "failure:encoding"})
	statser.Count("http.incoming", float64(requestFailureUnmarshal), []string{"result:failure", "failure:unmarshal"})
	statser.Count("http.incoming.metrics", float64(metricsProcessed), nil)
	statser.Count("http.incoming.events", float64(eventsProcessed), nil)
}

func (rhh *rawHttpHandlerV2) readBody(req *http.Request) ([]byte, int) {
	b, err := ioutil.ReadAll(req.Body)
	if err != nil {
		atomic.AddUint64(&rhh.requestFailureRead, 1)
		rhh.logger.WithError(err).Info("failed reading body")
		return nil, http.StatusInternalServerError
	}
	req.Body.Close()

	encoding := req.Header.Get("Content-Encoding")
	switch encoding {
	case "deflate":
		b, err = decompress(b)
		if err != nil {
			atomic.AddUint64(&rhh.requestFailureDecompress, 1)
			rhh.logger.WithError(err).Info("failed decompressing body")
			return nil, http.StatusBadRequest
		}
	case "identity", "":
		// no action
	default:
		atomic.AddUint64(&rhh.requestFailureEncoding, 1)
		if len(encoding) > 64 {
			encoding = encoding[0:64]
		}
		rhh.logger.WithField("encoding", encoding).Info("invalid encoding")
		return nil, http.StatusBadRequest
	}

	return b, 0
}

func (rhh *rawHttpHandlerV2) MetricHandler(w http.ResponseWriter, req *http.Request) {
	b, errCode := rhh.readBody(req)

	if errCode != 0 {
		w.WriteHeader(errCode)
		return
	}

	var msg pb.RawMessageV2
	err := proto.Unmarshal(b, &msg)
	if err != nil {
		atomic.AddUint64(&rhh.requestFailureUnmarshal, 1)
		rhh.logger.WithError(err).Error("failed to unmarshal")
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	mm := translateFromProtobufV2(&msg)
	rhh.handler.DispatchMetricMap(req.Context(), mm)

	atomic.AddUint64(&rhh.requestSuccess, 1)
	w.WriteHeader(http.StatusAccepted)
}

func (rhh *rawHttpHandlerV2) EventHandler(w http.ResponseWriter, req *http.Request) {
	b, errCode := rhh.readBody(req)

	if errCode != 0 {
		w.WriteHeader(errCode)
		return
	}

	var msg pb.EventV2
	err := proto.Unmarshal(b, &msg)
	if err != nil {
		atomic.AddUint64(&rhh.requestFailureUnmarshal, 1)
		rhh.logger.WithError(err).Error("failed to unmarshal")
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	event := &gostatsd.Event{
		Title:          msg.Title,
		Text:           msg.Text,
		DateHappened:   msg.DateHappened,
		Hostname:       msg.Hostname,
		AggregationKey: msg.AggregationKey,
		SourceTypeName: msg.SourceTypeName,
		Tags:           msg.Tags,
		SourceIP:       gostatsd.IP(msg.SourceIP),
	}

	switch msg.Priority {
	case pb.EventV2_Normal:
		event.Priority = gostatsd.PriNormal
	case pb.EventV2_Low:
		event.Priority = gostatsd.PriLow
	default:
		event.Priority = gostatsd.PriNormal
	}

	switch msg.Type {
	case pb.EventV2_Info:
		event.AlertType = gostatsd.AlertInfo
	case pb.EventV2_Warning:
		event.AlertType = gostatsd.AlertWarning
	case pb.EventV2_Error:
		event.AlertType = gostatsd.AlertError
	case pb.EventV2_Success:
		event.AlertType = gostatsd.AlertSuccess
	default:
		event.AlertType = gostatsd.AlertInfo
	}

	rhh.handler.DispatchEvent(req.Context(), event)

	atomic.AddUint64(&rhh.eventsProcessed, 1)
	atomic.AddUint64(&rhh.requestSuccess, 1)
	w.WriteHeader(http.StatusAccepted)
}

func translateFromProtobufV2(pbMetricMap *pb.RawMessageV2) *gostatsd.MetricMap {
	now := gostatsd.Nanotime(time.Now().UnixNano())
	mm := gostatsd.NewMetricMap()

	for metricName, tagMap := range pbMetricMap.Gauges {
		mm.Gauges[metricName] = map[string]gostatsd.Gauge{}
		for tagsKey, gauge := range tagMap.TagMap {
			mm.Gauges[metricName][tagsKey] = gostatsd.Gauge{
				Value:     gauge.Value,
				Timestamp: now,
				Hostname:  gauge.Hostname,
				Tags:      gauge.Tags,
			}
		}
	}

	for metricName, tagMap := range pbMetricMap.Counters {
		mm.Counters[metricName] = map[string]gostatsd.Counter{}
		for tagsKey, counter := range tagMap.TagMap {
			mm.Counters[metricName][tagsKey] = gostatsd.Counter{
				Value:     counter.Value,
				Timestamp: now,
				Tags:      counter.Tags,
				Hostname:  counter.Hostname,
			}
		}
	}

	for metricName, tagMap := range pbMetricMap.Timers {
		mm.Timers[metricName] = map[string]gostatsd.Timer{}
		for tagsKey, timer := range tagMap.TagMap {
			mm.Timers[metricName][tagsKey] = gostatsd.Timer{
				Values:       timer.Values,
				Timestamp:    now,
				Tags:         timer.Tags,
				Hostname:     timer.Hostname,
				SampledCount: timer.SampleCount,
			}
		}
	}

	for metricName, tagMap := range pbMetricMap.Sets {
		mm.Sets[metricName] = map[string]gostatsd.Set{}
		for tagsKey, set := range tagMap.TagMap {
			mm.Sets[metricName][tagsKey] = gostatsd.Set{
				Values:    map[string]struct{}{},
				Timestamp: now,
				Tags:      set.Tags,
				Hostname:  set.Hostname,
			}
			for _, value := range set.Values {
				mm.Sets[metricName][tagsKey].Values[value] = struct{}{}
			}
		}
	}

	return mm
}
<|endoftext|>


<commit_before>package main
import "fmt"
func main() {
	fmt.Println("Hello, 世界")
	fmt.Println("Mikael var her igjen")
	fmt.Println("Morten endret dette, wiiiiiiiiiiiiiiiiiiiiiiii")
	fmt.Println("Hei alle sammen")
	fmt.Println("Her er enda en endring")
	fmt.Println("Eirik sier hallo") // KOMMENTAR
	fmt.Println("yolo")
	fmt.Println("Det var ikke meg")
	fmt.Println("nils")
	fmt.Println("Abdi")
	fmt.Println("Brede")
	fmt.Println("Jan Kevin #")

}
<commit_msg>ICA01 helt ferdig<commit_after>package main

import "fmt"

func main() {
	fmt.Println("Hello, 世界")
	fmt.Println("Mikael var her igjen")
	fmt.Println("Morten endret dette, wiiiiiiiiiiiiiiiiiiiiiiii")
	fmt.Println("Hei alle sammen")
	fmt.Println("Her er enda en endring")
	fmt.Println("Eirik sier hallo")
	fmt.Println("yolo")
	fmt.Println("Det var ikke meg")
	fmt.Println("nils")
	fmt.Println("Abdi")
	fmt.Println("Brede")
	fmt.Println("Jan Kevin #")
	fmt.Println("Mats endret dette")

}
<|endoftext|>


<commit_before>package pgp

import (
	"bytes"

	"github.com/pkg/errors"
	"go.mozilla.org/autograph/signer"
	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/armor"
)

const (
	// Type of this signer is "pgp"
	Type = "pgp"
)

// PGPSigner holds the configuration of the signer
type PGPSigner struct {
	signer.Configuration
	entity *openpgp.Entity
}

// New initializes a pgp signer using a configuration
func New(conf signer.Configuration) (s *PGPSigner, err error) {
	s = new(PGPSigner)

	if conf.Type != Type {
		return nil, errors.Errorf("pgp: invalid type %q, must be %q", conf.Type, Type)
	}
	s.Type = conf.Type

	if conf.ID == "" {
		return nil, errors.New("pgp: missing signer ID in signer configuration")
	}
	s.ID = conf.ID

	if conf.PrivateKey == "" {
		return nil, errors.New("pgp: missing private key in signer configuration")
	}
	s.PrivateKey = conf.PrivateKey
	entities, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(s.PrivateKey))
	if err != nil {
		return nil, errors.Wrap(err, "pgp: failed to read armored keyring")
	}
	if len(entities) != 1 {
		return nil, errors.Errorf("pgp: found %d entities in armored keyring, expected one", len(entities))
	}
	s.entity = entities[0]

	// serialize the public key
	var pubkeybuf bytes.Buffer
	err = s.entity.Serialize(&pubkeybuf)
	if err != nil {
		return nil, errors.Wrap(err, "pgp: failed to serialize public key")
	}
	armoredbuf := bytes.NewBuffer(nil)
	ewrbuf, err := armor.Encode(armoredbuf, openpgp.PublicKeyType, nil)
	if err != nil {
		return nil, errors.Wrap(err, "pgp: failed to serialize public key")
	}
	_, err = ewrbuf.Write(pubkeybuf.Bytes())
	if err != nil {
		return nil, errors.Wrap(err, "pgp: failed to serialize public key")
	}
	ewrbuf.Close()
	s.PublicKey = armoredbuf.String()

	return
}

// Config returns the configuration of the current signer
func (s *PGPSigner) Config() signer.Configuration {
	return signer.Configuration{
		ID:         s.ID,
		Type:       s.Type,
		PrivateKey: s.PrivateKey,
		PublicKey:  s.PublicKey,
	}
}

// SignData takes data and returns an armored signature with pgp header and footer
func (s *PGPSigner) SignData(data []byte, options interface{}) (signer.Signature, error) {
	out := bytes.NewBuffer(nil)
	message := bytes.NewBuffer(data)
	err := openpgp.ArmoredDetachSign(out, s.entity, message, nil)
	if err != nil {
		return nil, errors.Wrap(err, "pgp: failed to sign")
	}
	sig := new(Signature)
	sig.Data = out.Bytes()
	return sig, nil
}

// Signature is a PGP signature
type Signature struct {
	Data []byte
}

// Marshal doesn't do much for this signer. sig.Data already contains
// an armored signature, so we simply convert it to a string and return it
func (sig *Signature) Marshal() (string, error) {
	return string(sig.Data), nil
}

// Unmarshal also does very little. It simply converts the armored signature
// from a string to an []byte, but doesn't attempt to parse it, and returns it
// as a Signature
func Unmarshal(sigstr string) (signer.Signature, error) {
	sig := new(Signature)
	sig.Data = []byte(sigstr)
	return sig, nil
}

// Options are not implemented for this signer
type Options struct {
}

// GetDefaultOptions returns default options of the signer
func (s *PGPSigner) GetDefaultOptions() interface{} {
	return Options{}
}
<commit_msg>signer: update docstring for pgp signer type<commit_after>package pgp

import (
	"bytes"

	"github.com/pkg/errors"
	"go.mozilla.org/autograph/signer"
	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/armor"
)

const (
	// Type of this signer is "pgp", which represents a signer
	// that uses the native golang.org/x/crypto/openpgp to sign
	// data
	Type = "pgp"
)

// PGPSigner holds the configuration of the signer
type PGPSigner struct {
	signer.Configuration
	entity *openpgp.Entity
}

// New initializes a pgp signer using a configuration
func New(conf signer.Configuration) (s *PGPSigner, err error) {
	s = new(PGPSigner)

	if conf.Type != Type {
		return nil, errors.Errorf("pgp: invalid type %q, must be %q", conf.Type, Type)
	}
	s.Type = conf.Type

	if conf.ID == "" {
		return nil, errors.New("pgp: missing signer ID in signer configuration")
	}
	s.ID = conf.ID

	if conf.PrivateKey == "" {
		return nil, errors.New("pgp: missing private key in signer configuration")
	}
	s.PrivateKey = conf.PrivateKey
	entities, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(s.PrivateKey))
	if err != nil {
		return nil, errors.Wrap(err, "pgp: failed to read armored keyring")
	}
	if len(entities) != 1 {
		return nil, errors.Errorf("pgp: found %d entities in armored keyring, expected one", len(entities))
	}
	s.entity = entities[0]

	// serialize the public key
	var pubkeybuf bytes.Buffer
	err = s.entity.Serialize(&pubkeybuf)
	if err != nil {
		return nil, errors.Wrap(err, "pgp: failed to serialize public key")
	}
	armoredbuf := bytes.NewBuffer(nil)
	ewrbuf, err := armor.Encode(armoredbuf, openpgp.PublicKeyType, nil)
	if err != nil {
		return nil, errors.Wrap(err, "pgp: failed to serialize public key")
	}
	_, err = ewrbuf.Write(pubkeybuf.Bytes())
	if err != nil {
		return nil, errors.Wrap(err, "pgp: failed to serialize public key")
	}
	ewrbuf.Close()
	s.PublicKey = armoredbuf.String()

	return
}

// Config returns the configuration of the current signer
func (s *PGPSigner) Config() signer.Configuration {
	return signer.Configuration{
		ID:         s.ID,
		Type:       s.Type,
		PrivateKey: s.PrivateKey,
		PublicKey:  s.PublicKey,
	}
}

// SignData takes data and returns an armored signature with pgp header and footer
func (s *PGPSigner) SignData(data []byte, options interface{}) (signer.Signature, error) {
	out := bytes.NewBuffer(nil)
	message := bytes.NewBuffer(data)
	err := openpgp.ArmoredDetachSign(out, s.entity, message, nil)
	if err != nil {
		return nil, errors.Wrap(err, "pgp: failed to sign")
	}
	sig := new(Signature)
	sig.Data = out.Bytes()
	return sig, nil
}

// Signature is a PGP signature
type Signature struct {
	Data []byte
}

// Marshal doesn't do much for this signer. sig.Data already contains
// an armored signature, so we simply convert it to a string and return it
func (sig *Signature) Marshal() (string, error) {
	return string(sig.Data), nil
}

// Unmarshal also does very little. It simply converts the armored signature
// from a string to an []byte, but doesn't attempt to parse it, and returns it
// as a Signature
func Unmarshal(sigstr string) (signer.Signature, error) {
	sig := new(Signature)
	sig.Data = []byte(sigstr)
	return sig, nil
}

// Options are not implemented for this signer
type Options struct {
}

// GetDefaultOptions returns default options of the signer
func (s *PGPSigner) GetDefaultOptions() interface{} {
	return Options{}
}
<|endoftext|>


<commit_before>package pagerduty

/*
 * Copyright 2016-2017 Netflix, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"strings"
	"time"

	"github.com/netflix/hal-9001/hal"
)

// TODO: add a timestamp-based cleanup for old edges/attrs/etc.

func pollerHandler(evt hal.Evt) {
	// nothing yet - TODO: add control code, e.g. force refresh
}

func pollerInit(inst *hal.Instance) {
	pf := hal.PeriodicFunc{
		Name:     "pagerduty-poller",
		Interval: time.Hour,
		Function: ingestPagerdutyAccount,
	}

	pf.Register()
	go pf.Start()
}

func ingestPagerdutyAccount() {
	token, err := getSecrets()
	if err != nil || token == "" {
		log.Printf("pagerduty: %s is not set up in hal.Secrets. Cannot continue.", PagerdutyTokenKey)
		return
	}

	ingestPDusers(token)
	ingestPDteams(token)
	ingestPDservices(token)
	ingestPDschedules(token)
}

func ingestPDusers(token string) {
	params := map[string][]string{"include[]": []string{"contact_methods"}}
	users, err := GetUsers(token, params)
	if err != nil {
		log.Printf("Could not retreive users from the Pagerduty API: %s", err)
		return
	}

	for _, user := range users {
		attrs := map[string]string{
			"pd-user-id": user.Id,
			"name":       user.Name,
			"email":      user.Email,
		}

		// plug in the contact methods
		for _, cm := range user.ContactMethods {
			if strings.HasSuffix(cm.Type, "_reference") {
				log.Printf("contact methods not included in data: try adding include[]=contact_methods to the request")
			} else {
				attrs[cm.Type+"-id"] = cm.Id
				attrs[cm.Type] = cm.Address
			}
		}

		edges := []string{"name", "email", "phone_contact_method", "sms_contact_method"}
		logit(hal.Directory().Put(user.Id, "pd-user", attrs, edges))

		for _, team := range user.Teams {
			logit(hal.Directory().PutNode(team.Id, "pd-team"))
			logit(hal.Directory().PutEdge(team.Id, "pd-team", user.Id, "pd-user"))
		}
	}
}

func ingestPDteams(token string) {
	teams, err := GetTeams(token, nil)
	if err != nil {
		log.Printf("Could not retreive teams from the Pagerduty API: %s", err)
		return
	}

	for _, team := range teams {
		attrs := map[string]string{
			"pd-team-id":      team.Id,
			"pd-team":         team.Name,
			"pd-team-summary": team.Summary,
		}

		logit(hal.Directory().Put(team.Id, "pd-team", attrs, []string{"pd-team-id"}))
	}
}

func ingestPDservices(token string) {
	params := map[string][]string{"include[]": []string{"integrations"}}
	services, err := GetServices(token, params)
	if err != nil {
		log.Printf("Could not retreive services from the Pagerduty API: %s", err)
		return
	}

	for _, service := range services {
		attrs := map[string]string{
			"pd-service-id":           service.Id,
			"pd-service":              service.Name,
			"pd-service-key":          service.ServiceKey,
			"pd-service-description":  service.Description,
			"pd-escalation-policy-id": service.EscalationPolicy.Id,
		}

		if len(service.Integrations) == 1 && service.Integrations[0].IntegrationKey != "" {
			attrs["pd-integration-key"] = service.Integrations[0].IntegrationKey
		}

		edges := []string{"pd-service-key", "pd-service-id", "pd-escalation-policy-id", "pd-integration-key"}
		logit(hal.Directory().Put(service.Id, "pd-service", attrs, edges))

		for _, team := range service.Teams {
			logit(hal.Directory().PutNode(team.Id, "pd-team"))
			logit(hal.Directory().PutEdge(team.Id, "pd-team", service.Id, "pd-service"))
		}

		for _, igr := range service.Integrations {
			if igr.IntegrationKey == "" {
				continue
			}

			logit(hal.Directory().PutNode(igr.IntegrationKey, "pd-integration-key"))
			logit(hal.Directory().PutEdge(igr.IntegrationKey, "pd-integration-key", service.Id, "pd-service"))
		}
	}
}

func ingestPDschedules(token string) {
	schedules, err := GetSchedules(token, nil)
	if err != nil {
		log.Printf("Could not retreive schedules from the Pagerduty API: %s", err)
		return
	}

	for _, schedule := range schedules {
		attrs := map[string]string{
			"pd-schedule-id":      schedule.Id,
			"pd-schedule":         schedule.Name,
			"pd-schedule-summary": schedule.Summary,
		}

		logit(hal.Directory().Put(schedule.Id, "pd-schedule", attrs, []string{"pd-schedule-id"}))

		for _, ep := range schedule.EscalationPolicies {
			logit(hal.Directory().PutNode(ep.Id, "pd-escalation-policy"))
			logit(hal.Directory().PutEdge(ep.Id, "pd-escalation-policy", schedule.Id, "pd-schedule"))
		}

		for _, user := range schedule.Users {
			logit(hal.Directory().PutNode(user.Id, "pd-user"))
			logit(hal.Directory().PutEdge(user.Id, "pd-user", schedule.Id, "pd-schedule"))
		}
	}
}

func logit(err error) {
	if err != nil {
		log.Println("pagerduty/hal_directory error: %s", err)
	}
}
<commit_msg>update directory attributes for pagerduty<commit_after>package pagerduty

/*
 * Copyright 2016-2017 Netflix, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"strings"
	"time"

	"github.com/netflix/hal-9001/hal"
)

// TODO: add a timestamp-based cleanup for old edges/attrs/etc.

func pollerHandler(evt hal.Evt) {
	// nothing yet - TODO: add control code, e.g. force refresh
}

func pollerInit(inst *hal.Instance) {
	pf := hal.PeriodicFunc{
		Name:     "pagerduty-poller",
		Interval: time.Hour,
		Function: ingestPagerdutyAccount,
	}

	pf.Register()
	go pf.Start()
}

func ingestPagerdutyAccount() {
	token, err := getSecrets()
	if err != nil || token == "" {
		log.Printf("pagerduty: %s is not set up in hal.Secrets. Cannot continue.", PagerdutyTokenKey)
		return
	}

	ingestPDusers(token)
	ingestPDteams(token)
	ingestPDservices(token)
	ingestPDschedules(token)
}

func ingestPDusers(token string) {
	params := map[string][]string{"include[]": []string{"contact_methods"}}
	users, err := GetUsers(token, params)
	if err != nil {
		log.Printf("Could not retreive users from the Pagerduty API: %s", err)
		return
	}

	for _, user := range users {
		attrs := map[string]string{
			"pd-user-id": user.Id,
			"name":       user.Name,
			"email":      user.Email,
		}

		// plug in the contact methods
		for _, cm := range user.ContactMethods {
			if strings.HasSuffix(cm.Type, "_reference") {
				log.Printf("contact methods not included in data: try adding include[]=contact_methods to the request")
			} else {
				attrs[cm.Type+"-id"] = cm.Id
				attrs[cm.Type] = cm.Address
			}
		}

		edges := []string{"name", "email", "phone_contact_method", "sms_contact_method"}
		logit(hal.Directory().Put(user.Id, "pd-user", attrs, edges))

		for _, team := range user.Teams {
			logit(hal.Directory().PutNode(team.Id, "pd-team"))
			logit(hal.Directory().PutEdge(team.Id, "pd-team", user.Id, "pd-user"))
		}
	}
}

func ingestPDteams(token string) {
	teams, err := GetTeams(token, nil)
	if err != nil {
		log.Printf("Could not retreive teams from the Pagerduty API: %s", err)
		return
	}

	for _, team := range teams {
		attrs := map[string]string{
			"pd-team-id":          team.Id,
			"pd-team":             team.Name,
			"pd-team-summary":     team.Summary,
			"pd-team-description": team.Description,
		}

		logit(hal.Directory().Put(team.Id, "pd-team", attrs, []string{"pd-team-id"}))
	}
}

func ingestPDservices(token string) {
	params := map[string][]string{"include[]": []string{"integrations"}}
	services, err := GetServices(token, params)
	if err != nil {
		log.Printf("Could not retreive services from the Pagerduty API: %s", err)
		return
	}

	for _, service := range services {
		attrs := map[string]string{
			"pd-service-id":           service.Id,
			"pd-service":              service.Name,
			"pd-service-description":  service.Description,
			"pd-escalation-policy-id": service.EscalationPolicy.Id,
		}

		if len(service.Integrations) == 1 && service.Integrations[0].IntegrationKey != "" {
			attrs["pd-integration-key"] = service.Integrations[0].IntegrationKey
		}

		edges := []string{"pd-service-key", "pd-service-id", "pd-escalation-policy-id", "pd-integration-key"}
		logit(hal.Directory().Put(service.Id, "pd-service", attrs, edges))

		for _, team := range service.Teams {
			logit(hal.Directory().PutNode(team.Id, "pd-team"))
			logit(hal.Directory().PutEdge(team.Id, "pd-team", service.Id, "pd-service"))
		}

		for _, igr := range service.Integrations {
			if igr.IntegrationKey == "" {
				continue
			}

			logit(hal.Directory().PutNode(igr.IntegrationKey, "pd-integration-key"))
			logit(hal.Directory().PutEdge(igr.IntegrationKey, "pd-integration-key", service.Id, "pd-service"))
		}
	}
}

func ingestPDschedules(token string) {
	schedules, err := GetSchedules(token, nil)
	if err != nil {
		log.Printf("Could not retreive schedules from the Pagerduty API: %s", err)
		return
	}

	for _, schedule := range schedules {
		attrs := map[string]string{
			"pd-schedule-id":      schedule.Id,
			"pd-schedule":         schedule.Name,
			"pd-schedule-summary": schedule.Summary,
		}

		logit(hal.Directory().Put(schedule.Id, "pd-schedule", attrs, []string{"pd-schedule-id"}))

		for _, ep := range schedule.EscalationPolicies {
			logit(hal.Directory().PutNode(ep.Id, "pd-escalation-policy"))
			logit(hal.Directory().PutEdge(ep.Id, "pd-escalation-policy", schedule.Id, "pd-schedule"))
		}

		for _, user := range schedule.Users {
			logit(hal.Directory().PutNode(user.Id, "pd-user"))
			logit(hal.Directory().PutEdge(user.Id, "pd-user", schedule.Id, "pd-schedule"))
		}
	}
}

func logit(err error) {
	if err != nil {
		log.Println("pagerduty/hal_directory error: %s", err)
	}
}
<|endoftext|>


<commit_before>package premailer

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func BenchmarkPremailerBasicHTMLBytes(b *testing.B) {
	html := []byte(`

        <title>Title</title>

Hi!

There

Hello

Yes!

Green color

        `)

	for n := 0; n < b.N; n++ {
		p, err := NewPremailerFromBytes(html, nil)
		assert.Nil(b, err)
		result_html, err := p.Transform()
		assert.NotNil(b, result_html)
		assert.Nil(b, err)
	}
}

func BenchmarkPremailerBasicHTML(b *testing.B) {
	html := `

        <title>Title</title>

Hi!

There

Hello

Yes!

Green color

        `

	for n := 0; n < b.N; n++ {
		p, err := NewPremailerFromString(html, nil)
		assert.Nil(b, err)
		result_html, err := p.Transform()
		assert.NotNil(b, result_html)
		assert.Nil(b, err)
	}
}

func BenchmarkPremailerBasicHTMLBytes2String(b *testing.B) {
	html := []byte(`

        <title>Title</title>

Hi!

There

Hello

Yes!

Green color

        `)

	for n := 0; n < b.N; n++ {
		p, err := NewPremailerFromString(string(html), nil)
		assert.Nil(b, err)
		result_html, err := p.Transform()
		assert.NotNil(b, result_html)
		assert.Nil(b, err)
	}
}
<commit_msg>Use same test string<commit_after>package premailer

import (
	"bytes"
	"testing"
)

var (
	testBenmarkBuf       *bytes.Buffer
	testBenmarkHtmlBytes []byte
	testBenmarkHtml      string
)

func init() {
	testBenmarkBuf = new(bytes.Buffer)
	testString := `

        <title>Title</title>

Hi!

There

Hello

Yes!

Green color

        `
	testBenmarkBuf.WriteString(testString)
	testBenmarkHtml = testString
	testBenmarkHtmlBytes = []byte(testString)
}

func BenchmarkPremailerBasicHTMLBytes(b *testing.B) {
	for n := 0; n < b.N; n++ {
		p, _ := NewPremailerFromBytes(testBenmarkHtmlBytes, nil)
		p.Transform()
	}
}

func BenchmarkPremailerBasicHTML(b *testing.B) {
	for n := 0; n < b.N; n++ {
		p, _ := NewPremailerFromString(testBenmarkHtml, nil)
		p.Transform()
	}
}

func BenchmarkPremailerBasicHTMLBytes2String(b *testing.B) {
	for n := 0; n < b.N; n++ {
		p, _ := NewPremailerFromString(testBenmarkBuf.String(), nil)
		p.Transform()
	}
}
<|endoftext|>


<commit_before>package serve2

import (
	"errors"
	"net"

	"github.com/joushou/serve2/utils"
)

const (
	// DefaultBytesToCheck default maximum amount of bytes to check
	DefaultBytesToCheck = 128
)

// Errors
var (
	ErrGreedyHandler = errors.New("remaining handlers too greedy")
)

// Protocol is the protocol detection and handling interface used by serve2.
type Protocol interface {
	// Check informs if the bytes match the protocol. If there is not enough
	// data yet, it should return false and the wanted amount of bytes, allowing
	// future calls when more data is available. It does not need to return the
	// same every time, and incrementally checking more and more data is
	// allowed. Returning false and 0 bytes needed means that the protocol
	// handler is 100% sure that this is not the proper protocol, and will not
	// result in any further calls.
	// Check, when called with nil, nil, must return false, N, where N is the
	// smallest amount of bytes that makes sense to call Check with.
	Check(header []byte, hints []interface{}) (ok bool, needed int)

	// Handle manages the protocol. In case of an encapsulating protocol, Handle
	// can return a net.Conn which will be thrown through the entire protocol
	// management show again.
	Handle(c net.Conn) (net.Conn, error)

	// String returns a pretty representation of the protocol to be used for
	// logging purposes.
	String() string
}

// ProtocolHandler is a legacy alias for Protocol
type ProtocolHandler Protocol

// Logger is used to provide logging functionality for serve2
type Logger func(format string, v ...interface{})

// Server handles a set of Protocols.
type Server struct {
	// DefaultProtocol is the protocol fallback if no match is made
	DefaultProtocol Protocol

	// Logger is used for logging if set
	Logger Logger

	// BytesToCheck is the max amount of bytes to check
	BytesToCheck int

	protocols   []Protocol
	minimumRead int
}

// AddHandler registers a Protocol
func (s *Server) AddHandler(p Protocol) {
	s.protocols = append(s.protocols, p)
}

// AddHandlers registers a set of Protocols
func (s *Server) AddHandlers(p ...Protocol) {
	for _, ph := range p {
		s.AddHandler(ph)
	}
}

// prepareHandlers sorts the protocols after how many bytes they require to
// detect their protocol (lowest first), and stores the highest number of bytes
// required.
func (s *Server) prepareHandlers() {
	var handlers []Protocol

	for range s.protocols {
		smallest := -1
		for i, v := range s.protocols {
			var contestant, current int
			_, contestant = v.Check(nil, nil)
			if smallest == -1 {
				smallest = i
			} else {
				_, current = s.protocols[smallest].Check(nil, nil)
				if contestant < current {
					smallest = i
				}
			}

		}
		handlers = append(handlers, s.protocols[smallest])
		s.protocols = append(s.protocols[:smallest], s.protocols[smallest+1:]...)
	}

	_, s.minimumRead = handlers[0].Check(nil, nil)

	s.protocols = handlers

	if s.Logger != nil {
		s.Logger("Sorted %d protocols:", len(s.protocols))

		for _, protocol := range s.protocols {
			s.Logger("\t%v", protocol)
		}
	}
}

func (s *Server) handle(h Protocol, c net.Conn, hints []interface{}, header []byte, readErr error) {
	proxy := utils.NewProxyConn(c, header, readErr)
	proxy.SetHints(hints)

	transport, err := h.Handle(proxy)
	if err != nil {
		s.Logger("Handling %v as %v failed: %v", c.RemoteAddr(), h, err)
	}

	if transport != nil {
		if s.Logger != nil {
			s.Logger("Handling %v as %v (transport)", c.RemoteAddr(), h)
		}
		if x, ok := transport.(utils.HintedConn); ok {
			hints = x.Hints()
		}
		s.HandleConn(transport, hints)
	} else {
		if s.Logger != nil {
			s.Logger("Handling %v as %v", c.RemoteAddr(), h)
		}
	}
}

// HandleConn runs a connection through protocol detection and handling as
// needed.
func (s *Server) HandleConn(c net.Conn, hints []interface{}) error {
	var (
		err      error
		n        int
		header   = make([]byte, 0, s.BytesToCheck)
		handlers = make([]Protocol, len(s.protocols))
	)

	if hints == nil {
		hints = make([]interface{}, 0)
	}

	copy(handlers, s.protocols)

	// This loop runs until we are out of candidate handlers, or until a handler
	// is selected.
	for len(handlers) > 0 {
		// Read the required data
		n, err = c.Read(header[len(header):cap(header)])
		header = header[:len(header)+n]

		if n == 0 && err == nil {
			// Nothing read, but connection isn't dead yet
			continue
		}

		if n == 0 && err != nil {
			// Can't read anything
			break
		}

		// We run the current data through all candidate handlers.
		for i := 0; i < len(handlers); i++ {
			handler := handlers[i]

			ok, required := handler.Check(header, hints)
			switch {
			case ok:
				// THe handler accepted the connection
				s.handle(handler, c, hints, header, err)
				return nil
			case required == 0:
				// The handler is sure that it doesn't match, so remove it.
			case required <= len(header):
				// The handler is broken, requesting less than we already gave it, so
				// we remove it.
				if s.Logger != nil {
					s.Logger("Handler %v is requesting %d bytes, but already read %d bytes. Skipping.",
						handler, required, len(header))
				}

			case required > s.BytesToCheck:
				// The handler is being greedy, so we remove it.
				if s.Logger != nil {
					s.Logger("Handler %v is requesting %d bytes, but maximum read size set to %d. Skipping.",
						handler, required, s.BytesToCheck)
				}

			default:
				// The handler is not certain, so we leave it be.
				continue
			}

			handlers = append(handlers[:i], handlers[i+1:]...)
			i--

		}
	}

	if err != nil && s.Logger != nil {
		s.Logger("Protocol detection failure: %v", err)
	}

	if s.DefaultProtocol != nil {
		if s.Logger != nil {
			s.Logger("Defaulting %v: [%q]", c.RemoteAddr(), header)
		}
		s.handle(s.DefaultProtocol, c, hints, header, err)
		return nil
	}

	// No one knew what was going on on this connection
	if s.Logger != nil {
		s.Logger("Handling %v failed: [%v]", c.RemoteAddr(), header)
	}

	c.Close()
	return err
}

// Serve accepts connections on a listener, handling them as appropriate.
func (s *Server) Serve(l net.Listener) {
	s.prepareHandlers()
	for {
		conn, err := l.Accept()
		if err != nil {
			panic(err)
		}

		go func() {
			s.HandleConn(conn, nil)
		}()
	}
}

// New returns a new Server.
func New() *Server {
	return &Server{
		BytesToCheck: DefaultBytesToCheck,
		Logger:       nil,
	}
}
<commit_msg>Remove String from Protocol interface<commit_after>package serve2

import (
	"errors"
	"net"

	"github.com/joushou/serve2/utils"
)

const (
	// DefaultBytesToCheck default maximum amount of bytes to check
	DefaultBytesToCheck = 128
)

// Errors
var (
	ErrGreedyHandler = errors.New("remaining handlers too greedy")
)

// Protocol is the protocol detection and handling interface used by serve2.
type Protocol interface {
	// Check informs if the bytes match the protocol. If there is not enough
	// data yet, it should return false and the wanted amount of bytes, allowing
	// future calls when more data is available. It does not need to return the
	// same every time, and incrementally checking more and more data is
	// allowed. Returning false and 0 bytes needed means that the protocol
	// handler is 100% sure that this is not the proper protocol, and will not
	// result in any further calls.
	// Check, when called with nil, nil, must return false, N, where N is the
	// smallest amount of bytes that makes sense to call Check with.
	Check(header []byte, hints []interface{}) (ok bool, needed int)

	// Handle manages the protocol. In case of an encapsulating protocol, Handle
	// can return a net.Conn which will be thrown through the entire protocol
	// management show again.
	Handle(c net.Conn) (net.Conn, error)
}

// ProtocolHandler is a legacy alias for Protocol
type ProtocolHandler Protocol

// Logger is used to provide logging functionality for serve2
type Logger func(format string, v ...interface{})

// Server handles a set of Protocols.
type Server struct {
	// DefaultProtocol is the protocol fallback if no match is made
	DefaultProtocol Protocol

	// Logger is used for logging if set
	Logger Logger

	// BytesToCheck is the max amount of bytes to check
	BytesToCheck int

	protocols   []Protocol
	minimumRead int
}

// AddHandler registers a Protocol
func (s *Server) AddHandler(p Protocol) {
	s.protocols = append(s.protocols, p)
}

// AddHandlers registers a set of Protocols
func (s *Server) AddHandlers(p ...Protocol) {
	for _, ph := range p {
		s.AddHandler(ph)
	}
}

// prepareHandlers sorts the protocols after how many bytes they require to
// detect their protocol (lowest first), and stores the highest number of bytes
// required.
func (s *Server) prepareHandlers() {
	var handlers []Protocol

	for range s.protocols {
		smallest := -1
		for i, v := range s.protocols {
			var contestant, current int
			_, contestant = v.Check(nil, nil)
			if smallest == -1 {
				smallest = i
			} else {
				_, current = s.protocols[smallest].Check(nil, nil)
				if contestant < current {
					smallest = i
				}
			}

		}
		handlers = append(handlers, s.protocols[smallest])
		s.protocols = append(s.protocols[:smallest], s.protocols[smallest+1:]...)
	}

	_, s.minimumRead = handlers[0].Check(nil, nil)

	s.protocols = handlers

	if s.Logger != nil {
		s.Logger("Sorted %d protocols:", len(s.protocols))

		for _, protocol := range s.protocols {
			s.Logger("\t%v", protocol)
		}
	}
}

func (s *Server) handle(h Protocol, c net.Conn, hints []interface{}, header []byte, readErr error) {
	proxy := utils.NewProxyConn(c, header, readErr)
	proxy.SetHints(hints)

	transport, err := h.Handle(proxy)
	if err != nil {
		s.Logger("Handling %v as %v failed: %v", c.RemoteAddr(), h, err)
	}

	if transport != nil {
		if s.Logger != nil {
			s.Logger("Handling %v as %v (transport)", c.RemoteAddr(), h)
		}
		if x, ok := transport.(utils.HintedConn); ok {
			hints = x.Hints()
		}
		s.HandleConn(transport, hints)
	} else {
		if s.Logger != nil {
			s.Logger("Handling %v as %v", c.RemoteAddr(), h)
		}
	}
}

// HandleConn runs a connection through protocol detection and handling as
// needed.
func (s *Server) HandleConn(c net.Conn, hints []interface{}) error {
	var (
		err      error
		n        int
		header   = make([]byte, 0, s.BytesToCheck)
		handlers = make([]Protocol, len(s.protocols))
	)

	if hints == nil {
		hints = make([]interface{}, 0)
	}

	copy(handlers, s.protocols)

	// This loop runs until we are out of candidate handlers, or until a handler
	// is selected.
	for len(handlers) > 0 {
		// Read the required data
		n, err = c.Read(header[len(header):cap(header)])
		header = header[:len(header)+n]

		if n == 0 && err == nil {
			// Nothing read, but connection isn't dead yet
			continue
		}

		if n == 0 && err != nil {
			// Can't read anything
			break
		}

		// We run the current data through all candidate handlers.
		for i := 0; i < len(handlers); i++ {
			handler := handlers[i]

			ok, required := handler.Check(header, hints)
			switch {
			case ok:
				// THe handler accepted the connection
				s.handle(handler, c, hints, header, err)
				return nil
			case required == 0:
				// The handler is sure that it doesn't match, so remove it.
			case required <= len(header):
				// The handler is broken, requesting less than we already gave it, so
				// we remove it.
				if s.Logger != nil {
					s.Logger("Handler %v is requesting %d bytes, but already read %d bytes. Skipping.",
						handler, required, len(header))
				}

			case required > s.BytesToCheck:
				// The handler is being greedy, so we remove it.
				if s.Logger != nil {
					s.Logger("Handler %v is requesting %d bytes, but maximum read size set to %d. Skipping.",
						handler, required, s.BytesToCheck)
				}

			default:
				// The handler is not certain, so we leave it be.
				continue
			}

			handlers = append(handlers[:i], handlers[i+1:]...)
			i--

		}
	}

	if err != nil && s.Logger != nil {
		s.Logger("Protocol detection failure: %v", err)
	}

	if s.DefaultProtocol != nil {
		if s.Logger != nil {
			s.Logger("Defaulting %v: [%q]", c.RemoteAddr(), header)
		}
		s.handle(s.DefaultProtocol, c, hints, header, err)
		return nil
	}

	// No one knew what was going on on this connection
	if s.Logger != nil {
		s.Logger("Handling %v failed: [%v]", c.RemoteAddr(), header)
	}

	c.Close()
	return err
}

// Serve accepts connections on a listener, handling them as appropriate.
func (s *Server) Serve(l net.Listener) {
	s.prepareHandlers()
	for {
		conn, err := l.Accept()
		if err != nil {
			panic(err)
		}

		go func() {
			s.HandleConn(conn, nil)
		}()
	}
}

// New returns a new Server.
func New() *Server {
	return &Server{
		BytesToCheck: DefaultBytesToCheck,
		Logger:       nil,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"math/rand"
	"sort"
	"strings"
	"unicode"
)

var AdminLog *log.Logger

var adminCommands = map[string]func(*Player){
	"TP": func(p *Player) {
		p.hud = &AdminTeleportHUD{Player: p}
	},
	"CHANGE EXAMINE": func(p *Player) {
		p.hud = &AdminChangeExamineHUD{Player: p, Input: []rune(p.Examine())}
	},
	"DELETE THE ENTIRE ZONE": func(p *Player) {
		p.lock.Lock()
		z := p.zone
		p.lock.Unlock()

		z.Lock()
		for i := range z.Tiles {
			for _, o := range z.Tiles[i].Objects {
				if _, ok := o.(*Player); !ok {
					z.Tiles[i].Remove(o)
				}
			}
		}
		z.Unlock()
		z.Save()
		z.Repaint()
	},
	"REGENERATE THE ENTIRE ZONE": func(p *Player) {
		p.lock.Lock()
		z := p.zone
		p.lock.Unlock()

		z.Lock()
		for i := range z.Tiles {
			for _, o := range z.Tiles[i].Objects {
				if _, ok := o.(*Player); !ok {
					z.Tiles[i].Remove(o)
				}
			}
		}
		z.Unlock()
		z.Generate()
		z.Save()
		z.Repaint()
	},
}

func init() {
	for t := range rockTypeInfo {
		rt := RockType(t)
		adminCommands["SPAWN "+strings.ToUpper(rockTypeInfo[rt].Name)+" ROCK"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&Rock{Type: rt})
			p.lock.Unlock()
		}
		adminCommands["SPAWN "+strings.ToUpper(rockTypeInfo[rt].Name)+" STONE"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&Stone{Type: rt})
			p.lock.Unlock()
		}
		adminCommands["SPAWN "+strings.ToUpper(rockTypeInfo[rt].Name)+" WALL"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&WallStone{Type: rt})
			p.lock.Unlock()
		}
		for m := range metalTypeInfo {
			if m == 0 {
				continue
			}
			mt := MetalType(m)
			adminCommands["SPAWN "+strings.ToUpper(rockTypeInfo[rt].Name)+" ROCK WITH "+strings.ToUpper(metalTypeInfo[mt].Name)+" ORE"] = func(p *Player) {
				p.lock.Lock()
				p.GiveItem(&Rock{Type: rt, Ore: mt})
				p.lock.Unlock()
			}
		}
	}
	for t := range metalTypeInfo {
		if t == 0 {
			continue
		}
		mt := MetalType(t)
		adminCommands["SPAWN "+strings.ToUpper(metalTypeInfo[mt].Name)+" ORE"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&Ore{Type: mt})
			p.lock.Unlock()
		}
		adminCommands["SPAWN "+strings.ToUpper(metalTypeInfo[mt].Name)+" WALL"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&WallMetal{Type: mt})
			p.lock.Unlock()
		}
	}
	for t := range woodTypeInfo {
		wt := WoodType(t)
		adminCommands["SPAWN "+strings.ToUpper(woodTypeInfo[wt].Name)+" TREE"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&Tree{Type: wt})
			p.lock.Unlock()
		}
		adminCommands["SPAWN "+strings.ToUpper(woodTypeInfo[wt].Name)+" LOGS"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&Logs{Type: wt})
			p.lock.Unlock()
		}
		adminCommands["SPAWN "+strings.ToUpper(woodTypeInfo[wt].Name)+" WALL"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&WallWood{Type: wt})
			p.lock.Unlock()
		}
	}
	adminCommands["SPAWN PLANT"] = func(p *Player) {
		p.lock.Lock()
		p.GiveItem(&Flora{Type: 0})
		p.lock.Unlock()
	}
	adminCommands["SPAWN HERO"] = func(p *Player) {
		p.lock.Lock()
		p.GiveItem(&Hero{Name_: GenerateName(rand.New(rand.NewSource(rand.Int63())), NameHero)})
		p.lock.Unlock()
	}
}

type AdminHUD struct {
	Player *Player
	Input  []rune
}

func (h *AdminHUD) Paint(setcell func(int, int, rune, Color)) {
	if !h.Player.Admin {
		h.Player.hud = nil
		return
	}

	setcell(0, 0, '>', "#00f")
	setcell(1, 0, ' ', "#00f")
	for i, r := range h.Input {
		setcell(i+2, 0, r, "#00f")
	}
}

func (h *AdminHUD) Key(code int, special bool) bool {
	if !h.Player.Admin {
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	if !special {
		if code != 0 && code != '`' {
			h.Input = append(h.Input, unicode.ToUpper(rune(code)))
			h.Player.Repaint()
		}
		return true
	}
	switch code {
	case 37, 38, 39, 40: // arrow keys
		return false
	case 9, 16, 17, 18: // tab shift ctrl alt
		return false
	case 8: // backspace
		if len(h.Input) > 0 {
			h.Input = h.Input[:len(h.Input)-1]
			h.Player.Repaint()
		}
		return true
	case 13: // enter
		if f, ok := adminCommands[string(h.Input)]; ok {
			h.Player.lock.Lock()
			AdminLog.Printf("COMMAND:%q [%d:%q] (%d:%d, %d:%d)", string(h.Input), h.Player.ID, h.Player.Name(), h.Player.ZoneX, h.Player.TileX, h.Player.ZoneY, h.Player.TileY)
			h.Player.lock.Unlock()

			h.Player.hud = nil
			f(h.Player)
			h.Player.Repaint()
		}
		return true
	case 27: // esc
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	return true
}

type PlayerList []*Player

func (l PlayerList) Len() int {
	return len(l)
}
func (l PlayerList) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}
func (l PlayerList) Less(i, j int) bool {
	return l[i].ID < l[j].ID
}

type AdminTeleportHUD struct {
	Player *Player
	List   PlayerList
	Offset int
}

func (h *AdminTeleportHUD) Paint(setcell func(int, int, rune, Color)) {
	if !h.Player.Admin {
		h.Player.hud = nil
		h.Player.Repaint()
		return
	}
	if h.List == nil {
		onlinePlayersLock.Lock()
		for _, p := range OnlinePlayers {
			h.List = append(h.List, p)
		}
		onlinePlayersLock.Unlock()
		sort.Sort(h.List)
	}
	for i, r := range "TELEPORT TO PLAYER" {
		setcell(i, 0, r, "#fff")
	}
	for i, p := range h.List[h.Offset:] {
		if i == 8 {
			break
		}
		setcell(0, i+1, '1'+rune(i), "#fff")
		setcell(1, i+1, ' ', "#fff")
		id := p.ID
		for k := 0; k < 16; k++ {
			setcell(17-k, i+1, rune("0123456789ABCDEF"[id&15]), "#44f")
			id >>= 4
		}
		setcell(18, i+1, ' ', "#00f")
		j := 19
		p.lock.Lock()
		name := p.Name()
		p.lock.Unlock()
		for _, r := range name {
			setcell(j, i+1, r, "#00f")
			j++
		}
	}
	if h.Offset > 0 {
		setcell(0, 9, '9', "#fff")
		setcell(1, 9, ' ', "#fff")
		j := 1
		for _, r := range "previous" {
			j++
			setcell(j, 9, r, "#fff")
		}
	}
	if len(h.List) > h.Offset+8 {
		setcell(0, 10, '0', "#fff")
		setcell(1, 10, ' ', "#fff")
		j := 2
		for _, r := range "next" {
			setcell(j, 10, r, "#fff")
			j++
		}
	}
}

func (h *AdminTeleportHUD) Key(code int, special bool) bool {
	if !h.Player.Admin {
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	if !special {
		return false
	}
	switch code {
	case '1', '2', '3', '4', '5', '6', '7', '8':
		i := code - '1' + h.Offset
		if i < len(h.List) {
			p := h.List[i]
			p.lock.Lock()
			zx, zy := p.ZoneX, p.ZoneY
			tx, ty := p.TileX, p.TileY
			name := p.Name()
			p.lock.Unlock()

			h.Player.lock.Lock()
			az := h.Player.zone
			azx, azy := h.Player.ZoneX, h.Player.ZoneY
			atx, aty := h.Player.TileX, h.Player.TileY
			aname := h.Player.Name()
			h.Player.lock.Unlock()

			AdminLog.Printf("TELEPORT [%d:%q] (%d:%d, %d:%d) => [%d:%q] (%d:%d, %d:%d)", h.Player.ID, aname, azx, atx, azy, aty, p.ID, name, zx, tx, zy, ty)

			az.Lock()
			az.Tile(atx, aty).Remove(h.Player)
			az.Repaint()
			az.Unlock()

			ReleaseZone(az)
			z := GrabZone(zx, zy)

			h.Player.lock.Lock()
			h.Player.zone = z
			h.Player.ZoneX, h.Player.ZoneY = zx, zy
			h.Player.TileX, h.Player.TileY = tx, ty
			h.Player.lock.Unlock()
			h.Player.Save()

			z.Lock()
			z.Tile(tx, ty).Add(h.Player)
			z.Repaint()
			z.Unlock()

			h.Player.hud = nil
			h.Player.Repaint()
		}
		return true
	case '9':
		if h.Offset > 0 {
			h.Offset--
			h.Player.Repaint()
		}
		return true
	case '0':
		if h.Offset+8 < len(h.List) {
			h.Offset++
			h.Player.Repaint()
		}
		return true

	case 27: // esc
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	return false
}

type AdminChangeExamineHUD struct {
	Player *Player
	Input  []rune
}

func (h *AdminChangeExamineHUD) Paint(setcell func(int, int, rune, Color)) {
	if !h.Player.Admin {
		h.Player.hud = nil
		return
	}

	h.Player.lock.Lock()
	name := strings.ToUpper(h.Player.Name())
	h.Player.lock.Unlock()

	for i, r := range []rune(name) {
		setcell(i, 0, r, "#fff")
	}
	setcell(0, 1, '>', "#00f")
	setcell(1, 1, ' ', "#00f")
	for i, r := range h.Input {
		setcell(i+2, 1, r, "#00f")
	}
}

func (h *AdminChangeExamineHUD) Key(code int, special bool) bool {
	if !h.Player.Admin {
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	if !special {
		if code != 0 {
			h.Input = append(h.Input, rune(code))
			h.Player.Repaint()
		}
		return true
	}
	switch code {
	case 37, 38, 39, 40: // arrow keys
		return false
	case 9, 16, 17, 18: // tab shift ctrl alt
		return false
	case 8: // backspace
		if len(h.Input) > 0 {
			h.Input = h.Input[:len(h.Input)-1]
			h.Player.Repaint()
		}
		return true
	case 13: // enter
		h.Player.lock.Lock()
		AdminLog.Printf("CHANGEEXAMINE:%q [%d:%q] (%d:%d, %d:%d)", string(h.Input), h.Player.ID, h.Player.Name(), h.Player.ZoneX, h.Player.TileX, h.Player.ZoneY, h.Player.TileY)
		h.Player.Examine_ = string(h.Input)
		h.Player.lock.Unlock()

		h.Player.hud = nil
		h.Player.Repaint()
		return true
	case 27: // esc
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	return true
}
<commit_msg>slightly different erase/regen code<commit_after>package main

import (
	"log"
	"math/rand"
	"sort"
	"strings"
	"unicode"
)

var AdminLog *log.Logger

var adminCommands = map[string]func(*Player){
	"TP": func(p *Player) {
		p.hud = &AdminTeleportHUD{Player: p}
	},
	"CHANGE EXAMINE": func(p *Player) {
		p.hud = &AdminChangeExamineHUD{Player: p, Input: []rune(p.Examine())}
	},
	"DELETE THE ENTIRE ZONE": func(p *Player) {
		p.lock.Lock()
		z := p.zone
		p.lock.Unlock()

		z.Lock()
		for i := range z.Tiles {
			t := &z.Tiles[i]
			for j := 0; j < len(t.Objects); j++ {
				if _, ok := t.Objects[j].(*Player); !ok {
					t.Objects = append(t.Objects[:j], t.Objects[j+1:]...)
					j--
				}
			}
			if len(t.Objects) == 0 {
				t.Objects = nil
			}
		}
		z.Unlock()
		z.Save()
		z.Repaint()
	},
	"REGENERATE THE ENTIRE ZONE": func(p *Player) {
		p.lock.Lock()
		z := p.zone
		p.lock.Unlock()

		z.Lock()
		for i := range z.Tiles {
			t := &z.Tiles[i]
			for j := 0; j < len(t.Objects); j++ {
				if _, ok := t.Objects[j].(*Player); !ok {
					t.Objects = append(t.Objects[:j], t.Objects[j+1:]...)
					j--
				}
			}
			if len(t.Objects) == 0 {
				t.Objects = nil
			}
		}
		z.Unlock()
		z.Generate()
		z.Save()
		z.Repaint()
	},
}

func init() {
	for t := range rockTypeInfo {
		rt := RockType(t)
		adminCommands["SPAWN "+strings.ToUpper(rockTypeInfo[rt].Name)+" ROCK"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&Rock{Type: rt})
			p.lock.Unlock()
		}
		adminCommands["SPAWN "+strings.ToUpper(rockTypeInfo[rt].Name)+" STONE"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&Stone{Type: rt})
			p.lock.Unlock()
		}
		adminCommands["SPAWN "+strings.ToUpper(rockTypeInfo[rt].Name)+" WALL"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&WallStone{Type: rt})
			p.lock.Unlock()
		}
		for m := range metalTypeInfo {
			if m == 0 {
				continue
			}
			mt := MetalType(m)
			adminCommands["SPAWN "+strings.ToUpper(rockTypeInfo[rt].Name)+" ROCK WITH "+strings.ToUpper(metalTypeInfo[mt].Name)+" ORE"] = func(p *Player) {
				p.lock.Lock()
				p.GiveItem(&Rock{Type: rt, Ore: mt})
				p.lock.Unlock()
			}
		}
	}
	for t := range metalTypeInfo {
		if t == 0 {
			continue
		}
		mt := MetalType(t)
		adminCommands["SPAWN "+strings.ToUpper(metalTypeInfo[mt].Name)+" ORE"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&Ore{Type: mt})
			p.lock.Unlock()
		}
		adminCommands["SPAWN "+strings.ToUpper(metalTypeInfo[mt].Name)+" WALL"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&WallMetal{Type: mt})
			p.lock.Unlock()
		}
	}
	for t := range woodTypeInfo {
		wt := WoodType(t)
		adminCommands["SPAWN "+strings.ToUpper(woodTypeInfo[wt].Name)+" TREE"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&Tree{Type: wt})
			p.lock.Unlock()
		}
		adminCommands["SPAWN "+strings.ToUpper(woodTypeInfo[wt].Name)+" LOGS"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&Logs{Type: wt})
			p.lock.Unlock()
		}
		adminCommands["SPAWN "+strings.ToUpper(woodTypeInfo[wt].Name)+" WALL"] = func(p *Player) {
			p.lock.Lock()
			p.GiveItem(&WallWood{Type: wt})
			p.lock.Unlock()
		}
	}
	adminCommands["SPAWN PLANT"] = func(p *Player) {
		p.lock.Lock()
		p.GiveItem(&Flora{Type: 0})
		p.lock.Unlock()
	}
	adminCommands["SPAWN HERO"] = func(p *Player) {
		p.lock.Lock()
		p.GiveItem(&Hero{Name_: GenerateName(rand.New(rand.NewSource(rand.Int63())), NameHero)})
		p.lock.Unlock()
	}
}

type AdminHUD struct {
	Player *Player
	Input  []rune
}

func (h *AdminHUD) Paint(setcell func(int, int, rune, Color)) {
	if !h.Player.Admin {
		h.Player.hud = nil
		return
	}

	setcell(0, 0, '>', "#00f")
	setcell(1, 0, ' ', "#00f")
	for i, r := range h.Input {
		setcell(i+2, 0, r, "#00f")
	}
}

func (h *AdminHUD) Key(code int, special bool) bool {
	if !h.Player.Admin {
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	if !special {
		if code != 0 && code != '`' {
			h.Input = append(h.Input, unicode.ToUpper(rune(code)))
			h.Player.Repaint()
		}
		return true
	}
	switch code {
	case 37, 38, 39, 40: // arrow keys
		return false
	case 9, 16, 17, 18: // tab shift ctrl alt
		return false
	case 8: // backspace
		if len(h.Input) > 0 {
			h.Input = h.Input[:len(h.Input)-1]
			h.Player.Repaint()
		}
		return true
	case 13: // enter
		if f, ok := adminCommands[string(h.Input)]; ok {
			h.Player.lock.Lock()
			AdminLog.Printf("COMMAND:%q [%d:%q] (%d:%d, %d:%d)", string(h.Input), h.Player.ID, h.Player.Name(), h.Player.ZoneX, h.Player.TileX, h.Player.ZoneY, h.Player.TileY)
			h.Player.lock.Unlock()

			h.Player.hud = nil
			f(h.Player)
			h.Player.Repaint()
		}
		return true
	case 27: // esc
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	return true
}

type PlayerList []*Player

func (l PlayerList) Len() int {
	return len(l)
}
func (l PlayerList) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}
func (l PlayerList) Less(i, j int) bool {
	return l[i].ID < l[j].ID
}

type AdminTeleportHUD struct {
	Player *Player
	List   PlayerList
	Offset int
}

func (h *AdminTeleportHUD) Paint(setcell func(int, int, rune, Color)) {
	if !h.Player.Admin {
		h.Player.hud = nil
		h.Player.Repaint()
		return
	}
	if h.List == nil {
		onlinePlayersLock.Lock()
		for _, p := range OnlinePlayers {
			h.List = append(h.List, p)
		}
		onlinePlayersLock.Unlock()
		sort.Sort(h.List)
	}
	for i, r := range "TELEPORT TO PLAYER" {
		setcell(i, 0, r, "#fff")
	}
	for i, p := range h.List[h.Offset:] {
		if i == 8 {
			break
		}
		setcell(0, i+1, '1'+rune(i), "#fff")
		setcell(1, i+1, ' ', "#fff")
		id := p.ID
		for k := 0; k < 16; k++ {
			setcell(17-k, i+1, rune("0123456789ABCDEF"[id&15]), "#44f")
			id >>= 4
		}
		setcell(18, i+1, ' ', "#00f")
		j := 19
		p.lock.Lock()
		name := p.Name()
		p.lock.Unlock()
		for _, r := range name {
			setcell(j, i+1, r, "#00f")
			j++
		}
	}
	if h.Offset > 0 {
		setcell(0, 9, '9', "#fff")
		setcell(1, 9, ' ', "#fff")
		j := 1
		for _, r := range "previous" {
			j++
			setcell(j, 9, r, "#fff")
		}
	}
	if len(h.List) > h.Offset+8 {
		setcell(0, 10, '0', "#fff")
		setcell(1, 10, ' ', "#fff")
		j := 2
		for _, r := range "next" {
			setcell(j, 10, r, "#fff")
			j++
		}
	}
}

func (h *AdminTeleportHUD) Key(code int, special bool) bool {
	if !h.Player.Admin {
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	if !special {
		return false
	}
	switch code {
	case '1', '2', '3', '4', '5', '6', '7', '8':
		i := code - '1' + h.Offset
		if i < len(h.List) {
			p := h.List[i]
			p.lock.Lock()
			zx, zy := p.ZoneX, p.ZoneY
			tx, ty := p.TileX, p.TileY
			name := p.Name()
			p.lock.Unlock()

			h.Player.lock.Lock()
			az := h.Player.zone
			azx, azy := h.Player.ZoneX, h.Player.ZoneY
			atx, aty := h.Player.TileX, h.Player.TileY
			aname := h.Player.Name()
			h.Player.lock.Unlock()

			AdminLog.Printf("TELEPORT [%d:%q] (%d:%d, %d:%d) => [%d:%q] (%d:%d, %d:%d)", h.Player.ID, aname, azx, atx, azy, aty, p.ID, name, zx, tx, zy, ty)

			az.Lock()
			az.Tile(atx, aty).Remove(h.Player)
			az.Repaint()
			az.Unlock()

			ReleaseZone(az)
			z := GrabZone(zx, zy)

			h.Player.lock.Lock()
			h.Player.zone = z
			h.Player.ZoneX, h.Player.ZoneY = zx, zy
			h.Player.TileX, h.Player.TileY = tx, ty
			h.Player.lock.Unlock()
			h.Player.Save()

			z.Lock()
			z.Tile(tx, ty).Add(h.Player)
			z.Repaint()
			z.Unlock()

			h.Player.hud = nil
			h.Player.Repaint()
		}
		return true
	case '9':
		if h.Offset > 0 {
			h.Offset--
			h.Player.Repaint()
		}
		return true
	case '0':
		if h.Offset+8 < len(h.List) {
			h.Offset++
			h.Player.Repaint()
		}
		return true

	case 27: // esc
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	return false
}

type AdminChangeExamineHUD struct {
	Player *Player
	Input  []rune
}

func (h *AdminChangeExamineHUD) Paint(setcell func(int, int, rune, Color)) {
	if !h.Player.Admin {
		h.Player.hud = nil
		return
	}

	h.Player.lock.Lock()
	name := strings.ToUpper(h.Player.Name())
	h.Player.lock.Unlock()

	for i, r := range []rune(name) {
		setcell(i, 0, r, "#fff")
	}
	setcell(0, 1, '>', "#00f")
	setcell(1, 1, ' ', "#00f")
	for i, r := range h.Input {
		setcell(i+2, 1, r, "#00f")
	}
}

func (h *AdminChangeExamineHUD) Key(code int, special bool) bool {
	if !h.Player.Admin {
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	if !special {
		if code != 0 {
			h.Input = append(h.Input, rune(code))
			h.Player.Repaint()
		}
		return true
	}
	switch code {
	case 37, 38, 39, 40: // arrow keys
		return false
	case 9, 16, 17, 18: // tab shift ctrl alt
		return false
	case 8: // backspace
		if len(h.Input) > 0 {
			h.Input = h.Input[:len(h.Input)-1]
			h.Player.Repaint()
		}
		return true
	case 13: // enter
		h.Player.lock.Lock()
		AdminLog.Printf("CHANGEEXAMINE:%q [%d:%q] (%d:%d, %d:%d)", string(h.Input), h.Player.ID, h.Player.Name(), h.Player.ZoneX, h.Player.TileX, h.Player.ZoneY, h.Player.TileY)
		h.Player.Examine_ = string(h.Input)
		h.Player.lock.Unlock()

		h.Player.hud = nil
		h.Player.Repaint()
		return true
	case 27: // esc
		h.Player.hud = nil
		h.Player.Repaint()
		return true
	}
	return true
}
<|endoftext|>