<commit_before>// Package trigram is a simple trigram index
package trigram

import (
	"sort"
)

// T is a trigram
type T uint32

func (t T) String() string {
	b := [3]byte{byte(t >> 16), byte(t >> 8), byte(t)}
	return string(b[:])
}

// DocID is a document ID
type DocID uint32

// Index is a trigram index
type Index map[T][]DocID

// a special (and invalid) trigram that holds all the document IDs
const tAllDocIDs T = 0xFFFFFFFF

// Extract returns a list of all the unique trigrams in s
func Extract(s string, trigrams []T) []T {

	for i := 0; i <= len(s)-3; i++ {
		t := T(uint32(s[i])<<16 | uint32(s[i+1])<<8 | uint32(s[i+2]))
		trigrams = appendIfUnique(trigrams, t)
	}

	return trigrams
}

// ExtractAll returns a list of all the trigrams in s
func ExtractAll(s string, trigrams []T) []T {

	for i := 0; i <= len(s)-3; i++ {
		t := T(uint32(s[i])<<16 | uint32(s[i+1])<<8 | uint32(s[i+2]))
		trigrams = append(trigrams, t)
	}

	return trigrams
}

func appendIfUnique(t []T, n T) []T {
	for _, v := range t {
		if v == n {
			return t
		}
	}

	return append(t, n)
}

// NewIndex returns an index for the strings in docs
func NewIndex(docs []string) Index {

	idx := make(Index)

	var allDocIDs []DocID

	var trigrams []T

	for id, d := range docs {
		ts := ExtractAll(d, trigrams)
		docid := DocID(id)
		allDocIDs = append(allDocIDs, docid)
		for _, t := range ts {
			idxt := idx[t]
			l := len(idxt)
			if l == 0 || idxt[l-1] != docid {
				idx[t] = append(idxt, docid)
			}
		}
		trigrams = trigrams[:0]
	}

	idx[tAllDocIDs] = allDocIDs

	return idx
}

// Add adds a new string to the search index
func (idx Index) Add(s string) DocID {
	id := DocID(len(idx[tAllDocIDs]))
	idx.Insert(s, id)
	return id
}

// AddTrigrams adds a set of trigrams to the search index
func (idx Index) AddTrigrams(ts []T) DocID {
	id := DocID(len(idx[tAllDocIDs]))
	idx.InsertTrigrams(ts, id)
	return id
}

// Insert adds a string with a given document ID
func (idx Index) Insert(s string, id DocID) {
	ts := ExtractAll(s, nil)
	idx.InsertTrigrams(ts, id)
}

// InsertTrigrams adds a set of trigrams with a given document ID
func (idx Index) InsertTrigrams(ts []T, id DocID) {
	for _, t := range ts {
		idxt, ok := idx[t]
		// this trigram post list has been pruned. we must keep it empty
		if ok && idxt == nil {
			continue
		}
		l := len(idxt)
		if l == 0 || idxt[l-1] != id {
			idx[t] = append(idxt, id)
		}
	}

	idx[tAllDocIDs] = append(idx[tAllDocIDs], id)
}

// Delete removes a document from the index
func (idx Index) Delete(s string, id DocID) {
	ts := ExtractAll(s, nil)
	for _, t := range ts {
		ids := idx[t]
		if ids == nil {
			continue
		}

		if len(ids) == 1 && ids[0] == id {
			delete(idx, t)
			continue
		}

		i := sort.Search(len(ids), func(i int) bool { return ids[i] >= id })

		if i != -1 && i < len(ids) && ids[i] == id {
			copy(ids[i:], ids[i+1:])
			idx[t] = ids[:len(ids)-1]
		}
	}
}

// for sorting
type docList []DocID

func (d docList) Len() int           { return len(d) }
func (d docList) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
func (d docList) Less(i, j int) bool { return d[i] < d[j] }

// Sort ensures all the document IDs are in order.
func (idx Index) Sort() {
	for _, v := range idx {
		dl := docList(v)
		if !sort.IsSorted(dl) {
			sort.Sort(dl)
		}
	}
}

// Prune removes all trigrams that are present in more than the specified percentage of the documents.
func (idx Index) Prune(pct float64) int {

	maxDocs := int(pct * float64(len(idx[tAllDocIDs])))

	var pruned int

	for k, v := range idx {
		if k != tAllDocIDs && len(v) > maxDocs {
			pruned++
			idx[k] = nil
		}
	}

	return pruned
}

// Query returns a list of document IDs that match the trigrams in the query s
func (idx Index) Query(s string) []DocID {
	ts := Extract(s, nil)
	return idx.QueryTrigrams(ts)
}

type tfList struct {
	tri  []T
	freq []int
}

func (tf tfList) Len() int { return len(tf.tri) }
func (tf tfList) Swap(i, j int) {
	tf.tri[i], tf.tri[j] = tf.tri[j], tf.tri[i]
	tf.freq[i], tf.freq[j] = tf.freq[j], tf.freq[i]
}
func (tf tfList) Less(i, j int) bool { return tf.freq[i] < tf.freq[j] }

// QueryTrigrams returns a list of document IDs that match the trigram set ts
func (idx Index) QueryTrigrams(ts []T) []DocID {

	if len(ts) == 0 {
		return idx[tAllDocIDs]
	}

	var freq []int

	for _, t := range ts {
		d, ok := idx[t]
		if !ok {
			return nil
		}
		freq = append(freq, len(d))
	}

	sort.Sort(tfList{ts, freq})

	var nonzero int
	for nonzero < len(freq) && freq[nonzero] == 0 {
		nonzero++
	}

	// query consists only of pruned trigrams -- return all documents
	if nonzero == len(freq) {
		return idx[tAllDocIDs]
	}

	ids := idx.Filter(idx[ts[nonzero]], ts[nonzero+1:])

	return ids
}

// FilterOr removes documents that don't contain any of the list of specified trigrams
// in other words, it's the union of the results of invidial filters
func (idx Index) FilterOr(docs []DocID, tss [][]T) []DocID {

	// no provided filter trigrams
	if len(tss) == 0 {
		return docs
	}
	maxDocs := len(docs)

	filtered := idx.Filter(docs, tss[0])

	for i := 1; i < len(tss); i++ {
		// docs can be a live postings list so we can't repurpose that array
		result := make([]DocID, 0, maxDocs)
		out := idx.Filter(docs, tss[i])
		filtered = union(result, filtered, out)
	}
	return filtered
}

// Filter removes documents that don't contain the specified trigrams
func (idx Index) Filter(docs []DocID, ts []T) []DocID {

	// no provided filter trigrams
	if len(ts) == 0 {
		return docs
	}

	// interesting implementation detail:
	// we don't want to repurpose/alter docs since it's typically
	// a live postings list, hence allocating a result slice
	// however, upon subsequent loop runs we do repurpose the input
	// as the output, because at that point its safe for reuse

	result := make([]DocID, len(docs))

	for _, t := range ts {
		d, ok := idx[t]
		// unknown trigram
		if !ok {
			return nil
		}

		if d == nil {
			// the trigram was removed via Prune()
			continue
		}

		result = intersect(result[:0], docs, d)
		docs = result
	}

	return docs
}

// intersect intersects the input slices and puts the output in result slice
// note that result may be backed by the same array as a or b, since
// we only add docs that also exist in both inputs, it's guaranteed that we
// never overwrite/clobber the input, as long as result's start and len are proper
func intersect(result, a, b []DocID) []DocID {

	var aidx, bidx int

scan:
	for aidx < len(a) && bidx < len(b) {
		if a[aidx] == b[bidx] {
			result = append(result, a[aidx])
			aidx++
			bidx++
			if aidx == len(a) || bidx == len(b) {
				break scan
			}
		}

		for a[aidx] < b[bidx] {
			aidx++
			if aidx == len(a) {
				break scan
			}
		}

		for a[aidx] > b[bidx] {
			bidx++
			if bidx == len(b) {
				break scan
			}
		}
	}

	return result
}

// union takes the union of the input slices
// result slice will be used for output
// specifying a result slice backed by the same array as a or b
// is almost always a bad idea and will clobber your input,
// unless you know what you're doing.
func union(result, a, b []DocID) []DocID {

	var aidx, bidx int

scan:
	for aidx < len(a) && bidx < len(b) {
		if a[aidx] == b[bidx] {
			result = append(result, a[aidx])
			aidx++
			bidx++
			if aidx == len(a) || bidx == len(b) {
				break scan
			}
		}

		for a[aidx] < b[bidx] {
			result = append(result, a[aidx])
			aidx++
			if aidx == len(a) {
				break scan
			}
		}

		for a[aidx] > b[bidx] {
			result = append(result, b[bidx])
			bidx++
			if bidx == len(b) {
				break scan
			}
		}
	}
	// we may have broken out because we either finished b, or a, or both
	// processes any remainders
	for aidx < len(a) {
		result = append(result, a[aidx])
		aidx++
	}

	for bidx < len(b) {
		result = append(result, b[bidx])
		bidx++
	}

	return result
}
<commit_msg>export allDocIDs (#18)<commit_after>// Package trigram is a simple trigram index
package trigram

import (
	"sort"
)

// T is a trigram
type T uint32

func (t T) String() string {
	b := [3]byte{byte(t >> 16), byte(t >> 8), byte(t)}
	return string(b[:])
}

// DocID is a document ID
type DocID uint32

// Index is a trigram index
type Index map[T][]DocID

// a special (and invalid) trigram that holds all the document IDs
const TAllDocIDs T = 0xFFFFFFFF

// Extract returns a list of all the unique trigrams in s
func Extract(s string, trigrams []T) []T {

	for i := 0; i <= len(s)-3; i++ {
		t := T(uint32(s[i])<<16 | uint32(s[i+1])<<8 | uint32(s[i+2]))
		trigrams = appendIfUnique(trigrams, t)
	}

	return trigrams
}

// ExtractAll returns a list of all the trigrams in s
func ExtractAll(s string, trigrams []T) []T {

	for i := 0; i <= len(s)-3; i++ {
		t := T(uint32(s[i])<<16 | uint32(s[i+1])<<8 | uint32(s[i+2]))
		trigrams = append(trigrams, t)
	}

	return trigrams
}

func appendIfUnique(t []T, n T) []T {
	for _, v := range t {
		if v == n {
			return t
		}
	}

	return append(t, n)
}

// NewIndex returns an index for the strings in docs
func NewIndex(docs []string) Index {

	idx := make(Index)

	var allDocIDs []DocID

	var trigrams []T

	for id, d := range docs {
		ts := ExtractAll(d, trigrams)
		docid := DocID(id)
		allDocIDs = append(allDocIDs, docid)
		for _, t := range ts {
			idxt := idx[t]
			l := len(idxt)
			if l == 0 || idxt[l-1] != docid {
				idx[t] = append(idxt, docid)
			}
		}
		trigrams = trigrams[:0]
	}

	idx[TAllDocIDs] = allDocIDs

	return idx
}

// Add adds a new string to the search index
func (idx Index) Add(s string) DocID {
	id := DocID(len(idx[TAllDocIDs]))
	idx.Insert(s, id)
	return id
}

// AddTrigrams adds a set of trigrams to the search index
func (idx Index) AddTrigrams(ts []T) DocID {
	id := DocID(len(idx[TAllDocIDs]))
	idx.InsertTrigrams(ts, id)
	return id
}

// Insert adds a string with a given document ID
func (idx Index) Insert(s string, id DocID) {
	ts := ExtractAll(s, nil)
	idx.InsertTrigrams(ts, id)
}

// InsertTrigrams adds a set of trigrams with a given document ID
func (idx Index) InsertTrigrams(ts []T, id DocID) {
	for _, t := range ts {
		idxt, ok := idx[t]
		// this trigram post list has been pruned. we must keep it empty
		if ok && idxt == nil {
			continue
		}
		l := len(idxt)
		if l == 0 || idxt[l-1] != id {
			idx[t] = append(idxt, id)
		}
	}

	idx[TAllDocIDs] = append(idx[TAllDocIDs], id)
}

// Delete removes a document from the index
func (idx Index) Delete(s string, id DocID) {
	ts := ExtractAll(s, nil)
	for _, t := range ts {
		ids := idx[t]
		if ids == nil {
			continue
		}

		if len(ids) == 1 && ids[0] == id {
			delete(idx, t)
			continue
		}

		i := sort.Search(len(ids), func(i int) bool { return ids[i] >= id })

		if i != -1 && i < len(ids) && ids[i] == id {
			copy(ids[i:], ids[i+1:])
			idx[t] = ids[:len(ids)-1]
		}
	}
}

// for sorting
type docList []DocID

func (d docList) Len() int           { return len(d) }
func (d docList) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
func (d docList) Less(i, j int) bool { return d[i] < d[j] }

// Sort ensures all the document IDs are in order.
func (idx Index) Sort() {
	for _, v := range idx {
		dl := docList(v)
		if !sort.IsSorted(dl) {
			sort.Sort(dl)
		}
	}
}

// Prune removes all trigrams that are present in more than the specified percentage of the documents.
func (idx Index) Prune(pct float64) int {

	maxDocs := int(pct * float64(len(idx[TAllDocIDs])))

	var pruned int

	for k, v := range idx {
		if k != TAllDocIDs && len(v) > maxDocs {
			pruned++
			idx[k] = nil
		}
	}

	return pruned
}

// Query returns a list of document IDs that match the trigrams in the query s
func (idx Index) Query(s string) []DocID {
	ts := Extract(s, nil)
	return idx.QueryTrigrams(ts)
}

type tfList struct {
	tri  []T
	freq []int
}

func (tf tfList) Len() int { return len(tf.tri) }
func (tf tfList) Swap(i, j int) {
	tf.tri[i], tf.tri[j] = tf.tri[j], tf.tri[i]
	tf.freq[i], tf.freq[j] = tf.freq[j], tf.freq[i]
}
func (tf tfList) Less(i, j int) bool { return tf.freq[i] < tf.freq[j] }

// QueryTrigrams returns a list of document IDs that match the trigram set ts
func (idx Index) QueryTrigrams(ts []T) []DocID {

	if len(ts) == 0 {
		return idx[TAllDocIDs]
	}

	var freq []int

	for _, t := range ts {
		d, ok := idx[t]
		if !ok {
			return nil
		}
		freq = append(freq, len(d))
	}

	sort.Sort(tfList{ts, freq})

	var nonzero int
	for nonzero < len(freq) && freq[nonzero] == 0 {
		nonzero++
	}

	// query consists only of pruned trigrams -- return all documents
	if nonzero == len(freq) {
		return idx[TAllDocIDs]
	}

	ids := idx.Filter(idx[ts[nonzero]], ts[nonzero+1:])

	return ids
}

// FilterOr removes documents that don't contain any of the list of specified trigrams
// in other words, it's the union of the results of invidial filters
func (idx Index) FilterOr(docs []DocID, tss [][]T) []DocID {

	// no provided filter trigrams
	if len(tss) == 0 {
		return docs
	}
	maxDocs := len(docs)

	filtered := idx.Filter(docs, tss[0])

	for i := 1; i < len(tss); i++ {
		// docs can be a live postings list so we can't repurpose that array
		result := make([]DocID, 0, maxDocs)
		out := idx.Filter(docs, tss[i])
		filtered = union(result, filtered, out)
	}
	return filtered
}

// Filter removes documents that don't contain the specified trigrams
func (idx Index) Filter(docs []DocID, ts []T) []DocID {

	// no provided filter trigrams
	if len(ts) == 0 {
		return docs
	}

	// interesting implementation detail:
	// we don't want to repurpose/alter docs since it's typically
	// a live postings list, hence allocating a result slice
	// however, upon subsequent loop runs we do repurpose the input
	// as the output, because at that point its safe for reuse

	result := make([]DocID, len(docs))

	for _, t := range ts {
		d, ok := idx[t]
		// unknown trigram
		if !ok {
			return nil
		}

		if d == nil {
			// the trigram was removed via Prune()
			continue
		}

		result = intersect(result[:0], docs, d)
		docs = result
	}

	return docs
}

// intersect intersects the input slices and puts the output in result slice
// note that result may be backed by the same array as a or b, since
// we only add docs that also exist in both inputs, it's guaranteed that we
// never overwrite/clobber the input, as long as result's start and len are proper
func intersect(result, a, b []DocID) []DocID {

	var aidx, bidx int

scan:
	for aidx < len(a) && bidx < len(b) {
		if a[aidx] == b[bidx] {
			result = append(result, a[aidx])
			aidx++
			bidx++
			if aidx == len(a) || bidx == len(b) {
				break scan
			}
		}

		for a[aidx] < b[bidx] {
			aidx++
			if aidx == len(a) {
				break scan
			}
		}

		for a[aidx] > b[bidx] {
			bidx++
			if bidx == len(b) {
				break scan
			}
		}
	}

	return result
}

// union takes the union of the input slices
// result slice will be used for output
// specifying a result slice backed by the same array as a or b
// is almost always a bad idea and will clobber your input,
// unless you know what you're doing.
func union(result, a, b []DocID) []DocID {

	var aidx, bidx int

scan:
	for aidx < len(a) && bidx < len(b) {
		if a[aidx] == b[bidx] {
			result = append(result, a[aidx])
			aidx++
			bidx++
			if aidx == len(a) || bidx == len(b) {
				break scan
			}
		}

		for a[aidx] < b[bidx] {
			result = append(result, a[aidx])
			aidx++
			if aidx == len(a) {
				break scan
			}
		}

		for a[aidx] > b[bidx] {
			result = append(result, b[bidx])
			bidx++
			if bidx == len(b) {
				break scan
			}
		}
	}
	// we may have broken out because we either finished b, or a, or both
	// processes any remainders
	for aidx < len(a) {
		result = append(result, a[aidx])
		aidx++
	}

	for bidx < len(b) {
		result = append(result, b[bidx])
		bidx++
	}

	return result
}
<|endoftext|>


<commit_before>package sources

import (
	"io/ioutil"
	"path/filepath"

	"github.com/anaminus/rbxmk"
	"github.com/robloxapi/types"
	lua "github.com/yuin/gopher-lua"
)

func init() { register(File) }
func File() rbxmk.Source {
	return rbxmk.Source{
		Name: "file",
		Read: func(s rbxmk.State) (b []byte, err error) {
			path := string(s.Pull(1, "string").(types.String))
			return ioutil.ReadFile(path)
		},
		Write: func(s rbxmk.State, b []byte) (err error) {
			path := string(s.Pull(1, "string").(types.String))
			return ioutil.WriteFile(path, b, 0666)
		},
		Library: rbxmk.Library{
			Open: func(s rbxmk.State) *lua.LTable {
				lib := s.L.CreateTable(0, 2)
				lib.RawSetString("read", s.WrapFunc(fileRead))
				lib.RawSetString("write", s.WrapFunc(fileWrite))
				return lib
			},
		},
	}
}

func fileRead(s rbxmk.State) int {
	fileName := string(s.Pull(1, "string").(types.String))
	formatName := string(s.PullOpt(2, "string", types.String("")).(types.String))
	if formatName == "" {
		f := s.Ext(fileName)
		if f == "" {
			return s.RaiseError("unknown format from %s", filepath.Base(fileName))
		}
		formatName = f
	}

	format := s.Format(formatName)
	if format.Name == "" {
		return s.RaiseError("unknown format %q", formatName)
	}
	if format.Decode == nil {
		return s.RaiseError("cannot decode with format %s", format.Name)
	}

	b, err := ioutil.ReadFile(fileName)
	if err != nil {
		return s.RaiseError(err.Error())
	}
	v, err := format.Decode(b)
	if err != nil {
		return s.RaiseError(err.Error())
	}
	return s.Push(v)
}

func fileWrite(s rbxmk.State) int {
	var fileName string
	var formatName string
	var value types.Value
	switch s.L.GetTop() {
	case 2:
		fileName = string(s.Pull(1, "string").(types.String))
		value = s.Pull(2, "Variant")
		f := s.Ext(fileName)
		if f == "" {
			return s.RaiseError("unknown format from %s", filepath.Base(fileName))
		}
		formatName = f
	case 3:
		fileName = string(s.Pull(1, "string").(types.String))
		formatName = string(s.Pull(2, "string").(types.String))
		value = s.Pull(3, "Variant")
	default:
		return s.RaiseError("expected 2 or 3 arguments")
	}
	format := s.Format(formatName)
	if format.Name == "" {
		return s.RaiseError("unknown format %q", formatName)
	}
	if format.Encode == nil {
		return s.RaiseError("cannot encode with format %s", format.Name)
	}

	b, err := format.Encode(value)
	if err != nil {
		return s.RaiseError(err.Error())
	}
	if err := ioutil.WriteFile(fileName, b, 0666); err != nil {
		return s.RaiseError(err.Error())
	}
	return 0
}
<commit_msg>Move format argument of file.write to end.<commit_after>package sources

import (
	"io/ioutil"
	"path/filepath"

	"github.com/anaminus/rbxmk"
	"github.com/robloxapi/types"
	lua "github.com/yuin/gopher-lua"
)

func init() { register(File) }
func File() rbxmk.Source {
	return rbxmk.Source{
		Name: "file",
		Read: func(s rbxmk.State) (b []byte, err error) {
			path := string(s.Pull(1, "string").(types.String))
			return ioutil.ReadFile(path)
		},
		Write: func(s rbxmk.State, b []byte) (err error) {
			path := string(s.Pull(1, "string").(types.String))
			return ioutil.WriteFile(path, b, 0666)
		},
		Library: rbxmk.Library{
			Open: func(s rbxmk.State) *lua.LTable {
				lib := s.L.CreateTable(0, 2)
				lib.RawSetString("read", s.WrapFunc(fileRead))
				lib.RawSetString("write", s.WrapFunc(fileWrite))
				return lib
			},
		},
	}
}

func fileRead(s rbxmk.State) int {
	fileName := string(s.Pull(1, "string").(types.String))
	formatName := string(s.PullOpt(2, "string", types.String("")).(types.String))
	if formatName == "" {
		if formatName = s.Ext(fileName); formatName == "" {
			return s.RaiseError("unknown format from %s", filepath.Base(fileName))
		}
	}

	format := s.Format(formatName)
	if format.Name == "" {
		return s.RaiseError("unknown format %q", formatName)
	}
	if format.Decode == nil {
		return s.RaiseError("cannot decode with format %s", format.Name)
	}

	b, err := ioutil.ReadFile(fileName)
	if err != nil {
		return s.RaiseError(err.Error())
	}
	v, err := format.Decode(b)
	if err != nil {
		return s.RaiseError(err.Error())
	}
	return s.Push(v)
}

func fileWrite(s rbxmk.State) int {
	fileName := string(s.Pull(1, "string").(types.String))
	value := s.Pull(2, "Variant")
	formatName := string(s.PullOpt(3, "string", types.String("")).(types.String))
	if formatName == "" {
		if formatName = s.Ext(fileName); formatName == "" {
			return s.RaiseError("unknown format from %s", filepath.Base(fileName))
		}
	}

	format := s.Format(formatName)
	if format.Name == "" {
		return s.RaiseError("unknown format %q", formatName)
	}
	if format.Encode == nil {
		return s.RaiseError("cannot encode with format %s", format.Name)
	}

	b, err := format.Encode(value)
	if err != nil {
		return s.RaiseError(err.Error())
	}
	if err := ioutil.WriteFile(fileName, b, 0666); err != nil {
		return s.RaiseError(err.Error())
	}
	return 0
}
<|endoftext|>


<commit_before>// Copyright 2013 The StudyGolang Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// http://studygolang.com
// Author: polaris	polaris@studygolang.com

package controller

import (
	"html/template"
	"net/http"

	"github.com/studygolang/studygolang/context"
	"github.com/studygolang/studygolang/logic"

	echo "github.com/labstack/echo/v4"
	"github.com/polaris1119/goutils"
	"github.com/polaris1119/slices"
)

type UserController struct{}

// 注册路由
func (self UserController) RegisterRoute(g *echo.Group) {
	g.GET("/user/:username", self.Home)
	g.GET("/user/:username/topics", self.Topics)
	g.GET("/user/:username/articles", self.Articles)
	g.GET("/user/:username/resources", self.Resources)
	g.GET("/user/:username/projects", self.Projects)
	g.GET("/user/:username/comments", self.Comments)
	g.GET("/users", self.ReadList)
	g.Match([]string{"GET", "POST"}, "/user/email/unsubscribe", self.EmailUnsub)
}

// Home 用户个人首页
func (UserController) Home(ctx echo.Context) error {
	username := ctx.Param("username")
	user := logic.DefaultUser.FindOne(context.EchoContext(ctx), "username", username)
	if user == nil || user.Uid == 0 {
		return ctx.Redirect(http.StatusSeeOther, "/users")
	}

	user.Weight = logic.DefaultRank.UserDAURank(context.EchoContext(ctx), user.Uid)

	topics := logic.DefaultTopic.FindRecent(5, user.Uid)

	articles := logic.DefaultArticle.FindByUser(context.EchoContext(ctx), user.Username, 5)

	resources := logic.DefaultResource.FindRecent(context.EchoContext(ctx), user.Uid)
	for _, resource := range resources {
		resource.CatName = logic.GetCategoryName(resource.Catid)
	}

	projects := logic.DefaultProject.FindRecent(context.EchoContext(ctx), user.Username)
	comments := logic.DefaultComment.FindRecent(context.EchoContext(ctx), user.Uid, -1, 5)

	user.IsOnline = logic.Book.RegUserIsOnline(user.Uid)

	return render(ctx, "user/profile.html", map[string]interface{}{
		"activeUsers": "active",
		"topics":      topics,
		"articles":    articles,
		"resources":   resources,
		"projects":    projects,
		"comments":    comments,
		"user":        user,
	})
}

// ReadList 会员列表
func (UserController) ReadList(ctx echo.Context) error {
	// 获取活跃会员
	// activeUsers := logic.DefaultUser.FindActiveUsers(ctx, 36)
	activeUsers := logic.DefaultRank.FindDAURank(context.EchoContext(ctx), 36)
	// 获取最新加入会员
	newUsers := logic.DefaultUser.FindNewUsers(context.EchoContext(ctx), 36)
	// 获取会员总数
	total := logic.DefaultUser.Total()

	return render(ctx, "user/users.html", map[string]interface{}{"activeUsers": "active", "actives": activeUsers, "news": newUsers, "total": total})
}

// EmailUnsub 邮件订阅/退订页面
func (UserController) EmailUnsub(ctx echo.Context) error {
	token := ctx.FormValue("u")
	if token == "" {
		return ctx.Redirect(http.StatusSeeOther, "/")
	}

	// 校验 token 的合法性
	email := ctx.FormValue("email")
	user := logic.DefaultUser.FindOne(context.EchoContext(ctx), "email", email)
	if user.Email == "" {
		return ctx.Redirect(http.StatusSeeOther, "/")
	}

	realToken := logic.DefaultEmail.GenUnsubscribeToken(user)
	if token != realToken {
		return ctx.Redirect(http.StatusSeeOther, "/")
	}

	if ctx.Request().Method != "POST" {
		data := map[string]interface{}{
			"email":       email,
			"token":       token,
			"unsubscribe": user.Unsubscribe,
		}

		return render(ctx, "user/email_unsub.html", data)
	}

	logic.DefaultUser.EmailSubscribe(context.EchoContext(ctx), user.Uid, goutils.MustInt(ctx.FormValue("unsubscribe")))

	return success(ctx, nil)
}

func (UserController) Topics(ctx echo.Context) error {
	username := ctx.Param("username")
	user := logic.DefaultUser.FindOne(context.EchoContext(ctx), "username", username)
	if user == nil || user.Uid == 0 {
		return ctx.Redirect(http.StatusSeeOther, "/users")
	}

	curPage := goutils.MustInt(ctx.QueryParam("p"), 1)
	paginator := logic.NewPaginator(curPage)

	querystring := "uid=?"
	topics := logic.DefaultTopic.FindAll(context.EchoContext(ctx), paginator, "topics.tid DESC", querystring, user.Uid)
	total := logic.DefaultTopic.Count(context.EchoContext(ctx), querystring, user.Uid)
	pageHtml := paginator.SetTotal(total).GetPageHtml(ctx.Request().URL.Path)

	return render(ctx, "user/topics.html", map[string]interface{}{
		"user":         user,
		"activeTopics": "active",
		"topics":       topics,
		"page":         template.HTML(pageHtml),
		"total":        total,
	})
}

func (UserController) Articles(ctx echo.Context) error {
	username := ctx.Param("username")
	user := logic.DefaultUser.FindOne(context.EchoContext(ctx), "username", username)
	if user == nil || user.Uid == 0 {
		return ctx.Redirect(http.StatusSeeOther, "/users")
	}

	curPage := goutils.MustInt(ctx.QueryParam("p"), 1)
	paginator := logic.NewPaginator(curPage)

	querystring := "author_txt=?"
	articles := logic.DefaultArticle.FindAll(context.EchoContext(ctx), paginator, "id DESC", querystring, user.Username)
	total := logic.DefaultArticle.Count(context.EchoContext(ctx), querystring, user.Username)
	pageHtml := paginator.SetTotal(total).GetPageHtml(ctx.Request().URL.Path)

	return render(ctx, "user/articles.html", map[string]interface{}{
		"user":           user,
		"activeArticles": "active",
		"articles":       articles,
		"page":           template.HTML(pageHtml),
		"total":          total,
	})
}

func (UserController) Resources(ctx echo.Context) error {
	username := ctx.Param("username")
	user := logic.DefaultUser.FindOne(context.EchoContext(ctx), "username", username)
	if user == nil || user.Uid == 0 {
		return ctx.Redirect(http.StatusSeeOther, "/users")
	}

	curPage := goutils.MustInt(ctx.QueryParam("p"), 1)
	paginator := logic.NewPaginator(curPage)

	querystring := "uid=?"
	resources, total := logic.DefaultResource.FindAll(context.EchoContext(ctx), paginator, "resource.id DESC", querystring, user.Uid)
	pageHtml := paginator.SetTotal(total).GetPageHtml(ctx.Request().URL.Path)

	return render(ctx, "user/resources.html", map[string]interface{}{
		"user":            user,
		"activeResources": "active",
		"resources":       resources,
		"page":            template.HTML(pageHtml),
		"total":           total,
	})
}

func (UserController) Projects(ctx echo.Context) error {
	username := ctx.Param("username")
	user := logic.DefaultUser.FindOne(context.EchoContext(ctx), "username", username)
	if user == nil || user.Uid == 0 {
		return ctx.Redirect(http.StatusSeeOther, "/users")
	}

	curPage := goutils.MustInt(ctx.QueryParam("p"), 1)
	paginator := logic.NewPaginator(curPage)

	querystring := "username=?"
	projects := logic.DefaultProject.FindAll(context.EchoContext(ctx), paginator, "id DESC", querystring, user.Username)
	total := logic.DefaultProject.Count(context.EchoContext(ctx), querystring, user.Username)
	pageHtml := paginator.SetTotal(total).GetPageHtml(ctx.Request().URL.Path)

	return render(ctx, "user/projects.html", map[string]interface{}{
		"user":           user,
		"activeProjects": "active",
		"projects":       projects,
		"page":           template.HTML(pageHtml),
		"total":          total,
	})
}
func (UserController) Comments(ctx echo.Context) error {

	username := ctx.Param("username")

	userid := 0
	querystring := ""

	if username != "0" {
		user := logic.DefaultUser.FindOne(context.EchoContext(ctx), "username", username)
		if user == nil || user.Uid == 0 {
			return ctx.Redirect(http.StatusSeeOther, "/users")
		}
		querystring = "uid=?"
		userid = user.Uid
		username = user.Username
	} else {
		username = ""
	}

	curPage := goutils.MustInt(ctx.QueryParam("p"), 1)
	paginator := logic.NewPaginator(curPage)

	comments := logic.DefaultComment.FindAll(context.EchoContext(ctx), paginator, "cid DESC", querystring, userid)

	total := logic.DefaultComment.Count(context.EchoContext(ctx), querystring, userid)

	pageHtml := paginator.SetTotal(total).GetPageHtml(ctx.Request().URL.Path)

	data := map[string]interface{}{
		"comments": comments,
		"page":     template.HTML(pageHtml),
		"total":    total,
	}

	if username == "" {
		uids := slices.StructsIntSlice(comments, "Uid")
		data["users"] = logic.DefaultUser.FindUserInfos(context.EchoContext(ctx), uids)
	}

	return render(ctx, "user/comments.html", data)

}
<commit_msg>停号后，个人首页不让查看<commit_after>// Copyright 2013 The StudyGolang Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// http://studygolang.com
// Author: polaris	polaris@studygolang.com

package controller

import (
	"html/template"
	"net/http"

	"github.com/studygolang/studygolang/context"
	"github.com/studygolang/studygolang/logic"
	"github.com/studygolang/studygolang/model"

	echo "github.com/labstack/echo/v4"
	"github.com/polaris1119/goutils"
	"github.com/polaris1119/slices"
)

type UserController struct{}

// 注册路由
func (self UserController) RegisterRoute(g *echo.Group) {
	g.GET("/user/:username", self.Home)
	g.GET("/user/:username/topics", self.Topics)
	g.GET("/user/:username/articles", self.Articles)
	g.GET("/user/:username/resources", self.Resources)
	g.GET("/user/:username/projects", self.Projects)
	g.GET("/user/:username/comments", self.Comments)
	g.GET("/users", self.ReadList)
	g.Match([]string{"GET", "POST"}, "/user/email/unsubscribe", self.EmailUnsub)
}

// Home 用户个人首页
func (UserController) Home(ctx echo.Context) error {
	username := ctx.Param("username")
	user := logic.DefaultUser.FindOne(context.EchoContext(ctx), "username", username)
	if user == nil || user.Uid == 0 || user.Status == model.UserStatusOutage {
		return ctx.Redirect(http.StatusSeeOther, "/users")
	}

	user.Weight = logic.DefaultRank.UserDAURank(context.EchoContext(ctx), user.Uid)

	topics := logic.DefaultTopic.FindRecent(5, user.Uid)

	articles := logic.DefaultArticle.FindByUser(context.EchoContext(ctx), user.Username, 5)

	resources := logic.DefaultResource.FindRecent(context.EchoContext(ctx), user.Uid)
	for _, resource := range resources {
		resource.CatName = logic.GetCategoryName(resource.Catid)
	}

	projects := logic.DefaultProject.FindRecent(context.EchoContext(ctx), user.Username)
	comments := logic.DefaultComment.FindRecent(context.EchoContext(ctx), user.Uid, -1, 5)

	user.IsOnline = logic.Book.RegUserIsOnline(user.Uid)

	return render(ctx, "user/profile.html", map[string]interface{}{
		"activeUsers": "active",
		"topics":      topics,
		"articles":    articles,
		"resources":   resources,
		"projects":    projects,
		"comments":    comments,
		"user":        user,
	})
}

// ReadList 会员列表
func (UserController) ReadList(ctx echo.Context) error {
	// 获取活跃会员
	// activeUsers := logic.DefaultUser.FindActiveUsers(ctx, 36)
	activeUsers := logic.DefaultRank.FindDAURank(context.EchoContext(ctx), 36)
	// 获取最新加入会员
	newUsers := logic.DefaultUser.FindNewUsers(context.EchoContext(ctx), 36)
	// 获取会员总数
	total := logic.DefaultUser.Total()

	return render(ctx, "user/users.html", map[string]interface{}{"activeUsers": "active", "actives": activeUsers, "news": newUsers, "total": total})
}

// EmailUnsub 邮件订阅/退订页面
func (UserController) EmailUnsub(ctx echo.Context) error {
	token := ctx.FormValue("u")
	if token == "" {
		return ctx.Redirect(http.StatusSeeOther, "/")
	}

	// 校验 token 的合法性
	email := ctx.FormValue("email")
	user := logic.DefaultUser.FindOne(context.EchoContext(ctx), "email", email)
	if user.Email == "" {
		return ctx.Redirect(http.StatusSeeOther, "/")
	}

	realToken := logic.DefaultEmail.GenUnsubscribeToken(user)
	if token != realToken {
		return ctx.Redirect(http.StatusSeeOther, "/")
	}

	if ctx.Request().Method != "POST" {
		data := map[string]interface{}{
			"email":       email,
			"token":       token,
			"unsubscribe": user.Unsubscribe,
		}

		return render(ctx, "user/email_unsub.html", data)
	}

	logic.DefaultUser.EmailSubscribe(context.EchoContext(ctx), user.Uid, goutils.MustInt(ctx.FormValue("unsubscribe")))

	return success(ctx, nil)
}

func (UserController) Topics(ctx echo.Context) error {
	username := ctx.Param("username")
	user := logic.DefaultUser.FindOne(context.EchoContext(ctx), "username", username)
	if user == nil || user.Uid == 0 {
		return ctx.Redirect(http.StatusSeeOther, "/users")
	}

	curPage := goutils.MustInt(ctx.QueryParam("p"), 1)
	paginator := logic.NewPaginator(curPage)

	querystring := "uid=?"
	topics := logic.DefaultTopic.FindAll(context.EchoContext(ctx), paginator, "topics.tid DESC", querystring, user.Uid)
	total := logic.DefaultTopic.Count(context.EchoContext(ctx), querystring, user.Uid)
	pageHtml := paginator.SetTotal(total).GetPageHtml(ctx.Request().URL.Path)

	return render(ctx, "user/topics.html", map[string]interface{}{
		"user":         user,
		"activeTopics": "active",
		"topics":       topics,
		"page":         template.HTML(pageHtml),
		"total":        total,
	})
}

func (UserController) Articles(ctx echo.Context) error {
	username := ctx.Param("username")
	user := logic.DefaultUser.FindOne(context.EchoContext(ctx), "username", username)
	if user == nil || user.Uid == 0 {
		return ctx.Redirect(http.StatusSeeOther, "/users")
	}

	curPage := goutils.MustInt(ctx.QueryParam("p"), 1)
	paginator := logic.NewPaginator(curPage)

	querystring := "author_txt=?"
	articles := logic.DefaultArticle.FindAll(context.EchoContext(ctx), paginator, "id DESC", querystring, user.Username)
	total := logic.DefaultArticle.Count(context.EchoContext(ctx), querystring, user.Username)
	pageHtml := paginator.SetTotal(total).GetPageHtml(ctx.Request().URL.Path)

	return render(ctx, "user/articles.html", map[string]interface{}{
		"user":           user,
		"activeArticles": "active",
		"articles":       articles,
		"page":           template.HTML(pageHtml),
		"total":          total,
	})
}

func (UserController) Resources(ctx echo.Context) error {
	username := ctx.Param("username")
	user := logic.DefaultUser.FindOne(context.EchoContext(ctx), "username", username)
	if user == nil || user.Uid == 0 {
		return ctx.Redirect(http.StatusSeeOther, "/users")
	}

	curPage := goutils.MustInt(ctx.QueryParam("p"), 1)
	paginator := logic.NewPaginator(curPage)

	querystring := "uid=?"
	resources, total := logic.DefaultResource.FindAll(context.EchoContext(ctx), paginator, "resource.id DESC", querystring, user.Uid)
	pageHtml := paginator.SetTotal(total).GetPageHtml(ctx.Request().URL.Path)

	return render(ctx, "user/resources.html", map[string]interface{}{
		"user":            user,
		"activeResources": "active",
		"resources":       resources,
		"page":            template.HTML(pageHtml),
		"total":           total,
	})
}

func (UserController) Projects(ctx echo.Context) error {
	username := ctx.Param("username")
	user := logic.DefaultUser.FindOne(context.EchoContext(ctx), "username", username)
	if user == nil || user.Uid == 0 {
		return ctx.Redirect(http.StatusSeeOther, "/users")
	}

	curPage := goutils.MustInt(ctx.QueryParam("p"), 1)
	paginator := logic.NewPaginator(curPage)

	querystring := "username=?"
	projects := logic.DefaultProject.FindAll(context.EchoContext(ctx), paginator, "id DESC", querystring, user.Username)
	total := logic.DefaultProject.Count(context.EchoContext(ctx), querystring, user.Username)
	pageHtml := paginator.SetTotal(total).GetPageHtml(ctx.Request().URL.Path)

	return render(ctx, "user/projects.html", map[string]interface{}{
		"user":           user,
		"activeProjects": "active",
		"projects":       projects,
		"page":           template.HTML(pageHtml),
		"total":          total,
	})
}
func (UserController) Comments(ctx echo.Context) error {

	username := ctx.Param("username")

	userid := 0
	querystring := ""

	if username != "0" {
		user := logic.DefaultUser.FindOne(context.EchoContext(ctx), "username", username)
		if user == nil || user.Uid == 0 {
			return ctx.Redirect(http.StatusSeeOther, "/users")
		}
		querystring = "uid=?"
		userid = user.Uid
		username = user.Username
	} else {
		username = ""
	}

	curPage := goutils.MustInt(ctx.QueryParam("p"), 1)
	paginator := logic.NewPaginator(curPage)

	comments := logic.DefaultComment.FindAll(context.EchoContext(ctx), paginator, "cid DESC", querystring, userid)

	total := logic.DefaultComment.Count(context.EchoContext(ctx), querystring, userid)

	pageHtml := paginator.SetTotal(total).GetPageHtml(ctx.Request().URL.Path)

	data := map[string]interface{}{
		"comments": comments,
		"page":     template.HTML(pageHtml),
		"total":    total,
	}

	if username == "" {
		uids := slices.StructsIntSlice(comments, "Uid")
		data["users"] = logic.DefaultUser.FindUserInfos(context.EchoContext(ctx), uids)
	}

	return render(ctx, "user/comments.html", data)

}
<|endoftext|>


<commit_before>package opentsdbfirehosenozzle_test

import (
	"encoding/json"
	"fmt"
	"github.com/cloudfoundry/sonde-go/events"
	"github.com/gogo/protobuf/proto"
	"github.com/gorilla/websocket"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/pivotal-cloudops/opentsdb-firehose-nozzle/nozzleconfig"
	"github.com/pivotal-cloudops/opentsdb-firehose-nozzle/opentsdbclient"
	"github.com/pivotal-cloudops/opentsdb-firehose-nozzle/opentsdbfirehosenozzle"
	. "github.com/pivotal-cloudops/opentsdb-firehose-nozzle/testhelpers"
	"github.com/pivotal-cloudops/opentsdb-firehose-nozzle/uaatokenfetcher"
	"log"
	"strings"
)

var _ = Describe("OpenTSDB Firehose Nozzle", func() {
	var fakeUAA *FakeUAA
	var fakeFirehose *FakeFirehose
	var fakeOpenTSDB *FakeOpenTSDB
	var config *nozzleconfig.NozzleConfig
	var nozzle *opentsdbfirehosenozzle.OpenTSDBFirehoseNozzle
	var logOutput *gbytes.Buffer

	BeforeEach(func() {
		fakeUAA = NewFakeUAA("bearer", "123456789")
		fakeToken := fakeUAA.AuthToken()
		fakeFirehose = NewFakeFirehose(fakeToken)
		fakeOpenTSDB = NewFakeOpenTSDB()

		fakeUAA.Start()
		fakeFirehose.Start()
		fakeOpenTSDB.Start()

		tokenFetcher := &uaatokenfetcher.UAATokenFetcher{
			UaaUrl: fakeUAA.URL(),
		}

		config = &nozzleconfig.NozzleConfig{
			UAAURL:               fakeUAA.URL(),
			OpenTSDBURL:          fakeOpenTSDB.URL(),
			FlushDurationSeconds: 10,
			TrafficControllerURL: strings.Replace(fakeFirehose.URL(), "http:", "ws:", 1),
			DisableAccessControl: false,
			MetricPrefix:         "opentsdb.nozzle.",
			MaxBufferSize:        50,
		}

		logOutput = gbytes.NewBuffer()
		log.SetOutput(logOutput)
		nozzle = opentsdbfirehosenozzle.NewOpenTSDBFirehoseNozzle(config, tokenFetcher)
	})

	AfterEach(func() {
		fakeUAA.Close()
		fakeFirehose.Close()
		fakeOpenTSDB.Close()
	})

	It("sends metrics when there are more than 50", func(done Done) {
		defer close(done)
		fakeFirehose.KeepConnectionAlive()

		for i := 0; i < 51; i++ {
			envelope := events.Envelope{
				Origin:    proto.String("origin"),
				Timestamp: proto.Int64(1000000000),
				EventType: events.Envelope_ValueMetric.Enum(),
				ValueMetric: &events.ValueMetric{
					Name:  proto.String(fmt.Sprintf("metricName-%d", i)),
					Value: proto.Float64(float64(i)),
					Unit:  proto.String("gauge"),
				},
				Deployment: proto.String("deployment-name"),
				Job:        proto.String("doppler"),
			}
			fakeFirehose.AddEvent(envelope)
		}

		go nozzle.Start()

		var contents []byte
		Eventually(fakeOpenTSDB.ReceivedContents).Should(Receive(&contents))

		var metrics []opentsdbclient.Metric
		err := json.Unmarshal(contents, &metrics)
		Expect(err).ToNot(HaveOccurred())

		Expect(logOutput).ToNot(gbytes.Say("Error while reading from the firehose"))

		// +3 internal metrics that show totalMessagesReceived, totalMetricSent, and slowConsumerAlert
		Expect(metrics).To(HaveLen(54))
		fakeFirehose.CloseAliveConnection()
	})

	It("receives data from the firehose", func(done Done) {
		defer close(done)

		for i := 0; i < 10; i++ {
			envelope := events.Envelope{
				Origin:    proto.String("origin"),
				Timestamp: proto.Int64(1000000000),
				EventType: events.Envelope_ValueMetric.Enum(),
				ValueMetric: &events.ValueMetric{
					Name:  proto.String(fmt.Sprintf("metricName-%d", i)),
					Value: proto.Float64(float64(i)),
					Unit:  proto.String("gauge"),
				},
				Deployment: proto.String("deployment-name"),
				Job:        proto.String("doppler"),
			}
			fakeFirehose.AddEvent(envelope)
		}

		go nozzle.Start()

		var contents []byte
		Eventually(fakeOpenTSDB.ReceivedContents).Should(Receive(&contents))

		var metrics []opentsdbclient.Metric
		err := json.Unmarshal(contents, &metrics)
		Expect(err).ToNot(HaveOccurred())

		Expect(logOutput).ToNot(gbytes.Say("Error while reading from the firehose"))

		// +3 internal metrics that show totalMessagesReceived, totalMetricSent, and slowConsumerAlert
		Expect(metrics).To(HaveLen(13))

	}, 2)

	It("sends a server disconnected metric when the server disconnects abnormally", func(done Done) {
		defer close(done)

		for i := 0; i < 10; i++ {
			envelope := events.Envelope{
				Origin:    proto.String("origin"),
				Timestamp: proto.Int64(1000000000),
				EventType: events.Envelope_ValueMetric.Enum(),
				ValueMetric: &events.ValueMetric{
					Name:  proto.String(fmt.Sprintf("metricName-%d", i)),
					Value: proto.Float64(float64(i)),
					Unit:  proto.String("gauge"),
				},
				Deployment: proto.String("deployment-name"),
				Job:        proto.String("doppler"),
			}
			fakeFirehose.AddEvent(envelope)
		}

		fakeFirehose.SetCloseMessage(websocket.FormatCloseMessage(websocket.CloseInternalServerErr, "Client did not respond to ping before keep-alive timeout expired."))

		go nozzle.Start()

		var contents []byte
		Eventually(fakeOpenTSDB.ReceivedContents).Should(Receive(&contents))

		var metrics []opentsdbclient.Metric
		err := json.Unmarshal(contents, &metrics)
		Expect(err).ToNot(HaveOccurred())

		slowConsumerMetric := findSlowConsumerMetric(metrics)
		Expect(slowConsumerMetric).NotTo(BeNil())
		Expect(slowConsumerMetric.Value).To(BeEquivalentTo(1))

		Expect(logOutput).To(gbytes.Say("Error while reading from the firehose"))
		Expect(logOutput).To(gbytes.Say("Client did not respond to ping before keep-alive timeout expired."))
		Expect(logOutput).To(gbytes.Say("Disconnected because nozzle couldn't keep up."))
	}, 2)

	It("does not report slow consumer error when closed for other reasons", func(done Done) {
		defer close(done)

		fakeFirehose.SetCloseMessage(websocket.FormatCloseMessage(websocket.CloseInvalidFramePayloadData, "Weird things happened."))

		go nozzle.Start()

		var contents []byte
		Eventually(fakeOpenTSDB.ReceivedContents).Should(Receive(&contents))

		var metrics []opentsdbclient.Metric
		err := json.Unmarshal(contents, &metrics)
		Expect(err).ToNot(HaveOccurred())

		errMetric := findSlowConsumerMetric(metrics)
		Expect(errMetric).NotTo(BeNil())
		Expect(errMetric.Value).To(BeEquivalentTo(0))

		Expect(logOutput).To(gbytes.Say("Error while reading from the firehose"))
		Expect(logOutput).NotTo(gbytes.Say("Client did not respond to ping before keep-alive timeout expired."))
		Expect(logOutput).NotTo(gbytes.Say("Disconnected because nozzle couldn't keep up."))
	}, 2)

	It("gets a valid authentication token", func() {
		go nozzle.Start()
		Eventually(fakeFirehose.Requested).Should(BeTrue())
		Consistently(fakeFirehose.LastAuthorization).Should(Equal("bearer 123456789"))
	})

	Context("receives a truncatingbuffer.droppedmessage value metric,", func() {
		It("sets a slow-consumer error", func() {
			slowConsumerError := events.Envelope{
				Origin:    proto.String("doppler"),
				Timestamp: proto.Int64(1000000000),
				EventType: events.Envelope_CounterEvent.Enum(),
				CounterEvent: &events.CounterEvent{
					Name:  proto.String("TruncatingBuffer.DroppedMessages"),
					Delta: proto.Uint64(1),
					Total: proto.Uint64(1),
				},
				Deployment: proto.String("deployment-name"),
				Job:        proto.String("doppler"),
			}
			fakeFirehose.AddEvent(slowConsumerError)

			go nozzle.Start()

			var contents []byte
			Eventually(fakeOpenTSDB.ReceivedContents).Should(Receive(&contents))

			var metrics []opentsdbclient.Metric
			err := json.Unmarshal(contents, &metrics)
			Expect(err).ToNot(HaveOccurred())

			Expect(findSlowConsumerMetric(metrics)).NotTo(BeNil())

			Expect(logOutput).To(gbytes.Say("We've intercepted an upstream message which indicates that the nozzle or the TrafficController is not keeping up. Please try scaling up the nozzle."))
		})
	})

	Context("when the DisableAccessControl is set to true", func() {
		var tokenFetcher *FakeTokenFetcher

		BeforeEach(func() {
			fakeUAA = NewFakeUAA("", "")
			fakeToken := fakeUAA.AuthToken()
			fakeFirehose = NewFakeFirehose(fakeToken)
			fakeOpenTSDB = NewFakeOpenTSDB()
			tokenFetcher = &FakeTokenFetcher{}

			fakeUAA.Start()
			fakeFirehose.Start()
			fakeOpenTSDB.Start()

			config = &nozzleconfig.NozzleConfig{
				FlushDurationSeconds: 1,
				OpenTSDBURL:          fakeOpenTSDB.URL(),
				TrafficControllerURL: strings.Replace(fakeFirehose.URL(), "http:", "ws:", 1),
				DisableAccessControl: true,
			}

			nozzle = opentsdbfirehosenozzle.NewOpenTSDBFirehoseNozzle(config, tokenFetcher)
		})

		AfterEach(func() {
			fakeUAA.Close()
			fakeFirehose.Close()
			fakeOpenTSDB.Close()
		})

		It("still tries to connect to the firehose", func() {
			go nozzle.Start()
			Eventually(fakeFirehose.Requested).Should(BeTrue())
		})

		It("gets an empty authentication token", func() {
			go nozzle.Start()
			Consistently(fakeUAA.Requested).Should(Equal(false))
			Consistently(fakeFirehose.LastAuthorization).Should(Equal(""))
		})

		It("does not rquire the presence of config.UAAURL", func() {
			nozzle.Start()
			Consistently(func() int { return tokenFetcher.NumCalls }).Should(Equal(0))
		})
	})
})

func findSlowConsumerMetric(metrics []opentsdbclient.Metric) *opentsdbclient.Metric {
	for _, metric := range metrics {
		if metric.Metric == "opentsdb.nozzle.slowConsumerAlert" {
			return &metric
		}
	}
	return nil
}
<commit_msg>Adding tests for max buffer size and flush duration<commit_after>package opentsdbfirehosenozzle_test

import (
	"encoding/json"
	"fmt"
	"github.com/cloudfoundry/sonde-go/events"
	"github.com/gogo/protobuf/proto"
	"github.com/gorilla/websocket"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/pivotal-cloudops/opentsdb-firehose-nozzle/nozzleconfig"
	"github.com/pivotal-cloudops/opentsdb-firehose-nozzle/opentsdbclient"
	"github.com/pivotal-cloudops/opentsdb-firehose-nozzle/opentsdbfirehosenozzle"
	. "github.com/pivotal-cloudops/opentsdb-firehose-nozzle/testhelpers"
	"github.com/pivotal-cloudops/opentsdb-firehose-nozzle/uaatokenfetcher"
	"log"
	"strings"
)

var _ = Describe("OpenTSDB Firehose Nozzle", func() {
	var fakeUAA *FakeUAA
	var fakeFirehose *FakeFirehose
	var fakeOpenTSDB *FakeOpenTSDB
	var config *nozzleconfig.NozzleConfig
	var nozzle *opentsdbfirehosenozzle.OpenTSDBFirehoseNozzle
	var logOutput *gbytes.Buffer
	var tokenFetcher *uaatokenfetcher.UAATokenFetcher

	BeforeEach(func() {
		fakeUAA = NewFakeUAA("bearer", "123456789")
		fakeToken := fakeUAA.AuthToken()
		fakeFirehose = NewFakeFirehose(fakeToken)
		fakeOpenTSDB = NewFakeOpenTSDB()

		fakeUAA.Start()
		fakeFirehose.Start()
		fakeOpenTSDB.Start()

		tokenFetcher = &uaatokenfetcher.UAATokenFetcher{
			UaaUrl: fakeUAA.URL(),
		}

		config = &nozzleconfig.NozzleConfig{
			UAAURL:               fakeUAA.URL(),
			OpenTSDBURL:          fakeOpenTSDB.URL(),
			FlushDurationSeconds: 10,
			TrafficControllerURL: strings.Replace(fakeFirehose.URL(), "http:", "ws:", 1),
			DisableAccessControl: false,
			MetricPrefix:         "opentsdb.nozzle.",
			MaxBufferSize:        50,
		}

		logOutput = gbytes.NewBuffer()
		log.SetOutput(logOutput)
		nozzle = opentsdbfirehosenozzle.NewOpenTSDBFirehoseNozzle(config, tokenFetcher)
	})

	AfterEach(func() {
		fakeUAA.Close()
		fakeFirehose.Close()
		fakeOpenTSDB.Close()
	})

	It("sends metrics when the FlushDurationTicker ticks", func(done Done) {
		defer close(done)
		fakeFirehose.KeepConnectionAlive()
		defer fakeFirehose.CloseAliveConnection()

		config.FlushDurationSeconds = 1
		nozzle = opentsdbfirehosenozzle.NewOpenTSDBFirehoseNozzle(config, tokenFetcher)

		envelope := events.Envelope{
			Origin:    proto.String("origin"),
			Timestamp: proto.Int64(1000000000),
			EventType: events.Envelope_ValueMetric.Enum(),
			ValueMetric: &events.ValueMetric{
				Name:  proto.String("metricName"),
				Value: proto.Float64(float64(1)),
				Unit:  proto.String("gauge"),
			},
			Deployment: proto.String("deployment-name"),
			Job:        proto.String("doppler"),
		}
		fakeFirehose.AddEvent(envelope)

		go nozzle.Start()

		var contents []byte
		Eventually(fakeOpenTSDB.ReceivedContents, 2).Should(Receive(&contents))

		var metrics []opentsdbclient.Metric
		err := json.Unmarshal(contents, &metrics)
		Expect(err).ToNot(HaveOccurred())

		Expect(logOutput).ToNot(gbytes.Say("Error while reading from the firehose"))

		// +3 internal metrics that show totalMessagesReceived, totalMetricSent, and slowConsumerAlert
		Expect(metrics).To(HaveLen(4))
	}, 3)

	It("sends metrics when there are more messages than buffersize", func(done Done) {
		defer close(done)
		fakeFirehose.KeepConnectionAlive()
		defer fakeFirehose.CloseAliveConnection()

		config.MaxBufferSize = 30
		nozzle = opentsdbfirehosenozzle.NewOpenTSDBFirehoseNozzle(config, tokenFetcher)

		for i := 0; i < 31; i++ {
			envelope := events.Envelope{
				Origin:    proto.String("origin"),
				Timestamp: proto.Int64(1000000000),
				EventType: events.Envelope_ValueMetric.Enum(),
				ValueMetric: &events.ValueMetric{
					Name:  proto.String(fmt.Sprintf("metricName-%d", i)),
					Value: proto.Float64(float64(i)),
					Unit:  proto.String("gauge"),
				},
				Deployment: proto.String("deployment-name"),
				Job:        proto.String("doppler"),
			}
			fakeFirehose.AddEvent(envelope)
		}

		go nozzle.Start()

		var contents []byte
		Eventually(fakeOpenTSDB.ReceivedContents).Should(Receive(&contents))

		var metrics []opentsdbclient.Metric
		err := json.Unmarshal(contents, &metrics)
		Expect(err).ToNot(HaveOccurred())

		Expect(logOutput).ToNot(gbytes.Say("Error while reading from the firehose"))

		// +3 internal metrics that show totalMessagesReceived, totalMetricSent, and slowConsumerAlert
		Expect(metrics).To(HaveLen(34))
	})

	It("receives data from the firehose", func(done Done) {
		defer close(done)

		for i := 0; i < 10; i++ {
			envelope := events.Envelope{
				Origin:    proto.String("origin"),
				Timestamp: proto.Int64(1000000000),
				EventType: events.Envelope_ValueMetric.Enum(),
				ValueMetric: &events.ValueMetric{
					Name:  proto.String(fmt.Sprintf("metricName-%d", i)),
					Value: proto.Float64(float64(i)),
					Unit:  proto.String("gauge"),
				},
				Deployment: proto.String("deployment-name"),
				Job:        proto.String("doppler"),
			}
			fakeFirehose.AddEvent(envelope)
		}

		go nozzle.Start()

		var contents []byte
		Eventually(fakeOpenTSDB.ReceivedContents).Should(Receive(&contents))

		var metrics []opentsdbclient.Metric
		err := json.Unmarshal(contents, &metrics)
		Expect(err).ToNot(HaveOccurred())

		Expect(logOutput).ToNot(gbytes.Say("Error while reading from the firehose"))

		// +3 internal metrics that show totalMessagesReceived, totalMetricSent, and slowConsumerAlert
		Expect(metrics).To(HaveLen(13))

	}, 2)

	It("sends a server disconnected metric when the server disconnects abnormally", func(done Done) {
		defer close(done)

		for i := 0; i < 10; i++ {
			envelope := events.Envelope{
				Origin:    proto.String("origin"),
				Timestamp: proto.Int64(1000000000),
				EventType: events.Envelope_ValueMetric.Enum(),
				ValueMetric: &events.ValueMetric{
					Name:  proto.String(fmt.Sprintf("metricName-%d", i)),
					Value: proto.Float64(float64(i)),
					Unit:  proto.String("gauge"),
				},
				Deployment: proto.String("deployment-name"),
				Job:        proto.String("doppler"),
			}
			fakeFirehose.AddEvent(envelope)
		}

		fakeFirehose.SetCloseMessage(websocket.FormatCloseMessage(websocket.CloseInternalServerErr, "Client did not respond to ping before keep-alive timeout expired."))

		go nozzle.Start()

		var contents []byte
		Eventually(fakeOpenTSDB.ReceivedContents).Should(Receive(&contents))

		var metrics []opentsdbclient.Metric
		err := json.Unmarshal(contents, &metrics)
		Expect(err).ToNot(HaveOccurred())

		slowConsumerMetric := findSlowConsumerMetric(metrics)
		Expect(slowConsumerMetric).NotTo(BeNil())
		Expect(slowConsumerMetric.Value).To(BeEquivalentTo(1))

		Expect(logOutput).To(gbytes.Say("Error while reading from the firehose"))
		Expect(logOutput).To(gbytes.Say("Client did not respond to ping before keep-alive timeout expired."))
		Expect(logOutput).To(gbytes.Say("Disconnected because nozzle couldn't keep up."))
	}, 2)

	It("does not report slow consumer error when closed for other reasons", func(done Done) {
		defer close(done)

		fakeFirehose.SetCloseMessage(websocket.FormatCloseMessage(websocket.CloseInvalidFramePayloadData, "Weird things happened."))

		go nozzle.Start()

		var contents []byte
		Eventually(fakeOpenTSDB.ReceivedContents).Should(Receive(&contents))

		var metrics []opentsdbclient.Metric
		err := json.Unmarshal(contents, &metrics)
		Expect(err).ToNot(HaveOccurred())

		errMetric := findSlowConsumerMetric(metrics)
		Expect(errMetric).NotTo(BeNil())
		Expect(errMetric.Value).To(BeEquivalentTo(0))

		Expect(logOutput).To(gbytes.Say("Error while reading from the firehose"))
		Expect(logOutput).NotTo(gbytes.Say("Client did not respond to ping before keep-alive timeout expired."))
		Expect(logOutput).NotTo(gbytes.Say("Disconnected because nozzle couldn't keep up."))
	}, 2)

	It("gets a valid authentication token", func() {
		go nozzle.Start()
		Eventually(fakeFirehose.Requested).Should(BeTrue())
		Consistently(fakeFirehose.LastAuthorization).Should(Equal("bearer 123456789"))
	})

	Context("receives a truncatingbuffer.droppedmessage value metric,", func() {
		It("sets a slow-consumer error", func() {
			slowConsumerError := events.Envelope{
				Origin:    proto.String("doppler"),
				Timestamp: proto.Int64(1000000000),
				EventType: events.Envelope_CounterEvent.Enum(),
				CounterEvent: &events.CounterEvent{
					Name:  proto.String("TruncatingBuffer.DroppedMessages"),
					Delta: proto.Uint64(1),
					Total: proto.Uint64(1),
				},
				Deployment: proto.String("deployment-name"),
				Job:        proto.String("doppler"),
			}
			fakeFirehose.AddEvent(slowConsumerError)

			go nozzle.Start()

			var contents []byte
			Eventually(fakeOpenTSDB.ReceivedContents).Should(Receive(&contents))

			var metrics []opentsdbclient.Metric
			err := json.Unmarshal(contents, &metrics)
			Expect(err).ToNot(HaveOccurred())

			Expect(findSlowConsumerMetric(metrics)).NotTo(BeNil())

			Expect(logOutput).To(gbytes.Say("We've intercepted an upstream message which indicates that the nozzle or the TrafficController is not keeping up. Please try scaling up the nozzle."))
		})
	})

	Context("when the DisableAccessControl is set to true", func() {
		var tokenFetcher *FakeTokenFetcher

		BeforeEach(func() {
			fakeUAA = NewFakeUAA("", "")
			fakeToken := fakeUAA.AuthToken()
			fakeFirehose = NewFakeFirehose(fakeToken)
			fakeOpenTSDB = NewFakeOpenTSDB()
			tokenFetcher = &FakeTokenFetcher{}

			fakeUAA.Start()
			fakeFirehose.Start()
			fakeOpenTSDB.Start()

			config = &nozzleconfig.NozzleConfig{
				FlushDurationSeconds: 1,
				OpenTSDBURL:          fakeOpenTSDB.URL(),
				TrafficControllerURL: strings.Replace(fakeFirehose.URL(), "http:", "ws:", 1),
				DisableAccessControl: true,
			}

			nozzle = opentsdbfirehosenozzle.NewOpenTSDBFirehoseNozzle(config, tokenFetcher)
		})

		AfterEach(func() {
			fakeUAA.Close()
			fakeFirehose.Close()
			fakeOpenTSDB.Close()
		})

		It("still tries to connect to the firehose", func() {
			go nozzle.Start()
			Eventually(fakeFirehose.Requested).Should(BeTrue())
		})

		It("gets an empty authentication token", func() {
			go nozzle.Start()
			Consistently(fakeUAA.Requested).Should(Equal(false))
			Consistently(fakeFirehose.LastAuthorization).Should(Equal(""))
		})

		It("does not rquire the presence of config.UAAURL", func() {
			nozzle.Start()
			Consistently(func() int { return tokenFetcher.NumCalls }).Should(Equal(0))
		})
	})
})

func findSlowConsumerMetric(metrics []opentsdbclient.Metric) *opentsdbclient.Metric {
	for _, metric := range metrics {
		if metric.Metric == "opentsdb.nozzle.slowConsumerAlert" {
			return &metric
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/yunge/sphinx"
        "fmt"
"time"
"strconv"
//	"github.com/yvasiyarov/newrelic_platform_go"
)

const (
    MIN_PAUSE_TIME = 30
)

type SphinxStatusData map[string]string
type MetricsDataSource struct {
	SphinxHost        string
	Port              int
	ConnectionTimeout int

        PreviousData SphinxStatusData
        LastData     SphinxStatusData
        LastUpdateTime time.Time
}

func NewMetricsDataSource(sphinxHost string, port int, connectionTimeout int) *MetricsDataSource {
	if port == 0 {
		port = 9312
	}
	ds := &MetricsDataSource{
		SphinxHost:        sphinxHost,
		Port:              port,
		ConnectionTimeout: connectionTimeout,
	}
	return ds
}

func (ds *MetricsDataSource) GetData(key string) (float64, error) {
    if err := ds.CheckAndUpdateData(); err != nil {
        return 0, err
    }

    prev, last, err := ds.GetOriginalData(key)

    if err != nil {
        return 0, err
    }
    return last - prev, nil
}

func (ds *MetricsDataSource) GetOriginalData(key string) (float64, float64, error) {
    previousValue, ok := ds.PreviousData[key]
    if !ok {
        return 0, 0, fmt.Errorf("Can not get data from source \n")
    }
    currentValue, ok := ds.LastData[key]
    if !ok {
        return 0, 0, fmt.Errorf("Can not get data from source \n")
    }

    previousValueConverted, err := strconv.ParseFloat(previousValue, 64)
    if err != nil {
        return 0, 0, fmt.Errorf("Can not convert previous value of %s to int \n", key)
    }
    currentValueConverted, err := strconv.ParseFloat(currentValue, 64)
    if err != nil {
        return 0, 0, fmt.Errorf("Can not convert current value of %s to int \n", key)
    }

    return previousValueConverted, currentValueConverted, nil
}

func (ds *MetricsDataSource) CheckAndUpdateData() error {
        startTime := time.Now()
        if startTime.Sub(ds.LastUpdateTime) > time.Second * MIN_PAUSE_TIME {
            newData, err := ds.QueryData()
            if err != nil {
                return err
            }

            if ds.PreviousData == nil {
                ds.PreviousData = newData
            } else {
                ds.PreviousData = ds.LastData
            }
            ds.LastData = newData
        }

        // check uptime
        //If uptime is less then in previous run - then server were restarted
        if prev, last, err := ds.GetOriginalData("uptime"); err != nil {
            return err
        } else {
            if last < prev {
               ds.PreviousData = ds.LastData
            }
        }
        return nil
}

func (ds *MetricsDataSource) QueryData() (SphinxStatusData, error) {
	client := sphinx.NewClient().SetServer(ds.SphinxHost, ds.Port)

	if ds.ConnectionTimeout != 0 {
		client.SetConnectTimeout(ds.ConnectionTimeout)
	}
	if err := client.Error(); err != nil {
		return nil, err
	}

	defer client.Close()
	status, err := client.Status()
	if err != nil {
		return nil, err
	}

        data := make(SphinxStatusData, len(status))
	for _, row := range status {
		data[row[0]] = row[1]
	}

	return data, nil
}

/*
type Metrica struct {
	Name       string
	Units      string
	DataKey    string
	Datasource *MetricsDataSource
}

func (metrica *Metrica) GetName() string {
	return metrica.Name
}
func (metrica *WaveMetrica) GetUnits() string {
	return metrica.Units
}
func (metrica *WaveMetrica) GetValue() (float64, error) {
	metrica.sawtoothCounter++
	if metrica.sawtoothCounter > metrica.sawtoothMax {
		metrica.sawtoothCounter = 0
	}
	return float64(metrica.sawtoothCounter), nil
}
*/

func main() {
	/*
		plugin := newrelic_platform_go.NewNewrelicPlugin("0.0.1", "7bceac019c7dcafae1ef95be3e3a3ff8866de246", 60)
		component := newrelic_platform_go.NewPluginComponent("Sphinx component", "com.github.yvasiyarov.Sphinx")
		plugin.AddComponent(component)

		m := &WaveMetrica{
			sawtoothMax:     10,
			sawtoothCounter: 5,
		}

		component.AddMetrica(m)
		plugin.Verbose = true
		plugin.Run()
	*/
	ds := NewMetricsDataSource("web-d5.butik.ru", 0, 0)
	v1, err := ds.GetData("command_search")
        fmt.Printf("V:%v err: %v\n", v1, err)
        fmt.Printf("DS %#v\n", ds)
}
<commit_msg>Code finished but not worked<commit_after>package main

import (
	"github.com/yunge/sphinx"
        "fmt"
"time"
"strconv"
	"github.com/yvasiyarov/newrelic_platform_go"
)

const (
    MIN_PAUSE_TIME = 30
)

type SphinxStatusData map[string]string
type MetricsDataSource struct {
	SphinxHost        string
	Port              int
	ConnectionTimeout int

        PreviousData SphinxStatusData
        LastData     SphinxStatusData
        LastUpdateTime time.Time
}

func NewMetricsDataSource(sphinxHost string, port int, connectionTimeout int) *MetricsDataSource {
	if port == 0 {
		port = 9312
	}
	ds := &MetricsDataSource{
		SphinxHost:        sphinxHost,
		Port:              port,
		ConnectionTimeout: connectionTimeout,
	}
	return ds
}

func (ds *MetricsDataSource) GetData(key string) (float64, error) {
    if err := ds.CheckAndUpdateData(); err != nil {
        return 0, err
    }

    prev, last, err := ds.GetOriginalData(key)

    if err != nil {
        return 0, err
    }
    return last - prev, nil
}

func (ds *MetricsDataSource) GetOriginalData(key string) (float64, float64, error) {
    previousValue, ok := ds.PreviousData[key]
    if !ok {
        return 0, 0, fmt.Errorf("Can not get data from source \n")
    }
    currentValue, ok := ds.LastData[key]
    if !ok {
        return 0, 0, fmt.Errorf("Can not get data from source \n")
    }

    //some metric calculation can be turned off by sphinx settings
    if previousValue == "OFF" || currentValue == "OFF" {
        return 0, 0, nil
    }

    previousValueConverted, err := strconv.ParseFloat(previousValue, 64)
    if err != nil {
        return 0, 0, fmt.Errorf("Can not convert previous value of %s to int \n", key)
    }
    currentValueConverted, err := strconv.ParseFloat(currentValue, 64)
    if err != nil {
        return 0, 0, fmt.Errorf("Can not convert current value of %s to int \n", key)
    }

    return previousValueConverted, currentValueConverted, nil
}

func (ds *MetricsDataSource) CheckAndUpdateData() error {
        startTime := time.Now()
        if startTime.Sub(ds.LastUpdateTime) > time.Second * MIN_PAUSE_TIME {
            newData, err := ds.QueryData()
            if err != nil {
                return err
            }

            if ds.PreviousData == nil {
                ds.PreviousData = newData
            } else {
                ds.PreviousData = ds.LastData
            }
            ds.LastData = newData
        }

        // check uptime
        //If uptime is less then in previous run - then server were restarted
        if prev, last, err := ds.GetOriginalData("uptime"); err != nil {
            return err
        } else {
            if last < prev {
               ds.PreviousData = ds.LastData
            }
        }
        return nil
}

func (ds *MetricsDataSource) QueryData() (SphinxStatusData, error) {
	client := sphinx.NewClient().SetServer(ds.SphinxHost, ds.Port)

	if ds.ConnectionTimeout != 0 {
		client.SetConnectTimeout(ds.ConnectionTimeout)
	}
	if err := client.Error(); err != nil {
		return nil, err
	}

	defer client.Close()
	status, err := client.Status()
	if err != nil {
		return nil, err
	}

        data := make(SphinxStatusData, len(status))
	for _, row := range status {
		data[row[0]] = row[1]
	}

	return data, nil
}

type Metrica struct {
	Name       string
	Units      string
	DataKey    string
	DataSource *MetricsDataSource
}

func (metrica *Metrica) GetName() string {
	return metrica.Name
}
func (metrica *Metrica) GetUnits() string {
	return metrica.Units
}
func (metrica *Metrica) GetValue() (float64, error) {
    return metrica.DataSource.GetData(metrica.DataKey)
}

func AddMetrcas(component newrelic_platform_go.IComponent, dataSource *MetricsDataSource) {
    keys := []string{
       "connections", "maxed_out", "command_search", "command_excerpt", "command_update", "command_keywords",
       "command_persist", "command_status", "command_flushattrs", "agent_connect", "agent_retry", "queries",
       "dist_queries", "query_wall", "query_cpu", "dist_wall", "dist_local", "dist_wait", "query_reads",
       "query_readkb", "query_readtime", "avg_query_wall", "avg_query_cpu", "avg_dist_wall", "avg_dist_local",
       "avg_dist_wait", "avg_query_reads", "avg_query_readkb", "avg_query_readtime",
    }

    names := []string{
       "Connections", "Maxed out", "Command search", "Command excerpt", "Command update", "Command keywords",
       "Command persist", "Command status", "Command flushattrs", "Agent connect", "Agent retry", "Queries",
       "Dist queries", "Query wall", "Query cpu", "Dist wall", "Dist local", "Dist wait", "Query reads",
       "Query readkb", "Query readtime", "Avg query wall", "Avg query cpu", "Avg dist wall", "Avg dist local",
       "Avg dist wait", "Avg query reads", "Avg query readkb", "Avg query readtime",
    }
    for i, key := range keys {
        m := &Metrica{
            DataKey: key,
            Name: names[i],
            Units: "val/second",
            DataSource: dataSource,
        }
        component.AddMetrica(m)
    }
}

func main() {
        plugin := newrelic_platform_go.NewNewrelicPlugin("0.0.1", "7bceac019c7dcafae1ef95be3e3a3ff8866de246", 60)
        component := newrelic_platform_go.NewPluginComponent("Sphinx component", "com.github.yvasiyarov.Sphinx")
        plugin.AddComponent(component)

	ds := NewMetricsDataSource("web-d5.butik.ru", 0, 0)
        AddMetrcas(component, ds)

        plugin.Harvest()
        plugin.Harvest()
}
<|endoftext|>


<commit_before>package memory

import (
	"errors"
	"math"
	"regexp"

	"github.com/grafana/metrictank/idx"
	"github.com/raintank/worldping-api/pkg/log"
)

var (
	errInvalidQuery = errors.New("invalid query")
)

const (
	EQUAL = iota
	NOT_EQUAL
	MATCH
	NOT_MATCH
)

type kv struct {
	key   string
	value string
}

type expression struct {
	kv
	operator int
}

type kvRe struct {
	key   string
	value *regexp.Regexp
}

type TagQuery struct {
	from      int64
	equal     []kv
	match     []kvRe
	notEqual  []kv
	notMatch  []kvRe
	startWith int
}

// parseExpression returns an expression that's been generated from the given
// string, in case of error the operator will be PARSING_ERROR.
func parseExpression(expr string) (expression, error) {
	var pos int
	regex, not := false, false
	res := expression{}

	// scan up to operator to get key
	for ; pos < len(expr); pos++ {
		// =
		if expr[pos] == 61 {
			break
		}

		// !
		if expr[pos] == 33 {
			not = true
			break
		}

		// disallow ; in key
		if expr[pos] == 59 {
			return res, errInvalidQuery
		}
	}

	// key must not be empty
	if pos == 0 {
		return res, errInvalidQuery
	}

	res.key = expr[:pos]

	// shift over the ! character
	if not {
		pos++
	}

	// expecting a =
	if len(expr) <= pos || expr[pos] != 61 {
		return res, errInvalidQuery
	}
	pos++

	// if ~
	if len(expr) > pos && expr[pos] == 126 {
		regex = true
		pos++
	}

	valuePos := pos
	for ; pos < len(expr); pos++ {
		// disallow ; in value
		if expr[pos] == 59 {
			return res, errInvalidQuery
		}
	}
	res.value = expr[valuePos:]

	if not {
		if regex {
			res.operator = NOT_MATCH
		} else {
			res.operator = NOT_EQUAL
		}
	} else {
		if regex {
			res.operator = MATCH
		} else {
			res.operator = EQUAL
		}
	}
	return res, nil
}

func NewTagQuery(expressions []string, from int64) (TagQuery, error) {
	q := TagQuery{from: from}

	if len(expressions) == 0 {
		return q, errInvalidQuery
	}

	for _, expr := range expressions {
		e, err := parseExpression(expr)
		if err != nil {
			return q, err
		}

		// special case of empty value
		if len(e.value) == 0 {
			if e.operator == EQUAL || e.operator == MATCH {
				q.notMatch = append(q.notMatch, kvRe{
					key:   e.key,
					value: nil,
				})
			} else {
				q.match = append(q.match, kvRe{
					key:   e.key,
					value: nil,
				})
			}
		} else {
			// always anchor all regular expressions at the beginning
			if (e.operator == MATCH || e.operator == NOT_MATCH) && e.value[0] != byte('^') {
				e.value = "^" + e.value
			}

			switch e.operator {
			case EQUAL:
				q.equal = append(q.equal, e.kv)
			case NOT_EQUAL:
				q.notEqual = append(q.notEqual, e.kv)
			case MATCH:
				var re *regexp.Regexp
				if e.value != "^.+" {
					re, err = regexp.Compile(e.value)
					if err != nil {
						return q, errInvalidQuery
					}
				}
				q.match = append(q.match, kvRe{key: e.key, value: re})
			case NOT_MATCH:
				var re *regexp.Regexp
				if e.value != "^.+" {
					re, err = regexp.Compile(e.value)
					if err != nil {
						return q, errInvalidQuery
					}
				}
				q.notMatch = append(q.notMatch, kvRe{key: e.key, value: re})
			}
		}
	}

	if len(q.equal) == 0 {
		if len(q.match) == 0 {
			return q, errInvalidQuery
		}
		q.startWith = MATCH
	} else {
		q.startWith = EQUAL
	}

	return q, nil
}

func (q *TagQuery) getInitialByMatch(index TagIndex) (int, TagIDs) {
	resultSet := make(TagIDs)
	lowestCount := math.MaxUint32
	startId := 0

	// choose key that has the smallest number of values because we want to
	// reduce the number of matches and start with the smallest possible resultSet.
	// the smaller the resultSet the less time will have to be spent to further
	// filter its values in the following expressions.
	for i := range q.match {
		l := len(index[q.match[i].key])
		if l < lowestCount {
			lowestCount = l
			startId = i
		}
	}

	// shortcut if value == nil.
	// this will simply match any value, like ^.+. since we know that every value
	// in the index must not be empty, we can skip the matching.
	if q.match[startId].value == nil {
		for _, ids := range index[q.match[startId].key] {
			for id := range ids {
				resultSet[id] = struct{}{}
			}
		}
		return startId, resultSet
	}

	for v, ids := range index[q.match[startId].key] {
		if !q.match[startId].value.MatchString(v) {
			continue
		}

		for id := range ids {
			resultSet[id] = struct{}{}
		}
	}
	return startId, resultSet
}

func (q *TagQuery) getInitialByEqual(index TagIndex) (int, TagIDs) {
	resultSet := make(TagIDs)
	lowestCount := math.MaxUint32
	startId := 0

	// choose key-value combo that has the smallest number of series IDs because
	// we want to start with the smallest possible resultSet.
	// the smaller the resultSet the less time will have to be spent to further
	// filter its values in the following expressions.
	for i := range q.equal {
		l := len(index[q.equal[i].key][q.equal[i].value])
		if l < lowestCount {
			lowestCount = l
			startId = i
		}
	}

	// copy the map, because we'll later delete items from it
	for k, v := range index[q.equal[startId].key][q.equal[startId].value] {
		resultSet[k] = v
	}

	return startId, resultSet
}

// filterByEqual filters a list of metric ids by the given expressions. if "not"
// is true it will remove all ids of which at least one expression is equal to
// one of its tags. if "not" is false the functionality is inverted, so it
// removes all the ids where no tag is equal to at least one of the expressions.
//
// skipEqual:   an integer specifying an expression index that should be skipped
// resultSet:   list of series IDs that should be filtered
// index:       the tag index based on which to evaluate the conditions
// not:         whether the resultSet shall be filtered by a = or != condition
//
func (q *TagQuery) filterByEqual(skipEqual int, resultSet TagIDs, index TagIndex, not bool) {
	var expressions []kv
	if not {
		expressions = q.notEqual
	} else {
		expressions = q.equal
	}

	for i, e := range expressions {
		if i == skipEqual {
			continue
		}

		indexIds := index[e.key][e.value]

		// shortcut if key=value combo does not exist at all
		if len(indexIds) == 0 && !not {
			for id := range resultSet {
				delete(resultSet, id)
			}
		}

		for id := range resultSet {
			if _, ok := indexIds[id]; ok == not {
				delete(resultSet, id)
			}
		}
	}
}

// filterByMatch filters a list of metric ids by the given expressions.
//
// skipMatch:   an integer specifying an expression index that should be skipped
// resultSet:   list of series IDs that should be filtered
// byId:        ID keyed index of metric definitions, used to lookup the tags of IDs
// not:         whether the resultSet shall be filtered by the =~ or !=~ condition
//
func (q *TagQuery) filterByMatch(skipMatch int, resultSet TagIDs, byId map[string]*idx.Archive, not bool) {
	var expressions []kvRe
	if not {
		expressions = q.notMatch
	} else {
		expressions = q.match
	}

	for i, e := range expressions {
		if i == skipMatch {
			continue
		}

		// cache all tags that have once matched the regular expression.
		// this is based on the assumption that many matching tags will be repeated
		// over multiple series, so there's no need to run the regex for each of them
		// because once we know that a tag matches we can just compare strings
		matchingTags := make(map[string]struct{})
		notMatchingTags := make(map[string]struct{})
	IDS:
		for id := range resultSet {
			var def *idx.Archive
			var ok bool
			if def, ok = byId[id.String()]; !ok {
				// should never happen because every ID in the tag index
				// must be present in the byId lookup table
				CorruptIndex.Inc()
				log.Error(3, "memory-idx: ID %q is in tag index but not in the byId lookup table", id.String())
				delete(resultSet, id)
				continue IDS
			}

			for _, tag := range def.Tags {
				// reduce regex matching by looking up cached matches
				if _, ok := matchingTags[tag]; ok {
					if not {
						delete(resultSet, id)
					}
					continue IDS
				}
			}

			for _, tag := range def.Tags {
				// length of key doesn't match
				if len(tag) <= len(e.key)+1 || tag[len(e.key)] != 61 {
					continue
				}

				// reduce regex matching by looking up cached non-matches
				if _, ok := notMatchingTags[tag]; ok {
					continue
				}

				// value == nil means that this expression can be short cut
				// by not evaluating it
				if e.key == tag[:len(e.key)] && (e.value == nil || e.value.MatchString(tag[len(e.key)+1:])) {
					if len(matchingTags) < matchCacheSize {
						matchingTags[tag] = struct{}{}
					}
					if not {
						delete(resultSet, id)
					}
					continue IDS
				} else {
					if len(notMatchingTags) < matchCacheSize {
						notMatchingTags[tag] = struct{}{}
					}
				}
			}
			if !not {
				delete(resultSet, id)
			}
		}
	}
}

func (q *TagQuery) filterByFrom(resultSet TagIDs, byId map[string]*idx.Archive) {
	if q.from <= 0 {
		return
	}

	for id := range resultSet {
		var def *idx.Archive
		var ok bool
		if def, ok = byId[id.String()]; !ok {
			// should never happen because every ID in the tag index
			// must be present in the byId lookup table
			CorruptIndex.Inc()
			log.Error(3, "memory-idx: ID %q is in tag index but not in the byId lookup table", id.String())
			delete(resultSet, id)
			continue
		}

		if def.LastUpdate < q.from {
			delete(resultSet, id)
		}
	}
}

func (q *TagQuery) Run(index TagIndex, byId map[string]*idx.Archive) TagIDs {
	var skipMatch, skipEqual = -1, -1
	var resultSet TagIDs

	if q.startWith == EQUAL {
		skipEqual, resultSet = q.getInitialByEqual(index)
	} else {
		skipMatch, resultSet = q.getInitialByMatch(index)
	}

	// filter the resultSet by the from condition and all other expressions given.
	// filters should be in ascending order by the cpu required to process them,
	// that way the most cpu intensive filters only get applied to the smallest
	// possible resultSet.
	q.filterByEqual(skipEqual, resultSet, index, false)
	q.filterByEqual(-1, resultSet, index, true)
	q.filterByFrom(resultSet, byId)
	q.filterByMatch(skipMatch, resultSet, byId, false)
	q.filterByMatch(-1, resultSet, byId, true)
	return resultSet
}
<commit_msg>sort all expressions by cost<commit_after>package memory

import (
	"errors"
	"regexp"
	"sort"

	"github.com/grafana/metrictank/idx"
	"github.com/raintank/worldping-api/pkg/log"
)

var (
	errInvalidQuery = errors.New("invalid query")
)

const (
	EQUAL = iota
	NOT_EQUAL
	MATCH
	NOT_MATCH
)

type expression struct {
	kv
	operator int
}

type kv struct {
	cost  uint // cost of evaluating expression, compared to other kv objects
	key   string
	value string
}

type kvRe struct {
	cost  uint // cost of evaluating expression, compared to other kvRe objects
	key   string
	value *regexp.Regexp
}

type KvByCost []kv

func (a KvByCost) Len() int           { return len(a) }
func (a KvByCost) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a KvByCost) Less(i, j int) bool { return a[i].cost < a[j].cost }

type KvReByCost []kvRe

func (a KvReByCost) Len() int           { return len(a) }
func (a KvReByCost) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a KvReByCost) Less(i, j int) bool { return a[i].cost < a[j].cost }

type TagQuery struct {
	from      int64
	equal     []kv
	match     []kvRe
	notEqual  []kv
	notMatch  []kvRe
	startWith int
}

// parseExpression returns an expression that's been generated from the given
// string, in case of error the operator will be PARSING_ERROR.
func parseExpression(expr string) (expression, error) {
	var pos int
	regex, not := false, false
	res := expression{}

	// scan up to operator to get key
	for ; pos < len(expr); pos++ {
		// =
		if expr[pos] == 61 {
			break
		}

		// !
		if expr[pos] == 33 {
			not = true
			break
		}

		// disallow ; in key
		if expr[pos] == 59 {
			return res, errInvalidQuery
		}
	}

	// key must not be empty
	if pos == 0 {
		return res, errInvalidQuery
	}

	res.key = expr[:pos]

	// shift over the ! character
	if not {
		pos++
	}

	// expecting a =
	if len(expr) <= pos || expr[pos] != 61 {
		return res, errInvalidQuery
	}
	pos++

	// if ~
	if len(expr) > pos && expr[pos] == 126 {
		regex = true
		pos++
	}

	valuePos := pos
	for ; pos < len(expr); pos++ {
		// disallow ; in value
		if expr[pos] == 59 {
			return res, errInvalidQuery
		}
	}
	res.value = expr[valuePos:]

	if not {
		if regex {
			res.operator = NOT_MATCH
		} else {
			res.operator = NOT_EQUAL
		}
	} else {
		if regex {
			res.operator = MATCH
		} else {
			res.operator = EQUAL
		}
	}
	return res, nil
}

func NewTagQuery(expressions []string, from int64) (TagQuery, error) {
	q := TagQuery{from: from}

	if len(expressions) == 0 {
		return q, errInvalidQuery
	}

	for _, expr := range expressions {
		e, err := parseExpression(expr)
		if err != nil {
			return q, err
		}

		// special case of empty value
		if len(e.value) == 0 {
			if e.operator == EQUAL || e.operator == MATCH {
				q.notMatch = append(q.notMatch, kvRe{
					key:   e.key,
					value: nil,
				})
			} else {
				q.match = append(q.match, kvRe{
					key:   e.key,
					value: nil,
				})
			}
		} else {
			// always anchor all regular expressions at the beginning
			if (e.operator == MATCH || e.operator == NOT_MATCH) && e.value[0] != byte('^') {
				e.value = "^" + e.value
			}

			switch e.operator {
			case EQUAL:
				q.equal = append(q.equal, e.kv)
			case NOT_EQUAL:
				q.notEqual = append(q.notEqual, e.kv)
			case MATCH:
				var re *regexp.Regexp
				if e.value != "^.+" {
					re, err = regexp.Compile(e.value)
					if err != nil {
						return q, errInvalidQuery
					}
				}
				q.match = append(q.match, kvRe{key: e.key, value: re})
			case NOT_MATCH:
				var re *regexp.Regexp
				if e.value != "^.+" {
					re, err = regexp.Compile(e.value)
					if err != nil {
						return q, errInvalidQuery
					}
				}
				q.notMatch = append(q.notMatch, kvRe{key: e.key, value: re})
			}
		}
	}

	if len(q.equal) == 0 {
		if len(q.match) == 0 {
			return q, errInvalidQuery
		}
		q.startWith = MATCH
	} else {
		q.startWith = EQUAL
	}

	return q, nil
}

func (q *TagQuery) getInitialByMatch(index TagIndex) (int, TagIDs) {
	resultSet := make(TagIDs)
	startId := 0

	// shortcut if value == nil.
	// this will simply match any value, like ^.+. since we know that every value
	// in the index must not be empty, we can skip the matching.
	if q.match[startId].value == nil {
		for _, ids := range index[q.match[startId].key] {
			for id := range ids {
				resultSet[id] = struct{}{}
			}
		}
		return startId, resultSet
	}

	for v, ids := range index[q.match[startId].key] {
		if !q.match[startId].value.MatchString(v) {
			continue
		}

		for id := range ids {
			resultSet[id] = struct{}{}
		}
	}
	return startId, resultSet
}

func (q *TagQuery) getInitialByEqual(index TagIndex) (int, TagIDs) {
	resultSet := make(TagIDs)
	startId := 0

	// copy the map, because we'll later delete items from it
	for k, v := range index[q.equal[startId].key][q.equal[startId].value] {
		resultSet[k] = v
	}

	return startId, resultSet
}

// filterByEqual filters a list of metric ids by the given expressions. if "not"
// is true it will remove all ids of which at least one expression is equal to
// one of its tags. if "not" is false the functionality is inverted, so it
// removes all the ids where no tag is equal to at least one of the expressions.
//
// skipEqual:   an integer specifying an expression index that should be skipped
// resultSet:   list of series IDs that should be filtered
// index:       the tag index based on which to evaluate the conditions
// not:         whether the resultSet shall be filtered by a = or != condition
//
func (q *TagQuery) filterByEqual(skipEqual int, resultSet TagIDs, index TagIndex, not bool) {
	var expressions []kv
	if not {
		expressions = q.notEqual
	} else {
		expressions = q.equal
	}

	for i, e := range expressions {
		if i == skipEqual {
			continue
		}

		indexIds := index[e.key][e.value]

		// shortcut if key=value combo does not exist at all
		if len(indexIds) == 0 && !not {
			for id := range resultSet {
				delete(resultSet, id)
			}
		}

		for id := range resultSet {
			if _, ok := indexIds[id]; ok == not {
				delete(resultSet, id)
			}
		}
	}
}

// filterByMatch filters a list of metric ids by the given expressions.
//
// skipMatch:   an integer specifying an expression index that should be skipped
// resultSet:   list of series IDs that should be filtered
// byId:        ID keyed index of metric definitions, used to lookup the tags of IDs
// not:         whether the resultSet shall be filtered by the =~ or !=~ condition
//
func (q *TagQuery) filterByMatch(skipMatch int, resultSet TagIDs, byId map[string]*idx.Archive, not bool) {
	var expressions []kvRe
	if not {
		expressions = q.notMatch
	} else {
		expressions = q.match
	}

	for i, e := range expressions {
		if i == skipMatch {
			continue
		}

		// cache all tags that have once matched the regular expression.
		// this is based on the assumption that many matching tags will be repeated
		// over multiple series, so there's no need to run the regex for each of them
		// because once we know that a tag matches we can just compare strings
		matchingTags := make(map[string]struct{})
		notMatchingTags := make(map[string]struct{})
	IDS:
		for id := range resultSet {
			var def *idx.Archive
			var ok bool
			if def, ok = byId[id.String()]; !ok {
				// should never happen because every ID in the tag index
				// must be present in the byId lookup table
				CorruptIndex.Inc()
				log.Error(3, "memory-idx: ID %q is in tag index but not in the byId lookup table", id.String())
				delete(resultSet, id)
				continue IDS
			}

			for _, tag := range def.Tags {
				// reduce regex matching by looking up cached matches
				if _, ok := matchingTags[tag]; ok {
					if not {
						delete(resultSet, id)
					}
					continue IDS
				}
			}

			for _, tag := range def.Tags {
				// length of key doesn't match
				if len(tag) <= len(e.key)+1 || tag[len(e.key)] != 61 {
					continue
				}

				// reduce regex matching by looking up cached non-matches
				if _, ok := notMatchingTags[tag]; ok {
					continue
				}

				// value == nil means that this expression can be short cut
				// by not evaluating it
				if e.key == tag[:len(e.key)] && (e.value == nil || e.value.MatchString(tag[len(e.key)+1:])) {
					if len(matchingTags) < matchCacheSize {
						matchingTags[tag] = struct{}{}
					}
					if not {
						delete(resultSet, id)
					}
					continue IDS
				} else {
					if len(notMatchingTags) < matchCacheSize {
						notMatchingTags[tag] = struct{}{}
					}
				}
			}
			if !not {
				delete(resultSet, id)
			}
		}
	}
}

func (q *TagQuery) filterByFrom(resultSet TagIDs, byId map[string]*idx.Archive) {
	if q.from <= 0 {
		return
	}

	for id := range resultSet {
		var def *idx.Archive
		var ok bool
		if def, ok = byId[id.String()]; !ok {
			// should never happen because every ID in the tag index
			// must be present in the byId lookup table
			CorruptIndex.Inc()
			log.Error(3, "memory-idx: ID %q is in tag index but not in the byId lookup table", id.String())
			delete(resultSet, id)
			continue
		}

		if def.LastUpdate < q.from {
			delete(resultSet, id)
		}
	}
}

func (q *TagQuery) Run(index TagIndex, byId map[string]*idx.Archive) TagIDs {
	var skipMatch, skipEqual = -1, -1
	var resultSet TagIDs

	for i := range q.equal {
		q.equal[i].cost = uint(len(index[q.equal[i].key][q.equal[i].value]))
	}

	for i := range q.match {
		q.match[i].cost = uint(len(index[q.match[i].key]))
	}

	sort.Sort(KvByCost(q.equal))
	sort.Sort(KvByCost(q.notEqual))
	sort.Sort(KvReByCost(q.match))
	sort.Sort(KvReByCost(q.notMatch))

	if q.startWith == EQUAL {
		skipEqual, resultSet = q.getInitialByEqual(index)
	} else {
		skipMatch, resultSet = q.getInitialByMatch(index)
	}

	// filter the resultSet by the from condition and all other expressions given.
	// filters should be in ascending order by the cpu required to process them,
	// that way the most cpu intensive filters only get applied to the smallest
	// possible resultSet.
	q.filterByEqual(skipEqual, resultSet, index, false)
	q.filterByEqual(-1, resultSet, index, true)
	q.filterByFrom(resultSet, byId)
	q.filterByMatch(skipMatch, resultSet, byId, false)
	q.filterByMatch(-1, resultSet, byId, true)
	return resultSet
}
<|endoftext|>


<commit_before><commit_msg>Return error from DecodeForm (don't panic)<commit_after><|endoftext|>


<commit_before>package shell

import (
	"context"
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"path/filepath"
	"strings"
)

func init() {
	Commands = append(Commands, &commandRemoteUncache{})
}

type commandRemoteUncache struct {
}

func (c *commandRemoteUncache) Name() string {
	return "remote.uncache"
}

func (c *commandRemoteUncache) Help() string {
	return `keep the metadata but remote cache the file content for mounted directories or files

	This is designed to run regularly. So you can add it to some cronjob.
	If a file is not synchronized with the remote copy, the file will be skipped to avoid loss of data.

	remote.uncache -dir=/xxx
	remote.uncache -dir=/xxx/some/sub/dir
	remote.uncache -dir=/xxx/some/sub/dir -include=*.pdf
	remote.uncache -dir=/xxx/some/sub/dir -exclude=*.txt
	remote.uncache -minSize=1024000    # uncache files larger than 100K
	remote.uncache -minAge=3600        # uncache files older than 1 hour

`
}

func (c *commandRemoteUncache) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	remoteUnmountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)

	dir := remoteUnmountCommand.String("dir", "", "a directory in filer")
	fileFiler := newFileFilter(remoteUnmountCommand)

	if err = remoteUnmountCommand.Parse(args); err != nil {
		return nil
	}

	mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress)
	if listErr != nil {
		return listErr
	}
	if *dir == "" {
		jsonPrintln(writer, mappings)
		fmt.Fprintln(writer, "need to specify '-dir' option")
		return nil
	}

	var localMountedDir string
	for k := range mappings.Mappings {
		if strings.HasPrefix(*dir, k) {
			localMountedDir = k
		}
	}
	if localMountedDir == "" {
		jsonPrintln(writer, mappings)
		fmt.Fprintf(writer, "%s is not mounted\n", *dir)
		return nil
	}

	// pull content from remote
	if err = c.uncacheContentData(commandEnv, writer, util.FullPath(*dir), fileFiler); err != nil {
		return fmt.Errorf("uncache content data: %v", err)
	}

	return nil
}

func (c *commandRemoteUncache) uncacheContentData(commandEnv *CommandEnv, writer io.Writer, dirToCache util.FullPath, fileFilter *FileFilter) error {

	return recursivelyTraverseDirectory(commandEnv, dirToCache, func(dir util.FullPath, entry *filer_pb.Entry) bool {

		if !mayHaveCachedToLocal(entry) {
			return true // true means recursive traversal should continue
		}

		if fileFilter.matches(entry) {
			return true
		}

		if entry.RemoteEntry.LastLocalSyncTsNs/1e9 < entry.Attributes.Mtime {
			return true // should not uncache an entry that is not synchronized with remote
		}

		entry.RemoteEntry.LastLocalSyncTsNs = 0
		entry.Chunks = nil

		fmt.Fprintf(writer, "Uncache %+v ... ", dir.Child(entry.Name))

		err := commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
			_, updateErr := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
				Directory: string(dir),
				Entry:     entry,
			})
			return updateErr
		})
		if err != nil {
			fmt.Fprintf(writer, "uncache %+v: %v\n", dir.Child(entry.Name), err)
			return false
		}
		fmt.Fprintf(writer, "Done\n")

		return true
	})
}

type FileFilter struct {
	include *string
	exclude *string
	minSize *int64
	maxSize *int64
	minAge  *int64
	maxAge  *int64
}

func newFileFilter(remoteMountCommand *flag.FlagSet) (ff *FileFilter) {
	ff = &FileFilter{}
	ff.include = remoteMountCommand.String("include", "", "pattens of file names, e.g., *.pdf, *.html, ab?d.txt")
	ff.exclude = remoteMountCommand.String("exclude", "", "pattens of file names, e.g., *.pdf, *.html, ab?d.txt")
	ff.minSize = remoteMountCommand.Int64("minSize", -1, "minimum file size in bytes")
	ff.maxSize = remoteMountCommand.Int64("maxSize", -1, "maximum file size in bytes")
	ff.minAge = remoteMountCommand.Int64("minAge", -1, "minimum file age in seconds")
	ff.maxAge = remoteMountCommand.Int64("maxAge", -1, "maximum file age in seconds")
	return
}

func (ff *FileFilter) matches(entry *filer_pb.Entry) bool {
	if *ff.include != "" {
		if ok, _ := filepath.Match(*ff.include, entry.Name); !ok {
			return true
		}
	}
	if *ff.exclude != "" {
		if ok, _ := filepath.Match(*ff.exclude, entry.Name); ok {
			return true
		}
	}
	if *ff.minSize != -1 {
		if int64(entry.Attributes.FileSize) < *ff.minSize {
			return false
		}
	}
	if *ff.maxSize != -1 {
		if int64(entry.Attributes.FileSize) > *ff.maxSize {
			return false
		}
	}
	if *ff.minAge != -1 {
		if entry.Attributes.Crtime < *ff.minAge {
			return false
		}
	}
	if *ff.maxAge != -1 {
		if entry.Attributes.Crtime > *ff.maxAge {
			return false
		}
	}
	return false
}
<commit_msg>fix cache/uncache filters<commit_after>package shell

import (
	"context"
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"path/filepath"
	"strings"
)

func init() {
	Commands = append(Commands, &commandRemoteUncache{})
}

type commandRemoteUncache struct {
}

func (c *commandRemoteUncache) Name() string {
	return "remote.uncache"
}

func (c *commandRemoteUncache) Help() string {
	return `keep the metadata but remote cache the file content for mounted directories or files

	This is designed to run regularly. So you can add it to some cronjob.
	If a file is not synchronized with the remote copy, the file will be skipped to avoid loss of data.

	remote.uncache -dir=/xxx
	remote.uncache -dir=/xxx/some/sub/dir
	remote.uncache -dir=/xxx/some/sub/dir -include=*.pdf
	remote.uncache -dir=/xxx/some/sub/dir -exclude=*.txt
	remote.uncache -minSize=1024000    # uncache files larger than 100K
	remote.uncache -minAge=3600        # uncache files older than 1 hour

`
}

func (c *commandRemoteUncache) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	remoteUnmountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)

	dir := remoteUnmountCommand.String("dir", "", "a directory in filer")
	fileFiler := newFileFilter(remoteUnmountCommand)

	if err = remoteUnmountCommand.Parse(args); err != nil {
		return nil
	}

	mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress)
	if listErr != nil {
		return listErr
	}
	if *dir == "" {
		jsonPrintln(writer, mappings)
		fmt.Fprintln(writer, "need to specify '-dir' option")
		return nil
	}

	var localMountedDir string
	for k := range mappings.Mappings {
		if strings.HasPrefix(*dir, k) {
			localMountedDir = k
		}
	}
	if localMountedDir == "" {
		jsonPrintln(writer, mappings)
		fmt.Fprintf(writer, "%s is not mounted\n", *dir)
		return nil
	}

	// pull content from remote
	if err = c.uncacheContentData(commandEnv, writer, util.FullPath(*dir), fileFiler); err != nil {
		return fmt.Errorf("uncache content data: %v", err)
	}

	return nil
}

func (c *commandRemoteUncache) uncacheContentData(commandEnv *CommandEnv, writer io.Writer, dirToCache util.FullPath, fileFilter *FileFilter) error {

	return recursivelyTraverseDirectory(commandEnv, dirToCache, func(dir util.FullPath, entry *filer_pb.Entry) bool {

		if !mayHaveCachedToLocal(entry) {
			return true // true means recursive traversal should continue
		}

		if fileFilter.matches(entry) {
			return true
		}

		if entry.RemoteEntry.LastLocalSyncTsNs/1e9 < entry.Attributes.Mtime {
			return true // should not uncache an entry that is not synchronized with remote
		}

		entry.RemoteEntry.LastLocalSyncTsNs = 0
		entry.Chunks = nil

		fmt.Fprintf(writer, "Uncache %+v ... ", dir.Child(entry.Name))

		err := commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
			_, updateErr := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
				Directory: string(dir),
				Entry:     entry,
			})
			return updateErr
		})
		if err != nil {
			fmt.Fprintf(writer, "uncache %+v: %v\n", dir.Child(entry.Name), err)
			return false
		}
		fmt.Fprintf(writer, "Done\n")

		return true
	})
}

type FileFilter struct {
	include *string
	exclude *string
	minSize *int64
	maxSize *int64
	minAge  *int64
	maxAge  *int64
}

func newFileFilter(remoteMountCommand *flag.FlagSet) (ff *FileFilter) {
	ff = &FileFilter{}
	ff.include = remoteMountCommand.String("include", "", "pattens of file names, e.g., *.pdf, *.html, ab?d.txt")
	ff.exclude = remoteMountCommand.String("exclude", "", "pattens of file names, e.g., *.pdf, *.html, ab?d.txt")
	ff.minSize = remoteMountCommand.Int64("minSize", -1, "minimum file size in bytes")
	ff.maxSize = remoteMountCommand.Int64("maxSize", -1, "maximum file size in bytes")
	ff.minAge = remoteMountCommand.Int64("minAge", -1, "minimum file age in seconds")
	ff.maxAge = remoteMountCommand.Int64("maxAge", -1, "maximum file age in seconds")
	return
}

func (ff *FileFilter) matches(entry *filer_pb.Entry) bool {
	if *ff.include != "" {
		if ok, _ := filepath.Match(*ff.include, entry.Name); !ok {
			return false
		}
	}
	if *ff.exclude != "" {
		if ok, _ := filepath.Match(*ff.exclude, entry.Name); ok {
			return false
		}
	}
	if *ff.minSize != -1 {
		if int64(entry.Attributes.FileSize) < *ff.minSize {
			return false
		}
	}
	if *ff.maxSize != -1 {
		if int64(entry.Attributes.FileSize) > *ff.maxSize {
			return false
		}
	}
	if *ff.minAge != -1 {
		if entry.Attributes.Crtime < *ff.minAge {
			return false
		}
	}
	if *ff.maxAge != -1 {
		if entry.Attributes.Crtime > *ff.maxAge {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before><commit_msg>Settings/warnings: dont panic if empty<commit_after><|endoftext|>