<commit_before>// Copyright 2013 Patrice FERLET
// Ue of this source code is governed by MIT-style
// license that can be found in the LICENSE file
package verbalexpressions

import (
	"log"
	"regexp"
	"strconv"
	"strings"
)

// VerbalExpression structure to create expression
type VerbalExpression struct {
	expression string
	anycase    bool
	oneline    bool
	suffixes   string
	prefixes   string
}

// quote is an alias to regexp.QuoteMeta
func quote(s string) string {
	return regexp.QuoteMeta(s)
}

// utility function to return only strings
func tostring(i interface{}) string {
	var r string
	switch x := i.(type) {
	case string:
		r = x
	case int64:
		r = strconv.FormatInt(x, 64)
	case uint:
		r = strconv.FormatUint(uint64(x), 64)
	case int:
		r = strconv.FormatInt(int64(x), 32)
	default:
		log.Panicf("Could not convert %v %t", x, x)
	}
	return r
}

// Instanciate a new VerbalExpression. You should use this method to
// initalize some internal var.
// Example:
//		v := verbalexpression.New().Find("foo")
func New() *VerbalExpression {
	r := new(VerbalExpression)
	r.anycase, r.oneline = false, true
	return r
}

// add method, append expresions to the internal string that will be parsed
func (v *VerbalExpression) add(s string) *VerbalExpression {
	v.expression += s
	return v
}

// Start to capture something, stop with EndCapture()
func (v *VerbalExpression) BeginCapture() *VerbalExpression {
	v.suffixes += ")"
	return v.add("(")
}

// Stop capturing expresions parts
func (v *VerbalExpression) EndCapture() *VerbalExpression {
	v.suffixes = strings.Replace(v.suffixes, ")", "", 1)
	return v.add(")")
}

// Anything will match any char
func (v *VerbalExpression) Anything() *VerbalExpression {
	return v.add(`(?:.*)`)
}

// AnythingBut will match anything excpeting the given string.
// Example:
//		s := "This is a simple test"
//		v := verbalexpressions.New().AnythingBut("ie").RegExp().FindAllString(s, -1)
//		[Th s  s a s mple t st]
func (v *VerbalExpression) AnythingBut(s string) *VerbalExpression {
	return v.add(`(?:[^` + quote(s) + `]*)`)
}

// EndOfLine tells verbalexpressions to match a end of line.
// Warning, to check multiple line, you must use SearchOneLine(true)
func (v *VerbalExpression) EndOfLine() *VerbalExpression {
	v.suffixes += "$"
	return v
}

// Maybe will search string zero on more times
func (v *VerbalExpression) Maybe(s string) *VerbalExpression {
	return v.add(`(?:` + quote(s) + `)?`)
}

// StartOfLine seeks the begining of a line. As EndOfLine you should use
// SearchOneLine(true) to test multiple lines
func (v *VerbalExpression) StartOfLine() *VerbalExpression {
	v.prefixes += `^`
	return v
}

// Find seeks string. The string MUST be there (unlike Maybe() method)
func (v *VerbalExpression) Find(s string) *VerbalExpression {
	return v.add(`(?:` + quote(s) + `)`)
}

// Alias to Find()
func (v *VerbalExpression) Then(s string) *VerbalExpression {
	return v.Find(s)
}

// Any accepts caracters to be matched
// Example:
//		s := "foo1 foo5 foobar"
//		v := New().Find("foo").Any("1234567890").Regex().FindAllString(s, -1)
//		[foo1 foo5]
func (v *VerbalExpression) Any(s string) *VerbalExpression {
	return v.add(`(?:[` + quote(s) + `])`)
}

//AnyOf is an alias to Any
func (v *VerbalExpression) AnyOf(s string) *VerbalExpression {
	return v.Any(s)
}

// LineBreak to find "\n" or "\r\n"
func (v *VerbalExpression) LineBreak() *VerbalExpression {
	return v.add(`(?:(?:\n)|(?:\r\n))`)
}

// Alias to LineBreak
func (v *VerbalExpression) Br() *VerbalExpression {
	return v.LineBreak()
}

// Range accepts an even number of arguments. Each pair of values defines start and end of range.
// Think like this: Range(from, to [, from, to ...])
// Example:
//		s := "This 1 is 55 a TEST"
//		v := verbalexpressions.New().Range("a","z",0,9)
func (v *VerbalExpression) Range(args ...interface{}) *VerbalExpression {
	if len(args)%2 != 0 {
		log.Panicf("Range: not even args number")
	}

	parts := make([]string, 3)
	app := ""
	for i := 0; i < len(args); i++ {
		app += tostring(args[i])
		if i%2 != 0 {
			parts = append(parts, quote(app))
			app = ""
		} else {
			app += "-"
		}
	}
	return v.add("[" + strings.Join(parts, "") + "]")
}

// Tab fetch tabulation char (\t)
func (v *VerbalExpression) Tab() *VerbalExpression {
	return v.add(`\t+`)
}

// Word matches any word (containing alpha char)
func (v *VerbalExpression) Word() *VerbalExpression {
	return v.add(`\w+`)
}

// Or, as the word is meaning...
func (v *VerbalExpression) Or() *VerbalExpression {
	v.prefixes += "(?:"
	v.suffixes = ")" + v.suffixes
	return v.add(")|(?:")
}

// WithAnyCase ask verbalexpressions to match with or without case sensitivity
func (v *VerbalExpression) WithAnyCase(sensitive bool) *VerbalExpression {
	v.anycase = sensitive
	return v
}

// SearchOneLine deactivate "multiline" mode if true
// Default is false
func (v *VerbalExpression) SearchOneLine(oneline bool) *VerbalExpression {
	v.oneline = !oneline
	return v
}

// Regex return the regular expression to use to test on string.
func (v *VerbalExpression) Regex() *regexp.Regexp {
	modifier := ""
	if v.anycase {
		modifier += "i"
	}

	if v.oneline {
		modifier += "m"
	}
	if len(modifier) > 0 {
		modifier = "(?" + modifier + ")"
	}

	return regexp.MustCompile(modifier + v.prefixes + v.expression + v.suffixes)
}

/* proxy and helpers to regexp.Regexp functions */

// Test return true if verbalexpressions matches something in string "s"
func (v *VerbalExpression) Test(s string) bool {
	return v.Regex().Match([]byte(s))
}

// Replace alias to regexp.ReplaceAllString. It replace the found expression from
// string src by string dst
func (v *VerbalExpression) Replace(src string, dst string) string {
	return v.Regex().ReplaceAllString(src, dst)
}

// Returns a slice of results from captures. If you didn't apply BeginCapture() and EnCapture(), the slices
// will return slice of []string where []string is length 1, and 0 index is the global capture
// Example:
//		s:="This should get barsystem and whatever..."
//		// get "bar" followed by a word
//		v := verbalexpressions.New().Anything().
//				BeginCatpure().
//				Find("bar").Word().
//				EndCapture()
//
//		res := v.Captures(s)
//		fmt.Println(res)
//		[["This should get barsystem", "barsystem"]]
//
// So, to range results, you can do:
//		for _, captures := range res {
//			fmt.Println(captures[1])
//		}
// Actualy, 1 matches first group, you can use several captures.
func (v *VerbalExpression) Captures(s string) [][]string {
	return v.Regex().FindAllStringSubmatch(s, -1)
}
<commit_msg>Reimplement modifiers<commit_after>// Copyright 2013 Patrice FERLET
// Ue of this source code is governed by MIT-style
// license that can be found in the LICENSE file
package verbalexpressions

import (
	"log"
	"regexp"
	"strconv"
	"strings"
)

// VerbalExpression structure to create expression
type VerbalExpression struct {
	expression string
	suffixes   string
	prefixes   string
	modifiers  string
}

// quote is an alias to regexp.QuoteMeta
func quote(s string) string {
	return regexp.QuoteMeta(s)
}

// utility function to return only strings
func tostring(i interface{}) string {
	var r string
	switch x := i.(type) {
	case string:
		r = x
	case int64:
		r = strconv.FormatInt(x, 64)
	case uint:
		r = strconv.FormatUint(uint64(x), 64)
	case int:
		r = strconv.FormatInt(int64(x), 32)
	default:
		log.Panicf("Could not convert %v %t", x, x)
	}
	return r
}

// Instanciate a new VerbalExpression. You should use this method to
// initalize some internal var.
// Example:
//		v := verbalexpression.New().Find("foo")
func New() *VerbalExpression {
	r := new(VerbalExpression)
	r.modifiers = "m"
	return r
}

// append a modifier
func (v *VerbalExpression) addmodifier(m string) *VerbalExpression {
	if !strings.Contains(v.modifiers, m) {
		v.modifiers += "m"
	}
	return v
}

// remove a modifier
func (v *VerbalExpression) removemodifier(m string) *VerbalExpression {
	v.modifiers = strings.Replace(v.modifiers, m, "", -1)
	return v
}

// add method, append expresions to the internal string that will be parsed
func (v *VerbalExpression) add(s string) *VerbalExpression {
	v.expression += s
	return v
}

// Start to capture something, stop with EndCapture()
func (v *VerbalExpression) BeginCapture() *VerbalExpression {
	v.suffixes += ")"
	return v.add("(")
}

// Stop capturing expresions parts
func (v *VerbalExpression) EndCapture() *VerbalExpression {
	v.suffixes = strings.Replace(v.suffixes, ")", "", 1)
	return v.add(")")
}

// Anything will match any char
func (v *VerbalExpression) Anything() *VerbalExpression {
	return v.add(`(?:.*)`)
}

// AnythingBut will match anything excpeting the given string.
// Example:
//		s := "This is a simple test"
//		v := verbalexpressions.New().AnythingBut("ie").RegExp().FindAllString(s, -1)
//		[Th s  s a s mple t st]
func (v *VerbalExpression) AnythingBut(s string) *VerbalExpression {
	return v.add(`(?:[^` + quote(s) + `]*)`)
}

// EndOfLine tells verbalexpressions to match a end of line.
// Warning, to check multiple line, you must use SearchOneLine(true)
func (v *VerbalExpression) EndOfLine() *VerbalExpression {
	v.suffixes += "$"
	return v
}

// Maybe will search string zero on more times
func (v *VerbalExpression) Maybe(s string) *VerbalExpression {
	return v.add(`(?:` + quote(s) + `)?`)
}

// StartOfLine seeks the begining of a line. As EndOfLine you should use
// SearchOneLine(true) to test multiple lines
func (v *VerbalExpression) StartOfLine() *VerbalExpression {
	v.prefixes += `^`
	return v
}

// Find seeks string. The string MUST be there (unlike Maybe() method)
func (v *VerbalExpression) Find(s string) *VerbalExpression {
	return v.add(`(?:` + quote(s) + `)`)
}

// Alias to Find()
func (v *VerbalExpression) Then(s string) *VerbalExpression {
	return v.Find(s)
}

// Any accepts caracters to be matched
// Example:
//		s := "foo1 foo5 foobar"
//		v := New().Find("foo").Any("1234567890").Regex().FindAllString(s, -1)
//		[foo1 foo5]
func (v *VerbalExpression) Any(s string) *VerbalExpression {
	return v.add(`(?:[` + quote(s) + `])`)
}

//AnyOf is an alias to Any
func (v *VerbalExpression) AnyOf(s string) *VerbalExpression {
	return v.Any(s)
}

// LineBreak to find "\n" or "\r\n"
func (v *VerbalExpression) LineBreak() *VerbalExpression {
	return v.add(`(?:(?:\n)|(?:\r\n))`)
}

// Alias to LineBreak
func (v *VerbalExpression) Br() *VerbalExpression {
	return v.LineBreak()
}

// Range accepts an even number of arguments. Each pair of values defines start and end of range.
// Think like this: Range(from, to [, from, to ...])
// Example:
//		s := "This 1 is 55 a TEST"
//		v := verbalexpressions.New().Range("a","z",0,9)
func (v *VerbalExpression) Range(args ...interface{}) *VerbalExpression {
	if len(args)%2 != 0 {
		log.Panicf("Range: not even args number")
	}

	parts := make([]string, 3)
	app := ""
	for i := 0; i < len(args); i++ {
		app += tostring(args[i])
		if i%2 != 0 {
			parts = append(parts, quote(app))
			app = ""
		} else {
			app += "-"
		}
	}
	return v.add("[" + strings.Join(parts, "") + "]")
}

// Tab fetch tabulation char (\t)
func (v *VerbalExpression) Tab() *VerbalExpression {
	return v.add(`\t+`)
}

// Word matches any word (containing alpha char)
func (v *VerbalExpression) Word() *VerbalExpression {
	return v.add(`\w+`)
}

// Or, chains a alternate expression
// Example:
//		v := Verbalexpression.New().
//				Find("foobarbaz").
//				Or().
//				Find("footestbaz")
func (v *VerbalExpression) Or() *VerbalExpression {
	v.prefixes += "(?:"
	v.suffixes = ")" + v.suffixes
	return v.add(")|(?:")
}

// WithAnyCase ask verbalexpressions to match with or without case sensitivity
func (v *VerbalExpression) WithAnyCase(sensitive bool) *VerbalExpression {
	return v.addmodifier("i")
}

// SearchOneLine deactivate "multiline" mode if true
// Default is false
func (v *VerbalExpression) SearchOneLine(oneline bool) *VerbalExpression {
	if oneline {
		return v.removemodifier("m")
	} else {
		return v.addmodifier("m")
	}
}

// Regex return the regular expression to use to test on string.
func (v *VerbalExpression) Regex() *regexp.Regexp {
	modifier := ""
	if len(v.modifiers) > 0 {
		modifier = "(?" + v.modifiers + ")"
	}

	return regexp.MustCompile(modifier + v.prefixes + v.expression + v.suffixes)
}

/* proxy and helpers to regexp.Regexp functions */

// Test return true if verbalexpressions matches something in string "s"
func (v *VerbalExpression) Test(s string) bool {
	return v.Regex().Match([]byte(s))
}

// Replace alias to regexp.ReplaceAllString. It replace the found expression from
// string src by string dst
func (v *VerbalExpression) Replace(src string, dst string) string {
	return v.Regex().ReplaceAllString(src, dst)
}

// Returns a slice of results from captures. If you didn't apply BeginCapture() and EnCapture(), the slices
// will return slice of []string where []string is length 1, and 0 index is the global capture
// Example:
//		s:="This should get barsystem and whatever..."
//		// get "bar" followed by a word
//		v := verbalexpressions.New().Anything().
//				BeginCatpure().
//				Find("bar").Word().
//				EndCapture()
//
//		res := v.Captures(s)
//		fmt.Println(res)
//		[["This should get barsystem", "barsystem"]]
//
// So, to range results, you can do:
//		for _, captures := range res {
//			fmt.Println(captures[1])
//		}
// Actualy, 1 matches first group, you can use several captures.
func (v *VerbalExpression) Captures(s string) [][]string {
	return v.Regex().FindAllStringSubmatch(s, -1)
}
<|endoftext|>


<commit_before>package federationapi

import (
	"bytes"
	"context"
	"crypto/ed25519"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"reflect"
	"testing"
	"time"

	"github.com/matrix-org/dendrite/federationapi/api"
	"github.com/matrix-org/dendrite/federationapi/routing"
	"github.com/matrix-org/dendrite/internal/caching"
	"github.com/matrix-org/dendrite/setup/base"
	"github.com/matrix-org/dendrite/setup/config"
	"github.com/matrix-org/gomatrixserverlib"
)

type server struct {
	name      gomatrixserverlib.ServerName        // server name
	validity  time.Duration                       // key validity duration from now
	config    *config.FederationAPI               // skeleton config, from TestMain
	fedclient *gomatrixserverlib.FederationClient // uses MockRoundTripper
	cache     *caching.Caches                     // server-specific cache
	api       api.FederationInternalAPI           // server-specific server key API
}

func (s *server) renew() {
	// This updates the validity period to be an hour in the
	// future, which is particularly useful in server A and
	// server C's cases which have validity either as now or
	// in the past.
	s.validity = time.Hour
	s.config.Matrix.KeyValidityPeriod = s.validity
}

var (
	serverKeyID = gomatrixserverlib.KeyID("ed25519:auto")
	serverA     = &server{name: "a.com", validity: time.Duration(0)} // expires now
	serverB     = &server{name: "b.com", validity: time.Hour}        // expires in an hour
	serverC     = &server{name: "c.com", validity: -time.Hour}       // expired an hour ago
)

var servers = map[string]*server{
	"a.com": serverA,
	"b.com": serverB,
	"c.com": serverC,
}

func TestMain(m *testing.M) {
	// Set up the server key API for each "server" that we
	// will use in our tests.
	for _, s := range servers {
		// Generate a new key.
		_, testPriv, err := ed25519.GenerateKey(nil)
		if err != nil {
			panic("can't generate identity key: " + err.Error())
		}

		// Create a new cache but don't enable prometheus!
		s.cache, err = caching.NewInMemoryLRUCache(false)
		if err != nil {
			panic("can't create cache: " + err.Error())
		}

		// Create a temporary directory for JetStream.
		d, err := ioutil.TempDir("./", "jetstream*")
		if err != nil {
			panic(err)
		}
		defer os.RemoveAll(d)

		// Draw up just enough Dendrite config for the server key
		// API to work.
		cfg := &config.Dendrite{}
		cfg.Defaults(true)
		cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
		cfg.Global.PrivateKey = testPriv
		cfg.Global.JetStream.InMemory = true
		cfg.Global.JetStream.TopicPrefix = string(s.name[:1])
		cfg.Global.JetStream.StoragePath = config.Path(d)
		cfg.Global.KeyID = serverKeyID
		cfg.Global.KeyValidityPeriod = s.validity
		cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:")
		s.config = &cfg.FederationAPI

		// Create a transport which redirects federation requests to
		// the mock round tripper. Since we're not *really* listening for
		// federation requests then this will return the key instead.
		transport := &http.Transport{}
		transport.RegisterProtocol("matrix", &MockRoundTripper{})

		// Create the federation client.
		s.fedclient = gomatrixserverlib.NewFederationClient(
			s.config.Matrix.ServerName, serverKeyID, testPriv,
			gomatrixserverlib.WithTransport(transport),
		)

		// Finally, build the server key APIs.
		sbase := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
		s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true)
	}

	// Now that we have built our server key APIs, start the
	// rest of the tests.
	os.Exit(m.Run())
}

type MockRoundTripper struct{}

func (m *MockRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) {
	// Check if the request is looking for keys from a server that
	// we know about in the test. The only reason this should go wrong
	// is if the test is broken.
	s, ok := servers[req.Host]
	if !ok {
		return nil, fmt.Errorf("server not known: %s", req.Host)
	}

	// We're intercepting /matrix/key/v2/server requests here, so check
	// that the URL supplied in the request is for that.
	if req.URL.Path != "/_matrix/key/v2/server" {
		return nil, fmt.Errorf("unexpected request path: %s", req.URL.Path)
	}

	// Get the keys and JSON-ify them.
	keys := routing.LocalKeys(s.config)
	body, err := json.MarshalIndent(keys.JSON, "", "  ")
	if err != nil {
		return nil, err
	}

	// And respond.
	res = &http.Response{
		StatusCode: 200,
		Body:       ioutil.NopCloser(bytes.NewReader(body)),
	}
	return
}

func TestServersRequestOwnKeys(t *testing.T) {
	// Each server will request its own keys. There's no reason
	// for this to fail as each server should know its own keys.

	for name, s := range servers {
		req := gomatrixserverlib.PublicKeyLookupRequest{
			ServerName: s.name,
			KeyID:      serverKeyID,
		}
		res, err := s.api.FetchKeys(
			context.Background(),
			map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{
				req: gomatrixserverlib.AsTimestamp(time.Now()),
			},
		)
		if err != nil {
			t.Fatalf("server could not fetch own key: %s", err)
		}
		if _, ok := res[req]; !ok {
			t.Fatalf("server didn't return its own key in the results")
		}
		t.Logf("%s's key expires at %s\n", name, res[req].ValidUntilTS.Time())
	}
}

func TestCachingBehaviour(t *testing.T) {
	// Server A will request Server B's key, which has a validity
	// period of an hour from now. We should retrieve the key and
	// it should make it into the cache automatically.

	req := gomatrixserverlib.PublicKeyLookupRequest{
		ServerName: serverB.name,
		KeyID:      serverKeyID,
	}
	ts := gomatrixserverlib.AsTimestamp(time.Now())

	res, err := serverA.api.FetchKeys(
		context.Background(),
		map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{
			req: ts,
		},
	)
	if err != nil {
		t.Fatalf("server A failed to retrieve server B key: %s", err)
	}
	if len(res) != 1 {
		t.Fatalf("server B should have returned one key but instead returned %d keys", len(res))
	}
	if _, ok := res[req]; !ok {
		t.Fatalf("server B isn't included in the key fetch response")
	}

	// At this point, if the previous key request was a success,
	// then the cache should now contain the key. Check if that's
	// the case - if it isn't then there's something wrong with
	// the cache implementation or we failed to get the key.

	cres, ok := serverA.cache.GetServerKey(req, ts)
	if !ok {
		t.Fatalf("server B key should be in cache but isn't")
	}
	if !reflect.DeepEqual(cres, res[req]) {
		t.Fatalf("the cached result from server B wasn't what server B gave us")
	}

	// If we ask the cache for the same key but this time for an event
	// that happened in +30 minutes. Since the validity period is for
	// another hour, then we should get a response back from the cache.

	_, ok = serverA.cache.GetServerKey(
		req,
		gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute*30)),
	)
	if !ok {
		t.Fatalf("server B key isn't in cache when it should be (+30 minutes)")
	}

	// If we ask the cache for the same key but this time for an event
	// that happened in +90 minutes then we should expect to get no
	// cache result. This is because the cache shouldn't return a result
	// that is obviously past the validity of the event.

	_, ok = serverA.cache.GetServerKey(
		req,
		gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute*90)),
	)
	if ok {
		t.Fatalf("server B key is in cache when it shouldn't be (+90 minutes)")
	}
}

func TestRenewalBehaviour(t *testing.T) {
	// Server A will request Server C's key but their validity period
	// is an hour in the past. We'll retrieve the key as, even though it's
	// past its validity, it will be able to verify past events.

	req := gomatrixserverlib.PublicKeyLookupRequest{
		ServerName: serverC.name,
		KeyID:      serverKeyID,
	}

	res, err := serverA.api.FetchKeys(
		context.Background(),
		map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{
			req: gomatrixserverlib.AsTimestamp(time.Now()),
		},
	)
	if err != nil {
		t.Fatalf("server A failed to retrieve server C key: %s", err)
	}
	if len(res) != 1 {
		t.Fatalf("server C should have returned one key but instead returned %d keys", len(res))
	}
	if _, ok := res[req]; !ok {
		t.Fatalf("server C isn't included in the key fetch response")
	}

	// If we ask the cache for the server key for an event that happened
	// 90 minutes ago then we should get a cache result, as the key hadn't
	// passed its validity by that point. The fact that the key is now in
	// the cache is, in itself, proof that we successfully retrieved the
	// key before.

	oldcached, ok := serverA.cache.GetServerKey(
		req,
		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*90)),
	)
	if !ok {
		t.Fatalf("server C key isn't in cache when it should be (-90 minutes)")
	}

	// If we now ask the cache for the same key but this time for an event
	// that only happened 30 minutes ago then we shouldn't get a cached
	// result, as the event happened after the key validity expired. This
	// is really just for sanity checking.

	_, ok = serverA.cache.GetServerKey(
		req,
		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*30)),
	)
	if ok {
		t.Fatalf("server B key is in cache when it shouldn't be (-30 minutes)")
	}

	// We're now going to kick server C into renewing its key. Since we're
	// happy at this point that the key that we already have is from the past
	// then repeating a key fetch should cause us to try and renew the key.
	// If so, then the new key will end up in our cache.

	serverC.renew()

	res, err = serverA.api.FetchKeys(
		context.Background(),
		map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{
			req: gomatrixserverlib.AsTimestamp(time.Now()),
		},
	)
	if err != nil {
		t.Fatalf("server A failed to retrieve server C key: %s", err)
	}
	if len(res) != 1 {
		t.Fatalf("server C should have returned one key but instead returned %d keys", len(res))
	}
	if _, ok = res[req]; !ok {
		t.Fatalf("server C isn't included in the key fetch response")
	}

	// We're now going to ask the cache what the new key validity is. If
	// it is still the same as the previous validity then we've failed to
	// retrieve the renewed key. If it's newer then we've successfully got
	// the renewed key.

	newcached, ok := serverA.cache.GetServerKey(
		req,
		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*30)),
	)
	if !ok {
		t.Fatalf("server B key isn't in cache when it shouldn't be (post-renewal)")
	}
	if oldcached.ValidUntilTS >= newcached.ValidUntilTS {
		t.Fatalf("the server B key should have been renewed but wasn't")
	}
	t.Log(res)
}
<commit_msg>Allow defers to run in `TestMain` in federation API tests<commit_after>package federationapi

import (
	"bytes"
	"context"
	"crypto/ed25519"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"reflect"
	"testing"
	"time"

	"github.com/matrix-org/dendrite/federationapi/api"
	"github.com/matrix-org/dendrite/federationapi/routing"
	"github.com/matrix-org/dendrite/internal/caching"
	"github.com/matrix-org/dendrite/setup/base"
	"github.com/matrix-org/dendrite/setup/config"
	"github.com/matrix-org/gomatrixserverlib"
)

type server struct {
	name      gomatrixserverlib.ServerName        // server name
	validity  time.Duration                       // key validity duration from now
	config    *config.FederationAPI               // skeleton config, from TestMain
	fedclient *gomatrixserverlib.FederationClient // uses MockRoundTripper
	cache     *caching.Caches                     // server-specific cache
	api       api.FederationInternalAPI           // server-specific server key API
}

func (s *server) renew() {
	// This updates the validity period to be an hour in the
	// future, which is particularly useful in server A and
	// server C's cases which have validity either as now or
	// in the past.
	s.validity = time.Hour
	s.config.Matrix.KeyValidityPeriod = s.validity
}

var (
	serverKeyID = gomatrixserverlib.KeyID("ed25519:auto")
	serverA     = &server{name: "a.com", validity: time.Duration(0)} // expires now
	serverB     = &server{name: "b.com", validity: time.Hour}        // expires in an hour
	serverC     = &server{name: "c.com", validity: -time.Hour}       // expired an hour ago
)

var servers = map[string]*server{
	"a.com": serverA,
	"b.com": serverB,
	"c.com": serverC,
}

func TestMain(m *testing.M) {
	// Set up the server key API for each "server" that we
	// will use in our tests.
	os.Exit(func() int {
		for _, s := range servers {
			// Generate a new key.
			_, testPriv, err := ed25519.GenerateKey(nil)
			if err != nil {
				panic("can't generate identity key: " + err.Error())
			}

			// Create a new cache but don't enable prometheus!
			s.cache, err = caching.NewInMemoryLRUCache(false)
			if err != nil {
				panic("can't create cache: " + err.Error())
			}

			// Create a temporary directory for JetStream.
			d, err := ioutil.TempDir("./", "jetstream*")
			if err != nil {
				panic(err)
			}
			defer os.RemoveAll(d)

			// Draw up just enough Dendrite config for the server key
			// API to work.
			cfg := &config.Dendrite{}
			cfg.Defaults(true)
			cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
			cfg.Global.PrivateKey = testPriv
			cfg.Global.JetStream.InMemory = true
			cfg.Global.JetStream.TopicPrefix = string(s.name[:1])
			cfg.Global.JetStream.StoragePath = config.Path(d)
			cfg.Global.KeyID = serverKeyID
			cfg.Global.KeyValidityPeriod = s.validity
			cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:")
			s.config = &cfg.FederationAPI

			// Create a transport which redirects federation requests to
			// the mock round tripper. Since we're not *really* listening for
			// federation requests then this will return the key instead.
			transport := &http.Transport{}
			transport.RegisterProtocol("matrix", &MockRoundTripper{})

			// Create the federation client.
			s.fedclient = gomatrixserverlib.NewFederationClient(
				s.config.Matrix.ServerName, serverKeyID, testPriv,
				gomatrixserverlib.WithTransport(transport),
			)

			// Finally, build the server key APIs.
			sbase := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
			s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true)
		}

		// Now that we have built our server key APIs, start the
		// rest of the tests.
		return m.Run()
	}())
}

type MockRoundTripper struct{}

func (m *MockRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) {
	// Check if the request is looking for keys from a server that
	// we know about in the test. The only reason this should go wrong
	// is if the test is broken.
	s, ok := servers[req.Host]
	if !ok {
		return nil, fmt.Errorf("server not known: %s", req.Host)
	}

	// We're intercepting /matrix/key/v2/server requests here, so check
	// that the URL supplied in the request is for that.
	if req.URL.Path != "/_matrix/key/v2/server" {
		return nil, fmt.Errorf("unexpected request path: %s", req.URL.Path)
	}

	// Get the keys and JSON-ify them.
	keys := routing.LocalKeys(s.config)
	body, err := json.MarshalIndent(keys.JSON, "", "  ")
	if err != nil {
		return nil, err
	}

	// And respond.
	res = &http.Response{
		StatusCode: 200,
		Body:       ioutil.NopCloser(bytes.NewReader(body)),
	}
	return
}

func TestServersRequestOwnKeys(t *testing.T) {
	// Each server will request its own keys. There's no reason
	// for this to fail as each server should know its own keys.

	for name, s := range servers {
		req := gomatrixserverlib.PublicKeyLookupRequest{
			ServerName: s.name,
			KeyID:      serverKeyID,
		}
		res, err := s.api.FetchKeys(
			context.Background(),
			map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{
				req: gomatrixserverlib.AsTimestamp(time.Now()),
			},
		)
		if err != nil {
			t.Fatalf("server could not fetch own key: %s", err)
		}
		if _, ok := res[req]; !ok {
			t.Fatalf("server didn't return its own key in the results")
		}
		t.Logf("%s's key expires at %s\n", name, res[req].ValidUntilTS.Time())
	}
}

func TestCachingBehaviour(t *testing.T) {
	// Server A will request Server B's key, which has a validity
	// period of an hour from now. We should retrieve the key and
	// it should make it into the cache automatically.

	req := gomatrixserverlib.PublicKeyLookupRequest{
		ServerName: serverB.name,
		KeyID:      serverKeyID,
	}
	ts := gomatrixserverlib.AsTimestamp(time.Now())

	res, err := serverA.api.FetchKeys(
		context.Background(),
		map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{
			req: ts,
		},
	)
	if err != nil {
		t.Fatalf("server A failed to retrieve server B key: %s", err)
	}
	if len(res) != 1 {
		t.Fatalf("server B should have returned one key but instead returned %d keys", len(res))
	}
	if _, ok := res[req]; !ok {
		t.Fatalf("server B isn't included in the key fetch response")
	}

	// At this point, if the previous key request was a success,
	// then the cache should now contain the key. Check if that's
	// the case - if it isn't then there's something wrong with
	// the cache implementation or we failed to get the key.

	cres, ok := serverA.cache.GetServerKey(req, ts)
	if !ok {
		t.Fatalf("server B key should be in cache but isn't")
	}
	if !reflect.DeepEqual(cres, res[req]) {
		t.Fatalf("the cached result from server B wasn't what server B gave us")
	}

	// If we ask the cache for the same key but this time for an event
	// that happened in +30 minutes. Since the validity period is for
	// another hour, then we should get a response back from the cache.

	_, ok = serverA.cache.GetServerKey(
		req,
		gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute*30)),
	)
	if !ok {
		t.Fatalf("server B key isn't in cache when it should be (+30 minutes)")
	}

	// If we ask the cache for the same key but this time for an event
	// that happened in +90 minutes then we should expect to get no
	// cache result. This is because the cache shouldn't return a result
	// that is obviously past the validity of the event.

	_, ok = serverA.cache.GetServerKey(
		req,
		gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute*90)),
	)
	if ok {
		t.Fatalf("server B key is in cache when it shouldn't be (+90 minutes)")
	}
}

func TestRenewalBehaviour(t *testing.T) {
	// Server A will request Server C's key but their validity period
	// is an hour in the past. We'll retrieve the key as, even though it's
	// past its validity, it will be able to verify past events.

	req := gomatrixserverlib.PublicKeyLookupRequest{
		ServerName: serverC.name,
		KeyID:      serverKeyID,
	}

	res, err := serverA.api.FetchKeys(
		context.Background(),
		map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{
			req: gomatrixserverlib.AsTimestamp(time.Now()),
		},
	)
	if err != nil {
		t.Fatalf("server A failed to retrieve server C key: %s", err)
	}
	if len(res) != 1 {
		t.Fatalf("server C should have returned one key but instead returned %d keys", len(res))
	}
	if _, ok := res[req]; !ok {
		t.Fatalf("server C isn't included in the key fetch response")
	}

	// If we ask the cache for the server key for an event that happened
	// 90 minutes ago then we should get a cache result, as the key hadn't
	// passed its validity by that point. The fact that the key is now in
	// the cache is, in itself, proof that we successfully retrieved the
	// key before.

	oldcached, ok := serverA.cache.GetServerKey(
		req,
		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*90)),
	)
	if !ok {
		t.Fatalf("server C key isn't in cache when it should be (-90 minutes)")
	}

	// If we now ask the cache for the same key but this time for an event
	// that only happened 30 minutes ago then we shouldn't get a cached
	// result, as the event happened after the key validity expired. This
	// is really just for sanity checking.

	_, ok = serverA.cache.GetServerKey(
		req,
		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*30)),
	)
	if ok {
		t.Fatalf("server B key is in cache when it shouldn't be (-30 minutes)")
	}

	// We're now going to kick server C into renewing its key. Since we're
	// happy at this point that the key that we already have is from the past
	// then repeating a key fetch should cause us to try and renew the key.
	// If so, then the new key will end up in our cache.

	serverC.renew()

	res, err = serverA.api.FetchKeys(
		context.Background(),
		map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{
			req: gomatrixserverlib.AsTimestamp(time.Now()),
		},
	)
	if err != nil {
		t.Fatalf("server A failed to retrieve server C key: %s", err)
	}
	if len(res) != 1 {
		t.Fatalf("server C should have returned one key but instead returned %d keys", len(res))
	}
	if _, ok = res[req]; !ok {
		t.Fatalf("server C isn't included in the key fetch response")
	}

	// We're now going to ask the cache what the new key validity is. If
	// it is still the same as the previous validity then we've failed to
	// retrieve the renewed key. If it's newer then we've successfully got
	// the renewed key.

	newcached, ok := serverA.cache.GetServerKey(
		req,
		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*30)),
	)
	if !ok {
		t.Fatalf("server B key isn't in cache when it shouldn't be (post-renewal)")
	}
	if oldcached.ValidUntilTS >= newcached.ValidUntilTS {
		t.Fatalf("the server B key should have been renewed but wasn't")
	}
	t.Log(res)
}
<|endoftext|>


<commit_before>package main

import (
	"strings"
	"testing"

	"gopkg.in/urfave/cli.v1"
)

func TestCommands_requirements(t *testing.T) {
	var cs, subcs []cli.Command
	for _, c := range Commands {
		if len(c.Subcommands) == 0 {
			cs = append(cs, c)
		} else {
			for _, sc := range c.Subcommands {
				cs = append(cs, sc)
			}
			subcs = append(subcs, c)
		}
	}
	for _, c := range cs {
		if !strings.HasPrefix(c.Description, "\n    ") {
			t.Errorf("%s: cli.Command.Description should start with '\\n    ', got:\n%s", c.Name, c.Description)
		}
		if !strings.HasSuffix(c.Description, "\n") {
			t.Errorf("%s: cli.Command.Description should end with '\\n', got:\n%s", c.Name, c.Description)
		}
		if len(c.Flags) > 0 && c.ArgsUsage == "" {
			t.Errorf("%s: cli.Command.ArgsUsage should not be empty. Describe flag options.", c.Name)
		}
	}
	for _, sc := range subcs {
		if sc.Description == "" {
			t.Errorf("%s: cli.Command.Description should not be empty", sc.Name)
		}
	}
}
<commit_msg>Remove Description convention tests<commit_after>package main

import (
	"testing"

	cli "gopkg.in/urfave/cli.v1"
)

func TestCommands_requirements(t *testing.T) {
	var cs, subcs []cli.Command
	for _, c := range Commands {
		if len(c.Subcommands) == 0 {
			cs = append(cs, c)
		} else {
			for _, sc := range c.Subcommands {
				cs = append(cs, sc)
			}
			subcs = append(subcs, c)
		}
	}
	for _, c := range cs {
		if len(c.Flags) > 0 && c.ArgsUsage == "" {
			t.Errorf("%s: cli.Command.ArgsUsage should not be empty. Describe flag options.", c.Name)
		}
	}
	for _, sc := range subcs {
		if sc.Description == "" {
			t.Errorf("%s: cli.Command.Description should not be empty", sc.Name)
		}
	}
}
<|endoftext|>


<commit_before>package system

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	httpd "github.com/rqlite/rqlite/http"
	"github.com/rqlite/rqlite/store"
)

// Node represents a node under test.
type Node struct {
	APIAddr  string
	RaftAddr string
	Dir      string
	Store    *store.Store
	Service  *httpd.Service
}

// SameAs returns true if this node is the same as node o.
func (n *Node) SameAs(o *Node) bool {
	return n.RaftAddr == o.RaftAddr
}

// Deprovision shuts down and removes all resources associated with the node.
func (n *Node) Deprovision() {
	n.Store.Close(false)
	n.Service.Close()
	os.RemoveAll(n.Dir)
}

// WaitForLeader blocks for up to 10 seconds until the node detects a leader.
func (n *Node) WaitForLeader() (string, error) {
	return n.Store.WaitForLeader(10 * time.Second)
}

// Execute executes a single statement against the node.
func (n *Node) Execute(stmt string) (string, error) {
	return n.ExecuteMulti([]string{stmt})
}

// ExecuteMulti executes multiple statements against the node.
func (n *Node) ExecuteMulti(stmts []string) (string, error) {
	j, err := json.Marshal(stmts)
	if err != nil {
		return "", err
	}
	return n.postExecute(string(j))
}

// Query runs a single query against the node.
func (n *Node) Query(stmt string) (string, error) {
	v, _ := url.Parse("http://" + n.APIAddr + "/db/query")
	v.RawQuery = url.Values{"q": []string{stmt}}.Encode()

	resp, err := http.Get(v.String())
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

// Join instructs this node to join the leader.
func (n *Node) Join(leader *Node) error {
	resp, err := DoJoinRequest(leader.APIAddr, n.RaftAddr)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return fmt.Errorf("failed to join, leader returned: %s", resp.Status)
	}
	defer resp.Body.Close()
	return nil
}

// Status returns the status and diagnostic output for node.
func (n *Node) Status() (string, error) {
	v, _ := url.Parse("http://" + n.APIAddr + "/status")

	resp, err := http.Get(v.String())
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

// Expvar returns the expvar output for node.
func (n *Node) Expvar() (string, error) {
	v, _ := url.Parse("http://" + n.APIAddr + "/debug/vars")

	resp, err := http.Get(v.String())
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

// ConfirmRedirect confirms that the node responds with a redirect to the given host.
func (n *Node) ConfirmRedirect(host string) bool {
	v, _ := url.Parse("http://" + n.APIAddr + "/db/query")
	v.RawQuery = url.Values{"q": []string{`SELECT * FROM foo`}}.Encode()

	resp, err := http.Get(v.String())
	if err != nil {
		return false
	}
	defer resp.Body.Close()
	fmt.Println(resp.StatusCode)
	if resp.StatusCode != http.StatusMovedPermanently {
		return false
	}
	if resp.Header.Get("location") != host {
		return false
	}
	return true
}

func (n *Node) postExecute(stmt string) (string, error) {
	resp, err := http.Post("http://"+n.APIAddr+"/db/execute", "application/json", strings.NewReader(stmt))
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

// Cluster represents a cluster of nodes.
type Cluster []*Node

// Leader returns the leader node of a cluster.
func (c Cluster) Leader() (*Node, error) {
	l, err := c[0].WaitForLeader()
	if err != nil {
		return nil, err
	}
	return c.FindNodeByRaftAddr(l)
}

// WaitForNewLeader waits for the leader to change from the node passed in.
func (c Cluster) WaitForNewLeader(old *Node) (*Node, error) {
	timer := time.NewTimer(30 * time.Second)
	defer timer.Stop()
	ticker := time.NewTicker(100 * time.Millisecond)
	defer ticker.Stop()

	for {
		select {
		case <-timer.C:
			return nil, fmt.Errorf("timed out waiting for new leader")
		case <-ticker.C:
			l, err := c.Leader()
			if err != nil {
				continue
			}
			if !l.SameAs(old) {
				return l, nil
			}
		}
	}
}

// Followers returns the slice of nodes in the cluster that are followers.
func (c Cluster) Followers() ([]*Node, error) {
	n, err := c[0].WaitForLeader()
	if err != nil {
		return nil, err
	}
	leader, err := c.FindNodeByRaftAddr(n)
	if err != nil {
		return nil, err
	}

	var followers []*Node
	for _, n := range c {
		if n != leader {
			followers = append(followers, n)
		}
	}
	return followers, nil
}

// RemoveNode removes the given node from the list of nodes representing
// a cluster.
func (c Cluster) RemoveNode(node *Node) {
	for i, n := range c {
		if n.RaftAddr == node.RaftAddr {
			c = append(c[:i], c[i+1:]...)
			return
		}
	}
}

// FindNodeByRaftAddr returns the node with the given Raft address.
func (c Cluster) FindNodeByRaftAddr(addr string) (*Node, error) {
	for _, n := range c {
		if n.RaftAddr == addr {
			return n, nil
		}
	}
	return nil, fmt.Errorf("node not found")
}

// Deprovision deprovisions every node in the cluster.
func (c Cluster) Deprovision() {
	for _, n := range c {
		n.Deprovision()
	}
}

// Remove tells the cluster at n to remove the node at addr. Assumes n is the leader.
func Remove(n *Node, addr string) error {
	b, err := json.Marshal(map[string]string{"addr": addr})
	if err != nil {
		return err
	}

	// Attempt to remove node from leader.
	resp, err := http.Post("http://"+n.APIAddr+"/remove", "application-type/json", bytes.NewReader(b))
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return fmt.Errorf("failed to remove node, leader returned: %s", resp.Status)
	}
	defer resp.Body.Close()
	return nil
}

// DoJoinRequest sends a join request to nodeAddr, for raftAddr.
func DoJoinRequest(nodeAddr, raftAddr string) (*http.Response, error) {
	b, err := json.Marshal(map[string]string{"addr": raftAddr})
	if err != nil {
		return nil, err
	}

	resp, err := http.Post("http://"+nodeAddr+"/join", "application-type/json", bytes.NewReader(b))
	if err != nil {
		return nil, err
	}

	return resp, nil
}

func mustNewNode(enableSingle bool) *Node {
	node := &Node{
		Dir: mustTempDir(),
	}

	dbConf := store.NewDBConfig("", false)
	node.Store = store.New(&store.StoreConfig{
		DBConf: dbConf,
		Dir:    node.Dir,
		Tn:     mustMockTransport("localhost:0"),
	})
	if err := node.Store.Open(enableSingle); err != nil {
		node.Deprovision()
		panic(fmt.Sprintf("failed to open store: %s", err.Error()))
	}
	node.RaftAddr = node.Store.Addr().String()

	node.Service = httpd.New("localhost:0", node.Store, nil)
	if err := node.Service.Start(); err != nil {
		node.Deprovision()
		panic(fmt.Sprintf("failed to start HTTP server: %s", err.Error()))
	}
	node.APIAddr = node.Service.Addr().String()

	return node
}

func mustNewLeaderNode() *Node {
	node := mustNewNode(true)
	if _, err := node.WaitForLeader(); err != nil {
		node.Deprovision()
		panic("node never became leader")
	}
	return node
}

type mockTransport struct {
	ln net.Listener
}

func mustMockTransport(addr string) *mockTransport {
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		panic("failed to create new transport")
	}
	return &mockTransport{ln}
}

func (m *mockTransport) Dial(addr string, timeout time.Duration) (net.Conn, error) {
	return net.DialTimeout("tcp", addr, timeout)
}

func (m *mockTransport) Accept() (net.Conn, error) { return m.ln.Accept() }

func (m *mockTransport) Close() error { return m.ln.Close() }

func (m *mockTransport) Addr() net.Addr { return m.ln.Addr() }

func mustTempDir() string {
	var err error
	path, err := ioutil.TempDir("", "rqlilte-system-test-")
	if err != nil {
		panic("failed to create temp dir")
	}
	return path
}

func isJSON(s string) bool {
	var js map[string]interface{}
	return json.Unmarshal([]byte(s), &js) == nil
}
<commit_msg>Enable expvar endpoint for testing<commit_after>package system

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	httpd "github.com/rqlite/rqlite/http"
	"github.com/rqlite/rqlite/store"
)

// Node represents a node under test.
type Node struct {
	APIAddr  string
	RaftAddr string
	Dir      string
	Store    *store.Store
	Service  *httpd.Service
}

// SameAs returns true if this node is the same as node o.
func (n *Node) SameAs(o *Node) bool {
	return n.RaftAddr == o.RaftAddr
}

// Deprovision shuts down and removes all resources associated with the node.
func (n *Node) Deprovision() {
	n.Store.Close(false)
	n.Service.Close()
	os.RemoveAll(n.Dir)
}

// WaitForLeader blocks for up to 10 seconds until the node detects a leader.
func (n *Node) WaitForLeader() (string, error) {
	return n.Store.WaitForLeader(10 * time.Second)
}

// Execute executes a single statement against the node.
func (n *Node) Execute(stmt string) (string, error) {
	return n.ExecuteMulti([]string{stmt})
}

// ExecuteMulti executes multiple statements against the node.
func (n *Node) ExecuteMulti(stmts []string) (string, error) {
	j, err := json.Marshal(stmts)
	if err != nil {
		return "", err
	}
	return n.postExecute(string(j))
}

// Query runs a single query against the node.
func (n *Node) Query(stmt string) (string, error) {
	v, _ := url.Parse("http://" + n.APIAddr + "/db/query")
	v.RawQuery = url.Values{"q": []string{stmt}}.Encode()

	resp, err := http.Get(v.String())
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

// Join instructs this node to join the leader.
func (n *Node) Join(leader *Node) error {
	resp, err := DoJoinRequest(leader.APIAddr, n.RaftAddr)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return fmt.Errorf("failed to join, leader returned: %s", resp.Status)
	}
	defer resp.Body.Close()
	return nil
}

// Status returns the status and diagnostic output for node.
func (n *Node) Status() (string, error) {
	v, _ := url.Parse("http://" + n.APIAddr + "/status")

	resp, err := http.Get(v.String())
	if err != nil {
		return "", err
	}
	if resp.StatusCode != 200 {
		return "", fmt.Errorf("status endpoint returned: %s", resp.Status)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

// Expvar returns the expvar output for node.
func (n *Node) Expvar() (string, error) {
	v, _ := url.Parse("http://" + n.APIAddr + "/debug/vars")

	resp, err := http.Get(v.String())
	if err != nil {
		return "", err
	}
	if resp.StatusCode != 200 {
		return "", fmt.Errorf("expvar endpoint returned: %s", resp.Status)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

// ConfirmRedirect confirms that the node responds with a redirect to the given host.
func (n *Node) ConfirmRedirect(host string) bool {
	v, _ := url.Parse("http://" + n.APIAddr + "/db/query")
	v.RawQuery = url.Values{"q": []string{`SELECT * FROM foo`}}.Encode()

	resp, err := http.Get(v.String())
	if err != nil {
		return false
	}
	defer resp.Body.Close()
	fmt.Println(resp.StatusCode)
	if resp.StatusCode != http.StatusMovedPermanently {
		return false
	}
	if resp.Header.Get("location") != host {
		return false
	}
	return true
}

func (n *Node) postExecute(stmt string) (string, error) {
	resp, err := http.Post("http://"+n.APIAddr+"/db/execute", "application/json", strings.NewReader(stmt))
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

// Cluster represents a cluster of nodes.
type Cluster []*Node

// Leader returns the leader node of a cluster.
func (c Cluster) Leader() (*Node, error) {
	l, err := c[0].WaitForLeader()
	if err != nil {
		return nil, err
	}
	return c.FindNodeByRaftAddr(l)
}

// WaitForNewLeader waits for the leader to change from the node passed in.
func (c Cluster) WaitForNewLeader(old *Node) (*Node, error) {
	timer := time.NewTimer(30 * time.Second)
	defer timer.Stop()
	ticker := time.NewTicker(100 * time.Millisecond)
	defer ticker.Stop()

	for {
		select {
		case <-timer.C:
			return nil, fmt.Errorf("timed out waiting for new leader")
		case <-ticker.C:
			l, err := c.Leader()
			if err != nil {
				continue
			}
			if !l.SameAs(old) {
				return l, nil
			}
		}
	}
}

// Followers returns the slice of nodes in the cluster that are followers.
func (c Cluster) Followers() ([]*Node, error) {
	n, err := c[0].WaitForLeader()
	if err != nil {
		return nil, err
	}
	leader, err := c.FindNodeByRaftAddr(n)
	if err != nil {
		return nil, err
	}

	var followers []*Node
	for _, n := range c {
		if n != leader {
			followers = append(followers, n)
		}
	}
	return followers, nil
}

// RemoveNode removes the given node from the list of nodes representing
// a cluster.
func (c Cluster) RemoveNode(node *Node) {
	for i, n := range c {
		if n.RaftAddr == node.RaftAddr {
			c = append(c[:i], c[i+1:]...)
			return
		}
	}
}

// FindNodeByRaftAddr returns the node with the given Raft address.
func (c Cluster) FindNodeByRaftAddr(addr string) (*Node, error) {
	for _, n := range c {
		if n.RaftAddr == addr {
			return n, nil
		}
	}
	return nil, fmt.Errorf("node not found")
}

// Deprovision deprovisions every node in the cluster.
func (c Cluster) Deprovision() {
	for _, n := range c {
		n.Deprovision()
	}
}

// Remove tells the cluster at n to remove the node at addr. Assumes n is the leader.
func Remove(n *Node, addr string) error {
	b, err := json.Marshal(map[string]string{"addr": addr})
	if err != nil {
		return err
	}

	// Attempt to remove node from leader.
	resp, err := http.Post("http://"+n.APIAddr+"/remove", "application-type/json", bytes.NewReader(b))
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return fmt.Errorf("failed to remove node, leader returned: %s", resp.Status)
	}
	defer resp.Body.Close()
	return nil
}

// DoJoinRequest sends a join request to nodeAddr, for raftAddr.
func DoJoinRequest(nodeAddr, raftAddr string) (*http.Response, error) {
	b, err := json.Marshal(map[string]string{"addr": raftAddr})
	if err != nil {
		return nil, err
	}

	resp, err := http.Post("http://"+nodeAddr+"/join", "application-type/json", bytes.NewReader(b))
	if err != nil {
		return nil, err
	}

	return resp, nil
}

func mustNewNode(enableSingle bool) *Node {
	node := &Node{
		Dir: mustTempDir(),
	}

	dbConf := store.NewDBConfig("", false)
	node.Store = store.New(&store.StoreConfig{
		DBConf: dbConf,
		Dir:    node.Dir,
		Tn:     mustMockTransport("localhost:0"),
	})
	if err := node.Store.Open(enableSingle); err != nil {
		node.Deprovision()
		panic(fmt.Sprintf("failed to open store: %s", err.Error()))
	}
	node.RaftAddr = node.Store.Addr().String()

	node.Service = httpd.New("localhost:0", node.Store, nil)
	node.Service.Expvar = true
	if err := node.Service.Start(); err != nil {
		node.Deprovision()
		panic(fmt.Sprintf("failed to start HTTP server: %s", err.Error()))
	}
	node.APIAddr = node.Service.Addr().String()

	return node
}

func mustNewLeaderNode() *Node {
	node := mustNewNode(true)
	if _, err := node.WaitForLeader(); err != nil {
		node.Deprovision()
		panic("node never became leader")
	}
	return node
}

type mockTransport struct {
	ln net.Listener
}

func mustMockTransport(addr string) *mockTransport {
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		panic("failed to create new transport")
	}
	return &mockTransport{ln}
}

func (m *mockTransport) Dial(addr string, timeout time.Duration) (net.Conn, error) {
	return net.DialTimeout("tcp", addr, timeout)
}

func (m *mockTransport) Accept() (net.Conn, error) { return m.ln.Accept() }

func (m *mockTransport) Close() error { return m.ln.Close() }

func (m *mockTransport) Addr() net.Addr { return m.ln.Addr() }

func mustTempDir() string {
	var err error
	path, err := ioutil.TempDir("", "rqlilte-system-test-")
	if err != nil {
		panic("failed to create temp dir")
	}
	return path
}

func isJSON(s string) bool {
	var js map[string]interface{}
	return json.Unmarshal([]byte(s), &js) == nil
}
<|endoftext|>


<commit_before>package bot

import (
	"fmt"

	"github.com/bwmarrin/discordgo"
	"go.uber.org/zap"
)

type EmojiRole struct {
	EmojiId string `yaml:"emojiId"`
	RoleId  string `yaml:"roleId"`
}

type DiscordEmojiRoleGroup struct {
	MessageTitle string      `yaml:"messageTitle"`
	MessageBody  string      `yaml:"messageBody"`
	Roles        []EmojiRole `yaml:"roles"`
}

type DiscordRoleCfg struct {
	ChannelId       string                  `yaml:"channelId"`
	EmojiRoleGroups []DiscordEmojiRoleGroup `yaml:"emojiRoles"`
}

func (d *DiscordAPI) StartRoleHandlers() {
	d.listRoles()
	d.clearRoleChannel()
	d.createRoleMessages()
	d.discord.AddHandler(d.roleAssignmentHandler)
	Logger.Info("Role assignment handler started")
}

func (d DiscordAPI) roleAssignmentHandler(s *discordgo.Session, event *discordgo.MessageReactionAdd) {

	// skip if the event was from the bot/app
	if event.UserID == d.Config.ClientId {
		return
	}

	// only handle if the message is one we have configured
	if roles, ok := d.assignmentMsgs[event.MessageID]; ok {

		// Unicode emojis use the unicode character (name) as the id. Others use the name and integer as the id.
		emojiId := event.Emoji.Name
		if event.Emoji.ID != "" {
			emojiId = fmt.Sprintf(":%s:%s", event.Emoji.Name, event.Emoji.ID)
		}

		if roleId, ok := roles[emojiId]; ok {
			// get the user from the server/guild
			member, err := d.discord.GuildMember(d.Config.GuildId, event.UserID)
			if err != nil {
				Logger.Error("Unable to get member", zap.Error(err))
				return
			}

			userHadRole := false
			// if the member has the mapped role, remove it
			for _, id := range member.Roles {
				if id == roleId {
					d.removeRoleFromUser(event.UserID, roleId)
					userHadRole = true // true, even if an attempt was made and failed
				}
			}

			// if the member does not have the mapped role, add it
			if !userHadRole {
				d.addRoleToUser(event.UserID, roleId)
			}

			// DM the user the confirmation
			role, err := d.FindGuildRole(roleId)
			if err != nil {
				Logger.Error("Unable to find role", zap.Error(err))
			}
			if userHadRole {
				d.SendDM(event.UserID, fmt.Sprintf("The %s role has been removed", role.Name))
			} else {
				d.SendDM(event.UserID, fmt.Sprintf("You now have the %s role!", role.Name))
			}

		}

		// remove the users reaction. If the add/remove failed, they can click it again to re-trigger
		err := d.discord.MessageReactionRemove(
			event.ChannelID,
			event.MessageID,
			emojiId,
			event.UserID,
		)

		if err != nil {
			Logger.Error("could not remove reaction from message",
				zap.String("user", event.UserID),
				zap.String("emoji", emojiId),
				zap.Error(err))
		}
	}

}

// removes all messages entered by this bot in the channel. Uses the ClientID of the bot/app
func (d *DiscordAPI) clearRoleChannel() {
	channelId := d.Config.RoleCfg.ChannelId

	messages, err := d.discord.ChannelMessages(channelId, 50, "", "", "")
	if err != nil {
		Logger.Error("Unable to access channel messages", zap.String("channelId", channelId), zap.Error(err))
		return
	}
	for _, message := range messages {
		if message.Author.ID == d.Config.ClientId {
			err := d.discord.ChannelMessageDelete(channelId, message.ID)
			if err == nil {
				Logger.Info("Removed bot message", zap.String("messageId", message.ID))
			} else {
				Logger.Error("Unable to remove message", zap.String("messageId", message.ID), zap.Error(err))
			}

		}
	}
}

// creates messages in the channel and adds the emojis
func (d *DiscordAPI) createRoleMessages() {
	d.assignmentMsgs = make(map[string]map[string]string)
	for _, group := range d.Config.RoleCfg.EmojiRoleGroups {
		messageEmbed := discordgo.MessageEmbed{
			Title:       group.MessageTitle,
			Description: group.MessageBody,
			Color:       15581239,
		}
		// create the message
		message, err := d.discord.ChannelMessageSendEmbed(d.Config.RoleCfg.ChannelId, &messageEmbed)
		if err != nil {
			Logger.Error("Unable to create message", zap.Error(err))
			continue
		}
		Logger.Info("Added role message", zap.String("message", group.MessageTitle), zap.String("messageId", message.ID))

		// add the emojis
		emojiRoles := make(map[string]string)
		for _, role := range group.Roles {
			emojiRoles[role.EmojiId] = role.RoleId
			err := d.discord.MessageReactionAdd(d.Config.RoleCfg.ChannelId, message.ID, role.EmojiId)
			if err != nil {
				Logger.Error("Unable to add emoji to message",
					zap.String("emojiId", role.EmojiId),
					zap.String("messageId", message.ID),
					zap.Error(err),
				)
			}
		}

		d.assignmentMsgs[message.ID] = emojiRoles
	}
}

func (d DiscordAPI) addRoleToUser(userId string, roleId string) {
	err := d.discord.GuildMemberRoleAdd(d.Config.GuildId, userId, roleId)
	if err != nil {
		Logger.Error("could not add role",
			zap.String("userId", userId),
			zap.String("roleId", roleId),
			zap.Error(err))
	} else {
		Logger.Info("added role to user",
			zap.String("userId", userId),
			zap.String("roleId", roleId),
		)
	}
}

func (d DiscordAPI) removeRoleFromUser(userId string, roleId string) {
	err := d.discord.GuildMemberRoleRemove(d.Config.GuildId, userId, roleId)
	if err != nil {
		Logger.Error("could not remove role", zap.Error(err))
	} else {
		Logger.Info("removed role from user",
			zap.String("userId", userId),
			zap.String("roleId", roleId),
		)
	}
}
func (d *DiscordAPI) listRoles() {
	roles, err := d.discord.GuildRoles(d.Config.GuildId)

	if err != nil {
		Logger.Error("Could not get roles", zap.Error(err))
		return
	}

	for i, role := range roles {
		Logger.Info(fmt.Sprintf("Role %d", i), zap.Any("role", role))
	}
}
<commit_msg>Refactoring role assignment for reusability<commit_after>package bot

import (
	"fmt"

	"github.com/bwmarrin/discordgo"
	"go.uber.org/zap"
)

type EmojiRole struct {
	EmojiId string `yaml:"emojiId"`
	RoleId  string `yaml:"roleId"`
}

type DiscordEmojiRoleGroup struct {
	MessageTitle string      `yaml:"messageTitle"`
	MessageBody  string      `yaml:"messageBody"`
	Roles        []EmojiRole `yaml:"roles"`
}

type DiscordRoleCfg struct {
	ChannelId       string                  `yaml:"channelId"`
	EmojiRoleGroups []DiscordEmojiRoleGroup `yaml:"emojiRoles"`
}

func (d *DiscordAPI) StartRoleHandlers() {
	d.listRoles()
	d.clearRoleChannel()
	d.createRoleMessages()
	d.discord.AddHandler(d.roleAssignmentHandler)
	Logger.Info("Role assignment handler started")
}

func (d DiscordAPI) roleAssignmentHandler(s *discordgo.Session, event *discordgo.MessageReactionAdd) {

	// skip if the event was from the bot/app
	if event.UserID == d.Config.ClientId {
		return
	}

	// only handle if the message is one we have configured
	if roles, ok := d.assignmentMsgs[event.MessageID]; ok {

		// Unicode emojis use the unicode character (name) as the id. Others use the name and integer as the id.
		emojiId := event.Emoji.Name
		if event.Emoji.ID != "" {
			emojiId = fmt.Sprintf(":%s:%s", event.Emoji.Name, event.Emoji.ID)
		}

		if roleId, ok := roles[emojiId]; ok {
			d.assignRoleToUser(event.UserID, roleId)
		}

		// remove the users reaction. If the add/remove failed, they can click it again to re-trigger
		err := d.discord.MessageReactionRemove(
			event.ChannelID,
			event.MessageID,
			emojiId,
			event.UserID,
		)

		if err != nil {
			Logger.Error("could not remove reaction from message",
				zap.String("user", event.UserID),
				zap.String("emoji", emojiId),
				zap.Error(err))
		}
	}

}

func (d *DiscordAPI) assignRoleToUser(userID, roleID string ) {
	// get the user from the server/guild
	member, err := d.discord.GuildMember(d.Config.GuildId, userID)
	if err != nil {
		Logger.Error("Unable to get member", zap.Error(err))
		return
	}

	userHadRole := false
	// if the member has the mapped role, remove it
	for _, id := range member.Roles {
		if id == roleID {
			d.removeRoleFromUser(userID, roleID)
			userHadRole = true // true, even if an attempt was made and failed
		}
	}

	// if the member does not have the mapped role, add it
	if !userHadRole {
		d.addRoleToUser(userID, roleID)
	}

	// DM the user the confirmation
	role, err := d.FindGuildRole(roleID)
	if err != nil {
		Logger.Error("Unable to find role", zap.Error(err))
	}
	if userHadRole {
		d.SendDM(userID, fmt.Sprintf("The %s role has been removed", role.Name))
	} else {
		d.SendDM(userID, fmt.Sprintf("You now have the %s role!", role.Name))
	}
}

// removes all messages entered by this bot in the channel. Uses the ClientID of the bot/app
func (d *DiscordAPI) clearRoleChannel() {
	channelId := d.Config.RoleCfg.ChannelId

	messages, err := d.discord.ChannelMessages(channelId, 50, "", "", "")
	if err != nil {
		Logger.Error("Unable to access channel messages", zap.String("channelId", channelId), zap.Error(err))
		return
	}
	for _, message := range messages {
		if message.Author.ID == d.Config.ClientId {
			err := d.discord.ChannelMessageDelete(channelId, message.ID)
			if err == nil {
				Logger.Info("Removed bot message", zap.String("messageId", message.ID))
			} else {
				Logger.Error("Unable to remove message", zap.String("messageId", message.ID), zap.Error(err))
			}

		}
	}
}

// creates messages in the channel and adds the emojis
func (d *DiscordAPI) createRoleMessages() {
	d.assignmentMsgs = make(map[string]map[string]string)
	for _, group := range d.Config.RoleCfg.EmojiRoleGroups {
		messageEmbed := discordgo.MessageEmbed{
			Title:       group.MessageTitle,
			Description: group.MessageBody,
			Color:       15581239,
		}
		// create the message
		message, err := d.discord.ChannelMessageSendEmbed(d.Config.RoleCfg.ChannelId, &messageEmbed)
		if err != nil {
			Logger.Error("Unable to create message", zap.Error(err))
			continue
		}
		Logger.Info("Added role message", zap.String("message", group.MessageTitle), zap.String("messageId", message.ID))

		// add the emojis
		emojiRoles := make(map[string]string)
		for _, role := range group.Roles {
			emojiRoles[role.EmojiId] = role.RoleId
			err := d.discord.MessageReactionAdd(d.Config.RoleCfg.ChannelId, message.ID, role.EmojiId)
			if err != nil {
				Logger.Error("Unable to add emoji to message",
					zap.String("emojiId", role.EmojiId),
					zap.String("messageId", message.ID),
					zap.Error(err),
				)
			}
		}

		d.assignmentMsgs[message.ID] = emojiRoles
	}
}

func (d DiscordAPI) addRoleToUser(userId string, roleId string) {
	err := d.discord.GuildMemberRoleAdd(d.Config.GuildId, userId, roleId)
	if err != nil {
		Logger.Error("could not add role",
			zap.String("userId", userId),
			zap.String("roleId", roleId),
			zap.Error(err))
	} else {
		Logger.Info("added role to user",
			zap.String("userId", userId),
			zap.String("roleId", roleId),
		)
	}
}

func (d DiscordAPI) removeRoleFromUser(userId string, roleId string) {
	err := d.discord.GuildMemberRoleRemove(d.Config.GuildId, userId, roleId)
	if err != nil {
		Logger.Error("could not remove role", zap.Error(err))
	} else {
		Logger.Info("removed role from user",
			zap.String("userId", userId),
			zap.String("roleId", roleId),
		)
	}
}
func (d *DiscordAPI) listRoles() {
	roles, err := d.discord.GuildRoles(d.Config.GuildId)

	if err != nil {
		Logger.Error("Could not get roles", zap.Error(err))
		return
	}

	for i, role := range roles {
		Logger.Info(fmt.Sprintf("Role %d", i), zap.Any("role", role))
	}
}
<|endoftext|>


<commit_before>package common

import (
	"encoding/base64"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"reflect"
	"strings"
)

func IsExist(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil
}

func MapReduce(map1, map2 map[string]interface{}) map[string]interface{} {
	for ia, va := range map1 {
		switch value := va.(type) {
		case string:
			if value != "" && map2[ia] == "" {
				map2[ia] = value
			}
		case []string:
			map2Value := reflect.ValueOf(map2[ia])
			if len(value) > 0 && map2Value.Len() == 0 {
				map2[ia] = value
			}
		case bool:
			map2Value := reflect.ValueOf(map2[ia])
			if value == true && map2Value.Bool() == false {
				map2[ia] = value
			}
		}
	}

	return map2
}

func StructToMap(val interface{}) (mapVal map[string]interface{}, ok bool) {
	structVal := reflect.Indirect(reflect.ValueOf(val))
	typ := structVal.Type()

	mapVal = make(map[string]interface{})

	for i := 0; i < typ.NumField(); i++ {
		field := structVal.Field(i)

		if field.CanSet() {
			mapVal[typ.Field(i).Name] = field.Interface()
		}
	}

	return
}

func MapToStruct(mapVal map[string]interface{}, val interface{}) (ok bool) {
	structVal := reflect.Indirect(reflect.ValueOf(val))
	for name, elem := range mapVal {
		structVal.FieldByName(name).Set(reflect.ValueOf(elem))
	}

	return
}

func GetFullPath(path string) (fullPath string) {
	usr, _ := user.Current()
	fullPath = strings.Replace(path, "~", usr.HomeDir, 1)
	fullPath, _ = filepath.Abs(fullPath)
	return fullPath
}

func GetMaxLength(list []string) (MaxLength int) {
	MaxLength = 0
	for _, elem := range list {
		if MaxLength < len(elem) {
			MaxLength = len(elem)
		}
	}
	return
}

func GetFilesBase64(list []string) (result string, err error) {
	var data []byte
	for _, path := range list {

		fullPath := GetFullPath(path)

		// open file
		file, err := os.Open(fullPath)
		if err != nil {
			return "", err
		}
		defer file.Close()

		file_data, err := ioutil.ReadAll(file)
		if err != nil {
			return "", err
		}

		data = append(data, file_data...)
	}

	result = base64.StdEncoding.EncodeToString(data)
	return result, err
}
<commit_msg>rcfile add newline<commit_after>package common

import (
	"encoding/base64"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"reflect"
	"strings"
)

func IsExist(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil
}

func MapReduce(map1, map2 map[string]interface{}) map[string]interface{} {
	for ia, va := range map1 {
		switch value := va.(type) {
		case string:
			if value != "" && map2[ia] == "" {
				map2[ia] = value
			}
		case []string:
			map2Value := reflect.ValueOf(map2[ia])
			if len(value) > 0 && map2Value.Len() == 0 {
				map2[ia] = value
			}
		case bool:
			map2Value := reflect.ValueOf(map2[ia])
			if value == true && map2Value.Bool() == false {
				map2[ia] = value
			}
		}
	}

	return map2
}

func StructToMap(val interface{}) (mapVal map[string]interface{}, ok bool) {
	structVal := reflect.Indirect(reflect.ValueOf(val))
	typ := structVal.Type()

	mapVal = make(map[string]interface{})

	for i := 0; i < typ.NumField(); i++ {
		field := structVal.Field(i)

		if field.CanSet() {
			mapVal[typ.Field(i).Name] = field.Interface()
		}
	}

	return
}

func MapToStruct(mapVal map[string]interface{}, val interface{}) (ok bool) {
	structVal := reflect.Indirect(reflect.ValueOf(val))
	for name, elem := range mapVal {
		structVal.FieldByName(name).Set(reflect.ValueOf(elem))
	}

	return
}

func GetFullPath(path string) (fullPath string) {
	usr, _ := user.Current()
	fullPath = strings.Replace(path, "~", usr.HomeDir, 1)
	fullPath, _ = filepath.Abs(fullPath)
	return fullPath
}

func GetMaxLength(list []string) (MaxLength int) {
	MaxLength = 0
	for _, elem := range list {
		if MaxLength < len(elem) {
			MaxLength = len(elem)
		}
	}
	return
}

func GetFilesBase64(list []string) (result string, err error) {
	var data []byte
	for _, path := range list {

		fullPath := GetFullPath(path)

		// open file
		file, err := os.Open(fullPath)
		if err != nil {
			return "", err
		}
		defer file.Close()

		file_data, err := ioutil.ReadAll(file)
		if err != nil {
			return "", err
		}

		data = append(data, file_data...)
		data = append(data, '\n')
	}

	result = base64.StdEncoding.EncodeToString(data)
	return result, err
}
<|endoftext|>


<commit_before>/*
	@author Robert
*/

package common

import (
	"errors"
	"math/rand"
	"reflect"
	"regexp"
	"strings"
	"unsafe"
)

var (
	letterAndNumberRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890")

	// email validation regular expression
	emailRegEx = regexp.MustCompile(`^[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,4}$`)
)

// IsEmailAddress returns true if str seems to be an email address
func IsEmailAddress(str string) bool {
	return emailRegEx.MatchString(str)
}

// UpdateStructFields() errors
var (
	ErrNotStruct = errors.New("Destination must by struct or a pointer to struct")
)

// UpdateStructFromMap can be used to update the fields of a structure by sending
// the new field values in a map, wherer the key is the field name as in the struct
// and the value is the new value that will be set.
//
// Only those values in the map will be evaluated and updated.
// Parameter destination must be a pointer, otherwise changes won't be reflected.
func UpdateStructFromMap(destination interface{}, source map[string]interface{}) (err error) {

	// to avoid panic, type of 'destination' must be a struct or a pointer
	if reflect.TypeOf(destination).Kind() == reflect.Struct || reflect.TypeOf(destination).Kind() == reflect.Ptr {
		if ps := reflect.ValueOf(destination); ps.IsValid() {
			s := ps.Elem()

			if s.Kind() == reflect.Struct {
				for k, v := range source {
					// retrieve field from struct
					if field := s.FieldByName(k); field.IsValid() {

						if field.CanSet() {

							// set field value based on the type
							// note: not all types are supported, so test and add as needed
							switch field.Kind() {

							case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
								field.SetInt(v.(int64))
							case reflect.String:
								field.SetString(v.(string))
							case reflect.Bool:
								field.SetBool(v.(bool))
							case reflect.Float32, reflect.Float64:
								field.SetFloat(v.(float64))
							case reflect.Ptr:
								field.SetPointer(v.(unsafe.Pointer))
							case reflect.Complex64, reflect.Complex128:
								field.SetComplex(v.(complex128))
							}
						}
					}
				}
			} else {
				err = ErrNotStruct
			}
		}
	} else {
		err = ErrNotStruct
	}

	return
}

// Random generates a random number between min and max.
// Keep in mind that random seed must be initialized before. Example:
// 		rand.Seed(time.Now().Unix())
func Random(min, max int) int {
	return rand.Intn(max-min) + min
}

// RandomString generates a random string of the specified length.
// Keep in mind that random seed must be initialized before. Example:
// 		rand.Seed(time.Now().Unix())
func RandomString(n int) string {
	b := make([]rune, n)
	for i := range b {
		b[i] = letterAndNumberRunes[rand.Intn(len(letterAndNumberRunes))]
	}
	return string(b)
}

// MaskString creates a mask with `maskChar` for the indicated string `s`.
// If noMaskLeft and noMaskRight equals -1, then all the string is masked.
func MaskString(s string, noMaskLeft, noMaskRight int, maskChar string) (masked string) {

	// return all masked string if applies
	if len(s) <= noMaskLeft+noMaskRight || (noMaskLeft == -1 && noMaskRight == -1) {
		masked = strings.Repeat(maskChar, len(s))
		return
	}

	if noMaskLeft == -1 {
		noMaskLeft = 0
	}

	if noMaskRight == -1 {
		noMaskRight = 0
	}

	sLen := len(s)

	leftStr := s[:noMaskLeft]
	rightStr := s[sLen-noMaskRight:]
	middle := strings.Repeat(maskChar, sLen-len(leftStr)-len(rightStr))

	masked = leftStr + middle + rightStr
	return
}

// FindInStringArray searches for the indicated element 'elem' in array 'a'
func FindInStringArray(elem string, a []string) (e string, found bool) {
	for i := range a {
		if a[i] == elem {
			return elem, true
		}
	}

	return "", false
}

// FindInIntgArray searches for the indicated element 'elem' in array 'a'
func FindInIntgArray(elem int64, a []int64) (e int64, found bool) {
	for i := range a {
		if a[i] == elem {
			return elem, true
		}
	}

	return 0, false
}

// GetNonNullFields returns an array with all the fields that
// aren't nil in the structure's instance
func GetNonNullFields(i interface{}, tagName string) (fields []string) {

	var e reflect.Value
	v := reflect.ValueOf(i)

	if v.Kind() == reflect.Struct {
		e = v
	} else if v.Kind() == reflect.Ptr {
		e = v.Elem()
	} else {
		// non applicable
		return
	}

	for f := 0; f < e.NumField(); f++ {
		process := false
		fieldInstance := e.Type().Field(f)
		fieldKind := e.Type().Kind()

		// skip structs and pointers to structs
		switch fieldKind {
		case reflect.Ptr:
			if e.Field(f).Elem().Kind() != reflect.Struct {
				process = true
			}
		default:
			process = true
		}

		if process {
			if !e.Field(f).IsNil() {
				if tagName != "" {
					if jsonTag := fieldInstance.Tag.Get(tagName); jsonTag != "" && jsonTag != "-" {
						fieldName := strings.Split(jsonTag, ",")[0]
						fields = append(fields, fieldName)
					}
				} else {
					fields = append(fields, fieldInstance.Name)
				}
			}
		}
	}

	return
}
<commit_msg>use random source for number generation<commit_after>/*
	@author Robert
*/

package common

import (
	"errors"
	"math/rand"
	"reflect"
	"regexp"
	"strings"
	"time"
	"unsafe"
)

var (
	letterAndNumberRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890")

	// email validation regular expression
	emailRegEx = regexp.MustCompile(`^[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,4}$`)

	randomGenerator = rand.New(rand.NewSource(time.Now().Unix()))
)

// IsEmailAddress returns true if str seems to be an email address
func IsEmailAddress(str string) bool {
	return emailRegEx.MatchString(str)
}

// UpdateStructFields() errors
var (
	ErrNotStruct = errors.New("Destination must by struct or a pointer to struct")
)

// UpdateStructFromMap can be used to update the fields of a structure by sending
// the new field values in a map, wherer the key is the field name as in the struct
// and the value is the new value that will be set.
//
// Only those values in the map will be evaluated and updated.
// Parameter destination must be a pointer, otherwise changes won't be reflected.
func UpdateStructFromMap(destination interface{}, source map[string]interface{}) (err error) {

	// to avoid panic, type of 'destination' must be a struct or a pointer
	if reflect.TypeOf(destination).Kind() == reflect.Struct || reflect.TypeOf(destination).Kind() == reflect.Ptr {
		if ps := reflect.ValueOf(destination); ps.IsValid() {
			s := ps.Elem()

			if s.Kind() == reflect.Struct {
				for k, v := range source {
					// retrieve field from struct
					if field := s.FieldByName(k); field.IsValid() {

						if field.CanSet() {

							// set field value based on the type
							// note: not all types are supported, so test and add as needed
							switch field.Kind() {

							case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
								field.SetInt(v.(int64))
							case reflect.String:
								field.SetString(v.(string))
							case reflect.Bool:
								field.SetBool(v.(bool))
							case reflect.Float32, reflect.Float64:
								field.SetFloat(v.(float64))
							case reflect.Ptr:
								field.SetPointer(v.(unsafe.Pointer))
							case reflect.Complex64, reflect.Complex128:
								field.SetComplex(v.(complex128))
							}
						}
					}
				}
			} else {
				err = ErrNotStruct
			}
		}
	} else {
		err = ErrNotStruct
	}

	return
}

// Random generates a random number between min and max.
// Keep in mind that random seed must be initialized before. Example:
// 		rand.Seed(time.Now().Unix())
func Random(min, max int) int {
	return rand.Intn(max-min) + min
}

// RandomString generates a random string of the specified length.
// Keep in mind that random seed must be initialized before. Example:
// 		rand.Seed(time.Now().Unix())
func RandomString(n int) string {
	b := make([]rune, n)
	for i := range b {
		b[i] = letterAndNumberRunes[rand.Intn(len(letterAndNumberRunes))]
	}
	return string(b)
}

// MaskString creates a mask with `maskChar` for the indicated string `s`.
// If noMaskLeft and noMaskRight equals -1, then all the string is masked.
func MaskString(s string, noMaskLeft, noMaskRight int, maskChar string) (masked string) {

	// return all masked string if applies
	if len(s) <= noMaskLeft+noMaskRight || (noMaskLeft == -1 && noMaskRight == -1) {
		masked = strings.Repeat(maskChar, len(s))
		return
	}

	if noMaskLeft == -1 {
		noMaskLeft = 0
	}

	if noMaskRight == -1 {
		noMaskRight = 0
	}

	sLen := len(s)

	leftStr := s[:noMaskLeft]
	rightStr := s[sLen-noMaskRight:]
	middle := strings.Repeat(maskChar, sLen-len(leftStr)-len(rightStr))

	masked = leftStr + middle + rightStr
	return
}

// FindInStringArray searches for the indicated element 'elem' in array 'a'
func FindInStringArray(elem string, a []string) (e string, found bool) {
	for i := range a {
		if a[i] == elem {
			return elem, true
		}
	}

	return "", false
}

// FindInIntgArray searches for the indicated element 'elem' in array 'a'
func FindInIntgArray(elem int64, a []int64) (e int64, found bool) {
	for i := range a {
		if a[i] == elem {
			return elem, true
		}
	}

	return 0, false
}

// GetNonNullFields returns an array with all the fields that
// aren't nil in the structure's instance
func GetNonNullFields(i interface{}, tagName string) (fields []string) {

	var e reflect.Value
	v := reflect.ValueOf(i)

	if v.Kind() == reflect.Struct {
		e = v
	} else if v.Kind() == reflect.Ptr {
		e = v.Elem()
	} else {
		// non applicable
		return
	}

	for f := 0; f < e.NumField(); f++ {
		process := false
		fieldInstance := e.Type().Field(f)
		fieldKind := e.Type().Kind()

		// skip structs and pointers to structs
		switch fieldKind {
		case reflect.Ptr:
			if e.Field(f).Elem().Kind() != reflect.Struct {
				process = true
			}
		default:
			process = true
		}

		if process {
			if !e.Field(f).IsNil() {
				if tagName != "" {
					if jsonTag := fieldInstance.Tag.Get(tagName); jsonTag != "" && jsonTag != "-" {
						fieldName := strings.Split(jsonTag, ",")[0]
						fields = append(fields, fieldName)
					}
				} else {
					fields = append(fields, fieldInstance.Name)
				}
			}
		}
	}

	return
}
<|endoftext|>