<commit_before>// +build debug

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"os"

	"restic"
	"restic/pack"
	"restic/repository"

	"restic/worker"
)

type CmdDump struct {
	global *GlobalOptions

	repo *repository.Repository
}

func init() {
	_, err := parser.AddCommand("dump",
		"dump data structures",
		"The dump command dumps data structures from a repository as JSON documents",
		&CmdDump{global: &globalOpts})
	if err != nil {
		panic(err)
	}
}

func (cmd CmdDump) Usage() string {
	return "[indexes|snapshots|trees|all|packs]"
}

func prettyPrintJSON(wr io.Writer, item interface{}) error {
	buf, err := json.MarshalIndent(item, "", "  ")
	if err != nil {
		return err
	}

	_, err = wr.Write(append(buf, '\n'))
	return err
}

func debugPrintSnapshots(repo *repository.Repository, wr io.Writer) error {
	done := make(chan struct{})
	defer close(done)

	for id := range repo.List(restic.SnapshotFile, done) {
		snapshot, err := restic.LoadSnapshot(repo, id)
		if err != nil {
			fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err)
			continue
		}

		fmt.Fprintf(wr, "snapshot_id: %v\n", id)

		err = prettyPrintJSON(wr, snapshot)
		if err != nil {
			return err
		}
	}

	return nil
}

const dumpPackWorkers = 10

// Pack is the struct used in printPacks.
type Pack struct {
	Name string `json:"name"`

	Blobs []Blob `json:"blobs"`
}

// Blob is the struct used in printPacks.
type Blob struct {
	Type   restic.BlobType `json:"type"`
	Length uint            `json:"length"`
	ID     restic.ID       `json:"id"`
	Offset uint            `json:"offset"`
}

func printPacks(repo *repository.Repository, wr io.Writer) error {
	done := make(chan struct{})
	defer close(done)

	f := func(job worker.Job, done <-chan struct{}) (interface{}, error) {
		name := job.Data.(string)

		h := restic.Handle{FileType: restic.DataFile, Name: name}

		blobInfo, err := repo.Backend().Stat(h)
		if err != nil {
			return nil, err
		}

		blobs, err := pack.List(repo.Key(), restic.ReaderAt(repo.Backend(), h), blobInfo.Size)
		if err != nil {
			return nil, err
		}

		return blobs, nil
	}

	jobCh := make(chan worker.Job)
	resCh := make(chan worker.Job)
	wp := worker.New(dumpPackWorkers, f, jobCh, resCh)

	go func() {
		for name := range repo.Backend().List(restic.DataFile, done) {
			jobCh <- worker.Job{Data: name}
		}
		close(jobCh)
	}()

	for job := range resCh {
		name := job.Data.(string)

		if job.Error != nil {
			fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", name, job.Error)
			continue
		}

		entries := job.Result.([]restic.Blob)
		p := Pack{
			Name:  name,
			Blobs: make([]Blob, len(entries)),
		}
		for i, blob := range entries {
			p.Blobs[i] = Blob{
				Type:   blob.Type,
				Length: blob.Length,
				ID:     blob.ID,
				Offset: blob.Offset,
			}
		}

		prettyPrintJSON(os.Stdout, p)
	}

	wp.Wait()

	return nil
}

func (cmd CmdDump) DumpIndexes() error {
	done := make(chan struct{})
	defer close(done)

	for id := range cmd.repo.List(restic.IndexFile, done) {
		fmt.Printf("index_id: %v\n", id)

		idx, err := repository.LoadIndex(cmd.repo, id)
		if err != nil {
			return err
		}

		err = idx.Dump(os.Stdout)
		if err != nil {
			return err
		}
	}

	return nil
}

func (cmd CmdDump) Execute(args []string) error {
	if len(args) != 1 {
		return restic.Fatalf("type not specified, Usage: %s", cmd.Usage())
	}

	repo, err := cmd.global.OpenRepository()
	if err != nil {
		return err
	}
	cmd.repo = repo

	lock, err := lockRepo(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	tpe := args[0]

	switch tpe {
	case "indexes":
		return cmd.DumpIndexes()
	case "snapshots":
		return debugPrintSnapshots(repo, os.Stdout)
	case "packs":
		return printPacks(repo, os.Stdout)
	case "all":
		fmt.Printf("snapshots:\n")
		err := debugPrintSnapshots(repo, os.Stdout)
		if err != nil {
			return err
		}

		fmt.Printf("\nindexes:\n")
		err = cmd.DumpIndexes()
		if err != nil {
			return err
		}

		return nil
	default:
		return restic.Fatalf("no such type %q", tpe)
	}
}
<commit_msg>Fix command 'dump'<commit_after>// +build debug

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"os"

	"restic"
	"restic/pack"
	"restic/repository"

	"restic/worker"
)

type CmdDump struct {
	global *GlobalOptions

	repo *repository.Repository
}

func init() {
	_, err := parser.AddCommand("dump",
		"dump data structures",
		"The dump command dumps data structures from a repository as JSON documents",
		&CmdDump{global: &globalOpts})
	if err != nil {
		panic(err)
	}
}

func (cmd CmdDump) Usage() string {
	return "[indexes|snapshots|trees|all|packs]"
}

func prettyPrintJSON(wr io.Writer, item interface{}) error {
	buf, err := json.MarshalIndent(item, "", "  ")
	if err != nil {
		return err
	}

	_, err = wr.Write(append(buf, '\n'))
	return err
}

func debugPrintSnapshots(repo *repository.Repository, wr io.Writer) error {
	done := make(chan struct{})
	defer close(done)

	for id := range repo.List(restic.SnapshotFile, done) {
		snapshot, err := restic.LoadSnapshot(repo, id)
		if err != nil {
			fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err)
			continue
		}

		fmt.Fprintf(wr, "snapshot_id: %v\n", id)

		err = prettyPrintJSON(wr, snapshot)
		if err != nil {
			return err
		}
	}

	return nil
}

const dumpPackWorkers = 10

// Pack is the struct used in printPacks.
type Pack struct {
	Name string `json:"name"`

	Blobs []Blob `json:"blobs"`
}

// Blob is the struct used in printPacks.
type Blob struct {
	Type   restic.BlobType `json:"type"`
	Length uint            `json:"length"`
	ID     restic.ID       `json:"id"`
	Offset uint            `json:"offset"`
}

func printPacks(repo *repository.Repository, wr io.Writer) error {
	done := make(chan struct{})
	defer close(done)

	f := func(job worker.Job, done <-chan struct{}) (interface{}, error) {
		name := job.Data.(string)

		h := restic.Handle{Type: restic.DataFile, Name: name}

		blobInfo, err := repo.Backend().Stat(h)
		if err != nil {
			return nil, err
		}

		blobs, err := pack.List(repo.Key(), restic.ReaderAt(repo.Backend(), h), blobInfo.Size)
		if err != nil {
			return nil, err
		}

		return blobs, nil
	}

	jobCh := make(chan worker.Job)
	resCh := make(chan worker.Job)
	wp := worker.New(dumpPackWorkers, f, jobCh, resCh)

	go func() {
		for name := range repo.Backend().List(restic.DataFile, done) {
			jobCh <- worker.Job{Data: name}
		}
		close(jobCh)
	}()

	for job := range resCh {
		name := job.Data.(string)

		if job.Error != nil {
			fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", name, job.Error)
			continue
		}

		entries := job.Result.([]restic.Blob)
		p := Pack{
			Name:  name,
			Blobs: make([]Blob, len(entries)),
		}
		for i, blob := range entries {
			p.Blobs[i] = Blob{
				Type:   blob.Type,
				Length: blob.Length,
				ID:     blob.ID,
				Offset: blob.Offset,
			}
		}

		prettyPrintJSON(os.Stdout, p)
	}

	wp.Wait()

	return nil
}

func (cmd CmdDump) DumpIndexes() error {
	done := make(chan struct{})
	defer close(done)

	for id := range cmd.repo.List(restic.IndexFile, done) {
		fmt.Printf("index_id: %v\n", id)

		idx, err := repository.LoadIndex(cmd.repo, id)
		if err != nil {
			return err
		}

		err = idx.Dump(os.Stdout)
		if err != nil {
			return err
		}
	}

	return nil
}

func (cmd CmdDump) Execute(args []string) error {
	if len(args) != 1 {
		return restic.Fatalf("type not specified, Usage: %s", cmd.Usage())
	}

	repo, err := cmd.global.OpenRepository()
	if err != nil {
		return err
	}
	cmd.repo = repo

	lock, err := lockRepo(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	tpe := args[0]

	switch tpe {
	case "indexes":
		return cmd.DumpIndexes()
	case "snapshots":
		return debugPrintSnapshots(repo, os.Stdout)
	case "packs":
		return printPacks(repo, os.Stdout)
	case "all":
		fmt.Printf("snapshots:\n")
		err := debugPrintSnapshots(repo, os.Stdout)
		if err != nil {
			return err
		}

		fmt.Printf("\nindexes:\n")
		err = cmd.DumpIndexes()
		if err != nil {
			return err
		}

		return nil
	default:
		return restic.Fatalf("no such type %q", tpe)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"restic"
	"restic/errors"
	"restic/index"

	"github.com/spf13/cobra"
)

var cmdList = &cobra.Command{
	Use:   "list [blobs|packs|index|snapshots|keys|locks]",
	Short: "list objects in the repository",
	Long: `

`,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runList(globalOptions, args)
	},
}

func init() {
	cmdRoot.AddCommand(cmdList)
}

func runList(opts GlobalOptions, args []string) error {
	if len(args) != 1 {
		return errors.Fatal("type not specified")
	}

	repo, err := OpenRepository(opts)
	if err != nil {
		return err
	}

	if !opts.NoLock {
		lock, err := lockRepo(repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	var t restic.FileType
	switch args[0] {
	case "packs":
		t = restic.DataFile
	case "index":
		t = restic.IndexFile
	case "snapshots":
		t = restic.SnapshotFile
	case "keys":
		t = restic.KeyFile
	case "locks":
		t = restic.LockFile
	case "blobs":
		idx, err := index.Load(repo, nil)
		if err != nil {
			return err
		}

		for _, pack := range idx.Packs {
			for _, entry := range pack.Entries {
				fmt.Printf("%v %v\n", entry.Type, entry.ID)
			}
		}

		return nil
	default:
		return errors.Fatal("invalid type")
	}

	for id := range repo.List(t, nil) {
		Printf("%s\n", id)
	}

	return nil
}
<commit_msg>Add long description of list command in help text<commit_after>package main

import (
	"fmt"
	"restic"
	"restic/errors"
	"restic/index"

	"github.com/spf13/cobra"
)

var cmdList = &cobra.Command{
	Use:   "list [blobs|packs|index|snapshots|keys|locks]",
	Short: "list objects in the repository",
	Long: `
The "list" command allows listing objects in the repository based on type.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runList(globalOptions, args)
	},
}

func init() {
	cmdRoot.AddCommand(cmdList)
}

func runList(opts GlobalOptions, args []string) error {
	if len(args) != 1 {
		return errors.Fatal("type not specified")
	}

	repo, err := OpenRepository(opts)
	if err != nil {
		return err
	}

	if !opts.NoLock {
		lock, err := lockRepo(repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	var t restic.FileType
	switch args[0] {
	case "packs":
		t = restic.DataFile
	case "index":
		t = restic.IndexFile
	case "snapshots":
		t = restic.SnapshotFile
	case "keys":
		t = restic.KeyFile
	case "locks":
		t = restic.LockFile
	case "blobs":
		idx, err := index.Load(repo, nil)
		if err != nil {
			return err
		}

		for _, pack := range idx.Packs {
			for _, entry := range pack.Entries {
				fmt.Printf("%v %v\n", entry.Type, entry.ID)
			}
		}

		return nil
	default:
		return errors.Fatal("invalid type")
	}

	for id := range repo.List(t, nil) {
		Printf("%s\n", id)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Oleku Konko All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

// This module is a Terminal  API for the Go Programming Language.
// The protocols were written in pure Go and works on windows and unix systems

/**

Simple go Application to get Terminal Size. So Many Implementations do not support windows but `ts` has full windows support.
Run `go get github.com/olekukonko/ts` to download and install

Installation

Minimum requirements are Go 1.1+ with fill Windows support

Example

	package main

	import (
		"fmt"
		"github.com/github/git-lfs/vendor/_nuts/github.com/olekukonko/ts"
	)

	func main() {
		size, _ := ts.GetSize()
		fmt.Println(size.Col())  // Get Width
		fmt.Println(size.Row())  // Get Height
		fmt.Println(size.PosX()) // Get X position
		fmt.Println(size.PosY()) // Get Y position
	}

**/

package ts
<commit_msg>no need to rewrite imports in comments<commit_after>// Copyright 2014 Oleku Konko All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

// This module is a Terminal  API for the Go Programming Language.
// The protocols were written in pure Go and works on windows and unix systems

/**

Simple go Application to get Terminal Size. So Many Implementations do not support windows but `ts` has full windows support.
Run `go get github.com/olekukonko/ts` to download and install

Installation

Minimum requirements are Go 1.1+ with fill Windows support

Example

	package main

	import (
		"fmt"
		"github.com/olekukonko/ts"
	)

	func main() {
		size, _ := ts.GetSize()
		fmt.Println(size.Col())  // Get Width
		fmt.Println(size.Row())  // Get Height
		fmt.Println(size.PosX()) // Get X position
		fmt.Println(size.PosY()) // Get Y position
	}

**/

package ts
<|endoftext|>


<commit_before>// Copyright (c) 2017 Alexander Eichhorn
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package xmp

import (
	"encoding"
	"fmt"
	"reflect"
	"strings"
	"sync"
)

// typeInfo holds details for the xml representation of a type.
type typeInfo struct {
	fields []fieldInfo
}

// fieldInfo holds details for the xmp representation of a single field.
type fieldInfo struct {
	idx        []int
	name       string
	minVersion Version
	maxVersion Version
	flags      fieldFlags
}

func (f fieldInfo) String() string {
	s := []string{fmt.Sprintf("FieldInfo: %s %v %v %v", f.name, f.idx, f.minVersion, f.maxVersion)}
	if f.flags&fAttr > 0 {
		s = append(s, "Attr")
	}
	if f.flags&fEmpty > 0 {
		s = append(s, "Empty")
	}
	if f.flags&fOmit > 0 {
		s = append(s, "Omit")
	}
	if f.flags&fAny > 0 {
		s = append(s, "Any")
	}
	if f.flags&fArray > 0 {
		s = append(s, "Array")
	}
	if f.flags&fBinaryMarshal > 0 {
		s = append(s, "BinaryMarshal")
	}
	if f.flags&fBinaryUnmarshal > 0 {
		s = append(s, "BinaryUnmarshal")
	}
	if f.flags&fTextMarshal > 0 {
		s = append(s, "TextMarshal")
	}
	if f.flags&fTextUnmarshal > 0 {
		s = append(s, "TextUnmarshal")
	}
	if f.flags&fMarshal > 0 {
		s = append(s, "Marshal")
	}
	if f.flags&fUnmarshal > 0 {
		s = append(s, "Unmarshal")
	}
	return strings.Join(s, " ")
}

type fieldFlags int

const (
	fElement fieldFlags = 1 << iota
	fAttr
	fEmpty
	fOmit
	fAny
	fArray
	fBinaryMarshal
	fBinaryUnmarshal
	fTextMarshal
	fTextUnmarshal
	fMarshal
	fUnmarshal
	fMarshalAttr
	fUnmarshalAttr
	fMode = fElement | fAttr | fEmpty | fOmit | fAny | fArray | fBinaryMarshal | fBinaryUnmarshal | fTextMarshal | fTextUnmarshal | fMarshal | fUnmarshal | fMarshalAttr | fUnmarshalAttr
)

type tinfoMap map[reflect.Type]*typeInfo

var tinfoNsMap = make(map[string]tinfoMap)
var tinfoLock sync.RWMutex

var (
	binaryUnmarshalerType = reflect.TypeOf((*encoding.BinaryUnmarshaler)(nil)).Elem()
	binaryMarshalerType   = reflect.TypeOf((*encoding.BinaryMarshaler)(nil)).Elem()
	textUnmarshalerType   = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem()
	textMarshalerType     = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem()
	marshalerType         = reflect.TypeOf((*Marshaler)(nil)).Elem()
	unmarshalerType       = reflect.TypeOf((*Unmarshaler)(nil)).Elem()
	attrMarshalerType     = reflect.TypeOf((*MarshalerAttr)(nil)).Elem()
	attrUnmarshalerType   = reflect.TypeOf((*UnmarshalerAttr)(nil)).Elem()
	arrayType             = reflect.TypeOf((*Array)(nil)).Elem()
	zeroType              = reflect.TypeOf((*Zero)(nil)).Elem()
	stringerType          = reflect.TypeOf((*fmt.Stringer)(nil)).Elem()
)

// getTypeInfo returns the typeInfo structure with details necessary
// for marshaling and unmarshaling typ.
func getTypeInfo(typ reflect.Type, ns string) (*typeInfo, error) {
	if ns == "" {
		ns = "xmp"
	}
	tinfoLock.RLock()
	m, ok := tinfoNsMap[ns]
	if !ok {
		m = make(tinfoMap)
		tinfoLock.RUnlock()
		tinfoLock.Lock()
		tinfoNsMap[ns] = m
		tinfoLock.Unlock()
		tinfoLock.RLock()
	}
	tinfo, ok := m[typ]
	tinfoLock.RUnlock()
	if ok {
		return tinfo, nil
	}
	tinfo = &typeInfo{}
	if typ.Kind() != reflect.Struct {
		return nil, fmt.Errorf("xmp: type %s is not a struct", typ.String())
	}
	n := typ.NumField()
	for i := 0; i < n; i++ {
		f := typ.Field(i)
		if (f.PkgPath != "" && !f.Anonymous) || f.Tag.Get(ns) == "-" {
			continue // Private field
		}

		// For embedded structs, embed its fields.
		if f.Anonymous {
			t := f.Type
			if t.Kind() == reflect.Ptr {
				t = t.Elem()
			}
			if t.Kind() == reflect.Struct {
				inner, err := getTypeInfo(t, ns)
				if err != nil {
					return nil, err
				}
				for _, finfo := range inner.fields {
					finfo.idx = append([]int{i}, finfo.idx...)
					if err := addFieldInfo(typ, tinfo, &finfo, ns); err != nil {
						return nil, err
					}
				}
				continue
			}
		}

		finfo, err := structFieldInfo(typ, &f, ns)
		if err != nil {
			return nil, err
		}

		// Add the field if it doesn't conflict with other fields.
		if err := addFieldInfo(typ, tinfo, finfo, ns); err != nil {
			return nil, err
		}
	}
	tinfoLock.Lock()
	m[typ] = tinfo
	tinfoLock.Unlock()
	return tinfo, nil
}

// structFieldInfo builds and returns a fieldInfo for f.
func structFieldInfo(typ reflect.Type, f *reflect.StructField, ns string) (*fieldInfo, error) {
	finfo := &fieldInfo{idx: f.Index}
	// Split the tag from the xml namespace if necessary.
	tag := f.Tag.Get(ns)

	// Parse flags.
	tokens := strings.Split(tag, ",")
	if len(tokens) == 1 {
		finfo.flags = fElement
	} else {
		tag = tokens[0]
		for _, flag := range tokens[1:] {
			switch flag {
			case "attr":
				finfo.flags |= fAttr
			case "empty":
				finfo.flags |= fEmpty
			case "omit":
				finfo.flags |= fOmit
			case "any":
				finfo.flags |= fAny
			}

			// dissect version(s)
			//   v1.0     - only write in version v1.0
			//   v1.0+    - starting at and after v1.0
			//   v1.0-    - only write before and including v1.0
			//   v1.0<1.2 - write from v1.0 until v1.2
			if strings.HasPrefix(flag, "v") {
				flag = flag[1:]
				var op rune
				tokens := strings.FieldsFunc(flag, func(r rune) bool {
					switch r {
					case '+', '-', '<':
						op = r
						return true
					default:
						return false
					}
				})
				var err error
				switch op {
				case '+':
					finfo.minVersion, err = ParseVersion(tokens[0])
				case '-':
					finfo.maxVersion, err = ParseVersion(tokens[0])
				case '<':
					finfo.minVersion, err = ParseVersion(tokens[0])
					if err == nil {
						finfo.maxVersion, err = ParseVersion(tokens[1])
					}
				default:
					finfo.minVersion, err = ParseVersion(flag)
					if err == nil {
						finfo.maxVersion, err = ParseVersion(flag)
					}
				}

				if err != nil {
					return nil, fmt.Errorf("invalid %s version on field %s of type %s (%q): %v", ns, f.Name, typ, f.Tag.Get(ns), err)
				}
			}
		}

		// When any flag except `attr` is used it defaults to `element`
		if finfo.flags&fAttr == 0 {
			finfo.flags |= fElement
		}
	}

	if tag != "" {
		finfo.name = tag
	} else {
		// Use field name as default.
		finfo.name = f.Name
	}

	// add static type info about interfaces the type implements
	if f.Type.Implements(arrayType) {
		finfo.flags |= fArray
	}
	if f.Type.Implements(binaryUnmarshalerType) {
		finfo.flags |= fBinaryUnmarshal
	}
	if f.Type.Implements(binaryMarshalerType) {
		finfo.flags |= fBinaryMarshal
	}
	if f.Type.Implements(textUnmarshalerType) {
		finfo.flags |= fTextUnmarshal
	}
	if f.Type.Implements(textMarshalerType) {
		finfo.flags |= fTextMarshal
	}
	if f.Type.Implements(unmarshalerType) {
		finfo.flags |= fUnmarshal
	}
	if f.Type.Implements(marshalerType) {
		finfo.flags |= fMarshal
	}
	if f.Type.Implements(attrUnmarshalerType) {
		finfo.flags |= fUnmarshalAttr
	}
	if f.Type.Implements(attrMarshalerType) {
		finfo.flags |= fMarshalAttr
	}

	return finfo, nil
}

func addFieldInfo(typ reflect.Type, tinfo *typeInfo, newf *fieldInfo, ns string) error {
	var conflicts []int
	// Find all conflicts.
	for i := range tinfo.fields {
		oldf := &tinfo.fields[i]

		// Same name is a conflict unless versions don't overlap.
		if newf.name == oldf.name {
			if !newf.minVersion.Between(oldf.minVersion, oldf.maxVersion) {
				continue
			}
			if !newf.maxVersion.Between(oldf.minVersion, oldf.maxVersion) {
				continue
			}
			conflicts = append(conflicts, i)
		}
	}

	// Return the first error.
	for _, i := range conflicts {
		oldf := &tinfo.fields[i]
		f1 := typ.FieldByIndex(oldf.idx)
		f2 := typ.FieldByIndex(newf.idx)
		return fmt.Errorf("xmp: %s field %q with tag %q conflicts with field %q with tag %q", typ, f1.Name, f1.Tag.Get(ns), f2.Name, f2.Tag.Get(ns))
	}

	// Without conflicts, add the new field and return.
	tinfo.fields = append(tinfo.fields, *newf)
	return nil
}

// value returns v's field value corresponding to finfo.
// It's equivalent to v.FieldByIndex(finfo.idx), but initializes
// and dereferences pointers as necessary.
func (finfo *fieldInfo) value(v reflect.Value) reflect.Value {
	for i, x := range finfo.idx {
		if i > 0 {
			t := v.Type()
			if t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct {
				if v.IsNil() {
					v.Set(reflect.New(v.Type().Elem()))
				}
				v = v.Elem()
			}
		}
		v = v.Field(x)
	}

	return v
}

// Load value from interface, but only if the result will be
// usefully addressable.
func derefIndirect(v interface{}) reflect.Value {
	return derefValue(reflect.ValueOf(v))
}

func derefValue(val reflect.Value) reflect.Value {
	if val.Kind() == reflect.Interface && !val.IsNil() {
		e := val.Elem()
		if e.Kind() == reflect.Ptr && !e.IsNil() {
			val = e
		}
	}

	if val.Kind() == reflect.Ptr {
		if val.IsNil() {
			val.Set(reflect.New(val.Type().Elem()))
		}
		val = val.Elem()
	}
	return val
}
<commit_msg>add new flag  for storing path segements as map keys<commit_after>// Copyright (c) 2017 Alexander Eichhorn
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package xmp

import (
	"encoding"
	"fmt"
	"reflect"
	"strings"
	"sync"
)

// typeInfo holds details for the xml representation of a type.
type typeInfo struct {
	fields []fieldInfo
}

// fieldInfo holds details for the xmp representation of a single field.
type fieldInfo struct {
	idx        []int
	name       string
	minVersion Version
	maxVersion Version
	flags      fieldFlags
}

func (f fieldInfo) String() string {
	s := []string{fmt.Sprintf("FieldInfo: %s %v %v %v", f.name, f.idx, f.minVersion, f.maxVersion)}
	if f.flags&fAttr > 0 {
		s = append(s, "Attr")
	}
	if f.flags&fEmpty > 0 {
		s = append(s, "Empty")
	}
	if f.flags&fOmit > 0 {
		s = append(s, "Omit")
	}
	if f.flags&fAny > 0 {
		s = append(s, "Any")
	}
	if f.flags&fFlat > 0 {
		s = append(s, "Flat")
	}
	if f.flags&fArray > 0 {
		s = append(s, "Array")
	}
	if f.flags&fBinaryMarshal > 0 {
		s = append(s, "BinaryMarshal")
	}
	if f.flags&fBinaryUnmarshal > 0 {
		s = append(s, "BinaryUnmarshal")
	}
	if f.flags&fTextMarshal > 0 {
		s = append(s, "TextMarshal")
	}
	if f.flags&fTextUnmarshal > 0 {
		s = append(s, "TextUnmarshal")
	}
	if f.flags&fMarshal > 0 {
		s = append(s, "Marshal")
	}
	if f.flags&fUnmarshal > 0 {
		s = append(s, "Unmarshal")
	}
	return strings.Join(s, " ")
}

type fieldFlags int

const (
	fElement fieldFlags = 1 << iota
	fAttr
	fEmpty
	fOmit
	fAny
	fFlat
	fArray
	fBinaryMarshal
	fBinaryUnmarshal
	fTextMarshal
	fTextUnmarshal
	fMarshal
	fUnmarshal
	fMarshalAttr
	fUnmarshalAttr
	fMode = fElement | fAttr | fEmpty | fOmit | fAny | fFlat | fArray | fBinaryMarshal | fBinaryUnmarshal | fTextMarshal | fTextUnmarshal | fMarshal | fUnmarshal | fMarshalAttr | fUnmarshalAttr
)

type tinfoMap map[reflect.Type]*typeInfo

var tinfoNsMap = make(map[string]tinfoMap)
var tinfoLock sync.RWMutex

var (
	binaryUnmarshalerType = reflect.TypeOf((*encoding.BinaryUnmarshaler)(nil)).Elem()
	binaryMarshalerType   = reflect.TypeOf((*encoding.BinaryMarshaler)(nil)).Elem()
	textUnmarshalerType   = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem()
	textMarshalerType     = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem()
	marshalerType         = reflect.TypeOf((*Marshaler)(nil)).Elem()
	unmarshalerType       = reflect.TypeOf((*Unmarshaler)(nil)).Elem()
	attrMarshalerType     = reflect.TypeOf((*MarshalerAttr)(nil)).Elem()
	attrUnmarshalerType   = reflect.TypeOf((*UnmarshalerAttr)(nil)).Elem()
	arrayType             = reflect.TypeOf((*Array)(nil)).Elem()
	zeroType              = reflect.TypeOf((*Zero)(nil)).Elem()
	stringerType          = reflect.TypeOf((*fmt.Stringer)(nil)).Elem()
)

// getTypeInfo returns the typeInfo structure with details necessary
// for marshaling and unmarshaling typ.
func getTypeInfo(typ reflect.Type, ns string) (*typeInfo, error) {
	if ns == "" {
		ns = "xmp"
	}
	tinfoLock.RLock()
	m, ok := tinfoNsMap[ns]
	if !ok {
		m = make(tinfoMap)
		tinfoLock.RUnlock()
		tinfoLock.Lock()
		tinfoNsMap[ns] = m
		tinfoLock.Unlock()
		tinfoLock.RLock()
	}
	tinfo, ok := m[typ]
	tinfoLock.RUnlock()
	if ok {
		return tinfo, nil
	}
	tinfo = &typeInfo{}
	if typ.Kind() != reflect.Struct {
		return nil, fmt.Errorf("xmp: type %s is not a struct", typ.String())
	}
	n := typ.NumField()
	for i := 0; i < n; i++ {
		f := typ.Field(i)
		if (f.PkgPath != "" && !f.Anonymous) || f.Tag.Get(ns) == "-" {
			continue // Private field
		}

		// For embedded structs, embed its fields.
		if f.Anonymous {
			t := f.Type
			if t.Kind() == reflect.Ptr {
				t = t.Elem()
			}
			if t.Kind() == reflect.Struct {
				inner, err := getTypeInfo(t, ns)
				if err != nil {
					return nil, err
				}
				for _, finfo := range inner.fields {
					finfo.idx = append([]int{i}, finfo.idx...)
					if err := addFieldInfo(typ, tinfo, &finfo, ns); err != nil {
						return nil, err
					}
				}
				continue
			}
		}

		finfo, err := structFieldInfo(typ, &f, ns)
		if err != nil {
			return nil, err
		}

		// Add the field if it doesn't conflict with other fields.
		if err := addFieldInfo(typ, tinfo, finfo, ns); err != nil {
			return nil, err
		}
	}
	tinfoLock.Lock()
	m[typ] = tinfo
	tinfoLock.Unlock()
	return tinfo, nil
}

// structFieldInfo builds and returns a fieldInfo for f.
func structFieldInfo(typ reflect.Type, f *reflect.StructField, ns string) (*fieldInfo, error) {
	finfo := &fieldInfo{idx: f.Index}
	// Split the tag from the xml namespace if necessary.
	tag := f.Tag.Get(ns)

	// Parse flags.
	tokens := strings.Split(tag, ",")
	if len(tokens) == 1 {
		finfo.flags = fElement
	} else {
		tag = tokens[0]
		for _, flag := range tokens[1:] {
			switch flag {
			case "attr":
				finfo.flags |= fAttr
			case "empty":
				finfo.flags |= fEmpty
			case "omit":
				finfo.flags |= fOmit
			case "any":
				finfo.flags |= fAny
			case "flat":
				finfo.flags |= fFlat
			}

			// dissect version(s)
			//   v1.0     - only write in version v1.0
			//   v1.0+    - starting at and after v1.0
			//   v1.0-    - only write before and including v1.0
			//   v1.0<1.2 - write from v1.0 until v1.2
			if strings.HasPrefix(flag, "v") {
				flag = flag[1:]
				var op rune
				tokens := strings.FieldsFunc(flag, func(r rune) bool {
					switch r {
					case '+', '-', '<':
						op = r
						return true
					default:
						return false
					}
				})
				var err error
				switch op {
				case '+':
					finfo.minVersion, err = ParseVersion(tokens[0])
				case '-':
					finfo.maxVersion, err = ParseVersion(tokens[0])
				case '<':
					finfo.minVersion, err = ParseVersion(tokens[0])
					if err == nil {
						finfo.maxVersion, err = ParseVersion(tokens[1])
					}
				default:
					finfo.minVersion, err = ParseVersion(flag)
					if err == nil {
						finfo.maxVersion, err = ParseVersion(flag)
					}
				}

				if err != nil {
					return nil, fmt.Errorf("invalid %s version on field %s of type %s (%q): %v", ns, f.Name, typ, f.Tag.Get(ns), err)
				}
			}
		}

		// When any flag except `attr` is used it defaults to `element`
		if finfo.flags&fAttr == 0 {
			finfo.flags |= fElement
		}
	}

	if tag != "" {
		finfo.name = tag
	} else {
		// Use field name as default.
		finfo.name = f.Name
	}

	// add static type info about interfaces the type implements
	if f.Type.Implements(arrayType) {
		finfo.flags |= fArray
	}
	if f.Type.Implements(binaryUnmarshalerType) {
		finfo.flags |= fBinaryUnmarshal
	}
	if f.Type.Implements(binaryMarshalerType) {
		finfo.flags |= fBinaryMarshal
	}
	if f.Type.Implements(textUnmarshalerType) {
		finfo.flags |= fTextUnmarshal
	}
	if f.Type.Implements(textMarshalerType) {
		finfo.flags |= fTextMarshal
	}
	if f.Type.Implements(unmarshalerType) {
		finfo.flags |= fUnmarshal
	}
	if f.Type.Implements(marshalerType) {
		finfo.flags |= fMarshal
	}
	if f.Type.Implements(attrUnmarshalerType) {
		finfo.flags |= fUnmarshalAttr
	}
	if f.Type.Implements(attrMarshalerType) {
		finfo.flags |= fMarshalAttr
	}

	return finfo, nil
}

func addFieldInfo(typ reflect.Type, tinfo *typeInfo, newf *fieldInfo, ns string) error {
	var conflicts []int
	// Find all conflicts.
	for i := range tinfo.fields {
		oldf := &tinfo.fields[i]

		// Same name is a conflict unless versions don't overlap.
		if newf.name == oldf.name {
			if !newf.minVersion.Between(oldf.minVersion, oldf.maxVersion) {
				continue
			}
			if !newf.maxVersion.Between(oldf.minVersion, oldf.maxVersion) {
				continue
			}
			conflicts = append(conflicts, i)
		}
	}

	// Return the first error.
	for _, i := range conflicts {
		oldf := &tinfo.fields[i]
		f1 := typ.FieldByIndex(oldf.idx)
		f2 := typ.FieldByIndex(newf.idx)
		return fmt.Errorf("xmp: %s field %q with tag %q conflicts with field %q with tag %q", typ, f1.Name, f1.Tag.Get(ns), f2.Name, f2.Tag.Get(ns))
	}

	// Without conflicts, add the new field and return.
	tinfo.fields = append(tinfo.fields, *newf)
	return nil
}

// value returns v's field value corresponding to finfo.
// It's equivalent to v.FieldByIndex(finfo.idx), but initializes
// and dereferences pointers as necessary.
func (finfo *fieldInfo) value(v reflect.Value) reflect.Value {
	for i, x := range finfo.idx {
		if i > 0 {
			t := v.Type()
			if t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct {
				if v.IsNil() {
					v.Set(reflect.New(v.Type().Elem()))
				}
				v = v.Elem()
			}
		}
		v = v.Field(x)
	}

	return v
}

// Load value from interface, but only if the result will be
// usefully addressable.
func derefIndirect(v interface{}) reflect.Value {
	return derefValue(reflect.ValueOf(v))
}

func derefValue(val reflect.Value) reflect.Value {
	if val.Kind() == reflect.Interface && !val.IsNil() {
		e := val.Elem()
		if e.Kind() == reflect.Ptr && !e.IsNil() {
			val = e
		}
	}

	if val.Kind() == reflect.Ptr {
		if val.IsNil() {
			val.Set(reflect.New(val.Type().Elem()))
		}
		val = val.Elem()
	}
	return val
}
<|endoftext|>


<commit_before>// Copyright 2018 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/nats-io/go-nats"
	"github.com/nats-io/jwt"
	"github.com/nats-io/nkeys"
)

func createAccount(s *Server) (*Account, nkeys.KeyPair) {
	okp, _ := nkeys.FromSeed(oSeed)
	akp, _ := nkeys.CreateAccount()
	pub, _ := akp.PublicKey()
	nac := jwt.NewAccountClaims(pub)
	jwt, _ := nac.Encode(okp)
	addAccountToMemResolver(s, pub, jwt)
	return s.LookupAccount(pub), akp
}

func TestSystemAccount(t *testing.T) {
	s := opTrustBasicSetup()
	defer s.Shutdown()
	buildMemAccResolver(s)

	acc, _ := createAccount(s)
	s.setSystemAccount(acc)

	s.mu.Lock()
	defer s.mu.Unlock()
	if s.sys == nil || s.sys.account == nil {
		t.Fatalf("Expected sys.account to be non-nil")
	}
	if s.sys.client == nil {
		t.Fatalf("Expected sys.client to be non-nil")
	}
	if s.sys.client.echo {
		t.Fatalf("Internal clients should always have echo false")
	}
}

func createUserCreds(t *testing.T, s *Server, akp nkeys.KeyPair) nats.Option {
	t.Helper()
	kp, _ := nkeys.CreateUser()
	pub, _ := kp.PublicKey()
	nuc := jwt.NewUserClaims(pub)
	ujwt, err := nuc.Encode(akp)
	if err != nil {
		t.Fatalf("Error generating user JWT: %v", err)
	}
	userCB := func() (string, error) {
		return ujwt, nil
	}
	sigCB := func(nonce []byte) ([]byte, error) {
		sig, _ := kp.Sign(nonce)
		return sig, nil
	}
	return nats.UserJWT(userCB, sigCB)
}

func runTrustedServer(t *testing.T) (*Server, *Options) {
	t.Helper()
	opts := DefaultOptions()
	kp, _ := nkeys.FromSeed(oSeed)
	pub, _ := kp.PublicKey()
	opts.TrustedNkeys = []string{pub}
	s := RunServer(opts)
	buildMemAccResolver(s)
	return s, opts
}

func TestSystemAccountNewConnection(t *testing.T) {
	s, opts := runTrustedServer(t)
	defer s.Shutdown()

	acc, akp := createAccount(s)
	s.setSystemAccount(acc)

	url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)

	ncs, err := nats.Connect(url, createUserCreds(t, s, akp))
	if err != nil {
		t.Fatalf("Error on connect: %v", err)
	}
	defer ncs.Close()

	sub, _ := ncs.SubscribeSync(">")
	defer sub.Unsubscribe()

	// We can't hear ourselves, so we need to create a second client to
	// trigger the connect/disconnect events.
	acc2, akp2 := createAccount(s)

	nc, err := nats.Connect(url, createUserCreds(t, s, akp2), nats.Name("TEST EVENTS"))
	if err != nil {
		t.Fatalf("Error on connect: %v", err)
	}
	defer nc.Close()

	msg, err := sub.NextMsg(time.Second)
	if err != nil {
		t.Fatalf("Error receiving msg: %v", err)
	}

	if !strings.HasPrefix(msg.Subject, fmt.Sprintf("$SYS.%s.CLIENT.CONNECT", acc2.Name)) {
		t.Fatalf("Expected subject to start with %q, got %q", "$SYS.<ACCOUNT>.CLIENT.CONNECT", msg.Subject)
	}
	tokens := strings.Split(msg.Subject, ".")
	if len(tokens) < 4 {
		t.Fatalf("Expected 4 tokens, got %d", len(tokens))
	}
	account := tokens[1]
	if account != acc2.Name {
		t.Fatalf("Expected %q for account, got %q", acc2.Name, account)
	}

	cem := ConnectEventMsg{}
	if err := json.Unmarshal(msg.Data, &cem); err != nil {
		t.Fatalf("Error unmarshalling connect event message: %v", err)
	}
	if cem.Server.ID != s.ID() {
		t.Fatalf("Expected server to be %q, got %q", s.ID(), cem.Server)
	}
	if cem.Server.Seq == 0 {
		t.Fatalf("Expected sequence to be non-zero")
	}
	if cem.Client.Name != "TEST EVENTS" {
		t.Fatalf("Expected client name to be %q, got %q", "TEST EVENTS", cem.Client.Name)
	}
	if cem.Client.Lang != "go" {
		t.Fatalf("Expected client lang to be \"go\", got %q", cem.Client.Lang)
	}

	// Now close the other client. Should fire a disconnect event.
	// First send and receive some messages.
	sub2, _ := nc.SubscribeSync("foo")
	defer sub2.Unsubscribe()
	sub3, _ := nc.SubscribeSync("*")
	defer sub3.Unsubscribe()

	for i := 0; i < 10; i++ {
		nc.Publish("foo", []byte("HELLO WORLD"))
	}
	nc.Flush()
	nc.Close()

	msg, err = sub.NextMsg(time.Second)
	if err != nil {
		t.Fatalf("Error receiving msg: %v", err)
	}

	if !strings.HasPrefix(msg.Subject, fmt.Sprintf("$SYS.%s.CLIENT.DISCONNECT", acc2.Name)) {
		t.Fatalf("Expected subject to start with %q, got %q", "$SYS.<ACCOUNT>.CLIENT.DISCONNECT", msg.Subject)
	}
	tokens = strings.Split(msg.Subject, ".")
	if len(tokens) < 4 {
		t.Fatalf("Expected 4 tokens, got %d", len(tokens))
	}
	account = tokens[1]
	if account != acc2.Name {
		t.Fatalf("Expected %q for account, got %q", acc2.Name, account)
	}

	dem := DisconnectEventMsg{}
	if err := json.Unmarshal(msg.Data, &dem); err != nil {
		t.Fatalf("Error unmarshalling disconnect event message: %v", err)
	}

	if dem.Server.ID != s.ID() {
		t.Fatalf("Expected server to be %q, got %q", s.ID(), dem.Server)
	}
	if dem.Server.Seq == 0 {
		t.Fatalf("Expected sequence to be non-zero")
	}
	if dem.Server.Seq <= cem.Server.Seq {
		t.Fatalf("Expected sequence to be increasing")
	}

	if cem.Client.Name != "TEST EVENTS" {
		t.Fatalf("Expected client name to be %q, got %q", "TEST EVENTS", dem.Client.Name)
	}
	if dem.Client.Lang != "go" {
		t.Fatalf("Expected client lang to be \"go\", got %q", dem.Client.Lang)
	}

	if dem.Sent.Msgs != 10 {
		t.Fatalf("Expected 10 msgs sent, got %d", dem.Sent.Msgs)
	}
	if dem.Sent.Bytes != 110 {
		t.Fatalf("Expected 110 bytes sent, got %d", dem.Sent.Bytes)
	}
	if dem.Received.Msgs != 20 {
		t.Fatalf("Expected 20 msgs received, got %d", dem.Sent.Msgs)
	}
	if dem.Received.Bytes != 220 {
		t.Fatalf("Expected 220 bytes sent, got %d", dem.Sent.Bytes)
	}
}

func TestSystemInternalSubscriptions(t *testing.T) {
	s, opts := runTrustedServer(t)
	defer s.Shutdown()

	sub, err := s.sysSubscribe("foo", nil)
	if sub != nil || err != ErrNoSysAccount {
		t.Fatalf("Expected to get proper error, got %v", err)
	}

	acc, akp := createAccount(s)
	s.setSystemAccount(acc)

	url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)

	nc, err := nats.Connect(url, createUserCreds(t, s, akp))
	if err != nil {
		t.Fatalf("Error on connect: %v", err)
	}
	defer nc.Close()

	sub, err = s.sysSubscribe("foo", nil)
	if sub != nil || err == nil {
		t.Fatalf("Expected to get error for no handler, got %v", err)
	}

	received := make(chan *nats.Msg)
	// Create message callback handler.
	cb := func(sub *subscription, subject, reply string, msg []byte) {
		copy := append([]byte(nil), msg...)
		received <- &nats.Msg{Subject: subject, Reply: reply, Data: copy}
	}

	// Now create an internal subscription
	sub, err = s.sysSubscribe("foo", cb)
	if sub == nil || err != nil {
		t.Fatalf("Expected to subscribe, got %v", err)
	}
	// Now send out a message from our normal client.
	nc.Publish("foo", []byte("HELLO WORLD"))

	var msg *nats.Msg

	select {
	case msg = <-received:
		if msg.Subject != "foo" {
			t.Fatalf("Expected \"foo\" as subject, got %q", msg.Subject)
		}
		if msg.Reply != "" {
			t.Fatalf("Expected no reply, got %q", msg.Reply)
		}
		if !bytes.Equal(msg.Data, []byte("HELLO WORLD")) {
			t.Fatalf("Got the wrong msg payload: %q", msg.Data)
		}
		break
	case <-time.After(time.Second):
		t.Fatalf("Did not receive the message")
	}
	s.sysUnsubscribe(sub)

	// Now send out a message from our normal client.
	// We should not see this one.
	nc.Publish("foo", []byte("You There?"))

	select {
	case <-received:
		t.Fatalf("Received a message when we should not have")
	case <-time.After(100 * time.Millisecond):
		break
	}

	// Now make sure we do not hear ourselves. We optimize this for internally
	// generated messages.
	r := SublistResult{psubs: []*subscription{sub}}
	s.sendInternalMsg(&r, "foo", msg.Data)

	select {
	case <-received:
		t.Fatalf("Received a message when we should not have")
	case <-time.After(100 * time.Millisecond):
		break
	}
}
<commit_msg>Make sure sub is processed<commit_after>// Copyright 2018 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/nats-io/go-nats"
	"github.com/nats-io/jwt"
	"github.com/nats-io/nkeys"
)

func createAccount(s *Server) (*Account, nkeys.KeyPair) {
	okp, _ := nkeys.FromSeed(oSeed)
	akp, _ := nkeys.CreateAccount()
	pub, _ := akp.PublicKey()
	nac := jwt.NewAccountClaims(pub)
	jwt, _ := nac.Encode(okp)
	addAccountToMemResolver(s, pub, jwt)
	return s.LookupAccount(pub), akp
}

func TestSystemAccount(t *testing.T) {
	s := opTrustBasicSetup()
	defer s.Shutdown()
	buildMemAccResolver(s)

	acc, _ := createAccount(s)
	s.setSystemAccount(acc)

	s.mu.Lock()
	defer s.mu.Unlock()
	if s.sys == nil || s.sys.account == nil {
		t.Fatalf("Expected sys.account to be non-nil")
	}
	if s.sys.client == nil {
		t.Fatalf("Expected sys.client to be non-nil")
	}
	if s.sys.client.echo {
		t.Fatalf("Internal clients should always have echo false")
	}
}

func createUserCreds(t *testing.T, s *Server, akp nkeys.KeyPair) nats.Option {
	t.Helper()
	kp, _ := nkeys.CreateUser()
	pub, _ := kp.PublicKey()
	nuc := jwt.NewUserClaims(pub)
	ujwt, err := nuc.Encode(akp)
	if err != nil {
		t.Fatalf("Error generating user JWT: %v", err)
	}
	userCB := func() (string, error) {
		return ujwt, nil
	}
	sigCB := func(nonce []byte) ([]byte, error) {
		sig, _ := kp.Sign(nonce)
		return sig, nil
	}
	return nats.UserJWT(userCB, sigCB)
}

func runTrustedServer(t *testing.T) (*Server, *Options) {
	t.Helper()
	opts := DefaultOptions()
	kp, _ := nkeys.FromSeed(oSeed)
	pub, _ := kp.PublicKey()
	opts.TrustedNkeys = []string{pub}
	s := RunServer(opts)
	buildMemAccResolver(s)
	return s, opts
}

func TestSystemAccountNewConnection(t *testing.T) {
	s, opts := runTrustedServer(t)
	defer s.Shutdown()

	acc, akp := createAccount(s)
	s.setSystemAccount(acc)

	url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)

	ncs, err := nats.Connect(url, createUserCreds(t, s, akp))
	if err != nil {
		t.Fatalf("Error on connect: %v", err)
	}
	defer ncs.Close()

	sub, _ := ncs.SubscribeSync(">")
	defer sub.Unsubscribe()
	ncs.Flush()

	// We can't hear ourselves, so we need to create a second client to
	// trigger the connect/disconnect events.
	acc2, akp2 := createAccount(s)

	nc, err := nats.Connect(url, createUserCreds(t, s, akp2), nats.Name("TEST EVENTS"))
	if err != nil {
		t.Fatalf("Error on connect: %v", err)
	}
	defer nc.Close()

	msg, err := sub.NextMsg(time.Second)
	if err != nil {
		t.Fatalf("Error receiving msg: %v", err)
	}

	if !strings.HasPrefix(msg.Subject, fmt.Sprintf("$SYS.%s.CLIENT.CONNECT", acc2.Name)) {
		t.Fatalf("Expected subject to start with %q, got %q", "$SYS.<ACCOUNT>.CLIENT.CONNECT", msg.Subject)
	}
	tokens := strings.Split(msg.Subject, ".")
	if len(tokens) < 4 {
		t.Fatalf("Expected 4 tokens, got %d", len(tokens))
	}
	account := tokens[1]
	if account != acc2.Name {
		t.Fatalf("Expected %q for account, got %q", acc2.Name, account)
	}

	cem := ConnectEventMsg{}
	if err := json.Unmarshal(msg.Data, &cem); err != nil {
		t.Fatalf("Error unmarshalling connect event message: %v", err)
	}
	if cem.Server.ID != s.ID() {
		t.Fatalf("Expected server to be %q, got %q", s.ID(), cem.Server)
	}
	if cem.Server.Seq == 0 {
		t.Fatalf("Expected sequence to be non-zero")
	}
	if cem.Client.Name != "TEST EVENTS" {
		t.Fatalf("Expected client name to be %q, got %q", "TEST EVENTS", cem.Client.Name)
	}
	if cem.Client.Lang != "go" {
		t.Fatalf("Expected client lang to be \"go\", got %q", cem.Client.Lang)
	}

	// Now close the other client. Should fire a disconnect event.
	// First send and receive some messages.
	sub2, _ := nc.SubscribeSync("foo")
	defer sub2.Unsubscribe()
	sub3, _ := nc.SubscribeSync("*")
	defer sub3.Unsubscribe()

	for i := 0; i < 10; i++ {
		nc.Publish("foo", []byte("HELLO WORLD"))
	}
	nc.Flush()
	nc.Close()

	msg, err = sub.NextMsg(time.Second)
	if err != nil {
		t.Fatalf("Error receiving msg: %v", err)
	}

	if !strings.HasPrefix(msg.Subject, fmt.Sprintf("$SYS.%s.CLIENT.DISCONNECT", acc2.Name)) {
		t.Fatalf("Expected subject to start with %q, got %q", "$SYS.<ACCOUNT>.CLIENT.DISCONNECT", msg.Subject)
	}
	tokens = strings.Split(msg.Subject, ".")
	if len(tokens) < 4 {
		t.Fatalf("Expected 4 tokens, got %d", len(tokens))
	}
	account = tokens[1]
	if account != acc2.Name {
		t.Fatalf("Expected %q for account, got %q", acc2.Name, account)
	}

	dem := DisconnectEventMsg{}
	if err := json.Unmarshal(msg.Data, &dem); err != nil {
		t.Fatalf("Error unmarshalling disconnect event message: %v", err)
	}

	if dem.Server.ID != s.ID() {
		t.Fatalf("Expected server to be %q, got %q", s.ID(), dem.Server)
	}
	if dem.Server.Seq == 0 {
		t.Fatalf("Expected sequence to be non-zero")
	}
	if dem.Server.Seq <= cem.Server.Seq {
		t.Fatalf("Expected sequence to be increasing")
	}

	if cem.Client.Name != "TEST EVENTS" {
		t.Fatalf("Expected client name to be %q, got %q", "TEST EVENTS", dem.Client.Name)
	}
	if dem.Client.Lang != "go" {
		t.Fatalf("Expected client lang to be \"go\", got %q", dem.Client.Lang)
	}

	if dem.Sent.Msgs != 10 {
		t.Fatalf("Expected 10 msgs sent, got %d", dem.Sent.Msgs)
	}
	if dem.Sent.Bytes != 110 {
		t.Fatalf("Expected 110 bytes sent, got %d", dem.Sent.Bytes)
	}
	if dem.Received.Msgs != 20 {
		t.Fatalf("Expected 20 msgs received, got %d", dem.Sent.Msgs)
	}
	if dem.Received.Bytes != 220 {
		t.Fatalf("Expected 220 bytes sent, got %d", dem.Sent.Bytes)
	}
}

func TestSystemInternalSubscriptions(t *testing.T) {
	s, opts := runTrustedServer(t)
	defer s.Shutdown()

	sub, err := s.sysSubscribe("foo", nil)
	if sub != nil || err != ErrNoSysAccount {
		t.Fatalf("Expected to get proper error, got %v", err)
	}

	acc, akp := createAccount(s)
	s.setSystemAccount(acc)

	url := fmt.Sprintf("nats://%s:%d", opts.Host, opts.Port)

	nc, err := nats.Connect(url, createUserCreds(t, s, akp))
	if err != nil {
		t.Fatalf("Error on connect: %v", err)
	}
	defer nc.Close()

	sub, err = s.sysSubscribe("foo", nil)
	if sub != nil || err == nil {
		t.Fatalf("Expected to get error for no handler, got %v", err)
	}

	received := make(chan *nats.Msg)
	// Create message callback handler.
	cb := func(sub *subscription, subject, reply string, msg []byte) {
		copy := append([]byte(nil), msg...)
		received <- &nats.Msg{Subject: subject, Reply: reply, Data: copy}
	}

	// Now create an internal subscription
	sub, err = s.sysSubscribe("foo", cb)
	if sub == nil || err != nil {
		t.Fatalf("Expected to subscribe, got %v", err)
	}
	// Now send out a message from our normal client.
	nc.Publish("foo", []byte("HELLO WORLD"))

	var msg *nats.Msg

	select {
	case msg = <-received:
		if msg.Subject != "foo" {
			t.Fatalf("Expected \"foo\" as subject, got %q", msg.Subject)
		}
		if msg.Reply != "" {
			t.Fatalf("Expected no reply, got %q", msg.Reply)
		}
		if !bytes.Equal(msg.Data, []byte("HELLO WORLD")) {
			t.Fatalf("Got the wrong msg payload: %q", msg.Data)
		}
		break
	case <-time.After(time.Second):
		t.Fatalf("Did not receive the message")
	}
	s.sysUnsubscribe(sub)

	// Now send out a message from our normal client.
	// We should not see this one.
	nc.Publish("foo", []byte("You There?"))

	select {
	case <-received:
		t.Fatalf("Received a message when we should not have")
	case <-time.After(100 * time.Millisecond):
		break
	}

	// Now make sure we do not hear ourselves. We optimize this for internally
	// generated messages.
	r := SublistResult{psubs: []*subscription{sub}}
	s.sendInternalMsg(&r, "foo", msg.Data)

	select {
	case <-received:
		t.Fatalf("Received a message when we should not have")
	case <-time.After(100 * time.Millisecond):
		break
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package indexers

import (
	"encoding/binary"
	"errors"

	"github.com/btcsuite/btcd/blockchain"
	"github.com/btcsuite/btcd/database"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcutil/gcs"
)

const (
	// cbfIndexName is the human-readable name for the index.
	cbfIndexName = "committed bloom filter index"
)

var (
	// cbfIndexKey is the name of the db bucket used to house the
	// block hash -> CBF index.
	cbfIndexKey = []byte("cbfbyhashidx")

	// errNoCBFEntry is an error that indicates a requested entry does
	// not exist in the CBF index.
	errCBFEntry = errors.New("no entry in the block ID index")
)

// The serialized format for keys and values in the block hash to CBF bucket is:
//   <hash> = <CBF>
//
//   Field           Type              Size
//   hash            chainhash.Hash    32 bytes
//   CBF             []byte            variable
//   -----
//   Total: > 32 bytes

// CBFIndex implements a CBF by hash index.
type CBFIndex struct {
	db database.DB
}

// Ensure the CBFIndex type implements the Indexer interface.
var _ Indexer = (*CBFIndex)(nil)

// Init initializes the hash-based CBF index.
//
// This is part of the Indexer interface.
func (idx *CBFIndex) Init() error {
	return nil
}

// Key returns the database key to use for the index as a byte slice.
//
// This is part of the Indexer interface.
func (idx *CBFIndex) Key() []byte {
	return cbfIndexKey
}

// Name returns the human-readable name of the index.
//
// This is part of the Indexer interface.
func (idx *CBFIndex) Name() string {
	return cbfIndexName
}

// Create is invoked when the indexer manager determines the index needs
// to be created for the first time.  It creates the buckets for the hash-based
// CBF index.
//
// This is part of the Indexer interface.
func (idx *CBFIndex) Create(dbTx database.Tx) error {
	meta := dbTx.Metadata()
	_, err := meta.CreateBucket(cbfIndexKey)
	return err
}

func generateFilterForBlock(block *btcutil.Block) ([]byte, error) {
	txSlice := block.Transactions() // XXX can this fail?
	txHashes := make([][]byte, len(txSlice))

	for i := 0; i < len(txSlice); i++ {
		txHash, err := block.TxHash(i)
		if err != nil {
			return nil, err
		}
		txHashes = append(txHashes, txHash.CloneBytes())
	}

	var key [gcs.KeySize]byte
	P := uint8(20) // collision probability

	for i := 0; i < gcs.KeySize; i += 4 {
		binary.BigEndian.PutUint32(key[i:], uint32(0xcafebabe))
	}

	filter, err := gcs.BuildGCSFilter(P, key, txHashes)
	if err != nil {
		return nil, err
	}

	return filter.Bytes(), nil
}

// ConnectBlock is invoked by the index manager when a new block has been
// connected to the main chain.  This indexer adds a hash-to-CBF mapping for
// every passed block.
//
// This is part of the Indexer interface.
func (idx *CBFIndex) ConnectBlock(dbTx database.Tx, block *btcutil.Block,
    view *blockchain.UtxoViewpoint) error {
	filterBytes, err := generateFilterForBlock(block)
	if err != nil {
		return err
	}

	meta := dbTx.Metadata()
	index := meta.Bucket(cbfIndexKey)
	err = index.Put(block.Hash().CloneBytes(), filterBytes)
	if err != nil {
		return err
	}

	return nil
}

// DisconnectBlock is invoked by the index manager when a block has been
// disconnected from the main chain.  This indexer removes the hash-to-CBF
// mapping for every passed block.
//
// This is part of the Indexer interface.
func (idx *CBFIndex) DisconnectBlock(dbTx database.Tx, block *btcutil.Block, view *blockchain.UtxoViewpoint) error {
	return nil
}

// NewCBFIndex returns a new instance of an indexer that is used to create a
// mapping of the hashes of all blocks in the blockchain to their respective
// committed bloom filters.
//
// It implements the Indexer interface which plugs into the IndexManager that in
// turn is used by the blockchain package.  This allows the index to be
// seamlessly maintained along with the chain.
func NewCBFIndex(db database.DB) *CBFIndex {
	return &CBFIndex{db: db}
}

// DropCBFIndex drops the CBF index from the provided database if exists.
func DropCBFIndex(db database.DB) error {
	return dropIndex(db, cbfIndexKey, cbfIndexName)
}
<commit_msg>Instrument basic logging<commit_after>// Copyright (c) 2017 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package indexers

import (
	"encoding/binary"
	"errors"
	"fmt"

	"github.com/btcsuite/btcd/blockchain"
	"github.com/btcsuite/btcd/database"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcutil/gcs"

	"os"
)

const (
	// cbfIndexName is the human-readable name for the index.
	cbfIndexName = "committed bloom filter index"
)

var (
	// cbfIndexKey is the name of the db bucket used to house the
	// block hash -> CBF index.
	cbfIndexKey = []byte("cbfbyhashidx")

	// errNoCBFEntry is an error that indicates a requested entry does
	// not exist in the CBF index.
	errCBFEntry = errors.New("no entry in the block ID index")
)

// The serialized format for keys and values in the block hash to CBF bucket is:
//   <hash> = <CBF>
//
//   Field           Type              Size
//   hash            chainhash.Hash    32 bytes
//   CBF             []byte            variable
//   -----
//   Total: > 32 bytes

// CBFIndex implements a CBF by hash index.
type CBFIndex struct {
	db database.DB
}

// Ensure the CBFIndex type implements the Indexer interface.
var _ Indexer = (*CBFIndex)(nil)

// Init initializes the hash-based CBF index.
//
// This is part of the Indexer interface.
func (idx *CBFIndex) Init() error {
	return nil
}

// Key returns the database key to use for the index as a byte slice.
//
// This is part of the Indexer interface.
func (idx *CBFIndex) Key() []byte {
	return cbfIndexKey
}

// Name returns the human-readable name of the index.
//
// This is part of the Indexer interface.
func (idx *CBFIndex) Name() string {
	return cbfIndexName
}

// Create is invoked when the indexer manager determines the index needs
// to be created for the first time.  It creates the buckets for the hash-based
// CBF index.
//
// This is part of the Indexer interface.
func (idx *CBFIndex) Create(dbTx database.Tx) error {
	meta := dbTx.Metadata()
	_, err := meta.CreateBucket(cbfIndexKey)
	return err
}

func generateFilterForBlock(block *btcutil.Block) ([]byte, error) {
	txSlice := block.Transactions() // XXX can this fail?
	txHashes := make([][]byte, len(txSlice))

	for i := 0; i < len(txSlice); i++ {
		txHash, err := block.TxHash(i)
		if err != nil {
			return nil, err
		}
		txHashes = append(txHashes, txHash.CloneBytes())
	}

	var key [gcs.KeySize]byte
	P := uint8(20) // collision probability

	for i := 0; i < gcs.KeySize; i += 4 {
		binary.BigEndian.PutUint32(key[i:], uint32(0xcafebabe))
	}

	filter, err := gcs.BuildGCSFilter(P, key, txHashes)
	if err != nil {
		return nil, err
	}

	fmt.Fprintf(os.Stderr, "Generated CBF for block %v", block.Hash())

	return filter.Bytes(), nil
}

// ConnectBlock is invoked by the index manager when a new block has been
// connected to the main chain.  This indexer adds a hash-to-CBF mapping for
// every passed block.
//
// This is part of the Indexer interface.
func (idx *CBFIndex) ConnectBlock(dbTx database.Tx, block *btcutil.Block,
    view *blockchain.UtxoViewpoint) error {
	filterBytes, err := generateFilterForBlock(block)
	if err != nil {
		return err
	}

	meta := dbTx.Metadata()
	index := meta.Bucket(cbfIndexKey)
	err = index.Put(block.Hash().CloneBytes(), filterBytes)
	if err != nil {
		return err
	}

	fmt.Fprintf(os.Stderr, "Stored CBF for block %v", block.Hash())

	return nil
}

// DisconnectBlock is invoked by the index manager when a block has been
// disconnected from the main chain.  This indexer removes the hash-to-CBF
// mapping for every passed block.
//
// This is part of the Indexer interface.
func (idx *CBFIndex) DisconnectBlock(dbTx database.Tx, block *btcutil.Block, view *blockchain.UtxoViewpoint) error {
	return nil
}

// NewCBFIndex returns a new instance of an indexer that is used to create a
// mapping of the hashes of all blocks in the blockchain to their respective
// committed bloom filters.
//
// It implements the Indexer interface which plugs into the IndexManager that in
// turn is used by the blockchain package.  This allows the index to be
// seamlessly maintained along with the chain.
func NewCBFIndex(db database.DB) *CBFIndex {
	return &CBFIndex{db: db}
}

// DropCBFIndex drops the CBF index from the provided database if exists.
func DropCBFIndex(db database.DB) error {
	return dropIndex(db, cbfIndexKey, cbfIndexName)
}
<|endoftext|>


<commit_before>package factory

import (
	"math/rand"
	"testing"

	"github.com/aurelien-rainone/evolve/bitstring"
	"github.com/aurelien-rainone/evolve/framework"
	"github.com/stretchr/testify/assert"
)

func TestBitStringFactory(t *testing.T) {
	const (
		candidateLength = 10
		populationSize  = 5
	)

	// local test function
	validatePopulation := func(population []framework.Candidate) {
		// Make sure the correct number of candidates were generated.
		assert.Lenf(t, population, populationSize, "want population size = %v, got %v", populationSize, len(population))
		// Make sure that each individual is the right length.
		for _, cand := range population {
			bitString := cand.(*bitstring.BitString)
			assert.Equalf(t, candidateLength, bitString.Len(), "want bitstring length = %v, got %v", candidateLength, bitString.Len())
		}
	}

	rng := rand.New(rand.NewSource(99))

	t.Run("unseed population", func(t *testing.T) {

		f := NewBitStringFactory(candidateLength)
		population := f.GenerateInitialPopulation(populationSize, rng)
		validatePopulation(population)
	})

	t.Run("seeded population", func(t *testing.T) {

		f := NewBitStringFactory(candidateLength)
		seed1, _ := bitstring.NewFromString("1111100000")
		seed2, _ := bitstring.NewFromString("1010101010")
		seeds := []framework.Candidate{seed1, seed2}
		population := f.SeedInitialPopulation(populationSize, seeds, rng)

		// Check that the seed candidates appear in the generated population.
		assert.Containsf(t, population, seed1, "Population does not contain seed candidate 1.")
		assert.Containsf(t, population, seed2, "Population does not contain seed candidate 2.")
		validatePopulation(population)
	})

	t.Run("too many seed candidates", func(t *testing.T) {

		f := NewBitStringFactory(candidateLength)
		candidate, _ := bitstring.New(candidateLength)
		// The following call should panic since the 3 seed candidates won't fit
		// into a population of size 2.
		assert.Panics(t, func() {
			f.SeedInitialPopulation(2,
				[]framework.Candidate{candidate, candidate, candidate},
				rng)
		})
	})

}
<commit_msg>Cosmetics<commit_after>package factory

import (
	"math/rand"
	"testing"

	"github.com/aurelien-rainone/evolve/bitstring"
	"github.com/aurelien-rainone/evolve/framework"
	"github.com/stretchr/testify/assert"
)

func TestBitStringFactory(t *testing.T) {
	const (
		candidateLength = 10
		populationSize  = 5
	)

	// local test function
	validatePopulation := func(population []framework.Candidate) {
		// Make sure the correct number of candidates were generated.
		assert.Lenf(t, population, populationSize, "want population size = %v, got %v", populationSize, len(population))
		// Make sure that each individual is the right length.
		for _, cand := range population {
			bitString := cand.(*bitstring.BitString)
			assert.Equalf(t, candidateLength, bitString.Len(), "want bitstring length = %v, got %v", candidateLength, bitString.Len())
		}
	}

	rng := rand.New(rand.NewSource(99))

	t.Run("unseed population", func(t *testing.T) {

		f := NewBitStringFactory(candidateLength)
		population := f.GenerateInitialPopulation(populationSize, rng)
		validatePopulation(population)
	})

	t.Run("seeded population", func(t *testing.T) {

		f := NewBitStringFactory(candidateLength)
		seed1, _ := bitstring.NewFromString("1111100000")
		seed2, _ := bitstring.NewFromString("1010101010")
		seeds := []framework.Candidate{seed1, seed2}
		population := f.SeedInitialPopulation(populationSize, seeds, rng)

		// Check that the seed candidates appear in the generated population.
		assert.Containsf(t, population, seed1, "Population does not contain seed candidate 1.")
		assert.Containsf(t, population, seed2, "Population does not contain seed candidate 2.")
		validatePopulation(population)
	})

	t.Run("too many seed candidates", func(t *testing.T) {

		f := NewBitStringFactory(candidateLength)
		candidate, _ := bitstring.New(candidateLength)
		// The following call should panic since the 3 seed candidates won't fit
		// into a population of size 2.
		assert.Panics(t, func() {
			f.SeedInitialPopulation(2,
				[]framework.Candidate{candidate, candidate, candidate},
				rng)
		})
	})
}
<|endoftext|>


<commit_before>package ziputils

import (
	"archive/zip"
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

//压缩文件
func ZipFile(srcPath, dstFileName string, bFlag bool) error {
	buf := new(bytes.Buffer)
	myzip := zip.NewWriter(buf)

	//fmt.Println("srcPath:", srcPath)
	//遍历目录下的所有文件，将文件写入到压缩包中
	err := filepath.Walk(srcPath, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return filepath.SkipDir
		}

		//fmt.Println("path:", path)

		//非目录才写入到压缩包
		if !fi.IsDir() {

			header, err := zip.FileInfoHeader(fi)
			if err != nil {
				return filepath.SkipDir
			}
			//判断是否需要把顶层目录写入到压缩包
			if bFlag {
				refFile, _ := filepath.Rel(filepath.Dir(srcPath), path)
				header.Name = strings.SplitN(refFile, `\`, 2)[1]
			} else {
				header.Name, _ = filepath.Rel(filepath.Dir(srcPath), path)
			}
			//fmt.Println("header.Name:", header.Name)

			//创建一个头
			w, err := myzip.CreateHeader(header)
			if err != nil {
				return err
			}

			fileData, err := ioutil.ReadFile(path)
			if err != nil {
				return filepath.SkipDir
			}
			//写入文件信息
			w.Write(fileData)
		}
		return nil
	})

	myzip.Close()

	//先检查目录存在不存在
	os.MkdirAll(filepath.Dir(dstFileName), 0666)
	// 建立zip文件
	retFile, err := os.Create(dstFileName)
	if err != nil {
		return err
	}
	defer retFile.Close()

	// 将buf中的数据写入文件
	_, err = buf.WriteTo(retFile)
	if err != nil {
		return err
	}
	return nil
}

//解压文件
func UnZip(srcFile, dstPath string) error {

	//创建一个目录
	err := os.MkdirAll(dstPath, 0666)
	if err != nil {
		fmt.Println("创建目录失败, path:", dstPath, ", err:", err)
		return err
	}

	//读取zip文件
	cf, err := zip.OpenReader(srcFile)
	if err != nil {
		fmt.Println(err)
		return err
	}
	defer cf.Close()

	for _, file := range cf.File {

		rc, err := file.Open()
		if err != nil {
			fmt.Println("open file failed, err:", err)
			return err
		}

		if file.FileInfo().IsDir() {
			//创建目录
			err = os.MkdirAll(filepath.Join(dstPath, file.Name), 0666)
			if err != nil && err != os.ErrExist {
				fmt.Println("创建目录失败, path:", filepath.Dir(filepath.Join(dstPath, file.Name)), ", err:", err)
				return err
			}
		} else {

			//创建文件所在目录
			err = os.MkdirAll(filepath.Dir(filepath.Join(dstPath, file.Name)), 0666)
			if err != nil && err != os.ErrExist {
				fmt.Println("创建目录失败, path:", filepath.Dir(filepath.Join(dstPath, file.Name)), ", err:", err)
				return err
			}
			//创建文件
			f, err := os.Create(filepath.Join(dstPath, file.Name))
			if err != nil {
				fmt.Println("create file failed, err:", err)
				return err
			}
			defer f.Close()
			_, err = io.Copy(f, rc)
			if err != nil {
				fmt.Println("copy file failed, err:", err)
				return err
			}
		}

	}
	return nil
}

func WriteComment(fileName string, comment ...string) error {
	//打开文件
	f, err := os.OpenFile(fileName, os.O_RDWR, 0666)
	if err != nil {
		fmt.Println("打开文件失败, err:", err)
		return err
	}
	defer f.Close()

	//将要写入的comment进行字节序列化
	commentList := make([]zipComment, 0, len(comment))
	item := zipComment{}
	for _, v := range comment {
		item.Data = v
		item.getLen()
		commentList = append(commentList, item)
	}
	byteComment := pack(commentList...)
	//	fmt.Println("comment bytes:", byteComment)

	f.Seek(0, os.SEEK_END)

	//写入comment字节流
	num, err := f.Write(byteComment)
	if err != nil {
		fmt.Println(err)
		return err
	}
	//写入comment长度 2字节
	err = binary.Write(f, binary.BigEndian, uint16(num))
	if err != nil {
		fmt.Println("err:", err)
		return err
	}
	//	fmt.Println("write num:", num)
	return nil
}

func ReadComment(fileName string) []string {
	//打开文件
	f, err := os.OpenFile(fileName, os.O_RDONLY, 0666)
	if err != nil {
		fmt.Println("打开文件失败, err:", err)
		return make([]string, 0)
	}
	defer f.Close()

	//定位到comment长度字节流的位置
	f.Seek(-2, os.SEEK_END)

	//获取comment长度
	var commentLen uint16
	err = binary.Read(f, binary.BigEndian, &commentLen)
	if err != nil {
		fmt.Println("获取comment长度错误, err:", err)
		return make([]string, 0)
	}

	//	fmt.Println("commentLen:", commentLen)

	//定位到comment字节流的开始位置
	seekLen := int64(0) - int64(commentLen) - int64(2)
	f.Seek(seekLen, os.SEEK_END)

	//读取comment字节流
	comment := make([]byte, commentLen)
	//	num, err := f.Read(comment)
	_, err = f.Read(comment)
	if err != nil {
		fmt.Println("读取comment数据失败, err:", err)
		return make([]string, 0)
	}

	//	fmt.Println("comment read len:", num)

	//解析comment字节流
	commentData := unPack(comment)
	//	fmt.Println("commentData:", commentData)

	retData := make([]string, 0, len(commentData))
	for _, v := range commentData {
		retData = append(retData, v.Data)
	}
	return retData
}

//comment单元素结构
type zipComment struct {
	Data string
	Len  uint16
}

func (this *zipComment) getLen() {
	this.Len = uint16(len([]byte(this.Data)))
}

//comment元素字节序列化
func pack(data ...zipComment) []byte {
	buf := new(bytes.Buffer)

	for _, v := range data {
		err := binary.Write(buf, binary.LittleEndian, v.Len)
		if err != nil {
			fmt.Println(err)
			return buf.Bytes()
		}
		buf.WriteString(v.Data)
	}

	return buf.Bytes()
}

//comment元素字节反序列化
func unPack(data []byte) []zipComment {
	buf := bytes.NewReader(data)

	var retData []zipComment

	buf.Seek(0, os.SEEK_SET)
	for {
		var val zipComment
		err := binary.Read(buf, binary.LittleEndian, &val.Len)
		if err == io.EOF {
			break
		}
		if err != nil {
			fmt.Println(err)
			break
		}

		item := make([]byte, val.Len)
		//		num, err := buf.Read(item)
		_, err = buf.Read(item)
		if err == io.EOF {
			break
		}
		if err != nil {

			fmt.Println("err:", err)
			return make([]zipComment, 0)
		}
		//		fmt.Println("num:", num)
		val.Data = string(item)
		retData = append(retData, val)
	}
	return retData
}
<commit_msg>修改BUG<commit_after>package ziputils

import (
	"archive/zip"
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

//压缩文件
func ZipFile(srcPath, dstFileName string, bFlag bool) error {
	buf := new(bytes.Buffer)
	myzip := zip.NewWriter(buf)

	//fmt.Println("srcPath:", srcPath)
	//遍历目录下的所有文件，将文件写入到压缩包中
	err := filepath.Walk(srcPath, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return filepath.SkipDir
		}

		//fmt.Println("path:", path)

		//非目录才写入到压缩包
		if !fi.IsDir() {

			header, err := zip.FileInfoHeader(fi)
			if err != nil {
				return filepath.SkipDir
			}
			//判断是否需要把顶层目录写入到压缩包
			if bFlag {
				refFile, _ := filepath.Rel(filepath.Dir(srcPath), path)
				header.Name = strings.SplitN(refFile, `\`, 2)[1]
			} else {
				header.Name, _ = filepath.Rel(filepath.Dir(srcPath), path)
			}
			//fmt.Println("header.Name:", header.Name)

			//创建一个头
			w, err := myzip.CreateHeader(header)
			if err != nil {
				return err
			}

			fileData, err := ioutil.ReadFile(path)
			if err != nil {
				return filepath.SkipDir
			}
			//写入文件信息
			w.Write(fileData)
		}
		return nil
	})

	myzip.Close()

	//先检查目录存在不存在
	os.MkdirAll(filepath.Dir(dstFileName), 0666)
	// 建立zip文件
	retFile, err := os.Create(dstFileName)
	if err != nil {
		return err
	}
	defer retFile.Close()

	// 将buf中的数据写入文件
	_, err = buf.WriteTo(retFile)
	if err != nil {
		return err
	}
	return nil
}

//解压文件
func UnZip(srcFile, dstPath string) error {

	//创建一个目录
	err := os.MkdirAll(dstPath, 0666)
	if err != nil {
		fmt.Println("创建目录失败, path:", dstPath, ", err:", err)
		return err
	}

	//读取zip文件
	cf, err := zip.OpenReader(srcFile)
	if err != nil {
		fmt.Println(err)
		return err
	}
	defer cf.Close()

	for _, file := range cf.File {

		rc, err := file.Open()
		if err != nil {
			fmt.Println("open file failed, err:", err)
			return err
		}

		if file.FileInfo().IsDir() {
			//创建目录
			err = os.MkdirAll(filepath.Join(dstPath, file.Name), 0666)
			if err != nil && err != os.ErrExist {
				fmt.Println("创建目录失败, path:", filepath.Dir(filepath.Join(dstPath, file.Name)), ", err:", err)
				return err
			}
		} else {

			//创建文件所在目录
			err = os.MkdirAll(filepath.Dir(filepath.Join(dstPath, file.Name)), 0666)
			if err != nil && err != os.ErrExist {
				fmt.Println("创建目录失败, path:", filepath.Dir(filepath.Join(dstPath, file.Name)), ", err:", err)
				return err
			}
			//创建文件
			f, err := os.Create(filepath.Join(dstPath, file.Name))
			if err != nil {
				fmt.Println("create file failed, err:", err)
				return err
			}
			defer f.Close()
			_, err = io.Copy(f, rc)
			if err != nil {
				fmt.Println("copy file failed, err:", err)
				return err
			}
		}

	}
	return nil
}

func WriteComment(fileName string, comment ...string) error {
	//打开文件
	f, err := os.OpenFile(fileName, os.O_RDWR, 0666)
	if err != nil {
		fmt.Println("打开文件失败, err:", err)
		return err
	}
	defer f.Close()

	//将要写入的comment进行字节序列化
	commentList := make([]zipComment, 0, len(comment))
	item := zipComment{}
	for _, v := range comment {
		item.Data = v
		item.getLen()
		commentList = append(commentList, item)
	}
	byteComment := pack(commentList...)
	//	fmt.Println("comment bytes:", byteComment)

	f.Seek(0, os.SEEK_END)

	//写入comment字节流
	num, err := f.Write(byteComment)
	if err != nil {
		fmt.Println(err)
		return err
	}
	//写入comment长度 2字节
	err = binary.Write(f, binary.BigEndian, uint16(num))
	if err != nil {
		fmt.Println("err:", err)
		return err
	}

	//将zip包的comment字段长度修改
	seekLen := int64(0) - int64(num) - int64(4)

	var commentLen uint16 = uint16(num) + 2
	f.Seek(seekLen, os.SEEK_END)
	err = binary.Write(f, binary.BigEndian, commentLen)
	if err != nil {
		fmt.Println("err:", err)
		return err
	}
	//	fmt.Println("write num:", num)
	return nil
}

func ReadComment(fileName string) []string {
	//打开文件
	f, err := os.OpenFile(fileName, os.O_RDONLY, 0666)
	if err != nil {
		fmt.Println("打开文件失败, err:", err)
		return make([]string, 0)
	}
	defer f.Close()

	//定位到comment长度字节流的位置
	f.Seek(-2, os.SEEK_END)

	//获取comment长度
	var commentLen uint16
	err = binary.Read(f, binary.BigEndian, &commentLen)
	if err != nil {
		fmt.Println("获取comment长度错误, err:", err)
		return make([]string, 0)
	}

	//	fmt.Println("commentLen:", commentLen)

	//定位到comment字节流的开始位置
	seekLen := int64(0) - int64(commentLen) - int64(2)
	f.Seek(seekLen, os.SEEK_END)

	//读取comment字节流
	comment := make([]byte, commentLen)
	//	num, err := f.Read(comment)
	_, err = f.Read(comment)
	if err != nil {
		fmt.Println("读取comment数据失败, err:", err)
		return make([]string, 0)
	}

	//	fmt.Println("comment read len:", num)

	//解析comment字节流
	commentData := unPack(comment)
	//	fmt.Println("commentData:", commentData)

	retData := make([]string, 0, len(commentData))
	for _, v := range commentData {
		retData = append(retData, v.Data)
	}
	return retData
}

//comment单元素结构
type zipComment struct {
	Data string
	Len  uint16
}

func (this *zipComment) getLen() {
	this.Len = uint16(len([]byte(this.Data)))
}

//comment元素字节序列化
func pack(data ...zipComment) []byte {
	buf := new(bytes.Buffer)

	for _, v := range data {
		err := binary.Write(buf, binary.LittleEndian, v.Len)
		if err != nil {
			fmt.Println(err)
			return buf.Bytes()
		}
		buf.WriteString(v.Data)
	}

	return buf.Bytes()
}

//comment元素字节反序列化
func unPack(data []byte) []zipComment {
	buf := bytes.NewReader(data)

	var retData []zipComment

	buf.Seek(0, os.SEEK_SET)
	for {
		var val zipComment
		err := binary.Read(buf, binary.LittleEndian, &val.Len)
		if err == io.EOF {
			break
		}
		if err != nil {
			fmt.Println(err)
			break
		}

		item := make([]byte, val.Len)
		//		num, err := buf.Read(item)
		_, err = buf.Read(item)
		if err == io.EOF {
			break
		}
		if err != nil {

			fmt.Println("err:", err)
			return make([]zipComment, 0)
		}
		//		fmt.Println("num:", num)
		val.Data = string(item)
		retData = append(retData, val)
	}
	return retData
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/absolute8511/ZanRedisDB/common"
	"github.com/absolute8511/ZanRedisDB/node"
	"github.com/absolute8511/ZanRedisDB/rockredis"
	"github.com/siddontang/goredis"
	"github.com/stretchr/testify/assert"
)

type testClusterInfo struct {
	server    *Server
	nsConf    *node.NamespaceConfig
	redisPort int
	replicaID uint64
}

var testClusterOnce sync.Once
var kvsCluster []testClusterInfo
var learnerServers []*Server
var gtmpClusterDir string
var seedNodes []node.ReplicaInfo

func TestMain(m *testing.M) {
	//SetLogger(2, newTestLogger(t))
	if testing.Verbose() {
		rockredis.SetLogLevel(4)
		node.SetLogLevel(4)
	}
	ret := m.Run()
	if kvs != nil {
		kvs.Stop()
	}
	if kvsMerge != nil {
		kvsMerge.Stop()
	}
	if kvsFullScan != nil {
		kvsFullScan.Stop()
	}
	for _, v := range kvsCluster {
		v.server.Stop()
	}
	if ret == 0 {
		if strings.Contains(gtmpClusterDir, "rocksdb-test") {
			fmt.Println("removing: ", gtmpClusterDir)
			os.RemoveAll(gtmpClusterDir)
		}
		if strings.Contains(gtmpMergeDir, "rocksdb-test") {
			fmt.Println("removing: ", gtmpMergeDir)
			os.RemoveAll(gtmpMergeDir)
		}
		if strings.Contains(gtmpScanDir, "rocksdb-test") {
			fmt.Println("removing: ", gtmpScanDir)
			os.RemoveAll(gtmpScanDir)
		}
		if strings.Contains(gtmpDir, "rocksdb-test") {
			fmt.Println("removing: ", gtmpDir)
			os.RemoveAll(gtmpDir)
		}
	}
	os.Exit(ret)
}

func startTestCluster(t *testing.T, replicaNum int, syncLearnerNum int) ([]testClusterInfo, []*Server, string) {
	ctmpDir, err := ioutil.TempDir("", fmt.Sprintf("rocksdb-test-%d", time.Now().UnixNano()))
	assert.Nil(t, err)
	SetLogger(2, newTestLogger(t))
	node.SetLogger(2, newTestLogger(t))
	rockredis.SetLogger(2, newTestLogger(t))
	t.Logf("dir:%v\n", ctmpDir)
	kvsClusterTmp := make([]testClusterInfo, 0, replicaNum)
	learnerServersTmp := make([]*Server, 0, syncLearnerNum)
	rport := 52845
	raftPort := 52745
	seedNodes = make([]node.ReplicaInfo, 0, replicaNum)
	for index := 0; index < replicaNum; index++ {
		raftAddr := "http://127.0.0.1:" + strconv.Itoa(raftPort+index)
		var replica node.ReplicaInfo
		replica.NodeID = uint64(1 + index)
		replica.ReplicaID = uint64(1 + index)
		replica.RaftAddr = raftAddr
		seedNodes = append(seedNodes, replica)
	}
	for index := 0; index < replicaNum+syncLearnerNum; index++ {
		tmpDir := path.Join(ctmpDir, strconv.Itoa(index))
		os.MkdirAll(tmpDir, 0700)
		ioutil.WriteFile(
			path.Join(tmpDir, "myid"),
			[]byte(strconv.FormatInt(int64(1+index), 10)),
			common.FILE_PERM)
		raftAddr := "http://127.0.0.1:" + strconv.Itoa(raftPort+index)
		redisport := rport + index
		var replica node.ReplicaInfo
		replica.NodeID = uint64(1 + index)
		replica.ReplicaID = uint64(1 + index)
		replica.RaftAddr = raftAddr
		kvOpts := ServerConfig{
			ClusterID:     "unit-test-cluster",
			DataDir:       tmpDir,
			RedisAPIPort:  redisport,
			LocalRaftAddr: raftAddr,
			BroadcastAddr: "127.0.0.1",
			TickMs:        20,
			ElectionTick:  20,
		}
		if index >= replicaNum {
			kvOpts.LearnerRole = common.LearnerRoleLogSyncer
		}
		if testing.Verbose() {
			rockredis.SetLogLevel(4)
			node.SetLogLevel(4)
		}
		nsConf := node.NewNSConfig()
		nsConf.Name = "default-0"
		nsConf.BaseName = "default"
		nsConf.EngType = rockredis.EngType
		nsConf.PartitionNum = 1
		nsConf.Replicator = replicaNum
		nsConf.RaftGroupConf.GroupID = 1000
		nsConf.RaftGroupConf.SeedNodes = seedNodes
		nsConf.ExpirationPolicy = "consistency_deletion"
		kv := NewServer(kvOpts)
		if _, err := kv.InitKVNamespace(replica.ReplicaID, nsConf, false); err != nil {
			t.Fatalf("failed to init namespace: %v", err)
		}
		kv.Start()
		if index >= replicaNum {
			learnerServersTmp = append(learnerServersTmp, kv)
		} else {
			kvsClusterTmp = append(kvsClusterTmp, testClusterInfo{server: kv,
				nsConf: nsConf, redisPort: redisport, replicaID: replica.ReplicaID})
		}
	}

	time.Sleep(time.Second * 3)
	return kvsClusterTmp, learnerServersTmp, ctmpDir
}

func getTestClusterConn(t *testing.T, needLeader bool) *goredis.PoolConn {
	testClusterOnce.Do(func() {
		kvsCluster, learnerServers, gtmpClusterDir = startTestCluster(t, 3, 1)
	},
	)
	rport := 0
	for _, n := range kvsCluster {
		replicaNode := n.server.GetNamespaceFromFullName("default-0")
		assert.NotNil(t, replicaNode)
		if needLeader {
			if replicaNode.Node.IsLead() {
				rport = n.redisPort
				break
			}
		} else {
			rport = n.redisPort
			break
		}
	}
	c := goredis.NewClient("127.0.0.1:"+strconv.Itoa(rport), "")
	c.SetMaxIdleConns(4)
	conn, err := c.Get()
	if err != nil {
		t.Fatal(err)
	}
	return conn
}

func TestStartCluster(t *testing.T) {
	c := getTestClusterConn(t, false)
	defer c.Close()

	assert.Equal(t, 3, len(kvsCluster))
	var leaderNode *node.NamespaceNode
	for _, n := range kvsCluster {
		replicaNode := n.server.GetNamespaceFromFullName("default-0")
		assert.NotNil(t, replicaNode)
		if replicaNode.Node.IsLead() {
			leaderNode = replicaNode
			break
		}
	}

	assert.Equal(t, 1, len(learnerServers))
	learnerNode := learnerServers[0].GetNamespaceFromFullName("default-0")
	assert.NotNil(t, learnerNode)
	m := learnerNode.Node.GetLocalMemberInfo()
	nsStats := learnerNode.Node.GetStats()
	assert.Equal(t, common.LearnerRoleLogSyncer, nsStats.InternalStats["role"])

	raftStats := leaderNode.Node.GetRaftStatus()
	_, ok := raftStats.Progress[m.ID]
	assert.Equal(t, false, ok)

	err := leaderNode.Node.ProposeAddLearner(*m)
	assert.Nil(t, err)
	time.Sleep(time.Second * 3)
	assert.Equal(t, true, learnerNode.IsReady())

	key := "default:test-cluster:a"
	rsp, err := goredis.String(c.Do("set", key, "1234"))
	assert.Nil(t, err)
	assert.Equal(t, OK, rsp)

	var sindex uint64
	start := time.Now()
	for {
		nsStats = learnerNode.Node.GetStats()
		if nsStats.InternalStats["synced"].(int64) >= 1 {
			assert.Equal(t, int64(1), nsStats.InternalStats["synced"])
			sindex = nsStats.InternalStats["synced_index"].(uint64)
			assert.Equal(t, true, sindex > uint64(3))
			break
		}
		time.Sleep(time.Second)
		if time.Since(start) > time.Minute {
			t.Errorf("\033[31m timed out %v for wait raft stats \033[39m\n", time.Since(start))
			break
		}
	}

	v, err := goredis.String(c.Do("get", key))
	assert.Nil(t, err)
	assert.Equal(t, "1234", v)
	_, err = goredis.Int(c.Do("del", key))
	assert.Nil(t, err)

	time.Sleep(time.Second * 3)
	nsStats = learnerNode.Node.GetStats()
	assert.Equal(t, int64(2), nsStats.InternalStats["synced"])
	assert.Equal(t, sindex+1, nsStats.InternalStats["synced_index"])
	sindex = nsStats.InternalStats["synced_index"].(uint64)

	n, err := goredis.Int(c.Do("exists", key))
	assert.Nil(t, err)
	assert.Equal(t, 0, n)

	raftStats = leaderNode.Node.GetRaftStatus()
	pr := raftStats.Progress[m.ID]
	assert.Equal(t, true, pr.IsLearner)

	learnerNode.Close()
	time.Sleep(time.Second)
	learnerNode = learnerServers[0].GetNamespaceFromFullName("default-0")
	assert.Nil(t, learnerNode)

	_, err = goredis.Int(c.Do("del", key))
	assert.Nil(t, err)

	// restart will replay all logs
	nsConf := node.NewNSConfig()
	nsConf.Name = "default-0"
	nsConf.BaseName = "default"
	nsConf.EngType = rockredis.EngType
	nsConf.PartitionNum = 1
	nsConf.Replicator = 3
	nsConf.RaftGroupConf.GroupID = 1000
	nsConf.ExpirationPolicy = "consistency_deletion"
	learnerNode, err = learnerServers[0].InitKVNamespace(m.ID, nsConf, true)
	assert.Nil(t, err)
	err = learnerNode.Start(false)
	assert.Nil(t, err)

	start = time.Now()
	for {
		time.Sleep(time.Second)
		nsStats = learnerNode.Node.GetStats()
		if nsStats.InternalStats["synced"].(int64) >= 3 {
			assert.Equal(t, int64(3), nsStats.InternalStats["synced"])
			assert.Equal(t, sindex+1, nsStats.InternalStats["synced_index"])
			break
		}

		if time.Since(start) > time.Minute {
			t.Errorf("\033[31m timed out %v for wait raft stats \033[39m\n", time.Since(start))
			break
		}
	}
}

func TestRestartFollower(t *testing.T) {
	c := getTestClusterConn(t, true)
	defer c.Close()

	assert.Equal(t, 3, len(kvsCluster))
	var leaderNode *node.NamespaceNode
	var followerS testClusterInfo
	var follower *node.NamespaceNode
	for _, n := range kvsCluster {
		replicaNode := n.server.GetNamespaceFromFullName("default-0")
		assert.NotNil(t, replicaNode)
		if replicaNode.Node.IsLead() {
			leaderNode = replicaNode
		} else {
			followerS = n
			follower = replicaNode
			break
		}
	}

	ci := follower.Node.GetCommittedIndex()
	m := follower.Node.GetLocalMemberInfo()
	follower.Close()
	_ = leaderNode
	key := "default:test-cluster:a"
	rsp, err := goredis.String(c.Do("set", key, "1234"))
	assert.Nil(t, err)
	assert.Equal(t, OK, rsp)

	follower, err = followerS.server.InitKVNamespace(m.ID, followerS.nsConf, true)
	assert.Nil(t, err)
	follower.Start(false)
	start := time.Now()
	for {
		time.Sleep(time.Second)
		if ci+1 >= follower.Node.GetCommittedIndex() {
			// restart follower should catchup with new committed
			assert.Equal(t, ci+1, follower.Node.GetCommittedIndex())
			break
		}
		if time.Since(start) > time.Minute {
			t.Errorf("\033[31m timed out %v for wait raft stats \033[39m\n", time.Since(start))
			break
		}
	}
}

func TestRestartCluster(t *testing.T) {
	// stop all nodes in cluster and start one by one
	c := getTestClusterConn(t, true)
	defer c.Close()

	assert.Equal(t, 3, len(kvsCluster))

	var leaderNode *node.NamespaceNode
	for _, n := range kvsCluster {
		replicaNode := n.server.GetNamespaceFromFullName("default-0")
		assert.NotNil(t, replicaNode)
		if replicaNode.Node.IsLead() {
			leaderNode = replicaNode
			break
		}
	}

	ci := leaderNode.Node.GetCommittedIndex()

	key := "default:test-cluster:a"
	rsp, err := goredis.String(c.Do("set", key, "1234"))
	assert.Nil(t, err)
	assert.Equal(t, OK, rsp)

	for _, s := range kvsCluster {
		node := s.server.GetNamespaceFromFullName("default-0")
		node.Close()
	}

	for _, s := range kvsCluster {
		node, err := s.server.InitKVNamespace(s.replicaID, s.nsConf, true)
		assert.Nil(t, err)
		assert.NotNil(t, node)
		err = node.Start(false)
		assert.Nil(t, err)
	}
	time.Sleep(time.Second * 2)

	hasLeader := false
	for _, s := range kvsCluster {
		replicaNode := s.server.GetNamespaceFromFullName("default-0")
		assert.NotNil(t, replicaNode)
		newci := replicaNode.Node.GetCommittedIndex()
		assert.Equal(t, ci+1+1, newci)
		if replicaNode.Node.IsLead() {
			hasLeader = true
		}
	}
	assert.Equal(t, true, hasLeader)
}
<commit_msg>fix test case<commit_after>package server

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/absolute8511/ZanRedisDB/common"
	"github.com/absolute8511/ZanRedisDB/node"
	"github.com/absolute8511/ZanRedisDB/rockredis"
	"github.com/siddontang/goredis"
	"github.com/stretchr/testify/assert"
)

type testClusterInfo struct {
	server    *Server
	nsConf    *node.NamespaceConfig
	redisPort int
	replicaID uint64
}

var testClusterOnce sync.Once
var kvsCluster []testClusterInfo
var learnerServers []*Server
var gtmpClusterDir string
var seedNodes []node.ReplicaInfo

func TestMain(m *testing.M) {
	//SetLogger(2, newTestLogger(t))
	if testing.Verbose() {
		rockredis.SetLogLevel(4)
		node.SetLogLevel(4)
	}
	ret := m.Run()
	if kvs != nil {
		kvs.Stop()
	}
	if kvsMerge != nil {
		kvsMerge.Stop()
	}
	if kvsFullScan != nil {
		kvsFullScan.Stop()
	}
	for _, v := range kvsCluster {
		v.server.Stop()
	}
	if ret == 0 {
		if strings.Contains(gtmpClusterDir, "rocksdb-test") {
			fmt.Println("removing: ", gtmpClusterDir)
			os.RemoveAll(gtmpClusterDir)
		}
		if strings.Contains(gtmpMergeDir, "rocksdb-test") {
			fmt.Println("removing: ", gtmpMergeDir)
			os.RemoveAll(gtmpMergeDir)
		}
		if strings.Contains(gtmpScanDir, "rocksdb-test") {
			fmt.Println("removing: ", gtmpScanDir)
			os.RemoveAll(gtmpScanDir)
		}
		if strings.Contains(gtmpDir, "rocksdb-test") {
			fmt.Println("removing: ", gtmpDir)
			os.RemoveAll(gtmpDir)
		}
	}
	os.Exit(ret)
}

func startTestCluster(t *testing.T, replicaNum int, syncLearnerNum int) ([]testClusterInfo, []*Server, string) {
	ctmpDir, err := ioutil.TempDir("", fmt.Sprintf("rocksdb-test-%d", time.Now().UnixNano()))
	assert.Nil(t, err)
	SetLogger(2, newTestLogger(t))
	node.SetLogger(2, newTestLogger(t))
	rockredis.SetLogger(2, newTestLogger(t))
	t.Logf("dir:%v\n", ctmpDir)
	kvsClusterTmp := make([]testClusterInfo, 0, replicaNum)
	learnerServersTmp := make([]*Server, 0, syncLearnerNum)
	rport := 52845
	raftPort := 52745
	seedNodes = make([]node.ReplicaInfo, 0, replicaNum)
	for index := 0; index < replicaNum; index++ {
		raftAddr := "http://127.0.0.1:" + strconv.Itoa(raftPort+index)
		var replica node.ReplicaInfo
		replica.NodeID = uint64(1 + index)
		replica.ReplicaID = uint64(1 + index)
		replica.RaftAddr = raftAddr
		seedNodes = append(seedNodes, replica)
	}
	for index := 0; index < replicaNum+syncLearnerNum; index++ {
		tmpDir := path.Join(ctmpDir, strconv.Itoa(index))
		os.MkdirAll(tmpDir, 0700)
		ioutil.WriteFile(
			path.Join(tmpDir, "myid"),
			[]byte(strconv.FormatInt(int64(1+index), 10)),
			common.FILE_PERM)
		raftAddr := "http://127.0.0.1:" + strconv.Itoa(raftPort+index)
		redisport := rport + index
		var replica node.ReplicaInfo
		replica.NodeID = uint64(1 + index)
		replica.ReplicaID = uint64(1 + index)
		replica.RaftAddr = raftAddr
		kvOpts := ServerConfig{
			ClusterID:     "unit-test-cluster",
			DataDir:       tmpDir,
			RedisAPIPort:  redisport,
			LocalRaftAddr: raftAddr,
			BroadcastAddr: "127.0.0.1",
			TickMs:        20,
			ElectionTick:  20,
		}
		if index >= replicaNum {
			kvOpts.LearnerRole = common.LearnerRoleLogSyncer
		}
		if testing.Verbose() {
			rockredis.SetLogLevel(4)
			node.SetLogLevel(4)
		}
		nsConf := node.NewNSConfig()
		nsConf.Name = "default-0"
		nsConf.BaseName = "default"
		nsConf.EngType = rockredis.EngType
		nsConf.PartitionNum = 1
		nsConf.Replicator = replicaNum
		nsConf.RaftGroupConf.GroupID = 1000
		nsConf.RaftGroupConf.SeedNodes = seedNodes
		nsConf.ExpirationPolicy = "consistency_deletion"
		kv := NewServer(kvOpts)
		if _, err := kv.InitKVNamespace(replica.ReplicaID, nsConf, false); err != nil {
			t.Fatalf("failed to init namespace: %v", err)
		}
		kv.Start()
		if index >= replicaNum {
			learnerServersTmp = append(learnerServersTmp, kv)
		} else {
			kvsClusterTmp = append(kvsClusterTmp, testClusterInfo{server: kv,
				nsConf: nsConf, redisPort: redisport, replicaID: replica.ReplicaID})
		}
	}

	time.Sleep(time.Second * 3)
	return kvsClusterTmp, learnerServersTmp, ctmpDir
}

func getTestClusterConn(t *testing.T, needLeader bool) *goredis.PoolConn {
	testClusterOnce.Do(func() {
		kvsCluster, learnerServers, gtmpClusterDir = startTestCluster(t, 3, 1)
	},
	)
	rport := 0
	for _, n := range kvsCluster {
		replicaNode := n.server.GetNamespaceFromFullName("default-0")
		assert.NotNil(t, replicaNode)
		if needLeader {
			if replicaNode.Node.IsLead() {
				rport = n.redisPort
				break
			}
		} else {
			rport = n.redisPort
			break
		}
	}
	c := goredis.NewClient("127.0.0.1:"+strconv.Itoa(rport), "")
	c.SetMaxIdleConns(4)
	conn, err := c.Get()
	if err != nil {
		t.Fatal(err)
	}
	return conn
}

func waitForLeader(t *testing.T, w time.Duration) *node.NamespaceNode {
	start := time.Now()
	for {
		for _, n := range kvsCluster {
			replicaNode := n.server.GetNamespaceFromFullName("default-0")
			assert.NotNil(t, replicaNode)
			if replicaNode.Node.IsLead() {
				leaderNode := replicaNode
				return leaderNode
			}
		}
		if time.Since(start) > w {
			t.Fatalf("\033[31m timed out %v for wait leader \033[39m\n", time.Since(start))
			break
		}
		time.Sleep(time.Second)
	}
	return nil
}
func TestStartCluster(t *testing.T) {
	c := getTestClusterConn(t, false)
	defer c.Close()

	assert.Equal(t, 3, len(kvsCluster))
	leaderNode := waitForLeader(t, time.Minute)
	assert.NotNil(t, leaderNode)

	assert.Equal(t, 1, len(learnerServers))
	learnerNode := learnerServers[0].GetNamespaceFromFullName("default-0")
	assert.NotNil(t, learnerNode)
	m := learnerNode.Node.GetLocalMemberInfo()
	nsStats := learnerNode.Node.GetStats()
	assert.Equal(t, common.LearnerRoleLogSyncer, nsStats.InternalStats["role"])

	raftStats := leaderNode.Node.GetRaftStatus()
	_, ok := raftStats.Progress[m.ID]
	assert.Equal(t, false, ok)

	err := leaderNode.Node.ProposeAddLearner(*m)
	assert.Nil(t, err)
	time.Sleep(time.Second * 3)
	assert.Equal(t, true, learnerNode.IsReady())

	key := "default:test-cluster:a"
	rsp, err := goredis.String(c.Do("set", key, "1234"))
	assert.Nil(t, err)
	assert.Equal(t, OK, rsp)

	var sindex uint64
	start := time.Now()
	for {
		nsStats = learnerNode.Node.GetStats()
		if nsStats.InternalStats["synced"].(int64) >= 1 {
			assert.Equal(t, int64(1), nsStats.InternalStats["synced"])
			sindex = nsStats.InternalStats["synced_index"].(uint64)
			assert.Equal(t, true, sindex > uint64(3))
			break
		}
		time.Sleep(time.Second)
		if time.Since(start) > time.Minute {
			t.Errorf("\033[31m timed out %v for wait raft stats \033[39m\n", time.Since(start))
			break
		}
	}

	v, err := goredis.String(c.Do("get", key))
	assert.Nil(t, err)
	assert.Equal(t, "1234", v)
	_, err = goredis.Int(c.Do("del", key))
	assert.Nil(t, err)

	time.Sleep(time.Second * 3)
	nsStats = learnerNode.Node.GetStats()
	assert.Equal(t, int64(2), nsStats.InternalStats["synced"])
	assert.Equal(t, sindex+1, nsStats.InternalStats["synced_index"])
	sindex = nsStats.InternalStats["synced_index"].(uint64)

	n, err := goredis.Int(c.Do("exists", key))
	assert.Nil(t, err)
	assert.Equal(t, 0, n)

	raftStats = leaderNode.Node.GetRaftStatus()
	pr := raftStats.Progress[m.ID]
	assert.Equal(t, true, pr.IsLearner)

	learnerNode.Close()
	time.Sleep(time.Second)
	learnerNode = learnerServers[0].GetNamespaceFromFullName("default-0")
	assert.Nil(t, learnerNode)

	_, err = goredis.Int(c.Do("del", key))
	assert.Nil(t, err)

	// restart will replay all logs
	nsConf := node.NewNSConfig()
	nsConf.Name = "default-0"
	nsConf.BaseName = "default"
	nsConf.EngType = rockredis.EngType
	nsConf.PartitionNum = 1
	nsConf.Replicator = 3
	nsConf.RaftGroupConf.GroupID = 1000
	nsConf.ExpirationPolicy = "consistency_deletion"
	learnerNode, err = learnerServers[0].InitKVNamespace(m.ID, nsConf, true)
	assert.Nil(t, err)
	err = learnerNode.Start(false)
	assert.Nil(t, err)

	start = time.Now()
	for {
		time.Sleep(time.Second)
		nsStats = learnerNode.Node.GetStats()
		newSindex := nsStats.InternalStats["synced_index"].(uint64)
		if newSindex > sindex {
			assert.Equal(t, true, nsStats.InternalStats["synced"].(int64) >= 1)
			assert.Equal(t, sindex+1, newSindex)
			break
		}

		if time.Since(start) > time.Minute {
			t.Errorf("\033[31m timed out %v for wait raft stats \033[39m\n", time.Since(start))
			break
		}
	}
}

func TestRestartFollower(t *testing.T) {
	c := getTestClusterConn(t, true)
	defer c.Close()

	assert.Equal(t, 3, len(kvsCluster))
	leaderNode := waitForLeader(t, time.Minute)
	assert.NotNil(t, leaderNode)
	var followerS testClusterInfo
	var follower *node.NamespaceNode
	for _, n := range kvsCluster {
		replicaNode := n.server.GetNamespaceFromFullName("default-0")
		assert.NotNil(t, replicaNode)
		if !replicaNode.Node.IsLead() {
			followerS = n
			follower = replicaNode
			break
		}
	}

	ci := follower.Node.GetCommittedIndex()
	m := follower.Node.GetLocalMemberInfo()
	follower.Close()
	_ = leaderNode
	key := "default:test-cluster:a"
	rsp, err := goredis.String(c.Do("set", key, "1234"))
	assert.Nil(t, err)
	assert.Equal(t, OK, rsp)

	follower, err = followerS.server.InitKVNamespace(m.ID, followerS.nsConf, true)
	assert.Nil(t, err)
	follower.Start(false)
	start := time.Now()
	for {
		time.Sleep(time.Second)
		if ci+1 >= follower.Node.GetCommittedIndex() {
			// restart follower should catchup with new committed
			assert.Equal(t, ci+1, follower.Node.GetCommittedIndex())
			break
		}
		if time.Since(start) > time.Minute {
			t.Errorf("\033[31m timed out %v for wait raft stats \033[39m\n", time.Since(start))
			break
		}
	}
}

func TestRestartCluster(t *testing.T) {
	// stop all nodes in cluster and start one by one
	c := getTestClusterConn(t, true)
	defer c.Close()

	assert.Equal(t, 3, len(kvsCluster))

	leaderNode := waitForLeader(t, time.Minute)
	assert.NotNil(t, leaderNode)

	ci := leaderNode.Node.GetCommittedIndex()

	key := "default:test-cluster:a"
	rsp, err := goredis.String(c.Do("set", key, "1234"))
	assert.Nil(t, err)
	assert.Equal(t, OK, rsp)

	for _, s := range kvsCluster {
		node := s.server.GetNamespaceFromFullName("default-0")
		node.Close()
	}

	for _, s := range kvsCluster {
		node, err := s.server.InitKVNamespace(s.replicaID, s.nsConf, true)
		assert.Nil(t, err)
		assert.NotNil(t, node)
		err = node.Start(false)
		assert.Nil(t, err)
	}
	time.Sleep(time.Second * 2)

	hasLeader := false
	for _, s := range kvsCluster {
		replicaNode := s.server.GetNamespaceFromFullName("default-0")
		assert.NotNil(t, replicaNode)
		newci := replicaNode.Node.GetCommittedIndex()
		assert.Equal(t, ci+1+1, newci)
		if replicaNode.Node.IsLead() {
			hasLeader = true
		}
	}
	assert.Equal(t, true, hasLeader)
}
<|endoftext|>


<commit_before>package templates

import (
	"fmt"
	"math"
	"math/rand"
	"testing"
)

func TestOneDims(t *testing.T) {
	test1d("tan", tan, d_tan, t)
	test1d("abs", abs, d_abs, t)
	test1d("atan", atan, d_atan, t)
	test1d("tanh", tanh, d_tanh, t)
	test1d("sin", sin, d_sin, t)
	test1d("asin", asin, d_asin, t)
	test1d("sinh", sinh, d_sinh, t)
	test1d("cos", cos, d_cos, t)
	test1d("acos", acos, d_acos, t)
	test1d("cosh", cosh, d_cosh, t)
	test1d("sqrt", sqrt, d_sqrt, t)
	test1d("exp", exp, d_exp, t)
	test1d("exp2", exp2, d_exp2, t)
	test1d("log", log, d_log, t)
	test1d("log2", log2, d_log2, t)
}

func TestTwoDims(t *testing.T) {
	test2d("add", add, d_add, t)
	test2d("multiply", multiply, d_multiply, t)
	test2d("subtract", subtract, d_subtract, t)
	test2d("divide", divide, d_divide, t)
	test2d("pow", pow, d_pow, t)
}

func test2d(name string, f Function2D, df DFunction2D, t *testing.T) {
	var n int
	for n < 10 {
		x := rand.NormFloat64()
		y := rand.NormFloat64()
		v := f(x, y)
		if math.IsNaN(v) || math.IsInf(v, 0) {
			continue
		}
		n++
		for i := 0; i < 2; i++ {
			if df := math.Abs(df(i, x, y) - f.Derivative()(i, x, y)); df > 0.0001 {
				t.Fatalf("oops: df(%d,%f,%f) = %f for %s", i, x, y, df, name)
			}
		}
	}
}

func test1d(name string, f, d Function, t *testing.T) {
	var n int
	for n < 10 {
		x := rand.NormFloat64()
		y := f(x)
		if math.IsNaN(y) || math.IsInf(y, 0) {
			continue
		}
		n++
		if df := math.Abs(d(x) - f.Derivative()(x)); df > 0.0001 {
			t.Fatalf("oops: df(%f) = %f for %s", x, df, name)
		}
	}
}

type Function2D func(a, b float64) float64
type DFunction2D func(i int, a, b float64) float64

type Function func(x float64) float64

const dx = 0.00000001

func (f Function) Derivative() Function {
	return func(x float64) float64 {
		return (f(x+dx) - f(x)) / dx
	}
}

func (f Function2D) Derivative() DFunction2D {
	return func(i int, a, b float64) float64 {
		var g Function
		var x float64
		switch i {
		case 0:
			g = func(x float64) float64 {
				return f(x, b)
			}
			x = a
		case 1:
			g = func(x float64) float64 {
				return f(a, x)
			}
			x = b
		default:
			panic(fmt.Sprintf("illegal index %d", i))
		}
		return g.Derivative()(x)
	}
}
<commit_msg>more rigorous and statistical testing<commit_after>package templates

import (
	"fmt"
	"math"
	"math/rand"
	"testing"
)

func TestOneDims(t *testing.T) {
	test1d("tan", tan, d_tan, t)
	test1d("abs", abs, d_abs, t)
	test1d("atan", atan, d_atan, t)
	test1d("tanh", tanh, d_tanh, t)
	test1d("sin", sin, d_sin, t)
	test1d("asin", asin, d_asin, t)
	test1d("sinh", sinh, d_sinh, t)
	test1d("cos", cos, d_cos, t)
	test1d("acos", acos, d_acos, t)
	test1d("cosh", cosh, d_cosh, t)
	test1d("sqrt", sqrt, d_sqrt, t)
	test1d("exp", exp, d_exp, t)
	test1d("exp2", exp2, d_exp2, t)
	test1d("log", log, d_log, t)
	test1d("log2", log2, d_log2, t)
}

func TestTwoDims(t *testing.T) {
	test2d("add", add, d_add, t)
	test2d("multiply", multiply, d_multiply, t)
	test2d("subtract", subtract, d_subtract, t)
	test2d("divide", divide, d_divide, t)
	test2d("pow", pow, d_pow, t)
}

func test1d(name string, f, df Function, t *testing.T) {
	df2 := f.Derivative()
	var n, failed int
	for n < n0 {
		a := rand.NormFloat64()
		if y := f(a); math.IsNaN(y) || math.IsInf(y, 0) {
			continue
		}
		n++
		if df := math.Abs(df(a) - df2(a)); df > dx2 {
			failed++
		}
	}
	eval(name, n, failed, t)
}

func test2d(name string, f Function2D, df DFunction2D, t *testing.T) {
	df2 := f.Derivative()
	var n, failed int
	for n < n0 {
		a, b := rand.NormFloat64(), rand.NormFloat64()
		if y := f(a, b); math.IsNaN(y) || math.IsInf(y, 0) {
			continue
		}
		n++
		for i := 0; i < 2; i++ {
			if df := math.Abs(df(i, a, b) - df2(i, a, b)); df > dx2 {
				failed++
			}
		}
	}
	eval(name, n, failed, t)
}

func eval(name string, n, failed int, t *testing.T) {
	if float64(failed)/float64(n) > threshold {
		t.Fatalf("oops: failed %d / %d for %s\n", failed, n, name)
	}
}

type Function func(x float64) float64
type Function2D func(a, b float64) float64
type DFunction2D func(i int, a, b float64) float64

const (
	threshold = 0.03
	n0        = 10000
	dx        = 0.000000001
	dx2       = 0.0001
)

func (f Function) Derivative() Function {
	return func(x float64) float64 {
		return (f(x+dx) - f(x)) / dx
	}
}

func (f Function2D) Derivative() DFunction2D {
	return func(i int, a, b float64) float64 {
		var g Function
		var x float64
		switch i {
		case 0:
			g = func(x float64) float64 {
				return f(x, b)
			}
			x = a
		case 1:
			g = func(x float64) float64 {
				return f(a, x)
			}
			x = b
		default:
			panic(fmt.Sprintf("illegal index %d", i))
		}
		return g.Derivative()(x)
	}
}
<|endoftext|>