<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package context

import (
	"go/ast"
	"go/parser"
	"go/token"
	"path"
	"strconv"
	"strings"

	"github.com/kardianos/govendor/internal/pathos"
	filepath "github.com/kardianos/govendor/internal/vfilepath"
	os "github.com/kardianos/govendor/internal/vos"
)

var knownOS = make(map[string]bool)
var knownArch = make(map[string]bool)

func init() {
	for _, v := range strings.Fields(goosList) {
		knownOS[v] = true
	}
	for _, v := range strings.Fields(goarchList) {
		knownArch[v] = true
	}
}

// loadPackage sets up the context with package information and
// is called before any initial operation is performed.
func (ctx *Context) loadPackage() error {
	ctx.loaded = true
	ctx.dirty = false
	ctx.Package = make(map[string]*Package, len(ctx.Package))
	err := filepath.Walk(ctx.RootDir, func(path string, info os.FileInfo, err error) error {
		if info == nil {
			return err
		}
		name := info.Name()
		// Still go into "_workspace" to aid godep migration.
		if info.IsDir() && (name[0] == '.' || name[0] == '_' || name == "testdata") && name != "_workspace" {
			return filepath.SkipDir
		}
		if info.IsDir() {
			return nil
		}
		return ctx.addFileImports(path, ctx.RootGopath)
	})
	if err != nil {
		return err
	}
	return ctx.determinePackageStatus()
}

func (ctx *Context) getFileTags(pathname string, f *ast.File) ([]string, error) {
	_, filenameExt := filepath.Split(pathname)

	if strings.HasSuffix(pathname, ".go") == false {
		return nil, nil
	}
	var err error
	if f == nil {
		f, err = parser.ParseFile(token.NewFileSet(), pathname, nil, parser.ImportsOnly|parser.ParseComments)
		if err != nil {
			return nil, err
		}
	}

	filename := filenameExt[:len(filenameExt)-3]

	l := strings.Split(filename, "_")
	tags := make([]string, 0)

	if n := len(l); n > 0 && l[n-1] == "test" {
		l = l[:n-1]
		tags = append(tags, "test")
	}
	n := len(l)
	if n >= 2 && knownOS[l[n-2]] && knownArch[l[n-1]] {
		tags = append(tags, l[n-2])
		tags = append(tags, l[n-1])
	}
	if n >= 1 && knownOS[l[n-1]] {
		tags = append(tags, l[n-1])
	}
	if n >= 1 && knownArch[l[n-1]] {
		tags = append(tags, l[n-1])
	}

	const buildPrefix = "// +build "
	for _, cc := range f.Comments {
		for _, c := range cc.List {
			if strings.HasPrefix(c.Text, buildPrefix) {
				text := strings.TrimPrefix(c.Text, buildPrefix)
				ss := strings.Fields(text)
				for _, s := range ss {
					tags = append(tags, strings.Split(s, ",")...)
				}
			}
		}
	}
	return tags, nil
}

// addFileImports is called from loadPackage and resolveUnknown.
func (ctx *Context) addFileImports(pathname, gopath string) error {
	dir, filenameExt := filepath.Split(pathname)
	importPath := pathos.FileTrimPrefix(dir, gopath)
	importPath = pathos.SlashToImportPath(importPath)
	importPath = strings.TrimPrefix(importPath, "/")
	importPath = strings.TrimSuffix(importPath, "/")

	if strings.HasSuffix(pathname, ".go") == false {
		return nil
	}
	f, err := parser.ParseFile(token.NewFileSet(), pathname, nil, parser.ImportsOnly|parser.ParseComments)
	if err != nil {
		return err
	}

	tags, err := ctx.getFileTags(pathname, f)
	if err != nil {
		return err
	}

	pkg, found := ctx.Package[importPath]
	if !found {
		status := StatusUnknown
		if f.Name.Name == "main" {
			status = StatusProgram
		}
		pkg = ctx.setPackage(dir, importPath, importPath, gopath, status)
		ctx.Package[importPath] = pkg
	}
	if pkg.Status != StatusLocal && pkg.Status != StatusProgram {
		for _, tag := range tags {
			for _, ignore := range ctx.ignoreTag {
				if tag == ignore {
					pkg.ignoreFile = append(pkg.ignoreFile, filenameExt)
					return nil
				}
			}
		}
	}
	pf := &File{
		Package: pkg,
		Path:    pathname,
		Imports: make([]string, len(f.Imports)),
	}
	pkg.Files = append(pkg.Files, pf)
	for i := range f.Imports {
		imp := f.Imports[i].Path.Value
		imp, err = strconv.Unquote(imp)
		if err != nil {
			return err
		}
		if strings.HasPrefix(imp, "./") {
			imp = path.Join(importPath, imp)
		}
		pf.Imports[i] = imp
		err = ctx.addSingleImport(pkg.Dir, imp)
		if err != nil {
			return err
		}
	}

	// Record any import comment for file.
	var ic *ast.Comment
	if f.Name != nil {
		pos := f.Name.Pos()
	big:
		// Find the next comment after the package name.
		for _, cblock := range f.Comments {
			for _, c := range cblock.List {
				if c.Pos() > pos {
					ic = c
					break big
				}
			}
		}
	}
	if ic != nil {
		// If it starts with the import text, assume it is the import comment and remove.
		if index := strings.Index(ic.Text, " import "); index > 0 && index < 5 {
			q := strings.TrimSpace(ic.Text[index+len(" import "):])
			pf.ImportComment, err = strconv.Unquote(q)
			if err != nil {
				pf.ImportComment = q
			}
		}
	}

	return nil
}

func (ctx *Context) setPackage(dir, canonical, local, gopath string, status Status) *Package {
	at := 0
	vMiddle := "/" + pathos.SlashToImportPath(ctx.VendorDiscoverFolder) + "/"
	vStart := pathos.SlashToImportPath(ctx.VendorDiscoverFolder) + "/"
	switch {
	case strings.Contains(canonical, vMiddle):
		at = strings.LastIndex(canonical, vMiddle) + len(vMiddle)
	case strings.HasPrefix(canonical, vStart):
		at = strings.LastIndex(canonical, vStart) + len(vStart)
	}

	inVendor := false
	if at > 0 {
		canonical = canonical[at:]
		inVendor = true
		if status == StatusUnknown {
			p := path.Join(ctx.RootImportPath, ctx.VendorDiscoverFolder)
			if strings.HasPrefix(local, p) {
				status = StatusVendor
			}
		}
	}
	if status == StatusUnknown && inVendor == false {
		if vp := ctx.VendorFilePackageLocal(local); vp != nil {
			status = StatusVendor
			inVendor = true
			canonical = vp.Path
		}
	}
	if status == StatusUnknown && strings.HasPrefix(canonical, ctx.RootImportPath) {
		status = StatusLocal
	}
	pkg := &Package{
		Dir:       dir,
		Canonical: canonical,
		Local:     local,
		Gopath:    gopath,
		Status:    status,
		inVendor:  inVendor,
	}
	ctx.Package[local] = pkg
	return pkg
}

func (ctx *Context) addSingleImport(pkgInDir, imp string) error {
	if _, found := ctx.Package[imp]; found {
		return nil
	}
	// Also need to check for vendor paths that won't use the local path in import path.
	for _, pkg := range ctx.Package {
		if pkg.Canonical == imp && pkg.inVendor && pathos.FileHasPrefix(pkg.Dir, pkgInDir) {
			return nil
		}
	}
	dir, gopath, err := ctx.findImportDir(pkgInDir, imp)
	if err != nil {
		if _, is := err.(ErrNotInGOPATH); is {
			ctx.setPackage("", imp, imp, "", StatusMissing)
			return nil
		}
		return err
	}
	if pathos.FileStringEquals(gopath, ctx.Goroot) {
		ctx.setPackage(dir, imp, imp, ctx.Goroot, StatusStandard)
		return nil
	}
	df, err := os.Open(dir)
	if err != nil {
		return err
	}
	info, err := df.Readdir(-1)
	df.Close()
	if err != nil {
		return err
	}
	for _, fi := range info {
		if fi.IsDir() {
			continue
		}
		switch fi.Name()[0] {
		case '.', '_':
			continue
		}
		if pathos.FileStringEquals(dir, pkgInDir) {
			continue
		}
		path := filepath.Join(dir, fi.Name())
		err = ctx.addFileImports(path, gopath)
		if err != nil {
			return err
		}
	}
	return nil
}

func (ctx *Context) determinePackageStatus() error {
	// Determine the status of remaining imports.
	for _, pkg := range ctx.Package {
		if pkg.Status != StatusUnknown {
			continue
		}
		if vp := ctx.VendorFilePackageLocal(pkg.Local); vp != nil {
			pkg.Status = StatusVendor
			pkg.inVendor = true
			pkg.Canonical = vp.Path
			continue
		}
		if strings.HasPrefix(pkg.Canonical, ctx.RootImportPath) {
			pkg.Status = StatusLocal
			continue
		}
		pkg.Status = StatusExternal
	}

	// Check all "external" packages for vendor.
	for _, pkg := range ctx.Package {
		if pkg.Status != StatusExternal {
			continue
		}
		root, err := findRoot(pkg.Dir, vendorFilename)
		if err != nil {
			// No vendor file found.
			if _, is := err.(ErrMissingVendorFile); is {
				continue
			}
			return err
		}
		vf, err := readVendorFile(filepath.Join(root, vendorFilename))
		if err != nil {
			return err
		}
		vpkg := vendorFileFindLocal(vf, root, pkg.Gopath, pkg.Local)
		if vpkg != nil {
			pkg.Canonical = vpkg.Path
		}
	}

	// Determine any un-used internal vendor imports.
	ctx.updatePackageReferences()
	for i := 0; i <= looplimit; i++ {
		altered := false
		for path, pkg := range ctx.Package {
			if len(pkg.referenced) == 0 && pkg.Status == StatusVendor {
				altered = true
				pkg.Status = StatusUnused
				for _, other := range ctx.Package {
					delete(other.referenced, path)
				}
			}
		}
		if !altered {
			break
		}
		if i == looplimit {
			panic("determinePackageStatus loop limit")
		}
	}
	return nil
}
<commit_msg>govendor/context: check if file has already been parsed<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package context

import (
	"go/ast"
	"go/parser"
	"go/token"
	"path"
	"strconv"
	"strings"

	"github.com/kardianos/govendor/internal/pathos"
	filepath "github.com/kardianos/govendor/internal/vfilepath"
	os "github.com/kardianos/govendor/internal/vos"
)

var knownOS = make(map[string]bool)
var knownArch = make(map[string]bool)

func init() {
	for _, v := range strings.Fields(goosList) {
		knownOS[v] = true
	}
	for _, v := range strings.Fields(goarchList) {
		knownArch[v] = true
	}
}

// loadPackage sets up the context with package information and
// is called before any initial operation is performed.
func (ctx *Context) loadPackage() error {
	ctx.loaded = true
	ctx.dirty = false
	ctx.Package = make(map[string]*Package, len(ctx.Package))
	err := filepath.Walk(ctx.RootDir, func(path string, info os.FileInfo, err error) error {
		if info == nil {
			return err
		}
		name := info.Name()
		// Still go into "_workspace" to aid godep migration.
		if info.IsDir() && (name[0] == '.' || name[0] == '_' || name == "testdata") && name != "_workspace" {
			return filepath.SkipDir
		}
		if info.IsDir() {
			return nil
		}
		return ctx.addFileImports(path, ctx.RootGopath)
	})
	if err != nil {
		return err
	}
	return ctx.determinePackageStatus()
}

func (ctx *Context) getFileTags(pathname string, f *ast.File) ([]string, error) {
	_, filenameExt := filepath.Split(pathname)

	if strings.HasSuffix(pathname, ".go") == false {
		return nil, nil
	}
	var err error
	if f == nil {
		f, err = parser.ParseFile(token.NewFileSet(), pathname, nil, parser.ImportsOnly|parser.ParseComments)
		if err != nil {
			return nil, err
		}
	}

	filename := filenameExt[:len(filenameExt)-3]

	l := strings.Split(filename, "_")
	tags := make([]string, 0)

	if n := len(l); n > 0 && l[n-1] == "test" {
		l = l[:n-1]
		tags = append(tags, "test")
	}
	n := len(l)
	if n >= 2 && knownOS[l[n-2]] && knownArch[l[n-1]] {
		tags = append(tags, l[n-2])
		tags = append(tags, l[n-1])
	}
	if n >= 1 && knownOS[l[n-1]] {
		tags = append(tags, l[n-1])
	}
	if n >= 1 && knownArch[l[n-1]] {
		tags = append(tags, l[n-1])
	}

	const buildPrefix = "// +build "
	for _, cc := range f.Comments {
		for _, c := range cc.List {
			if strings.HasPrefix(c.Text, buildPrefix) {
				text := strings.TrimPrefix(c.Text, buildPrefix)
				ss := strings.Fields(text)
				for _, s := range ss {
					tags = append(tags, strings.Split(s, ",")...)
				}
			}
		}
	}
	return tags, nil
}

// addFileImports is called from loadPackage and resolveUnknown.
func (ctx *Context) addFileImports(pathname, gopath string) error {
	dir, filenameExt := filepath.Split(pathname)
	importPath := pathos.FileTrimPrefix(dir, gopath)
	importPath = pathos.SlashToImportPath(importPath)
	importPath = strings.TrimPrefix(importPath, "/")
	importPath = strings.TrimSuffix(importPath, "/")

	if strings.HasSuffix(pathname, ".go") == false {
		return nil
	}
	// No need to add the same file more then once.
	for _, pkg := range ctx.Package {
		if pathos.FileStringEquals(pkg.Dir, dir) == false {
			continue
		}
		for _, f := range pkg.Files {
			if pathos.FileStringEquals(f.Path, pathname) {
				return nil
			}
		}
		for _, f := range pkg.ignoreFile {
			if pathos.FileStringEquals(f, filenameExt) {
				return nil
			}
		}
	}
	f, err := parser.ParseFile(token.NewFileSet(), pathname, nil, parser.ImportsOnly|parser.ParseComments)
	if err != nil {
		return err
	}

	tags, err := ctx.getFileTags(pathname, f)
	if err != nil {
		return err
	}

	pkg, found := ctx.Package[importPath]
	if !found {
		status := StatusUnknown
		if f.Name.Name == "main" {
			status = StatusProgram
		}
		pkg = ctx.setPackage(dir, importPath, importPath, gopath, status)
		ctx.Package[importPath] = pkg
	}
	if pkg.Status != StatusLocal && pkg.Status != StatusProgram {
		for _, tag := range tags {
			for _, ignore := range ctx.ignoreTag {
				if tag == ignore {
					pkg.ignoreFile = append(pkg.ignoreFile, filenameExt)
					return nil
				}
			}
		}
	}
	pf := &File{
		Package: pkg,
		Path:    pathname,
		Imports: make([]string, len(f.Imports)),
	}
	pkg.Files = append(pkg.Files, pf)
	for i := range f.Imports {
		imp := f.Imports[i].Path.Value
		imp, err = strconv.Unquote(imp)
		if err != nil {
			return err
		}
		if strings.HasPrefix(imp, "./") {
			imp = path.Join(importPath, imp)
		}
		pf.Imports[i] = imp
		err = ctx.addSingleImport(pkg.Dir, imp)
		if err != nil {
			return err
		}
	}

	// Record any import comment for file.
	var ic *ast.Comment
	if f.Name != nil {
		pos := f.Name.Pos()
	big:
		// Find the next comment after the package name.
		for _, cblock := range f.Comments {
			for _, c := range cblock.List {
				if c.Pos() > pos {
					ic = c
					break big
				}
			}
		}
	}
	if ic != nil {
		// If it starts with the import text, assume it is the import comment and remove.
		if index := strings.Index(ic.Text, " import "); index > 0 && index < 5 {
			q := strings.TrimSpace(ic.Text[index+len(" import "):])
			pf.ImportComment, err = strconv.Unquote(q)
			if err != nil {
				pf.ImportComment = q
			}
		}
	}

	return nil
}

func (ctx *Context) setPackage(dir, canonical, local, gopath string, status Status) *Package {
	at := 0
	vMiddle := "/" + pathos.SlashToImportPath(ctx.VendorDiscoverFolder) + "/"
	vStart := pathos.SlashToImportPath(ctx.VendorDiscoverFolder) + "/"
	switch {
	case strings.Contains(canonical, vMiddle):
		at = strings.LastIndex(canonical, vMiddle) + len(vMiddle)
	case strings.HasPrefix(canonical, vStart):
		at = strings.LastIndex(canonical, vStart) + len(vStart)
	}

	inVendor := false
	if at > 0 {
		canonical = canonical[at:]
		inVendor = true
		if status == StatusUnknown {
			p := path.Join(ctx.RootImportPath, ctx.VendorDiscoverFolder)
			if strings.HasPrefix(local, p) {
				status = StatusVendor
			}
		}
	}
	if status == StatusUnknown && inVendor == false {
		if vp := ctx.VendorFilePackageLocal(local); vp != nil {
			status = StatusVendor
			inVendor = true
			canonical = vp.Path
		}
	}
	if status == StatusUnknown && strings.HasPrefix(canonical, ctx.RootImportPath) {
		status = StatusLocal
	}
	pkg := &Package{
		Dir:       dir,
		Canonical: canonical,
		Local:     local,
		Gopath:    gopath,
		Status:    status,
		inVendor:  inVendor,
	}
	ctx.Package[local] = pkg
	return pkg
}

func (ctx *Context) addSingleImport(pkgInDir, imp string) error {
	if _, found := ctx.Package[imp]; found {
		return nil
	}
	// Also need to check for vendor paths that won't use the local path in import path.
	for _, pkg := range ctx.Package {
		if pkg.Canonical == imp && pkg.inVendor && pathos.FileHasPrefix(pkg.Dir, pkgInDir) {
			return nil
		}
	}
	dir, gopath, err := ctx.findImportDir(pkgInDir, imp)
	if err != nil {
		if _, is := err.(ErrNotInGOPATH); is {
			ctx.setPackage("", imp, imp, "", StatusMissing)
			return nil
		}
		return err
	}
	if pathos.FileStringEquals(gopath, ctx.Goroot) {
		ctx.setPackage(dir, imp, imp, ctx.Goroot, StatusStandard)
		return nil
	}
	df, err := os.Open(dir)
	if err != nil {
		return err
	}
	info, err := df.Readdir(-1)
	df.Close()
	if err != nil {
		return err
	}
	for _, fi := range info {
		if fi.IsDir() {
			continue
		}
		switch fi.Name()[0] {
		case '.', '_':
			continue
		}
		if pathos.FileStringEquals(dir, pkgInDir) {
			continue
		}
		path := filepath.Join(dir, fi.Name())
		err = ctx.addFileImports(path, gopath)
		if err != nil {
			return err
		}
	}
	return nil
}

func (ctx *Context) determinePackageStatus() error {
	// Determine the status of remaining imports.
	for _, pkg := range ctx.Package {
		if pkg.Status != StatusUnknown {
			continue
		}
		if vp := ctx.VendorFilePackageLocal(pkg.Local); vp != nil {
			pkg.Status = StatusVendor
			pkg.inVendor = true
			pkg.Canonical = vp.Path
			continue
		}
		if strings.HasPrefix(pkg.Canonical, ctx.RootImportPath) {
			pkg.Status = StatusLocal
			continue
		}
		pkg.Status = StatusExternal
	}

	// Check all "external" packages for vendor.
	for _, pkg := range ctx.Package {
		if pkg.Status != StatusExternal {
			continue
		}
		root, err := findRoot(pkg.Dir, vendorFilename)
		if err != nil {
			// No vendor file found.
			if _, is := err.(ErrMissingVendorFile); is {
				continue
			}
			return err
		}
		vf, err := readVendorFile(filepath.Join(root, vendorFilename))
		if err != nil {
			return err
		}
		vpkg := vendorFileFindLocal(vf, root, pkg.Gopath, pkg.Local)
		if vpkg != nil {
			pkg.Canonical = vpkg.Path
		}
	}

	// Determine any un-used internal vendor imports.
	ctx.updatePackageReferences()
	for i := 0; i <= looplimit; i++ {
		altered := false
		for path, pkg := range ctx.Package {
			if len(pkg.referenced) == 0 && pkg.Status == StatusVendor {
				altered = true
				pkg.Status = StatusUnused
				for _, other := range ctx.Package {
					delete(other.referenced, path)
				}
			}
		}
		if !altered {
			break
		}
		if i == looplimit {
			panic("determinePackageStatus loop limit")
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package consistence

import (
	"errors"
	"fmt"
	"github.com/absolute8511/nsq/internal/test"
	"github.com/absolute8511/nsq/nsqd"
	"io/ioutil"
	"os"
	"strconv"
	"testing"
	"time"
)

type fakeNsqdLeadership struct {
	clusterID            string
	regData              map[string]*NsqdNodeInfo
	fakeTopicsLeaderData map[string]map[int]*TopicCoordinator
	fakeTopicsInfo       map[string]map[int]*TopicPartionMetaInfo
}

func NewFakeNSQDLeadership() NSQDLeadership {
	return &fakeNsqdLeadership{
		regData:              make(map[string]*NsqdNodeInfo),
		fakeTopicsLeaderData: make(map[string]map[int]*TopicCoordinator),
		fakeTopicsInfo:       make(map[string]map[int]*TopicPartionMetaInfo),
	}
}

func (self *fakeNsqdLeadership) InitClusterID(id string) {
	self.clusterID = id
}

func (self *fakeNsqdLeadership) RegisterNsqd(nodeData *NsqdNodeInfo) error {
	self.regData[nodeData.GetID()] = nodeData
	return nil
}

func (self *fakeNsqdLeadership) UnregisterNsqd(nodeData *NsqdNodeInfo) error {
	delete(self.regData, nodeData.GetID())
	return nil
}

func (self *fakeNsqdLeadership) IsNodeTopicLeader(topic string, partition int, nodeData *NsqdNodeInfo) bool {
	t, ok := self.fakeTopicsLeaderData[topic]
	var tc *TopicCoordinator
	if ok {
		if tc, ok = t[partition]; ok {
			if tc.topicLeaderSession.LeaderNode != nil {
				if tc.topicLeaderSession.LeaderNode.GetID() == nodeData.GetID() {
					return true
				}
			}
		}
	}
	return false
}

func (self *fakeNsqdLeadership) GetAllLookupdNodes() ([]NsqLookupdNodeInfo, error) {
	v := make([]NsqLookupdNodeInfo, 0)
	return v, nil
}

func (self *fakeNsqdLeadership) AcquireTopicLeader(topic string, partition int, nodeData *NsqdNodeInfo, epoch EpochType) error {
	t, ok := self.fakeTopicsLeaderData[topic]
	var tc *TopicCoordinator
	if ok {
		if tc, ok = t[partition]; ok {
			if tc.topicLeaderSession.LeaderNode != nil {
				if tc.topicLeaderSession.LeaderNode.GetID() == nodeData.GetID() {
					return nil
				}
				return errors.New("topic leader already exist.")
			}
			tc.topicLeaderSession.LeaderNode = nodeData
			tc.topicLeaderSession.LeaderEpoch++
			tc.topicLeaderSession.Session = nodeData.GetID() + strconv.Itoa(int(tc.topicLeaderSession.LeaderEpoch))
			tc.topicInfo.ISR = append(tc.topicInfo.ISR, nodeData.GetID())
			tc.topicInfo.Leader = nodeData.GetID()
			tc.topicInfo.Epoch++
		} else {
			tc = &TopicCoordinator{}
			tc.topicInfo.Name = topic
			tc.topicInfo.Partition = partition
			tc.localDataLoaded = true
			tc.topicInfo.Leader = nodeData.GetID()
			tc.topicInfo.ISR = append(tc.topicInfo.ISR, nodeData.GetID())
			tc.topicInfo.Epoch++
			tc.topicLeaderSession.LeaderNode = nodeData
			tc.topicLeaderSession.LeaderEpoch++
			tc.topicLeaderSession.Session = nodeData.GetID() + strconv.Itoa(int(tc.topicLeaderSession.LeaderEpoch))
			t[partition] = tc
		}
	} else {
		tmp := make(map[int]*TopicCoordinator)
		tc = &TopicCoordinator{}
		tc.topicInfo.Name = topic
		tc.topicInfo.Partition = partition
		tc.localDataLoaded = true
		tc.topicInfo.Leader = nodeData.GetID()
		tc.topicInfo.ISR = append(tc.topicInfo.ISR, nodeData.GetID())
		tc.topicInfo.Epoch++
		tc.topicLeaderSession.LeaderNode = nodeData
		tc.topicLeaderSession.LeaderEpoch++
		tc.topicLeaderSession.Session = nodeData.GetID() + strconv.Itoa(int(tc.topicLeaderSession.LeaderEpoch))
		tmp[partition] = tc
		self.fakeTopicsLeaderData[topic] = tmp
	}
	return nil
}

func (self *fakeNsqdLeadership) ReleaseTopicLeader(topic string, partition int, session *TopicLeaderSession) error {
	t, ok := self.fakeTopicsLeaderData[topic]
	if ok {
		delete(t, partition)
	}
	return nil
}

func (self *fakeNsqdLeadership) WatchLookupdLeader(leader chan *NsqLookupdNodeInfo, stop chan struct{}) error {
	return nil
}

func (self *fakeNsqdLeadership) GetTopicInfo(topic string, partition int) (*TopicPartionMetaInfo, error) {
	t, ok := self.fakeTopicsInfo[topic]
	if ok {
		tp, ok2 := t[partition]
		if ok2 {
			return tp, nil
		}
	}
	return nil, errors.New("topic not exist")
}

func (self *fakeNsqdLeadership) GetTopicLeaderSession(topic string, partition int) (*TopicLeaderSession, error) {
	s, ok := self.fakeTopicsLeaderData[topic]
	if !ok {
		return nil, ErrMissingTopicLeaderSession
	}
	ss, ok := s[partition]
	if !ok {
		return nil, ErrMissingTopicLeaderSession
	}
	return &ss.topicLeaderSession, nil
}

func startNsqdCoord(t *testing.T, rpcport string, dataPath string, extraID string, nsqd *nsqd.NSQD, useFake bool) *NsqdCoordinator {
	nsqdCoord := NewNsqdCoordinator("test-cluster", "127.0.0.1", "0", rpcport, extraID, dataPath, nsqd)
	if useFake {
		nsqdCoord.leadership = NewFakeNSQDLeadership()
		nsqdCoord.lookupRemoteCreateFunc = func(addr string, to time.Duration) (INsqlookupRemoteProxy, error) {
			p, err := NewFakeLookupRemoteProxy(addr, to)
			if err == nil {
				p.(*fakeLookupRemoteProxy).t = t
			}
			return p, err
		}
	} else {
		nsqdCoord.leadership = NewNsqdEtcdMgr(testEtcdServers)
		nsqdCoord.leadership.UnregisterNsqd(&nsqdCoord.myNode)
	}
	nsqdCoord.lookupLeader = NsqLookupdNodeInfo{}
	return nsqdCoord
}

func startNsqdCoordWithFakeData(t *testing.T, rpcport string, dataPath string,
	extraID string, nsqd *nsqd.NSQD, fakeLeadership *fakeNsqdLeadership, fakeLookupProxy *fakeLookupRemoteProxy) *NsqdCoordinator {
	nsqdCoord := NewNsqdCoordinator(TEST_NSQ_CLUSTER_NAME, "127.0.0.1", "0", rpcport, extraID, dataPath, nsqd)
	nsqdCoord.leadership = fakeLeadership
	nsqdCoord.lookupRemoteCreateFunc = func(addr string, to time.Duration) (INsqlookupRemoteProxy, error) {
		fakeLookupProxy.t = t
		fakeLookupProxy.fakeNsqdCoords[nsqdCoord.myNode.GetID()] = nsqdCoord
		return fakeLookupProxy, nil
	}
	nsqdCoord.lookupLeader = NsqLookupdNodeInfo{}
	err := nsqdCoord.Start()
	if err != nil {
		panic(err)
	}
	time.Sleep(time.Second)
	return nsqdCoord
}

func TestNsqdRPCClient(t *testing.T) {
	coordLog.level = 2
	tmpDir, err := ioutil.TempDir("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano()))
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(tmpDir)

	nsqdCoord := startNsqdCoord(t, "0", tmpDir, "", nil, true)
	nsqdCoord.Start()
	defer nsqdCoord.Stop()
	time.Sleep(time.Second * 2)
	client, err := NewNsqdRpcClient(nsqdCoord.rpcServer.rpcListener.Addr().String(), time.Second)
	test.Nil(t, err)
	var rspInt int32
	err = client.CallWithRetry("NsqdCoordinator.TestRpcCallNotExist", "req", &rspInt)
	test.NotNil(t, err)

	rsp, rpcErr := client.CallRpcTest("reqdata")
	test.NotNil(t, rpcErr)
	test.Equal(t, rsp, "reqdata")
	test.Equal(t, rpcErr.ErrCode, RpcNoErr)
	test.Equal(t, rpcErr.ErrMsg, "reqdata")
	test.Equal(t, rpcErr.ErrType, CoordCommonErr)
}
<commit_msg>fix test cluster name<commit_after>package consistence

import (
	"errors"
	"fmt"
	"github.com/absolute8511/nsq/internal/test"
	"github.com/absolute8511/nsq/nsqd"
	"io/ioutil"
	"os"
	"strconv"
	"testing"
	"time"
)

type fakeNsqdLeadership struct {
	clusterID            string
	regData              map[string]*NsqdNodeInfo
	fakeTopicsLeaderData map[string]map[int]*TopicCoordinator
	fakeTopicsInfo       map[string]map[int]*TopicPartionMetaInfo
}

func NewFakeNSQDLeadership() NSQDLeadership {
	return &fakeNsqdLeadership{
		regData:              make(map[string]*NsqdNodeInfo),
		fakeTopicsLeaderData: make(map[string]map[int]*TopicCoordinator),
		fakeTopicsInfo:       make(map[string]map[int]*TopicPartionMetaInfo),
	}
}

func (self *fakeNsqdLeadership) InitClusterID(id string) {
	self.clusterID = id
}

func (self *fakeNsqdLeadership) RegisterNsqd(nodeData *NsqdNodeInfo) error {
	self.regData[nodeData.GetID()] = nodeData
	return nil
}

func (self *fakeNsqdLeadership) UnregisterNsqd(nodeData *NsqdNodeInfo) error {
	delete(self.regData, nodeData.GetID())
	return nil
}

func (self *fakeNsqdLeadership) IsNodeTopicLeader(topic string, partition int, nodeData *NsqdNodeInfo) bool {
	t, ok := self.fakeTopicsLeaderData[topic]
	var tc *TopicCoordinator
	if ok {
		if tc, ok = t[partition]; ok {
			if tc.topicLeaderSession.LeaderNode != nil {
				if tc.topicLeaderSession.LeaderNode.GetID() == nodeData.GetID() {
					return true
				}
			}
		}
	}
	return false
}

func (self *fakeNsqdLeadership) GetAllLookupdNodes() ([]NsqLookupdNodeInfo, error) {
	v := make([]NsqLookupdNodeInfo, 0)
	return v, nil
}

func (self *fakeNsqdLeadership) AcquireTopicLeader(topic string, partition int, nodeData *NsqdNodeInfo, epoch EpochType) error {
	t, ok := self.fakeTopicsLeaderData[topic]
	var tc *TopicCoordinator
	if ok {
		if tc, ok = t[partition]; ok {
			if tc.topicLeaderSession.LeaderNode != nil {
				if tc.topicLeaderSession.LeaderNode.GetID() == nodeData.GetID() {
					return nil
				}
				return errors.New("topic leader already exist.")
			}
			tc.topicLeaderSession.LeaderNode = nodeData
			tc.topicLeaderSession.LeaderEpoch++
			tc.topicLeaderSession.Session = nodeData.GetID() + strconv.Itoa(int(tc.topicLeaderSession.LeaderEpoch))
			tc.topicInfo.ISR = append(tc.topicInfo.ISR, nodeData.GetID())
			tc.topicInfo.Leader = nodeData.GetID()
			tc.topicInfo.Epoch++
		} else {
			tc = &TopicCoordinator{}
			tc.topicInfo.Name = topic
			tc.topicInfo.Partition = partition
			tc.localDataLoaded = true
			tc.topicInfo.Leader = nodeData.GetID()
			tc.topicInfo.ISR = append(tc.topicInfo.ISR, nodeData.GetID())
			tc.topicInfo.Epoch++
			tc.topicLeaderSession.LeaderNode = nodeData
			tc.topicLeaderSession.LeaderEpoch++
			tc.topicLeaderSession.Session = nodeData.GetID() + strconv.Itoa(int(tc.topicLeaderSession.LeaderEpoch))
			t[partition] = tc
		}
	} else {
		tmp := make(map[int]*TopicCoordinator)
		tc = &TopicCoordinator{}
		tc.topicInfo.Name = topic
		tc.topicInfo.Partition = partition
		tc.localDataLoaded = true
		tc.topicInfo.Leader = nodeData.GetID()
		tc.topicInfo.ISR = append(tc.topicInfo.ISR, nodeData.GetID())
		tc.topicInfo.Epoch++
		tc.topicLeaderSession.LeaderNode = nodeData
		tc.topicLeaderSession.LeaderEpoch++
		tc.topicLeaderSession.Session = nodeData.GetID() + strconv.Itoa(int(tc.topicLeaderSession.LeaderEpoch))
		tmp[partition] = tc
		self.fakeTopicsLeaderData[topic] = tmp
	}
	return nil
}

func (self *fakeNsqdLeadership) ReleaseTopicLeader(topic string, partition int, session *TopicLeaderSession) error {
	t, ok := self.fakeTopicsLeaderData[topic]
	if ok {
		delete(t, partition)
	}
	return nil
}

func (self *fakeNsqdLeadership) WatchLookupdLeader(leader chan *NsqLookupdNodeInfo, stop chan struct{}) error {
	return nil
}

func (self *fakeNsqdLeadership) GetTopicInfo(topic string, partition int) (*TopicPartionMetaInfo, error) {
	t, ok := self.fakeTopicsInfo[topic]
	if ok {
		tp, ok2 := t[partition]
		if ok2 {
			return tp, nil
		}
	}
	return nil, errors.New("topic not exist")
}

func (self *fakeNsqdLeadership) GetTopicLeaderSession(topic string, partition int) (*TopicLeaderSession, error) {
	s, ok := self.fakeTopicsLeaderData[topic]
	if !ok {
		return nil, ErrMissingTopicLeaderSession
	}
	ss, ok := s[partition]
	if !ok {
		return nil, ErrMissingTopicLeaderSession
	}
	return &ss.topicLeaderSession, nil
}

func startNsqdCoord(t *testing.T, rpcport string, dataPath string, extraID string, nsqd *nsqd.NSQD, useFake bool) *NsqdCoordinator {
	nsqdCoord := NewNsqdCoordinator(TEST_NSQ_CLUSTER_NAME, "127.0.0.1", "0", rpcport, extraID, dataPath, nsqd)
	if useFake {
		nsqdCoord.leadership = NewFakeNSQDLeadership()
		nsqdCoord.lookupRemoteCreateFunc = func(addr string, to time.Duration) (INsqlookupRemoteProxy, error) {
			p, err := NewFakeLookupRemoteProxy(addr, to)
			if err == nil {
				p.(*fakeLookupRemoteProxy).t = t
			}
			return p, err
		}
	} else {
		nsqdCoord.leadership = NewNsqdEtcdMgr(testEtcdServers)
		nsqdCoord.leadership.UnregisterNsqd(&nsqdCoord.myNode)
	}
	nsqdCoord.lookupLeader = NsqLookupdNodeInfo{}
	return nsqdCoord
}

func startNsqdCoordWithFakeData(t *testing.T, rpcport string, dataPath string,
	extraID string, nsqd *nsqd.NSQD, fakeLeadership *fakeNsqdLeadership, fakeLookupProxy *fakeLookupRemoteProxy) *NsqdCoordinator {
	nsqdCoord := NewNsqdCoordinator(TEST_NSQ_CLUSTER_NAME, "127.0.0.1", "0", rpcport, extraID, dataPath, nsqd)
	nsqdCoord.leadership = fakeLeadership
	nsqdCoord.lookupRemoteCreateFunc = func(addr string, to time.Duration) (INsqlookupRemoteProxy, error) {
		fakeLookupProxy.t = t
		fakeLookupProxy.fakeNsqdCoords[nsqdCoord.myNode.GetID()] = nsqdCoord
		return fakeLookupProxy, nil
	}
	nsqdCoord.lookupLeader = NsqLookupdNodeInfo{}
	err := nsqdCoord.Start()
	if err != nil {
		panic(err)
	}
	time.Sleep(time.Second)
	return nsqdCoord
}

func TestNsqdRPCClient(t *testing.T) {
	coordLog.level = 2
	tmpDir, err := ioutil.TempDir("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano()))
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(tmpDir)

	nsqdCoord := startNsqdCoord(t, "0", tmpDir, "", nil, true)
	nsqdCoord.Start()
	defer nsqdCoord.Stop()
	time.Sleep(time.Second * 2)
	client, err := NewNsqdRpcClient(nsqdCoord.rpcServer.rpcListener.Addr().String(), time.Second)
	test.Nil(t, err)
	var rspInt int32
	err = client.CallWithRetry("NsqdCoordinator.TestRpcCallNotExist", "req", &rspInt)
	test.NotNil(t, err)

	rsp, rpcErr := client.CallRpcTest("reqdata")
	test.NotNil(t, rpcErr)
	test.Equal(t, rsp, "reqdata")
	test.Equal(t, rpcErr.ErrCode, RpcNoErr)
	test.Equal(t, rpcErr.ErrMsg, "reqdata")
	test.Equal(t, rpcErr.ErrType, CoordCommonErr)
}
<|endoftext|>


<commit_before>package multiplexer

import (
	"reflect"
	"sort"
	"sync"

	"github.com/mailgun/kafka-pixy/actor"
	"github.com/mailgun/kafka-pixy/consumer"
	"github.com/mailgun/kafka-pixy/none"
	"github.com/mailgun/log"
)

// T fetches messages from inputs and multiplexes them to the output, giving
// preferences to inputs with higher lag. Multiplexes assumes ownership over
// inputs in the sense that it decides when an new input instance needs to
// started, or the old one stopped.
type T struct {
	actorID   *actor.ID
	spawnInFn SpawnInFn
	inputs    map[int32]*input
	output    Out
	isRunning bool
	stopCh    chan none.T
	wg        sync.WaitGroup
}

// In defines an interface of a multiplexer input.
type In interface {
	// Messages returns a channel that multiplexer receives messages from.
	// Read messages should NOT be considered as consumed by the input.
	Messages() <-chan consumer.Message

	// Stop signals the input to stop and blocks waiting for its goroutines to
	// complete.
	Stop()
}

// Out defines an interface of multiplexer output.
type Out interface {
	// Messages returns channel that multiplexer sends messages to.
	Messages() chan<- consumer.Message
}

// SpawnInFn is a function type that is used by multiplexer to spawn inputs for
// assigned partitions during rewiring.
type SpawnInFn func(partition int32) In

// New creates a new multiplexer instance.
func New(namespace *actor.ID, spawnInFn SpawnInFn) *T {
	return &T{
		actorID:   namespace.NewChild("mux"),
		inputs:    make(map[int32]*input),
		spawnInFn: spawnInFn,
		stopCh:    make(chan none.T),
	}
}

// input represents a multiplexer input along with a message to be fetched from
// that input next.
type input struct {
	In
	partition int32
	msg       consumer.Message
	msgOk     bool
}

// IsRunning returns `true` if multiplexer is running pumping events from the
// inputs to the output.
func (m *T) IsRunning() bool {
	return m.isRunning
}

// WireUp ensures that assigned inputs are spawned and multiplexed to the
// specified output. It stops inputs for partitions that are no longer
// assigned, spawns inputs for newly assigned partitions, and restarts the
// multiplexer, if either output or any of inputs has changed.
//
// The multiplexer may be stopped if either output or all inputs are removed.
//
// WARNING: do not ever pass (*T)(nil) in output, that will cause panic.
func (m *T) WireUp(output Out, assigned []int32) {
	var wg sync.WaitGroup

	if m.output != output {
		m.stopIfRunning()
		m.output = output
	}
	// If output is not provided, then stop all inputs and return.
	if output == nil {
		for p, in := range m.inputs {
			wg.Add(1)
			go func(in *input) {
				defer wg.Done()
				in.Stop()
			}(in)
			delete(m.inputs, p)
		}
		wg.Wait()
		return
	}
	// Stop inputs that are not assigned anymore.
	for p, in := range m.inputs {
		if !hasPartition(p, assigned) {
			m.stopIfRunning()
			wg.Add(1)
			go func(in *input) {
				defer wg.Done()
				in.Stop()
			}(in)
			delete(m.inputs, p)
		}
	}
	wg.Wait()
	// Spawn newly assigned inputs, but stop multiplexer before spawning the
	// first input.
	for _, p := range assigned {
		if _, ok := m.inputs[p]; !ok {
			m.stopIfRunning()
			m.inputs[p] = &input{In: m.spawnInFn(p), partition: p}
		}
	}
	if !m.IsRunning() && len(m.inputs) > 0 {
		m.start()
	}
}

// Stop synchronously stops the multiplexer.
func (m *T) Stop() {
	m.WireUp(nil, nil)
}

func (m *T) start() {
	actor.Spawn(m.actorID, &m.wg, m.run)
	m.isRunning = true
}

func (m *T) stopIfRunning() {
	if m.isRunning {
		m.stopCh <- none.V
		m.wg.Wait()
		m.isRunning = false
	}
}

func (m *T) run() {
reset:
	inputCount := len(m.inputs)
	if inputCount == 0 {
		return
	}
	sortedIns := makeSortedIns(m.inputs)
	// Prepare a list of reflective select cases. It is used when none of the
	// inputs has fetched messages and we need to wait on all of them. Yes,
	// reflection is slow, but it is only used when there is nothing to
	// consume anyway.
	selectCases := make([]reflect.SelectCase, inputCount+1)
	for i, in := range sortedIns {
		selectCases[i] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(in.Messages())}
	}
	selectCases[inputCount] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(m.stopCh)}

	inputIdx := -1
	for {
		// Collect next messages from inputs that have them available.
		isAtLeastOneAvailable := false
		for _, in := range sortedIns {
			if in.msgOk {
				isAtLeastOneAvailable = true
				continue
			}
			select {
			case msg, ok := <-in.Messages():
				// If a channel of an input is closed, then the input should be
				// removed from the list of multiplexed inputs.
				if !ok {
					log.Infof("<%s> input channel closed: partition=%d", m.actorID, in.partition)
					delete(m.inputs, in.partition)
					goto reset
				}
				in.msg = msg
				in.msgOk = true
				isAtLeastOneAvailable = true
			default:
			}
		}
		// If none of the inputs has a message available, then wait until
		// a message is fetched on any of them or a stop signal is received.
		if !isAtLeastOneAvailable {
			idx, value, _ := reflect.Select(selectCases)
			// Check if it is a stop signal.
			if idx == inputCount {
				return
			}
			sortedIns[idx].msg = value.Interface().(consumer.Message)
			sortedIns[idx].msgOk = true
		}
		// At this point there is at least one message available.
		inputIdx = selectInput(inputIdx, sortedIns)
		// Block until the output reads the next message of the selected input
		// or a stop signal is received.
		select {
		case <-m.stopCh:
			return
		case m.output.Messages() <- sortedIns[inputIdx].msg:
			sortedIns[inputIdx].msgOk = false
		}
	}
}

// makeSortedIns given a partition->input map returns a slice of all the inputs
// from the map sorted in ascending order of partition ids.
func makeSortedIns(inputs map[int32]*input) []*input {
	partitions := make([]int32, 0, len(inputs))
	for p := range inputs {
		partitions = append(partitions, p)
	}
	sort.Sort(Int32Slice(partitions))
	sortedIns := make([]*input, len(inputs))
	for i, p := range partitions {
		sortedIns[i] = inputs[p]
	}
	return sortedIns
}

func hasPartition(partition int32, partitions []int32) bool {
	count := len(partitions)
	if count == 0 {
		return false
	}
	return partitions[0] <= partition && partition <= partitions[count-1]
}

// selectInput picks an input that should be multiplexed next. It prefers the
// inputs with the largest lag. If there is more then one input with the same
// largest lag, then it picks the one that has index following prevSelectedIdx.
func selectInput(prevSelectedIdx int, sortedIns []*input) int {
	maxLag := int64(-1)
	selectedIdx := -1
	for i, input := range sortedIns {
		if !input.msgOk {
			continue
		}
		lag := input.msg.HighWaterMark - input.msg.Offset
		if lag > maxLag {
			maxLag = lag
			selectedIdx = i
			continue
		}
		if lag < maxLag {
			continue
		}
		if selectedIdx > prevSelectedIdx {
			continue
		}
		if i > prevSelectedIdx {
			selectedIdx = i
		}
	}
	return selectedIdx
}

type Int32Slice []int32

func (p Int32Slice) Len() int           { return len(p) }
func (p Int32Slice) Less(i, j int) bool { return p[i] < p[j] }
func (p Int32Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
<commit_msg>Start/stop mux inputs concurrently<commit_after>package multiplexer

import (
	"reflect"
	"sort"
	"sync"

	"github.com/mailgun/kafka-pixy/actor"
	"github.com/mailgun/kafka-pixy/consumer"
	"github.com/mailgun/kafka-pixy/none"
	"github.com/mailgun/log"
)

// T fetches messages from inputs and multiplexes them to the output, giving
// preferences to inputs with higher lag. Multiplexes assumes ownership over
// inputs in the sense that it decides when an new input instance needs to
// started, or the old one stopped.
type T struct {
	actorID   *actor.ID
	spawnInFn SpawnInFn
	inputs    map[int32]*input
	output    Out
	isRunning bool
	stopCh    chan none.T
	wg        sync.WaitGroup
}

// In defines an interface of a multiplexer input.
type In interface {
	// Messages returns a channel that multiplexer receives messages from.
	// Read messages should NOT be considered as consumed by the input.
	Messages() <-chan consumer.Message

	// Stop signals the input to stop and blocks waiting for its goroutines to
	// complete.
	Stop()
}

// Out defines an interface of multiplexer output.
type Out interface {
	// Messages returns channel that multiplexer sends messages to.
	Messages() chan<- consumer.Message
}

// SpawnInFn is a function type that is used by multiplexer to spawn inputs for
// assigned partitions during rewiring.
type SpawnInFn func(partition int32) In

// New creates a new multiplexer instance.
func New(namespace *actor.ID, spawnInFn SpawnInFn) *T {
	return &T{
		actorID:   namespace.NewChild("mux"),
		inputs:    make(map[int32]*input),
		spawnInFn: spawnInFn,
		stopCh:    make(chan none.T),
	}
}

// input represents a multiplexer input along with a message to be fetched from
// that input next.
type input struct {
	In
	partition int32
	msg       consumer.Message
	msgOk     bool
}

// IsRunning returns `true` if multiplexer is running pumping events from the
// inputs to the output.
func (m *T) IsRunning() bool {
	return m.isRunning
}

// WireUp ensures that assigned inputs are spawned and multiplexed to the
// specified output. It stops inputs for partitions that are no longer
// assigned, spawns inputs for newly assigned partitions, and restarts the
// multiplexer, if either output or any of inputs has changed.
//
// The multiplexer may be stopped if either output or all inputs are removed.
//
// WARNING: do not ever pass (*T)(nil) in output, that will cause panic.
func (m *T) WireUp(output Out, assigned []int32) {
	var wg sync.WaitGroup

	if m.output != output {
		m.stopIfRunning()
		m.output = output
	}
	// If output is not provided, then stop all inputs and return.
	if output == nil {
		for p, in := range m.inputs {
			wg.Add(1)
			go func(in *input) {
				defer wg.Done()
				in.Stop()
			}(in)
			delete(m.inputs, p)
		}
		wg.Wait()
		return
	}
	// Stop inputs that are not assigned anymore.
	for p, in := range m.inputs {
		if !hasPartition(p, assigned) {
			m.stopIfRunning()
			wg.Add(1)
			go func(in *input) {
				defer wg.Done()
				in.Stop()
			}(in)
			delete(m.inputs, p)
		}
	}
	// Spawn newly assigned inputs, but stop multiplexer before spawning the
	// first input.
	for _, p := range assigned {
		if _, ok := m.inputs[p]; !ok {
			m.stopIfRunning()
			m.inputs[p] = &input{In: m.spawnInFn(p), partition: p}
		}
	}
	if !m.IsRunning() && len(m.inputs) > 0 {
		m.start()
	}
	// Wait for stopping inputs to stop.
	wg.Wait()
}

// Stop synchronously stops the multiplexer.
func (m *T) Stop() {
	m.WireUp(nil, nil)
}

func (m *T) start() {
	actor.Spawn(m.actorID, &m.wg, m.run)
	m.isRunning = true
}

func (m *T) stopIfRunning() {
	if m.isRunning {
		m.stopCh <- none.V
		m.wg.Wait()
		m.isRunning = false
	}
}

func (m *T) run() {
reset:
	inputCount := len(m.inputs)
	if inputCount == 0 {
		return
	}
	sortedIns := makeSortedIns(m.inputs)
	// Prepare a list of reflective select cases. It is used when none of the
	// inputs has fetched messages and we need to wait on all of them. Yes,
	// reflection is slow, but it is only used when there is nothing to
	// consume anyway.
	selectCases := make([]reflect.SelectCase, inputCount+1)
	for i, in := range sortedIns {
		selectCases[i] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(in.Messages())}
	}
	selectCases[inputCount] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(m.stopCh)}

	inputIdx := -1
	for {
		// Collect next messages from inputs that have them available.
		isAtLeastOneAvailable := false
		for _, in := range sortedIns {
			if in.msgOk {
				isAtLeastOneAvailable = true
				continue
			}
			select {
			case msg, ok := <-in.Messages():
				// If a channel of an input is closed, then the input should be
				// removed from the list of multiplexed inputs.
				if !ok {
					log.Infof("<%s> input channel closed: partition=%d", m.actorID, in.partition)
					delete(m.inputs, in.partition)
					goto reset
				}
				in.msg = msg
				in.msgOk = true
				isAtLeastOneAvailable = true
			default:
			}
		}
		// If none of the inputs has a message available, then wait until
		// a message is fetched on any of them or a stop signal is received.
		if !isAtLeastOneAvailable {
			idx, value, _ := reflect.Select(selectCases)
			// Check if it is a stop signal.
			if idx == inputCount {
				return
			}
			sortedIns[idx].msg = value.Interface().(consumer.Message)
			sortedIns[idx].msgOk = true
		}
		// At this point there is at least one message available.
		inputIdx = selectInput(inputIdx, sortedIns)
		// Block until the output reads the next message of the selected input
		// or a stop signal is received.
		select {
		case <-m.stopCh:
			return
		case m.output.Messages() <- sortedIns[inputIdx].msg:
			sortedIns[inputIdx].msgOk = false
		}
	}
}

// makeSortedIns given a partition->input map returns a slice of all the inputs
// from the map sorted in ascending order of partition ids.
func makeSortedIns(inputs map[int32]*input) []*input {
	partitions := make([]int32, 0, len(inputs))
	for p := range inputs {
		partitions = append(partitions, p)
	}
	sort.Sort(Int32Slice(partitions))
	sortedIns := make([]*input, len(inputs))
	for i, p := range partitions {
		sortedIns[i] = inputs[p]
	}
	return sortedIns
}

func hasPartition(partition int32, partitions []int32) bool {
	count := len(partitions)
	if count == 0 {
		return false
	}
	return partitions[0] <= partition && partition <= partitions[count-1]
}

// selectInput picks an input that should be multiplexed next. It prefers the
// inputs with the largest lag. If there is more then one input with the same
// largest lag, then it picks the one that has index following prevSelectedIdx.
func selectInput(prevSelectedIdx int, sortedIns []*input) int {
	maxLag := int64(-1)
	selectedIdx := -1
	for i, input := range sortedIns {
		if !input.msgOk {
			continue
		}
		lag := input.msg.HighWaterMark - input.msg.Offset
		if lag > maxLag {
			maxLag = lag
			selectedIdx = i
			continue
		}
		if lag < maxLag {
			continue
		}
		if selectedIdx > prevSelectedIdx {
			continue
		}
		if i > prevSelectedIdx {
			selectedIdx = i
		}
	}
	return selectedIdx
}

type Int32Slice []int32

func (p Int32Slice) Len() int           { return len(p) }
func (p Int32Slice) Less(i, j int) bool { return p[i] < p[j] }
func (p Int32Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
<|endoftext|>


<commit_before>/*
Package "dense" provides an implementation of "Matrix" which stores elements in a slide.
*/
package dense

import (
	"errors"
	"fmt"

	"github.com/mitsuse/matrix-go"
)

type matrixImpl struct {
	rows     int
	columns  int
	elements []float64
}

func New(rows, columns int) func(elements ...float64) (matrix.Matrix, error) {
	rowsShouldBePositiveNumber(rows)
	columnShouldBePositiveNumber(rows)

	constructor := func(elements ...float64) (matrix.Matrix, error) {
		size := rows * columns

		if len(elements) != size {
			template := "The number of %q should equal to %q * %q."
			message := fmt.Sprintf(template, "elements", "rows", "columns")

			return nil, errors.New(message)
		}

		m := &matrixImpl{
			rows:     rows,
			columns:  columns,
			elements: make([]float64, size),
		}
		copy(m.elements, elements)

		return m, nil
	}

	return constructor
}

func (m *matrixImpl) Shape() (rows, columns int) {
	return m.Rows(), m.Columns()
}

func (m *matrixImpl) Rows() (rows int) {
	return m.rows
}

func (m *matrixImpl) Columns() (columns int) {
	return m.columns
}
<commit_msg>"Matrix" has two methods to return column or row.<commit_after>/*
Package "dense" provides an implementation of "Matrix" which stores elements in a slide.
*/
package dense

import (
	"errors"
	"fmt"

	"github.com/mitsuse/matrix-go"
)

type matrixImpl struct {
	rows     int
	columns  int
	elements []float64
}

func New(rows, columns int) func(elements ...float64) (matrix.Matrix, error) {
	rowsShouldBePositiveNumber(rows)
	columnShouldBePositiveNumber(rows)

	constructor := func(elements ...float64) (matrix.Matrix, error) {
		size := rows * columns

		if len(elements) != size {
			template := "The number of %q should equal to %q * %q."
			message := fmt.Sprintf(template, "elements", "rows", "columns")

			return nil, errors.New(message)
		}

		m := &matrixImpl{
			rows:     rows,
			columns:  columns,
			elements: make([]float64, size),
		}
		copy(m.elements, elements)

		return m, nil
	}

	return constructor
}

func (m *matrixImpl) Shape() (rows, columns int) {
	return m.Rows(), m.Columns()
}

func (m *matrixImpl) Rows() (rows int) {
	return m.rows
}

func (m *matrixImpl) Columns() (columns int) {
	return m.columns
}

func (m *matrixImpl) Row(row int) matrix.Row {
	// TODO: Implement.
	return nil
}

func (m *matrixImpl) Column(column int) matrix.Column {
	// TODO: Implement.
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	. "fmt"
	"sort"
)

func findLHS(nums []int) int {
	dic := map[int]int{}
	for _, v := range nums {
		if va, ok := dic[v]; ok {
			dic[v] = 1 + va
		} else {
			dic[v] = 1
		}
	}

	sortkey := []int{}
	for k := range dic {
		sortkey = append(sortkey, k)
	}
	sort.Ints(sortkey)

	result := 0
	for ind := 1; ind < len(sortkey); ind++ {
		if temp := dic[sortkey[ind]] + dic[sortkey[ind-1]]; sortkey[ind]-sortkey[ind-1] == 1 &&
			temp > result {
			result = temp
		}
	}

	return result
}

func main() {
	Println(findLHS([]int{1, 3, 2, 2, 5, 2, 3, 7}))

}
<commit_msg>well, go has += operation<commit_after>package main

import (
	. "fmt"
	"sort"
)

func findLHS(nums []int) int {
	dic := map[int]int{}
	for _, v := range nums {
		if _, ok := dic[v]; ok {
			dic[v] += 1
		} else {
			dic[v] = 1
		}
	}

	sortkey := []int{}
	for k := range dic {
		sortkey = append(sortkey, k)
	}
	sort.Ints(sortkey)

	result := 0
	for ind := 1; ind < len(sortkey); ind++ {
		if temp := dic[sortkey[ind]] + dic[sortkey[ind-1]]; sortkey[ind]-sortkey[ind-1] == 1 &&
			temp > result {
			result = temp
		}
	}

	return result
}

func main() {
	Println(findLHS([]int{1, 3, 2, 2, 5, 2, 3, 7}))

}
<|endoftext|>


<commit_before>package datastore

import (
	"errors"
	"fmt"
	"time"

	"github.com/SpirentOrion/trace"
	"github.com/goamz/goamz/aws"
	"github.com/goamz/goamz/dynamodb"
)

// DynamoParams holds AWS connection and auth properties for
// DynamoDB-based datastores.
type DynamoParams struct {
	Region    string
	TableName string
	AccessKey string
	SecretKey string
}

// NewDynamoParams extracts DynamoDB provider parameters from a
// generic string map and returns a DynamoParams structure.
func NewDynamoParams(params map[string]string) (*DynamoParams, error) {
	p := &DynamoParams{
		Region:    params["region"],
		TableName: params["table_name"],
		AccessKey: params["access_key"],
		SecretKey: params["secret_key"],
	}

	if p.Region == "" {
		return nil, errors.New("DynamoDB providers require a 'region' parameter")
	}
	if p.TableName == "" {
		return nil, errors.New("DynamoDB providers require a 'table_name' parameter")
	}

	return p, nil
}

type DynamoTable struct {
	*dynamodb.Table
}

func NewDynamoTable(params *DynamoParams) (*DynamoTable, error) {
	auth, err := aws.GetAuth(params.AccessKey, params.SecretKey, "", time.Time{})
	if err != nil {
		return nil, err
	}
	server := &dynamodb.Server{
		Auth:   auth,
		Region: aws.Regions[params.Region],
	}
	table := server.NewTable(params.TableName, dynamodb.PrimaryKey{KeyAttribute: dynamodb.NewStringAttribute("id", "")})
	return &DynamoTable{table}, nil
}

func (t *DynamoTable) String() string {
	return fmt.Sprintf("%s{%s:%s}", DYNAMODB_PROVIDER, t.Server.Region.Name, t.Name)
}

func (t *DynamoTable) Scan() (attrs []map[string]*dynamodb.Attribute, err error) {
	s, _ := trace.Continue(DYNAMODB_PROVIDER, t.String())
	trace.Run(s, func() {
		attrs, err = t.Table.Scan([]dynamodb.AttributeComparison{})
		if s != nil {
			data := s.Data()
			data["op"] = "Scan"
			if err != nil {
				data["error"] = err
			} else {
				data["items"] = len(attrs)
			}
		}
	})

	if err != nil {
		attrs = nil
		return
	}
	return
}

func (t *DynamoTable) GetItem(id string) (attrs map[string]*dynamodb.Attribute, ok bool, err error) {
	s, _ := trace.Continue(DYNAMODB_PROVIDER, t.String())
	trace.Run(s, func() {
		key := &dynamodb.Key{HashKey: id}
		attrs, err = t.Table.GetItem(key)
		if s != nil {
			data := s.Data()
			data["op"] = "GetItem"
			if err != nil && err != dynamodb.ErrNotFound {
				data["error"] = err
			}
		}
	})

	if err != nil {
		if err == dynamodb.ErrNotFound {
			err = nil
		}
		return
	} else {
		ok = true
	}
	return
}

func (t *DynamoTable) PutItem(id string, attrs []dynamodb.Attribute) (err error) {
	s, _ := trace.Continue(DYNAMODB_PROVIDER, t.String())
	trace.Run(s, func() {
		_, err = t.Table.PutItem(id, "", attrs)
		if s != nil {
			data := s.Data()
			data["op"] = "PutItem"
			if err != nil {
				data["error"] = err
			}
		}
	})

	return
}

func (t *DynamoTable) UpdateItem(id string, serial int64, attrs []dynamodb.Attribute) (err error) {
	s, _ := trace.Continue(DYNAMODB_PROVIDER, t.String())
	trace.Run(s, func() {
		key := &dynamodb.Key{HashKey: id}
		serialAttr := []dynamodb.Attribute{{
			Type:  dynamodb.TYPE_NUMBER,
			Name:  "serial",
			Value: fmt.Sprint(serial),
		}}
		_, err = t.Table.ConditionalUpdateAttributes(key, attrs, serialAttr)
		if s != nil {
			data := s.Data()
			data["op"] = "ConditionalUpdateAttributes"
			if err != nil {
				data["error"] = err
			}
		}
	})

	return
}

func (t *DynamoTable) DeleteItem(id string) (ok bool, err error) {
	s, _ := trace.Continue(DYNAMODB_PROVIDER, t.String())
	trace.Run(s, func() {
		key := &dynamodb.Key{HashKey: id}
		_, err = t.Table.DeleteItem(key)
		if s != nil {
			data := s.Data()
			data["op"] = "DeleteItem"
			if err != nil && err != dynamodb.ErrNotFound {
				data["error"] = err
			}
		}
	})

	if err != nil {
		if err == dynamodb.ErrNotFound {
			err = nil
		}
		return
	} else {
		ok = true
	}
	return
}
<commit_msg>Minor dynamodb helper method changes to facilitate ongoing service development.<commit_after>package datastore

import (
	"errors"
	"fmt"
	"time"

	"github.com/SpirentOrion/trace"
	"github.com/goamz/goamz/aws"
	"github.com/goamz/goamz/dynamodb"
)

// DynamoParams holds AWS connection and auth properties for
// DynamoDB-based datastores.
type DynamoParams struct {
	Region    string
	TableName string
	AccessKey string
	SecretKey string
}

// NewDynamoParams extracts DynamoDB provider parameters from a
// generic string map and returns a DynamoParams structure.
func NewDynamoParams(params map[string]string) (*DynamoParams, error) {
	p := &DynamoParams{
		Region:    params["region"],
		TableName: params["table_name"],
		AccessKey: params["access_key"],
		SecretKey: params["secret_key"],
	}

	if p.Region == "" {
		return nil, errors.New("DynamoDB providers require a 'region' parameter")
	}
	if p.TableName == "" {
		return nil, errors.New("DynamoDB providers require a 'table_name' parameter")
	}

	return p, nil
}

type DynamoTable struct {
	*dynamodb.Table
}

func NewDynamoTable(params *DynamoParams) (*DynamoTable, error) {
	auth, err := aws.GetAuth(params.AccessKey, params.SecretKey, "", time.Time{})
	if err != nil {
		return nil, err
	}
	server := &dynamodb.Server{
		Auth:   auth,
		Region: aws.Regions[params.Region],
	}
	table := server.NewTable(params.TableName, dynamodb.PrimaryKey{KeyAttribute: dynamodb.NewStringAttribute("id", "")})
	return &DynamoTable{table}, nil
}

func (t *DynamoTable) String() string {
	return fmt.Sprintf("%s{%s:%s}", DYNAMODB_PROVIDER, t.Server.Region.Name, t.Name)
}

func (t *DynamoTable) GetItem(id string) (attrs map[string]*dynamodb.Attribute, ok bool, err error) {
	s, _ := trace.Continue(DYNAMODB_PROVIDER, t.String())
	trace.Run(s, func() {
		key := &dynamodb.Key{HashKey: id}
		attrs, err = t.Table.GetItem(key)
		if s != nil {
			data := s.Data()
			data["op"] = "GetItem"
			if err != nil && err != dynamodb.ErrNotFound {
				data["error"] = err
			}
		}
	})

	if err != nil {
		if err == dynamodb.ErrNotFound {
			err = nil
		}
		return
	} else {
		ok = true
	}
	return
}

func (t *DynamoTable) PutItem(id string, attrs []dynamodb.Attribute) (err error) {
	s, _ := trace.Continue(DYNAMODB_PROVIDER, t.String())
	trace.Run(s, func() {
		_, err = t.Table.PutItem(id, "", attrs)
		if s != nil {
			data := s.Data()
			data["op"] = "PutItem"
			if err != nil {
				data["error"] = err
			}
		}
	})
	return
}

func (t *DynamoTable) UpdateItem(id string, serial int64, attrs []dynamodb.Attribute) (err error) {
	s, _ := trace.Continue(DYNAMODB_PROVIDER, t.String())
	trace.Run(s, func() {
		key := &dynamodb.Key{HashKey: id}
		serialAttr := []dynamodb.Attribute{{
			Type:  dynamodb.TYPE_NUMBER,
			Name:  "serial",
			Value: fmt.Sprint(serial),
		}}
		_, err = t.Table.ConditionalUpdateAttributes(key, attrs, serialAttr)
		if s != nil {
			data := s.Data()
			data["op"] = "ConditionalUpdateAttributes"
			if err != nil {
				data["error"] = err
			}
		}
	})
	return
}

func (t *DynamoTable) DeleteItem(id string) (ok bool, err error) {
	s, _ := trace.Continue(DYNAMODB_PROVIDER, t.String())
	trace.Run(s, func() {
		key := &dynamodb.Key{HashKey: id}
		_, err = t.Table.DeleteItem(key)
		if s != nil {
			data := s.Data()
			data["op"] = "DeleteItem"
			if err != nil && err != dynamodb.ErrNotFound {
				data["error"] = err
			}
		}
	})

	if err != nil {
		if err == dynamodb.ErrNotFound {
			err = nil
		}
		return
	} else {
		ok = true
	}
	return
}

func (t *DynamoTable) Scan(comps []dynamodb.AttributeComparison) (attrs []map[string]*dynamodb.Attribute, err error) {
	s, _ := trace.Continue(DYNAMODB_PROVIDER, t.String())
	trace.Run(s, func() {
		attrs, err = t.Table.Scan(comps)
		if s != nil {
			data := s.Data()
			data["op"] = "Scan"
			if err != nil {
				data["error"] = err
			} else {
				data["items"] = len(attrs)
			}
		}
	})

	if err != nil {
		attrs = nil
		return
	}
	return
}

func (t *DynamoTable) QueryOnIndex(comps []dynamodb.AttributeComparison, indexName string) (attrs []map[string]*dynamodb.Attribute, err error) {
	s, _ := trace.Continue(DYNAMODB_PROVIDER, t.String())
	trace.Run(s, func() {
		attrs, err = t.Table.QueryOnIndex(comps, indexName)
		if s != nil {
			data := s.Data()
			data["op"] = "QueryOnIndex"
			data["index"] = indexName
			if err != nil {
				data["error"] = err
			} else {
				data["items"] = len(attrs)
			}
		}
	})
	return
}
<|endoftext|>


<commit_before>package filer

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/cluster"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
)

type MetaAggregator struct {
	filer           *Filer
	self            pb.ServerAddress
	isLeader        bool
	grpcDialOption  grpc.DialOption
	MetaLogBuffer   *log_buffer.LogBuffer
	peerStatues     map[pb.ServerAddress]int
	peerStatuesLock sync.Mutex
	// notifying clients
	ListenersLock sync.Mutex
	ListenersCond *sync.Cond
}

// MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk.
// The old data comes from what each LocalMetadata persisted on disk.
func NewMetaAggregator(filer *Filer, self pb.ServerAddress, grpcDialOption grpc.DialOption) *MetaAggregator {
	t := &MetaAggregator{
		filer:          filer,
		self:           self,
		grpcDialOption: grpcDialOption,
		peerStatues:    make(map[pb.ServerAddress]int),
	}
	t.ListenersCond = sync.NewCond(&t.ListenersLock)
	t.MetaLogBuffer = log_buffer.NewLogBuffer("aggr", LogFlushInterval, nil, func() {
		t.ListenersCond.Broadcast()
	})
	return t
}

func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate) {
	if update.NodeType != cluster.FilerType {
		return
	}

	address := pb.ServerAddress(update.Address)
	if update.IsAdd {
		// every filer should subscribe to a new filer
		if ma.setActive(address, true) {
			go ma.subscribeToOneFiler(ma.filer, ma.self, address)
		}
	} else {
		ma.setActive(address, false)
	}
}

func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) (notDuplicated bool) {
	ma.peerStatuesLock.Lock()
	defer ma.peerStatuesLock.Unlock()
	if isActive {
		if _, found := ma.peerStatues[address]; found {
			ma.peerStatues[address] += 1
		} else {
			ma.peerStatues[address] = 1
			notDuplicated = true
		}
	} else {
		if _, found := ma.peerStatues[address]; found {
			delete(ma.peerStatues, address)
		}
	}
	return
}
func (ma *MetaAggregator) isActive(address pb.ServerAddress) (isActive bool) {
	ma.peerStatuesLock.Lock()
	defer ma.peerStatuesLock.Unlock()
	var count int
	count, isActive = ma.peerStatues[address]
	return count > 0 && isActive
}

func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress) {

	/*
		Each filer reads the "filer.store.id", which is the store's signature when filer starts.

		When reading from other filers' local meta changes:
		* if the received change does not contain signature from self, apply the change to current filer store.

		Upon connecting to other filers, need to remember their signature and their offsets.

	*/

	var maybeReplicateMetadataChange func(*filer_pb.SubscribeMetadataResponse)
	lastPersistTime := time.Now()
	lastTsNs := time.Now().Add(-LogFlushInterval).UnixNano()

	peerSignature, err := ma.readFilerStoreSignature(peer)
	for err != nil {
		glog.V(0).Infof("connecting to peer filer %s: %v", peer, err)
		time.Sleep(1357 * time.Millisecond)
		peerSignature, err = ma.readFilerStoreSignature(peer)
	}

	// when filer store is not shared by multiple filers
	if peerSignature != f.Signature {
		lastTsNs = 0
		if prevTsNs, err := ma.readOffset(f, peer, peerSignature); err == nil {
			lastTsNs = prevTsNs
		}

		glog.V(0).Infof("follow peer: %v, last %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
		var counter int64
		var synced bool
		maybeReplicateMetadataChange = func(event *filer_pb.SubscribeMetadataResponse) {
			if err := Replay(f.Store, event); err != nil {
				glog.Errorf("failed to reply metadata change from %v: %v", peer, err)
				return
			}
			counter++
			if lastPersistTime.Add(time.Minute).Before(time.Now()) {
				if err := ma.updateOffset(f, peer, peerSignature, event.TsNs); err == nil {
					if event.TsNs < time.Now().Add(-2*time.Minute).UnixNano() {
						glog.V(0).Infof("sync with %s progressed to: %v %0.2f/sec", peer, time.Unix(0, event.TsNs), float64(counter)/60.0)
					} else if !synced {
						synced = true
						glog.V(0).Infof("synced with %s", peer)
					}
					lastPersistTime = time.Now()
					counter = 0
				} else {
					glog.V(0).Infof("failed to update offset for %v: %v", peer, err)
				}
			}
		}
	}

	processEventFn := func(event *filer_pb.SubscribeMetadataResponse) error {
		data, err := proto.Marshal(event)
		if err != nil {
			glog.Errorf("failed to marshal subscribed filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
			return err
		}
		dir := event.Directory
		// println("received meta change", dir, "size", len(data))
		ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)
		if maybeReplicateMetadataChange != nil {
			maybeReplicateMetadataChange(event)
		}
		return nil
	}

	for {
		glog.V(4).Infof("subscribing remote %s meta change: %v", peer, time.Unix(0, lastTsNs))
		err := pb.WithFilerClient(true, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
			ctx, cancel := context.WithCancel(context.Background())
			defer cancel()
			stream, err := client.SubscribeLocalMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
				ClientName: "filer:" + string(self),
				PathPrefix: "/",
				SinceNs:    lastTsNs,
				ClientId:   int32(ma.filer.UniqueFileId),
			})
			if err != nil {
				return fmt.Errorf("subscribe: %v", err)
			}

			for {
				resp, listenErr := stream.Recv()
				if listenErr == io.EOF {
					return nil
				}
				if listenErr != nil {
					return listenErr
				}

				if err := processEventFn(resp); err != nil {
					return fmt.Errorf("process %v: %v", resp, err)
				}
				lastTsNs = resp.TsNs

				f.onMetadataChangeEvent(resp)

			}
		})
		if !ma.isActive(peer) {
			glog.V(0).Infof("stop subscribing remote %s meta change", peer)
			return
		}
		if err != nil {
			glog.V(0).Infof("subscribing remote %s meta change: %v", peer, err)
			time.Sleep(1733 * time.Millisecond)
		}
	}
}

func (ma *MetaAggregator) readFilerStoreSignature(peer pb.ServerAddress) (sig int32, err error) {
	err = pb.WithFilerClient(false, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
		if err != nil {
			return err
		}
		sig = resp.Signature
		return nil
	})
	return
}

const (
	MetaOffsetPrefix = "Meta"
)

func (ma *MetaAggregator) readOffset(f *Filer, peer pb.ServerAddress, peerSignature int32) (lastTsNs int64, err error) {

	key := []byte(MetaOffsetPrefix + "xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value, err := f.Store.KvGet(context.Background(), key)

	if err == ErrKvNotFound {
		glog.Warningf("readOffset %s not found", peer)
		return 0, nil
	}

	if err != nil {
		return 0, fmt.Errorf("readOffset %s : %v", peer, err)
	}

	lastTsNs = int64(util.BytesToUint64(value))

	glog.V(0).Infof("readOffset %s : %d", peer, lastTsNs)

	return
}

func (ma *MetaAggregator) updateOffset(f *Filer, peer pb.ServerAddress, peerSignature int32, lastTsNs int64) (err error) {

	key := []byte(MetaOffsetPrefix + "xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value := make([]byte, 8)
	util.Uint64toBytes(value, uint64(lastTsNs))

	err = f.Store.KvPut(context.Background(), key, value)

	if err != nil {
		return fmt.Errorf("updateOffset %s : %v", peer, err)
	}

	glog.V(4).Infof("updateOffset %s : %d", peer, lastTsNs)

	return
}
<commit_msg>reset sync offset if peer filer resets<commit_after>package filer

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/cluster"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
)

type MetaAggregator struct {
	filer           *Filer
	self            pb.ServerAddress
	isLeader        bool
	grpcDialOption  grpc.DialOption
	MetaLogBuffer   *log_buffer.LogBuffer
	peerStatues     map[pb.ServerAddress]int
	peerStatuesLock sync.Mutex
	// notifying clients
	ListenersLock sync.Mutex
	ListenersCond *sync.Cond
}

// MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk.
// The old data comes from what each LocalMetadata persisted on disk.
func NewMetaAggregator(filer *Filer, self pb.ServerAddress, grpcDialOption grpc.DialOption) *MetaAggregator {
	t := &MetaAggregator{
		filer:          filer,
		self:           self,
		grpcDialOption: grpcDialOption,
		peerStatues:    make(map[pb.ServerAddress]int),
	}
	t.ListenersCond = sync.NewCond(&t.ListenersLock)
	t.MetaLogBuffer = log_buffer.NewLogBuffer("aggr", LogFlushInterval, nil, func() {
		t.ListenersCond.Broadcast()
	})
	return t
}

func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate) {
	if update.NodeType != cluster.FilerType {
		return
	}

	address := pb.ServerAddress(update.Address)
	if update.IsAdd {
		// every filer should subscribe to a new filer
		if ma.setActive(address, true) {
			go ma.loopSubscribeToOnefiler(ma.filer, ma.self, address)
		}
	} else {
		ma.setActive(address, false)
	}
}

func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) (notDuplicated bool) {
	ma.peerStatuesLock.Lock()
	defer ma.peerStatuesLock.Unlock()
	if isActive {
		if _, found := ma.peerStatues[address]; found {
			ma.peerStatues[address] += 1
		} else {
			ma.peerStatues[address] = 1
			notDuplicated = true
		}
	} else {
		if _, found := ma.peerStatues[address]; found {
			delete(ma.peerStatues, address)
		}
	}
	return
}
func (ma *MetaAggregator) isActive(address pb.ServerAddress) (isActive bool) {
	ma.peerStatuesLock.Lock()
	defer ma.peerStatuesLock.Unlock()
	var count int
	count, isActive = ma.peerStatues[address]
	return count > 0 && isActive
}

func (ma *MetaAggregator) loopSubscribeToOnefiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress) {
	for {
		err := ma.doSubscribeToOneFiler(f, self, peer)
		if !ma.isActive(peer) {
			glog.V(0).Infof("stop subscribing remote %s meta change", peer)
			return
		}
		if err != nil {
			glog.V(0).Infof("subscribing remote %s meta change: %v", peer, err)
		}
		time.Sleep(1733 * time.Millisecond)
	}
}

func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress) error {

	/*
		Each filer reads the "filer.store.id", which is the store's signature when filer starts.

		When reading from other filers' local meta changes:
		* if the received change does not contain signature from self, apply the change to current filer store.

		Upon connecting to other filers, need to remember their signature and their offsets.

	*/

	var maybeReplicateMetadataChange func(*filer_pb.SubscribeMetadataResponse)
	lastPersistTime := time.Now()
	lastTsNs := time.Now().Add(-LogFlushInterval).UnixNano()

	peerSignature, err := ma.readFilerStoreSignature(peer)
	for err != nil {
		glog.V(0).Infof("connecting to peer filer %s: %v", peer, err)
		time.Sleep(1357 * time.Millisecond)
		peerSignature, err = ma.readFilerStoreSignature(peer)
	}

	// when filer store is not shared by multiple filers
	if peerSignature != f.Signature {
		lastTsNs = 0
		if prevTsNs, err := ma.readOffset(f, peer, peerSignature); err == nil {
			lastTsNs = prevTsNs
			defer func(prevTsNs int64) {
				if lastTsNs != prevTsNs && lastTsNs != lastPersistTime.UnixNano() {
					if err := ma.updateOffset(f, peer, peerSignature, lastTsNs); err == nil {
						glog.V(0).Infof("last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
					} else {
						glog.Errorf("failed to save last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
					}
				}
			}(prevTsNs)
		}

		glog.V(0).Infof("follow peer: %v, last %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
		var counter int64
		var synced bool
		maybeReplicateMetadataChange = func(event *filer_pb.SubscribeMetadataResponse) {
			if err := Replay(f.Store, event); err != nil {
				glog.Errorf("failed to reply metadata change from %v: %v", peer, err)
				return
			}
			counter++
			if lastPersistTime.Add(time.Minute).Before(time.Now()) {
				if err := ma.updateOffset(f, peer, peerSignature, event.TsNs); err == nil {
					if event.TsNs < time.Now().Add(-2*time.Minute).UnixNano() {
						glog.V(0).Infof("sync with %s progressed to: %v %0.2f/sec", peer, time.Unix(0, event.TsNs), float64(counter)/60.0)
					} else if !synced {
						synced = true
						glog.V(0).Infof("synced with %s", peer)
					}
					lastPersistTime = time.Now()
					counter = 0
				} else {
					glog.V(0).Infof("failed to update offset for %v: %v", peer, err)
				}
			}
		}
	}

	processEventFn := func(event *filer_pb.SubscribeMetadataResponse) error {
		data, err := proto.Marshal(event)
		if err != nil {
			glog.Errorf("failed to marshal subscribed filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
			return err
		}
		dir := event.Directory
		// println("received meta change", dir, "size", len(data))
		ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)
		if maybeReplicateMetadataChange != nil {
			maybeReplicateMetadataChange(event)
		}
		return nil
	}

	glog.V(4).Infof("subscribing remote %s meta change: %v", peer, time.Unix(0, lastTsNs))
	err = pb.WithFilerClient(true, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		stream, err := client.SubscribeLocalMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
			ClientName: "filer:" + string(self),
			PathPrefix: "/",
			SinceNs:    lastTsNs,
			ClientId:   int32(ma.filer.UniqueFileId),
		})
		if err != nil {
			return fmt.Errorf("subscribe: %v", err)
		}

		for {
			resp, listenErr := stream.Recv()
			if listenErr == io.EOF {
				return nil
			}
			if listenErr != nil {
				return listenErr
			}

			if err := processEventFn(resp); err != nil {
				return fmt.Errorf("process %v: %v", resp, err)
			}
			lastTsNs = resp.TsNs

			f.onMetadataChangeEvent(resp)

		}
	})
	return err
}

func (ma *MetaAggregator) readFilerStoreSignature(peer pb.ServerAddress) (sig int32, err error) {
	err = pb.WithFilerClient(false, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
		if err != nil {
			return err
		}
		sig = resp.Signature
		return nil
	})
	return
}

const (
	MetaOffsetPrefix = "Meta"
)

func (ma *MetaAggregator) readOffset(f *Filer, peer pb.ServerAddress, peerSignature int32) (lastTsNs int64, err error) {

	key := []byte(MetaOffsetPrefix + "xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value, err := f.Store.KvGet(context.Background(), key)

	if err == ErrKvNotFound {
		glog.Warningf("readOffset %s not found", peer)
		return 0, nil
	}

	if err != nil {
		return 0, fmt.Errorf("readOffset %s : %v", peer, err)
	}

	lastTsNs = int64(util.BytesToUint64(value))

	glog.V(0).Infof("readOffset %s : %d", peer, lastTsNs)

	return
}

func (ma *MetaAggregator) updateOffset(f *Filer, peer pb.ServerAddress, peerSignature int32, lastTsNs int64) (err error) {

	key := []byte(MetaOffsetPrefix + "xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value := make([]byte, 8)
	util.Uint64toBytes(value, uint64(lastTsNs))

	err = f.Store.KvPut(context.Background(), key, value)

	if err != nil {
		return fmt.Errorf("updateOffset %s : %v", peer, err)
	}

	glog.V(4).Infof("updateOffset %s : %d", peer, lastTsNs)

	return
}
<|endoftext|>


<commit_before>package mount

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/hanwen/go-fuse/v2/fuse"
	"math"
	"os"
	"sync"
)

type DirectoryHandleId uint64

type DirectoryHandle struct {
	isFinished    bool
	counter       uint32
	lastEntryName string
}

type DirectoryHandleToInode struct {
	// shares the file handle id sequencer with FileHandleToInode{nextFh}
	sync.Mutex
	dir2inode map[DirectoryHandleId]*DirectoryHandle
}

func NewDirectoryHandleToInode() *DirectoryHandleToInode {
	return &DirectoryHandleToInode{
		dir2inode: make(map[DirectoryHandleId]*DirectoryHandle),
	}
}

func (wfs *WFS) AcquireDirectoryHandle() (DirectoryHandleId, *DirectoryHandle) {
	wfs.fhmap.Lock()
	fh := wfs.fhmap.nextFh
	wfs.fhmap.nextFh++
	wfs.fhmap.Unlock()

	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	dh := &DirectoryHandle{
		isFinished:    false,
		lastEntryName: "",
	}
	wfs.dhmap.dir2inode[DirectoryHandleId(fh)] = dh
	return DirectoryHandleId(fh), dh
}

func (wfs *WFS) GetDirectoryHandle(dhid DirectoryHandleId) *DirectoryHandle {
	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	if dh, found := wfs.dhmap.dir2inode[dhid]; found {
		return dh
	}
	dh := &DirectoryHandle{
		isFinished:    false,
		lastEntryName: "",
	}

	wfs.dhmap.dir2inode[dhid] = dh
	return dh
}

func (wfs *WFS) ReleaseDirectoryHandle(dhid DirectoryHandleId) {
	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	delete(wfs.dhmap.dir2inode, dhid)
}

// Directory handling

/** Open directory
 *
 * Unless the 'default_permissions' mount option is given,
 * this method should check if opendir is permitted for this
 * directory. Optionally opendir may also return an arbitrary
 * filehandle in the fuse_file_info structure, which will be
 * passed to readdir, releasedir and fsyncdir.
 */
func (wfs *WFS) OpenDir(cancel <-chan struct{}, input *fuse.OpenIn, out *fuse.OpenOut) (code fuse.Status) {
	if !wfs.inodeToPath.HasInode(input.NodeId) {
		return fuse.ENOENT
	}
	dhid, _ := wfs.AcquireDirectoryHandle()
	out.Fh = uint64(dhid)
	return fuse.OK
}

/** Release directory
 *
 * If the directory has been removed after the call to opendir, the
 * path parameter will be NULL.
 */
func (wfs *WFS) ReleaseDir(input *fuse.ReleaseIn) {
	wfs.ReleaseDirectoryHandle(DirectoryHandleId(input.Fh))
}

/** Synchronize directory contents
 *
 * If the directory has been removed after the call to opendir, the
 * path parameter will be NULL.
 *
 * If the datasync parameter is non-zero, then only the user data
 * should be flushed, not the meta data
 */
func (wfs *WFS) FsyncDir(cancel <-chan struct{}, input *fuse.FsyncIn) (code fuse.Status) {
	return fuse.OK
}

/** Read directory
 *
 * The filesystem may choose between two modes of operation:
 *
 * 1) The readdir implementation ignores the offset parameter, and
 * passes zero to the filler function's offset.  The filler
 * function will not return '1' (unless an error happens), so the
 * whole directory is read in a single readdir operation.
 *
 * 2) The readdir implementation keeps track of the offsets of the
 * directory entries.  It uses the offset parameter and always
 * passes non-zero offset to the filler function.  When the buffer
 * is full (or an error happens) the filler function will return
 * '1'.
 */
func (wfs *WFS) ReadDir(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
	return wfs.doReadDirectory(input, out, false)
}

func (wfs *WFS) ReadDirPlus(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
	return wfs.doReadDirectory(input, out, true)
}

func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPlusMode bool) fuse.Status {

	dh := wfs.GetDirectoryHandle(DirectoryHandleId(input.Fh))
	if dh.isFinished {
		return fuse.OK
	}

	dirPath := wfs.inodeToPath.GetPath(input.NodeId)

	var dirEntry fuse.DirEntry
	if input.Offset == 0 && !isPlusMode {
		dh.counter++
		dirEntry.Ino = input.NodeId
		dirEntry.Name = "."
		dirEntry.Mode = toSystemMode(os.ModeDir)
		out.AddDirEntry(dirEntry)

		dh.counter++
		parentDir, _ := dirPath.DirAndName()
		parentInode := wfs.inodeToPath.GetInode(util.FullPath(parentDir))
		dirEntry.Ino = parentInode
		dirEntry.Name = ".."
		dirEntry.Mode = toSystemMode(os.ModeDir)
		out.AddDirEntry(dirEntry)

	}

	processEachEntryFn := func(entry *filer.Entry, isLast bool) bool {
		dh.counter++
		dirEntry.Name = entry.Name()
		inode := wfs.inodeToPath.GetInode(dirPath.Child(dirEntry.Name))
		dirEntry.Ino = inode
		dirEntry.Mode = toSystemMode(entry.Mode)
		if !isPlusMode {
			if !out.AddDirEntry(dirEntry) {
				return false
			}
		} else {
			entryOut := out.AddDirLookupEntry(dirEntry)
			if entryOut == nil {
				return false
			}
			wfs.outputFilerEntry(entryOut, inode, entry)
		}
		dh.lastEntryName = entry.Name()
		return true
	}

	if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath); err != nil {
		glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
		return fuse.EIO
	}
	listErr := wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, dh.lastEntryName, false, int64(math.MaxInt32), func(entry *filer.Entry) bool {
		return processEachEntryFn(entry, false)
	})
	if listErr != nil {
		glog.Errorf("list meta cache: %v", listErr)
		return fuse.EIO
	}
	if dh.counter < input.Length {
		dh.isFinished = true
	}

	return fuse.OK
}
<commit_msg>mount2: fix directory pagination<commit_after>package mount

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/hanwen/go-fuse/v2/fuse"
	"math"
	"os"
	"sync"
)

type DirectoryHandleId uint64

type DirectoryHandle struct {
	isFinished    bool
	lastEntryName string
}

type DirectoryHandleToInode struct {
	// shares the file handle id sequencer with FileHandleToInode{nextFh}
	sync.Mutex
	dir2inode map[DirectoryHandleId]*DirectoryHandle
}

func NewDirectoryHandleToInode() *DirectoryHandleToInode {
	return &DirectoryHandleToInode{
		dir2inode: make(map[DirectoryHandleId]*DirectoryHandle),
	}
}

func (wfs *WFS) AcquireDirectoryHandle() (DirectoryHandleId, *DirectoryHandle) {
	wfs.fhmap.Lock()
	fh := wfs.fhmap.nextFh
	wfs.fhmap.nextFh++
	wfs.fhmap.Unlock()

	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	dh := &DirectoryHandle{
		isFinished:    false,
		lastEntryName: "",
	}
	wfs.dhmap.dir2inode[DirectoryHandleId(fh)] = dh
	return DirectoryHandleId(fh), dh
}

func (wfs *WFS) GetDirectoryHandle(dhid DirectoryHandleId) *DirectoryHandle {
	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	if dh, found := wfs.dhmap.dir2inode[dhid]; found {
		return dh
	}
	dh := &DirectoryHandle{
		isFinished:    false,
		lastEntryName: "",
	}

	wfs.dhmap.dir2inode[dhid] = dh
	return dh
}

func (wfs *WFS) ReleaseDirectoryHandle(dhid DirectoryHandleId) {
	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	delete(wfs.dhmap.dir2inode, dhid)
}

// Directory handling

/** Open directory
 *
 * Unless the 'default_permissions' mount option is given,
 * this method should check if opendir is permitted for this
 * directory. Optionally opendir may also return an arbitrary
 * filehandle in the fuse_file_info structure, which will be
 * passed to readdir, releasedir and fsyncdir.
 */
func (wfs *WFS) OpenDir(cancel <-chan struct{}, input *fuse.OpenIn, out *fuse.OpenOut) (code fuse.Status) {
	if !wfs.inodeToPath.HasInode(input.NodeId) {
		return fuse.ENOENT
	}
	dhid, _ := wfs.AcquireDirectoryHandle()
	out.Fh = uint64(dhid)
	return fuse.OK
}

/** Release directory
 *
 * If the directory has been removed after the call to opendir, the
 * path parameter will be NULL.
 */
func (wfs *WFS) ReleaseDir(input *fuse.ReleaseIn) {
	wfs.ReleaseDirectoryHandle(DirectoryHandleId(input.Fh))
}

/** Synchronize directory contents
 *
 * If the directory has been removed after the call to opendir, the
 * path parameter will be NULL.
 *
 * If the datasync parameter is non-zero, then only the user data
 * should be flushed, not the meta data
 */
func (wfs *WFS) FsyncDir(cancel <-chan struct{}, input *fuse.FsyncIn) (code fuse.Status) {
	return fuse.OK
}

/** Read directory
 *
 * The filesystem may choose between two modes of operation:
 *
 * 1) The readdir implementation ignores the offset parameter, and
 * passes zero to the filler function's offset.  The filler
 * function will not return '1' (unless an error happens), so the
 * whole directory is read in a single readdir operation.
 *
 * 2) The readdir implementation keeps track of the offsets of the
 * directory entries.  It uses the offset parameter and always
 * passes non-zero offset to the filler function.  When the buffer
 * is full (or an error happens) the filler function will return
 * '1'.
 */
func (wfs *WFS) ReadDir(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
	return wfs.doReadDirectory(input, out, false)
}

func (wfs *WFS) ReadDirPlus(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
	return wfs.doReadDirectory(input, out, true)
}

func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPlusMode bool) fuse.Status {

	dh := wfs.GetDirectoryHandle(DirectoryHandleId(input.Fh))
	if dh.isFinished {
		return fuse.OK
	}

	isEarlyTerminated := false
	dirPath := wfs.inodeToPath.GetPath(input.NodeId)

	var dirEntry fuse.DirEntry
	if input.Offset == 0 && !isPlusMode {
		dirEntry.Ino = input.NodeId
		dirEntry.Name = "."
		dirEntry.Mode = toSystemMode(os.ModeDir)
		out.AddDirEntry(dirEntry)

		parentDir, _ := dirPath.DirAndName()
		parentInode := wfs.inodeToPath.GetInode(util.FullPath(parentDir))
		dirEntry.Ino = parentInode
		dirEntry.Name = ".."
		dirEntry.Mode = toSystemMode(os.ModeDir)
		out.AddDirEntry(dirEntry)

	}

	processEachEntryFn := func(entry *filer.Entry, isLast bool) bool {
		dirEntry.Name = entry.Name()
		inode := wfs.inodeToPath.GetInode(dirPath.Child(dirEntry.Name))
		dirEntry.Ino = inode
		dirEntry.Mode = toSystemMode(entry.Mode)
		if !isPlusMode {
			if !out.AddDirEntry(dirEntry) {
				isEarlyTerminated = true
				return false
			}
		} else {
			entryOut := out.AddDirLookupEntry(dirEntry)
			if entryOut == nil {
				isEarlyTerminated = true
				return false
			}
			wfs.outputFilerEntry(entryOut, inode, entry)
		}
		dh.lastEntryName = entry.Name()
		return true
	}

	if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath); err != nil {
		glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
		return fuse.EIO
	}
	listErr := wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, dh.lastEntryName, false, int64(math.MaxInt32), func(entry *filer.Entry) bool {
		return processEachEntryFn(entry, false)
	})
	if listErr != nil {
		glog.Errorf("list meta cache: %v", listErr)
		return fuse.EIO
	}
	if !isEarlyTerminated {
		dh.isFinished = true
	}

	return fuse.OK
}
<|endoftext|>


<commit_before>package pb

import (
	"context"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/keepalive"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
)

const (
	Max_Message_Size = 1 << 30 // 1 GB
)

var (
	// cache grpc connections
	grpcClients     = make(map[string]*grpc.ClientConn)
	grpcClientsLock sync.Mutex
)

func init() {
	http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 1024
}

func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
	var options []grpc.ServerOption
	options = append(options,
		grpc.KeepaliveParams(keepalive.ServerParameters{
			Time:    10 * time.Second, // wait time before ping if no activity
			Timeout: 20 * time.Second, // ping timeout
		}),
		grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
			MinTime:             60 * time.Second, // min time a client should wait before sending a ping
			PermitWithoutStream: true,
		}),
		grpc.MaxRecvMsgSize(Max_Message_Size),
		grpc.MaxSendMsgSize(Max_Message_Size),
	)
	for _, opt := range opts {
		if opt != nil {
			options = append(options, opt)
		}
	}
	return grpc.NewServer(options...)
}

func GrpcDial(ctx context.Context, address string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
	// opts = append(opts, grpc.WithBlock())
	// opts = append(opts, grpc.WithTimeout(time.Duration(5*time.Second)))
	var options []grpc.DialOption
	options = append(options,
		// grpc.WithInsecure(),
		grpc.WithDefaultCallOptions(
			grpc.MaxCallSendMsgSize(Max_Message_Size),
			grpc.MaxCallRecvMsgSize(Max_Message_Size),
		),
		grpc.WithKeepaliveParams(keepalive.ClientParameters{
			Time:                30 * time.Second, // client ping server if no activity for this long
			Timeout:             20 * time.Second,
			PermitWithoutStream: true,
		}))
	for _, opt := range opts {
		if opt != nil {
			options = append(options, opt)
		}
	}
	return grpc.DialContext(ctx, address, options...)
}

func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts ...grpc.DialOption) error {

	grpcClientsLock.Lock()

	existingConnection, found := grpcClients[address]
	if found {
		grpcClientsLock.Unlock()
		err := fn(existingConnection)
		if err != nil {
			grpcClientsLock.Lock()
			delete(grpcClients, address)
			grpcClientsLock.Unlock()
			existingConnection.Close()
		}
		return err
	}

	grpcConnection, err := GrpcDial(context.Background(), address, opts...)
	if err != nil {
		grpcClientsLock.Unlock()
		return fmt.Errorf("fail to dial %s: %v", address, err)
	}

	grpcClients[address] = grpcConnection
	grpcClientsLock.Unlock()

	err = fn(grpcConnection)
	if err != nil {
		grpcClientsLock.Lock()
		delete(grpcClients, address)
		grpcClientsLock.Unlock()
		grpcConnection.Close()
	}

	return err
}

func ParseServerToGrpcAddress(server string) (serverGrpcAddress string, err error) {
	colonIndex := strings.LastIndex(server, ":")
	if colonIndex < 0 {
		return "", fmt.Errorf("server should have hostname:port format: %v", server)
	}

	port, parseErr := strconv.ParseUint(server[colonIndex+1:], 10, 64)
	if parseErr != nil {
		return "", fmt.Errorf("server port parse error: %v", parseErr)
	}

	grpcPort := int(port) + 10000

	return fmt.Sprintf("%s:%d", server[:colonIndex], grpcPort), nil
}

func ServerToGrpcAddress(server string) (serverGrpcAddress string) {
	hostnameAndPort := strings.Split(server, ":")
	if len(hostnameAndPort) != 2 {
		return fmt.Sprintf("unexpected server address: %s", server)
	}

	port, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64)
	if parseErr != nil {
		return fmt.Sprintf("failed to parse port for %s:%s", hostnameAndPort[0], hostnameAndPort[1])
	}

	grpcPort := int(port) + 10000

	return fmt.Sprintf("%s:%d", hostnameAndPort[0], grpcPort)
}

func WithMasterClient(master string, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) error {

	masterGrpcAddress, parseErr := ParseServerToGrpcAddress(master)
	if parseErr != nil {
		return fmt.Errorf("failed to parse master grpc %v: %v", master, parseErr)
	}

	return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
		client := master_pb.NewSeaweedClient(grpcConnection)
		return fn(client)
	}, masterGrpcAddress, grpcDialOption)

}

func WithBrokerGrpcClient(brokerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client messaging_pb.SeaweedMessagingClient) error) error {

	return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
		client := messaging_pb.NewSeaweedMessagingClient(grpcConnection)
		return fn(client)
	}, brokerGrpcAddress, grpcDialOption)

}

func WithFilerClient(filer string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {

	filerGrpcAddress, parseErr := ParseServerToGrpcAddress(filer)
	if parseErr != nil {
		return fmt.Errorf("failed to parse filer grpc %v: %v", filer, parseErr)
	}

	return WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, fn)

}

func WithGrpcFilerClient(filerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {

	return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
		return fn(client)
	}, filerGrpcAddress, grpcDialOption)

}

func ParseFilerGrpcAddress(filer string) (filerGrpcAddress string, err error) {
	hostnameAndPort := strings.Split(filer, ":")
	if len(hostnameAndPort) != 2 {
		return "", fmt.Errorf("filer should have hostname:port format: %v", hostnameAndPort)
	}

	filerPort, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64)
	if parseErr != nil {
		return "", fmt.Errorf("filer port parse error: %v", parseErr)
	}

	filerGrpcPort := int(filerPort) + 10000

	return fmt.Sprintf("%s:%d", hostnameAndPort[0], filerGrpcPort), nil
}
<commit_msg>revert this grpc related change<commit_after>package pb

import (
	"context"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/keepalive"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
)

const (
	Max_Message_Size = 1 << 30 // 1 GB
)

var (
	// cache grpc connections
	grpcClients     = make(map[string]*grpc.ClientConn)
	grpcClientsLock sync.Mutex
)

func init() {
	http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 1024
}

func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
	var options []grpc.ServerOption
	options = append(options,
		grpc.KeepaliveParams(keepalive.ServerParameters{
			Time:    10 * time.Second, // wait time before ping if no activity
			Timeout: 20 * time.Second, // ping timeout
		}),
		grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
			MinTime:             60 * time.Second, // min time a client should wait before sending a ping
			PermitWithoutStream: false,
		}),
		grpc.MaxRecvMsgSize(Max_Message_Size),
		grpc.MaxSendMsgSize(Max_Message_Size),
	)
	for _, opt := range opts {
		if opt != nil {
			options = append(options, opt)
		}
	}
	return grpc.NewServer(options...)
}

func GrpcDial(ctx context.Context, address string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
	// opts = append(opts, grpc.WithBlock())
	// opts = append(opts, grpc.WithTimeout(time.Duration(5*time.Second)))
	var options []grpc.DialOption
	options = append(options,
		// grpc.WithInsecure(),
		grpc.WithDefaultCallOptions(
			grpc.MaxCallSendMsgSize(Max_Message_Size),
			grpc.MaxCallRecvMsgSize(Max_Message_Size),
		),
		grpc.WithKeepaliveParams(keepalive.ClientParameters{
			Time:                30 * time.Second, // client ping server if no activity for this long
			Timeout:             20 * time.Second,
			PermitWithoutStream: false,
		}))
	for _, opt := range opts {
		if opt != nil {
			options = append(options, opt)
		}
	}
	return grpc.DialContext(ctx, address, options...)
}

func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts ...grpc.DialOption) error {

	grpcClientsLock.Lock()

	existingConnection, found := grpcClients[address]
	if found {
		grpcClientsLock.Unlock()
		err := fn(existingConnection)
		if err != nil {
			grpcClientsLock.Lock()
			delete(grpcClients, address)
			grpcClientsLock.Unlock()
			existingConnection.Close()
		}
		return err
	}

	grpcConnection, err := GrpcDial(context.Background(), address, opts...)
	if err != nil {
		grpcClientsLock.Unlock()
		return fmt.Errorf("fail to dial %s: %v", address, err)
	}

	grpcClients[address] = grpcConnection
	grpcClientsLock.Unlock()

	err = fn(grpcConnection)
	if err != nil {
		grpcClientsLock.Lock()
		delete(grpcClients, address)
		grpcClientsLock.Unlock()
		grpcConnection.Close()
	}

	return err
}

func ParseServerToGrpcAddress(server string) (serverGrpcAddress string, err error) {
	colonIndex := strings.LastIndex(server, ":")
	if colonIndex < 0 {
		return "", fmt.Errorf("server should have hostname:port format: %v", server)
	}

	port, parseErr := strconv.ParseUint(server[colonIndex+1:], 10, 64)
	if parseErr != nil {
		return "", fmt.Errorf("server port parse error: %v", parseErr)
	}

	grpcPort := int(port) + 10000

	return fmt.Sprintf("%s:%d", server[:colonIndex], grpcPort), nil
}

func ServerToGrpcAddress(server string) (serverGrpcAddress string) {
	hostnameAndPort := strings.Split(server, ":")
	if len(hostnameAndPort) != 2 {
		return fmt.Sprintf("unexpected server address: %s", server)
	}

	port, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64)
	if parseErr != nil {
		return fmt.Sprintf("failed to parse port for %s:%s", hostnameAndPort[0], hostnameAndPort[1])
	}

	grpcPort := int(port) + 10000

	return fmt.Sprintf("%s:%d", hostnameAndPort[0], grpcPort)
}

func WithMasterClient(master string, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) error {

	masterGrpcAddress, parseErr := ParseServerToGrpcAddress(master)
	if parseErr != nil {
		return fmt.Errorf("failed to parse master grpc %v: %v", master, parseErr)
	}

	return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
		client := master_pb.NewSeaweedClient(grpcConnection)
		return fn(client)
	}, masterGrpcAddress, grpcDialOption)

}

func WithBrokerGrpcClient(brokerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client messaging_pb.SeaweedMessagingClient) error) error {

	return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
		client := messaging_pb.NewSeaweedMessagingClient(grpcConnection)
		return fn(client)
	}, brokerGrpcAddress, grpcDialOption)

}

func WithFilerClient(filer string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {

	filerGrpcAddress, parseErr := ParseServerToGrpcAddress(filer)
	if parseErr != nil {
		return fmt.Errorf("failed to parse filer grpc %v: %v", filer, parseErr)
	}

	return WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, fn)

}

func WithGrpcFilerClient(filerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {

	return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
		return fn(client)
	}, filerGrpcAddress, grpcDialOption)

}

func ParseFilerGrpcAddress(filer string) (filerGrpcAddress string, err error) {
	hostnameAndPort := strings.Split(filer, ":")
	if len(hostnameAndPort) != 2 {
		return "", fmt.Errorf("filer should have hostname:port format: %v", hostnameAndPort)
	}

	filerPort, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64)
	if parseErr != nil {
		return "", fmt.Errorf("filer port parse error: %v", parseErr)
	}

	filerGrpcPort := int(filerPort) + 10000

	return fmt.Sprintf("%s:%d", hostnameAndPort[0], filerGrpcPort), nil
}
<|endoftext|>