<commit_before>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//  Generate my.cnf files from templates.

package mysqlctl

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"io/ioutil"
	"math/big"
	"path"
	"text/template"

	"github.com/youtube/vitess/go/vt/env"
)

// This files handles the creation of Mycnf objects for the default 'vt'
// file structure. These path are used by the mysqlctl commands.
//
// The default 'vt' file structure is as follows:
// - the root is specified by the environment variable VTDATAROOT,
//   and defaults to /vt
// - each tablet with uid NNNNNNNNNN is located in <root>/vt_NNNNNNNNNN
// - in that tablet directory, there is a my.cnf file for the mysql instance,
//   and 'data', 'innodb', 'relay-logs', 'bin-logs' directories.
// - these sub-directories might be symlinks to other places,
//   see comment for createTopDir, to allow some data types to be on
//   different disk partitions.

const (
	dataDir          = "data"
	innodbDir        = "innodb"
	relayLogDir      = "relay-logs"
	binLogDir        = "bin-logs"
	innodbDataSubdir = "innodb/data"
	innodbLogSubdir  = "innodb/logs"
)

// NewMycnf fills the Mycnf structure with vt root paths and derived values.
// This is used to fill out the cnfTemplate values and generate my.cnf.
// uid is a unique id for a particular tablet - it must be unique within the
// tabletservers deployed within a keyspace, lest there be collisions on disk.
// mysqldPort needs to be unique per instance per machine.
func NewMycnf(tabletUID uint32, mysqlPort int32) *Mycnf {
	cnf := new(Mycnf)
	cnf.path = mycnfFile(tabletUID)
	tabletDir := TabletDir(tabletUID)
	cnf.ServerID = tabletUID
	cnf.MysqlPort = mysqlPort
	cnf.DataDir = path.Join(tabletDir, dataDir)
	cnf.InnodbDataHomeDir = path.Join(tabletDir, innodbDataSubdir)
	cnf.InnodbLogGroupHomeDir = path.Join(tabletDir, innodbLogSubdir)
	cnf.SocketFile = path.Join(tabletDir, "mysql.sock")
	cnf.GeneralLogPath = path.Join(tabletDir, "general.log")
	cnf.ErrorLogPath = path.Join(tabletDir, "error.log")
	cnf.SlowLogPath = path.Join(tabletDir, "slow-query.log")
	cnf.RelayLogPath = path.Join(tabletDir, relayLogDir,
		fmt.Sprintf("vt-%010d-relay-bin", tabletUID))
	cnf.RelayLogIndexPath = cnf.RelayLogPath + ".index"
	cnf.RelayLogInfoPath = path.Join(tabletDir, relayLogDir, "relay-log.info")
	cnf.BinLogPath = path.Join(tabletDir, binLogDir,
		fmt.Sprintf("vt-%010d-bin", tabletUID))
	cnf.MasterInfoFile = path.Join(tabletDir, "master.info")
	cnf.PidFile = path.Join(tabletDir, "mysql.pid")
	cnf.TmpDir = path.Join(tabletDir, "tmp")
	cnf.SlaveLoadTmpDir = cnf.TmpDir
	return cnf
}

// TabletDir returns the default directory for a tablet
func TabletDir(uid uint32) string {
	return fmt.Sprintf("%s/vt_%010d", env.VtDataRoot(), uid)
}

// mycnfFile returns the default location of the my.cnf file.
func mycnfFile(uid uint32) string {
	return path.Join(TabletDir(uid), "my.cnf")
}

// TopLevelDirs returns the list of directories in the toplevel tablet directory
// that might be located in a different place.
func TopLevelDirs() []string {
	return []string{dataDir, innodbDir, relayLogDir, binLogDir}
}

// directoryList returns the list of directories to create in an empty
// mysql instance.
func (cnf *Mycnf) directoryList() []string {
	return []string{
		cnf.DataDir,
		cnf.InnodbDataHomeDir,
		cnf.InnodbLogGroupHomeDir,
		cnf.TmpDir,
		path.Dir(cnf.RelayLogPath),
		path.Dir(cnf.BinLogPath),
	}
}

// makeMycnf will join cnf files cnfPaths and substitute in the right values.
func (cnf *Mycnf) makeMycnf(cnfFiles []string) (string, error) {
	myTemplateSource := new(bytes.Buffer)
	myTemplateSource.WriteString("[mysqld]\n")
	for _, path := range cnfFiles {
		data, dataErr := ioutil.ReadFile(path)
		if dataErr != nil {
			return "", dataErr
		}
		myTemplateSource.WriteString("## " + path + "\n")
		myTemplateSource.Write(data)
	}
	return cnf.fillMycnfTemplate(myTemplateSource.String())
}

// fillMycnfTemplate will fill in the passed in template with the values
// from Mycnf
func (cnf *Mycnf) fillMycnfTemplate(tmplSrc string) (string, error) {
	myTemplate, err := template.New("").Parse(tmplSrc)
	if err != nil {
		return "", err
	}
	mycnfData := new(bytes.Buffer)
	err = myTemplate.Execute(mycnfData, cnf)
	if err != nil {
		return "", err
	}
	return mycnfData.String(), nil
}

// RandomizeMysqlServerID generates a random MySQL server_id.
//
// The value assigned to ServerID will be in the range [100, 2^31):
// - It avoids 0 because that's reserved for mysqlbinlog dumps.
// - It also avoids 1-99 because low numbers are used for fake slave
// connections.  See NewSlaveConnection() in slave_connection.go for
// more on that.
// - It avoids the 2^31 - 2^32-1 range, as there seems to be some
// confusion there. The main MySQL documentation at:
// http://dev.mysql.com/doc/refman/5.7/en/replication-options.html
// implies serverID is a full 32 bits number. The semi-sync log line
// at startup '[Note] Start semi-sync binlog_dump to slave ...'
// interprets the server_id as signed 32-bit (shows negative numbers
// for that range).
// Such an ID may also be responsible for a mysqld crash in semi-sync code,
// although we haven't been able to verify that yet. The issue for that is:
// https://github.com/youtube/vitess/issues/2280
func (cnf *Mycnf) RandomizeMysqlServerID() error {
	// rand.Int(_, max) returns a value in the range [0, max).
	bigN, err := rand.Int(rand.Reader, big.NewInt(1<<31-100))
	if err != nil {
		return err
	}
	n := bigN.Uint64()
	// n is in the range [0, 2^31 - 100).
	// Add back 100 to put it in the range [100, 2^31).
	cnf.ServerID = uint32(n + 100)
	return nil
}
<commit_msg>Allow specifying the subdir to use for vttablet<commit_after>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//  Generate my.cnf files from templates.

package mysqlctl

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"io/ioutil"
	"math/big"
	"path"
	"text/template"

	"flag"

	"github.com/youtube/vitess/go/vt/env"
)

// This files handles the creation of Mycnf objects for the default 'vt'
// file structure. These path are used by the mysqlctl commands.
//
// The default 'vt' file structure is as follows:
// - the root is specified by the environment variable VTDATAROOT,
//   and defaults to /vt
// - each tablet with uid NNNNNNNNNN is located in <root>/vt_NNNNNNNNNN
// - in that tablet directory, there is a my.cnf file for the mysql instance,
//   and 'data', 'innodb', 'relay-logs', 'bin-logs' directories.
// - these sub-directories might be symlinks to other places,
//   see comment for createTopDir, to allow some data types to be on
//   different disk partitions.

const (
	dataDir          = "data"
	innodbDir        = "innodb"
	relayLogDir      = "relay-logs"
	binLogDir        = "bin-logs"
	innodbDataSubdir = "innodb/data"
	innodbLogSubdir  = "innodb/logs"
)

var (
	tabletDir = flag.String("tablet_dir", "", "The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid.")
)

// NewMycnf fills the Mycnf structure with vt root paths and derived values.
// This is used to fill out the cnfTemplate values and generate my.cnf.
// uid is a unique id for a particular tablet - it must be unique within the
// tabletservers deployed within a keyspace, lest there be collisions on disk.
// mysqldPort needs to be unique per instance per machine.
func NewMycnf(tabletUID uint32, mysqlPort int32) *Mycnf {
	cnf := new(Mycnf)
	cnf.path = mycnfFile(tabletUID)
	tabletDir := TabletDir(tabletUID)
	cnf.ServerID = tabletUID
	cnf.MysqlPort = mysqlPort
	cnf.DataDir = path.Join(tabletDir, dataDir)
	cnf.InnodbDataHomeDir = path.Join(tabletDir, innodbDataSubdir)
	cnf.InnodbLogGroupHomeDir = path.Join(tabletDir, innodbLogSubdir)
	cnf.SocketFile = path.Join(tabletDir, "mysql.sock")
	cnf.GeneralLogPath = path.Join(tabletDir, "general.log")
	cnf.ErrorLogPath = path.Join(tabletDir, "error.log")
	cnf.SlowLogPath = path.Join(tabletDir, "slow-query.log")
	cnf.RelayLogPath = path.Join(tabletDir, relayLogDir,
		fmt.Sprintf("vt-%010d-relay-bin", tabletUID))
	cnf.RelayLogIndexPath = cnf.RelayLogPath + ".index"
	cnf.RelayLogInfoPath = path.Join(tabletDir, relayLogDir, "relay-log.info")
	cnf.BinLogPath = path.Join(tabletDir, binLogDir,
		fmt.Sprintf("vt-%010d-bin", tabletUID))
	cnf.MasterInfoFile = path.Join(tabletDir, "master.info")
	cnf.PidFile = path.Join(tabletDir, "mysql.pid")
	cnf.TmpDir = path.Join(tabletDir, "tmp")
	cnf.SlaveLoadTmpDir = cnf.TmpDir
	return cnf
}

// TabletDir returns the default directory for a tablet
func TabletDir(uid uint32) string {
	if *tabletDir != "" {
		return fmt.Sprintf("%s/%s", env.VtDataRoot(), *tabletDir)
	}
	return fmt.Sprintf("%s/vt_%010d", env.VtDataRoot(), uid)
}

// mycnfFile returns the default location of the my.cnf file.
func mycnfFile(uid uint32) string {
	return path.Join(TabletDir(uid), "my.cnf")
}

// TopLevelDirs returns the list of directories in the toplevel tablet directory
// that might be located in a different place.
func TopLevelDirs() []string {
	return []string{dataDir, innodbDir, relayLogDir, binLogDir}
}

// directoryList returns the list of directories to create in an empty
// mysql instance.
func (cnf *Mycnf) directoryList() []string {
	return []string{
		cnf.DataDir,
		cnf.InnodbDataHomeDir,
		cnf.InnodbLogGroupHomeDir,
		cnf.TmpDir,
		path.Dir(cnf.RelayLogPath),
		path.Dir(cnf.BinLogPath),
	}
}

// makeMycnf will join cnf files cnfPaths and substitute in the right values.
func (cnf *Mycnf) makeMycnf(cnfFiles []string) (string, error) {
	myTemplateSource := new(bytes.Buffer)
	myTemplateSource.WriteString("[mysqld]\n")
	for _, path := range cnfFiles {
		data, dataErr := ioutil.ReadFile(path)
		if dataErr != nil {
			return "", dataErr
		}
		myTemplateSource.WriteString("## " + path + "\n")
		myTemplateSource.Write(data)
	}
	return cnf.fillMycnfTemplate(myTemplateSource.String())
}

// fillMycnfTemplate will fill in the passed in template with the values
// from Mycnf
func (cnf *Mycnf) fillMycnfTemplate(tmplSrc string) (string, error) {
	myTemplate, err := template.New("").Parse(tmplSrc)
	if err != nil {
		return "", err
	}
	mycnfData := new(bytes.Buffer)
	err = myTemplate.Execute(mycnfData, cnf)
	if err != nil {
		return "", err
	}
	return mycnfData.String(), nil
}

// RandomizeMysqlServerID generates a random MySQL server_id.
//
// The value assigned to ServerID will be in the range [100, 2^31):
// - It avoids 0 because that's reserved for mysqlbinlog dumps.
// - It also avoids 1-99 because low numbers are used for fake slave
// connections.  See NewSlaveConnection() in slave_connection.go for
// more on that.
// - It avoids the 2^31 - 2^32-1 range, as there seems to be some
// confusion there. The main MySQL documentation at:
// http://dev.mysql.com/doc/refman/5.7/en/replication-options.html
// implies serverID is a full 32 bits number. The semi-sync log line
// at startup '[Note] Start semi-sync binlog_dump to slave ...'
// interprets the server_id as signed 32-bit (shows negative numbers
// for that range).
// Such an ID may also be responsible for a mysqld crash in semi-sync code,
// although we haven't been able to verify that yet. The issue for that is:
// https://github.com/youtube/vitess/issues/2280
func (cnf *Mycnf) RandomizeMysqlServerID() error {
	// rand.Int(_, max) returns a value in the range [0, max).
	bigN, err := rand.Int(rand.Reader, big.NewInt(1<<31-100))
	if err != nil {
		return err
	}
	n := bigN.Uint64()
	// n is in the range [0, 2^31 - 100).
	// Add back 100 to put it in the range [100, 2^31).
	cnf.ServerID = uint32(n + 100)
	return nil
}
<|endoftext|>


<commit_before>package tableacl

var allAcl SimpleACL

const (
	ALL = "*"
)

// NewACL returns an ACL with the specified entries
func NewACL(entries []string) (ACL, error) {
	a := SimpleACL(map[string]bool{})
	for _, e := range entries {
		a[e] = true
	}
	return a, nil
}

// SimpleACL keeps all entries in a unique in-memory list
type SimpleACL map[string]bool

// Check checks the membership of a principal in this ACL
func (a SimpleACL) IsMember(principal string) bool {
	return a[principal] || a[ALL]
}

func all() ACL {
	if allAcl == nil {
		allAcl = SimpleACL(map[string]bool{ALL: true})
	}
	return allAcl
}
<commit_msg>Table ACL: make SimpleAcl private<commit_after>package tableacl

var allAcl simpleACL

const (
	ALL = "*"
)

// NewACL returns an ACL with the specified entries
func NewACL(entries []string) (ACL, error) {
	a := simpleACL(map[string]bool{})
	for _, e := range entries {
		a[e] = true
	}
	return a, nil
}

// simpleACL keeps all entries in a unique in-memory list
type simpleACL map[string]bool

// IsMember checks the membership of a principal in this ACL
func (a simpleACL) IsMember(principal string) bool {
	return a[principal] || a[ALL]
}

func all() ACL {
	if allAcl == nil {
		allAcl = simpleACL(map[string]bool{ALL: true})
	}
	return allAcl
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"runtime"

	"github.com/ready-steady/numan/basis/linhat"
	"github.com/ready-steady/numan/grid/newcot"
	"github.com/ready-steady/numan/interp/adhier"
)

type solver struct {
	problem *problem
	target  target

	ic uint32 // inputs
	oc uint32 // outputs
	fc uint32 // fake inputs like time

	interpolator *adhier.Self
}

func newSolver(problem *problem, target target) *solver {
	ic, oc := target.InputsOutputs()

	interpolator := adhier.New(newcot.NewOpen(uint16(ic)), linhat.NewOpen(uint16(ic)),
		adhier.Config(problem.config.Interpolation), uint16(oc))

	return &solver{
		problem: problem,
		target:  target,

		ic: ic,
		oc: oc,
		fc: ic - problem.zc,

		interpolator: interpolator,
	}
}

func (s *solver) hasFakeInputs() bool {
	return s.fc > 0
}

func (s *solver) construct() *adhier.Surrogate {
	if s.hasFakeInputs() {
		// NOTE: Caching only make sense when there are fake inputs.
		return s.constructCached()
	} else {
		return s.constructDirect()
	}
}

func (s *solver) compute(nodes []float64) []float64 {
	ic, oc := s.ic, s.oc
	nc := uint32(len(nodes)) / ic

	jobs := s.spawnWorkers()

	done := make(chan result, nc)
	values := make([]float64, oc*nc)

	jc, rc := uint32(0), uint32(0)
	nextJob := job{
		node:  nodes[jc*ic:],
		value: values[jc*oc:],
		done:  done,
	}

	for jc < nc || rc < nc {
		select {
		case jobs <- nextJob:
			jc++

			if jc >= nc {
				close(jobs)
				jobs = nil
				continue
			}

			nextJob = job{
				node:  nodes[jc*ic:],
				value: values[jc*oc:],
				done:  done,
			}
		case <-done:
			rc++
		}
	}

	return values
}

func (s *solver) evaluate(surrogate *adhier.Surrogate, points []float64) []float64 {
	return s.interpolator.Evaluate(surrogate, points)
}

func (s *solver) spawnWorkers() chan<- job {
	c := &s.problem.config

	wc := int(c.Workers)
	if wc <= 0 {
		wc = runtime.NumCPU()
	}

	if c.Verbose {
		fmt.Printf("Using %d workers...\n", wc)
	}

	runtime.GOMAXPROCS(wc)

	jobs := make(chan job)
	for i := 0; i < wc; i++ {
		go s.target.Serve(jobs)
	}

	return jobs
}
<commit_msg>Took into account an API change in numan/interp/adhier<commit_after>package main

import (
	"fmt"
	"runtime"

	"github.com/ready-steady/numan/basis/linhat"
	"github.com/ready-steady/numan/grid/newcot"
	"github.com/ready-steady/numan/interp/adhier"
)

type solver struct {
	problem *problem
	target  target

	ic uint32 // inputs
	oc uint32 // outputs
	fc uint32 // fake inputs like time

	interpolator *adhier.Interpolator
}

func newSolver(problem *problem, target target) *solver {
	ic, oc := target.InputsOutputs()

	interpolator := adhier.New(newcot.NewOpen(uint16(ic)), linhat.NewOpen(uint16(ic)),
		adhier.Config(problem.config.Interpolation), uint16(oc))

	return &solver{
		problem: problem,
		target:  target,

		ic: ic,
		oc: oc,
		fc: ic - problem.zc,

		interpolator: interpolator,
	}
}

func (s *solver) hasFakeInputs() bool {
	return s.fc > 0
}

func (s *solver) construct() *adhier.Surrogate {
	if s.hasFakeInputs() {
		// NOTE: Caching only make sense when there are fake inputs.
		return s.constructCached()
	} else {
		return s.constructDirect()
	}
}

func (s *solver) compute(nodes []float64) []float64 {
	ic, oc := s.ic, s.oc
	nc := uint32(len(nodes)) / ic

	jobs := s.spawnWorkers()

	done := make(chan result, nc)
	values := make([]float64, oc*nc)

	jc, rc := uint32(0), uint32(0)
	nextJob := job{
		node:  nodes[jc*ic:],
		value: values[jc*oc:],
		done:  done,
	}

	for jc < nc || rc < nc {
		select {
		case jobs <- nextJob:
			jc++

			if jc >= nc {
				close(jobs)
				jobs = nil
				continue
			}

			nextJob = job{
				node:  nodes[jc*ic:],
				value: values[jc*oc:],
				done:  done,
			}
		case <-done:
			rc++
		}
	}

	return values
}

func (s *solver) evaluate(surrogate *adhier.Surrogate, points []float64) []float64 {
	return s.interpolator.Evaluate(surrogate, points)
}

func (s *solver) spawnWorkers() chan<- job {
	c := &s.problem.config

	wc := int(c.Workers)
	if wc <= 0 {
		wc = runtime.NumCPU()
	}

	if c.Verbose {
		fmt.Printf("Using %d workers...\n", wc)
	}

	runtime.GOMAXPROCS(wc)

	jobs := make(chan job)
	for i := 0; i < wc; i++ {
		go s.target.Serve(jobs)
	}

	return jobs
}
<|endoftext|>


<commit_before>package iapi

import (
	"testing"
)

func TestHostgroups(t *testing.T) {
	icingaServer := Server{"root", ICINGA2_API_PASSWORD, "https://127.0.0.1:5665/v1", true, nil}
	t.Run("Create", func(t *testing.T) {
		t.Run("Hostgroup", func(t *testing.T) {
			name := "docker-servers"
			displayName := "Docker Host Servers"
			_, err := icingaServer.CreateHostgroup(name, displayName)
			if err != nil {
				t.Error(err)
			}
		})
	})

	t.Run("Read", func(t *testing.T) {
		t.Run("ValidHostgroup", func(t *testing.T) {
			name := "linux-servers"
			_, err := icingaServer.GetHostgroup(name)
			if err != nil {
				t.Error(err)
			}
		})

		t.Run("InvalidHostgroup", func(t *testing.T) {
			name := "irix-servers"
			_, err := icingaServer.GetHostgroup(name)
			if err != nil {
				t.Error(err)
			}
		})
	})

	t.Run("Delete", func(t *testing.T) {
		// Delete Hostgroup created via API. Should succeed
		t.Run("Hostgroup", func(t *testing.T) {
			name := "docker-servers"
			err := icingaServer.DeleteHostgroup(name)
			if err != nil {
				t.Error(err)
			}
		})

		t.Run("HostgroupNonAPI", func(t *testing.T) {
			name := "linux-servers"
			err := icingaServer.DeleteHostgroup(name)
			if err.Error() != "500 One or more objects could not be deleted" {
				t.Error(err)
			}
		})

		t.Run("HostgroupDNE", func(t *testing.T) {
			name := "docker-servers"
			err := icingaServer.DeleteHostgroup(name)
			if err.Error() != "No objects found." {
				t.Error(err)
			}
		})
	})
}
<commit_msg>hostgroups: improve error check on testing<commit_after>package iapi

import (
	"testing"
)

func TestHostgroups(t *testing.T) {
	icingaServer := Server{"root", ICINGA2_API_PASSWORD, "https://127.0.0.1:5665/v1", true, nil}
	t.Run("Create", func(t *testing.T) {
		t.Run("Hostgroup", func(t *testing.T) {
			name := "docker-servers"
			displayName := "Docker Host Servers"
			_, err := icingaServer.CreateHostgroup(name, displayName)
			if err != nil {
				t.Error(err)
			}
		})
	})

	t.Run("Read", func(t *testing.T) {
		t.Run("ValidHostgroup", func(t *testing.T) {
			name := "linux-servers"
			_, err := icingaServer.GetHostgroup(name)
			if err != nil {
				t.Error(err)
			}
		})

		t.Run("InvalidHostgroup", func(t *testing.T) {
			name := "irix-servers"
			_, err := icingaServer.GetHostgroup(name)
			if err != nil {
				t.Error(err)
			}
		})
	})

	t.Run("Delete", func(t *testing.T) {
		// Delete Hostgroup created via API. Should succeed
		t.Run("Hostgroup", func(t *testing.T) {
			name := "docker-servers"
			err := icingaServer.DeleteHostgroup(name)
			if err != nil {
				t.Error(err)
			}
		})

		t.Run("HostgroupNonAPI", func(t *testing.T) {
			name := "linux-servers"
			err := icingaServer.DeleteHostgroup(name)
			if err == nil {
				t.Error(err)
			}
		})

		t.Run("HostgroupDNE", func(t *testing.T) {
			name := "docker-servers"
			err := icingaServer.DeleteHostgroup(name)
			if err.Error() != "No objects found." {
				t.Error(err)
			}
		})
	})
}
<|endoftext|>


<commit_before>package rel

type InfixOperationNode BinaryNode
<commit_msg>Updated InfixOperationNode<commit_after>package rel

type InfixOperationNode struct {
	Operator SqlLiteralNode
	Left     Visitable
	Right    Visitable
}
<|endoftext|>


<commit_before>package docs

import (
	"fmt"
	"sort"

	"github.com/Jeffail/benthos/v3/internal/interop/plugins"
	"github.com/Jeffail/gabs/v2"
	"gopkg.in/yaml.v3"
)

var labelField = FieldCommon(
	"label", "An optional label to use as an identifier for observability data such as metrics and logging.",
).OmitWhen(func(field, parent interface{}) (string, bool) {
	gObj := gabs.Wrap(parent)
	if typeStr, exists := gObj.S("type").Data().(string); exists && typeStr == "resource" {
		return "label field should be omitted when pointing to a resource", true
	}
	if resourceStr, exists := gObj.S("resource").Data().(string); exists && resourceStr != "" {
		return "label field should be omitted when pointing to a resource", true
	}
	return "", false
}).AtVersion("3.44.0")

func reservedFieldsByType(t Type) map[string]FieldSpec {
	m := map[string]FieldSpec{
		"type":   FieldCommon("type", "").HasType(FieldString),
		"plugin": FieldCommon("plugin", "").HasType(FieldObject),
	}
	switch t {
	case TypeInput:
		fallthrough
	case TypeOutput:
		m["processors"] = FieldCommon("processors", "").Array().HasType(FieldProcessor).OmitWhen(func(field, _ interface{}) (string, bool) {
			if arr, ok := field.([]interface{}); ok && len(arr) == 0 {
				return "field processors is empty and can be removed", true
			}
			return "", false
		})
	}
	if _, isLabelType := map[Type]struct{}{
		TypeInput:     {},
		TypeProcessor: {},
		TypeOutput:    {},
	}[t]; isLabelType {
		m["label"] = labelField
	}
	return m
}

func refreshOldPlugins() {
	plugins.FlushNameTypes(func(nt [2]string) {
		RegisterDocs(ComponentSpec{
			Name:   nt[0],
			Type:   Type(nt[1]),
			Status: StatusPlugin,
		})
	})
}

// GetInferenceCandidate checks a generic config structure for a component and
// returns either the inferred type name or an error if one cannot be inferred.
func GetInferenceCandidate(t Type, defaultType string, raw interface{}) (string, ComponentSpec, error) {
	refreshOldPlugins()

	m, ok := raw.(map[string]interface{})
	if !ok {
		return "", ComponentSpec{}, fmt.Errorf("invalid config value %T, expected object", raw)
	}

	if tStr, ok := m["type"].(string); ok {
		spec, exists := GetDocs(tStr, t)
		if !exists {
			return "", ComponentSpec{}, fmt.Errorf("%v type '%v' was not recognised", string(t), tStr)
		}
		return tStr, spec, nil
	}

	var keys []string
	for k := range m {
		keys = append(keys, k)
	}

	return getInferenceCandidateFromList(t, defaultType, keys)
}

// GetInferenceCandidateFromNode checks a yaml node config structure for a
// component and returns either the inferred type name or an error if one cannot
// be inferred.
func GetInferenceCandidateFromNode(t Type, defaultType string, node *yaml.Node) (string, ComponentSpec, error) {
	refreshOldPlugins()

	if node.Kind != yaml.MappingNode {
		return "", ComponentSpec{}, fmt.Errorf("invalid type %v, expected object", node.Kind)
	}

	var keys []string
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == "type" {
			tStr := node.Content[i+1].Value
			spec, exists := GetDocs(tStr, t)
			if !exists {
				return "", ComponentSpec{}, fmt.Errorf("%v type '%v' was not recognised", string(t), tStr)
			}
			return tStr, spec, nil
		}
		keys = append(keys, node.Content[i].Value)
	}

	return getInferenceCandidateFromList(t, defaultType, keys)
}

func getInferenceCandidateFromList(t Type, defaultType string, l []string) (string, ComponentSpec, error) {
	ignore := reservedFieldsByType(t)

	var candidates []string
	var inferred string
	var inferredSpec ComponentSpec
	for _, k := range l {
		if _, exists := ignore[k]; exists {
			continue
		}
		candidates = append(candidates, k)
		if spec, exists := GetDocs(k, t); exists {
			if len(inferred) > 0 {
				candidates = []string{inferred, k}
				sort.Strings(candidates)
				return "", ComponentSpec{}, fmt.Errorf(
					"unable to infer %v type, multiple candidates '%v' and '%v'", string(t), candidates[0], candidates[1],
				)
			}
			inferred = k
			inferredSpec = spec
		}
	}

	if len(candidates) == 0 && len(defaultType) > 0 {
		// A totally empty component config results in the default.
		// TODO: V4 Disable this
		if spec, exists := GetDocs(defaultType, t); exists {
			return defaultType, spec, nil
		}
	}

	if len(inferred) == 0 {
		sort.Strings(candidates)
		return "", ComponentSpec{}, fmt.Errorf("unable to infer %v type, candidates were: %v", string(t), candidates)
	}
	return inferred, inferredSpec, nil
}

// TODO: V4 Remove this.
func sanitiseConditionConfig(raw interface{}, removeDeprecated bool) error {
	// This is a nasty hack until Benthos v4.
	m, ok := raw.(map[string]interface{})
	if !ok {
		return fmt.Errorf("expected object configuration type, found: %T", raw)
	}
	typeStr, ok := m["type"]
	if !ok {
		return nil
	}
	for k := range m {
		if k == typeStr || k == "type" || k == "plugin" {
			continue
		}
		delete(m, k)
	}
	return nil
}

// TODO: V4 Remove this.
func sanitiseConditionConfigNode(node *yaml.Node) error {
	// This is a nasty hack until Benthos v4.
	newNodes := []*yaml.Node{}

	var name string
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == "type" {
			name = node.Content[i+1].Value
			newNodes = append(newNodes, node.Content[i])
			newNodes = append(newNodes, node.Content[i+1])
			break
		}
	}

	if name == "" {
		return nil
	}

	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == name {
			newNodes = append(newNodes, node.Content[i])
			newNodes = append(newNodes, node.Content[i+1])
			break
		}
	}

	node.Content = newNodes
	return nil
}

// SanitiseComponentConfig reduces a raw component configuration into only the
// fields for the component name configured.
func SanitiseComponentConfig(componentType Type, raw interface{}, filter FieldFilter) error {
	if componentType == "condition" {
		return sanitiseConditionConfig(raw, false)
	}

	name, spec, err := GetInferenceCandidate(componentType, "", raw)
	if err != nil {
		return err
	}

	m, ok := raw.(map[string]interface{})
	if !ok {
		return fmt.Errorf("expected object configuration type, found: %T", raw)
	}

	if componentConfRaw, exists := m[name]; exists {
		spec.Config.sanitise(componentConfRaw, filter)
	}

	reservedFields := reservedFieldsByType(componentType)
	for k, v := range m {
		if k == name {
			continue
		}
		spec, exists := reservedFields[k]
		if !exists {
			delete(m, k)
		}
		if _, omit := spec.shouldOmit(v, m); omit {
			delete(m, k)
		}
	}

	for name, fieldSpec := range reservedFields {
		fieldSpec.sanitise(m[name], filter)
	}
	return nil
}

// SanitiseConfig contains fields describing the desired behaviour of the config
// sanitiser such as removing certain fields.
type SanitiseConfig struct {
	RemoveTypeField  bool
	RemoveDeprecated bool
	Filter           FieldFilter
}

// SanitiseNode takes a yaml.Node and a config spec and sorts the fields of the
// node according to the spec. Also optionally removes the `type` field from
// this and all nested components.
func SanitiseNode(cType Type, node *yaml.Node, conf SanitiseConfig) error {
	if cType == "condition" {
		return sanitiseConditionConfigNode(node)
	}

	newNodes := []*yaml.Node{}

	var name string
	var keys []string
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == "label" {
			if _, omit := labelField.shouldOmitNode(node.Content[i+1], node); !omit {
				newNodes = append(newNodes, node.Content[i])
				newNodes = append(newNodes, node.Content[i+1])
			}
			break
		}
	}
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == "type" {
			name = node.Content[i+1].Value
			if !conf.RemoveTypeField {
				newNodes = append(newNodes, node.Content[i])
				newNodes = append(newNodes, node.Content[i+1])
			}
			break
		} else {
			keys = append(keys, node.Content[i].Value)
		}
	}
	if len(name) == 0 {
		if len(node.Content) == 0 {
			return nil
		}
		var err error
		if name, _, err = getInferenceCandidateFromList(cType, "", keys); err != nil {
			return err
		}
	}

	cSpec, exists := GetDocs(name, cType)
	if !exists {
		return fmt.Errorf("failed to obtain docs for %v type %v", cType, name)
	}

	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == name {
			if err := cSpec.Config.SanitiseNode(node.Content[i+1], conf); err != nil {
				return err
			}
			newNodes = append(newNodes, node.Content[i])
			newNodes = append(newNodes, node.Content[i+1])
			break
		}
	}

	reservedFields := reservedFieldsByType(cType)
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == name || node.Content[i].Value == "type" || node.Content[i].Value == "label" {
			continue
		}
		if spec, exists := reservedFields[node.Content[i].Value]; exists {
			if _, omit := spec.shouldOmitNode(node.Content[i+1], node); omit {
				continue
			}
			if err := spec.SanitiseNode(node.Content[i+1], conf); err != nil {
				return err
			}
			newNodes = append(newNodes, node.Content[i])
			newNodes = append(newNodes, node.Content[i+1])
		}
	}

	node.Content = newNodes
	return nil
}

// LintNode takes a yaml.Node and a config spec and returns a list of linting
// errors found in the config.
func LintNode(cType Type, node *yaml.Node) []Lint {
	if cType == "condition" {
		return nil
	}

	var lints []Lint

	var name string
	var keys []string
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == "type" {
			name = node.Content[i+1].Value
			break
		} else {
			keys = append(keys, node.Content[i].Value)
		}
	}
	if len(name) == 0 {
		if len(node.Content) == 0 {
			return nil
		}
		var err error
		if name, _, err = getInferenceCandidateFromList(cType, "", keys); err != nil {
			lints = append(lints, NewLintWarning(node.Line, "unable to infer component type"))
			return lints
		}
	}

	cSpec, exists := GetDocs(name, cType)
	if !exists {
		lints = append(lints, NewLintWarning(node.Line, fmt.Sprintf("failed to obtain docs for %v type %v", cType, name)))
		return lints
	}

	lintTarget := name
	if cSpec.Status == StatusPlugin {
		lintTarget = "plugin"
	}
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == lintTarget {
			lints = append(lints, cSpec.Config.lintNode(node.Content[i+1])...)
			break
		}
	}

	reservedFields := reservedFieldsByType(cType)
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == name || node.Content[i].Value == "type" {
			continue
		}
		spec, exists := reservedFields[node.Content[i].Value]
		if exists {
			lints = append(lints, lintFromOmit(spec, node, node.Content[i+1])...)
			lints = append(lints, spec.lintNode(node.Content[i+1])...)
		} else {
			lints = append(lints, NewLintError(
				node.Content[i].Line,
				fmt.Sprintf("field %v is invalid when the component type is %v", node.Content[i].Value, name),
			))
		}
	}

	return lints
}
<commit_msg>Fix plugin sanitation<commit_after>package docs

import (
	"fmt"
	"sort"

	"github.com/Jeffail/benthos/v3/internal/interop/plugins"
	"github.com/Jeffail/gabs/v2"
	"gopkg.in/yaml.v3"
)

var labelField = FieldCommon(
	"label", "An optional label to use as an identifier for observability data such as metrics and logging.",
).OmitWhen(func(field, parent interface{}) (string, bool) {
	gObj := gabs.Wrap(parent)
	if typeStr, exists := gObj.S("type").Data().(string); exists && typeStr == "resource" {
		return "label field should be omitted when pointing to a resource", true
	}
	if resourceStr, exists := gObj.S("resource").Data().(string); exists && resourceStr != "" {
		return "label field should be omitted when pointing to a resource", true
	}
	return "", false
}).AtVersion("3.44.0")

func reservedFieldsByType(t Type) map[string]FieldSpec {
	m := map[string]FieldSpec{
		"type":   FieldCommon("type", "").HasType(FieldString),
		"plugin": FieldCommon("plugin", "").HasType(FieldObject),
	}
	switch t {
	case TypeInput:
		fallthrough
	case TypeOutput:
		m["processors"] = FieldCommon("processors", "").Array().HasType(FieldProcessor).OmitWhen(func(field, _ interface{}) (string, bool) {
			if arr, ok := field.([]interface{}); ok && len(arr) == 0 {
				return "field processors is empty and can be removed", true
			}
			return "", false
		})
	}
	if _, isLabelType := map[Type]struct{}{
		TypeInput:     {},
		TypeProcessor: {},
		TypeOutput:    {},
	}[t]; isLabelType {
		m["label"] = labelField
	}
	return m
}

func refreshOldPlugins() {
	plugins.FlushNameTypes(func(nt [2]string) {
		RegisterDocs(ComponentSpec{
			Name:   nt[0],
			Type:   Type(nt[1]),
			Status: StatusPlugin,
		})
	})
}

// GetInferenceCandidate checks a generic config structure for a component and
// returns either the inferred type name or an error if one cannot be inferred.
func GetInferenceCandidate(t Type, defaultType string, raw interface{}) (string, ComponentSpec, error) {
	refreshOldPlugins()

	m, ok := raw.(map[string]interface{})
	if !ok {
		return "", ComponentSpec{}, fmt.Errorf("invalid config value %T, expected object", raw)
	}

	if tStr, ok := m["type"].(string); ok {
		spec, exists := GetDocs(tStr, t)
		if !exists {
			return "", ComponentSpec{}, fmt.Errorf("%v type '%v' was not recognised", string(t), tStr)
		}
		return tStr, spec, nil
	}

	var keys []string
	for k := range m {
		keys = append(keys, k)
	}

	return getInferenceCandidateFromList(t, defaultType, keys)
}

// GetInferenceCandidateFromNode checks a yaml node config structure for a
// component and returns either the inferred type name or an error if one cannot
// be inferred.
func GetInferenceCandidateFromNode(t Type, defaultType string, node *yaml.Node) (string, ComponentSpec, error) {
	refreshOldPlugins()

	if node.Kind != yaml.MappingNode {
		return "", ComponentSpec{}, fmt.Errorf("invalid type %v, expected object", node.Kind)
	}

	var keys []string
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == "type" {
			tStr := node.Content[i+1].Value
			spec, exists := GetDocs(tStr, t)
			if !exists {
				return "", ComponentSpec{}, fmt.Errorf("%v type '%v' was not recognised", string(t), tStr)
			}
			return tStr, spec, nil
		}
		keys = append(keys, node.Content[i].Value)
	}

	return getInferenceCandidateFromList(t, defaultType, keys)
}

func getInferenceCandidateFromList(t Type, defaultType string, l []string) (string, ComponentSpec, error) {
	ignore := reservedFieldsByType(t)

	var candidates []string
	var inferred string
	var inferredSpec ComponentSpec
	for _, k := range l {
		if _, exists := ignore[k]; exists {
			continue
		}
		candidates = append(candidates, k)
		if spec, exists := GetDocs(k, t); exists {
			if len(inferred) > 0 {
				candidates = []string{inferred, k}
				sort.Strings(candidates)
				return "", ComponentSpec{}, fmt.Errorf(
					"unable to infer %v type, multiple candidates '%v' and '%v'", string(t), candidates[0], candidates[1],
				)
			}
			inferred = k
			inferredSpec = spec
		}
	}

	if len(candidates) == 0 && len(defaultType) > 0 {
		// A totally empty component config results in the default.
		// TODO: V4 Disable this
		if spec, exists := GetDocs(defaultType, t); exists {
			return defaultType, spec, nil
		}
	}

	if len(inferred) == 0 {
		sort.Strings(candidates)
		return "", ComponentSpec{}, fmt.Errorf("unable to infer %v type, candidates were: %v", string(t), candidates)
	}
	return inferred, inferredSpec, nil
}

// TODO: V4 Remove this.
func sanitiseConditionConfig(raw interface{}, removeDeprecated bool) error {
	// This is a nasty hack until Benthos v4.
	m, ok := raw.(map[string]interface{})
	if !ok {
		return fmt.Errorf("expected object configuration type, found: %T", raw)
	}
	typeStr, ok := m["type"]
	if !ok {
		return nil
	}
	for k := range m {
		if k == typeStr || k == "type" || k == "plugin" {
			continue
		}
		delete(m, k)
	}
	return nil
}

// TODO: V4 Remove this.
func sanitiseConditionConfigNode(node *yaml.Node) error {
	// This is a nasty hack until Benthos v4.
	newNodes := []*yaml.Node{}

	var name string
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == "type" {
			name = node.Content[i+1].Value
			newNodes = append(newNodes, node.Content[i])
			newNodes = append(newNodes, node.Content[i+1])
			break
		}
	}

	if name == "" {
		return nil
	}

	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == name {
			newNodes = append(newNodes, node.Content[i])
			newNodes = append(newNodes, node.Content[i+1])
			break
		}
	}

	node.Content = newNodes
	return nil
}

// SanitiseComponentConfig reduces a raw component configuration into only the
// fields for the component name configured.
func SanitiseComponentConfig(componentType Type, raw interface{}, filter FieldFilter) error {
	if componentType == "condition" {
		return sanitiseConditionConfig(raw, false)
	}

	name, spec, err := GetInferenceCandidate(componentType, "", raw)
	if err != nil {
		return err
	}

	m, ok := raw.(map[string]interface{})
	if !ok {
		return fmt.Errorf("expected object configuration type, found: %T", raw)
	}

	if componentConfRaw, exists := m[name]; exists {
		spec.Config.sanitise(componentConfRaw, filter)
	}

	reservedFields := reservedFieldsByType(componentType)
	for k, v := range m {
		if k == name {
			continue
		}
		spec, exists := reservedFields[k]
		if !exists {
			delete(m, k)
		}
		if _, omit := spec.shouldOmit(v, m); omit {
			delete(m, k)
		}
	}

	for name, fieldSpec := range reservedFields {
		fieldSpec.sanitise(m[name], filter)
	}
	return nil
}

// SanitiseConfig contains fields describing the desired behaviour of the config
// sanitiser such as removing certain fields.
type SanitiseConfig struct {
	RemoveTypeField  bool
	RemoveDeprecated bool
	Filter           FieldFilter
}

// SanitiseNode takes a yaml.Node and a config spec and sorts the fields of the
// node according to the spec. Also optionally removes the `type` field from
// this and all nested components.
func SanitiseNode(cType Type, node *yaml.Node, conf SanitiseConfig) error {
	if cType == "condition" {
		return sanitiseConditionConfigNode(node)
	}

	newNodes := []*yaml.Node{}

	var name string
	var keys []string
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == "label" {
			if _, omit := labelField.shouldOmitNode(node.Content[i+1], node); !omit {
				newNodes = append(newNodes, node.Content[i])
				newNodes = append(newNodes, node.Content[i+1])
			}
			break
		}
	}
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == "type" {
			name = node.Content[i+1].Value
			if !conf.RemoveTypeField {
				newNodes = append(newNodes, node.Content[i])
				newNodes = append(newNodes, node.Content[i+1])
			}
			break
		} else {
			keys = append(keys, node.Content[i].Value)
		}
	}
	if len(name) == 0 {
		if len(node.Content) == 0 {
			return nil
		}
		var err error
		if name, _, err = getInferenceCandidateFromList(cType, "", keys); err != nil {
			return err
		}
	}

	cSpec, exists := GetDocs(name, cType)
	if !exists {
		return fmt.Errorf("failed to obtain docs for %v type %v", cType, name)
	}

	nameFound := false
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == name {
			nameFound = true
			if err := cSpec.Config.SanitiseNode(node.Content[i+1], conf); err != nil {
				return err
			}
			newNodes = append(newNodes, node.Content[i])
			newNodes = append(newNodes, node.Content[i+1])
			break
		}
	}

	// If the type field was omitted but we didn't see a config under the name
	// then we need to add it back in as it cannot be inferred.
	if !nameFound && conf.RemoveTypeField {
		for i := 0; i < len(node.Content); i += 2 {
			if node.Content[i].Value == "type" {
				newNodes = append(newNodes, node.Content[i])
				newNodes = append(newNodes, node.Content[i+1])
				break
			}
		}
	}

	reservedFields := reservedFieldsByType(cType)
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == name || node.Content[i].Value == "type" || node.Content[i].Value == "label" {
			continue
		}
		if spec, exists := reservedFields[node.Content[i].Value]; exists {
			if _, omit := spec.shouldOmitNode(node.Content[i+1], node); omit {
				continue
			}
			if err := spec.SanitiseNode(node.Content[i+1], conf); err != nil {
				return err
			}
			newNodes = append(newNodes, node.Content[i])
			newNodes = append(newNodes, node.Content[i+1])
		}
	}

	node.Content = newNodes
	return nil
}

// LintNode takes a yaml.Node and a config spec and returns a list of linting
// errors found in the config.
func LintNode(cType Type, node *yaml.Node) []Lint {
	if cType == "condition" {
		return nil
	}

	var lints []Lint

	var name string
	var keys []string
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == "type" {
			name = node.Content[i+1].Value
			break
		} else {
			keys = append(keys, node.Content[i].Value)
		}
	}
	if len(name) == 0 {
		if len(node.Content) == 0 {
			return nil
		}
		var err error
		if name, _, err = getInferenceCandidateFromList(cType, "", keys); err != nil {
			lints = append(lints, NewLintWarning(node.Line, "unable to infer component type"))
			return lints
		}
	}

	cSpec, exists := GetDocs(name, cType)
	if !exists {
		lints = append(lints, NewLintWarning(node.Line, fmt.Sprintf("failed to obtain docs for %v type %v", cType, name)))
		return lints
	}

	lintTarget := name
	if cSpec.Status == StatusPlugin {
		lintTarget = "plugin"
	}
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == lintTarget {
			lints = append(lints, cSpec.Config.lintNode(node.Content[i+1])...)
			break
		}
	}

	reservedFields := reservedFieldsByType(cType)
	for i := 0; i < len(node.Content); i += 2 {
		if node.Content[i].Value == name || node.Content[i].Value == "type" {
			continue
		}
		spec, exists := reservedFields[node.Content[i].Value]
		if exists {
			lints = append(lints, lintFromOmit(spec, node, node.Content[i+1])...)
			lints = append(lints, spec.lintNode(node.Content[i+1])...)
		} else {
			lints = append(lints, NewLintError(
				node.Content[i].Line,
				fmt.Sprintf("field %v is invalid when the component type is %v", node.Content[i].Value, name),
			))
		}
	}

	return lints
}
<|endoftext|>


<commit_before>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package entry

import (
	"net/http"
	"regexp"
	"strings"
)

// Entry 路由项需要实现的接口
type Entry interface {
	// 匹配匹配字符串
	Pattern() string

	// 匹配程度
	//  -1 表示完全不匹配；
	//  0  表示完全匹配；
	//  >0 表示部分匹配，值越小表示匹配程度越高。
	Match(url string) int

	// 获取参数，只有正则表达式才有数据。
	Params(url string) map[string]string

	// 执行该路由项的函数
	ServeHTTP(w http.ResponseWriter, r *http.Request)

	// 是否为正则表达式
	IsRegexp() bool
}

//////////////// basic

// 最基本的字符串匹配路由项。
type basic struct {
	pattern string
	handler http.Handler
}

func (b *basic) Pattern() string {
	return b.pattern
}

func (b *basic) IsRegexp() bool {
	return false
}

func (b *basic) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	b.handler.ServeHTTP(w, r)
}

func (b *basic) Params(url string) map[string]string {
	return nil
}

func (b *basic) Match(url string) int {
	if url == b.pattern {
		return 0
	}
	return -1
}

//////////////// static

// 静态文件匹配路由项，只要路径中的开头字符串与 pattern 相同，即表示匹配成功。
// 根据match()的返回值来确定哪个最匹配。
type static struct {
	pattern string
	handler http.Handler
}

func (s *static) Pattern() string {
	return s.pattern
}

func (s *static) IsRegexp() bool {
	return false
}

func (s *static) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s.handler.ServeHTTP(w, r)
}

func (s *static) Params(url string) map[string]string {
	return nil
}

func (s *static) Match(url string) int {
	l := len(url) - len(s.pattern)
	switch {
	case l < 0:
		return -1
	case l >= 0:
		if (s.pattern[len(s.pattern)-1] == '/') &&
			(s.pattern == url[:len(s.pattern)]) {
			return l
		}
		return -1
	} // end switch

	return -1
}

//////////////////// regexpr

// 正则表达式匹配。
type regexpr struct {
	pattern   string
	expr      *regexp.Regexp
	hasParams bool
	handler   http.Handler
}

func (re *regexpr) Pattern() string {
	return re.pattern
}

func (re *regexpr) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	re.handler.ServeHTTP(w, r)
}

func (re *regexpr) IsRegexp() bool {
	return true
}

func (re *regexpr) Match(url string) int {
	// 正则匹配，没有部分匹配功能，匹配返回0,否则返回-1
	loc := re.expr.FindStringIndex(url)
	if loc == nil {
		return -1
	}
	if loc[0] == 0 && loc[1] == len(url) {
		return 0
	}
	return -1
}

// 将 url 与当前的表达式进行匹配，返回其命名路由参数的值。若不匹配，则返回 nil
func (re *regexpr) Params(url string) map[string]string {
	if !re.hasParams {
		return nil
	}

	// 正确匹配正则表达式，则获相关的正则表达式命名变量。
	mapped := make(map[string]string, 3)
	subexps := re.expr.SubexpNames()
	args := re.expr.FindStringSubmatch(url)
	for index, name := range subexps {
		if len(name) > 0 && index < len(args) {
			mapped[name] = args[index]
		}
	}
	return mapped
}

// New 根据内容，生成相应的 Entry 接口实例。
//
// pattern 匹配内容。
// h 对应的 http.Handler，外层调用者确保该值不能为nil.
func New(pattern string, h http.Handler) Entry {
	strs := split(pattern)

	if len(strs) > 1 { // 正则路由
		p, hasParams := toPattern(strs)
		return &regexpr{
			pattern:   pattern,
			handler:   h,
			hasParams: hasParams,
			expr:      regexp.MustCompile(p),
		}
	}

	if pattern[len(pattern)-1] == '/' {
		return &static{
			pattern: pattern,
			handler: h,
		}
	}

	return &basic{
		pattern: pattern,
		handler: h,
	}
}

// 将 strs 按照顺序合并成一个正则表达式
// 返回参数正则表达式的字符串和一个 bool 值用以表式正则中是否包含了命名匹配。
func toPattern(strs []string) (string, bool) {
	pattern := ""
	hasParams := false

	for _, v := range strs {
		lastIndex := len(v) - 1
		if v[0] != '{' || v[lastIndex] != '}' { // 普通字符串
			pattern += v
			continue
		}

		v = v[1:lastIndex] // 去掉首尾的{}符号

		index := strings.IndexByte(v, ':')
		if index < 0 { // 只存在命名，而不存在正则表达式，默认匹配[^/]
			pattern += "(?P<" + v + ">[^/]+)"
			hasParams = true
			continue
		}

		if index == 0 { // 不存在命名，但有正则表达式
			pattern += v[1:]
			continue
		}

		pattern += "(?P<" + v[:index] + ">" + v[index+1:] + ")"
		hasParams = true
	}

	return pattern, hasParams
}

// 将 str 以 { 和 } 为分隔符进行分隔。
// 符号 { 和 } 必须成对出现，且不能嵌套，否则结果是未知的。
//  /api/{id:\\d+}/users/ ==> {"/api/", "{id:\\d+}", "/users/"}
func split(str string) []string {
	ret := []string{}
	var seq byte = '{'

	for {
		if len(str) == 0 { // 没有更多字符了，结束
			break
		}

		index := strings.IndexByte(str, seq)
		if index < 0 { // 未找到分隔符，结束
			ret = append(ret, str)
			break
		}

		if seq == '}' { // 将 } 字符留在当前字符串中
			index++
		}

		if index > 0 { // 为零表示当前字符串为空，无须理会。
			ret = append(ret, str[:index])
			str = str[index:]
		}

		if seq == '{' {
			seq = '}'
		} else {
			seq = '{'
		}
	}

	return ret
}
<commit_msg>修正文档错误<commit_after>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package entry

import (
	"net/http"
	"regexp"
	"strings"
)

// Entry 路由项需要实现的接口
type Entry interface {
	// 匹配匹配字符串
	Pattern() string

	// 匹配程度
	//  -1 表示完全不匹配；
	//  0  表示完全匹配；
	//  >0 表示部分匹配，值越小表示匹配程度越高。
	Match(url string) int

	// 获取参数，只有正则表达式才有数据。
	Params(url string) map[string]string

	// 执行该路由项的函数
	ServeHTTP(w http.ResponseWriter, r *http.Request)

	// 是否为正则表达式
	IsRegexp() bool
}

//////////////// basic

// 最基本的字符串匹配路由项。
type basic struct {
	pattern string
	handler http.Handler
}

func (b *basic) Pattern() string {
	return b.pattern
}

func (b *basic) IsRegexp() bool {
	return false
}

func (b *basic) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	b.handler.ServeHTTP(w, r)
}

func (b *basic) Params(url string) map[string]string {
	return nil
}

func (b *basic) Match(url string) int {
	if url == b.pattern {
		return 0
	}
	return -1
}

//////////////// static

// 静态文件匹配路由项，只要路径中的开头字符串与 pattern 相同，即表示匹配成功。
// 根据 match() 的返回值来确定哪个最匹配。
type static struct {
	pattern string
	handler http.Handler
}

func (s *static) Pattern() string {
	return s.pattern
}

func (s *static) IsRegexp() bool {
	return false
}

func (s *static) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s.handler.ServeHTTP(w, r)
}

func (s *static) Params(url string) map[string]string {
	return nil
}

func (s *static) Match(url string) int {
	l := len(url) - len(s.pattern)
	switch {
	case l < 0:
		return -1
	case l >= 0:
		if (s.pattern[len(s.pattern)-1] == '/') &&
			(s.pattern == url[:len(s.pattern)]) {
			return l
		}
		return -1
	} // end switch

	return -1
}

//////////////////// regexpr

// 正则表达式匹配。
type regexpr struct {
	pattern   string
	expr      *regexp.Regexp
	hasParams bool
	handler   http.Handler
}

func (re *regexpr) Pattern() string {
	return re.pattern
}

func (re *regexpr) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	re.handler.ServeHTTP(w, r)
}

func (re *regexpr) IsRegexp() bool {
	return true
}

func (re *regexpr) Match(url string) int {
	// 正则匹配，没有部分匹配功能，匹配返回 0，否则返回 -1
	loc := re.expr.FindStringIndex(url)
	if loc == nil {
		return -1
	}
	if loc[0] == 0 && loc[1] == len(url) {
		return 0
	}
	return -1
}

// 将 url 与当前的表达式进行匹配，返回其命名路由参数的值。若不匹配，则返回 nil
func (re *regexpr) Params(url string) map[string]string {
	if !re.hasParams {
		return nil
	}

	// 正确匹配正则表达式，则获相关的正则表达式命名变量。
	mapped := make(map[string]string, 3)
	subexps := re.expr.SubexpNames()
	args := re.expr.FindStringSubmatch(url)
	for index, name := range subexps {
		if len(name) > 0 && index < len(args) {
			mapped[name] = args[index]
		}
	}
	return mapped
}

// New 根据内容，生成相应的 Entry 接口实例。
//
// pattern 匹配内容。
// h 对应的 http.Handler，外层调用者确保该值不能为 nil.
func New(pattern string, h http.Handler) Entry {
	strs := split(pattern)

	if len(strs) > 1 { // 正则路由
		p, hasParams := toPattern(strs)
		return &regexpr{
			pattern:   pattern,
			handler:   h,
			hasParams: hasParams,
			expr:      regexp.MustCompile(p),
		}
	}

	if pattern[len(pattern)-1] == '/' {
		return &static{
			pattern: pattern,
			handler: h,
		}
	}

	return &basic{
		pattern: pattern,
		handler: h,
	}
}

// 将 strs 按照顺序合并成一个正则表达式
// 返回参数正则表达式的字符串，和一个 bool 值用以表式正则中是否包含了命名匹配。
func toPattern(strs []string) (string, bool) {
	pattern := ""
	hasParams := false

	for _, v := range strs {
		lastIndex := len(v) - 1
		if v[0] != '{' || v[lastIndex] != '}' { // 普通字符串
			pattern += v
			continue
		}

		v = v[1:lastIndex] // 去掉首尾的{}符号

		index := strings.IndexByte(v, ':')
		if index < 0 { // 只存在命名，而不存在正则表达式，默认匹配[^/]
			pattern += "(?P<" + v + ">[^/]+)"
			hasParams = true
			continue
		}

		if index == 0 { // 不存在命名，但有正则表达式
			pattern += v[1:]
			continue
		}

		pattern += "(?P<" + v[:index] + ">" + v[index+1:] + ")"
		hasParams = true
	}

	return pattern, hasParams
}

// 将 str 以 { 和 } 为分隔符进行分隔。
// 符号 { 和 } 必须成对出现，且不能嵌套，否则结果是未知的。
//  /api/{id:\\d+}/users/ ==> {"/api/", "{id:\\d+}", "/users/"}
func split(str string) []string {
	ret := []string{}
	var seq byte = '{'

	for {
		if len(str) == 0 { // 没有更多字符了，结束
			break
		}

		index := strings.IndexByte(str, seq)
		if index < 0 { // 未找到分隔符，结束
			ret = append(ret, str)
			break
		}

		if seq == '}' { // 将 } 字符留在当前字符串中
			index++
		}

		if index > 0 { // 为零表示当前字符串为空，无须理会。
			ret = append(ret, str[:index])
			str = str[index:]
		}

		if seq == '{' {
			seq = '}'
		} else {
			seq = '{'
		}
	}

	return ret
}
<|endoftext|>


<commit_before>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

// Package entry 路由项的相关操作。
package entry

import (
	"net/http"

	"github.com/issue9/mux/internal/syntax"
)

// Entry 表示一类资源的进入点，拥有统一的路由匹配模式。
type Entry interface {
	// 返回路由的匹配字符串
	Pattern() string

	// 与当前路由项是否匹配，若匹配的话，也将同时返回参数。
	// params 参数仅在 matched 为 true 时，才会有意义
	Match(path string) (matched bool, params map[string]string)

	// 优先级，越小越靠前。
	// 当多个路由项对某个请求都匹配时，将根据优先级来确定哪条最终会获得匹配。
	Priority() int

	// 添加请求方法及其对应的处理函数。
	//
	// 若已经存在，则返回错误。
	// 若 method == http.MethodOptions，则可以去覆盖默认的处理方式。
	Add(handler http.Handler, methods ...string) error

	// 移除指定方法的处理函数。empty 表示当前路由项中已经没有任何处理方法。
	Remove(method ...string) (empty bool)

	// 根据参数生成一条路径。
	// params 为替换匹配字符串的参数；
	// path 在有通配符的情况下，会替代通配符所代码的内容。
	URL(params map[string]string, path string) (string, error)

	// 获取指定请求方法对应的处理函数，若不存在，则返回 nil。
	Handler(method string) http.Handler

	// 手动设置 OPTIONS 的 Allow 报头。不调用此函数，
	// 会自动根据当前的 Add 和 Remove 调整 Allow 报头，
	// 调用 SetAllow() 之后，这些自动设置不再启作用。
	SetAllow(string)
}

// New 声明一个 Entry 实例。
func New(pattern string) (Entry, error) {
	s, err := syntax.New(pattern)
	if err != nil {
		return nil, err

	}

	if s.Type == syntax.TypeRegexp {
		return newRegexp(s)
	} else if s.Type == syntax.TypeNamed {
		return newNamed(s), nil
	}

	return newBasic(s), nil
}
<commit_msg>[internal/entry] 改进 New 函数<commit_after>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

// Package entry 路由项的相关操作。
package entry

import (
	"fmt"
	"net/http"

	"github.com/issue9/mux/internal/syntax"
)

// Entry 表示一类资源的进入点，拥有统一的路由匹配模式。
type Entry interface {
	// 返回路由的匹配字符串
	Pattern() string

	// 与当前路由项是否匹配，若匹配的话，也将同时返回参数。
	// params 参数仅在 matched 为 true 时，才会有意义
	Match(path string) (matched bool, params map[string]string)

	// 优先级，越小越靠前。
	// 当多个路由项对某个请求都匹配时，将根据优先级来确定哪条最终会获得匹配。
	Priority() int

	// 添加请求方法及其对应的处理函数。
	//
	// 若已经存在，则返回错误。
	// 若 method == http.MethodOptions，则可以去覆盖默认的处理方式。
	Add(handler http.Handler, methods ...string) error

	// 移除指定方法的处理函数。empty 表示当前路由项中已经没有任何处理方法。
	Remove(method ...string) (empty bool)

	// 根据参数生成一条路径。
	// params 为替换匹配字符串的参数；
	// path 在有通配符的情况下，会替代通配符所代码的内容。
	URL(params map[string]string, path string) (string, error)

	// 获取指定请求方法对应的处理函数，若不存在，则返回 nil。
	Handler(method string) http.Handler

	// 手动设置 OPTIONS 的 Allow 报头。不调用此函数，
	// 会自动根据当前的 Add 和 Remove 调整 Allow 报头，
	// 调用 SetAllow() 之后，这些自动设置不再启作用。
	SetAllow(string)
}

// New 声明一个 Entry 实例。
func New(pattern string) (Entry, error) {
	s, err := syntax.New(pattern)
	if err != nil {
		return nil, err

	}

	switch s.Type {
	case syntax.TypeRegexp:
		return newRegexp(s)
	case syntax.TypeNamed:
		return newNamed(s), nil
	case syntax.TypeBasic:
		return newBasic(s), nil
	default:
		return nil, fmt.Errorf("未知的类型：%v", s.Type)
	}
}
<|endoftext|>


<commit_before>package msrpc

import (
	"encoding/binary"
	"encoding/hex"

	"github.com/hirochachacha/go-smb2/internal/utf16le"
)

var le = binary.LittleEndian

func roundup(x, align int) int {
	return (x + (align - 1)) &^ (align - 1)
}

const (
	RPC_VERSION       = 5
	RPC_VERSION_MINOR = 0

	RPC_TYPE_REQUEST  = 0
	RPC_TYPE_RESPONSE = 2
	RPC_TYPE_BIND     = 11
	RPC_TYPE_BIND_ACK = 12

	RPC_PACKET_FLAG_FIRST = 0x01
	RPC_PACKET_FLAG_LAST  = 0x02

	SRVSVC_VERSION       = 3
	SRVSVC_VERSION_MINOR = 0

	NDR_VERSION = 2

	OP_NET_SHARE_ENUM = 15
)

var (
	SRVSVC_UUID = []byte("c84f324b7016d30112785a47bf6ee188")
	NDR_UUID    = []byte("045d888aeb1cc9119fe808002b104860")
)

type Bind struct {
	CallId uint32
}

func (r *Bind) Size() int {
	return 72
}

func (r *Bind) Encode(b []byte) {
	b[0] = RPC_VERSION
	b[1] = RPC_VERSION_MINOR
	b[2] = RPC_TYPE_BIND
	b[3] = RPC_PACKET_FLAG_FIRST | RPC_PACKET_FLAG_LAST

	// order = Little-Endian, float = IEEE, char = ASCII
	b[4] = 0x10
	b[5] = 0
	b[6] = 0
	b[7] = 0

	le.PutUint16(b[8:10], 72)        // frag length
	le.PutUint16(b[10:12], 0)        // auth length
	le.PutUint32(b[12:16], r.CallId) // call id
	le.PutUint16(b[16:18], 4280)     // max xmit frag
	le.PutUint16(b[18:20], 4280)     // max recv frag
	le.PutUint32(b[20:24], 0)        // assoc group
	le.PutUint32(b[24:28], 1)        // num ctx items
	le.PutUint16(b[28:30], 0)        // ctx item[1] .context id
	le.PutUint16(b[30:32], 1)        // ctx item[1] .num trans items

	hex.Decode(b[32:48], SRVSVC_UUID)
	le.PutUint16(b[48:50], SRVSVC_VERSION)
	le.PutUint16(b[50:52], SRVSVC_VERSION_MINOR)

	hex.Decode(b[52:68], NDR_UUID)
	le.PutUint32(b[68:72], NDR_VERSION)
}

type BindAckDecoder []byte

func (c BindAckDecoder) IsInvalid() bool {
	if len(c) < 24 {
		return true
	}
	if c.Version() != RPC_VERSION {
		return true
	}
	if c.VersionMinor() != RPC_VERSION_MINOR {
		return true
	}
	if c.PacketType() != RPC_TYPE_BIND_ACK {
		return true
	}
	return false
}

func (c BindAckDecoder) Version() uint8 {
	return c[0]
}

func (c BindAckDecoder) VersionMinor() uint8 {
	return c[1]
}

func (c BindAckDecoder) PacketType() uint8 {
	return c[2]
}

func (c BindAckDecoder) PacketFlags() uint8 {
	return c[3]
}

func (c BindAckDecoder) DataRepresentation() []byte {
	return c[4:8]
}

func (c BindAckDecoder) FragLength() uint16 {
	return le.Uint16(c[8:10])
}

func (c BindAckDecoder) AuthLength() uint16 {
	return le.Uint16(c[10:12])
}

func (c BindAckDecoder) CallId() uint32 {
	return le.Uint32(c[12:16])
}

func (c BindAckDecoder) MaxXmitFrag() uint16 {
	return le.Uint16(c[16:18])
}

func (c BindAckDecoder) MaxRecvFrag() uint16 {
	return le.Uint16(c[18:20])
}

func (c BindAckDecoder) AssocGroupId() uint32 {
	return le.Uint32(c[20:24])
}

type NetShareEnumAllRequest struct {
	CallId     uint32
	ServerName string
	Level      uint32
}

func (r *NetShareEnumAllRequest) Size() int {
	off := 40 + utf16le.EncodedStringLen(r.ServerName) + 2
	off = roundup(off, 4)
	off += 24
	off += 4
	return off
}

func (r *NetShareEnumAllRequest) Encode(b []byte) {
	b[0] = RPC_VERSION
	b[1] = RPC_VERSION_MINOR
	b[2] = RPC_TYPE_REQUEST
	b[3] = 0x03

	// order = Little-Endian, float = IEEE, char = ASCII
	b[4] = 0x10
	b[5] = 0
	b[6] = 0
	b[7] = 0

	le.PutUint16(b[10:12], 0)                 // auth length
	le.PutUint32(b[12:16], r.CallId)          // call id
	le.PutUint16(b[20:22], 0)                 // context id
	le.PutUint16(b[22:24], OP_NET_SHARE_ENUM) // opnum

	// follwing parts will change if we use NDR64 instead of NDR

	// pointer to server unc

	le.PutUint32(b[24:28], 0x20000) // referent ID

	count := utf16le.EncodedStringLen(r.ServerName)/2 + 1

	le.PutUint32(b[28:32], uint32(count)) // max count
	le.PutUint32(b[32:36], 0)             // offset
	le.PutUint32(b[36:40], uint32(count)) // actual count

	utf16le.EncodeString(b[40:], r.ServerName) // server unc

	off := 40 + count*2
	off = roundup(off, 4)

	// pointer level

	le.PutUint32(b[off:off+4], r.Level)

	// pointer to ctr (srvsvc_NetShareCtr)

	le.PutUint32(b[off+4:off+8], 1)            // ctr
	le.PutUint32(b[off+8:off+12], 0x20004)     // referent ID
	le.PutUint32(b[off+12:off+16], 0)          // ctr1.count
	le.PutUint32(b[off+16:off+20], 0)          // ctr1.pointer
	le.PutUint32(b[off+20:off+24], 0xffffffff) // max buffer

	off += 24

	// pointer to resume handle

	le.PutUint32(b[off:off+4], 0) // null pointer
	// le.PutUint32(b[off:off+4], 0x20008) // referent ID
	// le.PutUint32(b[off+4:off+8], 0)     // resume handle

	off += 4

	le.PutUint16(b[8:10], uint16(off))     // frag length
	le.PutUint32(b[16:20], uint32(off-24)) // alloc hint
}

type NetShareEnumAllResponseDecoder []byte

func (c NetShareEnumAllResponseDecoder) IsInvalid() bool {
	if len(c) < 24 {
		return true
	}
	if c.Version() != RPC_VERSION {
		return true
	}
	if c.VersionMinor() != RPC_VERSION_MINOR {
		return true
	}
	if c.PacketType() != RPC_TYPE_RESPONSE {
		return true
	}

	return false
}

func (c NetShareEnumAllResponseDecoder) Version() uint8 {
	return c[0]
}

func (c NetShareEnumAllResponseDecoder) VersionMinor() uint8 {
	return c[1]
}

func (c NetShareEnumAllResponseDecoder) PacketType() uint8 {
	return c[2]
}

func (c NetShareEnumAllResponseDecoder) PacketFlags() uint8 {
	return c[3]
}

func (c NetShareEnumAllResponseDecoder) DataRepresentation() []byte {
	return c[4:8]
}

func (c NetShareEnumAllResponseDecoder) FragLength() uint16 {
	return le.Uint16(c[8:10])
}

func (c NetShareEnumAllResponseDecoder) AuthLength() uint16 {
	return le.Uint16(c[10:12])
}

func (c NetShareEnumAllResponseDecoder) CallId() uint32 {
	return le.Uint32(c[12:16])
}

func (c NetShareEnumAllResponseDecoder) AllocHint() uint32 {
	return le.Uint32(c[16:20])
}

func (c NetShareEnumAllResponseDecoder) ContextId() uint16 {
	return le.Uint16(c[20:22])
}

func (c NetShareEnumAllResponseDecoder) CancelCount() uint8 {
	return c[22]
}

func (c NetShareEnumAllResponseDecoder) IsIncomplete() bool {
	if len(c) < 48 {
		return true
	}

	level := le.Uint32(c[24:28])

	count := int(le.Uint32(c[36:40]))

	switch level {
	case 0:
		offset := 48 + count*4 // name pointer
		if len(c) < offset {
			return true
		}

		for i := 0; i < count; i++ {
			if len(c) < offset+12 {
				return true
			}

			noff := int(le.Uint32(c[offset+4 : offset+8]))    // offset
			nlen := int(le.Uint32(c[offset+8:offset+12])) * 2 // actual count
			offset = roundup(offset+12+noff+nlen, 4)

			if len(c) < offset {
				return true
			}
		}
	case 1:
		offset := 48 + count*12
		if len(c) < offset {
			return true
		}

		for i := 0; i < count; i++ {
			{ // name
				if len(c) < offset+12 {
					return true
				}

				noff := int(le.Uint32(c[offset+4 : offset+8]))    // offset
				nlen := int(le.Uint32(c[offset+8:offset+12])) * 2 // actual count
				offset = roundup(offset+12+noff+nlen, 4)

				if len(c) < offset {
					return true
				}
			}

			{ // comment
				if len(c) < offset+12 {
					return true
				}

				coff := int(le.Uint32(c[offset+4 : offset+8]))    // offset
				clen := int(le.Uint32(c[offset+8:offset+12])) * 2 // actual count
				offset = roundup(offset+12+coff+clen, 4)

				if len(c) < offset {
					return true
				}
			}
		}
	default:
		// TODO not supported yet
		return true
	}

	return false
}

func (c NetShareEnumAllResponseDecoder) Buffer() []byte {
	return c[24:]
}

func (c NetShareEnumAllResponseDecoder) ShareNameList() []string {
	level := le.Uint32(c[24:28])

	count := int(le.Uint32(c[36:40]))

	ss := make([]string, count)

	switch level {
	case 0:
		offset := 48 + count*4 // name pointer
		for i := 0; i < count; i++ {
			noff := int(le.Uint32(c[offset+4 : offset+8]))    // offset
			nlen := int(le.Uint32(c[offset+8:offset+12])) * 2 // actual count

			ss[i] = utf16le.DecodeToString(c[offset+12+noff : offset+12+noff+nlen])

			offset = roundup(offset+12+noff+nlen, 4)
		}
	case 1:
		offset := 48 + count*12
		for i := 0; i < count; i++ {
			{ // name
				noff := int(le.Uint32(c[offset+4 : offset+8]))    // offset
				nlen := int(le.Uint32(c[offset+8:offset+12])) * 2 // actual count

				ss[i] = utf16le.DecodeToString(c[offset+12+noff : offset+12+noff+nlen])

				offset = roundup(offset+12+noff+nlen, 4)
			}

			{ // comment
				coff := int(le.Uint32(c[offset+4 : offset+8]))    // offset
				clen := int(le.Uint32(c[offset+8:offset+12])) * 2 // actual count
				offset = roundup(offset+12+coff+clen, 4)
			}
		}
	default:
		// TODO not supported yet
		return nil
	}

	return ss
}
<commit_msg>refactor: make it simple<commit_after>package msrpc

import (
	"encoding/binary"
	"encoding/hex"

	"github.com/hirochachacha/go-smb2/internal/utf16le"
)

var le = binary.LittleEndian

func roundup(x, align int) int {
	return (x + (align - 1)) &^ (align - 1)
}

const (
	RPC_VERSION       = 5
	RPC_VERSION_MINOR = 0

	RPC_TYPE_REQUEST  = 0
	RPC_TYPE_RESPONSE = 2
	RPC_TYPE_BIND     = 11
	RPC_TYPE_BIND_ACK = 12

	RPC_PACKET_FLAG_FIRST = 0x01
	RPC_PACKET_FLAG_LAST  = 0x02

	SRVSVC_VERSION       = 3
	SRVSVC_VERSION_MINOR = 0

	NDR_VERSION = 2

	OP_NET_SHARE_ENUM = 15
)

var (
	SRVSVC_UUID = []byte("c84f324b7016d30112785a47bf6ee188")
	NDR_UUID    = []byte("045d888aeb1cc9119fe808002b104860")
)

type Bind struct {
	CallId uint32
}

func (r *Bind) Size() int {
	return 72
}

func (r *Bind) Encode(b []byte) {
	b[0] = RPC_VERSION
	b[1] = RPC_VERSION_MINOR
	b[2] = RPC_TYPE_BIND
	b[3] = RPC_PACKET_FLAG_FIRST | RPC_PACKET_FLAG_LAST

	// order = Little-Endian, float = IEEE, char = ASCII
	b[4] = 0x10
	b[5] = 0
	b[6] = 0
	b[7] = 0

	le.PutUint16(b[8:10], 72)        // frag length
	le.PutUint16(b[10:12], 0)        // auth length
	le.PutUint32(b[12:16], r.CallId) // call id
	le.PutUint16(b[16:18], 4280)     // max xmit frag
	le.PutUint16(b[18:20], 4280)     // max recv frag
	le.PutUint32(b[20:24], 0)        // assoc group
	le.PutUint32(b[24:28], 1)        // num ctx items
	le.PutUint16(b[28:30], 0)        // ctx item[1] .context id
	le.PutUint16(b[30:32], 1)        // ctx item[1] .num trans items

	hex.Decode(b[32:48], SRVSVC_UUID)
	le.PutUint16(b[48:50], SRVSVC_VERSION)
	le.PutUint16(b[50:52], SRVSVC_VERSION_MINOR)

	hex.Decode(b[52:68], NDR_UUID)
	le.PutUint32(b[68:72], NDR_VERSION)
}

type BindAckDecoder []byte

func (c BindAckDecoder) IsInvalid() bool {
	if len(c) < 24 {
		return true
	}
	if c.Version() != RPC_VERSION {
		return true
	}
	if c.VersionMinor() != RPC_VERSION_MINOR {
		return true
	}
	if c.PacketType() != RPC_TYPE_BIND_ACK {
		return true
	}
	return false
}

func (c BindAckDecoder) Version() uint8 {
	return c[0]
}

func (c BindAckDecoder) VersionMinor() uint8 {
	return c[1]
}

func (c BindAckDecoder) PacketType() uint8 {
	return c[2]
}

func (c BindAckDecoder) PacketFlags() uint8 {
	return c[3]
}

func (c BindAckDecoder) DataRepresentation() []byte {
	return c[4:8]
}

func (c BindAckDecoder) FragLength() uint16 {
	return le.Uint16(c[8:10])
}

func (c BindAckDecoder) AuthLength() uint16 {
	return le.Uint16(c[10:12])
}

func (c BindAckDecoder) CallId() uint32 {
	return le.Uint32(c[12:16])
}

func (c BindAckDecoder) MaxXmitFrag() uint16 {
	return le.Uint16(c[16:18])
}

func (c BindAckDecoder) MaxRecvFrag() uint16 {
	return le.Uint16(c[18:20])
}

func (c BindAckDecoder) AssocGroupId() uint32 {
	return le.Uint32(c[20:24])
}

type NetShareEnumAllRequest struct {
	CallId     uint32
	ServerName string
	Level      uint32
}

func (r *NetShareEnumAllRequest) Size() int {
	off := 40 + utf16le.EncodedStringLen(r.ServerName) + 2
	off = roundup(off, 4)
	off += 24
	off += 4
	return off
}

func (r *NetShareEnumAllRequest) Encode(b []byte) {
	b[0] = RPC_VERSION
	b[1] = RPC_VERSION_MINOR
	b[2] = RPC_TYPE_REQUEST
	b[3] = RPC_PACKET_FLAG_FIRST | RPC_PACKET_FLAG_LAST

	// order = Little-Endian, float = IEEE, char = ASCII
	b[4] = 0x10
	b[5] = 0
	b[6] = 0
	b[7] = 0

	le.PutUint16(b[10:12], 0)                 // auth length
	le.PutUint32(b[12:16], r.CallId)          // call id
	le.PutUint16(b[20:22], 0)                 // context id
	le.PutUint16(b[22:24], OP_NET_SHARE_ENUM) // opnum

	// follwing parts will change if we use NDR64 instead of NDR

	// pointer to server unc

	le.PutUint32(b[24:28], 0x20000) // referent ID

	count := utf16le.EncodedStringLen(r.ServerName)/2 + 1

	le.PutUint32(b[28:32], uint32(count)) // max count
	le.PutUint32(b[32:36], 0)             // offset
	le.PutUint32(b[36:40], uint32(count)) // actual count

	utf16le.EncodeString(b[40:], r.ServerName) // server unc

	off := 40 + count*2
	off = roundup(off, 4)

	// pointer level

	le.PutUint32(b[off:off+4], r.Level)

	// pointer to ctr (srvsvc_NetShareCtr)

	le.PutUint32(b[off+4:off+8], 1)            // ctr
	le.PutUint32(b[off+8:off+12], 0x20004)     // referent ID
	le.PutUint32(b[off+12:off+16], 0)          // ctr1.count
	le.PutUint32(b[off+16:off+20], 0)          // ctr1.pointer
	le.PutUint32(b[off+20:off+24], 0xffffffff) // max buffer

	off += 24

	// pointer to resume handle

	le.PutUint32(b[off:off+4], 0) // null pointer
	// le.PutUint32(b[off:off+4], 0x20008) // referent ID
	// le.PutUint32(b[off+4:off+8], 0)     // resume handle

	off += 4

	le.PutUint16(b[8:10], uint16(off))     // frag length
	le.PutUint32(b[16:20], uint32(off-24)) // alloc hint
}

type NetShareEnumAllResponseDecoder []byte

func (c NetShareEnumAllResponseDecoder) IsInvalid() bool {
	if len(c) < 24 {
		return true
	}
	if c.Version() != RPC_VERSION {
		return true
	}
	if c.VersionMinor() != RPC_VERSION_MINOR {
		return true
	}
	if c.PacketType() != RPC_TYPE_RESPONSE {
		return true
	}

	return false
}

func (c NetShareEnumAllResponseDecoder) Version() uint8 {
	return c[0]
}

func (c NetShareEnumAllResponseDecoder) VersionMinor() uint8 {
	return c[1]
}

func (c NetShareEnumAllResponseDecoder) PacketType() uint8 {
	return c[2]
}

func (c NetShareEnumAllResponseDecoder) PacketFlags() uint8 {
	return c[3]
}

func (c NetShareEnumAllResponseDecoder) DataRepresentation() []byte {
	return c[4:8]
}

func (c NetShareEnumAllResponseDecoder) FragLength() uint16 {
	return le.Uint16(c[8:10])
}

func (c NetShareEnumAllResponseDecoder) AuthLength() uint16 {
	return le.Uint16(c[10:12])
}

func (c NetShareEnumAllResponseDecoder) CallId() uint32 {
	return le.Uint32(c[12:16])
}

func (c NetShareEnumAllResponseDecoder) AllocHint() uint32 {
	return le.Uint32(c[16:20])
}

func (c NetShareEnumAllResponseDecoder) ContextId() uint16 {
	return le.Uint16(c[20:22])
}

func (c NetShareEnumAllResponseDecoder) CancelCount() uint8 {
	return c[22]
}

func (c NetShareEnumAllResponseDecoder) IsIncomplete() bool {
	if len(c) < 48 {
		return true
	}

	level := le.Uint32(c[24:28])

	count := int(le.Uint32(c[36:40]))

	switch level {
	case 0:
		offset := 48 + count*4 // name pointer
		if len(c) < offset {
			return true
		}

		for i := 0; i < count; i++ {
			if len(c) < offset+12 {
				return true
			}

			noff := int(le.Uint32(c[offset+4 : offset+8]))    // offset
			nlen := int(le.Uint32(c[offset+8:offset+12])) * 2 // actual count
			offset = roundup(offset+12+noff+nlen, 4)

			if len(c) < offset {
				return true
			}
		}
	case 1:
		offset := 48 + count*12
		if len(c) < offset {
			return true
		}

		for i := 0; i < count; i++ {
			{ // name
				if len(c) < offset+12 {
					return true
				}

				noff := int(le.Uint32(c[offset+4 : offset+8]))    // offset
				nlen := int(le.Uint32(c[offset+8:offset+12])) * 2 // actual count
				offset = roundup(offset+12+noff+nlen, 4)

				if len(c) < offset {
					return true
				}
			}

			{ // comment
				if len(c) < offset+12 {
					return true
				}

				coff := int(le.Uint32(c[offset+4 : offset+8]))    // offset
				clen := int(le.Uint32(c[offset+8:offset+12])) * 2 // actual count
				offset = roundup(offset+12+coff+clen, 4)

				if len(c) < offset {
					return true
				}
			}
		}
	default:
		// TODO not supported yet
		return true
	}

	return false
}

func (c NetShareEnumAllResponseDecoder) Buffer() []byte {
	return c[24:]
}

func (c NetShareEnumAllResponseDecoder) ShareNameList() []string {
	level := le.Uint32(c[24:28])

	count := int(le.Uint32(c[36:40]))

	ss := make([]string, count)

	switch level {
	case 0:
		offset := 48 + count*4 // name pointer
		for i := 0; i < count; i++ {
			noff := int(le.Uint32(c[offset+4 : offset+8]))    // offset
			nlen := int(le.Uint32(c[offset+8:offset+12])) * 2 // actual count

			ss[i] = utf16le.DecodeToString(c[offset+12+noff : offset+12+noff+nlen])

			offset = roundup(offset+12+noff+nlen, 4)
		}
	case 1:
		offset := 48 + count*12
		for i := 0; i < count; i++ {
			{ // name
				noff := int(le.Uint32(c[offset+4 : offset+8]))    // offset
				nlen := int(le.Uint32(c[offset+8:offset+12])) * 2 // actual count

				ss[i] = utf16le.DecodeToString(c[offset+12+noff : offset+12+noff+nlen])

				offset = roundup(offset+12+noff+nlen, 4)
			}

			{ // comment
				coff := int(le.Uint32(c[offset+4 : offset+8]))    // offset
				clen := int(le.Uint32(c[offset+8:offset+12])) * 2 // actual count
				offset = roundup(offset+12+coff+clen, 4)
			}
		}
	default:
		// TODO not supported yet
		return nil
	}

	return ss
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

// For regular files, mtail gets notified on modifications (i.e. appends) to
// log files that are being watched, in order to read the new lines. Log files
// can also be rotated, so mtail is also notified of creates in the log file
// directory.

import (
	"expvar"
	"html/template"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"

	"github.com/google/mtail/internal/logline"
	"github.com/google/mtail/internal/watcher"
)

var (
	// logCount records the number of logs that are being tailed
	logCount = expvar.NewInt("log_count")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	lines chan<- *logline.LogLine // Logfile lines being emitted.
	w     watcher.Watcher

	handlesMu sync.RWMutex     // protects `handles'
	handles   map[string]*File // File handles for each pathname.

	globPatternsMu sync.RWMutex        // protects `globPatterns'
	globPatterns   map[string]struct{} // glob patterns to match newly created files in dir paths against

	runDone chan struct{} // Signals termination of the run goroutine.

	eventsHandle int // record the handle with which to add new log files to the watcher

	oneShot bool
}

// OneShot puts the tailer in one-shot mode.
func OneShot(t *Tailer) error {
	t.oneShot = true
	return nil
}

// New creates a new Tailer.
func New(lines chan<- *logline.LogLine, w watcher.Watcher, options ...func(*Tailer) error) (*Tailer, error) {
	if lines == nil {
		return nil, errors.New("can't create tailer without lines channel")
	}
	if w == nil {
		return nil, errors.New("can't create tailer without W")
	}
	t := &Tailer{
		lines:        lines,
		w:            w,
		handles:      make(map[string]*File),
		globPatterns: make(map[string]struct{}),
		runDone:      make(chan struct{}),
	}
	if err := t.SetOption(options...); err != nil {
		return nil, err
	}
	handle, eventsChan := t.w.Events()
	t.eventsHandle = handle
	go t.run(eventsChan)
	return t, nil
}

// SetOption takes one or more option functions and applies them in order to Tailer.
func (t *Tailer) SetOption(options ...func(*Tailer) error) error {
	for _, option := range options {
		if err := option(t); err != nil {
			return err
		}
	}
	return nil
}

// setHandle sets a file handle under it's pathname
func (t *Tailer) setHandle(pathname string, f *File) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return errors.Wrapf(err, "Failed to lookup abspath of %q", pathname)
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	t.handles[absPath] = f
	return nil
}

// handleForPath retrives a file handle for a pathname.
func (t *Tailer) handleForPath(pathname string) (*File, bool) {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		glog.V(2).Infof("Couldn't resolve path %q: %s", pathname, err)
		return nil, false
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	fd, ok := t.handles[absPath]
	return fd, ok
}

func (t *Tailer) hasHandle(pathname string) bool {
	_, ok := t.handleForPath(pathname)
	return ok
}

// AddPattern adds a pattern to the list of patterns to filter filenames against.
func (t *Tailer) AddPattern(pattern string) error {
	absPath, err := filepath.Abs(pattern)
	if err != nil {
		glog.V(2).Infof("Couldn't canonicalize path %q: %s", pattern, err)
		return err
	}
	glog.V(2).Infof("AddPattern: %s", absPath)
	t.globPatternsMu.Lock()
	t.globPatterns[absPath] = struct{}{}
	t.globPatternsMu.Unlock()
	return nil
}

// TailPattern registers a pattern to be tailed.  If pattern is a plain
// file then it is watched for updates and opened.  If pattern is a glob, then
// all paths that match the glob are opened and watched, and the directories
// containing those matches, if any, are watched.
func (t *Tailer) TailPattern(pattern string) error {
	if err := t.AddPattern(pattern); err != nil {
		return err
	}
	// Add a watch on the containing directory, so we know when a rotation
	// occurs or something shows up that matches this pattern.
	if err := t.watchDirname(pattern); err != nil {
		return err
	}
	matches, err := filepath.Glob(pattern)
	if err != nil {
		return err
	}
	glog.V(1).Infof("glob matches: %v", matches)
	// Error if there are no matches, but if they show up later, they'll get picked up by the directory watch set above.
	if len(matches) == 0 {
		return errors.Errorf("No matches for pattern %q", pattern)
	}
	for _, pathname := range matches {
		err := t.TailPath(pathname)
		if err != nil {
			return errors.Wrapf(err, "attempting to tail %q", pathname)
		}
	}
	return nil
}

// TailPath registers a filesystem pathname to be tailed.
func (t *Tailer) TailPath(pathname string) error {
	if t.hasHandle(pathname) {
		glog.V(2).Infof("already watching %q", pathname)
		return nil
	}
	if err := t.w.Add(pathname, t.eventsHandle); err != nil {
		return err
	}
	// New file at start of program, seek to EOF.
	return t.openLogPath(pathname, false)
}

// handleLogEvent is dispatched when an Event is received, causing the tailer
// to read all available bytes from an already-opened file and send each log
// line onto lines channel.  Because we handle rotations and truncates when
// reaching EOF in the file reader itself, we don't care what the signal is
// from the filewatcher.
func (t *Tailer) handleLogEvent(pathname string) {
	glog.V(2).Infof("handleLogUpdate %s", pathname)
	fd, ok := t.handleForPath(pathname)
	if !ok {
		glog.V(1).Infof("No file handle found for %q, but is being watched", pathname)
		// We want to open files we have watches on in case the file was
		// unreadable before now; but we have to copmare against the glob to be
		// sure we don't just add all the files in a watched directory as they
		// get modified.
		t.handleCreateGlob(pathname)
		return
	}
	doFollow(fd)
}

// doFollow performs the Follow on an existing file descriptor, logging any errors
func doFollow(fd *File) {
	err := fd.Follow()
	if err != nil && err != io.EOF {
		glog.Info(err)
	}
}

// watchDirname adds the directory containing a path to be watched.
func (t *Tailer) watchDirname(pathname string) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return err
	}
	d := filepath.Dir(absPath)
	return t.w.Add(d, t.eventsHandle)
}

// openLogPath opens a log file named by pathname.
func (t *Tailer) openLogPath(pathname string, seekToStart bool) error {
	glog.V(2).Infof("openlogPath %s %v", pathname, seekToStart)
	if err := t.watchDirname(pathname); err != nil {
		return err
	}
	f, err := NewFile(pathname, t.lines, seekToStart || t.oneShot)
	if err != nil {
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			glog.V(1).Infof("pathname %q doesn't exist (yet?)", f.Pathname)
			return nil
		}
		return err
	}
	glog.V(2).Infof("Adding a file watch on %q", f.Pathname)
	if err := t.w.Add(f.Pathname, t.eventsHandle); err != nil {
		return err
	}
	if err := t.setHandle(pathname, f); err != nil {
		return err
	}
	if err := f.Read(); err != nil && err != io.EOF {
		return err
	}
	glog.Infof("Tailing %s", f.Pathname)
	logCount.Add(1)
	return nil
}

// handleCreateGlob matches the pathname against the glob patterns and starts tailing the file.
func (t *Tailer) handleCreateGlob(pathname string) {
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()

	for pattern := range t.globPatterns {
		matched, err := filepath.Match(pattern, pathname)
		if err != nil {
			glog.Warningf("Unexpected bad pattern %q not detected earlier", pattern)
			continue
		}
		if !matched {
			glog.V(2).Infof("%q did not match pattern %q", pathname, pattern)
			continue
		}
		glog.V(1).Infof("New file %q matched existing glob %q", pathname, pattern)
		// If this file was just created, read from the start of the file.
		if err := t.openLogPath(pathname, true); err != nil {
			glog.Infof("Failed to tail new file %q: %s", pathname, err)
		}
		glog.V(2).Infof("started tailing %q", pathname)
		return
	}
	glog.V(2).Infof("did not start tailing %q", pathname)
}

// run the main event loop for the Tailer.  It receives notification of
// log file changes from the watcher channel, and dispatches the log event
// handler.
func (t *Tailer) run(events <-chan watcher.Event) {
	defer close(t.runDone)

	for e := range events {
		glog.V(2).Infof("Event type %#v", e)
		t.handleLogEvent(e.Pathname)
	}
	glog.Infof("Closing lines channel.")
	close(t.lines)
	glog.Infof("Shutting down tailer.")
}

// Close signals termination to the watcher.
func (t *Tailer) Close() error {
	if err := t.w.Close(); err != nil {
		return err
	}
	<-t.runDone
	return nil
}

const tailerTemplate = `

Log Tailer

Patterns

{{range $name, $val := $.Patterns}}

{{$name}}

{{end}}

Log files watched

pathname
errors
rotations
truncations
lines read

{{range $name, $val := $.Handles}}

{{$name}}

{{index $.Errors $name}}
{{index $.Rotations $name}}
{{index $.Truncs $name}}
{{index $.Lines $name}}

{{end}}

`

// WriteStatusHTML emits the Tailer's state in HTML format to the io.Writer w.
func (t *Tailer) WriteStatusHTML(w io.Writer) error {
	tpl, err := template.New("tailer").Parse(tailerTemplate)
	if err != nil {
		return err
	}
	t.handlesMu.RLock()
	defer t.handlesMu.RUnlock()
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()
	data := struct {
		Handles   map[string]*File
		Patterns  map[string]struct{}
		Rotations map[string]string
		Lines     map[string]string
		Errors    map[string]string
		Truncs    map[string]string
	}{
		t.handles,
		t.globPatterns,
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
	}
	for _, pair := range []struct {
		v *expvar.Map
		m map[string]string
	}{
		{logErrors, data.Errors},
		{logRotations, data.Rotations},
		{logTruncs, data.Truncs},
		{lineCount, data.Lines},
	} {
		pair.v.Do(func(kv expvar.KeyValue) {
			pair.m[kv.Key] = kv.Value.String()
		})
	}
	return tpl.Execute(w, data)
}

// Gc removes file handles that have had no reads for 24h or more.
func (t *Tailer) Gc() error {
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	for k, v := range t.handles {
		if time.Since(v.LastRead) > (time.Hour * 24) {
			if err := t.w.Remove(v.Pathname); err != nil {
				glog.Info(err)
			}
			if err := v.Close(); err != nil {
				glog.Info(err)
			}
			delete(t.handles, k)
		}
	}
	return nil
}

// StartExpiryLoop runs a permanent goroutine to expire metrics every duration.
func (t *Tailer) StartGcLoop(duration time.Duration) {
	if duration <= 0 {
		glog.Info("Log handle expiration disabled")
		return
	}
	go func() {
		glog.Infof("Starting log handle expiry loop every %s", duration.String())
		ticker := time.NewTicker(duration)
		for range ticker.C {
			if err := t.Gc(); err != nil {
				glog.Info(err)
			}
		}
	}()
}
<commit_msg>Fix a nil pointer deref in tailer error handling.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

// For regular files, mtail gets notified on modifications (i.e. appends) to
// log files that are being watched, in order to read the new lines. Log files
// can also be rotated, so mtail is also notified of creates in the log file
// directory.

import (
	"expvar"
	"html/template"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"

	"github.com/google/mtail/internal/logline"
	"github.com/google/mtail/internal/watcher"
)

var (
	// logCount records the number of logs that are being tailed
	logCount = expvar.NewInt("log_count")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	lines chan<- *logline.LogLine // Logfile lines being emitted.
	w     watcher.Watcher

	handlesMu sync.RWMutex     // protects `handles'
	handles   map[string]*File // File handles for each pathname.

	globPatternsMu sync.RWMutex        // protects `globPatterns'
	globPatterns   map[string]struct{} // glob patterns to match newly created files in dir paths against

	runDone chan struct{} // Signals termination of the run goroutine.

	eventsHandle int // record the handle with which to add new log files to the watcher

	oneShot bool
}

// OneShot puts the tailer in one-shot mode.
func OneShot(t *Tailer) error {
	t.oneShot = true
	return nil
}

// New creates a new Tailer.
func New(lines chan<- *logline.LogLine, w watcher.Watcher, options ...func(*Tailer) error) (*Tailer, error) {
	if lines == nil {
		return nil, errors.New("can't create tailer without lines channel")
	}
	if w == nil {
		return nil, errors.New("can't create tailer without W")
	}
	t := &Tailer{
		lines:        lines,
		w:            w,
		handles:      make(map[string]*File),
		globPatterns: make(map[string]struct{}),
		runDone:      make(chan struct{}),
	}
	if err := t.SetOption(options...); err != nil {
		return nil, err
	}
	handle, eventsChan := t.w.Events()
	t.eventsHandle = handle
	go t.run(eventsChan)
	return t, nil
}

// SetOption takes one or more option functions and applies them in order to Tailer.
func (t *Tailer) SetOption(options ...func(*Tailer) error) error {
	for _, option := range options {
		if err := option(t); err != nil {
			return err
		}
	}
	return nil
}

// setHandle sets a file handle under it's pathname
func (t *Tailer) setHandle(pathname string, f *File) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return errors.Wrapf(err, "Failed to lookup abspath of %q", pathname)
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	t.handles[absPath] = f
	return nil
}

// handleForPath retrives a file handle for a pathname.
func (t *Tailer) handleForPath(pathname string) (*File, bool) {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		glog.V(2).Infof("Couldn't resolve path %q: %s", pathname, err)
		return nil, false
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	fd, ok := t.handles[absPath]
	return fd, ok
}

func (t *Tailer) hasHandle(pathname string) bool {
	_, ok := t.handleForPath(pathname)
	return ok
}

// AddPattern adds a pattern to the list of patterns to filter filenames against.
func (t *Tailer) AddPattern(pattern string) error {
	absPath, err := filepath.Abs(pattern)
	if err != nil {
		glog.V(2).Infof("Couldn't canonicalize path %q: %s", pattern, err)
		return err
	}
	glog.V(2).Infof("AddPattern: %s", absPath)
	t.globPatternsMu.Lock()
	t.globPatterns[absPath] = struct{}{}
	t.globPatternsMu.Unlock()
	return nil
}

// TailPattern registers a pattern to be tailed.  If pattern is a plain
// file then it is watched for updates and opened.  If pattern is a glob, then
// all paths that match the glob are opened and watched, and the directories
// containing those matches, if any, are watched.
func (t *Tailer) TailPattern(pattern string) error {
	if err := t.AddPattern(pattern); err != nil {
		return err
	}
	// Add a watch on the containing directory, so we know when a rotation
	// occurs or something shows up that matches this pattern.
	if err := t.watchDirname(pattern); err != nil {
		return err
	}
	matches, err := filepath.Glob(pattern)
	if err != nil {
		return err
	}
	glog.V(1).Infof("glob matches: %v", matches)
	// Error if there are no matches, but if they show up later, they'll get picked up by the directory watch set above.
	if len(matches) == 0 {
		return errors.Errorf("No matches for pattern %q", pattern)
	}
	for _, pathname := range matches {
		err := t.TailPath(pathname)
		if err != nil {
			return errors.Wrapf(err, "attempting to tail %q", pathname)
		}
	}
	return nil
}

// TailPath registers a filesystem pathname to be tailed.
func (t *Tailer) TailPath(pathname string) error {
	if t.hasHandle(pathname) {
		glog.V(2).Infof("already watching %q", pathname)
		return nil
	}
	if err := t.w.Add(pathname, t.eventsHandle); err != nil {
		return err
	}
	// New file at start of program, seek to EOF.
	return t.openLogPath(pathname, false)
}

// handleLogEvent is dispatched when an Event is received, causing the tailer
// to read all available bytes from an already-opened file and send each log
// line onto lines channel.  Because we handle rotations and truncates when
// reaching EOF in the file reader itself, we don't care what the signal is
// from the filewatcher.
func (t *Tailer) handleLogEvent(pathname string) {
	glog.V(2).Infof("handleLogUpdate %s", pathname)
	fd, ok := t.handleForPath(pathname)
	if !ok {
		glog.V(1).Infof("No file handle found for %q, but is being watched", pathname)
		// We want to open files we have watches on in case the file was
		// unreadable before now; but we have to copmare against the glob to be
		// sure we don't just add all the files in a watched directory as they
		// get modified.
		t.handleCreateGlob(pathname)
		return
	}
	doFollow(fd)
}

// doFollow performs the Follow on an existing file descriptor, logging any errors
func doFollow(fd *File) {
	err := fd.Follow()
	if err != nil && err != io.EOF {
		glog.Info(err)
	}
}

// watchDirname adds the directory containing a path to be watched.
func (t *Tailer) watchDirname(pathname string) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return err
	}
	d := filepath.Dir(absPath)
	return t.w.Add(d, t.eventsHandle)
}

// openLogPath opens a log file named by pathname.
func (t *Tailer) openLogPath(pathname string, seekToStart bool) error {
	glog.V(2).Infof("openlogPath %s %v", pathname, seekToStart)
	if err := t.watchDirname(pathname); err != nil {
		return err
	}
	f, err := NewFile(pathname, t.lines, seekToStart || t.oneShot)
	if err != nil {
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			glog.V(1).Infof("pathname %q doesn't exist (yet?)", pathname)
			return nil
		}
		return err
	}
	glog.V(2).Infof("Adding a file watch on %q", f.Pathname)
	if err := t.w.Add(f.Pathname, t.eventsHandle); err != nil {
		return err
	}
	if err := t.setHandle(pathname, f); err != nil {
		return err
	}
	if err := f.Read(); err != nil && err != io.EOF {
		return err
	}
	glog.Infof("Tailing %s", f.Pathname)
	logCount.Add(1)
	return nil
}

// handleCreateGlob matches the pathname against the glob patterns and starts tailing the file.
func (t *Tailer) handleCreateGlob(pathname string) {
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()

	for pattern := range t.globPatterns {
		matched, err := filepath.Match(pattern, pathname)
		if err != nil {
			glog.Warningf("Unexpected bad pattern %q not detected earlier", pattern)
			continue
		}
		if !matched {
			glog.V(2).Infof("%q did not match pattern %q", pathname, pattern)
			continue
		}
		glog.V(1).Infof("New file %q matched existing glob %q", pathname, pattern)
		// If this file was just created, read from the start of the file.
		if err := t.openLogPath(pathname, true); err != nil {
			glog.Infof("Failed to tail new file %q: %s", pathname, err)
		}
		glog.V(2).Infof("started tailing %q", pathname)
		return
	}
	glog.V(2).Infof("did not start tailing %q", pathname)
}

// run the main event loop for the Tailer.  It receives notification of
// log file changes from the watcher channel, and dispatches the log event
// handler.
func (t *Tailer) run(events <-chan watcher.Event) {
	defer close(t.runDone)

	for e := range events {
		glog.V(2).Infof("Event type %#v", e)
		t.handleLogEvent(e.Pathname)
	}
	glog.Infof("Closing lines channel.")
	close(t.lines)
	glog.Infof("Shutting down tailer.")
}

// Close signals termination to the watcher.
func (t *Tailer) Close() error {
	if err := t.w.Close(); err != nil {
		return err
	}
	<-t.runDone
	return nil
}

const tailerTemplate = `

Log Tailer

Patterns

{{range $name, $val := $.Patterns}}

{{$name}}

{{end}}

Log files watched

pathname
errors
rotations
truncations
lines read

{{range $name, $val := $.Handles}}

{{$name}}

{{index $.Errors $name}}
{{index $.Rotations $name}}
{{index $.Truncs $name}}
{{index $.Lines $name}}

{{end}}

`

// WriteStatusHTML emits the Tailer's state in HTML format to the io.Writer w.
func (t *Tailer) WriteStatusHTML(w io.Writer) error {
	tpl, err := template.New("tailer").Parse(tailerTemplate)
	if err != nil {
		return err
	}
	t.handlesMu.RLock()
	defer t.handlesMu.RUnlock()
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()
	data := struct {
		Handles   map[string]*File
		Patterns  map[string]struct{}
		Rotations map[string]string
		Lines     map[string]string
		Errors    map[string]string
		Truncs    map[string]string
	}{
		t.handles,
		t.globPatterns,
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
	}
	for _, pair := range []struct {
		v *expvar.Map
		m map[string]string
	}{
		{logErrors, data.Errors},
		{logRotations, data.Rotations},
		{logTruncs, data.Truncs},
		{lineCount, data.Lines},
	} {
		pair.v.Do(func(kv expvar.KeyValue) {
			pair.m[kv.Key] = kv.Value.String()
		})
	}
	return tpl.Execute(w, data)
}

// Gc removes file handles that have had no reads for 24h or more.
func (t *Tailer) Gc() error {
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	for k, v := range t.handles {
		if time.Since(v.LastRead) > (time.Hour * 24) {
			if err := t.w.Remove(v.Pathname); err != nil {
				glog.Info(err)
			}
			if err := v.Close(); err != nil {
				glog.Info(err)
			}
			delete(t.handles, k)
		}
	}
	return nil
}

// StartExpiryLoop runs a permanent goroutine to expire metrics every duration.
func (t *Tailer) StartGcLoop(duration time.Duration) {
	if duration <= 0 {
		glog.Info("Log handle expiration disabled")
		return
	}
	go func() {
		glog.Infof("Starting log handle expiry loop every %s", duration.String())
		ticker := time.NewTicker(duration)
		for range ticker.C {
			if err := t.Gc(); err != nil {
				glog.Info(err)
			}
		}
	}()
}
<|endoftext|>


<commit_before>// Copyright 2017, OpenCensus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package stats

import "sync"

var (
	exportersMu sync.RWMutex // guards exporters
	exporters   = make(map[Exporter]struct{})
)

// Exporter exports the collected records as view data.
//
// The Export method should return quickly; if an
// Exporter takes a significant amount of time to
// process a ViewData, that work should be done on another goroutine.
//
// The ViewData should not be modified.
type Exporter interface {
	Export(viewData *ViewData)
}

// RegisterExporter registers an exporter.
// Collected data will be reported via all the
// registered exporters. Once you don't want data
// to be expoter on the registered exporter, use
// UnregisterExporter.
func RegisterExporter(e Exporter) {
	exportersMu.Lock()
	defer exportersMu.Unlock()

	exporters[e] = struct{}{}
}

// UnregisterExporter unregisters an exporter.
func UnregisterExporter(e Exporter) {
	exportersMu.Lock()
	defer exportersMu.Unlock()

	delete(exporters, e)
}
<commit_msg>Fix stats doc typo (#262)<commit_after>// Copyright 2017, OpenCensus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package stats

import "sync"

var (
	exportersMu sync.RWMutex // guards exporters
	exporters   = make(map[Exporter]struct{})
)

// Exporter exports the collected records as view data.
//
// The Export method should return quickly; if an
// Exporter takes a significant amount of time to
// process a ViewData, that work should be done on another goroutine.
//
// The ViewData should not be modified.
type Exporter interface {
	Export(viewData *ViewData)
}

// RegisterExporter registers an exporter.
// Collected data will be reported via all the
// registered exporters. Once you no longer
// want data to be exported, invoke UnregisterExporter
// with the previously registered exporter.
func RegisterExporter(e Exporter) {
	exportersMu.Lock()
	defer exportersMu.Unlock()

	exporters[e] = struct{}{}
}

// UnregisterExporter unregisters an exporter.
func UnregisterExporter(e Exporter) {
	exportersMu.Lock()
	defer exportersMu.Unlock()

	delete(exporters, e)
}
<|endoftext|>