<commit_before>// Copyright 2018 Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"bufio"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"unicode"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

var excepFileFlag string

var rootCmd = &cobra.Command{
	Use:   "pkgchk <dir>",
	Short: "Performs package validation tasks against all packages found under the specified directory.",
	Long: `This tool will perform various package validation checks against all of the packages
found under the specified directory.  Failures can be baselined and thus ignored by
copying the failure text verbatim, pasting it into a text file then specifying that
file via the optional exceptions flag.
`,
	Args: func(cmd *cobra.Command, args []string) error {
		if err := cobra.ExactArgs(1)(cmd, args); err != nil {
			return err
		}
		return nil
	},
	RunE: func(cmd *cobra.Command, args []string) error {
		cmd.SilenceUsage = true
		return theCommand(args)
	},
}

func init() {
	rootCmd.PersistentFlags().StringVarP(&excepFileFlag, "exceptions", "e", "", "text file containing the list of exceptions")
}

// Execute executes the specified command.
func Execute() {
	if err := rootCmd.Execute(); err != nil {
		os.Exit(1)
	}
}

func theCommand(args []string) error {
	rootDir := args[0]
	if !filepath.IsAbs(rootDir) {
		asAbs, err := filepath.Abs(rootDir)
		if err != nil {
			return errors.Wrap(err, "failed to get absolute path")
		}
		rootDir = asAbs
	}

	pkgs, err := getPkgs(rootDir)
	if err != nil {
		return errors.Wrap(err, "failed to get packages")
	}

	var exceptions []string
	if excepFileFlag != "" {
		exceptions, err = loadExceptions(excepFileFlag)
		if err != nil {
			return errors.Wrap(err, "failed to load exceptions")
		}
	}
	verifiers := getVerifiers()
	count := 0
	for _, pkg := range pkgs {
		for _, v := range verifiers {
			if err = v(pkg); err != nil && !contains(exceptions, err.Error()) {
				fmt.Fprintln(os.Stderr, err)
				count++
			}
		}
	}

	var res error
	if count > 0 {
		res = fmt.Errorf("found %d errors", count)
	}
	return res
}

func contains(items []string, item string) bool {
	if items == nil {
		return false
	}
	for _, i := range items {
		if i == item {
			return true
		}
	}
	return false
}

func loadExceptions(excepFile string) ([]string, error) {
	f, err := os.Open(excepFile)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	exceps := []string{}
	for scanner := bufio.NewScanner(f); scanner.Scan(); {
		exceps = append(exceps, scanner.Text())
	}
	return exceps, nil
}

type pkg struct {
	// the directory where the package resides relative to the root dir
	d string

	// the AST of the package
	p *ast.Package
}

// returns true if the package directory corresponds to an ARM package
func (p pkg) isARMPkg() bool {
	return strings.Index(p.d, "/mgmt/") > -1
}

// walks the directory hierarchy from the specified root returning a slice of all the packages found
func getPkgs(rootDir string) ([]pkg, error) {
	pkgs := []pkg{}
	err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			// check if leaf dir
			fi, err := ioutil.ReadDir(path)
			if err != nil {
				return err
			}
			hasSubDirs := false
			for _, f := range fi {
				if f.IsDir() {
					hasSubDirs = true
					break
				}
			}
			if !hasSubDirs {
				fs := token.NewFileSet()
				packages, err := parser.ParseDir(fs, path, nil, parser.PackageClauseOnly)
				if err != nil {
					return err
				}
				if len(packages) < 1 {
					return errors.New("didn't find any packages which is unexpected")
				}
				if len(packages) > 1 {
					return errors.New("found more than one package which is unexpected")
				}
				var p *ast.Package
				for _, pkgs := range packages {
					p = pkgs
				}
				// normalize directory separator to '/' character
				pkgs = append(pkgs, pkg{
					d: strings.Replace(path[len(rootDir):], "\\", "/", -1),
					p: p,
				})
			}
		}
		return nil
	})
	return pkgs, err
}

type verifier func(p pkg) error

// returns a list of verifiers to execute
func getVerifiers() []verifier {
	return []verifier{
		verifyPkgMatchesDir,
		verifyLowerCase,
		verifyDirectorySturcture,
	}
}

// ensures that the leaf directory name matches the package name
func verifyPkgMatchesDir(p pkg) error {
	leaf := p.d[strings.LastIndex(p.d, "/")+1:]
	if strings.Compare(leaf, p.p.Name) != 0 {
		return fmt.Errorf("leaf directory of '%s' doesn't match package name '%s'", p.d, p.p.Name)
	}
	return nil
}

// ensures that there are no upper-case letters in a package's directory
func verifyLowerCase(p pkg) error {
	// walk the package directory looking for upper-case characters
	for _, r := range p.d {
		if r == '/' {
			continue
		}
		if unicode.IsUpper(r) {
			return fmt.Errorf("found upper-case character in directory '%s'", p.d)
		}
	}
	return nil
}

// ensures that the package's directory hierarchy is properly formed
func verifyDirectorySturcture(p pkg) error {
	// for ARM the package directory structure is highly deterministic:
	// /redis/mgmt/2015-08-01/redis
	// /resources/mgmt/2017-06-01-preview/policy
	// /preview/signalr/mgmt/2018-03-01-preview/signalr
	if !p.isARMPkg() {
		return nil
	}
	regexStr := strings.Join([]string{
		`^(?:/preview)?`,
		`[a-z0-9\-]+`,
		`mgmt`,
		`\d{4}-\d{2}-\d{2}(?:-preview)?`,
		`[a-z0-9]+`,
	}, "/")
	regex := regexp.MustCompile(regexStr)
	if !regex.MatchString(p.d) {
		return fmt.Errorf("bad directory structure '%s'", p.d)
	}
	return nil
}
<commit_msg>fix scanner<commit_after>// Copyright 2018 Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"bufio"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"unicode"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

var excepFileFlag string

var rootCmd = &cobra.Command{
	Use:   "pkgchk <dir>",
	Short: "Performs package validation tasks against all packages found under the specified directory.",
	Long: `This tool will perform various package validation checks against all of the packages
found under the specified directory.  Failures can be baselined and thus ignored by
copying the failure text verbatim, pasting it into a text file then specifying that
file via the optional exceptions flag.
`,
	Args: func(cmd *cobra.Command, args []string) error {
		if err := cobra.ExactArgs(1)(cmd, args); err != nil {
			return err
		}
		return nil
	},
	RunE: func(cmd *cobra.Command, args []string) error {
		cmd.SilenceUsage = true
		return theCommand(args)
	},
}

func init() {
	rootCmd.PersistentFlags().StringVarP(&excepFileFlag, "exceptions", "e", "", "text file containing the list of exceptions")
}

// Execute executes the specified command.
func Execute() {
	if err := rootCmd.Execute(); err != nil {
		os.Exit(1)
	}
}

func theCommand(args []string) error {
	rootDir := args[0]
	if !filepath.IsAbs(rootDir) {
		asAbs, err := filepath.Abs(rootDir)
		if err != nil {
			return errors.Wrap(err, "failed to get absolute path")
		}
		rootDir = asAbs
	}

	pkgs, err := getPkgs(rootDir)
	if err != nil {
		return errors.Wrap(err, "failed to get packages")
	}

	var exceptions []string
	if excepFileFlag != "" {
		exceptions, err = loadExceptions(excepFileFlag)
		if err != nil {
			return errors.Wrap(err, "failed to load exceptions")
		}
	}
	verifiers := getVerifiers()
	count := 0
	for _, pkg := range pkgs {
		for _, v := range verifiers {
			if err = v(pkg); err != nil && !contains(exceptions, err.Error()) {
				fmt.Fprintln(os.Stderr, err)
				count++
			}
		}
	}

	var res error
	if count > 0 {
		res = fmt.Errorf("found %d errors", count)
	}
	return res
}

func contains(items []string, item string) bool {
	if items == nil {
		return false
	}
	for _, i := range items {
		if i == item {
			return true
		}
	}
	return false
}

func loadExceptions(excepFile string) ([]string, error) {
	f, err := os.Open(excepFile)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	exceps := []string{}

	scanner := bufio.NewScanner(f)
	for  scanner.Scan(); {
		exceps = append(exceps, scanner.Text())
	}
	if err = scanner.Err(); err != nil {
		return nil, err
	}

	return exceps, nil
}

type pkg struct {
	// the directory where the package resides relative to the root dir
	d string

	// the AST of the package
	p *ast.Package
}

// returns true if the package directory corresponds to an ARM package
func (p pkg) isARMPkg() bool {
	return strings.Index(p.d, "/mgmt/") > -1
}

// walks the directory hierarchy from the specified root returning a slice of all the packages found
func getPkgs(rootDir string) ([]pkg, error) {
	pkgs := []pkg{}
	err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			// check if leaf dir
			fi, err := ioutil.ReadDir(path)
			if err != nil {
				return err
			}
			hasSubDirs := false
			for _, f := range fi {
				if f.IsDir() {
					hasSubDirs = true
					break
				}
			}
			if !hasSubDirs {
				fs := token.NewFileSet()
				packages, err := parser.ParseDir(fs, path, nil, parser.PackageClauseOnly)
				if err != nil {
					return err
				}
				if len(packages) < 1 {
					return errors.New("didn't find any packages which is unexpected")
				}
				if len(packages) > 1 {
					return errors.New("found more than one package which is unexpected")
				}
				var p *ast.Package
				for _, pkgs := range packages {
					p = pkgs
				}
				// normalize directory separator to '/' character
				pkgs = append(pkgs, pkg{
					d: strings.Replace(path[len(rootDir):], "\\", "/", -1),
					p: p,
				})
			}
		}
		return nil
	})
	return pkgs, err
}

type verifier func(p pkg) error

// returns a list of verifiers to execute
func getVerifiers() []verifier {
	return []verifier{
		verifyPkgMatchesDir,
		verifyLowerCase,
		verifyDirectorySturcture,
	}
}

// ensures that the leaf directory name matches the package name
func verifyPkgMatchesDir(p pkg) error {
	leaf := p.d[strings.LastIndex(p.d, "/")+1:]
	if strings.Compare(leaf, p.p.Name) != 0 {
		return fmt.Errorf("leaf directory of '%s' doesn't match package name '%s'", p.d, p.p.Name)
	}
	return nil
}

// ensures that there are no upper-case letters in a package's directory
func verifyLowerCase(p pkg) error {
	// walk the package directory looking for upper-case characters
	for _, r := range p.d {
		if r == '/' {
			continue
		}
		if unicode.IsUpper(r) {
			return fmt.Errorf("found upper-case character in directory '%s'", p.d)
		}
	}
	return nil
}

// ensures that the package's directory hierarchy is properly formed
func verifyDirectorySturcture(p pkg) error {
	// for ARM the package directory structure is highly deterministic:
	// /redis/mgmt/2015-08-01/redis
	// /resources/mgmt/2017-06-01-preview/policy
	// /preview/signalr/mgmt/2018-03-01-preview/signalr
	if !p.isARMPkg() {
		return nil
	}
	regexStr := strings.Join([]string{
		`^(?:/preview)?`,
		`[a-z0-9\-]+`,
		`mgmt`,
		`\d{4}-\d{2}-\d{2}(?:-preview)?`,
		`[a-z0-9]+`,
	}, "/")
	regex := regexp.MustCompile(regexStr)
	if !regex.MatchString(p.d) {
		return fmt.Errorf("bad directory structure '%s'", p.d)
	}
	return nil
}
<|endoftext|>


<commit_before>package multiverse

import (
	"fmt"
	"runtime"
)

func (r Rarity) Ok(c *Card) (bool, error) {
	for _, printing := range c.Printings {
		if printing.Rarity == r {
			return true, nil
		}
	}
	return false, nil
}

func (m ManaColor) Ok(c *Card) (bool, error) {
	if m == ManaColors.Colorless {
		return c.Colors == 0, nil
	}
	return c.Colors&m != 0, nil
}

type Filter interface {
	Ok(*Card) (bool, error)
}

func (m Multiverse) Search(f Filter) ([]*Card, error) {
	c := m.Cards.List
	cores := runtime.GOMAXPROCS(-1)
	sectionLen := c.Len() / cores

	cardChan := make(chan *Card, cores*4)
	doneChan := make(chan bool)
	errChan := make(chan error)
	list := make([]*Card, 0, 1)

	for i := 0; i < cores; i++ {
		start := sectionLen * i
		end := start + sectionLen

		if i == cores-1 {
			end = c.Len()
		}

		go func(start, end int) {
			for _, card := range c[start:end] {
				ok, err := f.Ok(card.Card)
				if err != nil {
					errChan <- err
					return
				}
				if ok {
					cardChan <- card.Card
				}
			}
			doneChan <- true
		}(start, end)
	}

	for cores > 0 {
		select {
		case <-doneChan:
			cores--
		case c := <-cardChan:
			appendNonDuplicateToCardList(&list, c)
		case err := <-errChan:
			return nil, err
		}
	}

	for len(cardChan) > 0 {
		c := <-cardChan
		appendNonDuplicateToCardList(&list, c)

	}

	shortList := make([]*Card, len(list))

	for i, card := range list {
		shortList[i] = card
	}

	return shortList, nil
}

func appendToCardList(list *[]*Card, c *Card) {
	for _, card := range *list {
		if card == c {
			return
		}
	}
	appendNonDuplicateToCardList(list, c)
}

func appendNonDuplicateToCardList(list *[]*Card, c *Card) {
	l := len(*list)
	if l < cap(*list) {
		*list = (*list)[:l+1]
	} else {
		newList := make([]*Card, l+1, l*2)
		copy(newList, *list)
		*list = newList
	}
	(*list)[l] = c
}

type Not struct {
	Filter
}

func (n Not) Ok(c *Card) (bool, error) {
	ok, err := n.Filter.Ok(c)
	return !ok, err
}

type And []Filter

func (a And) Ok(c *Card) (bool, error) {
	for _, f := range a {
		ok, err := f.Ok(c)
		if !ok || err != nil {
			return ok, err
		}
	}
	return true, nil
}

type Or []Filter

func (o Or) Ok(c *Card) (bool, error) {
	for _, f := range o {
		ok, err := f.Ok(c)
		if ok || err != nil {
			return ok, err
		}
	}
	return false, nil
}

type Cond map[string]interface{}

func (c Cond) Ok(card *Card) (bool, error) {
	for key, val := range c {
		switch key {
		case "color", "colors":
			same, err := handleColorSearch(card.Colors, val)
			if err != nil || !same {
				return same, err
			}
		}
	}
	return true, nil
}

func handleColorSearch(cardColor ManaColor, val interface{}) (bool, error) {
	switch val := val.(type) {
	case ManaColor:
		if cardColor&val != 0 {
			return true, nil
		}
		return false, nil
	}

	return false, fmt.Errorf("unexpected color type %T", val)
}
<commit_msg>Increased buffer size to improve performance.<commit_after>package multiverse

import (
	"fmt"
	"runtime"
)

func (r Rarity) Ok(c *Card) (bool, error) {
	for _, printing := range c.Printings {
		if printing.Rarity == r {
			return true, nil
		}
	}
	return false, nil
}

func (m ManaColor) Ok(c *Card) (bool, error) {
	if m == ManaColors.Colorless {
		return c.Colors == 0, nil
	}
	return c.Colors&m != 0, nil
}

type Filter interface {
	Ok(*Card) (bool, error)
}

func (m Multiverse) Search(f Filter) ([]*Card, error) {
	c := m.Cards.List
	cores := runtime.GOMAXPROCS(-1)
	sectionLen := c.Len() / cores

	cardChan := make(chan *Card, cores*16)
	doneChan := make(chan bool)
	errChan := make(chan error)
	list := make([]*Card, 0, 1)

	for i := 0; i < cores; i++ {
		start := sectionLen * i
		end := start + sectionLen

		if i == cores-1 {
			end = c.Len()
		}

		go func(start, end int) {
			for _, card := range c[start:end] {
				ok, err := f.Ok(card.Card)
				if err != nil {
					errChan <- err
					return
				}
				if ok {
					cardChan <- card.Card
				}
			}
			doneChan <- true
		}(start, end)
	}

	for cores > 0 {
		select {
		case <-doneChan:
			cores--
		case c := <-cardChan:
			appendNonDuplicateToCardList(&list, c)
		case err := <-errChan:
			return nil, err
		}
	}

	for len(cardChan) > 0 {
		c := <-cardChan
		appendNonDuplicateToCardList(&list, c)

	}

	shortList := make([]*Card, len(list))

	for i, card := range list {
		shortList[i] = card
	}

	return shortList, nil
}

func appendToCardList(list *[]*Card, c *Card) {
	for _, card := range *list {
		if card == c {
			return
		}
	}
	appendNonDuplicateToCardList(list, c)
}

func appendNonDuplicateToCardList(list *[]*Card, c *Card) {
	l := len(*list)
	if l < cap(*list) {
		*list = (*list)[:l+1]
	} else {
		newList := make([]*Card, l+1, l*2)
		copy(newList, *list)
		*list = newList
	}
	(*list)[l] = c
}

type Not struct {
	Filter
}

func (n Not) Ok(c *Card) (bool, error) {
	ok, err := n.Filter.Ok(c)
	return !ok, err
}

type And []Filter

func (a And) Ok(c *Card) (bool, error) {
	for _, f := range a {
		ok, err := f.Ok(c)
		if !ok || err != nil {
			return ok, err
		}
	}
	return true, nil
}

type Or []Filter

func (o Or) Ok(c *Card) (bool, error) {
	for _, f := range o {
		ok, err := f.Ok(c)
		if ok || err != nil {
			return ok, err
		}
	}
	return false, nil
}

type Cond map[string]interface{}

func (c Cond) Ok(card *Card) (bool, error) {
	for key, val := range c {
		switch key {
		case "color", "colors":
			same, err := handleColorSearch(card.Colors, val)
			if err != nil || !same {
				return same, err
			}
		}
	}
	return true, nil
}

func handleColorSearch(cardColor ManaColor, val interface{}) (bool, error) {
	switch val := val.(type) {
	case ManaColor:
		if cardColor&val != 0 {
			return true, nil
		}
		return false, nil
	}

	return false, fmt.Errorf("unexpected color type %T", val)
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package mysqldef

import . "github.com/pingcap/check"

var _ = Suite(&testBitSuite{})

type testBitSuite struct {
}

func (s *testBitSuite) TestBit(c *C) {
	tbl := []struct {
		Input     string
		Width     int
		Number    int64
		String    string
		BitString string
	}{
		{"0b01", 8, 1, "0b00000001", "\x01"},
		{"0b111111111", 16, 511, "0b0000000111111111", "\x01\xff"},
		{"0b01", -1, 1, "0b00000001", "\x01"},
	}

	for _, t := range tbl {
		b, err := ParseBit(t.Input, t.Width)
		c.Assert(err, IsNil)
		c.Assert(err, IsNil)
		c.Assert(b.ToNumber(), Equals, float64(t.Number))
		c.Assert(b.String(), Equals, t.String)
		c.Assert(b.ToString(), Equals, t.BitString)
	}

	tblErr := []struct {
		Input string
		Width int
	}{
		{"0b11", 1},
		{"0B11", 2},
	}

	for _, t := range tblErr {
		_, err := ParseBit(t.Input, t.Width)
		c.Assert(err, NotNil)
	}
}
<commit_msg>mysqldef: Address comment<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package mysqldef

import . "github.com/pingcap/check"

var _ = Suite(&testBitSuite{})

type testBitSuite struct {
}

func (s *testBitSuite) TestBit(c *C) {
	tbl := []struct {
		Input     string
		Width     int
		Number    int64
		String    string
		BitString string
	}{
		{"0b01", 8, 1, "0b00000001", "\x01"},
		{"0b111111111", 16, 511, "0b0000000111111111", "\x01\xff"},
		{"0b01", -1, 1, "0b00000001", "\x01"},
	}

	for _, t := range tbl {
		b, err := ParseBit(t.Input, t.Width)
		c.Assert(err, IsNil)
		c.Assert(b.ToNumber(), Equals, float64(t.Number))
		c.Assert(b.String(), Equals, t.String)
		c.Assert(b.ToString(), Equals, t.BitString)
	}

	tblErr := []struct {
		Input string
		Width int
	}{
		{"0b11", 1},
		{"0B11", 2},
	}

	for _, t := range tblErr {
		_, err := ParseBit(t.Input, t.Width)
		c.Assert(err, NotNil)
	}
}
<|endoftext|>


<commit_before>package myutil

import (
	"encoding/json"
	"fmt"
	"reflect"
	"strings"
)

func DifferenceJSON(jsonOld string, jsonNew string) []byte {
	mapOld, mapNew := jsonToMap(jsonOld), jsonToMap(jsonNew)
	mapDiff := DifferenceMap(mapOld, mapNew)

	jsonDiff, _ := json.Marshal(mapDiff)

	return jsonDiff
}

func DifferenceMap(mapOld []map[string]string, mapNew []map[string]string) []map[string]string {
	mapDiff := make([]map[string]string, 0)
	for _, objectNew := range mapNew {
		for index, objectOld := range mapOld {
			if reflect.DeepEqual(objectNew, objectOld) {
				break
			}
			if index == len(mapNew)-1 {
				mapDiff = append(mapDiff, objectNew)
			}
		}
	}
	return mapDiff
}

func jsonToMap(jsonString string) []map[string]string {
	var articles []map[string]string
	err := json.NewDecoder(strings.NewReader(jsonString)).Decode(&articles)
	if err != nil {
		fmt.Println(err)
	}
	return articles
}
<commit_msg>fix bug<commit_after>package myutil

import (
	"bytes"
	"encoding/json"
	"fmt"
	"reflect"
)

func DifferenceJSON(jsonOld []byte, jsonNew []byte) []byte {
	mapOld, mapNew := jsonToMap(jsonOld), jsonToMap(jsonNew)
	mapDiff := DifferenceMap(mapOld, mapNew)

	jsonDiff, _ := json.Marshal(mapDiff)

	return jsonDiff
}

func DifferenceMap(mapOld []map[string]string, mapNew []map[string]string) []map[string]string {
	mapDiff := make([]map[string]string, 0)
	for _, objectNew := range mapNew {
		for index, objectOld := range mapOld {
			if reflect.DeepEqual(objectNew, objectOld) {
				break
			}
			if index == len(mapOld)-1 {
				mapDiff = append(mapDiff, objectNew)
			}
		}
	}
	return mapDiff
}

func jsonToMap(jsonString []byte) []map[string]string {
	var articles []map[string]string
	err := json.NewDecoder(bytes.NewReader(jsonString)).Decode(&articles)
	if err != nil {
		fmt.Println(err)
	}
	return articles
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Bram Gruneir (bram+code@cockroachlabs.com)

package storage

import (
	"container/heap"
	"sort"
	"sync"
	"time"

	"github.com/cockroachdb/cockroach/gossip"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/hlc"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/stop"
)

const (
	// TestTimeUntilStoreDead is the test value for TimeUntilStoreDead to
	// quickly mark stores as dead.
	TestTimeUntilStoreDead = 5 * time.Millisecond

	// TestTimeUntilStoreDeadOff is the test value for TimeUntilStoreDead that
	// prevents the store pool from marking stores as dead.
	TestTimeUntilStoreDeadOff = 24 * time.Hour
)

type storeDetail struct {
	desc            *roachpb.StoreDescriptor
	dead            bool
	timesDied       int
	foundDeadOn     roachpb.Timestamp
	lastUpdatedTime roachpb.Timestamp // This is also the priority for the queue.
	index           int               // index of the item in the heap, required for heap.Interface
}

// markDead sets the storeDetail to dead(inactive).
func (sd *storeDetail) markDead(foundDeadOn roachpb.Timestamp) {
	sd.dead = true
	sd.foundDeadOn = foundDeadOn
	sd.timesDied++
	log.Warningf("store %s on node %s is now considered offline", sd.desc.StoreID, sd.desc.Node.NodeID)
}

// markAlive sets the storeDetail to alive(active) and saves the updated time
// and descriptor.
func (sd *storeDetail) markAlive(foundAliveOn roachpb.Timestamp, storeDesc *roachpb.StoreDescriptor) {
	sd.desc = storeDesc
	sd.dead = false
	sd.lastUpdatedTime = foundAliveOn
}

// storePoolPQ implements the heap.Interface (which includes sort.Interface)
// and holds storeDetail. storePoolPQ is not threadsafe.
type storePoolPQ []*storeDetail

// Len implements the sort.Interface.
func (pq storePoolPQ) Len() int {
	return len(pq)
}

// Less implements the sort.Interface.
func (pq storePoolPQ) Less(i, j int) bool {
	return pq[i].lastUpdatedTime.Less(pq[j].lastUpdatedTime)
}

// Swap implements the sort.Interface.
func (pq storePoolPQ) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
	pq[i].index, pq[j].index = i, j
}

// Push implements the heap.Interface.
func (pq *storePoolPQ) Push(x interface{}) {
	n := len(*pq)
	item := x.(*storeDetail)
	item.index = n
	*pq = append(*pq, item)
}

// Pop implements the heap.Interface.
func (pq *storePoolPQ) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	item.index = -1 // for safety
	*pq = old[0 : n-1]
	return item
}

// peek returns the next value in the priority queue without dequeuing it.
func (pq storePoolPQ) peek() *storeDetail {
	if len(pq) == 0 {
		return nil
	}
	return (pq)[0]
}

// enqueue either adds the detail to the queue or updates its location in the
// priority queue.
func (pq *storePoolPQ) enqueue(detail *storeDetail) {
	if detail.index < 0 {
		heap.Push(pq, detail)
	} else {
		heap.Fix(pq, detail.index)
	}
}

// dequeue removes the next detail from the priority queue.
func (pq *storePoolPQ) dequeue() *storeDetail {
	if len(*pq) == 0 {
		return nil
	}
	return heap.Pop(pq).(*storeDetail)
}

// StorePool maintains a list of all known stores in the cluster and
// information on their health.
type StorePool struct {
	clock              *hlc.Clock
	timeUntilStoreDead time.Duration

	// Each storeDetail is contained in both a map and a priorityQueue; pointers
	// are used so that data can be kept in sync.
	mu     sync.RWMutex // Protects stores and queue.
	stores map[roachpb.StoreID]*storeDetail
	queue  storePoolPQ
}

// NewStorePool creates a StorePool and registers the store updating callback
// with gossip.
func NewStorePool(g *gossip.Gossip, clock *hlc.Clock, timeUntilStoreDead time.Duration, stopper *stop.Stopper) *StorePool {
	sp := &StorePool{
		clock:              clock,
		timeUntilStoreDead: timeUntilStoreDead,
		stores:             make(map[roachpb.StoreID]*storeDetail),
	}
	heap.Init(&sp.queue)

	storeRegex := gossip.MakePrefixPattern(gossip.KeyStorePrefix)
	g.RegisterCallback(storeRegex, sp.storeGossipUpdate)

	sp.start(stopper)

	return sp
}

// storeGossipUpdate is the gossip callback used to keep the StorePool up to date.
func (sp *StorePool) storeGossipUpdate(_ string, content roachpb.Value) {
	var storeDesc roachpb.StoreDescriptor
	if err := content.GetProto(&storeDesc); err != nil {
		log.Error(err)
		return
	}

	sp.mu.Lock()
	defer sp.mu.Unlock()
	// Does this storeDetail exist yet?
	detail, ok := sp.stores[storeDesc.StoreID]
	if !ok {
		// Setting index to -1 ensures this gets added to the queue.
		detail = &storeDetail{index: -1}
		sp.stores[storeDesc.StoreID] = detail
	}
	detail.markAlive(sp.clock.Now(), &storeDesc)
	sp.queue.enqueue(detail)
}

// start will run continuously and mark stores as offline if they haven't been
// heard from in longer than timeUntilStoreDead.
func (sp *StorePool) start(stopper *stop.Stopper) {
	stopper.RunWorker(func() {
		var timeoutTimer util.Timer
		defer timeoutTimer.Stop()
		for {
			var timeout time.Duration
			sp.mu.Lock()
			detail := sp.queue.peek()
			if detail == nil {
				// No stores yet, wait the full timeout.
				timeout = sp.timeUntilStoreDead
			} else {
				// Check to see if the store should be marked as dead.
				deadAsOf := detail.lastUpdatedTime.GoTime().Add(sp.timeUntilStoreDead)
				now := sp.clock.Now()
				if now.GoTime().After(deadAsOf) {
					deadDetail := sp.queue.dequeue()
					deadDetail.markDead(now)
					// The next store might be dead as well, set the timeout to
					// 0 to process it immediately.
					timeout = 0
				} else {
					// Store is still alive, schedule the next check for when
					// it should timeout.
					timeout = deadAsOf.Sub(now.GoTime())
				}
			}
			sp.mu.Unlock()
			timeoutTimer.Reset(timeout)
			select {
			case <-timeoutTimer.C:
				timeoutTimer.Read = true
			case <-stopper.ShouldStop():
				return
			}
		}
	})
}

// getStoreDetailLocked returns the store detail for the given storeID.
// The lock must be held *in write mode* even though this looks like a
// read-only method.
func (sp *StorePool) getStoreDetailLocked(storeID roachpb.StoreID) storeDetail {
	detail, ok := sp.stores[storeID]
	if !ok {
		// We don't have this store yet (this is normal when we're
		// starting up and don't have full information from the gossip
		// network). The first time this occurs, presume the store is
		// alive, but start the clock so it will become dead if enough
		// time passes without updates from gossip.
		detail = &storeDetail{index: -1}
		sp.stores[storeID] = detail
		detail.markAlive(sp.clock.Now(), nil)
		sp.queue.enqueue(detail)
	}

	return *detail
}

// getStoreDescriptor returns the latest store descriptor for the given
// storeID.
func (sp *StorePool) getStoreDescriptor(storeID roachpb.StoreID) *roachpb.StoreDescriptor {
	sp.mu.RLock()
	defer sp.mu.RUnlock()

	detail, ok := sp.stores[storeID]
	if !ok {
		return nil
	}

	return detail.desc
}

// deadReplicas returns any replicas from the supplied slice that are
// located on dead stores.
func (sp *StorePool) deadReplicas(repls []roachpb.ReplicaDescriptor) []roachpb.ReplicaDescriptor {
	sp.mu.Lock()
	defer sp.mu.Unlock()

	var deadReplicas []roachpb.ReplicaDescriptor
	for _, repl := range repls {
		if sp.getStoreDetailLocked(repl.StoreID).dead {
			deadReplicas = append(deadReplicas, repl)
		}
	}
	return deadReplicas
}

// stat provides a running sample size and mean.
type stat struct {
	n, mean float64
}

// Update adds the specified value to the stat, augmenting the sample
// size & mean.
func (s *stat) update(x float64) {
	s.n++
	s.mean += (x - s.mean) / s.n
}

// StoreList holds a list of store descriptors and associated count and used
// stats for those stores.
type StoreList struct {
	stores      []*roachpb.StoreDescriptor
	count, used stat
}

// add includes the store descriptor to the list of stores and updates
// maintained statistics.
func (sl *StoreList) add(s *roachpb.StoreDescriptor) {
	sl.stores = append(sl.stores, s)
	sl.count.update(float64(s.Capacity.RangeCount))
	sl.used.update(s.Capacity.FractionUsed())
}

// GetStoreList returns a storeList that contains all active stores that
// contain the required attributes and their associated stats. It also returns
// the number of total alive stores.
// TODO(embark, spencer): consider using a reverse index map from
// Attr->stores, for efficiency. Ensure that entries in this map still
// have an opportunity to be garbage collected.
func (sp *StorePool) getStoreList(required roachpb.Attributes, deterministic bool) (StoreList, int) {
	sp.mu.RLock()
	defer sp.mu.RUnlock()

	var storeIDs roachpb.StoreIDSlice
	for storeID := range sp.stores {
		storeIDs = append(storeIDs, storeID)
	}
	// Sort the stores by key if deterministic is requested. This is only for
	// unit testing.
	if deterministic {
		sort.Sort(storeIDs)
	}
	sl := StoreList{}
	var aliveStoreCount int
	for _, storeID := range storeIDs {
		detail := sp.stores[roachpb.StoreID(storeID)]
		if !detail.dead && detail.desc != nil {
			aliveStoreCount++
			if required.IsSubset(*detail.desc.CombinedAttrs()) {
				sl.add(detail.desc)
			}
		}
	}
	return sl, aliveStoreCount
}
<commit_msg>storage: Fix nil pointer in markDead on log<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Bram Gruneir (bram+code@cockroachlabs.com)

package storage

import (
	"container/heap"
	"sort"
	"sync"
	"time"

	"github.com/cockroachdb/cockroach/gossip"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/hlc"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/stop"
)

const (
	// TestTimeUntilStoreDead is the test value for TimeUntilStoreDead to
	// quickly mark stores as dead.
	TestTimeUntilStoreDead = 5 * time.Millisecond

	// TestTimeUntilStoreDeadOff is the test value for TimeUntilStoreDead that
	// prevents the store pool from marking stores as dead.
	TestTimeUntilStoreDeadOff = 24 * time.Hour
)

type storeDetail struct {
	desc            *roachpb.StoreDescriptor
	dead            bool
	timesDied       int
	foundDeadOn     roachpb.Timestamp
	lastUpdatedTime roachpb.Timestamp // This is also the priority for the queue.
	index           int               // index of the item in the heap, required for heap.Interface
}

// markDead sets the storeDetail to dead(inactive).
func (sd *storeDetail) markDead(foundDeadOn roachpb.Timestamp) {
	sd.dead = true
	sd.foundDeadOn = foundDeadOn
	sd.timesDied++
	if sd.desc != nil {
		// sd.desc can still be nil if it was markedAlive and enqueued in getStoreDetailLocked
		// and never markedAlive again.
		log.Warningf("store %s on node %s is now considered offline", sd.desc.StoreID, sd.desc.Node.NodeID)
	}
}

// markAlive sets the storeDetail to alive(active) and saves the updated time
// and descriptor.
func (sd *storeDetail) markAlive(foundAliveOn roachpb.Timestamp, storeDesc *roachpb.StoreDescriptor) {
	sd.desc = storeDesc
	sd.dead = false
	sd.lastUpdatedTime = foundAliveOn
}

// storePoolPQ implements the heap.Interface (which includes sort.Interface)
// and holds storeDetail. storePoolPQ is not threadsafe.
type storePoolPQ []*storeDetail

// Len implements the sort.Interface.
func (pq storePoolPQ) Len() int {
	return len(pq)
}

// Less implements the sort.Interface.
func (pq storePoolPQ) Less(i, j int) bool {
	return pq[i].lastUpdatedTime.Less(pq[j].lastUpdatedTime)
}

// Swap implements the sort.Interface.
func (pq storePoolPQ) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
	pq[i].index, pq[j].index = i, j
}

// Push implements the heap.Interface.
func (pq *storePoolPQ) Push(x interface{}) {
	n := len(*pq)
	item := x.(*storeDetail)
	item.index = n
	*pq = append(*pq, item)
}

// Pop implements the heap.Interface.
func (pq *storePoolPQ) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	item.index = -1 // for safety
	*pq = old[0 : n-1]
	return item
}

// peek returns the next value in the priority queue without dequeuing it.
func (pq storePoolPQ) peek() *storeDetail {
	if len(pq) == 0 {
		return nil
	}
	return (pq)[0]
}

// enqueue either adds the detail to the queue or updates its location in the
// priority queue.
func (pq *storePoolPQ) enqueue(detail *storeDetail) {
	if detail.index < 0 {
		heap.Push(pq, detail)
	} else {
		heap.Fix(pq, detail.index)
	}
}

// dequeue removes the next detail from the priority queue.
func (pq *storePoolPQ) dequeue() *storeDetail {
	if len(*pq) == 0 {
		return nil
	}
	return heap.Pop(pq).(*storeDetail)
}

// StorePool maintains a list of all known stores in the cluster and
// information on their health.
type StorePool struct {
	clock              *hlc.Clock
	timeUntilStoreDead time.Duration

	// Each storeDetail is contained in both a map and a priorityQueue; pointers
	// are used so that data can be kept in sync.
	mu     sync.RWMutex // Protects stores and queue.
	stores map[roachpb.StoreID]*storeDetail
	queue  storePoolPQ
}

// NewStorePool creates a StorePool and registers the store updating callback
// with gossip.
func NewStorePool(g *gossip.Gossip, clock *hlc.Clock, timeUntilStoreDead time.Duration, stopper *stop.Stopper) *StorePool {
	sp := &StorePool{
		clock:              clock,
		timeUntilStoreDead: timeUntilStoreDead,
		stores:             make(map[roachpb.StoreID]*storeDetail),
	}
	heap.Init(&sp.queue)

	storeRegex := gossip.MakePrefixPattern(gossip.KeyStorePrefix)
	g.RegisterCallback(storeRegex, sp.storeGossipUpdate)

	sp.start(stopper)

	return sp
}

// storeGossipUpdate is the gossip callback used to keep the StorePool up to date.
func (sp *StorePool) storeGossipUpdate(_ string, content roachpb.Value) {
	var storeDesc roachpb.StoreDescriptor
	if err := content.GetProto(&storeDesc); err != nil {
		log.Error(err)
		return
	}

	sp.mu.Lock()
	defer sp.mu.Unlock()
	// Does this storeDetail exist yet?
	detail, ok := sp.stores[storeDesc.StoreID]
	if !ok {
		// Setting index to -1 ensures this gets added to the queue.
		detail = &storeDetail{index: -1}
		sp.stores[storeDesc.StoreID] = detail
	}
	detail.markAlive(sp.clock.Now(), &storeDesc)
	sp.queue.enqueue(detail)
}

// start will run continuously and mark stores as offline if they haven't been
// heard from in longer than timeUntilStoreDead.
func (sp *StorePool) start(stopper *stop.Stopper) {
	stopper.RunWorker(func() {
		var timeoutTimer util.Timer
		defer timeoutTimer.Stop()
		for {
			var timeout time.Duration
			sp.mu.Lock()
			detail := sp.queue.peek()
			if detail == nil {
				// No stores yet, wait the full timeout.
				timeout = sp.timeUntilStoreDead
			} else {
				// Check to see if the store should be marked as dead.
				deadAsOf := detail.lastUpdatedTime.GoTime().Add(sp.timeUntilStoreDead)
				now := sp.clock.Now()
				if now.GoTime().After(deadAsOf) {
					deadDetail := sp.queue.dequeue()
					deadDetail.markDead(now)
					// The next store might be dead as well, set the timeout to
					// 0 to process it immediately.
					timeout = 0
				} else {
					// Store is still alive, schedule the next check for when
					// it should timeout.
					timeout = deadAsOf.Sub(now.GoTime())
				}
			}
			sp.mu.Unlock()
			timeoutTimer.Reset(timeout)
			select {
			case <-timeoutTimer.C:
				timeoutTimer.Read = true
			case <-stopper.ShouldStop():
				return
			}
		}
	})
}

// getStoreDetailLocked returns the store detail for the given storeID.
// The lock must be held *in write mode* even though this looks like a
// read-only method.
func (sp *StorePool) getStoreDetailLocked(storeID roachpb.StoreID) storeDetail {
	detail, ok := sp.stores[storeID]
	if !ok {
		// We don't have this store yet (this is normal when we're
		// starting up and don't have full information from the gossip
		// network). The first time this occurs, presume the store is
		// alive, but start the clock so it will become dead if enough
		// time passes without updates from gossip.
		detail = &storeDetail{index: -1}
		sp.stores[storeID] = detail
		detail.markAlive(sp.clock.Now(), nil)
		sp.queue.enqueue(detail)
	}

	return *detail
}

// getStoreDescriptor returns the latest store descriptor for the given
// storeID.
func (sp *StorePool) getStoreDescriptor(storeID roachpb.StoreID) *roachpb.StoreDescriptor {
	sp.mu.RLock()
	defer sp.mu.RUnlock()

	detail, ok := sp.stores[storeID]
	if !ok {
		return nil
	}

	return detail.desc
}

// deadReplicas returns any replicas from the supplied slice that are
// located on dead stores.
func (sp *StorePool) deadReplicas(repls []roachpb.ReplicaDescriptor) []roachpb.ReplicaDescriptor {
	sp.mu.Lock()
	defer sp.mu.Unlock()

	var deadReplicas []roachpb.ReplicaDescriptor
	for _, repl := range repls {
		if sp.getStoreDetailLocked(repl.StoreID).dead {
			deadReplicas = append(deadReplicas, repl)
		}
	}
	return deadReplicas
}

// stat provides a running sample size and mean.
type stat struct {
	n, mean float64
}

// Update adds the specified value to the stat, augmenting the sample
// size & mean.
func (s *stat) update(x float64) {
	s.n++
	s.mean += (x - s.mean) / s.n
}

// StoreList holds a list of store descriptors and associated count and used
// stats for those stores.
type StoreList struct {
	stores      []*roachpb.StoreDescriptor
	count, used stat
}

// add includes the store descriptor to the list of stores and updates
// maintained statistics.
func (sl *StoreList) add(s *roachpb.StoreDescriptor) {
	sl.stores = append(sl.stores, s)
	sl.count.update(float64(s.Capacity.RangeCount))
	sl.used.update(s.Capacity.FractionUsed())
}

// GetStoreList returns a storeList that contains all active stores that
// contain the required attributes and their associated stats. It also returns
// the number of total alive stores.
// TODO(embark, spencer): consider using a reverse index map from
// Attr->stores, for efficiency. Ensure that entries in this map still
// have an opportunity to be garbage collected.
func (sp *StorePool) getStoreList(required roachpb.Attributes, deterministic bool) (StoreList, int) {
	sp.mu.RLock()
	defer sp.mu.RUnlock()

	var storeIDs roachpb.StoreIDSlice
	for storeID := range sp.stores {
		storeIDs = append(storeIDs, storeID)
	}
	// Sort the stores by key if deterministic is requested. This is only for
	// unit testing.
	if deterministic {
		sort.Sort(storeIDs)
	}
	sl := StoreList{}
	var aliveStoreCount int
	for _, storeID := range storeIDs {
		detail := sp.stores[roachpb.StoreID(storeID)]
		if !detail.dead && detail.desc != nil {
			aliveStoreCount++
			if required.IsSubset(*detail.desc.CombinedAttrs()) {
				sl.add(detail.desc)
			}
		}
	}
	return sl, aliveStoreCount
}
<|endoftext|>


<commit_before>// Mgmt
// Copyright (C) 2013-2016+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

// Package recwatch provides recursive file watching events via fsnotify.
package recwatch

import (
	"fmt"
	"log"
	"math"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"syscall"

	"github.com/purpleidea/mgmt/global" // XXX: package mgmtmain instead?
	"github.com/purpleidea/mgmt/util"

	"gopkg.in/fsnotify.v1"
	//"github.com/go-fsnotify/fsnotify" // git master of "gopkg.in/fsnotify.v1"
)

// Event represents a watcher event. These can include errors.
type Event struct {
	Error error
	Body  *fsnotify.Event
}

// RecWatcher is the struct for the recursive watcher. Run Init() on it.
type RecWatcher struct {
	Path     string // computed path
	Recurse  bool   // should we watch recursively?
	isDir    bool   // computed isDir
	safename string // safe path
	watcher  *fsnotify.Watcher
	watches  map[string]struct{}
	events   chan Event // one channel for events and err...
	once     sync.Once
	wg       sync.WaitGroup
	exit     chan struct{}
	closeErr error
}

// NewRecWatcher creates an initializes a new recursive watcher.
func NewRecWatcher(path string, recurse bool) (*RecWatcher, error) {
	obj := &RecWatcher{
		Path:    path,
		Recurse: recurse,
	}
	return obj, obj.Init()
}

// Init starts the recursive file watcher.
func (obj *RecWatcher) Init() error {
	obj.watcher = nil
	obj.watches = make(map[string]struct{})
	obj.events = make(chan Event)
	obj.exit = make(chan struct{})
	obj.isDir = strings.HasSuffix(obj.Path, "/") // dirs have trailing slashes
	obj.safename = path.Clean(obj.Path)          // no trailing slash

	var err error
	obj.watcher, err = fsnotify.NewWatcher()
	if err != nil {
		return err
	}

	if obj.isDir {
		if err := obj.addSubFolders(obj.safename); err != nil {
			return err
		}
	}

	go func() {
		if err := obj.Watch(); err != nil {
			obj.events <- Event{Error: err}
		}
		obj.Close()
	}()
	return nil
}

//func (obj *RecWatcher) Add(path string) error { // XXX implement me or not?
//
//}
//
//func (obj *RecWatcher) Remove(path string) error { // XXX implement me or not?
//
//}

// Close shuts down the watcher.
func (obj *RecWatcher) Close() error {
	obj.once.Do(obj.close) // don't cause the channel to close twice
	return obj.closeErr
}

// This close function is the function that actually does the close work. Don't
// call it more than once!
func (obj *RecWatcher) close() {
	var err error
	close(obj.exit) // send exit signal
	obj.wg.Wait()
	if obj.watcher != nil {
		err = obj.watcher.Close()
		obj.watcher = nil
		// TODO: should we send the close error?
		//if err != nil {
		//	obj.events <- Event{Error: err}
		//}
	}
	close(obj.events)
	obj.closeErr = err // set the error
}

// Events returns a channel of events. These include events for errors.
func (obj *RecWatcher) Events() chan Event { return obj.events }

// Watch is the primary listener for this resource and it outputs events.
func (obj *RecWatcher) Watch() error {
	if obj.watcher == nil {
		return fmt.Errorf("Watcher is not initialized!")
	}
	obj.wg.Add(1)
	defer obj.wg.Done()

	patharray := util.PathSplit(obj.safename) // tokenize the path
	var index = len(patharray)                // starting index
	var current string                        // current "watcher" location
	var deltaDepth int                        // depth delta between watcher and event
	var send = false                          // send event?

	for {
		current = strings.Join(patharray[0:index], "/")
		if current == "" { // the empty string top is the root dir ("/")
			current = "/"
		}
		if global.DEBUG {
			log.Printf("Watching: %s", current) // attempting to watch...
		}
		// initialize in the loop so that we can reset on rm-ed handles
		if err := obj.watcher.Add(current); err != nil {
			if global.DEBUG {
				log.Printf("watcher.Add(%s): Error: %v", current, err)
			}
			if err == syscall.ENOENT {
				index-- // usually not found, move up one dir
			} else if err == syscall.ENOSPC {
				// no space left on device, out of inotify watches
				// TODO: consider letting the user fall back to
				// polling if they hit this error very often...
				return fmt.Errorf("Out of inotify watches: %v", err)
			} else if os.IsPermission(err) {
				return fmt.Errorf("Permission denied adding a watch: %v", err)
			} else {
				return fmt.Errorf("Unknown error: %v", err)
			}
			index = int(math.Max(1, float64(index)))
			continue
		}

		select {
		case event := <-obj.watcher.Events:
			if global.DEBUG {
				log.Printf("Watch(%s), Event(%s): %v", current, event.Name, event.Op)
			}
			// the deeper you go, the bigger the deltaDepth is...
			// this is the difference between what we're watching,
			// and the event... doesn't mean we can't watch deeper
			if current == event.Name {
				deltaDepth = 0 // i was watching what i was looking for

			} else if util.HasPathPrefix(event.Name, current) {
				deltaDepth = len(util.PathSplit(current)) - len(util.PathSplit(event.Name)) // -1 or less

			} else if util.HasPathPrefix(current, event.Name) {
				deltaDepth = len(util.PathSplit(event.Name)) - len(util.PathSplit(current)) // +1 or more
				// if below me...
				if _, exists := obj.watches[event.Name]; exists {
					send = true
					if event.Op&fsnotify.Remove == fsnotify.Remove {
						obj.watcher.Remove(event.Name)
						delete(obj.watches, event.Name)
					}
					if (event.Op&fsnotify.Create == fsnotify.Create) && isDir(event.Name) {
						obj.watcher.Add(event.Name)
						obj.watches[event.Name] = struct{}{}
						if err := obj.addSubFolders(event.Name); err != nil {
							return err
						}
					}
				}

			} else {
				// TODO different watchers get each others events!
				// https://github.com/go-fsnotify/fsnotify/issues/95
				// this happened with two values such as:
				// event.Name: /tmp/mgmt/f3 and current: /tmp/mgmt/f2
				continue
			}
			//log.Printf("The delta depth is: %v", deltaDepth)

			// if we have what we wanted, awesome, send an event...
			if event.Name == obj.safename {
				//log.Println("Event!")
				// FIXME: should all these below cases trigger?
				send = true

				if obj.isDir {
					if err := obj.addSubFolders(obj.safename); err != nil {
						return err
					}
				}

				// file removed, move the watch upwards
				if deltaDepth >= 0 && (event.Op&fsnotify.Remove == fsnotify.Remove) {
					//log.Println("Removal!")
					obj.watcher.Remove(current)
					index--
				}

				// we must be a parent watcher, so descend in
				if deltaDepth < 0 {
					// XXX: we can block here due to: https://github.com/fsnotify/fsnotify/issues/123
					obj.watcher.Remove(current)
					index++
				}

				// if safename starts with event.Name, we're above, and no event should be sent
			} else if util.HasPathPrefix(obj.safename, event.Name) {
				//log.Println("Above!")

				if deltaDepth >= 0 && (event.Op&fsnotify.Remove == fsnotify.Remove) {
					log.Println("Removal!")
					obj.watcher.Remove(current)
					index--
				}

				if deltaDepth < 0 {
					log.Println("Parent!")
					if util.PathPrefixDelta(obj.safename, event.Name) == 1 { // we're the parent dir
						send = true
					}
					obj.watcher.Remove(current)
					index++
				}

				// if event.Name startswith safename, send event, we're already deeper
			} else if util.HasPathPrefix(event.Name, obj.safename) {
				//log.Println("Event2!")
				send = true
			}

			// do all our event sending all together to avoid duplicate msgs
			if send {
				send = false
				// only invalid state on certain types of events
				obj.events <- Event{Error: nil, Body: &event}
			}

		case err := <-obj.watcher.Errors:
			return fmt.Errorf("Unknown watcher error: %v", err)

		case <-obj.exit:
			return nil
		}
	}
}

// addSubFolders is a helper that is used to add recursive dirs to the watches.
func (obj *RecWatcher) addSubFolders(p string) error {
	if !obj.Recurse {
		return nil // if we're not watching recursively, just exit early
	}
	// look at all subfolders...
	walkFn := func(path string, info os.FileInfo, err error) error {
		if global.DEBUG {
			log.Printf("Walk: %s (%v): %v", path, info, err)
		}
		if err != nil {
			return nil
		}
		if info.IsDir() {
			obj.watches[path] = struct{}{} // add key
			err := obj.watcher.Add(path)
			if err != nil {
				return err // TODO: will this bubble up?
			}
		}
		return nil
	}
	err := filepath.Walk(p, walkFn)
	return err
}

func isDir(path string) bool {
	finfo, err := os.Stat(path)
	if err != nil {
		return false
	}
	return finfo.IsDir()
}
<commit_msg>recwatch: Rearrange condition to unconfuse golinter<commit_after>// Mgmt
// Copyright (C) 2013-2016+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

// Package recwatch provides recursive file watching events via fsnotify.
package recwatch

import (
	"fmt"
	"log"
	"math"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"syscall"

	"github.com/purpleidea/mgmt/global" // XXX: package mgmtmain instead?
	"github.com/purpleidea/mgmt/util"

	"gopkg.in/fsnotify.v1"
	//"github.com/go-fsnotify/fsnotify" // git master of "gopkg.in/fsnotify.v1"
)

// Event represents a watcher event. These can include errors.
type Event struct {
	Error error
	Body  *fsnotify.Event
}

// RecWatcher is the struct for the recursive watcher. Run Init() on it.
type RecWatcher struct {
	Path     string // computed path
	Recurse  bool   // should we watch recursively?
	isDir    bool   // computed isDir
	safename string // safe path
	watcher  *fsnotify.Watcher
	watches  map[string]struct{}
	events   chan Event // one channel for events and err...
	once     sync.Once
	wg       sync.WaitGroup
	exit     chan struct{}
	closeErr error
}

// NewRecWatcher creates an initializes a new recursive watcher.
func NewRecWatcher(path string, recurse bool) (*RecWatcher, error) {
	obj := &RecWatcher{
		Path:    path,
		Recurse: recurse,
	}
	return obj, obj.Init()
}

// Init starts the recursive file watcher.
func (obj *RecWatcher) Init() error {
	obj.watcher = nil
	obj.watches = make(map[string]struct{})
	obj.events = make(chan Event)
	obj.exit = make(chan struct{})
	obj.isDir = strings.HasSuffix(obj.Path, "/") // dirs have trailing slashes
	obj.safename = path.Clean(obj.Path)          // no trailing slash

	var err error
	obj.watcher, err = fsnotify.NewWatcher()
	if err != nil {
		return err
	}

	if obj.isDir {
		if err := obj.addSubFolders(obj.safename); err != nil {
			return err
		}
	}

	go func() {
		if err := obj.Watch(); err != nil {
			obj.events <- Event{Error: err}
		}
		obj.Close()
	}()
	return nil
}

//func (obj *RecWatcher) Add(path string) error { // XXX implement me or not?
//
//}
//
//func (obj *RecWatcher) Remove(path string) error { // XXX implement me or not?
//
//}

// Close shuts down the watcher.
func (obj *RecWatcher) Close() error {
	obj.once.Do(obj.close) // don't cause the channel to close twice
	return obj.closeErr
}

// This close function is the function that actually does the close work. Don't
// call it more than once!
func (obj *RecWatcher) close() {
	var err error
	close(obj.exit) // send exit signal
	obj.wg.Wait()
	if obj.watcher != nil {
		err = obj.watcher.Close()
		obj.watcher = nil
		// TODO: should we send the close error?
		//if err != nil {
		//	obj.events <- Event{Error: err}
		//}
	}
	close(obj.events)
	obj.closeErr = err // set the error
}

// Events returns a channel of events. These include events for errors.
func (obj *RecWatcher) Events() chan Event { return obj.events }

// Watch is the primary listener for this resource and it outputs events.
func (obj *RecWatcher) Watch() error {
	if obj.watcher == nil {
		return fmt.Errorf("Watcher is not initialized!")
	}
	obj.wg.Add(1)
	defer obj.wg.Done()

	patharray := util.PathSplit(obj.safename) // tokenize the path
	var index = len(patharray)                // starting index
	var current string                        // current "watcher" location
	var deltaDepth int                        // depth delta between watcher and event
	var send = false                          // send event?

	for {
		current = strings.Join(patharray[0:index], "/")
		if current == "" { // the empty string top is the root dir ("/")
			current = "/"
		}
		if global.DEBUG {
			log.Printf("Watching: %s", current) // attempting to watch...
		}
		// initialize in the loop so that we can reset on rm-ed handles
		if err := obj.watcher.Add(current); err != nil {
			if global.DEBUG {
				log.Printf("watcher.Add(%s): Error: %v", current, err)
			}

			if err == syscall.ENOENT {
				index-- // usually not found, move up one dir
				index = int(math.Max(1, float64(index)))
				continue
			}

			if err == syscall.ENOSPC {
				// no space left on device, out of inotify watches
				// TODO: consider letting the user fall back to
				// polling if they hit this error very often...
				return fmt.Errorf("Out of inotify watches: %v", err)
			} else if os.IsPermission(err) {
				return fmt.Errorf("Permission denied adding a watch: %v", err)
			}
			return fmt.Errorf("Unknown error: %v", err)
		}

		select {
		case event := <-obj.watcher.Events:
			if global.DEBUG {
				log.Printf("Watch(%s), Event(%s): %v", current, event.Name, event.Op)
			}
			// the deeper you go, the bigger the deltaDepth is...
			// this is the difference between what we're watching,
			// and the event... doesn't mean we can't watch deeper
			if current == event.Name {
				deltaDepth = 0 // i was watching what i was looking for

			} else if util.HasPathPrefix(event.Name, current) {
				deltaDepth = len(util.PathSplit(current)) - len(util.PathSplit(event.Name)) // -1 or less

			} else if util.HasPathPrefix(current, event.Name) {
				deltaDepth = len(util.PathSplit(event.Name)) - len(util.PathSplit(current)) // +1 or more
				// if below me...
				if _, exists := obj.watches[event.Name]; exists {
					send = true
					if event.Op&fsnotify.Remove == fsnotify.Remove {
						obj.watcher.Remove(event.Name)
						delete(obj.watches, event.Name)
					}
					if (event.Op&fsnotify.Create == fsnotify.Create) && isDir(event.Name) {
						obj.watcher.Add(event.Name)
						obj.watches[event.Name] = struct{}{}
						if err := obj.addSubFolders(event.Name); err != nil {
							return err
						}
					}
				}

			} else {
				// TODO different watchers get each others events!
				// https://github.com/go-fsnotify/fsnotify/issues/95
				// this happened with two values such as:
				// event.Name: /tmp/mgmt/f3 and current: /tmp/mgmt/f2
				continue
			}
			//log.Printf("The delta depth is: %v", deltaDepth)

			// if we have what we wanted, awesome, send an event...
			if event.Name == obj.safename {
				//log.Println("Event!")
				// FIXME: should all these below cases trigger?
				send = true

				if obj.isDir {
					if err := obj.addSubFolders(obj.safename); err != nil {
						return err
					}
				}

				// file removed, move the watch upwards
				if deltaDepth >= 0 && (event.Op&fsnotify.Remove == fsnotify.Remove) {
					//log.Println("Removal!")
					obj.watcher.Remove(current)
					index--
				}

				// we must be a parent watcher, so descend in
				if deltaDepth < 0 {
					// XXX: we can block here due to: https://github.com/fsnotify/fsnotify/issues/123
					obj.watcher.Remove(current)
					index++
				}

				// if safename starts with event.Name, we're above, and no event should be sent
			} else if util.HasPathPrefix(obj.safename, event.Name) {
				//log.Println("Above!")

				if deltaDepth >= 0 && (event.Op&fsnotify.Remove == fsnotify.Remove) {
					log.Println("Removal!")
					obj.watcher.Remove(current)
					index--
				}

				if deltaDepth < 0 {
					log.Println("Parent!")
					if util.PathPrefixDelta(obj.safename, event.Name) == 1 { // we're the parent dir
						send = true
					}
					obj.watcher.Remove(current)
					index++
				}

				// if event.Name startswith safename, send event, we're already deeper
			} else if util.HasPathPrefix(event.Name, obj.safename) {
				//log.Println("Event2!")
				send = true
			}

			// do all our event sending all together to avoid duplicate msgs
			if send {
				send = false
				// only invalid state on certain types of events
				obj.events <- Event{Error: nil, Body: &event}
			}

		case err := <-obj.watcher.Errors:
			return fmt.Errorf("Unknown watcher error: %v", err)

		case <-obj.exit:
			return nil
		}
	}
}

// addSubFolders is a helper that is used to add recursive dirs to the watches.
func (obj *RecWatcher) addSubFolders(p string) error {
	if !obj.Recurse {
		return nil // if we're not watching recursively, just exit early
	}
	// look at all subfolders...
	walkFn := func(path string, info os.FileInfo, err error) error {
		if global.DEBUG {
			log.Printf("Walk: %s (%v): %v", path, info, err)
		}
		if err != nil {
			return nil
		}
		if info.IsDir() {
			obj.watches[path] = struct{}{} // add key
			err := obj.watcher.Add(path)
			if err != nil {
				return err // TODO: will this bubble up?
			}
		}
		return nil
	}
	err := filepath.Walk(p, walkFn)
	return err
}

func isDir(path string) bool {
	finfo, err := os.Stat(path)
	if err != nil {
		return false
	}
	return finfo.IsDir()
}
<|endoftext|>


<commit_before>// +build !binary_log

package zerolog_test

import (
	"errors"
	"fmt"
	stdlog "log"
	"net"
	"os"
	"time"

	"github.com/rs/zerolog"
)

func ExampleNew() {
	log := zerolog.New(os.Stdout)

	log.Info().Msg("hello world")
	// Output: {"level":"info","message":"hello world"}
}

func ExampleLogger_With() {
	log := zerolog.New(os.Stdout).
		With().
		Str("foo", "bar").
		Logger()

	log.Info().Msg("hello world")

	// Output: {"level":"info","foo":"bar","message":"hello world"}
}

func ExampleLogger_Level() {
	log := zerolog.New(os.Stdout).Level(zerolog.WarnLevel)

	log.Info().Msg("filtered out message")
	log.Error().Msg("kept message")

	// Output: {"level":"error","message":"kept message"}
}

func ExampleLogger_Sample() {
	log := zerolog.New(os.Stdout).Sample(&zerolog.BasicSampler{N: 2})

	log.Info().Msg("message 1")
	log.Info().Msg("message 2")
	log.Info().Msg("message 3")
	log.Info().Msg("message 4")

	// Output: {"level":"info","message":"message 2"}
	// {"level":"info","message":"message 4"}
}

type LevelNameHook struct{}

func (h LevelNameHook) Run(e *zerolog.Event, l zerolog.Level, msg string) {
	if l != zerolog.NoLevel {
		e.Str("level_name", l.String())
	} else {
		e.Str("level_name", "NoLevel")
	}
}

type MessageHook string

func (h MessageHook) Run(e *zerolog.Event, l zerolog.Level, msg string) {
	e.Str("the_message", msg)
}

func ExampleLogger_Hook() {
	var levelNameHook LevelNameHook
	var messageHook MessageHook = "The message"

	log := zerolog.New(os.Stdout).Hook(levelNameHook).Hook(messageHook)

	log.Info().Msg("hello world")

	// Output: {"level":"info","level_name":"info","the_message":"hello world","message":"hello world"}
}

func ExampleLogger_Print() {
	log := zerolog.New(os.Stdout)

	log.Print("hello world")

	// Output: {"level":"debug","message":"hello world"}
}

func ExampleLogger_Printf() {
	log := zerolog.New(os.Stdout)

	log.Printf("hello %s", "world")

	// Output: {"level":"debug","message":"hello world"}
}

func ExampleLogger_Debug() {
	log := zerolog.New(os.Stdout)

	log.Debug().
		Str("foo", "bar").
		Int("n", 123).
		Msg("hello world")

	// Output: {"level":"debug","foo":"bar","n":123,"message":"hello world"}
}

func ExampleLogger_Info() {
	log := zerolog.New(os.Stdout)

	log.Info().
		Str("foo", "bar").
		Int("n", 123).
		Msg("hello world")

	// Output: {"level":"info","foo":"bar","n":123,"message":"hello world"}
}

func ExampleLogger_Warn() {
	log := zerolog.New(os.Stdout)

	log.Warn().
		Str("foo", "bar").
		Msg("a warning message")

	// Output: {"level":"warn","foo":"bar","message":"a warning message"}
}

func ExampleLogger_Error() {
	log := zerolog.New(os.Stdout)

	log.Error().
		Err(errors.New("some error")).
		Msg("error doing something")

	// Output: {"level":"error","error":"some error","message":"error doing something"}
}

func ExampleLogger_WithLevel() {
	log := zerolog.New(os.Stdout)

	log.WithLevel(zerolog.InfoLevel).
		Msg("hello world")

	// Output: {"level":"info","message":"hello world"}
}

func ExampleLogger_Write() {
	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Logger()

	stdlog.SetFlags(0)
	stdlog.SetOutput(log)

	stdlog.Print("hello world")

	// Output: {"foo":"bar","message":"hello world"}
}

func ExampleLogger_Log() {
	log := zerolog.New(os.Stdout)

	log.Log().
		Str("foo", "bar").
		Str("bar", "baz").
		Msg("")

	// Output: {"foo":"bar","bar":"baz"}
}

func ExampleEvent_Dict() {
	log := zerolog.New(os.Stdout)

	log.Log().
		Str("foo", "bar").
		Dict("dict", zerolog.Dict().
			Str("bar", "baz").
			Int("n", 1),
		).
		Msg("hello world")

	// Output: {"foo":"bar","dict":{"bar":"baz","n":1},"message":"hello world"}
}

type User struct {
	Name    string
	Age     int
	Created time.Time
}

func (u User) MarshalZerologObject(e *zerolog.Event) {
	e.Str("name", u.Name).
		Int("age", u.Age).
		Time("created", u.Created)
}

type Price struct {
	val  uint64
	prec int
	unit string
}

func (p Price) MarshalZerologObject(e *zerolog.Event) {
	denom := uint64(1)
	for i := 0; i < p.prec; i++ {
		denom *= 10
	}
	result := []byte(p.unit)
	result = append(result, fmt.Sprintf("%d.%d", p.val/denom, p.val%denom)...)
	e.Str("price", string(result))
}

type Users []User

func (uu Users) MarshalZerologArray(a *zerolog.Array) {
	for _, u := range uu {
		a.Object(u)
	}
}

func ExampleEvent_Array() {
	log := zerolog.New(os.Stdout)

	log.Log().
		Str("foo", "bar").
		Array("array", zerolog.Arr().
			Str("baz").
			Int(1),
		).
		Msg("hello world")

	// Output: {"foo":"bar","array":["baz",1],"message":"hello world"}
}

func ExampleEvent_Array_object() {
	log := zerolog.New(os.Stdout)

	// Users implements zerolog.LogArrayMarshaler
	u := Users{
		User{"John", 35, time.Time{}},
		User{"Bob", 55, time.Time{}},
	}

	log.Log().
		Str("foo", "bar").
		Array("users", u).
		Msg("hello world")

	// Output: {"foo":"bar","users":[{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},{"name":"Bob","age":55,"created":"0001-01-01T00:00:00Z"}],"message":"hello world"}
}

func ExampleEvent_Object() {
	log := zerolog.New(os.Stdout)

	// User implements zerolog.LogObjectMarshaler
	u := User{"John", 35, time.Time{}}

	log.Log().
		Str("foo", "bar").
		Object("user", u).
		Msg("hello world")

	// Output: {"foo":"bar","user":{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},"message":"hello world"}
}

func ExampleEvent_EmbedObject() {
	log := zerolog.New(os.Stdout)

	price := Price{val: 6449, prec: 2, unit: "$"}

	log.Log().
		Str("foo", "bar").
		EmbedObject(price).
		Msg("hello world")

	// Output: {"foo":"bar","price":"$64.49","message":"hello world"}
}

func ExampleEvent_Interface() {
	log := zerolog.New(os.Stdout)

	obj := struct {
		Name string `json:"name"`
	}{
		Name: "john",
	}

	log.Log().
		Str("foo", "bar").
		Interface("obj", obj).
		Msg("hello world")

	// Output: {"foo":"bar","obj":{"name":"john"},"message":"hello world"}
}

func ExampleEvent_Dur() {
	d := time.Duration(10 * time.Second)

	log := zerolog.New(os.Stdout)

	log.Log().
		Str("foo", "bar").
		Dur("dur", d).
		Msg("hello world")

	// Output: {"foo":"bar","dur":10000,"message":"hello world"}
}

func ExampleEvent_Durs() {
	d := []time.Duration{
		time.Duration(10 * time.Second),
		time.Duration(20 * time.Second),
	}

	log := zerolog.New(os.Stdout)

	log.Log().
		Str("foo", "bar").
		Durs("durs", d).
		Msg("hello world")

	// Output: {"foo":"bar","durs":[10000,20000],"message":"hello world"}
}

func ExampleContext_Dict() {
	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Dict("dict", zerolog.Dict().
			Str("bar", "baz").
			Int("n", 1),
		).Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","dict":{"bar":"baz","n":1},"message":"hello world"}
}

func ExampleContext_Array() {
	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Array("array", zerolog.Arr().
			Str("baz").
			Int(1),
		).Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","array":["baz",1],"message":"hello world"}
}

func ExampleContext_Array_object() {
	// Users implements zerolog.LogArrayMarshaler
	u := Users{
		User{"John", 35, time.Time{}},
		User{"Bob", 55, time.Time{}},
	}

	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Array("users", u).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","users":[{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},{"name":"Bob","age":55,"created":"0001-01-01T00:00:00Z"}],"message":"hello world"}
}

func ExampleContext_Object() {
	// User implements zerolog.LogObjectMarshaler
	u := User{"John", 35, time.Time{}}

	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Object("user", u).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","user":{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},"message":"hello world"}
}

func ExampleContext_EmbedObject() {

	price := Price{val: 6449, prec: 2, unit: "$"}

	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		EmbedObject(price).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","price":"$64.49","message":"hello world"}
}

func ExampleContext_Interface() {
	obj := struct {
		Name string `json:"name"`
	}{
		Name: "john",
	}

	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Interface("obj", obj).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","obj":{"name":"john"},"message":"hello world"}
}

func ExampleContext_Dur() {
	d := time.Duration(10 * time.Second)

	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Dur("dur", d).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","dur":10000,"message":"hello world"}
}

func ExampleContext_Durs() {
	d := []time.Duration{
		time.Duration(10 * time.Second),
		time.Duration(20 * time.Second),
	}

	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Durs("durs", d).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","durs":[10000,20000],"message":"hello world"}
}

func ExampleContext_IPAddr() {
	hostIP := net.IP{192, 168, 0, 100}
	log := zerolog.New(os.Stdout).With().
		IPAddr("HostIP", hostIP).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"HostIP":"192.168.0.100","message":"hello world"}
}

func ExampleContext_IPPrefix() {
	route := net.IPNet{IP: net.IP{192, 168, 0, 0}, Mask: net.CIDRMask(24, 32)}
	log := zerolog.New(os.Stdout).With().
		IPPrefix("Route", route).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"Route":"192.168.0.0/24","message":"hello world"}
}

func ExampleContext_MacAddr() {
	mac := net.HardwareAddr{0x00, 0x14, 0x22, 0x01, 0x23, 0x45}
	log := zerolog.New(os.Stdout).With().
		MACAddr("hostMAC", mac).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"hostMAC":"00:14:22:01:23:45","message":"hello world"}
}
<commit_msg>Fix failing test introduced by last commit<commit_after>// +build !binary_log

package zerolog_test

import (
	"errors"
	"fmt"
	stdlog "log"
	"net"
	"os"
	"time"

	"github.com/rs/zerolog"
)

func ExampleNew() {
	log := zerolog.New(os.Stdout)

	log.Info().Msg("hello world")
	// Output: {"level":"info","message":"hello world"}
}

func ExampleLogger_With() {
	log := zerolog.New(os.Stdout).
		With().
		Str("foo", "bar").
		Logger()

	log.Info().Msg("hello world")

	// Output: {"level":"info","foo":"bar","message":"hello world"}
}

func ExampleLogger_Level() {
	log := zerolog.New(os.Stdout).Level(zerolog.WarnLevel)

	log.Info().Msg("filtered out message")
	log.Error().Msg("kept message")

	// Output: {"level":"error","message":"kept message"}
}

func ExampleLogger_Sample() {
	log := zerolog.New(os.Stdout).Sample(&zerolog.BasicSampler{N: 2})

	log.Info().Msg("message 1")
	log.Info().Msg("message 2")
	log.Info().Msg("message 3")
	log.Info().Msg("message 4")

	// Output: {"level":"info","message":"message 1"}
	// {"level":"info","message":"message 3"}
}

type LevelNameHook struct{}

func (h LevelNameHook) Run(e *zerolog.Event, l zerolog.Level, msg string) {
	if l != zerolog.NoLevel {
		e.Str("level_name", l.String())
	} else {
		e.Str("level_name", "NoLevel")
	}
}

type MessageHook string

func (h MessageHook) Run(e *zerolog.Event, l zerolog.Level, msg string) {
	e.Str("the_message", msg)
}

func ExampleLogger_Hook() {
	var levelNameHook LevelNameHook
	var messageHook MessageHook = "The message"

	log := zerolog.New(os.Stdout).Hook(levelNameHook).Hook(messageHook)

	log.Info().Msg("hello world")

	// Output: {"level":"info","level_name":"info","the_message":"hello world","message":"hello world"}
}

func ExampleLogger_Print() {
	log := zerolog.New(os.Stdout)

	log.Print("hello world")

	// Output: {"level":"debug","message":"hello world"}
}

func ExampleLogger_Printf() {
	log := zerolog.New(os.Stdout)

	log.Printf("hello %s", "world")

	// Output: {"level":"debug","message":"hello world"}
}

func ExampleLogger_Debug() {
	log := zerolog.New(os.Stdout)

	log.Debug().
		Str("foo", "bar").
		Int("n", 123).
		Msg("hello world")

	// Output: {"level":"debug","foo":"bar","n":123,"message":"hello world"}
}

func ExampleLogger_Info() {
	log := zerolog.New(os.Stdout)

	log.Info().
		Str("foo", "bar").
		Int("n", 123).
		Msg("hello world")

	// Output: {"level":"info","foo":"bar","n":123,"message":"hello world"}
}

func ExampleLogger_Warn() {
	log := zerolog.New(os.Stdout)

	log.Warn().
		Str("foo", "bar").
		Msg("a warning message")

	// Output: {"level":"warn","foo":"bar","message":"a warning message"}
}

func ExampleLogger_Error() {
	log := zerolog.New(os.Stdout)

	log.Error().
		Err(errors.New("some error")).
		Msg("error doing something")

	// Output: {"level":"error","error":"some error","message":"error doing something"}
}

func ExampleLogger_WithLevel() {
	log := zerolog.New(os.Stdout)

	log.WithLevel(zerolog.InfoLevel).
		Msg("hello world")

	// Output: {"level":"info","message":"hello world"}
}

func ExampleLogger_Write() {
	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Logger()

	stdlog.SetFlags(0)
	stdlog.SetOutput(log)

	stdlog.Print("hello world")

	// Output: {"foo":"bar","message":"hello world"}
}

func ExampleLogger_Log() {
	log := zerolog.New(os.Stdout)

	log.Log().
		Str("foo", "bar").
		Str("bar", "baz").
		Msg("")

	// Output: {"foo":"bar","bar":"baz"}
}

func ExampleEvent_Dict() {
	log := zerolog.New(os.Stdout)

	log.Log().
		Str("foo", "bar").
		Dict("dict", zerolog.Dict().
			Str("bar", "baz").
			Int("n", 1),
		).
		Msg("hello world")

	// Output: {"foo":"bar","dict":{"bar":"baz","n":1},"message":"hello world"}
}

type User struct {
	Name    string
	Age     int
	Created time.Time
}

func (u User) MarshalZerologObject(e *zerolog.Event) {
	e.Str("name", u.Name).
		Int("age", u.Age).
		Time("created", u.Created)
}

type Price struct {
	val  uint64
	prec int
	unit string
}

func (p Price) MarshalZerologObject(e *zerolog.Event) {
	denom := uint64(1)
	for i := 0; i < p.prec; i++ {
		denom *= 10
	}
	result := []byte(p.unit)
	result = append(result, fmt.Sprintf("%d.%d", p.val/denom, p.val%denom)...)
	e.Str("price", string(result))
}

type Users []User

func (uu Users) MarshalZerologArray(a *zerolog.Array) {
	for _, u := range uu {
		a.Object(u)
	}
}

func ExampleEvent_Array() {
	log := zerolog.New(os.Stdout)

	log.Log().
		Str("foo", "bar").
		Array("array", zerolog.Arr().
			Str("baz").
			Int(1),
		).
		Msg("hello world")

	// Output: {"foo":"bar","array":["baz",1],"message":"hello world"}
}

func ExampleEvent_Array_object() {
	log := zerolog.New(os.Stdout)

	// Users implements zerolog.LogArrayMarshaler
	u := Users{
		User{"John", 35, time.Time{}},
		User{"Bob", 55, time.Time{}},
	}

	log.Log().
		Str("foo", "bar").
		Array("users", u).
		Msg("hello world")

	// Output: {"foo":"bar","users":[{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},{"name":"Bob","age":55,"created":"0001-01-01T00:00:00Z"}],"message":"hello world"}
}

func ExampleEvent_Object() {
	log := zerolog.New(os.Stdout)

	// User implements zerolog.LogObjectMarshaler
	u := User{"John", 35, time.Time{}}

	log.Log().
		Str("foo", "bar").
		Object("user", u).
		Msg("hello world")

	// Output: {"foo":"bar","user":{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},"message":"hello world"}
}

func ExampleEvent_EmbedObject() {
	log := zerolog.New(os.Stdout)

	price := Price{val: 6449, prec: 2, unit: "$"}

	log.Log().
		Str("foo", "bar").
		EmbedObject(price).
		Msg("hello world")

	// Output: {"foo":"bar","price":"$64.49","message":"hello world"}
}

func ExampleEvent_Interface() {
	log := zerolog.New(os.Stdout)

	obj := struct {
		Name string `json:"name"`
	}{
		Name: "john",
	}

	log.Log().
		Str("foo", "bar").
		Interface("obj", obj).
		Msg("hello world")

	// Output: {"foo":"bar","obj":{"name":"john"},"message":"hello world"}
}

func ExampleEvent_Dur() {
	d := time.Duration(10 * time.Second)

	log := zerolog.New(os.Stdout)

	log.Log().
		Str("foo", "bar").
		Dur("dur", d).
		Msg("hello world")

	// Output: {"foo":"bar","dur":10000,"message":"hello world"}
}

func ExampleEvent_Durs() {
	d := []time.Duration{
		time.Duration(10 * time.Second),
		time.Duration(20 * time.Second),
	}

	log := zerolog.New(os.Stdout)

	log.Log().
		Str("foo", "bar").
		Durs("durs", d).
		Msg("hello world")

	// Output: {"foo":"bar","durs":[10000,20000],"message":"hello world"}
}

func ExampleContext_Dict() {
	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Dict("dict", zerolog.Dict().
			Str("bar", "baz").
			Int("n", 1),
		).Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","dict":{"bar":"baz","n":1},"message":"hello world"}
}

func ExampleContext_Array() {
	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Array("array", zerolog.Arr().
			Str("baz").
			Int(1),
		).Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","array":["baz",1],"message":"hello world"}
}

func ExampleContext_Array_object() {
	// Users implements zerolog.LogArrayMarshaler
	u := Users{
		User{"John", 35, time.Time{}},
		User{"Bob", 55, time.Time{}},
	}

	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Array("users", u).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","users":[{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},{"name":"Bob","age":55,"created":"0001-01-01T00:00:00Z"}],"message":"hello world"}
}

func ExampleContext_Object() {
	// User implements zerolog.LogObjectMarshaler
	u := User{"John", 35, time.Time{}}

	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Object("user", u).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","user":{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},"message":"hello world"}
}

func ExampleContext_EmbedObject() {

	price := Price{val: 6449, prec: 2, unit: "$"}

	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		EmbedObject(price).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","price":"$64.49","message":"hello world"}
}

func ExampleContext_Interface() {
	obj := struct {
		Name string `json:"name"`
	}{
		Name: "john",
	}

	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Interface("obj", obj).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","obj":{"name":"john"},"message":"hello world"}
}

func ExampleContext_Dur() {
	d := time.Duration(10 * time.Second)

	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Dur("dur", d).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","dur":10000,"message":"hello world"}
}

func ExampleContext_Durs() {
	d := []time.Duration{
		time.Duration(10 * time.Second),
		time.Duration(20 * time.Second),
	}

	log := zerolog.New(os.Stdout).With().
		Str("foo", "bar").
		Durs("durs", d).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"foo":"bar","durs":[10000,20000],"message":"hello world"}
}

func ExampleContext_IPAddr() {
	hostIP := net.IP{192, 168, 0, 100}
	log := zerolog.New(os.Stdout).With().
		IPAddr("HostIP", hostIP).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"HostIP":"192.168.0.100","message":"hello world"}
}

func ExampleContext_IPPrefix() {
	route := net.IPNet{IP: net.IP{192, 168, 0, 0}, Mask: net.CIDRMask(24, 32)}
	log := zerolog.New(os.Stdout).With().
		IPPrefix("Route", route).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"Route":"192.168.0.0/24","message":"hello world"}
}

func ExampleContext_MacAddr() {
	mac := net.HardwareAddr{0x00, 0x14, 0x22, 0x01, 0x23, 0x45}
	log := zerolog.New(os.Stdout).With().
		MACAddr("hostMAC", mac).
		Logger()

	log.Log().Msg("hello world")

	// Output: {"hostMAC":"00:14:22:01:23:45","message":"hello world"}
}
<|endoftext|>


<commit_before>package utils

import (
	"log"
	"os"
)

func SetRight(file string, uid int32, gid int32) {
	f, err := os.Open(file)
	if err != nil {
		log.Fatalf("Open file %v error %v", file, err)
	}
	defer f.Close()
	err = f.Chown(int(uid), int(gid))
	if err != nil {
		log.Fatalf("Chown file %v error %v", file, err)
	}
}
<commit_msg>Fix bug of SetRight, enable chown with the sub files in containerend.<commit_after>package utils

import (
	"log"
	"os"
)

func SetRight(file string, uid int32, gid int32) {
	f, err := os.Open(file)
	if err != nil {
		log.Fatalf("Open file %v error %v", file, err)
	}
	defer f.Close()

	err = f.Chown(int(uid), int(gid))
	if err != nil {
		log.Fatalf("Chown file %v error %v", file, err)
	}

	// Read all files under file
	ff, _ := f.Readdirnames(0)
	for _, fi := range ff {
		subFile := file + "/" + fi
		err = os.Chown(subFile, int(uid), int(gid))
		if err != nil {
			log.Fatalf("Chown file %v error %v", subFile, err)
		}
	}
}
<|endoftext|>


<commit_before>package k8s_test

import (
	"github.com/onsi/gomega/gexec"

	. "github.com/concourse/concourse/topgun"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
)

var _ = Describe("Baggageclaim Drivers", func() {
	var (
		proxySession *gexec.Session
		atcEndpoint  string
	)

	AfterEach(func() {
		helmDestroy(releaseName)
		Wait(Start(nil, "kubectl", "delete", "namespace", namespace, "--wait=false"))

		if proxySession != nil {
			Wait(proxySession.Interrupt())
		}
	})

	type Case struct {
		Driver     string
		NodeImage  string
		ShouldWork bool
	}

	DescribeTable("across different node images",
		func(c Case) {
			setReleaseNameAndNamespace("bd-"+c.Driver+"-"+c.NodeImage)

			helmDeployTestFlags := []string{
				"--set=concourse.web.kubernetes.enabled=false",
				"--set=concourse.worker.baggageclaim.driver=" + c.Driver,
				"--set=worker.nodeSelector.nodeImage=" + c.NodeImage,
				"--set=worker.replicas=1",
			}

			deployConcourseChart(releaseName, helmDeployTestFlags...)

			if !c.ShouldWork {
				workerLogsSession := Start(nil, "kubectl", "logs",
					"--namespace="+namespace, "-lapp="+namespace+"-worker")
				<-workerLogsSession.Exited

				Expect(workerLogsSession.Out.Contents()).To(ContainSubstring("failed-to-set-up-driver"))
				return
			}

			waitAllPodsInNamespaceToBeReady(namespace)

			By("Creating the web proxy")
			proxySession, atcEndpoint = startPortForwarding(namespace, "service/"+releaseName+"-web", "8080")

			By("Logging in")
			fly.Login("test", "test", atcEndpoint)

			By("Setting and triggering a dumb pipeline")
			fly.Run("set-pipeline", "-n", "-c", "../pipelines/get-task.yml", "-p", "pipeline")
			fly.Run("trigger-job", "-j", "pipeline/simple-job")
		},
		Entry("with btrfs on cos", Case{
			Driver:     "btrfs",
			NodeImage:  "cos",
			ShouldWork: false,
		}),
		Entry("with btrfs on ubuntu", Case{
			Driver:     "btrfs",
			NodeImage:  "ubuntu",
			ShouldWork: true,
		}),
		Entry("with overlay on cos", Case{
			Driver:     "overlay",
			NodeImage:  "cos",
			ShouldWork: true,
		}),
		Entry("with overlay on ubuntu", Case{
			Driver:     "overlay",
			NodeImage:  "ubuntu",
			ShouldWork: true,
		}),
		Entry("with naive on cos", Case{
			Driver:     "naive",
			NodeImage:  "cos",
			ShouldWork: true,
		}),
		Entry("with naive on ubuntu", Case{
			Driver:     "naive",
			NodeImage:  "ubuntu",
			ShouldWork: true,
		}),
	)
})
<commit_msg>- Add unpause pipeline (gotcha!) - Add `-w` to the trigger command to make sure the task runs to completion.<commit_after>package k8s_test

import (
	"github.com/onsi/gomega/gexec"

	. "github.com/concourse/concourse/topgun"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
)

var _ = Describe("Baggageclaim Drivers", func() {
	var (
		proxySession *gexec.Session
		atcEndpoint  string
	)

	AfterEach(func() {
		helmDestroy(releaseName)
		Wait(Start(nil, "kubectl", "delete", "namespace", namespace, "--wait=false"))

		if proxySession != nil {
			Wait(proxySession.Interrupt())
		}
	})

	type Case struct {
		Driver     string
		NodeImage  string
		ShouldWork bool
	}

	DescribeTable("across different node images",
		func(c Case) {
			setReleaseNameAndNamespace("bd-" + c.Driver + "-" + c.NodeImage)

			helmDeployTestFlags := []string{
				"--set=concourse.web.kubernetes.enabled=false",
				"--set=concourse.worker.baggageclaim.driver=" + c.Driver,
				"--set=worker.nodeSelector.nodeImage=" + c.NodeImage,
				"--set=worker.replicas=1",
			}

			deployConcourseChart(releaseName, helmDeployTestFlags...)

			if !c.ShouldWork {
				Eventually(func() []byte {
					workerLogsSession := Start(nil, "kubectl", "logs",
						"--namespace="+namespace, "-lapp="+namespace+"-worker")
					<-workerLogsSession.Exited

					return workerLogsSession.Out.Contents()

				}).Should(ContainSubstring("failed-to-set-up-driver"))
				return
			}

			waitAllPodsInNamespaceToBeReady(namespace)

			By("Creating the web proxy")
			proxySession, atcEndpoint = startPortForwarding(namespace, "service/"+releaseName+"-web", "8080")

			By("Logging in")
			fly.Login("test", "test", atcEndpoint)

			By("Setting and triggering a dumb pipeline")
			fly.Run("set-pipeline", "-n", "-c", "../pipelines/get-task.yml", "-p", "some-pipeline")
			fly.Run("unpause-pipeline", "-p", "some-pipeline")
			fly.Run("trigger-job", "-w", "-j", "some-pipeline/simple-job")
		},
		Entry("with btrfs on cos", Case{
			Driver:     "btrfs",
			NodeImage:  "cos",
			ShouldWork: false,
		}),
		Entry("with btrfs on ubuntu", Case{
			Driver:     "btrfs",
			NodeImage:  "ubuntu",
			ShouldWork: true,
		}),
		Entry("with overlay on cos", Case{
			Driver:     "overlay",
			NodeImage:  "cos",
			ShouldWork: true,
		}),
		Entry("with overlay on ubuntu", Case{
			Driver:     "overlay",
			NodeImage:  "ubuntu",
			ShouldWork: true,
		}),
		Entry("with naive on cos", Case{
			Driver:     "naive",
			NodeImage:  "cos",
			ShouldWork: true,
		}),
		Entry("with naive on ubuntu", Case{
			Driver:     "naive",
			NodeImage:  "ubuntu",
			ShouldWork: true,
		}),
	)
})
<|endoftext|>


<commit_before>package main

import (
	"owl/common/tcp"
	"owl/common/types"
	"owl/common/utils"
	"strings"
	"time"
)

var (
	netCollect   *NetCollect
	AgentVersion string = "0.1"
)

type NetCollect struct {
	srv          *tcp.Server
	tsdBuffer    chan *types.TimeSeriesData
	histroyMap   map[string]float64
	repeater     *tcp.Session
	cfc          *tcp.Session
	metricBuffer chan *types.MetricConfig
	switchs      []*types.Switch
}

func InitNetCollect() error {
	s := tcp.NewServer("", &handle{})
	c := &NetCollect{}
	c.srv = s
	c.tsdBuffer = make(chan *types.TimeSeriesData, GlobalConfig.BUFFER_SIZE)
	c.metricBuffer = make(chan *types.MetricConfig, GlobalConfig.BUFFER_SIZE)
	c.switchs = []*types.Switch{}
	c.repeater = &tcp.Session{}
	c.cfc = &tcp.Session{}
	netCollect = c
	return nil
}

func InitIpRange() error {
	ipList := []string{}
	for _, ipRange := range GlobalConfig.IP_RANGE {
		var (
			ipStart string
			ipEnd   string
		)
		if len(ipRange) == 0 {
			continue
		}
		if strings.Contains(ipRange, "-") {
			field := strings.Split(ipRange, "-")
			if len(field) != 2 {
				continue
			}
			ipStart, ipEnd = field[0], field[1]
		} else {
			ipStart = ipRange
			ipEnd = ipRange
		}
		ips, err := utils.GetIPRange(ipStart, ipEnd)
		if err != nil {
			return err
		}
		ipList = append(ipList, ips...)
	}
	for _, ip := range ipList {
		s := &types.Switch{
			ID:              utils.Md5(ip)[:16],
			IP:              ip,
			Hostname:        "",
			LegalPrefix:     GlobalConfig.LEGAL_PREFIX,
			CollectInterval: GlobalConfig.COLLECT_INTERVAL,
			Snmp: types.SnmpConfig{
				Port:      GlobalConfig.SNMP_PORT,
				Version:   GlobalConfig.SNMP_VERSION,
				Community: GlobalConfig.SNMP_COMMUNITY,
			},
		}
		netCollect.switchs = append(netCollect.switchs, s)
		lg.Info("do %s, %#v", s.IP, s.Snmp)
		go s.Do(netCollect.tsdBuffer, netCollect.metricBuffer)
		go netCollect.SendConfig2CFC()
		go netCollect.SendMetri2CFC()
	}
	return nil
}

//TODO:需要优化
func (this *NetCollect) Dial(Type string) {
	var (
		tempDelay time.Duration
		err       error
		session   *tcp.Session
	)
retry:
	switch Type {
	case "cfc":
		session, err = this.srv.Connect(GlobalConfig.CFC_ADDR, nil)
	case "repeater":
		session, err = this.srv.Connect(GlobalConfig.REPEATER_ADDR, nil)
	default:

	}
	if err != nil {
		lg.Error("%s: %s", Type, err.Error())
		if tempDelay == 0 {
			tempDelay = 5 * time.Millisecond
		} else {
			tempDelay *= 2
		}
		if max := 5 * time.Second; tempDelay > max {
			tempDelay = max
		}
		time.Sleep(tempDelay)
		goto retry
	}
	switch Type {
	case "cfc":
		this.cfc = session
	case "repeater":
		this.repeater = session
	}
	for {
		switch Type {
		case "cfc":
			if this.cfc.IsClosed() {
				goto retry
			}
		case "repeater":
			if this.repeater.IsClosed() {
				goto retry
			}
		}
		time.Sleep(time.Second * 1)
	}
}

func (this *NetCollect) SendConfig2CFC() {
	for {
		if this.cfc.IsClosed() {
			goto sleep
		}
		for _, s := range this.switchs {
			if s.Hostname == "" || s.Hostname == "Unknown" {
				continue
			}
			this.cfc.Send(
				types.Pack(
					types.MESS_POST_HOST_CONFIG,
					&types.Host{
						ID:           s.ID,
						IP:           s.IP,
						Hostname:     s.Hostname,
						AgentVersion: AgentVersion,
					}))
		}
	sleep:
		time.Sleep(time.Minute * 1)
	}
}

func (this *NetCollect) SendMetri2CFC() {
	time.Sleep(time.Second * time.Duration(GlobalConfig.COLLECT_INTERVAL))
	for {
		if this.cfc.IsClosed() {
			time.Sleep(time.Second * 30)
			continue
		}
		select {
		case tsd, ok := <-this.metricBuffer:
			if ok {
				this.cfc.Send(types.Pack(types.MESS_POST_METRIC, tsd))
				lg.Info("sender to cfc %s", tsd)
			}
		}
	}
}

func (this *NetCollect) SendTSD2Repeater() {
	go func() {
		for {
			if this.repeater.IsClosed() {
				time.Sleep(time.Second * 5)
				continue
			}
			select {
			case tsd, ok := <-this.tsdBuffer:
				if ok {
					this.repeater.Send(
						types.Pack(types.MESS_POST_TSD,
							tsd,
						))
					lg.Info("sender to repeater %s", tsd)
				}
			}
		}
	}()
}
<commit_msg>增加网络设备心跳<commit_after>package main

import (
	"owl/common/tcp"
	"owl/common/types"
	"owl/common/utils"
	"strings"
	"time"
)

var (
	netCollect   *NetCollect
	AgentVersion string = "0.1"
)

type NetCollect struct {
	srv          *tcp.Server
	tsdBuffer    chan *types.TimeSeriesData
	histroyMap   map[string]float64
	repeater     *tcp.Session
	cfc          *tcp.Session
	metricBuffer chan *types.MetricConfig
	switchs      []*types.Switch
}

func InitNetCollect() error {
	s := tcp.NewServer("", &handle{})
	c := &NetCollect{}
	c.srv = s
	c.tsdBuffer = make(chan *types.TimeSeriesData, GlobalConfig.BUFFER_SIZE)
	c.metricBuffer = make(chan *types.MetricConfig, GlobalConfig.BUFFER_SIZE)
	c.switchs = []*types.Switch{}
	c.repeater = &tcp.Session{}
	c.cfc = &tcp.Session{}
	netCollect = c
	return nil
}

func InitIpRange() error {
	ipList := []string{}
	for _, ipRange := range GlobalConfig.IP_RANGE {
		var (
			ipStart string
			ipEnd   string
		)
		if len(ipRange) == 0 {
			continue
		}
		if strings.Contains(ipRange, "-") {
			field := strings.Split(ipRange, "-")
			if len(field) != 2 {
				continue
			}
			ipStart, ipEnd = field[0], field[1]
		} else {
			ipStart = ipRange
			ipEnd = ipRange
		}
		ips, err := utils.GetIPRange(ipStart, ipEnd)
		if err != nil {
			return err
		}
		ipList = append(ipList, ips...)
	}
	for _, ip := range ipList {
		s := &types.Switch{
			ID:              utils.Md5(ip)[:16],
			IP:              ip,
			Hostname:        "",
			LegalPrefix:     GlobalConfig.LEGAL_PREFIX,
			CollectInterval: GlobalConfig.COLLECT_INTERVAL,
			Snmp: types.SnmpConfig{
				Port:      GlobalConfig.SNMP_PORT,
				Version:   GlobalConfig.SNMP_VERSION,
				Community: GlobalConfig.SNMP_COMMUNITY,
			},
		}
		netCollect.switchs = append(netCollect.switchs, s)
		lg.Info("do %s, %#v", s.IP, s.Snmp)
		go s.Do(netCollect.tsdBuffer, netCollect.metricBuffer)
		go netCollect.SendConfig2CFC()
		go netCollect.SendMetri2CFC()
	}
	return nil
}

//TODO:需要优化
func (this *NetCollect) Dial(Type string) {
	var (
		tempDelay time.Duration
		err       error
		session   *tcp.Session
	)
retry:
	switch Type {
	case "cfc":
		session, err = this.srv.Connect(GlobalConfig.CFC_ADDR, nil)
	case "repeater":
		session, err = this.srv.Connect(GlobalConfig.REPEATER_ADDR, nil)
	default:

	}
	if err != nil {
		lg.Error("%s: %s", Type, err.Error())
		if tempDelay == 0 {
			tempDelay = 5 * time.Millisecond
		} else {
			tempDelay *= 2
		}
		if max := 5 * time.Second; tempDelay > max {
			tempDelay = max
		}
		time.Sleep(tempDelay)
		goto retry
	}
	switch Type {
	case "cfc":
		this.cfc = session
	case "repeater":
		this.repeater = session
	}
	for {
		switch Type {
		case "cfc":
			if this.cfc.IsClosed() {
				goto retry
			}
		case "repeater":
			if this.repeater.IsClosed() {
				goto retry
			}
		}
		time.Sleep(time.Second * 1)
	}
}

func (this *NetCollect) SendConfig2CFC() {
	for {
		if this.cfc.IsClosed() {
			goto sleep
		}
		for _, s := range this.switchs {
			if s.Hostname == "" || s.Hostname == "Unknown" {
				continue
			}
			h := &types.Host{
				ID:           s.ID,
				IP:           s.IP,
				Hostname:     s.Hostname,
				AgentVersion: AgentVersion,
			}
			//config
			this.cfc.Send(
				types.Pack(
					types.MESS_POST_HOST_CONFIG,
					h,
				))
			//heartbeat
			this.cfc.Send(
				types.Pack(
					types.MESS_POST_HOST_ALIVE,
					h,
				))
		}
	sleep:
		time.Sleep(time.Minute * 1)
	}
}

func (this *NetCollect) SendMetri2CFC() {
	time.Sleep(time.Second * time.Duration(GlobalConfig.COLLECT_INTERVAL))
	for {
		if this.cfc.IsClosed() {
			time.Sleep(time.Second * 30)
			continue
		}
		select {
		case tsd, ok := <-this.metricBuffer:
			if ok {
				this.cfc.Send(types.Pack(types.MESS_POST_METRIC, tsd))
				lg.Info("sender to cfc %s", tsd)
			}
		}
	}
}

func (this *NetCollect) SendTSD2Repeater() {
	go func() {
		for {
			if this.repeater.IsClosed() {
				time.Sleep(time.Second * 5)
				continue
			}
			select {
			case tsd, ok := <-this.tsdBuffer:
				if ok {
					this.repeater.Send(
						types.Pack(types.MESS_POST_TSD,
							tsd,
						))
					lg.Info("sender to repeater %s", tsd)
				}
			}
		}
	}()
}
<|endoftext|>