<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package filepath implements utility routines for manipulating filename paths
// in a way compatible with the target operating system-defined file paths.
package filepath

import (
	"bytes"
	"os"
	"runtime"
	"sort"
	"strings"
)

const (
	Separator     = os.PathSeparator
	ListSeparator = os.PathListSeparator
)

// Clean returns the shortest path name equivalent to path
// by purely lexical processing.  It applies the following rules
// iteratively until no further processing can be done:
//
//	1. Replace multiple Separator elements with a single one.
//	2. Eliminate each . path name element (the current directory).
//	3. Eliminate each inner .. path name element (the parent directory)
//	   along with the non-.. element that precedes it.
//	4. Eliminate .. elements that begin a rooted path:
//	   that is, replace "/.." by "/" at the beginning of a path,
//         assuming Separator is '/'.
//
// If the result of this process is an empty string, Clean
// returns the string ".".
//
// See also Rob Pike, ``Lexical File Names in Plan 9 or
// Getting Dot-Dot right,''
// http://plan9.bell-labs.com/sys/doc/lexnames.html
func Clean(path string) string {
	vol := VolumeName(path)
	path = path[len(vol):]
	if path == "" {
		if len(vol) > 1 && vol[1] != ':' {
			// should be UNC
			return FromSlash(vol)
		}
		return vol + "."
	}
	rooted := os.IsPathSeparator(path[0])

	// Invariants:
	//	reading from path; r is index of next byte to process.
	//	writing to buf; w is index of next byte to write.
	//	dotdot is index in buf where .. must stop, either because
	//		it is the leading slash or it is a leading ../../.. prefix.
	n := len(path)
	buf := []byte(path)
	r, w, dotdot := 0, 0, 0
	if rooted {
		buf[0] = Separator
		r, w, dotdot = 1, 1, 1
	}

	for r < n {
		switch {
		case os.IsPathSeparator(path[r]):
			// empty path element
			r++
		case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])):
			// . element
			r++
		case path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])):
			// .. element: remove to last separator
			r += 2
			switch {
			case w > dotdot:
				// can backtrack
				w--
				for w > dotdot && !os.IsPathSeparator(buf[w]) {
					w--
				}
			case !rooted:
				// cannot backtrack, but not rooted, so append .. element.
				if w > 0 {
					buf[w] = Separator
					w++
				}
				buf[w] = '.'
				w++
				buf[w] = '.'
				w++
				dotdot = w
			}
		default:
			// real path element.
			// add slash if needed
			if rooted && w != 1 || !rooted && w != 0 {
				buf[w] = Separator
				w++
			}
			// copy element
			for ; r < n && !os.IsPathSeparator(path[r]); r++ {
				buf[w] = path[r]
				w++
			}
		}
	}

	// Turn empty string into "."
	if w == 0 {
		buf[w] = '.'
		w++
	}

	return FromSlash(vol + string(buf[0:w]))
}

// ToSlash returns the result of replacing each separator character
// in path with a slash ('/') character.
func ToSlash(path string) string {
	if Separator == '/' {
		return path
	}
	return strings.Replace(path, string(Separator), "/", -1)
}

// FromSlash returns the result of replacing each slash ('/') character
// in path with a separator character.
func FromSlash(path string) string {
	if Separator == '/' {
		return path
	}
	return strings.Replace(path, "/", string(Separator), -1)
}

// SplitList splits a list of paths joined by the OS-specific ListSeparator.
func SplitList(path string) []string {
	if path == "" {
		return []string{}
	}
	return strings.Split(path, string(ListSeparator))
}

// Split splits path immediately following the final Separator,
// separating it into a directory and file name component.
// If there is no Separator in path, Split returns an empty dir
// and file set to path.
func Split(path string) (dir, file string) {
	vol := VolumeName(path)
	i := len(path) - 1
	for i >= len(vol) && !os.IsPathSeparator(path[i]) {
		i--
	}
	return path[:i+1], path[i+1:]
}

// Join joins any number of path elements into a single path, adding
// a Separator if necessary.  All empty strings are ignored.
func Join(elem ...string) string {
	for i, e := range elem {
		if e != "" {
			return Clean(strings.Join(elem[i:], string(Separator)))
		}
	}
	return ""
}

// Ext returns the file name extension used by path.
// The extension is the suffix beginning at the final dot
// in the final element of path; it is empty if there is
// no dot.
func Ext(path string) string {
	for i := len(path) - 1; i >= 0 && !os.IsPathSeparator(path[i]); i-- {
		if path[i] == '.' {
			return path[i:]
		}
	}
	return ""
}

// EvalSymlinks returns the path name after the evaluation of any symbolic
// links.
// If path is relative it will be evaluated relative to the current directory.
func EvalSymlinks(path string) (string, os.Error) {
	if runtime.GOOS == "windows" {
		// Symlinks are not supported under windows.
		_, err := os.Lstat(path)
		if err != nil {
			return "", err
		}
		return Clean(path), nil
	}
	const maxIter = 255
	originalPath := path
	// consume path by taking each frontmost path element,
	// expanding it if it's a symlink, and appending it to b
	var b bytes.Buffer
	for n := 0; path != ""; n++ {
		if n > maxIter {
			return "", os.NewError("EvalSymlinks: too many links in " + originalPath)
		}

		// find next path component, p
		i := strings.IndexRune(path, Separator)
		var p string
		if i == -1 {
			p, path = path, ""
		} else {
			p, path = path[:i], path[i+1:]
		}

		if p == "" {
			if b.Len() == 0 {
				// must be absolute path
				b.WriteRune(Separator)
			}
			continue
		}

		fi, err := os.Lstat(b.String() + p)
		if err != nil {
			return "", err
		}
		if !fi.IsSymlink() {
			b.WriteString(p)
			if path != "" {
				b.WriteRune(Separator)
			}
			continue
		}

		// it's a symlink, put it at the front of path
		dest, err := os.Readlink(b.String() + p)
		if err != nil {
			return "", err
		}
		if IsAbs(dest) {
			b.Reset()
		}
		path = dest + string(Separator) + path
	}
	return Clean(b.String()), nil
}

// Abs returns an absolute representation of path.
// If the path is not absolute it will be joined with the current
// working directory to turn it into an absolute path.  The absolute
// path name for a given file is not guaranteed to be unique.
func Abs(path string) (string, os.Error) {
	if IsAbs(path) {
		return Clean(path), nil
	}
	wd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	return Join(wd, path), nil
}

// SkipDir is used as a return value from WalkFuncs to indicate that
// the directory named in the call is to be skipped. It is not returned
// as an error by any function.
var SkipDir = os.NewError("skip this directory")

// WalkFunc is the type of the function called for each file or directory
// visited by Walk.  If there was a problem walking to the file or directory
// named by path, the incoming error will describe the problem and the
// function can decide how to handle that error (and Walk will not descend
// into that directory).  If an error is returned, processing stops.  The
// sole exception is that if path is a directory and the function returns the
// special value SkipDir, the contents of the directory are skipped
// and processing continues as usual on the next file.
type WalkFunc func(path string, info *os.FileInfo, err os.Error) os.Error

// walk recursively descends path, calling w.
func walk(path string, info *os.FileInfo, walkFn WalkFunc) os.Error {
	err := walkFn(path, info, nil)
	if err != nil {
		if info.IsDirectory() && err == SkipDir {
			return nil
		}
		return err
	}

	if !info.IsDirectory() {
		return nil
	}

	list, err := readDir(path)
	if err != nil {
		return walkFn(path, info, err)
	}

	for _, fileInfo := range list {
		if err = walk(Join(path, fileInfo.Name), fileInfo, walkFn); err != nil {
			return err
		}
	}
	return nil
}

// Walk walks the file tree rooted at root, calling walkFn for each file or
// directory in the tree, including root. All errors that arise visiting files
// and directories are filtered by walkFn.
func Walk(root string, walkFn WalkFunc) os.Error {
	info, err := os.Lstat(root)
	if err != nil {
		return walkFn(root, nil, err)
	}
	return walk(root, info, walkFn)
}

// readDir reads the directory named by dirname and returns
// a sorted list of directory entries.
// Copied from io/ioutil to avoid the circular import.
func readDir(dirname string) ([]*os.FileInfo, os.Error) {
	f, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	list, err := f.Readdir(-1)
	f.Close()
	if err != nil {
		return nil, err
	}
	fi := make(fileInfoList, len(list))
	for i := range list {
		fi[i] = &list[i]
	}
	sort.Sort(fi)
	return fi, nil
}

// A dirList implements sort.Interface.
type fileInfoList []*os.FileInfo

func (f fileInfoList) Len() int           { return len(f) }
func (f fileInfoList) Less(i, j int) bool { return f[i].Name < f[j].Name }
func (f fileInfoList) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }

// Base returns the last element of path.
// Trailing path separators are removed before extracting the last element.
// If the path is empty, Base returns ".".
// If the path consists entirely of separators, Base returns a single separator.
func Base(path string) string {
	if path == "" {
		return "."
	}
	// Strip trailing slashes.
	for len(path) > 0 && os.IsPathSeparator(path[len(path)-1]) {
		path = path[0 : len(path)-1]
	}
	// Find the last element
	i := len(path) - 1
	for i >= 0 && !os.IsPathSeparator(path[i]) {
		i--
	}
	if i >= 0 {
		path = path[i+1:]
	}
	// If empty now, it had only slashes.
	if path == "" {
		return string(Separator)
	}
	return path
}
<commit_msg>path/filepath: document that Walk sorts its output<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package filepath implements utility routines for manipulating filename paths
// in a way compatible with the target operating system-defined file paths.
package filepath

import (
	"bytes"
	"os"
	"runtime"
	"sort"
	"strings"
)

const (
	Separator     = os.PathSeparator
	ListSeparator = os.PathListSeparator
)

// Clean returns the shortest path name equivalent to path
// by purely lexical processing.  It applies the following rules
// iteratively until no further processing can be done:
//
//	1. Replace multiple Separator elements with a single one.
//	2. Eliminate each . path name element (the current directory).
//	3. Eliminate each inner .. path name element (the parent directory)
//	   along with the non-.. element that precedes it.
//	4. Eliminate .. elements that begin a rooted path:
//	   that is, replace "/.." by "/" at the beginning of a path,
//         assuming Separator is '/'.
//
// If the result of this process is an empty string, Clean
// returns the string ".".
//
// See also Rob Pike, ``Lexical File Names in Plan 9 or
// Getting Dot-Dot right,''
// http://plan9.bell-labs.com/sys/doc/lexnames.html
func Clean(path string) string {
	vol := VolumeName(path)
	path = path[len(vol):]
	if path == "" {
		if len(vol) > 1 && vol[1] != ':' {
			// should be UNC
			return FromSlash(vol)
		}
		return vol + "."
	}
	rooted := os.IsPathSeparator(path[0])

	// Invariants:
	//	reading from path; r is index of next byte to process.
	//	writing to buf; w is index of next byte to write.
	//	dotdot is index in buf where .. must stop, either because
	//		it is the leading slash or it is a leading ../../.. prefix.
	n := len(path)
	buf := []byte(path)
	r, w, dotdot := 0, 0, 0
	if rooted {
		buf[0] = Separator
		r, w, dotdot = 1, 1, 1
	}

	for r < n {
		switch {
		case os.IsPathSeparator(path[r]):
			// empty path element
			r++
		case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])):
			// . element
			r++
		case path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])):
			// .. element: remove to last separator
			r += 2
			switch {
			case w > dotdot:
				// can backtrack
				w--
				for w > dotdot && !os.IsPathSeparator(buf[w]) {
					w--
				}
			case !rooted:
				// cannot backtrack, but not rooted, so append .. element.
				if w > 0 {
					buf[w] = Separator
					w++
				}
				buf[w] = '.'
				w++
				buf[w] = '.'
				w++
				dotdot = w
			}
		default:
			// real path element.
			// add slash if needed
			if rooted && w != 1 || !rooted && w != 0 {
				buf[w] = Separator
				w++
			}
			// copy element
			for ; r < n && !os.IsPathSeparator(path[r]); r++ {
				buf[w] = path[r]
				w++
			}
		}
	}

	// Turn empty string into "."
	if w == 0 {
		buf[w] = '.'
		w++
	}

	return FromSlash(vol + string(buf[0:w]))
}

// ToSlash returns the result of replacing each separator character
// in path with a slash ('/') character.
func ToSlash(path string) string {
	if Separator == '/' {
		return path
	}
	return strings.Replace(path, string(Separator), "/", -1)
}

// FromSlash returns the result of replacing each slash ('/') character
// in path with a separator character.
func FromSlash(path string) string {
	if Separator == '/' {
		return path
	}
	return strings.Replace(path, "/", string(Separator), -1)
}

// SplitList splits a list of paths joined by the OS-specific ListSeparator.
func SplitList(path string) []string {
	if path == "" {
		return []string{}
	}
	return strings.Split(path, string(ListSeparator))
}

// Split splits path immediately following the final Separator,
// separating it into a directory and file name component.
// If there is no Separator in path, Split returns an empty dir
// and file set to path.
func Split(path string) (dir, file string) {
	vol := VolumeName(path)
	i := len(path) - 1
	for i >= len(vol) && !os.IsPathSeparator(path[i]) {
		i--
	}
	return path[:i+1], path[i+1:]
}

// Join joins any number of path elements into a single path, adding
// a Separator if necessary.  All empty strings are ignored.
func Join(elem ...string) string {
	for i, e := range elem {
		if e != "" {
			return Clean(strings.Join(elem[i:], string(Separator)))
		}
	}
	return ""
}

// Ext returns the file name extension used by path.
// The extension is the suffix beginning at the final dot
// in the final element of path; it is empty if there is
// no dot.
func Ext(path string) string {
	for i := len(path) - 1; i >= 0 && !os.IsPathSeparator(path[i]); i-- {
		if path[i] == '.' {
			return path[i:]
		}
	}
	return ""
}

// EvalSymlinks returns the path name after the evaluation of any symbolic
// links.
// If path is relative it will be evaluated relative to the current directory.
func EvalSymlinks(path string) (string, os.Error) {
	if runtime.GOOS == "windows" {
		// Symlinks are not supported under windows.
		_, err := os.Lstat(path)
		if err != nil {
			return "", err
		}
		return Clean(path), nil
	}
	const maxIter = 255
	originalPath := path
	// consume path by taking each frontmost path element,
	// expanding it if it's a symlink, and appending it to b
	var b bytes.Buffer
	for n := 0; path != ""; n++ {
		if n > maxIter {
			return "", os.NewError("EvalSymlinks: too many links in " + originalPath)
		}

		// find next path component, p
		i := strings.IndexRune(path, Separator)
		var p string
		if i == -1 {
			p, path = path, ""
		} else {
			p, path = path[:i], path[i+1:]
		}

		if p == "" {
			if b.Len() == 0 {
				// must be absolute path
				b.WriteRune(Separator)
			}
			continue
		}

		fi, err := os.Lstat(b.String() + p)
		if err != nil {
			return "", err
		}
		if !fi.IsSymlink() {
			b.WriteString(p)
			if path != "" {
				b.WriteRune(Separator)
			}
			continue
		}

		// it's a symlink, put it at the front of path
		dest, err := os.Readlink(b.String() + p)
		if err != nil {
			return "", err
		}
		if IsAbs(dest) {
			b.Reset()
		}
		path = dest + string(Separator) + path
	}
	return Clean(b.String()), nil
}

// Abs returns an absolute representation of path.
// If the path is not absolute it will be joined with the current
// working directory to turn it into an absolute path.  The absolute
// path name for a given file is not guaranteed to be unique.
func Abs(path string) (string, os.Error) {
	if IsAbs(path) {
		return Clean(path), nil
	}
	wd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	return Join(wd, path), nil
}

// SkipDir is used as a return value from WalkFuncs to indicate that
// the directory named in the call is to be skipped. It is not returned
// as an error by any function.
var SkipDir = os.NewError("skip this directory")

// WalkFunc is the type of the function called for each file or directory
// visited by Walk.  If there was a problem walking to the file or directory
// named by path, the incoming error will describe the problem and the
// function can decide how to handle that error (and Walk will not descend
// into that directory).  If an error is returned, processing stops.  The
// sole exception is that if path is a directory and the function returns the
// special value SkipDir, the contents of the directory are skipped
// and processing continues as usual on the next file.
type WalkFunc func(path string, info *os.FileInfo, err os.Error) os.Error

// walk recursively descends path, calling w.
func walk(path string, info *os.FileInfo, walkFn WalkFunc) os.Error {
	err := walkFn(path, info, nil)
	if err != nil {
		if info.IsDirectory() && err == SkipDir {
			return nil
		}
		return err
	}

	if !info.IsDirectory() {
		return nil
	}

	list, err := readDir(path)
	if err != nil {
		return walkFn(path, info, err)
	}

	for _, fileInfo := range list {
		if err = walk(Join(path, fileInfo.Name), fileInfo, walkFn); err != nil {
			return err
		}
	}
	return nil
}

// Walk walks the file tree rooted at root, calling walkFn for each file or
// directory in the tree, including root. All errors that arise visiting files
// and directories are filtered by walkFn. The files are walked in lexical
// order, which makes the output deterministic but means that for very
// large directories Walk can be inefficient.
func Walk(root string, walkFn WalkFunc) os.Error {
	info, err := os.Lstat(root)
	if err != nil {
		return walkFn(root, nil, err)
	}
	return walk(root, info, walkFn)
}

// readDir reads the directory named by dirname and returns
// a sorted list of directory entries.
// Copied from io/ioutil to avoid the circular import.
func readDir(dirname string) ([]*os.FileInfo, os.Error) {
	f, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	list, err := f.Readdir(-1)
	f.Close()
	if err != nil {
		return nil, err
	}
	fi := make(fileInfoList, len(list))
	for i := range list {
		fi[i] = &list[i]
	}
	sort.Sort(fi)
	return fi, nil
}

// A dirList implements sort.Interface.
type fileInfoList []*os.FileInfo

func (f fileInfoList) Len() int           { return len(f) }
func (f fileInfoList) Less(i, j int) bool { return f[i].Name < f[j].Name }
func (f fileInfoList) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }

// Base returns the last element of path.
// Trailing path separators are removed before extracting the last element.
// If the path is empty, Base returns ".".
// If the path consists entirely of separators, Base returns a single separator.
func Base(path string) string {
	if path == "" {
		return "."
	}
	// Strip trailing slashes.
	for len(path) > 0 && os.IsPathSeparator(path[len(path)-1]) {
		path = path[0 : len(path)-1]
	}
	// Find the last element
	i := len(path) - 1
	for i >= 0 && !os.IsPathSeparator(path[i]) {
		i--
	}
	if i >= 0 {
		path = path[i+1:]
	}
	// If empty now, it had only slashes.
	if path == "" {
		return string(Separator)
	}
	return path
}
<|endoftext|>


<commit_before>// Copyright 2015 FactomProject Authors. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.

package process

import (
	"encoding/hex"
	"errors"
	"github.com/FactomProject/FactomCode/common"
	cp "github.com/FactomProject/FactomCode/controlpanel"
	"github.com/FactomProject/FactomCode/database"
	"github.com/FactomProject/btcd/wire"
	"github.com/davecgh/go-spew/spew"
	"strconv"
	"time"
)

// processDirBlock validates dir block and save it to factom db.
// similar to blockChain.BC_ProcessBlock
func processDirBlock(msg *wire.MsgDirBlock) error {

	// Error condiftion for Milestone 1
	if nodeMode == common.SERVER_NODE {
		return errors.New("Server received msg:" + msg.Command())
	}

	blk, _ := db.FetchDBlockByHeight(msg.DBlk.Header.DBHeight)
	if blk != nil {
		procLog.Info("DBlock already exists for height:" + string(msg.DBlk.Header.DBHeight))
		cp.CP.AddUpdate(
			"DBOverlap",                                                          // tag
			"warning",                                                            // Category
			"Directory Block Overlap",                                            // Title
			"DBlock already exists for height:"+string(msg.DBlk.Header.DBHeight), // Message
			0) // Expire
		return nil
	}

	msg.DBlk.IsSealed = true
	dchain.AddDBlockToDChain(msg.DBlk)

	//Add it to mem pool before saving it in db
	fMemPool.addBlockMsg(msg, strconv.Itoa(int(msg.DBlk.Header.DBHeight))) // store in mempool with the height as the key

	procLog.Debug("SyncUp: MsgDirBlock DBHeight=", msg.DBlk.Header.DBHeight)
	cp.CP.AddUpdate(
		"DBSyncUp", // tag
		"Status",   // Category
		"SyncUp:",  // Title
		"MsgDirBlock DBHeigth=:"+string(msg.DBlk.Header.DBHeight), // Message
		0) // Expire

	return nil
}

// processFBlock validates admin block and save it to factom db.
// similar to blockChain.BC_ProcessBlock
func processFBlock(msg *wire.MsgFBlock) error {

	// Error condiftion for Milestone 1
	if nodeMode == common.SERVER_NODE {
		return errors.New("Server received msg:" + msg.Command())
	}

	key := hex.EncodeToString(msg.SC.GetHash().Bytes())
	//Add it to mem pool before saving it in db
	fMemPool.addBlockMsg(msg, string(key)) // stored in mem pool with the MR as the key

	procLog.Debug("SyncUp: MsgFBlock DBHeight=", msg.SC.GetDBHeight())

	return nil

}

// processABlock validates admin block and save it to factom db.
// similar to blockChain.BC_ProcessBlock
func processABlock(msg *wire.MsgABlock) error {

	// Error condiftion for Milestone 1
	if nodeMode == common.SERVER_NODE {
		return errors.New("Server received msg:" + msg.Command())
	}

	//Add it to mem pool before saving it in db
	abHash, err := msg.ABlk.PartialHash()
	if err != nil {
		return err
	}
	fMemPool.addBlockMsg(msg, abHash.String()) // store in mem pool with ABHash as key

	procLog.Debug("SyncUp: MsgABlock DBHeight=", msg.ABlk.Header.DBHeight)

	return nil
}

// procesFBlock validates entry credit block and save it to factom db.
// similar to blockChain.BC_ProcessBlock
func procesECBlock(msg *wire.MsgECBlock) error {

	// Error condiftion for Milestone 1
	if nodeMode == common.SERVER_NODE {
		return errors.New("Server received msg:" + msg.Command())
	}

	//Add it to mem pool before saving it in db
	hash, err := msg.ECBlock.HeaderHash()
	if err != nil {
		return err
	}
	fMemPool.addBlockMsg(msg, hash.String())

	procLog.Debug("SyncUp: MsgCBlock DBHeight=", msg.ECBlock.Header.DBHeight)

	return nil
}

// processEBlock validates entry block and save it to factom db.
// similar to blockChain.BC_ProcessBlock
func processEBlock(msg *wire.MsgEBlock) error {

	// Error condiftion for Milestone 1
	if nodeMode == common.SERVER_NODE {
		return errors.New("Server received msg:" + msg.Command())
	}
	/*
		if msg.EBlk.Header.DBHeight >= dchain.NextBlockHeight || msg.EBlk.Header.DBHeight < 0 {
			return errors.New("MsgEBlock has an invalid DBHeight:" + strconv.Itoa(int(msg.EBlk.Header.DBHeight)))
		}
	*/
	//Add it to mem pool before saving it in db
	keyMR, err := msg.EBlk.KeyMR()
	if err != nil {
		return err
	}
	fMemPool.addBlockMsg(msg, keyMR.String()) // store it in mem pool with MR as the key

	procLog.Debug("SyncUp: MsgEBlock DBHeight=", msg.EBlk.Header.DBHeight)

	return nil
}

// processEntry validates entry and save it to factom db.
// similar to blockChain.BC_ProcessBlock
func processEntry(msg *wire.MsgEntry) error {

	// Error condiftion for Milestone 1
	if nodeMode == common.SERVER_NODE {
		return errors.New("Server received msg:" + msg.Command())
	}

	// store the entry in mem pool
	h := msg.Entry.Hash()
	fMemPool.addBlockMsg(msg, h.String()) // store it in mem pool with hash as the key

	procLog.Debug("SyncUp: MsgEntry hash=", msg.Entry.Hash())

	return nil
}

// Validate the new blocks in mem pool and store them in db
func validateAndStoreBlocks(fMemPool *ftmMemPool, db database.Db, dchain *common.DChain, outCtlMsgQ chan wire.FtmInternalMsg) {
	var myDBHeight int64
	var sleeptime int
	var dblk *common.DirectoryBlock

	for true {
		dblk = nil
		_, myDBHeight, _ = db.FetchBlockHeightCache()

		adj := (len(dchain.Blocks) - int(myDBHeight))
		if adj <= 0 {
			adj = 1
		}
		// in milliseconds
		sleeptime = 100 + 1000/adj

		if len(dchain.Blocks) > int(myDBHeight+1) {
			dblk = dchain.Blocks[myDBHeight+1]
		}
		if dblk != nil {
			if validateBlocksFromMemPool(dblk, fMemPool, db) {
				err := storeBlocksFromMemPool(dblk, fMemPool, db)
				if err == nil {
					deleteBlocksFromMemPool(dblk, fMemPool)
				} else {
					panic("error in deleteBlocksFromMemPool.")
				}
			} else {
				time.Sleep(time.Duration(sleeptime * 1000000)) // Nanoseconds for duration
			}
		} else {
			time.Sleep(time.Duration(sleeptime * 1000000)) // Nanoseconds for duration

			//TODO: send an internal msg to sync up with peers
		}

	}

}

// Validate the new blocks in mem pool and store them in db
func validateBlocksFromMemPool(b *common.DirectoryBlock, fMemPool *ftmMemPool, db database.Db) bool {

	// Validate the genesis block
	if b.Header.DBHeight == 0 {
		h, _ := common.CreateHash(b)
		if h.String() != common.GENESIS_DIR_BLOCK_HASH {
			// panic for milestone 1
			//panic("Genesis dir block is not as expected: " + h.String())
			procLog.Errorf("Genesis dir block is not as expected: " + h.String())
		}
	}

	for _, dbEntry := range b.DBEntries {
		switch dbEntry.ChainID.String() {
		case ecchain.ChainID.String():
			if _, ok := fMemPool.blockpool[dbEntry.KeyMR.String()]; !ok {
				return false
			}
		case achain.ChainID.String():
			if msg, ok := fMemPool.blockpool[dbEntry.KeyMR.String()]; !ok {
				return false
			} else {
				// validate signature of the previous dir block
				aBlkMsg, _ := msg.(*wire.MsgABlock)
				if !validateDBSignature(aBlkMsg.ABlk, dchain) {
					return false
				}
			}
		case fchain.ChainID.String():
			if _, ok := fMemPool.blockpool[dbEntry.KeyMR.String()]; !ok {
				return false
			}
		default:
			if msg, ok := fMemPool.blockpool[dbEntry.KeyMR.String()]; !ok {
				return false
			} else {
				eBlkMsg, _ := msg.(*wire.MsgEBlock)
				// validate every entry in EBlock
				for _, ebEntry := range eBlkMsg.EBlk.Body.EBEntries {
					if _, foundInMemPool := fMemPool.blockpool[ebEntry.String()]; !foundInMemPool {
						// continue if the entry arleady exists in db
						entry, _ := db.FetchEntryByHash(ebEntry)
						if entry == nil {
							return false
						}
					}
				}
			}
		}
	}

	return true
}

// Validate the new blocks in mem pool and store them in db
// Need to make a batch insert in db in milestone 2
func storeBlocksFromMemPool(b *common.DirectoryBlock, fMemPool *ftmMemPool, db database.Db) error {

	for _, dbEntry := range b.DBEntries {
		switch dbEntry.ChainID.String() {
		case ecchain.ChainID.String():
			ecBlkMsg := fMemPool.blockpool[dbEntry.KeyMR.String()].(*wire.MsgECBlock)
			err := db.ProcessECBlockBatch(ecBlkMsg.ECBlock)
			if err != nil {
				return err
			}
			// needs to be improved??
			initializeECreditMap(ecBlkMsg.ECBlock)
			// for debugging
			exportECBlock(ecBlkMsg.ECBlock)
		case achain.ChainID.String():
			aBlkMsg := fMemPool.blockpool[dbEntry.KeyMR.String()].(*wire.MsgABlock)
			err := db.ProcessABlockBatch(aBlkMsg.ABlk)
			if err != nil {
				return err
			}
			// for debugging
			exportABlock(aBlkMsg.ABlk)
		case fchain.ChainID.String():
			fBlkMsg := fMemPool.blockpool[dbEntry.KeyMR.String()].(*wire.MsgFBlock)
			err := db.ProcessFBlockBatch(fBlkMsg.SC)
			if err != nil {
				return err
			}
			// Initialize the Factoid State
			err = common.FactoidState.AddTransactionBlock(fBlkMsg.SC)
			FactoshisPerCredit = fBlkMsg.SC.GetExchRate()
			if err != nil {
				return err
			}

			// for debugging
			exportFctBlock(fBlkMsg.SC)
		default:
			// handle Entry Block
			eBlkMsg, _ := fMemPool.blockpool[dbEntry.KeyMR.String()].(*wire.MsgEBlock)
			// store entry in db first
			for _, ebEntry := range eBlkMsg.EBlk.Body.EBEntries {
				if msg, foundInMemPool := fMemPool.blockpool[ebEntry.String()]; foundInMemPool {
					err := db.InsertEntry(msg.(*wire.MsgEntry).Entry)
					if err != nil {
						return err
					}
				}
			}
			// Store Entry Block in db
			err := db.ProcessEBlockBatch(eBlkMsg.EBlk)
			if err != nil {
				return err
			}

			// create a chain in db if it's not existing
			chain := chainIDMap[eBlkMsg.EBlk.Header.ChainID.String()]
			if chain == nil {
				chain = new(common.EChain)
				chain.ChainID = eBlkMsg.EBlk.Header.ChainID
				if eBlkMsg.EBlk.Header.EBSequence == 0 {
					chain.FirstEntry, _ = db.FetchEntryByHash(eBlkMsg.EBlk.Body.EBEntries[0])
				}
				db.InsertChain(chain)
				chainIDMap[chain.ChainID.String()] = chain
			} else if chain.FirstEntry == nil && eBlkMsg.EBlk.Header.EBSequence == 0 {
				chain.FirstEntry, _ = db.FetchEntryByHash(eBlkMsg.EBlk.Body.EBEntries[0])
				db.InsertChain(chain)
			}

			// for debugging
			exportEBlock(eBlkMsg.EBlk)
		}
	}

	// Store the dir block
	err := db.ProcessDBlockBatch(b)
	if err != nil {
		return err
	}

	// Update dir block height cache in db
	commonHash, _ := common.CreateHash(b)
	db.UpdateBlockHeightCache(b.Header.DBHeight, commonHash)

	// for debugging
	exportDBlock(b)

	return nil
}

// Validate the new blocks in mem pool and store them in db
func deleteBlocksFromMemPool(b *common.DirectoryBlock, fMemPool *ftmMemPool) error {

	for _, dbEntry := range b.DBEntries {
		switch dbEntry.ChainID.String() {
		case ecchain.ChainID.String():
			fMemPool.deleteBlockMsg(dbEntry.KeyMR.String())
		case achain.ChainID.String():
			fMemPool.deleteBlockMsg(dbEntry.KeyMR.String())
		case fchain.ChainID.String():
			fMemPool.deleteBlockMsg(dbEntry.KeyMR.String())
		default:
			eBlkMsg, _ := fMemPool.blockpool[dbEntry.KeyMR.String()].(*wire.MsgEBlock)
			for _, ebEntry := range eBlkMsg.EBlk.Body.EBEntries {
				fMemPool.deleteBlockMsg(ebEntry.String())
			}
			fMemPool.deleteBlockMsg(dbEntry.KeyMR.String())
		}
	}
	fMemPool.deleteBlockMsg(strconv.Itoa(int(b.Header.DBHeight)))

	return nil
}

func validateDBSignature(aBlock *common.AdminBlock, dchain *common.DChain) bool {

	dbSigEntry := aBlock.GetDBSignature()
	if dbSigEntry == nil {
		if aBlock.Header.DBHeight == 0 {
			return true
		} else {
			return false
		}
	} else {
		dbSig := dbSigEntry.(*common.DBSignatureEntry)
		if serverPubKey.String() != dbSig.PubKey.String() {
			return false
		} else {
			// obtain the previous directory block
			dblk := dchain.Blocks[aBlock.Header.DBHeight-1]
			if dblk == nil {
				return false
			} else {
				// validatet the signature
				bHeader, _ := dblk.Header.MarshalBinary()
				if !serverPubKey.Verify(bHeader, (*[64]byte)(dbSig.PrevDBSig)) {
					procLog.Infof("No valid signature found in Admin Block = %s\n", spew.Sdump(aBlock))
					return false
				}
			}
		}
	}

	return true
}
<commit_msg>fixed symptom of syncup issue<commit_after>// Copyright 2015 FactomProject Authors. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.

package process

import (
	"bytes"
	"encoding/hex"
	"errors"
	"github.com/FactomProject/FactomCode/common"
	cp "github.com/FactomProject/FactomCode/controlpanel"
	"github.com/FactomProject/FactomCode/database"
	"github.com/FactomProject/btcd/wire"
	"github.com/davecgh/go-spew/spew"
	"strconv"
	"time"
)

// processDirBlock validates dir block and save it to factom db.
// similar to blockChain.BC_ProcessBlock
func processDirBlock(msg *wire.MsgDirBlock) error {

	// Error condiftion for Milestone 1
	if nodeMode == common.SERVER_NODE {
		return errors.New("Server received msg:" + msg.Command())
	}

	blk, _ := db.FetchDBlockByHeight(msg.DBlk.Header.DBHeight)
	if blk != nil {
		procLog.Info("DBlock already exists for height:" + string(msg.DBlk.Header.DBHeight))
		cp.CP.AddUpdate(
			"DBOverlap",                                                          // tag
			"warning",                                                            // Category
			"Directory Block Overlap",                                            // Title
			"DBlock already exists for height:"+string(msg.DBlk.Header.DBHeight), // Message
			0) // Expire
		return nil
	}

	msg.DBlk.IsSealed = true
	dchain.AddDBlockToDChain(msg.DBlk)

	//Add it to mem pool before saving it in db
	fMemPool.addBlockMsg(msg, strconv.Itoa(int(msg.DBlk.Header.DBHeight))) // store in mempool with the height as the key

	procLog.Debug("SyncUp: MsgDirBlock DBHeight=", msg.DBlk.Header.DBHeight)
	cp.CP.AddUpdate(
		"DBSyncUp", // tag
		"Status",   // Category
		"SyncUp:",  // Title
		"MsgDirBlock DBHeigth=:"+string(msg.DBlk.Header.DBHeight), // Message
		0) // Expire

	return nil
}

// processFBlock validates admin block and save it to factom db.
// similar to blockChain.BC_ProcessBlock
func processFBlock(msg *wire.MsgFBlock) error {

	// Error condiftion for Milestone 1
	if nodeMode == common.SERVER_NODE {
		return errors.New("Server received msg:" + msg.Command())
	}

	key := hex.EncodeToString(msg.SC.GetHash().Bytes())
	//Add it to mem pool before saving it in db
	fMemPool.addBlockMsg(msg, string(key)) // stored in mem pool with the MR as the key

	procLog.Debug("SyncUp: MsgFBlock DBHeight=", msg.SC.GetDBHeight())

	return nil

}

// processABlock validates admin block and save it to factom db.
// similar to blockChain.BC_ProcessBlock
func processABlock(msg *wire.MsgABlock) error {

	// Error condiftion for Milestone 1
	if nodeMode == common.SERVER_NODE {
		return errors.New("Server received msg:" + msg.Command())
	}

	//Add it to mem pool before saving it in db
	abHash, err := msg.ABlk.PartialHash()
	if err != nil {
		return err
	}
	fMemPool.addBlockMsg(msg, abHash.String()) // store in mem pool with ABHash as key

	procLog.Debug("SyncUp: MsgABlock DBHeight=", msg.ABlk.Header.DBHeight)

	return nil
}

// procesFBlock validates entry credit block and save it to factom db.
// similar to blockChain.BC_ProcessBlock
func procesECBlock(msg *wire.MsgECBlock) error {

	// Error condiftion for Milestone 1
	if nodeMode == common.SERVER_NODE {
		return errors.New("Server received msg:" + msg.Command())
	}

	//Add it to mem pool before saving it in db
	hash, err := msg.ECBlock.HeaderHash()
	if err != nil {
		return err
	}
	fMemPool.addBlockMsg(msg, hash.String())

	procLog.Debug("SyncUp: MsgCBlock DBHeight=", msg.ECBlock.Header.DBHeight)

	return nil
}

// processEBlock validates entry block and save it to factom db.
// similar to blockChain.BC_ProcessBlock
func processEBlock(msg *wire.MsgEBlock) error {

	// Error condiftion for Milestone 1
	if nodeMode == common.SERVER_NODE {
		return errors.New("Server received msg:" + msg.Command())
	}
	/*
		if msg.EBlk.Header.DBHeight >= dchain.NextBlockHeight || msg.EBlk.Header.DBHeight < 0 {
			return errors.New("MsgEBlock has an invalid DBHeight:" + strconv.Itoa(int(msg.EBlk.Header.DBHeight)))
		}
	*/
	//Add it to mem pool before saving it in db
	keyMR, err := msg.EBlk.KeyMR()
	if err != nil {
		return err
	}
	fMemPool.addBlockMsg(msg, keyMR.String()) // store it in mem pool with MR as the key

	procLog.Debug("SyncUp: MsgEBlock DBHeight=", msg.EBlk.Header.DBHeight)

	return nil
}

// processEntry validates entry and save it to factom db.
// similar to blockChain.BC_ProcessBlock
func processEntry(msg *wire.MsgEntry) error {

	// Error condiftion for Milestone 1
	if nodeMode == common.SERVER_NODE {
		return errors.New("Server received msg:" + msg.Command())
	}

	// store the entry in mem pool
	h := msg.Entry.Hash()
	fMemPool.addBlockMsg(msg, h.String()) // store it in mem pool with hash as the key

	procLog.Debug("SyncUp: MsgEntry hash=", msg.Entry.Hash())

	return nil
}

// Validate the new blocks in mem pool and store them in db
func validateAndStoreBlocks(fMemPool *ftmMemPool, db database.Db, dchain *common.DChain, outCtlMsgQ chan wire.FtmInternalMsg) {
	var myDBHeight int64
	var sleeptime int
	var dblk *common.DirectoryBlock

	for true {
		dblk = nil
		_, myDBHeight, _ = db.FetchBlockHeightCache()

		adj := (len(dchain.Blocks) - int(myDBHeight))
		if adj <= 0 {
			adj = 1
		}
		// in milliseconds
		sleeptime = 100 + 1000/adj

		if len(dchain.Blocks) > int(myDBHeight+1) {
			dblk = dchain.Blocks[myDBHeight+1]
		}
		if dblk != nil {
			if validateBlocksFromMemPool(dblk, fMemPool, db) {
				err := storeBlocksFromMemPool(dblk, fMemPool, db)
				if err == nil {
					deleteBlocksFromMemPool(dblk, fMemPool)
				} else {
					panic("error in deleteBlocksFromMemPool.")
				}
			} else {
				time.Sleep(time.Duration(sleeptime * 1000000)) // Nanoseconds for duration
			}
		} else {
			time.Sleep(time.Duration(sleeptime * 1000000)) // Nanoseconds for duration

			//TODO: send an internal msg to sync up with peers
		}

	}

}

// Validate the new blocks in mem pool and store them in db
func validateBlocksFromMemPool(b *common.DirectoryBlock, fMemPool *ftmMemPool, db database.Db) bool {

	// Validate the genesis block
	if b.Header.DBHeight == 0 {
		h, _ := common.CreateHash(b)
		if h.String() != common.GENESIS_DIR_BLOCK_HASH {
			// panic for milestone 1
			//panic("Genesis dir block is not as expected: " + h.String())
			procLog.Errorf("Genesis dir block is not as expected: " + h.String())
		}
	}

	for _, dbEntry := range b.DBEntries {
		switch dbEntry.ChainID.String() {
		case ecchain.ChainID.String():
			if _, ok := fMemPool.blockpool[dbEntry.KeyMR.String()]; !ok {
				return false
			}
		case achain.ChainID.String():
			if msg, ok := fMemPool.blockpool[dbEntry.KeyMR.String()]; !ok {
				return false
			} else {
				// validate signature of the previous dir block
				aBlkMsg, _ := msg.(*wire.MsgABlock)
				if !validateDBSignature(aBlkMsg.ABlk, dchain) {
					return false
				}
			}
		case fchain.ChainID.String():
			if _, ok := fMemPool.blockpool[dbEntry.KeyMR.String()]; !ok {
				return false
			}
		default:
			if msg, ok := fMemPool.blockpool[dbEntry.KeyMR.String()]; !ok {
				return false
			} else {
				eBlkMsg, _ := msg.(*wire.MsgEBlock)
				// validate every entry in EBlock
				for _, ebEntry := range eBlkMsg.EBlk.Body.EBEntries {
					if _, foundInMemPool := fMemPool.blockpool[ebEntry.String()]; !foundInMemPool {
						if !bytes.Equal(ebEntry.Bytes()[:31],common.ZERO_HASH[:31]) {
							// continue if the entry arleady exists in db
							entry, _ := db.FetchEntryByHash(ebEntry)
							if entry == nil {
								return false
							}
						}
					}
				}
			}
		}
	}

	return true
}

// Validate the new blocks in mem pool and store them in db
// Need to make a batch insert in db in milestone 2
func storeBlocksFromMemPool(b *common.DirectoryBlock, fMemPool *ftmMemPool, db database.Db) error {

	for _, dbEntry := range b.DBEntries {
		switch dbEntry.ChainID.String() {
		case ecchain.ChainID.String():
			ecBlkMsg := fMemPool.blockpool[dbEntry.KeyMR.String()].(*wire.MsgECBlock)
			err := db.ProcessECBlockBatch(ecBlkMsg.ECBlock)
			if err != nil {
				return err
			}
			// needs to be improved??
			initializeECreditMap(ecBlkMsg.ECBlock)
			// for debugging
			exportECBlock(ecBlkMsg.ECBlock)
		case achain.ChainID.String():
			aBlkMsg := fMemPool.blockpool[dbEntry.KeyMR.String()].(*wire.MsgABlock)
			err := db.ProcessABlockBatch(aBlkMsg.ABlk)
			if err != nil {
				return err
			}
			// for debugging
			exportABlock(aBlkMsg.ABlk)
		case fchain.ChainID.String():
			fBlkMsg := fMemPool.blockpool[dbEntry.KeyMR.String()].(*wire.MsgFBlock)
			err := db.ProcessFBlockBatch(fBlkMsg.SC)
			if err != nil {
				return err
			}
			// Initialize the Factoid State
			err = common.FactoidState.AddTransactionBlock(fBlkMsg.SC)
			FactoshisPerCredit = fBlkMsg.SC.GetExchRate()
			if err != nil {
				return err
			}

			// for debugging
			exportFctBlock(fBlkMsg.SC)
		default:
			// handle Entry Block
			eBlkMsg, _ := fMemPool.blockpool[dbEntry.KeyMR.String()].(*wire.MsgEBlock)
			// store entry in db first
			for _, ebEntry := range eBlkMsg.EBlk.Body.EBEntries {
				if msg, foundInMemPool := fMemPool.blockpool[ebEntry.String()]; foundInMemPool {
					err := db.InsertEntry(msg.(*wire.MsgEntry).Entry)
					if err != nil {
						return err
					}
				}
			}
			// Store Entry Block in db
			err := db.ProcessEBlockBatch(eBlkMsg.EBlk)
			if err != nil {
				return err
			}

			// create a chain in db if it's not existing
			chain := chainIDMap[eBlkMsg.EBlk.Header.ChainID.String()]
			if chain == nil {
				chain = new(common.EChain)
				chain.ChainID = eBlkMsg.EBlk.Header.ChainID
				if eBlkMsg.EBlk.Header.EBSequence == 0 {
					chain.FirstEntry, _ = db.FetchEntryByHash(eBlkMsg.EBlk.Body.EBEntries[0])
				}
				db.InsertChain(chain)
				chainIDMap[chain.ChainID.String()] = chain
			} else if chain.FirstEntry == nil && eBlkMsg.EBlk.Header.EBSequence == 0 {
				chain.FirstEntry, _ = db.FetchEntryByHash(eBlkMsg.EBlk.Body.EBEntries[0])
				db.InsertChain(chain)
			}

			// for debugging
			exportEBlock(eBlkMsg.EBlk)
		}
	}

	// Store the dir block
	err := db.ProcessDBlockBatch(b)
	if err != nil {
		return err
	}

	// Update dir block height cache in db
	commonHash, _ := common.CreateHash(b)
	db.UpdateBlockHeightCache(b.Header.DBHeight, commonHash)

	// for debugging
	exportDBlock(b)

	return nil
}

// Validate the new blocks in mem pool and store them in db
func deleteBlocksFromMemPool(b *common.DirectoryBlock, fMemPool *ftmMemPool) error {

	for _, dbEntry := range b.DBEntries {
		switch dbEntry.ChainID.String() {
		case ecchain.ChainID.String():
			fMemPool.deleteBlockMsg(dbEntry.KeyMR.String())
		case achain.ChainID.String():
			fMemPool.deleteBlockMsg(dbEntry.KeyMR.String())
		case fchain.ChainID.String():
			fMemPool.deleteBlockMsg(dbEntry.KeyMR.String())
		default:
			eBlkMsg, _ := fMemPool.blockpool[dbEntry.KeyMR.String()].(*wire.MsgEBlock)
			for _, ebEntry := range eBlkMsg.EBlk.Body.EBEntries {
				fMemPool.deleteBlockMsg(ebEntry.String())
			}
			fMemPool.deleteBlockMsg(dbEntry.KeyMR.String())
		}
	}
	fMemPool.deleteBlockMsg(strconv.Itoa(int(b.Header.DBHeight)))

	return nil
}

func validateDBSignature(aBlock *common.AdminBlock, dchain *common.DChain) bool {

	dbSigEntry := aBlock.GetDBSignature()
	if dbSigEntry == nil {
		if aBlock.Header.DBHeight == 0 {
			return true
		} else {
			return false
		}
	} else {
		dbSig := dbSigEntry.(*common.DBSignatureEntry)
		if serverPubKey.String() != dbSig.PubKey.String() {
			return false
		} else {
			// obtain the previous directory block
			dblk := dchain.Blocks[aBlock.Header.DBHeight-1]
			if dblk == nil {
				return false
			} else {
				// validatet the signature
				bHeader, _ := dblk.Header.MarshalBinary()
				if !serverPubKey.Verify(bHeader, (*[64]byte)(dbSig.PrevDBSig)) {
					procLog.Infof("No valid signature found in Admin Block = %s\n", spew.Sdump(aBlock))
					return false
				}
			}
		}
	}

	return true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/url"
	"time"

	"github.com/elastic/libbeat/beat"
	"github.com/elastic/libbeat/cfgfile"
	"github.com/elastic/libbeat/logp"
	lbpublisher "github.com/elastic/libbeat/publisher"

	"github.com/mrkschan/nginxbeat/collector"
	"github.com/mrkschan/nginxbeat/publisher"
)

const selector = "nginxbeat"

// Nginxbeat implements Beater interface and sends Nginx status using libbeat.
type Nginxbeat struct {
	// NbConfig holds configurations of Nginxbeat parsed by libbeat.
	NbConfig ConfigSettings

	done   chan uint
	events lbpublisher.Client

	urls   []*url.URL
	format string
	period time.Duration
}

// Config Nginxbeat according to nginxbeat.yml.
func (nb *Nginxbeat) Config(b *beat.Beat) error {
	err := cfgfile.Read(&nb.NbConfig, "")
	if err != nil {
		logp.Err("Error reading configuration file: %v", err)
		return err
	}

	var urlConfig []string
	if nb.NbConfig.Input.URLs != nil {
		urlConfig = nb.NbConfig.Input.URLs
	} else {
		urlConfig = []string{"http://127.0.0.1/status"}
	}

	nb.urls = make([]*url.URL, len(urlConfig))
	for i := 0; i < len(urlConfig); i++ {
		u, err := url.Parse(urlConfig[i])
		if err != nil {
			logp.Err("Invalid Nginx status page: %v", err)
			return err
		}
		nb.urls[i] = u
	}

	var f string
	if nb.NbConfig.Input.Format != "" {
		f = nb.NbConfig.Input.Format
	} else {
		f = "stub"
	}
	if f != "stub" && f != "plus" {
		err = fmt.Errorf("%s is an unsupported format", f)
	}
	if err != nil {
		logp.Err("Invalid Nginx status format: %v", err)
		return err
	}
	nb.format = f

	if nb.NbConfig.Input.Period != nil {
		nb.period = time.Duration(*nb.NbConfig.Input.Period) * time.Second
	} else {
		nb.period = 10 * time.Second
	}

	logp.Debug(selector, "Init nginxbeat")
	logp.Debug(selector, "Watch %v", nb.urls)
	logp.Debug(selector, "Format %v", nb.format)
	logp.Debug(selector, "Period %v", nb.period)

	return nil
}

// Setup Nginxbeat.
func (nb *Nginxbeat) Setup(b *beat.Beat) error {
	nb.events = b.Events
	nb.done = make(chan uint)

	return nil
}

// Run Nginxbeat.
func (nb *Nginxbeat) Run(b *beat.Beat) error {
	logp.Debug(selector, "Run nginxbeat")

	for _, u := range nb.urls {
		go func() {
			var c collector.Collector
			var p publisher.Publisher

			switch nb.format {
			case "stub":
				c = collector.NewStubCollector()
				p = publisher.NewStubPublisher(nb.events)
			case "plus":
				c = collector.NewPlusCollector()
				p = publisher.NewPlusPublisher(nb.events)
			}

			ticker := time.NewTicker(nb.period)
			defer ticker.Stop()

			for {
				select {
				case <-nb.done:
					goto GotoFinish
				case <-ticker.C:
				}

				start := time.Now()

				s, err := c.Collect(*u)
				if err != nil {
					logp.Err("Fail to read Nginx status: %v", err)
					goto GotoNext
				}
				p.Publish(s)

			GotoNext:
				end := time.Now()
				duration := end.Sub(start)
				if duration.Nanoseconds() > nb.period.Nanoseconds() {
					logp.Warn("Ignoring tick(s) due to processing taking longer than one period")
				}
			}

		GotoFinish:
		}()
	}

	<-nb.done
	return nil
}

// Cleanup Nginxbeat.
func (nb *Nginxbeat) Cleanup(b *beat.Beat) error {
	return nil
}

// Stop Nginxbeat.
func (nb *Nginxbeat) Stop() {
	logp.Debug(selector, "Stop nginxbeat")
	close(nb.done)
}
<commit_msg>No longer keep a global reference to libbeat publisher client<commit_after>package main

import (
	"fmt"
	"net/url"
	"time"

	"github.com/elastic/libbeat/beat"
	"github.com/elastic/libbeat/cfgfile"
	"github.com/elastic/libbeat/logp"

	"github.com/mrkschan/nginxbeat/collector"
	"github.com/mrkschan/nginxbeat/publisher"
)

const selector = "nginxbeat"

// Nginxbeat implements Beater interface and sends Nginx status using libbeat.
type Nginxbeat struct {
	// NbConfig holds configurations of Nginxbeat parsed by libbeat.
	NbConfig ConfigSettings

	done chan uint

	urls   []*url.URL
	format string
	period time.Duration
}

// Config Nginxbeat according to nginxbeat.yml.
func (nb *Nginxbeat) Config(b *beat.Beat) error {
	err := cfgfile.Read(&nb.NbConfig, "")
	if err != nil {
		logp.Err("Error reading configuration file: %v", err)
		return err
	}

	var urlConfig []string
	if nb.NbConfig.Input.URLs != nil {
		urlConfig = nb.NbConfig.Input.URLs
	} else {
		urlConfig = []string{"http://127.0.0.1/status"}
	}

	nb.urls = make([]*url.URL, len(urlConfig))
	for i := 0; i < len(urlConfig); i++ {
		u, err := url.Parse(urlConfig[i])
		if err != nil {
			logp.Err("Invalid Nginx status page: %v", err)
			return err
		}
		nb.urls[i] = u
	}

	var f string
	if nb.NbConfig.Input.Format != "" {
		f = nb.NbConfig.Input.Format
	} else {
		f = "stub"
	}
	if f != "stub" && f != "plus" {
		err = fmt.Errorf("%s is an unsupported format", f)
	}
	if err != nil {
		logp.Err("Invalid Nginx status format: %v", err)
		return err
	}
	nb.format = f

	if nb.NbConfig.Input.Period != nil {
		nb.period = time.Duration(*nb.NbConfig.Input.Period) * time.Second
	} else {
		nb.period = 10 * time.Second
	}

	logp.Debug(selector, "Init nginxbeat")
	logp.Debug(selector, "Watch %v", nb.urls)
	logp.Debug(selector, "Format %v", nb.format)
	logp.Debug(selector, "Period %v", nb.period)

	return nil
}

// Setup Nginxbeat.
func (nb *Nginxbeat) Setup(b *beat.Beat) error {
	nb.done = make(chan uint)

	return nil
}

// Run Nginxbeat.
func (nb *Nginxbeat) Run(b *beat.Beat) error {
	logp.Debug(selector, "Run nginxbeat")

	for _, u := range nb.urls {
		go func() {
			var c collector.Collector
			var p publisher.Publisher

			switch nb.format {
			case "stub":
				c = collector.NewStubCollector()
				p = publisher.NewStubPublisher(b.Events)
			case "plus":
				c = collector.NewPlusCollector()
				p = publisher.NewPlusPublisher(b.Events)
			}

			ticker := time.NewTicker(nb.period)
			defer ticker.Stop()

			for {
				select {
				case <-nb.done:
					goto GotoFinish
				case <-ticker.C:
				}

				start := time.Now()

				s, err := c.Collect(*u)
				if err != nil {
					logp.Err("Fail to read Nginx status: %v", err)
					goto GotoNext
				}
				p.Publish(s)

			GotoNext:
				end := time.Now()
				duration := end.Sub(start)
				if duration.Nanoseconds() > nb.period.Nanoseconds() {
					logp.Warn("Ignoring tick(s) due to processing taking longer than one period")
				}
			}

		GotoFinish:
		}()
	}

	<-nb.done
	return nil
}

// Cleanup Nginxbeat.
func (nb *Nginxbeat) Cleanup(b *beat.Beat) error {
	return nil
}

// Stop Nginxbeat.
func (nb *Nginxbeat) Stop() {
	logp.Debug(selector, "Stop nginxbeat")
	close(nb.done)
}
<|endoftext|>


<commit_before>package readline

import (
	"bytes"
	"fmt"
	"io"
)

type AutoCompleter interface {
	Do(line []rune, pos int) (newLine [][]rune, offset int)
}

type opCompleter struct {
	w  io.Writer
	op *Operation
	ac AutoCompleter

	inCompleteMode  bool
	inSelectMode    bool
	candicate       [][]rune
	candicateSource []rune
	candicateOff    int
	candicateChoise int
	candicateColNum int
}

func newOpCompleter(w io.Writer, op *Operation) *opCompleter {
	return &opCompleter{
		w:  w,
		op: op,
		ac: op.cfg.AutoComplete,
	}
}

func (o *opCompleter) doSelect() {
	if len(o.candicate) == 1 {
		o.op.buf.WriteRunes(o.candicate[0])
		o.ExitCompleteMode(false)
		return
	}
	o.nextCandicate(1)
	o.CompleteRefresh()
}

func (o *opCompleter) nextCandicate(i int) {
	o.candicateChoise += i
	o.candicateChoise = o.candicateChoise % len(o.candicate)
	if o.candicateChoise < 0 {
		o.candicateChoise = len(o.candicate) + o.candicateChoise
	}
}

func (o *opCompleter) OnComplete() {
	if o.IsInCompleteSelectMode() {
		o.doSelect()
		return
	}

	buf := o.op.buf
	rs := buf.Runes()

	if o.IsInCompleteMode() && EqualRunes(rs, o.candicateSource) {
		o.EnterCompleteSelectMode()
		o.doSelect()
		return
	}

	o.ExitCompleteSelectMode()
	o.candicateSource = rs
	newLines, offset := o.ac.Do(rs, buf.idx)
	if len(newLines) == 0 {
		o.ExitCompleteMode(false)
		return
	}

	// only Aggregate candicates in non-complete mode
	if !o.IsInCompleteMode() {
		if len(newLines) == 1 {
			buf.WriteRunes(newLines[0])
			o.ExitCompleteMode(false)
			return
		}

		same, size := AggRunes(newLines)
		if size > 0 {
			buf.WriteRunes(same)
			o.ExitCompleteMode(false)
			return
		}
	}

	o.EnterCompleteMode(offset, newLines)
}

func (o *opCompleter) IsInCompleteSelectMode() bool {
	return o.inSelectMode
}

func (o *opCompleter) IsInCompleteMode() bool {
	return o.inCompleteMode
}

func (o *opCompleter) HandleCompleteSelect(r rune) bool {
	next := true
	switch r {
	case CharEnter, CharCtrlJ:
		next = false
		o.op.buf.WriteRunes(o.op.candicate[o.op.candicateChoise])
		o.ExitCompleteMode(false)
	case CharLineStart:
		num := o.candicateChoise % o.candicateColNum
		o.nextCandicate(-num)
	case CharLineEnd:
		num := o.candicateColNum - o.candicateChoise%o.candicateColNum - 1
		o.candicateChoise += num
		if o.candicateChoise >= len(o.candicate) {
			o.candicateChoise = len(o.candicate) - 1
		}
	case CharBackspace:
		o.ExitCompleteSelectMode()
		next = false
	case CharTab, CharForward:
		o.doSelect()
	case CharCancel, CharInterrupt:
		o.ExitCompleteMode(true)
		next = false
	case CharNext:
		tmpChoise := o.candicateChoise + o.candicateColNum
		if tmpChoise >= o.getMatrixSize() {
			tmpChoise -= o.getMatrixSize()
		} else if tmpChoise >= len(o.candicate) {
			tmpChoise += o.candicateColNum
			tmpChoise -= o.getMatrixSize()
		}
		o.candicateChoise = tmpChoise
	case CharBackward:
		if o.op.buf.Len() > 0 {
			o.nextCandicate(-1)
		}
	case CharPrev:
		tmpChoise := o.candicateChoise - o.candicateColNum
		if tmpChoise < 0 {
			tmpChoise += o.getMatrixSize()
			if tmpChoise >= len(o.candicate) {
				tmpChoise -= o.candicateColNum
			}
		}
		o.candicateChoise = tmpChoise
	default:
		next = false
		o.ExitCompleteSelectMode()
	}
	if next {
		o.CompleteRefresh()
		return true
	}
	return false
}

func (o *opCompleter) getMatrixSize() int {
	line := len(o.candicate) / o.candicateColNum
	if len(o.candicate)%o.candicateColNum != 0 {
		line++
	}
	return line * o.candicateColNum
}

func (o *opCompleter) CompleteRefresh() {
	if !o.inCompleteMode {
		return
	}
	lineCnt := o.op.buf.CursorLineCount()
	colWidth := 0
	for _, c := range o.candicate {
		w := RunesWidth(c)
		if w > colWidth {
			colWidth = w
		}
	}
	colNum := getWidth() / (colWidth + o.candicateOff + 2)
	o.candicateColNum = colNum
	buf := bytes.NewBuffer(nil)
	buf.Write(bytes.Repeat([]byte("\n"), lineCnt))
	same := o.op.buf.RuneSlice(-o.candicateOff)
	colIdx := 0
	lines := 1
	buf.WriteString("\033[J")
	for idx, c := range o.candicate {
		inSelect := idx == o.candicateChoise && o.IsInCompleteSelectMode()
		if inSelect {
			buf.WriteString("\033[30;47m")
		}
		buf.WriteString(string(same))
		buf.WriteString(string(c))
		buf.Write(bytes.Repeat([]byte(" "), colWidth-len(c)))
		if inSelect {
			buf.WriteString("\033[0m")
		}

		buf.WriteString("  ")
		colIdx++
		if colIdx == colNum {
			buf.WriteString("\n")
			lines++
			colIdx = 0
		}
	}

	// move back
	fmt.Fprintf(buf, "\033[%dA\r", lineCnt-1+lines)
	fmt.Fprintf(buf, "\033[%dC", o.op.buf.idx+o.op.buf.PromptLen())
	o.w.Write(buf.Bytes())
}

func (o *opCompleter) aggCandicate(candicate [][]rune) int {
	offset := 0
	for i := 0; i < len(candicate[0]); i++ {
		for j := 0; j < len(candicate)-1; j++ {
			if i > len(candicate[j]) {
				goto aggregate
			}
			if candicate[j][i] != candicate[j+1][i] {
				goto aggregate
			}
		}
		offset = i
	}
aggregate:
	return offset
}

func (o *opCompleter) EnterCompleteSelectMode() {
	o.inSelectMode = true
	o.candicateChoise = -1
	o.CompleteRefresh()
}

func (o *opCompleter) EnterCompleteMode(offset int, candicate [][]rune) {
	o.inCompleteMode = true
	o.candicate = candicate
	o.candicateOff = offset
	o.CompleteRefresh()
}

func (o *opCompleter) ExitCompleteSelectMode() {
	o.inSelectMode = false
	o.candicate = nil
	o.candicateChoise = -1
	o.candicateOff = -1
	o.candicateSource = nil
}

func (o *opCompleter) ExitCompleteMode(revent bool) {
	o.inCompleteMode = false
	o.ExitCompleteSelectMode()
}
<commit_msg>fix backward in complete select mode<commit_after>package readline

import (
	"bytes"
	"fmt"
	"io"
)

type AutoCompleter interface {
	Do(line []rune, pos int) (newLine [][]rune, offset int)
}

type opCompleter struct {
	w  io.Writer
	op *Operation
	ac AutoCompleter

	inCompleteMode  bool
	inSelectMode    bool
	candicate       [][]rune
	candicateSource []rune
	candicateOff    int
	candicateChoise int
	candicateColNum int
}

func newOpCompleter(w io.Writer, op *Operation) *opCompleter {
	return &opCompleter{
		w:  w,
		op: op,
		ac: op.cfg.AutoComplete,
	}
}

func (o *opCompleter) doSelect() {
	if len(o.candicate) == 1 {
		o.op.buf.WriteRunes(o.candicate[0])
		o.ExitCompleteMode(false)
		return
	}
	o.nextCandicate(1)
	o.CompleteRefresh()
}

func (o *opCompleter) nextCandicate(i int) {
	o.candicateChoise += i
	o.candicateChoise = o.candicateChoise % len(o.candicate)
	if o.candicateChoise < 0 {
		o.candicateChoise = len(o.candicate) + o.candicateChoise
	}
}

func (o *opCompleter) OnComplete() {
	if o.IsInCompleteSelectMode() {
		o.doSelect()
		return
	}

	buf := o.op.buf
	rs := buf.Runes()

	if o.IsInCompleteMode() && EqualRunes(rs, o.candicateSource) {
		o.EnterCompleteSelectMode()
		o.doSelect()
		return
	}

	o.ExitCompleteSelectMode()
	o.candicateSource = rs
	newLines, offset := o.ac.Do(rs, buf.idx)
	if len(newLines) == 0 {
		o.ExitCompleteMode(false)
		return
	}

	// only Aggregate candicates in non-complete mode
	if !o.IsInCompleteMode() {
		if len(newLines) == 1 {
			buf.WriteRunes(newLines[0])
			o.ExitCompleteMode(false)
			return
		}

		same, size := AggRunes(newLines)
		if size > 0 {
			buf.WriteRunes(same)
			o.ExitCompleteMode(false)
			return
		}
	}

	o.EnterCompleteMode(offset, newLines)
}

func (o *opCompleter) IsInCompleteSelectMode() bool {
	return o.inSelectMode
}

func (o *opCompleter) IsInCompleteMode() bool {
	return o.inCompleteMode
}

func (o *opCompleter) HandleCompleteSelect(r rune) bool {
	next := true
	switch r {
	case CharEnter, CharCtrlJ:
		next = false
		o.op.buf.WriteRunes(o.op.candicate[o.op.candicateChoise])
		o.ExitCompleteMode(false)
	case CharLineStart:
		num := o.candicateChoise % o.candicateColNum
		o.nextCandicate(-num)
	case CharLineEnd:
		num := o.candicateColNum - o.candicateChoise%o.candicateColNum - 1
		o.candicateChoise += num
		if o.candicateChoise >= len(o.candicate) {
			o.candicateChoise = len(o.candicate) - 1
		}
	case CharBackspace:
		o.ExitCompleteSelectMode()
		next = false
	case CharTab, CharForward:
		o.doSelect()
	case CharCancel, CharInterrupt:
		o.ExitCompleteMode(true)
		next = false
	case CharNext:
		tmpChoise := o.candicateChoise + o.candicateColNum
		if tmpChoise >= o.getMatrixSize() {
			tmpChoise -= o.getMatrixSize()
		} else if tmpChoise >= len(o.candicate) {
			tmpChoise += o.candicateColNum
			tmpChoise -= o.getMatrixSize()
		}
		o.candicateChoise = tmpChoise
	case CharBackward:
		o.nextCandicate(-1)
	case CharPrev:
		tmpChoise := o.candicateChoise - o.candicateColNum
		if tmpChoise < 0 {
			tmpChoise += o.getMatrixSize()
			if tmpChoise >= len(o.candicate) {
				tmpChoise -= o.candicateColNum
			}
		}
		o.candicateChoise = tmpChoise
	default:
		next = false
		o.ExitCompleteSelectMode()
	}
	if next {
		o.CompleteRefresh()
		return true
	}
	return false
}

func (o *opCompleter) getMatrixSize() int {
	line := len(o.candicate) / o.candicateColNum
	if len(o.candicate)%o.candicateColNum != 0 {
		line++
	}
	return line * o.candicateColNum
}

func (o *opCompleter) CompleteRefresh() {
	if !o.inCompleteMode {
		return
	}
	lineCnt := o.op.buf.CursorLineCount()
	colWidth := 0
	for _, c := range o.candicate {
		w := RunesWidth(c)
		if w > colWidth {
			colWidth = w
		}
	}
	colNum := getWidth() / (colWidth + o.candicateOff + 2)
	o.candicateColNum = colNum
	buf := bytes.NewBuffer(nil)
	buf.Write(bytes.Repeat([]byte("\n"), lineCnt))
	same := o.op.buf.RuneSlice(-o.candicateOff)
	colIdx := 0
	lines := 1
	buf.WriteString("\033[J")
	for idx, c := range o.candicate {
		inSelect := idx == o.candicateChoise && o.IsInCompleteSelectMode()
		if inSelect {
			buf.WriteString("\033[30;47m")
		}
		buf.WriteString(string(same))
		buf.WriteString(string(c))
		buf.Write(bytes.Repeat([]byte(" "), colWidth-len(c)))
		if inSelect {
			buf.WriteString("\033[0m")
		}

		buf.WriteString("  ")
		colIdx++
		if colIdx == colNum {
			buf.WriteString("\n")
			lines++
			colIdx = 0
		}
	}

	// move back
	fmt.Fprintf(buf, "\033[%dA\r", lineCnt-1+lines)
	fmt.Fprintf(buf, "\033[%dC", o.op.buf.idx+o.op.buf.PromptLen())
	o.w.Write(buf.Bytes())
}

func (o *opCompleter) aggCandicate(candicate [][]rune) int {
	offset := 0
	for i := 0; i < len(candicate[0]); i++ {
		for j := 0; j < len(candicate)-1; j++ {
			if i > len(candicate[j]) {
				goto aggregate
			}
			if candicate[j][i] != candicate[j+1][i] {
				goto aggregate
			}
		}
		offset = i
	}
aggregate:
	return offset
}

func (o *opCompleter) EnterCompleteSelectMode() {
	o.inSelectMode = true
	o.candicateChoise = -1
	o.CompleteRefresh()
}

func (o *opCompleter) EnterCompleteMode(offset int, candicate [][]rune) {
	o.inCompleteMode = true
	o.candicate = candicate
	o.candicateOff = offset
	o.CompleteRefresh()
}

func (o *opCompleter) ExitCompleteSelectMode() {
	o.inSelectMode = false
	o.candicate = nil
	o.candicateChoise = -1
	o.candicateOff = -1
	o.candicateSource = nil
}

func (o *opCompleter) ExitCompleteMode(revent bool) {
	o.inCompleteMode = false
	o.ExitCompleteSelectMode()
}
<|endoftext|>


<commit_before>package aws

import (
	"crypto/sha256"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/arn"
	"github.com/aws/aws-sdk-go/service/sns"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

var snsPlatformRequiresPlatformPrincipal = map[string]bool{
	"APNS":         true,
	"APNS_SANDBOX": true,
}

// Mutable attributes
// http://docs.aws.amazon.com/sns/latest/api/API_SetPlatformApplicationAttributes.html
var snsPlatformApplicationAttributeMap = map[string]string{
	"event_delivery_failure_topic_arn": "EventDeliveryFailure",
	"event_endpoint_created_topic_arn": "EventEndpointCreated",
	"event_endpoint_deleted_topic_arn": "EventEndpointDeleted",
	"event_endpoint_updated_topic_arn": "EventEndpointUpdated",
	"failure_feedback_role_arn":        "FailureFeedbackRoleArn",
	"platform_principal":               "PlatformPrincipal",
	"success_feedback_role_arn":        "SuccessFeedbackRoleArn",
	"success_feedback_sample_rate":     "SuccessFeedbackSampleRate",
}

func resourceAwsSnsPlatformApplication() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSnsPlatformApplicationCreate,
		Read:   resourceAwsSnsPlatformApplicationRead,
		Update: resourceAwsSnsPlatformApplicationUpdate,
		Delete: resourceAwsSnsPlatformApplicationDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		CustomizeDiff: func(diff *schema.ResourceDiff, v interface{}) error {
			return validateAwsSnsPlatformApplication(diff)
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"platform": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"platform_credential": {
				Type:      schema.TypeString,
				Required:  true,
				StateFunc: hashSum,
			},
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"event_delivery_failure_topic_arn": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"event_endpoint_created_topic_arn": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"event_endpoint_deleted_topic_arn": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"event_endpoint_updated_topic_arn": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"failure_feedback_role_arn": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"platform_principal": {
				Type:      schema.TypeString,
				Optional:  true,
				StateFunc: hashSum,
			},
			"success_feedback_role_arn": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"success_feedback_sample_rate": {
				Type:     schema.TypeString,
				Optional: true,
			},
		},
	}
}

func resourceAwsSnsPlatformApplicationCreate(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	attributes := make(map[string]*string)
	name := d.Get("name").(string)
	platform := d.Get("platform").(string)

	attributes["PlatformCredential"] = aws.String(d.Get("platform_credential").(string))
	if v, ok := d.GetOk("platform_principal"); ok {
		attributes["PlatformPrincipal"] = aws.String(v.(string))
	}

	req := &sns.CreatePlatformApplicationInput{
		Name:       aws.String(name),
		Platform:   aws.String(platform),
		Attributes: attributes,
	}

	log.Printf("[DEBUG] SNS create application: %s", req)

	output, err := snsconn.CreatePlatformApplication(req)
	if err != nil {
		return fmt.Errorf("Error creating SNS platform application: %s", err)
	}

	d.SetId(*output.PlatformApplicationArn)

	return resourceAwsSnsPlatformApplicationUpdate(d, meta)
}

func resourceAwsSnsPlatformApplicationUpdate(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	attributes := make(map[string]*string)

	for k := range resourceAwsSnsPlatformApplication().Schema {
		if attrKey, ok := snsPlatformApplicationAttributeMap[k]; ok {
			if d.HasChange(k) {
				log.Printf("[DEBUG] Updating %s", attrKey)
				_, n := d.GetChange(k)
				attributes[attrKey] = aws.String(n.(string))
			}
		}
	}

	if d.HasChange("platform_credential") {
		attributes["PlatformCredential"] = aws.String(d.Get("platform_credential").(string))
		// If the platform requires a principal it must also be specified, even if it didn't change
		// since credential is stored as a hash, the only way to update principal is to update both
		// as they must be specified together in the request.
		if v, ok := d.GetOk("platform_principal"); ok {
			attributes["PlatformPrincipal"] = aws.String(v.(string))
		}
	}

	// Make API call to update attributes
	req := &sns.SetPlatformApplicationAttributesInput{
		PlatformApplicationArn: aws.String(d.Id()),
		Attributes:             attributes,
	}

	err := resource.Retry(1*time.Minute, func() *resource.RetryError {
		_, err := snsconn.SetPlatformApplicationAttributes(req)
		if err != nil {
			if isAWSErr(err, sns.ErrCodeInvalidParameterException, "is not a valid role to allow SNS to write to Cloudwatch Logs") {
				return resource.RetryableError(err)
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})

	if err != nil {
		return fmt.Errorf("Error updating SNS platform application: %s", err)
	}

	return resourceAwsSnsPlatformApplicationRead(d, meta)
}

func resourceAwsSnsPlatformApplicationRead(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	// There is no SNS Describe/GetPlatformApplication to fetch attributes like name and platform
	// We will use the ID, which should be a platform application ARN, to:
	//  * Validate its an appropriate ARN on import
	//  * Parse out the name and platform
	arn, name, platform, err := decodeResourceAwsSnsPlatformApplicationID(d.Id())
	if err != nil {
		return err
	}

	d.Set("arn", arn)
	d.Set("name", name)
	d.Set("platform", platform)

	attributeOutput, err := snsconn.GetPlatformApplicationAttributes(&sns.GetPlatformApplicationAttributesInput{
		PlatformApplicationArn: aws.String(arn),
	})

	if err != nil {
		if isAWSErr(err, sns.ErrCodeNotFoundException, "") {
			log.Printf("[WARN] SNS Platform Application (%s) not found, removing from state", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	if attributeOutput.Attributes != nil && len(attributeOutput.Attributes) > 0 {
		attrmap := attributeOutput.Attributes
		resource := *resourceAwsSnsPlatformApplication()
		// iKey = internal struct key, oKey = AWS Attribute Map key
		for iKey, oKey := range snsPlatformApplicationAttributeMap {
			log.Printf("[DEBUG] Updating %s => %s", iKey, oKey)

			if attrmap[oKey] != nil {
				// Some of the fetched attributes are stateful properties such as
				// the number of subscriptions, the owner, etc. skip those
				if resource.Schema[iKey] != nil {
					value := *attrmap[oKey]
					log.Printf("[DEBUG] Updating %s => %s -> %s", iKey, oKey, value)
					d.Set(iKey, *attrmap[oKey])
				}
			}
		}
	}

	return nil
}

func resourceAwsSnsPlatformApplicationDelete(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	log.Printf("[DEBUG] SNS Delete Application: %s", d.Id())
	_, err := snsconn.DeletePlatformApplication(&sns.DeletePlatformApplicationInput{
		PlatformApplicationArn: aws.String(d.Id()),
	})
	return err
}

func decodeResourceAwsSnsPlatformApplicationID(input string) (arnS, name, platform string, err error) {
	platformApplicationArn, err := arn.Parse(input)
	if err != nil {
		err = fmt.Errorf(
			"SNS Platform Application ID must be of the form "+
				"arn:PARTITION:sns:REGION:ACCOUNTID:app/PLATFORM/NAME, "+
				"was provided %q and received error: %s", input, err)
		return
	}

	platformApplicationArnResourceParts := strings.Split(platformApplicationArn.Resource, "/")
	if len(platformApplicationArnResourceParts) != 3 || platformApplicationArnResourceParts[0] != "app" {
		err = fmt.Errorf(
			"SNS Platform Application ID must be of the form "+
				"arn:PARTITION:sns:REGION:ACCOUNTID:app/PLATFORM/NAME, "+
				"was provided: %s", input)
		return
	}

	arnS = platformApplicationArn.String()
	name = platformApplicationArnResourceParts[2]
	platform = platformApplicationArnResourceParts[1]
	return
}

func hashSum(contents interface{}) string {
	return fmt.Sprintf("%x", sha256.Sum256([]byte(contents.(string))))
}

func validateAwsSnsPlatformApplication(d *schema.ResourceDiff) error {
	platform := d.Get("platform").(string)
	if snsPlatformRequiresPlatformPrincipal[platform] {
		if v, ok := d.GetOk("platform_principal"); ok {
			value := v.(string)
			if len(value) == 0 {
				return fmt.Errorf("platform_principal must be non-empty when platform = %s", platform)
			}
			return nil
		}
		return fmt.Errorf("platform_principal is required when platform = %s", platform)
	}
	return nil
}
<commit_msg>Final retry after timeout updating SNS platform application<commit_after>package aws

import (
	"crypto/sha256"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/arn"
	"github.com/aws/aws-sdk-go/service/sns"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

var snsPlatformRequiresPlatformPrincipal = map[string]bool{
	"APNS":         true,
	"APNS_SANDBOX": true,
}

// Mutable attributes
// http://docs.aws.amazon.com/sns/latest/api/API_SetPlatformApplicationAttributes.html
var snsPlatformApplicationAttributeMap = map[string]string{
	"event_delivery_failure_topic_arn": "EventDeliveryFailure",
	"event_endpoint_created_topic_arn": "EventEndpointCreated",
	"event_endpoint_deleted_topic_arn": "EventEndpointDeleted",
	"event_endpoint_updated_topic_arn": "EventEndpointUpdated",
	"failure_feedback_role_arn":        "FailureFeedbackRoleArn",
	"platform_principal":               "PlatformPrincipal",
	"success_feedback_role_arn":        "SuccessFeedbackRoleArn",
	"success_feedback_sample_rate":     "SuccessFeedbackSampleRate",
}

func resourceAwsSnsPlatformApplication() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSnsPlatformApplicationCreate,
		Read:   resourceAwsSnsPlatformApplicationRead,
		Update: resourceAwsSnsPlatformApplicationUpdate,
		Delete: resourceAwsSnsPlatformApplicationDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		CustomizeDiff: func(diff *schema.ResourceDiff, v interface{}) error {
			return validateAwsSnsPlatformApplication(diff)
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"platform": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"platform_credential": {
				Type:      schema.TypeString,
				Required:  true,
				StateFunc: hashSum,
			},
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"event_delivery_failure_topic_arn": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"event_endpoint_created_topic_arn": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"event_endpoint_deleted_topic_arn": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"event_endpoint_updated_topic_arn": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"failure_feedback_role_arn": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"platform_principal": {
				Type:      schema.TypeString,
				Optional:  true,
				StateFunc: hashSum,
			},
			"success_feedback_role_arn": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"success_feedback_sample_rate": {
				Type:     schema.TypeString,
				Optional: true,
			},
		},
	}
}

func resourceAwsSnsPlatformApplicationCreate(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	attributes := make(map[string]*string)
	name := d.Get("name").(string)
	platform := d.Get("platform").(string)

	attributes["PlatformCredential"] = aws.String(d.Get("platform_credential").(string))
	if v, ok := d.GetOk("platform_principal"); ok {
		attributes["PlatformPrincipal"] = aws.String(v.(string))
	}

	req := &sns.CreatePlatformApplicationInput{
		Name:       aws.String(name),
		Platform:   aws.String(platform),
		Attributes: attributes,
	}

	log.Printf("[DEBUG] SNS create application: %s", req)

	output, err := snsconn.CreatePlatformApplication(req)
	if err != nil {
		return fmt.Errorf("Error creating SNS platform application: %s", err)
	}

	d.SetId(*output.PlatformApplicationArn)

	return resourceAwsSnsPlatformApplicationUpdate(d, meta)
}

func resourceAwsSnsPlatformApplicationUpdate(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	attributes := make(map[string]*string)

	for k := range resourceAwsSnsPlatformApplication().Schema {
		if attrKey, ok := snsPlatformApplicationAttributeMap[k]; ok {
			if d.HasChange(k) {
				log.Printf("[DEBUG] Updating %s", attrKey)
				_, n := d.GetChange(k)
				attributes[attrKey] = aws.String(n.(string))
			}
		}
	}

	if d.HasChange("platform_credential") {
		attributes["PlatformCredential"] = aws.String(d.Get("platform_credential").(string))
		// If the platform requires a principal it must also be specified, even if it didn't change
		// since credential is stored as a hash, the only way to update principal is to update both
		// as they must be specified together in the request.
		if v, ok := d.GetOk("platform_principal"); ok {
			attributes["PlatformPrincipal"] = aws.String(v.(string))
		}
	}

	// Make API call to update attributes
	req := &sns.SetPlatformApplicationAttributesInput{
		PlatformApplicationArn: aws.String(d.Id()),
		Attributes:             attributes,
	}

	err := resource.Retry(1*time.Minute, func() *resource.RetryError {
		_, err := snsconn.SetPlatformApplicationAttributes(req)
		if err != nil {
			if isAWSErr(err, sns.ErrCodeInvalidParameterException, "is not a valid role to allow SNS to write to Cloudwatch Logs") {
				return resource.RetryableError(err)
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})
	if isResourceTimeoutError(err) {
		_, err = snsconn.SetPlatformApplicationAttributes(req)
	}

	if err != nil {
		return fmt.Errorf("Error updating SNS platform application: %s", err)
	}

	return resourceAwsSnsPlatformApplicationRead(d, meta)
}

func resourceAwsSnsPlatformApplicationRead(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	// There is no SNS Describe/GetPlatformApplication to fetch attributes like name and platform
	// We will use the ID, which should be a platform application ARN, to:
	//  * Validate its an appropriate ARN on import
	//  * Parse out the name and platform
	arn, name, platform, err := decodeResourceAwsSnsPlatformApplicationID(d.Id())
	if err != nil {
		return err
	}

	d.Set("arn", arn)
	d.Set("name", name)
	d.Set("platform", platform)

	attributeOutput, err := snsconn.GetPlatformApplicationAttributes(&sns.GetPlatformApplicationAttributesInput{
		PlatformApplicationArn: aws.String(arn),
	})

	if err != nil {
		if isAWSErr(err, sns.ErrCodeNotFoundException, "") {
			log.Printf("[WARN] SNS Platform Application (%s) not found, removing from state", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	if attributeOutput.Attributes != nil && len(attributeOutput.Attributes) > 0 {
		attrmap := attributeOutput.Attributes
		resource := *resourceAwsSnsPlatformApplication()
		// iKey = internal struct key, oKey = AWS Attribute Map key
		for iKey, oKey := range snsPlatformApplicationAttributeMap {
			log.Printf("[DEBUG] Updating %s => %s", iKey, oKey)

			if attrmap[oKey] != nil {
				// Some of the fetched attributes are stateful properties such as
				// the number of subscriptions, the owner, etc. skip those
				if resource.Schema[iKey] != nil {
					value := *attrmap[oKey]
					log.Printf("[DEBUG] Updating %s => %s -> %s", iKey, oKey, value)
					d.Set(iKey, *attrmap[oKey])
				}
			}
		}
	}

	return nil
}

func resourceAwsSnsPlatformApplicationDelete(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	log.Printf("[DEBUG] SNS Delete Application: %s", d.Id())
	_, err := snsconn.DeletePlatformApplication(&sns.DeletePlatformApplicationInput{
		PlatformApplicationArn: aws.String(d.Id()),
	})
	return err
}

func decodeResourceAwsSnsPlatformApplicationID(input string) (arnS, name, platform string, err error) {
	platformApplicationArn, err := arn.Parse(input)
	if err != nil {
		err = fmt.Errorf(
			"SNS Platform Application ID must be of the form "+
				"arn:PARTITION:sns:REGION:ACCOUNTID:app/PLATFORM/NAME, "+
				"was provided %q and received error: %s", input, err)
		return
	}

	platformApplicationArnResourceParts := strings.Split(platformApplicationArn.Resource, "/")
	if len(platformApplicationArnResourceParts) != 3 || platformApplicationArnResourceParts[0] != "app" {
		err = fmt.Errorf(
			"SNS Platform Application ID must be of the form "+
				"arn:PARTITION:sns:REGION:ACCOUNTID:app/PLATFORM/NAME, "+
				"was provided: %s", input)
		return
	}

	arnS = platformApplicationArn.String()
	name = platformApplicationArnResourceParts[2]
	platform = platformApplicationArnResourceParts[1]
	return
}

func hashSum(contents interface{}) string {
	return fmt.Sprintf("%x", sha256.Sum256([]byte(contents.(string))))
}

func validateAwsSnsPlatformApplication(d *schema.ResourceDiff) error {
	platform := d.Get("platform").(string)
	if snsPlatformRequiresPlatformPrincipal[platform] {
		if v, ok := d.GetOk("platform_principal"); ok {
			value := v.(string)
			if len(value) == 0 {
				return fmt.Errorf("platform_principal must be non-empty when platform = %s", platform)
			}
			return nil
		}
		return fmt.Errorf("platform_principal is required when platform = %s", platform)
	}
	return nil
}
<|endoftext|>


<commit_before>package etcd

import (
	"context"
	"fmt"
	"strings"

	etcdv3 "github.com/coreos/etcd/clientv3"
	"github.com/hashicorp/terraform/backend"
	"github.com/hashicorp/terraform/state"
	"github.com/hashicorp/terraform/state/remote"
	"github.com/hashicorp/terraform/terraform"
)

const (
	keyEnvPrefix = "-env:"
)

func (b *Backend) States() ([]string, error) {
	client, err := b.rawClient()
	if err != nil {
		return nil, err
	}

	prefix := b.determineKey("")
	res, err := client.Get(context.TODO(), prefix, etcdv3.WithPrefix(), etcdv3.WithKeysOnly())
	if err != nil {
		return nil, err
	}

	result := make([]string, 1, len(res.Kvs)+1)
	result[0] = backend.DefaultStateName
	for _, kv := range res.Kvs {
		result = append(result, strings.TrimPrefix(string(kv.Key), prefix))
	}

	return result, nil
}

func (b *Backend) DeleteState(name string) error {
	if name == backend.DefaultStateName || name == "" {
		return fmt.Errorf("Can't delete default state.")
	}

	client, err := b.rawClient()
	if err != nil {
		return err
	}

	path := b.determineKey(name)

	_, err = client.Delete(context.TODO(), path)
	return err
}

func (b *Backend) State(name string) (state.State, error) {
	client, err := b.rawClient()
	if err != nil {
		return nil, err
	}

	var stateMgr state.State = &remote.State{
		Client: &RemoteClient{
			Client: client,
			DoLock: b.lock,
			Key:    b.determineKey(name),
		},
	}

	if !b.lock {
		stateMgr = &state.LockDisabled{Inner: stateMgr}
	}

	lockInfo := state.NewLockInfo()
	lockInfo.Operation = "init"
	lockId, err := stateMgr.Lock(lockInfo)
	if err != nil {
		return nil, fmt.Errorf("Failed to lock state in etcd: %s.", err)
	}

	lockUnlock := func(parent error) error {
		if err := stateMgr.Unlock(lockId); err != nil {
			return fmt.Errorf(strings.TrimSpace(errStateUnlock), lockId, err)
		}
		return parent
	}

	if err := stateMgr.RefreshState(); err != nil {
		err = lockUnlock(err)
		return nil, err
	}

	if v := stateMgr.State(); v == nil {
		if err := stateMgr.WriteState(terraform.NewState()); err != nil {
			err = lockUnlock(err)
			return nil, err
		}
		if err := stateMgr.PersistState(); err != nil {
			err = lockUnlock(err)
			return nil, err
		}
	}

	if err := lockUnlock(nil); err != nil {
		return nil, err
	}

	return stateMgr, nil
}

func (b *Backend) determineKey(name string) string {
	prefix := b.prefix
	if name != backend.DefaultStateName {
		prefix += fmt.Sprintf("%s%s", keyEnvPrefix, name)
	}
	return prefix
}

const errStateUnlock = `
Error unlocking etcd state. Lock ID: %s

Error: %s

You may have to force-unlock this state in order to use it again.
`
<commit_msg>Sort the result of 'Backend.States()'.<commit_after>package etcd

import (
	"context"
	"fmt"
	"sort"
	"strings"

	etcdv3 "github.com/coreos/etcd/clientv3"
	"github.com/hashicorp/terraform/backend"
	"github.com/hashicorp/terraform/state"
	"github.com/hashicorp/terraform/state/remote"
	"github.com/hashicorp/terraform/terraform"
)

const (
	keyEnvPrefix = "-env:"
)

func (b *Backend) States() ([]string, error) {
	client, err := b.rawClient()
	if err != nil {
		return nil, err
	}

	prefix := b.determineKey("")
	res, err := client.Get(context.TODO(), prefix, etcdv3.WithPrefix(), etcdv3.WithKeysOnly())
	if err != nil {
		return nil, err
	}

	result := make([]string, 1, len(res.Kvs)+1)
	result[0] = backend.DefaultStateName
	for _, kv := range res.Kvs {
		result = append(result, strings.TrimPrefix(string(kv.Key), prefix))
	}
	sort.Strings(result[1:])

	return result, nil
}

func (b *Backend) DeleteState(name string) error {
	if name == backend.DefaultStateName || name == "" {
		return fmt.Errorf("Can't delete default state.")
	}

	client, err := b.rawClient()
	if err != nil {
		return err
	}

	path := b.determineKey(name)

	_, err = client.Delete(context.TODO(), path)
	return err
}

func (b *Backend) State(name string) (state.State, error) {
	client, err := b.rawClient()
	if err != nil {
		return nil, err
	}

	var stateMgr state.State = &remote.State{
		Client: &RemoteClient{
			Client: client,
			DoLock: b.lock,
			Key:    b.determineKey(name),
		},
	}

	if !b.lock {
		stateMgr = &state.LockDisabled{Inner: stateMgr}
	}

	lockInfo := state.NewLockInfo()
	lockInfo.Operation = "init"
	lockId, err := stateMgr.Lock(lockInfo)
	if err != nil {
		return nil, fmt.Errorf("Failed to lock state in etcd: %s.", err)
	}

	lockUnlock := func(parent error) error {
		if err := stateMgr.Unlock(lockId); err != nil {
			return fmt.Errorf(strings.TrimSpace(errStateUnlock), lockId, err)
		}
		return parent
	}

	if err := stateMgr.RefreshState(); err != nil {
		err = lockUnlock(err)
		return nil, err
	}

	if v := stateMgr.State(); v == nil {
		if err := stateMgr.WriteState(terraform.NewState()); err != nil {
			err = lockUnlock(err)
			return nil, err
		}
		if err := stateMgr.PersistState(); err != nil {
			err = lockUnlock(err)
			return nil, err
		}
	}

	if err := lockUnlock(nil); err != nil {
		return nil, err
	}

	return stateMgr, nil
}

func (b *Backend) determineKey(name string) string {
	prefix := b.prefix
	if name != backend.DefaultStateName {
		prefix += fmt.Sprintf("%s%s", keyEnvPrefix, name)
	}
	return prefix
}

const errStateUnlock = `
Error unlocking etcd state. Lock ID: %s

Error: %s

You may have to force-unlock this state in order to use it again.
`
<|endoftext|>


<commit_before>package notes

import (
	"strings"
	"sync"
	"time"
)

var AllNotes *Agenda

type Agenda struct {
	Notebooks map[string]string
	Notes     map[string][]Note
	rMutex    sync.RWMutex
	lastSync  time.Time
}

func init() {
	AllNotes = NewAgenda()
}

func NewAgenda() *Agenda {
	a := new(Agenda)
	a.Notes = make(map[string][]Note)
	a.Notebooks = make(map[string]string)
	return a
}

func (a *Agenda) GetNote(id int64) *Note {
	a.rMutex.Lock()
	defer a.rMutex.Unlock()

	for _, notes := range a.Notes {
		for i := range notes {
			if notes[i].Id == id {
				return &notes[i]
			}
		}
	}
	return nil
}

func (a *Agenda) AddNotebook(name string, content string) error {
	a.rMutex.RLock()
	defer a.rMutex.RUnlock()

	notes := Parse(content)
	a.Notes[name] = make([]Note, 0, len(notes))
	for i := range notes {
		notes[i].Source = name
		a.Notes[name] = append(a.Notes[name], notes[i])
	}

	a.Notebooks[name] = content
	a.lastSync = time.Now()

	return nil
}

func (a *Agenda) AddNote(n Note) {
	a.rMutex.RLock()
	defer a.rMutex.RUnlock()

	if !n.IsValid() {
		return
	}

	if _, ok := a.Notes[n.Source]; !ok {
		a.Notes[n.Source] = make([]Note, 0)
	}

	a.Notes[n.Source] = append(a.Notes[n.Source], n)
}

func (a *Agenda) GetNotebooks() map[string]string {
	notebooks := make(map[string]string)
	a.rMutex.Lock()
	defer a.rMutex.Unlock()

	for k, v := range a.Notebooks {
		notebooks[k] = v
	}
	return notebooks
}

func (a *Agenda) GetNotesFromDate(daynotes *DayNotes) {
	a.rMutex.Lock()
	defer a.rMutex.Unlock()

	for _, notes := range a.Notes {
		for i := range notes {
			daynotes.Add(notes[i])
		}
	}
}

func (a *Agenda) GetNotesWithText(pattern string) []Note {
	a.rMutex.Lock()
	defer a.rMutex.Unlock()

	search := make([]Note, 0)
	pattern = strings.ToLower(pattern)

	for _, nbook := range a.Notes {
		for _, note := range nbook {
			title := strings.ToLower(note.Title)
			body := strings.ToLower(note.Body)
			if strings.Contains(title, pattern) ||
				strings.Contains(body, pattern) {
				search = append(search, note)
			}
		}
	}

	return search
}

func (a *Agenda) GetNotesFromNotebook(notebook string) []Note {
	a.rMutex.Lock()
	defer a.rMutex.Unlock()

	if _, ok := a.Notes[notebook]; !ok {
		return nil
	}

	return a.Notes[notebook]
}

func (a *Agenda) GetNotesToDo() []Note {
	a.rMutex.Lock()
	defer a.rMutex.Unlock()

	todo := make([]Note, 0)

	for _, notes := range a.Notes {
		for _, note := range notes {
			if note.IsTodo() && note.IsWarningTime() {
				todo = append(todo, note)
			}
		}
	}

	return todo
}

func (a *Agenda) GetBusyDates() []string {
	a.rMutex.Lock()
	defer a.rMutex.Unlock()

	dates := make(map[time.Time]bool)
	for _, notes := range a.Notes {
		for _, note := range notes {
			for _, stamp := range note.Stamps {
				if ok, _ := dates[stamp]; !ok {
					dates[stamp] = true
				}
			}
		}
	}

	alldates := make([]string, 0, len(dates))
	for date := range dates {
		sdate := date.Format(DATEFORMATPRINT)
		alldates = append(alldates, sdate)
	}

	return alldates
}
<commit_msg>notebooks views now show html<commit_after>package notes

import (
	"strings"
	"sync"
	"time"
)

var AllNotes *Agenda

type Agenda struct {
	Notebooks map[string]string
	Notes     map[string][]Note
	rMutex    sync.RWMutex
	lastSync  time.Time
}

func init() {
	AllNotes = NewAgenda()
}

func NewAgenda() *Agenda {
	a := new(Agenda)
	a.Notes = make(map[string][]Note)
	a.Notebooks = make(map[string]string)
	return a
}

func (a *Agenda) GetNote(id int64) *Note {
	a.rMutex.Lock()
	defer a.rMutex.Unlock()

	for _, notes := range a.Notes {
		for i := range notes {
			if notes[i].Id == id {
				return &notes[i]
			}
		}
	}
	return nil
}

func (a *Agenda) AddNotebook(name string, content string) error {
	a.rMutex.RLock()
	defer a.rMutex.RUnlock()

	notes := Parse(content)
	a.Notes[name] = make([]Note, 0, len(notes))
	for i := range notes {
		notes[i].Source = name
		a.Notes[name] = append(a.Notes[name], notes[i])
	}

	//a.Notebooks[name] = content
	a.Notebooks[name] = Org2HTML([]byte(content), "")
	a.lastSync = time.Now()

	return nil
}

func (a *Agenda) AddNote(n Note) {
	a.rMutex.RLock()
	defer a.rMutex.RUnlock()

	if !n.IsValid() {
		return
	}

	if _, ok := a.Notes[n.Source]; !ok {
		a.Notes[n.Source] = make([]Note, 0)
	}

	a.Notes[n.Source] = append(a.Notes[n.Source], n)
}

func (a *Agenda) GetNotebooks() map[string]string {
	notebooks := make(map[string]string)
	a.rMutex.Lock()
	defer a.rMutex.Unlock()

	for k, v := range a.Notebooks {
		notebooks[k] = v
	}
	return notebooks
}

func (a *Agenda) GetNotesFromDate(daynotes *DayNotes) {
	a.rMutex.Lock()
	defer a.rMutex.Unlock()

	for _, notes := range a.Notes {
		for i := range notes {
			daynotes.Add(notes[i])
		}
	}
}

func (a *Agenda) GetNotesWithText(pattern string) []Note {
	a.rMutex.Lock()
	defer a.rMutex.Unlock()

	search := make([]Note, 0)
	pattern = strings.ToLower(pattern)

	for _, nbook := range a.Notes {
		for _, note := range nbook {
			title := strings.ToLower(note.Title)
			body := strings.ToLower(note.Body)
			if strings.Contains(title, pattern) ||
				strings.Contains(body, pattern) {
				search = append(search, note)
			}
		}
	}

	return search
}

func (a *Agenda) GetNotesFromNotebook(notebook string) []Note {
	a.rMutex.Lock()
	defer a.rMutex.Unlock()

	if _, ok := a.Notes[notebook]; !ok {
		return nil
	}

	return a.Notes[notebook]
}

func (a *Agenda) GetNotesToDo() []Note {
	a.rMutex.Lock()
	defer a.rMutex.Unlock()

	todo := make([]Note, 0)

	for _, notes := range a.Notes {
		for _, note := range notes {
			if note.IsTodo() && note.IsWarningTime() {
				todo = append(todo, note)
			}
		}
	}

	return todo
}

func (a *Agenda) GetBusyDates() []string {
	a.rMutex.Lock()
	defer a.rMutex.Unlock()

	dates := make(map[time.Time]bool)
	for _, notes := range a.Notes {
		for _, note := range notes {
			for _, stamp := range note.Stamps {
				if ok, _ := dates[stamp]; !ok {
					dates[stamp] = true
				}
			}
		}
	}

	alldates := make([]string, 0, len(dates))
	for date := range dates {
		sdate := date.Format(DATEFORMATPRINT)
		alldates = append(alldates, sdate)
	}

	return alldates
}
<|endoftext|>


<commit_before>package main

import (
        "fmt"
        "io/ioutil"
        "flag"
        "os"
        "github.com/google/go-github/github"
        "golang.org/x/oauth2"
        "gopkg.in/yaml.v2"
)

type Config struct {
        Users  []string
        Token  string
}

func printIssue(issue github.Issue) {
        if *issue.Number < 10 {
                fmt.Printf("    [%d]  - %s\n", *issue.Number, *issue.Title)
        } else {
                fmt.Printf("    [%d] - %s\n", *issue.Number, *issue.Title)
        }
}

func ReadConfig(file string) Config {
        var config Config

        raw, err := ioutil.ReadFile(file)
        if err != nil {
                fmt.Println(err)
                os.Exit(1)
        }

        err = yaml.Unmarshal(raw, &config)
        if err != nil {
                fmt.Println(err)
                os.Exit(1)
        }

        return config
}

func BuildClient(config Config) *github.Client {
        ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: config.Token},)
        tc := oauth2.NewClient(oauth2.NoContext, ts)
        client := github.NewClient(tc)

        return client
}

func ProcessRepositories(config Config, client *github.Client) {
        for _, user := range config.Users {
                repos, _, err := client.Repositories.List(user, nil)

                if err != nil {
                        fmt.Println("Error:", err)
                } else {
                        fmt.Printf("Open issues for %s\n", user)
                        for _, repo := range repos {
                                issues, _, err := client.Issues.ListByRepo(user, *repo.Name, nil)
                                if err != nil {
                                        fmt.Println("Error:", err)
                                } else {
                                        if len(issues) != 0 {
                                                fmt.Printf("  %s\n", *repo.Name)
                                                for _, issue := range issues {
                                                        printIssue(issue)
                                                }
                                        }
                                }
                        }
                }
        }
}

func main() {
        configPtr := flag.String("config", "config.yml", "Path to configuration yaml file")
        flag.Parse()

        config := ReadConfig(*configPtr)
        client := BuildClient(config)
        ProcessRepositories(config, client)
}
<commit_msg>Fix print issue<commit_after>package main

import (
        "fmt"
        "io/ioutil"
        "flag"
        "os"
        "github.com/google/go-github/github"
        "golang.org/x/oauth2"
        "gopkg.in/yaml.v2"
)

type Config struct {
        Users  []string
        Token  string
}

func printIssue(issue *github.Issue) {
        if *issue.Number < 10 {
                fmt.Printf("    [%d]  - %s\n", *issue.Number, *issue.Title)
        } else {
                fmt.Printf("    [%d] - %s\n", *issue.Number, *issue.Title)
        }
}

func ReadConfig(file string) Config {
        var config Config

        raw, err := ioutil.ReadFile(file)
        if err != nil {
                fmt.Println(err)
                os.Exit(1)
        }

        err = yaml.Unmarshal(raw, &config)
        if err != nil {
                fmt.Println(err)
                os.Exit(1)
        }

        return config
}

func BuildClient(config Config) *github.Client {
        ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: config.Token},)
        tc := oauth2.NewClient(oauth2.NoContext, ts)
        client := github.NewClient(tc)

        return client
}

func ProcessRepositories(config Config, client *github.Client) {
        for _, user := range config.Users {
                repos, _, err := client.Repositories.List(user, nil)

                if err != nil {
                        fmt.Println("Error:", err)
                } else {
                        fmt.Printf("Open issues for %s\n", user)
                        for _, repo := range repos {
                                issues, _, err := client.Issues.ListByRepo(user, *repo.Name, nil)
                                if err != nil {
                                        fmt.Println("Error:", err)
                                } else {
                                        if len(issues) != 0 {
                                                fmt.Printf("  %s\n", *repo.Name)
                                                for _, issue := range issues {
                                                        printIssue(issue)
                                                }
                                        }
                                }
                        }
                }
        }
}

func main() {
        configPtr := flag.String("config", "config.yml", "Path to configuration yaml file")
        flag.Parse()

        config := ReadConfig(*configPtr)
        client := BuildClient(config)
        ProcessRepositories(config, client)
}
<|endoftext|>