<commit_before>// +build selinux,linux

package label

import (
	"fmt"
	"strings"

	"github.com/opencontainers/runc/libcontainer/selinux"
)

// Valid Label Options
var validOptions = map[string]bool{
	"disable": true,
	"type":    true,
	"user":    true,
	"role":    true,
	"level":   true,
}

var ErrIncompatibleLabel = fmt.Errorf("Bad SELinux option z and Z can not be used together")

// InitLabels returns the process label and file labels to be used within
// the container.  A list of options can be passed into this function to alter
// the labels.  The labels returned will include a random MCS String, that is
// guaranteed to be unique.
func InitLabels(options []string) (string, string, error) {
	if !selinux.SelinuxEnabled() {
		return "", "", nil
	}
	processLabel, mountLabel := selinux.GetLxcContexts()
	if processLabel != "" {
		pcon := selinux.NewContext(processLabel)
		mcon := selinux.NewContext(mountLabel)
		for _, opt := range options {
			if opt == "disable" {
				return "", "", nil
			}
			if i := strings.Index(opt, ":"); i == -1 {
				return "", "", fmt.Errorf("Bad label option %q, valid options 'disable' or \n'user, role, level, type' followed by ':' and a value", opt)
			}
			con := strings.SplitN(opt, ":", 2)
			if !validOptions[con[0]] {
				return "", "", fmt.Errorf("Bad label option %q, valid options 'disable, user, role, level, type'", con[0])

			}
			pcon[con[0]] = con[1]
			if con[0] == "level" || con[0] == "user" {
				mcon[con[0]] = con[1]
			}
		}
		processLabel = pcon.Get()
		mountLabel = mcon.Get()
	}
	return processLabel, mountLabel, nil
}

func GetROMountLabel() string {
	return selinux.GetROFileLabel()
}

// DEPRECATED: The GenLabels function is only to be used during the transition to the official API.
func GenLabels(options string) (string, string, error) {
	return InitLabels(strings.Fields(options))
}

// FormatMountLabel returns a string to be used by the mount command.
// The format of this string will be used to alter the labeling of the mountpoint.
// The string returned is suitable to be used as the options field of the mount command.
// If you need to have additional mount point options, you can pass them in as
// the first parameter.  Second parameter is the label that you wish to apply
// to all content in the mount point.
func FormatMountLabel(src, mountLabel string) string {
	if mountLabel != "" {
		switch src {
		case "":
			src = fmt.Sprintf("context=%q", mountLabel)
		default:
			src = fmt.Sprintf("%s,context=%q", src, mountLabel)
		}
	}
	return src
}

// SetProcessLabel takes a process label and tells the kernel to assign the
// label to the next program executed by the current process.
func SetProcessLabel(processLabel string) error {
	if processLabel == "" {
		return nil
	}
	return selinux.Setexeccon(processLabel)
}

// GetProcessLabel returns the process label that the kernel will assign
// to the next program executed by the current process.  If "" is returned
// this indicates that the default labeling will happen for the process.
func GetProcessLabel() (string, error) {
	return selinux.Getexeccon()
}

// GetFileLabel returns the label for specified path
func GetFileLabel(path string) (string, error) {
	return selinux.Getfilecon(path)
}

// SetFileLabel modifies the "path" label to the specified file label
func SetFileLabel(path string, fileLabel string) error {
	if selinux.SelinuxEnabled() && fileLabel != "" {
		return selinux.Setfilecon(path, fileLabel)
	}
	return nil
}

// SetFileCreateLabel tells the kernel the label for all files to be created
func SetFileCreateLabel(fileLabel string) error {
	if selinux.SelinuxEnabled() {
		return selinux.Setfscreatecon(fileLabel)
	}
	return nil
}

// Relabel changes the label of path to the filelabel string.
// It changes the MCS label to s0 if shared is true.
// This will allow all containers to share the content.
func Relabel(path string, fileLabel string, shared bool) error {
	if !selinux.SelinuxEnabled() {
		return nil
	}

	if fileLabel == "" {
		return nil
	}

	exclude_paths := map[string]bool{"/": true, "/usr": true, "/etc": true}
	if exclude_paths[path] {
		return fmt.Errorf("SELinux relabeling of %s is not allowed", path)
	}

	if shared {
		c := selinux.NewContext(fileLabel)
		c["level"] = "s0"
		fileLabel = c.Get()
	}
	if err := selinux.Chcon(path, fileLabel, true); err != nil {
		return fmt.Errorf("SELinux relabeling of %s is not allowed: %q", path, err)
	}
	return nil
}

// GetPidLabel will return the label of the process running with the specified pid
func GetPidLabel(pid int) (string, error) {
	return selinux.Getpidcon(pid)
}

// Init initialises the labeling system
func Init() {
	selinux.SelinuxEnabled()
}

// ReserveLabel will record the fact that the MCS label has already been used.
// This will prevent InitLabels from using the MCS label in a newly created
// container
func ReserveLabel(label string) error {
	selinux.ReserveLabel(label)
	return nil
}

// UnreserveLabel will remove the reservation of the MCS label.
// This will allow InitLabels to use the MCS label in a newly created
// containers
func UnreserveLabel(label string) error {
	selinux.FreeLxcContexts(label)
	return nil
}

// DupSecOpt takes a process label and returns security options that
// can be used to set duplicate labels on future container processes
func DupSecOpt(src string) []string {
	return selinux.DupSecOpt(src)
}

// DisableSecOpt returns a security opt that can disable labeling
// support for future container processes
func DisableSecOpt() []string {
	return selinux.DisableSecOpt()
}

// Validate checks that the label does not include unexpected options
func Validate(label string) error {
	if strings.Contains(label, "z") && strings.Contains(label, "Z") {
		return ErrIncompatibleLabel
	}
	return nil
}

// RelabelNeeded checks whether the user requested a relabel
func RelabelNeeded(label string) bool {
	return strings.Contains(label, "z") || strings.Contains(label, "Z")
}

// IsShared checks that the label includes a "shared" mark
func IsShared(label string) bool {
	return strings.Contains(label, "z")
}
<commit_msg>Don't override system error<commit_after>// +build selinux,linux

package label

import (
	"fmt"
	"strings"

	"github.com/opencontainers/runc/libcontainer/selinux"
)

// Valid Label Options
var validOptions = map[string]bool{
	"disable": true,
	"type":    true,
	"user":    true,
	"role":    true,
	"level":   true,
}

var ErrIncompatibleLabel = fmt.Errorf("Bad SELinux option z and Z can not be used together")

// InitLabels returns the process label and file labels to be used within
// the container.  A list of options can be passed into this function to alter
// the labels.  The labels returned will include a random MCS String, that is
// guaranteed to be unique.
func InitLabels(options []string) (string, string, error) {
	if !selinux.SelinuxEnabled() {
		return "", "", nil
	}
	processLabel, mountLabel := selinux.GetLxcContexts()
	if processLabel != "" {
		pcon := selinux.NewContext(processLabel)
		mcon := selinux.NewContext(mountLabel)
		for _, opt := range options {
			if opt == "disable" {
				return "", "", nil
			}
			if i := strings.Index(opt, ":"); i == -1 {
				return "", "", fmt.Errorf("Bad label option %q, valid options 'disable' or \n'user, role, level, type' followed by ':' and a value", opt)
			}
			con := strings.SplitN(opt, ":", 2)
			if !validOptions[con[0]] {
				return "", "", fmt.Errorf("Bad label option %q, valid options 'disable, user, role, level, type'", con[0])

			}
			pcon[con[0]] = con[1]
			if con[0] == "level" || con[0] == "user" {
				mcon[con[0]] = con[1]
			}
		}
		processLabel = pcon.Get()
		mountLabel = mcon.Get()
	}
	return processLabel, mountLabel, nil
}

func GetROMountLabel() string {
	return selinux.GetROFileLabel()
}

// DEPRECATED: The GenLabels function is only to be used during the transition to the official API.
func GenLabels(options string) (string, string, error) {
	return InitLabels(strings.Fields(options))
}

// FormatMountLabel returns a string to be used by the mount command.
// The format of this string will be used to alter the labeling of the mountpoint.
// The string returned is suitable to be used as the options field of the mount command.
// If you need to have additional mount point options, you can pass them in as
// the first parameter.  Second parameter is the label that you wish to apply
// to all content in the mount point.
func FormatMountLabel(src, mountLabel string) string {
	if mountLabel != "" {
		switch src {
		case "":
			src = fmt.Sprintf("context=%q", mountLabel)
		default:
			src = fmt.Sprintf("%s,context=%q", src, mountLabel)
		}
	}
	return src
}

// SetProcessLabel takes a process label and tells the kernel to assign the
// label to the next program executed by the current process.
func SetProcessLabel(processLabel string) error {
	if processLabel == "" {
		return nil
	}
	return selinux.Setexeccon(processLabel)
}

// GetProcessLabel returns the process label that the kernel will assign
// to the next program executed by the current process.  If "" is returned
// this indicates that the default labeling will happen for the process.
func GetProcessLabel() (string, error) {
	return selinux.Getexeccon()
}

// GetFileLabel returns the label for specified path
func GetFileLabel(path string) (string, error) {
	return selinux.Getfilecon(path)
}

// SetFileLabel modifies the "path" label to the specified file label
func SetFileLabel(path string, fileLabel string) error {
	if selinux.SelinuxEnabled() && fileLabel != "" {
		return selinux.Setfilecon(path, fileLabel)
	}
	return nil
}

// SetFileCreateLabel tells the kernel the label for all files to be created
func SetFileCreateLabel(fileLabel string) error {
	if selinux.SelinuxEnabled() {
		return selinux.Setfscreatecon(fileLabel)
	}
	return nil
}

// Relabel changes the label of path to the filelabel string.
// It changes the MCS label to s0 if shared is true.
// This will allow all containers to share the content.
func Relabel(path string, fileLabel string, shared bool) error {
	if !selinux.SelinuxEnabled() {
		return nil
	}

	if fileLabel == "" {
		return nil
	}

	exclude_paths := map[string]bool{"/": true, "/usr": true, "/etc": true}
	if exclude_paths[path] {
		return fmt.Errorf("SELinux relabeling of %s is not allowed", path)
	}

	if shared {
		c := selinux.NewContext(fileLabel)
		c["level"] = "s0"
		fileLabel = c.Get()
	}
	if err := selinux.Chcon(path, fileLabel, true); err != nil {
		return err
	}
	return nil
}

// GetPidLabel will return the label of the process running with the specified pid
func GetPidLabel(pid int) (string, error) {
	return selinux.Getpidcon(pid)
}

// Init initialises the labeling system
func Init() {
	selinux.SelinuxEnabled()
}

// ReserveLabel will record the fact that the MCS label has already been used.
// This will prevent InitLabels from using the MCS label in a newly created
// container
func ReserveLabel(label string) error {
	selinux.ReserveLabel(label)
	return nil
}

// UnreserveLabel will remove the reservation of the MCS label.
// This will allow InitLabels to use the MCS label in a newly created
// containers
func UnreserveLabel(label string) error {
	selinux.FreeLxcContexts(label)
	return nil
}

// DupSecOpt takes a process label and returns security options that
// can be used to set duplicate labels on future container processes
func DupSecOpt(src string) []string {
	return selinux.DupSecOpt(src)
}

// DisableSecOpt returns a security opt that can disable labeling
// support for future container processes
func DisableSecOpt() []string {
	return selinux.DisableSecOpt()
}

// Validate checks that the label does not include unexpected options
func Validate(label string) error {
	if strings.Contains(label, "z") && strings.Contains(label, "Z") {
		return ErrIncompatibleLabel
	}
	return nil
}

// RelabelNeeded checks whether the user requested a relabel
func RelabelNeeded(label string) bool {
	return strings.Contains(label, "z") || strings.Contains(label, "Z")
}

// IsShared checks that the label includes a "shared" mark
func IsShared(label string) bool {
	return strings.Contains(label, "z")
}
<|endoftext|>


<commit_before>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2013 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
    "database/sql/driver"
    )

type _result struct {
}
func (r *_result) LastInsertId() (int64, error) {
    return 0, nil
}

func (r *_result) RowsAffected() (int64, error) {
    return 0, nil
}

type firebirdsqlStmt struct {
    wp *wireProtocol
    stmtHandle int32
    tx *firebirdsqlTx
    xsqlda []xSQLVAR
    blr []byte
    stmtType int32
}

func (stmt *firebirdsqlStmt) Close() (err error) {
    stmt.wp.opFreeStatement(stmt.stmtHandle, 2)     // DSQL_drop
    return
}

func (stmt *firebirdsqlStmt) NumInput() int {
    return -1
}

func (stmt *firebirdsqlStmt) Exec(args []driver.Value) (result driver.Result, err error) {
    stmt.wp.opExecute(stmt.stmtHandle, stmt.tx.transHandle, args)
    _, _, _, err = stmt.wp.opResponse()

    result = new(_result)
    return
}

func (stmt *firebirdsqlStmt) Query(args []driver.Value) (rows driver.Rows, err error) {
    stmt.wp.opExecute(stmt.stmtHandle, stmt.tx.transHandle, args)
    _, _, _, err = stmt.wp.opResponse()
    rows = newFirebirdsqlRows(stmt)
    return
}

func newFirebirdsqlStmt(fc *firebirdsqlConn, query string) (stmt *firebirdsqlStmt, err error) {
    stmt = new(firebirdsqlStmt)
    stmt.wp = fc.wp
    stmt.tx = fc.tx
    fc.wp.opAllocateStatement()
    stmt.stmtHandle, _, _, err = fc.wp.opResponse()
    if err != nil {
        return
    }
    fc.wp.opPrepareStatement(stmt.stmtHandle, stmt.tx.transHandle, query)
    _, _, buf, err := fc.wp.opResponse()

    stmt.stmtType, stmt.xsqlda, err = fc.wp.parse_xsqlda(buf, stmt.stmtHandle)
    stmt.blr = calcBlr(stmt.xsqlda)

    return
}
<commit_msg>remove 1 line<commit_after>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2013 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
    "database/sql/driver"
    )

type _result struct {
}
func (r *_result) LastInsertId() (int64, error) {
    return 0, nil
}

func (r *_result) RowsAffected() (int64, error) {
    return 0, nil
}

type firebirdsqlStmt struct {
    wp *wireProtocol
    stmtHandle int32
    tx *firebirdsqlTx
    xsqlda []xSQLVAR
    blr []byte
    stmtType int32
}

func (stmt *firebirdsqlStmt) Close() (err error) {
    stmt.wp.opFreeStatement(stmt.stmtHandle, 2)     // DSQL_drop
    return
}

func (stmt *firebirdsqlStmt) NumInput() int {
    return -1
}

func (stmt *firebirdsqlStmt) Exec(args []driver.Value) (result driver.Result, err error) {
    stmt.wp.opExecute(stmt.stmtHandle, stmt.tx.transHandle, args)
    _, _, _, err = stmt.wp.opResponse()

    result = new(_result)
    return
}

func (stmt *firebirdsqlStmt) Query(args []driver.Value) (rows driver.Rows, err error) {
    stmt.wp.opExecute(stmt.stmtHandle, stmt.tx.transHandle, args)
    rows = newFirebirdsqlRows(stmt)
    return
}

func newFirebirdsqlStmt(fc *firebirdsqlConn, query string) (stmt *firebirdsqlStmt, err error) {
    stmt = new(firebirdsqlStmt)
    stmt.wp = fc.wp
    stmt.tx = fc.tx
    fc.wp.opAllocateStatement()
    stmt.stmtHandle, _, _, err = fc.wp.opResponse()
    if err != nil {
        return
    }
    fc.wp.opPrepareStatement(stmt.stmtHandle, stmt.tx.transHandle, query)
    _, _, buf, err := fc.wp.opResponse()

    stmt.stmtType, stmt.xsqlda, err = fc.wp.parse_xsqlda(buf, stmt.stmtHandle)
    stmt.blr = calcBlr(stmt.xsqlda)

    return
}
<|endoftext|>


<commit_before>package build

import (
	"fmt"
	"neon/util"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

const (
	PLUGIN_SITE = "github.com"
)

// InstallPlugin installs given plugin in repository:
// - plugin: the plugin name such as c4s4/build. First part us Github user name
//   and second is repository name for the plugin.
// - repository: plugin repository, defaults to ~/.neon.
// Return: an error if something went wrong downloading plugin.
func InstallPlugin(plugin, repository string) error {
	re := regexp.MustCompile(`^` + RegexpPlugin + `$`)
	if !re.MatchString(plugin) {
		return fmt.Errorf("plugin name '%s' is invalid", plugin)
	}
	path := filepath.Join(repository, plugin)
	if util.DirExists(path) {
		Message("Plugin '%s' already installed in '%s'", plugin, path)
		return nil
	}
	absolute := util.ExpandUserHome(path)
	repo := "git://" + PLUGIN_SITE + "/" + plugin + ".git"
	command := exec.Command("git", "clone", repo, absolute)
	Message("Running command '%s'...", strings.Join(command.Args, " "))
	output, err := command.CombinedOutput()
	if err != nil {
		re = regexp.MustCompile("\n\n")
		message := re.ReplaceAllString(string(output), "\n")
		message = strings.TrimSpace(message)
		Message(message)
		return fmt.Errorf("installing plugin '%s'", plugin)
	} else {
		Message("Plugin '%s' installed in '%s'", plugin, path)
	}
	return nil
}
<commit_msg>Renamed constant<commit_after>package build

import (
	"fmt"
	"neon/util"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

const (
	PluginSite = "github.com"
)

// InstallPlugin installs given plugin in repository:
// - plugin: the plugin name such as c4s4/build. First part us Github user name
//   and second is repository name for the plugin.
// - repository: plugin repository, defaults to ~/.neon.
// Return: an error if something went wrong downloading plugin.
func InstallPlugin(plugin, repository string) error {
	re := regexp.MustCompile(`^` + RegexpPlugin + `$`)
	if !re.MatchString(plugin) {
		return fmt.Errorf("plugin name '%s' is invalid", plugin)
	}
	path := filepath.Join(repository, plugin)
	if util.DirExists(path) {
		Message("Plugin '%s' already installed in '%s'", plugin, path)
		return nil
	}
	absolute := util.ExpandUserHome(path)
	repo := "git://" + PluginSite + "/" + plugin + ".git"
	command := exec.Command("git", "clone", repo, absolute)
	Message("Running command '%s'...", strings.Join(command.Args, " "))
	output, err := command.CombinedOutput()
	if err != nil {
		re = regexp.MustCompile("\n\n")
		message := re.ReplaceAllString(string(output), "\n")
		message = strings.TrimSpace(message)
		Message(message)
		return fmt.Errorf("installing plugin '%s'", plugin)
	} else {
		Message("Plugin '%s' installed in '%s'", plugin, path)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"io"
	"os"
	"strings"

	"github.com/coocood/freecache"
	"github.com/miekg/dns"
	"github.com/op/go-logging"
)

var logger = logging.MustGetLogger("dnsblocker")
var logFormatter = logging.MustStringFormatter(
	"%{color}%{time} [%{level:.5s}] %{message}%{color:reset}",
)

var dnsServer string
var listen string
var hostsPath string
var logLevelStr string
var blocked = []string{}
var cacheSize int
var cacheDuration int
var cache *freecache.Cache

func init() {
	flag.StringVar(&dnsServer, "dns-server", "192.168.1.1:domain",
		"DNS server for proxy not blocked queries")
	flag.StringVar(&listen, "listen", ":domain", "Listen is pair 'ip:port'")
	flag.StringVar(&hostsPath, "hosts-path", "hosts", "Path to hosts file")
	flag.StringVar(&logLevelStr, "log-level", "INFO", "Set minimum log level")
	flag.IntVar(&cacheSize, "cache-size", 1024*1024, "Set cache size into bytes")
	flag.IntVar(&cacheDuration, "cache-duration", 5*60, "Set cache duration into seconds")
	flag.Parse()

	configureLogger()
	configureCache()

	logger.Infof("Start with: listen: %s; dns-server: %s; hosts-path: %s",
		listen, dnsServer, hostsPath)

	loadBlocked()
}

func configureLogger() {
	logging.SetBackend(
		logging.MultiLogger(
			logging.NewBackendFormatter(
				logging.NewLogBackend(os.Stdout, "", 0),
				logFormatter,
			),
		),
	)
	normalizedLogLevel := strings.ToUpper(logLevelStr)
	if logLevel, levelErr := logging.LogLevel(normalizedLogLevel); nil == levelErr {
		logging.SetLevel(logLevel, "")
	} else {
		logger.Fatal(levelErr)
	}
}

func loadBlocked() {
	file, openErr := os.Open(hostsPath)
	if nil != openErr {
		panic(openErr)
	}
	defer file.Close()
	buffer := bufio.NewReader(file)
	for {
		line, readErr := buffer.ReadString('\n')
		if readErr == io.EOF {
			break
		} else if nil != readErr {
			panic(readErr)
		}

		host := dns.Fqdn(line[0 : len(line)-1])
		blocked = append(blocked, host)
	}
	logger.Infof("Loaded %d domains", len(blocked))
}

func configureCache() {
	cache = freecache.NewCache(cacheSize)
}

func isBlocked(requestedName string) bool {
	for _, name := range blocked {
		if dns.IsSubDomain(name, requestedName) {
			return true
		}
	}
	return false
}

func messageCacheKey(message *dns.Msg) []byte {
	questions := make([]string, 0, len(message.Question))
	for _, question := range message.Question {
		questions = append(questions, question.String())
	}
	return ([]byte)(strings.Join(questions, "\n"))
}

func dnsExchangeWithCache(requestMessage *dns.Msg) (*dns.Msg, error) {
	cacheKey := messageCacheKey(requestMessage)

	if cachedResponseData, cacheGetErr := cache.Get(cacheKey); nil == cacheGetErr {
		logger.Info("Message found in cache")
		cachedResponseMessage := new(dns.Msg)
		if unpackErr := cachedResponseMessage.Unpack(cachedResponseData); nil != unpackErr {
			logger.Errorf("Unpack error: %s", unpackErr)
			return nil, unpackErr
		}
		logger.Debug("Success unpack message")
		cachedResponseMessage.SetReply(requestMessage)
		return cachedResponseMessage, nil
	} else if freecache.ErrNotFound == cacheGetErr {
		logger.Infof("Message not found in cache: %s", cacheGetErr)
		responseMessage, exchangeErr := dns.Exchange(requestMessage, dnsServer)
		if nil != exchangeErr {
			logger.Errorf("Exchange error: %s", exchangeErr)
			return nil, exchangeErr
		}
		logger.Debug("Exchange success")
		responseData, packErr := responseMessage.Pack()
		if nil != packErr {
			logger.Errorf("Error pack message: %s", packErr)
			return nil, packErr
		}
		logger.Debug("Pack success")
		if cacheSetErr := cache.Set(cacheKey, responseData, cacheDuration); nil != cacheSetErr {
			logger.Error("Error set cache")
			return nil, cacheSetErr
		}
		logger.Debug("Write cache success")
		return responseMessage, nil
	} else {
		logger.Errorf("Cache error: %s", cacheGetErr)
		return nil, cacheGetErr
	}
}

func proxyRequest(writer dns.ResponseWriter, requestMessage *dns.Msg) {
	responseMessage, fetchErr := dnsExchangeWithCache(requestMessage)

	if nil == fetchErr {
		logger.Debug("Response message: %+v", responseMessage)
		writeResponse(writer, responseMessage)
	} else {
		logger.Errorf("Fetch error: %s", fetchErr)

		errorMessage := new(dns.Msg)
		errorMessage.SetRcode(requestMessage, dns.RcodeServerFailure)
		logger.Errorf("Error message: %+v", errorMessage)

		writeResponse(writer, errorMessage)
	}
}

func errorResponse(writer dns.ResponseWriter, requestMessage *dns.Msg, responseCode int) {
	responseMessage := new(dns.Msg)
	responseMessage.SetRcode(requestMessage, responseCode)
	logger.Debugf("Block response: %+v", responseMessage)
	writeResponse(writer, responseMessage)
}

func handler(writer dns.ResponseWriter, requestMessage *dns.Msg) {
	logger.Debugf("Query message: %+v", requestMessage)
	logger.Infof("Accept request from %v", writer.RemoteAddr())

	if 1 != len(requestMessage.Question) {
		logger.Warning("Can not process message with multiple questions")
		errorResponse(writer, requestMessage, dns.RcodeFormatError)
		return
	}

	question := requestMessage.Question[0]
	logger.Infof("Request name: %s", question.Name)

	if isBlocked(question.Name) {
		logger.Infof("Block name: %s", question.Name)
		errorResponse(writer, requestMessage, dns.RcodeRefused)
	} else {
		logger.Infof("Request name %s is proxed", question.Name)
		proxyRequest(writer, requestMessage)
	}
}

func writeResponse(writer dns.ResponseWriter, message *dns.Msg) {
	if writeErr := writer.WriteMsg(message); nil == writeErr {
		logger.Debug("Writer success")
	} else {
		logger.Errorf("Writer error: %s", writeErr)
	}
}

func main() {
	server := &dns.Server{
		Addr:    listen,
		Net:     "udp",
		Handler: dns.HandlerFunc(handler),
	}

	if serverErr := server.ListenAndServe(); nil != serverErr {
		logger.Fatal(serverErr)
	}
}
<commit_msg>Refactore caching<commit_after>package main

import (
	"bufio"
	"flag"
	"io"
	"os"
	"strings"

	"github.com/coocood/freecache"
	"github.com/miekg/dns"
	"github.com/op/go-logging"
)

var logger = logging.MustGetLogger("dnsblocker")
var logFormatter = logging.MustStringFormatter(
	"%{color}%{time} [%{level:.5s}] %{message}%{color:reset}",
)

var dnsServer string
var listen string
var hostsPath string
var logLevelStr string
var blocked = []string{}
var cacheSize int
var cacheDuration int
var cache *freecache.Cache

func init() {
	flag.StringVar(&dnsServer, "dns-server", "192.168.1.1:domain",
		"DNS server for proxy not blocked queries")
	flag.StringVar(&listen, "listen", ":domain", "Listen is pair 'ip:port'")
	flag.StringVar(&hostsPath, "hosts-path", "hosts", "Path to hosts file")
	flag.StringVar(&logLevelStr, "log-level", "INFO", "Set minimum log level")
	flag.IntVar(&cacheSize, "cache-size", 1024*1024, "Set cache size into bytes")
	flag.IntVar(&cacheDuration, "cache-duration", 5*60, "Set cache duration into seconds")
	flag.Parse()

	configureLogger()
	configureCache()

	logger.Infof("Start listen on: %s", listen)
	logger.Infof("DNS source: %s", dnsServer)
	logger.Infof("Hosts file: %s", hostsPath)
	logger.Infof("Log level: %s", logLevelStr)
	logger.Infof("Cache size: %d", cacheSize)
	logger.Infof("Cache duration: %d", cacheDuration)

	loadBlocked()
}

func configureLogger() {
	logging.SetBackend(
		logging.MultiLogger(
			logging.NewBackendFormatter(
				logging.NewLogBackend(os.Stdout, "", 0),
				logFormatter,
			),
		),
	)
	normalizedLogLevel := strings.ToUpper(logLevelStr)
	if logLevel, levelErr := logging.LogLevel(normalizedLogLevel); nil == levelErr {
		logging.SetLevel(logLevel, "")
	} else {
		logger.Fatal(levelErr)
	}
}

func loadBlocked() {
	file, openErr := os.Open(hostsPath)
	if nil != openErr {
		panic(openErr)
	}
	defer file.Close()
	buffer := bufio.NewReader(file)
	for {
		line, readErr := buffer.ReadString('\n')
		if readErr == io.EOF {
			break
		} else if nil != readErr {
			panic(readErr)
		}

		host := dns.Fqdn(line[0 : len(line)-1])
		blocked = append(blocked, host)
	}
	logger.Infof("Loaded %d domains", len(blocked))
}

func configureCache() {
	cache = freecache.NewCache(cacheSize)
}

func isBlocked(requestedName string) bool {
	for _, name := range blocked {
		if dns.IsSubDomain(name, requestedName) {
			return true
		}
	}
	return false
}

func messageCacheKey(message *dns.Msg) string {
	questions := make([]string, 0, len(message.Question))
	for _, question := range message.Question {
		questions = append(questions, question.String())
	}
	return strings.Join(questions, ",")
}

func fetchCache(cacheKey string) (*dns.Msg, error) {
	messageData, cacheErr := cache.Get([]byte(cacheKey))
	if freecache.ErrNotFound == cacheErr {
		logger.Infof("Message for key %s not found in cache", cacheKey)
		return nil, cacheErr
	}
	if nil != cacheErr {
		logger.Errorf("Cache error for key %s: %s", cacheKey, cacheErr)
		return nil, cacheErr
	}

	logger.Infof("Message for key %sfound in cache", cacheKey)

	dnsMessage := new(dns.Msg)
	if unpackErr := dnsMessage.Unpack(messageData); nil != unpackErr {
		logger.Errorf("Error unpack mesasge with key %s : %s", cacheKey, unpackErr)
		return nil, unpackErr
	}

	logger.Debug("Success unpack message")
	return dnsMessage, nil
}

func writeCache(cacheKey string, dnsMessage *dns.Msg) error {
	messageData, packErr := dnsMessage.Pack()
	if nil != packErr {
		logger.Errorf("Error pack message with key %s: %s", cacheKey, packErr)
		return packErr
	}

	logger.Debugf("Pack message with key %s is success", cacheKey)

	if cacheErr := cache.Set([]byte(cacheKey), messageData, cacheDuration); nil != cacheErr {
		logger.Errorf("Error write cache for key %s: %s", cacheKey, cacheErr)
		return cacheErr
	}
	logger.Debugf("Write cache for key %s is success", cacheKey)
	return nil
}

func dnsExchangeWithCache(requestMessage *dns.Msg) (*dns.Msg, error) {
	cacheKey := messageCacheKey(requestMessage)

	if cachedResponseMessage, fetchErr := fetchCache(cacheKey); nil == fetchErr {
		cachedResponseMessage.SetReply(requestMessage)
		return cachedResponseMessage, nil
	} else if freecache.ErrNotFound != fetchErr {
		return nil, fetchErr
	}

	responseMessage, exchangeErr := dns.Exchange(requestMessage, dnsServer)
	if nil != exchangeErr {
		logger.Errorf("Exchange error for key %s: %s", cacheKey, exchangeErr)
		return nil, exchangeErr
	}

	logger.Debugf("Exchange for key %s is success", cacheKey)

	if writeErr := writeCache(cacheKey, responseMessage); nil != writeErr {
		return nil, writeErr
	}

	return responseMessage, nil
}

func proxyRequest(writer dns.ResponseWriter, requestMessage *dns.Msg) {
	responseMessage, fetchErr := dnsExchangeWithCache(requestMessage)

	if nil == fetchErr {
		logger.Debug("Response message: %+v", responseMessage)
		writeResponse(writer, responseMessage)
	} else {
		logger.Errorf("Fetch error: %s", fetchErr)

		errorMessage := new(dns.Msg)
		errorMessage.SetRcode(requestMessage, dns.RcodeServerFailure)
		logger.Errorf("Error message: %+v", errorMessage)

		writeResponse(writer, errorMessage)
	}
}

func errorResponse(writer dns.ResponseWriter, requestMessage *dns.Msg, responseCode int) {
	responseMessage := new(dns.Msg)
	responseMessage.SetRcode(requestMessage, responseCode)
	logger.Debugf("Block response: %+v", responseMessage)
	writeResponse(writer, responseMessage)
}

func handler(writer dns.ResponseWriter, requestMessage *dns.Msg) {
	logger.Debugf("Query message: %+v", requestMessage)
	logger.Infof("Accept request from %v", writer.RemoteAddr())

	if 1 != len(requestMessage.Question) {
		logger.Warning("Can not process message with multiple questions")
		errorResponse(writer, requestMessage, dns.RcodeFormatError)
		return
	}

	question := requestMessage.Question[0]
	logger.Infof("Request name: %s", question.Name)

	if isBlocked(question.Name) {
		logger.Infof("Block name: %s", question.Name)
		errorResponse(writer, requestMessage, dns.RcodeRefused)
	} else {
		logger.Infof("Request name %s is proxed", question.Name)
		proxyRequest(writer, requestMessage)
	}
}

func writeResponse(writer dns.ResponseWriter, message *dns.Msg) {
	if writeErr := writer.WriteMsg(message); nil == writeErr {
		logger.Debug("Writer success")
	} else {
		logger.Errorf("Writer error: %s", writeErr)
	}
}

func main() {
	server := &dns.Server{
		Addr:    listen,
		Net:     "udp",
		Handler: dns.HandlerFunc(handler),
	}

	if serverErr := server.ListenAndServe(); nil != serverErr {
		logger.Fatal(serverErr)
	}
}
<|endoftext|>


<commit_before>package search

import (
	"fmt"
	"github.com/balzaczyy/golucene/index"
	"github.com/balzaczyy/golucene/util"
)

type TermQuery struct {
	*AbstractQuery
	term               index.Term
	docFreq            int
	perReaderTermState *index.TermContext
}

func NewTermQuery(t index.Term) *TermQuery {
	return NewTermQueryWithDocFreq(t, -1)
}

func NewTermQueryWithDocFreq(t index.Term, docFreq int) *TermQuery {
	ans := &TermQuery{}
	ans.AbstractQuery = NewAbstractQuery(ans)
	ans.term = t
	ans.docFreq = docFreq
	return ans
}

func (q *TermQuery) CreateWeight(ss IndexSearcher) Weight {
	ctx := ss.TopReaderContext()
	var termState *index.TermContext
	if q.perReaderTermState == nil || q.perReaderTermState.TopReaderContext != ctx {
		// make TermQuery single-pass if we don't have a PRTS or if the context differs!
		termState = index.NewTermContextFromTerm(ctx, q.term)
	} else {
		// PRTS was pre-build for this IS
		termState = q.perReaderTermState
	}

	// we must not ignore the given docFreq - if set use the given value (lie)
	if q.docFreq != -1 {
		termState.DocFreq = q.docFreq
	}

	return NewTermWeight(q, ss, *termState)
}

func (q *TermQuery) String() string {
	boost := ""
	if q.boost != 1.0 {
		boost = fmt.Sprintf("^%v", q.boost)
	}
	return fmt.Sprintf("%v:%v%v", q.term.Field, string(q.term.Bytes), boost)
}

type TermWeight struct {
	query      *TermQuery
	similarity Similarity
	stats      SimWeight
	termStates index.TermContext
}

func NewTermWeight(q *TermQuery, ss IndexSearcher, termStates index.TermContext) TermWeight {
	sim := ss.similarity
	return TermWeight{q, sim, sim.computeWeight(
		q.AbstractQuery.boost,
		ss.CollectionStatistics(q.term.Field),
		ss.TermStatistics(q.term, termStates)), termStates}
}

func (tw TermWeight) ValueForNormalization() float32 {
	return tw.stats.ValueForNormalization()
}

func (tw TermWeight) Normalize(norm float64, topLevelBoost float32) float32 {
	return tw.stats.Normalize(norm, topLevelBoost)
}

func (tw TermWeight) IsScoresDocsOutOfOrder() bool {
	return false
}

func (tw TermWeight) Scorer(context index.AtomicReaderContext,
	inOrder bool, topScorer bool, acceptDocs util.Bits) (sc Scorer, ok bool) {
	// assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight (" + termStates.topReaderContext + ") is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
	termsEnum, ok := tw.termsEnum(context)
	if !ok {
		return Scorer{}, false
	}
	docs := termsEnum.Docs(acceptDocs, index.DOCS_ENUM_EMPTY)
	// assert docs != null;
	return *(newTermScorer(tw, docs, tw.similarity.exactSimScorer(tw.stats, context)).Scorer), true
}

func (tw TermWeight) termsEnum(ctx index.AtomicReaderContext) (te index.TermsEnum, ok bool) {
	state := tw.termStates.State(ctx.Ord)
	if state == nil { // term is not present in that reader
		// assert termNotInReader(ctx.Reader(), tw.Term)
		// : "no termstate found but term exists in reader term=" + term;
		return index.TERMS_ENUM_EMPTY, false
	}
	te = ctx.Reader().(index.AtomicReader).Terms(tw.query.term.Field).Iterator(index.TERMS_ENUM_EMPTY)
	te.SeekExactFromLast(tw.query.term.Bytes, *state)
	return te, true
}

type TermScorer struct {
	*Scorer
	docScorer ExactSimScorer
	docsEnum  index.DocsEnum
}

func newTermScorer(w Weight, td index.DocsEnum, docScorer ExactSimScorer) TermScorer {
	ans := &TermScorer{}
	scorer := newScorer(ans, w, func() float64 {
		// assert docID() != NO_MORE_DOCS
		return ans.docScorer.Score(ans.docsEnum.DocId(), ans.docsEnum.Freq())
	})
	ans.Scorer = &scorer
	ans.docScorer = docScorer
	ans.docsEnum = td
	return *ans
}
<commit_msg>fix rename<commit_after>package search

import (
	"fmt"
	"github.com/balzaczyy/golucene/index"
	"github.com/balzaczyy/golucene/util"
)

type TermQuery struct {
	*AbstractQuery
	term               index.Term
	docFreq            int
	perReaderTermState *index.TermContext
}

func NewTermQuery(t index.Term) *TermQuery {
	return NewTermQueryWithDocFreq(t, -1)
}

func NewTermQueryWithDocFreq(t index.Term, docFreq int) *TermQuery {
	ans := &TermQuery{}
	ans.AbstractQuery = NewAbstractQuery(ans)
	ans.term = t
	ans.docFreq = docFreq
	return ans
}

func (q *TermQuery) CreateWeight(ss IndexSearcher) Weight {
	ctx := ss.TopReaderContext()
	var termState *index.TermContext
	if q.perReaderTermState == nil || q.perReaderTermState.TopReaderContext != ctx {
		// make TermQuery single-pass if we don't have a PRTS or if the context differs!
		termState = index.NewTermContextFromTerm(ctx, q.term)
	} else {
		// PRTS was pre-build for this IS
		termState = q.perReaderTermState
	}

	// we must not ignore the given docFreq - if set use the given value (lie)
	if q.docFreq != -1 {
		termState.DocFreq = q.docFreq
	}

	return NewTermWeight(q, ss, *termState)
}

func (q *TermQuery) String() string {
	boost := ""
	if q.boost != 1.0 {
		boost = fmt.Sprintf("^%v", q.boost)
	}
	return fmt.Sprintf("%v:%v%v", q.term.Field, string(q.term.Bytes), boost)
}

type TermWeight struct {
	query      *TermQuery
	similarity Similarity
	stats      SimWeight
	termStates index.TermContext
}

func NewTermWeight(q *TermQuery, ss IndexSearcher, termStates index.TermContext) TermWeight {
	sim := ss.similarity
	return TermWeight{q, sim, sim.computeWeight(
		q.AbstractQuery.boost,
		ss.CollectionStatistics(q.term.Field),
		ss.TermStatistics(q.term, termStates)), termStates}
}

func (tw TermWeight) ValueForNormalization() float32 {
	return tw.stats.ValueForNormalization()
}

func (tw TermWeight) Normalize(norm float64, topLevelBoost float32) float32 {
	return tw.stats.Normalize(norm, topLevelBoost)
}

func (tw TermWeight) IsScoresDocsOutOfOrder() bool {
	return false
}

func (tw TermWeight) Scorer(context index.AtomicReaderContext,
	inOrder bool, topScorer bool, acceptDocs util.Bits) (sc Scorer, ok bool) {
	// assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight (" + termStates.topReaderContext + ") is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
	termsEnum, ok := tw.termsEnum(context)
	if !ok {
		return Scorer{}, false
	}
	docs := termsEnum.Docs(acceptDocs, index.DOCS_ENUM_EMPTY)
	// assert docs != null;
	return *(newTermScorer(tw, docs, tw.similarity.exactSimScorer(tw.stats, context)).Scorer), true
}

func (tw TermWeight) termsEnum(ctx index.AtomicReaderContext) (te index.TermsEnum, ok bool) {
	state := tw.termStates.State(ctx.Ord)
	if state == nil { // term is not present in that reader
		// assert termNotInReader(ctx.Reader(), tw.Term)
		// : "no termstate found but term exists in reader term=" + term;
		return index.EMPTY_TERMS_ENUM, false
	}
	te = ctx.Reader().(index.AtomicReader).Terms(tw.query.term.Field).Iterator(index.EMPTY_TERMS_ENUM)
	te.SeekExactFromLast(tw.query.term.Bytes, *state)
	return te, true
}

type TermScorer struct {
	*Scorer
	docScorer ExactSimScorer
	docsEnum  index.DocsEnum
}

func newTermScorer(w Weight, td index.DocsEnum, docScorer ExactSimScorer) TermScorer {
	ans := &TermScorer{}
	scorer := newScorer(ans, w, func() float64 {
		// assert docID() != NO_MORE_DOCS
		return ans.docScorer.Score(ans.docsEnum.DocId(), ans.docsEnum.Freq())
	})
	ans.Scorer = &scorer
	ans.docScorer = docScorer
	ans.docsEnum = td
	return *ans
}
<|endoftext|>


<commit_before><commit_msg>term_test: add TestTermSetFlowControl()<commit_after><|endoftext|>


<commit_before>// +build linux

package linux

import (
	"bufio"
	"io"
	"os"
	"regexp"
	"strings"

	"github.com/mackerelio/mackerel-agent/logging"
)

// CPUGenerator Collects CPU specs
type CPUGenerator struct {
}

// Key return "cpu"
func (g *CPUGenerator) Key() string {
	return "cpu"
}

var cpuLogger = logging.GetLogger("spec.cpu")
var cpuinfoLineReg = regexp.MustCompile(`^([^:]+?)\s+:\s+(.*)$`)

func (g *CPUGenerator) generate(file io.Reader) (interface{}, error) {
	scanner := bufio.NewScanner(file)

	var results []map[string]interface{}
	var cur map[string]interface{}
	var modelName string

	for scanner.Scan() {
		line := scanner.Text()
		if matches := cpuinfoLineReg.FindStringSubmatch(line); matches != nil {
			key := matches[1]
			val := matches[2]
			switch key {
			case "processor":
				cur = make(map[string]interface{})
				if modelName != "" {
					cur["model_name"] = modelName
				}
				results = append(results, cur)
			case "Processor":
				modelName = val
			case "vendor_id", "model", "stepping", "physical id", "core id", "model name", "cache size":
				cur[strings.Replace(key, " ", "_", -1)] = val
			case "cpu family":
				cur["family"] = val
			case "cpu cores":
				cur["cores"] = val
			case "cpu MHz":
				cur["mhz"] = val
			}
		}
	}
	if err := scanner.Err(); err != nil {
		cpuLogger.Errorf("Failed (skip this spec): %s", err)
		return nil, err
	}

	// Old kernels with CONFIG_SMP disabled has no "processor: " line
	if len(results) == 0 && modelName != "" {
		cur = make(map[string]interface{})
		cur["model_name"] = modelName
		results = append(results, cur)
	}

	return results, nil
}

// Generate cpu specs
func (g *CPUGenerator) Generate() (interface{}, error) {
	file, err := os.Open("/proc/cpuinfo")
	if err != nil {
		cpuLogger.Errorf("Failed (skip this spec): %s", err)
		return nil, err
	}
	defer file.Close()

	return g.generate(file)
}
<commit_msg>remove regexp from spec/linux/cpu<commit_after>// +build linux

package linux

import (
	"bufio"
	"io"
	"os"
	"strings"

	"github.com/mackerelio/mackerel-agent/logging"
)

// CPUGenerator Collects CPU specs
type CPUGenerator struct {
}

// Key return "cpu"
func (g *CPUGenerator) Key() string {
	return "cpu"
}

var cpuLogger = logging.GetLogger("spec.cpu")

func (g *CPUGenerator) generate(file io.Reader) (interface{}, error) {
	scanner := bufio.NewScanner(file)

	var results []map[string]interface{}
	var cur map[string]interface{}
	var modelName string

	for scanner.Scan() {
		line := scanner.Text()
		kv := strings.SplitN(line, ":", 2)
		if len(kv) < 2 {
			continue
		}
		key := strings.TrimSpace(kv[0])
		val := strings.TrimSpace(kv[1])
		switch key {
		case "processor":
			cur = make(map[string]interface{})
			if modelName != "" {
				cur["model_name"] = modelName
			}
			results = append(results, cur)
		case "Processor":
			modelName = val
		case "vendor_id", "model", "stepping", "physical id", "core id", "model name", "cache size":
			cur[strings.Replace(key, " ", "_", -1)] = val
		case "cpu family":
			cur["family"] = val
		case "cpu cores":
			cur["cores"] = val
		case "cpu MHz":
			cur["mhz"] = val
		}
	}
	if err := scanner.Err(); err != nil {
		cpuLogger.Errorf("Failed (skip this spec): %s", err)
		return nil, err
	}

	// Old kernels with CONFIG_SMP disabled has no "processor: " line
	if len(results) == 0 && modelName != "" {
		cur = make(map[string]interface{})
		cur["model_name"] = modelName
		results = append(results, cur)
	}

	return results, nil
}

// Generate cpu specs
func (g *CPUGenerator) Generate() (interface{}, error) {
	file, err := os.Open("/proc/cpuinfo")
	if err != nil {
		cpuLogger.Errorf("Failed (skip this spec): %s", err)
		return nil, err
	}
	defer file.Close()

	return g.generate(file)
}
<|endoftext|>


<commit_before>package rawgeo

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math"
	"os"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/TomiHiltunen/geohash-golang"
	"github.com/cznic/kv"
	log "gopkg.in/inconshreveable/log15.v2"
)

// TODO(tsileo): implement remove

var (
	Log               = log.New()
	ErrNotFound       = errors.New("key does not exist")
	ErrMissingID      = errors.New("missing ID")
	ErrInvalidLatLong = errors.New("invalid lat/long")
)

var (
	indexKeyFmt = "%s:%s"   // {geohash}:{id}
	earthRadius = 6378137.0 // As defined by WGS 84
)

func init() {
	Log.SetHandler(log.DiscardHandler())
}

type Point struct {
	ID      string
	Geohash string

	// These fields are computed query time and will be ignored when indexing
	Distance float64
	Lat, Lng float64
}

func decodeLatLong(p *Point) {
	latlng := geohash.Decode(p.Geohash).Center()
	p.Lat, p.Lng = latlng.Lat(), latlng.Lng()
}

func NewPointFromGeohash(id, geohash string) *Point {
	p := &Point{
		ID:      id,
		Geohash: geohash,
	}
	decodeLatLong(p)
	return p
}

// Implements the equirectangular approximation from www.movable-type.co.uk/scripts/latlong.html
// (Pythagoras’ theorem on an equirectangular projection)
//
// Returns the approximate distance in meters
func (p *Point) DistanceFrom(point *Point) float64 {
	x := (point.Lng - p.Lng) * math.Pi / 180 * math.Cos((point.Lat+p.Lat)*math.Pi/360)
	y := (point.Lat - p.Lat) * math.Pi / 180
	return math.Sqrt(x*x+y*y) * earthRadius
}

type byDistance []*Point

func (s byDistance) Len() int           { return len(s) }
func (s byDistance) Less(i, j int) bool { return s[i].Distance < s[j].Distance }
func (s byDistance) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

type RawGeo struct {
	db   *kv.DB
	path string
	mu   sync.Mutex
	log  log.Logger
}

// New initializes/loads a `RawGeo` index at the given path
func New(path string) (*RawGeo, error) {
	createOpen := kv.Open
	if _, err := os.Stat(path); os.IsNotExist(err) {
		createOpen = kv.Create
	}
	kvdb, err := createOpen(path, &kv.Options{})
	if err != nil {
		return nil, err
	}
	return &RawGeo{
		db:   kvdb,
		path: path,
		log:  Log.New("path", path),
	}, nil
}

// Close gracefully closes the opened index
func (rg *RawGeo) Close() error {
	return rg.db.Close()
}

// Destroy will try to remove the index file
func (rg *RawGeo) Destroy() error {
	if rg.path != "" {
		rg.Close()
		return os.RemoveAll(rg.path)
	}
	return nil
}

// Put index the given entry in the given index
// (you have to cared about duplicate IDs)
func (rg *RawGeo) Index(point *Point) error {
	// Ensure the entry contains a latitude and a longitude
	if point.Lat == 0 || point.Lng == 0 {
		return ErrInvalidLatLong
	}
	// Compute the Geohash if needed
	if point.Geohash == "" {
		point.Geohash = geohash.Encode(point.Lat, point.Lng)
	}
	if point.ID == "" {
		return ErrMissingID
	}
	if err := rg.db.Set([]byte(fmt.Sprintf(indexKeyFmt, point.Geohash, point.ID)), nil); err != nil {
		return err
	}
	return nil
}

// Query will returns all the points found in the given radius (in meters) sorted by distance to the query
func (rg *RawGeo) Query(lat, lng, radius float64) ([]*Point, error) {
	start := time.Now()
	qlog := rg.log.New("lat", lat, "lng", lng, "radius", radius)
	qlog.Debug("new query")
	refPoint := &Point{
		Lat: lat,
		Lng: lng,
	}
	precision := radiusToPrecision(radius)
	qlog.Debug("converted radius to precision", "radius", radius, "precision", precision)
	gh := geohash.EncodeWithPrecision(lat, lng, precision)
	qlog.Debug("query geohash", "geohash", gh)

	res := []*Point{}

	// Search in the given geohash box, along with all the 8 adjacent boxes
	for _, geohash := range append(geohash.CalculateAllAdjacent(gh), gh) {
		subres, err := rg.findPrefix(geohash)
		qlog.Debug("new subquery", "geohash", geohash, "cnt", len(subres))
		if err != nil {
			return nil, err
		}
		for _, point := range subres {
			// Compute the distance from the reference point (i.e. the query)
			point.Distance = refPoint.DistanceFrom(point)
			// Since the geohashes window is not a radius, we still need to check
			// if the point is located within the given radius.
			if point.Distance > radius {
				continue
			}
			res = append(res, point)
		}
	}
	sort.Sort(byDistance(res))
	qlog.Debug("query took", "duration", time.Since(start))
	return res, nil
}

func (rg *RawGeo) findPrefix(geoPrefix string) ([]*Point, error) {
	var limit int
	points := []*Point{}
	enum, _, err := rg.db.Seek([]byte(geoPrefix))
	if err != nil {
		return nil, err
	}
	// TODO(tsileo): check/fix the endBytes
	endBytes := []byte("\xff")
	i := 0
	for {
		k, _, err := enum.Next()
		sk := string(k)
		if err == io.EOF {
			break
		}
		if !strings.HasPrefix(sk, geoPrefix) {
			break
		}
		if bytes.Compare(k, endBytes) > 0 || (limit != 0 && i > limit) {
			return points, nil
		}
		parts := strings.SplitN(sk, ":", 2)
		p := NewPointFromGeohash(parts[1], parts[0])
		points = append(points, p)
		i++
	}
	return points, nil
}

// Data comes from http://www.movable-type.co.uk/scripts/geohash.html
var geoPrecision = []struct {
	radius    float64 // in meters
	precision int     // Geohash precision
}{
	{0.074, 11},
	{0.6, 10},
	{2.4, 9},
	{19, 8},
	{76, 7},
	{610, 6},
	{2400, 5},
	{20000, 4},
	{78000, 3},
	{630000, 2},
	{2500000, 1},
}

// Convert radius meters to Geohash precision
func radiusToPrecision(r float64) int {
	for _, gp := range geoPrecision {
		if r <= gp.radius {
			return gp.precision
		}
	}
	return 2
}
<commit_msg>Tweaked the log<commit_after>package rawgeo

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/TomiHiltunen/geohash-golang"
	"github.com/cznic/kv"
	log "gopkg.in/inconshreveable/log15.v2"
)

// TODO(tsileo): implement remove

var (
	Log               = log.New()
	ErrNotFound       = errors.New("key does not exist")
	ErrMissingID      = errors.New("missing ID")
	ErrInvalidLatLong = errors.New("invalid lat/long")
)

var (
	indexKeyFmt = "%s:%s"   // {geohash}:{id}
	earthRadius = 6378137.0 // As defined by WGS 84
)

func init() {
	Log.SetHandler(log.DiscardHandler())
}

type Point struct {
	ID      string
	Geohash string

	// These fields are computed query time and will be ignored when indexing
	Distance float64
	Lat, Lng float64
}

func decodeLatLong(p *Point) {
	latlng := geohash.Decode(p.Geohash).Center()
	p.Lat, p.Lng = latlng.Lat(), latlng.Lng()
}

func NewPointFromGeohash(id, geohash string) *Point {
	p := &Point{
		ID:      id,
		Geohash: geohash,
	}
	decodeLatLong(p)
	return p
}

// Implements the equirectangular approximation from www.movable-type.co.uk/scripts/latlong.html
// (Pythagoras’ theorem on an equirectangular projection)
//
// Returns the approximate distance in meters
func (p *Point) DistanceFrom(point *Point) float64 {
	x := (point.Lng - p.Lng) * math.Pi / 180 * math.Cos((point.Lat+p.Lat)*math.Pi/360)
	y := (point.Lat - p.Lat) * math.Pi / 180
	return math.Sqrt(x*x+y*y) * earthRadius
}

type byDistance []*Point

func (s byDistance) Len() int           { return len(s) }
func (s byDistance) Less(i, j int) bool { return s[i].Distance < s[j].Distance }
func (s byDistance) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

type RawGeo struct {
	db   *kv.DB
	path string
	mu   sync.Mutex
	log  log.Logger
}

// New initializes/loads a `RawGeo` index at the given path
func New(path string) (*RawGeo, error) {
	createOpen := kv.Open
	if _, err := os.Stat(path); os.IsNotExist(err) {
		createOpen = kv.Create
	}
	kvdb, err := createOpen(path, &kv.Options{})
	if err != nil {
		return nil, err
	}
	return &RawGeo{
		db:   kvdb,
		path: path,
		log:  Log.New("path", filepath.Base(path)),
	}, nil
}

// Close gracefully closes the opened index
func (rg *RawGeo) Close() error {
	return rg.db.Close()
}

// Destroy will try to remove the index file
func (rg *RawGeo) Destroy() error {
	if rg.path != "" {
		rg.Close()
		return os.RemoveAll(rg.path)
	}
	return nil
}

// Put index the given entry in the given index
// (you have to cared about duplicate IDs)
func (rg *RawGeo) Index(point *Point) error {
	// Ensure the entry contains a latitude and a longitude
	if point.Lat == 0 || point.Lng == 0 {
		return ErrInvalidLatLong
	}
	// Compute the Geohash if needed
	if point.Geohash == "" {
		point.Geohash = geohash.Encode(point.Lat, point.Lng)
	}
	if point.ID == "" {
		return ErrMissingID
	}
	if err := rg.db.Set([]byte(fmt.Sprintf(indexKeyFmt, point.Geohash, point.ID)), nil); err != nil {
		return err
	}
	rg.log.Debug("indexed new point", "id", point.ID, "lat", point.Lat, "lng", point.Lng, "geohash", point.Geohash)
	return nil
}

// Query will returns all the points found in the given radius (in meters) sorted by distance to the query
func (rg *RawGeo) Query(lat, lng, radius float64) ([]*Point, error) {
	start := time.Now()
	qlog := rg.log.New("lat", lat, "lng", lng, "radius", radius)
	qlog.Debug("new query")
	refPoint := &Point{
		Lat: lat,
		Lng: lng,
	}
	precision := radiusToPrecision(radius)
	qlog.Debug("converted radius to precision", "radius", radius, "precision", precision)
	gh := geohash.EncodeWithPrecision(lat, lng, precision)
	qlog.Debug("query geohash", "geohash", gh)

	res := []*Point{}

	// Search in the given geohash box, along with all the 8 adjacent boxes
	for _, geohash := range append(geohash.CalculateAllAdjacent(gh), gh) {
		subres, err := rg.findPrefix(geohash)
		qlog.Debug("new subquery", "geohash", geohash, "cnt", len(subres))
		if err != nil {
			return nil, err
		}
		for _, point := range subres {
			// Compute the distance from the reference point (i.e. the query)
			point.Distance = refPoint.DistanceFrom(point)
			// Since the geohashes window is not a radius, we still need to check
			// if the point is located within the given radius.
			if point.Distance > radius {
				continue
			}
			res = append(res, point)
		}
	}
	sort.Sort(byDistance(res))
	qlog.Debug("query took", "duration", time.Since(start))
	return res, nil
}

func (rg *RawGeo) findPrefix(geoPrefix string) ([]*Point, error) {
	var limit int
	points := []*Point{}
	enum, _, err := rg.db.Seek([]byte(geoPrefix))
	if err != nil {
		return nil, err
	}
	// TODO(tsileo): check/fix the endBytes
	endBytes := []byte("\xff")
	i := 0
	for {
		k, _, err := enum.Next()
		sk := string(k)
		if err == io.EOF {
			break
		}
		if !strings.HasPrefix(sk, geoPrefix) {
			break
		}
		if bytes.Compare(k, endBytes) > 0 || (limit != 0 && i > limit) {
			return points, nil
		}
		parts := strings.SplitN(sk, ":", 2)
		p := NewPointFromGeohash(parts[1], parts[0])
		points = append(points, p)
		i++
	}
	return points, nil
}

// Data comes from http://www.movable-type.co.uk/scripts/geohash.html
var geoPrecision = []struct {
	radius    float64 // in meters
	precision int     // Geohash precision
}{
	{0.074, 11},
	{0.6, 10},
	{2.4, 9},
	{19, 8},
	{76, 7},
	{610, 6},
	{2400, 5},
	{20000, 4},
	{78000, 3},
	{630000, 2},
	{2500000, 1},
}

// Convert radius meters to Geohash precision
func radiusToPrecision(r float64) int {
	for _, gp := range geoPrecision {
		if r <= gp.radius {
			return gp.precision
		}
	}
	return 2
}
<|endoftext|>


<commit_before>package ipam

import (
	"context"
	"encoding/json"
	"net"

	"github.com/giantswarm/apiextensions/pkg/apis/cluster/v1alpha1"
	"github.com/giantswarm/microerror"
	"github.com/giantswarm/micrologger"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset"
)

type ClusterPersisterConfig struct {
	CMAClient clientset.Interface
	Logger    micrologger.Logger
}

type ClusterPersister struct {
	cmaClient clientset.Interface
	logger    micrologger.Logger
}

func NewClusterPersister(config ClusterPersisterConfig) (*ClusterPersister, error) {
	if config.CMAClient == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.CMAClient must not be empty", config)
	}
	if config.Logger == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config)
	}

	p := &ClusterPersister{
		cmaClient: config.CMAClient,
		logger:    config.Logger,
	}

	return p, nil
}

func (p *ClusterPersister) Persist(ctx context.Context, subnet net.IPNet, namespace string, name string) error {
	cr, err := p.cmaClient.ClusterV1alpha1().Clusters(namespace).Get(name, metav1.GetOptions{})
	if err != nil {
		return microerror.Mask(err)
	}

	var providerStatus v1alpha1.AWSClusterStatus
	{
		if cr.Status.ProviderStatus == nil {
			cr.Status.ProviderStatus = &runtime.RawExtension{}
		}

		err := json.Unmarshal(cr.Status.ProviderStatus.Raw, &providerStatus)
		if err != nil {
			return microerror.Mask(err)
		}

		providerStatus.Provider.Network.CIDR = subnet.String()
	}

	{
		b, err := json.Marshal(providerStatus)
		if err != nil {
			return microerror.Mask(err)
		}

		cr.Status.ProviderStatus.Raw = b
	}

	{
		_, err := p.cmaClient.ClusterV1alpha1().Clusters(cr.Namespace).UpdateStatus(cr)
		if err != nil {
			return microerror.Mask(err)
		}
	}

	return nil
}
<commit_msg>fix unexpected end of json input error on nil byte slice (#1876)<commit_after>package ipam

import (
	"context"
	"encoding/json"
	"net"

	"github.com/giantswarm/apiextensions/pkg/apis/cluster/v1alpha1"
	"github.com/giantswarm/microerror"
	"github.com/giantswarm/micrologger"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset"
)

type ClusterPersisterConfig struct {
	CMAClient clientset.Interface
	Logger    micrologger.Logger
}

type ClusterPersister struct {
	cmaClient clientset.Interface
	logger    micrologger.Logger
}

func NewClusterPersister(config ClusterPersisterConfig) (*ClusterPersister, error) {
	if config.CMAClient == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.CMAClient must not be empty", config)
	}
	if config.Logger == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config)
	}

	p := &ClusterPersister{
		cmaClient: config.CMAClient,
		logger:    config.Logger,
	}

	return p, nil
}

func (p *ClusterPersister) Persist(ctx context.Context, subnet net.IPNet, namespace string, name string) error {
	cr, err := p.cmaClient.ClusterV1alpha1().Clusters(namespace).Get(name, metav1.GetOptions{})
	if err != nil {
		return microerror.Mask(err)
	}

	var providerStatus v1alpha1.AWSClusterStatus
	{
		if cr.Status.ProviderStatus == nil {
			cr.Status.ProviderStatus = &runtime.RawExtension{}
		}
		if cr.Status.ProviderStatus.Raw == nil {
			cr.Status.ProviderStatus.Raw = []byte{}
		}

		err := json.Unmarshal(cr.Status.ProviderStatus.Raw, &providerStatus)
		if err != nil {
			return microerror.Mask(err)
		}

		providerStatus.Provider.Network.CIDR = subnet.String()
	}

	{
		b, err := json.Marshal(providerStatus)
		if err != nil {
			return microerror.Mask(err)
		}

		cr.Status.ProviderStatus.Raw = b
	}

	{
		_, err := p.cmaClient.ClusterV1alpha1().Clusters(cr.Namespace).UpdateStatus(cr)
		if err != nil {
			return microerror.Mask(err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package goraygun

import (
  "encoding/json"
  "errors"
  "log"
  "net/http"
  "strings"
)

const ENDPOINT = "https://api.raygun.io/entries"

const (
  ClientName    = "Go-Raygun"
  ClientVersion = "0.0.1"
  ClientRepo    = "http://github.com/sditools/go-raygun"
)

type Settings struct {
  ApiKey   string
  Enabled  bool
  Endpoint string
}

type Client struct {
  settings Settings
  Entry    Entry
}

func Init(s Settings, e Entry) (c Client) {
  // provide user the option to override (for testing)
  if s.Endpoint == "" {
    s.Endpoint = ENDPOINT
  }

  c.settings = s
  e.Details.Environment.populate()
  c.Entry = e
  return
}

func (c *Client) Recover() {
  if err := recover(); err != nil {
    c.Report(getError(err), c.Entry)
  }
}

func (c *Client) Report(err error, entry Entry) {
  if !c.settings.Enabled {
    return
  }
  st, stErr := GetStackTrace(3)
  if stErr != nil {
    // handle stErr
    return
  }
  entry.populate(err, st)
  c.post(entry, c.settings.Endpoint)
}

func (c *Client) post(e Entry, uri string) {
  data, err := json.Marshal(e)
  if err != nil {
    log.Printf("Error Marshalling RayGun Message: %v:", err)
  }

  req, err := http.NewRequest("POST", uri, strings.NewReader(string(data)))
  if err != nil {
    log.Printf("Error creating POST request: %v", err)
  }

  req.Header.Set("X-ApiKey", c.settings.ApiKey)
  req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

  resp, err := http.DefaultClient.Do(req)
  if err != nil {
    log.Printf("Error sending Request: %v", err)
  }

  if resp.StatusCode != http.StatusAccepted {
    log.Printf("Error status sent back: %v", resp.StatusCode)
  }

  defer resp.Body.Close()
}

func getError(err interface{}) error {
  switch err := err.(type) {
  case error:
    return err
  case string:
    return errors.New(err)
  default:
    return errors.New("")
  }
}
<commit_msg>logging<commit_after>package goraygun

import (
  "encoding/json"
  "errors"
  "log"
  "net/http"
  "strconv"
  "strings"
)

const ENDPOINT = "https://api.raygun.io/entries"

const (
  ClientName    = "Go-Raygun"
  ClientVersion = "0.0.1"
  ClientRepo    = "http://github.com/sditools/go-raygun"
)

type Settings struct {
  ApiKey   string
  Enabled  bool
  Endpoint string
}

type Client struct {
  settings Settings
  Entry    Entry
}

func Init(s Settings, e Entry) (c Client) {
  // provide user the option to override (for testing)
  if s.Endpoint == "" {
    s.Endpoint = ENDPOINT
  }

  c.settings = s
  e.Details.Environment.populate()
  c.Entry = e
  return
}

func (c *Client) Recover() {
  if err := recover(); err != nil {
    c.Report(getError(err), c.Entry)
  }
}

func (c *Client) Report(err error, entry Entry) {
  st, stErr := GetStackTrace(3)
  if stErr != nil {
    // handle stErr
    return
  }
  if !c.settings.Enabled {
    log.Println("Error:", err.Error())
    log.Println(st[0].FileName + ":" + strconv.Itoa(st[0].LineNumber))
    return
  }
  entry.populate(err, st)
  c.post(entry, c.settings.Endpoint)
}

func (c *Client) post(e Entry, uri string) {
  data, err := json.Marshal(e)
  if err != nil {
    log.Printf("Error Marshalling RayGun Message: %v:", err)
  }

  req, err := http.NewRequest("POST", uri, strings.NewReader(string(data)))
  if err != nil {
    log.Printf("Error creating POST request: %v", err)
  }

  req.Header.Set("X-ApiKey", c.settings.ApiKey)
  req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

  resp, err := http.DefaultClient.Do(req)
  if err != nil {
    log.Printf("Error sending Request: %v", err)
  }

  if resp.StatusCode != http.StatusAccepted {
    log.Printf("Error status sent back: %v", resp.StatusCode)
  }

  defer resp.Body.Close()
}

func getError(err interface{}) error {
  switch err := err.(type) {
  case error:
    return err
  case string:
    return errors.New(err)
  default:
    return errors.New("")
  }
}
<|endoftext|>


<commit_before>// Package gedcom implements a parser to read genealogical data in a standard format
package gedcom

import (
	"io"
	"strconv"

	"github.com/MJKWoolnough/errors"
	"github.com/MJKWoolnough/parser"
)

type line struct {
	level  uint64
	xrefID string
	tag    string
	value  string
}

// Reader reads Records from the underlying GEDCOM file
type Reader struct {
	t       *tokeniser
	options options

	peeked bool
	line   line
	err    error
	done   bool

	hadHeader, hadRecord bool
}

// NewReader creates a new Reader, setting the given options
func NewReader(r io.Reader, opts ...Option) *Reader {
	var o options
	for _, opt := range opts {
		opt(&o)
	}
	return &Reader{
		t:       newTokeniser(r, o),
		options: o,
	}
}

func (r *Reader) readLine() {
	if r.err != nil {
		return
	}
	var t parser.Token
	t, r.err = r.t.GetToken()
	if r.err != nil {
		return
	}
	if t.Type != tokenLevel {
		if t.Type == parser.TokenDone {
			r.done = true
			return
		}
		r.err = ErrNotLevel
		return
	}
	r.line.level, r.err = strconv.ParseUint(t.Data, 10, 64)
	if r.err != nil {
		return
	}
	t, r.err = r.t.GetToken()
	if r.err != nil {
		return
	}
	if t.Type == tokenXref {
		r.line.xrefID = t.Data
		t, r.err = r.t.GetToken()
		if r.err != nil {
			return
		}
	} else {
		r.line.xrefID = ""
	}
	if t.Type != tokenTag {
		r.err = ErrNotTag
		return
	}
	r.line.tag = t.Data
	t, r.err = r.t.GetToken()
	if r.err != nil {
		return
	}
	if t.Type == tokenEndLine {
		r.line.value = ""
		return
	}
	if t.Type != tokenLine && t.Type != tokenPointer {
		r.err = ErrNotLine
		return
	}
	r.line.value = t.Data
}

// Record returns a GEDCOM Record.
// Record types are: -
// 	Header
// 	Submission
// 	Family
// 	Invididual
// 	MultimediaNote
// 	Repository
// 	Source
// 	Submitter
//	Trailer
func (r *Reader) Record() (Record, error) {
	if !r.peeked {
		r.readLine()
		r.peeked = true
	}
	if r.done {
		return nil, io.EOF
	}
	if r.err != nil {
		return nil, r.err
	}
	if !r.hadHeader {
		if r.line.tag != "HEAD" {
			r.peeked = false
			return nil, ErrNoHeader
		}
		r.hadHeader = true
	} else if !r.hadRecord {
		switch r.line.tag {
		case "FAM", "INDI", "OBJE", "NOTE", "REPO", "SOUR", "SUBN":
			r.hadRecord = true
		case "TRLR":
			r.peeked = false
			return nil, ErrNoRecords
		}
	} else if r.line.tag == "TRLR" {
		r.peeked = false
		return &Trailer{}, nil
	}

	lines := make([]line, 1, 32)
	lines[0] = r.line
	var lastlevel uint64
	for {
		if r.err != nil {
			return nil, r.err
		}
		if r.line.level > lastlevel+1 {
			return nil, ErrInvalidLevel
		}
		lastlevel = r.line.level
		lines = append(lines, r.line)
		r.readLine()
		if r.line.level == 0 {
			plines := parseLines(lines)
			var record Record
			switch lines[0].tag {
			case "HEAD":
				record = &Header{}
			case "SUBM":
				record = &SubmissionRecord{}
			case "FAM":
				record = &Family{}
			case "INDI":
				record = &Individual{}
			case "OBJE":
				record = &MultimediaRecord{}
			case "NOTE":
				record = &NoteRecord{}
			case "REPO":
				record = &RepositoryRecord{}
			case "SOUR":
				record = &SourceRecord{}
			case "SUBN":
				record = &SubmitterRecord{}
			default:
				if lines[0].tag[0] == '_' {
					return plines, nil
				}
				return plines, ErrContext{"root", lines[0].tag, ErrUnknownTag}
			}
			err := record.parse(&plines, r.options)
			if err != nil {
				return nil, ErrContext{"root", lines[0].tag, err}
			}
			return record, nil
		}
	}

}

// Errors
var (
	ErrNoHeader  errors.Error = "no header"
	ErrNoRecords errors.Error = "no records"
	ErrNotLevel  errors.Error = "not level token"
	ErrNotTag    errors.Error = "not tag token"
	ErrNotLine   errors.Error = "not line_value token"
)
<commit_msg>updated comment to show types returned are pointers<commit_after>// Package gedcom implements a parser to read genealogical data in a standard format
package gedcom

import (
	"io"
	"strconv"

	"github.com/MJKWoolnough/errors"
	"github.com/MJKWoolnough/parser"
)

type line struct {
	level  uint64
	xrefID string
	tag    string
	value  string
}

// Reader reads Records from the underlying GEDCOM file
type Reader struct {
	t       *tokeniser
	options options

	peeked bool
	line   line
	err    error
	done   bool

	hadHeader, hadRecord bool
}

// NewReader creates a new Reader, setting the given options
func NewReader(r io.Reader, opts ...Option) *Reader {
	var o options
	for _, opt := range opts {
		opt(&o)
	}
	return &Reader{
		t:       newTokeniser(r, o),
		options: o,
	}
}

func (r *Reader) readLine() {
	if r.err != nil {
		return
	}
	var t parser.Token
	t, r.err = r.t.GetToken()
	if r.err != nil {
		return
	}
	if t.Type != tokenLevel {
		if t.Type == parser.TokenDone {
			r.done = true
			return
		}
		r.err = ErrNotLevel
		return
	}
	r.line.level, r.err = strconv.ParseUint(t.Data, 10, 64)
	if r.err != nil {
		return
	}
	t, r.err = r.t.GetToken()
	if r.err != nil {
		return
	}
	if t.Type == tokenXref {
		r.line.xrefID = t.Data
		t, r.err = r.t.GetToken()
		if r.err != nil {
			return
		}
	} else {
		r.line.xrefID = ""
	}
	if t.Type != tokenTag {
		r.err = ErrNotTag
		return
	}
	r.line.tag = t.Data
	t, r.err = r.t.GetToken()
	if r.err != nil {
		return
	}
	if t.Type == tokenEndLine {
		r.line.value = ""
		return
	}
	if t.Type != tokenLine && t.Type != tokenPointer {
		r.err = ErrNotLine
		return
	}
	r.line.value = t.Data
}

// Record returns a GEDCOM Record.
// Record types are: -
// 	*Header
// 	*Submission
// 	*Family
// 	*Invididual
// 	*MultimediaNote
// 	*Repository
// 	*Source
// 	*Submitter
//	*Trailer
func (r *Reader) Record() (Record, error) {
	if !r.peeked {
		r.readLine()
		r.peeked = true
	}
	if r.done {
		return nil, io.EOF
	}
	if r.err != nil {
		return nil, r.err
	}
	if !r.hadHeader {
		if r.line.tag != "HEAD" {
			r.peeked = false
			return nil, ErrNoHeader
		}
		r.hadHeader = true
	} else if !r.hadRecord {
		switch r.line.tag {
		case "FAM", "INDI", "OBJE", "NOTE", "REPO", "SOUR", "SUBN":
			r.hadRecord = true
		case "TRLR":
			r.peeked = false
			return nil, ErrNoRecords
		}
	} else if r.line.tag == "TRLR" {
		r.peeked = false
		return &Trailer{}, nil
	}

	lines := make([]line, 1, 32)
	lines[0] = r.line
	var lastlevel uint64
	for {
		if r.err != nil {
			return nil, r.err
		}
		if r.line.level > lastlevel+1 {
			return nil, ErrInvalidLevel
		}
		lastlevel = r.line.level
		lines = append(lines, r.line)
		r.readLine()
		if r.line.level == 0 {
			plines := parseLines(lines)
			var record Record
			switch lines[0].tag {
			case "HEAD":
				record = &Header{}
			case "SUBM":
				record = &SubmissionRecord{}
			case "FAM":
				record = &Family{}
			case "INDI":
				record = &Individual{}
			case "OBJE":
				record = &MultimediaRecord{}
			case "NOTE":
				record = &NoteRecord{}
			case "REPO":
				record = &RepositoryRecord{}
			case "SOUR":
				record = &SourceRecord{}
			case "SUBN":
				record = &SubmitterRecord{}
			default:
				if lines[0].tag[0] == '_' {
					return plines, nil
				}
				return plines, ErrContext{"root", lines[0].tag, ErrUnknownTag}
			}
			err := record.parse(&plines, r.options)
			if err != nil {
				return nil, ErrContext{"root", lines[0].tag, err}
			}
			return record, nil
		}
	}

}

// Errors
var (
	ErrNoHeader  errors.Error = "no header"
	ErrNoRecords errors.Error = "no records"
	ErrNotLevel  errors.Error = "not level token"
	ErrNotTag    errors.Error = "not tag token"
	ErrNotLine   errors.Error = "not line_value token"
)
<|endoftext|>


<commit_before>package builder

// nginx
const (
	NginxVersion           = "1.22.0"
	NginxDownloadURLPrefix = "https://nginx.org/download"
)

// pcre
const (
	PcreVersion           = "10.40"
	PcreDownloadURLPrefix = "https://github.com/PhilipHazel/pcre2/releases/download"
)

// openssl
const (
	OpenSSLVersion           = "1.1.1o"
	OpenSSLDownloadURLPrefix = "https://www.openssl.org/source"
)

// libressl
const (
	LibreSSLVersion           = "3.5.3"
	LibreSSLDownloadURLPrefix = "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL"
)

// zlib
const (
	ZlibVersion           = "1.2.12"
	ZlibDownloadURLPrefix = "https://zlib.net"
)

// openResty
const (
	OpenRestyVersion           = "1.21.4.1"
	OpenRestyDownloadURLPrefix = "https://openresty.org/download"
)

// tengine
const (
	TengineVersion           = "2.3.3"
	TengineDownloadURLPrefix = "https://tengine.taobao.org/download"
)

// component enumerations
const (
	ComponentNginx = iota
	ComponentOpenResty
	ComponentTengine
	ComponentPcre
	ComponentOpenSSL
	ComponentLibreSSL
	ComponentZlib
	ComponentMax
)
<commit_msg>bumped default openssl version to 1.1.1p.<commit_after>package builder

// nginx
const (
	NginxVersion           = "1.22.0"
	NginxDownloadURLPrefix = "https://nginx.org/download"
)

// pcre
const (
	PcreVersion           = "10.40"
	PcreDownloadURLPrefix = "https://github.com/PhilipHazel/pcre2/releases/download"
)

// openssl
const (
	OpenSSLVersion           = "1.1.1p"
	OpenSSLDownloadURLPrefix = "https://www.openssl.org/source"
)

// libressl
const (
	LibreSSLVersion           = "3.5.3"
	LibreSSLDownloadURLPrefix = "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL"
)

// zlib
const (
	ZlibVersion           = "1.2.12"
	ZlibDownloadURLPrefix = "https://zlib.net"
)

// openResty
const (
	OpenRestyVersion           = "1.21.4.1"
	OpenRestyDownloadURLPrefix = "https://openresty.org/download"
)

// tengine
const (
	TengineVersion           = "2.3.3"
	TengineDownloadURLPrefix = "https://tengine.taobao.org/download"
)

// component enumerations
const (
	ComponentNginx = iota
	ComponentOpenResty
	ComponentTengine
	ComponentPcre
	ComponentOpenSSL
	ComponentLibreSSL
	ComponentZlib
	ComponentMax
)
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package cbft

import (
	"encoding/json"
	"fmt"
	"hash"
	"hash/crc32"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	log "github.com/couchbase/clog"
)

const FILES_FEED_SLEEP_START_MS = 5000
const FILES_FEED_BACKOFF_FACTOR = 1.5
const FILES_FEED_MAX_SLEEP_MS = 1000 * 60 * 5 // 5 minutes.

func init() {
	RegisterFeedType("files", &FeedType{
		Start:      StartFilesFeed,
		Partitions: FilesFeedPartitions,
		Public:     true,
		Description: "general/files" +
			" - files under a dataDir subdirectory tree will be the data source",
		StartSample: &FilesFeedParams{
			RegExps:       []string{".txt$", ".md$"},
			SleepStartMS:  FILES_FEED_SLEEP_START_MS,
			BackoffFactor: FILES_FEED_BACKOFF_FACTOR,
			MaxSleepMS:    FILES_FEED_MAX_SLEEP_MS,
		},
	})
}

// FilesFeed is a feed implementation that that emits file contents
// from a local subdirectory tree.
//
// The subdirectory tree lives under the dataDir...
//
//    <dataDir>/<sourceName/**
//
// FilesFeed supports optional regexp patterns to allow you to filter
// for only the file paths that you want.
//
// Limitations:
//
// - Only a small number of files will work well (hundreds to low
// thousands, not millions).
//
// - FilesFeed uses file modification timestamps as a poor-man's
// approach instead of properly tracking sequence numbers.  That has
// implications such as whenever a FilesFeed (re-)starts (e.g., the
// process restarts), the FilesFeed will re-emits all files and then
// track the max modification timestamp going forwards as it regularly
// polls for file changes.
type FilesFeed struct {
	mgr        *Manager
	name       string
	indexName  string
	sourceName string
	params     *FilesFeedParams
	dests      map[string]Dest
	disable    bool

	m       sync.Mutex
	closeCh chan struct{}
}

type FilesFeedParams struct {
	RegExps       []string `json:"regExps"`
	MaxFileSize   int64    `json:"maxFileSize"`
	NumPartitions int      `json:"numPartitions"`
	SleepStartMS  int      `json:"sleepStartMS"`
	BackoffFactor float32  `json:"backoffFactor"`
	MaxSleepMS    int      `json:"maxSleepMS"`
}

type FileDoc struct {
	Name     string `json:"name"`
	Path     string `json:"path"` // Path relative to the source name.
	Contents string `json:"contents"`
}

func StartFilesFeed(mgr *Manager, feedName, indexName, indexUUID,
	sourceType, sourceName, sourceUUID, params string,
	dests map[string]Dest) error {
	feed, err := NewFilesFeed(mgr, feedName, indexName, sourceName,
		params, dests, mgr.tagsMap != nil && !mgr.tagsMap["feed"])
	if err != nil {
		return fmt.Errorf("feed_files: NewFilesFeed,"+
			" feedName: %s, err: %v", feedName, err)
	}
	err = feed.Start()
	if err != nil {
		return fmt.Errorf("feed_files: could not start,"+
			" feedName: %s, err: %v", feedName, err)
	}
	err = mgr.registerFeed(feed)
	if err != nil {
		feed.Close()
		return err
	}
	return nil
}

func NewFilesFeed(mgr *Manager, name, indexName, sourceName,
	paramsStr string, dests map[string]Dest, disable bool) (
	*FilesFeed, error) {
	if sourceName == "" {
		return nil, fmt.Errorf("feed_files: missing source name")
	}

	if strings.Index(sourceName, "..") >= 0 {
		return nil, fmt.Errorf("feed_files: disallowed source name,"+
			" name: %s, sourceName: %q", name, sourceName)
	}

	params := &FilesFeedParams{}
	if paramsStr != "" {
		err := json.Unmarshal([]byte(paramsStr), params)
		if err != nil {
			return nil, err
		}
	}

	return &FilesFeed{
		mgr:        mgr,
		name:       name,
		indexName:  indexName,
		sourceName: sourceName,
		params:     params,
		dests:      dests,
		disable:    disable,
		closeCh:    make(chan struct{}),
	}, nil
}

func (t *FilesFeed) Name() string {
	return t.name
}

func (t *FilesFeed) IndexName() string {
	return t.indexName
}

func (t *FilesFeed) Start() error {
	if t.disable {
		log.Printf("feed_files: disable, name: %s", t.Name())
		return nil
	}

	startSleepMS := t.params.SleepStartMS
	if startSleepMS <= 0 {
		startSleepMS = FILES_FEED_SLEEP_START_MS
	}

	backoffFactor := t.params.BackoffFactor
	if backoffFactor <= 0 {
		backoffFactor = FILES_FEED_BACKOFF_FACTOR
	}

	maxSleepMS := t.params.MaxSleepMS
	if maxSleepMS <= 0 {
		maxSleepMS = FILES_FEED_MAX_SLEEP_MS
	}

	numPartitions := t.params.NumPartitions
	if numPartitions < 0 {
		numPartitions = 0
	}

	partitions := make([]string, numPartitions)
	for i := 0; i < len(partitions); i++ {
		partitions[i] = strconv.Itoa(i)
	}

	go func() {
		initTime := time.Now()
		initTimeMicroSecs := initTime.UnixNano() / int64(1000)

		// TODO: NOTE: We're assuming (lazily, incorrectly) that this
		// way of initializing a sequence number never goes downwards,
		// even during fast restarts or clock changes or node
		// rebalances/reassignments.
		seqs := map[string]uint64{}
		for partition, _ := range t.dests {
			seqs[partition] = uint64(initTimeMicroSecs)
		}

		var prevStartTime time.Time

		ExponentialBackoffLoop(t.Name(),
			func() int {
				select {
				case <-t.closeCh:
					return -1
				default:
				}

				h := crc32.NewIEEE()

				startTime := time.Now()

				progress := false

				paths, err := FilesFindMatches(t.mgr.DataDir(),
					t.sourceName, t.params.RegExps, prevStartTime,
					t.params.MaxFileSize)
				if err != nil {
					log.Printf("feed_files, FilesFindMatches, err: %v", err)
					return -1
				}

				seqDeltaMax := uint64(0)

				seqEnds := map[string]uint64{}

				for _, path := range paths {
					partition := FilesPathToPartition(h, partitions, path)

					if t.dests[partition] == nil {
						continue
					}

					seq := seqs[partition]

					seqEnd, exists := seqEnds[partition]
					if exists {
						seqEnd = seqEnd + 1
					} else {
						seqEnd = seq
					}
					seqEnds[partition] = seqEnd

					if seqDeltaMax < seqEnd-seq {
						seqDeltaMax = seqEnd - seq
					}
				}

				snapshotSent := map[string]bool{}

				for _, path := range paths {
					select {
					case <-t.closeCh:
						return -1
					default:
					}

					partition := FilesPathToPartition(h, partitions, path)

					dest := t.dests[partition]
					if dest == nil {
						continue
					}

					seqCur := seqs[partition]
					seqs[partition] = seqCur + 1

					buf, err := ioutil.ReadFile(path)
					if err != nil {
						log.Printf("feed_files: read file,"+
							" name: %s, path: %s, err: %v",
							t.Name(), path, err)
						continue
					}

					jbuf, err := json.Marshal(FileDoc{
						Name:     filepath.Base(path),
						Path:     path,
						Contents: string(buf),
					})
					if err != nil {
						log.Printf("feed_files: json marshal file,"+
							" name: %s, path: %s, err: %v",
							t.Name(), path, err)
						continue
					}

					if !snapshotSent[partition] {
						err = dest.SnapshotStart(partition, seqCur,
							seqEnds[partition])
						if err != nil {
							log.Printf("feed_files: SnapshotStart,"+
								" name: %s, partition: %s,"+
								" seqCur: %d, seqEnd: %s, err: %v",
								t.Name(), partition,
								seqCur, seqEnds[partition], err)
							return -1
						}

						snapshotSent[partition] = true
					}

					pathBuf := []byte(path)

					err = dest.DataUpdate(partition, pathBuf, seqCur, jbuf)
					if err != nil {
						log.Printf("feed_files: DataUpdate,"+
							" name: %s, path: %s, partition: %s,"+
							" seqCur: %d, err: %v",
							t.Name(), path, partition, seqCur, err)
						return -1
					}

					progress = true
				}

				prevStartTime = startTime

				// NOTE: We may need to sleep a certain amount in case
				// there were tons of file updates/mutations, and we
				// want to reduce the window of potentially repeating
				// sequence numbers.  The window still exists if we
				// crash and quickly restart during the sleep, where
				// the restarted process might have a
				// lower-than-wanted initTime.
				wantTime := initTime // Copy, because Add() mutates.
				wantTime.Add(time.Duration(int64(seqDeltaMax)))

				currTime := time.Now()
				if wantTime.After(currTime) {
					time.Sleep(wantTime.Sub(currTime))
				}

				if progress {
					return 1
				}
				return 0
			},
			startSleepMS,
			backoffFactor,
			maxSleepMS)
	}()

	return nil
}

func (t *FilesFeed) Close() error {
	t.m.Lock()
	if t.closeCh != nil {
		close(t.closeCh)
		t.closeCh = nil
	}
	t.m.Unlock()

	return nil
}

func (t *FilesFeed) Dests() map[string]Dest {
	return t.dests
}

func (t *FilesFeed) Stats(w io.Writer) error {
	_, err := w.Write([]byte("{}"))
	return err
}

// -----------------------------------------------------

func FilesFeedPartitions(sourceType, sourceName, sourceUUID, sourceParams,
	server string) ([]string, error) {
	ffp := &FilesFeedParams{}
	if sourceParams != "" {
		err := json.Unmarshal([]byte(sourceParams), ffp)
		if err != nil {
			return nil, fmt.Errorf("feed_files:"+
				" could not parse sourceParams: %s, err: %v",
				sourceParams, err)
		}
	}
	rv := make([]string, ffp.NumPartitions)
	for i := 0; i < ffp.NumPartitions; i++ {
		rv[i] = strconv.Itoa(i)
	}
	return rv, nil
}

// -----------------------------------------------------

func FilesFindMatches(dataDir, sourceName string,
	regExps []string, modTimeGTE time.Time, maxSize int64) (
	[]string, error) {
	walkPath, err := filepath.EvalSymlinks(dataDir +
		string(os.PathSeparator) + "files" +
		string(os.PathSeparator) + sourceName)
	if err != nil {
		return nil, err
	}

	pathsOk := []string(nil)

	err = filepath.Walk(walkPath,
		func(path string, fi os.FileInfo, err error) error {
			if err != nil ||
				fi.IsDir() ||
				fi.ModTime().Before(modTimeGTE) ||
				(maxSize > 0 && fi.Size() > maxSize) {
				return nil
			}

			if len(regExps) <= 0 {
				pathsOk = append(pathsOk, path)
				return nil
			}

			for _, reStr := range regExps {
				matched, err := regexp.MatchString(reStr, path)
				if err != nil {
					return fmt.Errorf("feed_files, MatchString,"+
						" reStr: %s, path: %s, err: %v",
						reStr, path, err)
				}
				if matched {
					pathsOk = append(pathsOk, path)
					return nil
				}
			}

			return nil
		})
	if err != nil {
		return nil, err
	}

	return pathsOk, nil
}

func FilesPathToPartition(h hash.Hash32,
	partitions []string, path string) string {
	if len(partitions) <= 0 {
		return ""
	}

	h.Reset()
	io.WriteString(h, path)
	i := h.Sum32() % uint32(len(partitions))
	return partitions[i]
}
<commit_msg>printf fmt type fixed<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package cbft

import (
	"encoding/json"
	"fmt"
	"hash"
	"hash/crc32"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	log "github.com/couchbase/clog"
)

const FILES_FEED_SLEEP_START_MS = 5000
const FILES_FEED_BACKOFF_FACTOR = 1.5
const FILES_FEED_MAX_SLEEP_MS = 1000 * 60 * 5 // 5 minutes.

func init() {
	RegisterFeedType("files", &FeedType{
		Start:      StartFilesFeed,
		Partitions: FilesFeedPartitions,
		Public:     true,
		Description: "general/files" +
			" - files under a dataDir subdirectory tree will be the data source",
		StartSample: &FilesFeedParams{
			RegExps:       []string{".txt$", ".md$"},
			SleepStartMS:  FILES_FEED_SLEEP_START_MS,
			BackoffFactor: FILES_FEED_BACKOFF_FACTOR,
			MaxSleepMS:    FILES_FEED_MAX_SLEEP_MS,
		},
	})
}

// FilesFeed is a feed implementation that that emits file contents
// from a local subdirectory tree.
//
// The subdirectory tree lives under the dataDir...
//
//    <dataDir>/<sourceName/**
//
// FilesFeed supports optional regexp patterns to allow you to filter
// for only the file paths that you want.
//
// Limitations:
//
// - Only a small number of files will work well (hundreds to low
// thousands, not millions).
//
// - FilesFeed uses file modification timestamps as a poor-man's
// approach instead of properly tracking sequence numbers.  That has
// implications such as whenever a FilesFeed (re-)starts (e.g., the
// process restarts), the FilesFeed will re-emits all files and then
// track the max modification timestamp going forwards as it regularly
// polls for file changes.
type FilesFeed struct {
	mgr        *Manager
	name       string
	indexName  string
	sourceName string
	params     *FilesFeedParams
	dests      map[string]Dest
	disable    bool

	m       sync.Mutex
	closeCh chan struct{}
}

type FilesFeedParams struct {
	RegExps       []string `json:"regExps"`
	MaxFileSize   int64    `json:"maxFileSize"`
	NumPartitions int      `json:"numPartitions"`
	SleepStartMS  int      `json:"sleepStartMS"`
	BackoffFactor float32  `json:"backoffFactor"`
	MaxSleepMS    int      `json:"maxSleepMS"`
}

type FileDoc struct {
	Name     string `json:"name"`
	Path     string `json:"path"` // Path relative to the source name.
	Contents string `json:"contents"`
}

func StartFilesFeed(mgr *Manager, feedName, indexName, indexUUID,
	sourceType, sourceName, sourceUUID, params string,
	dests map[string]Dest) error {
	feed, err := NewFilesFeed(mgr, feedName, indexName, sourceName,
		params, dests, mgr.tagsMap != nil && !mgr.tagsMap["feed"])
	if err != nil {
		return fmt.Errorf("feed_files: NewFilesFeed,"+
			" feedName: %s, err: %v", feedName, err)
	}
	err = feed.Start()
	if err != nil {
		return fmt.Errorf("feed_files: could not start,"+
			" feedName: %s, err: %v", feedName, err)
	}
	err = mgr.registerFeed(feed)
	if err != nil {
		feed.Close()
		return err
	}
	return nil
}

func NewFilesFeed(mgr *Manager, name, indexName, sourceName,
	paramsStr string, dests map[string]Dest, disable bool) (
	*FilesFeed, error) {
	if sourceName == "" {
		return nil, fmt.Errorf("feed_files: missing source name")
	}

	if strings.Index(sourceName, "..") >= 0 {
		return nil, fmt.Errorf("feed_files: disallowed source name,"+
			" name: %s, sourceName: %q", name, sourceName)
	}

	params := &FilesFeedParams{}
	if paramsStr != "" {
		err := json.Unmarshal([]byte(paramsStr), params)
		if err != nil {
			return nil, err
		}
	}

	return &FilesFeed{
		mgr:        mgr,
		name:       name,
		indexName:  indexName,
		sourceName: sourceName,
		params:     params,
		dests:      dests,
		disable:    disable,
		closeCh:    make(chan struct{}),
	}, nil
}

func (t *FilesFeed) Name() string {
	return t.name
}

func (t *FilesFeed) IndexName() string {
	return t.indexName
}

func (t *FilesFeed) Start() error {
	if t.disable {
		log.Printf("feed_files: disable, name: %s", t.Name())
		return nil
	}

	startSleepMS := t.params.SleepStartMS
	if startSleepMS <= 0 {
		startSleepMS = FILES_FEED_SLEEP_START_MS
	}

	backoffFactor := t.params.BackoffFactor
	if backoffFactor <= 0 {
		backoffFactor = FILES_FEED_BACKOFF_FACTOR
	}

	maxSleepMS := t.params.MaxSleepMS
	if maxSleepMS <= 0 {
		maxSleepMS = FILES_FEED_MAX_SLEEP_MS
	}

	numPartitions := t.params.NumPartitions
	if numPartitions < 0 {
		numPartitions = 0
	}

	partitions := make([]string, numPartitions)
	for i := 0; i < len(partitions); i++ {
		partitions[i] = strconv.Itoa(i)
	}

	go func() {
		initTime := time.Now()
		initTimeMicroSecs := initTime.UnixNano() / int64(1000)

		// TODO: NOTE: We're assuming (lazily, incorrectly) that this
		// way of initializing a sequence number never goes downwards,
		// even during fast restarts or clock changes or node
		// rebalances/reassignments.
		seqs := map[string]uint64{}
		for partition, _ := range t.dests {
			seqs[partition] = uint64(initTimeMicroSecs)
		}

		var prevStartTime time.Time

		ExponentialBackoffLoop(t.Name(),
			func() int {
				select {
				case <-t.closeCh:
					return -1
				default:
				}

				h := crc32.NewIEEE()

				startTime := time.Now()

				progress := false

				paths, err := FilesFindMatches(t.mgr.DataDir(),
					t.sourceName, t.params.RegExps, prevStartTime,
					t.params.MaxFileSize)
				if err != nil {
					log.Printf("feed_files, FilesFindMatches, err: %v", err)
					return -1
				}

				seqDeltaMax := uint64(0)

				seqEnds := map[string]uint64{}

				for _, path := range paths {
					partition := FilesPathToPartition(h, partitions, path)

					if t.dests[partition] == nil {
						continue
					}

					seq := seqs[partition]

					seqEnd, exists := seqEnds[partition]
					if exists {
						seqEnd = seqEnd + 1
					} else {
						seqEnd = seq
					}
					seqEnds[partition] = seqEnd

					if seqDeltaMax < seqEnd-seq {
						seqDeltaMax = seqEnd - seq
					}
				}

				snapshotSent := map[string]bool{}

				for _, path := range paths {
					select {
					case <-t.closeCh:
						return -1
					default:
					}

					partition := FilesPathToPartition(h, partitions, path)

					dest := t.dests[partition]
					if dest == nil {
						continue
					}

					seqCur := seqs[partition]
					seqs[partition] = seqCur + 1

					buf, err := ioutil.ReadFile(path)
					if err != nil {
						log.Printf("feed_files: read file,"+
							" name: %s, path: %s, err: %v",
							t.Name(), path, err)
						continue
					}

					jbuf, err := json.Marshal(FileDoc{
						Name:     filepath.Base(path),
						Path:     path,
						Contents: string(buf),
					})
					if err != nil {
						log.Printf("feed_files: json marshal file,"+
							" name: %s, path: %s, err: %v",
							t.Name(), path, err)
						continue
					}

					if !snapshotSent[partition] {
						err = dest.SnapshotStart(partition, seqCur,
							seqEnds[partition])
						if err != nil {
							log.Printf("feed_files: SnapshotStart,"+
								" name: %s, partition: %s,"+
								" seqCur: %d, seqEnd: %d, err: %v",
								t.Name(), partition,
								seqCur, seqEnds[partition], err)
							return -1
						}

						snapshotSent[partition] = true
					}

					pathBuf := []byte(path)

					err = dest.DataUpdate(partition, pathBuf, seqCur, jbuf)
					if err != nil {
						log.Printf("feed_files: DataUpdate,"+
							" name: %s, path: %s, partition: %s,"+
							" seqCur: %d, err: %v",
							t.Name(), path, partition, seqCur, err)
						return -1
					}

					progress = true
				}

				prevStartTime = startTime

				// NOTE: We may need to sleep a certain amount in case
				// there were tons of file updates/mutations, and we
				// want to reduce the window of potentially repeating
				// sequence numbers.  The window still exists if we
				// crash and quickly restart during the sleep, where
				// the restarted process might have a
				// lower-than-wanted initTime.
				wantTime := initTime // Copy, because Add() mutates.
				wantTime.Add(time.Duration(int64(seqDeltaMax)))

				currTime := time.Now()
				if wantTime.After(currTime) {
					time.Sleep(wantTime.Sub(currTime))
				}

				if progress {
					return 1
				}
				return 0
			},
			startSleepMS,
			backoffFactor,
			maxSleepMS)
	}()

	return nil
}

func (t *FilesFeed) Close() error {
	t.m.Lock()
	if t.closeCh != nil {
		close(t.closeCh)
		t.closeCh = nil
	}
	t.m.Unlock()

	return nil
}

func (t *FilesFeed) Dests() map[string]Dest {
	return t.dests
}

func (t *FilesFeed) Stats(w io.Writer) error {
	_, err := w.Write([]byte("{}"))
	return err
}

// -----------------------------------------------------

func FilesFeedPartitions(sourceType, sourceName, sourceUUID, sourceParams,
	server string) ([]string, error) {
	ffp := &FilesFeedParams{}
	if sourceParams != "" {
		err := json.Unmarshal([]byte(sourceParams), ffp)
		if err != nil {
			return nil, fmt.Errorf("feed_files:"+
				" could not parse sourceParams: %s, err: %v",
				sourceParams, err)
		}
	}
	rv := make([]string, ffp.NumPartitions)
	for i := 0; i < ffp.NumPartitions; i++ {
		rv[i] = strconv.Itoa(i)
	}
	return rv, nil
}

// -----------------------------------------------------

func FilesFindMatches(dataDir, sourceName string,
	regExps []string, modTimeGTE time.Time, maxSize int64) (
	[]string, error) {
	walkPath, err := filepath.EvalSymlinks(dataDir +
		string(os.PathSeparator) + "files" +
		string(os.PathSeparator) + sourceName)
	if err != nil {
		return nil, err
	}

	pathsOk := []string(nil)

	err = filepath.Walk(walkPath,
		func(path string, fi os.FileInfo, err error) error {
			if err != nil ||
				fi.IsDir() ||
				fi.ModTime().Before(modTimeGTE) ||
				(maxSize > 0 && fi.Size() > maxSize) {
				return nil
			}

			if len(regExps) <= 0 {
				pathsOk = append(pathsOk, path)
				return nil
			}

			for _, reStr := range regExps {
				matched, err := regexp.MatchString(reStr, path)
				if err != nil {
					return fmt.Errorf("feed_files, MatchString,"+
						" reStr: %s, path: %s, err: %v",
						reStr, path, err)
				}
				if matched {
					pathsOk = append(pathsOk, path)
					return nil
				}
			}

			return nil
		})
	if err != nil {
		return nil, err
	}

	return pathsOk, nil
}

func FilesPathToPartition(h hash.Hash32,
	partitions []string, path string) string {
	if len(partitions) <= 0 {
		return ""
	}

	h.Reset()
	io.WriteString(h, path)
	i := h.Sum32() % uint32(len(partitions))
	return partitions[i]
}
<|endoftext|>