<commit_before>package pwh

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha512"
	"math/rand"
)

const pwTable = "*?0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

type PWHasher struct {
	publicSalt []byte
	complexity int
}

func New(publicSalt string, complexity int) *PWHasher {
	if complexity < 1 {
		complexity = 1
	}
	publicSaltHasher := sha512.New()
	publicSaltHasher.Write([]byte(publicSalt))
	return &PWHasher{publicSaltHasher.Sum(nil), complexity}
}

func (pwh *PWHasher) Hash(word, salt string) string {
	return string(pwh.hash(rand.Int()%pwh.complexity, word, salt))
}

func (pwh *PWHasher) Match(word, salt, hash string) bool {
	for i := 0; i < pwh.complexity; i++ {
		if bytes.Equal([]byte(hash), pwh.hash(i, word, salt)) {
			return true
		}
	}
	return false
}

func (pwh *PWHasher) MatchX(word, salt, hash string, routines int) bool {
	if routines < 2 {
		return pwh.Match(word, salt, hash)
	}
	groups := (pwh.complexity + routines - 1) / routines
	matchc := make(chan bool, routines)
	matched := 0
	for i := 0; i < routines; i++ {
		go func(i int) {
			for s, e := i*groups, (i+1)*groups; s < e; s++ {
				if matched == routines {
					return
				}
				if bytes.Equal([]byte(hash), pwh.hash(s, word, salt)) {
					matchc <- true
					return
				}
			}
			matchc <- false
		}(i)
	}
	for {
		if <-matchc {
			matched = routines // Use for stoping all hasher goroutines
			return true
		} else if matched++; matched == routines {
			return false
		}
	}
}

func (pwh *PWHasher) hash(r int, word, salt string) []byte {
	codeTable := make([]byte, 64)
	for i, p := 0, rand.New(rand.NewSource(int64(r))).Perm(64); i < 64; i++ {
		codeTable[i] = pwTable[p[i]]
	}
	hmac := hmac.New(sha512.New384, pwh.publicSalt)
	hmac.Write([]byte(word))
	hmac.Write([]byte(salt))
	hashBytes := hmac.Sum(nil)
	hash := make([]byte, 64)
	for i, j := 0, 0; i < 48; i += 3 {
		j = i * 4 / 3
		hash[j] = codeTable[hashBytes[i]>>2]
		hash[j+1] = codeTable[(hashBytes[i]&0x3)<<4|hashBytes[i+1]>>4]
		hash[j+2] = codeTable[(hashBytes[i+1]&0xf)<<2|hashBytes[i+2]>>6]
		hash[j+3] = codeTable[hashBytes[i+2]&0x3f]
	}
	return hash
}
<commit_msg>add default hasher<commit_after>package pwh

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha512"
	"math/rand"
	"sync"
)

const pwTable = "*?0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

type PWHasher struct {
	lock       sync.RWMutex
	publicSalt []byte
	complexity int
}

func New(publicSalt string, complexity int) (pwh *PWHasher) {
	pwh = &PWHasher{}
	pwh.Config(publicSalt, complexity)
	return
}

func (pwh *PWHasher) Config(publicSalt string, complexity int) {
	if complexity < 1 {
		complexity = 1
	}
	publicSaltHasher := sha512.New()
	publicSaltHasher.Write([]byte(publicSalt))

	pwh.lock.Lock()
	defer pwh.lock.Unlock()

	pwh.complexity = complexity
	pwh.publicSalt = publicSaltHasher.Sum(nil)
}

func (pwh *PWHasher) Hash(word, salt string) string {
	pwh.lock.RLock()
	defer pwh.lock.RUnlock()

	return string(pwh.hash(rand.Int()%pwh.complexity, word, salt))
}

func (pwh *PWHasher) Match(word, salt, hash string) bool {
	pwh.lock.RLock()
	defer pwh.lock.RUnlock()

	for i := 0; i < pwh.complexity; i++ {
		if bytes.Equal([]byte(hash), pwh.hash(i, word, salt)) {
			return true
		}
	}
	return false
}

func (pwh *PWHasher) MatchX(word, salt, hash string, routines int) bool {
	if routines < 2 {
		return pwh.Match(word, salt, hash)
	}

	pwh.lock.RLock()
	defer pwh.lock.RUnlock()

	groups := (pwh.complexity + routines - 1) / routines
	matchc := make(chan bool, routines)
	matched := 0
	for i := 0; i < routines; i++ {
		go func(i int) {
			for s, e := i*groups, (i+1)*groups; s < e; s++ {
				if matched == routines {
					return
				}
				if bytes.Equal([]byte(hash), pwh.hash(s, word, salt)) {
					matchc <- true
					return
				}
			}
			matchc <- false
		}(i)
	}
	for {
		if <-matchc {
			matched = routines // Use for stoping all hasher goroutines
			return true
		} else if matched++; matched == routines {
			return false
		}
	}
}

func (pwh *PWHasher) hash(r int, word, salt string) []byte {
	codeTable := make([]byte, 64)
	for i, p := 0, rand.New(rand.NewSource(int64(r))).Perm(64); i < 64; i++ {
		codeTable[i] = pwTable[p[i]]
	}
	hmac := hmac.New(sha512.New384, pwh.publicSalt)
	hmac.Write([]byte(word))
	hmac.Write([]byte(salt))
	hashBytes := hmac.Sum(nil)
	hash := make([]byte, 64)
	for i, j := 0, 0; i < 48; i += 3 {
		j = i * 4 / 3
		hash[j] = codeTable[hashBytes[i]>>2]
		hash[j+1] = codeTable[(hashBytes[i]&0x3)<<4|hashBytes[i+1]>>4]
		hash[j+2] = codeTable[(hashBytes[i+1]&0xf)<<2|hashBytes[i+2]>>6]
		hash[j+3] = codeTable[hashBytes[i+2]&0x3f]
	}
	return hash
}

var defaultPWHasher *PWHasher

func Config(publicSalt string, complexity int) {
	if complexity < 1 {
		complexity = 1
	}
	publicSaltHasher := sha512.New()
	publicSaltHasher.Write([]byte(publicSalt))
	defaultPWHasher.complexity = complexity
	defaultPWHasher.publicSalt = publicSaltHasher.Sum(nil)
}

func Hash(word, salt string) string {
	return defaultPWHasher.Hash(word, salt)
}

func Match(word, salt, hash string) bool {
	return defaultPWHasher.Match(word, salt, hash)
}

func MatchX(word, salt, hash string, routines int) bool {
	return defaultPWHasher.MatchX(word, salt, hash, routines)
}

func init() {
	defaultPWHasher = New("", 512)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"time"

	"github.com/mschwager/riplink/src/requests"
)

func main() {
	var queryUrl string
	flag.StringVar(&queryUrl, "url", "https://google.com", "URL to query")

	var timeout int
	flag.IntVar(&timeout, "timeout", 5, "Timeout in seconds")

	var verbose bool
	flag.BoolVar(&verbose, "verbose", false, "Verbose output")

	var depth uint
	flag.UintVar(&depth, "depth", 1, "Follow discovered links this deep")

	var sameDomain bool
	flag.BoolVar(&sameDomain, "same-domain", false, "Only query links of the same domain as the initial URL")

	var httpCode int
	var httpCodeDefault int = 0
	flag.IntVar(&httpCode, "http-code", httpCodeDefault, "Only print results that received this HTTP return code (default not HTTP 2XX)")

	flag.Parse()

	client := &http.Client{
		Timeout: time.Second * time.Duration(timeout),
	}

	results := make(chan *requests.Result)

	go requests.RecursiveQueryToChan(client, queryUrl, depth, sameDomain, results)

	printPredicate := func(code int) bool {
		return code < 200 || code > 299
	}

	if httpCode != httpCodeDefault {
		printPredicate = func(code int) bool {
			return code == httpCode
		}
	}

	for result := range results {
		if result.Err != nil {
			fmt.Println(result.Err)
			continue
		}

		if verbose || printPredicate(result.Code) {
			fmt.Println(result.Url, result.Code)
		}
	}
}
<commit_msg>Only print errors when verbose is on<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"time"

	"github.com/mschwager/riplink/src/requests"
)

func main() {
	var queryUrl string
	flag.StringVar(&queryUrl, "url", "https://google.com", "URL to query")

	var timeout int
	flag.IntVar(&timeout, "timeout", 5, "Timeout in seconds")

	var verbose bool
	flag.BoolVar(&verbose, "verbose", false, "Verbose output")

	var depth uint
	flag.UintVar(&depth, "depth", 1, "Follow discovered links this deep")

	var sameDomain bool
	flag.BoolVar(&sameDomain, "same-domain", false, "Only query links of the same domain as the initial URL")

	var httpCode int
	var httpCodeDefault int = 0
	flag.IntVar(&httpCode, "http-code", httpCodeDefault, "Only print results that received this HTTP return code (default not HTTP 2XX)")

	flag.Parse()

	client := &http.Client{
		Timeout: time.Second * time.Duration(timeout),
	}

	results := make(chan *requests.Result)

	go requests.RecursiveQueryToChan(client, queryUrl, depth, sameDomain, results)

	printPredicate := func(code int) bool {
		return code < 200 || code > 299
	}

	if httpCode != httpCodeDefault {
		printPredicate = func(code int) bool {
			return code == httpCode
		}
	}

	for result := range results {
		if result.Err != nil {
			if verbose {
				fmt.Println(result.Err)
			}
			continue
		}

		if verbose || printPredicate(result.Code) {
			fmt.Println(result.Url, result.Code)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package syslog

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"
	"text/template"
)

// tagOffset represents the substring start value for the tag to return
// the logfileName value from the syslogtag.  Substrings in syslog are
// indexed from 1, hence the + 1.
const tagOffset = len("juju-") + 1

// The rsyslog conf for state server nodes.
// Messages are gathered from other nodes and accumulated in an all-machines.log file.
//
// The apparmor profile is quite strict about where rsyslog can write files.
// Instead of poking with the profile, the local provider now logs to
// {{logDir}}-{{user}}-{{env name}}/all-machines.log, and a symlink is made
// in the local provider log dir to point to that file. The file is also
// created with 0644 so the user can read it without poking permissions. By
// default rsyslog creates files with 0644, but in the ubuntu package, the
// setting is changed to 0640, which means normal users can't read the log
// file. Using a new action directive (new as in not-legacy), we can specify
// the file create mode so it doesn't use the default.
//
// I would dearly love to write the filtering action as follows to avoid setting
// and resetting the global $FileCreateMode, but alas, precise doesn't support it
//
// if $syslogtag startswith "juju{{namespace}}-" then
//   action(type="omfile"
//          File="{{logDir}}{{namespace}}/all-machines.log"
//          Template="JujuLogFormat{{namespace}}"
//          FileCreateMode="0644")
// & stop
//
// Instead we need to mess with the global FileCreateMode.  We set it back
// to the ubuntu default after defining our rule.
const stateServerRsyslogTemplate = `
$ModLoad imfile

$InputFilePersistStateInterval 50
$InputFilePollInterval 5
$InputFileName {{logfilePath}}
$InputFileTag juju{{namespace}}-{{logfileName}}:
$InputFileStateFile {{logfileName}}{{namespace}}
$InputRunFileMonitor

$ModLoad imtcp
$DefaultNetstreamDriver gtls
$DefaultNetstreamDriverCAFile {{tlsCACertPath}}
$DefaultNetstreamDriverCertFile {{tlsCertPath}}
$DefaultNetstreamDriverKeyFile {{tlsKeyPath}}
$InputTCPServerStreamDriverAuthMode anon
$InputTCPServerStreamDriverMode 1 # run driver in TLS-only mode
$InputTCPServerRun {{portNumber}}

# Messages received from remote rsyslog machines have messages prefixed with a space,
# so add one in for local messages too if needed.
$template JujuLogFormat{{namespace}},"%syslogtag:{{tagStart}}:$%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"

$FileCreateMode 0644
:syslogtag, startswith, "juju{{namespace}}-" {{logDir}}/all-machines.log;JujuLogFormat{{namespace}}
& ~
$FileCreateMode 0640
`

// The rsyslog conf for non-state server nodes.
// Messages are forwarded to the state server node.
const nodeRsyslogTemplate = `
$ModLoad imfile

{{range $i, $bootstrapIP := bootstrapHosts}}
{{if $i}}

{{end}}
# Enable reliable forwarding.
$ActionQueueType LinkedList
$ActionQueueFileName {{logfileName}}{{namespace}}
$ActionResumeRetryCount -1
$ActionQueueSaveOnShutdown on

$InputFilePersistStateInterval 50
$InputFilePollInterval 5
$InputFileName {{logfilePath}}
$InputFileTag juju{{namespace}}-{{logfileName}}:
$InputFileStateFile {{logfileName}}{{namespace}}
$InputRunFileMonitor

$DefaultNetstreamDriver gtls
$DefaultNetstreamDriverCAFile {{tlsCACertPath}}
$ActionSendStreamDriverAuthMode anon
$ActionSendStreamDriverMode 1 # run driver in TLS-only mode

$template LongTagForwardFormat,"<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%"

:syslogtag, startswith, "juju{{namespace}}-" @@{{$bootstrapIP}}:{{portNumber}};LongTagForwardFormat
{{end}}
& ~
`

// nodeRsyslogTemplateTLSHeader is prepended to
// nodeRsyslogTemplate if TLS is to be used.
const nodeRsyslogTemplateTLSHeader = `
`

const (
	defaultConfigDir          = "/etc/rsyslog.d"
	defaultCACertFileName     = "ca-cert.pem"
	defaultServerCertFileName = "rsyslog-cert.pem"
	defaultServerKeyFileName  = "rsyslog-key.pem"
)

// SyslogConfigRenderer instances are used to generate a rsyslog conf file.
type SyslogConfigRenderer interface {
	Render() ([]byte, error)
}

// SyslogConfig provides a means to configure and generate rsyslog conf files for
// the state server nodes and unit nodes.
// rsyslog is configured to tail the specified log file.
type SyslogConfig struct {
	// the template representing the config file contents.
	configTemplate string
	// the directory where the config file is written.
	ConfigDir string
	// the config file name.
	ConfigFileName string
	// the name of the log file to tail.
	LogFileName string
	// the addresses of the state server to which messages should be forwarded.
	StateServerAddresses []string
	// CA certificate file name.
	CACertFileName string
	// Server certificate file name.
	ServerCertFileName string
	// Server private key file name.
	ServerKeyFileName string
	// the port number for the listener
	Port int
	// the directory for the logfiles
	LogDir string
	// namespace is used when there are multiple environments on one machine
	Namespace string
}

// NewForwardConfig creates a SyslogConfig instance used on unit nodes to forward log entries
// to the state server nodes.
func NewForwardConfig(logFile, logDir string, port int, namespace string, stateServerAddresses []string) *SyslogConfig {
	conf := &SyslogConfig{
		configTemplate:       nodeRsyslogTemplate,
		StateServerAddresses: stateServerAddresses,
		LogFileName:          logFile,
		Port:                 port,
		LogDir:               logDir,
	}
	if namespace != "" {
		conf.Namespace = "-" + namespace
	}
	return conf
}

// NewAccumulateConfig creates a SyslogConfig instance used to accumulate log entries from the
// various unit nodes.
func NewAccumulateConfig(logFile, logDir string, port int, namespace string) *SyslogConfig {
	conf := &SyslogConfig{
		configTemplate: stateServerRsyslogTemplate,
		LogFileName:    logFile,
		Port:           port,
		LogDir:         logDir,
	}
	if namespace != "" {
		conf.Namespace = "-" + namespace
	}
	return conf
}

func either(a, b string) string {
	if a != "" {
		return a
	}
	return b
}

func (slConfig *SyslogConfig) ConfigFilePath() string {
	dir := either(slConfig.ConfigDir, defaultConfigDir)
	return filepath.Join(dir, slConfig.ConfigFileName)
}

func (slConfig *SyslogConfig) CACertPath() string {
	filename := either(slConfig.CACertFileName, defaultCACertFileName)
	return filepath.Join(slConfig.LogDir, filename)
}

func (slConfig *SyslogConfig) ServerCertPath() string {
	filename := either(slConfig.ServerCertFileName, defaultServerCertFileName)
	return filepath.Join(slConfig.LogDir, filename)
}

func (slConfig *SyslogConfig) ServerKeyPath() string {
	filename := either(slConfig.ServerCertFileName, defaultServerKeyFileName)
	return filepath.Join(slConfig.LogDir, filename)
}

// Render generates the rsyslog config.
func (slConfig *SyslogConfig) Render() ([]byte, error) {
	var bootstrapHosts = func() []string {
		var hosts []string
		for _, addr := range slConfig.StateServerAddresses {
			parts := strings.Split(addr, ":")
			hosts = append(hosts, parts[0])
		}
		return hosts
	}

	var logFilePath = func() string {
		return fmt.Sprintf("%s/%s.log", slConfig.LogDir, slConfig.LogFileName)
	}

	t := template.New("")
	t.Funcs(template.FuncMap{
		"logfileName":    func() string { return slConfig.LogFileName },
		"bootstrapHosts": bootstrapHosts,
		"logfilePath":    logFilePath,
		"portNumber":     func() int { return slConfig.Port },
		"logDir":         func() string { return slConfig.LogDir },
		"namespace":      func() string { return slConfig.Namespace },
		"tagStart":       func() int { return tagOffset + len(slConfig.Namespace) },
		"tlsCACertPath":  slConfig.CACertPath,
		"tlsCertPath":    slConfig.ServerCertPath,
		"tlsKeyPath":     slConfig.ServerKeyPath,
	})

	// Process the rsyslog config template and echo to the conf file.
	p, err := t.Parse(slConfig.configTemplate)
	if err != nil {
		return nil, err
	}
	var confBuf bytes.Buffer
	if err := p.Execute(&confBuf, nil); err != nil {
		return nil, err
	}
	return confBuf.Bytes(), nil
}

// Write generates and writes the rsyslog config.
func (slConfig *SyslogConfig) Write() error {
	data, err := slConfig.Render()
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(slConfig.ConfigFilePath(), data, 0644)
	return err
}
<commit_msg>utils/syslog: fixed stray output<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package syslog

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"
	"text/template"
)

// tagOffset represents the substring start value for the tag to return
// the logfileName value from the syslogtag.  Substrings in syslog are
// indexed from 1, hence the + 1.
const tagOffset = len("juju-") + 1

// The rsyslog conf for state server nodes.
// Messages are gathered from other nodes and accumulated in an all-machines.log file.
//
// The apparmor profile is quite strict about where rsyslog can write files.
// Instead of poking with the profile, the local provider now logs to
// {{logDir}}-{{user}}-{{env name}}/all-machines.log, and a symlink is made
// in the local provider log dir to point to that file. The file is also
// created with 0644 so the user can read it without poking permissions. By
// default rsyslog creates files with 0644, but in the ubuntu package, the
// setting is changed to 0640, which means normal users can't read the log
// file. Using a new action directive (new as in not-legacy), we can specify
// the file create mode so it doesn't use the default.
//
// I would dearly love to write the filtering action as follows to avoid setting
// and resetting the global $FileCreateMode, but alas, precise doesn't support it
//
// if $syslogtag startswith "juju{{namespace}}-" then
//   action(type="omfile"
//          File="{{logDir}}{{namespace}}/all-machines.log"
//          Template="JujuLogFormat{{namespace}}"
//          FileCreateMode="0644")
// & stop
//
// Instead we need to mess with the global FileCreateMode.  We set it back
// to the ubuntu default after defining our rule.
const stateServerRsyslogTemplate = `
$ModLoad imfile

$InputFilePersistStateInterval 50
$InputFilePollInterval 5
$InputFileName {{logfilePath}}
$InputFileTag juju{{namespace}}-{{logfileName}}:
$InputFileStateFile {{logfileName}}{{namespace}}
$InputRunFileMonitor

$ModLoad imtcp
$DefaultNetstreamDriver gtls
$DefaultNetstreamDriverCAFile {{tlsCACertPath}}
$DefaultNetstreamDriverCertFile {{tlsCertPath}}
$DefaultNetstreamDriverKeyFile {{tlsKeyPath}}
$InputTCPServerStreamDriverAuthMode anon
$InputTCPServerStreamDriverMode 1 # run driver in TLS-only mode
$InputTCPServerRun {{portNumber}}

# Messages received from remote rsyslog machines have messages prefixed with a space,
# so add one in for local messages too if needed.
$template JujuLogFormat{{namespace}},"%syslogtag:{{tagStart}}:$%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"

$FileCreateMode 0644
:syslogtag, startswith, "juju{{namespace}}-" {{logDir}}/all-machines.log;JujuLogFormat{{namespace}}
& ~
$FileCreateMode 0640
`

// The rsyslog conf for non-state server nodes.
// Messages are forwarded to the state server node.
const nodeRsyslogTemplate = `
$ModLoad imfile
{{range $i, $bootstrapIP := bootstrapHosts}}{{if $i}}\n{{end}}
# Enable reliable forwarding.
$ActionQueueType LinkedList
$ActionQueueFileName {{logfileName}}{{namespace}}
$ActionResumeRetryCount -1
$ActionQueueSaveOnShutdown on

$InputFilePersistStateInterval 50
$InputFilePollInterval 5
$InputFileName {{logfilePath}}
$InputFileTag juju{{namespace}}-{{logfileName}}:
$InputFileStateFile {{logfileName}}{{namespace}}
$InputRunFileMonitor

$DefaultNetstreamDriver gtls
$DefaultNetstreamDriverCAFile {{tlsCACertPath}}
$ActionSendStreamDriverAuthMode anon
$ActionSendStreamDriverMode 1 # run driver in TLS-only mode

$template LongTagForwardFormat,"<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%"

:syslogtag, startswith, "juju{{namespace}}-" @@{{$bootstrapIP}}:{{portNumber}};LongTagForwardFormat{{end}}
& ~
`

// nodeRsyslogTemplateTLSHeader is prepended to
// nodeRsyslogTemplate if TLS is to be used.
const nodeRsyslogTemplateTLSHeader = `
`

const (
	defaultConfigDir          = "/etc/rsyslog.d"
	defaultCACertFileName     = "ca-cert.pem"
	defaultServerCertFileName = "rsyslog-cert.pem"
	defaultServerKeyFileName  = "rsyslog-key.pem"
)

// SyslogConfigRenderer instances are used to generate a rsyslog conf file.
type SyslogConfigRenderer interface {
	Render() ([]byte, error)
}

// SyslogConfig provides a means to configure and generate rsyslog conf files for
// the state server nodes and unit nodes.
// rsyslog is configured to tail the specified log file.
type SyslogConfig struct {
	// the template representing the config file contents.
	configTemplate string
	// the directory where the config file is written.
	ConfigDir string
	// the config file name.
	ConfigFileName string
	// the name of the log file to tail.
	LogFileName string
	// the addresses of the state server to which messages should be forwarded.
	StateServerAddresses []string
	// CA certificate file name.
	CACertFileName string
	// Server certificate file name.
	ServerCertFileName string
	// Server private key file name.
	ServerKeyFileName string
	// the port number for the listener
	Port int
	// the directory for the logfiles
	LogDir string
	// namespace is used when there are multiple environments on one machine
	Namespace string
}

// NewForwardConfig creates a SyslogConfig instance used on unit nodes to forward log entries
// to the state server nodes.
func NewForwardConfig(logFile, logDir string, port int, namespace string, stateServerAddresses []string) *SyslogConfig {
	conf := &SyslogConfig{
		configTemplate:       nodeRsyslogTemplate,
		StateServerAddresses: stateServerAddresses,
		LogFileName:          logFile,
		Port:                 port,
		LogDir:               logDir,
	}
	if namespace != "" {
		conf.Namespace = "-" + namespace
	}
	return conf
}

// NewAccumulateConfig creates a SyslogConfig instance used to accumulate log entries from the
// various unit nodes.
func NewAccumulateConfig(logFile, logDir string, port int, namespace string) *SyslogConfig {
	conf := &SyslogConfig{
		configTemplate: stateServerRsyslogTemplate,
		LogFileName:    logFile,
		Port:           port,
		LogDir:         logDir,
	}
	if namespace != "" {
		conf.Namespace = "-" + namespace
	}
	return conf
}

func either(a, b string) string {
	if a != "" {
		return a
	}
	return b
}

func (slConfig *SyslogConfig) ConfigFilePath() string {
	dir := either(slConfig.ConfigDir, defaultConfigDir)
	return filepath.Join(dir, slConfig.ConfigFileName)
}

func (slConfig *SyslogConfig) CACertPath() string {
	filename := either(slConfig.CACertFileName, defaultCACertFileName)
	return filepath.Join(slConfig.LogDir, filename)
}

func (slConfig *SyslogConfig) ServerCertPath() string {
	filename := either(slConfig.ServerCertFileName, defaultServerCertFileName)
	return filepath.Join(slConfig.LogDir, filename)
}

func (slConfig *SyslogConfig) ServerKeyPath() string {
	filename := either(slConfig.ServerCertFileName, defaultServerKeyFileName)
	return filepath.Join(slConfig.LogDir, filename)
}

// Render generates the rsyslog config.
func (slConfig *SyslogConfig) Render() ([]byte, error) {
	var bootstrapHosts = func() []string {
		var hosts []string
		for _, addr := range slConfig.StateServerAddresses {
			parts := strings.Split(addr, ":")
			hosts = append(hosts, parts[0])
		}
		return hosts
	}

	var logFilePath = func() string {
		return fmt.Sprintf("%s/%s.log", slConfig.LogDir, slConfig.LogFileName)
	}

	t := template.New("")
	t.Funcs(template.FuncMap{
		"logfileName":    func() string { return slConfig.LogFileName },
		"bootstrapHosts": bootstrapHosts,
		"logfilePath":    logFilePath,
		"portNumber":     func() int { return slConfig.Port },
		"logDir":         func() string { return slConfig.LogDir },
		"namespace":      func() string { return slConfig.Namespace },
		"tagStart":       func() int { return tagOffset + len(slConfig.Namespace) },
		"tlsCACertPath":  slConfig.CACertPath,
		"tlsCertPath":    slConfig.ServerCertPath,
		"tlsKeyPath":     slConfig.ServerKeyPath,
	})

	// Process the rsyslog config template and echo to the conf file.
	p, err := t.Parse(slConfig.configTemplate)
	if err != nil {
		return nil, err
	}
	var confBuf bytes.Buffer
	if err := p.Execute(&confBuf, nil); err != nil {
		return nil, err
	}
	return confBuf.Bytes(), nil
}

// Write generates and writes the rsyslog config.
func (slConfig *SyslogConfig) Write() error {
	data, err := slConfig.Render()
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(slConfig.ConfigFilePath(), data, 0644)
	return err
}
<|endoftext|>


<commit_before>package backends

import (
	"encoding/json"
	"reflect"
	"strings"
	"time"

	"github.com/RichardKnop/machinery/v1/config"
	"github.com/RichardKnop/machinery/v1/log"
	"github.com/RichardKnop/machinery/v1/tasks"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// MongodbBackend represents a MongoDB result backend
type MongodbBackend struct {
	Backend
	session              *mgo.Session
	tasksCollection      *mgo.Collection
	groupMetasCollection *mgo.Collection
}

// NewMongodbBackend creates MongodbBackend instance
func NewMongodbBackend(cnf *config.Config) Interface {
	return &MongodbBackend{Backend: New(cnf)}
}

// InitGroup creates and saves a group meta data object
func (b *MongodbBackend) InitGroup(groupUUID string, taskUUIDs []string) error {
	if err := b.connect(); err != nil {
		return err
	}

	groupMeta := &tasks.GroupMeta{
		GroupUUID: groupUUID,
		TaskUUIDs: taskUUIDs,
		CreatedAt: time.Now().UTC(),
	}
	return b.groupMetasCollection.Insert(groupMeta)
}

// GroupCompleted returns true if all tasks in a group finished
func (b *MongodbBackend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) {
	groupMeta, err := b.getGroupMeta(groupUUID)
	if err != nil {
		return false, err
	}

	taskStates, err := b.getStates(groupMeta.TaskUUIDs...)
	if err != nil {
		return false, err
	}

	var countSuccessTasks = 0
	for _, taskState := range taskStates {
		if taskState.IsCompleted() {
			countSuccessTasks++
		}
	}

	return countSuccessTasks == groupTaskCount, nil
}

// GroupTaskStates returns states of all tasks in the group
func (b *MongodbBackend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) {
	groupMeta, err := b.getGroupMeta(groupUUID)
	if err != nil {
		return []*tasks.TaskState{}, err
	}

	return b.getStates(groupMeta.TaskUUIDs...)
}

// TriggerChord flags chord as triggered in the backend storage to make sure
// chord is never triggered multiple times. Returns a boolean flag to indicate
// whether the worker should trigger chord (true) or no if it has been triggered
// already (false)
func (b *MongodbBackend) TriggerChord(groupUUID string) (bool, error) {
	if err := b.connect(); err != nil {
		return false, err
	}
	query := bson.M{
		"_id":             groupUUID,
		"chord_triggered": false,
	}
	change := mgo.Change{
		Update: bson.M{
			"$set": bson.M{
				"chord_triggered": true,
			},
		},
		ReturnNew: false,
	}
	_, err := b.groupMetasCollection.
		Find(query).
		Apply(change, nil)
	if err != nil {
		if err == mgo.ErrNotFound {
			log.WARNING.Printf("Chord already triggered for group %s", groupUUID)
			return false, nil
		}
		return false, err
	}
	return true, nil
}

// SetStatePending updates task state to PENDING
func (b *MongodbBackend) SetStatePending(signature *tasks.Signature) error {
	update := bson.M{
		"state":      tasks.StatePending,
		"task_name":  signature.Name,
		"created_at": time.Now().UTC(),
	}
	return b.updateState(signature, update)
}

// SetStateReceived updates task state to RECEIVED
func (b *MongodbBackend) SetStateReceived(signature *tasks.Signature) error {
	update := bson.M{"state": tasks.StateReceived}
	return b.updateState(signature, update)
}

// SetStateStarted updates task state to STARTED
func (b *MongodbBackend) SetStateStarted(signature *tasks.Signature) error {
	update := bson.M{"state": tasks.StateStarted}
	return b.updateState(signature, update)
}

// SetStateRetry updates task state to RETRY
func (b *MongodbBackend) SetStateRetry(signature *tasks.Signature) error {
	update := bson.M{"state": tasks.StateRetry}
	return b.updateState(signature, update)
}

// SetStateSuccess updates task state to SUCCESS
func (b *MongodbBackend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error {
	decodedResults := b.decodeResults(results)
	update := bson.M{
		"state":   tasks.StateSuccess,
		"results": decodedResults,
	}
	return b.updateState(signature, update)
}

// decodeResults detects & decodes json strings in TaskResult.Value and returns a new slice
func (b *MongodbBackend) decodeResults(results []*tasks.TaskResult) []*tasks.TaskResult {
	l := len(results)
	jsonResults := make([]*tasks.TaskResult, l, l)
	for i, result := range results {
		jsonResult := new(bson.M)
		resultType := reflect.TypeOf(result.Value).Kind()
		if resultType == reflect.String {
			err := json.NewDecoder(strings.NewReader(result.Value.(string))).Decode(&jsonResult)
			if err == nil {
				jsonResults[i] = &tasks.TaskResult{
					Type:  "json",
					Value: jsonResult,
				}
				continue
			}
		}
		jsonResults[i] = result
	}
	return jsonResults
}

// SetStateFailure updates task state to FAILURE
func (b *MongodbBackend) SetStateFailure(signature *tasks.Signature, err string) error {
	update := bson.M{"state": tasks.StateFailure, "error": err}
	return b.updateState(signature, update)
}

// GetState returns the latest task state
func (b *MongodbBackend) GetState(taskUUID string) (*tasks.TaskState, error) {
	if err := b.connect(); err != nil {
		return nil, err
	}

	state := new(tasks.TaskState)
	if err := b.tasksCollection.FindId(taskUUID).One(state); err != nil {
		return nil, err
	}
	return state, nil
}

// PurgeState deletes stored task state
func (b *MongodbBackend) PurgeState(taskUUID string) error {
	if err := b.connect(); err != nil {
		return err
	}

	return b.tasksCollection.RemoveId(taskUUID)
}

// PurgeGroupMeta deletes stored group meta data
func (b *MongodbBackend) PurgeGroupMeta(groupUUID string) error {
	if err := b.connect(); err != nil {
		return err
	}

	return b.groupMetasCollection.RemoveId(groupUUID)
}

// lockGroupMeta acquires lock on groupUUID document
func (b *MongodbBackend) lockGroupMeta(groupUUID string) error {
	query := bson.M{
		"_id":  groupUUID,
		"lock": false,
	}
	change := mgo.Change{
		Update: bson.M{
			"$set": bson.M{
				"lock": true,
			},
		},
		Upsert:    true,
		ReturnNew: false,
	}
	_, err := b.groupMetasCollection.
		Find(query).
		Apply(change, nil)
	return err
}

// unlockGroupMeta releases lock on groupUUID document
func (b *MongodbBackend) unlockGroupMeta(groupUUID string) error {
	update := bson.M{"$set": bson.M{"lock": false}}
	_, err := b.groupMetasCollection.UpsertId(groupUUID, update)
	return err
}

// getGroupMeta retrieves group meta data, convenience function to avoid repetition
func (b *MongodbBackend) getGroupMeta(groupUUID string) (*tasks.GroupMeta, error) {
	if err := b.connect(); err != nil {
		return nil, err
	}

	query := bson.M{"_id": groupUUID}

	groupMeta := new(tasks.GroupMeta)
	if err := b.groupMetasCollection.Find(query).One(groupMeta); err != nil {
		return nil, err
	}
	return groupMeta, nil
}

// getStates returns multiple task states
func (b *MongodbBackend) getStates(taskUUIDs ...string) ([]*tasks.TaskState, error) {
	if err := b.connect(); err != nil {
		return nil, err
	}

	states := make([]*tasks.TaskState, 0, len(taskUUIDs))

	iter := b.tasksCollection.Find(bson.M{"_id": bson.M{"$in": taskUUIDs}}).Iter()

	state := new(tasks.TaskState)
	for iter.Next(state) {
		states = append(states, state)

		// otherwise we would end up with the last task being every element of the slice
		state = new(tasks.TaskState)
	}

	return states, nil
}

// updateState saves current task state
func (b *MongodbBackend) updateState(signature *tasks.Signature, update bson.M) error {
	if err := b.connect(); err != nil {
		return err
	}

	update = bson.M{"$set": update}
	_, err := b.tasksCollection.UpsertId(signature.UUID, update)
	if err != nil {
		return err
	}
	return nil
}

// connect returns a session if we are already connected to mongo, otherwise
// (when called for the first time) it will open a new session and ensure
// all required indexes for our collections exist
func (b *MongodbBackend) connect() error {
	if b.session != nil {
		return nil
	}

	session, err := mgo.Dial(b.cnf.ResultBackend)
	if err != nil {
		return err
	}
	b.session = session

	b.tasksCollection = b.session.DB("").C("tasks")
	b.groupMetasCollection = b.session.DB("").C("group_metas")

	return b.createMongoIndexes()
}

// createMongoIndexes ensures all indexes are in place
func (b *MongodbBackend) createMongoIndexes() error {
	indexes := []mgo.Index{
		{
			Key:         []string{"state"},
			Background:  true, // can be used while index is being built
			ExpireAfter: time.Duration(b.cnf.ResultsExpireIn) * time.Second,
		},
		{
			Key:         []string{"lock"},
			Background:  true, // can be used while index is being built
			ExpireAfter: time.Duration(b.cnf.ResultsExpireIn) * time.Second,
		},
	}

	for _, index := range indexes {
		// Check if index already exists, if it does, skip
		if err := b.tasksCollection.EnsureIndex(index); err == nil {
			log.INFO.Printf("%s index already exist, skipping create step", index.Key[0])
			continue
		}

		// Create index (keep in mind EnsureIndex is blocking operation)
		log.INFO.Printf("Creating %s index", index.Key[0])
		if err := b.tasksCollection.DropIndex(index.Key[0]); err != nil {
			return err
		}
		if err := b.tasksCollection.EnsureIndex(index); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>mongobackend TLS support + sessions are copied<commit_after>package backends

import (
	"crypto/tls"
	"encoding/json"
	"net"
	"reflect"
	"strings"
	"time"

	"github.com/RichardKnop/machinery/v1/config"
	"github.com/RichardKnop/machinery/v1/log"
	"github.com/RichardKnop/machinery/v1/tasks"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// MongodbBackend represents a MongoDB result backend
type MongodbBackend struct {
	Backend
	session *mgo.Session
}

// NewMongodbBackend creates MongodbBackend instance
func NewMongodbBackend(cnf *config.Config) Interface {
	return &MongodbBackend{Backend: New(cnf)}
}

// Op represents a mongo operation using a copied session
type Op struct {
	session              *mgo.Session
	tasksCollection      *mgo.Collection
	groupMetasCollection *mgo.Collection
}

// Do wraps a func using op & defers session close
func (op *Op) Do(f func() error) error {
	defer op.session.Close()
	return f()
}

// newOp returns an Op pointer w/ copied session
// and task & groupMetas collections
func (b *MongodbBackend) newOp() *Op {
	session := b.session.Copy()
	return &Op{
		session:              session,
		tasksCollection:      session.DB("").C("tasks"),
		groupMetasCollection: session.DB("").C("group_metas"),
	}
}

// InitGroup creates and saves a group meta data object
func (b *MongodbBackend) InitGroup(groupUUID string, taskUUIDs []string) error {
	op, err := b.connect()
	if err != nil {
		return err
	}
	return op.Do(func() error {
		groupMeta := &tasks.GroupMeta{
			GroupUUID: groupUUID,
			TaskUUIDs: taskUUIDs,
			CreatedAt: time.Now().UTC(),
		}
		return op.groupMetasCollection.Insert(groupMeta)
	})
}

// GroupCompleted returns true if all tasks in a group finished
func (b *MongodbBackend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) {
	groupMeta, err := b.getGroupMeta(groupUUID)
	if err != nil {
		return false, err
	}

	taskStates, err := b.getStates(groupMeta.TaskUUIDs...)
	if err != nil {
		return false, err
	}

	var countSuccessTasks = 0
	for _, taskState := range taskStates {
		if taskState.IsCompleted() {
			countSuccessTasks++
		}
	}

	return countSuccessTasks == groupTaskCount, nil
}

// GroupTaskStates returns states of all tasks in the group
func (b *MongodbBackend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) {
	groupMeta, err := b.getGroupMeta(groupUUID)
	if err != nil {
		return []*tasks.TaskState{}, err
	}

	return b.getStates(groupMeta.TaskUUIDs...)
}

// TriggerChord flags chord as triggered in the backend storage to make sure
// chord is never triggered multiple times. Returns a boolean flag to indicate
// whether the worker should trigger chord (true) or no if it has been triggered
// already (false)
func (b *MongodbBackend) TriggerChord(groupUUID string) (bool, error) {
	op, err := b.connect()
	if err != nil {
		return false, err
	}
	err = op.Do(func() error {
		query := bson.M{
			"_id":             groupUUID,
			"chord_triggered": false,
		}
		change := mgo.Change{
			Update: bson.M{
				"$set": bson.M{
					"chord_triggered": true,
				},
			},
			ReturnNew: false,
		}
		_, err := op.groupMetasCollection.
			Find(query).
			Apply(change, nil)
		return err
	})
	if err != nil {
		if err == mgo.ErrNotFound {
			log.WARNING.Printf("Chord already triggered for group %s", groupUUID)
			return false, nil
		}
		return false, err
	}
	return true, nil
}

// SetStatePending updates task state to PENDING
func (b *MongodbBackend) SetStatePending(signature *tasks.Signature) error {
	update := bson.M{
		"state":      tasks.StatePending,
		"task_name":  signature.Name,
		"created_at": time.Now().UTC(),
	}
	return b.updateState(signature, update)
}

// SetStateReceived updates task state to RECEIVED
func (b *MongodbBackend) SetStateReceived(signature *tasks.Signature) error {
	update := bson.M{"state": tasks.StateReceived}
	return b.updateState(signature, update)
}

// SetStateStarted updates task state to STARTED
func (b *MongodbBackend) SetStateStarted(signature *tasks.Signature) error {
	update := bson.M{"state": tasks.StateStarted}
	return b.updateState(signature, update)
}

// SetStateRetry updates task state to RETRY
func (b *MongodbBackend) SetStateRetry(signature *tasks.Signature) error {
	update := bson.M{"state": tasks.StateRetry}
	return b.updateState(signature, update)
}

// SetStateSuccess updates task state to SUCCESS
func (b *MongodbBackend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error {
	decodedResults := b.decodeResults(results)
	update := bson.M{
		"state":   tasks.StateSuccess,
		"results": decodedResults,
	}
	return b.updateState(signature, update)
}

// decodeResults detects & decodes json strings in TaskResult.Value and returns a new slice
func (b *MongodbBackend) decodeResults(results []*tasks.TaskResult) []*tasks.TaskResult {
	l := len(results)
	jsonResults := make([]*tasks.TaskResult, l, l)
	for i, result := range results {
		jsonResult := new(bson.M)
		resultType := reflect.TypeOf(result.Value).Kind()
		if resultType == reflect.String {
			err := json.NewDecoder(strings.NewReader(result.Value.(string))).Decode(&jsonResult)
			if err == nil {
				jsonResults[i] = &tasks.TaskResult{
					Type:  "json",
					Value: jsonResult,
				}
				continue
			}
		}
		jsonResults[i] = result
	}
	return jsonResults
}

// SetStateFailure updates task state to FAILURE
func (b *MongodbBackend) SetStateFailure(signature *tasks.Signature, err string) error {
	update := bson.M{"state": tasks.StateFailure, "error": err}
	return b.updateState(signature, update)
}

// GetState returns the latest task state
func (b *MongodbBackend) GetState(taskUUID string) (*tasks.TaskState, error) {
	op, err := b.connect()
	if err != nil {
		return nil, err
	}
	state := new(tasks.TaskState)
	err = op.Do(func() error {
		return op.tasksCollection.FindId(taskUUID).One(state)
	})
	if err != nil {
		return nil, err
	}
	return state, nil
}

// PurgeState deletes stored task state
func (b *MongodbBackend) PurgeState(taskUUID string) error {
	op, err := b.connect()
	if err != nil {
		return err
	}
	return op.Do(func() error {
		return op.tasksCollection.RemoveId(taskUUID)
	})
}

// PurgeGroupMeta deletes stored group meta data
func (b *MongodbBackend) PurgeGroupMeta(groupUUID string) error {
	op, err := b.connect()
	if err != nil {
		return err
	}
	return op.Do(func() error {
		return op.groupMetasCollection.RemoveId(groupUUID)
	})
}

// lockGroupMeta acquires lock on groupUUID document
func (b *MongodbBackend) lockGroupMeta(groupUUID string) error {
	op, err := b.connect()
	if err != nil {
		return err
	}
	return op.Do(func() error {
		query := bson.M{
			"_id":  groupUUID,
			"lock": false,
		}
		change := mgo.Change{
			Update: bson.M{
				"$set": bson.M{
					"lock": true,
				},
			},
			Upsert:    true,
			ReturnNew: false,
		}
		_, err := op.groupMetasCollection.
			Find(query).
			Apply(change, nil)
		return err
	})
}

// unlockGroupMeta releases lock on groupUUID document
func (b *MongodbBackend) unlockGroupMeta(groupUUID string) error {
	op, err := b.connect()
	if err != nil {
		return err
	}
	return op.Do(func() error {
		update := bson.M{"$set": bson.M{"lock": false}}
		_, err := op.groupMetasCollection.UpsertId(groupUUID, update)
		return err
	})
}

// getGroupMeta retrieves group meta data, convenience function to avoid repetition
func (b *MongodbBackend) getGroupMeta(groupUUID string) (*tasks.GroupMeta, error) {
	op, err := b.connect()
	if err != nil {
		return nil, err
	}
	groupMeta := new(tasks.GroupMeta)
	err = op.Do(func() error {
		query := bson.M{"_id": groupUUID}
		return op.groupMetasCollection.Find(query).One(groupMeta)
	})
	if err != nil {
		return nil, err
	}
	return groupMeta, nil
}

// getStates returns multiple task states
func (b *MongodbBackend) getStates(taskUUIDs ...string) ([]*tasks.TaskState, error) {
	op, err := b.connect()
	if err != nil {
		return nil, err
	}
	states := make([]*tasks.TaskState, 0, len(taskUUIDs))
	op.Do(func() error {
		iter := op.tasksCollection.Find(bson.M{"_id": bson.M{"$in": taskUUIDs}}).Iter()
		state := new(tasks.TaskState)
		for iter.Next(state) {
			states = append(states, state)
			// otherwise we would end up with the last task being every element of the slice
			state = new(tasks.TaskState)
		}
		return nil
	})
	return states, nil
}

// updateState saves current task state
func (b *MongodbBackend) updateState(signature *tasks.Signature, update bson.M) error {
	op, err := b.connect()
	if err != nil {
		return err
	}
	return op.Do(func() error {
		update = bson.M{"$set": update}
		_, err := op.tasksCollection.UpsertId(signature.UUID, update)
		return err
	})
}

// connect creates the underlying mgo session if it doesn't exist
// creates required indexes for our collections
// and returns a a new Op
func (b *MongodbBackend) connect() (*Op, error) {
	if b.session != nil {
		b.session.Refresh()
		return b.newOp(), nil
	}
	session, err := b.dial()
	if err != nil {
		return nil, err
	}
	b.session = session
	err = b.createMongoIndexes()
	if err != nil {
		return nil, err
	}
	return b.newOp(), nil
}

// dial connects to mongo with TLSConfig if provided
// else connects via ResultBackend uri
func (b *MongodbBackend) dial() (*mgo.Session, error) {
	if b.cnf.TLSConfig == nil {
		return mgo.Dial(b.cnf.ResultBackend)
	}
	dialInfo, err := mgo.ParseURL(b.cnf.ResultBackend)
	if err != nil {
		return nil, err
	}
	dialInfo.Timeout = 5 * time.Second
	dialInfo.DialServer = func(addr *mgo.ServerAddr) (net.Conn, error) {
		return tls.Dial("tcp", addr.String(), b.cnf.TLSConfig)
	}
	return mgo.DialWithInfo(dialInfo)
}

// createMongoIndexes ensures all indexes are in place
func (b *MongodbBackend) createMongoIndexes() error {
	op, err := b.connect()
	if err != nil {
		return err
	}
	return op.Do(func() error {
		indexes := []mgo.Index{
			{
				Key:         []string{"state"},
				Background:  true, // can be used while index is being built
				ExpireAfter: time.Duration(b.cnf.ResultsExpireIn) * time.Second,
			},
			{
				Key:         []string{"lock"},
				Background:  true, // can be used while index is being built
				ExpireAfter: time.Duration(b.cnf.ResultsExpireIn) * time.Second,
			},
		}

		for _, index := range indexes {
			// Check if index already exists, if it does, skip
			if err := op.tasksCollection.EnsureIndex(index); err == nil {
				log.INFO.Printf("%s index already exist, skipping create step", index.Key[0])
				continue
			}

			// Create index (keep in mind EnsureIndex is blocking operation)
			log.INFO.Printf("Creating %s index", index.Key[0])
			if err := op.tasksCollection.DropIndex(index.Key[0]); err != nil {
				return err
			}
			if err := op.tasksCollection.EnsureIndex(index); err != nil {
				return err
			}
		}
		return nil
	})
}
<|endoftext|>


<commit_before>// Copyright 2017 The clang-server Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpc

import (
	"net"
	"time"

	"github.com/go-clang/v3.9/clang"
	flatbuffers "github.com/google/flatbuffers/go"
	"github.com/zchee/clang-server/indexdb"
	"github.com/zchee/clang-server/internal/log"
	"github.com/zchee/clang-server/internal/pathutil"
	"github.com/zchee/clang-server/symbol"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

const port = ":50051"

type server struct {
	db       *indexdb.IndexDB
	filename string
	idx      clang.Index
	tu       clang.TranslationUnit
}

// GRPCServer represents a clang-server gRPC server.
type GRPCServer struct {
	server
}

// NewGRPCServer return the new Server with initialize idx.
func NewGRPCServer() *GRPCServer {
	return &GRPCServer{
		server{
			idx: clang.NewIndex(0, 0),
		},
	}
}

// Serve serve clang-server server with flatbuffers gRPC custom codec.
func (s *server) Serve() {
	println("Serve")
	l, err := net.Listen("tcp", port)
	if err != nil {
		log.Fatal(err)
	}

	grpcServer := grpc.NewServer(grpc.CustomCodec(flatbuffers.FlatbuffersCodec{}))
	symbol.RegisterClangServer(grpcServer, s)
	if err := grpcServer.Serve(l); err != nil {
		log.Fatal(err)
	}
}

// Completion implements symbol.ClangServer Completion interface.
func (s *server) Completion(ctx context.Context, loc *symbol.SymbolLocation) (*flatbuffers.Builder, error) {
	defer profile(time.Now(), "Completion")
	f := string(loc.FileName())

	if s.filename != f {
		s.filename = f
		dir, _ := pathutil.FindProjectRoot(f)
		db, err := indexdb.NewIndexDB(dir)
		if err != nil {
			return nil, err
		}
		s.db = db
		defer db.Close()

		buf, err := db.Get(f)
		if err != nil {
			return nil, err
		}

		file := symbol.GetRootAsFile(buf, 0)

		if cErr := s.idx.ParseTranslationUnit2(file.Name(), file.Flags(), nil, clang.DefaultEditingTranslationUnitOptions()|clang.DefaultCodeCompleteOptions()|uint32(clang.TranslationUnit_KeepGoing), &s.tu); clang.ErrorCode(cErr) != clang.Error_Success {
			log.Fatal(cErr)
		}
	}

	codeCompleteResults := new(symbol.CodeCompleteResults)
	result := codeCompleteResults.Marshal(s.tu.CodeCompleteAt(f, loc.Line(), loc.Col(), nil, clang.DefaultCodeCompleteOptions()))

	return result, nil
}

func profile(start time.Time, name string) {
	elapsed := time.Since(start).Seconds()
	log.Debugf("%s: %fsec\n", name, elapsed)
}
<commit_msg>rpc: use hashutil.NewHashString for db.Get<commit_after>// Copyright 2017 The clang-server Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpc

import (
	"net"
	"time"

	"github.com/go-clang/v3.9/clang"
	flatbuffers "github.com/google/flatbuffers/go"
	"github.com/zchee/clang-server/indexdb"
	"github.com/zchee/clang-server/internal/hashutil"
	"github.com/zchee/clang-server/internal/log"
	"github.com/zchee/clang-server/internal/pathutil"
	"github.com/zchee/clang-server/symbol"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

const port = ":50051"

type server struct {
	db       *indexdb.IndexDB
	filename string
	idx      clang.Index
	tu       clang.TranslationUnit
}

// GRPCServer represents a clang-server gRPC server.
type GRPCServer struct {
	server
}

// NewGRPCServer return the new Server with initialize idx.
func NewGRPCServer() *GRPCServer {
	return &GRPCServer{
		server{
			idx: clang.NewIndex(0, 0),
		},
	}
}

// Serve serve clang-server server with flatbuffers gRPC custom codec.
func (s *server) Serve() {
	println("Serve")
	l, err := net.Listen("tcp", port)
	if err != nil {
		log.Fatal(err)
	}

	grpcServer := grpc.NewServer(grpc.CustomCodec(flatbuffers.FlatbuffersCodec{}))
	symbol.RegisterClangServer(grpcServer, s)
	if err := grpcServer.Serve(l); err != nil {
		log.Fatal(err)
	}
}

// Completion implements symbol.ClangServer Completion interface.
func (s *server) Completion(ctx context.Context, loc *symbol.SymbolLocation) (*flatbuffers.Builder, error) {
	defer profile(time.Now(), "Completion")
	f := string(loc.FileName())

	if s.filename != f {
		s.filename = f
		dir, _ := pathutil.FindProjectRoot(f)
		db, err := indexdb.NewIndexDB(dir)
		if err != nil {
			return nil, err
		}
		s.db = db
		defer db.Close()

		fhash := hashutil.NewHashString(f)
		buf, err := db.Get(fhash[:])
		if err != nil {
			return nil, err
		}

		file := symbol.GetRootAsFile(buf, 0)

		if cErr := s.idx.ParseTranslationUnit2(file.Name(), file.Flags(), nil, clang.DefaultEditingTranslationUnitOptions()|clang.DefaultCodeCompleteOptions()|uint32(clang.TranslationUnit_KeepGoing), &s.tu); clang.ErrorCode(cErr) != clang.Error_Success {
			log.Fatal(cErr)
		}
	}

	codeCompleteResults := new(symbol.CodeCompleteResults)
	result := codeCompleteResults.Marshal(s.tu.CodeCompleteAt(f, loc.Line(), loc.Col(), nil, clang.DefaultCodeCompleteOptions()))

	return result, nil
}

func profile(start time.Time, name string) {
	elapsed := time.Since(start).Seconds()
	log.Debugf("%s: %fsec\n", name, elapsed)
}
<|endoftext|>


<commit_before>package csv

import (
	"bytes"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/matryer/is"
)

type csvRow struct {
	Name string
}

func ExampleTable_Iter() {
	table, _ := NewTable(FromString("\"name\"\nfoo\nbar"), LoadHeaders())
	iter, _ := table.Iter()
	defer iter.Close()
	for iter.Next() {
		fmt.Println(iter.Row())
	}
	// Output:[foo]
	// [bar]
}

func ExampleTable_ReadAll() {
	table, _ := NewTable(FromString("\"name\"\nfoo\nbar"), LoadHeaders())
	rows, _ := table.ReadAll()
	fmt.Print(rows)
	// Output:[[foo] [bar]]
}

func ExampleNewWriter() {
	var buf bytes.Buffer
	w := NewWriter(&buf)
	w.Write([]string{"foo", "bar"})
	w.Flush()
	fmt.Println(buf.String())
	// Output:foo,bar
}

func TestRemote(t *testing.T) {
	is := is.New(t)
	h := func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "\"name\"\nfoo\nbar")
	}
	ts := httptest.NewServer(http.HandlerFunc(h))
	defer ts.Close()
	table, _ := NewTable(Remote(ts.URL), LoadHeaders())
	got, _ := table.ReadAll()
	want := [][]string{{"foo"}, {"bar"}}
	is.Equal(want, got)

	t.Run("Error", func(t *testing.T) {
		is := is.New(t)
		_, err := NewTable(Remote("invalidURL"), LoadHeaders())
		is.True(err != nil)
	})
}

func TestLoadHeaders(t *testing.T) {
	t.Run("EmptyString", func(t *testing.T) {
		is := is.New(t)
		table, err := NewTable(FromString(""), LoadHeaders())
		is.NoErr(err)
		is.Equal(len(table.Headers()), 0)
	})
	t.Run("SimpleCase", func(t *testing.T) {
		is := is.New(t)
		in := `"name"
"bar"`
		table, err := NewTable(FromString(in), LoadHeaders())
		is.NoErr(err)

		want := []string{"name"}
		is.Equal(want, table.Headers())

		iter, _ := table.Iter()
		iter.Next()
		want = []string{"bar"}
		is.Equal(want, iter.Row())
		is.True(!iter.Next())
	})
}

func TestNewTable(t *testing.T) {
	t.Run("ErrorOpts", func(t *testing.T) {
		is := is.New(t)
		table, err := NewTable(FromString(""), errorOpts())
		is.True(table == nil)
		is.True(err != nil)
	})
	t.Run("ErrorSource", func(t *testing.T) {
		is := is.New(t)
		_, err := NewTable(errorSource(), LoadHeaders())
		is.True(err != nil)
	})
}

func TestSetHeaders(t *testing.T) {
	is := is.New(t)
	in := "Foo"
	table, err := NewTable(FromString(in), SetHeaders("name"))
	is.NoErr(err)
	want := []string{"name"}
	is.Equal(want, table.Headers())

	iter, _ := table.Iter()
	iter.Next()
	want = []string{"Foo"}
	is.Equal(want, iter.Row())
	is.True(!iter.Next())
}

func TestDelimiter(t *testing.T) {
	is := is.New(t)
	in := "Foo;Bar"
	table, err := NewTable(FromString(in), Delimiter(';'))
	is.NoErr(err)
	contents, err := table.ReadAll()
	is.NoErr(err)
	is.Equal(contents, [][]string{{"Foo", "Bar"}})
}

func TestConsiderInitialSpace(t *testing.T) {
	is := is.New(t)
	in := " Foo"
	table, err := NewTable(FromString(in), ConsiderInitialSpace())
	is.NoErr(err)
	contents, err := table.ReadAll()
	is.NoErr(err)
	is.Equal(contents, [][]string{{" Foo"}})
}
<commit_msg>Add TestString.<commit_after>package csv

import (
	"bytes"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/matryer/is"
)

type csvRow struct {
	Name string
}

func ExampleTable_Iter() {
	table, _ := NewTable(FromString("\"name\"\nfoo\nbar"), LoadHeaders())
	iter, _ := table.Iter()
	defer iter.Close()
	for iter.Next() {
		fmt.Println(iter.Row())
	}
	// Output:[foo]
	// [bar]
}

func ExampleTable_ReadAll() {
	table, _ := NewTable(FromString("\"name\"\nfoo\nbar"), LoadHeaders())
	rows, _ := table.ReadAll()
	fmt.Print(rows)
	// Output:[[foo] [bar]]
}

func ExampleNewWriter() {
	var buf bytes.Buffer
	w := NewWriter(&buf)
	w.Write([]string{"foo", "bar"})
	w.Flush()
	fmt.Println(buf.String())
	// Output:foo,bar
}

func TestRemote(t *testing.T) {
	is := is.New(t)
	h := func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "\"name\"\nfoo\nbar")
	}
	ts := httptest.NewServer(http.HandlerFunc(h))
	defer ts.Close()
	table, _ := NewTable(Remote(ts.URL), LoadHeaders())
	got, _ := table.ReadAll()
	want := [][]string{{"foo"}, {"bar"}}
	is.Equal(want, got)

	t.Run("Error", func(t *testing.T) {
		is := is.New(t)
		_, err := NewTable(Remote("invalidURL"), LoadHeaders())
		is.True(err != nil)
	})
}

func TestLoadHeaders(t *testing.T) {
	t.Run("EmptyString", func(t *testing.T) {
		is := is.New(t)
		table, err := NewTable(FromString(""), LoadHeaders())
		is.NoErr(err)
		is.Equal(len(table.Headers()), 0)
	})
	t.Run("SimpleCase", func(t *testing.T) {
		is := is.New(t)
		in := `"name"
"bar"`
		table, err := NewTable(FromString(in), LoadHeaders())
		is.NoErr(err)

		want := []string{"name"}
		is.Equal(want, table.Headers())

		iter, _ := table.Iter()
		iter.Next()
		want = []string{"bar"}
		is.Equal(want, iter.Row())
		is.True(!iter.Next())
	})
}

func TestNewTable(t *testing.T) {
	t.Run("ErrorOpts", func(t *testing.T) {
		is := is.New(t)
		table, err := NewTable(FromString(""), errorOpts())
		is.True(table == nil)
		is.True(err != nil)
	})
	t.Run("ErrorSource", func(t *testing.T) {
		is := is.New(t)
		_, err := NewTable(errorSource(), LoadHeaders())
		is.True(err != nil)
	})
}

func TestSetHeaders(t *testing.T) {
	is := is.New(t)
	in := "Foo"
	table, err := NewTable(FromString(in), SetHeaders("name"))
	is.NoErr(err)
	want := []string{"name"}
	is.Equal(want, table.Headers())

	iter, _ := table.Iter()
	iter.Next()
	want = []string{"Foo"}
	is.Equal(want, iter.Row())
	is.True(!iter.Next())
}

func TestDelimiter(t *testing.T) {
	is := is.New(t)
	in := "Foo;Bar"
	table, err := NewTable(FromString(in), Delimiter(';'))
	is.NoErr(err)
	contents, err := table.ReadAll()
	is.NoErr(err)
	is.Equal(contents, [][]string{{"Foo", "Bar"}})
}

func TestConsiderInitialSpace(t *testing.T) {
	is := is.New(t)
	in := " Foo"
	table, err := NewTable(FromString(in), ConsiderInitialSpace())
	is.NoErr(err)
	contents, err := table.ReadAll()
	is.NoErr(err)
	is.Equal(contents, [][]string{{" Foo"}})
}

func TestString(t *testing.T) {
	is := is.New(t)
	in := "name\nfoo\nbar"
	want := "name\nfoo\nbar\n"
	table, err := NewTable(FromString(in))
	is.NoErr(err)
	is.Equal(want, table.String())
}
<|endoftext|>


<commit_before>/*
 *    Copyright (C) 2014 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

package modules

import (
	"bytes"
	"log"
	"strings"
	"text/template"

	"github.com/muesli/beehive/filters"
)

// An element in a Chain
type ChainElement struct {
	Action Action
	Filter Filter
}

// A user defined Chain
type Chain struct {
	Name        string
	Description string
	Event       *Event
	Elements    []ChainElement
}

// Execute a filter. Returns whether the filter passed or not.
func execFilter(filter Filter, opts map[string]interface{}) bool {
	f := *filters.GetFilter(filter.Name)
	log.Println("\tExecuting filter:", f.Name(), "-", f.Description())

	for _, opt := range filter.Options {
		log.Println("\t\tOptions:", opt)
		origVal := opts[opt.Name]
		cleanVal := opt.Value
		if opt.Trimmed {
			switch v := origVal.(type) {
			case string:
				origVal = strings.TrimSpace(v)
			}
			switch v := cleanVal.(type) {
			case string:
				cleanVal = strings.TrimSpace(v)
			}
		}
		if opt.CaseInsensitive {
			switch v := origVal.(type) {
			case string:
				origVal = strings.ToLower(v)
			}
			switch v := cleanVal.(type) {
			case string:
				cleanVal = strings.ToLower(v)
			}
		}

		if f.Passes(origVal, cleanVal) == opt.Inverse {
			return false
		}
	}

	return true
}

// Execute an action and map its ins & outs.
func execAction(action Action, opts map[string]interface{}) bool {
	a := Action{
		Bee:  action.Bee,
		Name: action.Name,
	}

	for _, opt := range action.Options {
		ph := Placeholder{
			Name: opt.Name,
		}

		switch opt.Value.(type) {
		case string:
			var value bytes.Buffer

			funcMap := template.FuncMap{
				"Left": func(values...interface{}) string {
					return values[0].(string)[:values[1].(int)]
				},
				"Right": func(values...interface{}) string {
					return values[0].(string)[values[1].(int):]
				},
			}

			tmpl, err := template.New(action.Bee + "_" + action.Name + "_" + opt.Name).Funcs(funcMap).Parse(opt.Value.(string))
			if err == nil {
				err = tmpl.Execute(&value, opts)
			}
			if err != nil {
				panic(err)
			}

			ph.Type = "string"
			ph.Value = value.String()

		default:
			ph.Type = opt.Type
			ph.Value = opt.Value
		}
		a.Options = append(a.Options, ph)
	}

	log.Println("\tExecuting action:", a.Bee, "/", a.Name, "-", GetActionDescriptor(&a).Description)
	for _, v := range a.Options {
		log.Println("\t\tOptions:", v)
	}
	(*GetModule(a.Bee)).Action(a)

	return true
}

// Execute chains for an event we received.
func execChains(event *Event) {
	for _, c := range chains {
		if c.Event.Name != event.Name || c.Event.Bee != event.Bee {
			continue
		}

		log.Println("Executing chain:", c.Name, "-", c.Description)
		for _, el := range c.Elements {
			m := make(map[string]interface{})
			for _, opt := range event.Options {
				m[opt.Name] = opt.Value
			}

			if el.Filter.Name != "" {
				if execFilter(el.Filter, m) {
					log.Println("\t\tPassed filter!")
				} else {
					log.Println("\t\tDid not pass filter!")
					break
				}
			}
			if el.Action.Name != "" {
				execAction(el.Action, m)
			}
		}
	}
}
<commit_msg>* Fixed Right func, introduced Mid.<commit_after>/*
 *    Copyright (C) 2014 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

package modules

import (
	"bytes"
	"log"
	"strings"
	"text/template"

	"github.com/muesli/beehive/filters"
)

// An element in a Chain
type ChainElement struct {
	Action Action
	Filter Filter
}

// A user defined Chain
type Chain struct {
	Name        string
	Description string
	Event       *Event
	Elements    []ChainElement
}

// Execute a filter. Returns whether the filter passed or not.
func execFilter(filter Filter, opts map[string]interface{}) bool {
	f := *filters.GetFilter(filter.Name)
	log.Println("\tExecuting filter:", f.Name(), "-", f.Description())

	for _, opt := range filter.Options {
		log.Println("\t\tOptions:", opt)
		origVal := opts[opt.Name]
		cleanVal := opt.Value
		if opt.Trimmed {
			switch v := origVal.(type) {
			case string:
				origVal = strings.TrimSpace(v)
			}
			switch v := cleanVal.(type) {
			case string:
				cleanVal = strings.TrimSpace(v)
			}
		}
		if opt.CaseInsensitive {
			switch v := origVal.(type) {
			case string:
				origVal = strings.ToLower(v)
			}
			switch v := cleanVal.(type) {
			case string:
				cleanVal = strings.ToLower(v)
			}
		}

		if f.Passes(origVal, cleanVal) == opt.Inverse {
			return false
		}
	}

	return true
}

// Execute an action and map its ins & outs.
func execAction(action Action, opts map[string]interface{}) bool {
	a := Action{
		Bee:  action.Bee,
		Name: action.Name,
	}

	for _, opt := range action.Options {
		ph := Placeholder{
			Name: opt.Name,
		}

		switch opt.Value.(type) {
		case string:
			var value bytes.Buffer

			funcMap := template.FuncMap{
				"Left": func(values...interface{}) string {
					return values[0].(string)[:values[1].(int)]
				},
				"Mid": func(values...interface{}) string {
					if len(values) > 2 {
						return values[0].(string)[values[1].(int):values[2].(int)]
					} else {
						return values[0].(string)[values[1].(int):]
					}
				},
				"Right": func(values...interface{}) string {
					return values[0].(string)[len(values[0].(string)) - values[1].(int):]
				},
			}

			tmpl, err := template.New(action.Bee + "_" + action.Name + "_" + opt.Name).Funcs(funcMap).Parse(opt.Value.(string))
			if err == nil {
				err = tmpl.Execute(&value, opts)
			}
			if err != nil {
				panic(err)
			}

			ph.Type = "string"
			ph.Value = value.String()

		default:
			ph.Type = opt.Type
			ph.Value = opt.Value
		}
		a.Options = append(a.Options, ph)
	}

	log.Println("\tExecuting action:", a.Bee, "/", a.Name, "-", GetActionDescriptor(&a).Description)
	for _, v := range a.Options {
		log.Println("\t\tOptions:", v)
	}
	(*GetModule(a.Bee)).Action(a)

	return true
}

// Execute chains for an event we received.
func execChains(event *Event) {
	for _, c := range chains {
		if c.Event.Name != event.Name || c.Event.Bee != event.Bee {
			continue
		}

		log.Println("Executing chain:", c.Name, "-", c.Description)
		for _, el := range c.Elements {
			m := make(map[string]interface{})
			for _, opt := range event.Options {
				m[opt.Name] = opt.Value
			}

			if el.Filter.Name != "" {
				if execFilter(el.Filter, m) {
					log.Println("\t\tPassed filter!")
				} else {
					log.Println("\t\tDid not pass filter!")
					break
				}
			}
			if el.Action.Name != "" {
				execAction(el.Action, m)
			}
		}
	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package catalog

// Error will be moved to a shared package
type Error error

// Item will be moved to a shared package
type Item interface {
}

// Site represents a cluster or single-node server
type Site interface {
	URL() string
}

// Pool represents a logical authentication, query, and resource
// allocation boundary, as well as a grouping of buckets.
type Pool interface {
	Name() string
	Buckets() (map[string]*Bucket, Error)
}

// Bucket is a collection of key-value entries (typically
// key-document, but not always).
type Bucket interface {
	Name() string
	Count() (int64, Error) // why is this needed?
	AccessPaths() ([]*Scanner, Error)
	Fetch(id string) (*Item, Error)
}

// IndexStatistics captures statistics for a range index (view or
// btree index).
type IndexStatistics interface {
	Count() (int64, Error)
	Min() (*Item, Error)
	Max() (*Item, Error)
	DistinctCount(int64, Error)
	Bins() ([]*Bin, Error)
}

// Bin represents a range bin within IndexStatistics.
type Bin interface {
	Count() (int64, Error)
	Min() (*Item, Error)
	Max() (*Item, Error)
	DistinctCount(int64, Error)
}

type ItemChannel chan *Item

// Scanner is the base type for full and various index scanners.
type Scanner interface {
	Channel() (ItemChannel, Error)
}

// FullScanner performs full bucket scans.
type FullScanner interface {
	Scanner
}

// Direction represents ASC and DESC
// TODO: Is this needed?
type Direction int

const (
	ASC  Direction = 1
	DESC           = 2
)

// RangeScanner is the base type for view and declarative index
// scanners.
type RangeScanner interface {
	Scanner
	Key() []string
	Direction() Direction
	Statistics() (*IndexStatistics, Error)
}

// ViewScanner represents Couchbase view indexes.
type ViewScanner interface {
	RangeScanner
}

// IndexScanner represents declarative btree indexes.
type IndexScanner interface {
	RangeScanner
}

// SearchScanner represents full text search indexes.
type SearchScanner interface {
	Scanner
}
<commit_msg>Renamed Bucket.Fetch() to Bucket.Lookup()<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package catalog

// Error will be moved to a shared package.
type Error error

// Item will be moved to a shared package.
type Item interface {
}

// Site represents a cluster or single-node server.
type Site interface {
	URL() string
}

// Pool represents a logical authentication, query, and resource
// allocation boundary, as well as a grouping of buckets.
type Pool interface {
	Name() string
	Buckets() (map[string]*Bucket, Error)
}

// Bucket is a collection of key-value entries (typically
// key-document, but not always).
type Bucket interface {
	Name() string
	Count() (int64, Error) // why is this needed?
	AccessPaths() ([]*Scanner, Error)
	Lookup(id string) (*Item, Error)
}

// IndexStatistics captures statistics for a range index (view or
// btree index).
type IndexStatistics interface {
	Count() (int64, Error)
	Min() (*Item, Error)
	Max() (*Item, Error)
	DistinctCount(int64, Error)
	Bins() ([]*Bin, Error)
}

// Bin represents a range bin within IndexStatistics.
type Bin interface {
	Count() (int64, Error)
	Min() (*Item, Error)
	Max() (*Item, Error)
	DistinctCount(int64, Error)
}

type ItemChannel chan *Item

// Scanner is the base type for all scanners.
type Scanner interface {
	Channel() (ItemChannel, Error)
}

// FullScanner performs full bucket scans.
type FullScanner interface {
	Scanner
}

// Direction represents ASC and DESC
// TODO: Is this needed?
type Direction int

const (
	ASC  Direction = 1
	DESC           = 2
)

// RangeScanner is the base type for view and declarative index
// scanners.
type RangeScanner interface {
	Scanner
	Key() []string
	Direction() Direction
	Statistics() (*IndexStatistics, Error)
}

// ViewScanner represents Couchbase view indexes.
type ViewScanner interface {
	RangeScanner
}

// IndexScanner represents declarative btree indexes.
type IndexScanner interface {
	RangeScanner
}

// SearchScanner represents full text search indexes.
type SearchScanner interface {
	Scanner
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"
)

type CredentialsCacher interface {
	Store(c *AssumeRoleCredentials) error
}

// This should be compatible with the Credentials portion
// of the awscli credential cache json file.
type AssumeRoleCredentials struct {
	AccessKeyId     string
	SecretAccessKey string
	SessionToken    string
	Expiration      time.Time
}

type CredentialsCacherProvider struct {
	credentials.Expiry
	CacheFilename string
	Credentials   AssumeRoleCredentials
}

func (p *CredentialsCacherProvider) Store(c *AssumeRoleCredentials) error {
	data, err := json.Marshal(*c)
	if err != nil {
		return err
	}

	err = os.MkdirAll(filepath.Dir(p.CacheFilename), 0750)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(p.CacheFilename, data, 0600)
	if err != nil {
		return err
	}

	return nil
}

func (p *CredentialsCacherProvider) Retrieve() (credentials.Value, error) {
	val := credentials.Value{ProviderName: "CredentialsCacherProvider"}

	data, err := ioutil.ReadFile(p.CacheFilename)
	if err != nil {
		return val, err
	}

	err = json.Unmarshal(data, &p.Credentials)
	if err != nil {
		return val, err
	}

	val.AccessKeyID = p.Credentials.AccessKeyId
	val.SecretAccessKey = p.Credentials.SecretAccessKey
	val.SessionToken = p.Credentials.SessionToken

	// Flag credentials to refresh after ~90% of the actual expiration time (6 minutes for default/max
	// credential lifetime of 1h, 90 seconds for minimum credential lifetime of 15m), using the ModTime()
	// of the credential cache file as the anchor for the calculation
	cache_s, err := os.Stat(p.CacheFilename)
	if err == nil {
		window := p.Credentials.Expiration.Sub(cache_s.ModTime()) / 10
		p.Expiry.SetExpiration(p.Credentials.Expiration, window)
	}

	return val, nil
}

func (p *CredentialsCacherProvider) IsExpired() bool {
	return p.Expiry.IsExpired()
}
<commit_msg>update cred cacher provider so that it's compatible with python aws-runas<commit_after>package main

import (
	"encoding/json"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"
)

type CredentialsCacher interface {
	Store(c *CacheableCredentials) error
	ExpirationTime() time.Time
}

type CacheableCredentials struct {
	AccessKeyId     string
	SecretAccessKey string
	SessionToken    string
	Expiration      int64
}

type CachedCredentials struct {
	Credentials CacheableCredentials
}

type CredentialsCacherProvider struct {
	CacheFilename string
	credentials.Expiry
	CachedCredentials
}

func (p *CredentialsCacherProvider) Store(c *CacheableCredentials) error {
	data, err := json.Marshal(CachedCredentials{Credentials: *c})
	if err != nil {
		return err
	}

	err = os.MkdirAll(filepath.Dir(p.CacheFilename), 0750)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(p.CacheFilename, data, 0600)
	if err != nil {
		return err
	}

	return nil
}

func (p *CredentialsCacherProvider) ExpirationTime() time.Time {
	return time.Unix(p.Credentials.Expiration, 0)
}

func (p *CredentialsCacherProvider) Retrieve() (credentials.Value, error) {
	val := credentials.Value{ProviderName: "CredentialsCacherProvider"}

	data, err := ioutil.ReadFile(p.CacheFilename)
	if err != nil {
		return val, err
	}

	err = json.Unmarshal(data, &p.CachedCredentials)
	if err != nil {
		return val, err
	}

	val.AccessKeyID = p.Credentials.AccessKeyId
	val.SecretAccessKey = p.Credentials.SecretAccessKey
	val.SessionToken = p.Credentials.SessionToken
	exp_t := p.ExpirationTime()

	// Flag credentials to refresh after ~90% of the actual expiration time (6 minutes for default/max
	// credential lifetime of 1h, 90 seconds for minimum credential lifetime of 15m), using the ModTime()
	// of the credential cache file as the anchor for the calculation
	cache_s, err := os.Stat(p.CacheFilename)
	if err == nil {
		window := exp_t.Sub(cache_s.ModTime()) / 10
		p.Expiry.SetExpiration(exp_t, window)
	}

	return val, nil
}

func (p *CredentialsCacherProvider) IsExpired() bool {
	return p.Expiry.IsExpired()
}
<|endoftext|>


<commit_before>package cbreaker

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/mailgun/log"
	"github.com/mailgun/vulcan/netutils"
)

type SideEffect interface {
	Exec() error
}

type Webhook struct {
	URL     string
	Method  string
	Headers http.Header
	Form    url.Values
	Body    []byte
}

type WebhookSideEffect struct {
	w Webhook
}

func NewWebhookSideEffect(w Webhook) (*WebhookSideEffect, error) {
	if w.Method == "" {
		return nil, fmt.Errorf("Supply method")
	}
	_, err := netutils.ParseUrl(w.URL)
	if err != nil {
		return nil, err
	}

	return &WebhookSideEffect{w: w}, nil
}

func (w *WebhookSideEffect) getBody() io.Reader {
	if len(w.w.Form) != 0 {
		return strings.NewReader(w.w.Form.Encode())
	}
	if len(w.w.Body) != 0 {
		return bytes.NewBuffer(w.w.Body)
	}
	return nil
}

func (w *WebhookSideEffect) Exec() error {
	r, err := http.NewRequest(w.w.Method, w.w.URL, w.getBody())
	if err != nil {
		return err
	}
	if len(w.w.Headers) != 0 {
		netutils.CopyHeaders(r.Header, w.w.Headers)
	}
	if len(w.w.Form) != 0 {
		r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	}
	re, err := http.DefaultClient.Do(r)
	if err != nil {
		return err
	}
	if re.Body != nil {
		defer re.Body.Close()
	}
	body, err := ioutil.ReadAll(re.Body)
	if err != nil {
		return err
	}
	log.Infof("%v got response: (%s): %s", w, re.Status, string(body))
	return nil
}
<commit_msg>Remove dependency on vulcan<commit_after>package cbreaker

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/mailgun/log"
	"github.com/mailgun/oxy/utils"
)

type SideEffect interface {
	Exec() error
}

type Webhook struct {
	URL     string
	Method  string
	Headers http.Header
	Form    url.Values
	Body    []byte
}

type WebhookSideEffect struct {
	w Webhook
}

func NewWebhookSideEffect(w Webhook) (*WebhookSideEffect, error) {
	if w.Method == "" {
		return nil, fmt.Errorf("Supply method")
	}
	_, err := url.Parse(w.URL)
	if err != nil {
		return nil, err
	}

	return &WebhookSideEffect{w: w}, nil
}

func (w *WebhookSideEffect) getBody() io.Reader {
	if len(w.w.Form) != 0 {
		return strings.NewReader(w.w.Form.Encode())
	}
	if len(w.w.Body) != 0 {
		return bytes.NewBuffer(w.w.Body)
	}
	return nil
}

func (w *WebhookSideEffect) Exec() error {
	r, err := http.NewRequest(w.w.Method, w.w.URL, w.getBody())
	if err != nil {
		return err
	}
	if len(w.w.Headers) != 0 {
		utils.CopyHeaders(r.Header, w.w.Headers)
	}
	if len(w.w.Form) != 0 {
		r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	}
	re, err := http.DefaultClient.Do(r)
	if err != nil {
		return err
	}
	if re.Body != nil {
		defer re.Body.Close()
	}
	body, err := ioutil.ReadAll(re.Body)
	if err != nil {
		return err
	}
	log.Infof("%v got response: (%s): %s", w, re.Status, string(body))
	return nil
}
<|endoftext|>


<commit_before>package charneoapo

import "testing"

func TestSimple(t *testing.T) {
	c := NewNeoapo()

	err := c.Do("2100101")

	if err != nil {
		t.Fatal(err)
	}

	if c.Name() != "本田未央" {
		t.Errorf("Unexpected Neoapo.Name: %s", c.Name())
	}

	if c.Kana() != "ほんだみお" {
		t.Errorf("Unexpected Neoapo.Kana: %s", c.Kana())
	}

	if c.Product() != "THE IDOLM@STER CINDERELLA GIRLS アイドルマスターシンデレラガールズ" {
		t.Errorf("Unexpected Neoapo.Product: %s", c.Product())
	}

	if c.Birthday().Unix() != 849398400 {
		t.Errorf("Unexpected Neoapo.Birthday: %s", c.Birthday())
	}

	if c.Blood() != "B" {
		t.Errorf("Unexpected Neoapo.Blood: %s", c.Blood())
	}

	if c.Height() != 161 {
		t.Errorf("Unexpected Neoapo.Height: %d", c.Height())
	}
	if c.Weight() != 46 {
		t.Errorf("Unexpected Neoapo.Weight: %d", c.Weight())
	}

	if c.BWH() != "B84(D)/W58/H87" {
		t.Errorf("Unexpected Neoapo.BWH: %s", c.BWH())
	}
	if c.Bust() != 84 {
		t.Errorf("Unexpected Neoapo.Bust: %d", c.Bust())
	}
	if c.Waist() != 58 {
		t.Errorf("Unexpected Neoapo.Waist: %d", c.Waist())
	}
	if c.Hip() != 87 {
		t.Errorf("Unexpected Neoapo.Hip: %d", c.Hip())
	}

	if c.Bracup() != "D" {
		t.Errorf("Unexpected Neoapo.Bracup: %s", c.Bracup())
	}

	if c.Comment() != "パッションを選ぶとチュートリアルで仲間になる。覚醒美希っぽい髪型とピンクのジャージが特徴的で、明るく元気一杯。口癖は「えへへっ」" {
		t.Errorf("Unexpected Neoapo.Comment: %s", c.Comment())
	}
}
<commit_msg>Fixed testing<commit_after>package charneoapo

import "testing"

func TestSimple(t *testing.T) {
	c := NewNeoapo()

	err := c.Do("characters", "16222")

	if err != nil {
		t.Fatal(err)
	}

	if c.Name() != "本田未央" {
		t.Errorf("Unexpected Neoapo.Name: %s", c.Name())
	}

	if c.Kana() != "ほんだみお" {
		t.Errorf("Unexpected Neoapo.Kana: %s", c.Kana())
	}

	if c.Product() != "THE IDOLM@STER CINDERELLA GIRLS アイドルマスターシンデレラガールズ" {
		t.Errorf("Unexpected Neoapo.Product: %s", c.Product())
	}

	if c.Birthday().Unix() != 849398400 {
		t.Errorf("Unexpected Neoapo.Birthday: %s", c.Birthday())
	}

	if c.Blood() != "B" {
		t.Errorf("Unexpected Neoapo.Blood: %s", c.Blood())
	}

	if c.Height() != 161 {
		t.Errorf("Unexpected Neoapo.Height: %d", c.Height())
	}
	if c.Weight() != 46 {
		t.Errorf("Unexpected Neoapo.Weight: %d", c.Weight())
	}

	if c.BWH() != "B84(D)/W58/H87" {
		t.Errorf("Unexpected Neoapo.BWH: %s", c.BWH())
	}
	if c.Bust() != 84 {
		t.Errorf("Unexpected Neoapo.Bust: %d", c.Bust())
	}
	if c.Waist() != 58 {
		t.Errorf("Unexpected Neoapo.Waist: %d", c.Waist())
	}
	if c.Hip() != 87 {
		t.Errorf("Unexpected Neoapo.Hip: %d", c.Hip())
	}

	if c.Bracup() != "D" {
		t.Errorf("Unexpected Neoapo.Bracup: %s", c.Bracup())
	}

	if c.Comment() != "パッションを選ぶとチュートリアルで仲間になる。覚醒美希っぽい髪型とピンクのジャージが特徴的で、明るく元気一杯。口癖は「えへへっ」" {
		t.Errorf("Unexpected Neoapo.Comment: %s", c.Comment())
	}
}
<|endoftext|>


<commit_before>// Package charset provides functions to decode and encode charsets.
package charset

import (
	"fmt"
	"io"
	"strings"

	"github.com/emersion/go-message"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/encoding/japanese"
	"golang.org/x/text/encoding/simplifiedchinese"
	"golang.org/x/text/encoding/traditionalchinese"
)

var charsets = map[string]encoding.Encoding{
	"big5":             traditionalchinese.Big5,
	"euc-jp":           japanese.EUCJP,
	"gbk":              simplifiedchinese.GBK,
	"gb2312":           simplifiedchinese.GBK,     // as GBK is a superset of HZGB2312, so just use GBK
	"gb18030":          simplifiedchinese.GB18030, // GB18030 Use for parse QQ business mail message
	"iso-2022-jp":      japanese.ISO2022JP,
	"iso-8859-1":       charmap.ISO8859_1,
	"iso-8859-2":       charmap.ISO8859_2,
	"iso-8859-3":       charmap.ISO8859_3,
	"iso-8859-4":       charmap.ISO8859_4,
	"iso-8859-9":       charmap.ISO8859_9,
	"iso-8859-10":      charmap.ISO8859_10,
	"iso-8859-13":      charmap.ISO8859_13,
	"iso-8859-14":      charmap.ISO8859_14,
	"iso-8859-15":      charmap.ISO8859_15,
	"iso-8859-16":      charmap.ISO8859_16,
	"koi8-r":           charmap.KOI8R,
	"shift_jis":        japanese.ShiftJIS,
	"windows-1250":     charmap.Windows1250,
	"windows-1251":     charmap.Windows1251,
	"windows-1252":     charmap.Windows1252,
	"cp1250":           charmap.Windows1250,
	"cp1251":           charmap.Windows1251,
	"cp1252":           charmap.Windows1252,
	"ansi_x3.110-1983": charmap.ISO8859_1,
}

func init() {
	message.CharsetReader = Reader
}

// Reader returns an io.Reader that converts the provided charset to UTF-8.
func Reader(charset string, input io.Reader) (io.Reader, error) {
	charset = strings.ToLower(charset)
	// QUIRK: "ascii" and "utf8" are not in the spec but are common
	if charset == "utf-8" || charset == "utf8" || charset == "us-ascii" || charset == "ascii" {
		return input, nil
	}
	if enc, ok := charsets[charset]; ok {
		return enc.NewDecoder().Reader(input), nil
	}
	return nil, fmt.Errorf("unhandled charset %q", charset)
}

// RegisterEncoding registers an encoding. This is intended to be called from
// the init function in packages that want to support additional charsets.
func RegisterEncoding(name string, enc encoding.Encoding) {
	charsets[name] = enc
}
<commit_msg>Support ANSI x3.4 as a name for ASCII<commit_after>// Package charset provides functions to decode and encode charsets.
package charset

import (
	"fmt"
	"io"
	"strings"

	"github.com/emersion/go-message"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/encoding/japanese"
	"golang.org/x/text/encoding/simplifiedchinese"
	"golang.org/x/text/encoding/traditionalchinese"
)

var charsets = map[string]encoding.Encoding{
	"big5":             traditionalchinese.Big5,
	"euc-jp":           japanese.EUCJP,
	"gbk":              simplifiedchinese.GBK,
	"gb2312":           simplifiedchinese.GBK,     // as GBK is a superset of HZGB2312, so just use GBK
	"gb18030":          simplifiedchinese.GB18030, // GB18030 Use for parse QQ business mail message
	"iso-2022-jp":      japanese.ISO2022JP,
	"iso-8859-1":       charmap.ISO8859_1,
	"iso-8859-2":       charmap.ISO8859_2,
	"iso-8859-3":       charmap.ISO8859_3,
	"iso-8859-4":       charmap.ISO8859_4,
	"iso-8859-9":       charmap.ISO8859_9,
	"iso-8859-10":      charmap.ISO8859_10,
	"iso-8859-13":      charmap.ISO8859_13,
	"iso-8859-14":      charmap.ISO8859_14,
	"iso-8859-15":      charmap.ISO8859_15,
	"iso-8859-16":      charmap.ISO8859_16,
	"koi8-r":           charmap.KOI8R,
	"shift_jis":        japanese.ShiftJIS,
	"windows-1250":     charmap.Windows1250,
	"windows-1251":     charmap.Windows1251,
	"windows-1252":     charmap.Windows1252,
	"cp1250":           charmap.Windows1250,
	"cp1251":           charmap.Windows1251,
	"cp1252":           charmap.Windows1252,
	"ansi_x3.110-1983": charmap.ISO8859_1,
}

func init() {
	message.CharsetReader = Reader
}

// Reader returns an io.Reader that converts the provided charset to UTF-8.
func Reader(charset string, input io.Reader) (io.Reader, error) {
	charset = strings.ToLower(charset)
	// QUIRK: "ascii" and "utf8" are not in the spec but are common
	if charset == "utf-8" || charset == "utf8" || charset == "us-ascii" || charset == "ascii" || strings.HasPrefix(charset, "ansi_x3.4-") {
		return input, nil
	}
	if enc, ok := charsets[charset]; ok {
		return enc.NewDecoder().Reader(input), nil
	}
	return nil, fmt.Errorf("unhandled charset %q", charset)
}

// RegisterEncoding registers an encoding. This is intended to be called from
// the init function in packages that want to support additional charsets.
func RegisterEncoding(name string, enc encoding.Encoding) {
	charsets[name] = enc
}
<|endoftext|>


<commit_before>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import (
	"errors"
	"fmt"
	"reflect"

	ignTypes "github.com/coreos/ignition/config/v2_1/types"
	"github.com/coreos/ignition/config/validate/report"

	"github.com/coreos/container-linux-config-transpiler/config/platform"
	"github.com/coreos/container-linux-config-transpiler/config/templating"
	"github.com/coreos/container-linux-config-transpiler/config/types/util"
	"github.com/coreos/ignition/config/validate/astnode"
)

var (
	ErrPlatformUnspecified = fmt.Errorf("platform must be specified to use templating")
	ErrInvalidKey          = errors.New("Key is invalid (wrong type or not found")
	ErrNilNode             = errors.New("Ast node is nil")
	ErrKeyNotFound         = errors.New("Key not found")
)

func init() {
	register2_0(func(in Config, ast astnode.AstNode, out ignTypes.Config, p string) (ignTypes.Config, report.Report, astnode.AstNode) {
		if p == platform.OpenStackMetadata || p == platform.CloudStackConfigDrive {
			out.Systemd.Units = append(out.Systemd.Units, ignTypes.Unit{
				Name: "coreos-metadata.service",
				Dropins: []ignTypes.Dropin{{
					Name:     "20-clct-provider-override.conf",
					Contents: fmt.Sprintf("[Service]\nEnvironment=COREOS_METADATA_OPT_PROVIDER=--provider=%s", p),
				}},
			})
			out.Systemd.Units = append(out.Systemd.Units, ignTypes.Unit{
				Name: "coreos-metadata-sshkeys@.service",
				Dropins: []ignTypes.Dropin{{
					Name:     "20-clct-provider-override.conf",
					Contents: fmt.Sprintf("[Service]\nEnvironment=COREOS_METADATA_OPT_PROVIDER=--provider=%s", p),
				}},
			})
		}
		return out, report.Report{}, ast
	})
}

func isZero(v interface{}) bool {
	if v == nil {
		return true
	}
	zv := reflect.Zero(reflect.TypeOf(v))
	return reflect.DeepEqual(v, zv.Interface())
}

// assembleUnit will assemble the contents of a systemd unit dropin that will
// have the given environment variables, and call the given exec line with the
// provided args prepended to it
func assembleUnit(exec string, args, vars []string, p string) (util.SystemdUnit, error) {
	hasTemplating := templating.HasTemplating(args)

	out := util.NewSystemdUnit()
	if hasTemplating {
		if p == "" {
			return util.SystemdUnit{}, ErrPlatformUnspecified
		}
		out.Unit.Add("Requires=coreos-metadata.service")
		out.Unit.Add("After=coreos-metadata.service")
		out.Service.Add("EnvironmentFile=/run/metadata/coreos")
		var err error
		args, err = templating.PerformTemplating(p, args)
		if err != nil {
			return util.SystemdUnit{}, err
		}
	}

	for _, v := range vars {
		out.Service.Add(fmt.Sprintf("Environment=\"%s\"", v))
	}
	for _, a := range args {
		exec += fmt.Sprintf(" \\\n  %s", a)
	}
	out.Service.Add("ExecStart=")
	out.Service.Add(fmt.Sprintf("ExecStart=%s", exec))
	return out, nil
}

func getArgs(format, tagName string, e interface{}) []string {
	if e == nil {
		return nil
	}
	et := reflect.TypeOf(e)
	ev := reflect.ValueOf(e)

	vars := []string{}
	for i := 0; i < et.NumField(); i++ {
		if val := ev.Field(i).Interface(); !isZero(val) {
			if et.Field(i).Anonymous {
				vars = append(vars, getCliArgs(val)...)
			} else {
				key := et.Field(i).Tag.Get(tagName)
				if _, ok := val.(string); ok {
					// to handle whitespace characters
					val = fmt.Sprintf("%q", val)
				}
				vars = append(vars, fmt.Sprintf(format, key, val))
			}
		}
	}

	return vars
}

// getCliArgs builds a list of --ARG=VAL from a struct with cli: tags on its members.
func getCliArgs(e interface{}) []string {
	return getArgs("--%s=%v", "cli", e)
}

// Get returns the value for key, where key is an int or string and n is a
// sequence node or mapping node, respectively.
func getNodeChild(n astnode.AstNode, key interface{}) (astnode.AstNode, error) {
	if n == nil {
		return nil, ErrNilNode
	}
	switch k := key.(type) {
	case int:
		if child, ok := n.SliceChild(k); ok {
			return child, nil
		}
	case string:
		kvmap, ok := n.KeyValueMap()
		if !ok {
			return nil, ErrInvalidKey
		}
		if v, ok := kvmap[k]; ok {
			return v, nil
		}
	default:
		return nil, ErrInvalidKey
	}
	// not found
	return nil, ErrKeyNotFound
}

// GetPath works like calling Get() repeatly with each argument.
func getNodeChildPath(n astnode.AstNode, key ...interface{}) (astnode.AstNode, error) {
	if len(key) == 0 {
		return n, nil
	}
	next, err := getNodeChild(n, key[0])
	if err != nil {
		return nil, err
	}
	return getNodeChildPath(next, key[1:]...)
}
<commit_msg>config: enable coreos-metadata-sshkeys serice on openstack<commit_after>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import (
	"errors"
	"fmt"
	"reflect"

	ignTypes "github.com/coreos/ignition/config/v2_1/types"
	"github.com/coreos/ignition/config/validate/report"

	"github.com/coreos/container-linux-config-transpiler/config/platform"
	"github.com/coreos/container-linux-config-transpiler/config/templating"
	"github.com/coreos/container-linux-config-transpiler/config/types/util"
	iutil "github.com/coreos/container-linux-config-transpiler/internal/util"
	"github.com/coreos/ignition/config/validate/astnode"
)

var (
	ErrPlatformUnspecified = fmt.Errorf("platform must be specified to use templating")
	ErrInvalidKey          = errors.New("Key is invalid (wrong type or not found")
	ErrNilNode             = errors.New("Ast node is nil")
	ErrKeyNotFound         = errors.New("Key not found")
)

func init() {
	register2_0(func(in Config, ast astnode.AstNode, out ignTypes.Config, p string) (ignTypes.Config, report.Report, astnode.AstNode) {
		if p == platform.OpenStackMetadata || p == platform.CloudStackConfigDrive {
			out.Systemd.Units = append(out.Systemd.Units, ignTypes.Unit{
				Name: "coreos-metadata.service",
				Dropins: []ignTypes.Dropin{{
					Name:     "20-clct-provider-override.conf",
					Contents: fmt.Sprintf("[Service]\nEnvironment=COREOS_METADATA_OPT_PROVIDER=--provider=%s", p),
				}},
			})
			out.Systemd.Units = append(out.Systemd.Units, ignTypes.Unit{
				Name:    "coreos-metadata-sshkeys@.service",
				Enabled: iutil.BoolToPtr(true),
				Dropins: []ignTypes.Dropin{{
					Name:     "20-clct-provider-override.conf",
					Contents: fmt.Sprintf("[Service]\nEnvironment=COREOS_METADATA_OPT_PROVIDER=--provider=%s", p),
				}},
			})
		}
		return out, report.Report{}, ast
	})
}

func isZero(v interface{}) bool {
	if v == nil {
		return true
	}
	zv := reflect.Zero(reflect.TypeOf(v))
	return reflect.DeepEqual(v, zv.Interface())
}

// assembleUnit will assemble the contents of a systemd unit dropin that will
// have the given environment variables, and call the given exec line with the
// provided args prepended to it
func assembleUnit(exec string, args, vars []string, p string) (util.SystemdUnit, error) {
	hasTemplating := templating.HasTemplating(args)

	out := util.NewSystemdUnit()
	if hasTemplating {
		if p == "" {
			return util.SystemdUnit{}, ErrPlatformUnspecified
		}
		out.Unit.Add("Requires=coreos-metadata.service")
		out.Unit.Add("After=coreos-metadata.service")
		out.Service.Add("EnvironmentFile=/run/metadata/coreos")
		var err error
		args, err = templating.PerformTemplating(p, args)
		if err != nil {
			return util.SystemdUnit{}, err
		}
	}

	for _, v := range vars {
		out.Service.Add(fmt.Sprintf("Environment=\"%s\"", v))
	}
	for _, a := range args {
		exec += fmt.Sprintf(" \\\n  %s", a)
	}
	out.Service.Add("ExecStart=")
	out.Service.Add(fmt.Sprintf("ExecStart=%s", exec))
	return out, nil
}

func getArgs(format, tagName string, e interface{}) []string {
	if e == nil {
		return nil
	}
	et := reflect.TypeOf(e)
	ev := reflect.ValueOf(e)

	vars := []string{}
	for i := 0; i < et.NumField(); i++ {
		if val := ev.Field(i).Interface(); !isZero(val) {
			if et.Field(i).Anonymous {
				vars = append(vars, getCliArgs(val)...)
			} else {
				key := et.Field(i).Tag.Get(tagName)
				if _, ok := val.(string); ok {
					// to handle whitespace characters
					val = fmt.Sprintf("%q", val)
				}
				vars = append(vars, fmt.Sprintf(format, key, val))
			}
		}
	}

	return vars
}

// getCliArgs builds a list of --ARG=VAL from a struct with cli: tags on its members.
func getCliArgs(e interface{}) []string {
	return getArgs("--%s=%v", "cli", e)
}

// Get returns the value for key, where key is an int or string and n is a
// sequence node or mapping node, respectively.
func getNodeChild(n astnode.AstNode, key interface{}) (astnode.AstNode, error) {
	if n == nil {
		return nil, ErrNilNode
	}
	switch k := key.(type) {
	case int:
		if child, ok := n.SliceChild(k); ok {
			return child, nil
		}
	case string:
		kvmap, ok := n.KeyValueMap()
		if !ok {
			return nil, ErrInvalidKey
		}
		if v, ok := kvmap[k]; ok {
			return v, nil
		}
	default:
		return nil, ErrInvalidKey
	}
	// not found
	return nil, ErrKeyNotFound
}

// GetPath works like calling Get() repeatly with each argument.
func getNodeChildPath(n astnode.AstNode, key ...interface{}) (astnode.AstNode, error) {
	if len(key) == 0 {
		return n, nil
	}
	next, err := getNodeChild(n, key[0])
	if err != nil {
		return nil, err
	}
	return getNodeChildPath(next, key[1:]...)
}
<|endoftext|>


<commit_before>package checker

import (
	"net/http"
	"strings"
	"time"

	"github.com/asciimoo/privacyscore/result"
)

const USER_AGENT = "PrivacyScore Checker v0.1.0"
const TIMEOUT = 3

type Checker interface {
	Check(*result.Result)
}

var checkers []Checker = []Checker{
	&CookieChecker{},
	&HTMLChecker{},
	&HTTPSChecker{},
	&SecureHeaderChecker{},
}

func Run(URL string) (*result.Result, bool) {
	if !strings.HasPrefix(URL, "http://") && !strings.HasPrefix(URL, "https://") {
		URL = "http://" + URL
	}
	var r *result.Result
	client := http.Client{Timeout: time.Duration(TIMEOUT * time.Second)}
	req, err := http.NewRequest("GET", URL, nil)
	if err != nil {
		r = &result.Result{}
		r.AddError(err)
		return r, false
	}
	req.Header.Set("User-Agent", USER_AGENT)
	response, err := client.Do(req)
	if err != nil {
		r = &result.Result{}
		r.AddError(err)
		return r, false
	}
	r, err = result.New(URL, response)
	if err != nil {
		r.AddError(err)
	}
	for _, c := range checkers {
		c.Check(r)
	}
	return r, true
}
<commit_msg>[mod] browser like useragent<commit_after>package checker

import (
	"net/http"
	"strings"
	"time"

	"github.com/asciimoo/privacyscore/result"
)

const USER_AGENT = "Mozilla/5.0 (compatible) PrivacyScore Checker v0.1.0"
const TIMEOUT = 3

type Checker interface {
	Check(*result.Result)
}

var checkers []Checker = []Checker{
	&CookieChecker{},
	&HTMLChecker{},
	&HTTPSChecker{},
	&SecureHeaderChecker{},
}

func Run(URL string) (*result.Result, bool) {
	if !strings.HasPrefix(URL, "http://") && !strings.HasPrefix(URL, "https://") {
		URL = "http://" + URL
	}
	var r *result.Result
	client := http.Client{Timeout: time.Duration(TIMEOUT * time.Second)}
	req, err := http.NewRequest("GET", URL, nil)
	if err != nil {
		r = &result.Result{}
		r.AddError(err)
		return r, false
	}
	req.Header.Set("User-Agent", USER_AGENT)
	response, err := client.Do(req)
	if err != nil {
		r = &result.Result{}
		r.AddError(err)
		return r, false
	}
	r, err = result.New(URL, response)
	if err != nil {
		r.AddError(err)
	}
	for _, c := range checkers {
		c.Check(r)
	}
	return r, true
}
<|endoftext|>


<commit_before>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build darwin freebsd linux windows
// +build !android
// +build !ios

package ebitenutil

import (
	"image"

	"github.com/hajimehoshi/ebiten"
)

// NewImageFromFile loads the file with path and returns ebiten.Image and image.Image.
//
// Image decoders must be imported when using NewImageFromFile. For example,
// if you want to load a PNG image, you'd need to add `_ "image/png"` to the import section.
//
// How to solve path depends on your environment. This varies on your desktop or web browser.
// Note that this doesn't work on mobiles.
//
// For productions, instead of using NewImageFromFile, it is safer to embed your resources, e.g., with github.com/rakyll/statik .
func NewImageFromFile(path string, filter ebiten.Filter) (*ebiten.Image, image.Image, error) {
	file, err := OpenFile(path)
	if err != nil {
		return nil, nil, err
	}
	defer func() {
		_ = file.Close()
	}()
	img, _, err := image.Decode(file)
	if err != nil {
		return nil, nil, err
	}
	img2, err := ebiten.NewImageFromImage(img, filter)
	if err != nil {
		return nil, nil, err
	}
	return img2, img, err
}
<commit_msg>Fix missing js build tag (#1088)<commit_after>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build darwin freebsd js linux windows
// +build !android
// +build !ios

package ebitenutil

import (
	"image"

	"github.com/hajimehoshi/ebiten"
)

// NewImageFromFile loads the file with path and returns ebiten.Image and image.Image.
//
// Image decoders must be imported when using NewImageFromFile. For example,
// if you want to load a PNG image, you'd need to add `_ "image/png"` to the import section.
//
// How to solve path depends on your environment. This varies on your desktop or web browser.
// Note that this doesn't work on mobiles.
//
// For productions, instead of using NewImageFromFile, it is safer to embed your resources, e.g., with github.com/rakyll/statik .
func NewImageFromFile(path string, filter ebiten.Filter) (*ebiten.Image, image.Image, error) {
	file, err := OpenFile(path)
	if err != nil {
		return nil, nil, err
	}
	defer func() {
		_ = file.Close()
	}()
	img, _, err := image.Decode(file)
	if err != nil {
		return nil, nil, err
	}
	img2, err := ebiten.NewImageFromImage(img, filter)
	if err != nil {
		return nil, nil, err
	}
	return img2, img, err
}
<|endoftext|>