<commit_before>// Copyright (c) 2014 The SkyDNS Authors. All rights reserved.
// Use of this source code is governed by The MIT License (MIT) that can be
// found in the LICENSE file.

package server

import (
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/miekg/dns"
)

const (
	SCacheCapacity = 10000
	RCacheCapacity = 100000
	RCacheTtl      = 60
)

// Config provides options to the SkyDNS resolver.
type Config struct {
	// The ip:port SkyDNS should be listening on for incoming DNS requests.
	DnsAddr string `json:"dns_addr,omitempty"`
	// bind to port(s) activated by systemd. If set to true, this overrides DnsAddr.
	Systemd bool `json:"systemd,omitempty"`
	// The domain SkyDNS is authoritative for, defaults to skydns.local.
	Domain string `json:"domain,omitempty"`
	// Domain pointing to a key where service info is stored when being queried
	// for local.dns.skydns.local.
	Local string `json:"local,omitempty"`
	// The hostmaster responsible for this domain, defaults to hostmaster.<Domain>.
	Hostmaster string `json:"hostmaster,omitempty"`
	DNSSEC     string `json:"dnssec,omitempty"`
	// Round robin A/AAAA replies. Default is true.
	RoundRobin bool `json:"round_robin,omitempty"`
	// List of ip:port, seperated by commas of recursive nameservers to forward queries to.
	Nameservers []string `json:"nameservers,omitempty"`
	// Never provide a recursive service.
	NoRec       bool          `json:norec,omitempty"`
	ReadTimeout time.Duration `json:"read_timeout,omitempty"`
	// Default priority on SRV records when none is given. Defaults to 10.
	Priority uint16 `json:"priority"`
	// Default TTL, in seconds, when none is given in etcd. Defaults to 3600.
	Ttl uint32 `json:"ttl,omitempty"`
	// Minimum TTL, in seconds, for NXDOMAIN responses. Defaults to 300.
	MinTtl uint32 `json:"min_ttl,omitempty"`
	// SCache, capacity of the signature cache in signatures stored.
	SCache int `json:"scache,omitempty"`
	// RCache, capacity of response cache in resource records stored.
	RCache int `json:"rcache,omitempty"`
	// RCacheTtl, how long to cache in seconds.
	RCacheTtl int `json:"rcache_ttl,omitempty"`
	// How many labels a name should have before we allow forwarding. Default to 2.
	Ndots int `json:"ndot,omitempty"`

	// DNSSEC key material
	PubKey  *dns.DNSKEY    `json:"-"`
	KeyTag  uint16         `json:"-"`
	PrivKey dns.PrivateKey `json:"-"`

	Verbose bool `json:"-"`

	// some predefined string "constants"
	localDomain string // "local.dns." + config.Domain
	dnsDomain   string // "ns.dns". + config.Domain

	// Stub zones support. Pointer to a map that we refresh when we see
	// an update. Map contains domainname -> nameserver:port
	stub *map[string][]string
}

func SetDefaults(config *Config) error {
	if config.ReadTimeout == 0 {
		config.ReadTimeout = 2 * time.Second
	}
	if config.DnsAddr == "" {
		config.DnsAddr = "127.0.0.1:53"
	}
	if config.Domain == "" {
		config.Domain = "skydns.local."
	}
	if config.Hostmaster == "" {
		config.Hostmaster = appendDomain("hostmaster", config.Domain)
	}
	// People probably don't know that SOA's email addresses cannot
	// contain @-signs, replace them with dots
	config.Hostmaster = dns.Fqdn(strings.Replace(config.Hostmaster, "@", ".", -1))
	if config.MinTtl == 0 {
		config.MinTtl = 60
	}
	if config.Ttl == 0 {
		config.Ttl = 3600
	}
	if config.Priority == 0 {
		config.Priority = 10
	}
	if config.RCache < 0 {
		config.RCache = 0
	}
	if config.SCache < 0 {
		config.SCache = 0
	}
	if config.RCacheTtl == 0 {
		config.RCacheTtl = RCacheTtl
	}
	if config.Ndots <= 0 {
		config.Ndots = 2
	}

	if len(config.Nameservers) == 0 {
		c, err := dns.ClientConfigFromFile("/etc/resolv.conf")
		if err != nil {
			return err
		}
		for _, s := range c.Servers {
			config.Nameservers = append(config.Nameservers, net.JoinHostPort(s, c.Port))
		}
	}
	config.Domain = dns.Fqdn(strings.ToLower(config.Domain))
	if config.DNSSEC != "" {
		// For some reason the + are replaces by spaces in etcd. Re-replace them
		keyfile := strings.Replace(config.DNSSEC, " ", "+", -1)
		k, p, err := ParseKeyFile(keyfile)
		if err != nil {
			return err
		}
		if k.Header().Name != dns.Fqdn(config.Domain) {
			return fmt.Errorf("ownername of DNSKEY must match SkyDNS domain")
		}
		k.Header().Ttl = config.Ttl
		config.PubKey = k
		config.KeyTag = k.KeyTag()
		config.PrivKey = p
	}
	config.localDomain = appendDomain("local.dns", config.Domain)
	config.dnsDomain = appendDomain("ns.dns", config.Domain)
	stubmap := make(map[string][]string)
	config.stub = &stubmap
	return nil
}

func appendDomain(s1, s2 string) string {
	if len(s2) > 0 && s2[0] == '.' {
		return s1 + s2
	}
	return s1 + "." + s2
}
<commit_msg>UPSTREAM: Handle missing resolv.conf<commit_after>// Copyright (c) 2014 The SkyDNS Authors. All rights reserved.
// Use of this source code is governed by The MIT License (MIT) that can be
// found in the LICENSE file.

package server

import (
	"fmt"
	"net"
	"os"
	"strings"
	"time"

	"github.com/miekg/dns"
)

const (
	SCacheCapacity = 10000
	RCacheCapacity = 100000
	RCacheTtl      = 60
)

// Config provides options to the SkyDNS resolver.
type Config struct {
	// The ip:port SkyDNS should be listening on for incoming DNS requests.
	DnsAddr string `json:"dns_addr,omitempty"`
	// bind to port(s) activated by systemd. If set to true, this overrides DnsAddr.
	Systemd bool `json:"systemd,omitempty"`
	// The domain SkyDNS is authoritative for, defaults to skydns.local.
	Domain string `json:"domain,omitempty"`
	// Domain pointing to a key where service info is stored when being queried
	// for local.dns.skydns.local.
	Local string `json:"local,omitempty"`
	// The hostmaster responsible for this domain, defaults to hostmaster.<Domain>.
	Hostmaster string `json:"hostmaster,omitempty"`
	DNSSEC     string `json:"dnssec,omitempty"`
	// Round robin A/AAAA replies. Default is true.
	RoundRobin bool `json:"round_robin,omitempty"`
	// List of ip:port, seperated by commas of recursive nameservers to forward queries to.
	Nameservers []string `json:"nameservers,omitempty"`
	// Never provide a recursive service.
	NoRec       bool          `json:norec,omitempty"`
	ReadTimeout time.Duration `json:"read_timeout,omitempty"`
	// Default priority on SRV records when none is given. Defaults to 10.
	Priority uint16 `json:"priority"`
	// Default TTL, in seconds, when none is given in etcd. Defaults to 3600.
	Ttl uint32 `json:"ttl,omitempty"`
	// Minimum TTL, in seconds, for NXDOMAIN responses. Defaults to 300.
	MinTtl uint32 `json:"min_ttl,omitempty"`
	// SCache, capacity of the signature cache in signatures stored.
	SCache int `json:"scache,omitempty"`
	// RCache, capacity of response cache in resource records stored.
	RCache int `json:"rcache,omitempty"`
	// RCacheTtl, how long to cache in seconds.
	RCacheTtl int `json:"rcache_ttl,omitempty"`
	// How many labels a name should have before we allow forwarding. Default to 2.
	Ndots int `json:"ndot,omitempty"`

	// DNSSEC key material
	PubKey  *dns.DNSKEY    `json:"-"`
	KeyTag  uint16         `json:"-"`
	PrivKey dns.PrivateKey `json:"-"`

	Verbose bool `json:"-"`

	// some predefined string "constants"
	localDomain string // "local.dns." + config.Domain
	dnsDomain   string // "ns.dns". + config.Domain

	// Stub zones support. Pointer to a map that we refresh when we see
	// an update. Map contains domainname -> nameserver:port
	stub *map[string][]string
}

func SetDefaults(config *Config) error {
	if config.ReadTimeout == 0 {
		config.ReadTimeout = 2 * time.Second
	}
	if config.DnsAddr == "" {
		config.DnsAddr = "127.0.0.1:53"
	}
	if config.Domain == "" {
		config.Domain = "skydns.local."
	}
	if config.Hostmaster == "" {
		config.Hostmaster = appendDomain("hostmaster", config.Domain)
	}
	// People probably don't know that SOA's email addresses cannot
	// contain @-signs, replace them with dots
	config.Hostmaster = dns.Fqdn(strings.Replace(config.Hostmaster, "@", ".", -1))
	if config.MinTtl == 0 {
		config.MinTtl = 60
	}
	if config.Ttl == 0 {
		config.Ttl = 3600
	}
	if config.Priority == 0 {
		config.Priority = 10
	}
	if config.RCache < 0 {
		config.RCache = 0
	}
	if config.SCache < 0 {
		config.SCache = 0
	}
	if config.RCacheTtl == 0 {
		config.RCacheTtl = RCacheTtl
	}
	if config.Ndots <= 0 {
		config.Ndots = 2
	}

	if len(config.Nameservers) == 0 {
		c, err := dns.ClientConfigFromFile("/etc/resolv.conf")
		if os.IsNotExist(err) {
			c = &dns.ClientConfig{
				Port:     "53",
				Ndots:    1,
				Timeout:  1,
				Attempts: 2,
			}
		} else if err != nil {
			return err
		}
		for _, s := range c.Servers {
			config.Nameservers = append(config.Nameservers, net.JoinHostPort(s, c.Port))
		}
	}
	config.Domain = dns.Fqdn(strings.ToLower(config.Domain))
	if config.DNSSEC != "" {
		// For some reason the + are replaces by spaces in etcd. Re-replace them
		keyfile := strings.Replace(config.DNSSEC, " ", "+", -1)
		k, p, err := ParseKeyFile(keyfile)
		if err != nil {
			return err
		}
		if k.Header().Name != dns.Fqdn(config.Domain) {
			return fmt.Errorf("ownername of DNSKEY must match SkyDNS domain")
		}
		k.Header().Ttl = config.Ttl
		config.PubKey = k
		config.KeyTag = k.KeyTag()
		config.PrivKey = p
	}
	config.localDomain = appendDomain("local.dns", config.Domain)
	config.dnsDomain = appendDomain("ns.dns", config.Domain)
	stubmap := make(map[string][]string)
	config.stub = &stubmap
	return nil
}

func appendDomain(s1, s2 string) string {
	if len(s2) > 0 && s2[0] == '.' {
		return s1 + s2
	}
	return s1 + "." + s2
}
<|endoftext|>


<commit_before>package logical

import (
	"encoding/json"
	"errors"

	"github.com/hashicorp/vault/helper/wrapping"
)

const (
	// HTTPContentType can be specified in the Data field of a Response
	// so that the HTTP front end can specify a custom Content-Type associated
	// with the HTTPRawBody. This can only be used for non-secrets, and should
	// be avoided unless absolutely necessary, such as implementing a specification.
	// The value must be a string.
	HTTPContentType = "http_content_type"

	// HTTPRawBody is the raw content of the HTTP body that goes with the HTTPContentType.
	// This can only be specified for non-secrets, and should should be similarly
	// avoided like the HTTPContentType. The value must be a byte slice.
	HTTPRawBody = "http_raw_body"

	// HTTPStatusCode is the response code of the HTTP body that goes with the HTTPContentType.
	// This can only be specified for non-secrets, and should should be similarly
	// avoided like the HTTPContentType. The value must be an integer.
	HTTPStatusCode = "http_status_code"

	// For unwrapping we may need to know whether the value contained in the
	// raw body is already JSON-unmarshaled. The presence of this key indicates
	// that it has already been unmarshaled. That way we don't need to simply
	// ignore errors.
	HTTPRawBodyAlreadyJSONDecoded = "http_raw_body_already_json_decoded"
)

// Response is a struct that stores the response of a request.
// It is used to abstract the details of the higher level request protocol.
type Response struct {
	// Secret, if not nil, denotes that this response represents a secret.
	Secret *Secret `json:"secret" structs:"secret" mapstructure:"secret"`

	// Auth, if not nil, contains the authentication information for
	// this response. This is only checked and means something for
	// credential backends.
	Auth *Auth `json:"auth" structs:"auth" mapstructure:"auth"`

	// Response data is an opaque map that must have string keys. For
	// secrets, this data is sent down to the user as-is. To store internal
	// data that you don't want the user to see, store it in
	// Secret.InternalData.
	Data map[string]interface{} `json:"data" structs:"data" mapstructure:"data"`

	// Redirect is an HTTP URL to redirect to for further authentication.
	// This is only valid for credential backends. This will be blanked
	// for any logical backend and ignored.
	Redirect string `json:"redirect" structs:"redirect" mapstructure:"redirect"`

	// Warnings allow operations or backends to return warnings in response
	// to user actions without failing the action outright.
	Warnings []string `json:"warnings" structs:"warnings" mapstructure:"warnings"`

	// Information for wrapping the response in a cubbyhole
	WrapInfo *wrapping.ResponseWrapInfo `json:"wrap_info" structs:"wrap_info" mapstructure:"wrap_info"`
}

// AddWarning adds a warning into the response's warning list
func (r *Response) AddWarning(warning string) {
	if r.Warnings == nil {
		r.Warnings = make([]string, 0, 1)
	}
	r.Warnings = append(r.Warnings, warning)
}

// IsError returns true if this response seems to indicate an error.
func (r *Response) IsError() bool {
	return r != nil && r.Data != nil && len(r.Data) == 1 && r.Data["error"] != nil
}

func (r *Response) Error() error {
	if !r.IsError() {
		return nil
	}
	switch r.Data["error"].(type) {
	case string:
		return errors.New(r.Data["error"].(string))
	case error:
		return r.Data["error"].(error)
	}
	return nil
}

// HelpResponse is used to format a help response
func HelpResponse(text string, seeAlso []string, oapiDoc interface{}) *Response {
	return &Response{
		Data: map[string]interface{}{
			"help":     text,
			"see_also": seeAlso,
			"openapi":  oapiDoc,
		},
	}
}

// ErrorResponse is used to format an error response
func ErrorResponse(text string) *Response {
	return &Response{
		Data: map[string]interface{}{
			"error": text,
		},
	}
}

// ListResponse is used to format a response to a list operation.
func ListResponse(keys []string) *Response {
	resp := &Response{
		Data: map[string]interface{}{},
	}
	if len(keys) != 0 {
		resp.Data["keys"] = keys
	}
	return resp
}

// ListResponseWithInfo is used to format a response to a list operation and
// return the keys as well as a map with corresponding key info.
func ListResponseWithInfo(keys []string, keyInfo map[string]interface{}) *Response {
	resp := ListResponse(keys)

	keyInfoData := make(map[string]interface{})
	for _, key := range keys {
		val, ok := keyInfo[key]
		if ok {
			keyInfoData[key] = val
		}
	}

	if len(keyInfoData) > 0 {
		resp.Data["key_info"] = keyInfoData
	}

	return resp
}

// RespondWithStatusCode takes a response and converts it to a raw response with
// the provided Status Code.
func RespondWithStatusCode(resp *Response, req *Request, code int) (*Response, error) {
	ret := &Response{
		Data: map[string]interface{}{
			HTTPContentType: "application/json",
			HTTPStatusCode:  code,
		},
	}

	if resp != nil {
		httpResp := LogicalResponseToHTTPResponse(resp)

		if req != nil {
			httpResp.RequestID = req.ID
		}

		body, err := json.Marshal(httpResp)
		if err != nil {
			return nil, err
		}

		// We default to string here so that the value is HMAC'd via audit.
		// Since this function is always marshaling to JSON, this is
		// appropriate.
		ret.Data[HTTPRawBody] = string(body)
	}

	return ret, nil
}
<commit_msg>Add Sprintf capability to logical.ErrorResponse (#6076)<commit_after>package logical

import (
	"encoding/json"
	"errors"
	"fmt"

	"github.com/hashicorp/vault/helper/wrapping"
)

const (
	// HTTPContentType can be specified in the Data field of a Response
	// so that the HTTP front end can specify a custom Content-Type associated
	// with the HTTPRawBody. This can only be used for non-secrets, and should
	// be avoided unless absolutely necessary, such as implementing a specification.
	// The value must be a string.
	HTTPContentType = "http_content_type"

	// HTTPRawBody is the raw content of the HTTP body that goes with the HTTPContentType.
	// This can only be specified for non-secrets, and should should be similarly
	// avoided like the HTTPContentType. The value must be a byte slice.
	HTTPRawBody = "http_raw_body"

	// HTTPStatusCode is the response code of the HTTP body that goes with the HTTPContentType.
	// This can only be specified for non-secrets, and should should be similarly
	// avoided like the HTTPContentType. The value must be an integer.
	HTTPStatusCode = "http_status_code"

	// For unwrapping we may need to know whether the value contained in the
	// raw body is already JSON-unmarshaled. The presence of this key indicates
	// that it has already been unmarshaled. That way we don't need to simply
	// ignore errors.
	HTTPRawBodyAlreadyJSONDecoded = "http_raw_body_already_json_decoded"
)

// Response is a struct that stores the response of a request.
// It is used to abstract the details of the higher level request protocol.
type Response struct {
	// Secret, if not nil, denotes that this response represents a secret.
	Secret *Secret `json:"secret" structs:"secret" mapstructure:"secret"`

	// Auth, if not nil, contains the authentication information for
	// this response. This is only checked and means something for
	// credential backends.
	Auth *Auth `json:"auth" structs:"auth" mapstructure:"auth"`

	// Response data is an opaque map that must have string keys. For
	// secrets, this data is sent down to the user as-is. To store internal
	// data that you don't want the user to see, store it in
	// Secret.InternalData.
	Data map[string]interface{} `json:"data" structs:"data" mapstructure:"data"`

	// Redirect is an HTTP URL to redirect to for further authentication.
	// This is only valid for credential backends. This will be blanked
	// for any logical backend and ignored.
	Redirect string `json:"redirect" structs:"redirect" mapstructure:"redirect"`

	// Warnings allow operations or backends to return warnings in response
	// to user actions without failing the action outright.
	Warnings []string `json:"warnings" structs:"warnings" mapstructure:"warnings"`

	// Information for wrapping the response in a cubbyhole
	WrapInfo *wrapping.ResponseWrapInfo `json:"wrap_info" structs:"wrap_info" mapstructure:"wrap_info"`
}

// AddWarning adds a warning into the response's warning list
func (r *Response) AddWarning(warning string) {
	if r.Warnings == nil {
		r.Warnings = make([]string, 0, 1)
	}
	r.Warnings = append(r.Warnings, warning)
}

// IsError returns true if this response seems to indicate an error.
func (r *Response) IsError() bool {
	return r != nil && r.Data != nil && len(r.Data) == 1 && r.Data["error"] != nil
}

func (r *Response) Error() error {
	if !r.IsError() {
		return nil
	}
	switch r.Data["error"].(type) {
	case string:
		return errors.New(r.Data["error"].(string))
	case error:
		return r.Data["error"].(error)
	}
	return nil
}

// HelpResponse is used to format a help response
func HelpResponse(text string, seeAlso []string, oapiDoc interface{}) *Response {
	return &Response{
		Data: map[string]interface{}{
			"help":     text,
			"see_also": seeAlso,
			"openapi":  oapiDoc,
		},
	}
}

// ErrorResponse is used to format an error response
func ErrorResponse(text string, vargs ...interface{}) *Response {
	if len(vargs) > 0 {
		text = fmt.Sprintf(text, vargs...)
	}
	return &Response{
		Data: map[string]interface{}{
			"error": text,
		},
	}
}

// ListResponse is used to format a response to a list operation.
func ListResponse(keys []string) *Response {
	resp := &Response{
		Data: map[string]interface{}{},
	}
	if len(keys) != 0 {
		resp.Data["keys"] = keys
	}
	return resp
}

// ListResponseWithInfo is used to format a response to a list operation and
// return the keys as well as a map with corresponding key info.
func ListResponseWithInfo(keys []string, keyInfo map[string]interface{}) *Response {
	resp := ListResponse(keys)

	keyInfoData := make(map[string]interface{})
	for _, key := range keys {
		val, ok := keyInfo[key]
		if ok {
			keyInfoData[key] = val
		}
	}

	if len(keyInfoData) > 0 {
		resp.Data["key_info"] = keyInfoData
	}

	return resp
}

// RespondWithStatusCode takes a response and converts it to a raw response with
// the provided Status Code.
func RespondWithStatusCode(resp *Response, req *Request, code int) (*Response, error) {
	ret := &Response{
		Data: map[string]interface{}{
			HTTPContentType: "application/json",
			HTTPStatusCode:  code,
		},
	}

	if resp != nil {
		httpResp := LogicalResponseToHTTPResponse(resp)

		if req != nil {
			httpResp.RequestID = req.ID
		}

		body, err := json.Marshal(httpResp)
		if err != nil {
			return nil, err
		}

		// We default to string here so that the value is HMAC'd via audit.
		// Since this function is always marshaling to JSON, this is
		// appropriate.
		ret.Data[HTTPRawBody] = string(body)
	}

	return ret, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2016 Magnus Bäck <magnus@noun.se>

package logstash

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
)

// Process represents the invocation and execution of a Logstash child
// process that emits JSON events from the input and filter
// configuration files supplied by the caller.
type Process struct {
	// Input will be connected to the stdin stream of the started
	// Logstash process. Make sure to close it when all data has
	// been written so that the process will terminate.
	Input io.WriteCloser

	child  *exec.Cmd
	log    io.ReadCloser
	output io.ReadCloser
}

// NewProcess prepares for the execution of a new Logstash process but
// doesn't actually start it. logstashPath is the path to the Logstash
// executable (typically /opt/logstash/bin/logstash), inputCodec is
// the desired codec for the stdin input and inputType the value of
// the "type" field for ingested events. The configs parameter is
// one or more configuration files containing Logstash filters.
func NewProcess(logstashPath, inputCodec string, fields FieldSet, configs ...string) (*Process, error) {
	if len(configs) == 0 {
		return nil, errors.New("Must provide non-empty list of configuration file or directory names.")
	}

	// Unfortunately Logstash doesn't make it easy to just read
	// events from a stdout-connected pipe and the log from a
	// stderr-connected pipe. Stdout can contain other garbage (at
	// the very least "future logs will be sent to ...") and error
	// messages could very well be sent there too. Mitigate by
	// having Logstash write output logs to a temporary file and
	// its own logs to a different temporary file.
	outputFile, err := NewDeletedTempFile("", "")
	if err != nil {
		return nil, err
	}
	logFile, err := NewDeletedTempFile("", "")
	if err != nil {
		outputFile.Close()
		return nil, err
	}

	fieldHash, err := fields.LogstashHash()
	if err != nil {
		return nil, err
	}
	args := []string{
		"--debug",
		"-e",
		fmt.Sprintf(
			"input { stdin { codec => %q add_field => %s } } "+
				"output { file { path => %q codec => \"json_lines\" } }",
			inputCodec, fieldHash, outputFile.Name()),
		"--log",
		logFile.Name(),
	}
	for _, c := range configs {
		args = append(args, "--config")
		args = append(args, c)
	}
	c := exec.Command(logstashPath, args...)

	// The test cases must be written to be stable and independent
	// of the current timezone to there's no risk of a @timestamp
	// mismatch just because we've gone into daylight savings time.
	c.Env = []string{
		"TZ=UTC",
	}

	inputPipe, err := c.StdinPipe()
	if err != nil {
		outputFile.Close()
		logFile.Close()
		return nil, err
	}

	return &Process{
		Input:  inputPipe,
		child:  c,
		output: outputFile,
		log:    logFile,
	}, nil
}

// Start starts a Logstash child process with the previously supplied
// configuration.
func (p *Process) Start() error {
	log.Info("Starting %q with args %q.", p.child.Path, p.child.Args)
	return p.child.Start()
}

// Wait blocks until the started Logstash process terminates and
// returns the result of the execution.
func (p *Process) Wait() (*Result, error) {
	log.Debug("Waiting for child with pid %d to terminate.", p.child.Process.Pid)

	waiterr := p.child.Wait()

	// Save the log output regardless of whether the child process
	// succeeded or not.
	logbuf, logerr := ioutil.ReadAll(p.log)
	if logerr != nil {
		// Log this weird error condition but don't let it
		// fail the function. We don't care about the log
		// contents unless Logstash fails, in which we'll
		// report that problem anyway.
		log.Error("Error reading the Logstash logfile: %s", logerr.Error())
	}
	result := Result{
		Log:     string(logbuf),
		Success: waiterr == nil,
	}
	if waiterr != nil {
		return &result, waiterr
	}

	scanner := bufio.NewScanner(p.output)
	for scanner.Scan() {
		var event Event
		err := json.Unmarshal([]byte(scanner.Text()), &event)
		if err != nil {
			return &result, fmt.Errorf("Logstash succeeded, but this output line couldn't be parsed as JSON: %s", scanner.Text())
		}
		result.Events = append(result.Events, event)
	}
	return &result, scanner.Err()
}

// Release frees all allocated resources connected to this process.
func (p *Process) Release() {
	p.output.Close()
	p.log.Close()
}
<commit_msg>Force single filter worker to avoid pipeline races.<commit_after>// Copyright (c) 2015-2016 Magnus Bäck <magnus@noun.se>

package logstash

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
)

// Process represents the invocation and execution of a Logstash child
// process that emits JSON events from the input and filter
// configuration files supplied by the caller.
type Process struct {
	// Input will be connected to the stdin stream of the started
	// Logstash process. Make sure to close it when all data has
	// been written so that the process will terminate.
	Input io.WriteCloser

	child  *exec.Cmd
	log    io.ReadCloser
	output io.ReadCloser
}

// NewProcess prepares for the execution of a new Logstash process but
// doesn't actually start it. logstashPath is the path to the Logstash
// executable (typically /opt/logstash/bin/logstash), inputCodec is
// the desired codec for the stdin input and inputType the value of
// the "type" field for ingested events. The configs parameter is
// one or more configuration files containing Logstash filters.
func NewProcess(logstashPath, inputCodec string, fields FieldSet, configs ...string) (*Process, error) {
	if len(configs) == 0 {
		return nil, errors.New("Must provide non-empty list of configuration file or directory names.")
	}

	// Unfortunately Logstash doesn't make it easy to just read
	// events from a stdout-connected pipe and the log from a
	// stderr-connected pipe. Stdout can contain other garbage (at
	// the very least "future logs will be sent to ...") and error
	// messages could very well be sent there too. Mitigate by
	// having Logstash write output logs to a temporary file and
	// its own logs to a different temporary file.
	outputFile, err := NewDeletedTempFile("", "")
	if err != nil {
		return nil, err
	}
	logFile, err := NewDeletedTempFile("", "")
	if err != nil {
		outputFile.Close()
		return nil, err
	}

	fieldHash, err := fields.LogstashHash()
	if err != nil {
		return nil, err
	}
	args := []string{
		"--filterworkers", // Make messages arrive in order.
		"1",
		"--debug",
		"-e",
		fmt.Sprintf(
			"input { stdin { codec => %q add_field => %s } } "+
				"output { file { path => %q codec => \"json_lines\" } }",
			inputCodec, fieldHash, outputFile.Name()),
		"--log",
		logFile.Name(),
	}
	for _, c := range configs {
		args = append(args, "--config")
		args = append(args, c)
	}
	c := exec.Command(logstashPath, args...)

	// The test cases must be written to be stable and independent
	// of the current timezone to there's no risk of a @timestamp
	// mismatch just because we've gone into daylight savings time.
	c.Env = []string{
		"TZ=UTC",
	}

	inputPipe, err := c.StdinPipe()
	if err != nil {
		outputFile.Close()
		logFile.Close()
		return nil, err
	}

	return &Process{
		Input:  inputPipe,
		child:  c,
		output: outputFile,
		log:    logFile,
	}, nil
}

// Start starts a Logstash child process with the previously supplied
// configuration.
func (p *Process) Start() error {
	log.Info("Starting %q with args %q.", p.child.Path, p.child.Args)
	return p.child.Start()
}

// Wait blocks until the started Logstash process terminates and
// returns the result of the execution.
func (p *Process) Wait() (*Result, error) {
	log.Debug("Waiting for child with pid %d to terminate.", p.child.Process.Pid)

	waiterr := p.child.Wait()

	// Save the log output regardless of whether the child process
	// succeeded or not.
	logbuf, logerr := ioutil.ReadAll(p.log)
	if logerr != nil {
		// Log this weird error condition but don't let it
		// fail the function. We don't care about the log
		// contents unless Logstash fails, in which we'll
		// report that problem anyway.
		log.Error("Error reading the Logstash logfile: %s", logerr.Error())
	}
	result := Result{
		Log:     string(logbuf),
		Success: waiterr == nil,
	}
	if waiterr != nil {
		return &result, waiterr
	}

	scanner := bufio.NewScanner(p.output)
	for scanner.Scan() {
		var event Event
		err := json.Unmarshal([]byte(scanner.Text()), &event)
		if err != nil {
			return &result, fmt.Errorf("Logstash succeeded, but this output line couldn't be parsed as JSON: %s", scanner.Text())
		}
		result.Events = append(result.Events, event)
	}
	return &result, scanner.Err()
}

// Release frees all allocated resources connected to this process.
func (p *Process) Release() {
	p.output.Close()
	p.log.Close()
}
<|endoftext|>


<commit_before>// Package promgrpc is an instrumentation package that allows capturing metrics of your gRPC based services, both the server and the client side.
// The main goal of version 4 was to make it modular without sacrificing the simplicity of use.
//
// It is still possible to integrate the package in just a few lines.
// However, if necessary, metrics can be added, removed or modified freely.
//
// Design
//
// The package does not introduce any new concepts to an already complicated environment.
// Instead, it focuses on providing implementations of interfaces exported by gRPC and Prometheus libraries.
//
// It causes no side effects nor has global state.
// Instead, it comes with handy one-liners to reduce integration overhead.
//
// The package achieved high modularity by using Inversion of Control.
// We can define three layers of abstraction, where each is configurable or if necessary replaceable.
//
// Collectors serve one purpose, storing metrics.
// These are types well known from Prometheus ecosystem, like counters, gauges, histograms or summaries.
// This package comes with a set of predefined functions that create a specific instances for each use case. For example:
//
//  func NewRequestsTotalCounterVec(Subsystem, ...CollectorOption) *prometheus.CounterVec
//
// Level higher consist of stats handlers. This layer is responsible for metrics collection.
// It is aware of a collector and knows how to use it to record event occurrences.
// Each implementation satisfies stats.Handler and prometheus.Collector interface and knows how to monitor a single dimension, e.g. a total number of received/sent requests:
//
//  func NewRequestsStatsHandler(Subsystem, *prometheus.GaugeVec, ...StatsHandlerOption) *RequestsStatsHandler {
//
// Above all, there is a coordinator.
// StatsHandler combines multiple stats handlers into a single instance.
package promgrpc
<commit_msg>v4 - doc Metrics section<commit_after>// Package promgrpc is an instrumentation package that allows capturing metrics of your gRPC based services, both the server and the client side.
// The main goal of version 4 was to make it modular without sacrificing the simplicity of use.
//
// It is still possible to integrate the package in just a few lines.
// However, if necessary, metrics can be added, removed or modified freely.
//
// Design
//
// The package does not introduce any new concepts to an already complicated environment.
// Instead, it focuses on providing implementations of interfaces exported by gRPC and Prometheus libraries.
//
// It causes no side effects nor has global state.
// Instead, it comes with handy one-liners to reduce integration overhead.
//
// The package achieved high modularity by using Inversion of Control.
// We can define three layers of abstraction, where each is configurable or if necessary replaceable.
//
// Collectors serve one purpose, storing metrics.
// These are types well known from Prometheus ecosystem, like counters, gauges, histograms or summaries.
// This package comes with a set of predefined functions that create a specific instances for each use case. For example:
//
//  func NewRequestsTotalCounterVec(Subsystem, ...CollectorOption) *prometheus.CounterVec
//
// Level higher consist of stats handlers. This layer is responsible for metrics collection.
// It is aware of a collector and knows how to use it to record event occurrences.
// Each implementation satisfies stats.Handler and prometheus.Collector interface and knows how to monitor a single dimension, e.g. a total number of received/sent requests:
//
//  func NewRequestsStatsHandler(Subsystem, *prometheus.GaugeVec, ...StatsHandlerOption) *RequestsStatsHandler {
//
// Above all, there is a coordinator.
// StatsHandler combines multiple stats handlers into a single instance.
//
// Metrics
//
// The package comes with eighteen predefined metrics — nine for server and nine for client side:
//
//  grpc_client_connections
//  grpc_client_message_received_size_histogram_bytes
//  grpc_client_message_sent_size_histogram_bytes
//  grpc_client_messages_received_total
//  grpc_client_messages_sent_total
//  grpc_client_request_duration_histogram_seconds
//  grpc_client_requests_in_flight
//  grpc_client_requests_sent_total
//  grpc_client_responses_received_total
//  grpc_server_connections
//  grpc_server_message_received_size_histogram_bytes
//  grpc_server_message_sent_size_histogram_bytes
//  grpc_server_messages_received_total
//  grpc_server_messages_sent_total
//  grpc_server_request_duration_histogram_seconds
//  grpc_server_requests_in_flight
//  grpc_server_requests_received_total
//  grpc_server_responses_sent_total
package promgrpc
<|endoftext|>


<commit_before>package skiplist

import "sync"

// CNode is a node for the concurrent list
type CNode struct {
	sync.RWMutex
	forward     []*CNode
	key         int
	val         []byte
	marked      bool
	fullyLinked bool
	topLayer    int
}

// CList is a skip list built for high concurrency
type CList struct {
	sync.RWMutex
	MaxLevel int
	level    int
	length   int
	header   *CNode
	footer   *CNode
}

// var _ SkipList = (*CList)(nil)

// NewDupeList initializes a new skiplist with
// max level of 32 or 2^32 elements that allows duplicates
func NewCList() *CList {
	return NewCListWithLevel(ListMaxLevel)
}

// NewDupeListWithLevel initializes a new skiplist with a custom
// max level. Level is defaulted to 32 to allow
// for 2^32 max elements
func NewCListWithLevel(level int) *CList {
	return &CList{
		MaxLevel: level,
		header:   &CNode{forward: make([]*CNode, level)},
		level:    0,
	}
}

// Search will look for a node by the key passed in
// and return a Node if found otherwise nil
func (cl *CList) Search(key int) *CNode {
	x := cl.header
	for i := cl.level; i >= 0; i-- {
		for x.forward[i] != nil && x.forward[i].key < key {
			x = x.forward[i]
		}
	}
	x = x.forward[0]
	if x != nil && x.key == key {
		return x
	}
	return nil
}
<commit_msg>Fix comment description for golint<commit_after>package skiplist

import "sync"

// CNode is a node for the concurrent list
type CNode struct {
	sync.RWMutex
	forward     []*CNode
	key         int
	val         []byte
	marked      bool
	fullyLinked bool
	topLayer    int
}

// CList is a skip list built for high concurrency
type CList struct {
	sync.RWMutex
	MaxLevel int
	level    int
	length   int
	header   *CNode
	footer   *CNode
}

// var _ SkipList = (*CList)(nil)

// NewCList initializes a new skiplist with
// max level of 32 or 2^32 elements that allows duplicates
func NewCList() *CList {
	return NewCListWithLevel(ListMaxLevel)
}

// NewCListWithLevel initializes a new skiplist with a custom
// max level. Level is defaulted to 32 to allow
// for 2^32 max elements
func NewCListWithLevel(level int) *CList {
	return &CList{
		MaxLevel: level,
		header:   &CNode{forward: make([]*CNode, level)},
		level:    0,
	}
}

// Search will look for a node by the key passed in
// and return a Node if found otherwise nil
func (cl *CList) Search(key int) *CNode {
	x := cl.header
	for i := cl.level; i >= 0; i-- {
		for x.forward[i] != nil && x.forward[i].key < key {
			x = x.forward[i]
		}
	}
	x = x.forward[0]
	if x != nil && x.key == key {
		return x
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"

	"github.com/FactomProject/factom"
)

var ack = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli ack fct|e TxID|FullTx"
	cmd.description = "Returns information about a factoid transaction, or an entry / entry credit transaction"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) < 2 {
			fmt.Println(cmd.helpMsg)
			return
		}
		ackType := args[0]
		tx := args[1]

		txID := ""
		fullTx := ""

		if len(tx) == 64 {
			txID = tx
		} else {
			fullTx = tx
		}

		if ackType == "fct" {
			resp, err := factom.FactoidACK(txID, fullTx)
			if err != nil {
				errorln(err)
				return
			}
			str, err := json.MarshalIndent(resp, "", "\t")
			if err != nil {
				errorln(err)
				return
			}
			fmt.Printf("%s\n", str)
		}
		if ackType == "e" {
			resp, err := factom.EntryACK(txID, fullTx)
			if err != nil {
				errorln(err)
				return
			}
			str, err := json.MarshalIndent(resp, "", "\t")
			if err != nil {
				errorln(err)
				return
			}
			fmt.Printf("%s\n", str)
		}
	}
	help.Add("ack", cmd)
	return cmd
}()
<commit_msg>Added a way to get transaction by name<commit_after>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"os"

	"github.com/FactomProject/factom"
)

var ack = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli ack fct|e TxID|FullTx"
	cmd.description = "Returns information about a factoid transaction, or an entry / entry credit transaction"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) < 2 {
			fmt.Println(cmd.helpMsg)
			return
		}
		ackType := args[0]
		tx := args[1]

		txID := ""
		fullTx := ""

		if len(tx) == 64 {
			txID = tx
		} else {
			_, err := hex.DecodeString(tx)
			if len(tx) < 64 || err != nil {
				h, err := factom.TransactionHash(tx)
				if err != nil {
					errorln(err)
					return
				}
				txID = h
			} else {
				fullTx = tx
			}
		}

		if ackType == "fct" {
			resp, err := factom.FactoidACK(txID, fullTx)
			if err != nil {
				errorln(err)
				return
			}
			str, err := json.MarshalIndent(resp, "", "\t")
			if err != nil {
				errorln(err)
				return
			}
			fmt.Printf("%s\n", str)
		}
		if ackType == "e" {
			resp, err := factom.EntryACK(txID, fullTx)
			if err != nil {
				errorln(err)
				return
			}
			str, err := json.MarshalIndent(resp, "", "\t")
			if err != nil {
				errorln(err)
				return
			}
			fmt.Printf("%s\n", str)
		}
	}
	help.Add("ack", cmd)
	return cmd
}()
<|endoftext|>


<commit_before>package chevalier

import (
	zmq "github.com/pebbe/zmq4"
)

// GetContents list for origin from a Vaultaire
// readerd listening on endpoint, returning it as a DataSourceBurst.
func GetContents(endpoint, origin string) (*DataSourceBurst, error) {
	sources := make([]*DataSourceResponse, 0)
	sock, err := zmq.NewSocket(zmq.REQ)
	if err != nil {
		return nil, err
	}
	err = sock.Connect(endpoint)
	if err != nil {
		return nil, err
	}
	request := make([][]byte, 2)
	request[0] = make([]byte, len(origin))
	for idx, chr := range origin {
		request[1][idx] = byte(chr)
	}
	request[1] = make([]byte, 1)
	request[1][0] = byte(ContentsListRequest)
	_, err = sock.SendMessage(request[0], request[1])
	if err != nil {
		return nil, err
	}
	for res := new(ContentsResponse); notStopResponse(res); res, err = recvContentsMessage(sock) {
		if err != nil {
			return nil, err
		}
		res_ := unpackSourceResponse(res)
		sources = append(sources, res_)
	}
	return nil, nil
}

func notStopResponse(res *ContentsResponse) bool {
	if res == nil {
		return false // first iteration or error
	}
	if res.opCode == ContentsListEntry {
		return false // data response, continue
	}
	return true
}

func recvContentsMessage(sock *zmq.Socket) (*ContentsResponse, error) {
	bs, err := sock.RecvBytes(0)
	if err != nil {
		return nil, err
	}
	return unpackContentsResponse(bs)
}

func unpackSourceResponse(res *ContentsResponse) *DataSourceResponse {
	return nil
}
<commit_msg>How did that ever work?<commit_after>package chevalier

import (
	zmq "github.com/pebbe/zmq4"
)

// GetContents list for origin from a Vaultaire
// readerd listening on endpoint, returning it as a DataSourceBurst.
func GetContents(endpoint, origin string) (*DataSourceBurst, error) {
	sources := make([]*DataSourceResponse, 0)
	sock, err := zmq.NewSocket(zmq.REQ)
	if err != nil {
		return nil, err
	}
	err = sock.Connect(endpoint)
	if err != nil {
		return nil, err
	}
	request := make([][]byte, 2)
	request[0] = make([]byte, len(origin))
	for idx, chr := range origin {
		request[0][idx] = byte(chr)
	}
	request[1] = make([]byte, 1)
	request[1][0] = byte(ContentsListRequest)
	_, err = sock.SendMessage(request[0], request[1])
	if err != nil {
		return nil, err
	}
	for res := new(ContentsResponse); notStopResponse(res); res, err = recvContentsMessage(sock) {
		if err != nil {
			return nil, err
		}
		res_ := unpackSourceResponse(res)
		sources = append(sources, res_)
	}
	return nil, nil
}

func notStopResponse(res *ContentsResponse) bool {
	if res == nil {
		return false // first iteration or error
	}
	if res.opCode == ContentsListEntry {
		return false // data response, continue
	}
	return true
}

func recvContentsMessage(sock *zmq.Socket) (*ContentsResponse, error) {
	bs, err := sock.RecvBytes(0)
	if err != nil {
		return nil, err
	}
	return unpackContentsResponse(bs)
}

func unpackSourceResponse(res *ContentsResponse) *DataSourceResponse {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"time"

	"github.com/nightlyone/lockfile"
)

var updateTopic = &Topic{
	Name:        "update",
	Description: "update heroku-cli",
}

var updateCmd = &Command{
	Topic:       "update",
	Description: "updates heroku-cli",
	Args:        []Arg{{Name: "channel", Optional: true}},
	Run: func(ctx *Context) {
		channel := ctx.Args.(map[string]string)["channel"]
		if channel == "" {
			channel = "master"
		}
		Errf("updating plugins... ")
		if err := node.UpdatePackages(); err != nil {
			panic(err)
		}
		Errln("done")
		manifest := getUpdateManifest(channel)
		build := manifest.Builds[runtime.GOOS][runtime.GOARCH]
		Errf("updating to %s (%s)... ", manifest.Version, manifest.Channel)
		update(build.URL, build.Sha1)
		Errln("done")
	},
}

var binPath = filepath.Join(AppDir, "heroku-cli")
var updateLockPath = filepath.Join(AppDir, "updating.lock")
var autoupdateFile = filepath.Join(AppDir, "autoupdate")

func init() {
	if runtime.GOOS == "windows" {
		binPath = binPath + ".exe"
	}
}

// UpdateIfNeeded checks for and performs an autoupdate if there is a new version out.
func UpdateIfNeeded() {
	lock := getUpdateLock()
	defer lock.Unlock()
	if !updateNeeded() {
		return
	}
	defer touchAutoupdateFile()
	manifest := getUpdateManifest(Channel)
	node.UpdatePackages()
	if manifest.Version == Version {
		return
	}
	if !updatable() {
		Errf("Out of date: You are running %s but %s is out.\n", Version, manifest.Version)
		return
	}
	// Leave out updating text until heroku-cli is used in place of ruby cli
	// So it doesn't confuse users with 2 different version numbers
	//Errf("Updating to %s... ", manifest.Version)
	build := manifest.Builds[runtime.GOOS][runtime.GOARCH]
	update(build.URL, build.Sha1)
	//Errln("done")

	// these are deferred but won't be called because of os.Exit
	touchAutoupdateFile()
	lock.Unlock()
	execBin()
	os.Exit(0)
}

// Attempts to get the lockfile
// Blocks if unavailable
func getUpdateLock() lockfile.Lockfile {
	lock, err := lockfile.New(updateLockPath)
	if err != nil {
		Errln("Cannot initialize update lockfile.")
		panic(err)
	}
	start := time.Now()
	for {
		if lock.TryLock() == nil {
			return lock
		}
		time.Sleep(10 * time.Millisecond)
		if time.Since(start) > 30*time.Second {
			// In a timeout, assume the last updating process timed out
			os.Remove(updateLockPath)
		}
	}
}

func updateNeeded() bool {
	if Version == "dev" {
		return false
	}
	f, err := os.Stat(autoupdateFile)
	if err != nil {
		return true
	}
	return time.Since(f.ModTime()) > 1*time.Hour
}

func touchAutoupdateFile() {
	out, err := os.OpenFile(autoupdateFile, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		panic(err)
	}
	out.WriteString(time.Now().String())
}

type manifest struct {
	Channel, Version string
	Builds           map[string]map[string]struct {
		URL, Sha1 string
	}
}

func getUpdateManifest(channel string) manifest {
	res, err := http.Get("http://d1gvo455cekpjp.cloudfront.net/" + channel + "/manifest.json")
	if err != nil {
		panic(err)
	}
	var m manifest
	json.NewDecoder(res.Body).Decode(&m)
	return m
}

func updatable() bool {
	path, err := filepath.Abs(os.Args[0])
	if err != nil {
		Errln(err)
	}
	return path == binPath
}

func update(url, sha1 string) {
	// on windows we can't remove an existing file or remove the running binary
	// so we download the file to binName.new
	// move the running binary to binName.old (deleting any existing file first)
	// rename the downloaded file to binName
	if err := downloadBin(binPath+".new", url); err != nil {
		panic(err)
	}
	if fileSha1(binPath+".new") != sha1 {
		panic("SHA mismatch")
	}
	os.Remove(binPath + ".old")
	if err := os.Rename(binPath, binPath+".old"); err != nil {
		panic(err)
	}
	if err := os.Rename(binPath+".new", binPath); err != nil {
		panic(err)
	}
}

func downloadBin(path, url string) error {
	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
	if err != nil {
		return err
	}
	defer out.Close()
	client := &http.Client{}
	req, err := http.NewRequest("GET", url+".gz", nil)
	if err != nil {
		return err
	}
	req.Header.Add("Accept-Encoding", "gzip")
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	uncompressed, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}
	_, err = io.Copy(out, uncompressed)
	return err
}

func fileSha1(path string) string {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", sha1.Sum(data))
}

func execBin() {
	cmd := exec.Command(binPath, os.Args[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Run()
}
<commit_msg>speed up updating by synchronously updating plugins<commit_after>package main

import (
	"compress/gzip"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"time"

	"github.com/nightlyone/lockfile"
)

var updateTopic = &Topic{
	Name:        "update",
	Description: "update heroku-cli",
}

var updateCmd = &Command{
	Topic:       "update",
	Description: "updates heroku-cli",
	Args:        []Arg{{Name: "channel", Optional: true}},
	Run: func(ctx *Context) {
		channel := ctx.Args.(map[string]string)["channel"]
		if channel == "" {
			channel = "master"
		}
		Errf("updating plugins... ")
		if err := node.UpdatePackages(); err != nil {
			panic(err)
		}
		Errln("done")
		manifest := getUpdateManifest(channel)
		build := manifest.Builds[runtime.GOOS][runtime.GOARCH]
		Errf("updating to %s (%s)... ", manifest.Version, manifest.Channel)
		update(build.URL, build.Sha1)
		Errln("done")
	},
}

var binPath = filepath.Join(AppDir, "heroku-cli")
var updateLockPath = filepath.Join(AppDir, "updating.lock")
var autoupdateFile = filepath.Join(AppDir, "autoupdate")

func init() {
	if runtime.GOOS == "windows" {
		binPath = binPath + ".exe"
	}
}

// UpdateIfNeeded checks for and performs an autoupdate if there is a new version out.
func UpdateIfNeeded() {
	lock := getUpdateLock()
	if !updateNeeded() {
		lock.Unlock()
		return
	}
	manifest := getUpdateManifest(Channel)
	doneUpdatingPlugins := make(chan bool)
	go func() {
		node.UpdatePackages()
		doneUpdatingPlugins <- true
	}()
	if manifest.Version == Version {
		<-doneUpdatingPlugins
		touchAutoupdateFile()
		lock.Unlock()
		return
	}
	if !updatable() {
		Errf("Out of date: You are running %s but %s is out.\n", Version, manifest.Version)
		<-doneUpdatingPlugins
		lock.Unlock()
		return
	}
	// Leave out updating text until heroku-cli is used in place of ruby cli
	// So it doesn't confuse users with 2 different version numbers
	//Errf("Updating to %s... ", manifest.Version)
	build := manifest.Builds[runtime.GOOS][runtime.GOARCH]
	update(build.URL, build.Sha1)
	<-doneUpdatingPlugins
	//Errln("done")

	// these are deferred but won't be called because of os.Exit
	touchAutoupdateFile()
	lock.Unlock()
	execBin()
	os.Exit(0)
}

// Attempts to get the lockfile
// Blocks if unavailable
func getUpdateLock() lockfile.Lockfile {
	lock, err := lockfile.New(updateLockPath)
	if err != nil {
		Errln("Cannot initialize update lockfile.")
		panic(err)
	}
	start := time.Now()
	for {
		if lock.TryLock() == nil {
			return lock
		}
		time.Sleep(10 * time.Millisecond)
		if time.Since(start) > 30*time.Second {
			// In a timeout, assume the last updating process timed out
			os.Remove(updateLockPath)
		}
	}
}

func updateNeeded() bool {
	if Version == "dev" {
		return false
	}
	f, err := os.Stat(autoupdateFile)
	if err != nil {
		return true
	}
	return time.Since(f.ModTime()) > 1*time.Hour
}

func touchAutoupdateFile() {
	out, err := os.OpenFile(autoupdateFile, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		panic(err)
	}
	out.WriteString(time.Now().String())
}

type manifest struct {
	Channel, Version string
	Builds           map[string]map[string]struct {
		URL, Sha1 string
	}
}

func getUpdateManifest(channel string) manifest {
	res, err := http.Get("http://d1gvo455cekpjp.cloudfront.net/" + channel + "/manifest.json")
	if err != nil {
		panic(err)
	}
	var m manifest
	json.NewDecoder(res.Body).Decode(&m)
	return m
}

func updatable() bool {
	path, err := filepath.Abs(os.Args[0])
	if err != nil {
		Errln(err)
	}
	return path == binPath
}

func update(url, sha1 string) {
	// on windows we can't remove an existing file or remove the running binary
	// so we download the file to binName.new
	// move the running binary to binName.old (deleting any existing file first)
	// rename the downloaded file to binName
	if err := downloadBin(binPath+".new", url); err != nil {
		panic(err)
	}
	if fileSha1(binPath+".new") != sha1 {
		panic("SHA mismatch")
	}
	os.Remove(binPath + ".old")
	if err := os.Rename(binPath, binPath+".old"); err != nil {
		panic(err)
	}
	if err := os.Rename(binPath+".new", binPath); err != nil {
		panic(err)
	}
}

func downloadBin(path, url string) error {
	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
	if err != nil {
		return err
	}
	defer out.Close()
	client := &http.Client{}
	req, err := http.NewRequest("GET", url+".gz", nil)
	if err != nil {
		return err
	}
	req.Header.Add("Accept-Encoding", "gzip")
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	uncompressed, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}
	_, err = io.Copy(out, uncompressed)
	return err
}

func fileSha1(path string) string {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", sha1.Sum(data))
}

func execBin() {
	cmd := exec.Command(binPath, os.Args[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Run()
}
<|endoftext|>


<commit_before>package ipset

import (
	"bytes"
	"os/exec"
	"strings"

	"github.com/pkg/errors"
)

type Name string

type Type string

const (
	ListSet = Type("list:set")
	HashIP  = Type("hash:ip")
)

type Interface interface {
	Create(ipsetName Name, ipsetType Type) error
	AddEntry(ipsetName Name, entry string) error
	DelEntry(ipsetName Name, entry string) error
	Flush(ipsetName Name) error
	Destroy(ipsetName Name) error

	List(prefix string) ([]Name, error)

	FlushAll() error
	DestroyAll() error
}

type ipset struct {
	refCount
}

func New() Interface {
	return &ipset{refCount: newRefCount()}
}

func (i *ipset) Create(ipsetName Name, ipsetType Type) error {
	err, _ := doExec("create", string(ipsetName), string(ipsetType))
	return err
}

func (i *ipset) AddEntry(ipsetName Name, entry string) error {
	if i.inc(ipsetName, entry) > 1 { // already in the set
		return nil
	}
	err, _ := doExec("add", string(ipsetName), entry)
	return err
}

func (i *ipset) DelEntry(ipsetName Name, entry string) error {
	if i.dec(ipsetName, entry) > 0 { // still needed
		return nil
	}
	err, _ := doExec("del", string(ipsetName), entry)
	return err
}

func (i *ipset) Flush(ipsetName Name) error {
	i.removeSet(ipsetName)
	err, _ := doExec("flush", string(ipsetName))
	return err
}

func (i *ipset) FlushAll() error {
	i.refCount = newRefCount()
	err, _ := doExec("flush")
	return err
}

func (i *ipset) Destroy(ipsetName Name) error {
	i.removeSet(ipsetName)
	err, _ := doExec("destroy", string(ipsetName))
	return err
}

func (i *ipset) DestroyAll() error {
	i.refCount = newRefCount()
	err, _ := doExec("destroy")
	return err
}

// Fetch a list of all existing sets
func (i *ipset) List(prefix string) ([]Name, error) {
	err, output := doExec("list","-name","-output","plain")

	var selected []Name
	if err == nil && len(output) > 0 {
		output = bytes.TrimRight(output, "\n")
		sets := strings.Split(string(output[:]), "\n")

		plen := len(prefix)
		for _, v := range sets {
			if (plen <= len(v)) && (prefix == v[:len(prefix)]) {
				selected = append(selected, Name(v))
			}
		}

	}

	return selected, err
}

func doExec(args ...string) (error, []byte) {
	output, err := exec.Command("ipset", args...).CombinedOutput()
	if err != nil {
		return errors.Wrapf(err, "ipset %v failed: %s", args, output), output
	}
	return nil, output
}

// Reference-counting
type key struct {
	ipsetName Name
	entry     string
}

// note no locking is required as all operations are serialised in the controller
type refCount struct {
	ref map[key]int
}

func newRefCount() refCount {
	return refCount{ref: make(map[key]int)}
}

func (rc *refCount) inc(ipsetName Name, entry string) int {
	k := key{ipsetName, entry}
	rc.ref[k]++
	return rc.ref[k]
}

func (rc *refCount) dec(ipsetName Name, entry string) int {
	k := key{ipsetName, entry}
	rc.ref[k]--
	return rc.ref[k]
}

func (rc *refCount) removeSet(ipsetName Name) {
	for k := range rc.ref {
		if k.ipsetName == ipsetName {
			delete(rc.ref, k)
		}
	}
}
<commit_msg>Simplify List()<commit_after>package ipset

import (
	"os/exec"
	"strings"

	"github.com/pkg/errors"
)

type Name string

type Type string

const (
	ListSet = Type("list:set")
	HashIP  = Type("hash:ip")
)

type Interface interface {
	Create(ipsetName Name, ipsetType Type) error
	AddEntry(ipsetName Name, entry string) error
	DelEntry(ipsetName Name, entry string) error
	Flush(ipsetName Name) error
	Destroy(ipsetName Name) error

	List(prefix string) ([]Name, error)

	FlushAll() error
	DestroyAll() error
}

type ipset struct {
	refCount
}

func New() Interface {
	return &ipset{refCount: newRefCount()}
}

func (i *ipset) Create(ipsetName Name, ipsetType Type) error {
	err, _ := doExec("create", string(ipsetName), string(ipsetType))
	return err
}

func (i *ipset) AddEntry(ipsetName Name, entry string) error {
	if i.inc(ipsetName, entry) > 1 { // already in the set
		return nil
	}
	err, _ := doExec("add", string(ipsetName), entry)
	return err
}

func (i *ipset) DelEntry(ipsetName Name, entry string) error {
	if i.dec(ipsetName, entry) > 0 { // still needed
		return nil
	}
	err, _ := doExec("del", string(ipsetName), entry)
	return err
}

func (i *ipset) Flush(ipsetName Name) error {
	i.removeSet(ipsetName)
	err, _ := doExec("flush", string(ipsetName))
	return err
}

func (i *ipset) FlushAll() error {
	i.refCount = newRefCount()
	err, _ := doExec("flush")
	return err
}

func (i *ipset) Destroy(ipsetName Name) error {
	i.removeSet(ipsetName)
	err, _ := doExec("destroy", string(ipsetName))
	return err
}

func (i *ipset) DestroyAll() error {
	i.refCount = newRefCount()
	err, _ := doExec("destroy")
	return err
}

// Fetch a list of all existing sets with a given prefix
func (i *ipset) List(prefix string) ([]Name, error) {
	err, output := doExec("list","-name","-output","plain")
	if err != nil {
		return nil, err
	}

	var selected []Name
	sets := strings.Split(string(output), "\n")
	for _, v := range sets {
		if strings.HasPrefix(v, prefix) {
			selected = append(selected, Name(v))
		}
	}

	return selected, err
}

func doExec(args ...string) (error, []byte) {
	output, err := exec.Command("ipset", args...).CombinedOutput()
	if err != nil {
		return errors.Wrapf(err, "ipset %v failed: %s", args, output), output
	}
	return nil, output
}

// Reference-counting
type key struct {
	ipsetName Name
	entry     string
}

// note no locking is required as all operations are serialised in the controller
type refCount struct {
	ref map[key]int
}

func newRefCount() refCount {
	return refCount{ref: make(map[key]int)}
}

func (rc *refCount) inc(ipsetName Name, entry string) int {
	k := key{ipsetName, entry}
	rc.ref[k]++
	return rc.ref[k]
}

func (rc *refCount) dec(ipsetName Name, entry string) int {
	k := key{ipsetName, entry}
	rc.ref[k]--
	return rc.ref[k]
}

func (rc *refCount) removeSet(ipsetName Name) {
	for k := range rc.ref {
		if k.ipsetName == ipsetName {
			delete(rc.ref, k)
		}
	}
}
<|endoftext|>


<commit_before>package dec

import (
	"bytes"
	"io"
	"runtime"
	"testing"

	"gopkg.in/kothar/brotli-go.v0/enc"
)

func TestStreamDecompression(T *testing.T) {

	input1 := bytes.Repeat([]byte("The quick brown fox jumps over the lazy dog. "), 100000)

	output1 := make([]byte, len(input1)*2)
	params := enc.NewBrotliParams()
	params.SetQuality(4)

	_, err := enc.CompressBuffer(params, input1, output1)
	if err != nil {
		T.Fatal(err)
	}

	// Decompress as a stream
	reader := NewBrotliReader(bytes.NewReader(output1))
	decoded := make([]byte, len(input1))

	read, err := io.ReadFull(reader, decoded)
	if err != nil {
		T.Fatal(err)
	}
	if read != len(input1) {
		T.Errorf("Length of decoded stream (%d) doesn't match input (%d)", read, len(input1))
	}

	T.Logf("Input:  %s", input1[:50])
	T.Logf("Output: %s", decoded[:50])
	if !bytes.Equal(decoded, input1) {
		T.Error("Decoded output does not match original input")
	}

	// Decompress using a shorter buffer
	reader = NewBrotliReader(bytes.NewReader(output1))
	decoded = make([]byte, 500)

	read, err = reader.Read(decoded)
	if err != nil {
		T.Fatal(err)
	}
	if read != len(decoded) {
		T.Errorf("Length of decoded stream (%d) shorter than requested (%d)", read, len(decoded))
	}

	T.Logf("Input:  %s", input1[:50])
	T.Logf("Output: %s", decoded[:50])
	if !bytes.Equal(decoded, input1[:len(decoded)]) {
		T.Error("Decoded output does not match original input")
	}

	// Read next buffer
	read, err = reader.Read(decoded)
	if err != nil {
		T.Fatal(err)
	}
	if read != len(decoded) {
		T.Errorf("Length of decoded stream (%d) shorter than requested (%d)", read, len(decoded))
	}

	T.Logf("Input:  %s", input1[len(decoded):len(decoded)+50])
	T.Logf("Output: %s", decoded[:50])
	if !bytes.Equal(decoded, input1[len(decoded):2*len(decoded)]) {
		T.Error("Decoded output does not match original input")
	}
}

// Attempt to GC error in decoder
func TestGCErrors(T *testing.T) {
	input := bytes.Repeat([]byte("The quick brown fox jumps over the lazy dog. "), 10000)

	buffer := make([]byte, len(input)*2)
	params := enc.NewBrotliParams()
	params.SetQuality(4)

	output, err := enc.CompressBuffer(params, input, buffer)
	if err != nil {
		T.Fatal(err)
	}

	// Decompress stream
	reader := NewBrotliReader(bytes.NewReader(output))
	decoded := make([]byte, 18123)
	var count int
	for read, err := reader.Read(decoded); err != io.EOF; {
		if err != nil {
			T.Fatal(err)
		}

		count += read
		T.Logf("Read %d/%d bytes\n", count, len(input))
		if count > len(input) {
			T.Error("Too many bytes read from stream without EOF")
			break
		}

		// Force garbage collection
		runtime.GC()
	}
	reader.Close()
}
<commit_msg>Fix GC test<commit_after>package dec

import (
	"bytes"
	"io"
	"runtime"
	"testing"

	"gopkg.in/kothar/brotli-go.v0/enc"
)

func TestStreamDecompression(T *testing.T) {

	input1 := bytes.Repeat([]byte("The quick brown fox jumps over the lazy dog. "), 100000)

	output1 := make([]byte, len(input1)*2)
	params := enc.NewBrotliParams()
	params.SetQuality(4)

	_, err := enc.CompressBuffer(params, input1, output1)
	if err != nil {
		T.Fatal(err)
	}

	// Decompress as a stream
	reader := NewBrotliReader(bytes.NewReader(output1))
	decoded := make([]byte, len(input1))

	read, err := io.ReadFull(reader, decoded)
	if err != nil {
		T.Fatal(err)
	}
	if read != len(input1) {
		T.Errorf("Length of decoded stream (%d) doesn't match input (%d)", read, len(input1))
	}

	T.Logf("Input:  %s", input1[:50])
	T.Logf("Output: %s", decoded[:50])
	if !bytes.Equal(decoded, input1) {
		T.Error("Decoded output does not match original input")
	}

	// Decompress using a shorter buffer
	reader = NewBrotliReader(bytes.NewReader(output1))
	decoded = make([]byte, 500)

	read, err = reader.Read(decoded)
	if err != nil {
		T.Fatal(err)
	}
	if read != len(decoded) {
		T.Errorf("Length of decoded stream (%d) shorter than requested (%d)", read, len(decoded))
	}

	T.Logf("Input:  %s", input1[:50])
	T.Logf("Output: %s", decoded[:50])
	if !bytes.Equal(decoded, input1[:len(decoded)]) {
		T.Error("Decoded output does not match original input")
	}

	// Read next buffer
	read, err = reader.Read(decoded)
	if err != nil {
		T.Fatal(err)
	}
	if read != len(decoded) {
		T.Errorf("Length of decoded stream (%d) shorter than requested (%d)", read, len(decoded))
	}

	T.Logf("Input:  %s", input1[len(decoded):len(decoded)+50])
	T.Logf("Output: %s", decoded[:50])
	if !bytes.Equal(decoded, input1[len(decoded):2*len(decoded)]) {
		T.Error("Decoded output does not match original input")
	}
}

// Attempt to GC error in decoder
func TestGCErrors(T *testing.T) {
	input := bytes.Repeat([]byte("The quick brown fox jumps over the lazy dog. "), 10000)

	buffer := make([]byte, len(input)*2)
	params := enc.NewBrotliParams()
	params.SetQuality(4)

	output, err := enc.CompressBuffer(params, input, buffer)
	if err != nil {
		T.Fatal(err)
	}

	// Decompress stream
	reader := NewBrotliReader(bytes.NewReader(output))
	decoded := make([]byte, 18123)
	var count int
	for {
		read, err := reader.Read(decoded)
		if err != nil {
			if err == io.EOF {
				if read == 0 {
					break
				}
			} else {
				T.Fatal(err)
			}
		}

		count += read
		T.Logf("Read %d/%d bytes\n", count, len(input))
		if count > len(input) {
			T.Error("Too many bytes read from stream without EOF")
			break
		}

		// Force garbage collection
		runtime.GC()
	}
	reader.Close()
}
<|endoftext|>


<commit_before>package contextx

import (
	"context"
	"os"
	"syscall"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestContextCancellationWhenSignalsAreNotified(t *testing.T) {
	assert := assert.New(t)

	testCases := []struct {
		signal      syscall.Signal
		isCanceled  bool
		description string
	}{
		{syscall.SIGHUP, false, "config signal handling does nothing, so context won't be canceled"},
		{syscall.SIGWINCH, false, "this signal is not even handled, same thing than sighup"},
	}

	for _, testCase := range testCases {

		finished := make(chan struct{})

		runner := func() RunnerFunc {
			return func(ctx context.Context) {
				<-ctx.Done()
				close(finished)
			}
		}

		ctx := context.Background()
		c := make(chan os.Signal)

		go signalsAdapter(c).Adapt(runner()).Run(ctx)

		// send the signal
		c <- testCase.signal

		// wait till runner has finished running
		select {
		case <-finished:
			assert.Equal(testCase.isCanceled, true, testCase.description)
		case <-time.After(10 * time.Millisecond):
			assert.Equal(testCase.isCanceled, false, testCase.description)
		}
	}
}
<commit_msg>Fix POSIX signal test<commit_after>package contextx

import (
	"context"
	"os"
	"syscall"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestContextCancellationWhenSignalsAreNotified(t *testing.T) {
	a := assert.New(t)

	testCases := []struct {
		signal      syscall.Signal
		isCanceled  bool
		description string
	}{
		{syscall.SIGINT, true, "SIGINT should cancel the context"},
		{syscall.SIGHUP, false, "config signal handling does nothing, so context won't be canceled"},
		{syscall.SIGWINCH, false, "this signal is not even handled, same thing than sighup"},
	}

	for _, testCase := range testCases {
		finished := make(chan struct{})

		runner := func() RunnerFunc {
			return func(ctx context.Context) {
				<-ctx.Done()
				close(finished)
			}
		}

		ctx := context.Background()
		c := make(chan os.Signal)

		go signalsAdapter(c).Adapt(runner()).Run(ctx)

		// send the signal
		c <- testCase.signal

		// wait till runner has finished running
		select {
		case <-finished:
			a.Equal(testCase.isCanceled, true, testCase.description)
		case <-time.After(10 * time.Millisecond):
			a.Equal(testCase.isCanceled, false, testCase.description)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2018-2019 The grok_exporter Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fswatcher

import (
	"fmt"
	"github.com/fstab/grok_exporter/tailer/glob"
	"github.com/prometheus/common/log"
	"github.com/sirupsen/logrus"
	"io"
	"os"
	"path/filepath"
	"time"
)

type FileTailer interface {
	Lines() chan *Line
	Errors() chan Error
	Close()
}

type Line struct {
	Line  string
	File  string
	Extra interface{}
}

// ideas how this might look like in the config file:
//
// * input section may specify multiple inputs and use globs
//
// * metrics may define filters to specify which files they apply to:
//   - filename_filter: filter file names, like *server1*
//   - filepath_filter: filter path, like /logs/server1/*
// Heads up: filters use globs while matches use regular expressions.
// Moreover, we should provide vars {{.filename}} and {{.filepath}} for labels.

type fileTailer struct {
	globs        []glob.Glob
	watchedDirs  []*Dir
	watchedFiles map[string]*fileWithReader // path -> fileWithReader
	osSpecific   fswatcher
	lines        chan *Line
	errors       chan Error
	done         chan struct{}
}

type fswatcher interface {
	io.Closer
	runFseventProducerLoop() fseventProducerLoop
	processEvent(t *fileTailer, event fsevent, log logrus.FieldLogger) Error
	watchDir(path string) (*Dir, Error)
	unwatchDir(dir *Dir) error
	watchFile(file fileMeta) Error
}

type fseventProducerLoop interface {
	Close()
	Events() chan fsevent
	Errors() chan Error
}

type fsevent interface{}

type fileMeta interface {
	Fd() uintptr
	Name() string
}

func (t *fileTailer) Lines() chan *Line {
	return t.lines
}

func (t *fileTailer) Errors() chan Error {
	return t.errors
}

// Close() triggers the shutdown of the file tailer.
// The file tailer will eventually terminate,
// but after Close() returns it might still be running in the background for a few milliseconds.
func (t *fileTailer) Close() {
	// Closing the done channel will stop the consumer loop.
	// Deferred functions within the consumer loop will close the producer loop.
	close(t.done)
}

func RunFileTailer(globs []glob.Glob, readall bool, failOnMissingFile bool, log logrus.FieldLogger) (FileTailer, error) {
	return runFileTailer(initWatcher, globs, readall, failOnMissingFile, log)
}

func RunPollingFileTailer(globs []glob.Glob, readall bool, failOnMissingFile bool, pollInterval time.Duration, log logrus.FieldLogger) (FileTailer, error) {
	initFunc := func() (fswatcher, Error) {
		return initPollingWatcher(pollInterval)
	}
	return runFileTailer(initFunc, globs, readall, failOnMissingFile, log)
}

func runFileTailer(initFunc func() (fswatcher, Error), globs []glob.Glob, readall bool, failOnMissingFile bool, log logrus.FieldLogger) (FileTailer, error) {

	var (
		t   *fileTailer
		Err Error
	)

	t = &fileTailer{
		globs:        globs,
		watchedFiles: make(map[string]*fileWithReader),
		lines:        make(chan *Line),
		errors:       make(chan Error),
		done:         make(chan struct{}),
	}

	t.osSpecific, Err = initFunc()
	if Err != nil {
		return nil, Err
	}

	go func() {

		defer t.shutdown()

		Err = t.watchDirs(log)
		if Err != nil {
			select {
			case <-t.done:
			case t.errors <- Err:
			}
			return
		}

		eventProducerLoop := t.osSpecific.runFseventProducerLoop()
		defer eventProducerLoop.Close()

		for _, dir := range t.watchedDirs {
			dirLogger := log.WithField("directory", dir.Path())
			dirLogger.Debugf("initializing directory")
			Err = t.syncFilesInDir(dir, readall, dirLogger) // This may already write lines to the lines channel, so we will not go past this line unless the consumer starts reading lines.
			if Err != nil {
				select {
				case <-t.done:
				case t.errors <- Err:
				}
				return
			}
		}

		// make sure at least one logfile was found for each glob
		if failOnMissingFile {
			missingFileError := t.checkMissingFile()
			if missingFileError != nil {
				select {
				case <-t.done:
				case t.errors <- missingFileError:
				}
				return
			}
		}

		for { // event consumer loop
			select {
			case <-t.done:
				return
			case event, open := <-eventProducerLoop.Events():
				if !open {
					return
				}
				processEventError := t.osSpecific.processEvent(t, event, log)
				if processEventError != nil {
					select {
					case <-t.done:
					case t.errors <- processEventError:
					}
					return
				}
			case err, open := <-eventProducerLoop.Errors():
				if !open {
					return
				}
				select {
				case <-t.done:
				case t.errors <- NewError(NotSpecified, err, "error reading file system events"):
				}
				return
			}
		}
	}()
	return t, nil
}

func (t *fileTailer) shutdown() {

	close(t.lines)
	close(t.errors)

	warnf := func(format string, args ...interface{}) {
		log.Warnf("error while shutting down the file system watcher: %v", fmt.Sprintf(format, args))
	}

	for _, dir := range t.watchedDirs {
		err := t.osSpecific.unwatchDir(dir)
		if err != nil {
			warnf("%v", err)
		}
	}

	err := t.osSpecific.Close()
	if err != nil {
		warnf("%v", err)
	}

	for _, file := range t.watchedFiles {
		err = file.file.Close()
		if err != nil {
			warnf("close(%q) failed: %v", file.file.Name(), err)
		}
	}
}

func (t *fileTailer) watchDirs(log logrus.FieldLogger) Error {
	var (
		err      error
		Err      Error
		dirPaths []string
		dirPath  string
	)
	dirPaths, Err = uniqueDirs(t.globs)
	if Err != nil {
		return Err
	}
	for _, dirPath = range dirPaths {
		log.Debugf("watching directory %v", dirPath)
		dir, Err := t.osSpecific.watchDir(dirPath)
		if err != nil {
			return Err
		}
		t.watchedDirs = append(t.watchedDirs, dir)
	}
	return nil
}

func (t *fileTailer) syncFilesInDir(dir *Dir, readall bool, log logrus.FieldLogger) Error {
	watchedFilesAfter := make(map[string]*fileWithReader)
	for path, file := range t.watchedFiles {
		if filepath.Dir(path) != dir.Path() {
			watchedFilesAfter[path] = file
		}
	}
	fileInfos, Err := dir.ls()
	if Err != nil {
		return Err
	}
	for _, fileInfo := range fileInfos {
		filePath := filepath.Join(dir.Path(), fileInfo.Name())
		fileLogger := log.WithField("file", fileInfo.Name())
		if !anyGlobMatches(t.globs, filePath) {
			fileLogger.Debug("skipping file, because file name does not match")
			continue
		}
		if fileInfo.IsDir() {
			fileLogger.Debug("skipping, because it is a directory")
			continue
		}
		alreadyWatched, Err := findSameFile(t, fileInfo, filePath)
		if Err != nil {
			return Err
		}
		if alreadyWatched != nil {
			if alreadyWatched.file.Name() != filePath { // file is already watched but renamed
				renamedFile, err := NewFile(alreadyWatched.file, filePath)
				if err != nil {
					return NewErrorf(NotSpecified, err, "%v: failed to follow moved file", filePath)
				}
				fileLogger.WithField("fd", renamedFile.Fd()).Infof("file with old_fd=%v was moved from old_path=%v", alreadyWatched.file.Fd(), alreadyWatched.file.Name())
				alreadyWatched.file.Close()
				Err = t.osSpecific.watchFile(renamedFile)
				if Err != nil {
					renamedFile.Close()
					return Err
				}
				alreadyWatched.file = renamedFile // re-use lineReader
				Err = t.readNewLines(alreadyWatched, fileLogger)
				if Err != nil {
					alreadyWatched.file.Close()
					return Err
				}
				watchedFilesAfter[filePath] = alreadyWatched
			} else {
				fileLogger.Debug("skipping, because file is already watched")
				watchedFilesAfter[filePath] = alreadyWatched
			}
			continue
		}
		newFile, Err := open(filePath)
		if Err != nil {
			if Err.Type() == FileNotFound {
				fileLogger.Debug("skipping, because file does no longer exist")
				continue
			} else {
				return Err
			}
		}
		if !readall {
			_, err := newFile.Seek(0, io.SeekEnd)
			if err != nil {
				newFile.Close()
				return NewError(NotSpecified, os.NewSyscallError("seek", err), filePath)
			}
		}
		fileLogger = fileLogger.WithField("fd", newFile.Fd())
		fileLogger.Info("watching new file")

		Err = t.osSpecific.watchFile(newFile)
		if Err != nil {
			newFile.Close()
			return Err
		}

		newFileWithReader := &fileWithReader{file: newFile, reader: NewLineReader()}
		Err = t.readNewLines(newFileWithReader, fileLogger)
		if Err != nil {
			newFile.Close()
			return Err
		}
		watchedFilesAfter[filePath] = newFileWithReader
	}
	for _, f := range t.watchedFiles {
		if !contains(watchedFilesAfter, f) {
			fileLogger := log.WithField("file", filepath.Base(f.file.Name())).WithField("fd", f.file.Fd())
			fileLogger.Info("file was removed, closing and un-watching")
			f.file.Close()
		}
	}
	t.watchedFiles = watchedFilesAfter
	return nil
}

func (t *fileTailer) readNewLines(file *fileWithReader, log logrus.FieldLogger) Error {
	var (
		line string
		eof  bool
		err  error
	)
	for {
		line, eof, err = file.reader.ReadLine(file.file)
		if err != nil {
			return NewErrorf(NotSpecified, err, "%v: read() failed", file.file.Name())
		}
		if eof {
			return nil
		}
		log.Debugf("read line %q", line)
		select {
		case <-t.done:
			return nil
		case t.lines <- &Line{Line: line, File: file.file.Name()}:
		}
	}
}

func (t *fileTailer) checkMissingFile() Error {
OUTER:
	for _, g := range t.globs {
		for watchedFileName := range t.watchedFiles {
			if g.Match(watchedFileName) {
				continue OUTER
			}
		}
		// Error message must be phrased so that it makes sense for globs,
		// but also if g is a plain path without wildcards.
		return NewErrorf(FileNotFound, nil, "%v: no such file", g)
	}
	return nil
}

// Gets the directory paths from the glob expressions,
// and makes sure these directories exist.
func uniqueDirs(globs []glob.Glob) ([]string, Error) {
	var (
		result  = make([]string, 0, len(globs))
		g       glob.Glob
		dirInfo os.FileInfo
		err     error
	)
	for _, g = range globs {
		if containsString(result, g.Dir()) {
			continue
		}
		dirInfo, err = os.Stat(g.Dir())
		if err != nil {
			if os.IsNotExist(err) {
				return nil, NewErrorf(DirectoryNotFound, nil, "%q: no such directory", g.Dir())
			}
			return nil, NewErrorf(NotSpecified, err, "%q: stat() failed", g.Dir())
		}
		if !dirInfo.IsDir() {
			return nil, NewErrorf(NotSpecified, nil, "%q is not a directory", g.Dir())
		}
		result = append(result, g.Dir())
	}
	return result, nil
}

func anyGlobMatches(globs []glob.Glob, path string) bool {
	for _, pattern := range globs {
		if pattern.Match(path) {
			return true
		}
	}
	return false
}

func containsString(list []string, s string) bool {
	for _, existing := range list {
		if existing == s {
			return true
		}
	}
	return false
}

func contains(list map[string]*fileWithReader, f *fileWithReader) bool {
	for _, existing := range list {
		if existing == f {
			return true
		}
	}
	return false
}
<commit_msg>#115 error handling when log dir cannot be opened<commit_after>// Copyright 2018-2019 The grok_exporter Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fswatcher

import (
	"fmt"
	"github.com/fstab/grok_exporter/tailer/glob"
	"github.com/prometheus/common/log"
	"github.com/sirupsen/logrus"
	"io"
	"os"
	"path/filepath"
	"time"
)

type FileTailer interface {
	Lines() chan *Line
	Errors() chan Error
	Close()
}

type Line struct {
	Line  string
	File  string
	Extra interface{}
}

// ideas how this might look like in the config file:
//
// * input section may specify multiple inputs and use globs
//
// * metrics may define filters to specify which files they apply to:
//   - filename_filter: filter file names, like *server1*
//   - filepath_filter: filter path, like /logs/server1/*
// Heads up: filters use globs while matches use regular expressions.
// Moreover, we should provide vars {{.filename}} and {{.filepath}} for labels.

type fileTailer struct {
	globs        []glob.Glob
	watchedDirs  []*Dir
	watchedFiles map[string]*fileWithReader // path -> fileWithReader
	osSpecific   fswatcher
	lines        chan *Line
	errors       chan Error
	done         chan struct{}
}

type fswatcher interface {
	io.Closer
	runFseventProducerLoop() fseventProducerLoop
	processEvent(t *fileTailer, event fsevent, log logrus.FieldLogger) Error
	watchDir(path string) (*Dir, Error)
	unwatchDir(dir *Dir) error
	watchFile(file fileMeta) Error
}

type fseventProducerLoop interface {
	Close()
	Events() chan fsevent
	Errors() chan Error
}

type fsevent interface{}

type fileMeta interface {
	Fd() uintptr
	Name() string
}

func (t *fileTailer) Lines() chan *Line {
	return t.lines
}

func (t *fileTailer) Errors() chan Error {
	return t.errors
}

// Close() triggers the shutdown of the file tailer.
// The file tailer will eventually terminate,
// but after Close() returns it might still be running in the background for a few milliseconds.
func (t *fileTailer) Close() {
	// Closing the done channel will stop the consumer loop.
	// Deferred functions within the consumer loop will close the producer loop.
	close(t.done)
}

func RunFileTailer(globs []glob.Glob, readall bool, failOnMissingFile bool, log logrus.FieldLogger) (FileTailer, error) {
	return runFileTailer(initWatcher, globs, readall, failOnMissingFile, log)
}

func RunPollingFileTailer(globs []glob.Glob, readall bool, failOnMissingFile bool, pollInterval time.Duration, log logrus.FieldLogger) (FileTailer, error) {
	initFunc := func() (fswatcher, Error) {
		return initPollingWatcher(pollInterval)
	}
	return runFileTailer(initFunc, globs, readall, failOnMissingFile, log)
}

func runFileTailer(initFunc func() (fswatcher, Error), globs []glob.Glob, readall bool, failOnMissingFile bool, log logrus.FieldLogger) (FileTailer, error) {

	var (
		t   *fileTailer
		Err Error
	)

	t = &fileTailer{
		globs:        globs,
		watchedFiles: make(map[string]*fileWithReader),
		lines:        make(chan *Line),
		errors:       make(chan Error),
		done:         make(chan struct{}),
	}

	t.osSpecific, Err = initFunc()
	if Err != nil {
		return nil, Err
	}

	go func() {

		defer t.shutdown()

		Err = t.watchDirs(log)
		if Err != nil {
			select {
			case <-t.done:
			case t.errors <- Err:
			}
			return
		}

		eventProducerLoop := t.osSpecific.runFseventProducerLoop()
		defer eventProducerLoop.Close()

		for _, dir := range t.watchedDirs {
			dirLogger := log.WithField("directory", dir.Path())
			dirLogger.Debugf("initializing directory")
			Err = t.syncFilesInDir(dir, readall, dirLogger) // This may already write lines to the lines channel, so we will not go past this line unless the consumer starts reading lines.
			if Err != nil {
				select {
				case <-t.done:
				case t.errors <- Err:
				}
				return
			}
		}

		// make sure at least one logfile was found for each glob
		if failOnMissingFile {
			missingFileError := t.checkMissingFile()
			if missingFileError != nil {
				select {
				case <-t.done:
				case t.errors <- missingFileError:
				}
				return
			}
		}

		for { // event consumer loop
			select {
			case <-t.done:
				return
			case event, open := <-eventProducerLoop.Events():
				if !open {
					return
				}
				processEventError := t.osSpecific.processEvent(t, event, log)
				if processEventError != nil {
					select {
					case <-t.done:
					case t.errors <- processEventError:
					}
					return
				}
			case err, open := <-eventProducerLoop.Errors():
				if !open {
					return
				}
				select {
				case <-t.done:
				case t.errors <- NewError(NotSpecified, err, "error reading file system events"):
				}
				return
			}
		}
	}()
	return t, nil
}

func (t *fileTailer) shutdown() {

	close(t.lines)
	close(t.errors)

	warnf := func(format string, args ...interface{}) {
		log.Warnf("error while shutting down the file system watcher: %v", fmt.Sprintf(format, args))
	}

	for _, dir := range t.watchedDirs {
		err := t.osSpecific.unwatchDir(dir)
		if err != nil {
			warnf("%v", err)
		}
	}

	err := t.osSpecific.Close()
	if err != nil {
		warnf("%v", err)
	}

	for _, file := range t.watchedFiles {
		err = file.file.Close()
		if err != nil {
			warnf("close(%q) failed: %v", file.file.Name(), err)
		}
	}
}

func (t *fileTailer) watchDirs(log logrus.FieldLogger) Error {
	var (
		Err      Error
		dirPaths []string
		dirPath  string
	)
	dirPaths, Err = uniqueDirs(t.globs)
	if Err != nil {
		return Err
	}
	for _, dirPath = range dirPaths {
		log.Debugf("watching directory %v", dirPath)
		dir, Err := t.osSpecific.watchDir(dirPath)
		if Err != nil {
			return Err
		}
		t.watchedDirs = append(t.watchedDirs, dir)
	}
	return nil
}

func (t *fileTailer) syncFilesInDir(dir *Dir, readall bool, log logrus.FieldLogger) Error {
	watchedFilesAfter := make(map[string]*fileWithReader)
	for path, file := range t.watchedFiles {
		if filepath.Dir(path) != dir.Path() {
			watchedFilesAfter[path] = file
		}
	}
	fileInfos, Err := dir.ls()
	if Err != nil {
		return Err
	}
	for _, fileInfo := range fileInfos {
		filePath := filepath.Join(dir.Path(), fileInfo.Name())
		fileLogger := log.WithField("file", fileInfo.Name())
		if !anyGlobMatches(t.globs, filePath) {
			fileLogger.Debug("skipping file, because file name does not match")
			continue
		}
		if fileInfo.IsDir() {
			fileLogger.Debug("skipping, because it is a directory")
			continue
		}
		alreadyWatched, Err := findSameFile(t, fileInfo, filePath)
		if Err != nil {
			return Err
		}
		if alreadyWatched != nil {
			if alreadyWatched.file.Name() != filePath { // file is already watched but renamed
				renamedFile, err := NewFile(alreadyWatched.file, filePath)
				if err != nil {
					return NewErrorf(NotSpecified, err, "%v: failed to follow moved file", filePath)
				}
				fileLogger.WithField("fd", renamedFile.Fd()).Infof("file with old_fd=%v was moved from old_path=%v", alreadyWatched.file.Fd(), alreadyWatched.file.Name())
				alreadyWatched.file.Close()
				Err = t.osSpecific.watchFile(renamedFile)
				if Err != nil {
					renamedFile.Close()
					return Err
				}
				alreadyWatched.file = renamedFile // re-use lineReader
				Err = t.readNewLines(alreadyWatched, fileLogger)
				if Err != nil {
					alreadyWatched.file.Close()
					return Err
				}
				watchedFilesAfter[filePath] = alreadyWatched
			} else {
				fileLogger.Debug("skipping, because file is already watched")
				watchedFilesAfter[filePath] = alreadyWatched
			}
			continue
		}
		newFile, Err := open(filePath)
		if Err != nil {
			if Err.Type() == FileNotFound {
				fileLogger.Debug("skipping, because file does no longer exist")
				continue
			} else {
				return Err
			}
		}
		if !readall {
			_, err := newFile.Seek(0, io.SeekEnd)
			if err != nil {
				newFile.Close()
				return NewError(NotSpecified, os.NewSyscallError("seek", err), filePath)
			}
		}
		fileLogger = fileLogger.WithField("fd", newFile.Fd())
		fileLogger.Info("watching new file")

		Err = t.osSpecific.watchFile(newFile)
		if Err != nil {
			newFile.Close()
			return Err
		}

		newFileWithReader := &fileWithReader{file: newFile, reader: NewLineReader()}
		Err = t.readNewLines(newFileWithReader, fileLogger)
		if Err != nil {
			newFile.Close()
			return Err
		}
		watchedFilesAfter[filePath] = newFileWithReader
	}
	for _, f := range t.watchedFiles {
		if !contains(watchedFilesAfter, f) {
			fileLogger := log.WithField("file", filepath.Base(f.file.Name())).WithField("fd", f.file.Fd())
			fileLogger.Info("file was removed, closing and un-watching")
			f.file.Close()
		}
	}
	t.watchedFiles = watchedFilesAfter
	return nil
}

func (t *fileTailer) readNewLines(file *fileWithReader, log logrus.FieldLogger) Error {
	var (
		line string
		eof  bool
		err  error
	)
	for {
		line, eof, err = file.reader.ReadLine(file.file)
		if err != nil {
			return NewErrorf(NotSpecified, err, "%v: read() failed", file.file.Name())
		}
		if eof {
			return nil
		}
		log.Debugf("read line %q", line)
		select {
		case <-t.done:
			return nil
		case t.lines <- &Line{Line: line, File: file.file.Name()}:
		}
	}
}

func (t *fileTailer) checkMissingFile() Error {
OUTER:
	for _, g := range t.globs {
		for watchedFileName := range t.watchedFiles {
			if g.Match(watchedFileName) {
				continue OUTER
			}
		}
		// Error message must be phrased so that it makes sense for globs,
		// but also if g is a plain path without wildcards.
		return NewErrorf(FileNotFound, nil, "%v: no such file", g)
	}
	return nil
}

// Gets the directory paths from the glob expressions,
// and makes sure these directories exist.
func uniqueDirs(globs []glob.Glob) ([]string, Error) {
	var (
		result  = make([]string, 0, len(globs))
		g       glob.Glob
		dirInfo os.FileInfo
		err     error
	)
	for _, g = range globs {
		if containsString(result, g.Dir()) {
			continue
		}
		dirInfo, err = os.Stat(g.Dir())
		if err != nil {
			if os.IsNotExist(err) {
				return nil, NewErrorf(DirectoryNotFound, nil, "%q: no such directory", g.Dir())
			}
			return nil, NewErrorf(NotSpecified, err, "%q: stat() failed", g.Dir())
		}
		if !dirInfo.IsDir() {
			return nil, NewErrorf(NotSpecified, nil, "%q is not a directory", g.Dir())
		}
		result = append(result, g.Dir())
	}
	return result, nil
}

func anyGlobMatches(globs []glob.Glob, path string) bool {
	for _, pattern := range globs {
		if pattern.Match(path) {
			return true
		}
	}
	return false
}

func containsString(list []string, s string) bool {
	for _, existing := range list {
		if existing == s {
			return true
		}
	}
	return false
}

func contains(list map[string]*fileWithReader, f *fileWithReader) bool {
	for _, existing := range list {
		if existing == f {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The go-marathon Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package marathon

// PodVolume describes a volume on the host
type PodVolume struct {
	Name       string            `json:"name,omitempty"`
	Host       string            `json:"host,omitempty"`
	Persistent *PersistentVolume `json:"persistent,omitempty"`
}

// PodVolumeMount describes how to mount a volume into a task
type PodVolumeMount struct {
	Name      string `json:"name,omitempty"`
	MountPath string `json:"mountPath,omitempty"`
	ReadOnly  bool   `json:"readOnly,omitempty"`
}

// NewPodVolume creates a new PodVolume
func NewPodVolume(name, path string) *PodVolume {
	return &PodVolume{
		Name: name,
		Host: path,
	}
}

// NewPodVolumeMount creates a new PodVolumeMount
func NewPodVolumeMount(name, mount string) *PodVolumeMount {
	return &PodVolumeMount{
		Name:      name,
		MountPath: mount,
	}
}

// SetPersistentVolume sets the persistence settings of a PodVolume
func (pv *PodVolume) SetPersistentVolume(p *PersistentVolume) *PodVolume {
	pv.Persistent = p
	return pv
}
<commit_msg>allow file based secrets on pods<commit_after>/*
Copyright 2017 The go-marathon Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package marathon

// PodVolume describes a volume on the host
type PodVolume struct {
	Name       string            `json:"name,omitempty"`
	Host       string            `json:"host,omitempty"`
	Secret     string            `json:"secret,omitempty"`
	Persistent *PersistentVolume `json:"persistent,omitempty"`
}

// PodVolumeMount describes how to mount a volume into a task
type PodVolumeMount struct {
	Name      string `json:"name,omitempty"`
	MountPath string `json:"mountPath,omitempty"`
	ReadOnly  bool   `json:"readOnly,omitempty"`
}

// NewPodVolume creates a new PodVolume
func NewPodVolume(name, path string) *PodVolume {
	return &PodVolume{
		Name: name,
		Host: path,
	}
}

// NewPodVolume creates a new PodVolume for file based secrets
func NewPodVolumeSecret(name, secretPath string) *PodVolume {
	return &PodVolume{
		Name:   name,
		Secret: secretPath,
	}
}

// NewPodVolumeMount creates a new PodVolumeMount
func NewPodVolumeMount(name, mount string) *PodVolumeMount {
	return &PodVolumeMount{
		Name:      name,
		MountPath: mount,
	}
}

// SetPersistentVolume sets the persistence settings of a PodVolume
func (pv *PodVolume) SetPersistentVolume(p *PersistentVolume) *PodVolume {
	pv.Persistent = p
	return pv
}
<|endoftext|>