<commit_before>package eventmaster

import (
	"time"

	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	httpReqLatencies = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "eventmaster",
		Subsystem: "http_server",
		Name:      "request_latency",
		Help:      "Latency of http requests grouped by req path",
		Buckets:   prometheus.ExponentialBuckets(1, 10, 10),
	}, []string{"path"})

	reqLatency = prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Name: "http_request_latency_microseconds",
		Help: "http request duration (microseconds).",
	}, []string{"path"})

	httpReqCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "eventmaster",
		Subsystem: "http_server",
		Name:      "request_total",
		Help:      "The count of http requests received grouped by req path",
	}, []string{"path"})

	httpRespCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "eventmaster",
		Subsystem: "http_server",
		Name:      "response_total",
		Help:      "The count of http responses issued classified by code and api endpoint",
	}, []string{"path", "code"})

	grpcReqLatencies = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "eventmaster",
		Subsystem: "grpc_server",
		Name:      "request_latency",
		Help:      "Latency of grpc requests grouped by method name",
	}, []string{"method"})

	grpcReqCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "eventmaster",
		Subsystem: "grpc_server",
		Name:      "request_total",
		Help:      "The count of grpc requests received grouped by method name",
	}, []string{"method"})

	grpcRespCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "eventmaster",
		Subsystem: "grpc_server",
		Name:      "response_total",
		Help:      "The count of grpc responses issued classified by method name and success",
	}, []string{"method", "success"})

	rsyslogReqLatencies = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "eventmaster",
		Subsystem: "rsyslog_server",
		Name:      "request_latency",
		Help:      "Latency of rsyslog requests",
	}, []string{})

	rsyslogReqCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "eventmaster",
		Subsystem: "rsyslog_server",
		Name:      "request_total",
		Help:      "The count of rsyslog requests received",
	}, []string{})

	eventStoreTimer = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "eventmaster",
		Subsystem: "event_store",
		Name:      "method_time",
		Help:      "Time of event store methods by method name",
	}, []string{"method"})

	eventStoreDbErrCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "eventmaster",
		Subsystem: "event_store",
		Name:      "db_error",
		Help:      "The count of db errors by db name and type of operation",
	}, []string{"db_name", "operation"})
)

// RegisterPromMetrics registers all the metrics that eventmanger uses.
func RegisterPromMetrics() error {
	regErr := prometheus.Register(httpReqLatencies)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			httpReqLatencies = c.ExistingCollector.(*prometheus.HistogramVec)
		} else {
			return regErr
		}
	}

	if err := prometheus.Register(reqLatency); err != nil {
		return errors.Wrap(err, "registering request latency")
	}

	regErr = prometheus.Register(httpReqCounter)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			httpReqCounter = c.ExistingCollector.(*prometheus.CounterVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(httpRespCounter)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			httpRespCounter = c.ExistingCollector.(*prometheus.CounterVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(grpcReqLatencies)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			grpcReqLatencies = c.ExistingCollector.(*prometheus.HistogramVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(grpcReqCounter)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			grpcReqCounter = c.ExistingCollector.(*prometheus.CounterVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(grpcRespCounter)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			grpcRespCounter = c.ExistingCollector.(*prometheus.CounterVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(rsyslogReqLatencies)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			rsyslogReqLatencies = c.ExistingCollector.(*prometheus.HistogramVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(rsyslogReqCounter)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			rsyslogReqCounter = c.ExistingCollector.(*prometheus.CounterVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(eventStoreTimer)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			eventStoreTimer = c.ExistingCollector.(*prometheus.HistogramVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(eventStoreDbErrCounter)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			eventStoreDbErrCounter = c.ExistingCollector.(*prometheus.CounterVec)
		} else {
			return regErr
		}
	}
	return nil
}

// msSince returns milliseconds since start.
func msSince(start time.Time) float64 {
	return float64(time.Since(start) / time.Millisecond)
}
<commit_msg>Specify all latencies in ms<commit_after>package eventmaster

import (
	"time"

	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	httpReqLatencies = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "eventmaster",
		Subsystem: "http_server",
		Name:      "request_latency_ms",
		Help:      "Latency in ms of http requests grouped by req path",
		Buckets:   prometheus.ExponentialBuckets(1, 10, 10),
	}, []string{"path"})

	reqLatency = prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Name: "http_request_latency_ms",
		Help: "http request duration (ms).",
	}, []string{"path"})

	httpReqCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "eventmaster",
		Subsystem: "http_server",
		Name:      "request_total",
		Help:      "The count of http requests received grouped by req path",
	}, []string{"path"})

	httpRespCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "eventmaster",
		Subsystem: "http_server",
		Name:      "response_total",
		Help:      "The count of http responses issued classified by code and api endpoint",
	}, []string{"path", "code"})

	grpcReqLatencies = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "eventmaster",
		Subsystem: "grpc_server",
		Name:      "request_latency",
		Help:      "Latency of grpc requests grouped by method name",
	}, []string{"method"})

	grpcReqCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "eventmaster",
		Subsystem: "grpc_server",
		Name:      "request_total",
		Help:      "The count of grpc requests received grouped by method name",
	}, []string{"method"})

	grpcRespCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "eventmaster",
		Subsystem: "grpc_server",
		Name:      "response_total",
		Help:      "The count of grpc responses issued classified by method name and success",
	}, []string{"method", "success"})

	rsyslogReqLatencies = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "eventmaster",
		Subsystem: "rsyslog_server",
		Name:      "request_latency",
		Help:      "Latency of rsyslog requests",
	}, []string{})

	rsyslogReqCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "eventmaster",
		Subsystem: "rsyslog_server",
		Name:      "request_total",
		Help:      "The count of rsyslog requests received",
	}, []string{})

	eventStoreTimer = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "eventmaster",
		Subsystem: "event_store",
		Name:      "method_time",
		Help:      "Time of event store methods by method name",
	}, []string{"method"})

	eventStoreDbErrCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "eventmaster",
		Subsystem: "event_store",
		Name:      "db_error",
		Help:      "The count of db errors by db name and type of operation",
	}, []string{"db_name", "operation"})
)

// RegisterPromMetrics registers all the metrics that eventmanger uses.
func RegisterPromMetrics() error {
	regErr := prometheus.Register(httpReqLatencies)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			httpReqLatencies = c.ExistingCollector.(*prometheus.HistogramVec)
		} else {
			return regErr
		}
	}

	if err := prometheus.Register(reqLatency); err != nil {
		return errors.Wrap(err, "registering request latency")
	}

	regErr = prometheus.Register(httpReqCounter)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			httpReqCounter = c.ExistingCollector.(*prometheus.CounterVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(httpRespCounter)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			httpRespCounter = c.ExistingCollector.(*prometheus.CounterVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(grpcReqLatencies)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			grpcReqLatencies = c.ExistingCollector.(*prometheus.HistogramVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(grpcReqCounter)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			grpcReqCounter = c.ExistingCollector.(*prometheus.CounterVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(grpcRespCounter)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			grpcRespCounter = c.ExistingCollector.(*prometheus.CounterVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(rsyslogReqLatencies)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			rsyslogReqLatencies = c.ExistingCollector.(*prometheus.HistogramVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(rsyslogReqCounter)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			rsyslogReqCounter = c.ExistingCollector.(*prometheus.CounterVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(eventStoreTimer)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			eventStoreTimer = c.ExistingCollector.(*prometheus.HistogramVec)
		} else {
			return regErr
		}
	}

	regErr = prometheus.Register(eventStoreDbErrCounter)
	if regErr != nil {
		if c, ok := regErr.(prometheus.AlreadyRegisteredError); ok {
			eventStoreDbErrCounter = c.ExistingCollector.(*prometheus.CounterVec)
		} else {
			return regErr
		}
	}
	return nil
}

// msSince returns milliseconds since start.
func msSince(start time.Time) float64 {
	return float64(time.Since(start) / time.Millisecond)
}
<|endoftext|>


<commit_before>package elastilog4g

import (
	"errors"
	"github.com/dspasibenko/log4g"
	elastigo "github.com/mattbaird/elastigo/lib"
	"strconv"
	"strings"
)

// log4g the appender registration name
const esAppenderName = "log4g/elastilog4g"

// retry - specifies number of seconds to repeat indexing attempt in case of error
// the parameter is OPTIONAL, default value is 1 second.
const ESAParamRetry = "retry"

// index - specifies ES index name. Must be provided
const ESAParamIndexName = "index"

// index - specifies ES type name. Must be provided
const ESAParamTypeName = "_type"

// Specifies ES hosts the appender will connect to. Should be comma separated like:
// "192.168.1.1, 192.168.1.2"
// the parameter is OPTIONAL, default value is localhost
const ESAParamHosts = "hosts"

// Specifies ES port the appender will connect to
// the parameter is OPTIONAL, default value is 9200
const ESAParamPort = "port"

// Specifies the record TTL
// the parameter is OPTIONAL, default value is ""
const ESAParamTTL = "ttl"

type esAppender struct {
	conn     *elastigo.Conn
	indexer  *elastigo.BulkIndexer
	index    string
	typeName string
	ttl      string
}

type esAppenderFactory struct {
}

func Init() error {
	return log4g.RegisterAppender(&esAppenderFactory{})
}

// -------- Factory functions --------

func (*esAppenderFactory) Name() string {
	return esAppenderName
}

func (*esAppenderFactory) NewAppender(params map[string]string) (log4g.Appender, error) {
	retrySec, err := log4g.ParseInt(params[ESAParamRetry], 0, 60, 1)
	if err != nil {
		return nil, errors.New("Invalid " + ESAParamRetry + " value: " + err.Error())
	}

	index := strings.Trim(params[ESAParamIndexName], " ")
	if len(index) == 0 {
		return nil, errors.New("Mandatory appender index name setting should be provided")
	}

	typeName := strings.Trim(params[ESAParamTypeName], " ")
	if len(typeName) == 0 {
		return nil, errors.New("Mandatory appender index type setting should be provided")
	}

	hosts := strings.Split(params[ESAParamHosts], ",")
	for idx, host := range hosts {
		hosts[idx] = strings.Trim(host, " ")
	}

	port, err := log4g.ParseInt(params[ESAParamPort], 1000, 65535, 9200)
	if err != nil {
		return nil, errors.New("Invalid " + ESAParamPort + " value: " + err.Error())
	}

	ttl := params[ESAParamTTL]

	conn := elastigo.NewConn()
	if len(hosts) > 0 {
		conn.ClusterDomains = hosts
	}
	conn.Port = strconv.Itoa(port)
	esa := &esAppender{conn, conn.NewBulkIndexerErrors(1, int(retrySec)), index, typeName, ttl}
	esa.indexer.Start()

	return esa, nil
}

func (f *esAppenderFactory) Shutdown() {
}

// -------- Appender functions --------
func (esa *esAppender) Append(ev *log4g.LogEvent) (ok bool) {
	ok = false
	defer log4g.EndQuietly()
	esa.indexer.Index(esa.index, esa.typeName, "", esa.ttl, &ev.Timestamp, ev, false)
	ok = true
	return ok
}

func (esa *esAppender) Shutdown() {
	esa.indexer.Stop()
	esa.conn.Flush()
}
<commit_msg>switch to jrivets<commit_after>package elastilog4g

import (
	"errors"
	"github.com/jrivets/log4g"
	elastigo "github.com/mattbaird/elastigo/lib"
	"strconv"
	"strings"
)

// log4g the appender registration name
const esAppenderName = "log4g/elastilog4g"

// retry - specifies number of seconds to repeat indexing attempt in case of error
// the parameter is OPTIONAL, default value is 1 second.
const ESAParamRetry = "retry"

// index - specifies ES index name. Must be provided
const ESAParamIndexName = "index"

// index - specifies ES type name. Must be provided
const ESAParamTypeName = "_type"

// Specifies ES hosts the appender will connect to. Should be comma separated like:
// "192.168.1.1, 192.168.1.2"
// the parameter is OPTIONAL, default value is localhost
const ESAParamHosts = "hosts"

// Specifies ES port the appender will connect to
// the parameter is OPTIONAL, default value is 9200
const ESAParamPort = "port"

// Specifies the record TTL
// the parameter is OPTIONAL, default value is ""
const ESAParamTTL = "ttl"

type esAppender struct {
	conn     *elastigo.Conn
	indexer  *elastigo.BulkIndexer
	index    string
	typeName string
	ttl      string
}

type esAppenderFactory struct {
}

func Init() error {
	return log4g.RegisterAppender(&esAppenderFactory{})
}

// -------- Factory functions --------

func (*esAppenderFactory) Name() string {
	return esAppenderName
}

func (*esAppenderFactory) NewAppender(params map[string]string) (log4g.Appender, error) {
	retrySec, err := log4g.ParseInt(params[ESAParamRetry], 0, 60, 1)
	if err != nil {
		return nil, errors.New("Invalid " + ESAParamRetry + " value: " + err.Error())
	}

	index := strings.Trim(params[ESAParamIndexName], " ")
	if len(index) == 0 {
		return nil, errors.New("Mandatory appender index name setting should be provided")
	}

	typeName := strings.Trim(params[ESAParamTypeName], " ")
	if len(typeName) == 0 {
		return nil, errors.New("Mandatory appender index type setting should be provided")
	}

	hosts := strings.Split(params[ESAParamHosts], ",")
	for idx, host := range hosts {
		hosts[idx] = strings.Trim(host, " ")
	}

	port, err := log4g.ParseInt(params[ESAParamPort], 1000, 65535, 9200)
	if err != nil {
		return nil, errors.New("Invalid " + ESAParamPort + " value: " + err.Error())
	}

	ttl := params[ESAParamTTL]

	conn := elastigo.NewConn()
	if len(hosts) > 0 {
		conn.ClusterDomains = hosts
	}
	conn.Port = strconv.Itoa(port)
	esa := &esAppender{conn, conn.NewBulkIndexerErrors(1, int(retrySec)), index, typeName, ttl}
	esa.indexer.Start()

	return esa, nil
}

func (f *esAppenderFactory) Shutdown() {
}

// -------- Appender functions --------
func (esa *esAppender) Append(ev *log4g.LogEvent) (ok bool) {
	ok = false
	defer log4g.EndQuietly()
	esa.indexer.Index(esa.index, esa.typeName, "", esa.ttl, &ev.Timestamp, ev, false)
	ok = true
	return ok
}

func (esa *esAppender) Shutdown() {
	esa.indexer.Stop()
	esa.conn.Flush()
}
<|endoftext|>


<commit_before>package semver

import "strings"

func containsOnly(s string, set string) bool {
	return strings.IndexFunc(s, func(r rune) bool {
		return !strings.ContainsRune(set, r)
	}) == -1
}

func hasLeadingZero(number string) bool {
	if len(number) > 1 {
		if strings.HasPrefix(number, "0") {
			return true
		}
	}
	return false
}

func gt(main, other *Version) bool {
	return main.Compare(other) > 0
}

func gte(main, other *Version) bool {
	return main.Compare(other) >= 0
}

func lt(main, other *Version) bool {
	return main.Compare(other) < 0
}

func lte(main, other *Version) bool {
	return main.Compare(other) <= 0
}

func eq(main, other *Version) bool {
	return main.Compare(other) == 0
}

func rng(main, first, second *Version) bool {
	return false
}
<commit_msg>Added range check helper.<commit_after>package semver

import "strings"

func containsOnly(s string, set string) bool {
	return strings.IndexFunc(s, func(r rune) bool {
		return !strings.ContainsRune(set, r)
	}) == -1
}

func hasLeadingZero(number string) bool {
	if len(number) > 1 {
		if strings.HasPrefix(number, "0") {
			return true
		}
	}
	return false
}

func gt(main, other *Version) bool {
	return main.Compare(other) > 0
}

func gte(main, other *Version) bool {
	return main.Compare(other) >= 0
}

func lt(main, other *Version) bool {
	return main.Compare(other) < 0
}

func lte(main, other *Version) bool {
	return main.Compare(other) <= 0
}

func eq(main, other *Version) bool {
	return main.Compare(other) == 0
}

func rng(main, first, second *Version) bool {
	return gte(main, first) && lte(main, second)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"strings"
	"text/template"

	"github.com/Masterminds/sprig"
	"github.com/huandu/xstrings"

	"github.com/golang/protobuf/protoc-gen-go/descriptor"
)

var ProtoHelpersFuncMap = template.FuncMap{
	"string": func(i interface {
		String() string
	}) string {
		return i.String()
	},
	"json": func(v interface{}) string {
		a, _ := json.Marshal(v)
		return string(a)
	},
	"prettyjson": func(v interface{}) string {
		a, _ := json.MarshalIndent(v, "", "  ")
		return string(a)
	},
	"splitArray": func(sep string, s string) []string {
		return strings.Split(s, sep)
	},
	"first": func(a []string) string {
		return a[0]
	},
	"last": func(a []string) string {
		return a[len(a)-1]
	},
	"upperFirst": func(s string) string {
		return strings.ToUpper(s[:1]) + s[1:]
	},
	"lowerFirst": func(s string) string {
		return strings.ToLower(s[:1]) + s[1:]
	},
	"camelCase": func(s string) string {
		return xstrings.ToCamelCase(s)
	},
	"lowerCamelCase": func(s string) string {
		cc := xstrings.ToCamelCase(s)
		return strings.ToLower(cc[:1]) + cc[1:]
	},
	"snakeCase": func(s string) string {
		return xstrings.ToSnakeCase(s)
	},
	"kebabCase": func(s string) string {
		return strings.Replace(xstrings.ToSnakeCase(s), "_", "-", -1)
	},
	"getMessageType": getMessageType,
	"isFieldMessage": isFieldMessage,
}

func init() {
	for k, v := range sprig.TxtFuncMap() {
		ProtoHelpersFuncMap[k] = v
	}
}

func getMessageType(f *descriptor.FileDescriptorProto, name string) *descriptor.DescriptorProto {
	for _, m := range f.MessageType {
		// name usually contains the package name
		if strings.HasSuffix(name, *m.Name) {
			return m
		}
	}

	return nil
}

func isFieldMessage(f *descriptor.FieldDescriptorProto) bool {
	if f.Type != nil && *f.Type == descriptor.FieldDescriptorProto_TYPE_MESSAGE {
		return true
	}
	if f.TypeName != nil && (*f.TypeName)[0] == '.' {
		return true
	}

	return false
}
<commit_msg>fix (helper): fix isFieldMessage<commit_after>package main

import (
	"encoding/json"
	"strings"
	"text/template"

	"github.com/Masterminds/sprig"
	"github.com/huandu/xstrings"

	"github.com/golang/protobuf/protoc-gen-go/descriptor"
)

var ProtoHelpersFuncMap = template.FuncMap{
	"string": func(i interface {
		String() string
	}) string {
		return i.String()
	},
	"json": func(v interface{}) string {
		a, _ := json.Marshal(v)
		return string(a)
	},
	"prettyjson": func(v interface{}) string {
		a, _ := json.MarshalIndent(v, "", "  ")
		return string(a)
	},
	"splitArray": func(sep string, s string) []string {
		return strings.Split(s, sep)
	},
	"first": func(a []string) string {
		return a[0]
	},
	"last": func(a []string) string {
		return a[len(a)-1]
	},
	"upperFirst": func(s string) string {
		return strings.ToUpper(s[:1]) + s[1:]
	},
	"lowerFirst": func(s string) string {
		return strings.ToLower(s[:1]) + s[1:]
	},
	"camelCase": func(s string) string {
		return xstrings.ToCamelCase(s)
	},
	"lowerCamelCase": func(s string) string {
		cc := xstrings.ToCamelCase(s)
		return strings.ToLower(cc[:1]) + cc[1:]
	},
	"snakeCase": func(s string) string {
		return xstrings.ToSnakeCase(s)
	},
	"kebabCase": func(s string) string {
		return strings.Replace(xstrings.ToSnakeCase(s), "_", "-", -1)
	},
	"getMessageType": getMessageType,
	"isFieldMessage": isFieldMessage,
}

func init() {
	for k, v := range sprig.TxtFuncMap() {
		ProtoHelpersFuncMap[k] = v
	}
}

func getMessageType(f *descriptor.FileDescriptorProto, name string) *descriptor.DescriptorProto {
	for _, m := range f.MessageType {
		// name usually contains the package name
		if strings.HasSuffix(name, *m.Name) {
			return m
		}
	}

	return nil
}

func isFieldMessage(f *descriptor.FieldDescriptorProto) bool {
	if f.Type != nil && *f.Type == descriptor.FieldDescriptorProto_TYPE_MESSAGE &&
		f.Label != nil && *f.Label != descriptor.FieldDescriptorProto_LABEL_REPEATED {
		return true
	}

	return false
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2017, AverageSecurityGuy
# All rights reserved.

Demonstrate function usage in Go

Usage:

$ go run functions.go string
*/

package main

import (
	"fmt"
	"os"
)

// Declare a function that takes two parameters and does not return a value.
// The variable is declared before the type. Variables of the same type can be
// declared at the same time before stating the type:
// func some_func(a, b, c string, i, j int)
func write_string(s string, n int) {
	for i := 0; i < n; i++ {
		fmt.Println(s)
	}
}

// Declare a function that takes a string and returns a boolean value.
func short_string(s string) bool {
	return len(s) < 5
}

// Declare a function that returns two values. You can return as many value as
// needed by placing them in the comma delimited return list.
func test_string(s string) (int, bool) {
	l := len(s)
	return l, l < 5
}

func main() {

	// Check the number of arguments we have. The name of the script is the
	// first argument.
	if len(os.Args) != 2 {

		// If we don't have the correct number of arguments then print a
		// message. Println will automatically add a newline at the end of the
		// string. Always use double quotes for strings, single quotes have a
		// different meaning, which we are not going to discuss.
		fmt.Println("Usage: go run functions.go string")

		// Exit the program
		os.Exit(1)
	}

	// Declare and assign a variable. The alternative is to do it in two steps:
	// var str string
	// str = os.Args[1]
	str := os.Args[1]

	// Call a function with two parameters.
	write_string(str, 10)

	// Call a function and catch the return value.
	ans := short_string(str)
	if ans == true {
		fmt.Println("String is short")
	} else {
		fmt.Println("String is not short")
	}

	// Call a function and get both return values. If you don't need all of the
	// return values from a function, you can throw away the return values you
	// do not want or need. using an underscore:
	// _, b := test_string(str)
	n, b := test_string(str)
	if b == true {
		fmt.Printf("The string is short: %d characters.\n", n)
	} else {
		fmt.Printf("The string is not short: %d characters.\n", n)
	}
}
<commit_msg>Day 3<commit_after>/*
Copyright (c) 2017, AverageSecurityGuy
# All rights reserved.

Demonstrate function usage in Go

Usage:

$ go run functions.go string
*/

package main

import (
	"fmt"
	"os"
)

// Declare a function that takes two parameters and does not return a value.
// The variable is declared before the type. Variables of the same type can be
// declared at the same time before stating the type:
// func some_func(a, b, c string, i, j int)
func write_string(s string, n int) {
	for i := 0; i < n; i++ {
		fmt.Println(s)
	}
}

// Declare a function that takes a string and returns a boolean value.
func short_string(s string) bool {
	return len(s) < 5
}

// Declare a function that returns two values. You can return as many value as
// needed by placing them in the comma delimited return list.
func test_string(s string) (int, bool) {
	l := len(s)
	return l, l < 5
}

func main() {

	// Check the number of arguments we have. The name of the script is the
	// first argument.
	if len(os.Args) != 2 {

		// If we don't have the correct number of arguments then print a
		// message. Println will automatically add a newline at the end of the
		// string. Always use double quotes for strings, single quotes have a
		// different meaning, which we are not going to discuss.
		fmt.Println("Usage: go run functions.go string")

		// Exit the program
		os.Exit(1)
	}

	// Declare and assign a variable. The alternative is to do it in two steps:
	// var str string
	// str = os.Args[1]
	str := os.Args[1]

	// Call a function with two parameters.
	write_string(str, 10)

	// Call a function and catch the return value.
	ans := short_string(str)
	if ans == true {
		fmt.Println("String is short")
	} else {
		fmt.Println("String is not short")
	}

	// Call a function and get both return values. If you don't need all of the
	// return values from a function, you can throw away the return values you
	// do not want or need. using an underscore:
	// _, b := test_string(str)
	n, b := test_string(str)
	if b == true {
		fmt.Printf("The string is short: %d characters.\n", n)
	} else {
		fmt.Printf("The string is not short: %d characters.\n", n)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"math/big"
	"net/http"
	"os"
	"path"
	"strings"

	"archive/tar"
	"crypto/rand"

	"github.com/fsouza/go-dockerclient"
	"golang.org/x/net/webdav"
)

const (
	VERSION_TAG        = "v1"
	DOCKER_TAR_PREFIX  = "rootfs/"
	OWNER_PERM_RW      = 0600
	HEALTHZ_URL_PATH   = "/healthz"
	CONTENT_URL_PREFIX = "/api/" + VERSION_TAG + "/content/"
	METADATA_URL_PATH  = "/api/" + VERSION_TAG + "/metadata"
)

func handleTarStream(reader io.ReadCloser, destination string) {
	tr := tar.NewReader(reader)
	if tr != nil {
		err := processTarStream(tr, destination)
		if err != nil {
			log.Print(err)
		}
	} else {
		log.Printf("Unable to create image tar reader")
	}
	reader.Close()
}

func processTarStream(tr *tar.Reader, destination string) error {
	for {
		hdr, err := tr.Next()
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return fmt.Errorf("Unable to extract container: %v\n", err)
		}

		hdrInfo := hdr.FileInfo()

		path := path.Join(destination, strings.TrimPrefix(hdr.Name, DOCKER_TAR_PREFIX))
		// Overriding permissions to allow writing content
		mode := hdrInfo.Mode() | OWNER_PERM_RW

		switch hdr.Typeflag {
		case tar.TypeDir:
			if err := os.Mkdir(path, mode); err != nil {
				if !os.IsExist(err) {
					return fmt.Errorf("Unable to create directory: %v", err)
				}
				err = os.Chmod(path, mode)
				if err != nil {
					return fmt.Errorf("Unable to update directory mode: %v", err)
				}
			}
		case tar.TypeReg, tar.TypeRegA:
			file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
			if err != nil {
				return fmt.Errorf("Unable to create file: %v", err)
			}
			if _, err := io.Copy(file, tr); err != nil {
				file.Close()
				return fmt.Errorf("Unable to write into file: %v", err)
			}
			file.Close()
		default:
			// For now we're skipping anything else. Special device files and
			// symlinks are not needed or anyway probably incorrect.
		}
	}
}

func generateRandomName() string {
	n, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		log.Fatalf("Unable to generate random container name: %v\n", err)
	}
	return fmt.Sprintf("image-inspector-%016x", n)
}

func main() {
	uri := flag.String("docker", "unix:///var/run/docker.sock", "Daemon socket to connect to")
	image := flag.String("image", "", "Docker image to inspect")
	path := flag.String("path", "", "Destination path for the image files")
	serve := flag.String("serve", "", "Host and port where to serve the image with webdav")

	flag.Parse()

	if *uri == "" {
		log.Fatalf("Docker socket connection must be specified")
	}
	if *image == "" {
		log.Fatalf("Docker image to inspect must be specified")
	}

	client, err := docker.NewClient(*uri)
	if err != nil {
		log.Fatalf("Unable to connect to docker daemon: %v\n", err)
	}

	if _, err := client.InspectImage(*image); err != nil {
		log.Printf("Pulling image %s", *image)
		imagePullOption := docker.PullImageOptions{Repository: *image}
		imagePullAuth := docker.AuthConfiguration{} // TODO: support authentication
		if err := client.PullImage(imagePullOption, imagePullAuth); err != nil {
			log.Fatalf("Unable to pull docker image: %v\n", err)
		}
	} else {
		log.Printf("Image %s is available, skipping image pull", *image)
	}

	// For security purpose we don't define any entrypoint and command
	container, err := client.CreateContainer(docker.CreateContainerOptions{
		Name: generateRandomName(),
		Config: &docker.Config{
			Image:      *image,
			Entrypoint: []string{""},
			Cmd:        []string{""},
		},
	})
	if err != nil {
		log.Fatalf("Unable to create docker container: %v\n", err)
	}

	containerMetadata, err := client.InspectContainer(container.ID)
	if err != nil {
		log.Fatalf("Unable to get docker container information: %v\n", err)
	}

	imageMetadata, err := client.InspectImage(containerMetadata.Image)
	if err != nil {
		log.Fatalf("Unable to get docker image information: %v\n", err)
	}

	if path != nil && *path != "" {
		err = os.Mkdir(*path, 0755)
		if err != nil {
			if !os.IsExist(err) {
				log.Fatalf("Unable to create destination path: %v\n", err)
			}
		}
	} else {
		// forcing to use /var/tmp because often it's not an in-memory tmpfs
		*path, err = ioutil.TempDir("/var/tmp", "image-inspector-")
		if err != nil {
			log.Fatalf("Unable to create temporary path: %v\n", err)
		}
	}

	reader, writer := io.Pipe()
	go handleTarStream(reader, *path)

	log.Printf("Extracting image %s to %s", *image, *path)
	_ = client.CopyFromContainer(docker.CopyFromContainerOptions{
		Container:    container.ID,
		OutputStream: writer,
		Resource:     "/",
	})

	_ = client.RemoveContainer(docker.RemoveContainerOptions{
		ID: container.ID,
	})

	if serve != nil && *serve != "" {
		log.Printf("Serving image content %s on webdav://%s%s", *path, *serve, CONTENT_URL_PREFIX)

		http.Handle(CONTENT_URL_PREFIX, &webdav.Handler{
			Prefix:     CONTENT_URL_PREFIX,
			FileSystem: webdav.Dir(*path),
			LockSystem: webdav.NewMemLS(),
		})

		http.HandleFunc(HEALTHZ_URL_PATH, func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte("ok\n"))
		})

		http.HandleFunc(METADATA_URL_PATH, func(w http.ResponseWriter, r *http.Request) {
			body, err := json.Marshal(imageMetadata)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			w.Write(body)
		})

		log.Fatal(http.ListenAndServe(*serve, nil))
	}
}
<commit_msg>server: reorder handlers by function<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"math/big"
	"net/http"
	"os"
	"path"
	"strings"

	"archive/tar"
	"crypto/rand"

	"github.com/fsouza/go-dockerclient"
	"golang.org/x/net/webdav"
)

const (
	VERSION_TAG        = "v1"
	DOCKER_TAR_PREFIX  = "rootfs/"
	OWNER_PERM_RW      = 0600
	HEALTHZ_URL_PATH   = "/healthz"
	CONTENT_URL_PREFIX = "/api/" + VERSION_TAG + "/content/"
	METADATA_URL_PATH  = "/api/" + VERSION_TAG + "/metadata"
)

func handleTarStream(reader io.ReadCloser, destination string) {
	tr := tar.NewReader(reader)
	if tr != nil {
		err := processTarStream(tr, destination)
		if err != nil {
			log.Print(err)
		}
	} else {
		log.Printf("Unable to create image tar reader")
	}
	reader.Close()
}

func processTarStream(tr *tar.Reader, destination string) error {
	for {
		hdr, err := tr.Next()
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return fmt.Errorf("Unable to extract container: %v\n", err)
		}

		hdrInfo := hdr.FileInfo()

		path := path.Join(destination, strings.TrimPrefix(hdr.Name, DOCKER_TAR_PREFIX))
		// Overriding permissions to allow writing content
		mode := hdrInfo.Mode() | OWNER_PERM_RW

		switch hdr.Typeflag {
		case tar.TypeDir:
			if err := os.Mkdir(path, mode); err != nil {
				if !os.IsExist(err) {
					return fmt.Errorf("Unable to create directory: %v", err)
				}
				err = os.Chmod(path, mode)
				if err != nil {
					return fmt.Errorf("Unable to update directory mode: %v", err)
				}
			}
		case tar.TypeReg, tar.TypeRegA:
			file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
			if err != nil {
				return fmt.Errorf("Unable to create file: %v", err)
			}
			if _, err := io.Copy(file, tr); err != nil {
				file.Close()
				return fmt.Errorf("Unable to write into file: %v", err)
			}
			file.Close()
		default:
			// For now we're skipping anything else. Special device files and
			// symlinks are not needed or anyway probably incorrect.
		}
	}
}

func generateRandomName() string {
	n, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		log.Fatalf("Unable to generate random container name: %v\n", err)
	}
	return fmt.Sprintf("image-inspector-%016x", n)
}

func main() {
	uri := flag.String("docker", "unix:///var/run/docker.sock", "Daemon socket to connect to")
	image := flag.String("image", "", "Docker image to inspect")
	path := flag.String("path", "", "Destination path for the image files")
	serve := flag.String("serve", "", "Host and port where to serve the image with webdav")

	flag.Parse()

	if *uri == "" {
		log.Fatalf("Docker socket connection must be specified")
	}
	if *image == "" {
		log.Fatalf("Docker image to inspect must be specified")
	}

	client, err := docker.NewClient(*uri)
	if err != nil {
		log.Fatalf("Unable to connect to docker daemon: %v\n", err)
	}

	if _, err := client.InspectImage(*image); err != nil {
		log.Printf("Pulling image %s", *image)
		imagePullOption := docker.PullImageOptions{Repository: *image}
		imagePullAuth := docker.AuthConfiguration{} // TODO: support authentication
		if err := client.PullImage(imagePullOption, imagePullAuth); err != nil {
			log.Fatalf("Unable to pull docker image: %v\n", err)
		}
	} else {
		log.Printf("Image %s is available, skipping image pull", *image)
	}

	// For security purpose we don't define any entrypoint and command
	container, err := client.CreateContainer(docker.CreateContainerOptions{
		Name: generateRandomName(),
		Config: &docker.Config{
			Image:      *image,
			Entrypoint: []string{""},
			Cmd:        []string{""},
		},
	})
	if err != nil {
		log.Fatalf("Unable to create docker container: %v\n", err)
	}

	containerMetadata, err := client.InspectContainer(container.ID)
	if err != nil {
		log.Fatalf("Unable to get docker container information: %v\n", err)
	}

	imageMetadata, err := client.InspectImage(containerMetadata.Image)
	if err != nil {
		log.Fatalf("Unable to get docker image information: %v\n", err)
	}

	if path != nil && *path != "" {
		err = os.Mkdir(*path, 0755)
		if err != nil {
			if !os.IsExist(err) {
				log.Fatalf("Unable to create destination path: %v\n", err)
			}
		}
	} else {
		// forcing to use /var/tmp because often it's not an in-memory tmpfs
		*path, err = ioutil.TempDir("/var/tmp", "image-inspector-")
		if err != nil {
			log.Fatalf("Unable to create temporary path: %v\n", err)
		}
	}

	reader, writer := io.Pipe()
	go handleTarStream(reader, *path)

	log.Printf("Extracting image %s to %s", *image, *path)
	_ = client.CopyFromContainer(docker.CopyFromContainerOptions{
		Container:    container.ID,
		OutputStream: writer,
		Resource:     "/",
	})

	_ = client.RemoveContainer(docker.RemoveContainerOptions{
		ID: container.ID,
	})

	if serve != nil && *serve != "" {
		log.Printf("Serving image content %s on webdav://%s%s", *path, *serve, CONTENT_URL_PREFIX)

		http.HandleFunc(HEALTHZ_URL_PATH, func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte("ok\n"))
		})

		http.HandleFunc(METADATA_URL_PATH, func(w http.ResponseWriter, r *http.Request) {
			body, err := json.Marshal(imageMetadata)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			w.Write(body)
		})

		http.Handle(CONTENT_URL_PREFIX, &webdav.Handler{
			Prefix:     CONTENT_URL_PREFIX,
			FileSystem: webdav.Dir(*path),
			LockSystem: webdav.NewMemLS(),
		})

		log.Fatal(http.ListenAndServe(*serve, nil))
	}
}
<|endoftext|>


<commit_before>package hostess

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"sort"
	"strings"
)

const default_osx = `
##
# Host Database
#
# localhost is used to configure the loopback interface
# when the system is booting.  Do not change this entry.
##

127.0.0.1       localhost
255.255.255.255 broadcasthost
::1             localhost
fe80::1%lo0     localhost
`

const default_linux = `
127.0.0.1   localhost
127.0.1.1   HOSTNAME

# The following lines are desirable for IPv6 capable hosts
::1     localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts
`

type Hostname struct {
	Domain  string
	Ip      string
	Enabled bool
	// Ipv6    bool
}

func (h *Hostname) Format() string {
	r := fmt.Sprintf("%s %s", h.Ip, h.Domain)
	if !h.Enabled {
		r = "# " + r
	}
	return r
}

// Hostfile represents /etc/hosts (or a similar file, depending on OS), and
// includes a list of Hostnames. Hostfile includes
type Hostfile struct {
	Path  string
	Hosts map[string]*Hostname
	data  string
}

func TrimWS(s string) string {
	return strings.Trim(s, " \n\t")
}

// NewHostFile creates a new Hostfile object from the specified file.
func NewHostfile(path string) *Hostfile {
	return &Hostfile{path, make(map[string]*Hostname), ""}
}

func (h *Hostfile) ReadFile(path string) string {
	data, err := ioutil.ReadFile(h.Path)
	if err != nil {
		fmt.Println("Can't read ", h.Path)
		os.Exit(1)
	}
	h.data = string(data)
	return h.data
}

var line_parser = regexp.MustCompile(``)

func parseLine(line string) (Hostname, error) {
	// 1. Split on # to discard comments.
	// 2. Split on first space to find the IP
	// 3. Split remainder of line on whitespace to find
	//    domain names
	// 4. Validate the IP (maybe -- could be ipv4 or ipv6)
	hostname := Hostname{}
	if false {
		return hostname, errors.New("Can't parse hostname")
	}
	return hostname, nil
}

func (h *Hostfile) Read(hostfile string) []Hostname {
	var hosts = make([]Hostname, 0)
	return hosts
}

func getSortedMapKeys(m map[string][]string) []string {
	keys := make([]string, len(m))
	i := 0
	for k := range m {
		keys[i] = k
		i += 1
	}
	sort.Strings(keys)
	return keys
}

func (h *Hostfile) ListDomainsByIp(ip string) []string {
	names := make([]string, 0)
	for _, v := range h.Hosts {
		if v.Ip == ip {
			names = append(names, v.Domain)
		}
	}
	sort.Strings(names)

	// Magic for localhost only, to make sure it's the first domain on its line
	if ip == "127.0.0.1" {
		for k, v := range names {
			if v == "localhost" {
				names = append(names[:k], names[k+1:]...)
			}
		}
		names = append([]string{"localhost"}, names...)
	}

	return names
}

// Format takes the current list of Hostnames in this Hostfile and turns it
// into a string suitable for use as an /etc/hosts file.
// Sorting uses the following logic:
// 1. List is sorted by IP address
// 2. Commented items are left in place
// 3. 127.* appears at the top of the list (so boot resolvers don't break)
// 4. When present, localhost will always appear first in the domain list
func (h *Hostfile) Format() string {
	// localhost := "127.0.0.1 localhost"

	localhosts := make(map[string][]string)
	ips := make(map[string][]string)

	// Map domains and IPs into slices of domains keyd by IP
	// 127.0.0.1 = [localhost, blah, blah2]
	// 2.2.2.3 = [domain1, domain2]
	for _, hostname := range h.Hosts {
		if hostname.Ip[0:4] == "127." {
			localhosts[hostname.Ip] = append(localhosts[hostname.Ip], hostname.Domain)
		} else {
			ips[hostname.Ip] = append(ips[hostname.Ip], hostname.Domain)
		}
	}

	localhosts_keys := getSortedMapKeys(localhosts)
	ips_keys := getSortedMapKeys(ips)
	out := make([]string, 0)

	for _, ip := range localhosts_keys {
		enabled := ip
		enabled_b := false
		disabled := "# " + ip
		disabled_b := false
		for _, hostname := range h.Hosts {
			if hostname.Ip == ip {
				if hostname.Enabled {
					enabled += " " + hostname.Domain
					enabled_b = true
				} else {
					disabled += " " + hostname.Domain
					disabled_b = true
				}
			}
		}
		if enabled_b {
			out = append(out, enabled)
		}
		if disabled_b {
			out = append(out, disabled)
		}
	}

	for _, ip := range ips_keys {
		enabled := ip
		enabled_b := false
		disabled := "# " + ip
		disabled_b := false
		for _, hostname := range h.Hosts {
			if hostname.Ip == ip {
				if hostname.Enabled {
					enabled += " " + hostname.Domain
					enabled_b = true
				} else {
					disabled += " " + hostname.Domain
					disabled_b = true
				}
			}
		}
		if enabled_b {
			out = append(out, enabled)
		}
		if disabled_b {
			out = append(out, disabled)
		}
	}

	return strings.Join(out, "\n")
}

func (h *Hostfile) Save() error {
	// h.Format(h.Path)
	return nil
}

func (h *Hostfile) Add(host Hostname) error {
	host_f, found := h.Hosts[host.Domain]
	if found {
		if host_f.Ip == host.Ip {
			return errors.New(fmt.Sprintf("Duplicate hostname entry for %s -> %s",
				host.Domain, host.Ip))
		} else {
			return errors.New(fmt.Sprintf("Conflicting hostname entries for %s -> %s and -> %s",
				host.Domain, host.Ip, host_f.Ip))
		}
	} else {
		h.Hosts[host.Domain] = &host
	}
	return nil
}

func (h *Hostfile) Delete(domain string) {
	delete(h.Hosts, domain)
}

func (h *Hostfile) Enable(domain string) {
	_, ok := h.Hosts[domain]
	if ok {
		h.Hosts[domain].Enabled = true
	}
}

func (h *Hostfile) Disable(domain string) {
	_, ok := h.Hosts[domain]
	if ok {
		h.Hosts[domain].Enabled = false
	}
}

func GetHostsPath() string {
	path := os.Getenv("HOSTESS_FILE")
	if path == "" {
		path = "/etc/hosts"
	}
	return path
}

func Hostess() {
	hostfile := NewHostfile(GetHostsPath())
	hostfile.ReadFile(hostfile.Path)
}
<commit_msg>Moved special localhost logic into a reusable funciton<commit_after>package hostess

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"sort"
	"strings"
)

const default_osx = `
##
# Host Database
#
# localhost is used to configure the loopback interface
# when the system is booting.  Do not change this entry.
##

127.0.0.1       localhost
255.255.255.255 broadcasthost
::1             localhost
fe80::1%lo0     localhost
`

const default_linux = `
127.0.0.1   localhost
127.0.1.1   HOSTNAME

# The following lines are desirable for IPv6 capable hosts
::1     localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts
`

type Hostname struct {
	Domain  string
	Ip      string
	Enabled bool
	// Ipv6    bool
}

func (h *Hostname) Format() string {
	r := fmt.Sprintf("%s %s", h.Ip, h.Domain)
	if !h.Enabled {
		r = "# " + r
	}
	return r
}

// Hostfile represents /etc/hosts (or a similar file, depending on OS), and
// includes a list of Hostnames. Hostfile includes
type Hostfile struct {
	Path  string
	Hosts map[string]*Hostname
	data  string
}

func TrimWS(s string) string {
	return strings.Trim(s, " \n\t")
}

// NewHostFile creates a new Hostfile object from the specified file.
func NewHostfile(path string) *Hostfile {
	return &Hostfile{path, make(map[string]*Hostname), ""}
}

func (h *Hostfile) ReadFile(path string) string {
	data, err := ioutil.ReadFile(h.Path)
	if err != nil {
		fmt.Println("Can't read ", h.Path)
		os.Exit(1)
	}
	h.data = string(data)
	return h.data
}

var line_parser = regexp.MustCompile(``)

func parseLine(line string) (Hostname, error) {
	// 1. Split on # to discard comments.
	// 2. Split on first space to find the IP
	// 3. Split remainder of line on whitespace to find
	//    domain names
	// 4. Validate the IP (maybe -- could be ipv4 or ipv6)
	hostname := Hostname{}
	if false {
		return hostname, errors.New("Can't parse hostname")
	}
	return hostname, nil
}

func (h *Hostfile) Read(hostfile string) []Hostname {
	var hosts = make([]Hostname, 0)
	return hosts
}

func getSortedMapKeys(m map[string][]string) []string {
	keys := make([]string, len(m))
	i := 0
	for k := range m {
		keys[i] = k
		i += 1
	}
	sort.Strings(keys)
	return keys
}

// MoveToFront looks for string in a slice of strings and if it finds it, moves
// it to the front of the slice.
// Note: this could probably be made faster using pointers to switch the values
// instead of copying a bunch of crap, but it works and speed is not a problem.
func MoveToFront(list []string, search string) []string {
	for k, v := range list {
		if v == search {
			list = append(list[:k], list[k+1:]...)
		}
	}
	return append([]string{search}, list...)
}

// ListDomainsByIp will look through Hostfile to find domains that match the
// specified Ip and return them in a sorted slice.
func (h *Hostfile) ListDomainsByIp(ip string) []string {
	names := make([]string, 0)
	for _, v := range h.Hosts {
		if v.Ip == ip {
			names = append(names, v.Domain)
		}
	}
	sort.Strings(names)

	// Magic for localhost only, to make sure it's the first domain on its line
	if ip == "127.0.0.1" {
		names = MoveToFront(names, "localhost")
	}

	return names
}

// Format takes the current list of Hostnames in this Hostfile and turns it
// into a string suitable for use as an /etc/hosts file.
// Sorting uses the following logic:
// 1. List is sorted by IP address
// 2. Commented items are left in place
// 3. 127.* appears at the top of the list (so boot resolvers don't break)
// 4. When present, localhost will always appear first in the domain list
func (h *Hostfile) Format() string {
	// localhost := "127.0.0.1 localhost"

	localhosts := make(map[string][]string)
	ips := make(map[string][]string)

	// Map domains and IPs into slices of domains keyd by IP
	// 127.0.0.1 = [localhost, blah, blah2]
	// 2.2.2.3 = [domain1, domain2]
	for _, hostname := range h.Hosts {
		if hostname.Ip[0:4] == "127." {
			localhosts[hostname.Ip] = append(localhosts[hostname.Ip], hostname.Domain)
		} else {
			ips[hostname.Ip] = append(ips[hostname.Ip], hostname.Domain)
		}
	}

	localhosts_keys := getSortedMapKeys(localhosts)
	ips_keys := getSortedMapKeys(ips)
	out := make([]string, 0)

	for _, ip := range localhosts_keys {
		enabled := ip
		enabled_b := false
		disabled := "# " + ip
		disabled_b := false
		for _, hostname := range h.Hosts {
			if hostname.Ip == ip {
				if hostname.Enabled {
					enabled += " " + hostname.Domain
					enabled_b = true
				} else {
					disabled += " " + hostname.Domain
					disabled_b = true
				}
			}
		}
		if enabled_b {
			out = append(out, enabled)
		}
		if disabled_b {
			out = append(out, disabled)
		}
	}

	for _, ip := range ips_keys {
		enabled := ip
		enabled_b := false
		disabled := "# " + ip
		disabled_b := false
		for _, hostname := range h.Hosts {
			if hostname.Ip == ip {
				if hostname.Enabled {
					enabled += " " + hostname.Domain
					enabled_b = true
				} else {
					disabled += " " + hostname.Domain
					disabled_b = true
				}
			}
		}
		if enabled_b {
			out = append(out, enabled)
		}
		if disabled_b {
			out = append(out, disabled)
		}
	}

	return strings.Join(out, "\n")
}

func (h *Hostfile) Save() error {
	// h.Format(h.Path)
	return nil
}

func (h *Hostfile) Add(host Hostname) error {
	host_f, found := h.Hosts[host.Domain]
	if found {
		if host_f.Ip == host.Ip {
			return errors.New(fmt.Sprintf("Duplicate hostname entry for %s -> %s",
				host.Domain, host.Ip))
		} else {
			return errors.New(fmt.Sprintf("Conflicting hostname entries for %s -> %s and -> %s",
				host.Domain, host.Ip, host_f.Ip))
		}
	} else {
		h.Hosts[host.Domain] = &host
	}
	return nil
}

func (h *Hostfile) Delete(domain string) {
	delete(h.Hosts, domain)
}

func (h *Hostfile) Enable(domain string) {
	_, ok := h.Hosts[domain]
	if ok {
		h.Hosts[domain].Enabled = true
	}
}

func (h *Hostfile) Disable(domain string) {
	_, ok := h.Hosts[domain]
	if ok {
		h.Hosts[domain].Enabled = false
	}
}

func GetHostsPath() string {
	path := os.Getenv("HOSTESS_FILE")
	if path == "" {
		path = "/etc/hosts"
	}
	return path
}

func Hostess() {
	hostfile := NewHostfile(GetHostsPath())
	hostfile.ReadFile(hostfile.Path)
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"io/ioutil"
	"os"
	"runtime"

	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
)

type StepKeyPair struct {
	Debug                bool
	SSHAgentAuth         bool
	DebugKeyPath         string
	TemporaryKeyPairName string
	KeyPairName          string
	PrivateKeyFile       string

	keyName string
}

func (s *StepKeyPair) Run(state multistep.StateBag) multistep.StepAction {
	ui := state.Get("ui").(packer.Ui)

	if s.PrivateKeyFile != "" {
		ui.Say("Using existing SSH private key")
		privateKeyBytes, err := ioutil.ReadFile(s.PrivateKeyFile)
		if err != nil {
			state.Put("error", fmt.Errorf(
				"Error loading configured private key file: %s", err))
			return multistep.ActionHalt
		}

		state.Put("keyPair", s.KeyPairName)
		state.Put("privateKey", string(privateKeyBytes))

		return multistep.ActionContinue
	}

	if s.SSHAgentAuth && s.KeyPairName == "" {
		ui.Say("Using SSH Agent with key pair in Source AMI")
		return multistep.ActionContinue
	}

	if s.SSHAgentAuth && s.KeyPairName != "" {
		ui.Say(fmt.Sprintf("Using SSH Agent for existing key pair %s", s.KeyPairName))
		state.Put("keyPair", s.KeyPairName)
		return multistep.ActionContinue
	}

	if s.TemporaryKeyPairName == "" {
		ui.Say("Not using temporary keypair")
		state.Put("keyPair", "")
		return multistep.ActionContinue
	}

	ec2conn := state.Get("ec2").(*ec2.EC2)

	ui.Say(fmt.Sprintf("Creating temporary keypair: %s", s.TemporaryKeyPairName))
	keyResp, err := ec2conn.CreateKeyPair(&ec2.CreateKeyPairInput{
		KeyName: &s.TemporaryKeyPairName})
	if err != nil {
		state.Put("error", fmt.Errorf("Error creating temporary keypair: %s", err))
		return multistep.ActionHalt
	}

	// Set the keyname so we know to delete it later
	s.keyName = s.TemporaryKeyPairName

	// Set some state data for use in future steps
	state.Put("keyPair", s.keyName)
	state.Put("privateKey", *keyResp.KeyMaterial)

	// If we're in debug mode, output the private key to the working
	// directory.
	if s.Debug {
		ui.Message(fmt.Sprintf("Saving key for debug purposes: %s", s.DebugKeyPath))
		f, err := os.Create(s.DebugKeyPath)
		if err != nil {
			state.Put("error", fmt.Errorf("Error saving debug key: %s", err))
			return multistep.ActionHalt
		}
		defer f.Close()

		// Write the key out
		if _, err := f.Write([]byte(*keyResp.KeyMaterial)); err != nil {
			state.Put("error", fmt.Errorf("Error saving debug key: %s", err))
			return multistep.ActionHalt
		}

		// Chmod it so that it is SSH ready
		if runtime.GOOS != "windows" {
			if err := f.Chmod(0600); err != nil {
				state.Put("error", fmt.Errorf("Error setting permissions of debug key: %s", err))
				return multistep.ActionHalt
			}
		}
	}

	return multistep.ActionContinue
}

func (s *StepKeyPair) Cleanup(state multistep.StateBag) {
	// If no key name is set, then we never created it, so just return
	// If we used an SSH private key file, do not go about deleting
	// keypairs
	if s.PrivateKeyFile != "" || s.KeyPairName != "" {
		return
	}

	ec2conn := state.Get("ec2").(*ec2.EC2)
	ui := state.Get("ui").(packer.Ui)

	// Remove the keypair
	ui.Say("Deleting temporary keypair...")
	_, err := ec2conn.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: &s.keyName})
	if err != nil {
		ui.Error(fmt.Sprintf(
			"Error cleaning up keypair. Please delete the key manually: %s", s.keyName))
	}

	// Also remove the physical key if we're debugging.
	if s.Debug {
		if err := os.Remove(s.DebugKeyPath); err != nil {
			ui.Error(fmt.Sprintf(
				"Error removing debug key '%s': %s", s.DebugKeyPath, err))
		}
	}
}
<commit_msg>don't attempt to delete non-existant key when using agent auth<commit_after>package common

import (
	"fmt"
	"io/ioutil"
	"os"
	"runtime"

	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
)

type StepKeyPair struct {
	Debug                bool
	SSHAgentAuth         bool
	DebugKeyPath         string
	TemporaryKeyPairName string
	KeyPairName          string
	PrivateKeyFile       string

	keyName string
}

func (s *StepKeyPair) Run(state multistep.StateBag) multistep.StepAction {
	ui := state.Get("ui").(packer.Ui)

	if s.PrivateKeyFile != "" {
		ui.Say("Using existing SSH private key")
		privateKeyBytes, err := ioutil.ReadFile(s.PrivateKeyFile)
		if err != nil {
			state.Put("error", fmt.Errorf(
				"Error loading configured private key file: %s", err))
			return multistep.ActionHalt
		}

		state.Put("keyPair", s.KeyPairName)
		state.Put("privateKey", string(privateKeyBytes))

		return multistep.ActionContinue
	}

	if s.SSHAgentAuth && s.KeyPairName == "" {
		ui.Say("Using SSH Agent with key pair in Source AMI")
		return multistep.ActionContinue
	}

	if s.SSHAgentAuth && s.KeyPairName != "" {
		ui.Say(fmt.Sprintf("Using SSH Agent for existing key pair %s", s.KeyPairName))
		state.Put("keyPair", s.KeyPairName)
		return multistep.ActionContinue
	}

	if s.TemporaryKeyPairName == "" {
		ui.Say("Not using temporary keypair")
		state.Put("keyPair", "")
		return multistep.ActionContinue
	}

	ec2conn := state.Get("ec2").(*ec2.EC2)

	ui.Say(fmt.Sprintf("Creating temporary keypair: %s", s.TemporaryKeyPairName))
	keyResp, err := ec2conn.CreateKeyPair(&ec2.CreateKeyPairInput{
		KeyName: &s.TemporaryKeyPairName})
	if err != nil {
		state.Put("error", fmt.Errorf("Error creating temporary keypair: %s", err))
		return multistep.ActionHalt
	}

	// Set the keyname so we know to delete it later
	s.keyName = s.TemporaryKeyPairName

	// Set some state data for use in future steps
	state.Put("keyPair", s.keyName)
	state.Put("privateKey", *keyResp.KeyMaterial)

	// If we're in debug mode, output the private key to the working
	// directory.
	if s.Debug {
		ui.Message(fmt.Sprintf("Saving key for debug purposes: %s", s.DebugKeyPath))
		f, err := os.Create(s.DebugKeyPath)
		if err != nil {
			state.Put("error", fmt.Errorf("Error saving debug key: %s", err))
			return multistep.ActionHalt
		}
		defer f.Close()

		// Write the key out
		if _, err := f.Write([]byte(*keyResp.KeyMaterial)); err != nil {
			state.Put("error", fmt.Errorf("Error saving debug key: %s", err))
			return multistep.ActionHalt
		}

		// Chmod it so that it is SSH ready
		if runtime.GOOS != "windows" {
			if err := f.Chmod(0600); err != nil {
				state.Put("error", fmt.Errorf("Error setting permissions of debug key: %s", err))
				return multistep.ActionHalt
			}
		}
	}

	return multistep.ActionContinue
}

func (s *StepKeyPair) Cleanup(state multistep.StateBag) {
	// If no key name is set, then we never created it, so just return
	// If we used an SSH private key file, do not go about deleting
	// keypairs
	if s.PrivateKeyFile != "" || (s.KeyPairName == "" && s.keyName == "") {
		return
	}

	ec2conn := state.Get("ec2").(*ec2.EC2)
	ui := state.Get("ui").(packer.Ui)

	// Remove the keypair
	ui.Say("Deleting temporary keypair...")
	_, err := ec2conn.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: &s.keyName})
	if err != nil {
		ui.Error(fmt.Sprintf(
			"Error cleaning up keypair. Please delete the key manually: %s", s.keyName))
	}

	// Also remove the physical key if we're debugging.
	if s.Debug {
		if err := os.Remove(s.DebugKeyPath); err != nil {
			ui.Error(fmt.Sprintf(
				"Error removing debug key '%s': %s", s.DebugKeyPath, err))
		}
	}
}
<|endoftext|>


<commit_before>package gocql

import (
	"net"
	"sync"
	"testing"
)

func TestEventDebounce(t *testing.T) {
	const eventCount = 150
	wg := &sync.WaitGroup{}
	wg.Add(1)

	eventsSeen := 0
	debouncer := newEventDeouncer("testDebouncer", func(events []frame) {
		defer wg.Done()
		eventsSeen += len(events)
	})

	for i := 0; i < eventCount; i++ {
		debouncer.debounce(&statusChangeEventFrame{
			change: "UP",
			host:   net.IPv4(127, 0, 0, 1),
			port:   9042,
		})
	}

	wg.Wait()
	if eventCount != eventsSeen {
		t.Fatalf("expected to see %d events but got %d", eventCount, eventsSeen)
	}
	debouncer.stop()
}
<commit_msg>defer ending debouncer.stop()<commit_after>package gocql

import (
	"net"
	"sync"
	"testing"
)

func TestEventDebounce(t *testing.T) {
	const eventCount = 150
	wg := &sync.WaitGroup{}
	wg.Add(1)

	eventsSeen := 0
	debouncer := newEventDeouncer("testDebouncer", func(events []frame) {
		defer wg.Done()
		eventsSeen += len(events)
	})
	defer debouncer.stop()

	for i := 0; i < eventCount; i++ {
		debouncer.debounce(&statusChangeEventFrame{
			change: "UP",
			host:   net.IPv4(127, 0, 0, 1),
			port:   9042,
		})
	}

	wg.Wait()
	if eventCount != eventsSeen {
		t.Fatalf("expected to see %d events but got %d", eventCount, eventsSeen)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"flag"
	"fmt"
	"github.com/Arachnid/evmdis"
	"io/ioutil"
	"log"
	"os"
)

const swarmHashLength = 43

var swarmHashProgramTrailer = [...]byte{0x00, 0x29}
var swarmHashHeader = [...]byte{0xa1, 0x65}

func main() {

	withSwarmHash := flag.Bool("swarm", true, "solc adds a reference to the Swarm API description to the generated bytecode, if this flag is set it removes this reference before analysis")
	ctorMode := flag.Bool("ctor", false, "Indicates that the provided bytecode has construction(ctor) code included. (needs to be analyzed seperatly)")
	logging := flag.Bool("log", false, "print logging output")

	flag.Parse()

	if !*logging {
		log.SetOutput(ioutil.Discard)
	}

	hexdata, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		panic(fmt.Sprintf("Could not read from stdin: %v", err))
	}

	bytecodeLength := uint64(hex.DecodedLen(len(hexdata)))
	bytecode := make([]byte, bytecodeLength)

	hex.Decode(bytecode, hexdata)

	// detect swarm hash and remove it from bytecode, see http://solidity.readthedocs.io/en/latest/miscellaneous.html?highlight=swarm#encoding-of-the-metadata-hash-in-the-bytecode
	if bytecode[bytecodeLength-1] == swarmHashProgramTrailer[1] &&
		bytecode[bytecodeLength-2] == swarmHashProgramTrailer[0] &&
		bytecode[bytecodeLength-43] == swarmHashHeader[0] &&
		bytecode[bytecodeLength-42] == swarmHashHeader[1] &&
		*withSwarmHash {
		bytecodeLength -= swarmHashLength // remove swarm part
	}

	program := evmdis.NewProgram(bytecode[:bytecodeLength])
	AnalyzeProgram(program)

	if *ctorMode {
		var codeEntryPoint = FindNextCodeEntryPoint(program)

		if codeEntryPoint == 0 {
			panic("no code entrypoint found in ctor")
		} else if codeEntryPoint >= bytecodeLength {
			panic("code entrypoint outside of currently available code")
		}

		ctor := evmdis.NewProgram(bytecode[:codeEntryPoint])
		code := evmdis.NewProgram(bytecode[codeEntryPoint:bytecodeLength])

		AnalyzeProgram(ctor)
		fmt.Println("# Constructor part -------------------------")
		PrintAnalysisResult(ctor)

		AnalyzeProgram(code)
		fmt.Println("# Code part -------------------------")
		PrintAnalysisResult(code)

	} else {
		PrintAnalysisResult(program)
	}
}

func FindNextCodeEntryPoint(program *evmdis.Program) uint64 {
	var lastPos uint64 = 0
	for _, block := range program.Blocks {
		for _, instruction := range block.Instructions {
			if instruction.Op == evmdis.CODECOPY {
				var expression evmdis.Expression

				instruction.Annotations.Get(&expression)

				arg := expression.(*evmdis.InstructionExpression).Arguments[1].Eval()

				if arg != nil {
					lastPos = arg.Uint64()
				}
			}
		}
	}
	return lastPos
}

func PrintAnalysisResult(program *evmdis.Program) {
	for _, block := range program.Blocks {
		offset := block.Offset

		// Print out the jump label for the block, if there is one
		var label *evmdis.JumpLabel
		block.Annotations.Get(&label)
		if label != nil {
			fmt.Printf("%v\n", label)
		}

		// Print out the stack prestate for this block
		var reaching evmdis.ReachingDefinition
		block.Annotations.Get(&reaching)
		fmt.Printf("# Stack: %v\n", reaching)

		for _, instruction := range block.Instructions {
			var expression evmdis.Expression
			instruction.Annotations.Get(&expression)

			if expression != nil {
				if instruction.Op.StackWrites() == 1 && !instruction.Op.IsDup() {
					fmt.Printf("0x%X\tPUSH(%v)\n", offset, expression)
				} else {
					fmt.Printf("0x%X\t%v\n", offset, expression)
				}
			}
			offset += instruction.Op.OperandSize() + 1
		}
		fmt.Printf("\n")
	}
}

func AnalyzeProgram(program *evmdis.Program) {
	if err := evmdis.PerformReachingAnalysis(program); err != nil {
		panic(fmt.Sprintf("Error performing reaching analysis: %v", err))
	}
	evmdis.PerformReachesAnalysis(program)
	evmdis.CreateLabels(program)
	if err := evmdis.BuildExpressions(program); err != nil {
		panic(fmt.Sprintf("Error building expressions: %v", err))
	}
}
<commit_msg>Fix typo in main.go<commit_after>package main

import (
	"encoding/hex"
	"flag"
	"fmt"
	"github.com/Arachnid/evmdis"
	"io/ioutil"
	"log"
	"os"
)

const swarmHashLength = 43

var swarmHashProgramTrailer = [...]byte{0x00, 0x29}
var swarmHashHeader = [...]byte{0xa1, 0x65}

func main() {

	withSwarmHash := flag.Bool("swarm", true, "solc adds a reference to the Swarm API description to the generated bytecode, if this flag is set it removes this reference before analysis")
	ctorMode := flag.Bool("ctor", false, "Indicates that the provided bytecode has construction(ctor) code included. (needs to be analyzed separately)")
	logging := flag.Bool("log", false, "print logging output")

	flag.Parse()

	if !*logging {
		log.SetOutput(ioutil.Discard)
	}

	hexdata, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		panic(fmt.Sprintf("Could not read from stdin: %v", err))
	}

	bytecodeLength := uint64(hex.DecodedLen(len(hexdata)))
	bytecode := make([]byte, bytecodeLength)

	hex.Decode(bytecode, hexdata)

	// detect swarm hash and remove it from bytecode, see http://solidity.readthedocs.io/en/latest/miscellaneous.html?highlight=swarm#encoding-of-the-metadata-hash-in-the-bytecode
	if bytecode[bytecodeLength-1] == swarmHashProgramTrailer[1] &&
		bytecode[bytecodeLength-2] == swarmHashProgramTrailer[0] &&
		bytecode[bytecodeLength-43] == swarmHashHeader[0] &&
		bytecode[bytecodeLength-42] == swarmHashHeader[1] &&
		*withSwarmHash {
		bytecodeLength -= swarmHashLength // remove swarm part
	}

	program := evmdis.NewProgram(bytecode[:bytecodeLength])
	AnalyzeProgram(program)

	if *ctorMode {
		var codeEntryPoint = FindNextCodeEntryPoint(program)

		if codeEntryPoint == 0 {
			panic("no code entrypoint found in ctor")
		} else if codeEntryPoint >= bytecodeLength {
			panic("code entrypoint outside of currently available code")
		}

		ctor := evmdis.NewProgram(bytecode[:codeEntryPoint])
		code := evmdis.NewProgram(bytecode[codeEntryPoint:bytecodeLength])

		AnalyzeProgram(ctor)
		fmt.Println("# Constructor part -------------------------")
		PrintAnalysisResult(ctor)

		AnalyzeProgram(code)
		fmt.Println("# Code part -------------------------")
		PrintAnalysisResult(code)

	} else {
		PrintAnalysisResult(program)
	}
}

func FindNextCodeEntryPoint(program *evmdis.Program) uint64 {
	var lastPos uint64 = 0
	for _, block := range program.Blocks {
		for _, instruction := range block.Instructions {
			if instruction.Op == evmdis.CODECOPY {
				var expression evmdis.Expression

				instruction.Annotations.Get(&expression)

				arg := expression.(*evmdis.InstructionExpression).Arguments[1].Eval()

				if arg != nil {
					lastPos = arg.Uint64()
				}
			}
		}
	}
	return lastPos
}

func PrintAnalysisResult(program *evmdis.Program) {
	for _, block := range program.Blocks {
		offset := block.Offset

		// Print out the jump label for the block, if there is one
		var label *evmdis.JumpLabel
		block.Annotations.Get(&label)
		if label != nil {
			fmt.Printf("%v\n", label)
		}

		// Print out the stack prestate for this block
		var reaching evmdis.ReachingDefinition
		block.Annotations.Get(&reaching)
		fmt.Printf("# Stack: %v\n", reaching)

		for _, instruction := range block.Instructions {
			var expression evmdis.Expression
			instruction.Annotations.Get(&expression)

			if expression != nil {
				if instruction.Op.StackWrites() == 1 && !instruction.Op.IsDup() {
					fmt.Printf("0x%X\tPUSH(%v)\n", offset, expression)
				} else {
					fmt.Printf("0x%X\t%v\n", offset, expression)
				}
			}
			offset += instruction.Op.OperandSize() + 1
		}
		fmt.Printf("\n")
	}
}

func AnalyzeProgram(program *evmdis.Program) {
	if err := evmdis.PerformReachingAnalysis(program); err != nil {
		panic(fmt.Sprintf("Error performing reaching analysis: %v", err))
	}
	evmdis.PerformReachesAnalysis(program)
	evmdis.CreateLabels(program)
	if err := evmdis.BuildExpressions(program); err != nil {
		panic(fmt.Sprintf("Error building expressions: %v", err))
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"bytes"
	"fmt"
	"sync"
	"time"

	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/tsuru/iaas"
	"github.com/tsuru/tsuru/log"
)

type nodeHealer struct {
	sync.Mutex
	locks                 map[string]*sync.Mutex
	provisioner           *dockerProvisioner
	disabledTime          time.Duration
	waitTimeNewMachine    time.Duration
	failuresBeforeHealing int
}

func (h *nodeHealer) healNode(node *cluster.Node) (cluster.Node, error) {
	emptyNode := cluster.Node{}
	failingAddr := node.Address
	nodeMetadata := node.CleanMetadata()
	failingHost := urlToHost(failingAddr)
	failures := node.FailureCount()
	machine, err := iaas.CreateMachineForIaaS(nodeMetadata["iaas"], nodeMetadata)
	if err != nil {
		node.ResetFailures()
		return emptyNode, fmt.Errorf("Can't auto-heal after %d failures for node %s: error creating new machine: %s", failures, failingHost, err.Error())
	}
	err = h.provisioner.getCluster().Unregister(failingAddr)
	if err != nil {
		machine.Destroy()
		return emptyNode, fmt.Errorf("Can't auto-heal after %d failures for node %s: error unregistering old node: %s", failures, failingHost, err.Error())
	}
	newAddr := machine.FormatNodeAddress()
	log.Debugf("New machine created during healing process: %s - Waiting for docker to start...", newAddr)
	createdNode, err := h.provisioner.getCluster().WaitAndRegister(newAddr, nodeMetadata, h.waitTimeNewMachine)
	if err != nil {
		node.ResetFailures()
		h.provisioner.getCluster().Register(failingAddr, nodeMetadata)
		machine.Destroy()
		return emptyNode, fmt.Errorf("Can't auto-heal after %d failures for node %s: error registering new node: %s", failures, failingHost, err.Error())
	}
	var buf bytes.Buffer
	err = h.provisioner.moveContainers(failingHost, "", &buf)
	if err != nil {
		log.Errorf("Unable to move containers, skipping containers healing %q -> %q: %s: %s", failingHost, machine.Address, err.Error(), buf.String())
	}
	failingMachine, err := iaas.FindMachineByIdOrAddress(node.Metadata["iaas-id"], failingHost)
	if err != nil {
		return createdNode, fmt.Errorf("Unable to find failing machine %s in IaaS: %s", failingHost, err.Error())
	}
	err = failingMachine.Destroy()
	if err != nil {
		return createdNode, fmt.Errorf("Unable to destroy machine %s from IaaS: %s", failingHost, err.Error())
	}
	log.Debugf("Done auto-healing node %q, node %q created in its place.", failingHost, machine.Address)
	return createdNode, nil
}

func (h *nodeHealer) HandleError(node *cluster.Node) time.Duration {
	h.Lock()
	if h.locks[node.Address] == nil {
		h.locks[node.Address] = &sync.Mutex{}
	}
	h.locks[node.Address].Lock()
	h.Unlock()
	defer func() {
		h.Lock()
		defer h.Unlock()
		h.locks[node.Address].Unlock()
	}()
	failures := node.FailureCount()
	if failures < h.failuresBeforeHealing {
		log.Debugf("%d failures detected in node %q, waiting for more failures before healing.", failures, node.Address)
		return h.disabledTime
	}
	if !node.HasSuccess() {
		log.Debugf("Node %q has never been successfully reached, healing won't run on it.", node.Address)
		return h.disabledTime
	}
	_, hasIaas := node.Metadata["iaas"]
	if !hasIaas {
		log.Debugf("Node %q doesn't have IaaS information, healing won't run on it.", node.Address)
		return h.disabledTime
	}
	healingCounter, err := healingCountFor("node", node.Address, consecutiveHealingsTimeframe)
	if err != nil {
		log.Errorf("Node healing: couldn't verify number of previous healings for %s: %s", node.Address, err.Error())
		return h.disabledTime
	}
	if healingCounter > consecutiveHealingsLimitInTimeframe {
		log.Errorf("Node healing: number of healings for node %s in the last %d minutes exceeds limit of %d: %d",
			node.Address, consecutiveHealingsTimeframe/time.Minute, consecutiveHealingsLimitInTimeframe, healingCounter)
		return h.disabledTime
	}
	log.Errorf("Initiating healing process for node %q after %d failures.", node.Address, failures)
	evt, err := newHealingEvent(*node)
	if err != nil {
		log.Errorf("Error trying to insert healing event: %s", err.Error())
		return h.disabledTime
	}
	createdNode, err := h.healNode(node)
	if err != nil {
		log.Errorf("Error healing: %s", err.Error())
	}
	err = evt.update(createdNode, err)
	if err != nil {
		log.Errorf("Error trying to update healing event: %s", err.Error())
	}
	if createdNode.Address != "" {
		return 0
	}
	return h.disabledTime
}

func (h *nodeHealer) Shutdown() {
	h.Lock()
	for _, lock := range h.locks {
		lock.Lock()
	}
}

func (h *nodeHealer) String() string {
	return "node healer"
}
<commit_msg>provision/docker: fix locking in node healer<commit_after>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"bytes"
	"fmt"
	"sync"
	"time"

	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/tsuru/iaas"
	"github.com/tsuru/tsuru/log"
)

type nodeHealer struct {
	sync.Mutex
	locks                 map[string]*sync.Mutex
	provisioner           *dockerProvisioner
	disabledTime          time.Duration
	waitTimeNewMachine    time.Duration
	failuresBeforeHealing int
}

func (h *nodeHealer) healNode(node *cluster.Node) (cluster.Node, error) {
	emptyNode := cluster.Node{}
	failingAddr := node.Address
	nodeMetadata := node.CleanMetadata()
	failingHost := urlToHost(failingAddr)
	failures := node.FailureCount()
	machine, err := iaas.CreateMachineForIaaS(nodeMetadata["iaas"], nodeMetadata)
	if err != nil {
		node.ResetFailures()
		return emptyNode, fmt.Errorf("Can't auto-heal after %d failures for node %s: error creating new machine: %s", failures, failingHost, err.Error())
	}
	err = h.provisioner.getCluster().Unregister(failingAddr)
	if err != nil {
		machine.Destroy()
		return emptyNode, fmt.Errorf("Can't auto-heal after %d failures for node %s: error unregistering old node: %s", failures, failingHost, err.Error())
	}
	newAddr := machine.FormatNodeAddress()
	log.Debugf("New machine created during healing process: %s - Waiting for docker to start...", newAddr)
	createdNode, err := h.provisioner.getCluster().WaitAndRegister(newAddr, nodeMetadata, h.waitTimeNewMachine)
	if err != nil {
		node.ResetFailures()
		h.provisioner.getCluster().Register(failingAddr, nodeMetadata)
		machine.Destroy()
		return emptyNode, fmt.Errorf("Can't auto-heal after %d failures for node %s: error registering new node: %s", failures, failingHost, err.Error())
	}
	var buf bytes.Buffer
	err = h.provisioner.moveContainers(failingHost, "", &buf)
	if err != nil {
		log.Errorf("Unable to move containers, skipping containers healing %q -> %q: %s: %s", failingHost, machine.Address, err.Error(), buf.String())
	}
	failingMachine, err := iaas.FindMachineByIdOrAddress(node.Metadata["iaas-id"], failingHost)
	if err != nil {
		return createdNode, fmt.Errorf("Unable to find failing machine %s in IaaS: %s", failingHost, err.Error())
	}
	err = failingMachine.Destroy()
	if err != nil {
		return createdNode, fmt.Errorf("Unable to destroy machine %s from IaaS: %s", failingHost, err.Error())
	}
	log.Debugf("Done auto-healing node %q, node %q created in its place.", failingHost, machine.Address)
	return createdNode, nil
}

func (h *nodeHealer) HandleError(node *cluster.Node) time.Duration {
	h.Lock()
	if h.locks[node.Address] == nil {
		h.locks[node.Address] = &sync.Mutex{}
	}
	h.Unlock()
	h.locks[node.Address].Lock()
	defer h.locks[node.Address].Unlock()
	failures := node.FailureCount()
	if failures < h.failuresBeforeHealing {
		log.Debugf("%d failures detected in node %q, waiting for more failures before healing.", failures, node.Address)
		return h.disabledTime
	}
	if !node.HasSuccess() {
		log.Debugf("Node %q has never been successfully reached, healing won't run on it.", node.Address)
		return h.disabledTime
	}
	_, hasIaas := node.Metadata["iaas"]
	if !hasIaas {
		log.Debugf("Node %q doesn't have IaaS information, healing won't run on it.", node.Address)
		return h.disabledTime
	}
	healingCounter, err := healingCountFor("node", node.Address, consecutiveHealingsTimeframe)
	if err != nil {
		log.Errorf("Node healing: couldn't verify number of previous healings for %s: %s", node.Address, err.Error())
		return h.disabledTime
	}
	if healingCounter > consecutiveHealingsLimitInTimeframe {
		log.Errorf("Node healing: number of healings for node %s in the last %d minutes exceeds limit of %d: %d",
			node.Address, consecutiveHealingsTimeframe/time.Minute, consecutiveHealingsLimitInTimeframe, healingCounter)
		return h.disabledTime
	}
	log.Errorf("Initiating healing process for node %q after %d failures.", node.Address, failures)
	evt, err := newHealingEvent(*node)
	if err != nil {
		log.Errorf("Error trying to insert healing event: %s", err.Error())
		return h.disabledTime
	}
	createdNode, err := h.healNode(node)
	if err != nil {
		log.Errorf("Error healing: %s", err.Error())
	}
	err = evt.update(createdNode, err)
	if err != nil {
		log.Errorf("Error trying to update healing event: %s", err.Error())
	}
	if createdNode.Address != "" {
		return 0
	}
	return h.disabledTime
}

func (h *nodeHealer) Shutdown() {
	h.Lock()
	for _, lock := range h.locks {
		lock.Lock()
	}
}

func (h *nodeHealer) String() string {
	return "node healer"
}
<|endoftext|>


<commit_before>package cluster

import (
	"context"
	"sync"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/lxd/task"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
)

var listeners = map[string]*lxd.EventListener{}
var listenersLock sync.Mutex

// Events starts a task that continuously monitors the list of cluster nodes and
// maintains a pool of websocket connections against all of them, in order to
// get notified about events.
//
// Whenever an event is received the given callback is invoked.
func Events(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, f func(int64, api.Event)) (task.Func, task.Schedule) {
	// Update our pool of event listeners. Since database queries are
	// blocking, we spawn the actual logic in a goroutine, to abort
	// immediately when we receive the stop signal.
	update := func(ctx context.Context) {
		ch := make(chan struct{})
		go func() {
			eventsUpdateListeners(endpoints, cluster, serverCert, f)
			ch <- struct{}{}
		}()
		select {
		case <-ch:
		case <-ctx.Done():
		}
	}

	schedule := task.Every(time.Second)

	return update, schedule
}

func eventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, f func(int64, api.Event)) {
	// Get the current cluster nodes.
	var nodes []db.NodeInfo
	var offlineThreshold time.Duration

	err := cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error

		nodes, err = tx.GetNodes()
		if err != nil {
			return err
		}

		offlineThreshold, err = tx.GetNodeOfflineThreshold()
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		logger.Warn("Failed to get current cluster members", log.Ctx{"err": err})
		return
	}
	if len(nodes) == 1 {
		return // Either we're not clustered or this is a single-node cluster
	}

	address := endpoints.NetworkAddress()

	addresses := make([]string, len(nodes))
	for i, node := range nodes {
		addresses[i] = node.Address

		// Don't bother trying to connect to offline nodes, or to ourselves.
		if node.IsOffline(offlineThreshold) || node.Address == address {
			continue
		}

		listenersLock.Lock()
		listener, ok := listeners[node.Address]

		// The node has already a listener associated to it.
		if ok {
			// Double check that the listener is still
			// connected. If it is, just move on, other
			// we'll try to connect again.
			if listeners[node.Address].IsActive() {
				listenersLock.Unlock()
				continue
			}

			delete(listeners, node.Address)
		}
		listenersLock.Unlock()

		listener, err := eventsConnect(node.Address, endpoints.NetworkCert(), serverCert())
		if err != nil {
			logger.Warn("Failed to get events from member", log.Ctx{"address": node.Address, "err": err})
			continue
		}
		logger.Debug("Listening for events on member", log.Ctx{"address": node.Address})
		listener.AddHandler(nil, func(event api.Event) { f(node.ID, event) })

		listenersLock.Lock()
		listeners[node.Address] = listener
		listenersLock.Unlock()
	}

	listenersLock.Lock()
	for address, listener := range listeners {
		if !shared.StringInSlice(address, addresses) {
			listener.Disconnect()
			delete(listeners, address)
		}
	}
	listenersLock.Unlock()
}

// Establish a client connection to get events from the given node.
func eventsConnect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo) (*lxd.EventListener, error) {
	client, err := Connect(address, networkCert, serverCert, nil, true)
	if err != nil {
		return nil, err
	}

	// Set the project to the special wildcard in order to get notified
	// about all events across all projects.
	client = client.UseProject("*")

	return client.GetEvents()
}
<commit_msg>lxd/cluster/events: Disconnect event listeners for offline members in eventsUpdateListeners<commit_after>package cluster

import (
	"context"
	"sync"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/lxd/task"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
)

var listeners = map[string]*lxd.EventListener{}
var listenersLock sync.Mutex

// Events starts a task that continuously monitors the list of cluster nodes and
// maintains a pool of websocket connections against all of them, in order to
// get notified about events.
//
// Whenever an event is received the given callback is invoked.
func Events(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, f func(int64, api.Event)) (task.Func, task.Schedule) {
	// Update our pool of event listeners. Since database queries are
	// blocking, we spawn the actual logic in a goroutine, to abort
	// immediately when we receive the stop signal.
	update := func(ctx context.Context) {
		ch := make(chan struct{})
		go func() {
			eventsUpdateListeners(endpoints, cluster, serverCert, f)
			ch <- struct{}{}
		}()
		select {
		case <-ch:
		case <-ctx.Done():
		}
	}

	schedule := task.Every(time.Second)

	return update, schedule
}

func eventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, f func(int64, api.Event)) {
	// Get the current cluster nodes.
	var nodes []db.NodeInfo
	var offlineThreshold time.Duration

	err := cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error

		nodes, err = tx.GetNodes()
		if err != nil {
			return err
		}

		offlineThreshold, err = tx.GetNodeOfflineThreshold()
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		logger.Warn("Failed to get current cluster members", log.Ctx{"err": err})
		return
	}
	if len(nodes) == 1 {
		return // Either we're not clustered or this is a single-node cluster
	}

	address := endpoints.NetworkAddress()

	addresses := make([]string, len(nodes))
	for i, node := range nodes {
		addresses[i] = node.Address

		if node.Address == address {
			continue
		}

		listenersLock.Lock()
		listener, ok := listeners[node.Address]

		// Don't bother trying to connect to offline nodes, or to ourselves.
		if node.IsOffline(offlineThreshold) {
			if ok {
				listener.Disconnect()
			}

			listenersLock.Unlock()
			continue
		}

		// The node has already a listener associated to it.
		if ok {
			// Double check that the listener is still
			// connected. If it is, just move on, other
			// we'll try to connect again.
			if listeners[node.Address].IsActive() {
				listenersLock.Unlock()
				continue
			}

			delete(listeners, node.Address)
		}
		listenersLock.Unlock()

		listener, err := eventsConnect(node.Address, endpoints.NetworkCert(), serverCert())
		if err != nil {
			logger.Warn("Failed to get events from member", log.Ctx{"address": node.Address, "err": err})
			continue
		}
		logger.Debug("Listening for events on member", log.Ctx{"address": node.Address})
		listener.AddHandler(nil, func(event api.Event) { f(node.ID, event) })

		listenersLock.Lock()
		listeners[node.Address] = listener
		listenersLock.Unlock()
	}

	listenersLock.Lock()
	for address, listener := range listeners {
		if !shared.StringInSlice(address, addresses) {
			listener.Disconnect()
			delete(listeners, address)
		}
	}
	listenersLock.Unlock()
}

// Establish a client connection to get events from the given node.
func eventsConnect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo) (*lxd.EventListener, error) {
	client, err := Connect(address, networkCert, serverCert, nil, true)
	if err != nil {
		return nil, err
	}

	// Set the project to the special wildcard in order to get notified
	// about all events across all projects.
	client = client.UseProject("*")

	return client.GetEvents()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/db/query"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/version"
	"github.com/pkg/errors"
)

func containersGet(d *Daemon, r *http.Request) Response {
	for i := 0; i < 100; i++ {
		result, err := doContainersGet(d, r)
		if err == nil {
			return SyncResponse(true, result)
		}
		if !query.IsRetriableError(err) {
			logger.Debugf("DBERR: containersGet: error %q", err)
			return SmartError(err)
		}
		// 100 ms may seem drastic, but we really don't want to thrash
		// perhaps we should use a random amount
		time.Sleep(100 * time.Millisecond)
	}

	logger.Debugf("DBERR: containersGet, db is locked")
	logger.Debugf(logger.GetStack())
	return InternalError(fmt.Errorf("DB is locked"))
}

func doContainersGet(d *Daemon, r *http.Request) (interface{}, error) {
	resultString := []string{}
	resultList := []*api.Container{}
	resultFullList := []*api.ContainerFull{}
	resultMu := sync.Mutex{}

	// Parse the recursion field
	recursionStr := r.FormValue("recursion")

	recursion, err := strconv.Atoi(recursionStr)
	if err != nil {
		recursion = 0
	}

	// Parse the project field
	project := projectParam(r)

	// Get the list and location of all containers
	var result map[string][]string // Containers by node address
	var nodes map[string]string    // Node names by container
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error

		result, err = tx.ContainersListByNodeAddress(project)
		if err != nil {
			return err
		}

		nodes, err = tx.ContainersByNodeName(project)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return []string{}, err
	}

	// Get the local containers
	nodeCts := map[string]container{}
	if recursion > 0 {
		cts, err := containerLoadNodeProjectAll(d.State(), project)
		if err != nil {
			return nil, err
		}

		for _, ct := range cts {
			nodeCts[ct.Name()] = ct
		}
	}

	// Append containers to list and handle errors
	resultListAppend := func(name string, c api.Container, err error) {
		if err != nil {
			c = api.Container{
				Name:       name,
				Status:     api.Error.String(),
				StatusCode: api.Error,
				Location:   nodes[name],
			}
		}
		resultMu.Lock()
		resultList = append(resultList, &c)
		resultMu.Unlock()
	}

	resultFullListAppend := func(name string, c api.ContainerFull, err error) {
		if err != nil {
			c = api.ContainerFull{Container: api.Container{
				Name:       name,
				Status:     api.Error.String(),
				StatusCode: api.Error,
				Location:   nodes[name],
			}}
		}
		resultMu.Lock()
		resultFullList = append(resultFullList, &c)
		resultMu.Unlock()
	}

	// Get the data
	wg := sync.WaitGroup{}
	for address, containers := range result {
		// If this is an internal request from another cluster node,
		// ignore containers from other nodes, and return only the ones
		// on this node
		if isClusterNotification(r) && address != "" {
			continue
		}

		// Mark containers on unavailable nodes as down
		if recursion > 0 && address == "0.0.0.0" {
			for _, container := range containers {
				if recursion == 1 {
					resultListAppend(container, api.Container{}, fmt.Errorf("unavailable"))
				} else {
					resultFullListAppend(container, api.ContainerFull{}, fmt.Errorf("unavailable"))
				}
			}

			continue
		}

		// For recursion requests we need to fetch the state of remote
		// containers from their respective nodes.
		if recursion > 0 && address != "" && !isClusterNotification(r) {
			wg.Add(1)
			go func(address string, containers []string) {
				defer wg.Done()
				cert := d.endpoints.NetworkCert()

				if recursion == 1 {
					cs, err := doContainersGetFromNode(project, address, cert)
					if err != nil {
						for _, name := range containers {
							resultListAppend(name, api.Container{}, err)
						}

						return
					}

					for _, c := range cs {
						resultListAppend(c.Name, c, nil)
					}

					return
				}

				cs, err := doContainersFullGetFromNode(project, address, cert)
				if err != nil {
					for _, name := range containers {
						resultFullListAppend(name, api.ContainerFull{}, err)
					}

					return
				}

				for _, c := range cs {
					resultFullListAppend(c.Name, c, nil)
				}
			}(address, containers)

			continue
		}

		if recursion == 0 {
			for _, container := range containers {
				url := fmt.Sprintf("/%s/containers/%s", version.APIVersion, container)
				resultString = append(resultString, url)
			}
		} else {
			threads := 4
			if len(containers) < threads {
				threads = len(containers)
			}

			queue := make(chan string, threads)

			for i := 0; i < threads; i++ {
				wg.Add(1)

				go func() {
					for {
						container, more := <-queue
						if !more {
							break
						}

						if recursion == 1 {
							c, _, err := nodeCts[container].Render()
							if err != nil {
								resultListAppend(container, api.Container{}, err)
							} else {
								resultListAppend(container, *c.(*api.Container), err)
							}

							continue
						}

						c, _, err := nodeCts[container].RenderFull()
						if err != nil {
							resultFullListAppend(container, api.ContainerFull{}, err)
						} else {
							resultFullListAppend(container, *c, err)
						}
					}

					wg.Done()
				}()
			}

			for _, container := range containers {
				queue <- container
			}

			close(queue)
		}
	}
	wg.Wait()

	if recursion == 0 {
		return resultString, nil
	}

	if recursion == 1 {
		// Sort the result list by name.
		sort.Slice(resultList, func(i, j int) bool {
			return resultList[i].Name < resultList[j].Name
		})

		return resultList, nil
	}

	// Sort the result list by name.
	sort.Slice(resultFullList, func(i, j int) bool {
		return resultFullList[i].Name < resultFullList[j].Name
	})

	return resultFullList, nil
}

// Fetch information about the containers on the given remote node, using the
// rest API and with a timeout of 30 seconds.
func doContainersGetFromNode(project, node string, cert *shared.CertInfo) ([]api.Container, error) {
	f := func() ([]api.Container, error) {
		client, err := cluster.Connect(node, cert, true)
		if err != nil {
			return nil, errors.Wrapf(err, "Failed to connect to node %s", node)
		}

		client = client.UseProject(project)

		containers, err := client.GetContainers()
		if err != nil {
			return nil, errors.Wrapf(err, "Failed to get containers from node %s", node)
		}

		return containers, nil
	}

	timeout := time.After(30 * time.Second)
	done := make(chan struct{})

	var containers []api.Container
	var err error

	go func() {
		containers, err = f()
		done <- struct{}{}
	}()

	select {
	case <-timeout:
		err = fmt.Errorf("Timeout getting containers from node %s", node)
	case <-done:
	}

	return containers, err
}

func doContainersFullGetFromNode(project, node string, cert *shared.CertInfo) ([]api.ContainerFull, error) {
	f := func() ([]api.ContainerFull, error) {
		client, err := cluster.Connect(node, cert, true)
		if err != nil {
			return nil, errors.Wrapf(err, "Failed to connect to node %s", node)
		}

		client = client.UseProject(project)

		containers, err := client.GetContainersFull()
		if err != nil {
			return nil, errors.Wrapf(err, "Failed to get containers from node %s", node)
		}

		return containers, nil
	}

	timeout := time.After(30 * time.Second)
	done := make(chan struct{})

	var containers []api.ContainerFull
	var err error

	go func() {
		containers, err = f()
		done <- struct{}{}
	}()

	select {
	case <-timeout:
		err = fmt.Errorf("Timeout getting containers from node %s", node)
	case <-done:
	}

	return containers, err
}
<commit_msg>lxd/containers/get: Makes /1.0/containers filter by instance type container<commit_after>package main

import (
	"fmt"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/gorilla/mux"
	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/db/query"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/version"
)

func containersGet(d *Daemon, r *http.Request) Response {
	for i := 0; i < 100; i++ {
		result, err := doContainersGet(d, r)
		if err == nil {
			return SyncResponse(true, result)
		}
		if !query.IsRetriableError(err) {
			logger.Debugf("DBERR: containersGet: error %q", err)
			return SmartError(err)
		}
		// 100 ms may seem drastic, but we really don't want to thrash
		// perhaps we should use a random amount
		time.Sleep(100 * time.Millisecond)
	}

	logger.Debugf("DBERR: containersGet, db is locked")
	logger.Debugf(logger.GetStack())
	return InternalError(fmt.Errorf("DB is locked"))
}

func doContainersGet(d *Daemon, r *http.Request) (interface{}, error) {
	resultString := []string{}
	resultList := []*api.Container{}
	resultFullList := []*api.ContainerFull{}
	resultMu := sync.Mutex{}

	// Instance type.
	instanceType := instance.TypeAny
	if strings.HasPrefix(mux.CurrentRoute(r).GetName(), "container") {
		instanceType = instance.TypeContainer
	}

	// Parse the recursion field
	recursionStr := r.FormValue("recursion")

	recursion, err := strconv.Atoi(recursionStr)
	if err != nil {
		recursion = 0
	}

	// Parse the project field
	project := projectParam(r)

	// Get the list and location of all containers
	var result map[string][]string // Containers by node address
	var nodes map[string]string    // Node names by container
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error

		result, err = tx.ContainersListByNodeAddress(project, instanceType)
		if err != nil {
			return err
		}

		nodes, err = tx.ContainersByNodeName(project, instanceType)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return []string{}, err
	}

	// Get the local containers
	nodeCts := map[string]container{}
	if recursion > 0 {
		cts, err := containerLoadNodeProjectAll(d.State(), project, instanceType)
		if err != nil {
			return nil, err
		}

		for _, ct := range cts {
			nodeCts[ct.Name()] = ct
		}
	}

	// Append containers to list and handle errors
	resultListAppend := func(name string, c api.Container, err error) {
		if err != nil {
			c = api.Container{
				Name:       name,
				Status:     api.Error.String(),
				StatusCode: api.Error,
				Location:   nodes[name],
			}
		}
		resultMu.Lock()
		resultList = append(resultList, &c)
		resultMu.Unlock()
	}

	resultFullListAppend := func(name string, c api.ContainerFull, err error) {
		if err != nil {
			c = api.ContainerFull{Container: api.Container{
				Name:       name,
				Status:     api.Error.String(),
				StatusCode: api.Error,
				Location:   nodes[name],
			}}
		}
		resultMu.Lock()
		resultFullList = append(resultFullList, &c)
		resultMu.Unlock()
	}

	// Get the data
	wg := sync.WaitGroup{}
	for address, containers := range result {
		// If this is an internal request from another cluster node,
		// ignore containers from other nodes, and return only the ones
		// on this node
		if isClusterNotification(r) && address != "" {
			continue
		}

		// Mark containers on unavailable nodes as down
		if recursion > 0 && address == "0.0.0.0" {
			for _, container := range containers {
				if recursion == 1 {
					resultListAppend(container, api.Container{}, fmt.Errorf("unavailable"))
				} else {
					resultFullListAppend(container, api.ContainerFull{}, fmt.Errorf("unavailable"))
				}
			}

			continue
		}

		// For recursion requests we need to fetch the state of remote
		// containers from their respective nodes.
		if recursion > 0 && address != "" && !isClusterNotification(r) {
			wg.Add(1)
			go func(address string, containers []string) {
				defer wg.Done()
				cert := d.endpoints.NetworkCert()

				if recursion == 1 {
					cs, err := doContainersGetFromNode(project, address, cert, instanceType)
					if err != nil {
						for _, name := range containers {
							resultListAppend(name, api.Container{}, err)
						}

						return
					}

					for _, c := range cs {
						resultListAppend(c.Name, c, nil)
					}

					return
				}

				cs, err := doContainersFullGetFromNode(project, address, cert, instanceType)
				if err != nil {
					for _, name := range containers {
						resultFullListAppend(name, api.ContainerFull{}, err)
					}

					return
				}

				for _, c := range cs {
					resultFullListAppend(c.Name, c, nil)
				}
			}(address, containers)

			continue
		}

		if recursion == 0 {
			for _, container := range containers {
				instancePath := "instances"
				if instanceType == instance.TypeContainer {
					instancePath = "containers"
				}
				url := fmt.Sprintf("/%s/%s/%s", version.APIVersion, instancePath, container)
				resultString = append(resultString, url)
			}
		} else {
			threads := 4
			if len(containers) < threads {
				threads = len(containers)
			}

			queue := make(chan string, threads)

			for i := 0; i < threads; i++ {
				wg.Add(1)

				go func() {
					for {
						container, more := <-queue
						if !more {
							break
						}

						if recursion == 1 {
							c, _, err := nodeCts[container].Render()
							if err != nil {
								resultListAppend(container, api.Container{}, err)
							} else {
								resultListAppend(container, *c.(*api.Container), err)
							}

							continue
						}

						c, _, err := nodeCts[container].RenderFull()
						if err != nil {
							resultFullListAppend(container, api.ContainerFull{}, err)
						} else {
							resultFullListAppend(container, *c, err)
						}
					}

					wg.Done()
				}()
			}

			for _, container := range containers {
				queue <- container
			}

			close(queue)
		}
	}
	wg.Wait()

	if recursion == 0 {
		return resultString, nil
	}

	if recursion == 1 {
		// Sort the result list by name.
		sort.Slice(resultList, func(i, j int) bool {
			return resultList[i].Name < resultList[j].Name
		})

		return resultList, nil
	}

	// Sort the result list by name.
	sort.Slice(resultFullList, func(i, j int) bool {
		return resultFullList[i].Name < resultFullList[j].Name
	})

	return resultFullList, nil
}

// Fetch information about the containers on the given remote node, using the
// rest API and with a timeout of 30 seconds.
func doContainersGetFromNode(project, node string, cert *shared.CertInfo, instanceType instance.Type) ([]api.Container, error) {
	f := func() ([]api.Container, error) {
		client, err := cluster.Connect(node, cert, true)
		if err != nil {
			return nil, errors.Wrapf(err, "Failed to connect to node %s", node)
		}

		client = client.UseProject(project)

		containers, err := client.GetContainers()
		if err != nil {
			return nil, errors.Wrapf(err, "Failed to get containers from node %s", node)
		}

		return containers, nil
	}

	timeout := time.After(30 * time.Second)
	done := make(chan struct{})

	var containers []api.Container
	var err error

	go func() {
		containers, err = f()
		done <- struct{}{}
	}()

	select {
	case <-timeout:
		err = fmt.Errorf("Timeout getting containers from node %s", node)
	case <-done:
	}

	return containers, err
}

func doContainersFullGetFromNode(project, node string, cert *shared.CertInfo, instanceType instance.Type) ([]api.ContainerFull, error) {
	f := func() ([]api.ContainerFull, error) {
		client, err := cluster.Connect(node, cert, true)
		if err != nil {
			return nil, errors.Wrapf(err, "Failed to connect to node %s", node)
		}

		client = client.UseProject(project)

		containers, err := client.GetContainersFull()
		if err != nil {
			return nil, errors.Wrapf(err, "Failed to get containers from node %s", node)
		}

		return containers, nil
	}

	timeout := time.After(30 * time.Second)
	done := make(chan struct{})

	var containers []api.ContainerFull
	var err error

	go func() {
		containers, err = f()
		done <- struct{}{}
	}()

	select {
	case <-timeout:
		err = fmt.Errorf("Timeout getting containers from node %s", node)
	case <-done:
	}

	return containers, err
}
<|endoftext|>