<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package httplog

import (
	"bufio"
	"context"
	"fmt"
	"net"
	"net/http"
	"runtime"
	"strings"
	"time"

	"k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/klog/v2"
)

// StacktracePred returns true if a stacktrace should be logged for this status.
type StacktracePred func(httpStatus int) (logStacktrace bool)

type logger interface {
	Addf(format string, data ...interface{})
}

type respLoggerContextKeyType int

// respLoggerContextKey is used to store the respLogger pointer in the request context.
const respLoggerContextKey respLoggerContextKeyType = iota

// Add a layer on top of ResponseWriter, so we can track latency and error
// message sources.
//
// TODO now that we're using go-restful, we shouldn't need to be wrapping
// the http.ResponseWriter. We can recover panics from go-restful, and
// the logging value is questionable.
type respLogger struct {
	hijacked       bool
	statusRecorded bool
	status         int
	statusStack    string
	addedInfo      strings.Builder
	startTime      time.Time

	captureErrorOutput bool

	req *http.Request
	w   http.ResponseWriter

	logStacktracePred StacktracePred
}

// Simple logger that logs immediately when Addf is called
type passthroughLogger struct{}

//lint:ignore SA1019 Interface implementation check to make sure we don't drop CloseNotifier again
var _ http.CloseNotifier = &respLogger{}

// Addf logs info immediately.
func (passthroughLogger) Addf(format string, data ...interface{}) {
	klog.V(2).Info(fmt.Sprintf(format, data...))
}

// DefaultStacktracePred is the default implementation of StacktracePred.
func DefaultStacktracePred(status int) bool {
	return (status < http.StatusOK || status >= http.StatusInternalServerError) && status != http.StatusSwitchingProtocols
}

// WithLogging wraps the handler with logging.
func WithLogging(handler http.Handler, pred StacktracePred) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		ctx := req.Context()
		if old := respLoggerFromRequest(req); old != nil {
			panic("multiple WithLogging calls!")
		}

		startTime := time.Now()
		if receivedTimestamp, ok := request.ReceivedTimestampFrom(ctx); ok {
			startTime = receivedTimestamp
		}

		rl := newLoggedWithStartTime(req, w, startTime).StacktraceWhen(pred)
		req = req.WithContext(context.WithValue(ctx, respLoggerContextKey, rl))

		if klog.V(3).Enabled() {
			defer func() { klog.InfoS("HTTP", rl.LogArgs()...) }()
		}
		handler.ServeHTTP(rl, req)
	})
}

// respLoggerFromContext returns the respLogger or nil.
func respLoggerFromContext(ctx context.Context) *respLogger {
	val := ctx.Value(respLoggerContextKey)
	if rl, ok := val.(*respLogger); ok {
		return rl
	}
	return nil
}

func respLoggerFromRequest(req *http.Request) *respLogger {
	return respLoggerFromContext(req.Context())
}

func newLoggedWithStartTime(req *http.Request, w http.ResponseWriter, startTime time.Time) *respLogger {
	return &respLogger{
		startTime:         startTime,
		req:               req,
		w:                 w,
		logStacktracePred: DefaultStacktracePred,
	}
}

// newLogged turns a normal response writer into a logged response writer.
func newLogged(req *http.Request, w http.ResponseWriter) *respLogger {
	return newLoggedWithStartTime(req, w, time.Now())
}

// LogOf returns the logger hiding in w. If there is not an existing logger
// then a passthroughLogger will be created which will log to stdout immediately
// when Addf is called.
func LogOf(req *http.Request, w http.ResponseWriter) logger {
	if rl := respLoggerFromRequest(req); rl != nil {
		return rl
	}
	return &passthroughLogger{}
}

// Unlogged returns the original ResponseWriter, or w if it is not our inserted logger.
func Unlogged(req *http.Request, w http.ResponseWriter) http.ResponseWriter {
	if rl := respLoggerFromRequest(req); rl != nil {
		return rl.w
	}
	return w
}

// StacktraceWhen sets the stacktrace logging predicate, which decides when to log a stacktrace.
// There's a default, so you don't need to call this unless you don't like the default.
func (rl *respLogger) StacktraceWhen(pred StacktracePred) *respLogger {
	rl.logStacktracePred = pred
	return rl
}

// StatusIsNot returns a StacktracePred which will cause stacktraces to be logged
// for any status *not* in the given list.
func StatusIsNot(statuses ...int) StacktracePred {
	statusesNoTrace := map[int]bool{}
	for _, s := range statuses {
		statusesNoTrace[s] = true
	}
	return func(status int) bool {
		_, ok := statusesNoTrace[status]
		return !ok
	}
}

// Addf adds additional data to be logged with this request.
func (rl *respLogger) Addf(format string, data ...interface{}) {
	rl.addedInfo.WriteString("\n")
	rl.addedInfo.WriteString(fmt.Sprintf(format, data...))
}

func AddInfof(ctx context.Context, format string, data ...interface{}) {
	if rl := respLoggerFromContext(ctx); rl != nil {
		rl.Addf(format, data...)
	}
}

func (rl *respLogger) LogArgs() []interface{} {
	latency := time.Since(rl.startTime)
	auditID := request.GetAuditIDTruncated(rl.req)
	if rl.hijacked {
		return []interface{}{
			"verb", rl.req.Method,
			"URI", rl.req.RequestURI,
			"latency", latency,
			"userAgent", rl.req.UserAgent(),
			"audit-ID", auditID,
			"srcIP", rl.req.RemoteAddr,
			"hijacked", true,
		}
	}
	args := []interface{}{
		"verb", rl.req.Method,
		"URI", rl.req.RequestURI,
		"latency", latency,
		"userAgent", rl.req.UserAgent(),
		"audit-ID", auditID,
		"srcIP", rl.req.RemoteAddr,
		"resp", rl.status,
	}
	if len(rl.statusStack) > 0 {
		args = append(args, "statusStack", rl.statusStack)
	}

	info := rl.addedInfo.String()
	if len(info) > 0 {
		args = append(args, "addedInfo", info)
	}
	return args
}

// Header implements http.ResponseWriter.
func (rl *respLogger) Header() http.Header {
	return rl.w.Header()
}

// Write implements http.ResponseWriter.
func (rl *respLogger) Write(b []byte) (int, error) {
	if !rl.statusRecorded {
		rl.recordStatus(http.StatusOK) // Default if WriteHeader hasn't been called
	}
	if rl.captureErrorOutput {
		rl.Addf("logging error output: %q\n", string(b))
	}
	return rl.w.Write(b)
}

// Flush implements http.Flusher even if the underlying http.Writer doesn't implement it.
// Flush is used for streaming purposes and allows to flush buffered data to the client.
func (rl *respLogger) Flush() {
	if flusher, ok := rl.w.(http.Flusher); ok {
		flusher.Flush()
	} else if klog.V(2).Enabled() {
		klog.InfoDepth(1, fmt.Sprintf("Unable to convert %+v into http.Flusher", rl.w))
	}
}

// WriteHeader implements http.ResponseWriter.
func (rl *respLogger) WriteHeader(status int) {
	rl.recordStatus(status)
	rl.w.WriteHeader(status)
}

// Hijack implements http.Hijacker.
func (rl *respLogger) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	rl.hijacked = true
	return rl.w.(http.Hijacker).Hijack()
}

// CloseNotify implements http.CloseNotifier
func (rl *respLogger) CloseNotify() <-chan bool {
	//lint:ignore SA1019 There are places in the code base requiring the CloseNotifier interface to be implemented.
	return rl.w.(http.CloseNotifier).CloseNotify()
}

func (rl *respLogger) recordStatus(status int) {
	rl.status = status
	rl.statusRecorded = true
	if rl.logStacktracePred(status) {
		// Only log stacks for errors
		stack := make([]byte, 50*1024)
		stack = stack[:runtime.Stack(stack, false)]
		rl.statusStack = "\n" + string(stack)
		rl.captureErrorOutput = true
	} else {
		rl.statusStack = ""
	}
}
<commit_msg>hotfix(staging_apiserver_pkg_httplog): restore depth to log calls<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package httplog

import (
	"bufio"
	"context"
	"fmt"
	"net"
	"net/http"
	"runtime"
	"strings"
	"time"

	"k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/klog/v2"
)

// StacktracePred returns true if a stacktrace should be logged for this status.
type StacktracePred func(httpStatus int) (logStacktrace bool)

type logger interface {
	Addf(format string, data ...interface{})
}

type respLoggerContextKeyType int

// respLoggerContextKey is used to store the respLogger pointer in the request context.
const respLoggerContextKey respLoggerContextKeyType = iota

// Add a layer on top of ResponseWriter, so we can track latency and error
// message sources.
//
// TODO now that we're using go-restful, we shouldn't need to be wrapping
// the http.ResponseWriter. We can recover panics from go-restful, and
// the logging value is questionable.
type respLogger struct {
	hijacked       bool
	statusRecorded bool
	status         int
	statusStack    string
	addedInfo      strings.Builder
	startTime      time.Time

	captureErrorOutput bool

	req *http.Request
	w   http.ResponseWriter

	logStacktracePred StacktracePred
}

// Simple logger that logs immediately when Addf is called
type passthroughLogger struct{}

//lint:ignore SA1019 Interface implementation check to make sure we don't drop CloseNotifier again
var _ http.CloseNotifier = &respLogger{}

// Addf logs info immediately.
func (passthroughLogger) Addf(format string, data ...interface{}) {
	klog.V(2).Info(fmt.Sprintf(format, data...))
}

// DefaultStacktracePred is the default implementation of StacktracePred.
func DefaultStacktracePred(status int) bool {
	return (status < http.StatusOK || status >= http.StatusInternalServerError) && status != http.StatusSwitchingProtocols
}

// WithLogging wraps the handler with logging.
func WithLogging(handler http.Handler, pred StacktracePred) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		ctx := req.Context()
		if old := respLoggerFromRequest(req); old != nil {
			panic("multiple WithLogging calls!")
		}

		startTime := time.Now()
		if receivedTimestamp, ok := request.ReceivedTimestampFrom(ctx); ok {
			startTime = receivedTimestamp
		}

		rl := newLoggedWithStartTime(req, w, startTime).StacktraceWhen(pred)
		req = req.WithContext(context.WithValue(ctx, respLoggerContextKey, rl))

		if klog.V(3).Enabled() {
			defer rl.Log()
		}
		handler.ServeHTTP(rl, req)
	})
}

// respLoggerFromContext returns the respLogger or nil.
func respLoggerFromContext(ctx context.Context) *respLogger {
	val := ctx.Value(respLoggerContextKey)
	if rl, ok := val.(*respLogger); ok {
		return rl
	}
	return nil
}

func respLoggerFromRequest(req *http.Request) *respLogger {
	return respLoggerFromContext(req.Context())
}

func newLoggedWithStartTime(req *http.Request, w http.ResponseWriter, startTime time.Time) *respLogger {
	return &respLogger{
		startTime:         startTime,
		req:               req,
		w:                 w,
		logStacktracePred: DefaultStacktracePred,
	}
}

// newLogged turns a normal response writer into a logged response writer.
func newLogged(req *http.Request, w http.ResponseWriter) *respLogger {
	return newLoggedWithStartTime(req, w, time.Now())
}

// LogOf returns the logger hiding in w. If there is not an existing logger
// then a passthroughLogger will be created which will log to stdout immediately
// when Addf is called.
func LogOf(req *http.Request, w http.ResponseWriter) logger {
	if rl := respLoggerFromRequest(req); rl != nil {
		return rl
	}
	return &passthroughLogger{}
}

// Unlogged returns the original ResponseWriter, or w if it is not our inserted logger.
func Unlogged(req *http.Request, w http.ResponseWriter) http.ResponseWriter {
	if rl := respLoggerFromRequest(req); rl != nil {
		return rl.w
	}
	return w
}

// StacktraceWhen sets the stacktrace logging predicate, which decides when to log a stacktrace.
// There's a default, so you don't need to call this unless you don't like the default.
func (rl *respLogger) StacktraceWhen(pred StacktracePred) *respLogger {
	rl.logStacktracePred = pred
	return rl
}

// StatusIsNot returns a StacktracePred which will cause stacktraces to be logged
// for any status *not* in the given list.
func StatusIsNot(statuses ...int) StacktracePred {
	statusesNoTrace := map[int]bool{}
	for _, s := range statuses {
		statusesNoTrace[s] = true
	}
	return func(status int) bool {
		_, ok := statusesNoTrace[status]
		return !ok
	}
}

// Addf adds additional data to be logged with this request.
func (rl *respLogger) Addf(format string, data ...interface{}) {
	rl.addedInfo.WriteString("\n")
	rl.addedInfo.WriteString(fmt.Sprintf(format, data...))
}

func AddInfof(ctx context.Context, format string, data ...interface{}) {
	if rl := respLoggerFromContext(ctx); rl != nil {
		rl.Addf(format, data...)
	}
}

// Log is intended to be called once at the end of your request handler, via defer
func (rl *respLogger) Log() {
	latency := time.Since(rl.startTime)
	auditID := request.GetAuditIDTruncated(rl.req)
	keysAndValues := []interface{}{
		"verb", rl.req.Method,
		"URI", rl.req.RequestURI,
		"latency", latency,
		"userAgent", rl.req.UserAgent(),
		"audit-ID", auditID,
		"srcIP", rl.req.RemoteAddr,
	}

	if rl.hijacked {
		keysAndValues = append(keysAndValues, "hijacked", true)
	} else {
		keysAndValues = append(keysAndValues, "resp", rl.status)
		if len(rl.statusStack) > 0 {
			keysAndValues = append(keysAndValues, "statusStack", rl.statusStack)
		}
		info := rl.addedInfo.String()
		if len(info) > 0 {
			keysAndValues = append(keysAndValues, "addedInfo", info)
		}
	}

	klog.InfoSDepth(1, "HTTP", keysAndValues...)
}

// Header implements http.ResponseWriter.
func (rl *respLogger) Header() http.Header {
	return rl.w.Header()
}

// Write implements http.ResponseWriter.
func (rl *respLogger) Write(b []byte) (int, error) {
	if !rl.statusRecorded {
		rl.recordStatus(http.StatusOK) // Default if WriteHeader hasn't been called
	}
	if rl.captureErrorOutput {
		rl.Addf("logging error output: %q\n", string(b))
	}
	return rl.w.Write(b)
}

// Flush implements http.Flusher even if the underlying http.Writer doesn't implement it.
// Flush is used for streaming purposes and allows to flush buffered data to the client.
func (rl *respLogger) Flush() {
	if flusher, ok := rl.w.(http.Flusher); ok {
		flusher.Flush()
	} else if klog.V(2).Enabled() {
		klog.InfoDepth(1, fmt.Sprintf("Unable to convert %+v into http.Flusher", rl.w))
	}
}

// WriteHeader implements http.ResponseWriter.
func (rl *respLogger) WriteHeader(status int) {
	rl.recordStatus(status)
	rl.w.WriteHeader(status)
}

// Hijack implements http.Hijacker.
func (rl *respLogger) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	rl.hijacked = true
	return rl.w.(http.Hijacker).Hijack()
}

// CloseNotify implements http.CloseNotifier
func (rl *respLogger) CloseNotify() <-chan bool {
	//lint:ignore SA1019 There are places in the code base requiring the CloseNotifier interface to be implemented.
	return rl.w.(http.CloseNotifier).CloseNotify()
}

func (rl *respLogger) recordStatus(status int) {
	rl.status = status
	rl.statusRecorded = true
	if rl.logStacktracePred(status) {
		// Only log stacks for errors
		stack := make([]byte, 50*1024)
		stack = stack[:runtime.Stack(stack, false)]
		rl.statusStack = "\n" + string(stack)
		rl.captureErrorOutput = true
	} else {
		rl.statusStack = ""
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"net/http"
	"os"

	"github.com/gorilla/mux"
	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
	"github.com/matrix-org/dendrite/clientapi/producers"
	"github.com/matrix-org/dendrite/common"
	"github.com/matrix-org/dendrite/common/config"
	"github.com/matrix-org/dendrite/common/keydb"
	"github.com/matrix-org/gomatrixserverlib"

	mediaapi_routing "github.com/matrix-org/dendrite/mediaapi/routing"
	mediaapi_storage "github.com/matrix-org/dendrite/mediaapi/storage"

	roomserver_alias "github.com/matrix-org/dendrite/roomserver/alias"
	roomserver_input "github.com/matrix-org/dendrite/roomserver/input"
	roomserver_query "github.com/matrix-org/dendrite/roomserver/query"
	roomserver_storage "github.com/matrix-org/dendrite/roomserver/storage"

	clientapi_consumers "github.com/matrix-org/dendrite/clientapi/consumers"
	clientapi_routing "github.com/matrix-org/dendrite/clientapi/routing"

	syncapi_consumers "github.com/matrix-org/dendrite/syncapi/consumers"
	syncapi_routing "github.com/matrix-org/dendrite/syncapi/routing"
	syncapi_storage "github.com/matrix-org/dendrite/syncapi/storage"
	syncapi_sync "github.com/matrix-org/dendrite/syncapi/sync"
	syncapi_types "github.com/matrix-org/dendrite/syncapi/types"

	federationapi_routing "github.com/matrix-org/dendrite/federationapi/routing"

	federationsender_consumers "github.com/matrix-org/dendrite/federationsender/consumers"
	"github.com/matrix-org/dendrite/federationsender/queue"
	federationsender_storage "github.com/matrix-org/dendrite/federationsender/storage"

	log "github.com/Sirupsen/logrus"
	sarama "gopkg.in/Shopify/sarama.v1"
)

var (
	logDir        = os.Getenv("LOG_DIR")
	configPath    = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.")
	httpBindAddr  = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server")
	httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server")
)

func main() {
	common.SetupLogging(logDir)

	flag.Parse()

	if *configPath == "" {
		log.Fatal("--config must be supplied")
	}
	cfg, err := config.Load(*configPath)
	if err != nil {
		log.Fatalf("Invalid config file: %s", err)
	}

	m := newMonolith(cfg)
	m.setupDatabases()
	m.setupFederation()
	m.setupRoomServer()
	m.setupProducers()
	m.setupNotifiers()
	m.setupConsumers()
	m.setupAPIs()

	// Expose the matrix APIs directly rather than putting them under a /api path.
	log.Fatal(http.ListenAndServe(*httpBindAddr, m.api))
}

// A monolith contains all the dendrite components.
// Some of the setup functions depend on previous setup functions, so they must
// be called in the same order as they are defined in the file.
type monolith struct {
	cfg *config.Dendrite
	api *mux.Router

	roomServerDB       *roomserver_storage.Database
	accountDB          *accounts.Database
	deviceDB           *devices.Database
	keyDB              *keydb.Database
	mediaAPIDB         *mediaapi_storage.Database
	syncAPIDB          *syncapi_storage.SyncServerDatabase
	federationSenderDB *federationsender_storage.Database

	federation *gomatrixserverlib.FederationClient
	keyRing    gomatrixserverlib.KeyRing

	inputAPI *roomserver_input.RoomserverInputAPI
	queryAPI *roomserver_query.RoomserverQueryAPI
	aliasAPI *roomserver_alias.RoomserverAliasAPI

	roomServerProducer *producers.RoomserverProducer
	userUpdateProducer *producers.UserUpdateProducer
	syncProducer       *producers.SyncAPIProducer

	syncAPINotifier *syncapi_sync.Notifier
}

func newMonolith(cfg *config.Dendrite) *monolith {
	return &monolith{cfg: cfg, api: mux.NewRouter()}
}

func (m *monolith) setupDatabases() {
	var err error
	m.roomServerDB, err = roomserver_storage.Open(string(m.cfg.Database.RoomServer))
	if err != nil {
		panic(err)
	}
	m.accountDB, err = accounts.NewDatabase(string(m.cfg.Database.Account), m.cfg.Matrix.ServerName)
	if err != nil {
		log.Panicf("Failed to setup account database(%q): %s", m.cfg.Database.Account, err.Error())
	}
	m.deviceDB, err = devices.NewDatabase(string(m.cfg.Database.Device), m.cfg.Matrix.ServerName)
	if err != nil {
		log.Panicf("Failed to setup device database(%q): %s", m.cfg.Database.Device, err.Error())
	}
	m.keyDB, err = keydb.NewDatabase(string(m.cfg.Database.ServerKey))
	if err != nil {
		log.Panicf("Failed to setup key database(%q): %s", m.cfg.Database.ServerKey, err.Error())
	}
	m.mediaAPIDB, err = mediaapi_storage.Open(string(m.cfg.Database.MediaAPI))
	if err != nil {
		log.Panicf("Failed to setup sync api database(%q): %s", m.cfg.Database.MediaAPI, err.Error())
	}
	m.syncAPIDB, err = syncapi_storage.NewSyncServerDatabase(string(m.cfg.Database.SyncAPI))
	if err != nil {
		log.Panicf("Failed to setup sync api database(%q): %s", m.cfg.Database.SyncAPI, err.Error())
	}
	m.federationSenderDB, err = federationsender_storage.NewDatabase(string(m.cfg.Database.FederationSender))
	if err != nil {
		log.Panicf("startup: failed to create federation sender database with data source %s : %s", m.cfg.Database.FederationSender, err)
	}
}

func (m *monolith) setupFederation() {
	m.federation = gomatrixserverlib.NewFederationClient(
		m.cfg.Matrix.ServerName, m.cfg.Matrix.KeyID, m.cfg.Matrix.PrivateKey,
	)

	m.keyRing = gomatrixserverlib.KeyRing{
		KeyFetchers: []gomatrixserverlib.KeyFetcher{
			// TODO: Use perspective key fetchers for production.
			&gomatrixserverlib.DirectKeyFetcher{Client: m.federation.Client},
		},
		KeyDatabase: m.keyDB,
	}
}

func (m *monolith) setupRoomServer() {
	kafkaProducer, err := sarama.NewSyncProducer(m.cfg.Kafka.Addresses, nil)
	if err != nil {
		panic(err)
	}

	m.inputAPI = &roomserver_input.RoomserverInputAPI{
		DB:                   m.roomServerDB,
		Producer:             kafkaProducer,
		OutputRoomEventTopic: string(m.cfg.Kafka.Topics.OutputRoomEvent),
	}

	m.queryAPI = &roomserver_query.RoomserverQueryAPI{
		DB: m.roomServerDB,
	}

	m.aliasAPI = &roomserver_alias.RoomserverAliasAPI{
		DB:       m.roomServerDB,
		Cfg:      m.cfg,
		InputAPI: m.inputAPI,
		QueryAPI: m.queryAPI,
	}
}

func (m *monolith) setupProducers() {
	var err error
	m.roomServerProducer = producers.NewRoomserverProducer(m.inputAPI)
	m.userUpdateProducer, err = producers.NewUserUpdateProducer(
		m.cfg.Kafka.Addresses, string(m.cfg.Kafka.Topics.UserUpdates),
	)
	if err != nil {
		log.Panicf("Failed to setup kafka producers(%q): %s", m.cfg.Kafka.Addresses, err)
	}
	m.syncProducer, err = producers.NewSyncAPIProducer(
		m.cfg.Kafka.Addresses, string(m.cfg.Kafka.Topics.OutputClientData),
	)
	if err != nil {
		log.Panicf("Failed to setup kafka producers(%q): %s", m.cfg.Kafka.Addresses, err)
	}
}

func (m *monolith) setupNotifiers() {
	pos, err := m.syncAPIDB.SyncStreamPosition()
	if err != nil {
		log.Panicf("startup: failed to get latest sync stream position : %s", err)
	}

	n := syncapi_sync.NewNotifier(syncapi_types.StreamPosition(pos))
	if err = n.Load(m.syncAPIDB); err != nil {
		log.Panicf("startup: failed to set up notifier: %s", err)
	}
}

func (m *monolith) setupConsumers() {
	clientAPIConsumer, err := clientapi_consumers.NewOutputRoomEvent(m.cfg, m.accountDB)
	if err != nil {
		log.Panicf("startup: failed to create room server consumer: %s", err)
	}
	if err = clientAPIConsumer.Start(); err != nil {
		log.Panicf("startup: failed to start room server consumer")
	}

	syncAPIRoomConsumer, err := syncapi_consumers.NewOutputRoomEvent(
		m.cfg, m.syncAPINotifier, m.syncAPIDB,
	)
	if err != nil {
		log.Panicf("startup: failed to create room server consumer: %s", err)
	}
	if err = syncAPIRoomConsumer.Start(); err != nil {
		log.Panicf("startup: failed to start room server consumer: %s", err)
	}

	syncAPIClientConsumer, err := syncapi_consumers.NewOutputClientData(
		m.cfg, m.syncAPINotifier, m.syncAPIDB,
	)
	if err != nil {
		log.Panicf("startup: failed to create client API server consumer: %s", err)
	}
	if err = syncAPIClientConsumer.Start(); err != nil {
		log.Panicf("startup: failed to start client API server consumer: %s", err)
	}

	federationSenderQueues := queue.NewOutgoingQueues(m.cfg.Matrix.ServerName, m.federation)

	federationSenderRoomConsumer, err := federationsender_consumers.NewOutputRoomEvent(
		m.cfg, federationSenderQueues, m.federationSenderDB,
	)
	if err != nil {
		log.WithError(err).Panicf("startup: failed to create room server consumer")
	}
	if err = federationSenderRoomConsumer.Start(); err != nil {
		log.WithError(err).Panicf("startup: failed to start room server consumer")
	}
}

func (m *monolith) setupAPIs() {
	clientapi_routing.Setup(
		m.api, http.DefaultClient, *m.cfg, m.roomServerProducer,
		m.queryAPI, m.aliasAPI, m.accountDB, m.deviceDB, m.federation, m.keyRing,
		m.userUpdateProducer, m.syncProducer,
	)

	mediaapi_routing.Setup(
		m.api, http.DefaultClient, m.cfg, m.mediaAPIDB,
	)

	syncapi_routing.Setup(m.api, syncapi_sync.NewRequestPool(
		m.syncAPIDB, m.syncAPINotifier, m.accountDB,
	), m.deviceDB)

	federationapi_routing.Setup(
		m.api, *m.cfg, m.queryAPI, m.roomServerProducer, m.keyRing, m.federation,
	)
}
<commit_msg>Fix bad initialisation of sync API notifier in monolith (#176)<commit_after>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"net/http"
	"os"

	"github.com/gorilla/mux"
	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
	"github.com/matrix-org/dendrite/clientapi/producers"
	"github.com/matrix-org/dendrite/common"
	"github.com/matrix-org/dendrite/common/config"
	"github.com/matrix-org/dendrite/common/keydb"
	"github.com/matrix-org/gomatrixserverlib"

	mediaapi_routing "github.com/matrix-org/dendrite/mediaapi/routing"
	mediaapi_storage "github.com/matrix-org/dendrite/mediaapi/storage"

	roomserver_alias "github.com/matrix-org/dendrite/roomserver/alias"
	roomserver_input "github.com/matrix-org/dendrite/roomserver/input"
	roomserver_query "github.com/matrix-org/dendrite/roomserver/query"
	roomserver_storage "github.com/matrix-org/dendrite/roomserver/storage"

	clientapi_consumers "github.com/matrix-org/dendrite/clientapi/consumers"
	clientapi_routing "github.com/matrix-org/dendrite/clientapi/routing"

	syncapi_consumers "github.com/matrix-org/dendrite/syncapi/consumers"
	syncapi_routing "github.com/matrix-org/dendrite/syncapi/routing"
	syncapi_storage "github.com/matrix-org/dendrite/syncapi/storage"
	syncapi_sync "github.com/matrix-org/dendrite/syncapi/sync"
	syncapi_types "github.com/matrix-org/dendrite/syncapi/types"

	federationapi_routing "github.com/matrix-org/dendrite/federationapi/routing"

	federationsender_consumers "github.com/matrix-org/dendrite/federationsender/consumers"
	"github.com/matrix-org/dendrite/federationsender/queue"
	federationsender_storage "github.com/matrix-org/dendrite/federationsender/storage"

	log "github.com/Sirupsen/logrus"
	sarama "gopkg.in/Shopify/sarama.v1"
)

var (
	logDir        = os.Getenv("LOG_DIR")
	configPath    = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.")
	httpBindAddr  = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server")
	httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server")
)

func main() {
	common.SetupLogging(logDir)

	flag.Parse()

	if *configPath == "" {
		log.Fatal("--config must be supplied")
	}
	cfg, err := config.Load(*configPath)
	if err != nil {
		log.Fatalf("Invalid config file: %s", err)
	}

	m := newMonolith(cfg)
	m.setupDatabases()
	m.setupFederation()
	m.setupRoomServer()
	m.setupProducers()
	m.setupNotifiers()
	m.setupConsumers()
	m.setupAPIs()

	// Expose the matrix APIs directly rather than putting them under a /api path.
	log.Fatal(http.ListenAndServe(*httpBindAddr, m.api))
}

// A monolith contains all the dendrite components.
// Some of the setup functions depend on previous setup functions, so they must
// be called in the same order as they are defined in the file.
type monolith struct {
	cfg *config.Dendrite
	api *mux.Router

	roomServerDB       *roomserver_storage.Database
	accountDB          *accounts.Database
	deviceDB           *devices.Database
	keyDB              *keydb.Database
	mediaAPIDB         *mediaapi_storage.Database
	syncAPIDB          *syncapi_storage.SyncServerDatabase
	federationSenderDB *federationsender_storage.Database

	federation *gomatrixserverlib.FederationClient
	keyRing    gomatrixserverlib.KeyRing

	inputAPI *roomserver_input.RoomserverInputAPI
	queryAPI *roomserver_query.RoomserverQueryAPI
	aliasAPI *roomserver_alias.RoomserverAliasAPI

	roomServerProducer *producers.RoomserverProducer
	userUpdateProducer *producers.UserUpdateProducer
	syncProducer       *producers.SyncAPIProducer

	syncAPINotifier *syncapi_sync.Notifier
}

func newMonolith(cfg *config.Dendrite) *monolith {
	return &monolith{cfg: cfg, api: mux.NewRouter()}
}

func (m *monolith) setupDatabases() {
	var err error
	m.roomServerDB, err = roomserver_storage.Open(string(m.cfg.Database.RoomServer))
	if err != nil {
		panic(err)
	}
	m.accountDB, err = accounts.NewDatabase(string(m.cfg.Database.Account), m.cfg.Matrix.ServerName)
	if err != nil {
		log.Panicf("Failed to setup account database(%q): %s", m.cfg.Database.Account, err.Error())
	}
	m.deviceDB, err = devices.NewDatabase(string(m.cfg.Database.Device), m.cfg.Matrix.ServerName)
	if err != nil {
		log.Panicf("Failed to setup device database(%q): %s", m.cfg.Database.Device, err.Error())
	}
	m.keyDB, err = keydb.NewDatabase(string(m.cfg.Database.ServerKey))
	if err != nil {
		log.Panicf("Failed to setup key database(%q): %s", m.cfg.Database.ServerKey, err.Error())
	}
	m.mediaAPIDB, err = mediaapi_storage.Open(string(m.cfg.Database.MediaAPI))
	if err != nil {
		log.Panicf("Failed to setup sync api database(%q): %s", m.cfg.Database.MediaAPI, err.Error())
	}
	m.syncAPIDB, err = syncapi_storage.NewSyncServerDatabase(string(m.cfg.Database.SyncAPI))
	if err != nil {
		log.Panicf("Failed to setup sync api database(%q): %s", m.cfg.Database.SyncAPI, err.Error())
	}
	m.federationSenderDB, err = federationsender_storage.NewDatabase(string(m.cfg.Database.FederationSender))
	if err != nil {
		log.Panicf("startup: failed to create federation sender database with data source %s : %s", m.cfg.Database.FederationSender, err)
	}
}

func (m *monolith) setupFederation() {
	m.federation = gomatrixserverlib.NewFederationClient(
		m.cfg.Matrix.ServerName, m.cfg.Matrix.KeyID, m.cfg.Matrix.PrivateKey,
	)

	m.keyRing = gomatrixserverlib.KeyRing{
		KeyFetchers: []gomatrixserverlib.KeyFetcher{
			// TODO: Use perspective key fetchers for production.
			&gomatrixserverlib.DirectKeyFetcher{Client: m.federation.Client},
		},
		KeyDatabase: m.keyDB,
	}
}

func (m *monolith) setupRoomServer() {
	kafkaProducer, err := sarama.NewSyncProducer(m.cfg.Kafka.Addresses, nil)
	if err != nil {
		panic(err)
	}

	m.inputAPI = &roomserver_input.RoomserverInputAPI{
		DB:                   m.roomServerDB,
		Producer:             kafkaProducer,
		OutputRoomEventTopic: string(m.cfg.Kafka.Topics.OutputRoomEvent),
	}

	m.queryAPI = &roomserver_query.RoomserverQueryAPI{
		DB: m.roomServerDB,
	}

	m.aliasAPI = &roomserver_alias.RoomserverAliasAPI{
		DB:       m.roomServerDB,
		Cfg:      m.cfg,
		InputAPI: m.inputAPI,
		QueryAPI: m.queryAPI,
	}
}

func (m *monolith) setupProducers() {
	var err error
	m.roomServerProducer = producers.NewRoomserverProducer(m.inputAPI)
	m.userUpdateProducer, err = producers.NewUserUpdateProducer(
		m.cfg.Kafka.Addresses, string(m.cfg.Kafka.Topics.UserUpdates),
	)
	if err != nil {
		log.Panicf("Failed to setup kafka producers(%q): %s", m.cfg.Kafka.Addresses, err)
	}
	m.syncProducer, err = producers.NewSyncAPIProducer(
		m.cfg.Kafka.Addresses, string(m.cfg.Kafka.Topics.OutputClientData),
	)
	if err != nil {
		log.Panicf("Failed to setup kafka producers(%q): %s", m.cfg.Kafka.Addresses, err)
	}
}

func (m *monolith) setupNotifiers() {
	pos, err := m.syncAPIDB.SyncStreamPosition()
	if err != nil {
		log.Panicf("startup: failed to get latest sync stream position : %s", err)
	}

	m.syncAPINotifier = syncapi_sync.NewNotifier(syncapi_types.StreamPosition(pos))
	if err = m.syncAPINotifier.Load(m.syncAPIDB); err != nil {
		log.Panicf("startup: failed to set up notifier: %s", err)
	}
}

func (m *monolith) setupConsumers() {
	clientAPIConsumer, err := clientapi_consumers.NewOutputRoomEvent(m.cfg, m.accountDB)
	if err != nil {
		log.Panicf("startup: failed to create room server consumer: %s", err)
	}
	if err = clientAPIConsumer.Start(); err != nil {
		log.Panicf("startup: failed to start room server consumer")
	}

	syncAPIRoomConsumer, err := syncapi_consumers.NewOutputRoomEvent(
		m.cfg, m.syncAPINotifier, m.syncAPIDB,
	)
	if err != nil {
		log.Panicf("startup: failed to create room server consumer: %s", err)
	}
	if err = syncAPIRoomConsumer.Start(); err != nil {
		log.Panicf("startup: failed to start room server consumer: %s", err)
	}

	syncAPIClientConsumer, err := syncapi_consumers.NewOutputClientData(
		m.cfg, m.syncAPINotifier, m.syncAPIDB,
	)
	if err != nil {
		log.Panicf("startup: failed to create client API server consumer: %s", err)
	}
	if err = syncAPIClientConsumer.Start(); err != nil {
		log.Panicf("startup: failed to start client API server consumer: %s", err)
	}

	federationSenderQueues := queue.NewOutgoingQueues(m.cfg.Matrix.ServerName, m.federation)

	federationSenderRoomConsumer, err := federationsender_consumers.NewOutputRoomEvent(
		m.cfg, federationSenderQueues, m.federationSenderDB,
	)
	if err != nil {
		log.WithError(err).Panicf("startup: failed to create room server consumer")
	}
	if err = federationSenderRoomConsumer.Start(); err != nil {
		log.WithError(err).Panicf("startup: failed to start room server consumer")
	}
}

func (m *monolith) setupAPIs() {
	clientapi_routing.Setup(
		m.api, http.DefaultClient, *m.cfg, m.roomServerProducer,
		m.queryAPI, m.aliasAPI, m.accountDB, m.deviceDB, m.federation, m.keyRing,
		m.userUpdateProducer, m.syncProducer,
	)

	mediaapi_routing.Setup(
		m.api, http.DefaultClient, m.cfg, m.mediaAPIDB,
	)

	syncapi_routing.Setup(m.api, syncapi_sync.NewRequestPool(
		m.syncAPIDB, m.syncAPINotifier, m.accountDB,
	), m.deviceDB)

	federationapi_routing.Setup(
		m.api, *m.cfg, m.queryAPI, m.roomServerProducer, m.keyRing, m.federation,
	)
}
<|endoftext|>


<commit_before>package kato

//-----------------------------------------------------------------------------
// Package factored import statement:
//-----------------------------------------------------------------------------

import (

	// Stdlib:
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
)

//-----------------------------------------------------------------------------
// WaitChan stuff:
//-----------------------------------------------------------------------------

// WaitChan is used to handle errors that occur in some goroutines.
type WaitChan struct {
	WaitGrp sync.WaitGroup
	ErrChan chan error
	EndChan chan bool
}

// NewWaitChan initializes a WaitChan struct.
func NewWaitChan(len int) *WaitChan {
	wch := new(WaitChan)
	wch.WaitGrp.Add(len)
	wch.ErrChan = make(chan error, 1)
	wch.EndChan = make(chan bool, 1)
	return wch
}

// WaitErr waits for any error or for all go routines to finish.
func (wch *WaitChan) WaitErr() error {

	// Put the wait group in a go routine:
	go func() {
		wch.WaitGrp.Wait()
		close(wch.EndChan)
	}()

	// This select will block:
	select {
	case <-wch.EndChan:
		return nil
	case err := <-wch.ErrChan:
		return err
	}
}

//-----------------------------------------------------------------------------
// func: CountNodes
//-----------------------------------------------------------------------------

// CountNodes returns the count of <role> nodes defined in <quads>.
func CountNodes(quads []string, role string) (count int) {

	// Default to zero:
	count = 0

	// Get the role count:
	for _, q := range quads {
		if strings.Contains(q, role) {
			s := strings.Split(q, ":")
			count, _ = strconv.Atoi(s[0])
			break
		}
	}

	return
}

//-----------------------------------------------------------------------------
// func: CreateDNSZones
//-----------------------------------------------------------------------------

// CreateDNSZones creates (int|ext).<domain> zones using <provider>.
func CreateDNSZones(wch *WaitChan, provider, apiKey, domain string) error {

	// Decrement:
	if wch != nil {
		defer wch.WaitGrp.Done()
	}

	// Forge the zone command:
	cmd := exec.Command("katoctl", provider,
		"--api-key", apiKey, "zone", "add",
		domain, "int."+domain, "ext."+domain)

	// Execute the zone command:
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	return nil
}

//-----------------------------------------------------------------------------
// func: ExecutePipeline
//-----------------------------------------------------------------------------

// ExecutePipeline takes two commands and pipes the stdout of the first one
// into the stdin of the second one. Returns the output as []byte.
func ExecutePipeline(cmd1, cmd2 *exec.Cmd) ([]byte, error) {

	var err error

	// Adjust the stderr:
	cmd1.Stderr = os.Stderr
	cmd2.Stderr = os.Stderr

	// Connect both commands:
	cmd2.Stdin, err = cmd1.StdoutPipe()
	if err != nil {
		return nil, err
	}

	// Get cmd2 stdout:
	stdout, err := cmd2.StdoutPipe()
	if err != nil {
		return nil, err
	}

	// Execute the pipeline:
	if err = cmd2.Start(); err != nil {
		return nil, err
	}
	if err = cmd1.Run(); err != nil {
		return nil, err
	}

	// Read the cmd2 output:
	out, err := ioutil.ReadAll(stdout)
	if err != nil {
		return nil, err
	}

	// Wait and return:
	if err = cmd2.Wait(); err != nil {
		return nil, err
	}

	return out, nil
}

//-----------------------------------------------------------------------------
// func: NewEtcdToken
//-----------------------------------------------------------------------------

// NewEtcdToken takes quorumCount and returns a valid etcd bootstrap token:
func NewEtcdToken(wch *WaitChan, quorumCount int, token *string) error {

	// Decrement:
	if wch != nil {
		defer wch.WaitGrp.Done()
	}

	// Request an etcd bootstrap token:
	res, err := http.Get("https://discovery.etcd.io/new?size=" + strconv.Itoa(quorumCount))
	if err != nil {
		return err
	}

	// Retrieve the token URL:
	tokenURL, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}

	// Call the close method:
	_ = res.Body.Close()

	// Return the token ID:
	slice := strings.Split(string(tokenURL), "/")
	*token = slice[len(slice)-1]
	return nil
}

//-----------------------------------------------------------------------------
// func: LoadState
//-----------------------------------------------------------------------------

// LoadState reads the current ClusterID state file and decodes its content
// into a data structure:
func LoadState(clusterID string) ([]byte, error) {

	// Load data from state file:
	stateFile := os.Getenv("HOME") + "/.kato/" + clusterID + ".json"
	raw, err := ioutil.ReadFile(stateFile)
	if err != nil {
		return nil, err
	}

	return raw, nil
}

//-----------------------------------------------------------------------------
// func: OffsetIP
//-----------------------------------------------------------------------------

// OffsetIP takes a CIDR and an offset and returns the IP address at the offset
// position starting at the beginning of the CIDR's subnet:
func OffsetIP(cidr string, offset int) string {

	// Parse the CIDR:
	ip1, ipnet, err := net.ParseCIDR(cidr)
	if err != nil {
		return ""
	}

	// Compute the IP:
	ip2 := ip1.Mask(ipnet.Mask)
	a := int(ipToI32(ip2[len(ip2)-4:]))

	// Return:
	return i32ToIP(int32(a + offset)).String()
}

func ipToI32(ip net.IP) int32 {
	ip = ip.To4()
	return int32(ip[0])<<24 | int32(ip[1])<<16 | int32(ip[2])<<8 | int32(ip[3])
}

func i32ToIP(a int32) net.IP {
	return net.IPv4(byte(a>>24), byte(a>>16), byte(a>>8), byte(a))
}
<commit_msg>Do not close the channel (reuse it)<commit_after>package kato

//-----------------------------------------------------------------------------
// Package factored import statement:
//-----------------------------------------------------------------------------

import (

	// Stdlib:
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
)

//-----------------------------------------------------------------------------
// WaitChan stuff:
//-----------------------------------------------------------------------------

// WaitChan is used to handle errors that occur in some goroutines.
type WaitChan struct {
	WaitGrp sync.WaitGroup
	ErrChan chan error
	EndChan chan bool
}

// NewWaitChan initializes a WaitChan struct.
func NewWaitChan(len int) *WaitChan {
	wch := new(WaitChan)
	wch.WaitGrp.Add(len)
	wch.ErrChan = make(chan error, 1)
	wch.EndChan = make(chan bool, 1)
	return wch
}

// WaitErr waits for any error or for all go routines to finish.
func (wch *WaitChan) WaitErr() error {

	// Put the wait group in a go routine:
	go func() {
		wch.WaitGrp.Wait()
		wch.EndChan <- true
	}()

	// This select will block:
	select {
	case <-wch.EndChan:
		return nil
	case err := <-wch.ErrChan:
		return err
	}
}

//-----------------------------------------------------------------------------
// func: CountNodes
//-----------------------------------------------------------------------------

// CountNodes returns the count of <role> nodes defined in <quads>.
func CountNodes(quads []string, role string) (count int) {

	// Default to zero:
	count = 0

	// Get the role count:
	for _, q := range quads {
		if strings.Contains(q, role) {
			s := strings.Split(q, ":")
			count, _ = strconv.Atoi(s[0])
			break
		}
	}

	return
}

//-----------------------------------------------------------------------------
// func: CreateDNSZones
//-----------------------------------------------------------------------------

// CreateDNSZones creates (int|ext).<domain> zones using <provider>.
func CreateDNSZones(wch *WaitChan, provider, apiKey, domain string) error {

	// Decrement:
	if wch != nil {
		defer wch.WaitGrp.Done()
	}

	// Forge the zone command:
	cmd := exec.Command("katoctl", provider,
		"--api-key", apiKey, "zone", "add",
		domain, "int."+domain, "ext."+domain)

	// Execute the zone command:
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	return nil
}

//-----------------------------------------------------------------------------
// func: ExecutePipeline
//-----------------------------------------------------------------------------

// ExecutePipeline takes two commands and pipes the stdout of the first one
// into the stdin of the second one. Returns the output as []byte.
func ExecutePipeline(cmd1, cmd2 *exec.Cmd) ([]byte, error) {

	var err error

	// Adjust the stderr:
	cmd1.Stderr = os.Stderr
	cmd2.Stderr = os.Stderr

	// Connect both commands:
	cmd2.Stdin, err = cmd1.StdoutPipe()
	if err != nil {
		return nil, err
	}

	// Get cmd2 stdout:
	stdout, err := cmd2.StdoutPipe()
	if err != nil {
		return nil, err
	}

	// Execute the pipeline:
	if err = cmd2.Start(); err != nil {
		return nil, err
	}
	if err = cmd1.Run(); err != nil {
		return nil, err
	}

	// Read the cmd2 output:
	out, err := ioutil.ReadAll(stdout)
	if err != nil {
		return nil, err
	}

	// Wait and return:
	if err = cmd2.Wait(); err != nil {
		return nil, err
	}

	return out, nil
}

//-----------------------------------------------------------------------------
// func: NewEtcdToken
//-----------------------------------------------------------------------------

// NewEtcdToken takes quorumCount and returns a valid etcd bootstrap token:
func NewEtcdToken(wch *WaitChan, quorumCount int, token *string) error {

	// Decrement:
	if wch != nil {
		defer wch.WaitGrp.Done()
	}

	// Request an etcd bootstrap token:
	res, err := http.Get("https://discovery.etcd.io/new?size=" + strconv.Itoa(quorumCount))
	if err != nil {
		return err
	}

	// Retrieve the token URL:
	tokenURL, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}

	// Call the close method:
	_ = res.Body.Close()

	// Return the token ID:
	slice := strings.Split(string(tokenURL), "/")
	*token = slice[len(slice)-1]
	return nil
}

//-----------------------------------------------------------------------------
// func: LoadState
//-----------------------------------------------------------------------------

// LoadState reads the current ClusterID state file and decodes its content
// into a data structure:
func LoadState(clusterID string) ([]byte, error) {

	// Load data from state file:
	stateFile := os.Getenv("HOME") + "/.kato/" + clusterID + ".json"
	raw, err := ioutil.ReadFile(stateFile)
	if err != nil {
		return nil, err
	}

	return raw, nil
}

//-----------------------------------------------------------------------------
// func: OffsetIP
//-----------------------------------------------------------------------------

// OffsetIP takes a CIDR and an offset and returns the IP address at the offset
// position starting at the beginning of the CIDR's subnet:
func OffsetIP(cidr string, offset int) string {

	// Parse the CIDR:
	ip1, ipnet, err := net.ParseCIDR(cidr)
	if err != nil {
		return ""
	}

	// Compute the IP:
	ip2 := ip1.Mask(ipnet.Mask)
	a := int(ipToI32(ip2[len(ip2)-4:]))

	// Return:
	return i32ToIP(int32(a + offset)).String()
}

func ipToI32(ip net.IP) int32 {
	ip = ip.To4()
	return int32(ip[0])<<24 | int32(ip[1])<<16 | int32(ip[2])<<8 | int32(ip[3])
}

func i32ToIP(a int32) net.IP {
	return net.IPv4(byte(a>>24), byte(a>>16), byte(a>>8), byte(a))
}
<|endoftext|>


<commit_before>package driver

import (
	// "github.com/docker/libnetwork/ipams/remote"
	// "github.com/docker/libnetwork/ipams/remote/api"
	log "github.com/Sirupsen/logrus"
	"github.com/docker/go-plugins-helpers/ipam"
	"github.com/docker/libnetwork/datastore"
	"net"
)

type IPAMDriver struct {
	Addresses ipam.AddressSpacesResponse
	Alloc     *Allocator
}

func (ipd *IPAMDriver) GetCapabilities() (*ipam.CapabilitiesResponse, error) {
	log.Debugln("GetCapabilities() called")
	return &ipam.CapabilitiesResponse{RequiresMACAddress: true}, nil
}

// GetDefaultAddressSpaces returns the default local and global address spaces for this ipam
func (ipd *IPAMDriver) GetDefaultAddressSpaces() (*ipam.AddressSpacesResponse, error) {
	log.Debugln("GetDefaultAddressSpaces() called")
	return &ipd.Addresses, nil
}

// RequestPool returns an address pool along with its unique id. Address space is a mandatory field
// which denotes a set of non-overlapping pools. pool describes the pool of addresses in CIDR notation.
// subpool indicates a smaller range of addresses from the pool, for now it is specified in CIDR notation.
// Both pool and subpool are non mandatory fields. When they are not specified, Ipam driver may choose to
// return a self chosen pool for this request. In such case the v6 flag needs to be set appropriately so
// that the driver would return the expected ip version pool.
func (ipd *IPAMDriver) RequestPool(pool *ipam.RequestPoolRequest) (*ipam.RequestPoolResponse, error) {
	log.Debugf("RequestPool called with argument %#v", pool)
	key, nw, data, err := ipd.Alloc.RequestPool(pool.AddressSpace, pool.Pool, pool.SubPool, pool.Options, pool.V6)
	if err != nil {
		log.Errorf("RequestPool returned error: %s", err.Error())
		return nil, err
	}
	resp := new(ipam.RequestPoolResponse)
	resp.Pool = nw.String()
	resp.PoolID = key
	resp.Data = data
	log.Debugf("RequestPool returning %#v", resp)
	return resp, nil

}

// Release the address from the specified pool ID
func (ipd *IPAMDriver) ReleasePool(req *ipam.ReleasePoolRequest) error {
	log.Debugf("ReleasePool called with %#v", req)
	err := ipd.Alloc.ReleasePool(req.PoolID)
	if err != nil {
		log.Errorf("Error returned from ReleasePool: %s", err.Error())
	}
	return err
}

// Request an Address
func (ipd *IPAMDriver) RequestAddress(req *ipam.RequestAddressRequest) (*ipam.RequestAddressResponse, error) {
	log.Debugf("RequestAddress called with %#v", req)
	ip := net.ParseIP(req.Address)
	newip, data, err := ipd.Alloc.RequestAddress(req.PoolID, ip, req.Options)
	if err != nil {
		log.Errorf("error returned from RequestAddress: %s", err.Error())
		return nil, err
	}
	resp := new(ipam.RequestAddressResponse)
	resp.Address = newip.String()
	resp.Data = data
	log.Debugf("RequestAddress returning %#v", resp)
	return resp, nil
}

func (ipd *IPAMDriver) ReleaseAddress(req *ipam.ReleaseAddressRequest) error {
	log.Debugf("ReleaseAddress called with %#v", req)

	err := ipd.Alloc.ReleaseAddress(req.PoolID, net.ParseIP(req.Address))
	if err != nil {
		log.Errorf("Error returned from ReleaseAddress: %s", err.Error())
	}
	return err
}

func Init(Addresses *ipam.AddressSpacesResponse, cfg *datastore.ScopeCfg) (*IPAMDriver, error) {
	var err error
	ipd := new(IPAMDriver)
	log.Debugf("Init called")
	//	cfg := new(datastore.ScopeCfg)
	//	cfg.Client.Address = "cnllab01.dev.ena.net"
	//	cfg.Client.Provider = "consul"
	//	cfg.Client.Config = new(store.Config)
	//	cfg.Client.Config.ConnectionTimeout = 10 * time.Second
	if Addresses != nil {
		ipd.Addresses = *Addresses
	}
	ds, err := datastore.NewDataStore(datastore.GlobalScope, cfg)
	if err != nil {
		log.Errorf("Error returned from init: %s", err.Error())
		return nil, err
	}

	ipd.Alloc, err = NewAllocator(ds, ds)
	if err != nil {
		log.Errorf("NewAllocator returned error: %s", err.Error())
		return nil, err
	}
	log.Debug("Init success")
	return ipd, nil

}
<commit_msg>fix issue with datastore initialization<commit_after>package driver

import (
	// "github.com/docker/libnetwork/ipams/remote"
	// "github.com/docker/libnetwork/ipams/remote/api"
	log "github.com/Sirupsen/logrus"
	"github.com/docker/go-plugins-helpers/ipam"
	"github.com/docker/libnetwork/datastore"
	"net"
)

type IPAMDriver struct {
	Addresses ipam.AddressSpacesResponse
	Alloc     *Allocator
}

func (ipd *IPAMDriver) GetCapabilities() (*ipam.CapabilitiesResponse, error) {
	log.Debugln("GetCapabilities() called")
	return &ipam.CapabilitiesResponse{RequiresMACAddress: true}, nil
}

// GetDefaultAddressSpaces returns the default local and global address spaces for this ipam
func (ipd *IPAMDriver) GetDefaultAddressSpaces() (*ipam.AddressSpacesResponse, error) {
	log.Debugln("GetDefaultAddressSpaces() called")
	return &ipd.Addresses, nil
}

// RequestPool returns an address pool along with its unique id. Address space is a mandatory field
// which denotes a set of non-overlapping pools. pool describes the pool of addresses in CIDR notation.
// subpool indicates a smaller range of addresses from the pool, for now it is specified in CIDR notation.
// Both pool and subpool are non mandatory fields. When they are not specified, Ipam driver may choose to
// return a self chosen pool for this request. In such case the v6 flag needs to be set appropriately so
// that the driver would return the expected ip version pool.
func (ipd *IPAMDriver) RequestPool(pool *ipam.RequestPoolRequest) (*ipam.RequestPoolResponse, error) {
	log.Debugf("RequestPool called with argument %#v", pool)
	key, nw, data, err := ipd.Alloc.RequestPool(pool.AddressSpace, pool.Pool, pool.SubPool, pool.Options, pool.V6)
	if err != nil {
		log.Errorf("RequestPool returned error: %s", err.Error())
		return nil, err
	}
	resp := new(ipam.RequestPoolResponse)
	resp.Pool = nw.String()
	resp.PoolID = key
	resp.Data = data
	log.Debugf("RequestPool returning %#v", resp)
	return resp, nil

}

// Release the address from the specified pool ID
func (ipd *IPAMDriver) ReleasePool(req *ipam.ReleasePoolRequest) error {
	log.Debugf("ReleasePool called with %#v", req)
	err := ipd.Alloc.ReleasePool(req.PoolID)
	if err != nil {
		log.Errorf("Error returned from ReleasePool: %s", err.Error())
	}
	return err
}

// Request an Address
func (ipd *IPAMDriver) RequestAddress(req *ipam.RequestAddressRequest) (*ipam.RequestAddressResponse, error) {
	log.Debugf("RequestAddress called with %#v", req)
	ip := net.ParseIP(req.Address)
	newip, data, err := ipd.Alloc.RequestAddress(req.PoolID, ip, req.Options)
	if err != nil {
		log.Errorf("error returned from RequestAddress: %s", err.Error())
		return nil, err
	}
	resp := new(ipam.RequestAddressResponse)
	resp.Address = newip.String()
	resp.Data = data
	log.Debugf("RequestAddress returning %#v", resp)
	return resp, nil
}

func (ipd *IPAMDriver) ReleaseAddress(req *ipam.ReleaseAddressRequest) error {
	log.Debugf("ReleaseAddress called with %#v", req)

	err := ipd.Alloc.ReleaseAddress(req.PoolID, net.ParseIP(req.Address))
	if err != nil {
		log.Errorf("Error returned from ReleaseAddress: %s", err.Error())
	}
	return err
}

func Init(Addresses *ipam.AddressSpacesResponse, cfg *datastore.ScopeCfg) (*IPAMDriver, error) {
	var err error
	ipd := new(IPAMDriver)
	log.Debugf("Init called")
	//	cfg := new(datastore.ScopeCfg)
	//	cfg.Client.Address = "cnllab01.dev.ena.net"
	//	cfg.Client.Provider = "consul"
	//	cfg.Client.Config = new(store.Config)
	//	cfg.Client.Config.ConnectionTimeout = 10 * time.Second
	if Addresses != nil {
		ipd.Addresses = *Addresses
	}
	dsg, err := datastore.NewDataStore(datastore.GlobalScope, cfg)

	if err != nil {
		log.Errorf("Error returned from init: %s", err.Error())
		return nil, err
	}

 	dsl, err := datastore.NewDataStore(datastore.LocalScope, cfg)

	if err != nil {
		log.Errorf("Error returned from init: %s", err.Error())
		return nil, err
	}

	ipd.Alloc, err = NewAllocator(dsg, dsl)
	if err != nil {
		log.Errorf("NewAllocator returned error: %s", err.Error())
		return nil, err
	}
	log.Debug("Init success")
	return ipd, nil

}
<|endoftext|>


<commit_before>package aws

import (
	mach "github.com/jeffjen/machine/lib/machine"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/codegangsta/cli"

	"fmt"
	"math/rand"
	"net"
	"os"
	"time"
)

func init() {
	rand.Seed(time.Now().Unix())
}

var (
	// AWS EC2 client object for establishing command
	svc *ec2.EC2

	// Instance Roster
	instList = make(mach.RegisteredInstances)
)

func NewCommand() cli.Command {
	return cli.Command{
		Name:  "aws",
		Usage: "Create and Manage AWS machine",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "region", EnvVar: "AWS_REGION", Usage: "AWS Region"},
			cli.StringFlag{Name: "key", EnvVar: "AWS_ACCESS_KEY_ID", Usage: "AWS access key"},
			cli.StringFlag{Name: "secret", EnvVar: "AWS_SECRET_ACCESS_KEY", Usage: "AWS secret key"},
			cli.StringFlag{Name: "token", EnvVar: "AWS_SESSION_TOKEN", Usage: "session token for temporary credentials"},
			cli.StringFlag{Name: "user", EnvVar: "MACHINE_USER", Usage: "Run command as user"},
			cli.StringFlag{Name: "cert", EnvVar: "MACHINE_CERT_FILE", Usage: "Private key to use in Authentication"},
		},
		Before: func(c *cli.Context) error {
			// bootstrap EC2 client with command line args
			cfg := aws.NewConfig()
			if region := c.String("region"); region != "" {
				cfg = cfg.WithRegion(region)
			}
			if id, secret, token := c.String("key"), c.String("secret"), c.String("token"); id != "" && secret != "" {
				cfg = cfg.WithCredentials(credentials.NewStaticCredentials(id, secret, token))
			}
			svc = ec2.New(session.New(cfg))
			return nil
		},
		Subcommands: []cli.Command{
			newConfigCommand(),
			newCreateCommand(),
			newStartCommand(),
			newStopCommand(),
			newRmCommand(),
			newImageCommand(),
		},
	}
}

func newStartCommand() cli.Command {
	return cli.Command{
		Name:  "start",
		Usage: "Start instance",
		Action: func(c *cli.Context) error {
			var name = c.Args().First()

			// Load from Instance Roster
			instList.Load()

			info, ok := instList[name]
			if !ok {
				fmt.Fprintln(os.Stderr, "Target machine not found")
				os.Exit(1)
			}

			_, err := svc.StartInstances(&ec2.StartInstancesInput{
				InstanceIds: []*string{
					aws.String(info.Id),
				},
			})
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}

			return nil
		},
	}
}

func newStopCommand() cli.Command {
	return cli.Command{
		Name:  "stop",
		Usage: "Stop instance",
		Action: func(c *cli.Context) error {
			var name = c.Args().First()

			// Load from Instance Roster
			instList.Load()

			info, ok := instList[name]
			if !ok {
				fmt.Fprintln(os.Stderr, "Target machine not found")
				os.Exit(1)
			}

			_, err := svc.StopInstances(&ec2.StopInstancesInput{
				InstanceIds: []*string{
					aws.String(info.Id),
				},
			})
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}

			return nil
		},
	}
}

func newRmCommand() cli.Command {
	return cli.Command{
		Name:  "rm",
		Usage: "Remove and Terminate instance",
		Action: func(c *cli.Context) error {
			var name = c.Args().First()

			// Load from Instance Roster to register and defer write back
			defer instList.Load().Dump()

			info, ok := instList[name]
			if !ok {
				fmt.Fprintln(os.Stderr, "Target machine not found")
				os.Exit(1)
			}

			_, err := svc.TerminateInstances(&ec2.TerminateInstancesInput{
				InstanceIds: []*string{
					aws.String(info.Id),
				},
			})
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}

			delete(instList, name)

			return nil
		},
	}
}

func newConfigCommand() cli.Command {
	return cli.Command{
		Name:  "config",
		Usage: "Configure AWS environment",
		Subcommands: []cli.Command{
			syncFromAWS(),
			getFromAWSConfig(),
		},
	}
}

func newCreateCommand() cli.Command {
	return cli.Command{
		Name:  "create",
		Usage: "Create a new EC2 instance",
		Flags: []cli.Flag{
			cli.BoolTFlag{Name: "use-docker", Usage: "Opt in to use Docker Engine"},
			cli.StringFlag{Name: "ami-id", Usage: "EC2 instance AMI ID"},
			cli.IntFlag{Name: "count", Value: 1, Usage: "EC2 instances to launch in this request"},
			cli.StringSliceFlag{Name: "group", Usage: "Network security group for user"},
			cli.StringFlag{Name: "iam-role", Usage: "EC2 IAM Role to apply"},
			cli.StringFlag{Name: "profile", Value: "default", Usage: "Name of the profile"},
			cli.IntFlag{Name: "root-size", Value: 8, Usage: "EC2 root volume size"},
			cli.StringFlag{Name: "ssh-key", Usage: "EC2 instance SSH KeyPair"},
			cli.BoolFlag{Name: "subnet-private", Usage: "Launch EC2 instance to internal subnet"},
			cli.StringFlag{Name: "subnet-id", Usage: "Launch EC2 instance to the specified subnet"},
			cli.StringSliceFlag{Name: "tag", Usage: "EC2 instance tag in the form field=value"},
			cli.StringFlag{Name: "type", Value: "t2.micro", Usage: "EC2 instance type"},
			cli.IntSliceFlag{Name: "volume-size", Usage: "EC2 EBS volume size"},
		},
		Action: func(c *cli.Context) error {
			var (
				profile = make(AWSProfile)

				user = c.GlobalString("user")
				cert = c.GlobalString("cert")

				name = c.Args().First()

				useDocker = c.Bool("use-docker")
			)

			if user == "" || cert == "" {
				fmt.Fprintln(os.Stderr, "Missing required remote auth info")
				os.Exit(1)
			}

			// Load from AWS configuration from last sync
			profile.Load()

			// Load from Instance Roster to register and defer write back
			defer instList.Load().Dump()

			if name == "" {
				fmt.Fprintln(os.Stderr, "Required argument `name` missing")
				os.Exit(1)
			} else if _, ok := instList[name]; ok {
				fmt.Fprintln(os.Stderr, "Machine exist")
				os.Exit(1)
			}

			region, ok := profile[c.GlobalString("region")]
			if !ok {
				fmt.Fprintln(os.Stderr, "Please run sync in the region of choice")
				os.Exit(1)
			}
			p, ok := region[c.String("profile")]
			if !ok {
				fmt.Fprintln(os.Stderr, "Unable to find matching VPC profile")
				os.Exit(1)
			}

			// Invoke EC2 launch procedure
			for state := range newEC2Inst(c, p, user, cert, name, useDocker) {
				if addr, _ := net.ResolveTCPAddr("tcp", *state.PublicIpAddress+":2376"); state.err == nil {
					fmt.Printf("%s - %s - Instance ID: %s\n", *state.PublicIpAddress, *state.PrivateIpAddress, *state.InstanceId)
					if useDocker {
						instList[state.name] = &mach.Instance{
							Id:         *state.InstanceId,
							Driver:     "aws",
							DockerHost: addr,
							State:      "running",
						}
					}
				} else {
					fmt.Fprintln(os.Stderr, state.err)
				}
			}

			return nil
		},
	}
}

func newImageCommand() cli.Command {
	return cli.Command{
		Name:  "register-ami",
		Usage: "Register an AMI from specification",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "instance-id", Usage: "EC2 instance ID"},
			cli.StringFlag{Name: "name", Usage: "EC2 AMI Name"},
			cli.StringFlag{Name: "desc", Usage: "EC2 AMI Description"},
		},
		Action: func(c *cli.Context) error {
			var (
				instId = c.String("instance-id")
				name   = c.String("name")
				desc   = c.String("desc")
			)

			resp, err := svc.CreateImage(&ec2.CreateImageInput{
				InstanceId:  aws.String(instId),
				Name:        aws.String(name),
				Description: aws.String(desc),
			})
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			} else {
				fmt.Println(*resp.ImageId)
			}

			return nil
		},
	}
}
<commit_msg>UPDATE: start/stop to AWS instance updates DockerHost and State<commit_after>package aws

import (
	mach "github.com/jeffjen/machine/lib/machine"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/codegangsta/cli"

	"fmt"
	"math/rand"
	"net"
	"os"
	"time"
)

func init() {
	rand.Seed(time.Now().Unix())
}

var (
	// AWS EC2 client object for establishing command
	svc *ec2.EC2

	// Instance Roster
	instList = make(mach.RegisteredInstances)
)

func NewCommand() cli.Command {
	return cli.Command{
		Name:  "aws",
		Usage: "Create and Manage AWS machine",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "region", EnvVar: "AWS_REGION", Usage: "AWS Region"},
			cli.StringFlag{Name: "key", EnvVar: "AWS_ACCESS_KEY_ID", Usage: "AWS access key"},
			cli.StringFlag{Name: "secret", EnvVar: "AWS_SECRET_ACCESS_KEY", Usage: "AWS secret key"},
			cli.StringFlag{Name: "token", EnvVar: "AWS_SESSION_TOKEN", Usage: "session token for temporary credentials"},
			cli.StringFlag{Name: "user", EnvVar: "MACHINE_USER", Usage: "Run command as user"},
			cli.StringFlag{Name: "cert", EnvVar: "MACHINE_CERT_FILE", Usage: "Private key to use in Authentication"},
		},
		Before: func(c *cli.Context) error {
			// bootstrap EC2 client with command line args
			cfg := aws.NewConfig()
			if region := c.String("region"); region != "" {
				cfg = cfg.WithRegion(region)
			}
			if id, secret, token := c.String("key"), c.String("secret"), c.String("token"); id != "" && secret != "" {
				cfg = cfg.WithCredentials(credentials.NewStaticCredentials(id, secret, token))
			}
			svc = ec2.New(session.New(cfg))
			return nil
		},
		Subcommands: []cli.Command{
			newConfigCommand(),
			newCreateCommand(),
			newStartCommand(),
			newStopCommand(),
			newRmCommand(),
			newImageCommand(),
		},
	}
}

func newStartCommand() cli.Command {
	return cli.Command{
		Name:  "start",
		Usage: "Start instance",
		Action: func(c *cli.Context) error {
			var name = c.Args().First()

			// Load from Instance Roster
			defer instList.Load().Dump()

			info, ok := instList[name]
			if !ok {
				fmt.Fprintln(os.Stderr, "Target machine not found")
				os.Exit(1)
			}

			_, err := svc.StartInstances(&ec2.StartInstancesInput{
				InstanceIds: []*string{
					aws.String(info.Id),
				},
			})
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}

			if state := <-ec2_WaitForReady(&info.Id); state.err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			} else {
				addr, _ := net.ResolveTCPAddr("tcp", *state.PublicIpAddress+":2376")
				info.DockerHost = addr
				info.State = "running"
			}

			return nil
		},
	}
}

func newStopCommand() cli.Command {
	return cli.Command{
		Name:  "stop",
		Usage: "Stop instance",
		Action: func(c *cli.Context) error {
			var name = c.Args().First()

			// Load from Instance Roster
			defer instList.Load().Dump()

			info, ok := instList[name]
			if !ok {
				fmt.Fprintln(os.Stderr, "Target machine not found")
				os.Exit(1)
			}

			_, err := svc.StopInstances(&ec2.StopInstancesInput{
				InstanceIds: []*string{
					aws.String(info.Id),
				},
			})
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}

			info.DockerHost = nil
			info.State = "stopped"

			return nil
		},
	}
}

func newRmCommand() cli.Command {
	return cli.Command{
		Name:  "rm",
		Usage: "Remove and Terminate instance",
		Action: func(c *cli.Context) error {
			var name = c.Args().First()

			// Load from Instance Roster to register and defer write back
			defer instList.Load().Dump()

			info, ok := instList[name]
			if !ok {
				fmt.Fprintln(os.Stderr, "Target machine not found")
				os.Exit(1)
			}

			_, err := svc.TerminateInstances(&ec2.TerminateInstancesInput{
				InstanceIds: []*string{
					aws.String(info.Id),
				},
			})
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}

			delete(instList, name)

			return nil
		},
	}
}

func newConfigCommand() cli.Command {
	return cli.Command{
		Name:  "config",
		Usage: "Configure AWS environment",
		Subcommands: []cli.Command{
			syncFromAWS(),
			getFromAWSConfig(),
		},
	}
}

func newCreateCommand() cli.Command {
	return cli.Command{
		Name:  "create",
		Usage: "Create a new EC2 instance",
		Flags: []cli.Flag{
			cli.BoolTFlag{Name: "use-docker", Usage: "Opt in to use Docker Engine"},
			cli.StringFlag{Name: "ami-id", Usage: "EC2 instance AMI ID"},
			cli.IntFlag{Name: "count", Value: 1, Usage: "EC2 instances to launch in this request"},
			cli.StringSliceFlag{Name: "group", Usage: "Network security group for user"},
			cli.StringFlag{Name: "iam-role", Usage: "EC2 IAM Role to apply"},
			cli.StringFlag{Name: "profile", Value: "default", Usage: "Name of the profile"},
			cli.IntFlag{Name: "root-size", Value: 8, Usage: "EC2 root volume size"},
			cli.StringFlag{Name: "ssh-key", Usage: "EC2 instance SSH KeyPair"},
			cli.BoolFlag{Name: "subnet-private", Usage: "Launch EC2 instance to internal subnet"},
			cli.StringFlag{Name: "subnet-id", Usage: "Launch EC2 instance to the specified subnet"},
			cli.StringSliceFlag{Name: "tag", Usage: "EC2 instance tag in the form field=value"},
			cli.StringFlag{Name: "type", Value: "t2.micro", Usage: "EC2 instance type"},
			cli.IntSliceFlag{Name: "volume-size", Usage: "EC2 EBS volume size"},
		},
		Action: func(c *cli.Context) error {
			var (
				profile = make(AWSProfile)

				user = c.GlobalString("user")
				cert = c.GlobalString("cert")

				name = c.Args().First()

				useDocker = c.Bool("use-docker")
			)

			if user == "" || cert == "" {
				fmt.Fprintln(os.Stderr, "Missing required remote auth info")
				os.Exit(1)
			}

			// Load from AWS configuration from last sync
			profile.Load()

			// Load from Instance Roster to register and defer write back
			defer instList.Load().Dump()

			if name == "" {
				fmt.Fprintln(os.Stderr, "Required argument `name` missing")
				os.Exit(1)
			} else if _, ok := instList[name]; ok {
				fmt.Fprintln(os.Stderr, "Machine exist")
				os.Exit(1)
			}

			region, ok := profile[c.GlobalString("region")]
			if !ok {
				fmt.Fprintln(os.Stderr, "Please run sync in the region of choice")
				os.Exit(1)
			}
			p, ok := region[c.String("profile")]
			if !ok {
				fmt.Fprintln(os.Stderr, "Unable to find matching VPC profile")
				os.Exit(1)
			}

			// Invoke EC2 launch procedure
			for state := range newEC2Inst(c, p, user, cert, name, useDocker) {
				if addr, _ := net.ResolveTCPAddr("tcp", *state.PublicIpAddress+":2376"); state.err == nil {
					fmt.Printf("%s - %s - Instance ID: %s\n", *state.PublicIpAddress, *state.PrivateIpAddress, *state.InstanceId)
					if useDocker {
						instList[state.name] = &mach.Instance{
							Id:         *state.InstanceId,
							Driver:     "aws",
							DockerHost: addr,
							State:      "running",
						}
					}
				} else {
					fmt.Fprintln(os.Stderr, state.err)
				}
			}

			return nil
		},
	}
}

func newImageCommand() cli.Command {
	return cli.Command{
		Name:  "register-ami",
		Usage: "Register an AMI from specification",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "instance-id", Usage: "EC2 instance ID"},
			cli.StringFlag{Name: "name", Usage: "EC2 AMI Name"},
			cli.StringFlag{Name: "desc", Usage: "EC2 AMI Description"},
		},
		Action: func(c *cli.Context) error {
			var (
				instId = c.String("instance-id")
				name   = c.String("name")
				desc   = c.String("desc")
			)

			resp, err := svc.CreateImage(&ec2.CreateImageInput{
				InstanceId:  aws.String(instId),
				Name:        aws.String(name),
				Description: aws.String(desc),
			})
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			} else {
				fmt.Println(*resp.ImageId)
			}

			return nil
		},
	}
}
<|endoftext|>


<commit_before>package imap

import (
	"errors"
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/mitchellh/mapstructure"
	"github.com/yesnault/go-imap/imap"

	"github.com/ovh/venom"
	"github.com/ovh/venom/executors"
)

// Name for test imap
const Name = "imap"

var imapLogMask = imap.LogNone
var imapSafeLogMask = imap.LogNone

// New returns a new Test Exec
func New() venom.Executor {
	return &Executor{}
}

// Executor represents a Test Exec
type Executor struct {
	IMAPHost        string `json:"imaphost,omitempty" yaml:"imaphost,omitempty"`
	IMAPPort        string `json:"imapport,omitempty" yaml:"imapport,omitempty"`
	IMAPUser        string `json:"imapuser,omitempty" yaml:"imapuser,omitempty"`
	IMAPPassword    string `json:"imappassword,omitempty" yaml:"imappassword,omitempty"`
	MBox            string `json:"mbox,omitempty" yaml:"mbox,omitempty"`
	MBoxOnSuccess   string `json:"mboxonsuccess,omitempty" yaml:"mboxonsuccess,omitempty"`
	DeleteOnSuccess bool   `json:"deleteonsuccess,omitempty" yaml:"deleteonsuccess,omitempty"`
	SearchFrom      string `json:"searchfrom,omitempty" yaml:"searchfrom,omitempty"`
	SearchSubject   string `json:"searchsubject,omitempty" yaml:"searchsubject,omitempty"`
	SearchBody      string `json:"searchbody,omitempty" yaml:"searchbody,omitempty"`
}

// Mail contains an analyzed mail
type Mail struct {
	From    string
	Subject string
	UID     uint32
	Body    string
}

// Result represents a step result
type Result struct {
	Executor    Executor `json:"executor,omitempty" yaml:"executor,omitempty"`
	Err         string   `json:"error" yaml:"error"`
	Subject     string   `json:"subject,omitempty" yaml:"subject,omitempty"`
	Body        string   `json:"body,omitempty" yaml:"body,omitempty"`
	TimeSeconds float64  `json:"timeSeconds,omitempty" yaml:"timeSeconds,omitempty"`
	TimeHuman   string   `json:"timeHuman,omitempty" yaml:"timeHuman,omitempty"`
}

// GetDefaultAssertions return default assertions for type exec
func (Executor) GetDefaultAssertions() *venom.StepAssertions {
	return &venom.StepAssertions{Assertions: []string{"result.err ShouldNotExist"}}
}

// Run execute TestStep of type exec
func (Executor) Run(ctx venom.TestCaseContext, l venom.Logger, step venom.TestStep) (venom.ExecutorResult, error) {
	var t Executor
	if err := mapstructure.Decode(step, &t); err != nil {
		return nil, err
	}

	start := time.Now()

	result := Result{Executor: t}
	find, errs := t.getMail(l)
	if errs != nil {
		result.Err = errs.Error()
	}
	if find != nil {
		result.Subject = find.Subject
		result.Body = find.Body
	} else if result.Err == "" {
		result.Err = "searched mail not found"
	}

	elapsed := time.Since(start)
	result.TimeSeconds = elapsed.Seconds()
	result.TimeHuman = fmt.Sprintf("%s", elapsed)
	result.Executor.IMAPPassword = "****hidden****" // do not output password

	return executors.Dump(result)
}

func (e *Executor) getMail(l venom.Logger) (*Mail, error) {
	if e.SearchFrom == "" && e.SearchSubject == "" && e.SearchBody == "" {
		return nil, fmt.Errorf("You have to use searchfrom and/or searchsubject and/or searchbody")
	}

	c, errc := connect(e.IMAPHost, e.IMAPPort, e.IMAPUser, e.IMAPPassword)
	if errc != nil {
		return nil, fmt.Errorf("Error while connecting:%s", errc.Error())
	}
	defer c.Logout(5 * time.Second)

	var box string

	if e.MBox == "" {
		box = "INBOX"
	} else {
		box = e.MBox
	}

	count, err := queryCount(c, box)
	if err != nil {
		return nil, fmt.Errorf("Error while queryCount:%s", err.Error())
	}

	l.Debugf("count messages:%d", count)

	if count == 0 {
		return nil, errors.New("No message to fetch")
	}

	messages, err := fetch(c, box, count, l)
	if err != nil {
		return nil, fmt.Errorf("Error while feching messages:%s", err.Error())
	}

	for _, msg := range messages {
		m, erre := extract(msg, l)
		if erre != nil {
			return nil, erre
		}

		found, errs := e.isSearched(m)
		if errs != nil {
			return nil, errs
		}

		if found {
			if e.DeleteOnSuccess {
				l.Debugf("Delete message %s", m.UID)
				if err := m.delete(c); err != nil {
					return nil, err
				}
			} else if e.MBoxOnSuccess != "" {
				l.Debugf("Move to %s", e.MBoxOnSuccess)
				if err := m.move(c, e.MBoxOnSuccess); err != nil {
					return nil, err
				}
			}
			return m, nil
		}
	}

	return nil, errors.New("Mail not found")
}

func (e *Executor) isSearched(m *Mail) (bool, error) {
	if e.SearchFrom != "" {
		ma, erra := regexp.MatchString(e.SearchFrom, m.From)
		if erra != nil || !ma {
			return false, erra
		}
	}
	if e.SearchSubject != "" {
		mb, errb := regexp.MatchString(e.SearchSubject, m.Subject)
		if errb != nil || !mb {
			return false, errb
		}
	}
	if e.SearchBody != "" {
		mc, errc := regexp.MatchString(e.SearchBody, m.Body)
		if errc != nil || !mc {
			return false, errc
		}
	}
	return true, nil
}

func (m *Mail) move(c *imap.Client, mbox string) error {
	seq, _ := imap.NewSeqSet("")
	seq.AddNum(m.UID)

	if _, err := c.UIDMove(seq, mbox); err != nil {
		return fmt.Errorf("Error while move msg to %s, err:%s", mbox, err.Error())
	}
	return nil
}

func (m *Mail) delete(c *imap.Client) error {
	seq, _ := imap.NewSeqSet("")
	seq.AddNum(m.UID)

	if _, err := c.UIDStore(seq, "+FLAGS.SILENT", imap.NewFlagSet(`\Deleted`)); err != nil {
		return fmt.Errorf("Error while deleting msg, err: %s", err.Error())
	}
	if _, err := c.Expunge(nil); err != nil {
		return fmt.Errorf("Error while expunging messages: err: %s", err.Error())
	}
	return nil
}

func connect(host, port, imapUsername, imapPassword string) (*imap.Client, error) {
	if !strings.Contains(host, ":") {
		if port == "" {
			port = ":993"
		} else if port != "" && !strings.HasPrefix(port, ":") {
			port = ":" + port
		}
	}

	c, errd := imap.DialTLS(host+port, nil)
	if errd != nil {
		return nil, fmt.Errorf("Unable to dial: %s", errd)
	}

	if c.Caps["STARTTLS"] {
		if _, err := check(c.StartTLS(nil)); err != nil {
			return nil, fmt.Errorf("Unable to start TLS: %s\n", err)
		}
	}

	c.SetLogMask(imapSafeLogMask)
	if _, err := check(c.Login(imapUsername, imapPassword)); err != nil {
		return nil, fmt.Errorf("Unable to login: %s", err)
	}
	c.SetLogMask(imapLogMask)

	return c, nil
}

func fetch(c *imap.Client, box string, nb uint32, l venom.Logger) ([]imap.Response, error) {
	l.Debugf("call Select")
	if _, err := c.Select(box, false); err != nil {
		l.Errorf("Error with select %s", err.Error())
		return []imap.Response{}, err
	}
	defer c.Close(false)

	seqset, _ := imap.NewSeqSet("1:*")

	cmd, err := c.Fetch(seqset, "ENVELOPE", "RFC822.HEADER", "RFC822.TEXT", "UID")
	if err != nil {
		l.Errorf("Error with fetch:%s", err)
		return []imap.Response{}, err
	}

	messages := []imap.Response{}
	for cmd.InProgress() {
		// Wait for the next response (no timeout)
		c.Recv(-1)

		// Process command data
		for _, rsp := range cmd.Data {
			messages = append(messages, *rsp)
		}
		cmd.Data = nil
		c.Data = nil
	}
	l.Debugf("Nb messages fetch:%d", len(messages))
	return messages, nil
}

func queryCount(imapClient *imap.Client, box string) (uint32, error) {
	cmd, errc := check(imapClient.Status(box))
	if errc != nil {
		return 0, errc
	}

	var count uint32
	for _, result := range cmd.Data {
		mailboxStatus := result.MailboxStatus()
		if mailboxStatus != nil {
			count += mailboxStatus.Messages
		}
	}

	return count, nil
}

func check(cmd *imap.Command, erri error) (*imap.Command, error) {
	if erri != nil {
		return nil, erri
	}

	if _, err := cmd.Result(imap.OK); err != nil {
		return nil, err
	}

	return cmd, nil
}
<commit_msg>fix (plugin-venom): update vendor (#1479)<commit_after>package imap

import (
	"errors"
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/mitchellh/mapstructure"
	"github.com/yesnault/go-imap/imap"

	"github.com/ovh/venom"
	"github.com/ovh/venom/executors"
)

// Name for test imap
const Name = "imap"

var imapLogMask = imap.LogNone
var imapSafeLogMask = imap.LogNone

// New returns a new Test Exec
func New() venom.Executor {
	return &Executor{}
}

// Executor represents a Test Exec
type Executor struct {
	IMAPHost        string `json:"imaphost,omitempty" yaml:"imaphost,omitempty"`
	IMAPPort        string `json:"imapport,omitempty" yaml:"imapport,omitempty"`
	IMAPUser        string `json:"imapuser,omitempty" yaml:"imapuser,omitempty"`
	IMAPPassword    string `json:"imappassword,omitempty" yaml:"imappassword,omitempty"`
	MBox            string `json:"mbox,omitempty" yaml:"mbox,omitempty"`
	MBoxOnSuccess   string `json:"mboxonsuccess,omitempty" yaml:"mboxonsuccess,omitempty"`
	DeleteOnSuccess bool   `json:"deleteonsuccess,omitempty" yaml:"deleteonsuccess,omitempty"`
	SearchFrom      string `json:"searchfrom,omitempty" yaml:"searchfrom,omitempty"`
	SearchSubject   string `json:"searchsubject,omitempty" yaml:"searchsubject,omitempty"`
	SearchBody      string `json:"searchbody,omitempty" yaml:"searchbody,omitempty"`
}

// Mail contains an analyzed mail
type Mail struct {
	From    string
	Subject string
	UID     uint32
	Body    string
}

// Result represents a step result
type Result struct {
	Executor    Executor `json:"executor,omitempty" yaml:"executor,omitempty"`
	Err         string   `json:"error" yaml:"error"`
	Subject     string   `json:"subject,omitempty" yaml:"subject,omitempty"`
	Body        string   `json:"body,omitempty" yaml:"body,omitempty"`
	TimeSeconds float64  `json:"timeSeconds,omitempty" yaml:"timeSeconds,omitempty"`
	TimeHuman   string   `json:"timeHuman,omitempty" yaml:"timeHuman,omitempty"`
}

// GetDefaultAssertions return default assertions for type exec
func (Executor) GetDefaultAssertions() *venom.StepAssertions {
	return &venom.StepAssertions{Assertions: []string{"result.err ShouldNotExist"}}
}

// Run execute TestStep of type exec
func (Executor) Run(ctx venom.TestCaseContext, l venom.Logger, step venom.TestStep) (venom.ExecutorResult, error) {
	var t Executor
	if err := mapstructure.Decode(step, &t); err != nil {
		return nil, err
	}

	start := time.Now()

	result := Result{Executor: t}
	find, errs := t.getMail(l)
	if errs != nil {
		result.Err = errs.Error()
	}
	if find != nil {
		result.Subject = find.Subject
		result.Body = find.Body
	} else if result.Err == "" {
		result.Err = "searched mail not found"
	}

	elapsed := time.Since(start)
	result.TimeSeconds = elapsed.Seconds()
	result.TimeHuman = fmt.Sprintf("%s", elapsed)
	result.Executor.IMAPPassword = "****hidden****" // do not output password

	return executors.Dump(result)
}

func (e *Executor) getMail(l venom.Logger) (*Mail, error) {
	if e.SearchFrom == "" && e.SearchSubject == "" && e.SearchBody == "" {
		return nil, fmt.Errorf("You have to use searchfrom and/or searchsubject and/or searchbody")
	}

	c, errc := connect(e.IMAPHost, e.IMAPPort, e.IMAPUser, e.IMAPPassword)
	if errc != nil {
		return nil, fmt.Errorf("Error while connecting:%s", errc.Error())
	}
	defer c.Logout(5 * time.Second)

	var box string

	if e.MBox == "" {
		box = "INBOX"
	} else {
		box = e.MBox
	}

	count, err := queryCount(c, box)
	if err != nil {
		return nil, fmt.Errorf("Error while queryCount:%s", err.Error())
	}

	l.Debugf("count messages:%d", count)

	if count == 0 {
		return nil, errors.New("No message to fetch")
	}

	messages, err := fetch(c, box, count, l)
	if err != nil {
		return nil, fmt.Errorf("Error while feching messages:%s", err.Error())
	}
	defer c.Close(false)

	for _, msg := range messages {
		m, erre := extract(msg, l)
		if erre != nil {
			return nil, erre
		}

		found, errs := e.isSearched(m)
		if errs != nil {
			return nil, errs
		}

		if found {
			if e.DeleteOnSuccess {
				l.Debugf("Delete message %s", m.UID)
				if err := m.delete(c); err != nil {
					return nil, err
				}
			} else if e.MBoxOnSuccess != "" {
				l.Debugf("Move to %s", e.MBoxOnSuccess)
				if err := m.move(c, e.MBoxOnSuccess); err != nil {
					return nil, err
				}
			}
			return m, nil
		}
	}

	return nil, errors.New("Mail not found")
}

func (e *Executor) isSearched(m *Mail) (bool, error) {
	if e.SearchFrom != "" {
		ma, erra := regexp.MatchString(e.SearchFrom, m.From)
		if erra != nil || !ma {
			return false, erra
		}
	}
	if e.SearchSubject != "" {
		mb, errb := regexp.MatchString(e.SearchSubject, m.Subject)
		if errb != nil || !mb {
			return false, errb
		}
	}
	if e.SearchBody != "" {
		mc, errc := regexp.MatchString(e.SearchBody, m.Body)
		if errc != nil || !mc {
			return false, errc
		}
	}
	return true, nil
}

func (m *Mail) move(c *imap.Client, mbox string) error {
	seq, _ := imap.NewSeqSet("")
	seq.AddNum(m.UID)

	if _, err := c.UIDMove(seq, mbox); err != nil {
		return fmt.Errorf("Error while move msg to %s, err:%s", mbox, err.Error())
	}
	return nil
}

func (m *Mail) delete(c *imap.Client) error {
	seq, _ := imap.NewSeqSet("")
	seq.AddNum(m.UID)

	if _, err := c.UIDStore(seq, "+FLAGS.SILENT", imap.NewFlagSet(`\Deleted`)); err != nil {
		return fmt.Errorf("Error while deleting msg, err: %s", err.Error())
	}
	if _, err := c.Expunge(nil); err != nil {
		return fmt.Errorf("Error while expunging messages: err: %s", err.Error())
	}
	return nil
}

func connect(host, port, imapUsername, imapPassword string) (*imap.Client, error) {
	if !strings.Contains(host, ":") {
		if port == "" {
			port = ":993"
		} else if port != "" && !strings.HasPrefix(port, ":") {
			port = ":" + port
		}
	}

	c, errd := imap.DialTLS(host+port, nil)
	if errd != nil {
		return nil, fmt.Errorf("Unable to dial: %s", errd)
	}

	if c.Caps["STARTTLS"] {
		if _, err := check(c.StartTLS(nil)); err != nil {
			return nil, fmt.Errorf("Unable to start TLS: %s\n", err)
		}
	}

	c.SetLogMask(imapSafeLogMask)
	if _, err := check(c.Login(imapUsername, imapPassword)); err != nil {
		return nil, fmt.Errorf("Unable to login: %s", err)
	}
	c.SetLogMask(imapLogMask)

	return c, nil
}

func fetch(c *imap.Client, box string, nb uint32, l venom.Logger) ([]imap.Response, error) {
	l.Debugf("call Select")
	if _, err := c.Select(box, false); err != nil {
		l.Errorf("Error with select %s", err.Error())
		return []imap.Response{}, err
	}

	seqset, _ := imap.NewSeqSet("1:*")

	cmd, err := c.Fetch(seqset, "ENVELOPE", "RFC822.HEADER", "RFC822.TEXT", "UID")
	if err != nil {
		l.Errorf("Error with fetch:%s", err)
		return []imap.Response{}, err
	}

	messages := []imap.Response{}
	for cmd.InProgress() {
		// Wait for the next response (no timeout)
		c.Recv(-1)

		// Process command data
		for _, rsp := range cmd.Data {
			messages = append(messages, *rsp)
		}
		cmd.Data = nil
		c.Data = nil
	}
	l.Debugf("Nb messages fetch:%d", len(messages))
	return messages, nil
}

func queryCount(imapClient *imap.Client, box string) (uint32, error) {
	cmd, errc := check(imapClient.Status(box))
	if errc != nil {
		return 0, errc
	}

	var count uint32
	for _, result := range cmd.Data {
		mailboxStatus := result.MailboxStatus()
		if mailboxStatus != nil {
			count += mailboxStatus.Messages
		}
	}

	return count, nil
}

func check(cmd *imap.Command, erri error) (*imap.Command, error) {
	if erri != nil {
		return nil, erri
	}

	if _, err := cmd.Result(imap.OK); err != nil {
		return nil, err
	}

	return cmd, nil
}
<|endoftext|>


<commit_before><commit_msg>improve codegen verbose stuf<commit_after><|endoftext|>


<commit_before>package crypto

import (
	"encoding/binary"
	"errors"

	"github.com/lucas-clemente/fnv128a"
	"github.com/lucas-clemente/quic-go/internal/protocol"
)

// nullAEAD handles not-yet encrypted packets
type nullAEADFNV128a struct {
	perspective protocol.Perspective
}

var _ AEAD = &nullAEADFNV128a{}

// Open and verify the ciphertext
func (n *nullAEADFNV128a) Open(dst, src []byte, packetNumber protocol.PacketNumber, associatedData []byte) ([]byte, error) {
	if len(src) < 12 {
		return nil, errors.New("NullAEAD: ciphertext cannot be less than 12 bytes long")
	}

	hash := fnv128a.New()
	hash.Write(associatedData)
	hash.Write(src[12:])
	if n.perspective == protocol.PerspectiveServer {
		hash.Write([]byte("Client"))
	} else {
		hash.Write([]byte("Server"))
	}
	testHigh, testLow := hash.Sum128()

	low := binary.LittleEndian.Uint64(src)
	high := binary.LittleEndian.Uint32(src[8:])

	if uint32(testHigh&0xffffffff) != high || testLow != low {
		return nil, errors.New("NullAEAD: failed to authenticate received data")
	}
	return src[12:], nil
}

// Seal writes hash and ciphertext to the buffer
func (n *nullAEADFNV128a) Seal(dst, src []byte, packetNumber protocol.PacketNumber, associatedData []byte) []byte {
	if cap(dst) < 12+len(src) {
		dst = make([]byte, 12+len(src))
	} else {
		dst = dst[:12+len(src)]
	}

	hash := fnv128a.New()
	hash.Write(associatedData)
	hash.Write(src)

	if n.perspective == protocol.PerspectiveServer {
		hash.Write([]byte("Server"))
	} else {
		hash.Write([]byte("Client"))
	}

	high, low := hash.Sum128()

	copy(dst[12:], src)
	binary.LittleEndian.PutUint64(dst, low)
	binary.LittleEndian.PutUint32(dst[8:], uint32(high))
	return dst
}

func (n *nullAEADFNV128a) Overhead() int {
	return 12
}
<commit_msg>use the FNV 128a from the standard library<commit_after>package crypto

import (
	"bytes"
	"errors"
	"fmt"
	"hash/fnv"

	"github.com/lucas-clemente/quic-go/internal/protocol"
)

// nullAEAD handles not-yet encrypted packets
type nullAEADFNV128a struct {
	perspective protocol.Perspective
}

var _ AEAD = &nullAEADFNV128a{}

// Open and verify the ciphertext
func (n *nullAEADFNV128a) Open(dst, src []byte, packetNumber protocol.PacketNumber, associatedData []byte) ([]byte, error) {
	if len(src) < 12 {
		return nil, errors.New("NullAEAD: ciphertext cannot be less than 12 bytes long")
	}

	hash := fnv.New128a()
	hash.Write(associatedData)
	hash.Write(src[12:])
	if n.perspective == protocol.PerspectiveServer {
		hash.Write([]byte("Client"))
	} else {
		hash.Write([]byte("Server"))
	}
	sum := make([]byte, 0, 16)
	sum = hash.Sum(sum)
	// The tag is written in little endian, so we need to reverse the slice.
	reverse(sum)

	if !bytes.Equal(sum[:12], src[:12]) {
		return nil, fmt.Errorf("NullAEAD: failed to authenticate received data (%#v vs %#v)", sum[:12], src[:12])
	}
	return src[12:], nil
}

// Seal writes hash and ciphertext to the buffer
func (n *nullAEADFNV128a) Seal(dst, src []byte, packetNumber protocol.PacketNumber, associatedData []byte) []byte {
	if cap(dst) < 12+len(src) {
		dst = make([]byte, 12+len(src))
	} else {
		dst = dst[:12+len(src)]
	}

	hash := fnv.New128a()
	hash.Write(associatedData)
	hash.Write(src)

	if n.perspective == protocol.PerspectiveServer {
		hash.Write([]byte("Server"))
	} else {
		hash.Write([]byte("Client"))
	}
	sum := make([]byte, 0, 16)
	sum = hash.Sum(sum)
	// The tag is written in little endian, so we need to reverse the slice.
	reverse(sum)

	copy(dst[12:], src)
	copy(dst, sum[:12])
	return dst
}

func (n *nullAEADFNV128a) Overhead() int {
	return 12
}

func reverse(a []byte) {
	for left, right := 0, len(a)-1; left < right; left, right = left+1, right-1 {
		a[left], a[right] = a[right], a[left]
	}
}
<|endoftext|>


<commit_before>package main

import (
	"./cachectl"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"path/filepath"
	"regexp"
	"syscall"
	"time"
)

func scheduledPurgePages(target cachectl.SectionTarget) {

	re := regexp.MustCompile(target.Filter)

	for {
		timer := time.NewTimer(time.Second * time.Duration(target.PurgeInterval))
		<-timer.C

		fi, err := os.Stat(target.Path)
		if err != nil {
			fmt.Println(err.Error())
			continue
		}

		if fi.IsDir() {
			err := filepath.Walk(target.Path,
				func(path string, info os.FileInfo, err error) error {
					if !info.Mode().IsRegular() {
						return nil
					}

					if re.MatchString(path) {
						cachectl.RunPurgePages(path, info.Size(), target.Rate)
					}
					return nil
				})

			if err != nil {
				fmt.Printf("failed to walk in %s.", fi.Name())
			}
		} else {
			if !fi.Mode().IsRegular() {
				fmt.Printf("%s is not regular file\n", fi.Name())
				continue
			}

			cachectl.RunPurgePages(target.Path, fi.Size(), target.Rate)
		}
	}
}

func main() {

	// Parse flags
	version := flag.Bool("v", false, "show version")
	confPath := flag.String("c", "", "configuration file for cachectld")
	flag.Parse()

	if *version {
		cachectl.PrintVersion(cachectl.Cachectld)
		os.Exit(0)
	}

	var confCachectld cachectl.ConfToml
	err := cachectl.LoadConf(*confPath, &confCachectld)
	if err != nil {
		panic(err.Error())
	}

	err = cachectl.ValidateConf(&confCachectld)
	if err != nil {
		panic(err.Error())
	}

	for _, target := range confCachectld.Targets {
		go scheduledPurgePages(target)
	}

	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	exitchan := make(chan int)
	go func() {
		for {
			s := <-sigchan
			switch s {
			case syscall.SIGHUP:
				fallthrough
			case syscall.SIGINT:
				fallthrough
			case syscall.SIGTERM:
				fallthrough
			case syscall.SIGQUIT:
				exitchan <- 0
			default:
				exitchan <- 1
			}
		}
	}()

	code := <-exitchan
	os.Exit(code)
}
<commit_msg>cut redundant loop.<commit_after>package main

import (
	"./cachectl"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"path/filepath"
	"regexp"
	"syscall"
	"time"
)

func scheduledPurgePages(target cachectl.SectionTarget) {

	re := regexp.MustCompile(target.Filter)

	for {
		timer := time.NewTimer(time.Second * time.Duration(target.PurgeInterval))
		<-timer.C

		fi, err := os.Stat(target.Path)
		if err != nil {
			fmt.Println(err.Error())
			continue
		}

		if fi.IsDir() {
			err := filepath.Walk(target.Path,
				func(path string, info os.FileInfo, err error) error {
					if !info.Mode().IsRegular() {
						return nil
					}

					if re.MatchString(path) {
						cachectl.RunPurgePages(path, info.Size(), target.Rate)
					}
					return nil
				})

			if err != nil {
				fmt.Printf("failed to walk in %s.", fi.Name())
			}
		} else {
			if !fi.Mode().IsRegular() {
				fmt.Printf("%s is not regular file\n", fi.Name())
				continue
			}

			cachectl.RunPurgePages(target.Path, fi.Size(), target.Rate)
		}
	}
}

func main() {

	// Parse flags
	version := flag.Bool("v", false, "show version")
	confPath := flag.String("c", "", "configuration file for cachectld")
	flag.Parse()

	if *version {
		cachectl.PrintVersion(cachectl.Cachectld)
		os.Exit(0)
	}

	var confCachectld cachectl.ConfToml
	err := cachectl.LoadConf(*confPath, &confCachectld)
	if err != nil {
		panic(err.Error())
	}

	err = cachectl.ValidateConf(&confCachectld)
	if err != nil {
		panic(err.Error())
	}

	for _, target := range confCachectld.Targets {
		go scheduledPurgePages(target)
	}

	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	exitchan := make(chan int)
	go func() {
		s := <-sigchan
		switch s {
		case syscall.SIGHUP:
			fallthrough
		case syscall.SIGINT:
			fallthrough
		case syscall.SIGTERM:
			fallthrough
		case syscall.SIGQUIT:
			exitchan <- 0
		default:
			exitchan <- 1
		}
	}()

	code := <-exitchan
	os.Exit(code)
}
<|endoftext|>