<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/tilezen/tapalcatl"
	"io"
	"log"
	"net"
	"net/http"
	"time"
)

type ParseResult struct {
	Coord       tapalcatl.TileCoord
	Cond        Condition
	ContentType string
}

type Parser interface {
	Parse(*http.Request) (*ParseResult, error)
}

type BufferManager interface {
	Get() *bytes.Buffer
	Put(*bytes.Buffer)
}

// the reqState structures and string generation serve to emit a single log entry line
// a log parser will pick this up and use it to persist metrics
// the string functions here are specific to the format used and should be updated with care

type reqResponseState int32

const (
	ResponseState_Nil reqResponseState = iota
	ResponseState_Success
	ResponseState_NotModified
	ResponseState_NotFound
	ResponseState_BadRequest
	ResponseState_Error
	ResponseState_Count
)

func (rrs reqResponseState) String() string {
	switch rrs {
	case ResponseState_Nil:
		return "nil"
	case ResponseState_Success:
		return "ok"
	case ResponseState_NotModified:
		return "notmod"
	case ResponseState_NotFound:
		return "notfound"
	case ResponseState_BadRequest:
		return "badreq"
	case ResponseState_Error:
		return "err"
	default:
		return "unknown"
	}
}

type reqFetchState int32

const (
	FetchState_Nil reqFetchState = iota
	FetchState_Success
	FetchState_NotFound
	FetchState_FetchError
	FetchState_ReadError
	FetchState_Count
)

func (rfs reqFetchState) String() string {
	switch rfs {
	case FetchState_Nil:
		return "nil"
	case FetchState_Success:
		return "ok"
	case FetchState_NotFound:
		return "notfound"
	case FetchState_FetchError:
		return "fetcherr"
	case FetchState_ReadError:
		return "readerr"
	default:
		return "unknown"
	}
}

type reqFetchSize struct {
	bodySize, bytesLength, bytesCap int64
}

type reqStorageMetadata struct {
	hasLastModified, hasEtag bool
}

type requestState struct {
	responseState        reqResponseState
	fetchState           reqFetchState
	fetchSize            reqFetchSize
	storageMetadata      reqStorageMetadata
	isZipError           bool
	isResponseWriteError bool
	isCondError          bool
}

func logBool(x bool) string {
	if x {
		return "1"
	} else {
		return "0"
	}
}

func boolAsInt(x bool) int {
	if x {
		return 1
	} else {
		return 0
	}
}

// create a log string
func (reqState *requestState) String() string {

	var fetchSize string
	if reqState.fetchSize.bodySize > 0 {
		fetchSize = fmt.Sprintf(
			"%d %d %d",
			reqState.fetchSize.bodySize,
			reqState.fetchSize.bytesLength,
			reqState.fetchSize.bytesCap,
		)
	} else {
		fetchSize = "nil"
	}

	hasLastMod := logBool(reqState.storageMetadata.hasLastModified)
	hasEtag := logBool(reqState.storageMetadata.hasEtag)

	isZipErr := logBool(reqState.isZipError)
	isRespErr := logBool(reqState.isResponseWriteError)
	isCondErr := logBool(reqState.isCondError)

	result := fmt.Sprintf(
		"METRICS: respstate(%s) fetchstate(%s) fetchsize(%s) ziperr(%s) resperr(%s) conderr(%s) lastmod(%s) etag(%s)",
		reqState.responseState,
		reqState.fetchState,
		fetchSize,
		isZipErr,
		isRespErr,
		isCondErr,
		hasLastMod,
		hasEtag,
	)

	return result
}

type metricsWriter interface {
	Write(*requestState)
}

type nilMetricsWriter struct{}

func (_ *nilMetricsWriter) Write(reqState *requestState) {}

type statsdMetricsWriter struct {
	addr   *net.UDPAddr
	prefix string
	logger *log.Logger
}

func makeMetricPrefix(prefix string, metric string) string {
	if prefix == "" {
		return metric
	} else {
		return fmt.Sprintf("%s.%s", prefix, metric)
	}
}

func makeStatsdLineCount(prefix string, metric string, value int) string {
	return fmt.Sprintf("%s.count:%d|c\n", makeMetricPrefix(prefix, metric), value)
}

func makeStatsdLineGauge(prefix string, metric string, value int) string {
	return fmt.Sprintf("%s.value:%d|g\n", makeMetricPrefix(prefix, metric), value)
}

func writeStatsdCount(w io.Writer, prefix string, metric string, value int) {
	w.Write([]byte(makeStatsdLineCount(prefix, metric, value)))
}

func writeStatsdGauge(w io.Writer, prefix string, metric string, value int) {
	w.Write([]byte(makeStatsdLineGauge(prefix, metric, value)))
}

type prefixedStatsdWriter struct {
	prefix string
	w      io.Writer
}

func (psw *prefixedStatsdWriter) WriteCount(metric string, value int) {
	writeStatsdCount(psw.w, psw.prefix, metric, value)
}

func (psw *prefixedStatsdWriter) WriteGauge(metric string, value int) {
	writeStatsdGauge(psw.w, psw.prefix, metric, value)
}

func (smw *statsdMetricsWriter) Write(reqState *requestState) {
	go func() {

		conn, err := net.DialUDP("udp", nil, smw.addr)
		if err != nil {
			smw.logger.Printf("ERROR: Metrics Writer failed to connect to %s: %s\n", smw.addr, err)
		}
		defer conn.Close()

		w := bufio.NewWriter(conn)
		defer w.Flush()

		psw := prefixedStatsdWriter{
			prefix: smw.prefix,
			w:      w,
		}

		respStateInt := int32(reqState.responseState)
		if respStateInt > 0 && respStateInt < int32(ResponseState_Count) {
			respStateName := reqState.responseState.String()
			respMetricName := fmt.Sprintf("responsestate.%s", respStateName)
			psw.WriteCount(respMetricName, 1)
		} else {
			smw.logger.Printf("ERROR: Invalid response state: %s", reqState.responseState)
		}

		fetchStateInt := int32(reqState.fetchState)
		if fetchStateInt > 0 && fetchStateInt < int32(FetchState_Count) {
			fetchStateName := reqState.fetchState.String()
			fetchMetricName := fmt.Sprintf("fetchstate.%s", fetchStateName)
			psw.WriteCount(fetchMetricName, 1)
		} else {
			smw.logger.Printf("ERROR: Invalid fetch state: %s", reqState.responseState)
		}

		if reqState.fetchSize.bodySize > 0 {
			psw.WriteGauge("fetchsize.body-size", int(reqState.fetchSize.bodySize))
			psw.WriteGauge("fetchsize.buffer-length", int(reqState.fetchSize.bytesLength))
			psw.WriteGauge("fetchsize.buffer-capacity", int(reqState.fetchSize.bytesCap))
		}

		psw.WriteCount("lastmodified", boolAsInt(reqState.storageMetadata.hasLastModified))
		psw.WriteCount("etag", boolAsInt(reqState.storageMetadata.hasEtag))

		psw.WriteCount("zip-error", boolAsInt(reqState.isZipError))
		psw.WriteCount("response-write-error", boolAsInt(reqState.isResponseWriteError))
		psw.WriteCount("condition-parse-error", boolAsInt(reqState.isCondError))

	}()
}

func NewStatsdMetricsWriter(addr *net.UDPAddr, metricsPrefix string, logger *log.Logger) metricsWriter {
	return &statsdMetricsWriter{
		addr:   addr,
		prefix: metricsPrefix,
		logger: logger,
	}
}

func MetatileHandler(p Parser, metatileSize int, mimeMap map[string]string, storage Storage, bufferManager BufferManager, mw metricsWriter, logger *log.Logger) http.Handler {

	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {

		reqState := requestState{}

		numRequests.Add(1)
		start_time := time.Now()
		defer func() {
			// update expvar state
			updateCounters(time.Since(start_time))

			if reqState.responseState == ResponseState_Nil {
				logger.Printf("ERROR: Code error: handler did not set response state")
			}

			// relies on the Stringer implementation to format the record correctly
			logger.Printf("INFO: %s", &reqState)

			// write out metrics
			mw.Write(&reqState)

		}()

		parseResult, err := p.Parse(req)
		if err != nil {
			requestParseErrors.Add(1)
			var sc int
			var response string

			if pe, ok := err.(*ParseError); ok {
				logger.Printf("WARNING: Parse failure: %s", err.Error())
				if pe.MimeError != nil {
					sc = http.StatusNotFound
					reqState.responseState = ResponseState_NotFound
					response = pe.MimeError.Error()
				} else if pe.CoordError != nil {
					sc = http.StatusBadRequest
					reqState.responseState = ResponseState_BadRequest
					response = pe.CoordError.Error()
				} else if pe.CondError != nil {
					reqState.isCondError = true
					logger.Printf("WARNING: Condition Error: %s", pe.CondError)
				}
			} else {
				logger.Printf("ERROR: Unknown parse error: %#v\n", err)
				sc = http.StatusInternalServerError
				response = "Internal server error"
				reqState.responseState = ResponseState_Error
			}

			// only return an error response when not a condition parse error
			// NOTE: maybe it's better to not consider this an error, but
			// capture it in the parse result state and handle it that way?
			if sc > 0 {
				http.Error(rw, response, sc)
				return
			}
		}

		metaCoord, offset := parseResult.Coord.MetaAndOffset(metatileSize)

		storageResult, err := storage.Fetch(metaCoord, parseResult.Cond)
		if err != nil || storageResult.NotFound {
			if err != nil {
				storageFetchErrors.Add(1)
				logger.Printf("WARNING: Metatile storage fetch failure: %s", err.Error())
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				reqState.fetchState = FetchState_FetchError
				reqState.responseState = ResponseState_Error
			} else {
				numStorageMisses.Add(1)
				http.NotFound(rw, req)
				reqState.fetchState = FetchState_NotFound
				reqState.responseState = ResponseState_NotFound
			}
			return
		}
		numStorageHits.Add(1)
		reqState.fetchState = FetchState_Success

		if storageResult.NotModified {
			numStorageNotModified.Add(1)
			rw.WriteHeader(http.StatusNotModified)
			reqState.responseState = ResponseState_NotModified
			return
		}
		numStorageReads.Add(1)

		// metatile reader needs to be able to seek in the buffer and know
		// its size. the easiest way to ensure that is to buffer the whole
		// thing into memory.
		storageResp := storageResult.Response

		buf := bufferManager.Get()
		defer bufferManager.Put(buf)

		bodySize, err := io.Copy(buf, storageResp.Body)
		if err != nil {
			storageReadErrors.Add(1)
			logger.Printf("ERROR: Failed to read storage body: %s", err.Error())
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			reqState.fetchState = FetchState_ReadError
			reqState.responseState = ResponseState_Error
			return
		}
		reqState.fetchState = FetchState_Success

		storageBytes := buf.Bytes()
		reqState.fetchSize.bodySize = bodySize
		reqState.fetchSize.bytesLength = int64(len(storageBytes))
		reqState.fetchSize.bytesCap = int64(cap(storageBytes))

		headers := rw.Header()
		headers.Set("Content-Type", parseResult.ContentType)
		if lastMod := storageResp.LastModified; lastMod != nil {
			lastModifiedFormatted := lastMod.Format(time.RFC1123Z)
			headers.Set("Last-Modified", lastModifiedFormatted)
			reqState.storageMetadata.hasLastModified = true
		}
		if etag := storageResp.ETag; etag != nil {
			headers.Set("ETag", *etag)
			reqState.storageMetadata.hasEtag = true
		}

		reader, err := tapalcatl.NewMetatileReader(offset, bytes.NewReader(storageBytes), bodySize)
		if err != nil {
			metatileReadErrors.Add(1)
			logger.Printf("ERROR: Failed to read metatile: %s", err.Error())
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			reqState.isZipError = true
			reqState.responseState = ResponseState_Error
			return
		}

		rw.WriteHeader(http.StatusOK)
		reqState.responseState = ResponseState_Success
		_, err = io.Copy(rw, reader)
		if err != nil {
			responseWriteErrors.Add(1)
			logger.Printf("ERROR: Failed to write response body: %s", err.Error())
			reqState.isResponseWriteError = true
		}
	})
}
<commit_msg>Use a separate goroutine with a bounded queue<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/tilezen/tapalcatl"
	"io"
	"log"
	"net"
	"net/http"
	"time"
)

type ParseResult struct {
	Coord       tapalcatl.TileCoord
	Cond        Condition
	ContentType string
}

type Parser interface {
	Parse(*http.Request) (*ParseResult, error)
}

type BufferManager interface {
	Get() *bytes.Buffer
	Put(*bytes.Buffer)
}

// the reqState structures and string generation serve to emit a single log entry line
// a log parser will pick this up and use it to persist metrics
// the string functions here are specific to the format used and should be updated with care

type reqResponseState int32

const (
	ResponseState_Nil reqResponseState = iota
	ResponseState_Success
	ResponseState_NotModified
	ResponseState_NotFound
	ResponseState_BadRequest
	ResponseState_Error
	ResponseState_Count
)

func (rrs reqResponseState) String() string {
	switch rrs {
	case ResponseState_Nil:
		return "nil"
	case ResponseState_Success:
		return "ok"
	case ResponseState_NotModified:
		return "notmod"
	case ResponseState_NotFound:
		return "notfound"
	case ResponseState_BadRequest:
		return "badreq"
	case ResponseState_Error:
		return "err"
	default:
		return "unknown"
	}
}

type reqFetchState int32

const (
	FetchState_Nil reqFetchState = iota
	FetchState_Success
	FetchState_NotFound
	FetchState_FetchError
	FetchState_ReadError
	FetchState_Count
)

func (rfs reqFetchState) String() string {
	switch rfs {
	case FetchState_Nil:
		return "nil"
	case FetchState_Success:
		return "ok"
	case FetchState_NotFound:
		return "notfound"
	case FetchState_FetchError:
		return "fetcherr"
	case FetchState_ReadError:
		return "readerr"
	default:
		return "unknown"
	}
}

type reqFetchSize struct {
	bodySize, bytesLength, bytesCap int64
}

type reqStorageMetadata struct {
	hasLastModified, hasEtag bool
}

type requestState struct {
	responseState        reqResponseState
	fetchState           reqFetchState
	fetchSize            reqFetchSize
	storageMetadata      reqStorageMetadata
	isZipError           bool
	isResponseWriteError bool
	isCondError          bool
}

func logBool(x bool) string {
	if x {
		return "1"
	} else {
		return "0"
	}
}

func boolAsInt(x bool) int {
	if x {
		return 1
	} else {
		return 0
	}
}

// create a log string
func (reqState *requestState) String() string {

	var fetchSize string
	if reqState.fetchSize.bodySize > 0 {
		fetchSize = fmt.Sprintf(
			"%d %d %d",
			reqState.fetchSize.bodySize,
			reqState.fetchSize.bytesLength,
			reqState.fetchSize.bytesCap,
		)
	} else {
		fetchSize = "nil"
	}

	hasLastMod := logBool(reqState.storageMetadata.hasLastModified)
	hasEtag := logBool(reqState.storageMetadata.hasEtag)

	isZipErr := logBool(reqState.isZipError)
	isRespErr := logBool(reqState.isResponseWriteError)
	isCondErr := logBool(reqState.isCondError)

	result := fmt.Sprintf(
		"METRICS: respstate(%s) fetchstate(%s) fetchsize(%s) ziperr(%s) resperr(%s) conderr(%s) lastmod(%s) etag(%s)",
		reqState.responseState,
		reqState.fetchState,
		fetchSize,
		isZipErr,
		isRespErr,
		isCondErr,
		hasLastMod,
		hasEtag,
	)

	return result
}

type metricsWriter interface {
	Write(*requestState)
}

type nilMetricsWriter struct{}

func (_ *nilMetricsWriter) Write(reqState *requestState) {}

type statsdMetricsWriter struct {
	addr   *net.UDPAddr
	prefix string
	logger *log.Logger
	queue  chan *requestState
}

func makeMetricPrefix(prefix string, metric string) string {
	if prefix == "" {
		return metric
	} else {
		return fmt.Sprintf("%s.%s", prefix, metric)
	}
}

func makeStatsdLineCount(prefix string, metric string, value int) string {
	return fmt.Sprintf("%s.count:%d|c\n", makeMetricPrefix(prefix, metric), value)
}

func makeStatsdLineGauge(prefix string, metric string, value int) string {
	return fmt.Sprintf("%s.value:%d|g\n", makeMetricPrefix(prefix, metric), value)
}

func writeStatsdCount(w io.Writer, prefix string, metric string, value int) {
	w.Write([]byte(makeStatsdLineCount(prefix, metric, value)))
}

func writeStatsdGauge(w io.Writer, prefix string, metric string, value int) {
	w.Write([]byte(makeStatsdLineGauge(prefix, metric, value)))
}

type prefixedStatsdWriter struct {
	prefix string
	w      io.Writer
}

func (psw *prefixedStatsdWriter) WriteCount(metric string, value int) {
	writeStatsdCount(psw.w, psw.prefix, metric, value)
}

func (psw *prefixedStatsdWriter) WriteGauge(metric string, value int) {
	writeStatsdGauge(psw.w, psw.prefix, metric, value)
}

func (smw *statsdMetricsWriter) Process(reqState *requestState) {
	conn, err := net.DialUDP("udp", nil, smw.addr)
	if err != nil {
		smw.logger.Printf("ERROR: Metrics Writer failed to connect to %s: %s\n", smw.addr, err)
	}
	defer conn.Close()

	w := bufio.NewWriter(conn)
	defer w.Flush()

	psw := prefixedStatsdWriter{
		prefix: smw.prefix,
		w:      w,
	}

	respStateInt := int32(reqState.responseState)
	if respStateInt > 0 && respStateInt < int32(ResponseState_Count) {
		respStateName := reqState.responseState.String()
		respMetricName := fmt.Sprintf("responsestate.%s", respStateName)
		psw.WriteCount(respMetricName, 1)
	} else {
		smw.logger.Printf("ERROR: Invalid response state: %s", reqState.responseState)
	}

	fetchStateInt := int32(reqState.fetchState)
	if fetchStateInt > 0 && fetchStateInt < int32(FetchState_Count) {
		fetchStateName := reqState.fetchState.String()
		fetchMetricName := fmt.Sprintf("fetchstate.%s", fetchStateName)
		psw.WriteCount(fetchMetricName, 1)
	} else {
		smw.logger.Printf("ERROR: Invalid fetch state: %s", reqState.responseState)
	}

	if reqState.fetchSize.bodySize > 0 {
		psw.WriteGauge("fetchsize.body-size", int(reqState.fetchSize.bodySize))
		psw.WriteGauge("fetchsize.buffer-length", int(reqState.fetchSize.bytesLength))
		psw.WriteGauge("fetchsize.buffer-capacity", int(reqState.fetchSize.bytesCap))
	}

	psw.WriteCount("lastmodified", boolAsInt(reqState.storageMetadata.hasLastModified))
	psw.WriteCount("etag", boolAsInt(reqState.storageMetadata.hasEtag))

	psw.WriteCount("zip-error", boolAsInt(reqState.isZipError))
	psw.WriteCount("response-write-error", boolAsInt(reqState.isResponseWriteError))
	psw.WriteCount("condition-parse-error", boolAsInt(reqState.isCondError))
}

func (smw *statsdMetricsWriter) Write(reqState *requestState) {
	select {
	case smw.queue <- reqState:
	default:
		smw.logger.Printf("WARNING: Metrics Writer queue full\n")
	}
}

func NewStatsdMetricsWriter(addr *net.UDPAddr, metricsPrefix string, logger *log.Logger) metricsWriter {
	maxQueueSize := 4096
	queue := make(chan *requestState, maxQueueSize)

	smw := &statsdMetricsWriter{
		addr:   addr,
		prefix: metricsPrefix,
		logger: logger,
		queue:  queue,
	}

	go func(smw *statsdMetricsWriter) {
		for reqState := range smw.queue {
			smw.Process(reqState)
		}
	}(smw)

	return smw
}

func MetatileHandler(p Parser, metatileSize int, mimeMap map[string]string, storage Storage, bufferManager BufferManager, mw metricsWriter, logger *log.Logger) http.Handler {

	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {

		reqState := requestState{}

		numRequests.Add(1)
		start_time := time.Now()
		defer func() {
			// update expvar state
			updateCounters(time.Since(start_time))

			if reqState.responseState == ResponseState_Nil {
				logger.Printf("ERROR: Code error: handler did not set response state")
			}

			// relies on the Stringer implementation to format the record correctly
			logger.Printf("INFO: %s", &reqState)

			// write out metrics
			mw.Write(&reqState)

		}()

		parseResult, err := p.Parse(req)
		if err != nil {
			requestParseErrors.Add(1)
			var sc int
			var response string

			if pe, ok := err.(*ParseError); ok {
				logger.Printf("WARNING: Parse failure: %s", err.Error())
				if pe.MimeError != nil {
					sc = http.StatusNotFound
					reqState.responseState = ResponseState_NotFound
					response = pe.MimeError.Error()
				} else if pe.CoordError != nil {
					sc = http.StatusBadRequest
					reqState.responseState = ResponseState_BadRequest
					response = pe.CoordError.Error()
				} else if pe.CondError != nil {
					reqState.isCondError = true
					logger.Printf("WARNING: Condition Error: %s", pe.CondError)
				}
			} else {
				logger.Printf("ERROR: Unknown parse error: %#v\n", err)
				sc = http.StatusInternalServerError
				response = "Internal server error"
				reqState.responseState = ResponseState_Error
			}

			// only return an error response when not a condition parse error
			// NOTE: maybe it's better to not consider this an error, but
			// capture it in the parse result state and handle it that way?
			if sc > 0 {
				http.Error(rw, response, sc)
				return
			}
		}

		metaCoord, offset := parseResult.Coord.MetaAndOffset(metatileSize)

		storageResult, err := storage.Fetch(metaCoord, parseResult.Cond)
		if err != nil || storageResult.NotFound {
			if err != nil {
				storageFetchErrors.Add(1)
				logger.Printf("WARNING: Metatile storage fetch failure: %s", err.Error())
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				reqState.fetchState = FetchState_FetchError
				reqState.responseState = ResponseState_Error
			} else {
				numStorageMisses.Add(1)
				http.NotFound(rw, req)
				reqState.fetchState = FetchState_NotFound
				reqState.responseState = ResponseState_NotFound
			}
			return
		}
		numStorageHits.Add(1)
		reqState.fetchState = FetchState_Success

		if storageResult.NotModified {
			numStorageNotModified.Add(1)
			rw.WriteHeader(http.StatusNotModified)
			reqState.responseState = ResponseState_NotModified
			return
		}
		numStorageReads.Add(1)

		// metatile reader needs to be able to seek in the buffer and know
		// its size. the easiest way to ensure that is to buffer the whole
		// thing into memory.
		storageResp := storageResult.Response

		buf := bufferManager.Get()
		defer bufferManager.Put(buf)

		bodySize, err := io.Copy(buf, storageResp.Body)
		if err != nil {
			storageReadErrors.Add(1)
			logger.Printf("ERROR: Failed to read storage body: %s", err.Error())
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			reqState.fetchState = FetchState_ReadError
			reqState.responseState = ResponseState_Error
			return
		}
		reqState.fetchState = FetchState_Success

		storageBytes := buf.Bytes()
		reqState.fetchSize.bodySize = bodySize
		reqState.fetchSize.bytesLength = int64(len(storageBytes))
		reqState.fetchSize.bytesCap = int64(cap(storageBytes))

		headers := rw.Header()
		headers.Set("Content-Type", parseResult.ContentType)
		if lastMod := storageResp.LastModified; lastMod != nil {
			lastModifiedFormatted := lastMod.Format(time.RFC1123Z)
			headers.Set("Last-Modified", lastModifiedFormatted)
			reqState.storageMetadata.hasLastModified = true
		}
		if etag := storageResp.ETag; etag != nil {
			headers.Set("ETag", *etag)
			reqState.storageMetadata.hasEtag = true
		}

		reader, err := tapalcatl.NewMetatileReader(offset, bytes.NewReader(storageBytes), bodySize)
		if err != nil {
			metatileReadErrors.Add(1)
			logger.Printf("ERROR: Failed to read metatile: %s", err.Error())
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			reqState.isZipError = true
			reqState.responseState = ResponseState_Error
			return
		}

		rw.WriteHeader(http.StatusOK)
		reqState.responseState = ResponseState_Success
		_, err = io.Copy(rw, reader)
		if err != nil {
			responseWriteErrors.Add(1)
			logger.Printf("ERROR: Failed to write response body: %s", err.Error())
			reqState.isResponseWriteError = true
		}
	})
}
<|endoftext|>


<commit_before>package model

import (
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"time"

	"github.com/intelsdi-x/snap/mgmt/rest/rbody"
)

var (
	MetricAlreadyExists = errors.New("Metric already exists.")
)

type Metric struct {
	Id        string
	Owner     int64
	Public    bool
	Namespace string
	Version   int64
	Policy    []rbody.PolicyTable
	Created   time.Time
}

func (m *Metric) SetId() {
	var buffer bytes.Buffer
	buffer.WriteString(fmt.Sprintf("%t:%d:%s:%d", m.Public, m.Owner, m.Namespace, m.Version))
	m.Id = fmt.Sprintf("%x", md5.Sum(buffer.Bytes()))
}

// "url" tag is used by github.com/google/go-querystring/query
// "form" tag is used by is ued by github.com/go-macaron/binding
type GetMetricsQuery struct {
	Namespace string `form:"namespace" url:"namespace,omitempty"`
	Version   int64  `form:"version" url:"version, omitempty"`
	Owner     int64  `form:"-" url:"-"`
}
<commit_msg>polish metric model<commit_after>package model

import (
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"time"

	"github.com/intelsdi-x/snap/mgmt/rest/rbody"
)

var (
	MetricAlreadyExists = errors.New("Metric already exists.")
)

type Metric struct {
	Id        string              `json:"-"`
	Owner     int64               `json:"-"`
	Public    bool                `json:"public"`
	Namespace string              `json:"namespace"`
	Version   int64               `json:"version"`
	Policy    []rbody.PolicyTable `json:"policy"`
	Created   time.Time           `json:"created"`
}

func (m *Metric) SetId() {
	var buffer bytes.Buffer
	buffer.WriteString(fmt.Sprintf("%t:%d:%s:%d", m.Public, m.Owner, m.Namespace, m.Version))
	m.Id = fmt.Sprintf("%x", md5.Sum(buffer.Bytes()))
}

// "url" tag is used by github.com/google/go-querystring/query
// "form" tag is used by is ued by github.com/go-macaron/binding
type GetMetricsQuery struct {
	Namespace string `form:"namespace" url:"namespace,omitempty"`
	Version   int64  `form:"version" url:"version, omitempty"`
	Owner     int64  `form:"-" url:"-"`
	OrderBy   string `form:"orderBy" url:"orderBy,omitempty"`
	Limit     int    `form:"limit" url:"limit,omitempty"`
	Page      int    `form:"page" url:"page,omitempty"`
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"strconv"

	"github.com/dynport/urknall"
)

type Firewall struct {
	Interface string `urknall:"default=eth0"`
	WithVPN   bool
	Paranoid  bool
	Rules     []*FirewallRule
	IPSets    []*FirewallIPSet // List of ipsets for the firewall.
}

func (fw *Firewall) Render(pkg urknall.Package) {
	var ipsetsCmd Command
	if len(fw.IPSets) > 0 {
		ipsetsCmd = WriteFile("/etc/iptables/ipsets", fwIpset, "root", 0644)
	} else {
		ipsetsCmd = Shell("rm -f /etc/iptables/ipsets")
	}
	pkg.AddCommands("base",
		InstallPackages("iptables", "ipset"),
		WriteFile("/etc/network/if-pre-up.d/iptables", firewallUpstart, "root", 0744),
		WriteFile("/etc/iptables/rules_ipv4", fw_rules_ipv4, "root", 0644),
		WriteFile("/etc/iptables/rules_ipv6", fw_rules_ipv6, "root", 0644),
		ipsetsCmd,
		Shell("{ modprobe iptable_filter && modprobe iptable_nat; }; /bin/true"), // here to make sure next command succeeds.
		Shell("IFACE={{ .Interface }} /etc/network/if-pre-up.d/iptables"),
	)
}

// IPSets are the possibility to change a rule, without actually rewriting the rules. That is they add some sort of
// flexibility with regard to dynamic entities like a load balancer, which must have access to the different machines
// that should take the load.
//
// A set is defined by a name, that is used in iptables rule (see "Rule.(Source|Destination).IPSet") to reference the
// contained entities. The type defines what parameters must match an entry (see "ipset --help" output and the man page
// for a list of allowed values), for example a set could define hosts and ports.
//
// The family defines the type of IP address to handle, either IPv4 or IPv6. The allowed values are "inet" and "inet6"
// respectively.
//
// There are some ipset internal parameters that shouldn't need to be changed often. Those are "HashSize" that defines
// the size of the underlying hash. This value defaults to 1024. The "MaxElem" number determines how much elements there
// can be in the set at most.
//
// An initial set of members can be defined, if reasonable.
type FirewallIPSet struct {
	Name     string   // Name of the ipset.
	Type     string   // Type of the ipset.
	Family   string   // Network address family.
	HashSize int      // Size of the hash.
	MaxElem  int      // Max number of elements of the set.
	Members  []net.IP // Initial set of members.
}

func (ips *FirewallIPSet) IPSetRestore() (cmd string) {
	cmd = fmt.Sprintf("create %s %s family %s hashsize %d maxelem %d\n",
		ips.Name, ips.Type, ips.family(), ips.hashsize(), ips.maxelem())
	for idx := range ips.Members {
		cmd += fmt.Sprintf("add %s %s\n", ips.Name, ips.Members[idx].String())
	}
	return cmd + "\n"
}

func (ips *FirewallIPSet) family() string {
	if ips.Family == "" {
		return "inet"
	}
	return ips.Family
}

func (i *FirewallIPSet) hashsize() int {
	if i.HashSize == 0 {
		return 1024
	}
	return i.HashSize
}

func (i *FirewallIPSet) maxelem() int {
	if i.MaxElem == 0 {
		return 65536
	}
	return i.MaxElem
}

// A rule defines what is allowed to flow from some source to some destination. A description can be added to make the
// resulting scripts more readable.
//
// The "Chain" field determines which chain the rule is added to. This should be either "INPUT", "OUTPUT", or "FORWARD",
// with the names of the chains mostly speaking for themselves.
//
// The protocol setting is another easy match for the rule and especially required for some of the target's settings,
// i.e. if a port is specified the protocol must be given too.
//
// Source and destination are the two communicating entities. For the input chain the local host is destination and for
// output it is the source.
type FirewallRule struct {
	Description string
	Chain       string // Chain to add the rule to.
	Protocol    string // The protocol used.

	Source      *FirewallTarget // The source of the packet.
	Destination *FirewallTarget // The destination of the packet.
}

// Method to create something digestable for IPtablesRestore (aka users might well ignore this).
func (r *FirewallRule) Filter() (cmd string) {
	cfg := &iptConfig{rule: r, moduleConfig: map[string]iptModConfig{}}

	if r.Source != nil {
		r.Source.convert(cfg, "src")
	}

	if r.Destination != nil {
		r.Destination.convert(cfg, "dest")
	}

	return cfg.FilterTableRule()
}

func (r *FirewallRule) isNATRule() bool {
	return r.Chain == "FORWARD" &&
		((r.Source != nil && r.Source.NAT != "") ||
			(r.Destination != nil && r.Destination.NAT != ""))
}

func (r *FirewallRule) NAT() (cmd string) {
	if !r.isNATRule() {
		return ""
	}

	cfg := &iptConfig{rule: r, moduleConfig: map[string]iptModConfig{}}

	if r.Source != nil {
		r.Source.convert(cfg, "src")
	}

	if r.Destination != nil {
		r.Destination.convert(cfg, "dest")
	}

	return cfg.NATTableRule()
}

func (r *FirewallRule) IPsets() {
}

type iptModConfig map[string]string

type iptConfig struct {
	rule *FirewallRule

	sourceIP string
	destIP   string

	sourceIface string
	destIface   string

	sourceNAT string
	destNAT   string

	moduleConfig map[string]iptModConfig
}

func (cfg *iptConfig) basicSettings(natRule bool) (s string) {
	if cfg.rule.Protocol != "" {
		s += " --protocol " + cfg.rule.Protocol
	}
	if cfg.sourceIP != "" {
		s += " --source " + cfg.sourceIP
	}
	if cfg.sourceIface != "" {
		if cfg.rule.Chain == "FORWARD" {
			if !natRule || cfg.destNAT != "" {
				s += " --in-interface " + cfg.sourceIface
			}
		} else {
			s += " --out-interface " + cfg.sourceIface
		}
	}
	if cfg.destIP != "" {
		s += " --destination " + cfg.destIP
	}
	if cfg.destIface != "" {
		if cfg.rule.Chain == "FORWARD" {
			if !natRule || cfg.sourceNAT != "" {
				s += " --out-interface " + cfg.destIface
			}
		} else {
			s += " --in-interface " + cfg.destIface
		}
	}
	return s
}

func (cfg *iptConfig) FilterTableRule() (s string) {
	if cfg.rule.Description != "" {
		s = "# " + cfg.rule.Description + "\n"
	}
	s += "-A " + cfg.rule.Chain

	s += cfg.basicSettings(false)

	for module, modOptions := range cfg.moduleConfig {
		s += " " + module
		for option, value := range modOptions {
			s += " " + option + " " + value
		}
	}

	s += " -m state --state NEW -j ACCEPT\n"
	return s
}

func (cfg *iptConfig) NATTableRule() (s string) {
	if cfg.rule.Description != "" {
		s = "# " + cfg.rule.Description + "\n"
	}

	switch {
	case cfg.sourceNAT != "" && cfg.destNAT == "":
		s += "-A POSTROUTING"
	case cfg.sourceNAT == "" && cfg.destNAT != "":
		s += "-A PREROUTING"
	default:
		panic("but you said NAT would be configured?!")
	}

	s += cfg.basicSettings(true)

	switch {
	case cfg.sourceNAT == "MASQ":
		s += " -j MASQUERADE"
	case cfg.sourceNAT != "":
		s += " -j SNAT --to " + cfg.sourceNAT
	case cfg.destNAT != "":
		s += " -j DNAT --to " + cfg.destNAT
	}

	return s
}

func (t *FirewallTarget) convert(cfg *iptConfig, tType string) {
	if t.Port != 0 {
		if cfg.rule.Protocol == "" {
			panic("port requires the protocol to be specified")
		}

		module := "-m " + cfg.rule.Protocol
		if _, found := cfg.moduleConfig[module]; !found {
			cfg.moduleConfig[module] = iptModConfig{}
		}
		switch tType {
		case "src":
			cfg.moduleConfig[module]["--source-port"] = strconv.Itoa(t.Port)
		case "dest":
			cfg.moduleConfig[module]["--destination-port"] = strconv.Itoa(t.Port)
		}
	}

	if t.IP != nil {
		switch tType {
		case "src":
			cfg.sourceIP = t.IP.String()
		case "dest":
			cfg.destIP = t.IP.String()
		}
	}

	if t.IPSet != "" {
		module := "-m set"
		if _, found := cfg.moduleConfig[module]; !found {
			cfg.moduleConfig[module] = iptModConfig{}
		}
		value := cfg.moduleConfig[module]["--match-set "+t.IPSet]
		set := ""
		switch tType {
		case "src":
			set = "src"
		case "dest":
			set = "dst"
		}
		if value != "" {
			cfg.moduleConfig[module]["--match-set "+t.IPSet] = value + "," + set
		} else {
			cfg.moduleConfig[module]["--match-set "+t.IPSet] = set
		}
	}

	if t.Interface != "" {
		switch tType {
		case "src":
			cfg.sourceIface = t.Interface
		case "dest":
			cfg.destIface = t.Interface
		}
	}

	if t.NAT != "" {
		switch tType {
		case "src": // for input on the source the destination address can be modified.
			cfg.destNAT = t.NAT
		case "dest": // for output on the destination the source address can be modified.
			cfg.sourceNAT = t.NAT
		}

		if cfg.sourceNAT != "" && cfg.destNAT != "" {
			panic("only source or destination NAT allowed!")
		}
	}
}

// The target of a rule. It can be specified either by IP or the name of an IPSet. Additional parameters are the port
// and interface used. It's totally valid to only specify a subset (or even none) of the fields. For example IP and
// IPSet must not be given for the host the rule is applied on.
type FirewallTarget struct {
	IP        net.IP // IP of the target.
	IPSet     string // IPSet used for matching.
	Port      int    // Port packets must use to match.
	Interface string // Interface the packet goes through.
	NAT       string // NAT configuration (empty, "MASQ", or Interface's IP).
}

const fw_rules_ipv6 = `
*filter
:INPUT DROP [0:0]
:FORWARD DROP [0:0]
:OUTPUT DROP [0:0]

COMMIT
`

const fw_rules_ipv4 = `*filter
:INPUT DROP [0:0]
:FORWARD DROP [0:0]
:OUTPUT DROP [0:0]

# Accept any related or established connections.
-I INPUT 1 -m state --state RELATED,ESTABLISHED -j ACCEPT
-I FORWARD 1 -m state --state RELATED,ESTABLISHED -j ACCEPT
-I OUTPUT 1 -m state --state {{ if not .Paranoid }}NEW,{{ end }}RELATED,ESTABLISHED -j ACCEPT

# Allow all traffic on the loopback interface.
-A INPUT -i lo -j ACCEPT
-A OUTPUT -o lo -j ACCEPT

{{ if .WithVPN }}
# Allow all traffic on the VPN interface.
-A INPUT -i tun0 -j ACCEPT
-A OUTPUT -o tun0 -j ACCEPT
{{ end }}

{{ if .Paranoid }}
# Outbound DNS lookups
-A OUTPUT -o {{ .Interface }} -p udp -m udp --dport 53 -j ACCEPT

# Outbound PING requests
-A OUTPUT -p icmp -j ACCEPT

# Outbound Network Time Protocol (NTP) request
-A OUTPUT -p udp --dport 123 --sport 123 -j ACCEPT

# Allow outbound DHCP request - Some hosts (Linode) automatically assign the primary IP
-A OUTPUT -p udp --dport 67:68 --sport 67:68 -m state --state NEW -j ACCEPT

# Outbound HTTP
-A OUTPUT -o {{ .Interface }} -p tcp -m tcp --dport 80 -m state --state NEW -j ACCEPT
-A OUTPUT -o {{ .Interface }} -p tcp -m tcp --dport 443 -m state --state NEW -j ACCEPT
{{ end }}

# SSH
-A INPUT -i {{ .Interface }} -p tcp -m tcp --dport 22 -m state --state NEW -j ACCEPT

{{ range .Rules }}{{ .Filter }}{{ end }}

{{ if .WithVPN }}
# Outbound OpenVPN traffic (required to connect to the VPN).
-A OUTPUT -o {{ .Interface }} -p tcp -m tcp --dport 1194 -m state --state NEW -j ACCEPT
-A OUTPUT -o {{ .Interface }} -p udp -m udp --dport 1194 -m state --state NEW -j ACCEPT
{{ end }}
COMMIT

*nat
{{ range .Rules }}{{ .NAT }}{{ end }}
COMMIT
`

const fwIpset = `# IPSet configuration
{{ range .IPSets }}{{ .IPSetRestore }}{{ end }}`
const firewallUpstart = `#!/bin/sh
set -e

case "$IFACE" in
	{{ .Interface }})
		test -e /etc/iptables/ipsets && /usr/sbin/ipset restore -! < /etc/iptables/ipsets
		/sbin/iptables-restore < /etc/iptables/rules_ipv4
		/sbin/ip6tables-restore < /etc/iptables/rules_ipv6
		;;
esac

`
<commit_msg>fix firewall template<commit_after>package main

import (
	"fmt"
	"net"
	"strconv"

	"github.com/dynport/urknall"
)

type Firewall struct {
	Interface string `urknall:"default=eth0"`
	WithVPN   bool
	Paranoid  bool
	Rules     []*FirewallRule
	IPSets    []*FirewallIPSet // List of ipsets for the firewall.
}

func (fw *Firewall) Render(pkg urknall.Package) {
	var ipsetsCmd urknall.Command
	if len(fw.IPSets) > 0 {
		ipsetsCmd = WriteFile("/etc/iptables/ipsets", fwIpset, "root", 0644)
	} else {
		ipsetsCmd = Shell("rm -f /etc/iptables/ipsets")
	}
	pkg.AddCommands("base",
		InstallPackages("iptables", "ipset"),
		WriteFile("/etc/network/if-pre-up.d/iptables", firewallUpstart, "root", 0744),
		WriteFile("/etc/iptables/rules_ipv4", fw_rules_ipv4, "root", 0644),
		WriteFile("/etc/iptables/rules_ipv6", fw_rules_ipv6, "root", 0644),
		ipsetsCmd,
		Shell("{ modprobe iptable_filter && modprobe iptable_nat; }; /bin/true"), // here to make sure next command succeeds.
		Shell("IFACE={{ .Interface }} /etc/network/if-pre-up.d/iptables"),
	)
}

// IPSets are the possibility to change a rule, without actually rewriting the rules. That is they add some sort of
// flexibility with regard to dynamic entities like a load balancer, which must have access to the different machines
// that should take the load.
//
// A set is defined by a name, that is used in iptables rule (see "Rule.(Source|Destination).IPSet") to reference the
// contained entities. The type defines what parameters must match an entry (see "ipset --help" output and the man page
// for a list of allowed values), for example a set could define hosts and ports.
//
// The family defines the type of IP address to handle, either IPv4 or IPv6. The allowed values are "inet" and "inet6"
// respectively.
//
// There are some ipset internal parameters that shouldn't need to be changed often. Those are "HashSize" that defines
// the size of the underlying hash. This value defaults to 1024. The "MaxElem" number determines how much elements there
// can be in the set at most.
//
// An initial set of members can be defined, if reasonable.
type FirewallIPSet struct {
	Name     string   // Name of the ipset.
	Type     string   // Type of the ipset.
	Family   string   // Network address family.
	HashSize int      // Size of the hash.
	MaxElem  int      // Max number of elements of the set.
	Members  []net.IP // Initial set of members.
}

func (ips *FirewallIPSet) IPSetRestore() (cmd string) {
	cmd = fmt.Sprintf("create %s %s family %s hashsize %d maxelem %d\n",
		ips.Name, ips.Type, ips.family(), ips.hashsize(), ips.maxelem())
	for idx := range ips.Members {
		cmd += fmt.Sprintf("add %s %s\n", ips.Name, ips.Members[idx].String())
	}
	return cmd + "\n"
}

func (ips *FirewallIPSet) family() string {
	if ips.Family == "" {
		return "inet"
	}
	return ips.Family
}

func (i *FirewallIPSet) hashsize() int {
	if i.HashSize == 0 {
		return 1024
	}
	return i.HashSize
}

func (i *FirewallIPSet) maxelem() int {
	if i.MaxElem == 0 {
		return 65536
	}
	return i.MaxElem
}

// A rule defines what is allowed to flow from some source to some destination. A description can be added to make the
// resulting scripts more readable.
//
// The "Chain" field determines which chain the rule is added to. This should be either "INPUT", "OUTPUT", or "FORWARD",
// with the names of the chains mostly speaking for themselves.
//
// The protocol setting is another easy match for the rule and especially required for some of the target's settings,
// i.e. if a port is specified the protocol must be given too.
//
// Source and destination are the two communicating entities. For the input chain the local host is destination and for
// output it is the source.
type FirewallRule struct {
	Description string
	Chain       string // Chain to add the rule to.
	Protocol    string // The protocol used.

	Source      *FirewallTarget // The source of the packet.
	Destination *FirewallTarget // The destination of the packet.
}

// Method to create something digestable for IPtablesRestore (aka users might well ignore this).
func (r *FirewallRule) Filter() (cmd string) {
	cfg := &iptConfig{rule: r, moduleConfig: map[string]iptModConfig{}}

	if r.Source != nil {
		r.Source.convert(cfg, "src")
	}

	if r.Destination != nil {
		r.Destination.convert(cfg, "dest")
	}

	return cfg.FilterTableRule()
}

func (r *FirewallRule) isNATRule() bool {
	return r.Chain == "FORWARD" &&
		((r.Source != nil && r.Source.NAT != "") ||
			(r.Destination != nil && r.Destination.NAT != ""))
}

func (r *FirewallRule) NAT() (cmd string) {
	if !r.isNATRule() {
		return ""
	}

	cfg := &iptConfig{rule: r, moduleConfig: map[string]iptModConfig{}}

	if r.Source != nil {
		r.Source.convert(cfg, "src")
	}

	if r.Destination != nil {
		r.Destination.convert(cfg, "dest")
	}

	return cfg.NATTableRule()
}

func (r *FirewallRule) IPsets() {
}

type iptModConfig map[string]string

type iptConfig struct {
	rule *FirewallRule

	sourceIP string
	destIP   string

	sourceIface string
	destIface   string

	sourceNAT string
	destNAT   string

	moduleConfig map[string]iptModConfig
}

func (cfg *iptConfig) basicSettings(natRule bool) (s string) {
	if cfg.rule.Protocol != "" {
		s += " --protocol " + cfg.rule.Protocol
	}
	if cfg.sourceIP != "" {
		s += " --source " + cfg.sourceIP
	}
	if cfg.sourceIface != "" {
		if cfg.rule.Chain == "FORWARD" {
			if !natRule || cfg.destNAT != "" {
				s += " --in-interface " + cfg.sourceIface
			}
		} else {
			s += " --out-interface " + cfg.sourceIface
		}
	}
	if cfg.destIP != "" {
		s += " --destination " + cfg.destIP
	}
	if cfg.destIface != "" {
		if cfg.rule.Chain == "FORWARD" {
			if !natRule || cfg.sourceNAT != "" {
				s += " --out-interface " + cfg.destIface
			}
		} else {
			s += " --in-interface " + cfg.destIface
		}
	}
	return s
}

func (cfg *iptConfig) FilterTableRule() (s string) {
	if cfg.rule.Description != "" {
		s = "# " + cfg.rule.Description + "\n"
	}
	s += "-A " + cfg.rule.Chain

	s += cfg.basicSettings(false)

	for module, modOptions := range cfg.moduleConfig {
		s += " " + module
		for option, value := range modOptions {
			s += " " + option + " " + value
		}
	}

	s += " -m state --state NEW -j ACCEPT\n"
	return s
}

func (cfg *iptConfig) NATTableRule() (s string) {
	if cfg.rule.Description != "" {
		s = "# " + cfg.rule.Description + "\n"
	}

	switch {
	case cfg.sourceNAT != "" && cfg.destNAT == "":
		s += "-A POSTROUTING"
	case cfg.sourceNAT == "" && cfg.destNAT != "":
		s += "-A PREROUTING"
	default:
		panic("but you said NAT would be configured?!")
	}

	s += cfg.basicSettings(true)

	switch {
	case cfg.sourceNAT == "MASQ":
		s += " -j MASQUERADE"
	case cfg.sourceNAT != "":
		s += " -j SNAT --to " + cfg.sourceNAT
	case cfg.destNAT != "":
		s += " -j DNAT --to " + cfg.destNAT
	}

	return s
}

func (t *FirewallTarget) convert(cfg *iptConfig, tType string) {
	if t.Port != 0 {
		if cfg.rule.Protocol == "" {
			panic("port requires the protocol to be specified")
		}

		module := "-m " + cfg.rule.Protocol
		if _, found := cfg.moduleConfig[module]; !found {
			cfg.moduleConfig[module] = iptModConfig{}
		}
		switch tType {
		case "src":
			cfg.moduleConfig[module]["--source-port"] = strconv.Itoa(t.Port)
		case "dest":
			cfg.moduleConfig[module]["--destination-port"] = strconv.Itoa(t.Port)
		}
	}

	if t.IP != nil {
		switch tType {
		case "src":
			cfg.sourceIP = t.IP.String()
		case "dest":
			cfg.destIP = t.IP.String()
		}
	}

	if t.IPSet != "" {
		module := "-m set"
		if _, found := cfg.moduleConfig[module]; !found {
			cfg.moduleConfig[module] = iptModConfig{}
		}
		value := cfg.moduleConfig[module]["--match-set "+t.IPSet]
		set := ""
		switch tType {
		case "src":
			set = "src"
		case "dest":
			set = "dst"
		}
		if value != "" {
			cfg.moduleConfig[module]["--match-set "+t.IPSet] = value + "," + set
		} else {
			cfg.moduleConfig[module]["--match-set "+t.IPSet] = set
		}
	}

	if t.Interface != "" {
		switch tType {
		case "src":
			cfg.sourceIface = t.Interface
		case "dest":
			cfg.destIface = t.Interface
		}
	}

	if t.NAT != "" {
		switch tType {
		case "src": // for input on the source the destination address can be modified.
			cfg.destNAT = t.NAT
		case "dest": // for output on the destination the source address can be modified.
			cfg.sourceNAT = t.NAT
		}

		if cfg.sourceNAT != "" && cfg.destNAT != "" {
			panic("only source or destination NAT allowed!")
		}
	}
}

// The target of a rule. It can be specified either by IP or the name of an IPSet. Additional parameters are the port
// and interface used. It's totally valid to only specify a subset (or even none) of the fields. For example IP and
// IPSet must not be given for the host the rule is applied on.
type FirewallTarget struct {
	IP        net.IP // IP of the target.
	IPSet     string // IPSet used for matching.
	Port      int    // Port packets must use to match.
	Interface string // Interface the packet goes through.
	NAT       string // NAT configuration (empty, "MASQ", or Interface's IP).
}

const fw_rules_ipv6 = `
*filter
:INPUT DROP [0:0]
:FORWARD DROP [0:0]
:OUTPUT DROP [0:0]

COMMIT
`

const fw_rules_ipv4 = `*filter
:INPUT DROP [0:0]
:FORWARD DROP [0:0]
:OUTPUT DROP [0:0]

# Accept any related or established connections.
-I INPUT 1 -m state --state RELATED,ESTABLISHED -j ACCEPT
-I FORWARD 1 -m state --state RELATED,ESTABLISHED -j ACCEPT
-I OUTPUT 1 -m state --state {{ if not .Paranoid }}NEW,{{ end }}RELATED,ESTABLISHED -j ACCEPT

# Allow all traffic on the loopback interface.
-A INPUT -i lo -j ACCEPT
-A OUTPUT -o lo -j ACCEPT

{{ if .WithVPN }}
# Allow all traffic on the VPN interface.
-A INPUT -i tun0 -j ACCEPT
-A OUTPUT -o tun0 -j ACCEPT
{{ end }}

{{ if .Paranoid }}
# Outbound DNS lookups
-A OUTPUT -o {{ .Interface }} -p udp -m udp --dport 53 -j ACCEPT

# Outbound PING requests
-A OUTPUT -p icmp -j ACCEPT

# Outbound Network Time Protocol (NTP) request
-A OUTPUT -p udp --dport 123 --sport 123 -j ACCEPT

# Allow outbound DHCP request - Some hosts (Linode) automatically assign the primary IP
-A OUTPUT -p udp --dport 67:68 --sport 67:68 -m state --state NEW -j ACCEPT

# Outbound HTTP
-A OUTPUT -o {{ .Interface }} -p tcp -m tcp --dport 80 -m state --state NEW -j ACCEPT
-A OUTPUT -o {{ .Interface }} -p tcp -m tcp --dport 443 -m state --state NEW -j ACCEPT
{{ end }}

# SSH
-A INPUT -i {{ .Interface }} -p tcp -m tcp --dport 22 -m state --state NEW -j ACCEPT

{{ range .Rules }}{{ .Filter }}{{ end }}

{{ if .WithVPN }}
# Outbound OpenVPN traffic (required to connect to the VPN).
-A OUTPUT -o {{ .Interface }} -p tcp -m tcp --dport 1194 -m state --state NEW -j ACCEPT
-A OUTPUT -o {{ .Interface }} -p udp -m udp --dport 1194 -m state --state NEW -j ACCEPT
{{ end }}
COMMIT

*nat
{{ range .Rules }}{{ .NAT }}{{ end }}
COMMIT
`

const fwIpset = `# IPSet configuration
{{ range .IPSets }}{{ .IPSetRestore }}{{ end }}`
const firewallUpstart = `#!/bin/sh
set -e

case "$IFACE" in
	{{ .Interface }})
		test -e /etc/iptables/ipsets && /usr/sbin/ipset restore -! < /etc/iptables/ipsets
		/sbin/iptables-restore < /etc/iptables/rules_ipv4
		/sbin/ip6tables-restore < /etc/iptables/rules_ipv6
		;;
esac

`
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// +build go1.3

package lxd

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"os/exec"
	"strings"
	"syscall"

	"github.com/juju/errors"
	"github.com/juju/utils/packaging/config"
	"github.com/juju/utils/packaging/manager"
	"github.com/juju/utils/proxy"
	"github.com/juju/utils/set"
	"github.com/lxc/lxd/shared"

	"github.com/juju/juju/container"
	"github.com/juju/juju/tools/lxdclient"
)

const lxdBridgeFile = "/etc/default/lxd-bridge"

var requiredPackages = []string{
	"lxd",
}

type containerInitialiser struct {
	series         string
	getExecCommand func(string, ...string) *exec.Cmd
}

// containerInitialiser implements container.Initialiser.
var _ container.Initialiser = (*containerInitialiser)(nil)

// NewContainerInitialiser returns an instance used to perform the steps
// required to allow a host machine to run a LXC container.
func NewContainerInitialiser(series string) container.Initialiser {
	return &containerInitialiser{
		series,
		exec.Command,
	}
}

// Initialise is specified on the container.Initialiser interface.
func (ci *containerInitialiser) Initialise() error {
	err := ensureDependencies(ci.series)
	if err != nil {
		return err
	}

	err = configureLXDBridge()
	if err != nil {
		return err
	}
	proxies := proxy.DetectProxies()
	err = ConfigureLXDProxies(proxies)
	if err != nil {
		return err
	}

	// Well... this will need to change soon once we are passed 17.04 as who
	// knows what the series name will be.
	if ci.series < "xenial" {
		return nil
	}

	output, err := ci.getExecCommand(
		"lxd",
		"init",
		"--auto",
	).CombinedOutput()

	if err == nil {
		return nil
	}

	out := string(output)
	if strings.Contains(out, "You have existing containers or images. lxd init requires an empty LXD.") {
		// this error means we've already run lxd init, which is ok, so just
		// ignore it.
		return nil
	}

	return errors.Annotate(err, "while running lxd init --auto: "+out)
}

// getPackageManager is a helper function which returns the
// package manager implementation for the current system.
func getPackageManager(series string) (manager.PackageManager, error) {
	return manager.NewPackageManager(series)
}

// getPackagingConfigurer is a helper function which returns the
// packaging configuration manager for the current system.
func getPackagingConfigurer(series string) (config.PackagingConfigurer, error) {
	return config.NewPackagingConfigurer(series)
}

// ConfigureLXDProxies will try to set the lxc config core.proxy_http and core.proxy_https
// configuration values based on the current environment.
func ConfigureLXDProxies(proxies proxy.Settings) error {
	setter, err := getLXDConfigSetter()
	if err != nil {
		return errors.Trace(err)
	}
	return errors.Trace(configureLXDProxies(setter, proxies))
}

var getLXDConfigSetter = getConfigSetterConnect

func getConfigSetterConnect() (configSetter, error) {
	return ConnectLocal()
}

type configSetter interface {
	SetServerConfig(key, value string) error
}

func configureLXDProxies(setter configSetter, proxies proxy.Settings) error {
	err := setter.SetServerConfig("core.proxy_http", proxies.Http)
	if err != nil {
		return errors.Trace(err)
	}
	err = setter.SetServerConfig("core.proxy_https", proxies.Https)
	if err != nil {
		return errors.Trace(err)
	}
	err = setter.SetServerConfig("core.proxy_ignore_hosts", proxies.NoProxy)
	if err != nil {
		return errors.Trace(err)
	}
	return nil
}

// df returns the number of free bytes on the file system at the given path
var df = func(path string) (uint64, error) {
	// Note: do not use golang.org/x/sys/unix for this, it is
	// the best solution but will break the build in s390x
	// and introduce cgo dependency lp:1632541
	statfs := syscall.Statfs_t{}
	err := syscall.Statfs(path, &statfs)
	if err != nil {
		return 0, err
	}
	return uint64(statfs.Bsize) * statfs.Bfree, nil
}

var configureLXDBridge = func() error {
	client, err := ConnectLocal()
	if err != nil {
		return errors.Trace(err)
	}

	status, err := client.ServerStatus()
	if err != nil {
		return errors.Trace(err)
	}

	// If LXD itself supports managing networks (added in LXD 2.3) we can allow
	// it to do all of the network configuration.
	if shared.StringInSlice("network", status.APIExtensions) {
		return lxdclient.CreateDefaultBridgeInDefaultProfile(client)
	}
	return configureLXDBridgeForOlderLXD()
}

// configureLXDBridgeForOlderLXD is used for LXD agents that don't support the
// Network API (pre 2.3)
func configureLXDBridgeForOlderLXD() error {
	f, err := os.OpenFile(lxdBridgeFile, os.O_RDWR, 0777)
	if err != nil {
		/* We're using an old version of LXD which doesn't have
		 * lxd-bridge; let's not fail here.
		 */
		if os.IsNotExist(err) {
			logger.Debugf("couldn't find %s, not configuring it", lxdBridgeFile)
			return nil
		}
		return errors.Trace(err)
	}
	defer f.Close()

	existing, err := ioutil.ReadAll(f)
	if err != nil {
		return errors.Trace(err)
	}

	newBridgeCfg, err := bridgeConfiguration(string(existing))
	if err != nil {
		return errors.Trace(err)
	}

	if newBridgeCfg == string(existing) {
		return nil
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		return errors.Trace(err)
	}
	_, err = f.WriteString(newBridgeCfg)
	if err != nil {
		return errors.Trace(err)
	}

	/* non-systemd systems don't have the lxd-bridge service, so this always fails */
	_ = exec.Command("service", "lxd-bridge", "restart").Run()
	return exec.Command("service", "lxd", "restart").Run()
}

var interfaceAddrs = func() ([]net.Addr, error) {
	return net.InterfaceAddrs()
}

func editLXDBridgeFile(input string, subnet string) string {
	buffer := bytes.Buffer{}

	newValues := map[string]string{
		"USE_LXD_BRIDGE":      "true",
		"EXISTING_BRIDGE":     "",
		"LXD_BRIDGE":          "lxdbr0",
		"LXD_IPV4_ADDR":       fmt.Sprintf("10.0.%s.1", subnet),
		"LXD_IPV4_NETMASK":    "255.255.255.0",
		"LXD_IPV4_NETWORK":    fmt.Sprintf("10.0.%s.1/24", subnet),
		"LXD_IPV4_DHCP_RANGE": fmt.Sprintf("10.0.%s.2,10.0.%s.254", subnet, subnet),
		"LXD_IPV4_DHCP_MAX":   "253",
		"LXD_IPV4_NAT":        "true",
		"LXD_IPV6_PROXY":      "false",
	}
	found := map[string]bool{}

	for _, line := range strings.Split(input, "\n") {
		out := line

		for prefix, value := range newValues {
			if strings.HasPrefix(line, prefix+"=") {
				out = fmt.Sprintf(`%s="%s"`, prefix, value)
				found[prefix] = true
				break
			}
		}

		buffer.WriteString(out)
		buffer.WriteString("\n")
	}

	for prefix, value := range newValues {
		if !found[prefix] {
			buffer.WriteString(prefix)
			buffer.WriteString("=")
			buffer.WriteString(value)
			buffer.WriteString("\n")
			found[prefix] = true // not necessary but keeps "found" logically consistent
		}
	}

	return buffer.String()
}

// ensureDependencies creates a set of install packages using
// apt.GetPreparePackages and runs each set of packages through
// apt.GetInstall.
func ensureDependencies(series string) error {
	if series == "precise" {
		return fmt.Errorf("LXD is not supported in precise.")
	}

	pacman, err := getPackageManager(series)
	if err != nil {
		return err
	}
	pacconfer, err := getPackagingConfigurer(series)
	if err != nil {
		return err
	}

	for _, pack := range requiredPackages {
		pkg := pack
		if config.SeriesRequiresCloudArchiveTools(series) &&
			pacconfer.IsCloudArchivePackage(pack) {
			pkg = strings.Join(pacconfer.ApplyCloudArchiveTarget(pack), " ")
		}

		if config.RequiresBackports(series, pack) {
			pkg = fmt.Sprintf("--target-release %s-backports %s", series, pkg)
		}

		if err := pacman.Install(pkg); err != nil {
			return err
		}
	}

	return err
}

// randomizedOctetRange is a variable for testing purposes.
var randomizedOctetRange = func() []int { return rand.Perm(255) }

// getKnownV4IPsAndCIDRs iterates all of the known Addresses on this machine
// and groups them up into known CIDRs and IP addresses.
func getKnownV4IPsAndCIDRs(addrFunc func() ([]net.Addr, error)) ([]net.IP, []*net.IPNet, error) {
	addrs, err := addrFunc()
	if err != nil {
		return nil, nil, errors.Annotate(err, "cannot get network interface addresses")
	}

	knownIPs := []net.IP{}
	seenIPs := set.NewStrings()
	knownCIDRs := []*net.IPNet{}
	seenCIDRs := set.NewStrings()
	for _, netAddr := range addrs {
		ip, ipNet, err := net.ParseCIDR(netAddr.String())
		if err != nil {
			continue
		}
		if ip.To4() == nil {
			continue
		}
		if !seenIPs.Contains(ip.String()) {
			knownIPs = append(knownIPs, ip)
			seenIPs.Add(ip.String())
		}
		if !seenCIDRs.Contains(ipNet.String()) {
			knownCIDRs = append(knownCIDRs, ipNet)
			seenCIDRs.Add(ipNet.String())
		}
	}
	return knownIPs, knownCIDRs, nil
}

// findNextAvailableIPv4Subnet scans the list of interfaces on the machine
// looking for 10.0.0.0/16 networks and returns the next subnet not in
// use, having first detected the highest subnet. The next subnet can
// actually be lower if we overflowed 255 whilst seeking out the next
// unused subnet. If all subnets are in use an error is returned.
//
// TODO(frobware): this is not an ideal solution as it doesn't take
// into account any static routes that may be set up on the machine.
//
// TODO(frobware): this only caters for IPv4 setups.
func findNextAvailableIPv4Subnet() (string, error) {
	knownIPs, knownCIDRs, err := getKnownV4IPsAndCIDRs(interfaceAddrs)
	if err != nil {
		return "", errors.Trace(err)
	}

	randomized3rdSegment := randomizedOctetRange()
	for _, i := range randomized3rdSegment {
		// lxd randomizes the 2nd and 3rd segments, we should be fine with the
		// 3rd only
		ip, ip10network, err := net.ParseCIDR(fmt.Sprintf("10.0.%d.0/24", i))
		if err != nil {
			return "", errors.Trace(err)
		}

		collides := false
		for _, kIP := range knownIPs {
			if ip10network.Contains(kIP) {
				collides = true
				break
			}
		}
		if !collides {
			for _, kNet := range knownCIDRs {
				if kNet.Contains(ip) || ip10network.Contains(kNet.IP) {
					collides = true
					break
				}
			}
		}
		if !collides {
			return fmt.Sprintf("%d", i), nil

		}
	}
	return "", errors.New("could not find unused subnet")
}

func parseLXDBridgeConfigValues(input string) map[string]string {
	values := make(map[string]string)

	for _, line := range strings.Split(input, "\n") {
		line = strings.TrimSpace(line)

		if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") {
			continue
		}

		tokens := strings.Split(line, "=")

		if tokens[0] == "" {
			continue // no key
		}

		value := ""

		if len(tokens) > 1 {
			value = tokens[1]
			if strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`) {
				value = strings.Trim(value, `"`)
			}
		}

		values[tokens[0]] = value
	}
	return values
}

// bridgeConfiguration ensures that input has a valid setting for
// LXD_IPV4_ADDR, returning the existing input if is already set, and
// allocating the next available subnet if it is not.
func bridgeConfiguration(input string) (string, error) {
	values := parseLXDBridgeConfigValues(input)
	ipAddr := net.ParseIP(values["LXD_IPV4_ADDR"])

	if ipAddr == nil || ipAddr.To4() == nil {
		logger.Infof("LXD_IPV4_ADDR is not set; searching for unused subnet")
		subnet, err := findNextAvailableIPv4Subnet()
		if err != nil {
			return "", errors.Trace(err)
		}
		logger.Infof("setting LXD_IPV4_ADDR=10.0.%s.1", subnet)
		return editLXDBridgeFile(input, subnet), nil
	}
	return input, nil
}
<commit_msg>Added a time based seed  value.<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// +build go1.3

package lxd

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"

	"github.com/juju/errors"
	"github.com/juju/utils/packaging/config"
	"github.com/juju/utils/packaging/manager"
	"github.com/juju/utils/proxy"
	"github.com/juju/utils/set"
	"github.com/lxc/lxd/shared"

	"github.com/juju/juju/container"
	"github.com/juju/juju/tools/lxdclient"
)

const lxdBridgeFile = "/etc/default/lxd-bridge"

var requiredPackages = []string{
	"lxd",
}

type containerInitialiser struct {
	series         string
	getExecCommand func(string, ...string) *exec.Cmd
}

// containerInitialiser implements container.Initialiser.
var _ container.Initialiser = (*containerInitialiser)(nil)

// NewContainerInitialiser returns an instance used to perform the steps
// required to allow a host machine to run a LXC container.
func NewContainerInitialiser(series string) container.Initialiser {
	return &containerInitialiser{
		series,
		exec.Command,
	}
}

// Initialise is specified on the container.Initialiser interface.
func (ci *containerInitialiser) Initialise() error {
	err := ensureDependencies(ci.series)
	if err != nil {
		return err
	}

	err = configureLXDBridge()
	if err != nil {
		return err
	}
	proxies := proxy.DetectProxies()
	err = ConfigureLXDProxies(proxies)
	if err != nil {
		return err
	}

	// Well... this will need to change soon once we are passed 17.04 as who
	// knows what the series name will be.
	if ci.series < "xenial" {
		return nil
	}

	output, err := ci.getExecCommand(
		"lxd",
		"init",
		"--auto",
	).CombinedOutput()

	if err == nil {
		return nil
	}

	out := string(output)
	if strings.Contains(out, "You have existing containers or images. lxd init requires an empty LXD.") {
		// this error means we've already run lxd init, which is ok, so just
		// ignore it.
		return nil
	}

	return errors.Annotate(err, "while running lxd init --auto: "+out)
}

// getPackageManager is a helper function which returns the
// package manager implementation for the current system.
func getPackageManager(series string) (manager.PackageManager, error) {
	return manager.NewPackageManager(series)
}

// getPackagingConfigurer is a helper function which returns the
// packaging configuration manager for the current system.
func getPackagingConfigurer(series string) (config.PackagingConfigurer, error) {
	return config.NewPackagingConfigurer(series)
}

// ConfigureLXDProxies will try to set the lxc config core.proxy_http and core.proxy_https
// configuration values based on the current environment.
func ConfigureLXDProxies(proxies proxy.Settings) error {
	setter, err := getLXDConfigSetter()
	if err != nil {
		return errors.Trace(err)
	}
	return errors.Trace(configureLXDProxies(setter, proxies))
}

var getLXDConfigSetter = getConfigSetterConnect

func getConfigSetterConnect() (configSetter, error) {
	return ConnectLocal()
}

type configSetter interface {
	SetServerConfig(key, value string) error
}

func configureLXDProxies(setter configSetter, proxies proxy.Settings) error {
	err := setter.SetServerConfig("core.proxy_http", proxies.Http)
	if err != nil {
		return errors.Trace(err)
	}
	err = setter.SetServerConfig("core.proxy_https", proxies.Https)
	if err != nil {
		return errors.Trace(err)
	}
	err = setter.SetServerConfig("core.proxy_ignore_hosts", proxies.NoProxy)
	if err != nil {
		return errors.Trace(err)
	}
	return nil
}

// df returns the number of free bytes on the file system at the given path
var df = func(path string) (uint64, error) {
	// Note: do not use golang.org/x/sys/unix for this, it is
	// the best solution but will break the build in s390x
	// and introduce cgo dependency lp:1632541
	statfs := syscall.Statfs_t{}
	err := syscall.Statfs(path, &statfs)
	if err != nil {
		return 0, err
	}
	return uint64(statfs.Bsize) * statfs.Bfree, nil
}

var configureLXDBridge = func() error {
	client, err := ConnectLocal()
	if err != nil {
		return errors.Trace(err)
	}

	status, err := client.ServerStatus()
	if err != nil {
		return errors.Trace(err)
	}

	// If LXD itself supports managing networks (added in LXD 2.3) we can allow
	// it to do all of the network configuration.
	if shared.StringInSlice("network", status.APIExtensions) {
		return lxdclient.CreateDefaultBridgeInDefaultProfile(client)
	}
	return configureLXDBridgeForOlderLXD()
}

// configureLXDBridgeForOlderLXD is used for LXD agents that don't support the
// Network API (pre 2.3)
func configureLXDBridgeForOlderLXD() error {
	f, err := os.OpenFile(lxdBridgeFile, os.O_RDWR, 0777)
	if err != nil {
		/* We're using an old version of LXD which doesn't have
		 * lxd-bridge; let's not fail here.
		 */
		if os.IsNotExist(err) {
			logger.Debugf("couldn't find %s, not configuring it", lxdBridgeFile)
			return nil
		}
		return errors.Trace(err)
	}
	defer f.Close()

	existing, err := ioutil.ReadAll(f)
	if err != nil {
		return errors.Trace(err)
	}

	newBridgeCfg, err := bridgeConfiguration(string(existing))
	if err != nil {
		return errors.Trace(err)
	}

	if newBridgeCfg == string(existing) {
		return nil
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		return errors.Trace(err)
	}
	_, err = f.WriteString(newBridgeCfg)
	if err != nil {
		return errors.Trace(err)
	}

	/* non-systemd systems don't have the lxd-bridge service, so this always fails */
	_ = exec.Command("service", "lxd-bridge", "restart").Run()
	return exec.Command("service", "lxd", "restart").Run()
}

var interfaceAddrs = func() ([]net.Addr, error) {
	return net.InterfaceAddrs()
}

func editLXDBridgeFile(input string, subnet string) string {
	buffer := bytes.Buffer{}

	newValues := map[string]string{
		"USE_LXD_BRIDGE":      "true",
		"EXISTING_BRIDGE":     "",
		"LXD_BRIDGE":          "lxdbr0",
		"LXD_IPV4_ADDR":       fmt.Sprintf("10.0.%s.1", subnet),
		"LXD_IPV4_NETMASK":    "255.255.255.0",
		"LXD_IPV4_NETWORK":    fmt.Sprintf("10.0.%s.1/24", subnet),
		"LXD_IPV4_DHCP_RANGE": fmt.Sprintf("10.0.%s.2,10.0.%s.254", subnet, subnet),
		"LXD_IPV4_DHCP_MAX":   "253",
		"LXD_IPV4_NAT":        "true",
		"LXD_IPV6_PROXY":      "false",
	}
	found := map[string]bool{}

	for _, line := range strings.Split(input, "\n") {
		out := line

		for prefix, value := range newValues {
			if strings.HasPrefix(line, prefix+"=") {
				out = fmt.Sprintf(`%s="%s"`, prefix, value)
				found[prefix] = true
				break
			}
		}

		buffer.WriteString(out)
		buffer.WriteString("\n")
	}

	for prefix, value := range newValues {
		if !found[prefix] {
			buffer.WriteString(prefix)
			buffer.WriteString("=")
			buffer.WriteString(value)
			buffer.WriteString("\n")
			found[prefix] = true // not necessary but keeps "found" logically consistent
		}
	}

	return buffer.String()
}

// ensureDependencies creates a set of install packages using
// apt.GetPreparePackages and runs each set of packages through
// apt.GetInstall.
func ensureDependencies(series string) error {
	if series == "precise" {
		return fmt.Errorf("LXD is not supported in precise.")
	}

	pacman, err := getPackageManager(series)
	if err != nil {
		return err
	}
	pacconfer, err := getPackagingConfigurer(series)
	if err != nil {
		return err
	}

	for _, pack := range requiredPackages {
		pkg := pack
		if config.SeriesRequiresCloudArchiveTools(series) &&
			pacconfer.IsCloudArchivePackage(pack) {
			pkg = strings.Join(pacconfer.ApplyCloudArchiveTarget(pack), " ")
		}

		if config.RequiresBackports(series, pack) {
			pkg = fmt.Sprintf("--target-release %s-backports %s", series, pkg)
		}

		if err := pacman.Install(pkg); err != nil {
			return err
		}
	}

	return err
}

// randomizedOctetRange is a variable for testing purposes.
var randomizedOctetRange = func() []int {
	rand.Seed(time.Now().UnixNano())
	return rand.Perm(255)
}

// getKnownV4IPsAndCIDRs iterates all of the known Addresses on this machine
// and groups them up into known CIDRs and IP addresses.
func getKnownV4IPsAndCIDRs(addrFunc func() ([]net.Addr, error)) ([]net.IP, []*net.IPNet, error) {
	addrs, err := addrFunc()
	if err != nil {
		return nil, nil, errors.Annotate(err, "cannot get network interface addresses")
	}

	knownIPs := []net.IP{}
	seenIPs := set.NewStrings()
	knownCIDRs := []*net.IPNet{}
	seenCIDRs := set.NewStrings()
	for _, netAddr := range addrs {
		ip, ipNet, err := net.ParseCIDR(netAddr.String())
		if err != nil {
			continue
		}
		if ip.To4() == nil {
			continue
		}
		if !seenIPs.Contains(ip.String()) {
			knownIPs = append(knownIPs, ip)
			seenIPs.Add(ip.String())
		}
		if !seenCIDRs.Contains(ipNet.String()) {
			knownCIDRs = append(knownCIDRs, ipNet)
			seenCIDRs.Add(ipNet.String())
		}
	}
	return knownIPs, knownCIDRs, nil
}

// findNextAvailableIPv4Subnet scans the list of interfaces on the machine
// looking for 10.0.0.0/16 networks and returns the next subnet not in
// use, having first detected the highest subnet. The next subnet can
// actually be lower if we overflowed 255 whilst seeking out the next
// unused subnet. If all subnets are in use an error is returned.
//
// TODO(frobware): this is not an ideal solution as it doesn't take
// into account any static routes that may be set up on the machine.
//
// TODO(frobware): this only caters for IPv4 setups.
func findNextAvailableIPv4Subnet() (string, error) {
	knownIPs, knownCIDRs, err := getKnownV4IPsAndCIDRs(interfaceAddrs)
	if err != nil {
		return "", errors.Trace(err)
	}

	randomized3rdSegment := randomizedOctetRange()
	for _, i := range randomized3rdSegment {
		// lxd randomizes the 2nd and 3rd segments, we should be fine with the
		// 3rd only
		ip, ip10network, err := net.ParseCIDR(fmt.Sprintf("10.0.%d.0/24", i))
		if err != nil {
			return "", errors.Trace(err)
		}

		collides := false
		for _, kIP := range knownIPs {
			if ip10network.Contains(kIP) {
				collides = true
				break
			}
		}
		if !collides {
			for _, kNet := range knownCIDRs {
				if kNet.Contains(ip) || ip10network.Contains(kNet.IP) {
					collides = true
					break
				}
			}
		}
		if !collides {
			return fmt.Sprintf("%d", i), nil
		}
	}
	return "", errors.New("could not find unused subnet")
}

func parseLXDBridgeConfigValues(input string) map[string]string {
	values := make(map[string]string)

	for _, line := range strings.Split(input, "\n") {
		line = strings.TrimSpace(line)

		if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") {
			continue
		}

		tokens := strings.Split(line, "=")

		if tokens[0] == "" {
			continue // no key
		}

		value := ""

		if len(tokens) > 1 {
			value = tokens[1]
			if strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`) {
				value = strings.Trim(value, `"`)
			}
		}

		values[tokens[0]] = value
	}
	return values
}

// bridgeConfiguration ensures that input has a valid setting for
// LXD_IPV4_ADDR, returning the existing input if is already set, and
// allocating the next available subnet if it is not.
func bridgeConfiguration(input string) (string, error) {
	values := parseLXDBridgeConfigValues(input)
	ipAddr := net.ParseIP(values["LXD_IPV4_ADDR"])

	if ipAddr == nil || ipAddr.To4() == nil {
		logger.Infof("LXD_IPV4_ADDR is not set; searching for unused subnet")
		subnet, err := findNextAvailableIPv4Subnet()
		if err != nil {
			return "", errors.Trace(err)
		}
		logger.Infof("setting LXD_IPV4_ADDR=10.0.%s.1", subnet)
		return editLXDBridgeFile(input, subnet), nil
	}
	return input, nil
}
<|endoftext|>


<commit_before>package main

import "testing"

func TestFibonacci(t *testing.T) {
	tcs := map[string]struct {
		n        int
		expected int
	}{
		"n = 1":   {1, 1},
		"n = 2":   {2, 1},
		"n = 50":  {50, 12586269025},
		"n = 200": {200, 280571172992510140037611932413038677189525},
	}

	for name, tc := range tcs {
		t.Run(name, func(t *testing.T) {
			res := fib(tc.n)
			if res != tc.expected {
				t.Fatalf("want %v, got %v\n", tc.expected, res)
			}
		})
	}
}
<commit_msg>Change expected value in test cases to string to allow for values that don't fit in an int64<commit_after>package main

import (
	"math/big"
	"testing"
)

func TestFibonacci(t *testing.T) {
	tcs := map[string]struct {
		n        int
		expected string //Use string to allow for values that overflow an int64
	}{
		"n = 0":   {0, "0"},
		"n = 1":   {1, "1"},
		"n = 2":   {2, "1"},
		"n = 50":  {50, "12586269025"},
		"n = 100": {100, "354224848179261915075"},
		"n = 200": {200, "280571172992510140037611932413038677189525"},
	}

	for name, tc := range tcs {
		t.Run(name, func(t *testing.T) {
			expected, ok := big.NewInt(0).SetString(tc.expected, 10)
			if !ok {
				t.Fatalf("Bad expected value in test case: %s", tc.expected)
			}
			res := fib(tc.n)
			if res != expected {
				t.Fatalf("want %v, got %v\n", tc.expected, res)
			}
		})
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/iam"

	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsIamSamlProvider() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsIamSamlProviderCreate,
		Read:   resourceAwsIamSamlProviderRead,
		Update: resourceAwsIamSamlProviderUpdate,
		Delete: resourceAwsIamSamlProviderDelete,

		Schema: map[string]*schema.Schema{
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"valid_until": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"saml_metadata_document": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
		},
	}
}

func resourceAwsIamSamlProviderCreate(d *schema.ResourceData, meta interface{}) error {
	iamconn := meta.(*AWSClient).iamconn

	input := &iam.CreateSAMLProviderInput{
		Name:                 aws.String(d.Get("name").(string)),
		SAMLMetadataDocument: aws.String(d.Get("saml_metadata_document").(string)),
	}

	out, err := iamconn.CreateSAMLProvider(input)
	if err != nil {
		return err
	}

	d.SetId(*out.SAMLProviderArn)

	return resourceAwsIamSamlProviderRead(d, meta)
}

func resourceAwsIamSamlProviderRead(d *schema.ResourceData, meta interface{}) error {
	iamconn := meta.(*AWSClient).iamconn

	input := &iam.GetSAMLProviderInput{
		SAMLProviderArn: aws.String(d.Id()),
	}
	out, err := iamconn.GetSAMLProvider(input)
	if err != nil {
		return err
	}

	validUntil := out.ValidUntil.Format(time.RFC1123)
	d.Set("valid_until", validUntil)
	d.Set("saml_metadata_document", *out.SAMLMetadataDocument)

	return nil
}

func resourceAwsIamSamlProviderUpdate(d *schema.ResourceData, meta interface{}) error {
	iamconn := meta.(*AWSClient).iamconn

	input := &iam.UpdateSAMLProviderInput{
		SAMLProviderArn:      aws.String(d.Id()),
		SAMLMetadataDocument: aws.String(d.Get("saml_metadata_document").(string)),
	}
	_, err := iamconn.UpdateSAMLProvider(input)
	if err != nil {
		return err
	}

	return resourceAwsIamSamlProviderRead(d, meta)
}

func resourceAwsIamSamlProviderDelete(d *schema.ResourceData, meta interface{}) error {
	iamconn := meta.(*AWSClient).iamconn

	input := &iam.DeleteSAMLProviderInput{
		SAMLProviderArn: aws.String(d.Id()),
	}
	_, err := iamconn.DeleteSAMLProvider(input)

	return err
}
<commit_msg>provider/aws: Set previously missing ARN in iam_saml_provider<commit_after>package aws

import (
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/iam"

	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsIamSamlProvider() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsIamSamlProviderCreate,
		Read:   resourceAwsIamSamlProviderRead,
		Update: resourceAwsIamSamlProviderUpdate,
		Delete: resourceAwsIamSamlProviderDelete,

		Schema: map[string]*schema.Schema{
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"valid_until": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"saml_metadata_document": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
		},
	}
}

func resourceAwsIamSamlProviderCreate(d *schema.ResourceData, meta interface{}) error {
	iamconn := meta.(*AWSClient).iamconn

	input := &iam.CreateSAMLProviderInput{
		Name:                 aws.String(d.Get("name").(string)),
		SAMLMetadataDocument: aws.String(d.Get("saml_metadata_document").(string)),
	}

	out, err := iamconn.CreateSAMLProvider(input)
	if err != nil {
		return err
	}

	d.SetId(*out.SAMLProviderArn)

	return resourceAwsIamSamlProviderRead(d, meta)
}

func resourceAwsIamSamlProviderRead(d *schema.ResourceData, meta interface{}) error {
	iamconn := meta.(*AWSClient).iamconn

	input := &iam.GetSAMLProviderInput{
		SAMLProviderArn: aws.String(d.Id()),
	}
	out, err := iamconn.GetSAMLProvider(input)
	if err != nil {
		return err
	}

	validUntil := out.ValidUntil.Format(time.RFC1123)
	d.Set("arn", d.Id())
	d.Set("valid_until", validUntil)
	d.Set("saml_metadata_document", *out.SAMLMetadataDocument)

	return nil
}

func resourceAwsIamSamlProviderUpdate(d *schema.ResourceData, meta interface{}) error {
	iamconn := meta.(*AWSClient).iamconn

	input := &iam.UpdateSAMLProviderInput{
		SAMLProviderArn:      aws.String(d.Id()),
		SAMLMetadataDocument: aws.String(d.Get("saml_metadata_document").(string)),
	}
	_, err := iamconn.UpdateSAMLProvider(input)
	if err != nil {
		return err
	}

	return resourceAwsIamSamlProviderRead(d, meta)
}

func resourceAwsIamSamlProviderDelete(d *schema.ResourceData, meta interface{}) error {
	iamconn := meta.(*AWSClient).iamconn

	input := &iam.DeleteSAMLProviderInput{
		SAMLProviderArn: aws.String(d.Id()),
	}
	_, err := iamconn.DeleteSAMLProvider(input)

	return err
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package agent

import (
	"fmt"
	"net"
	"strings"
	"sync"

	"github.com/romana/core/common"
	"github.com/romana/core/common/log/trace"
	log "github.com/romana/rlog"
)

// NetworkConfig holds the agent's current configuration.
// This consists of data parsed from the config file as well as
// runtime or discovered configuration, such as the network
// config of the current host.
// NetworkConfig public methods are used to implement
// firewall.NetConfig interface.
type NetworkConfig struct {
	// Current host network configuration
	sync.Mutex
	romanaGW        net.IP
	romanaGWMask    net.IPMask
	oldRomanaGW     net.IP
	oldRomanaGWMask net.IPMask
	otherHosts      []common.Host
	dc              common.Datacenter
}

// EndpointNetmaskSize returns integer value (aka size) of endpoint netmask.
func (c *NetworkConfig) EndpointNetmaskSize() uint64 {
	// TODO make this depend on the IP version
	return 32 - uint64(c.dc.EndpointSpaceBits)
}

// PNetCIDR returns pseudo net cidr in net.IPNet format.
func (c *NetworkConfig) PNetCIDR() (cidr *net.IPNet, err error) {
	_, cidr, err = net.ParseCIDR(c.dc.Cidr)
	return
}

// PrefixBits returns tenant bits value from POC config.
func (c *NetworkConfig) PrefixBits() uint {
	return c.dc.PrefixBits
}

// PortBits returns tenant bits value from POC config.
func (c *NetworkConfig) PortBits() uint {
	return c.dc.PortBits
}

// TenantBits returns tenant bits value from POC config.
func (c *NetworkConfig) TenantBits() uint {
	return c.dc.TenantBits
}

// SegmentBits returns segment bits value from POC config.
func (c *NetworkConfig) SegmentBits() uint {
	return c.dc.SegmentBits
}

// EndpointBits returns endpoint bits value from POC config.
func (c *NetworkConfig) EndpointBits() uint {
	return c.dc.EndpointBits
}

// RomanaGW returns current romana gateway.
func (c *NetworkConfig) RomanaGW() net.IP {
	return c.romanaGW
}

// RomanaGWMask returns current romana gateway mask.
func (c *NetworkConfig) RomanaGWMask() net.IPMask {
	return c.romanaGWMask
}

// updateRomanaGWMask updates romana gateway and mask to new
// value and moves the old one to old* variables. The old
// address and mask are used later to identify the address
// we need to delete from the Romana GW interface.
func (c *NetworkConfig) updateRomanaGWMask(ipnet *net.IPNet) {
	c.Lock()
	defer c.Unlock()

	log.Tracef(4, "Current Romana Gateway: %s/%s", c.romanaGW, c.romanaGWMask)
	log.Tracef(4, "New Romana Gateway: %s", ipnet.String())

	c.oldRomanaGW = c.romanaGW
	c.oldRomanaGWMask = c.romanaGWMask
	c.romanaGW = ipnet.IP
	c.romanaGWMask = ipnet.Mask
}

// identifyCurrentHost contacts topology service and get
// details about the node the agent is running on, it then
// populates IP and Mask fields, by comparing hostname with
// one received from topology service.
func (a Agent) identifyCurrentHost() error {
	log.Trace(trace.Private, "In Agent identifyCurrentHost()")

	topologyURL, err := a.client.GetServiceUrl("topology")
	if err != nil {
		return agentError(err)
	}
	index := common.IndexResponse{}
	err = a.client.Get(topologyURL, &index)
	if err != nil {
		return agentError(err)
	}
	dcURL := index.Links.FindByRel("datacenter")
	a.networkConfig.dc = common.Datacenter{}
	err = a.client.Get(dcURL, &a.networkConfig.dc)
	if err != nil {
		return agentError(err)
	}

	hostURL := index.Links.FindByRel("host-list")
	hosts := []common.Host{}
	err = a.client.Get(hostURL, &hosts)
	if err != nil {
		return agentError(err)
	}
	log.Trace(trace.Inside, "Retrieved hosts list, found", len(hosts), "hosts")

	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return err
	}

	log.Tracef(trace.Inside, "Searching %d interfaces for a matching host configuration: %v", len(addrs), addrs)

	for _, host := range hosts {
		for _, addr := range addrs {
			if strings.Contains(addr.String(), host.Ip) {

				_, ipnet, err := net.ParseCIDR(host.RomanaIp)
				if err != nil {
					return fmt.Errorf("Unable to parse Romana IP Address: %s", host.RomanaIp)
				}

				romanaIP, err := GetFirstIPinCIDR(ipnet)
				if err != nil {
					fmt.Println(err)
				}
				log.Debug("Assigning Romana IP: ", romanaIP)

				a.networkConfig.updateRomanaGWMask(romanaIP)

				log.Trace(trace.Inside, "Found match for host", host, "using address", addr)

				return nil
			}
		}
	}

	return agentErrorString("Unable to identify current host, host not added to romana cluster.")
}

// updateRoutes discovers network configuration
// of the host we are running on.
// We need to know public IP and Romana gateway IP of the current host.
// This is done by matching current host IP addresses against what topology
// service thinks the host address is.
// If no match is found we assume we are running on host which is not
// part of the Romana setup and spit error out.
func (a Agent) updateRoutes() error {
	log.Trace(trace.Private, "In Agent updateRoutes()")

	topologyURL, err := a.client.GetServiceUrl("topology")
	if err != nil {
		return agentError(err)
	}
	index := common.IndexResponse{}
	err = a.client.Get(topologyURL, &index)
	if err != nil {
		return agentError(err)
	}
	dcURL := index.Links.FindByRel("datacenter")
	a.networkConfig.dc = common.Datacenter{}
	err = a.client.Get(dcURL, &a.networkConfig.dc)
	if err != nil {
		return agentError(err)
	}

	hostURL := index.Links.FindByRel("host-list")
	hosts := []common.Host{}
	err = a.client.Get(hostURL, &hosts)
	if err != nil {
		return agentError(err)
	}
	log.Trace(trace.Inside, "Retrieved hosts list, found", len(hosts), "hosts")

	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return err
	}

	log.Tracef(trace.Inside, "Searching %d interfaces for a matching host configuration: %v", len(addrs), addrs)

	// Find an interface that matches a Romana CIDR
	// and store that interface's IP address.
	// It will be used when configuring iptables and routes to tap interfaces.
	for i, host := range hosts {
		_, romanaCIDR, err := net.ParseCIDR(host.RomanaIp)
		if err != nil {
			log.Tracef(trace.Inside, "Unable to parse '%s' (%s). Skipping.", host.RomanaIp, err)
			continue
		}
		for _, addr := range addrs {
			ipnet, ok := addr.(*net.IPNet)
			if !ok {
				continue
			}
			if romanaCIDR.Contains(ipnet.IP) {
				// Check that it's the same subnet size
				s1, _ := romanaCIDR.Mask.Size()
				s2, _ := ipnet.Mask.Size()
				if s1 != s2 {
					continue
				}

				// Check if romanaGW address was changed recently, if it was
				// then remove the old one and set the new one we received.
				//
				// romana-gw can change due to multiple reasons:
				// * when node is cordoned/un-cordoned thus no pods except
				//   daemon set are there.
				// * when node is rebooted and it takes it long enough that the
				//   current cidr was re-allocated to other node.
				//
				romanaGWMaskSize, _ := a.networkConfig.romanaGWMask.Size()
				ipnetMaskSize, _ := ipnet.Mask.Size()
				if !a.networkConfig.romanaGW.Equal(ipnet.IP) ||
					romanaGWMaskSize != ipnetMaskSize {

					// For now we don't need updating romanaGW address
					// and mask, so disable it, but do log a message
					// about it.
					if true {
						log.Error("Agent: romana gateway IP Address/Mask changed on the node.")
					} else {
						a.networkConfig.updateRomanaGWMask(ipnet)

						// Update Romana Gateway with new config.
						if err := a.createRomanaGW(); err != nil {
							log.Error("Agent: Failed to update romana gateway IP Address on the node:", err)
							return err
						}
					}
				}

				log.Trace(trace.Private, "Acquiring mutex identifyCurrentHost")
				a.Helper.ensureInterHostRoutesMutex.Lock()

				// Retain the other hosts that were listed.
				// This will be used for creating inter-host routes.
				a.networkConfig.otherHosts = append(a.networkConfig.otherHosts, hosts[0:i]...)
				a.networkConfig.otherHosts = append(a.networkConfig.otherHosts, hosts[i+1:]...)
				log.Trace(trace.Inside, "Found match for CIDR", romanaCIDR, "using address", ipnet.IP)

				log.Trace(trace.Private, "Releasing mutex identifyCurrentHost")
				a.Helper.ensureInterHostRoutesMutex.Unlock()

				return nil
			}
		}
	}
	return agentErrorString("Unable to find interface matching any Romana CIDR")
}

func GetFirstIPinCIDR(ipnet *net.IPNet) (*net.IPNet, error) {
	if ipnet == nil {
		return nil, fmt.Errorf("Agent Error, no input provided to GetFirstIPinCIDR")
	}
	if ipnet.IP.String() == "<nil>" || ipnet.Mask.String() == "<nil>" {
		return nil, fmt.Errorf("Agent Error, invalid IP/Mask provided to GetFirstIPinCIDR")
	}

	ip := ipnet.IP.Mask(ipnet.Mask)
	for i := len(ip) - 1; i >= 0; i-- {
		ip[i]++
		if ip[i] > 0 {
			break
		}
	}

	if !ipnet.Contains(ip) {
		return nil, fmt.Errorf("Error, no more IP address left in the network provided: %s", ipnet.String())
	}

	return &net.IPNet{IP: ip, Mask: ipnet.Mask}, nil
}
<commit_msg>Fixes memory leak in romana agent<commit_after>// Copyright (c) 2017 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package agent

import (
	"fmt"
	"net"
	"strings"
	"sync"

	"github.com/romana/core/common"
	"github.com/romana/core/common/log/trace"
	log "github.com/romana/rlog"
)

// NetworkConfig holds the agent's current configuration.
// This consists of data parsed from the config file as well as
// runtime or discovered configuration, such as the network
// config of the current host.
// NetworkConfig public methods are used to implement
// firewall.NetConfig interface.
type NetworkConfig struct {
	// Current host network configuration
	sync.Mutex
	romanaGW        net.IP
	romanaGWMask    net.IPMask
	oldRomanaGW     net.IP
	oldRomanaGWMask net.IPMask
	otherHosts      []common.Host
	dc              common.Datacenter
}

// EndpointNetmaskSize returns integer value (aka size) of endpoint netmask.
func (c *NetworkConfig) EndpointNetmaskSize() uint64 {
	// TODO make this depend on the IP version
	return 32 - uint64(c.dc.EndpointSpaceBits)
}

// PNetCIDR returns pseudo net cidr in net.IPNet format.
func (c *NetworkConfig) PNetCIDR() (cidr *net.IPNet, err error) {
	_, cidr, err = net.ParseCIDR(c.dc.Cidr)
	return
}

// PrefixBits returns tenant bits value from POC config.
func (c *NetworkConfig) PrefixBits() uint {
	return c.dc.PrefixBits
}

// PortBits returns tenant bits value from POC config.
func (c *NetworkConfig) PortBits() uint {
	return c.dc.PortBits
}

// TenantBits returns tenant bits value from POC config.
func (c *NetworkConfig) TenantBits() uint {
	return c.dc.TenantBits
}

// SegmentBits returns segment bits value from POC config.
func (c *NetworkConfig) SegmentBits() uint {
	return c.dc.SegmentBits
}

// EndpointBits returns endpoint bits value from POC config.
func (c *NetworkConfig) EndpointBits() uint {
	return c.dc.EndpointBits
}

// RomanaGW returns current romana gateway.
func (c *NetworkConfig) RomanaGW() net.IP {
	return c.romanaGW
}

// RomanaGWMask returns current romana gateway mask.
func (c *NetworkConfig) RomanaGWMask() net.IPMask {
	return c.romanaGWMask
}

// updateRomanaGWMask updates romana gateway and mask to new
// value and moves the old one to old* variables. The old
// address and mask are used later to identify the address
// we need to delete from the Romana GW interface.
func (c *NetworkConfig) updateRomanaGWMask(ipnet *net.IPNet) {
	c.Lock()
	defer c.Unlock()

	log.Tracef(4, "Current Romana Gateway: %s/%s", c.romanaGW, c.romanaGWMask)
	log.Tracef(4, "New Romana Gateway: %s", ipnet.String())

	c.oldRomanaGW = c.romanaGW
	c.oldRomanaGWMask = c.romanaGWMask
	c.romanaGW = ipnet.IP
	c.romanaGWMask = ipnet.Mask
}

// identifyCurrentHost contacts topology service and get
// details about the node the agent is running on, it then
// populates IP and Mask fields, by comparing hostname with
// one received from topology service.
func (a Agent) identifyCurrentHost() error {
	log.Trace(trace.Private, "In Agent identifyCurrentHost()")

	topologyURL, err := a.client.GetServiceUrl("topology")
	if err != nil {
		return agentError(err)
	}
	index := common.IndexResponse{}
	err = a.client.Get(topologyURL, &index)
	if err != nil {
		return agentError(err)
	}
	dcURL := index.Links.FindByRel("datacenter")
	a.networkConfig.dc = common.Datacenter{}
	err = a.client.Get(dcURL, &a.networkConfig.dc)
	if err != nil {
		return agentError(err)
	}

	hostURL := index.Links.FindByRel("host-list")
	hosts := []common.Host{}
	err = a.client.Get(hostURL, &hosts)
	if err != nil {
		return agentError(err)
	}
	log.Trace(trace.Inside, "Retrieved hosts list, found", len(hosts), "hosts")

	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return err
	}

	log.Tracef(trace.Inside, "Searching %d interfaces for a matching host configuration: %v", len(addrs), addrs)

	for _, host := range hosts {
		for _, addr := range addrs {
			if strings.Contains(addr.String(), host.Ip) {

				_, ipnet, err := net.ParseCIDR(host.RomanaIp)
				if err != nil {
					return fmt.Errorf("Unable to parse Romana IP Address: %s", host.RomanaIp)
				}

				romanaIP, err := GetFirstIPinCIDR(ipnet)
				if err != nil {
					fmt.Println(err)
				}
				log.Debug("Assigning Romana IP: ", romanaIP)

				a.networkConfig.updateRomanaGWMask(romanaIP)

				log.Trace(trace.Inside, "Found match for host", host, "using address", addr)

				return nil
			}
		}
	}

	return agentErrorString("Unable to identify current host, host not added to romana cluster.")
}

// updateRoutes discovers network configuration
// of the host we are running on.
// We need to know public IP and Romana gateway IP of the current host.
// This is done by matching current host IP addresses against what topology
// service thinks the host address is.
// If no match is found we assume we are running on host which is not
// part of the Romana setup and spit error out.
func (a Agent) updateRoutes() error {
	log.Trace(trace.Private, "In Agent updateRoutes()")

	topologyURL, err := a.client.GetServiceUrl("topology")
	if err != nil {
		return agentError(err)
	}
	index := common.IndexResponse{}
	err = a.client.Get(topologyURL, &index)
	if err != nil {
		return agentError(err)
	}
	dcURL := index.Links.FindByRel("datacenter")
	a.networkConfig.dc = common.Datacenter{}
	err = a.client.Get(dcURL, &a.networkConfig.dc)
	if err != nil {
		return agentError(err)
	}

	hostURL := index.Links.FindByRel("host-list")
	hosts := []common.Host{}
	err = a.client.Get(hostURL, &hosts)
	if err != nil {
		return agentError(err)
	}
	log.Trace(trace.Inside, "Retrieved hosts list, found", len(hosts), "hosts")

	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return err
	}

	log.Tracef(trace.Inside, "Searching %d interfaces for a matching host configuration: %v", len(addrs), addrs)

	var otherHosts []common.Host

	// Find an interface that matches a Romana CIDR
	// and store that interface's IP address.
	// It will be used when configuring iptables and routes to tap interfaces.
	for i, host := range hosts {
		_, romanaCIDR, err := net.ParseCIDR(host.RomanaIp)
		if err != nil {
			log.Tracef(trace.Inside, "Unable to parse '%s' (%s). Skipping.", host.RomanaIp, err)
			continue
		}
		for _, addr := range addrs {
			ipnet, ok := addr.(*net.IPNet)
			if !ok {
				continue
			}
			if romanaCIDR.Contains(ipnet.IP) {
				// Check that it's the same subnet size
				s1, _ := romanaCIDR.Mask.Size()
				s2, _ := ipnet.Mask.Size()
				if s1 != s2 {
					continue
				}

				// Check if romanaGW address was changed recently, if it was
				// then remove the old one and set the new one we received.
				//
				// romana-gw can change due to multiple reasons:
				// * when node is cordoned/un-cordoned thus no pods except
				//   daemon set are there.
				// * when node is rebooted and it takes it long enough that the
				//   current cidr was re-allocated to other node.
				//
				romanaGWMaskSize, _ := a.networkConfig.romanaGWMask.Size()
				ipnetMaskSize, _ := ipnet.Mask.Size()
				if !a.networkConfig.romanaGW.Equal(ipnet.IP) ||
					romanaGWMaskSize != ipnetMaskSize {

					// For now we don't need updating romanaGW address
					// and mask, so disable it, but do log a message
					// about it.
					if true {
						log.Error("Agent: romana gateway IP Address/Mask changed on the node.")
					} else {
						a.networkConfig.updateRomanaGWMask(ipnet)

						// Update Romana Gateway with new config.
						if err := a.createRomanaGW(); err != nil {
							log.Error("Agent: Failed to update romana gateway IP Address on the node:", err)
							return err
						}
					}
				}

				log.Trace(trace.Private, "Acquiring mutex identifyCurrentHost")
				a.Helper.ensureInterHostRoutesMutex.Lock()

				// Retain the other hosts that were listed.
				// This will be used for creating inter-host routes.
				otherHosts = append(a.networkConfig.otherHosts, hosts[0:i]...)
				otherHosts = append(a.networkConfig.otherHosts, hosts[i+1:]...)
				a.networkConfig.otherHosts = otherHosts
				log.Trace(trace.Inside, "Found match for CIDR", romanaCIDR, "using address", ipnet.IP)

				log.Trace(trace.Private, "Releasing mutex identifyCurrentHost")
				a.Helper.ensureInterHostRoutesMutex.Unlock()

				return nil
			}
		}
	}
	return agentErrorString("Unable to find interface matching any Romana CIDR")
}

func GetFirstIPinCIDR(ipnet *net.IPNet) (*net.IPNet, error) {
	if ipnet == nil {
		return nil, fmt.Errorf("Agent Error, no input provided to GetFirstIPinCIDR")
	}
	if ipnet.IP.String() == "<nil>" || ipnet.Mask.String() == "<nil>" {
		return nil, fmt.Errorf("Agent Error, invalid IP/Mask provided to GetFirstIPinCIDR")
	}

	ip := ipnet.IP.Mask(ipnet.Mask)
	for i := len(ip) - 1; i >= 0; i-- {
		ip[i]++
		if ip[i] > 0 {
			break
		}
	}

	if !ipnet.Contains(ip) {
		return nil, fmt.Errorf("Error, no more IP address left in the network provided: %s", ipnet.String())
	}

	return &net.IPNet{IP: ip, Mask: ipnet.Mask}, nil
}
<|endoftext|>


<commit_before>package agent

import (
	"github.com/gravitational/trace"
	"google.golang.org/grpc"
	grpcerrors "google.golang.org/grpc/codes"
)

// ConvertGRPCError maps grpc error to one of trace type classes.
// Returns original error if no mapping is possible
func ConvertGRPCError(err error) error {
	if err == nil {
		return nil
	}

	switch grpc.Code(trace.Unwrap(err)) {
	case grpcerrors.InvalidArgument, grpcerrors.OutOfRange:
		return trace.BadParameter(err.Error())
	case grpcerrors.DeadlineExceeded:
		return trace.LimitExceeded(err.Error())
	case grpcerrors.AlreadyExists:
		return trace.AlreadyExists(err.Error())
	case grpcerrors.NotFound:
		return trace.NotFound(err.Error())
	case grpcerrors.PermissionDenied, grpcerrors.Unauthenticated:
		return trace.AccessDenied(err.Error())
	}
	return trace.Wrap(err)
}
<commit_msg>+ IsUnavailableError<commit_after>package agent

import (
	"github.com/gravitational/trace"
	"google.golang.org/grpc"
	grpcerrors "google.golang.org/grpc/codes"
)

// ConvertGRPCError maps grpc error to one of trace type classes.
// Returns original error if no mapping is possible
func ConvertGRPCError(err error) error {
	if err == nil {
		return nil
	}

	switch grpc.Code(trace.Unwrap(err)) {
	case grpcerrors.InvalidArgument, grpcerrors.OutOfRange:
		return trace.BadParameter(err.Error())
	case grpcerrors.DeadlineExceeded:
		return trace.LimitExceeded(err.Error())
	case grpcerrors.AlreadyExists:
		return trace.AlreadyExists(err.Error())
	case grpcerrors.NotFound:
		return trace.NotFound(err.Error())
	case grpcerrors.PermissionDenied, grpcerrors.Unauthenticated:
		return trace.AccessDenied(err.Error())
	}
	return trace.Wrap(err)
}

// IsUnavailableError determines if the specified error
// is a temporary agent availability error
func IsUnavailableError(err error) bool {
	err = ConvertGRPCError(err)
	switch {
	case grpc.Code(trace.Unwrap(err)) == grpcerrors.Unavailable:
		return true
	case trace.IsLimitExceeded(err):
		return true
	}
	return false
}
<|endoftext|>