<commit_before>package sparta

import (
	"encoding/json"
	"fmt"
	"net/http"

	"strings"

	"github.com/Sirupsen/logrus"
)

type dispatchMap map[string]*LambdaAWSInfo

// LambdaHTTPHandler is an HTTP compliant handler that implements
// ServeHTTP
type LambdaHTTPHandler struct {
	lambdaDispatchMap dispatchMap
	logger            *logrus.Logger
}

func (handler *LambdaHTTPHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Remove the leading slash and dispatch it to the golang handler
	lambdaFunc := strings.TrimLeft(req.URL.Path, "/")
	decoder := json.NewDecoder(req.Body)
	var request lambdaRequest
	defer func() {
		if r := recover(); r != nil {
			err, ok := r.(error)
			if !ok {
				err = fmt.Errorf("%v", r)
			}
			errorString := fmt.Sprintf("Lambda handler panic: %#v", err)
			http.Error(w, errorString, http.StatusBadRequest)
		}
	}()

	err := decoder.Decode(&request)
	if nil != err {
		errorString := fmt.Sprintf("Failed to decode proxy request: %s", err.Error())
		http.Error(w, errorString, http.StatusBadRequest)
		return
	}
	handler.logger.WithFields(logrus.Fields{
		"Request":    request,
		"LookupName": lambdaFunc,
	}).Debug("Dispatching")

	lambdaAWSInfo := handler.lambdaDispatchMap[lambdaFunc]
	var lambdaFn LambdaFunction
	if nil != lambdaAWSInfo {
		lambdaFn = lambdaAWSInfo.lambdaFn
	} else if strings.Contains(lambdaFunc, "::") {
		// Not the most exhaustive guard, but the CloudFormation custom resources
		// all have "::" delimiters in their type field.  Even if there is a false
		// positive, the customResourceForwarder will simply error out.
		lambdaFn = customResourceForwarder
	}

	if nil == lambdaFn {
		http.Error(w, "Unsupported path: "+lambdaFunc, http.StatusBadRequest)
		return
	}
	lambdaFn(&request.Event, &request.Context, w, handler.logger)
}

// NewLambdaHTTPHandler returns an initialized LambdaHTTPHandler instance.  The returned value
// can be provided to https://golang.org/pkg/net/http/httptest/#NewServer to perform
// localhost testing.
func NewLambdaHTTPHandler(lambdaAWSInfos []*LambdaAWSInfo, logger *logrus.Logger) *LambdaHTTPHandler {
	lookupMap := make(dispatchMap, 0)
	for _, eachLambdaInfo := range lambdaAWSInfos {
		logger.WithFields(logrus.Fields{
			"Path": eachLambdaInfo.lambdaFnName,
		}).Debug("Registering lambda URL")

		lookupMap[eachLambdaInfo.lambdaFnName] = eachLambdaInfo
	}

	return &LambdaHTTPHandler{
		lambdaDispatchMap: lookupMap,
		logger:            logger,
	}
}
<commit_msg>Correct import path<commit_after>package sparta

import (
	"encoding/json"
	"fmt"
	"github.com/mweagle/cloudformationresources"
	"net/http"

	"strings"

	"github.com/Sirupsen/logrus"
)

type dispatchMap map[string]*LambdaAWSInfo

// LambdaHTTPHandler is an HTTP compliant handler that implements
// ServeHTTP
type LambdaHTTPHandler struct {
	lambdaDispatchMap dispatchMap
	logger            *logrus.Logger
}

func (handler *LambdaHTTPHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Remove the leading slash and dispatch it to the golang handler
	lambdaFunc := strings.TrimLeft(req.URL.Path, "/")
	decoder := json.NewDecoder(req.Body)
	var request lambdaRequest
	defer func() {
		if r := recover(); r != nil {
			err, ok := r.(error)
			if !ok {
				err = fmt.Errorf("%v", r)
			}
			errorString := fmt.Sprintf("Lambda handler panic: %#v", err)
			http.Error(w, errorString, http.StatusBadRequest)
		}
	}()

	err := decoder.Decode(&request)
	if nil != err {
		errorString := fmt.Sprintf("Failed to decode proxy request: %s", err.Error())
		http.Error(w, errorString, http.StatusBadRequest)
		return
	}
	handler.logger.WithFields(logrus.Fields{
		"Request":    request,
		"LookupName": lambdaFunc,
	}).Debug("Dispatching")

	lambdaAWSInfo := handler.lambdaDispatchMap[lambdaFunc]
	var lambdaFn LambdaFunction
	if nil != lambdaAWSInfo {
		lambdaFn = lambdaAWSInfo.lambdaFn
	} else if strings.Contains(lambdaFunc, "::") {
		// Not the most exhaustive guard, but the CloudFormation custom resources
		// all have "::" delimiters in their type field.  Even if there is a false
		// positive, the customResourceForwarder will simply error out.
		lambdaFn = customResourceForwarder
	}

	if nil == lambdaFn {
		http.Error(w, "Unsupported path: "+lambdaFunc, http.StatusBadRequest)
		return
	}
	lambdaFn(&request.Event, &request.Context, w, handler.logger)
}

// NewLambdaHTTPHandler returns an initialized LambdaHTTPHandler instance.  The returned value
// can be provided to https://golang.org/pkg/net/http/httptest/#NewServer to perform
// localhost testing.
func NewLambdaHTTPHandler(lambdaAWSInfos []*LambdaAWSInfo, logger *logrus.Logger) *LambdaHTTPHandler {
	lookupMap := make(dispatchMap, 0)
	for _, eachLambdaInfo := range lambdaAWSInfos {
		logger.WithFields(logrus.Fields{
			"Path": eachLambdaInfo.lambdaFnName,
		}).Debug("Registering lambda URL")

		lookupMap[eachLambdaInfo.lambdaFnName] = eachLambdaInfo
	}

	return &LambdaHTTPHandler{
		lambdaDispatchMap: lookupMap,
		logger:            logger,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/danward79/sunrise"
)

func main() {
	//Melbourne -37.81, 144.96
	melbourne := sunrise.NewLocation(-37.81, 144.96)

	formatStr := "Jan 2 15:04:05"

	melbourne.Today()
	fmt.Println(melbourne.Sunrise().Format(formatStr))
	fmt.Println(melbourne.Sunset().Format(formatStr))
}
<commit_msg>Tidy up<commit_after>package main

import (
	"fmt"
	"github.com/danward79/sunrise"
)

func main() {
	//Melbourne -37.81, 144.96
	melbourne := sunrise.NewLocation(-37.81, 144.96)

	formatStr := "Jan 2 15:04:05"

	melbourne.Today()

	fmt.Println(melbourne.Sunset().Format(formatStr))

	melbourne.AddDays(1)

	fmt.Println(melbourne.Sunrise().Format(formatStr))
}
<|endoftext|>


<commit_before>package rest

import (
	"errors"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server/router"

	"github.com/rs/xid"

	"bytes"
	"io/ioutil"
	"net/http"
	"strings"
)

const xHeaderPrefix = "x-guble-"

var errNotFound = errors.New("Not Found.")

type RestMessageAPI struct {
	router router.Router
	prefix string
}

func NewRestMessageAPI(router router.Router, prefix string) *RestMessageAPI {
	return &RestMessageAPI{router, prefix}
}

func (api *RestMessageAPI) GetPrefix() string {
	return api.prefix
}

func (api *RestMessageAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, `Can not read body`, http.StatusBadRequest)
		return
	}

	topic, err := api.extractTopic(r.URL.Path)
	if err != nil {
		if err == errNotFound {
			http.NotFound(w, r)
			return
		}
		http.Error(w, "Server error.", http.StatusInternalServerError)
		return
	}

	msg := &protocol.Message{
		Path:          protocol.Path(topic),
		Body:          body,
		UserID:        q(r, `userId`),
		ApplicationID: xid.New().String(),
		OptionalID:    q(r, `messageId`),
		HeaderJSON:    headersToJSON(r.Header),
	}

	api.router.HandleMessage(msg)
}

func (api *RestMessageAPI) Check() error {
	return nil
}

func (api *RestMessageAPI) extractTopic(path string) (string, error) {
	p := removeTrailingSlash(api.prefix) + "/message"
	if !strings.HasPrefix(path, p) {
		return "", errNotFound
	}

	// Remove "`api.prefix` + /message" and we remain with the topic
	topic := strings.TrimPrefix(path, p)
	if topic == "/" || topic == "" {
		return "", errNotFound
	}

	return topic, nil
}

// returns a query parameter
func q(r *http.Request, name string) string {
	params := r.URL.Query()[name]
	if len(params) > 0 {
		return params[0]
	}
	return ""
}

func headersToJSON(header http.Header) string {
	buff := &bytes.Buffer{}
	buff.WriteString("{")

	count := 0
	for key, valueList := range header {
		if strings.HasPrefix(strings.ToLower(key), xHeaderPrefix) && len(valueList) > 0 {
			if count > 0 {
				buff.WriteString(",")
			}
			buff.WriteString(`"`)
			buff.WriteString(key[len(xHeaderPrefix):])
			buff.WriteString(`":`)
			buff.WriteString(`"`)
			buff.WriteString(valueList[0])
			buff.WriteString(`"`)
			count++
		}
	}
	buff.WriteString("}")
	return string(buff.Bytes())
}

func removeTrailingSlash(path string) string {
	if len(path) > 1 && path[len(path)-1] == '/' {
		return path[:len(path)-1]
	}
	return path
}
<commit_msg>removing useless Check()<commit_after>package rest

import (
	"errors"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server/router"

	"github.com/rs/xid"

	"bytes"
	"io/ioutil"
	"net/http"
	"strings"
)

const xHeaderPrefix = "x-guble-"

var errNotFound = errors.New("Not Found.")

type RestMessageAPI struct {
	router router.Router
	prefix string
}

func NewRestMessageAPI(router router.Router, prefix string) *RestMessageAPI {
	return &RestMessageAPI{router, prefix}
}

func (api *RestMessageAPI) GetPrefix() string {
	return api.prefix
}

func (api *RestMessageAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, `Can not read body`, http.StatusBadRequest)
		return
	}

	topic, err := api.extractTopic(r.URL.Path)
	if err != nil {
		if err == errNotFound {
			http.NotFound(w, r)
			return
		}
		http.Error(w, "Server error.", http.StatusInternalServerError)
		return
	}

	msg := &protocol.Message{
		Path:          protocol.Path(topic),
		Body:          body,
		UserID:        q(r, `userId`),
		ApplicationID: xid.New().String(),
		OptionalID:    q(r, `messageId`),
		HeaderJSON:    headersToJSON(r.Header),
	}

	api.router.HandleMessage(msg)
}

func (api *RestMessageAPI) extractTopic(path string) (string, error) {
	p := removeTrailingSlash(api.prefix) + "/message"
	if !strings.HasPrefix(path, p) {
		return "", errNotFound
	}

	// Remove "`api.prefix` + /message" and we remain with the topic
	topic := strings.TrimPrefix(path, p)
	if topic == "/" || topic == "" {
		return "", errNotFound
	}

	return topic, nil
}

// returns a query parameter
func q(r *http.Request, name string) string {
	params := r.URL.Query()[name]
	if len(params) > 0 {
		return params[0]
	}
	return ""
}

func headersToJSON(header http.Header) string {
	buff := &bytes.Buffer{}
	buff.WriteString("{")

	count := 0
	for key, valueList := range header {
		if strings.HasPrefix(strings.ToLower(key), xHeaderPrefix) && len(valueList) > 0 {
			if count > 0 {
				buff.WriteString(",")
			}
			buff.WriteString(`"`)
			buff.WriteString(key[len(xHeaderPrefix):])
			buff.WriteString(`":`)
			buff.WriteString(`"`)
			buff.WriteString(valueList[0])
			buff.WriteString(`"`)
			count++
		}
	}
	buff.WriteString("}")
	return string(buff.Bytes())
}

func removeTrailingSlash(path string) string {
	if len(path) > 1 && path[len(path)-1] == '/' {
		return path[:len(path)-1]
	}
	return path
}
<|endoftext|>


<commit_before>package service

import (
	"archive/zip"
	"bytes"
	"github.com/google/go-querystring/query"
	"github.com/mdreizin/smartling/model"
	"github.com/mdreizin/smartling/rest"
	"gopkg.in/resty.v0"
	"io/ioutil"
	"net/url"
	"path/filepath"
	"strings"
)

func extractFile(zf *zip.File, locales []string) (file *model.File, err error) {
	file = &model.File{}

	if rc, err := zf.Open(); err == nil {
		if b, err := ioutil.ReadAll(rc); err == nil {
			for _, locale := range locales {
				name := zf.Name
				localeSuffix := "_" + locale

				if strings.Contains(name, localeSuffix) {
					file.Path = strings.Replace(name, localeSuffix, "", -1)
					file.LocaleID = locale
					break
				}
			}

			file.Content = b
		}

		rc.Close()
	}

	return file, err
}

type DefaultFileService struct{}

func (s *DefaultFileService) Pull(params *FilePullParams) ([]*model.File, error) {
	var (
		q      url.Values
		err    error
		resp   *resty.Response
		_url   string
		reader *zip.Reader
	)

	files := []*model.File{}

	if _url, err = rest.DynamicURL(rest.FilePullURL, &params); err == nil {
		if q, err = query.Values(params); err == nil {
			q.Add("fileNameMode", "UNCHANGED")
			q.Add("localeMode", "LOCALE_IN_NAME")

			client := resty.R().SetMultiValueQueryParams(q).SetAuthToken(params.AuthToken)

			if resp, err = client.Get(_url); err == nil {
				body := resp.Body()

				if reader, err = zip.NewReader(bytes.NewReader(body), int64(len(body))); err == nil {
					for _, zf := range reader.File {
						if file, err := extractFile(zf, params.LocaleIDs); err == nil {
							files = append(files, file)
						}
					}
				}
			}
		}
	}

	return files, err
}

func (s *DefaultFileService) Push(params *FilePushParams) (*model.FileStats, error) {
	var (
		q        url.Values
		err      error
		resp     *resty.Response
		_url     string
		filename string
		content  []byte
	)

	stats := &model.FileStats{}

	if _url, err = rest.DynamicURL(rest.FilePushURL, &params); err == nil {
		if q, err = query.Values(params); err == nil {
			if filename, err = filepath.Abs(params.FilePath); err == nil {
				if content, err = ioutil.ReadFile(filename); err == nil {
					client := resty.R().
						SetResult(rest.Model{}).
						SetError(rest.Model{}).
						SetAuthToken(params.AuthToken).
						SetFileReader("file", "", bytes.NewReader(content))

					for p, v := range q {
						for _, pv := range v {
							client.FormData.Add(p, pv)
						}
					}

					client.SetFormData(params.Directives)

					if resp, err = client.Post(_url); err == nil {
						err = rest.Result(resp, &stats)
					}
				}
			}
		}
	}

	return stats, err
}
<commit_msg>refactor(DefaultFileService): renames `client` to `req`<commit_after>package service

import (
	"archive/zip"
	"bytes"
	"github.com/google/go-querystring/query"
	"github.com/mdreizin/smartling/model"
	"github.com/mdreizin/smartling/rest"
	"gopkg.in/resty.v0"
	"io/ioutil"
	"net/url"
	"path/filepath"
	"strings"
)

func extractFile(zf *zip.File, locales []string) (file *model.File, err error) {
	file = &model.File{}

	if rc, err := zf.Open(); err == nil {
		if b, err := ioutil.ReadAll(rc); err == nil {
			for _, locale := range locales {
				name := zf.Name
				localeSuffix := "_" + locale

				if strings.Contains(name, localeSuffix) {
					file.Path = strings.Replace(name, localeSuffix, "", -1)
					file.LocaleID = locale
					break
				}
			}

			file.Content = b
		}

		rc.Close()
	}

	return file, err
}

type DefaultFileService struct{}

func (s *DefaultFileService) Pull(params *FilePullParams) ([]*model.File, error) {
	var (
		q      url.Values
		err    error
		resp   *resty.Response
		_url   string
		reader *zip.Reader
	)

	files := []*model.File{}

	if _url, err = rest.DynamicURL(rest.FilePullURL, &params); err == nil {
		if q, err = query.Values(params); err == nil {
			q.Add("fileNameMode", "UNCHANGED")
			q.Add("localeMode", "LOCALE_IN_NAME")

			req := resty.R().SetMultiValueQueryParams(q).SetAuthToken(params.AuthToken)

			if resp, err = req.Get(_url); err == nil {
				body := resp.Body()

				if reader, err = zip.NewReader(bytes.NewReader(body), int64(len(body))); err == nil {
					for _, zf := range reader.File {
						if file, err := extractFile(zf, params.LocaleIDs); err == nil {
							files = append(files, file)
						}
					}
				}
			}
		}
	}

	return files, err
}

func (s *DefaultFileService) Push(params *FilePushParams) (*model.FileStats, error) {
	var (
		q        url.Values
		err      error
		resp     *resty.Response
		_url     string
		filename string
		content  []byte
	)

	stats := &model.FileStats{}

	if _url, err = rest.DynamicURL(rest.FilePushURL, &params); err == nil {
		if q, err = query.Values(params); err == nil {
			if filename, err = filepath.Abs(params.FilePath); err == nil {
				if content, err = ioutil.ReadFile(filename); err == nil {
					req := resty.R().
						SetResult(rest.Model{}).
						SetError(rest.Model{}).
						SetAuthToken(params.AuthToken).
						SetFileReader("file", "", bytes.NewReader(content))

					for p, v := range q {
						for _, pv := range v {
							req.FormData.Add(p, pv)
						}
					}

					req.SetFormData(params.Directives)

					if resp, err = req.Post(_url); err == nil {
						err = rest.Result(resp, &stats)
					}
				}
			}
		}
	}

	return stats, err
}
<|endoftext|>


<commit_before>package middleware

import (
	"github.com/gin-gonic/gin"
	"strings"
	"time"

	"github.com/eirka/eirka-libs/auth"
	"github.com/eirka/eirka-libs/db"
)

// list of keys to skip
var analyticsKeyList = map[string]bool{
	"taginfo":     true,
	"tags":        true,
	"post":        true,
	"tagtypes":    true,
	"imageboards": true,
	"new":         true,
	"favorited":   true,
}

// requesttype holds the data we want to capture
type RequestType struct {
	Ib        string
	Ip        string
	User      uint
	Path      string
	ItemKey   string
	ItemValue string
	Status    int
	Latency   time.Duration
	Cached    bool
}

// Analytics will log requests in the database
func Analytics() gin.HandlerFunc {
	return func(c *gin.Context) {
		req := c.Request
		// get request path
		path := req.URL.Path
		// Start timer
		start := time.Now()

		// Process request
		c.Next()

		// Stop timer
		end := time.Now()
		// get request latency
		latency := end.Sub(start)

		// Check if there was an error from the controller
		_, controllerError := c.Get("controllerError")
		if controllerError {
			c.Abort()
			return
		}

		// get userdata from session middleware
		userdata := c.MustGet("userdata").(auth.User)

		// get cached state from cache middleware
		cached := c.MustGet("cached").(bool)

		// get the ib
		ib := c.Param("ib")

		// Trim leading / from path and split
		params := strings.Split(strings.Trim(path, "/"), "/")

		// Make key from path
		key := itemKey{}
		key.generateKey(params...)

		if !skipKey(params[0]) {

			// set our data
			request := RequestType{
				Ib:        ib,
				Ip:        c.ClientIP(),
				User:      userdata.Id,
				Path:      path,
				Status:    c.Writer.Status(),
				Latency:   latency,
				ItemKey:   key.Key,
				ItemValue: key.Value,
				Cached:    cached,
			}

			// Get Database handle
			dbase, err := db.GetDb()
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

			// prepare query for analytics table
			ps1, err := dbase.Prepare("INSERT INTO analytics (ib_id, user_id, request_ip, request_path, request_status, request_latency, request_itemkey, request_itemvalue, request_cached, request_time) VALUES (?,?,?,?,?,?,?,?,?,NOW())")
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}
			defer ps1.Close()

			// input data
			_, err = ps1.Exec(request.Ib, request.User, request.Ip, request.Path, request.Status, request.Latency, request.ItemKey, request.ItemValue, request.Cached)
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

		}

	}
}

type itemKey struct {
	Key   string
	Value string
}

// Will take the params from the request and turn them into a key
func (r *itemKey) generateKey(params ...string) {

	switch {
	case len(params) <= 2:
		r.Key = params[0]
		r.Value = "1"
	case len(params) >= 3:
		r.Key = params[0]
		r.Value = params[2]
	}

	return

}

// Check if key should be skipped
func skipKey(key string) bool {

	if analyticsKeyList[strings.ToLower(key)] {
		return true
	}

	return false

}
<commit_msg>new user lib<commit_after>package middleware

import (
	"github.com/gin-gonic/gin"
	"strings"
	"time"

	"github.com/eirka/eirka-libs/db"
	"github.com/eirka/eirka-libs/user"
)

// list of keys to skip
var analyticsKeyList = map[string]bool{
	"taginfo":     true,
	"tags":        true,
	"post":        true,
	"tagtypes":    true,
	"imageboards": true,
	"new":         true,
	"favorited":   true,
}

// requesttype holds the data we want to capture
type RequestType struct {
	Ib        string
	Ip        string
	User      uint
	Path      string
	ItemKey   string
	ItemValue string
	Status    int
	Latency   time.Duration
	Cached    bool
}

// Analytics will log requests in the database
func Analytics() gin.HandlerFunc {
	return func(c *gin.Context) {
		req := c.Request
		// get request path
		path := req.URL.Path
		// Start timer
		start := time.Now()

		// Process request
		c.Next()

		// Stop timer
		end := time.Now()
		// get request latency
		latency := end.Sub(start)

		// Check if there was an error from the controller
		_, controllerError := c.Get("controllerError")
		if controllerError {
			c.Abort()
			return
		}

		// get userdata from session middleware
		userdata := c.MustGet("userdata").(user.User)

		// get cached state from cache middleware
		cached := c.MustGet("cached").(bool)

		// get the ib
		ib := c.Param("ib")

		// Trim leading / from path and split
		params := strings.Split(strings.Trim(path, "/"), "/")

		// Make key from path
		key := itemKey{}
		key.generateKey(params...)

		if !skipKey(params[0]) {

			// set our data
			request := RequestType{
				Ib:        ib,
				Ip:        c.ClientIP(),
				User:      userdata.Id,
				Path:      path,
				Status:    c.Writer.Status(),
				Latency:   latency,
				ItemKey:   key.Key,
				ItemValue: key.Value,
				Cached:    cached,
			}

			// Get Database handle
			dbase, err := db.GetDb()
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

			// prepare query for analytics table
			ps1, err := dbase.Prepare("INSERT INTO analytics (ib_id, user_id, request_ip, request_path, request_status, request_latency, request_itemkey, request_itemvalue, request_cached, request_time) VALUES (?,?,?,?,?,?,?,?,?,NOW())")
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}
			defer ps1.Close()

			// input data
			_, err = ps1.Exec(request.Ib, request.User, request.Ip, request.Path, request.Status, request.Latency, request.ItemKey, request.ItemValue, request.Cached)
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

		}

	}
}

type itemKey struct {
	Key   string
	Value string
}

// Will take the params from the request and turn them into a key
func (r *itemKey) generateKey(params ...string) {

	switch {
	case len(params) <= 2:
		r.Key = params[0]
		r.Value = "1"
	case len(params) >= 3:
		r.Key = params[0]
		r.Value = params[2]
	}

	return

}

// Check if key should be skipped
func skipKey(key string) bool {

	if analyticsKeyList[strings.ToLower(key)] {
		return true
	}

	return false

}
<|endoftext|>


<commit_before>package aws

import (
	"errors"
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
)

func dataSourceAwsEc2TransitGateway() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsEc2TransitGatewayRead,

		Schema: map[string]*schema.Schema{
			"amazon_side_asn": {
				Type:     schema.TypeInt,
				Computed: true,
			},
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"association_default_route_table_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"auto_accept_shared_attachments": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"default_route_table_association": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"default_route_table_propagation": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"description": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"dns_support": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"filter": dataSourceFiltersSchema(),
			"id": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"owner_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"propagation_default_route_table_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"tags": tagsSchemaComputed(),
			"vpn_ecmp_support": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func dataSourceAwsEc2TransitGatewayRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	input := &ec2.DescribeTransitGatewaysInput{}

	if v, ok := d.GetOk("filter"); ok {
		input.Filters = buildAwsDataSourceFilters(v.(*schema.Set))
	}

	if v, ok := d.GetOk("id"); ok {
		input.TransitGatewayIds = []*string{aws.String(v.(string))}
	}

	log.Printf("[DEBUG] Reading EC2 Transit Gateways: %s", input)
	output, err := conn.DescribeTransitGateways(input)

	if err != nil {
		return fmt.Errorf("error reading EC2 Transit Gateway: %s", err)
	}

	if output == nil || len(output.TransitGateways) == 0 {
		return errors.New("error reading EC2 Transit Gateway: no results found")
	}

	if len(output.TransitGateways) > 1 {
		return errors.New("error reading EC2 Transit Gateway: multiple results found, try adjusting search criteria")
	}

	transitGateway := output.TransitGateways[0]

	if transitGateway == nil {
		return errors.New("error reading EC2 Transit Gateway: empty result")
	}

	if transitGateway.Options == nil {
		return errors.New("error reading EC2 Transit Gateway: missing options")
	}

	d.Set("amazon_side_asn", aws.Int64Value(transitGateway.Options.AmazonSideAsn))
	d.Set("arn", transitGateway.TransitGatewayArn)
	d.Set("association_default_route_table_id", transitGateway.Options.AssociationDefaultRouteTableId)
	d.Set("auto_accept_shared_attachments", transitGateway.Options.AutoAcceptSharedAttachments)
	d.Set("default_route_table_association", transitGateway.Options.DefaultRouteTableAssociation)
	d.Set("default_route_table_propagation", transitGateway.Options.DefaultRouteTablePropagation)
	d.Set("description", transitGateway.Description)
	d.Set("dns_support", transitGateway.Options.DnsSupport)
	d.Set("owner_id", transitGateway.OwnerId)
	d.Set("propagation_default_route_table_id", transitGateway.Options.PropagationDefaultRouteTableId)

	if err := d.Set("tags", tagsToMap(transitGateway.Tags)); err != nil {
		return fmt.Errorf("error setting tags: %s", err)
	}

	d.Set("vpn_ecmp_support", transitGateway.Options.VpnEcmpSupport)

	d.SetId(aws.StringValue(transitGateway.TransitGatewayId))

	return nil
}
<commit_msg>d/aws_ec2_transit_gateway: Refactor tagging logic to keyvaluetags package.<commit_after>package aws

import (
	"errors"
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
)

func dataSourceAwsEc2TransitGateway() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsEc2TransitGatewayRead,

		Schema: map[string]*schema.Schema{
			"amazon_side_asn": {
				Type:     schema.TypeInt,
				Computed: true,
			},
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"association_default_route_table_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"auto_accept_shared_attachments": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"default_route_table_association": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"default_route_table_propagation": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"description": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"dns_support": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"filter": dataSourceFiltersSchema(),
			"id": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"owner_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"propagation_default_route_table_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"tags": tagsSchemaComputed(),
			"vpn_ecmp_support": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func dataSourceAwsEc2TransitGatewayRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	input := &ec2.DescribeTransitGatewaysInput{}

	if v, ok := d.GetOk("filter"); ok {
		input.Filters = buildAwsDataSourceFilters(v.(*schema.Set))
	}

	if v, ok := d.GetOk("id"); ok {
		input.TransitGatewayIds = []*string{aws.String(v.(string))}
	}

	log.Printf("[DEBUG] Reading EC2 Transit Gateways: %s", input)
	output, err := conn.DescribeTransitGateways(input)

	if err != nil {
		return fmt.Errorf("error reading EC2 Transit Gateway: %s", err)
	}

	if output == nil || len(output.TransitGateways) == 0 {
		return errors.New("error reading EC2 Transit Gateway: no results found")
	}

	if len(output.TransitGateways) > 1 {
		return errors.New("error reading EC2 Transit Gateway: multiple results found, try adjusting search criteria")
	}

	transitGateway := output.TransitGateways[0]

	if transitGateway == nil {
		return errors.New("error reading EC2 Transit Gateway: empty result")
	}

	if transitGateway.Options == nil {
		return errors.New("error reading EC2 Transit Gateway: missing options")
	}

	d.Set("amazon_side_asn", aws.Int64Value(transitGateway.Options.AmazonSideAsn))
	d.Set("arn", transitGateway.TransitGatewayArn)
	d.Set("association_default_route_table_id", transitGateway.Options.AssociationDefaultRouteTableId)
	d.Set("auto_accept_shared_attachments", transitGateway.Options.AutoAcceptSharedAttachments)
	d.Set("default_route_table_association", transitGateway.Options.DefaultRouteTableAssociation)
	d.Set("default_route_table_propagation", transitGateway.Options.DefaultRouteTablePropagation)
	d.Set("description", transitGateway.Description)
	d.Set("dns_support", transitGateway.Options.DnsSupport)
	d.Set("owner_id", transitGateway.OwnerId)
	d.Set("propagation_default_route_table_id", transitGateway.Options.PropagationDefaultRouteTableId)

	if err := d.Set("tags", keyvaluetags.Ec2KeyValueTags(transitGateway.Tags).IgnoreAws().Map()); err != nil {
		return fmt.Errorf("error setting tags: %s", err)
	}

	d.Set("vpn_ecmp_support", transitGateway.Options.VpnEcmpSupport)

	d.SetId(aws.StringValue(transitGateway.TransitGatewayId))

	return nil
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package tests_test

import (
	"strings"
	"time"

	expect "github.com/google/goexpect"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/client-go/kubecli"
	"kubevirt.io/kubevirt/tests"
)

var _ = Describe("[rfe_id:588][crit:medium][vendor:cnv-qe@redhat.com][level:component]ContainerDisk", func() {

	tests.FlagParse()

	virtClient, err := kubecli.GetKubevirtClient()
	tests.PanicOnError(err)

	BeforeEach(func() {
		tests.BeforeTestCleanup()
	})

	LaunchVMI := func(vmi *v1.VirtualMachineInstance) runtime.Object {
		By("Starting a VirtualMachineInstance")
		obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
		Expect(err).To(BeNil())
		return obj
	}

	VerifyContainerDiskVMI := func(vmi *v1.VirtualMachineInstance, obj runtime.Object, ignoreWarnings bool) {
		_, ok := obj.(*v1.VirtualMachineInstance)
		Expect(ok).To(BeTrue(), "Object is not of type *v1.VirtualMachineInstance")
		if ignoreWarnings == true {
			tests.WaitForSuccessfulVMIStartIgnoreWarnings(obj)
		} else {
			tests.WaitForSuccessfulVMIStart(obj)
		}

		// Verify Registry Disks are Online
		pods, err := virtClient.CoreV1().Pods(tests.NamespaceTestDefault).List(tests.UnfinishedVMIPodSelector(vmi))
		Expect(err).To(BeNil())

		By("Checking the number of VirtualMachineInstance disks")
		disksFound := 0
		for _, pod := range pods.Items {
			if pod.ObjectMeta.DeletionTimestamp != nil {
				continue
			}
			for _, containerStatus := range pod.Status.ContainerStatuses {
				if strings.HasPrefix(containerStatus.Name, "volume") == false {
					// only check readiness of disk containers
					continue
				}
				disksFound++
			}
			break
		}
		Expect(disksFound).To(Equal(1))
	}

	table.DescribeTable("should", func(image string, policy k8sv1.PullPolicy, expectedPolicy k8sv1.PullPolicy) {
		vmi := tests.NewRandomVMIWithEphemeralDisk(image)
		vmi.Spec.Volumes[0].ContainerDisk.ImagePullPolicy = policy
		vmi = tests.RunVMIAndExpectScheduling(vmi, 60)
		Expect(vmi.Spec.Volumes[0].ContainerDisk.ImagePullPolicy).To(Equal(expectedPolicy))
		pod := tests.GetPodByVirtualMachineInstance(vmi, vmi.Namespace)
		container := tests.GetContainerDiskContainerOfPod(pod, vmi.Spec.Volumes[0].Name)
		Expect(container.ImagePullPolicy).To(Equal(expectedPolicy))
	},
		table.Entry("[test_id:3246]generate and set Always pull policy", "test", k8sv1.PullPolicy(""), k8sv1.PullAlways),
		table.Entry("[test_id:3247]generate and set Always pull policy", "test:latest", k8sv1.PullPolicy(""), k8sv1.PullAlways),
		table.Entry("[test_id:3248]generate and set IfNotPresent pull policy", "test@sha256:9c2b78e11c25b3fd0b24b0ed684a112052dff03eee4ca4bdcc4f3168f9a14396", k8sv1.PullPolicy(""), k8sv1.PullIfNotPresent),
		table.Entry("[test_id:3249]pass through Never pull policy to the pod", "test@sha256:9c2b78e11c25b3fd0b24b0ed684a112052dff03eee4ca4bdcc4f3168f9a14396", k8sv1.PullNever, k8sv1.PullNever),
		table.Entry("[test_id:3250]pass through IfNotPresent pull policy to the pod", "test:latest", k8sv1.PullIfNotPresent, k8sv1.PullIfNotPresent),
	)

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting and stopping the same VirtualMachineInstance", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:1463]should success multiple times", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
				num := 2
				for i := 0; i < num; i++ {
					By("Starting the VirtualMachineInstance")
					obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
					Expect(err).To(BeNil())
					tests.WaitForSuccessfulVMIStart(obj)

					By("Stopping the VirtualMachineInstance")
					_, err = virtClient.RestClient().Delete().Resource("virtualmachineinstances").Namespace(vmi.GetObjectMeta().GetNamespace()).Name(vmi.GetObjectMeta().GetName()).Do().Get()
					Expect(err).To(BeNil())
					By("Waiting until the VirtualMachineInstance is gone")
					tests.WaitForVirtualMachineToDisappearWithTimeout(vmi, 120)
				}
			})
		})
	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting a VirtualMachineInstance", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:1464]should not modify the spec on status update", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
				v1.SetObjectDefaults_VirtualMachineInstance(vmi)

				By("Starting the VirtualMachineInstance")
				vmi, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
				Expect(err).To(BeNil())
				tests.WaitForSuccessfulVMIStart(vmi)
				startedVMI, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.ObjectMeta.Name, &metav1.GetOptions{})
				Expect(err).To(BeNil())
				By("Checking that the VirtualMachineInstance spec did not change")
				Expect(startedVMI.Spec).To(Equal(vmi.Spec))
			})
		})
	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting multiple VMIs", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:1465]should success", func() {
				num := 5
				vmis := make([]*v1.VirtualMachineInstance, 0, num)
				objs := make([]runtime.Object, 0, num)
				for i := 0; i < num; i++ {
					vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
					// FIXME if we give too much ram, the vmis really boot and eat all our memory (cache?)
					vmi.Spec.Domain.Resources.Requests[k8sv1.ResourceMemory] = resource.MustParse("1M")
					obj := LaunchVMI(vmi)
					vmis = append(vmis, vmi)
					objs = append(objs, obj)
				}

				for idx, vmi := range vmis {
					// TODO once networking is implemented properly set ignoreWarnings == false here.
					// We have to ignore warnings because VMIs started in parallel
					// may cause libvirt to fail to create the macvtap device in
					// the host network.
					// The new network implementation we're working on should resolve this.
					// NOTE the VirtualMachineInstance still starts successfully regardless of this warning.
					// It just requires virt-handler to retry the Start command at the moment.
					VerifyContainerDiskVMI(vmi, objs[idx], true)
				}
			}) // Timeout is long because this test involves multiple parallel VirtualMachineInstance launches.
		})
	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting from custom image location", func() {
		Context("with disk at /custom-disk/downloaded", func() {
			It("[test_id:1466]should boot normally", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirrosCustomLocation), "#!/bin/bash\necho 'hello'\n")
				for ind, volume := range vmi.Spec.Volumes {
					if volume.ContainerDisk != nil {
						vmi.Spec.Volumes[ind].ContainerDisk.Path = "/custom-disk/downloaded"
					}
				}
				By("Starting the VirtualMachineInstance")
				obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
				Expect(err).To(BeNil())
				tests.WaitForSuccessfulVMIStart(obj)
			})
		})

	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting with virtio-win", func() {
		Context("with virtio-win as secondary disk", func() {
			It("[test_id:1467]should boot and have the virtio as sata CDROM", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(tests.ContainerDiskFor(tests.ContainerDiskAlpine))
				tests.AddEphemeralCdrom(vmi, "disk4", "sata", tests.ContainerDiskFor(tests.ContainerDiskVirtio))

				By("Starting the VirtualMachineInstance")
				obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
				Expect(err).To(BeNil(), "expected vmi to start with no problem")
				tests.WaitForSuccessfulVMIStart(obj)

				By("Checking whether the second disk really contains virtio drivers")
				expecter, err := tests.LoggedInAlpineExpecter(vmi)
				Expect(err).ToNot(HaveOccurred(), "expected alpine to login properly")
				defer expecter.Close()

				_, err = expecter.ExpectBatch([]expect.Batcher{
					// mount virtio cdrom and check files are there
					&expect.BSnd{S: "mount -t iso9600 /dev/cdrom\n"},
					&expect.BSnd{S: "echo $?\n"},
					&expect.BExp{R: "0"},
					&expect.BSnd{S: "cd /media/cdrom\n"},
					&expect.BSnd{S: "ls virtio-win_license.txt guest-agent\n"},
					&expect.BSnd{S: "echo $?\n"},
					&expect.BExp{R: "0"},
				}, 200*time.Second)
				Expect(err).ToNot(HaveOccurred(), "expected virtio files to be mounted properly")
			})
		})
	})
})
<commit_msg>Add test to check kubevirt disk perms<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package tests_test

import (
	"fmt"
	"strings"
	"time"

	expect "github.com/google/goexpect"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/client-go/kubecli"
	"kubevirt.io/kubevirt/tests"
)

var _ = Describe("[rfe_id:588][crit:medium][vendor:cnv-qe@redhat.com][level:component]ContainerDisk", func() {

	tests.FlagParse()

	virtClient, err := kubecli.GetKubevirtClient()
	tests.PanicOnError(err)

	BeforeEach(func() {
		tests.BeforeTestCleanup()
	})

	LaunchVMI := func(vmi *v1.VirtualMachineInstance) runtime.Object {
		By("Starting a VirtualMachineInstance")
		obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
		Expect(err).To(BeNil())
		return obj
	}

	VerifyContainerDiskVMI := func(vmi *v1.VirtualMachineInstance, obj runtime.Object, ignoreWarnings bool) {
		_, ok := obj.(*v1.VirtualMachineInstance)
		Expect(ok).To(BeTrue(), "Object is not of type *v1.VirtualMachineInstance")
		if ignoreWarnings == true {
			tests.WaitForSuccessfulVMIStartIgnoreWarnings(obj)
		} else {
			tests.WaitForSuccessfulVMIStart(obj)
		}

		// Verify Registry Disks are Online
		pods, err := virtClient.CoreV1().Pods(tests.NamespaceTestDefault).List(tests.UnfinishedVMIPodSelector(vmi))
		Expect(err).To(BeNil())

		By("Checking the number of VirtualMachineInstance disks")
		disksFound := 0
		for _, pod := range pods.Items {
			if pod.ObjectMeta.DeletionTimestamp != nil {
				continue
			}
			for _, containerStatus := range pod.Status.ContainerStatuses {
				if strings.HasPrefix(containerStatus.Name, "volume") == false {
					// only check readiness of disk containers
					continue
				}
				disksFound++
			}
			break
		}
		Expect(disksFound).To(Equal(1))
	}

	table.DescribeTable("should", func(image string, policy k8sv1.PullPolicy, expectedPolicy k8sv1.PullPolicy) {
		vmi := tests.NewRandomVMIWithEphemeralDisk(image)
		vmi.Spec.Volumes[0].ContainerDisk.ImagePullPolicy = policy
		vmi = tests.RunVMIAndExpectScheduling(vmi, 60)
		Expect(vmi.Spec.Volumes[0].ContainerDisk.ImagePullPolicy).To(Equal(expectedPolicy))
		pod := tests.GetPodByVirtualMachineInstance(vmi, vmi.Namespace)
		container := tests.GetContainerDiskContainerOfPod(pod, vmi.Spec.Volumes[0].Name)
		Expect(container.ImagePullPolicy).To(Equal(expectedPolicy))
	},
		table.Entry("[test_id:3246]generate and set Always pull policy", "test", k8sv1.PullPolicy(""), k8sv1.PullAlways),
		table.Entry("[test_id:3247]generate and set Always pull policy", "test:latest", k8sv1.PullPolicy(""), k8sv1.PullAlways),
		table.Entry("[test_id:3248]generate and set IfNotPresent pull policy", "test@sha256:9c2b78e11c25b3fd0b24b0ed684a112052dff03eee4ca4bdcc4f3168f9a14396", k8sv1.PullPolicy(""), k8sv1.PullIfNotPresent),
		table.Entry("[test_id:3249]pass through Never pull policy to the pod", "test@sha256:9c2b78e11c25b3fd0b24b0ed684a112052dff03eee4ca4bdcc4f3168f9a14396", k8sv1.PullNever, k8sv1.PullNever),
		table.Entry("[test_id:3250]pass through IfNotPresent pull policy to the pod", "test:latest", k8sv1.PullIfNotPresent, k8sv1.PullIfNotPresent),
	)

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting and stopping the same VirtualMachineInstance", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:1463]should success multiple times", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
				num := 2
				for i := 0; i < num; i++ {
					By("Starting the VirtualMachineInstance")
					obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
					Expect(err).To(BeNil())
					tests.WaitForSuccessfulVMIStart(obj)

					By("Stopping the VirtualMachineInstance")
					_, err = virtClient.RestClient().Delete().Resource("virtualmachineinstances").Namespace(vmi.GetObjectMeta().GetNamespace()).Name(vmi.GetObjectMeta().GetName()).Do().Get()
					Expect(err).To(BeNil())
					By("Waiting until the VirtualMachineInstance is gone")
					tests.WaitForVirtualMachineToDisappearWithTimeout(vmi, 120)
				}
			})
		})
	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting a VirtualMachineInstance", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:1464]should not modify the spec on status update", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
				v1.SetObjectDefaults_VirtualMachineInstance(vmi)

				By("Starting the VirtualMachineInstance")
				vmi, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
				Expect(err).To(BeNil())
				tests.WaitForSuccessfulVMIStart(vmi)
				startedVMI, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.ObjectMeta.Name, &metav1.GetOptions{})
				Expect(err).To(BeNil())
				By("Checking that the VirtualMachineInstance spec did not change")
				Expect(startedVMI.Spec).To(Equal(vmi.Spec))
			})
		})
	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting multiple VMIs", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:1465]should success", func() {
				num := 5
				vmis := make([]*v1.VirtualMachineInstance, 0, num)
				objs := make([]runtime.Object, 0, num)
				for i := 0; i < num; i++ {
					vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
					// FIXME if we give too much ram, the vmis really boot and eat all our memory (cache?)
					vmi.Spec.Domain.Resources.Requests[k8sv1.ResourceMemory] = resource.MustParse("1M")
					obj := LaunchVMI(vmi)
					vmis = append(vmis, vmi)
					objs = append(objs, obj)
				}

				for idx, vmi := range vmis {
					// TODO once networking is implemented properly set ignoreWarnings == false here.
					// We have to ignore warnings because VMIs started in parallel
					// may cause libvirt to fail to create the macvtap device in
					// the host network.
					// The new network implementation we're working on should resolve this.
					// NOTE the VirtualMachineInstance still starts successfully regardless of this warning.
					// It just requires virt-handler to retry the Start command at the moment.
					VerifyContainerDiskVMI(vmi, objs[idx], true)
				}
			}) // Timeout is long because this test involves multiple parallel VirtualMachineInstance launches.
		})
	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting from custom image location", func() {
		Context("with disk at /custom-disk/downloaded", func() {
			It("[test_id:1466]should boot normally", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirrosCustomLocation), "#!/bin/bash\necho 'hello'\n")
				for ind, volume := range vmi.Spec.Volumes {
					if volume.ContainerDisk != nil {
						vmi.Spec.Volumes[ind].ContainerDisk.Path = "/custom-disk/downloaded"
					}
				}
				By("Starting the VirtualMachineInstance")
				obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
				Expect(err).To(BeNil())
				tests.WaitForSuccessfulVMIStart(obj)
			})
		})

	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting with virtio-win", func() {
		Context("with virtio-win as secondary disk", func() {
			It("[test_id:1467]should boot and have the virtio as sata CDROM", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(tests.ContainerDiskFor(tests.ContainerDiskAlpine))
				tests.AddEphemeralCdrom(vmi, "disk4", "sata", tests.ContainerDiskFor(tests.ContainerDiskVirtio))

				By("Starting the VirtualMachineInstance")
				obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
				Expect(err).To(BeNil(), "expected vmi to start with no problem")
				tests.WaitForSuccessfulVMIStart(obj)

				By("Checking whether the second disk really contains virtio drivers")
				expecter, err := tests.LoggedInAlpineExpecter(vmi)
				Expect(err).ToNot(HaveOccurred(), "expected alpine to login properly")
				defer expecter.Close()

				_, err = expecter.ExpectBatch([]expect.Batcher{
					// mount virtio cdrom and check files are there
					&expect.BSnd{S: "mount -t iso9600 /dev/cdrom\n"},
					&expect.BSnd{S: "echo $?\n"},
					&expect.BExp{R: "0"},
					&expect.BSnd{S: "cd /media/cdrom\n"},
					&expect.BSnd{S: "ls virtio-win_license.txt guest-agent\n"},
					&expect.BSnd{S: "echo $?\n"},
					&expect.BExp{R: "0"},
				}, 200*time.Second)
				Expect(err).ToNot(HaveOccurred(), "expected virtio files to be mounted properly")
			})
		})
	})

	Describe("[rfe_id:4052][crit:high][vendor:cnv-qe@redhat.com][level:component]VMI disk permissions", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:4299]should not have world write permissions", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(tests.ContainerDiskFor(tests.ContainerDiskAlpine))

				By("Starting a New VMI")
				vmi, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
				Expect(err).ToNot(HaveOccurred())
				tests.WaitForSuccessfulVMIStart(vmi)

				By("Ensuring VMI is running by logging in")
				tests.WaitUntilVMIReady(vmi, tests.LoggedInAlpineExpecter)

				By("Fetching virt-launcher Pod")
				pod := tests.GetPodByVirtualMachineInstance(vmi, tests.NamespaceTestDefault)

				writableImagePath := fmt.Sprintf("/var/run/kubevirt-ephemeral-disks/disk-data/%v/disk.qcow2", vmi.Spec.Domain.Devices.Disks[0].Name)

				writableImageOctalMode, err := tests.ExecuteCommandOnPod(
					virtClient,
					pod,
					"compute",
					[]string{"/usr/bin/bash", "-c", fmt.Sprintf("stat -c %%a %s", writableImagePath)},
				)
				Expect(err).ToNot(HaveOccurred())

				By("Checking the writable Image Octal mode")
				Expect(strings.Trim(writableImageOctalMode, "\n")).To(Equal("644"), "Octal Mode of writable Image should be 644")

				readonlyImageOctalMode, err := tests.ExecuteCommandOnPod(
					virtClient,
					pod,
					"compute",
					[]string{"/usr/bin/bash", "-c", "stat -c %a /var/run/kubevirt/container-disks/disk_0.img"},
				)
				Expect(err).ToNot(HaveOccurred())

				By("Checking the read-only Image Octal mode")
				Expect(strings.Trim(readonlyImageOctalMode, "\n")).To(Equal("555"), "Octal Mode of read-only Image should be 555")
			})
		})
	})
})
<|endoftext|>


<commit_before>package h2spec

import (
	"github.com/bradfitz/http2"
	"github.com/bradfitz/http2/hpack"
)

func FrameSizeTestGroup() *TestGroup {
	tg := NewTestGroup("4.2", "Frame Size")

	tg.AddTestCase(NewTestCase(
		"Sends large size frame that exceeds the SETTINGS_MAX_FRAME_SIZE",
		"The endpoint MUST send a FRAME_SIZE_ERROR error.",
		func(ctx *Context) (expected []Result, actual Result) {
			http2Conn := CreateHttp2Conn(ctx, false)
			defer http2Conn.conn.Close()

			http2Conn.fr.WriteSettings()

			hdrs := []hpack.HeaderField{
				pair(":method", "GET"),
				pair(":scheme", "http"),
				pair(":path", "/"),
				pair(":authority", ctx.Authority()),
			}

			var hp http2.HeadersFrameParam
			hp.StreamID = 1
			hp.EndStream = false
			hp.EndHeaders = true
			hp.BlockFragment = http2Conn.EncodeHeader(hdrs)
			http2Conn.fr.WriteHeaders(hp)
			http2Conn.fr.WriteData(1, true, []byte(dummyData(16385)))

			actualCodes := []http2.ErrCode{http2.ErrCodeFrameSize}
			return TestStreamError(ctx, http2Conn, actualCodes)
		},
	))

	return tg
}
<commit_msg>Test 4.2: use the max frame size from the server's SETTINGS frame<commit_after>package h2spec

import (
	"github.com/bradfitz/http2"
	"github.com/bradfitz/http2/hpack"
)

func FrameSizeTestGroup() *TestGroup {
	tg := NewTestGroup("4.2", "Frame Size")

	tg.AddTestCase(NewTestCase(
		"Sends large size frame that exceeds the SETTINGS_MAX_FRAME_SIZE",
		"The endpoint MUST send a FRAME_SIZE_ERROR error.",
		func(ctx *Context) (expected []Result, actual Result) {
			http2Conn := CreateHttp2Conn(ctx, true)
			defer http2Conn.conn.Close()

			hdrs := []hpack.HeaderField{
				pair(":method", "GET"),
				pair(":scheme", "http"),
				pair(":path", "/"),
				pair(":authority", ctx.Authority()),
			}

			var hp http2.HeadersFrameParam
			hp.StreamID = 1
			hp.EndStream = false
			hp.EndHeaders = true
			hp.BlockFragment = http2Conn.EncodeHeader(hdrs)
			http2Conn.fr.WriteHeaders(hp)
			max_size, ok := http2Conn.Settings[http2.SettingMaxFrameSize]
			if( !ok ) {
				max_size = 18384
			}

			http2Conn.fr.WriteData(1, true, []byte(dummyData(int(max_size) + 1)))

			actualCodes := []http2.ErrCode{http2.ErrCodeFrameSize}
			return TestStreamError(ctx, http2Conn, actualCodes)
		},
	))

	return tg
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"

	"github.com/ClusterHQ/dvol/pkg/datalayer"
)

/*
User
 | "dvol checkout -b foo"
 v
CLI
 | "what is the current active volume?", "oh, it's 'bar'"
 | "create branch 'foo' from active volume 'bar'"
 v
internal API
 | "create variant from snapshot at tip of volume bar"
 v
DataLayer (swappable for another implementation)

*/

/*

A dvol volume is:

* a forest of snapshots (aka commits, immutable snapshots of the volume at a certain point in time), with inherited branch labels
* a set of writeable working copies (writeable paths which get mounted into the container), one per branch

A data layer volume is what we call a writeable working copy.

current directory structure
---------------------------

What should go where?

STRUCTURE                                   WHAT
------------------------------------------------
current_volume.json                         dvol
volumes/
  foo/
    current_branch.json                     dvol
	running_point -> branches/bar           dvol (docker integration)
	commits/                                data layer commits
	  deadbeefdeadbeef/
	    <copy of data>
	branches/
	  bar/                                  data layer volume (one per branch), writeable working copy
	    <writeable data>
	  bar.json                              data layer commit metadata database (currently per branch, should be migrated into commits eventually, but not yet)

*/

const MAX_NAME_LENGTH int = 40

func ValidName(name string) bool {
	var validNameRegex = regexp.MustCompile(`^[a-zA-Z]+[a-zA-Z0-9-]*$`)
	return validNameRegex.MatchString(name) && len(name) <= MAX_NAME_LENGTH
}

type DvolAPI struct {
	basePath string
	dl       *datalayer.DataLayer
}

func NewDvolAPI(basePath string) *DvolAPI {
	dl := &datalayer.DataLayer{BasePath: basePath}
	return &DvolAPI{basePath, dl}
}

func (dvol *DvolAPI) CreateVolume(volumeName string) error {
	err := dvol.dl.CreateVolume(volumeName)
	if err != nil {
		return err
	}
	return dvol.setActiveVolume(volumeName)
}

func (dvol *DvolAPI) RemoveVolume(volumeName string) error {
	return dvol.dl.RemoveVolume(volumeName)
}

func (dvol *DvolAPI) setActiveVolume(volumeName string) error {
	currentVolumeJsonPath := filepath.FromSlash(dvol.basePath + "/current_volume.json")
	currentVolumeContent := map[string]string{
		"current_volume": volumeName,
	}
	// Create or update this file
	file, err := os.Create(currentVolumeJsonPath)
	if err != nil {
		return err
	}
	defer file.Close()
	encoder := json.NewEncoder(file)
	encoder.Encode(currentVolumeContent)
	return nil
}

func (dvol *DvolAPI) CreateVariant(volumeName, variantName string) error {
	return dvol.dl.CreateVariant(volumeName, variantName)
}

func (dvol *DvolAPI) ActiveVolume() (string, error) {
	currentVolumeJsonPath := filepath.FromSlash(dvol.basePath + "/current_volume.json")
	file, err := os.Open(currentVolumeJsonPath)
	if err != nil {
		return "", err
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	var store map[string]interface{}
	err = decoder.Decode(&store)
	if err != nil {
		return "", err
	}
	return store["current_volume"].(string), nil
}

func (dvol *DvolAPI) VolumeExists(volumeName string) bool {
	volumePath := filepath.FromSlash(dvol.basePath + "/" + volumeName)
	_, err := os.Stat(volumePath)
	return err == nil
}

func (dvol *DvolAPI) SwitchVolume(volumeName string) error {
	return dvol.setActiveVolume(volumeName)
}

func (dvol *DvolAPI) CurrentBranch(volumeName string) (string, error) {
	currentBranchJsonPath := filepath.FromSlash(dvol.basePath + "/" + volumeName + "/current_branch.json")
	file, err := os.Open(currentBranchJsonPath)
	if err != nil {
		// The error type should be checked here.
		// Only return master if no volume information is found.
		return "master", nil
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	var store map[string]interface{}
	err = decoder.Decode(&store)
	if err != nil {
		return "", err
	}
	return store["current_branch"].(string), nil
}

func (dvol *DvolAPI) AllVolumes() ([]string, error) {
	files, err := ioutil.ReadDir(dvol.basePath)
	if err != nil {
		return []string{}, err
	}
	volumes := make([]string, 0)
	for _, file := range files {
		if file.IsDir() {
			volumes = append(volumes, file.Name())
		}
	}
	return volumes, nil
}
<commit_msg>add comments<commit_after>package api

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"

	"github.com/ClusterHQ/dvol/pkg/datalayer"
)

/*
User
 | "dvol checkout -b foo"
 v
CLI
 | "what is the current active volume?", "oh, it's 'bar'"
 | "create branch 'foo' from active volume 'bar'"
 v
internal API
 | "create variant from snapshot at tip of volume bar"
 v
DataLayer (swappable for another implementation)

*/

/*

A dvol volume is:

* a forest of snapshots (aka commits, immutable snapshots of the volume at a certain point in time), with inherited branch labels
* a set of writeable working copies (writeable paths which get mounted into the container), one per branch

A data layer volume is what we call a writeable working copy.

current directory structure
---------------------------

What should go where?

STRUCTURE                                   WHAT
------------------------------------------------
current_volume.json                         dvol api
volumes/
  foo/
    current_branch.json                     dvol api
	running_point -> branches/bar           dvol docker integration
	commits/                                data layer commits
	  deadbeefdeadbeef/
	    <copy of data>
	branches/
	  bar/                                  data layer volume (one per branch), writeable working copy
	    <writeable data>
	  bar.json                              data layer commit metadata database (currently per branch, should be migrated into commits eventually, but not yet)

*/

const MAX_NAME_LENGTH int = 40

func ValidName(name string) bool {
	var validNameRegex = regexp.MustCompile(`^[a-zA-Z]+[a-zA-Z0-9-]*$`)
	return validNameRegex.MatchString(name) && len(name) <= MAX_NAME_LENGTH
}

type DvolAPI struct {
	basePath string
	dl       *datalayer.DataLayer
}

func NewDvolAPI(basePath string) *DvolAPI {
	dl := &datalayer.DataLayer{BasePath: basePath}
	return &DvolAPI{basePath, dl}
}

func (dvol *DvolAPI) CreateVolume(volumeName string) error {
	err := dvol.dl.CreateVolume(volumeName)
	if err != nil {
		return err
	}
	return dvol.setActiveVolume(volumeName)
}

func (dvol *DvolAPI) RemoveVolume(volumeName string) error {
	return dvol.dl.RemoveVolume(volumeName)
}

func (dvol *DvolAPI) setActiveVolume(volumeName string) error {
	currentVolumeJsonPath := filepath.FromSlash(dvol.basePath + "/current_volume.json")
	currentVolumeContent := map[string]string{
		"current_volume": volumeName,
	}
	// Create or update this file
	file, err := os.Create(currentVolumeJsonPath)
	if err != nil {
		return err
	}
	defer file.Close()
	encoder := json.NewEncoder(file)
	encoder.Encode(currentVolumeContent)
	return nil
}

func (dvol *DvolAPI) CreateVariant(volumeName, variantName string) error {
	return dvol.dl.CreateVariant(volumeName, variantName)
}

func (dvol *DvolAPI) ActiveVolume() (string, error) {
	currentVolumeJsonPath := filepath.FromSlash(dvol.basePath + "/current_volume.json")
	file, err := os.Open(currentVolumeJsonPath)
	if err != nil {
		return "", err
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	var store map[string]interface{}
	err = decoder.Decode(&store)
	if err != nil {
		return "", err
	}
	return store["current_volume"].(string), nil
}

func (dvol *DvolAPI) VolumeExists(volumeName string) bool {
	volumePath := filepath.FromSlash(dvol.basePath + "/" + volumeName)
	_, err := os.Stat(volumePath)
	return err == nil
}

func (dvol *DvolAPI) SwitchVolume(volumeName string) error {
	return dvol.setActiveVolume(volumeName)
}

func (dvol *DvolAPI) CurrentBranch(volumeName string) (string, error) {
	currentBranchJsonPath := filepath.FromSlash(dvol.basePath + "/" + volumeName + "/current_branch.json")
	file, err := os.Open(currentBranchJsonPath)
	if err != nil {
		// The error type should be checked here.
		// Only return master if no volume information is found.
		return "master", nil
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	var store map[string]interface{}
	err = decoder.Decode(&store)
	if err != nil {
		return "", err
	}
	return store["current_branch"].(string), nil
}

func (dvol *DvolAPI) AllVolumes() ([]string, error) {
	files, err := ioutil.ReadDir(dvol.basePath)
	if err != nil {
		return []string{}, err
	}
	volumes := make([]string, 0)
	for _, file := range files {
		if file.IsDir() {
			volumes = append(volumes, file.Name())
		}
	}
	return volumes, nil
}
<|endoftext|>


<commit_before>package linux_backend

import (
	"bytes"
	"fmt"
	"log"
	"os/exec"
	"path"
	"strconv"
	"sync"

	"github.com/vito/garden/backend"
	"github.com/vito/garden/backend/linux_backend/cgroups_manager"
	"github.com/vito/garden/backend/linux_backend/job_tracker"
	"github.com/vito/garden/command_runner"
)

type LinuxContainer struct {
	id   string
	path string

	spec backend.ContainerSpec

	runner         command_runner.CommandRunner
	cgroupsManager cgroups_manager.CgroupsManager

	jobTracker *job_tracker.JobTracker

	oomLock     sync.RWMutex
	oomNotifier *exec.Cmd
}

func NewLinuxContainer(id, path string, spec backend.ContainerSpec, runner command_runner.CommandRunner, cgroupsManager cgroups_manager.CgroupsManager) *LinuxContainer {
	return &LinuxContainer{
		id:   id,
		path: path,

		spec: spec,

		runner:         runner,
		cgroupsManager: cgroupsManager,

		jobTracker: job_tracker.New(path, runner),
	}
}

func (c *LinuxContainer) ID() string {
	return c.id
}

func (c *LinuxContainer) Handle() string {
	if c.spec.Handle != "" {
		return c.spec.Handle
	}

	return c.ID()
}

func (c *LinuxContainer) Start() error {
	start := exec.Command(path.Join(c.path, "start.sh"))

	start.Env = []string{
		"id=" + c.id,
		"container_iface_mtu=1500",
		"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
	}

	return c.runner.Run(start)
}

func (c *LinuxContainer) Stop(kill bool) error {
	stop := exec.Command(path.Join(c.path, "stop.sh"))

	if kill {
		stop.Args = append(stop.Args, "-w", "0")
	}

	err := c.runner.Run(stop)
	if err != nil {
		return err
	}

	c.stopOomNotifier()

	return nil
}

func (c *LinuxContainer) Info() (backend.ContainerInfo, error) {
	return backend.ContainerInfo{}, nil
}

func (c *LinuxContainer) CopyIn(src, dst string) error {
	return c.rsync(src, "vcap@container:"+dst)
}

func (c *LinuxContainer) CopyOut(src, dst, owner string) error {
	err := c.rsync("vcap@container:"+src, dst)
	if err != nil {
		return err
	}

	if owner != "" {
		chown := exec.Command("chown", "-R", owner, dst)

		err := c.runner.Run(chown)
		if err != nil {
			return err
		}
	}

	return nil
}

func (c *LinuxContainer) LimitBandwidth(limits backend.BandwidthLimits) (backend.BandwidthLimits, error) {
	limit := exec.Command(path.Join(c.path, "net_rate.sh"))

	limit.Env = []string{
		fmt.Sprintf("BURST=%d", limits.BurstRateInBytesPerSecond),
		fmt.Sprintf("RATE=%d", limits.RateInBytesPerSecond*8),
	}

	err := c.runner.Run(limit)
	if err != nil {
		return backend.BandwidthLimits{}, err
	}

	return limits, nil
}

func (c *LinuxContainer) LimitDisk(backend.DiskLimits) (backend.DiskLimits, error) {
	return backend.DiskLimits{}, nil
}

func (c *LinuxContainer) LimitMemory(limits backend.MemoryLimits) (backend.MemoryLimits, error) {
	err := c.startOomNotifier()
	if err != nil {
		return backend.MemoryLimits{}, err
	}

	limit := fmt.Sprintf("%d", limits.LimitInBytes)

	// memory.memsw.limit_in_bytes must be >= memory.limit_in_bytes
	//
	// however, it must be set after memory.limit_in_bytes, and if we're
	// increasing the limit, writing memory.limit_in_bytes first will fail.
	//
	// so, write memory.limit_in_bytes before and after
	c.cgroupsManager.Set("memory", "memory.limit_in_bytes", limit)

	err = c.cgroupsManager.Set("memory", "memory.memsw.limit_in_bytes", limit)
	if err != nil {
		return backend.MemoryLimits{}, err
	}

	err = c.cgroupsManager.Set("memory", "memory.limit_in_bytes", limit)
	if err != nil {
		return backend.MemoryLimits{}, err
	}

	actualLimitInBytes, err := c.cgroupsManager.Get("memory", "memory.limit_in_bytes")

	numericLimit, err := strconv.ParseUint(actualLimitInBytes, 10, 0)
	if err != nil {
		return backend.MemoryLimits{}, err
	}

	return backend.MemoryLimits{uint64(numericLimit)}, nil
}

func (c *LinuxContainer) Spawn(spec backend.JobSpec) (uint32, error) {
	wshPath := path.Join(c.path, "bin", "wsh")
	sockPath := path.Join(c.path, "run", "wshd.sock")

	user := "vcap"
	if spec.Privileged {
		user = "root"
	}

	wsh := exec.Command(wshPath, "--socket", sockPath, "--user", user, "/bin/bash")

	wsh.Stdin = bytes.NewBufferString(spec.Script)

	return c.jobTracker.Spawn(wsh)
}

func (c *LinuxContainer) Stream(uint32) (<-chan backend.JobStream, error) {
	return nil, nil
}

func (c *LinuxContainer) Link(jobID uint32) (backend.JobResult, error) {
	exitStatus, stdout, stderr, err := c.jobTracker.Link(jobID)
	if err != nil {
		return backend.JobResult{}, err
	}

	return backend.JobResult{
		ExitStatus: exitStatus,
		Stdout:     stdout,
		Stderr:     stderr,
	}, nil
}

func (c *LinuxContainer) Run(backend.JobSpec) (backend.JobResult, error) {
	return backend.JobResult{}, nil
}

func (c *LinuxContainer) NetIn(uint32, uint32) (uint32, uint32, error) {
	return 0, 0, nil
}

func (c *LinuxContainer) NetOut(string, uint32) error {
	return nil
}

func (c *LinuxContainer) rsync(src, dst string) error {
	wshPath := path.Join(c.path, "bin", "wsh")
	sockPath := path.Join(c.path, "run", "wshd.sock")

	rsync := exec.Command(
		"rsync",
		"-e", wshPath+" --socket "+sockPath+" --rsh",
		"-r",
		"-p",
		"--links",
		src,
		dst,
	)

	return c.runner.Run(rsync)
}

func (c *LinuxContainer) startOomNotifier() error {
	c.oomLock.Lock()
	defer c.oomLock.Unlock()

	if c.oomNotifier != nil {
		return nil
	}

	oomPath := path.Join(c.path, "bin", "oom")

	c.oomNotifier = exec.Command(oomPath, c.cgroupsManager.SubsystemPath("memory"))

	err := c.runner.Start(c.oomNotifier)
	if err != nil {
		return err
	}

	go c.watchForOom(c.oomNotifier)

	return nil
}

func (c *LinuxContainer) stopOomNotifier() {
	c.oomLock.RLock()
	defer c.oomLock.RUnlock()

	if c.oomNotifier != nil {
		c.runner.Kill(c.oomNotifier)
	}
}

func (c *LinuxContainer) watchForOom(oom *exec.Cmd) {
	err := c.runner.Wait(oom)
	if err == nil {
		c.Stop(false)
	}

	// TODO: handle case where oom notifier itself failed? kill container?
}
<commit_msg>add logging to linux container<commit_after>package linux_backend

import (
	"bytes"
	"fmt"
	"log"
	"os/exec"
	"path"
	"strconv"
	"sync"

	"github.com/vito/garden/backend"
	"github.com/vito/garden/backend/linux_backend/cgroups_manager"
	"github.com/vito/garden/backend/linux_backend/job_tracker"
	"github.com/vito/garden/command_runner"
)

type LinuxContainer struct {
	id   string
	path string

	spec backend.ContainerSpec

	runner         command_runner.CommandRunner
	cgroupsManager cgroups_manager.CgroupsManager

	jobTracker *job_tracker.JobTracker

	oomLock     sync.RWMutex
	oomNotifier *exec.Cmd
}

func NewLinuxContainer(id, path string, spec backend.ContainerSpec, runner command_runner.CommandRunner, cgroupsManager cgroups_manager.CgroupsManager) *LinuxContainer {
	return &LinuxContainer{
		id:   id,
		path: path,

		spec: spec,

		runner:         runner,
		cgroupsManager: cgroupsManager,

		jobTracker: job_tracker.New(path, runner),
	}
}

func (c *LinuxContainer) ID() string {
	return c.id
}

func (c *LinuxContainer) Handle() string {
	if c.spec.Handle != "" {
		return c.spec.Handle
	}

	return c.ID()
}

func (c *LinuxContainer) Start() error {
	log.Println(c.id, "starting")

	start := exec.Command(path.Join(c.path, "start.sh"))

	start.Env = []string{
		"id=" + c.id,
		"container_iface_mtu=1500",
		"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
	}

	return c.runner.Run(start)
}

func (c *LinuxContainer) Stop(kill bool) error {
	log.Println(c.id, "stopping")

	stop := exec.Command(path.Join(c.path, "stop.sh"))

	if kill {
		stop.Args = append(stop.Args, "-w", "0")
	}

	err := c.runner.Run(stop)
	if err != nil {
		return err
	}

	c.stopOomNotifier()

	return nil
}

func (c *LinuxContainer) Info() (backend.ContainerInfo, error) {
	return backend.ContainerInfo{}, nil
}

func (c *LinuxContainer) CopyIn(src, dst string) error {
	log.Println(c.id, "copying in from", src, "to", dst)
	return c.rsync(src, "vcap@container:"+dst)
}

func (c *LinuxContainer) CopyOut(src, dst, owner string) error {
	log.Println(c.id, "copying out from", src, "to", dst)

	err := c.rsync("vcap@container:"+src, dst)
	if err != nil {
		return err
	}

	if owner != "" {
		chown := exec.Command("chown", "-R", owner, dst)

		err := c.runner.Run(chown)
		if err != nil {
			return err
		}
	}

	return nil
}

func (c *LinuxContainer) LimitBandwidth(limits backend.BandwidthLimits) (backend.BandwidthLimits, error) {
	log.Println(
		c.id,
		"limiting bandwidth to",
		limits.RateInBytesPerSecond,
		"bytes per second; burst",
		limits.BurstRateInBytesPerSecond,
	)

	limit := exec.Command(path.Join(c.path, "net_rate.sh"))

	limit.Env = []string{
		fmt.Sprintf("BURST=%d", limits.BurstRateInBytesPerSecond),
		fmt.Sprintf("RATE=%d", limits.RateInBytesPerSecond*8),
	}

	err := c.runner.Run(limit)
	if err != nil {
		return backend.BandwidthLimits{}, err
	}

	return limits, nil
}

func (c *LinuxContainer) LimitDisk(backend.DiskLimits) (backend.DiskLimits, error) {
	return backend.DiskLimits{}, nil
}

func (c *LinuxContainer) LimitMemory(limits backend.MemoryLimits) (backend.MemoryLimits, error) {
	log.Println(c.id, "limiting memory to", limits.LimitInBytes, "bytes")

	err := c.startOomNotifier()
	if err != nil {
		return backend.MemoryLimits{}, err
	}

	limit := fmt.Sprintf("%d", limits.LimitInBytes)

	// memory.memsw.limit_in_bytes must be >= memory.limit_in_bytes
	//
	// however, it must be set after memory.limit_in_bytes, and if we're
	// increasing the limit, writing memory.limit_in_bytes first will fail.
	//
	// so, write memory.limit_in_bytes before and after
	c.cgroupsManager.Set("memory", "memory.limit_in_bytes", limit)

	err = c.cgroupsManager.Set("memory", "memory.memsw.limit_in_bytes", limit)
	if err != nil {
		return backend.MemoryLimits{}, err
	}

	err = c.cgroupsManager.Set("memory", "memory.limit_in_bytes", limit)
	if err != nil {
		return backend.MemoryLimits{}, err
	}

	actualLimitInBytes, err := c.cgroupsManager.Get("memory", "memory.limit_in_bytes")

	numericLimit, err := strconv.ParseUint(actualLimitInBytes, 10, 0)
	if err != nil {
		return backend.MemoryLimits{}, err
	}

	return backend.MemoryLimits{uint64(numericLimit)}, nil
}

func (c *LinuxContainer) Spawn(spec backend.JobSpec) (uint32, error) {
	log.Println(c.id, "spawning job:", spec.Script)

	wshPath := path.Join(c.path, "bin", "wsh")
	sockPath := path.Join(c.path, "run", "wshd.sock")

	user := "vcap"
	if spec.Privileged {
		user = "root"
	}

	wsh := exec.Command(wshPath, "--socket", sockPath, "--user", user, "/bin/bash")

	wsh.Stdin = bytes.NewBufferString(spec.Script)

	return c.jobTracker.Spawn(wsh)
}

func (c *LinuxContainer) Stream(uint32) (<-chan backend.JobStream, error) {
	return nil, nil
}

func (c *LinuxContainer) Link(jobID uint32) (backend.JobResult, error) {
	log.Println(c.id, "linking to job", jobID)

	exitStatus, stdout, stderr, err := c.jobTracker.Link(jobID)
	if err != nil {
		return backend.JobResult{}, err
	}

	return backend.JobResult{
		ExitStatus: exitStatus,
		Stdout:     stdout,
		Stderr:     stderr,
	}, nil
}

func (c *LinuxContainer) Run(backend.JobSpec) (backend.JobResult, error) {
	return backend.JobResult{}, nil
}

func (c *LinuxContainer) NetIn(uint32, uint32) (uint32, uint32, error) {
	return 0, 0, nil
}

func (c *LinuxContainer) NetOut(string, uint32) error {
	return nil
}

func (c *LinuxContainer) rsync(src, dst string) error {
	wshPath := path.Join(c.path, "bin", "wsh")
	sockPath := path.Join(c.path, "run", "wshd.sock")

	rsync := exec.Command(
		"rsync",
		"-e", wshPath+" --socket "+sockPath+" --rsh",
		"-r",
		"-p",
		"--links",
		src,
		dst,
	)

	return c.runner.Run(rsync)
}

func (c *LinuxContainer) startOomNotifier() error {
	c.oomLock.Lock()
	defer c.oomLock.Unlock()

	if c.oomNotifier != nil {
		return nil
	}

	oomPath := path.Join(c.path, "bin", "oom")

	c.oomNotifier = exec.Command(oomPath, c.cgroupsManager.SubsystemPath("memory"))

	err := c.runner.Start(c.oomNotifier)
	if err != nil {
		return err
	}

	go c.watchForOom(c.oomNotifier)

	return nil
}

func (c *LinuxContainer) stopOomNotifier() {
	c.oomLock.RLock()
	defer c.oomLock.RUnlock()

	if c.oomNotifier != nil {
		c.runner.Kill(c.oomNotifier)
	}
}

func (c *LinuxContainer) watchForOom(oom *exec.Cmd) {
	err := c.runner.Wait(oom)
	if err == nil {
		log.Println(c.id, "out of memory")
		c.Stop(false)
	} else {
		log.Println(c.id, "oom failed:", err)
	}

	// TODO: handle case where oom notifier itself failed? kill container?
}
<|endoftext|>


<commit_before>package api

import (
	"expvar"
	"net/http"
	"os"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"strconv"
	"time"

	"github.com/zserge/metric"

	"nimona.io/pkg/context"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/daemon/config"
	"nimona.io/pkg/exchange"
	"nimona.io/pkg/http/router"
	"nimona.io/pkg/keychain"
	"nimona.io/pkg/log"
	"nimona.io/pkg/net"
	"nimona.io/pkg/object"
	"nimona.io/pkg/orchestrator"
	"nimona.io/pkg/resolver"
	"nimona.io/pkg/sqlobjectstore"
)

// API for HTTP
type API struct {
	config *config.Config

	router   *router.Router
	keychain keychain.Keychain
	net      net.Network
	resolver resolver.Resolver
	exchange exchange.Exchange

	objectStore  *sqlobjectstore.Store
	orchestrator orchestrator.Orchestrator

	token string

	version      string
	commit       string
	buildDate    string
	gracefulStop chan bool
	srv          *http.Server
}

// New HTTP API
func New(
	cfg *config.Config,
	k crypto.PrivateKey,
	kc keychain.Keychain,
	n net.Network,
	d resolver.Resolver,
	x exchange.Exchange,
	sst *sqlobjectstore.Store,
	or orchestrator.Orchestrator,
	version string,
	commit string,
	buildDate string,
	token string,
) *API {
	r := router.New()

	api := &API{
		config: cfg,

		router:      r,
		keychain:    kc,
		net:         n,
		resolver:    d,
		exchange:    x,
		objectStore: sst,

		orchestrator: or,

		version:      version,
		commit:       commit,
		buildDate:    buildDate,
		token:        token,
		gracefulStop: make(chan bool),
	}

	r.Use(api.Cors())
	r.Use(api.TokenAuth())

	r.Handle(
		"GET",
		"/api/v1/version$",
		api.HandleVersion,
	)
	r.Handle(
		"GET",
		"/api/v1/local$",
		api.HandleGetLocal,
	)
	// r.Handle(
	// 	"GET",
	// 	"/api/v1/peers",
	// 	api.HandleGetLookup,
	// )

	r.Handle(
		"GET",
		"/api/v1/identities$",
		api.HandleGetIdentities,
	)
	r.Handle(
		"GET",
		"/api/v1/identities/(?P<fingerprint>.+)$",
		api.HandleGetIdentity,
	)
	r.Handle(
		"POST",
		"/api/v1/identities$",
		api.HandlePostIdentities,
	)

	r.Handle(
		"GET",
		"/api/v1/peers$",
		api.HandleGetPeers,
	)
	r.Handle(
		"GET",
		"/api/v1/peers/(?P<fingerprint>.+)$",
		api.HandleGetPeer,
	)

	r.Handle(
		"GET",
		"/api/v1/objects$",
		api.HandleGetObjects,
	)
	// r.Handle(
	// 	"GET",
	// 	"/api/v1/objects/(?P<objectHash>.+)$",
	// 	api.HandleGetObject,
	// )
	r.Handle(
		"POST",
		"/api/v1/objects$",
		api.HandlePostObjects,
	)
	// r.Handle(
	// 	"POST",
	// 	"/api/v1/objects/(?P<rootObjectHash>.+)$",
	// 	api.HandlePostObject,
	// )

	// r.Handle(
	// 	"GET",
	// 	"/api/v1/streams/(?P<ns>.+)/(?P<pattern>.*)$",
	// 	api.HandleGetStreams,
	// )

	r.Handle(
		"POST",
		"/api/v1/stop$",
		api.Stop,
	)

	if y, _ := strconv.ParseBool(os.Getenv("NIMONA_EXPVAR")); y {
		expvar.Publish(
			"go:goroutine",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		expvar.Publish(
			"go:cgocall",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		expvar.Publish(
			"go:alloc",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		expvar.Publish(
			"go:alloc.total",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		go func() {
			for range time.Tick(100 * time.Millisecond) {
				m := &runtime.MemStats{}
				runtime.ReadMemStats(m)
				expvar.Get("go:goroutine").(metric.Metric).Add(
					float64(runtime.NumGoroutine()),
				)
				expvar.Get("go:cgocall").(metric.Metric).Add(
					float64(runtime.NumCgoCall()),
				)
				expvar.Get("go:alloc").(metric.Metric).Add(
					float64(m.Alloc) / 1000000,
				)
				expvar.Get("go:alloc.total").(metric.Metric).Add(
					float64(m.TotalAlloc) / 1000000,
				)
			}
		}()
		r.Handle(
			"GET",
			"/debug/metrics",
			func(c *router.Context) {
				metric.Handler(metric.Exposed).ServeHTTP(c.Writer, c.Request)
			},
		)
		goroutineStackHandler := func(c *router.Context) {
			stack := debug.Stack()
			c.Writer.Write(stack)                          // nolint: errcheck
			pprof.Lookup("goroutine").WriteTo(c.Writer, 2) // nolint: errcheck
		}
		r.Handle(
			"GET",
			"/debug/stack/goroutine",
			goroutineStackHandler,
		)
		r.Handle(
			"GET",
			"/debug/expvar",
			func(c *router.Context) {
				expvar.Handler().ServeHTTP(c.Writer, c.Request)
			},
		)
	}

	return api
}

// Serve HTTP API
func (api *API) Serve(address string) error {
	ctx := context.Background()
	logger := log.FromContext(ctx).Named("api")

	api.srv = &http.Server{
		Addr:    address,
		Handler: api.router,
	}

	go func() {
		if err := api.srv.ListenAndServe(); err != nil &&
			err != http.ErrServerClosed {
			logger.Error("Error serving", log.Error(err))
		}
	}()

	<-api.gracefulStop

	if err := api.srv.Shutdown(ctx); err != nil {
		logger.Error("Failed to shutdown", log.Error(err))
	}

	return nil
}

func (api *API) Stop(c *router.Context) {
	c.Status(http.StatusOK)
	go func() {
		api.gracefulStop <- true
	}()
}

func (api *API) mapObject(o object.Object) map[string]interface{} {
	m := o.ToMap()
	m["_hash:s"] = object.NewHash(o).String()
	return m
}

func (api *API) mapObjects(objects []object.Object) []map[string]interface{} {
	ms := []map[string]interface{}{}
	for _, o := range objects {
		ms = append(ms, api.mapObject(o))
	}
	return ms
}
<commit_msg>chore(api): remove commented out routes<commit_after>package api

import (
	"expvar"
	"net/http"
	"os"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"strconv"
	"time"

	"github.com/zserge/metric"

	"nimona.io/pkg/context"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/daemon/config"
	"nimona.io/pkg/exchange"
	"nimona.io/pkg/http/router"
	"nimona.io/pkg/keychain"
	"nimona.io/pkg/log"
	"nimona.io/pkg/net"
	"nimona.io/pkg/object"
	"nimona.io/pkg/orchestrator"
	"nimona.io/pkg/resolver"
	"nimona.io/pkg/sqlobjectstore"
)

// API for HTTP
type API struct {
	config *config.Config

	router   *router.Router
	keychain keychain.Keychain
	net      net.Network
	resolver resolver.Resolver
	exchange exchange.Exchange

	objectStore  *sqlobjectstore.Store
	orchestrator orchestrator.Orchestrator

	token string

	version      string
	commit       string
	buildDate    string
	gracefulStop chan bool
	srv          *http.Server
}

// New HTTP API
func New(
	cfg *config.Config,
	k crypto.PrivateKey,
	kc keychain.Keychain,
	n net.Network,
	d resolver.Resolver,
	x exchange.Exchange,
	sst *sqlobjectstore.Store,
	or orchestrator.Orchestrator,
	version string,
	commit string,
	buildDate string,
	token string,
) *API {
	r := router.New()

	api := &API{
		config: cfg,

		router:      r,
		keychain:    kc,
		net:         n,
		resolver:    d,
		exchange:    x,
		objectStore: sst,

		orchestrator: or,

		version:      version,
		commit:       commit,
		buildDate:    buildDate,
		token:        token,
		gracefulStop: make(chan bool),
	}

	r.Use(api.Cors())
	r.Use(api.TokenAuth())

	r.Handle(
		"GET",
		"/api/v1/version$",
		api.HandleVersion,
	)
	r.Handle(
		"GET",
		"/api/v1/local$",
		api.HandleGetLocal,
	)

	r.Handle(
		"GET",
		"/api/v1/identities$",
		api.HandleGetIdentities,
	)
	r.Handle(
		"GET",
		"/api/v1/identities/(?P<fingerprint>.+)$",
		api.HandleGetIdentity,
	)
	r.Handle(
		"POST",
		"/api/v1/identities$",
		api.HandlePostIdentities,
	)

	r.Handle(
		"GET",
		"/api/v1/peers$",
		api.HandleGetPeers,
	)
	r.Handle(
		"GET",
		"/api/v1/peers/(?P<fingerprint>.+)$",
		api.HandleGetPeer,
	)

	r.Handle(
		"GET",
		"/api/v1/objects$",
		api.HandleGetObjects,
	)
	r.Handle(
		"POST",
		"/api/v1/objects$",
		api.HandlePostObjects,
	)
	r.Handle(
		"POST",
		"/api/v1/stop$",
		api.Stop,
	)

	if y, _ := strconv.ParseBool(os.Getenv("NIMONA_EXPVAR")); y {
		expvar.Publish(
			"go:goroutine",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		expvar.Publish(
			"go:cgocall",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		expvar.Publish(
			"go:alloc",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		expvar.Publish(
			"go:alloc.total",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		go func() {
			for range time.Tick(100 * time.Millisecond) {
				m := &runtime.MemStats{}
				runtime.ReadMemStats(m)
				expvar.Get("go:goroutine").(metric.Metric).Add(
					float64(runtime.NumGoroutine()),
				)
				expvar.Get("go:cgocall").(metric.Metric).Add(
					float64(runtime.NumCgoCall()),
				)
				expvar.Get("go:alloc").(metric.Metric).Add(
					float64(m.Alloc) / 1000000,
				)
				expvar.Get("go:alloc.total").(metric.Metric).Add(
					float64(m.TotalAlloc) / 1000000,
				)
			}
		}()
		r.Handle(
			"GET",
			"/debug/metrics",
			func(c *router.Context) {
				metric.Handler(metric.Exposed).ServeHTTP(c.Writer, c.Request)
			},
		)
		goroutineStackHandler := func(c *router.Context) {
			stack := debug.Stack()
			c.Writer.Write(stack)                          // nolint: errcheck
			pprof.Lookup("goroutine").WriteTo(c.Writer, 2) // nolint: errcheck
		}
		r.Handle(
			"GET",
			"/debug/stack/goroutine",
			goroutineStackHandler,
		)
		r.Handle(
			"GET",
			"/debug/expvar",
			func(c *router.Context) {
				expvar.Handler().ServeHTTP(c.Writer, c.Request)
			},
		)
	}

	return api
}

// Serve HTTP API
func (api *API) Serve(address string) error {
	ctx := context.Background()
	logger := log.FromContext(ctx).Named("api")

	api.srv = &http.Server{
		Addr:    address,
		Handler: api.router,
	}

	go func() {
		if err := api.srv.ListenAndServe(); err != nil &&
			err != http.ErrServerClosed {
			logger.Error("Error serving", log.Error(err))
		}
	}()

	<-api.gracefulStop

	if err := api.srv.Shutdown(ctx); err != nil {
		logger.Error("Failed to shutdown", log.Error(err))
	}

	return nil
}

func (api *API) Stop(c *router.Context) {
	c.Status(http.StatusOK)
	go func() {
		api.gracefulStop <- true
	}()
}

func (api *API) mapObject(o object.Object) map[string]interface{} {
	m := o.ToMap()
	m["_hash:s"] = object.NewHash(o).String()
	return m
}

func (api *API) mapObjects(objects []object.Object) []map[string]interface{} {
	ms := []map[string]interface{}{}
	for _, o := range objects {
		ms = append(ms, api.mapObject(o))
	}
	return ms
}
<|endoftext|>


<commit_before>package app

import (
	"context"
	"fmt"
	"log"
	"strings"

	"github.com/ash2k/smith"
	"github.com/ash2k/smith/pkg/client"
	"github.com/ash2k/smith/pkg/processor"
)

type App struct {
	Version   string
	GitCommit string
}

func (a *App) Run(ctx context.Context) error {
	c, err := client.NewInCluster()
	if err != nil {
		return err
	}
	c.Agent = "smith/" + a.Version + "/" + a.GitCommit

	// 1. Ensure ThirdPartyResource TEMPLATE exists
	err = retryUntilSuccessOrDone(ctx, func() error {
		return ensureResourceExists(ctx, c)
	}, func(e error) bool {
		// TODO be smarter about what is retried
		log.Printf("Failed to create resource %s: %v", smith.TemplateResourceName, e)
		return false
	})
	if err != nil {
		return err
	}

	// 2. TODO watch supported built-in resource types for events.

	// 3. List Third Party Resources to figure out list of supported ones
	var tprList smith.ThirdPartyResourceList
	err = retryUntilSuccessOrDone(ctx, func() error {
		tprList = smith.ThirdPartyResourceList{}
		return c.List(ctx, smith.ThirdPartyResourceGroupVersion, smith.AllNamespaces, smith.AllResources, nil, nil, &tprList)
	}, func(e error) bool {
		// TODO be smarter about what is retried
		log.Printf("Failed to list Third Party Resources %s: %v", smith.TemplateResourceName, e)
		return false
	})
	if err != nil {
		return err
	}
	allEvents := make(chan interface{})
	watcherCtx, watcherCancel := context.WithCancel(ctx)
	defer watcherCancel() // if NewWatcher panics, we still want to release context
	watcher := NewWatcher(watcherCtx, c, allEvents)
	defer watcher.Join()  // await termination
	defer watcherCancel() // cancel ctx to signal done to watcher

	for _, tpr := range tprList.Items {
		watchTpr(watcher, &tpr)
	}

	// 4. Watch for addition/removal of TPRs to start/stop watches
	watcher.Watch(smith.ThirdPartyResourceGroupVersion, smith.AllNamespaces, smith.AllResources, tprList.ResourceVersion, newTprEvent)

	// 5. List existing templates
	var templateList smith.TemplateList
	err = retryUntilSuccessOrDone(ctx, func() error {
		templateList = smith.TemplateList{}
		return c.List(ctx, smith.TemplateResourceGroupVersion, smith.AllNamespaces, smith.TemplateResourcePath, nil, nil, &templateList)
	}, func(e error) bool {
		// TODO be smarter about what is retried
		log.Printf("Failed to list resources %s: %v", smith.TemplateResourceName, e)
		return false
	})
	if err != nil {
		return err
	}

	// 6. Start rebuilds for existing templates to re-assert their state
	tp := processor.New(c)
	for _, template := range templateList.Items {
		tp.Rebuild(template)
	}

	// 7. Watch for template-related events
	watcher.Watch(smith.TemplateResourceGroupVersion, smith.AllNamespaces, smith.TemplateResourcePath, templateList.ResourceVersion, newTemplateEvent)

	// 8. Process events and trigger rebuilds as necessary
	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case event := <-allEvents:
			switch ev := event.(type) {
			case error:
				log.Printf("Something went wrong with watch: %v", ev)
			case *smith.TemplateWatchEvent:
				switch ev.Type {
				case smith.Added, smith.Modified:
					tp.Rebuild(*ev.Object)
				case smith.Deleted:
				// TODO Somehow use finalizers to prevent direct deletion?
				// "No direct deletion" convention? Use ObjectMeta.DeletionTimestamp like Namespace does?
				// Somehow implement GC to do cleanup after template is deleted?
				// Maybe store template in annotation on each resource to help reconstruct the dependency graph for GC?
				case smith.Error:
					// TODO what to do with it?
					log.Printf("Watch returned an Error event: %#v", ev)
				}
			case *smith.TprInstanceWatchEvent:
				switch ev.Type {
				case smith.Added, smith.Modified, smith.Deleted:
					templateName := ev.Object.Labels[smith.TemplateNameLabel]
					if templateName != "" {
						tp.RebuildByName(ev.Object.Namespace, templateName)
					}
				case smith.Error:
					// TODO what to do with it?
					log.Printf("Watch returned an Error event: %#v", ev)
				}
			case *smith.TprWatchEvent:
				switch ev.Type {
				case smith.Added:
					watchTpr(watcher, ev.Object)
				// TODO rebuild all templates containing resources of this type
				case smith.Modified:
					forgetTpr(watcher, ev.Object)
					watchTpr(watcher, ev.Object)
				// TODO rebuild all templates containing resources of this type
				case smith.Deleted:
					forgetTpr(watcher, ev.Object)
					// TODO rebuild all templates containing resources of this type
				case smith.Error:
					// TODO what to do with it?
					log.Printf("Watch returned an Error event: %#v", ev)
				}
			default:
				log.Printf("Unexpected event type: %T", event)
			}
		}
	}
}

func newTemplateEvent() interface{} {
	return &smith.TemplateWatchEvent{}
}

func newTprInstanceEvent() interface{} {
	return &smith.TprInstanceWatchEvent{}
}

func newTprEvent() interface{} {
	return &smith.TprWatchEvent{}
}

func ensureResourceExists(ctx context.Context, c *client.ResourceClient) error {
	log.Printf("Creating ThirdPartyResource %s", smith.TemplateResourceName)
	res := &smith.ThirdPartyResource{}
	err := c.Create(ctx, smith.ThirdPartyResourceGroupVersion, "", "thirdpartyresources", &smith.ThirdPartyResource{
		TypeMeta: smith.TypeMeta{
			Kind:       "ThirdPartyResource",
			APIVersion: smith.ThirdPartyResourceGroupVersion,
		},
		ObjectMeta: smith.ObjectMeta{
			Name: smith.TemplateResourceName,
		},
		Description: "Smith resource manager",
		Versions: []smith.APIVersion{
			{Name: smith.TemplateResourceVersion},
		},
	}, res)
	if err != nil {
		log.Printf("%#v", err)
		if !client.IsConflict(err) {
			return fmt.Errorf("failed to create ThirdPartyResource: %v", err)
		}
		log.Printf("ThirdPartyResource %s already exists", smith.TemplateResourceName)
	} else {
		log.Printf("ThirdPartyResource %s created: %+v", smith.TemplateResourceName, res)
	}
	return nil
}

func watchTpr(watcher *Watcher, tpr *smith.ThirdPartyResource) {
	// TODO only watch supported TPRs (inspect annotations?)
	path, group := splitTprName(tpr.Name)
	for _, version := range tpr.Versions {
		watcher.Watch(group+"/"+version.Name, smith.AllNamespaces, path, "", newTprInstanceEvent)
	}
}

func forgetTpr(watcher *Watcher, tpr *smith.ThirdPartyResource) {
	path, group := splitTprName(tpr.Name)
	for _, version := range tpr.Versions {
		watcher.Forget(group+"/"+version.Name, smith.AllNamespaces, path)
	}
}

// splitTprName splits TPR's name into resource name and group name.
// e.g. "postgresql-resource.smith-sql.ash2k.com" is split into "postgresqlresources" and "smith-sql.ash2k.com".
// See https://github.com/kubernetes/kubernetes/blob/master/docs/design/extending-api.md
// See k8s.io/pkg/api/meta/restmapper.go:147 KindToResource()
func splitTprName(name string) (string, string) {
	pos := strings.IndexByte(name, '.')
	if pos == -1 || pos == 0 {
		panic(fmt.Errorf("invalid resource name: %q", name))
	}
	resourcePath := strings.Replace(name[:pos], "-", "", -1)
	switch string(resourcePath[len(resourcePath)-1]) {
	case "s":
		resourcePath += "es"
	case "y":
		resourcePath = resourcePath[:len(resourcePath)-1] + "ies"
	}
	return resourcePath, name[pos+1:]
}
<commit_msg>Fix pluaralization<commit_after>package app

import (
	"context"
	"fmt"
	"log"
	"strings"

	"github.com/ash2k/smith"
	"github.com/ash2k/smith/pkg/client"
	"github.com/ash2k/smith/pkg/processor"
)

type App struct {
	Version   string
	GitCommit string
}

func (a *App) Run(ctx context.Context) error {
	c, err := client.NewInCluster()
	if err != nil {
		return err
	}
	c.Agent = "smith/" + a.Version + "/" + a.GitCommit

	// 1. Ensure ThirdPartyResource TEMPLATE exists
	err = retryUntilSuccessOrDone(ctx, func() error {
		return ensureResourceExists(ctx, c)
	}, func(e error) bool {
		// TODO be smarter about what is retried
		log.Printf("Failed to create resource %s: %v", smith.TemplateResourceName, e)
		return false
	})
	if err != nil {
		return err
	}

	// 2. TODO watch supported built-in resource types for events.

	// 3. List Third Party Resources to figure out list of supported ones
	var tprList smith.ThirdPartyResourceList
	err = retryUntilSuccessOrDone(ctx, func() error {
		tprList = smith.ThirdPartyResourceList{}
		return c.List(ctx, smith.ThirdPartyResourceGroupVersion, smith.AllNamespaces, smith.AllResources, nil, nil, &tprList)
	}, func(e error) bool {
		// TODO be smarter about what is retried
		log.Printf("Failed to list Third Party Resources %s: %v", smith.TemplateResourceName, e)
		return false
	})
	if err != nil {
		return err
	}
	allEvents := make(chan interface{})
	watcherCtx, watcherCancel := context.WithCancel(ctx)
	defer watcherCancel() // if NewWatcher panics, we still want to release context
	watcher := NewWatcher(watcherCtx, c, allEvents)
	defer watcher.Join()  // await termination
	defer watcherCancel() // cancel ctx to signal done to watcher

	for _, tpr := range tprList.Items {
		watchTpr(watcher, &tpr)
	}

	// 4. Watch for addition/removal of TPRs to start/stop watches
	watcher.Watch(smith.ThirdPartyResourceGroupVersion, smith.AllNamespaces, smith.AllResources, tprList.ResourceVersion, newTprEvent)

	// 5. List existing templates
	var templateList smith.TemplateList
	err = retryUntilSuccessOrDone(ctx, func() error {
		templateList = smith.TemplateList{}
		return c.List(ctx, smith.TemplateResourceGroupVersion, smith.AllNamespaces, smith.TemplateResourcePath, nil, nil, &templateList)
	}, func(e error) bool {
		// TODO be smarter about what is retried
		log.Printf("Failed to list resources %s: %v", smith.TemplateResourceName, e)
		return false
	})
	if err != nil {
		return err
	}

	// 6. Start rebuilds for existing templates to re-assert their state
	tp := processor.New(c)
	for _, template := range templateList.Items {
		tp.Rebuild(template)
	}

	// 7. Watch for template-related events
	watcher.Watch(smith.TemplateResourceGroupVersion, smith.AllNamespaces, smith.TemplateResourcePath, templateList.ResourceVersion, newTemplateEvent)

	// 8. Process events and trigger rebuilds as necessary
	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case event := <-allEvents:
			switch ev := event.(type) {
			case error:
				log.Printf("Something went wrong with watch: %v", ev)
			case *smith.TemplateWatchEvent:
				switch ev.Type {
				case smith.Added, smith.Modified:
					tp.Rebuild(*ev.Object)
				case smith.Deleted:
				// TODO Somehow use finalizers to prevent direct deletion?
				// "No direct deletion" convention? Use ObjectMeta.DeletionTimestamp like Namespace does?
				// Somehow implement GC to do cleanup after template is deleted?
				// Maybe store template in annotation on each resource to help reconstruct the dependency graph for GC?
				case smith.Error:
					// TODO what to do with it?
					log.Printf("Watch returned an Error event: %#v", ev)
				}
			case *smith.TprInstanceWatchEvent:
				switch ev.Type {
				case smith.Added, smith.Modified, smith.Deleted:
					templateName := ev.Object.Labels[smith.TemplateNameLabel]
					if templateName != "" {
						tp.RebuildByName(ev.Object.Namespace, templateName)
					}
				case smith.Error:
					// TODO what to do with it?
					log.Printf("Watch returned an Error event: %#v", ev)
				}
			case *smith.TprWatchEvent:
				switch ev.Type {
				case smith.Added:
					watchTpr(watcher, ev.Object)
				// TODO rebuild all templates containing resources of this type
				case smith.Modified:
					forgetTpr(watcher, ev.Object)
					watchTpr(watcher, ev.Object)
				// TODO rebuild all templates containing resources of this type
				case smith.Deleted:
					forgetTpr(watcher, ev.Object)
					// TODO rebuild all templates containing resources of this type
				case smith.Error:
					// TODO what to do with it?
					log.Printf("Watch returned an Error event: %#v", ev)
				}
			default:
				log.Printf("Unexpected event type: %T", event)
			}
		}
	}
}

func newTemplateEvent() interface{} {
	return &smith.TemplateWatchEvent{}
}

func newTprInstanceEvent() interface{} {
	return &smith.TprInstanceWatchEvent{}
}

func newTprEvent() interface{} {
	return &smith.TprWatchEvent{}
}

func ensureResourceExists(ctx context.Context, c *client.ResourceClient) error {
	log.Printf("Creating ThirdPartyResource %s", smith.TemplateResourceName)
	res := &smith.ThirdPartyResource{}
	err := c.Create(ctx, smith.ThirdPartyResourceGroupVersion, "", "thirdpartyresources", &smith.ThirdPartyResource{
		TypeMeta: smith.TypeMeta{
			Kind:       "ThirdPartyResource",
			APIVersion: smith.ThirdPartyResourceGroupVersion,
		},
		ObjectMeta: smith.ObjectMeta{
			Name: smith.TemplateResourceName,
		},
		Description: "Smith resource manager",
		Versions: []smith.APIVersion{
			{Name: smith.TemplateResourceVersion},
		},
	}, res)
	if err != nil {
		log.Printf("%#v", err)
		if !client.IsConflict(err) {
			return fmt.Errorf("failed to create ThirdPartyResource: %v", err)
		}
		log.Printf("ThirdPartyResource %s already exists", smith.TemplateResourceName)
	} else {
		log.Printf("ThirdPartyResource %s created: %+v", smith.TemplateResourceName, res)
	}
	return nil
}

func watchTpr(watcher *Watcher, tpr *smith.ThirdPartyResource) {
	// TODO only watch supported TPRs (inspect annotations?)
	path, group := splitTprName(tpr.Name)
	for _, version := range tpr.Versions {
		watcher.Watch(group+"/"+version.Name, smith.AllNamespaces, path, "", newTprInstanceEvent)
	}
}

func forgetTpr(watcher *Watcher, tpr *smith.ThirdPartyResource) {
	path, group := splitTprName(tpr.Name)
	for _, version := range tpr.Versions {
		watcher.Forget(group+"/"+version.Name, smith.AllNamespaces, path)
	}
}

// splitTprName splits TPR's name into resource name and group name.
// e.g. "postgresql-resource.smith-sql.ash2k.com" is split into "postgresqlresources" and "smith-sql.ash2k.com".
// See https://github.com/kubernetes/kubernetes/blob/master/docs/design/extending-api.md
// See k8s.io/pkg/api/meta/restmapper.go:147 KindToResource()
func splitTprName(name string) (string, string) {
	pos := strings.IndexByte(name, '.')
	if pos == -1 || pos == 0 {
		panic(fmt.Errorf("invalid resource name: %q", name))
	}
	resourcePath := strings.Replace(name[:pos], "-", "", -1)
	switch string(resourcePath[len(resourcePath)-1]) {
	case "s":
		resourcePath += "es"
	case "y":
		resourcePath = resourcePath[:len(resourcePath)-1] + "ies"
	default:
		resourcePath += "s"
	}
	return resourcePath, name[pos+1:]
}
<|endoftext|>