<commit_before>package query

import (
	"context"
	"fmt"
	"sync"

	"github.com/rs/rest-layer/schema"
)

// Resource represents type that can be queried by Projection.Eval.
type Resource interface {
	// Find executes the query and returns the matching items.
	Find(ctx context.Context, query *Query) ([]map[string]interface{}, error)

	// MultiGet get some items by their id and return them in the same order. If one
	// or more item(s) is not found, their slot in the response is set to nil.
	MultiGet(ctx context.Context, ids []interface{}) ([]map[string]interface{}, error)

	// SubResource returns the sub-resource at path. If path starts with a
	// dot, the lookup is performed relative to the current resource.
	SubResource(ctx context.Context, path string) (Resource, error)

	// Validator returns the schema.Validator associated with the resource.
	Validator() schema.Validator
}

// Eval evaluate the projection on the given payload with the help of the
// validator. The resolver is used to fetch payload of references outside of the
// provided payload.
func (p Projection) Eval(ctx context.Context, payload map[string]interface{}, rsc Resource) (map[string]interface{}, error) {
	rbr := &referenceBatchResolver{}
	validator := rsc.Validator()
	payload, err := evalProjection(ctx, p, payload, validator, rbr)
	if err == nil {
		// Execute the batched reference resolutions.
		err = rbr.execute(ctx, rsc)
	}
	return payload, err
}

func prepareProjection(p Projection, payload map[string]interface{}) (Projection, error) {
	var proj Projection
	if len(p) == 0 {
		// When the Projection is empty, it's like saying "all fields".
		// This allows notations like id,user{} to embed all fields of the user
		// sub-resource.
		for fn := range payload {
			proj = append(proj, ProjectionField{Name: fn})
		}
		return proj, nil
	}

	hasStar := false
	var starChildren Projection
	for _, pf := range p {
		if pf.Name == "*" {
			if hasStar {
				return nil, fmt.Errorf("only one * in projection allowed")
			}
			hasStar = true
			starChildren = pf.Children
		} else {
			proj = append(proj, pf)
		}
	}
	if hasStar {
		for fn := range payload {
			exists := false
			for _, pf := range proj {
				if fn == pf.Name && pf.Alias == "" {
					exists = true
				}
			}
			if !exists {
				proj = append(proj, ProjectionField{Name: fn, Children: starChildren})
			}
		}
	}
	return proj, nil
}

func evalProjectionArray(ctx context.Context, pf ProjectionField, payload []interface{}, def *schema.Field, rbr *referenceBatchResolver) (*[]interface{}, error) {
	res := make([]interface{}, 0, len(payload))
	// Return pointer to res, because it may be populated after this function ends, by referenceBatchResolver
	// in `schema.Reference` case
	resp := &res
	resMu := sync.Mutex{}

	validator := def.Validator
	name := pf.Name
	if pf.Alias != "" {
		name = pf.Alias
	}

	if ref, ok := validator.(*schema.Reference); ok {
		// Execute sub-request in batch
		e := &In{Field: "id", Values: payload}
		q := &Query{
			Projection: pf.Children,
			Predicate:  Predicate{e},
		}
		rbr.request(ref.Path, q, func(payloads []map[string]interface{}, validator schema.Validator) error {
			var v interface{}
			var err error
			for i := range payloads {
				if payloads[i], err = evalProjection(ctx, pf.Children, payloads[i], validator, rbr); err != nil {
					return fmt.Errorf("%s: error applying projection on sub-field item #%d: %v", pf.Name, i, err)
				}
			}
			if v, err = resolveFieldHandler(ctx, pf, def, payloads); err != nil {
				return fmt.Errorf("%s: error resolving field handler on sub-field: %v", name, err)
			}
			vv := v.([]map[string]interface{})
			resMu.Lock()
			for _, item := range vv {
				res = append(res, item)
			}
			resMu.Unlock()
			return nil
		})
	} else if dict, ok := validator.(*schema.Dict); ok {
		for _, val := range payload {
			subval, ok := val.(map[string]interface{})
			if !ok {
				return nil, fmt.Errorf("%s: invalid value: not a dict", pf.Name)
			}
			var err error
			if subval, err = evalProjection(ctx, pf.Children, subval, dict, rbr); err != nil {
				return nil, fmt.Errorf("%s.%v", pf.Name, err)
			}
			var v interface{}
			if v, err = resolveFieldHandler(ctx, pf, def, subval); err != nil {
				return nil, err
			}
			res = append(res, v)
		}
	} else if array, ok := validator.(*schema.Array); ok {
		for i, val := range payload {
			if subval, ok := val.([]interface{}); ok {
				var err error
				var subvalp *[]interface{}
				if subvalp, err = evalProjectionArray(ctx, pf, subval, &array.Values, rbr); err != nil {
					return nil, fmt.Errorf("%s: error applying projection on array item #%d: %v", pf.Name, i, err)
				}
				var v interface{}
				if v, err = resolveFieldHandler(ctx, pf, def, *subvalp); err != nil {
					return nil, fmt.Errorf("%s: error resolving field handler on array: %v", name, err)
				}
				res = append(res, v)
			} else {
				return nil, fmt.Errorf("%s. is not an array", pf.Name)
			}
		}
	} else {
		for _, val := range payload {
			subval, ok := val.(map[string]interface{})
			if !ok {
				return nil, fmt.Errorf("%s: invalid value: not a dict", pf.Name)
			}
			if fg, ok := validator.(schema.FieldGetter); ok {
				var err error
				if subval, err = evalProjection(ctx, pf.Children, subval, fg, rbr); err != nil {
					return nil, fmt.Errorf("%s.%v", pf.Name, err)
				}
				var v interface{}
				if v, err = resolveFieldHandler(ctx, pf, def, subval); err != nil {
					return nil, err
				}
				res = append(res, v)
			} else {
				return nil, fmt.Errorf("%s. is not an object", pf.Name)
			}
		}
	}

	return resp, nil
}

func evalProjection(ctx context.Context, p Projection, payload map[string]interface{}, fg schema.FieldGetter, rbr *referenceBatchResolver) (map[string]interface{}, error) {
	res := map[string]interface{}{}
	resMu := sync.Mutex{}
	var err error
	p, err = prepareProjection(p, payload)
	if err != nil {
		return nil, err
	}
	for i := range p {
		pf := p[i]
		name := pf.Name
		// Handle aliasing
		if pf.Alias != "" {
			name = pf.Alias
		}
		def := fg.GetField(pf.Name)
		// Skip hidden fields
		if def != nil && def.Hidden {
			continue
		}
		if val, found := payload[pf.Name]; found {
			// Handle sub field selection (if field has a value)
			if len(pf.Children) > 0 && val != nil {
				if def != nil && def.Schema != nil {
					subval, ok := val.(map[string]interface{})
					if !ok {
						return nil, fmt.Errorf("%s: invalid value: not a dict", pf.Name)
					}
					var err error
					if subval, err = evalProjection(ctx, pf.Children, subval, def.Schema, rbr); err != nil {
						return nil, fmt.Errorf("%s.%v", pf.Name, err)
					}
					if res[name], err = resolveFieldHandler(ctx, pf, def, subval); err != nil {
						return nil, err
					}
				} else if ref, ok := def.Validator.(*schema.Reference); ok {
					// Execute sub-request in batch
					q := &Query{
						Projection: pf.Children,
						Predicate:  Predicate{&Equal{Field: "id", Value: val}},
					}
					rbr.request(ref.Path, q, func(payloads []map[string]interface{}, validator schema.Validator) error {
						var v interface{}
						if len(payloads) == 1 {
							payload, err := evalProjection(ctx, pf.Children, payloads[0], validator, rbr)
							if err != nil {
								return fmt.Errorf("%s: error applying Projection on sub-field: %v", name, err)
							}
							if v, err = resolveFieldHandler(ctx, pf, def, payload); err != nil {
								return fmt.Errorf("%s: error resolving field handler on sub-field: %v", name, err)
							}
						}
						// Return `null` for missing result instead of empty object
						if m, ok := v.(map[string]interface{}); ok && len(m) == 0 {
							v = nil
						}
						resMu.Lock()
						res[name] = v
						resMu.Unlock()
						return nil
					})
				} else if dict, ok := def.Validator.(*schema.Dict); ok {
					subval, ok := val.(map[string]interface{})
					if !ok {
						return nil, fmt.Errorf("%s: invalid value: not a dict", pf.Name)
					}
					var err error
					if subval, err = evalProjection(ctx, pf.Children, subval, dict, rbr); err != nil {
						return nil, fmt.Errorf("%s.%v", pf.Name, err)
					}
					if res[name], err = resolveFieldHandler(ctx, pf, def, subval); err != nil {
						return nil, err
					}
				} else if array, ok := def.Validator.(*schema.Array); ok {
					if payload, ok := val.([]interface{}); ok {
						var err error
						var subvalp *[]interface{}
						if subvalp, err = evalProjectionArray(ctx, pf, payload, &array.Values, rbr); err != nil {
							return nil, fmt.Errorf("%s: error applying projection on array item #%d: %v", pf.Name, i, err)
						}
						if res[name], err = resolveFieldHandler(ctx, pf, &array.Values, subvalp); err != nil {
							return nil, fmt.Errorf("%s: error resolving field handler on array: %v", name, err)
						}
					} else {
						return nil, fmt.Errorf("%s: invalid value: not an array", pf.Name)
					}
				} else {
					return nil, fmt.Errorf("%s: field has no children", pf.Name)
				}
			} else {
				var err error
				if res[name], err = resolveFieldHandler(ctx, pf, def, val); err != nil {
					return nil, err
				}
			}
		} else if def != nil {
			// If field is not found, it may be a connection
			if ref, ok := def.Validator.(*schema.Connection); ok {
				id, ok := payload["id"]
				if !ok {
					return nil, fmt.Errorf("%s: error applying projection on sub-resource: item lacks ID field", pf.Name)
				}
				q, err := connectionQuery(pf, ref.Field, id, ref.Validator)
				if err != nil {
					return nil, err
				}
				rbr.request(ref.Path, q, func(payloads []map[string]interface{}, validator schema.Validator) (err error) {
					for i := range payloads {
						if payloads[i], err = evalProjection(ctx, pf.Children, payloads[i], validator, rbr); err != nil {
							return fmt.Errorf("%s: error applying projection on sub-resource item #%d: %v", pf.Name, i, err)
						}
					}
					var v interface{}
					if v, err = resolveFieldHandler(ctx, pf, def, payloads); err != nil {
						return fmt.Errorf("%s: error resolving field handler on sub-resource: %v", name, err)
					}
					resMu.Lock()
					res[name] = v
					resMu.Unlock()
					return nil
				})
			}
		}
	}
	return res, nil
}

// connectionQuery builds a query from a projection field on a schema.Connection type field.
func connectionQuery(pf ProjectionField, field string, id interface{}, validator schema.Validator) (*Query, error) {
	q := &Query{
		Projection: pf.Children,
		Predicate:  Predicate{&Equal{Field: field, Value: id}},
	}
	if filter, ok := pf.Params["filter"].(string); ok {
		p, err := ParsePredicate(filter)
		if err != nil {
			return nil, fmt.Errorf("%s: invalid filter: %v", pf.Name, err)
		}
		err = p.Prepare(validator)
		if err != nil {
			return nil, fmt.Errorf("%s: invalid filter: %v", pf.Name, err)
		}
		q.Predicate = append(q.Predicate, p...)
	}
	if sort, ok := pf.Params["sort"].(string); ok {
		s, err := ParseSort(sort)
		if err != nil {
			return nil, fmt.Errorf("%s: invalid sort: %v", pf.Name, err)
		}
		q.Sort = s
	}
	skip := 0
	if v, ok := pf.Params["skip"].(int); ok {
		skip = v
	}
	page := 1
	if v, ok := pf.Params["page"].(int); ok {
		page = v
	}
	limit := 20
	if v, ok := pf.Params["limit"].(int); ok {
		limit = v
	}
	q.Window = Page(page, limit, skip)
	return q, nil
}

// resolveFieldHandler calls the field handler with the provided params (if any).
func resolveFieldHandler(ctx context.Context, pf ProjectionField, def *schema.Field, val interface{}) (interface{}, error) {
	if def == nil {
		return val, nil
	}
	var err error
	if def.Handler != nil && len(pf.Params) > 0 {
		val, err = def.Handler(ctx, val, pf.Params)
		if err != nil {
			return nil, fmt.Errorf("%s: %v", pf.Name, err)
		}
	}
	if s, ok := def.Validator.(schema.FieldSerializer); ok {
		val, err = s.Serialize(val)
		if err != nil {
			return nil, fmt.Errorf("%s: %v", pf.Name, err)
		}
	}
	return val, nil
}
<commit_msg>Use proper type switch in `evalProjectionArray`.<commit_after>package query

import (
	"context"
	"fmt"
	"sync"

	"github.com/rs/rest-layer/schema"
)

// Resource represents type that can be queried by Projection.Eval.
type Resource interface {
	// Find executes the query and returns the matching items.
	Find(ctx context.Context, query *Query) ([]map[string]interface{}, error)

	// MultiGet get some items by their id and return them in the same order. If one
	// or more item(s) is not found, their slot in the response is set to nil.
	MultiGet(ctx context.Context, ids []interface{}) ([]map[string]interface{}, error)

	// SubResource returns the sub-resource at path. If path starts with a
	// dot, the lookup is performed relative to the current resource.
	SubResource(ctx context.Context, path string) (Resource, error)

	// Validator returns the schema.Validator associated with the resource.
	Validator() schema.Validator
}

// Eval evaluate the projection on the given payload with the help of the
// validator. The resolver is used to fetch payload of references outside of the
// provided payload.
func (p Projection) Eval(ctx context.Context, payload map[string]interface{}, rsc Resource) (map[string]interface{}, error) {
	rbr := &referenceBatchResolver{}
	validator := rsc.Validator()
	payload, err := evalProjection(ctx, p, payload, validator, rbr)
	if err == nil {
		// Execute the batched reference resolutions.
		err = rbr.execute(ctx, rsc)
	}
	return payload, err
}

func prepareProjection(p Projection, payload map[string]interface{}) (Projection, error) {
	var proj Projection
	if len(p) == 0 {
		// When the Projection is empty, it's like saying "all fields".
		// This allows notations like id,user{} to embed all fields of the user
		// sub-resource.
		for fn := range payload {
			proj = append(proj, ProjectionField{Name: fn})
		}
		return proj, nil
	}

	hasStar := false
	var starChildren Projection
	for _, pf := range p {
		if pf.Name == "*" {
			if hasStar {
				return nil, fmt.Errorf("only one * in projection allowed")
			}
			hasStar = true
			starChildren = pf.Children
		} else {
			proj = append(proj, pf)
		}
	}
	if hasStar {
		for fn := range payload {
			exists := false
			for _, pf := range proj {
				if fn == pf.Name && pf.Alias == "" {
					exists = true
				}
			}
			if !exists {
				proj = append(proj, ProjectionField{Name: fn, Children: starChildren})
			}
		}
	}
	return proj, nil
}

func evalProjectionArray(ctx context.Context, pf ProjectionField, payload []interface{}, def *schema.Field, rbr *referenceBatchResolver) (*[]interface{}, error) {
	res := make([]interface{}, 0, len(payload))
	// Return pointer to res, because it may be populated after this function ends, by referenceBatchResolver
	// in `schema.Reference` case
	resp := &res
	resMu := sync.Mutex{}

	validator := def.Validator
	name := pf.Name
	if pf.Alias != "" {
		name = pf.Alias
	}

	switch fieldType := validator.(type) {
	// schema.Reference has higher priority than schema.FieldGetter
	case *schema.Reference:
		// Execute sub-request in batch
		e := &In{Field: "id", Values: payload}
		q := &Query{
			Projection: pf.Children,
			Predicate:  Predicate{e},
		}
		rbr.request(fieldType.Path, q, func(payloads []map[string]interface{}, validator schema.Validator) error {
			var v interface{}
			var err error
			for i := range payloads {
				if payloads[i], err = evalProjection(ctx, pf.Children, payloads[i], validator, rbr); err != nil {
					return fmt.Errorf("%s: error applying projection on sub-field item #%d: %v", pf.Name, i, err)
				}
			}
			if v, err = resolveFieldHandler(ctx, pf, def, payloads); err != nil {
				return fmt.Errorf("%s: error resolving field handler on sub-field: %v", name, err)
			}
			vv := v.([]map[string]interface{})
			resMu.Lock()
			for _, item := range vv {
				res = append(res, item)
			}
			resMu.Unlock()
			return nil
		})
	// schema.Array has higher priority than schema.FieldGetter
	case *schema.Array:
		for i, val := range payload {
			if subval, ok := val.([]interface{}); ok {
				var err error
				var subvalp *[]interface{}
				if subvalp, err = evalProjectionArray(ctx, pf, subval, &fieldType.Values, rbr); err != nil {
					return nil, fmt.Errorf("%s: error applying projection on array item #%d: %v", pf.Name, i, err)
				}
				var v interface{}
				if v, err = resolveFieldHandler(ctx, pf, def, *subvalp); err != nil {
					return nil, fmt.Errorf("%s: error resolving field handler on array: %v", name, err)
				}
				res = append(res, v)
			} else {
				return nil, fmt.Errorf("%s. is not an array", pf.Name)
			}
		}
	case schema.FieldGetter:
		for _, val := range payload {
			subval, ok := val.(map[string]interface{})
			if !ok {
				return nil, fmt.Errorf("%s: invalid value: not a dict/object", pf.Name)
			}
			var err error
			if subval, err = evalProjection(ctx, pf.Children, subval, fieldType, rbr); err != nil {
				return nil, fmt.Errorf("%s.%v", pf.Name, err)
			}
			var v interface{}
			if v, err = resolveFieldHandler(ctx, pf, def, subval); err != nil {
				return nil, err
			}
			res = append(res, v)
		}
	default:
		return nil, fmt.Errorf("%s. unknown field type", pf.Name)
	}

	return resp, nil
}

func evalProjection(ctx context.Context, p Projection, payload map[string]interface{}, fg schema.FieldGetter, rbr *referenceBatchResolver) (map[string]interface{}, error) {
	res := map[string]interface{}{}
	resMu := sync.Mutex{}
	var err error
	p, err = prepareProjection(p, payload)
	if err != nil {
		return nil, err
	}
	for i := range p {
		pf := p[i]
		name := pf.Name
		// Handle aliasing
		if pf.Alias != "" {
			name = pf.Alias
		}
		def := fg.GetField(pf.Name)
		// Skip hidden fields
		if def != nil && def.Hidden {
			continue
		}
		if val, found := payload[pf.Name]; found {
			// Handle sub field selection (if field has a value)
			if len(pf.Children) > 0 && val != nil {
				if def != nil && def.Schema != nil {
					subval, ok := val.(map[string]interface{})
					if !ok {
						return nil, fmt.Errorf("%s: invalid value: not a dict", pf.Name)
					}
					var err error
					if subval, err = evalProjection(ctx, pf.Children, subval, def.Schema, rbr); err != nil {
						return nil, fmt.Errorf("%s.%v", pf.Name, err)
					}
					if res[name], err = resolveFieldHandler(ctx, pf, def, subval); err != nil {
						return nil, err
					}
				} else if ref, ok := def.Validator.(*schema.Reference); ok {
					// Execute sub-request in batch
					q := &Query{
						Projection: pf.Children,
						Predicate:  Predicate{&Equal{Field: "id", Value: val}},
					}
					rbr.request(ref.Path, q, func(payloads []map[string]interface{}, validator schema.Validator) error {
						var v interface{}
						if len(payloads) == 1 {
							payload, err := evalProjection(ctx, pf.Children, payloads[0], validator, rbr)
							if err != nil {
								return fmt.Errorf("%s: error applying Projection on sub-field: %v", name, err)
							}
							if v, err = resolveFieldHandler(ctx, pf, def, payload); err != nil {
								return fmt.Errorf("%s: error resolving field handler on sub-field: %v", name, err)
							}
						}
						// Return `null` for missing result instead of empty object
						if m, ok := v.(map[string]interface{}); ok && len(m) == 0 {
							v = nil
						}
						resMu.Lock()
						res[name] = v
						resMu.Unlock()
						return nil
					})
				} else if array, ok := def.Validator.(*schema.Array); ok {
					if payload, ok := val.([]interface{}); ok {
						var err error
						var subvalp *[]interface{}
						if subvalp, err = evalProjectionArray(ctx, pf, payload, &array.Values, rbr); err != nil {
							return nil, fmt.Errorf("%s: error applying projection on array item #%d: %v", pf.Name, i, err)
						}
						if res[name], err = resolveFieldHandler(ctx, pf, &array.Values, subvalp); err != nil {
							return nil, fmt.Errorf("%s: error resolving field handler on array: %v", name, err)
						}
					} else {
						return nil, fmt.Errorf("%s: invalid value: not an array", pf.Name)
					}
				} else if fg, ok := def.Validator.(schema.FieldGetter); ok {
					subval, ok := val.(map[string]interface{})
					if !ok {
						return nil, fmt.Errorf("%s: invalid value: not a dict", pf.Name)
					}
					var err error
					if subval, err = evalProjection(ctx, pf.Children, subval, fg, rbr); err != nil {
						return nil, fmt.Errorf("%s.%v", pf.Name, err)
					}
					if res[name], err = resolveFieldHandler(ctx, pf, def, subval); err != nil {
						return nil, err
					}
				} else {
					return nil, fmt.Errorf("%s: field has no children", pf.Name)
				}
			} else {
				var err error
				if res[name], err = resolveFieldHandler(ctx, pf, def, val); err != nil {
					return nil, err
				}
			}
		} else if def != nil {
			// If field is not found, it may be a connection
			if ref, ok := def.Validator.(*schema.Connection); ok {
				id, ok := payload["id"]
				if !ok {
					return nil, fmt.Errorf("%s: error applying projection on sub-resource: item lacks ID field", pf.Name)
				}
				q, err := connectionQuery(pf, ref.Field, id, ref.Validator)
				if err != nil {
					return nil, err
				}
				rbr.request(ref.Path, q, func(payloads []map[string]interface{}, validator schema.Validator) (err error) {
					for i := range payloads {
						if payloads[i], err = evalProjection(ctx, pf.Children, payloads[i], validator, rbr); err != nil {
							return fmt.Errorf("%s: error applying projection on sub-resource item #%d: %v", pf.Name, i, err)
						}
					}
					var v interface{}
					if v, err = resolveFieldHandler(ctx, pf, def, payloads); err != nil {
						return fmt.Errorf("%s: error resolving field handler on sub-resource: %v", name, err)
					}
					resMu.Lock()
					res[name] = v
					resMu.Unlock()
					return nil
				})
			}
		}
	}
	return res, nil
}

// connectionQuery builds a query from a projection field on a schema.Connection type field.
func connectionQuery(pf ProjectionField, field string, id interface{}, validator schema.Validator) (*Query, error) {
	q := &Query{
		Projection: pf.Children,
		Predicate:  Predicate{&Equal{Field: field, Value: id}},
	}
	if filter, ok := pf.Params["filter"].(string); ok {
		p, err := ParsePredicate(filter)
		if err != nil {
			return nil, fmt.Errorf("%s: invalid filter: %v", pf.Name, err)
		}
		err = p.Prepare(validator)
		if err != nil {
			return nil, fmt.Errorf("%s: invalid filter: %v", pf.Name, err)
		}
		q.Predicate = append(q.Predicate, p...)
	}
	if sort, ok := pf.Params["sort"].(string); ok {
		s, err := ParseSort(sort)
		if err != nil {
			return nil, fmt.Errorf("%s: invalid sort: %v", pf.Name, err)
		}
		q.Sort = s
	}
	skip := 0
	if v, ok := pf.Params["skip"].(int); ok {
		skip = v
	}
	page := 1
	if v, ok := pf.Params["page"].(int); ok {
		page = v
	}
	limit := 20
	if v, ok := pf.Params["limit"].(int); ok {
		limit = v
	}
	q.Window = Page(page, limit, skip)
	return q, nil
}

// resolveFieldHandler calls the field handler with the provided params (if any).
func resolveFieldHandler(ctx context.Context, pf ProjectionField, def *schema.Field, val interface{}) (interface{}, error) {
	if def == nil {
		return val, nil
	}
	var err error
	if def.Handler != nil && len(pf.Params) > 0 {
		val, err = def.Handler(ctx, val, pf.Params)
		if err != nil {
			return nil, fmt.Errorf("%s: %v", pf.Name, err)
		}
	}
	if s, ok := def.Validator.(schema.FieldSerializer); ok {
		val, err = s.Serialize(val)
		if err != nil {
			return nil, fmt.Errorf("%s: %v", pf.Name, err)
		}
	}
	return val, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"fmt"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/util/uuid"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
)

var _ = framework.KubeDescribe("Secrets", func() {
	f := framework.NewDefaultFramework("secrets")

	It("should be consumable from pods in volume [Conformance]", func() {
		name := "secret-test-" + string(uuid.NewUUID())
		volumeName := "secret-volume"
		volumeMountPath := "/etc/secret-volume"
		secret := secretForTest(f.Namespace.Name, name)

		By(fmt.Sprintf("Creating secret with name %s", secret.Name))
		defer func() {
			By("Cleaning up the secret")
			if err := f.Client.Secrets(f.Namespace.Name).Delete(secret.Name); err != nil {
				framework.Failf("unable to delete secret %v: %v", secret.Name, err)
			}
		}()
		var err error
		if secret, err = f.Client.Secrets(f.Namespace.Name).Create(secret); err != nil {
			framework.Failf("unable to create test secret %s: %v", secret.Name, err)
		}

		pod := &api.Pod{
			ObjectMeta: api.ObjectMeta{
				Name: "pod-secrets-" + string(uuid.NewUUID()),
			},
			Spec: api.PodSpec{
				Volumes: []api.Volume{
					{
						Name: volumeName,
						VolumeSource: api.VolumeSource{
							Secret: &api.SecretVolumeSource{
								SecretName: name,
							},
						},
					},
				},
				Containers: []api.Container{
					{
						Name:  "secret-volume-test",
						Image: "gcr.io/google_containers/mounttest:0.7",
						Args: []string{
							"--file_content=/etc/secret-volume/data-1",
							"--file_mode=/etc/secret-volume/data-1"},
						VolumeMounts: []api.VolumeMount{
							{
								Name:      volumeName,
								MountPath: volumeMountPath,
							},
						},
					},
				},
				RestartPolicy: api.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume secrets", pod, 0, []string{
			"content of file \"/etc/secret-volume/data-1\": value-1",
			"mode of file \"/etc/secret-volume/data-1\": -rw-r--r--",
		})
	})

	It("should be consumable from pods in volume with defaultMode set [Conformance]", func() {
		name := "secret-test-defaultmode-" + string(uuid.NewUUID())
		volumeName := "secret-volume"
		volumeMountPath := "/etc/secret-volume"
		secret := secretForTest(f.Namespace.Name, name)

		By(fmt.Sprintf("Creating secret with name %s", secret.Name))
		defer func() {
			By("Cleaning up the secret")
			if err := f.Client.Secrets(f.Namespace.Name).Delete(secret.Name); err != nil {
				framework.Failf("unable to delete secret %v: %v", secret.Name, err)
			}
		}()
		var err error
		if secret, err = f.Client.Secrets(f.Namespace.Name).Create(secret); err != nil {
			framework.Failf("unable to create test secret %s: %v", secret.Name, err)
		}

		defaultMode := int32(0400)
		pod := &api.Pod{
			ObjectMeta: api.ObjectMeta{
				Name: "pod-secrets-" + string(uuid.NewUUID()),
			},
			Spec: api.PodSpec{
				Volumes: []api.Volume{
					{
						Name: volumeName,
						VolumeSource: api.VolumeSource{
							Secret: &api.SecretVolumeSource{
								SecretName:  name,
								DefaultMode: &defaultMode,
							},
						},
					},
				},
				Containers: []api.Container{
					{
						Name:  "secret-volume-test",
						Image: "gcr.io/google_containers/mounttest:0.7",
						Args: []string{
							"--file_content=/etc/secret-volume/data-1",
							"--file_mode=/etc/secret-volume/data-1"},
						VolumeMounts: []api.VolumeMount{
							{
								Name:      volumeName,
								MountPath: volumeMountPath,
								ReadOnly:  true,
							},
						},
					},
				},
				RestartPolicy: api.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume secrets", pod, 0, []string{
			"content of file \"/etc/secret-volume/data-1\": value-1",
			"mode of file \"/etc/secret-volume/data-1\": -r--------",
		})
	})

	It("should be consumable from pods in volume with Mode set in the item [Conformance]", func() {
		name := "secret-test-itemmode-" + string(uuid.NewUUID())
		volumeName := "secret-volume"
		volumeMountPath := "/etc/secret-volume"
		secret := secretForTest(f.Namespace.Name, name)

		By(fmt.Sprintf("Creating secret with name %s", secret.Name))
		defer func() {
			By("Cleaning up the secret")
			if err := f.Client.Secrets(f.Namespace.Name).Delete(secret.Name); err != nil {
				framework.Failf("unable to delete secret %v: %v", secret.Name, err)
			}
		}()
		var err error
		if secret, err = f.Client.Secrets(f.Namespace.Name).Create(secret); err != nil {
			framework.Failf("unable to create test secret %s: %v", secret.Name, err)
		}

		mode := int32(0400)
		pod := &api.Pod{
			ObjectMeta: api.ObjectMeta{
				Name: "pod-secrets-" + string(uuid.NewUUID()),
			},
			Spec: api.PodSpec{
				Volumes: []api.Volume{
					{
						Name: volumeName,
						VolumeSource: api.VolumeSource{
							Secret: &api.SecretVolumeSource{
								SecretName: name,
								Items: []api.KeyToPath{
									{
										Key:  "data-1",
										Path: "data-1",
										Mode: &mode,
									},
								},
							},
						},
					},
				},
				Containers: []api.Container{
					{
						Name:  "secret-volume-test",
						Image: "gcr.io/google_containers/mounttest:0.7",
						Args: []string{
							"--file_content=/etc/secret-volume/data-1",
							"--file_mode=/etc/secret-volume/data-1"},
						VolumeMounts: []api.VolumeMount{
							{
								Name:      volumeName,
								MountPath: volumeMountPath,
							},
						},
					},
				},
				RestartPolicy: api.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume secrets", pod, 0, []string{
			"content of file \"/etc/secret-volume/data-1\": value-1",
			"mode of file \"/etc/secret-volume/data-1\": -r--------",
		})
	})

	It("should be consumable in multiple volumes in a pod [Conformance]", func() {
		// This test ensures that the same secret can be mounted in multiple
		// volumes in the same pod.  This test case exists to prevent
		// regressions that break this use-case.
		var (
			name             = "secret-test-" + string(uuid.NewUUID())
			volumeName       = "secret-volume"
			volumeMountPath  = "/etc/secret-volume"
			volumeName2      = "secret-volume-2"
			volumeMountPath2 = "/etc/secret-volume-2"
			secret           = secretForTest(f.Namespace.Name, name)
		)

		By(fmt.Sprintf("Creating secret with name %s", secret.Name))
		defer func() {
			By("Cleaning up the secret")
			if err := f.Client.Secrets(f.Namespace.Name).Delete(secret.Name); err != nil {
				framework.Failf("unable to delete secret %v: %v", secret.Name, err)
			}
		}()
		var err error
		if secret, err = f.Client.Secrets(f.Namespace.Name).Create(secret); err != nil {
			framework.Failf("unable to create test secret %s: %v", secret.Name, err)
		}

		pod := &api.Pod{
			ObjectMeta: api.ObjectMeta{
				Name: "pod-secrets-" + string(uuid.NewUUID()),
			},
			Spec: api.PodSpec{
				Volumes: []api.Volume{
					{
						Name: volumeName,
						VolumeSource: api.VolumeSource{
							Secret: &api.SecretVolumeSource{
								SecretName: name,
							},
						},
					},
					{
						Name: volumeName2,
						VolumeSource: api.VolumeSource{
							Secret: &api.SecretVolumeSource{
								SecretName: name,
							},
						},
					},
				},
				Containers: []api.Container{
					{
						Name:  "secret-volume-test",
						Image: "gcr.io/google_containers/mounttest:0.7",
						Args: []string{
							"--file_content=/etc/secret-volume/data-1",
							"--file_mode=/etc/secret-volume/data-1"},
						VolumeMounts: []api.VolumeMount{
							{
								Name:      volumeName,
								MountPath: volumeMountPath,
								ReadOnly:  true,
							},
							{
								Name:      volumeName2,
								MountPath: volumeMountPath2,
								ReadOnly:  true,
							},
						},
					},
				},
				RestartPolicy: api.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume secrets", pod, 0, []string{
			"content of file \"/etc/secret-volume/data-1\": value-1",
			"mode of file \"/etc/secret-volume/data-1\": -rw-r--r--",
		})
	})

	It("should be consumable from pods in env vars [Conformance]", func() {
		name := "secret-test-" + string(uuid.NewUUID())
		secret := secretForTest(f.Namespace.Name, name)

		By(fmt.Sprintf("Creating secret with name %s", secret.Name))
		defer func() {
			By("Cleaning up the secret")
			if err := f.Client.Secrets(f.Namespace.Name).Delete(secret.Name); err != nil {
				framework.Failf("unable to delete secret %v: %v", secret.Name, err)
			}
		}()
		var err error
		if secret, err = f.Client.Secrets(f.Namespace.Name).Create(secret); err != nil {
			framework.Failf("unable to create test secret %s: %v", secret.Name, err)
		}

		pod := &api.Pod{
			ObjectMeta: api.ObjectMeta{
				Name: "pod-secrets-" + string(uuid.NewUUID()),
			},
			Spec: api.PodSpec{
				Containers: []api.Container{
					{
						Name:    "secret-env-test",
						Image:   "gcr.io/google_containers/busybox:1.24",
						Command: []string{"sh", "-c", "env"},
						Env: []api.EnvVar{
							{
								Name: "SECRET_DATA",
								ValueFrom: &api.EnvVarSource{
									SecretKeyRef: &api.SecretKeySelector{
										LocalObjectReference: api.LocalObjectReference{
											Name: name,
										},
										Key: "data-1",
									},
								},
							},
						},
					},
				},
				RestartPolicy: api.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume secrets", pod, 0, []string{
			"SECRET_DATA=value-1",
		})
	})
})

func secretForTest(namespace, name string) *api.Secret {
	return &api.Secret{
		ObjectMeta: api.ObjectMeta{
			Namespace: namespace,
			Name:      name,
		},
		Data: map[string][]byte{
			"data-1": []byte("value-1\n"),
			"data-2": []byte("value-2\n"),
			"data-3": []byte("value-3\n"),
		},
	}
}
<commit_msg>Remove duplicated code in secret e2e tests<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"fmt"
	"os"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/util/uuid"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
)

var _ = framework.KubeDescribe("Secrets", func() {
	f := framework.NewDefaultFramework("secrets")

	It("should be consumable from pods in volume [Conformance]", func() {
		doSecretE2E(f, nil)
	})

	It("should be consumable from pods in volume with defaultMode set [Conformance]", func() {
		defaultMode := int32(0400)
		doSecretE2E(f, &defaultMode)
	})

	It("should be consumable from pods in volume with Mode set in the item [Conformance]", func() {
		name := "secret-test-itemmode-" + string(uuid.NewUUID())
		volumeName := "secret-volume"
		volumeMountPath := "/etc/secret-volume"
		secret := secretForTest(f.Namespace.Name, name)

		By(fmt.Sprintf("Creating secret with name %s", secret.Name))
		defer func() {
			By("Cleaning up the secret")
			if err := f.Client.Secrets(f.Namespace.Name).Delete(secret.Name); err != nil {
				framework.Failf("unable to delete secret %v: %v", secret.Name, err)
			}
		}()
		var err error
		if secret, err = f.Client.Secrets(f.Namespace.Name).Create(secret); err != nil {
			framework.Failf("unable to create test secret %s: %v", secret.Name, err)
		}

		mode := int32(0400)
		pod := &api.Pod{
			ObjectMeta: api.ObjectMeta{
				Name: "pod-secrets-" + string(uuid.NewUUID()),
			},
			Spec: api.PodSpec{
				Volumes: []api.Volume{
					{
						Name: volumeName,
						VolumeSource: api.VolumeSource{
							Secret: &api.SecretVolumeSource{
								SecretName: name,
								Items: []api.KeyToPath{
									{
										Key:  "data-1",
										Path: "data-1",
										Mode: &mode,
									},
								},
							},
						},
					},
				},
				Containers: []api.Container{
					{
						Name:  "secret-volume-test",
						Image: "gcr.io/google_containers/mounttest:0.7",
						Args: []string{
							"--file_content=/etc/secret-volume/data-1",
							"--file_mode=/etc/secret-volume/data-1"},
						VolumeMounts: []api.VolumeMount{
							{
								Name:      volumeName,
								MountPath: volumeMountPath,
							},
						},
					},
				},
				RestartPolicy: api.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume secrets", pod, 0, []string{
			"content of file \"/etc/secret-volume/data-1\": value-1",
			"mode of file \"/etc/secret-volume/data-1\": -r--------",
		})
	})

	It("should be consumable in multiple volumes in a pod [Conformance]", func() {
		// This test ensures that the same secret can be mounted in multiple
		// volumes in the same pod.  This test case exists to prevent
		// regressions that break this use-case.
		var (
			name             = "secret-test-" + string(uuid.NewUUID())
			volumeName       = "secret-volume"
			volumeMountPath  = "/etc/secret-volume"
			volumeName2      = "secret-volume-2"
			volumeMountPath2 = "/etc/secret-volume-2"
			secret           = secretForTest(f.Namespace.Name, name)
		)

		By(fmt.Sprintf("Creating secret with name %s", secret.Name))
		defer func() {
			By("Cleaning up the secret")
			if err := f.Client.Secrets(f.Namespace.Name).Delete(secret.Name); err != nil {
				framework.Failf("unable to delete secret %v: %v", secret.Name, err)
			}
		}()
		var err error
		if secret, err = f.Client.Secrets(f.Namespace.Name).Create(secret); err != nil {
			framework.Failf("unable to create test secret %s: %v", secret.Name, err)
		}

		pod := &api.Pod{
			ObjectMeta: api.ObjectMeta{
				Name: "pod-secrets-" + string(uuid.NewUUID()),
			},
			Spec: api.PodSpec{
				Volumes: []api.Volume{
					{
						Name: volumeName,
						VolumeSource: api.VolumeSource{
							Secret: &api.SecretVolumeSource{
								SecretName: name,
							},
						},
					},
					{
						Name: volumeName2,
						VolumeSource: api.VolumeSource{
							Secret: &api.SecretVolumeSource{
								SecretName: name,
							},
						},
					},
				},
				Containers: []api.Container{
					{
						Name:  "secret-volume-test",
						Image: "gcr.io/google_containers/mounttest:0.7",
						Args: []string{
							"--file_content=/etc/secret-volume/data-1",
							"--file_mode=/etc/secret-volume/data-1"},
						VolumeMounts: []api.VolumeMount{
							{
								Name:      volumeName,
								MountPath: volumeMountPath,
								ReadOnly:  true,
							},
							{
								Name:      volumeName2,
								MountPath: volumeMountPath2,
								ReadOnly:  true,
							},
						},
					},
				},
				RestartPolicy: api.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume secrets", pod, 0, []string{
			"content of file \"/etc/secret-volume/data-1\": value-1",
			"mode of file \"/etc/secret-volume/data-1\": -rw-r--r--",
		})
	})

	It("should be consumable from pods in env vars [Conformance]", func() {
		name := "secret-test-" + string(uuid.NewUUID())
		secret := secretForTest(f.Namespace.Name, name)

		By(fmt.Sprintf("Creating secret with name %s", secret.Name))
		defer func() {
			By("Cleaning up the secret")
			if err := f.Client.Secrets(f.Namespace.Name).Delete(secret.Name); err != nil {
				framework.Failf("unable to delete secret %v: %v", secret.Name, err)
			}
		}()
		var err error
		if secret, err = f.Client.Secrets(f.Namespace.Name).Create(secret); err != nil {
			framework.Failf("unable to create test secret %s: %v", secret.Name, err)
		}

		pod := &api.Pod{
			ObjectMeta: api.ObjectMeta{
				Name: "pod-secrets-" + string(uuid.NewUUID()),
			},
			Spec: api.PodSpec{
				Containers: []api.Container{
					{
						Name:    "secret-env-test",
						Image:   "gcr.io/google_containers/busybox:1.24",
						Command: []string{"sh", "-c", "env"},
						Env: []api.EnvVar{
							{
								Name: "SECRET_DATA",
								ValueFrom: &api.EnvVarSource{
									SecretKeyRef: &api.SecretKeySelector{
										LocalObjectReference: api.LocalObjectReference{
											Name: name,
										},
										Key: "data-1",
									},
								},
							},
						},
					},
				},
				RestartPolicy: api.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume secrets", pod, 0, []string{
			"SECRET_DATA=value-1",
		})
	})
})

func secretForTest(namespace, name string) *api.Secret {
	return &api.Secret{
		ObjectMeta: api.ObjectMeta{
			Namespace: namespace,
			Name:      name,
		},
		Data: map[string][]byte{
			"data-1": []byte("value-1\n"),
			"data-2": []byte("value-2\n"),
			"data-3": []byte("value-3\n"),
		},
	}
}

func doSecretE2E(f *framework.Framework, defaultMode *int32) {
	var (
		name            = "secret-test-" + string(uuid.NewUUID())
		volumeName      = "secret-volume"
		volumeMountPath = "/etc/secret-volume"
		secret          = secretForTest(f.Namespace.Name, name)
	)

	By(fmt.Sprintf("Creating secret with name %s", secret.Name))
	defer func() {
		By("Cleaning up the secret")
		if err := f.Client.Secrets(f.Namespace.Name).Delete(secret.Name); err != nil {
			framework.Failf("unable to delete secret %v: %v", secret.Name, err)
		}
	}()
	var err error
	if secret, err = f.Client.Secrets(f.Namespace.Name).Create(secret); err != nil {
		framework.Failf("unable to create test secret %s: %v", secret.Name, err)
	}

	pod := &api.Pod{
		ObjectMeta: api.ObjectMeta{
			Name: "pod-secrets-" + string(uuid.NewUUID()),
		},
		Spec: api.PodSpec{
			Volumes: []api.Volume{
				{
					Name: volumeName,
					VolumeSource: api.VolumeSource{
						Secret: &api.SecretVolumeSource{
							SecretName: name,
						},
					},
				},
			},
			Containers: []api.Container{
				{
					Name:  "secret-volume-test",
					Image: "gcr.io/google_containers/mounttest:0.7",
					Args: []string{
						"--file_content=/etc/secret-volume/data-1",
						"--file_mode=/etc/secret-volume/data-1"},
					VolumeMounts: []api.VolumeMount{
						{
							Name:      volumeName,
							MountPath: volumeMountPath,
						},
					},
				},
			},
			RestartPolicy: api.RestartPolicyNever,
		},
	}

	if defaultMode != nil {
		pod.Spec.Volumes[0].VolumeSource.Secret.DefaultMode = defaultMode
	} else {
		mode := int32(0644)
		defaultMode = &mode
	}

	modeString := fmt.Sprintf("%v", os.FileMode(*defaultMode))
	expectedOutput := []string{
		"content of file \"/etc/secret-volume/data-1\": value-1",
		"mode of file \"/etc/secret-volume/data-1\": " + modeString,
	}

	f.TestContainerOutput("consume secrets", pod, 0, expectedOutput)
}
<|endoftext|>


<commit_before><commit_msg>update example.<commit_after><|endoftext|>


<commit_before>package main

import (
    "fmt"
    "os"
    "time"
    "labix.org/v2/mgo"
    "labix.org/v2/mgo/bson"
    "strconv"
)

type FindType string
const (
    Recently FindType = "-last_access"
    Frequently FindType = "-count"
)

type Log struct {
    Id    bson.ObjectId `bson:"_id"`
    Path  string        `bson:"path"`
    Last  time.Time     `bson:"last_access"`
    Count float32       `bson:"count"`
}

type Target struct {
  index int
  regex string
}

func LogCurrent(c *mgo.Collection){
    path := os.Getenv("PWD")
    if os.Getenv("HOME") != path {
        c.Upsert(bson.M{"path": path}, bson.M{"$set": bson.M{"path": path, "last_access": time.Now()}, "$inc":bson.M{"count":1}})
    }
}

func RecentlyFrequently(c *mgo.Collection, sort FindType){
    target := getTarget()
    if (target.index >= 0 && "" == target.regex) {
        printTarget(c,sort,target)
    } else if (target.index < 0 && "" != target.regex) {
      target.index = 0
        printTarget(c,sort,target)
    } else {
        listRecentyFrequently(c,sort)
    }
}

func printTarget(c *mgo.Collection, sort FindType, target Target) {
    item := &Log{}
    if "" == target.regex {
      c.Find(nil).Sort(string(sort)).Skip(target.index).One(item)
      } else {
      c.Find( bson.M{"path": bson.M{"$regex": target.regex} } ).Sort(string(sort)).Skip(target.index).One(item)
      }
    fmt.Println(item.Path)
}

func listRecentyFrequently(c *mgo.Collection, sort FindType) {
    iter := c.Find(nil).Sort(string(sort)).Limit(20).Iter()
    item := &Log{}
    i := 0
    fmt.Printf("usage: %s target_index|path regex\n",os.Args[0])
    for iter.Next(&item) {
        fmt.Printf("%2d- %s\n",i,item.Path)
        i+=1
    }
    if err := iter.Close(); err != nil {
        fmt.Fprintf(os.Stderr,"Error closing iter: %v\n",err)
    }
}

func getTarget() Target {
    target := Target{index:-1, regex:""}
    if len(os.Args) > 1 {
        var err error
        target.index, err = strconv.Atoi(os.Args[1])
        if nil != err {
          target.index = -1
          target.regex = os.Args[1]
        }
    }
    return target
}

func DampenFrequency(c *mgo.Collection) {
    iter := c.Find(nil).Iter()
    item := &Log{}
    for iter.Next(&item) {
        item.Count /= 2.0
        c.Update(bson.M{"_id": item.Id},item)
    }
    if err := iter.Close(); err != nil {
        fmt.Fprintf(os.Stderr,"Error closing iter: %v\n",err)
    }
}

func RemoveDead(c *mgo.Collection) {
    count, err := c.Count()
    if nil != err {
        fmt.Fprintf(os.Stderr,"Can't get collection count: %v\n",err)
        return
    }
    limit := float32(count) * 0.1
    if limit < 20 {
        return
    }
    iter := c.Find(bson.M{"count": bson.M{"$lte": 0.25}}).Sort("last_access").Limit(int(limit)).Iter()
    item := &Log{}
    for iter.Next(&item) {
        fmt.Printf("Forgetting about %s\n",item.Path)
        if err := c.Remove(bson.M{"_id": item.Id}); nil != err {
            fmt.Fprintf(os.Stderr,"Error removing item (%v) iter: %v\n",item.Id,err)
        }
    }
    if err := iter.Close(); err != nil {
        fmt.Fprintf(os.Stderr,"Error closing iter: %v\n",err)
    }
}

func main () {
    uri := os.Getenv("CWD_LOGGER_URI")
    if "" == uri {
        // mongodb://user:pass@server.mongohq.com/db_name
        uri = "mongodb://localhost/" + os.Getenv("USER")
    }
    sess, err := mgo.DialWithTimeout(uri,500*time.Millisecond)
    if nil != err {
        fmt.Fprintf(os.Stderr,"Can't connect to mongo: %v\n", err)
        os.Exit(1)
    }
    defer sess.Close()

    collection := sess.DB("").C("logged_dirs")

    switch os.Args[0] {
    case "cwd_logger", "log_cwd", "go_cwd_logger":
        LogCurrent(collection)
    case "cwd_recently":
        RecentlyFrequently(collection, Recently)
    case "cwd_frequency":
        RecentlyFrequently(collection, Frequently)
    case "cwd_dampen_frequency":
        DampenFrequency(collection)
        RemoveDead(collection)
    default:
        LogCurrent(collection)
    }
}
<commit_msg>gofmt, I should add it to my save operation automatically<commit_after>package main

import (
	"fmt"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"os"
	"strconv"
	"time"
)

type FindType string

const (
	Recently   FindType = "-last_access"
	Frequently FindType = "-count"
)

type Log struct {
	Id    bson.ObjectId `bson:"_id"`
	Path  string        `bson:"path"`
	Last  time.Time     `bson:"last_access"`
	Count float32       `bson:"count"`
}

type Target struct {
	index int
	regex string
}

func LogCurrent(c *mgo.Collection) {
	path := os.Getenv("PWD")
	if os.Getenv("HOME") != path {
		c.Upsert(bson.M{"path": path}, bson.M{"$set": bson.M{"path": path, "last_access": time.Now()}, "$inc": bson.M{"count": 1}})
	}
}

func RecentlyFrequently(c *mgo.Collection, sort FindType) {
	target := getTarget()
	if target.index >= 0 && "" == target.regex {
		printTarget(c, sort, target)
	} else if target.index < 0 && "" != target.regex {
		target.index = 0
		printTarget(c, sort, target)
	} else {
		listRecentyFrequently(c, sort)
	}
}

func printTarget(c *mgo.Collection, sort FindType, target Target) {
	item := &Log{}
	if "" == target.regex {
		c.Find(nil).Sort(string(sort)).Skip(target.index).One(item)
	} else {
		c.Find(bson.M{"path": bson.M{"$regex": target.regex}}).Sort(string(sort)).Skip(target.index).One(item)
	}
	fmt.Println(item.Path)
}

func listRecentyFrequently(c *mgo.Collection, sort FindType) {
	iter := c.Find(nil).Sort(string(sort)).Limit(20).Iter()
	item := &Log{}
	i := 0
	fmt.Printf("usage: %s target_index|path regex\n", os.Args[0])
	for iter.Next(&item) {
		fmt.Printf("%2d- %s\n", i, item.Path)
		i += 1
	}
	if err := iter.Close(); err != nil {
		fmt.Fprintf(os.Stderr, "Error closing iter: %v\n", err)
	}
}

func getTarget() Target {
	target := Target{index: -1, regex: ""}
	if len(os.Args) > 1 {
		var err error
		target.index, err = strconv.Atoi(os.Args[1])
		if nil != err {
			target.index = -1
			target.regex = os.Args[1]
		}
	}
	return target
}

func DampenFrequency(c *mgo.Collection) {
	iter := c.Find(nil).Iter()
	item := &Log{}
	for iter.Next(&item) {
		item.Count /= 2.0
		c.Update(bson.M{"_id": item.Id}, item)
	}
	if err := iter.Close(); err != nil {
		fmt.Fprintf(os.Stderr, "Error closing iter: %v\n", err)
	}
}

func RemoveDead(c *mgo.Collection) {
	count, err := c.Count()
	if nil != err {
		fmt.Fprintf(os.Stderr, "Can't get collection count: %v\n", err)
		return
	}
	limit := float32(count) * 0.1
	if limit < 20 {
		return
	}
	iter := c.Find(bson.M{"count": bson.M{"$lte": 0.25}}).Sort("last_access").Limit(int(limit)).Iter()
	item := &Log{}
	for iter.Next(&item) {
		fmt.Printf("Forgetting about %s\n", item.Path)
		if err := c.Remove(bson.M{"_id": item.Id}); nil != err {
			fmt.Fprintf(os.Stderr, "Error removing item (%v) iter: %v\n", item.Id, err)
		}
	}
	if err := iter.Close(); err != nil {
		fmt.Fprintf(os.Stderr, "Error closing iter: %v\n", err)
	}
}

func main() {
	uri := os.Getenv("CWD_LOGGER_URI")
	if "" == uri {
		// mongodb://user:pass@server.mongohq.com/db_name
		uri = "mongodb://localhost/" + os.Getenv("USER")
	}
	sess, err := mgo.DialWithTimeout(uri, 500*time.Millisecond)
	if nil != err {
		fmt.Fprintf(os.Stderr, "Can't connect to mongo: %v\n", err)
		os.Exit(1)
	}
	defer sess.Close()

	collection := sess.DB("").C("logged_dirs")

	switch os.Args[0] {
	case "cwd_logger", "log_cwd", "go_cwd_logger":
		LogCurrent(collection)
	case "cwd_recently":
		RecentlyFrequently(collection, Recently)
	case "cwd_frequency":
		RecentlyFrequently(collection, Frequently)
	case "cwd_dampen_frequency":
		DampenFrequency(collection)
		RemoveDead(collection)
	default:
		LogCurrent(collection)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package memfs

import (
	"fmt"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
)

type memFS struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock timeutil.Clock

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// The collection of all inodes that have ever been created, indexed by inode
	// ID. Some inodes are not in use if they have been unlinked, and no inode
	// with ID less than fuse.RootInodeID is ever used.
	//
	// INVARIANT: len(inodes) > fuse.RootInodeID
	// INVARIANT: For all i < fuse.RootInodeID, inodes[i].impl == nil
	// INVARIANT: inodes[fuse.RootInodeID].impl is of type *memDir
	inodes []inode // GUARDED_BY(mu)

	// A list of inode IDs within inodes available for reuse, not including the
	// reserved IDs less than fuse.RootInodeID.
	//
	// INVARIANT: This is all and only indices i of inodes such that i >
	// fuse.RootInodeID and inodes[i].impl == nil
	freeInodes []fuse.InodeID // GUARDED_BY(mu)
}

// Create a file system that stores data and metadata in memory.
func NewMemFS(
	clock timeutil.Clock) (fs fuse.FileSystem) {
	fs = &memFS{
		clock: clock,
	}

	fs.(*memFS).mu = syncutil.NewInvariantMutex(fs.(*memFS).checkInvariants)
	return
}

func (fs *memFS) checkInvariants() {
	panic("TODO")
}

func (fs *memFS) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}
	return
}

func (fs *memFS) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// We don't mutate spontaneosuly, so if the VFS layer has asked for an
	// inode that doesn't exist, something screwed up earlier (a lookup, a
	// cache invalidation, etc.).
	if req.Inode >= fuse.InodeID(len(fs.inodes)) {
		panic(fmt.Sprintf("Inode out of range: %v vs. %v", req.Inode, len(fs.inodes)))
	}

	var inode *inode = &fs.inodes[req.Inode]
	if inode.impl == nil {
		panic(fmt.Sprintf("Dead inode requested: %v", req.Inode))
	}

	// All is good.
	return
}
<commit_msg>Implemented some of memFS.checkInvariants.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package memfs

import (
	"fmt"
	"reflect"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
)

type memFS struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock timeutil.Clock

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// The collection of all inodes that have ever been created, indexed by inode
	// ID. Some inodes are not in use if they have been unlinked, and no inode
	// with ID less than fuse.RootInodeID is ever used.
	//
	// INVARIANT: len(inodes) > fuse.RootInodeID
	// INVARIANT: For all i < fuse.RootInodeID, inodes[i].impl == nil
	// INVARIANT: inodes[fuse.RootInodeID].impl is of type *memDir
	inodes []inode // GUARDED_BY(mu)

	// A list of inode IDs within inodes available for reuse, not including the
	// reserved IDs less than fuse.RootInodeID.
	//
	// INVARIANT: This is all and only indices i of inodes such that i >
	// fuse.RootInodeID and inodes[i].impl == nil
	freeInodes []fuse.InodeID // GUARDED_BY(mu)
}

// Create a file system that stores data and metadata in memory.
func NewMemFS(
	clock timeutil.Clock) (fs fuse.FileSystem) {
	fs = &memFS{
		clock: clock,
	}

	fs.(*memFS).mu = syncutil.NewInvariantMutex(fs.(*memFS).checkInvariants)
	return
}

func (fs *memFS) checkInvariants() {
	// Check reserved inodes.
	for i := 0; i < fuse.RootInodeID; i++ {
		var inode *inode = &fs.inodes[i]
		if inode.impl != nil {
			panic(fmt.Sprintf("Non-nil impl for ID: %v", i))
		}
	}

	// Check the root inode.
	fs.inodes[fuse.RootInodeID].impl.(*dir)

	// Check inodes, building our own set of free IDs.
	freeIDsEncountered := make(map[fuse.InodeID]struct{})
	for i := range fs.inodes {
		var inode *inode = &fs.inodes[i]
		if inode.impl == nil {
			freeIDsEncountered[i] = struct{}{}
			continue
		}

		// Check for known types.
		switch inode.impl.(type) {
		case *memFile:
		case *memDir:
		default:
			panic(fmt.Sprintf("Unknown inode type: %v", reflect.TypeOf(inode.impl)))
		}
	}

	panic("TODO")
}

func (fs *memFS) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}
	return
}

func (fs *memFS) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// We don't mutate spontaneosuly, so if the VFS layer has asked for an
	// inode that doesn't exist, something screwed up earlier (a lookup, a
	// cache invalidation, etc.).
	if req.Inode >= fuse.InodeID(len(fs.inodes)) {
		panic(fmt.Sprintf("Inode out of range: %v vs. %v", req.Inode, len(fs.inodes)))
	}

	var inode *inode = &fs.inodes[req.Inode]
	if inode.impl == nil {
		panic(fmt.Sprintf("Dead inode requested: %v", req.Inode))
	}

	// All is good.
	return
}
<|endoftext|>


<commit_before>package compute

import (
	"encoding/xml"
	"fmt"
	"net/http"
)

// Account represents the details for a compute account.
type Account struct {
	// The XML name for the "Account" data contract
	XMLName xml.Name `xml:"Account"`

	// The compute API user name.
	UserName string `xml:"userName"`

	// The user's full name.
	FullName string `xml:"fullName"`

	// The user's first name.
	FirstName string `xml:"firstName"`

	// The user's last name.
	LastName string `xml:"lastName"`

	// The user's email address.
	EmailAddress string `xml:"emailAddress"`

	// The user's department.
	Department string `xml:"department"`

	// The Id of the user's organisation.
	OrganizationID string `xml:"orgId"`

	// The user's assigned roles.
	AssignedRoles []Role `xml:"roles>role"`
}

// Role represents a role assigned to a compute account.
type Role struct {
	// The XML name for the "Role" data contract
	XMLName xml.Name `xml:"role"`

	// The role name.
	Name string `xml:"name"`
}

// GetAccount retrieves the current user's account information
func (client *Client) GetAccount() (*Account, error) {
	client.stateLock.Lock()
	defer client.stateLock.Unlock()

	if client.account != nil {
		return client.account, nil
	}

	request, err := client.newRequestV1("myaccount", http.MethodGet, nil)
	if err != nil {
		return nil, err
	}

	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	if statusCode == 401 {
		return nil, fmt.Errorf("cannot connect to compute API (invalid credentials)")
	}

	account := &Account{}
	err = xml.Unmarshal(responseBody, account)
	if err != nil {
		return nil, err
	}

	client.account = account

	return account, nil
}
<commit_msg>Log response body if myaccount API returns invalid XML.<commit_after>package compute

import (
	"encoding/xml"
	"fmt"
	"log"
	"net/http"
)

// Account represents the details for a compute account.
type Account struct {
	// The XML name for the "Account" data contract
	XMLName xml.Name `xml:"Account"`

	// The compute API user name.
	UserName string `xml:"userName"`

	// The user's full name.
	FullName string `xml:"fullName"`

	// The user's first name.
	FirstName string `xml:"firstName"`

	// The user's last name.
	LastName string `xml:"lastName"`

	// The user's email address.
	EmailAddress string `xml:"emailAddress"`

	// The user's department.
	Department string `xml:"department"`

	// The Id of the user's organisation.
	OrganizationID string `xml:"orgId"`

	// The user's assigned roles.
	AssignedRoles []Role `xml:"roles>role"`
}

// Role represents a role assigned to a compute account.
type Role struct {
	// The XML name for the "Role" data contract
	XMLName xml.Name `xml:"role"`

	// The role name.
	Name string `xml:"name"`
}

// GetAccount retrieves the current user's account information
func (client *Client) GetAccount() (*Account, error) {
	client.stateLock.Lock()
	defer client.stateLock.Unlock()

	if client.account != nil {
		return client.account, nil
	}

	request, err := client.newRequestV1("myaccount", http.MethodGet, nil)
	if err != nil {
		return nil, err
	}

	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	if statusCode == 401 {
		return nil, fmt.Errorf("cannot connect to compute API (invalid credentials)")
	}

	account := &Account{}
	err = xml.Unmarshal(responseBody, account)
	if err != nil {
		log.Printf("Received invalid XML response from the CloudControl account-details API: %s\n'%s'",
			err,
			string(responseBody),
		)

		return nil, err
	}

	client.account = account

	return account, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 CloudByte, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package daemon

import (
	"fmt"
	"os"
	"os/exec"
	"github.com/Sirupsen/logrus"
	"github.com/openebs/openebs/types"
)

// Vsms returns the list of VSMs to show given the user's filtering.
func (daemon *Daemon) Vsms(config *types.VSMListOptions) ([]*types.Vsm, error) {
	vsms := []*types.Vsm{}

	//TODO - Fetch this data from a registry
	for i := 1; i < 2; i++ {
		vsm := &types.Vsm {
			Name:		"vsm",
			IPAddress:	"10.10.1.1",
			IOPS:		"100",
			Volumes:	"1",
			Status:		"active",
		}
		vsms = append(vsms, vsm)
	}

	logrus.Debugf("Total VSMs %d\n", len(vsms))
	return vsms, nil
}

// This returns the newly created VSM.
func (daemon *Daemon) VsmCreate(opts *types.VSMCreateOptions) (*types.Vsm, error) {

	fmt.Printf("VSM Create at server ...\n")
	fmt.Printf("Provided ip: %s\n", opts.IP)
	fmt.Printf("Provided vsm name: %s\n", opts.Name)
	fmt.Printf("Provided iface: %s\n", opts.Interface)
	fmt.Printf("Provided subnet: %s\n", opts.Subnet)
	fmt.Printf("Provided router: %s\n", opts.Router)
	fmt.Printf("Provided volume name: %s\n", opts.Volume)
	fmt.Printf("Provided mount name: %s\n", opts.Mount)

	name := opts.Name
	ip := opts.IP
	netface := opts.Interface
	subnet := opts.Subnet
	router := opts.Router
	volume := opts.Volume
	mount := opts.Mount

	// This is make based !!!
	// Base path of Makefile
	makefilename := MAKEFILEBASE + "vsm_create"

	// Subcommand i.e. makefile's target name
	subcommand := "create"

	// Non-silent mode to execute make commands
	makeopts := "-sf"

	// Preparing the arguments
	args := []string{makeopts, makefilename,
		subcommand,
		"name=" + name,
		"interface=" + netface,
		"ip=" + ip,
		"subnet=" + subnet,
		"volume=" + volume,
		"router=" + router,
		"mount=" + mount,
		"debug=1"}

	// Preparing the final command
	finalcmd := exec.Command(MAKECMD, args...)

	PrintCommand(finalcmd)

	// We want to see what's going on
	finalcmd.Stdout = os.Stdout
	finalcmd.Stderr = os.Stderr

	// Actual execution
	err := finalcmd.Run()
	if err != nil {
		return nil, err
	}

	vsm := &types.Vsm{
		Name: name,
		IPAddress: netface,
		IOPS: "0",
		Volumes: "1",
		Status: "Active",
	}

	return vsm, err
}
<commit_msg>Iterate through the vsms directory for vsm-ls<commit_after>// Copyright 2016 CloudByte, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package daemon

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/openebs/openebs/types"
	"os"
	"os/exec"
	"path/filepath"
)

var (
	// directory contains the list of VSMs created via OpenEBS
	vsmsDir = "/etc/openebs/.vsms/"
)

// Vsms returns the list of VSMs to show given the user's filtering.
func (daemon *Daemon) Vsms(config *types.VSMListOptions) ([]*types.Vsm, error) {
	vsms := []*types.Vsm{}

	err := filepath.Walk(vsmsDir, func(d string, fileInfo os.FileInfo, err error) error {
		if err != nil {
			if os.IsNotExist(err) && d != vsmsDir {
				return nil
			}
			return err
		}
		if fileInfo == nil || fileInfo.IsDir() {
			return nil
		}
		logrus.Debugf("Found VSM %s\n", fileInfo.Name())

		//TODO - Fetch the details of the VSM from the name
		vsm := &types.Vsm{
			Name:      fileInfo.Name(),
			IPAddress: "10.10.1.1",
			IOPS:      "100",
			Volumes:   "1",
			Status:    "active",
		}
		vsms = append(vsms, vsm)

		return nil
	})

	if err != nil {
		logrus.Errorf("Unable to fetch VSMs from %s\n", vsmsDir)
	}

	logrus.Debugf("Total VSMs %d\n", len(vsms))
	return vsms, nil
}

// This returns the newly created VSM.
func (daemon *Daemon) VsmCreate(opts *types.VSMCreateOptions) (*types.Vsm, error) {

	fmt.Printf("VSM Create at server ...\n")
	fmt.Printf("Provided ip: %s\n", opts.IP)
	fmt.Printf("Provided vsm name: %s\n", opts.Name)
	fmt.Printf("Provided iface: %s\n", opts.Interface)
	fmt.Printf("Provided subnet: %s\n", opts.Subnet)
	fmt.Printf("Provided router: %s\n", opts.Router)
	fmt.Printf("Provided volume name: %s\n", opts.Volume)
	fmt.Printf("Provided mount name: %s\n", opts.Mount)

	name := opts.Name
	ip := opts.IP
	netface := opts.Interface
	subnet := opts.Subnet
	router := opts.Router
	volume := opts.Volume
	mount := opts.Mount

	// This is make based !!!
	// Base path of Makefile
	makefilename := MAKEFILEBASE + "vsm_create"

	// Subcommand i.e. makefile's target name
	subcommand := "create"

	// Non-silent mode to execute make commands
	makeopts := "-sf"

	// Preparing the arguments
	args := []string{makeopts, makefilename,
		subcommand,
		"name=" + name,
		"interface=" + netface,
		"ip=" + ip,
		"subnet=" + subnet,
		"volume=" + volume,
		"router=" + router,
		"mount=" + mount,
		"debug=1"}

	// Preparing the final command
	finalcmd := exec.Command(MAKECMD, args...)

	PrintCommand(finalcmd)

	// We want to see what's going on
	finalcmd.Stdout = os.Stdout
	finalcmd.Stderr = os.Stderr

	// Actual execution
	err := finalcmd.Run()
	if err != nil {
		return nil, err
	}

	vsm := &types.Vsm{
		Name:      name,
		IPAddress: netface,
		IOPS:      "0",
		Volumes:   "1",
		Status:    "Active",
	}

	return vsm, err
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	context2 "context"
	"fmt"
	"os"
	"time"

	"github.com/onsi/ginkgo"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/clientcmd/api"
)

const (
	// Poll how often to poll for conditions
	Poll = 2 * time.Second

	// DefaultTimeout time to wait for operations to complete
	DefaultTimeout = 90 * time.Second
)

func nowStamp() string {
	return time.Now().Format(time.StampMilli)
}

func log(level string, format string, args ...interface{}) {
	fmt.Fprintf(ginkgo.GinkgoWriter, nowStamp()+": "+level+": "+format+"\n", args...)
}

// Logf logs to the INFO logs.
func Logf(format string, args ...interface{}) {
	log("INFO", format, args...)
}

// Failf logs to the INFO logs and fails the test.
func Failf(format string, args ...interface{}) {
	msg := fmt.Sprintf(format, args...)
	log("INFO", msg)
	ginkgo.Fail(nowStamp()+": "+msg, 1)
}

// RestclientConfig deserializes the contents of a kubeconfig file into a Config object.
func RestclientConfig(config, context string) (*api.Config, error) {
	Logf(">>> config: %s\n", config)
	if config == "" {
		return nil, fmt.Errorf("config file must be specified to load client config")
	}
	c, err := clientcmd.LoadFromFile(config)
	if err != nil {
		return nil, fmt.Errorf("error loading config: %v", err.Error())
	}
	if context != "" {
		Logf(">>> context: %s\n", context)
		c.CurrentContext = context
	}
	return c, nil
}

// RunID unique identifier of the e2e run
var RunID = uuid.NewUUID()

// CreateKubeNamespace creates a new namespace in the cluster
func CreateKubeNamespace(baseName string, c kubernetes.Interface) (string, error) {
	ts := time.Now().UnixNano()
	ns := &corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: fmt.Sprintf("e2e-tests-%v-%v-", baseName, ts),
		},
	}
	// Be robust about making the namespace creation call.
	var got *corev1.Namespace
	var err error

	err = wait.PollImmediate(Poll, DefaultTimeout, func() (bool, error) {
		got, err = c.CoreV1().Namespaces().Create(context2.TODO(), ns, metav1.CreateOptions{})
		if err != nil {
			Logf("Unexpected error while creating namespace: %v", err)
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		return "", err
	}
	return got.Name, nil
}

// DeleteKubeNamespace deletes a namespace and all the objects inside
func DeleteKubeNamespace(c kubernetes.Interface, namespace string) error {
	grace := int64(0)
	pb := metav1.DeletePropagationBackground
	return c.CoreV1().Namespaces().Delete(context2.TODO(), namespace, metav1.DeleteOptions{
		GracePeriodSeconds: &grace,
		PropagationPolicy:  &pb,
	})
}

// WaitForKubeNamespaceNotExist waits until a namespaces is not present in the cluster
func WaitForKubeNamespaceNotExist(c kubernetes.Interface, namespace string) error {
	return wait.PollImmediate(Poll, DefaultTimeout, namespaceNotExist(c, namespace))
}

func namespaceNotExist(c kubernetes.Interface, namespace string) wait.ConditionFunc {
	return func() (bool, error) {
		_, err := c.CoreV1().Namespaces().Get(context2.TODO(), namespace, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return true, nil
		}
		if err != nil {
			return false, err
		}
		return false, nil
	}
}

// WaitForNoPodsInNamespace waits until there are no pods running in a namespace
func WaitForNoPodsInNamespace(c kubernetes.Interface, namespace string) error {
	return wait.PollImmediate(Poll, DefaultTimeout, noPodsInNamespace(c, namespace))
}

func noPodsInNamespace(c kubernetes.Interface, namespace string) wait.ConditionFunc {
	return func() (bool, error) {
		items, err := c.CoreV1().Pods(namespace).List(context2.TODO(), metav1.ListOptions{})
		if apierrors.IsNotFound(err) {
			return true, nil
		}
		if err != nil {
			return false, err
		}

		if len(items.Items) == 0 {
			return true, nil
		}
		return false, nil
	}
}

// WaitForPodRunningInNamespace waits a default amount of time (PodStartTimeout) for the specified pod to become running.
// Returns an error if timeout occurs first, or pod goes in to failed state.
func WaitForPodRunningInNamespace(c kubernetes.Interface, pod *corev1.Pod) error {
	if pod.Status.Phase == corev1.PodRunning {
		return nil
	}
	return waitTimeoutForPodRunningInNamespace(c, pod.Name, pod.Namespace, DefaultTimeout)
}

func waitTimeoutForPodRunningInNamespace(c kubernetes.Interface, podName, namespace string, timeout time.Duration) error {
	return wait.PollImmediate(Poll, DefaultTimeout, podRunning(c, podName, namespace))
}

// WaitForSecretInNamespace waits a default amount of time for the specified secret is present in a particular namespace
func WaitForSecretInNamespace(c kubernetes.Interface, namespace, name string) error {
	return wait.PollImmediate(Poll, DefaultTimeout, secretInNamespace(c, namespace, name))
}

func secretInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc {
	return func() (bool, error) {
		s, err := c.CoreV1().Secrets(namespace).Get(context2.TODO(), name, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return false, nil
		}
		if err != nil {
			return false, err
		}

		if s != nil {
			return true, nil
		}
		return false, nil
	}
}

// WaitForFileInFS waits a default amount of time for the specified file is present in the filesystem
func WaitForFileInFS(file string) error {
	return wait.PollImmediate(Poll, DefaultTimeout, fileInFS(file))
}

func fileInFS(file string) wait.ConditionFunc {
	return func() (bool, error) {
		stat, err := os.Stat(file)
		if err != nil {
			return false, err
		}

		if stat == nil {
			return false, fmt.Errorf("file %v does not exist", file)
		}

		if stat.Size() > 0 {
			return true, nil
		}

		return false, fmt.Errorf("the file %v exists but it is empty", file)
	}
}

// WaitForNoIngressInNamespace waits until there is no ingress object in a particular namespace
func WaitForNoIngressInNamespace(c kubernetes.Interface, namespace, name string) error {
	return wait.PollImmediate(Poll, DefaultTimeout, noIngressInNamespace(c, namespace, name))
}

func noIngressInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc {
	return func() (bool, error) {
		ing, err := c.NetworkingV1beta1().Ingresses(namespace).Get(context2.TODO(), name, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return true, nil
		}
		if err != nil {
			return false, err
		}

		if ing == nil {
			return true, nil
		}
		return false, nil
	}
}

// WaitForIngressInNamespace waits until a particular ingress object exists namespace
func WaitForIngressInNamespace(c kubernetes.Interface, namespace, name string) error {
	return wait.PollImmediate(Poll, DefaultTimeout, ingressInNamespace(c, namespace, name))
}

func ingressInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc {
	return func() (bool, error) {
		ing, err := c.NetworkingV1beta1().Ingresses(namespace).Get(context2.TODO(), name, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return false, nil
		}
		if err != nil {
			return false, err
		}

		if ing != nil {
			return true, nil
		}
		return false, nil
	}
}

func podRunning(c kubernetes.Interface, podName, namespace string) wait.ConditionFunc {
	return func() (bool, error) {
		pod, err := c.CoreV1().Pods(namespace).Get(context2.TODO(), podName, metav1.GetOptions{})
		if err != nil {
			return false, nil
		}
		switch pod.Status.Phase {
		case corev1.PodRunning:
			return true, nil
		case corev1.PodFailed, corev1.PodSucceeded:
			return false, fmt.Errorf("pod ran to completion")
		}
		return false, nil
	}
}

func labelSelectorToString(labels map[string]string) string {
	var str string
	for k, v := range labels {
		str += fmt.Sprintf("%s=%s,", k, v)
	}
	return str[:len(str)-1]
}

// NewInt32 converts int32 to a pointer
func NewInt32(val int32) *int32 {
	p := new(int32)
	*p = val
	return p
}

// NewInt64 converts int64 to a pointer
func NewInt64(val int64) *int64 {
	p := new(int64)
	*p = val
	return p
}
<commit_msg>Adjust default e2e timeout<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	context2 "context"
	"fmt"
	"os"
	"time"

	"github.com/onsi/ginkgo"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/clientcmd/api"
)

const (
	// Poll how often to poll for conditions
	Poll = 2 * time.Second

	// DefaultTimeout time to wait for operations to complete
	DefaultTimeout = 180 * time.Second
)

func nowStamp() string {
	return time.Now().Format(time.StampMilli)
}

func log(level string, format string, args ...interface{}) {
	fmt.Fprintf(ginkgo.GinkgoWriter, nowStamp()+": "+level+": "+format+"\n", args...)
}

// Logf logs to the INFO logs.
func Logf(format string, args ...interface{}) {
	log("INFO", format, args...)
}

// Failf logs to the INFO logs and fails the test.
func Failf(format string, args ...interface{}) {
	msg := fmt.Sprintf(format, args...)
	log("INFO", msg)
	ginkgo.Fail(nowStamp()+": "+msg, 1)
}

// RestclientConfig deserializes the contents of a kubeconfig file into a Config object.
func RestclientConfig(config, context string) (*api.Config, error) {
	Logf(">>> config: %s\n", config)
	if config == "" {
		return nil, fmt.Errorf("config file must be specified to load client config")
	}
	c, err := clientcmd.LoadFromFile(config)
	if err != nil {
		return nil, fmt.Errorf("error loading config: %v", err.Error())
	}
	if context != "" {
		Logf(">>> context: %s\n", context)
		c.CurrentContext = context
	}
	return c, nil
}

// RunID unique identifier of the e2e run
var RunID = uuid.NewUUID()

// CreateKubeNamespace creates a new namespace in the cluster
func CreateKubeNamespace(baseName string, c kubernetes.Interface) (string, error) {
	ts := time.Now().UnixNano()
	ns := &corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: fmt.Sprintf("e2e-tests-%v-%v-", baseName, ts),
		},
	}
	// Be robust about making the namespace creation call.
	var got *corev1.Namespace
	var err error

	err = wait.PollImmediate(Poll, DefaultTimeout, func() (bool, error) {
		got, err = c.CoreV1().Namespaces().Create(context2.TODO(), ns, metav1.CreateOptions{})
		if err != nil {
			Logf("Unexpected error while creating namespace: %v", err)
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		return "", err
	}
	return got.Name, nil
}

// DeleteKubeNamespace deletes a namespace and all the objects inside
func DeleteKubeNamespace(c kubernetes.Interface, namespace string) error {
	grace := int64(0)
	pb := metav1.DeletePropagationBackground
	return c.CoreV1().Namespaces().Delete(context2.TODO(), namespace, metav1.DeleteOptions{
		GracePeriodSeconds: &grace,
		PropagationPolicy:  &pb,
	})
}

// WaitForKubeNamespaceNotExist waits until a namespaces is not present in the cluster
func WaitForKubeNamespaceNotExist(c kubernetes.Interface, namespace string) error {
	return wait.PollImmediate(Poll, DefaultTimeout, namespaceNotExist(c, namespace))
}

func namespaceNotExist(c kubernetes.Interface, namespace string) wait.ConditionFunc {
	return func() (bool, error) {
		_, err := c.CoreV1().Namespaces().Get(context2.TODO(), namespace, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return true, nil
		}
		if err != nil {
			return false, err
		}
		return false, nil
	}
}

// WaitForNoPodsInNamespace waits until there are no pods running in a namespace
func WaitForNoPodsInNamespace(c kubernetes.Interface, namespace string) error {
	return wait.PollImmediate(Poll, DefaultTimeout, noPodsInNamespace(c, namespace))
}

func noPodsInNamespace(c kubernetes.Interface, namespace string) wait.ConditionFunc {
	return func() (bool, error) {
		items, err := c.CoreV1().Pods(namespace).List(context2.TODO(), metav1.ListOptions{})
		if apierrors.IsNotFound(err) {
			return true, nil
		}
		if err != nil {
			return false, err
		}

		if len(items.Items) == 0 {
			return true, nil
		}
		return false, nil
	}
}

// WaitForPodRunningInNamespace waits a default amount of time (PodStartTimeout) for the specified pod to become running.
// Returns an error if timeout occurs first, or pod goes in to failed state.
func WaitForPodRunningInNamespace(c kubernetes.Interface, pod *corev1.Pod) error {
	if pod.Status.Phase == corev1.PodRunning {
		return nil
	}
	return waitTimeoutForPodRunningInNamespace(c, pod.Name, pod.Namespace, DefaultTimeout)
}

func waitTimeoutForPodRunningInNamespace(c kubernetes.Interface, podName, namespace string, timeout time.Duration) error {
	return wait.PollImmediate(Poll, DefaultTimeout, podRunning(c, podName, namespace))
}

// WaitForSecretInNamespace waits a default amount of time for the specified secret is present in a particular namespace
func WaitForSecretInNamespace(c kubernetes.Interface, namespace, name string) error {
	return wait.PollImmediate(Poll, DefaultTimeout, secretInNamespace(c, namespace, name))
}

func secretInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc {
	return func() (bool, error) {
		s, err := c.CoreV1().Secrets(namespace).Get(context2.TODO(), name, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return false, nil
		}
		if err != nil {
			return false, err
		}

		if s != nil {
			return true, nil
		}
		return false, nil
	}
}

// WaitForFileInFS waits a default amount of time for the specified file is present in the filesystem
func WaitForFileInFS(file string) error {
	return wait.PollImmediate(Poll, DefaultTimeout, fileInFS(file))
}

func fileInFS(file string) wait.ConditionFunc {
	return func() (bool, error) {
		stat, err := os.Stat(file)
		if err != nil {
			return false, err
		}

		if stat == nil {
			return false, fmt.Errorf("file %v does not exist", file)
		}

		if stat.Size() > 0 {
			return true, nil
		}

		return false, fmt.Errorf("the file %v exists but it is empty", file)
	}
}

// WaitForNoIngressInNamespace waits until there is no ingress object in a particular namespace
func WaitForNoIngressInNamespace(c kubernetes.Interface, namespace, name string) error {
	return wait.PollImmediate(Poll, DefaultTimeout, noIngressInNamespace(c, namespace, name))
}

func noIngressInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc {
	return func() (bool, error) {
		ing, err := c.NetworkingV1beta1().Ingresses(namespace).Get(context2.TODO(), name, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return true, nil
		}
		if err != nil {
			return false, err
		}

		if ing == nil {
			return true, nil
		}
		return false, nil
	}
}

// WaitForIngressInNamespace waits until a particular ingress object exists namespace
func WaitForIngressInNamespace(c kubernetes.Interface, namespace, name string) error {
	return wait.PollImmediate(Poll, DefaultTimeout, ingressInNamespace(c, namespace, name))
}

func ingressInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc {
	return func() (bool, error) {
		ing, err := c.NetworkingV1beta1().Ingresses(namespace).Get(context2.TODO(), name, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return false, nil
		}
		if err != nil {
			return false, err
		}

		if ing != nil {
			return true, nil
		}
		return false, nil
	}
}

func podRunning(c kubernetes.Interface, podName, namespace string) wait.ConditionFunc {
	return func() (bool, error) {
		pod, err := c.CoreV1().Pods(namespace).Get(context2.TODO(), podName, metav1.GetOptions{})
		if err != nil {
			return false, nil
		}
		switch pod.Status.Phase {
		case corev1.PodRunning:
			return true, nil
		case corev1.PodFailed, corev1.PodSucceeded:
			return false, fmt.Errorf("pod ran to completion")
		}
		return false, nil
	}
}

func labelSelectorToString(labels map[string]string) string {
	var str string
	for k, v := range labels {
		str += fmt.Sprintf("%s=%s,", k, v)
	}
	return str[:len(str)-1]
}

// NewInt32 converts int32 to a pointer
func NewInt32(val int32) *int32 {
	p := new(int32)
	*p = val
	return p
}

// NewInt64 converts int64 to a pointer
func NewInt64(val int64) *int64 {
	p := new(int64)
	*p = val
	return p
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
	"github.com/gophercloud/gophercloud/openstack/identity/v3/tokens"
	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/containers"
	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/objects"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"k8s.io/apimachinery/pkg/util/wait"

	"github.com/sapcc/kubernikus/pkg/api/client/operations"
	etcd_util "github.com/sapcc/kubernikus/pkg/util/etcd"
	"github.com/sapcc/kubernikus/test/e2e/framework"
)

const (
	CleanupBackupContainerDeleteInterval = 1 * time.Second
	CleanupBackupContainerDeleteTimeout  = 1 * time.Minute
)

type PyrolisisTests struct {
	Kubernikus *framework.Kubernikus
	OpenStack  *framework.OpenStack
	Reuse      bool
}

func (p *PyrolisisTests) Run(t *testing.T) {
	if p.Reuse == false {
		quota := t.Run("SettingKlustersOnFire", p.SettingKlustersOnFire)
		require.True(t, quota, "Klusters must burn")

		t.Run("Wait", func(t *testing.T) {
			t.Run("Klusters", p.WaitForE2EKlustersTerminated)
		})

		cleanupStorageContainer := t.Run("CleanupBackupStorageContainers", p.CleanupBackupStorageContainers)
		require.True(t, cleanupStorageContainer, "Etcd backup storage container cleanup failed")

		t.Run("CleanupVolumes", p.CleanupVolumes)
		t.Run("CleanupInstances", p.CleanupInstances)
	}
}

func (p *PyrolisisTests) SettingKlustersOnFire(t *testing.T) {
	res, err := p.Kubernikus.Client.Operations.ListClusters(
		operations.NewListClustersParams(),
		p.Kubernikus.AuthInfo,
	)
	require.NoError(t, err, "There should be no error while listing klusters")

	for _, kluster := range res.Payload {
		if strings.HasPrefix(kluster.Name, "e2e-") {
			t.Run(fmt.Sprintf("TerminatingKluster-%v", kluster.Name), func(t *testing.T) {
				_, err := p.Kubernikus.Client.Operations.TerminateCluster(
					operations.NewTerminateClusterParams().WithName(kluster.Name),
					p.Kubernikus.AuthInfo,
				)
				assert.NoError(t, err, "There should be no error while terminating klusters")
			})
		}
	}
}

func (p *PyrolisisTests) WaitForE2EKlustersTerminated(t *testing.T) {
	err := p.Kubernikus.WaitForKlusters("e2e-", 0, WaitForKlusterToBeDeletedTimeout)
	assert.NoError(t, err, "E2E Klusters didn't burn down in time")
}

func (p *PyrolisisTests) CleanupBackupStorageContainers(t *testing.T) {
	storageClient, err := openstack.NewObjectStorageV1(p.OpenStack.Provider, gophercloud.EndpointOpts{})
	require.NoError(t, err, "Could not create object storage client")

	containersListOpts := containers.ListOpts{
		Full: false,
	}
	allPages, err := containers.List(storageClient, containersListOpts).AllPages()
	require.NoError(t, err, "There should be no error while listing storage containers")

	allContainers, err := containers.ExtractNames(allPages)
	require.NoError(t, err, "There should be no error while extracting storage containers")

	objectsListOpts := objects.ListOpts{
		Full: false,
	}
	for _, container := range allContainers {
		if strings.HasPrefix(container, etcd_util.BackupStorageContainerBase) {
			allPages, err := objects.List(storageClient, container, objectsListOpts).AllPages()
			require.NoError(t, err, "There should be no error while lising objetcs in container %s", container)

			allObjects, err := objects.ExtractNames(allPages)
			require.NoError(t, err, "There should be no error while extracting objetcs names for container %s", container)

			for _, object := range allObjects {
				_, err := objects.Delete(storageClient, container, object, objects.DeleteOpts{}).Extract()
				//Ignore 404 from swift, this can happen for a successful delete becase of the eventual consistency
				if _, ok := err.(gophercloud.ErrDefault404); ok {
					continue
				}
				require.NoError(t, err, "There should be no error while deleting object %s/%s", container, object)
			}

			err = wait.PollImmediate(CleanupBackupContainerDeleteInterval, CleanupBackupContainerDeleteTimeout,
				func() (bool, error) {
					_, err := containers.Delete(storageClient, container).Extract()
					if errResponseCode, ok := err.(gophercloud.ErrUnexpectedResponseCode); ok && errResponseCode.Actual == 409 {
						return false, nil
					}
					//Ignore 404 from swift, this can happen for a successful delete becase of the eventual consistency
					if _, ok := err.(gophercloud.ErrDefault404); ok {
						return true, nil
					}
					return true, err
				})
			require.NoError(t, err, "There should be no error while deleting storage container: %s", container)
		}
	}
}

func (p *PyrolisisTests) CleanupVolumes(t *testing.T) {
	storageClient, err := openstack.NewBlockStorageV3(p.OpenStack.Provider, gophercloud.EndpointOpts{})
	require.NoError(t, err, "Could not create block storage client")

	project, err := tokens.Get(p.OpenStack.Identity, p.OpenStack.Provider.Token()).ExtractProject()
	require.NoError(t, err, "There should be no error while extracting the project")

	volumeListOpts := volumes.ListOpts{
		TenantID: project.ID,
	}

	allPages, err := volumes.List(storageClient, volumeListOpts).AllPages()
	require.NoError(t, err, "There should be no error while retrieving volume pages")

	allVolumes, err := volumes.ExtractVolumes(allPages)
	require.NoError(t, err, "There should be no error while extracting volumes")

	for _, vol := range allVolumes {
		if strings.HasPrefix(vol.Name, "kubernetes-dynamic-pvc-") &&
			strings.HasPrefix(vol.Metadata["kubernetes.io/created-for/pvc/namespace"], "e2e-volumes-") {
			err := volumes.Delete(storageClient, vol.ID, volumes.DeleteOpts{}).ExtractErr()
			require.NoError(t, err, "There should be no error while deleting volume %s (%s)", vol.Name, vol.ID)
		}
	}
}

func (p *PyrolisisTests) CleanupInstances(t *testing.T) {
	computeClient, err := openstack.NewComputeV2(p.OpenStack.Provider, gophercloud.EndpointOpts{})
	require.NoError(t, err, "There should be no error creating compute client")

	project, err := tokens.Get(p.OpenStack.Identity, p.OpenStack.Provider.Token()).ExtractProject()
	require.NoError(t, err, "There should be no error while extracting the project")

	serversListOpts := servers.ListOpts{
		Name:     "e2e-",
		TenantID: project.ID,
	}

	allPages, err := servers.List(computeClient, serversListOpts).AllPages()
	require.NoError(t, err, "There should be no error while listing all servers")

	allServers, err := servers.ExtractServers(allPages)
	require.NoError(t, err, "There should be no error while extracting all servers")

	for _, srv := range allServers {
		err := servers.Delete(computeClient, srv.ID).ExtractErr()
		require.NoError(t, err, "There should be no error while deleting server %s (%s)", srv.Name, srv.ID)
	}
}
<commit_msg>Clean up volumes first in e2e test<commit_after>package main

import (
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
	"github.com/gophercloud/gophercloud/openstack/identity/v3/tokens"
	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/containers"
	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/objects"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"k8s.io/apimachinery/pkg/util/wait"

	"github.com/sapcc/kubernikus/pkg/api/client/operations"
	etcd_util "github.com/sapcc/kubernikus/pkg/util/etcd"
	"github.com/sapcc/kubernikus/test/e2e/framework"
)

const (
	CleanupBackupContainerDeleteInterval = 1 * time.Second
	CleanupBackupContainerDeleteTimeout  = 1 * time.Minute
)

type PyrolisisTests struct {
	Kubernikus *framework.Kubernikus
	OpenStack  *framework.OpenStack
	Reuse      bool
}

func (p *PyrolisisTests) Run(t *testing.T) {
	if p.Reuse == false {
		quota := t.Run("SettingKlustersOnFire", p.SettingKlustersOnFire)
		require.True(t, quota, "Klusters must burn")

		t.Run("Wait", func(t *testing.T) {
			t.Run("Klusters", p.WaitForE2EKlustersTerminated)
		})

		t.Run("CleanupVolumes", p.CleanupVolumes)
		t.Run("CleanupInstances", p.CleanupInstances)

		cleanupStorageContainer := t.Run("CleanupBackupStorageContainers", p.CleanupBackupStorageContainers)
		require.True(t, cleanupStorageContainer, "Etcd backup storage container cleanup failed")
	}
}

func (p *PyrolisisTests) SettingKlustersOnFire(t *testing.T) {
	res, err := p.Kubernikus.Client.Operations.ListClusters(
		operations.NewListClustersParams(),
		p.Kubernikus.AuthInfo,
	)
	require.NoError(t, err, "There should be no error while listing klusters")

	for _, kluster := range res.Payload {
		if strings.HasPrefix(kluster.Name, "e2e-") {
			t.Run(fmt.Sprintf("TerminatingKluster-%v", kluster.Name), func(t *testing.T) {
				_, err := p.Kubernikus.Client.Operations.TerminateCluster(
					operations.NewTerminateClusterParams().WithName(kluster.Name),
					p.Kubernikus.AuthInfo,
				)
				assert.NoError(t, err, "There should be no error while terminating klusters")
			})
		}
	}
}

func (p *PyrolisisTests) WaitForE2EKlustersTerminated(t *testing.T) {
	err := p.Kubernikus.WaitForKlusters("e2e-", 0, WaitForKlusterToBeDeletedTimeout)
	assert.NoError(t, err, "E2E Klusters didn't burn down in time")
}

func (p *PyrolisisTests) CleanupBackupStorageContainers(t *testing.T) {
	storageClient, err := openstack.NewObjectStorageV1(p.OpenStack.Provider, gophercloud.EndpointOpts{})
	require.NoError(t, err, "Could not create object storage client")

	containersListOpts := containers.ListOpts{
		Full: false,
	}
	allPages, err := containers.List(storageClient, containersListOpts).AllPages()
	require.NoError(t, err, "There should be no error while listing storage containers")

	allContainers, err := containers.ExtractNames(allPages)
	require.NoError(t, err, "There should be no error while extracting storage containers")

	objectsListOpts := objects.ListOpts{
		Full: false,
	}
	for _, container := range allContainers {
		if strings.HasPrefix(container, etcd_util.BackupStorageContainerBase) {
			allPages, err := objects.List(storageClient, container, objectsListOpts).AllPages()
			require.NoError(t, err, "There should be no error while lising objetcs in container %s", container)

			allObjects, err := objects.ExtractNames(allPages)
			require.NoError(t, err, "There should be no error while extracting objetcs names for container %s", container)

			for _, object := range allObjects {
				_, err := objects.Delete(storageClient, container, object, objects.DeleteOpts{}).Extract()
				//Ignore 404 from swift, this can happen for a successful delete becase of the eventual consistency
				if _, ok := err.(gophercloud.ErrDefault404); ok {
					continue
				}
				require.NoError(t, err, "There should be no error while deleting object %s/%s", container, object)
			}

			err = wait.PollImmediate(CleanupBackupContainerDeleteInterval, CleanupBackupContainerDeleteTimeout,
				func() (bool, error) {
					_, err := containers.Delete(storageClient, container).Extract()
					if errResponseCode, ok := err.(gophercloud.ErrUnexpectedResponseCode); ok && errResponseCode.Actual == 409 {
						return false, nil
					}
					//Ignore 404 from swift, this can happen for a successful delete becase of the eventual consistency
					if _, ok := err.(gophercloud.ErrDefault404); ok {
						return true, nil
					}
					return true, err
				})
			require.NoError(t, err, "There should be no error while deleting storage container: %s", container)
		}
	}
}

func (p *PyrolisisTests) CleanupVolumes(t *testing.T) {
	storageClient, err := openstack.NewBlockStorageV3(p.OpenStack.Provider, gophercloud.EndpointOpts{})
	require.NoError(t, err, "Could not create block storage client")

	project, err := tokens.Get(p.OpenStack.Identity, p.OpenStack.Provider.Token()).ExtractProject()
	require.NoError(t, err, "There should be no error while extracting the project")

	volumeListOpts := volumes.ListOpts{
		TenantID: project.ID,
	}

	allPages, err := volumes.List(storageClient, volumeListOpts).AllPages()
	require.NoError(t, err, "There should be no error while retrieving volume pages")

	allVolumes, err := volumes.ExtractVolumes(allPages)
	require.NoError(t, err, "There should be no error while extracting volumes")

	for _, vol := range allVolumes {
		if strings.HasPrefix(vol.Name, "kubernetes-dynamic-pvc-") &&
			strings.HasPrefix(vol.Metadata["kubernetes.io/created-for/pvc/namespace"], "e2e-volumes-") {
			err := volumes.Delete(storageClient, vol.ID, volumes.DeleteOpts{}).ExtractErr()
			require.NoError(t, err, "There should be no error while deleting volume %s (%s)", vol.Name, vol.ID)
		}
	}
}

func (p *PyrolisisTests) CleanupInstances(t *testing.T) {
	computeClient, err := openstack.NewComputeV2(p.OpenStack.Provider, gophercloud.EndpointOpts{})
	require.NoError(t, err, "There should be no error creating compute client")

	project, err := tokens.Get(p.OpenStack.Identity, p.OpenStack.Provider.Token()).ExtractProject()
	require.NoError(t, err, "There should be no error while extracting the project")

	serversListOpts := servers.ListOpts{
		Name:     "e2e-",
		TenantID: project.ID,
	}

	allPages, err := servers.List(computeClient, serversListOpts).AllPages()
	require.NoError(t, err, "There should be no error while listing all servers")

	allServers, err := servers.ExtractServers(allPages)
	require.NoError(t, err, "There should be no error while extracting all servers")

	for _, srv := range allServers {
		err := servers.Delete(computeClient, srv.ID).ExtractErr()
		require.NoError(t, err, "There should be no error while deleting server %s (%s)", srv.Name, srv.ID)
	}
}
<|endoftext|>