<commit_before>package resize

import (
	"image"
	"image/color"
	"runtime"
	"testing"
)

var img = image.NewGray16(image.Rect(0, 0, 3, 3))

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	img.Set(1, 1, color.White)
}

func Test_Nearest(t *testing.T) {
	m := Resize(6, 0, img, NearestNeighbor)
	if m.At(1, 1) == m.At(2, 2) {
		t.Fail()
	}
}

func Test_Param1(t *testing.T) {
	m := Resize(0, 0, img, NearestNeighbor)
	if m.Bounds() != img.Bounds() {
		t.Fail()
	}
}

func Test_Param2(t *testing.T) {
	m := Resize(100, 0, img, NearestNeighbor)
	if m.Bounds() != image.Rect(0, 0, 100, 100) {
		t.Fail()
	}
}

func Test_ZeroImg(t *testing.T) {
	zeroImg := image.NewGray16(image.Rect(0, 0, 0, 0))

	m := Resize(0, 0, zeroImg, NearestNeighbor)
	if m.Bounds() != zeroImg.Bounds() {
		t.Fail()
	}
}

func Test_CorrectResize(t *testing.T) {
	zeroImg := image.NewGray16(image.Rect(0, 0, 256, 256))

	m := Resize(60, 0, zeroImg, NearestNeighbor)
	if m.Bounds() != image.Rect(0, 0, 60, 60) {
		t.Fail()
	}
}

func Benchmark_BigResizeLanczos3(b *testing.B) {
	var m image.Image
	for i := 0; i < b.N; i++ {
		m = Resize(1000, 1000, img, Lanczos3)
	}
	m.At(0, 0)
}

func Benchmark_BigResizeLanczos3Lut(b *testing.B) {
	var m image.Image
	for i := 0; i < b.N; i++ {
		m = Resize(1000, 1000, img, Lanczos3Lut)
	}
	m.At(0, 0)
}

func Benchmark_Reduction(b *testing.B) {
	largeImg := image.NewRGBA(image.Rect(0, 0, 1000, 1000))

	var m image.Image
	for i := 0; i < b.N; i++ {
		m = Resize(300, 300, largeImg, Bicubic)
	}
	m.At(0, 0)
}
<commit_msg>Add benchmarks to compare execution times.<commit_after>package resize

import (
	"image"
	"image/color"
	"runtime"
	"testing"
)

var img = image.NewGray16(image.Rect(0, 0, 3, 3))

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	img.Set(1, 1, color.White)
}

func Test_Nearest(t *testing.T) {
	m := Resize(6, 0, img, NearestNeighbor)
	if m.At(1, 1) == m.At(2, 2) {
		t.Fail()
	}
}

func Test_Param1(t *testing.T) {
	m := Resize(0, 0, img, NearestNeighbor)
	if m.Bounds() != img.Bounds() {
		t.Fail()
	}
}

func Test_Param2(t *testing.T) {
	m := Resize(100, 0, img, NearestNeighbor)
	if m.Bounds() != image.Rect(0, 0, 100, 100) {
		t.Fail()
	}
}

func Test_ZeroImg(t *testing.T) {
	zeroImg := image.NewGray16(image.Rect(0, 0, 0, 0))

	m := Resize(0, 0, zeroImg, NearestNeighbor)
	if m.Bounds() != zeroImg.Bounds() {
		t.Fail()
	}
}

func Test_CorrectResize(t *testing.T) {
	zeroImg := image.NewGray16(image.Rect(0, 0, 256, 256))

	m := Resize(60, 0, zeroImg, NearestNeighbor)
	if m.Bounds() != image.Rect(0, 0, 60, 60) {
		t.Fail()
	}
}

func Benchmark_BigResizeLanczos3(b *testing.B) {
	var m image.Image
	for i := 0; i < b.N; i++ {
		m = Resize(1000, 1000, img, Lanczos3)
	}
	m.At(0, 0)
}

func Benchmark_BigResizeLanczos3Lut(b *testing.B) {
	var m image.Image
	for i := 0; i < b.N; i++ {
		m = Resize(1000, 1000, img, Lanczos3Lut)
	}
	m.At(0, 0)
}

func Benchmark_Reduction(b *testing.B) {
	largeImg := image.NewRGBA(image.Rect(0, 0, 1000, 1000))

	var m image.Image
	for i := 0; i < b.N; i++ {
		m = Resize(300, 300, largeImg, Bicubic)
	}
	m.At(0, 0)
}

// Benchmark resize of 16 MPix jpeg image to 800px width.
func jpegThumb(b *testing.B, interp InterpolationFunction) {
	input := image.NewYCbCr(image.Rect(0, 0, 4896, 3264), image.YCbCrSubsampleRatio422)

	var output image.Image
	for i := 0; i < b.N; i++ {
		output = Resize(800, 0, input, interp)
	}

	output.At(0, 0)
}

func Benchmark_LargeJpegThumbNearestNeighbor(b *testing.B) {
	jpegThumb(b, NearestNeighbor)
}

func Benchmark_LargeJpegThumbBilinear(b *testing.B) {
	jpegThumb(b, Bilinear)
}

func Benchmark_LargeJpegThumbBicubic(b *testing.B) {
	jpegThumb(b, Bicubic)
}

func Benchmark_LargeJpegThumbMitchellNetravali(b *testing.B) {
	jpegThumb(b, MitchellNetravali)
}

func Benchmark_LargeJpegThumbLanczos2Lut(b *testing.B) {
	jpegThumb(b, Lanczos2Lut)
}

func Benchmark_LargeJpegThumbLanczos2(b *testing.B) {
	jpegThumb(b, Lanczos2)
}

func Benchmark_LargeJpegThumbLanczos3Lut(b *testing.B) {
	jpegThumb(b, Lanczos3Lut)
}

func Benchmark_LargeJpegThumbLanczos3(b *testing.B) {
	jpegThumb(b, Lanczos3)
}
<|endoftext|>


<commit_before>package runtime

import (
	"fmt"
)

type TypeError string

func (te TypeError) Error() string {
	return string(te)
}

func NewTypeError(t, op string) TypeError {
	return TypeError(fmt.Sprintf("type error: %s not allowed for type %s", op, t))
}

// Converter declares the required methods to convert a value
// to any one of the supported types (except Object and Func).
type Converter interface {
	Int() int64
	Float() float64
	String() string
	Bool() bool
	Native() interface{}
}

// Arithmetic defines the methods required to compute all
// the supported arithmetic operations.
type Arithmetic interface {
	Add(Val) Val
	Sub(Val) Val
	Mul(Val) Val
	Div(Val) Val
	Mod(Val) Val
	Unm() Val
}

type defaultArithmetic struct{}

func (ar *defaultArithmetic) binaryOp(l, r Val, op string, allowStrings bool) Val {
	lt, rt := Type(l), Type(r)
	mm := "__" + op
	if lt == "number" && rt == "number" {
		// Two numbers, standard arithmetic operation
		switch op {
		case "add":
			return Number(l.Float() + r.Float())
		case "sub":
			return Number(l.Float() - r.Float())
		case "mul":
			return Number(l.Float() * r.Float())
		case "div":
			return Number(l.Float() / r.Float())
		case "mod":
			return Number(l.Float() % r.Float())
		}
	} else if allowStrings && lt == "string" && rt == "string" {
		// Two strings
		switch op {
		case "add":
			return String(l.String() + r.String())
		}
	} else if lt == "object" {
		// If left operand is an object with a meta-method
		lo := l.(Object)
		if v, ok := lo.callMetaMethod(mm, r, Bool(true)); ok {
			return v
		}
	}
	// Last chance: if right operand is an object with a meta-method
	if rt == object {
		ro := r.(Object)
		if v, ok := ro.callMetaMethod(mm, l, Bool(false)); ok {
			return v
		}
	}
	panic(NewTypeError(lt, op))
}

func (ar *defaultArithmetic) Add(l, r Val) Val {
	return ar.binaryOp(l, r, "add", true)
}

func (ar *defaultArithmetic) Sub(l, r Val) Val {
	return ar.binaryOp(l, r, "sub", false)
}

func (ar *defaultArithmetic) Mul(l, r Val) Val {
	return ar.binaryOp(l, r, "mul", false)
}

func (ar *defaultArithmetic) Div(l, r Val) Val {
	return ar.binaryOp(l, r, "div", false)
}

func (ar *defaultArithmetic) Mod(l, r Val) Val {
	return ar.binaryOp(l, r, "mod", false)
}

func (ar *defaultArithmetic) Unm(l Val) Val {
	lt := Type(l)
	if lt == "number" {
		return Number(-l.Float())
	} else if lt == "object" {
		lo := l.(Object)
		if v, ok := lo.callMetaMethod("__unm"); ok {
			return v
		}
	}
	panic(NewTypeError(lt, "unm"))
}

// Comparer defines the method required to compare two Values.
// Cmp() returns 1 if the method receiver value is greater, 0 if
// it is equal, and -1 if it is lower.
type Comparer interface {
	Cmp(Val) int
}

type defaultComparer struct{}

func (dc *defaultComparer) Cmp(l, r Val) int {
	lt, rt := Type(l), Type(r)
	if lt == rt {
		// Comparable types
		switch lt {
		case "nil":
			return 0
		case "number":
			lf, rf := l.Float(), r.Float()
			if lf == rf {
				return 0
			} else if lf < rf {
				return -1
			} else {
				return 1
			}
		case "string":
			ls, rs := l.String(), r.String()
			if ls == rs {
				return 0
			} else if ls < rs {
				return -1
			} else {
				return 1
			}
		case "bool":
			lb, rb := l.Bool(), r.Bool()
			if lb == rb {
				return 0
			} else if lb {
				return 1 // true is greater than false (0)
			} else {
				-1
			}
		case "func":
			lf, rf := l.Native(), r.Native()
			if lf == rf {
				return 0
			} else {
				return -1
			}
		}
	}
}

// The dumper interface defines the required behaviour to pretty-print
// the values.
type dumper interface {
	dump() string
}

// Val is the representation of a value, any value, in the language.
// The supported value types are the following:
// * Number (float64)
// * String
// * Boolean (Bool)
// * Nil (null)
// * Object
// * Func
type Val interface {
	Converter
	dumper
}

func Type(v Val) string {
	switch v.(type) {
	case runtime.String:
		return runtime.String("string")
	case runtime.Number:
		return runtime.String("number")
	case runtime.Bool:
		return runtime.String("bool")
	case runtime.Func:
		return runtime.String("func")
	case runtime.Object:
		return runtime.String("object")
	default:
		if v == runtime.Nil {
			return runtime.String("nil")
		} else {
			return runtime.String("custom")
		}
	}
}

// The LogicProcessor interface defines the method required to implement
// boolean logic. It is defined as an interface for pluggable replacement
// for testing.
type LogicProcessor interface {
	Not(v Val) Bool
	And(x, y Val) Bool
	Or(x, y Val) Bool
}

// The default implementation of the logic processor.
type defaultLogic struct{}

func (d defaultLogic) Not(v Val) Bool {
	return Bool(!v.Bool())
}

func (d defaultLogic) And(x, y Val) Bool {
	return Bool(x.Bool() && y.Bool())
}

func (d defaultLogic) Or(x, y Val) Bool {
	return Bool(x.Bool() || y.Bool())
}
<commit_msg>complete Cmp based on matrix<commit_after>package runtime

import (
	"fmt"
)

type TypeError string

func (te TypeError) Error() string {
	return string(te)
}

func NewTypeError(t, op string) TypeError {
	return TypeError(fmt.Sprintf("type error: %s not allowed for type %s", op, t))
}

// Converter declares the required methods to convert a value
// to any one of the supported types (except Object and Func).
type Converter interface {
	Int() int64
	Float() float64
	String() string
	Bool() bool
	Native() interface{}
}

// Arithmetic defines the methods required to compute all
// the supported arithmetic operations.
type Arithmetic interface {
	Add(Val) Val
	Sub(Val) Val
	Mul(Val) Val
	Div(Val) Val
	Mod(Val) Val
	Unm() Val
}

type defaultArithmetic struct{}

func (ar *defaultArithmetic) binaryOp(l, r Val, op string, allowStrings bool) Val {
	lt, rt := Type(l), Type(r)
	mm := "__" + op
	if lt == "number" && rt == "number" {
		// Two numbers, standard arithmetic operation
		switch op {
		case "add":
			return Number(l.Float() + r.Float())
		case "sub":
			return Number(l.Float() - r.Float())
		case "mul":
			return Number(l.Float() * r.Float())
		case "div":
			return Number(l.Float() / r.Float())
		case "mod":
			return Number(l.Float() % r.Float())
		}
	} else if allowStrings && lt == "string" && rt == "string" {
		// Two strings
		switch op {
		case "add":
			return String(l.String() + r.String())
		}
	} else if lt == "object" {
		// If left operand is an object with a meta-method
		lo := l.(Object)
		if v, ok := lo.callMetaMethod(mm, r, Bool(true)); ok {
			return v
		}
	}
	// Last chance: if right operand is an object with a meta-method
	if rt == object {
		ro := r.(Object)
		if v, ok := ro.callMetaMethod(mm, l, Bool(false)); ok {
			return v
		}
	}
	panic(NewTypeError(lt, op))
}

func (ar *defaultArithmetic) Add(l, r Val) Val {
	return ar.binaryOp(l, r, "add", true)
}

func (ar *defaultArithmetic) Sub(l, r Val) Val {
	return ar.binaryOp(l, r, "sub", false)
}

func (ar *defaultArithmetic) Mul(l, r Val) Val {
	return ar.binaryOp(l, r, "mul", false)
}

func (ar *defaultArithmetic) Div(l, r Val) Val {
	return ar.binaryOp(l, r, "div", false)
}

func (ar *defaultArithmetic) Mod(l, r Val) Val {
	return ar.binaryOp(l, r, "mod", false)
}

func (ar *defaultArithmetic) Unm(l Val) Val {
	lt := Type(l)
	if lt == "number" {
		return Number(-l.Float())
	} else if lt == "object" {
		lo := l.(Object)
		if v, ok := lo.callMetaMethod("__unm"); ok {
			return v
		}
	}
	panic(NewTypeError(lt, "unm"))
}

// Comparer defines the method required to compare two Values.
// Cmp() returns 1 if the method receiver value is greater, 0 if
// it is equal, and -1 if it is lower.
type Comparer interface {
	Cmp(Val) int
}

var (
	// Unmutable, this would be a const if it was possible
	uneqMatrix = map[string]map[string]int{
		"nil": map[string]int{
			"number": -1,
			"string": -1,
			"bool":   -1,
			"object": -1,
			"func":   -1,
			"custom": -1,
		},
		"number": map[string]int{
			"nil":    1,
			"string": -1,
			"bool":   1,
			"object": -1,
			"func":   1,
			"custom": 1,
		},
		"string": map[string]int{
			"number": 1,
			"nil":    1,
			"bool":   1,
			"object": 1,
			"func":   1,
			"custom": 1,
		},
		"bool": map[string]int{
			"number": -1,
			"string": -1,
			"nil":    1,
			"object": -1,
			"func":   -1,
			"custom": -1,
		},
		"object": map[string]int{
			"number": 1,
			"string": -1,
			"bool":   1,
			"nil":    1,
			"func":   1,
			"custom": 1,
		},
		"func": map[string]int{
			"number": -1,
			"string": -1,
			"bool":   1,
			"object": -1,
			"nil":    1,
			"custom": 1,
		},
		"custom": map[string]int{
			"number": -1,
			"string": -1,
			"bool":   1,
			"object": -1,
			"func":   -1,
			"nil":    1,
		},
	}
)

type defaultComparer struct{}

func (dc *defaultComparer) Cmp(l, r Val) int {
	lt, rt := Type(l), Type(r)
	if lt == rt {
		// Comparable types
		switch lt {
		case "nil":
			return 0
		case "number":
			lf, rf := l.Float(), r.Float()
			if lf == rf {
				return 0
			} else if lf < rf {
				return -1
			} else {
				return 1
			}
		case "string":
			ls, rs := l.String(), r.String()
			if ls == rs {
				return 0
			} else if ls < rs {
				return -1
			} else {
				return 1
			}
		case "bool":
			lb, rb := l.Bool(), r.Bool()
			if lb == rb {
				return 0
			} else if lb {
				return 1 // true is greater than false (0)
			} else {
				-1
			}
		case "func":
			lf, rf := l.Native(), r.Native()
			if lf == rf {
				return 0
			} else {
				// "greater" or "lower" has no sense for funcs, return -1
				return -1
			}
		case "object":
			// If left has meta method, use left, otherwise right, else compare
			lo, ro := l.(Object), r.(Object)
			if v, ok := lo.callMetaMethod("__cmp", r, Bool(true)); ok {
				return v.Int()
			}
			if v, ok := ro.callMetaMethod("__cmp", l, Bool(false)); ok {
				return v.Int()
			}
			if lo == ro {
				return 0
			} else {
				// "greater" or "lower" has no sense for objects, return -1
				return -1
			}
		case "custom":
			if l == r {
				return 0
			} else {
				// "greater" or "lower" has no sense for custom vals, return -1
				return -1
			}
		default:
			panic(NewTypeError(lt, "cmp"))
		}
	} else {
		// Uncomparable types, first check for meta-methods
		var o Object
		var isLeft bool
		var otherv Val
		if lt == "object" {
			o = l.(Object)
			isLeft = true
			otherv = r
		} else if rt == "object" {
			o = r.(Object)
			isLeft = false
			otherv = l
		}
		if o != nil {
			if v, ok := o.callMetaMethod("__cmp", otherv, Bool(isLeft)); ok {
				return v.Int()
			}
		}
		// Else, return arbitrary but constant result
		return uneqMatrix[lt][rt]
	}
}

// The dumper interface defines the required behaviour to pretty-print
// the values.
// TODO : Should provide a default impl or make it public, so that custom types
// can actually be built from outside the package.
type dumper interface {
	dump() string
}

// Val is the representation of a value, any value, in the language.
// The supported value types are the following:
// * Number (float64)
// * String
// * Boolean (Bool)
// * Nil (null)
// * Object
// * Func
type Val interface {
	Converter
	dumper
}

func Type(v Val) string {
	switch v.(type) {
	case runtime.String:
		return runtime.String("string")
	case runtime.Number:
		return runtime.String("number")
	case runtime.Bool:
		return runtime.String("bool")
	case runtime.Func:
		return runtime.String("func")
	case runtime.Object:
		return runtime.String("object")
	default:
		if v == runtime.Nil {
			return runtime.String("nil")
		} else {
			return runtime.String("custom")
		}
	}
}

// The LogicProcessor interface defines the method required to implement
// boolean logic. It is defined as an interface for pluggable replacement
// for testing.
type LogicProcessor interface {
	Not(v Val) Bool
	And(x, y Val) Bool
	Or(x, y Val) Bool
}

// The default implementation of the logic processor.
type defaultLogic struct{}

func (d defaultLogic) Not(v Val) Bool {
	return Bool(!v.Bool())
}

func (d defaultLogic) And(x, y Val) Bool {
	return Bool(x.Bool() && y.Bool())
}

func (d defaultLogic) Or(x, y Val) Bool {
	return Bool(x.Bool() || y.Bool())
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v0

import (
	"k8s.io/apimachinery/pkg/conversion"
	"k8s.io/apimachinery/pkg/runtime"
	api "k8s.io/kubernetes/pkg/apis/abac"
)

// allAuthenticated matches k8s.io/apiserver/pkg/authentication/user.AllAuthenticated,
// but we don't want an client library (which must include types), depending on a server library
const allAuthenticated = "system:authenticated"

func addConversionFuncs(scheme *runtime.Scheme) error {
	return scheme.AddConversionFuncs(
		func(in *Policy, out *api.Policy, s conversion.Scope) error {
			// Begin by copying all fields
			out.Spec.User = in.User
			out.Spec.Group = in.Group
			out.Spec.Namespace = in.Namespace
			out.Spec.Resource = in.Resource
			out.Spec.Readonly = in.Readonly

			// In v0, unspecified user and group matches all authenticated subjects
			if len(in.User) == 0 && len(in.Group) == 0 {
				out.Spec.Group = allAuthenticated
			}
			// In v0, user or group of * matches all authenticated subjects
			if in.User == "*" || in.Group == "*" {
				out.Spec.Group = allAuthenticated
				out.Spec.User = ""
			}

			// In v0, leaving namespace empty matches all namespaces
			if len(in.Namespace) == 0 {
				out.Spec.Namespace = "*"
			}
			// In v0, leaving resource empty matches all resources
			if len(in.Resource) == 0 {
				out.Spec.Resource = "*"
			}
			// Any rule in v0 should match all API groups
			out.Spec.APIGroup = "*"

			// In v0, leaving namespace and resource blank allows non-resource paths
			if len(in.Namespace) == 0 && len(in.Resource) == 0 {
				out.Spec.NonResourcePath = "*"
			}

			return nil
		},
	)
}
<commit_msg>change an to a<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v0

import (
	"k8s.io/apimachinery/pkg/conversion"
	"k8s.io/apimachinery/pkg/runtime"
	api "k8s.io/kubernetes/pkg/apis/abac"
)

// allAuthenticated matches k8s.io/apiserver/pkg/authentication/user.AllAuthenticated,
// but we don't want a client library (which must include types), depending on a server library
const allAuthenticated = "system:authenticated"

func addConversionFuncs(scheme *runtime.Scheme) error {
	return scheme.AddConversionFuncs(
		func(in *Policy, out *api.Policy, s conversion.Scope) error {
			// Begin by copying all fields
			out.Spec.User = in.User
			out.Spec.Group = in.Group
			out.Spec.Namespace = in.Namespace
			out.Spec.Resource = in.Resource
			out.Spec.Readonly = in.Readonly

			// In v0, unspecified user and group matches all authenticated subjects
			if len(in.User) == 0 && len(in.Group) == 0 {
				out.Spec.Group = allAuthenticated
			}
			// In v0, user or group of * matches all authenticated subjects
			if in.User == "*" || in.Group == "*" {
				out.Spec.Group = allAuthenticated
				out.Spec.User = ""
			}

			// In v0, leaving namespace empty matches all namespaces
			if len(in.Namespace) == 0 {
				out.Spec.Namespace = "*"
			}
			// In v0, leaving resource empty matches all resources
			if len(in.Resource) == 0 {
				out.Spec.Resource = "*"
			}
			// Any rule in v0 should match all API groups
			out.Spec.APIGroup = "*"

			// In v0, leaving namespace and resource blank allows non-resource paths
			if len(in.Namespace) == 0 && len(in.Resource) == 0 {
				out.Spec.NonResourcePath = "*"
			}

			return nil
		},
	)
}
<|endoftext|>


<commit_before>package mxnet

import (
	"fmt"
	"time"

	log "github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	metav1unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/client-go/kubernetes/scheme"

	mxv1 "github.com/kubeflow/mxnet-operator/pkg/apis/mxnet/v1"
	"github.com/kubeflow/mxnet-operator/pkg/util/k8sutil"
	mxlogger "github.com/kubeflow/tf-operator/pkg/logger"
	"k8s.io/apimachinery/pkg/runtime"
)

const (
	failedMarshalMXJobReason = "FailedInvalidMXJobSpec"
	inspectFailMXJobReason   = "InspectFailedInvalidMXReplicaSpec"
)

// When a pod is added, set the defaults and enqueue the current mxjob.
func (tc *MXController) addMXJob(obj interface{}) {
	// Convert from unstructured object.
	mxJob, err := mxJobFromUnstructured(obj)
	if err != nil {
		un, ok := obj.(*metav1unstructured.Unstructured)
		logger := &log.Entry{}
		if ok {
			logger = mxlogger.LoggerForUnstructured(un, mxv1.Kind)
		}
		logger.Errorf("Failed to convert the MXJob: %v", err)
		// Log the failure to conditions.
		if err == errFailedMarshal {
			errMsg := fmt.Sprintf("Failed to marshal the object to MXJob; the spec is invalid: %v", err)
			logger.Warn(errMsg)
			// TODO(jlewi): v1 doesn't appear to define an error type.
			tc.Recorder.Event(un, v1.EventTypeWarning, failedMarshalMXJobReason, errMsg)

			status := mxv1.MXJobStatus{
				Conditions: []mxv1.MXJobCondition{
					{
						Type:               mxv1.MXJobFailed,
						Status:             v1.ConditionTrue,
						LastUpdateTime:     metav1.Now(),
						LastTransitionTime: metav1.Now(),
						Reason:             failedMarshalMXJobReason,
						Message:            errMsg,
					},
				},
			}
			statusMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&status)
			if err != nil {
				logger.Errorf("Could not covert the MXJobStatus to unstructured; %v", err)
				return
			}
			client, err := k8sutil.NewCRDRestClient(&mxv1.SchemeGroupVersion)
			if err == nil {
				if err1 := metav1unstructured.SetNestedField(un.Object, statusMap, "status"); err1 != nil {
					logger.Errorf("Could not set nested field: %v", err1)
				}
				logger.Infof("Updating the job to; %+v", un.Object)
				err = client.UpdateStatus(un, mxv1.Plural)
				if err != nil {
					logger.Errorf("Could not update the MXJob; %v", err)
				}
			} else {
				logger.Errorf("Could not create a REST client to update the MXJob")
			}
		}
		return
	}

	// Set default for the new mxjob.
	scheme.Scheme.Default(mxJob)

	msg := fmt.Sprintf("MXJob %s is created.", mxJob.Name)
	logger := mxlogger.LoggerForJob(mxJob)
	logger.Info(msg)

	// Add a created condition.
	err = updateMXJobConditions(mxJob, mxv1.MXJobCreated, mxJobCreatedReason, msg)
	if err != nil {
		logger.Errorf("Append mxJob condition error: %v", err)
		return
	}

	// Convert from mxjob object
	err = unstructuredFromMXJob(obj, mxJob)
	if err != nil {
		logger.Errorf("Failed to convert the obj: %v", err)
		return
	}
	tc.enqueueMXJob(obj)
}

// When a pod is updated, enqueue the current mxjob.
func (tc *MXController) updateMXJob(old, cur interface{}) {
	oldMXJob, err := mxJobFromUnstructured(old)
	if err != nil {
		return
	}
	curMXJob, err := mxJobFromUnstructured(cur)
	if err != nil {
		return
	}

	// never return error
	key, err := KeyFunc(curMXJob)
	if err != nil {
		return
	}

	log.Infof("Updating mxjob: %s", oldMXJob.Name)
	tc.enqueueMXJob(cur)

	// check if need to add a new rsync for ActiveDeadlineSeconds
	if curMXJob.Status.StartTime != nil {
		curMXJobADS := curMXJob.Spec.ActiveDeadlineSeconds
		if curMXJobADS == nil {
			return
		}
		oldMXJobADS := oldMXJob.Spec.ActiveDeadlineSeconds
		if oldMXJobADS == nil || *oldMXJobADS != *curMXJobADS {
			now := metav1.Now()
			start := curMXJob.Status.StartTime.Time
			passed := now.Time.Sub(start)
			total := time.Duration(*curMXJobADS) * time.Second
			// AddAfter will handle total < passed
			tc.WorkQueue.AddAfter(key, total-passed)
			log.Infof("job ActiveDeadlineSeconds updated, will rsync after %d seconds", total-passed)
		}
	}
}

func (tc *MXController) deletePodsAndServices(mxJob *mxv1.MXJob, pods []*v1.Pod) error {
	if len(pods) == 0 {
		return nil
	}

	// Delete nothing when the cleanPodPolicy is None.
	if *mxJob.Spec.CleanPodPolicy == mxv1.CleanPodPolicyNone {
		return nil
	}

	for _, pod := range pods {
		if *mxJob.Spec.CleanPodPolicy == mxv1.CleanPodPolicyRunning && pod.Status.Phase != v1.PodRunning {
			continue
		}

		if err := tc.PodControl.DeletePod(pod.Namespace, pod.Name, mxJob); err != nil {
			return err
		}
		// Pod and service have the same name, thus the service could be deleted using pod's name.
		if err := tc.ServiceControl.DeleteService(pod.Namespace, pod.Name, mxJob); err != nil {
			return err
		}
	}
	return nil
}

func (tc *MXController) cleanupMXJob(mxJob *mxv1.MXJob) error {
	currentTime := time.Now()
	ttl := mxJob.Spec.TTLSecondsAfterFinished
	if ttl == nil {
		// do nothing if the cleanup delay is not set
		return nil
	}
	duration := time.Second * time.Duration(*ttl)
	if currentTime.After(mxJob.Status.CompletionTime.Add(duration)) {
		err := tc.deleteMXJobHandler(mxJob)
		if err != nil {
			mxlogger.LoggerForJob(mxJob).Warnf("Cleanup MXJob error: %v.", err)
			return err
		}
		return nil
	}
	key, err := KeyFunc(mxJob)
	if err != nil {
		mxlogger.LoggerForJob(mxJob).Warnf("Couldn't get key for mxjob object: %v", err)
		return err
	}
	tc.WorkQueue.AddRateLimited(key)
	return nil
}

// deleteMXJob deletes the given MXJob.
func (tc *MXController) deleteMXJob(mxJob *mxv1.MXJob) error {
	return tc.mxJobClientSet.KubeflowV1beta1().MXJobs(mxJob.Namespace).Delete(mxJob.Name, &metav1.DeleteOptions{})
}

func getTotalReplicas(mxjob *mxv1.MXJob) int32 {
	mxjobReplicas := int32(0)
	for _, r := range mxjob.Spec.MXReplicaSpecs {
		mxjobReplicas += *r.Replicas
	}
	return mxjobReplicas
}

func getTotalFailedReplicas(mxjob *mxv1.MXJob) int32 {
	totalFailedReplicas := int32(0)
	for rtype := range mxjob.Status.MXReplicaStatuses {
		totalFailedReplicas += mxjob.Status.MXReplicaStatuses[rtype].Failed
	}
	return totalFailedReplicas
}
<commit_msg>fix wrong api version when delete mxjob (#46)<commit_after>package mxnet

import (
	"fmt"
	"time"

	log "github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	metav1unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/client-go/kubernetes/scheme"

	mxv1 "github.com/kubeflow/mxnet-operator/pkg/apis/mxnet/v1"
	"github.com/kubeflow/mxnet-operator/pkg/util/k8sutil"
	mxlogger "github.com/kubeflow/tf-operator/pkg/logger"
	"k8s.io/apimachinery/pkg/runtime"
)

const (
	failedMarshalMXJobReason = "FailedInvalidMXJobSpec"
	inspectFailMXJobReason   = "InspectFailedInvalidMXReplicaSpec"
)

// When a pod is added, set the defaults and enqueue the current mxjob.
func (tc *MXController) addMXJob(obj interface{}) {
	// Convert from unstructured object.
	mxJob, err := mxJobFromUnstructured(obj)
	if err != nil {
		un, ok := obj.(*metav1unstructured.Unstructured)
		logger := &log.Entry{}
		if ok {
			logger = mxlogger.LoggerForUnstructured(un, mxv1.Kind)
		}
		logger.Errorf("Failed to convert the MXJob: %v", err)
		// Log the failure to conditions.
		if err == errFailedMarshal {
			errMsg := fmt.Sprintf("Failed to marshal the object to MXJob; the spec is invalid: %v", err)
			logger.Warn(errMsg)
			// TODO(jlewi): v1 doesn't appear to define an error type.
			tc.Recorder.Event(un, v1.EventTypeWarning, failedMarshalMXJobReason, errMsg)

			status := mxv1.MXJobStatus{
				Conditions: []mxv1.MXJobCondition{
					{
						Type:               mxv1.MXJobFailed,
						Status:             v1.ConditionTrue,
						LastUpdateTime:     metav1.Now(),
						LastTransitionTime: metav1.Now(),
						Reason:             failedMarshalMXJobReason,
						Message:            errMsg,
					},
				},
			}
			statusMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&status)
			if err != nil {
				logger.Errorf("Could not covert the MXJobStatus to unstructured; %v", err)
				return
			}
			client, err := k8sutil.NewCRDRestClient(&mxv1.SchemeGroupVersion)
			if err == nil {
				if err1 := metav1unstructured.SetNestedField(un.Object, statusMap, "status"); err1 != nil {
					logger.Errorf("Could not set nested field: %v", err1)
				}
				logger.Infof("Updating the job to; %+v", un.Object)
				err = client.UpdateStatus(un, mxv1.Plural)
				if err != nil {
					logger.Errorf("Could not update the MXJob; %v", err)
				}
			} else {
				logger.Errorf("Could not create a REST client to update the MXJob")
			}
		}
		return
	}

	// Set default for the new mxjob.
	scheme.Scheme.Default(mxJob)

	msg := fmt.Sprintf("MXJob %s is created.", mxJob.Name)
	logger := mxlogger.LoggerForJob(mxJob)
	logger.Info(msg)

	// Add a created condition.
	err = updateMXJobConditions(mxJob, mxv1.MXJobCreated, mxJobCreatedReason, msg)
	if err != nil {
		logger.Errorf("Append mxJob condition error: %v", err)
		return
	}

	// Convert from mxjob object
	err = unstructuredFromMXJob(obj, mxJob)
	if err != nil {
		logger.Errorf("Failed to convert the obj: %v", err)
		return
	}
	tc.enqueueMXJob(obj)
}

// When a pod is updated, enqueue the current mxjob.
func (tc *MXController) updateMXJob(old, cur interface{}) {
	oldMXJob, err := mxJobFromUnstructured(old)
	if err != nil {
		return
	}
	curMXJob, err := mxJobFromUnstructured(cur)
	if err != nil {
		return
	}

	// never return error
	key, err := KeyFunc(curMXJob)
	if err != nil {
		return
	}

	log.Infof("Updating mxjob: %s", oldMXJob.Name)
	tc.enqueueMXJob(cur)

	// check if need to add a new rsync for ActiveDeadlineSeconds
	if curMXJob.Status.StartTime != nil {
		curMXJobADS := curMXJob.Spec.ActiveDeadlineSeconds
		if curMXJobADS == nil {
			return
		}
		oldMXJobADS := oldMXJob.Spec.ActiveDeadlineSeconds
		if oldMXJobADS == nil || *oldMXJobADS != *curMXJobADS {
			now := metav1.Now()
			start := curMXJob.Status.StartTime.Time
			passed := now.Time.Sub(start)
			total := time.Duration(*curMXJobADS) * time.Second
			// AddAfter will handle total < passed
			tc.WorkQueue.AddAfter(key, total-passed)
			log.Infof("job ActiveDeadlineSeconds updated, will rsync after %d seconds", total-passed)
		}
	}
}

func (tc *MXController) deletePodsAndServices(mxJob *mxv1.MXJob, pods []*v1.Pod) error {
	if len(pods) == 0 {
		return nil
	}

	// Delete nothing when the cleanPodPolicy is None.
	if *mxJob.Spec.CleanPodPolicy == mxv1.CleanPodPolicyNone {
		return nil
	}

	for _, pod := range pods {
		if *mxJob.Spec.CleanPodPolicy == mxv1.CleanPodPolicyRunning && pod.Status.Phase != v1.PodRunning {
			continue
		}

		if err := tc.PodControl.DeletePod(pod.Namespace, pod.Name, mxJob); err != nil {
			return err
		}
		// Pod and service have the same name, thus the service could be deleted using pod's name.
		if err := tc.ServiceControl.DeleteService(pod.Namespace, pod.Name, mxJob); err != nil {
			return err
		}
	}
	return nil
}

func (tc *MXController) cleanupMXJob(mxJob *mxv1.MXJob) error {
	currentTime := time.Now()
	ttl := mxJob.Spec.TTLSecondsAfterFinished
	if ttl == nil {
		// do nothing if the cleanup delay is not set
		return nil
	}
	duration := time.Second * time.Duration(*ttl)
	if currentTime.After(mxJob.Status.CompletionTime.Add(duration)) {
		err := tc.deleteMXJobHandler(mxJob)
		if err != nil {
			mxlogger.LoggerForJob(mxJob).Warnf("Cleanup MXJob error: %v.", err)
			return err
		}
		return nil
	}
	key, err := KeyFunc(mxJob)
	if err != nil {
		mxlogger.LoggerForJob(mxJob).Warnf("Couldn't get key for mxjob object: %v", err)
		return err
	}
	tc.WorkQueue.AddRateLimited(key)
	return nil
}

// deleteMXJob deletes the given MXJob.
func (tc *MXController) deleteMXJob(mxJob *mxv1.MXJob) error {
	return tc.mxJobClientSet.KubeflowV1().MXJobs(mxJob.Namespace).Delete(mxJob.Name, &metav1.DeleteOptions{})
}

func getTotalReplicas(mxjob *mxv1.MXJob) int32 {
	mxjobReplicas := int32(0)
	for _, r := range mxjob.Spec.MXReplicaSpecs {
		mxjobReplicas += *r.Replicas
	}
	return mxjobReplicas
}

func getTotalFailedReplicas(mxjob *mxv1.MXJob) int32 {
	totalFailedReplicas := int32(0)
	for rtype := range mxjob.Status.MXReplicaStatuses {
		totalFailedReplicas += mxjob.Status.MXReplicaStatuses[rtype].Failed
	}
	return totalFailedReplicas
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright 2017-2018 Authors of Cilium

package loader

import (
	"context"
	"fmt"
	"net"

	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/command/exec"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/mac"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/sysctl"

	"github.com/vishvananda/netlink"
)

type baseDeviceMode string

const (
	ipvlanMode = baseDeviceMode("ipvlan")
	directMode = baseDeviceMode("direct")
	tunnelMode = baseDeviceMode("tunnel")

	libbpfFixupMsg = "struct bpf_elf_map fixup performed due to size mismatch!"
)

func replaceQdisc(ifName string) error {
	link, err := netlink.LinkByName(ifName)
	if err != nil {
		return err
	}
	attrs := netlink.QdiscAttrs{
		LinkIndex: link.Attrs().Index,
		Handle:    netlink.MakeHandle(0xffff, 0),
		Parent:    netlink.HANDLE_CLSACT,
	}

	qdisc := &netlink.GenericQdisc{
		QdiscAttrs: attrs,
		QdiscType:  "clsact",
	}

	if err = netlink.QdiscReplace(qdisc); err != nil {
		return fmt.Errorf("netlink: Replacing qdisc for %s failed: %s", ifName, err)
	} else {
		log.Debugf("netlink: Replacing qdisc for %s succeeded", ifName)
	}

	return nil
}

// replaceDatapath the qdisc and BPF program for a endpoint or XDP program.
func replaceDatapath(ctx context.Context, ifName, objPath, progSec, progDirection string, xdp bool, xdpMode string) error {
	var (
		err        error
		loaderProg string
		args       []string
	)

	if !xdp {
		if err = replaceQdisc(ifName); err != nil {
			return fmt.Errorf("Failed to replace Qdisc for %s: %s", ifName, err)
		}
	}

	// FIXME: Replace cilium-map-migrate with Golang map migration
	cmd := exec.CommandContext(ctx, "cilium-map-migrate", "-s", objPath)
	cmd.Env = bpf.Environment()
	if _, err = cmd.CombinedOutput(log, true); err != nil {
		return err
	}
	defer func() {
		var retCode string
		if err == nil {
			retCode = "0"
		} else {
			retCode = "1"
		}
		args := []string{"-e", objPath, "-r", retCode}
		cmd := exec.CommandContext(ctx, "cilium-map-migrate", args...)
		cmd.Env = bpf.Environment()
		_, _ = cmd.CombinedOutput(log, true) // ignore errors
	}()

	// FIXME: replace exec with native call
	if xdp {
		loaderProg = "ip"
		args = []string{"-force", "link", "set", "dev", ifName, xdpMode,
			"obj", objPath, "sec", progSec}
	} else {
		loaderProg = "tc"
		args = []string{"filter", "replace", "dev", ifName, progDirection,
			"prio", "1", "handle", "1", "bpf", "da", "obj", objPath,
			"sec", progSec,
		}
	}
	cmd = exec.CommandContext(ctx, loaderProg, args...).WithFilters(libbpfFixupMsg)
	_, err = cmd.CombinedOutput(log, true)
	if err != nil {
		return fmt.Errorf("Failed to load prog with %s: %w", loaderProg, err)
	}

	return nil
}

// graftDatapath replaces obj in tail call map
func graftDatapath(ctx context.Context, mapPath, objPath, progSec string) error {
	var err error

	// FIXME: Replace cilium-map-migrate with Golang map migration
	cmd := exec.CommandContext(ctx, "cilium-map-migrate", "-s", objPath)
	cmd.Env = bpf.Environment()
	if _, err = cmd.CombinedOutput(log, true); err != nil {
		return err
	}
	defer func() {
		var retCode string
		if err == nil {
			retCode = "0"
		} else {
			retCode = "1"
		}
		args := []string{"-e", objPath, "-r", retCode}
		cmd := exec.CommandContext(ctx, "cilium-map-migrate", args...)
		cmd.Env = bpf.Environment()
		_, _ = cmd.CombinedOutput(log, true) // ignore errors
	}()

	// FIXME: replace exec with native call
	// FIXME: only key 0 right now, could be made more flexible
	args := []string{"exec", "bpf", "graft", mapPath, "key", "0",
		"obj", objPath, "sec", progSec,
	}
	cmd = exec.CommandContext(ctx, "tc", args...).WithFilters(libbpfFixupMsg)
	_, err = cmd.CombinedOutput(log, true)
	if err != nil {
		return fmt.Errorf("Failed to graft tc object: %s", err)
	}

	return nil
}

// RemoveTCFilters removes all tc filters from the given interface.
// Direction is passed as netlink.HANDLE_MIN_{INGRESS,EGRESS} via tcDir.
func RemoveTCFilters(ifName string, tcDir uint32) error {
	link, err := netlink.LinkByName(ifName)
	if err != nil {
		return err
	}

	filters, err := netlink.FilterList(link, tcDir)
	if err != nil {
		return err
	}

	for _, f := range filters {
		if err := netlink.FilterDel(f); err != nil {
			return err
		}
	}

	return nil
}

func setupDev(link netlink.Link) error {
	ifName := link.Attrs().Name

	if err := netlink.LinkSetUp(link); err != nil {
		log.WithError(err).WithField("device", ifName).Warn("Could not set up the link")
		return err
	}

	sysSettings := make([]sysctl.Setting, 0, 5)
	if option.Config.EnableIPv6 {
		sysSettings = append(sysSettings, sysctl.Setting{
			Name: fmt.Sprintf("net.ipv6.conf.%s.forwarding", ifName), Val: "1", IgnoreErr: false})
	}
	if option.Config.EnableIPv4 {
		sysSettings = append(sysSettings, []sysctl.Setting{
			{Name: fmt.Sprintf("net.ipv4.conf.%s.forwarding", ifName), Val: "1", IgnoreErr: false},
			{Name: fmt.Sprintf("net.ipv4.conf.%s.rp_filter", ifName), Val: "0", IgnoreErr: false},
			{Name: fmt.Sprintf("net.ipv4.conf.%s.accept_local", ifName), Val: "1", IgnoreErr: false},
			{Name: fmt.Sprintf("net.ipv4.conf.%s.send_redirects", ifName), Val: "0", IgnoreErr: false},
		}...)
	}
	if err := sysctl.ApplySettings(sysSettings); err != nil {
		return err
	}

	return nil
}

func setupVethPair(name, peerName string) error {
	// Create the veth pair if it doesn't exist.
	if _, err := netlink.LinkByName(name); err != nil {
		hostMac, err := mac.GenerateRandMAC()
		if err != nil {
			return err
		}
		peerMac, err := mac.GenerateRandMAC()
		if err != nil {
			return err
		}

		veth := &netlink.Veth{
			LinkAttrs: netlink.LinkAttrs{
				Name:         name,
				HardwareAddr: net.HardwareAddr(hostMac),
			},
			PeerName:         peerName,
			PeerHardwareAddr: net.HardwareAddr(peerMac),
		}
		if err := netlink.LinkAdd(veth); err != nil {
			return err
		}
	}

	veth, err := netlink.LinkByName(name)
	if err != nil {
		return err
	}
	if err := setupDev(veth); err != nil {
		return err
	}
	peer, err := netlink.LinkByName(peerName)
	if err != nil {
		return err
	}
	if err := setupDev(peer); err != nil {
		return err
	}

	return nil
}

func setupIpvlan(name string, nativeLink netlink.Link) (*netlink.IPVlan, error) {
	hostLink, err := netlink.LinkByName(name)
	if err == nil {
		// Ignore the error.
		netlink.LinkDel(hostLink)
	}

	ipvlan := &netlink.IPVlan{
		LinkAttrs: netlink.LinkAttrs{
			Name:        name,
			ParentIndex: nativeLink.Attrs().Index,
		},
		Mode: netlink.IPVLAN_MODE_L3,
	}
	if err := netlink.LinkAdd(ipvlan); err != nil {
		return nil, err
	}

	if err := setupDev(ipvlan); err != nil {
		return nil, err
	}

	return ipvlan, nil
}

// setupBaseDevice decides which and what kind of interfaces should be set up as
// the first step of datapath initialization, then performs the setup (and
// creation, if needed) of those interfaces. It returns two links and an error.
// By default, it sets up the veth pair - cilium_host and cilium_net.
// In ipvlan mode, it creates the cilium_host ipvlan with the native device as a
// parent.
func setupBaseDevice(nativeDevs []netlink.Link, mode baseDeviceMode, mtu int) (netlink.Link, netlink.Link, error) {
	switch mode {
	case ipvlanMode:
		ipvlan, err := setupIpvlan(defaults.HostDevice, nativeDevs[0])
		if err != nil {
			return nil, nil, err
		}
		if err := netlink.LinkSetMTU(ipvlan, mtu); err != nil {
			return nil, nil, err
		}

		return ipvlan, ipvlan, nil
	default:
		if err := setupVethPair(defaults.HostDevice, defaults.SecondHostDevice); err != nil {
			return nil, nil, err
		}

		linkHost, err := netlink.LinkByName(defaults.HostDevice)
		if err != nil {
			return nil, nil, err
		}
		linkNet, err := netlink.LinkByName(defaults.SecondHostDevice)
		if err != nil {
			return nil, nil, err
		}

		if err := netlink.LinkSetARPOff(linkHost); err != nil {
			return nil, nil, err
		}
		if err := netlink.LinkSetARPOff(linkNet); err != nil {
			return nil, nil, err
		}

		if err := netlink.LinkSetMTU(linkHost, mtu); err != nil {
			return nil, nil, err
		}
		if err := netlink.LinkSetMTU(linkNet, mtu); err != nil {
			return nil, nil, err
		}

		return linkHost, linkNet, nil
	}
}
<commit_msg>loader: replaceQdisc/Datapath - perform bpffs map migrations using Go<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright 2017-2018 Authors of Cilium

package loader

import (
	"context"
	"fmt"
	"net"

	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/command/exec"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/mac"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/sysctl"

	"github.com/vishvananda/netlink"
)

type baseDeviceMode string

const (
	ipvlanMode = baseDeviceMode("ipvlan")
	directMode = baseDeviceMode("direct")
	tunnelMode = baseDeviceMode("tunnel")

	libbpfFixupMsg = "struct bpf_elf_map fixup performed due to size mismatch!"
)

func replaceQdisc(ifName string) error {
	link, err := netlink.LinkByName(ifName)
	if err != nil {
		return err
	}
	attrs := netlink.QdiscAttrs{
		LinkIndex: link.Attrs().Index,
		Handle:    netlink.MakeHandle(0xffff, 0),
		Parent:    netlink.HANDLE_CLSACT,
	}

	qdisc := &netlink.GenericQdisc{
		QdiscAttrs: attrs,
		QdiscType:  "clsact",
	}

	if err = netlink.QdiscReplace(qdisc); err != nil {
		return fmt.Errorf("netlink: Replacing qdisc for %s failed: %s", ifName, err)
	} else {
		log.Debugf("netlink: Replacing qdisc for %s succeeded", ifName)
	}

	return nil
}

// replaceDatapath the qdisc and BPF program for a endpoint or XDP program.
func replaceDatapath(ctx context.Context, ifName, objPath, progSec, progDirection string, xdp bool, xdpMode string) error {
	var (
		loaderProg string
		args       []string
	)

	if !xdp {
		if err := replaceQdisc(ifName); err != nil {
			return fmt.Errorf("Failed to replace Qdisc for %s: %s", ifName, err)
		}
	}

	// Temporarily rename bpffs pins of maps whose definitions have changed in
	// a new version of a datapath ELF.
	if err := bpf.StartBPFFSMigration(bpf.GetMapRoot(), objPath); err != nil {
		return fmt.Errorf("Failed to start bpffs map migration: %w", err)
	}

	// Schedule finalizing the migration. If the iproute2 call below is successful,
	// any 'pending' pins will be removed. If not, any pending maps will be re-pinned
	// back to their initial paths.
	var revert bool
	defer func() {
		if err := bpf.FinalizeBPFFSMigration(bpf.GetMapRoot(), objPath, revert); err != nil {
			log.WithError(err).WithField("device", ifName).WithField("objPath", objPath).
				Error("Could not finalize bpffs map migration")
		}
	}()

	// FIXME: replace exec with native call
	if xdp {
		loaderProg = "ip"
		args = []string{"-force", "link", "set", "dev", ifName, xdpMode,
			"obj", objPath, "sec", progSec}
	} else {
		loaderProg = "tc"
		args = []string{"filter", "replace", "dev", ifName, progDirection,
			"prio", "1", "handle", "1", "bpf", "da", "obj", objPath,
			"sec", progSec,
		}
	}

	cmd := exec.CommandContext(ctx, loaderProg, args...).WithFilters(libbpfFixupMsg)
	if _, err := cmd.CombinedOutput(log, true); err != nil {
		// Program/object replacement unsuccessful, revert bpffs migration.
		revert = true
		return fmt.Errorf("Failed to load prog with %s: %w", loaderProg, err)
	}

	return nil
}

// graftDatapath replaces obj in tail call map
func graftDatapath(ctx context.Context, mapPath, objPath, progSec string) error {
	if err := bpf.StartBPFFSMigration(bpf.GetMapRoot(), objPath); err != nil {
		return fmt.Errorf("Failed to start bpffs map migration: %w", err)
	}

	var revert bool
	defer func() {
		if err := bpf.FinalizeBPFFSMigration(bpf.GetMapRoot(), objPath, revert); err != nil {
			log.WithError(err).WithField("mapPath", mapPath).WithField("objPath", objPath).
				Error("Could not finalize bpffs map migration")
		}
	}()

	// FIXME: replace exec with native call
	// FIXME: only key 0 right now, could be made more flexible
	args := []string{"exec", "bpf", "graft", mapPath, "key", "0",
		"obj", objPath, "sec", progSec,
	}
	cmd := exec.CommandContext(ctx, "tc", args...).WithFilters(libbpfFixupMsg)
	if _, err := cmd.CombinedOutput(log, true); err != nil {
		revert = true
		return fmt.Errorf("Failed to graft tc object: %s", err)
	}

	return nil
}

// RemoveTCFilters removes all tc filters from the given interface.
// Direction is passed as netlink.HANDLE_MIN_{INGRESS,EGRESS} via tcDir.
func RemoveTCFilters(ifName string, tcDir uint32) error {
	link, err := netlink.LinkByName(ifName)
	if err != nil {
		return err
	}

	filters, err := netlink.FilterList(link, tcDir)
	if err != nil {
		return err
	}

	for _, f := range filters {
		if err := netlink.FilterDel(f); err != nil {
			return err
		}
	}

	return nil
}

func setupDev(link netlink.Link) error {
	ifName := link.Attrs().Name

	if err := netlink.LinkSetUp(link); err != nil {
		log.WithError(err).WithField("device", ifName).Warn("Could not set up the link")
		return err
	}

	sysSettings := make([]sysctl.Setting, 0, 5)
	if option.Config.EnableIPv6 {
		sysSettings = append(sysSettings, sysctl.Setting{
			Name: fmt.Sprintf("net.ipv6.conf.%s.forwarding", ifName), Val: "1", IgnoreErr: false})
	}
	if option.Config.EnableIPv4 {
		sysSettings = append(sysSettings, []sysctl.Setting{
			{Name: fmt.Sprintf("net.ipv4.conf.%s.forwarding", ifName), Val: "1", IgnoreErr: false},
			{Name: fmt.Sprintf("net.ipv4.conf.%s.rp_filter", ifName), Val: "0", IgnoreErr: false},
			{Name: fmt.Sprintf("net.ipv4.conf.%s.accept_local", ifName), Val: "1", IgnoreErr: false},
			{Name: fmt.Sprintf("net.ipv4.conf.%s.send_redirects", ifName), Val: "0", IgnoreErr: false},
		}...)
	}
	if err := sysctl.ApplySettings(sysSettings); err != nil {
		return err
	}

	return nil
}

func setupVethPair(name, peerName string) error {
	// Create the veth pair if it doesn't exist.
	if _, err := netlink.LinkByName(name); err != nil {
		hostMac, err := mac.GenerateRandMAC()
		if err != nil {
			return err
		}
		peerMac, err := mac.GenerateRandMAC()
		if err != nil {
			return err
		}

		veth := &netlink.Veth{
			LinkAttrs: netlink.LinkAttrs{
				Name:         name,
				HardwareAddr: net.HardwareAddr(hostMac),
			},
			PeerName:         peerName,
			PeerHardwareAddr: net.HardwareAddr(peerMac),
		}
		if err := netlink.LinkAdd(veth); err != nil {
			return err
		}
	}

	veth, err := netlink.LinkByName(name)
	if err != nil {
		return err
	}
	if err := setupDev(veth); err != nil {
		return err
	}
	peer, err := netlink.LinkByName(peerName)
	if err != nil {
		return err
	}
	if err := setupDev(peer); err != nil {
		return err
	}

	return nil
}

func setupIpvlan(name string, nativeLink netlink.Link) (*netlink.IPVlan, error) {
	hostLink, err := netlink.LinkByName(name)
	if err == nil {
		// Ignore the error.
		netlink.LinkDel(hostLink)
	}

	ipvlan := &netlink.IPVlan{
		LinkAttrs: netlink.LinkAttrs{
			Name:        name,
			ParentIndex: nativeLink.Attrs().Index,
		},
		Mode: netlink.IPVLAN_MODE_L3,
	}
	if err := netlink.LinkAdd(ipvlan); err != nil {
		return nil, err
	}

	if err := setupDev(ipvlan); err != nil {
		return nil, err
	}

	return ipvlan, nil
}

// setupBaseDevice decides which and what kind of interfaces should be set up as
// the first step of datapath initialization, then performs the setup (and
// creation, if needed) of those interfaces. It returns two links and an error.
// By default, it sets up the veth pair - cilium_host and cilium_net.
// In ipvlan mode, it creates the cilium_host ipvlan with the native device as a
// parent.
func setupBaseDevice(nativeDevs []netlink.Link, mode baseDeviceMode, mtu int) (netlink.Link, netlink.Link, error) {
	switch mode {
	case ipvlanMode:
		ipvlan, err := setupIpvlan(defaults.HostDevice, nativeDevs[0])
		if err != nil {
			return nil, nil, err
		}
		if err := netlink.LinkSetMTU(ipvlan, mtu); err != nil {
			return nil, nil, err
		}

		return ipvlan, ipvlan, nil
	default:
		if err := setupVethPair(defaults.HostDevice, defaults.SecondHostDevice); err != nil {
			return nil, nil, err
		}

		linkHost, err := netlink.LinkByName(defaults.HostDevice)
		if err != nil {
			return nil, nil, err
		}
		linkNet, err := netlink.LinkByName(defaults.SecondHostDevice)
		if err != nil {
			return nil, nil, err
		}

		if err := netlink.LinkSetARPOff(linkHost); err != nil {
			return nil, nil, err
		}
		if err := netlink.LinkSetARPOff(linkNet); err != nil {
			return nil, nil, err
		}

		if err := netlink.LinkSetMTU(linkHost, mtu); err != nil {
			return nil, nil, err
		}
		if err := netlink.LinkSetMTU(linkNet, mtu); err != nil {
			return nil, nil, err
		}

		return linkHost, linkNet, nil
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubectl

import (
	"fmt"
	"io"
	"reflect"
	"sort"

	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/api/v1"
	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/integer"
	"k8s.io/kubernetes/pkg/util/jsonpath"

	"github.com/golang/glog"
)

// Sorting printer sorts list types before delegating to another printer.
// Non-list types are simply passed through
type SortingPrinter struct {
	SortField string
	Delegate  ResourcePrinter
	Decoder   runtime.Decoder
}

func (s *SortingPrinter) AfterPrint(w io.Writer, res string) error {
	return nil
}

func (s *SortingPrinter) PrintObj(obj runtime.Object, out io.Writer) error {
	if !meta.IsListType(obj) {
		return s.Delegate.PrintObj(obj, out)
	}

	if err := s.sortObj(obj); err != nil {
		return err
	}
	return s.Delegate.PrintObj(obj, out)
}

// TODO: implement HandledResources()
func (p *SortingPrinter) HandledResources() []string {
	return []string{}
}

func (s *SortingPrinter) sortObj(obj runtime.Object) error {
	objs, err := meta.ExtractList(obj)
	if err != nil {
		return err
	}
	if len(objs) == 0 {
		return nil
	}

	sorter, err := SortObjects(s.Decoder, objs, s.SortField)
	if err != nil {
		return err
	}

	switch list := obj.(type) {
	case *v1.List:
		outputList := make([]runtime.RawExtension, len(objs))
		for ix := range objs {
			outputList[ix] = list.Items[sorter.OriginalPosition(ix)]
		}
		list.Items = outputList
		return nil
	}
	return meta.SetList(obj, objs)
}

func SortObjects(decoder runtime.Decoder, objs []runtime.Object, fieldInput string) (*RuntimeSort, error) {
	parser := jsonpath.New("sorting")

	field, err := massageJSONPath(fieldInput)
	if err != nil {
		return nil, err
	}

	if err := parser.Parse(field); err != nil {
		return nil, err
	}

	for ix := range objs {
		item := objs[ix]
		switch u := item.(type) {
		case *runtime.Unknown:
			var err error
			if objs[ix], _, err = decoder.Decode(u.Raw, nil, nil); err != nil {
				return nil, err
			}
		}
	}

	var values [][]reflect.Value
	if unstructured, ok := objs[0].(*runtime.Unstructured); ok {
		values, err = parser.FindResults(unstructured.Object)
	} else {
		values, err = parser.FindResults(reflect.ValueOf(objs[0]).Elem().Interface())
	}

	if err != nil {
		return nil, err
	}
	if len(values) == 0 {
		return nil, fmt.Errorf("couldn't find any field with path: %s", field)
	}

	sorter := NewRuntimeSort(field, objs)
	sort.Sort(sorter)
	return sorter, nil
}

// RuntimeSort is an implementation of the golang sort interface that knows how to sort
// lists of runtime.Object
type RuntimeSort struct {
	field        string
	objs         []runtime.Object
	origPosition []int
}

func NewRuntimeSort(field string, objs []runtime.Object) *RuntimeSort {
	sorter := &RuntimeSort{field: field, objs: objs, origPosition: make([]int, len(objs))}
	for ix := range objs {
		sorter.origPosition[ix] = ix
	}
	return sorter
}

func (r *RuntimeSort) Len() int {
	return len(r.objs)
}

func (r *RuntimeSort) Swap(i, j int) {
	r.objs[i], r.objs[j] = r.objs[j], r.objs[i]
	r.origPosition[i], r.origPosition[j] = r.origPosition[j], r.origPosition[i]
}

func isLess(i, j reflect.Value) (bool, error) {
	switch i.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return i.Int() < j.Int(), nil
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return i.Uint() < j.Uint(), nil
	case reflect.Float32, reflect.Float64:
		return i.Float() < j.Float(), nil
	case reflect.String:
		return i.String() < j.String(), nil
	case reflect.Ptr:
		return isLess(i.Elem(), j.Elem())
	case reflect.Struct:
		// sort metav1.Time
		in := i.Interface()
		if t, ok := in.(metav1.Time); ok {
			return t.Before(j.Interface().(metav1.Time)), nil
		}
		// fallback to the fields comparison
		for idx := 0; idx < i.NumField(); idx++ {
			less, err := isLess(i.Field(idx), j.Field(idx))
			if err != nil || !less {
				return less, err
			}
		}
		return true, nil
	case reflect.Array, reflect.Slice:
		// note: the length of i and j may be different
		for idx := 0; idx < integer.IntMin(i.Len(), j.Len()); idx++ {
			less, err := isLess(i.Index(idx), j.Index(idx))
			if err != nil || !less {
				return less, err
			}
		}
		return true, nil

	case reflect.Interface:
		switch itype := i.Interface().(type) {
		case uint8:
			if jtype, ok := j.Interface().(uint8); ok {
				return itype < jtype, nil
			}
		case uint16:
			if jtype, ok := j.Interface().(uint16); ok {
				return itype < jtype, nil
			}
		case uint32:
			if jtype, ok := j.Interface().(uint32); ok {
				return itype < jtype, nil
			}
		case uint64:
			if jtype, ok := j.Interface().(uint64); ok {
				return itype < jtype, nil
			}
		case int8:
			if jtype, ok := j.Interface().(int8); ok {
				return itype < jtype, nil
			}
		case int16:
			if jtype, ok := j.Interface().(int16); ok {
				return itype < jtype, nil
			}
		case int32:
			if jtype, ok := j.Interface().(int32); ok {
				return itype < jtype, nil
			}
		case int64:
			if jtype, ok := j.Interface().(int64); ok {
				return itype < jtype, nil
			}
		case uint:
			if jtype, ok := j.Interface().(uint); ok {
				return itype < jtype, nil
			}
		case int:
			if jtype, ok := j.Interface().(int); ok {
				return itype < jtype, nil
			}
		case float32:
			if jtype, ok := j.Interface().(float32); ok {
				return itype < jtype, nil
			}
		case float64:
			if jtype, ok := j.Interface().(float64); ok {
				return itype < jtype, nil
			}
		case string:
			if jtype, ok := j.Interface().(string); ok {
				return itype < jtype, nil
			}
		default:
			return false, fmt.Errorf("unsortable type: %T", itype)
		}
		return false, fmt.Errorf("unsortable interface: %v", i.Kind())

	default:
		return false, fmt.Errorf("unsortable type: %v", i.Kind())
	}
}

func (r *RuntimeSort) Less(i, j int) bool {
	iObj := r.objs[i]
	jObj := r.objs[j]

	parser := jsonpath.New("sorting")
	parser.Parse(r.field)

	var iValues [][]reflect.Value
	var jValues [][]reflect.Value
	var err error

	if unstructured, ok := iObj.(*runtime.Unstructured); ok {
		iValues, err = parser.FindResults(unstructured.Object)
	} else {
		iValues, err = parser.FindResults(reflect.ValueOf(iObj).Elem().Interface())
	}
	if err != nil {
		glog.Fatalf("Failed to get i values for %#v using %s (%#v)", iObj, r.field, err)
	}

	if unstructured, ok := jObj.(*runtime.Unstructured); ok {
		jValues, err = parser.FindResults(unstructured.Object)
	} else {
		jValues, err = parser.FindResults(reflect.ValueOf(jObj).Elem().Interface())
	}
	if err != nil {
		glog.Fatalf("Failed to get j values for %#v using %s (%v)", jObj, r.field, err)
	}

	iField := iValues[0][0]
	jField := jValues[0][0]

	less, err := isLess(iField, jField)
	if err != nil {
		glog.Fatalf("Field %s in %v is an unsortable type: %s, err: %v", r.field, iObj, iField.Kind().String(), err)
	}
	return less
}

// Returns the starting (original) position of a particular index.  e.g. If OriginalPosition(0) returns 5 than the
// the item currently at position 0 was at position 5 in the original unsorted array.
func (r *RuntimeSort) OriginalPosition(ix int) int {
	if ix < 0 || ix > len(r.origPosition) {
		return -1
	}
	return r.origPosition[ix]
}
<commit_msg>Make sorting work on versioned objects<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubectl

import (
	"fmt"
	"io"
	"reflect"
	"sort"

	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/api/v1"
	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/integer"
	"k8s.io/kubernetes/pkg/util/jsonpath"

	"github.com/golang/glog"
)

// Sorting printer sorts list types before delegating to another printer.
// Non-list types are simply passed through
type SortingPrinter struct {
	SortField string
	Delegate  ResourcePrinter
	Decoder   runtime.Decoder
}

func (s *SortingPrinter) AfterPrint(w io.Writer, res string) error {
	return nil
}

func (s *SortingPrinter) PrintObj(obj runtime.Object, out io.Writer) error {
	if !meta.IsListType(obj) {
		return s.Delegate.PrintObj(obj, out)
	}

	if err := s.sortObj(obj); err != nil {
		return err
	}
	return s.Delegate.PrintObj(obj, out)
}

// TODO: implement HandledResources()
func (p *SortingPrinter) HandledResources() []string {
	return []string{}
}

func (s *SortingPrinter) sortObj(obj runtime.Object) error {
	objs, err := meta.ExtractList(obj)
	if err != nil {
		return err
	}
	if len(objs) == 0 {
		return nil
	}

	sorter, err := SortObjects(s.Decoder, objs, s.SortField)
	if err != nil {
		return err
	}

	switch list := obj.(type) {
	case *v1.List:
		outputList := make([]runtime.RawExtension, len(objs))
		for ix := range objs {
			outputList[ix] = list.Items[sorter.OriginalPosition(ix)]
		}
		list.Items = outputList
		return nil
	}
	return meta.SetList(obj, objs)
}

func SortObjects(decoder runtime.Decoder, objs []runtime.Object, fieldInput string) (*RuntimeSort, error) {
	parser := jsonpath.New("sorting")

	field, err := massageJSONPath(fieldInput)
	if err != nil {
		return nil, err
	}

	if err := parser.Parse(field); err != nil {
		return nil, err
	}

	for ix := range objs {
		item := objs[ix]
		switch u := item.(type) {
		case *runtime.Unknown:
			var err error
			// decode runtime.Unknown to runtime.Unstructured for sorting.
			// we don't actually want the internal versions of known types.
			if objs[ix], _, err = decoder.Decode(u.Raw, nil, &runtime.Unstructured{}); err != nil {
				return nil, err
			}
		}
	}

	var values [][]reflect.Value
	if unstructured, ok := objs[0].(*runtime.Unstructured); ok {
		values, err = parser.FindResults(unstructured.Object)
	} else {
		values, err = parser.FindResults(reflect.ValueOf(objs[0]).Elem().Interface())
	}

	if err != nil {
		return nil, err
	}
	if len(values) == 0 {
		return nil, fmt.Errorf("couldn't find any field with path: %s", field)
	}

	sorter := NewRuntimeSort(field, objs)
	sort.Sort(sorter)
	return sorter, nil
}

// RuntimeSort is an implementation of the golang sort interface that knows how to sort
// lists of runtime.Object
type RuntimeSort struct {
	field        string
	objs         []runtime.Object
	origPosition []int
}

func NewRuntimeSort(field string, objs []runtime.Object) *RuntimeSort {
	sorter := &RuntimeSort{field: field, objs: objs, origPosition: make([]int, len(objs))}
	for ix := range objs {
		sorter.origPosition[ix] = ix
	}
	return sorter
}

func (r *RuntimeSort) Len() int {
	return len(r.objs)
}

func (r *RuntimeSort) Swap(i, j int) {
	r.objs[i], r.objs[j] = r.objs[j], r.objs[i]
	r.origPosition[i], r.origPosition[j] = r.origPosition[j], r.origPosition[i]
}

func isLess(i, j reflect.Value) (bool, error) {
	switch i.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return i.Int() < j.Int(), nil
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return i.Uint() < j.Uint(), nil
	case reflect.Float32, reflect.Float64:
		return i.Float() < j.Float(), nil
	case reflect.String:
		return i.String() < j.String(), nil
	case reflect.Ptr:
		return isLess(i.Elem(), j.Elem())
	case reflect.Struct:
		// sort metav1.Time
		in := i.Interface()
		if t, ok := in.(metav1.Time); ok {
			return t.Before(j.Interface().(metav1.Time)), nil
		}
		// fallback to the fields comparison
		for idx := 0; idx < i.NumField(); idx++ {
			less, err := isLess(i.Field(idx), j.Field(idx))
			if err != nil || !less {
				return less, err
			}
		}
		return true, nil
	case reflect.Array, reflect.Slice:
		// note: the length of i and j may be different
		for idx := 0; idx < integer.IntMin(i.Len(), j.Len()); idx++ {
			less, err := isLess(i.Index(idx), j.Index(idx))
			if err != nil || !less {
				return less, err
			}
		}
		return true, nil

	case reflect.Interface:
		switch itype := i.Interface().(type) {
		case uint8:
			if jtype, ok := j.Interface().(uint8); ok {
				return itype < jtype, nil
			}
		case uint16:
			if jtype, ok := j.Interface().(uint16); ok {
				return itype < jtype, nil
			}
		case uint32:
			if jtype, ok := j.Interface().(uint32); ok {
				return itype < jtype, nil
			}
		case uint64:
			if jtype, ok := j.Interface().(uint64); ok {
				return itype < jtype, nil
			}
		case int8:
			if jtype, ok := j.Interface().(int8); ok {
				return itype < jtype, nil
			}
		case int16:
			if jtype, ok := j.Interface().(int16); ok {
				return itype < jtype, nil
			}
		case int32:
			if jtype, ok := j.Interface().(int32); ok {
				return itype < jtype, nil
			}
		case int64:
			if jtype, ok := j.Interface().(int64); ok {
				return itype < jtype, nil
			}
		case uint:
			if jtype, ok := j.Interface().(uint); ok {
				return itype < jtype, nil
			}
		case int:
			if jtype, ok := j.Interface().(int); ok {
				return itype < jtype, nil
			}
		case float32:
			if jtype, ok := j.Interface().(float32); ok {
				return itype < jtype, nil
			}
		case float64:
			if jtype, ok := j.Interface().(float64); ok {
				return itype < jtype, nil
			}
		case string:
			if jtype, ok := j.Interface().(string); ok {
				return itype < jtype, nil
			}
		default:
			return false, fmt.Errorf("unsortable type: %T", itype)
		}
		return false, fmt.Errorf("unsortable interface: %v", i.Kind())

	default:
		return false, fmt.Errorf("unsortable type: %v", i.Kind())
	}
}

func (r *RuntimeSort) Less(i, j int) bool {
	iObj := r.objs[i]
	jObj := r.objs[j]

	parser := jsonpath.New("sorting")
	parser.Parse(r.field)

	var iValues [][]reflect.Value
	var jValues [][]reflect.Value
	var err error

	if unstructured, ok := iObj.(*runtime.Unstructured); ok {
		iValues, err = parser.FindResults(unstructured.Object)
	} else {
		iValues, err = parser.FindResults(reflect.ValueOf(iObj).Elem().Interface())
	}
	if err != nil {
		glog.Fatalf("Failed to get i values for %#v using %s (%#v)", iObj, r.field, err)
	}

	if unstructured, ok := jObj.(*runtime.Unstructured); ok {
		jValues, err = parser.FindResults(unstructured.Object)
	} else {
		jValues, err = parser.FindResults(reflect.ValueOf(jObj).Elem().Interface())
	}
	if err != nil {
		glog.Fatalf("Failed to get j values for %#v using %s (%v)", jObj, r.field, err)
	}

	iField := iValues[0][0]
	jField := jValues[0][0]

	less, err := isLess(iField, jField)
	if err != nil {
		glog.Fatalf("Field %s in %v is an unsortable type: %s, err: %v", r.field, iObj, iField.Kind().String(), err)
	}
	return less
}

// Returns the starting (original) position of a particular index.  e.g. If OriginalPosition(0) returns 5 than the
// the item currently at position 0 was at position 5 in the original unsorted array.
func (r *RuntimeSort) OriginalPosition(ix int) int {
	if ix < 0 || ix > len(r.origPosition) {
		return -1
	}
	return r.origPosition[ix]
}
<|endoftext|>


<commit_before>// Copyright 2015-present Oursky Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package skyconfig

import (
	"errors"
	"fmt"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/joho/godotenv"
	"github.com/skygeario/skygear-server/pkg/server/uuid"
)

type PluginConfig struct {
	Transport string
	Path      string
	Args      []string
}

// Configuration is Skygear's configuration
// The configuration will load in following order:
// 1. The ENV
// 2. The key/value in .env file
// 3. The config in *.ini (To-be depreacted)
type Configuration struct {
	HTTP struct {
		Host string `json:"host"`
	} `json:"http"`
	App struct {
		Name          string `json:"name"`
		APIKey        string `json:"api_key"`
		MasterKey     string `json:"master_key"`
		AccessControl string `json:"access_control"`
		DevMode       bool   `json:"dev_mode"`
		CORSHost      string `json:"cors_host"`
		Slave         bool   `json:"slave"`
	} `json:"app"`
	DB struct {
		ImplName string `json:"implementation"`
		Option   string `json:"option"`
	} `json:"database"`
	TokenStore struct {
		ImplName string `json:"implementation"`
		Path     string `json:"path"`
		Prefix   string `json:"prefix"`
		Expiry   int64  `json:"expiry"`
		Secret   string `json:"secret"`
	} `json:"-"`
	AssetStore struct {
		ImplName string `json:"implementation"`
		Public   bool   `json:"public"`

		// followings only used when ImplName = fs
		Path string `json:"-"`

		// followings only used when ImplName = s3
		AccessToken string `json:"access_key"`
		SecretToken string `json:"secret_key"`
		Region      string `json:"region"`
		Bucket      string `json:"bucket"`

		// followings only used when ImplName = cloud
		CloudAssetHost          string `json:"cloud_asset_host"`
		CloudAssetToken         string `json:"cloud_asset_token"`
		CloudAssetPublicPrefix  string `json:"cloud_asset_public_prefix"`
		CloudAssetPrivatePrefix string `json:"cloud_asset_private_prefix"`
	} `json:"asset_store"`
	AssetURLSigner struct {
		URLPrefix string `json:"url_prefix"`
		Secret    string `json:"secret"`
	} `json:"asset_signer"`
	APNS struct {
		Enable   bool   `json:"enable"`
		Env      string `json:"env"`
		Cert     string `json:"cert"`
		Key      string `json:"key"`
		CertPath string `json:"-"`
		KeyPath  string `json:"-"`
	} `json:"apns"`
	GCM struct {
		Enable bool   `json:"enable"`
		APIKey string `json:"api_key"`
	} `json:"gcm"`
	LOG struct {
		Level        string            `json:"-"`
		LoggersLevel map[string]string `json:"-"`
	} `json:"log"`
	LogHook struct {
		SentryDSN   string
		SentryLevel string
	} `json:"-"`
	Plugin map[string]*PluginConfig `json:"-"`
}

func NewConfiguration() Configuration {
	config := Configuration{}
	config.HTTP.Host = ":3000"
	config.App.Name = "myapp"
	config.App.AccessControl = "role"
	config.App.DevMode = true
	config.App.CORSHost = "*"
	config.App.Slave = false
	config.DB.ImplName = "pq"
	config.DB.Option = "postgres://postgres:@localhost/postgres?sslmode=disable"
	config.TokenStore.ImplName = "fs"
	config.TokenStore.Path = "data/token"
	config.TokenStore.Expiry = 0
	config.AssetStore.ImplName = "fs"
	config.AssetStore.Path = "data/asset"
	config.AssetURLSigner.URLPrefix = "http://localhost:3000/files"
	config.APNS.Enable = false
	config.APNS.Env = "sandbox"
	config.GCM.Enable = false
	config.LOG.Level = "debug"
	config.LOG.LoggersLevel = map[string]string{
		"plugin": "info",
	}
	config.Plugin = map[string]*PluginConfig{}
	return config
}

func NewConfigurationWithKeys() Configuration {
	config := NewConfiguration()
	config.App.APIKey = uuid.New()
	config.App.MasterKey = uuid.New()
	return config
}

func (config *Configuration) Validate() error {
	if config.App.Name == "" {
		return errors.New("APP_NAME is not set")
	}
	if config.App.APIKey == "" {
		return errors.New("API_KEY is not set")
	}
	if config.App.MasterKey == "" {
		return errors.New("MASTER_KEY is not set")
	}
	if !regexp.MustCompile("^[A-Za-z0-9_]+$").MatchString(config.App.Name) {
		return fmt.Errorf("APP_NAME '%s' contains invalid characters other than alphanumerics or underscores", config.App.Name)
	}
	if config.APNS.Enable && !regexp.MustCompile("^(sandbox|production)$").MatchString(config.APNS.Env) {
		return fmt.Errorf("APNS_ENV must be sandbox or production")
	}
	return nil
}

func (config *Configuration) ReadFromEnv() {
	envErr := godotenv.Load()
	if envErr != nil {
		log.Print("Error in loading .env file")
	}

	config.readHost()

	appAPIKey := os.Getenv("API_KEY")
	if appAPIKey != "" {
		config.App.APIKey = appAPIKey
	}

	appMasterKey := os.Getenv("MASTER_KEY")
	if appMasterKey != "" {
		config.App.MasterKey = appMasterKey
	}

	appName := os.Getenv("APP_NAME")
	if appName != "" {
		config.App.Name = appName
	}

	corsHost := os.Getenv("CORS_HOST")
	if corsHost != "" {
		config.App.CORSHost = corsHost
	}

	accessControl := os.Getenv("ACCESS_CONRTOL")
	if accessControl != "" {
		config.App.AccessControl = accessControl
	}

	DevMode := os.Getenv("DEV_MODE")
	if DevMode == "YES" {
		config.App.DevMode = true
	}
	if DevMode == "NO" {
		config.App.DevMode = false
	}

	dbImplName := os.Getenv("DB_IMPL_NAME")
	if dbImplName != "" {
		config.DB.ImplName = dbImplName
	}

	if config.DB.ImplName == "pq" && os.Getenv("DATABASE_URL") != "" {
		config.DB.Option = os.Getenv("DATABASE_URL")
	}

	slave := os.Getenv("SLAVE")
	if slave == "YES" {
		config.App.Slave = true
	} else if slave == "NO" {
		config.App.Slave = false
	} else if slaveBool, err := strconv.ParseBool(slave); err == nil {
		config.App.Slave = slaveBool
	}

	config.readTokenStore()
	config.readAssetStore()
	config.readAPNS()
	config.readGCM()
	config.readLog()
	config.readPlugins()
}

func (config *Configuration) readHost() {
	// Default to :3000 if both HOST and PORT is missing
	host := os.Getenv("HOST")
	if host != "" {
		config.HTTP.Host = host
	}
	if config.HTTP.Host == "" {
		port := os.Getenv("PORT")
		if port != "" {
			config.HTTP.Host = ":" + port
		}
	}
}

func (config *Configuration) readTokenStore() {
	tokenStore := os.Getenv("TOKEN_STORE")
	if tokenStore != "" {
		config.TokenStore.ImplName = tokenStore
	}
	tokenStorePath := os.Getenv("TOKEN_STORE_PATH")
	if tokenStorePath != "" {
		config.TokenStore.Path = tokenStorePath
	}

	tokenStorePrefix := os.Getenv("TOKEN_STORE_PREFIX")
	if tokenStorePrefix != "" {
		config.TokenStore.Prefix = tokenStorePrefix
	}

	tokenStoreExpiry := os.Getenv("TOKEN_STORE_EXPIRY")
	if tokenStoreExpiry != "" {
		expiry, err := strconv.ParseInt(tokenStoreExpiry, 10, 64)
		if err == nil {
			config.TokenStore.Expiry = expiry
		}
	}

	tokenStoreSecret := os.Getenv("TOKEN_STORE_SECRET")
	if tokenStoreSecret != "" {
		config.TokenStore.Secret = tokenStoreSecret
	} else {
		config.TokenStore.Secret = config.App.MasterKey
	}
}

func (config *Configuration) readAssetStore() {
	assetStore := os.Getenv("ASSET_STORE")
	if assetStore != "" {
		config.AssetStore.ImplName = assetStore
	}

	assetStorePublic := os.Getenv("ASSET_STORE_PUBLIC")
	if assetStorePublic != "" {
		config.AssetStore.Public = assetStorePublic == "YES" || assetStorePublic == "1"
	}

	// Local Storage related
	assetStorePath := os.Getenv("ASSET_STORE_PATH")
	if assetStorePath != "" {
		config.AssetStore.Path = assetStorePath
	}
	assetStorePrefix := os.Getenv("ASSET_STORE_URL_PREFIX")
	if assetStorePrefix != "" {
		config.AssetURLSigner.URLPrefix = assetStorePrefix
	}
	assetStoreSecret := os.Getenv("ASSET_STORE_SECRET")
	if assetStoreSecret != "" {
		config.AssetURLSigner.Secret = assetStoreSecret
	}

	// S3 related
	assetStoreAccessKey := os.Getenv("ASSET_STORE_ACCESS_KEY")
	if assetStoreAccessKey != "" {
		config.AssetStore.AccessToken = assetStoreAccessKey
	}
	assetStoreSecretKey := os.Getenv("ASSET_STORE_SECRET_KEY")
	if assetStoreSecretKey != "" {
		config.AssetStore.SecretToken = assetStoreSecretKey
	}
	assetStoreRegion := os.Getenv("ASSET_STORE_REGION")
	if assetStoreRegion != "" {
		config.AssetStore.Region = assetStoreRegion
	}
	assetStoreBucket := os.Getenv("ASSET_STORE_BUCKET")
	if assetStoreBucket != "" {
		config.AssetStore.Bucket = assetStoreBucket
	}

	// Cloud Asset related
	cloudAssetHost := os.Getenv("CLOUD_ASSET_HOST")
	if cloudAssetHost != "" {
		config.AssetStore.CloudAssetHost = cloudAssetHost
	}
	cloudAssetToken := os.Getenv("CLOUD_ASSET_TOKEN")
	if cloudAssetToken != "" {
		config.AssetStore.CloudAssetToken = cloudAssetToken
	}
	cloudAssetPublicPrefix := os.Getenv("CLOUD_ASSET_PUBLIC_PREFIX")
	if cloudAssetPublicPrefix != "" {
		config.AssetStore.CloudAssetPublicPrefix = cloudAssetPublicPrefix
	}
	cloudAssetPrivatePrefix := os.Getenv("CLOUD_ASSET_PRIVATE_PREFIX")
	if cloudAssetPrivatePrefix != "" {
		config.AssetStore.CloudAssetPrivatePrefix = cloudAssetPrivatePrefix
	}
}

func (config *Configuration) readAPNS() {
	shouldEnableAPNS := os.Getenv("APNS_ENABLE")
	if shouldEnableAPNS != "" {
		config.APNS.Enable = shouldEnableAPNS == "1" || shouldEnableAPNS == "YES"
	}

	if !config.APNS.Enable {
		return
	}

	env := os.Getenv("APNS_ENV")
	if env != "" {
		config.APNS.Env = env
	}

	cert, key := os.Getenv("APNS_CERTIFICATE"), os.Getenv("APNS_PRIVATE_KEY")
	if cert != "" {
		config.APNS.Cert = cert
	}
	if key != "" {
		config.APNS.Key = key
	}

	certPath, keyPath := os.Getenv("APNS_CERTIFICATE_PATH"), os.Getenv("APNS_PRIVATE_KEY_PATH")
	if certPath != "" {
		config.APNS.CertPath = certPath
	}
	if keyPath != "" {
		config.APNS.KeyPath = keyPath
	}

}

func (config *Configuration) readGCM() {
	shouldEnableGCM := os.Getenv("GCM_ENABLE")
	if shouldEnableGCM != "" {
		config.GCM.Enable = shouldEnableGCM == "1" || shouldEnableGCM == "YES"
	}

	gcmAPIKey := os.Getenv("GCM_APIKEY")
	if gcmAPIKey != "" {
		config.GCM.APIKey = gcmAPIKey
	}
}

func (config *Configuration) readLog() {
	logLevel := os.Getenv("LOG_LEVEL")
	if logLevel != "" {
		config.LOG.Level = logLevel
	}

	for _, environ := range os.Environ() {
		if !strings.HasPrefix(environ, "LOG_LEVEL_") {
			continue
		}

		components := strings.SplitN(environ, "=", 2)
		loggerName := strings.ToLower(strings.TrimPrefix(components[0], "LOG_LEVEL_"))
		loggerLevel := components[1]
		config.LOG.LoggersLevel[loggerName] = loggerLevel
	}

	sentry := os.Getenv("SENTRY_DSN")
	if sentry != "" {
		config.LogHook.SentryDSN = sentry
	}

	sentryLevel := os.Getenv("SENTRY_LEVEL")
	if sentryLevel != "" {
		config.LogHook.SentryLevel = sentryLevel
	}
}

func (config *Configuration) readPlugins() {
	plugin := os.Getenv("PLUGINS")
	if plugin == "" {
		return
	}

	plugins := strings.Split(plugin, ",")
	for _, p := range plugins {
		pluginConfig := &PluginConfig{}
		pluginConfig.Transport = os.Getenv(p + "_TRANSPORT")
		pluginConfig.Path = os.Getenv(p + "_PATH")
		args := os.Getenv(p + "_ARGS")
		if args != "" {
			pluginConfig.Args = strings.Split(args, ",")
		}
		config.Plugin[p] = pluginConfig
	}
}
<commit_msg>Make consistent parsing of envvar bools<commit_after>// Copyright 2015-present Oursky Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package skyconfig

import (
	"errors"
	"fmt"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/joho/godotenv"
	"github.com/skygeario/skygear-server/pkg/server/uuid"
)

// parseBool parses a string representation of boolean value into a boolean
// type. In addition to what strconv.ParseBool recognize, it also recognize
// Yes, yes, YES, y, No, no, NO, n. Any other value returns an error.
func parseBool(str string) (bool, error) {
	switch str {
	case "Yes", "yes", "YES", "y":
		return true, nil
	case "No", "no", "NO", "n":
		return false, nil
	default:
		return strconv.ParseBool(str)
	}
}

type PluginConfig struct {
	Transport string
	Path      string
	Args      []string
}

// Configuration is Skygear's configuration
// The configuration will load in following order:
// 1. The ENV
// 2. The key/value in .env file
// 3. The config in *.ini (To-be depreacted)
type Configuration struct {
	HTTP struct {
		Host string `json:"host"`
	} `json:"http"`
	App struct {
		Name          string `json:"name"`
		APIKey        string `json:"api_key"`
		MasterKey     string `json:"master_key"`
		AccessControl string `json:"access_control"`
		DevMode       bool   `json:"dev_mode"`
		CORSHost      string `json:"cors_host"`
		Slave         bool   `json:"slave"`
	} `json:"app"`
	DB struct {
		ImplName string `json:"implementation"`
		Option   string `json:"option"`
	} `json:"database"`
	TokenStore struct {
		ImplName string `json:"implementation"`
		Path     string `json:"path"`
		Prefix   string `json:"prefix"`
		Expiry   int64  `json:"expiry"`
		Secret   string `json:"secret"`
	} `json:"-"`
	AssetStore struct {
		ImplName string `json:"implementation"`
		Public   bool   `json:"public"`

		// followings only used when ImplName = fs
		Path string `json:"-"`

		// followings only used when ImplName = s3
		AccessToken string `json:"access_key"`
		SecretToken string `json:"secret_key"`
		Region      string `json:"region"`
		Bucket      string `json:"bucket"`

		// followings only used when ImplName = cloud
		CloudAssetHost          string `json:"cloud_asset_host"`
		CloudAssetToken         string `json:"cloud_asset_token"`
		CloudAssetPublicPrefix  string `json:"cloud_asset_public_prefix"`
		CloudAssetPrivatePrefix string `json:"cloud_asset_private_prefix"`
	} `json:"asset_store"`
	AssetURLSigner struct {
		URLPrefix string `json:"url_prefix"`
		Secret    string `json:"secret"`
	} `json:"asset_signer"`
	APNS struct {
		Enable   bool   `json:"enable"`
		Env      string `json:"env"`
		Cert     string `json:"cert"`
		Key      string `json:"key"`
		CertPath string `json:"-"`
		KeyPath  string `json:"-"`
	} `json:"apns"`
	GCM struct {
		Enable bool   `json:"enable"`
		APIKey string `json:"api_key"`
	} `json:"gcm"`
	LOG struct {
		Level        string            `json:"-"`
		LoggersLevel map[string]string `json:"-"`
	} `json:"log"`
	LogHook struct {
		SentryDSN   string
		SentryLevel string
	} `json:"-"`
	Plugin map[string]*PluginConfig `json:"-"`
}

func NewConfiguration() Configuration {
	config := Configuration{}
	config.HTTP.Host = ":3000"
	config.App.Name = "myapp"
	config.App.AccessControl = "role"
	config.App.DevMode = true
	config.App.CORSHost = "*"
	config.App.Slave = false
	config.DB.ImplName = "pq"
	config.DB.Option = "postgres://postgres:@localhost/postgres?sslmode=disable"
	config.TokenStore.ImplName = "fs"
	config.TokenStore.Path = "data/token"
	config.TokenStore.Expiry = 0
	config.AssetStore.ImplName = "fs"
	config.AssetStore.Path = "data/asset"
	config.AssetURLSigner.URLPrefix = "http://localhost:3000/files"
	config.APNS.Enable = false
	config.APNS.Env = "sandbox"
	config.GCM.Enable = false
	config.LOG.Level = "debug"
	config.LOG.LoggersLevel = map[string]string{
		"plugin": "info",
	}
	config.Plugin = map[string]*PluginConfig{}
	return config
}

func NewConfigurationWithKeys() Configuration {
	config := NewConfiguration()
	config.App.APIKey = uuid.New()
	config.App.MasterKey = uuid.New()
	return config
}

func (config *Configuration) Validate() error {
	if config.App.Name == "" {
		return errors.New("APP_NAME is not set")
	}
	if config.App.APIKey == "" {
		return errors.New("API_KEY is not set")
	}
	if config.App.MasterKey == "" {
		return errors.New("MASTER_KEY is not set")
	}
	if !regexp.MustCompile("^[A-Za-z0-9_]+$").MatchString(config.App.Name) {
		return fmt.Errorf("APP_NAME '%s' contains invalid characters other than alphanumerics or underscores", config.App.Name)
	}
	if config.APNS.Enable && !regexp.MustCompile("^(sandbox|production)$").MatchString(config.APNS.Env) {
		return fmt.Errorf("APNS_ENV must be sandbox or production")
	}
	return nil
}

func (config *Configuration) ReadFromEnv() {
	envErr := godotenv.Load()
	if envErr != nil {
		log.Print("Error in loading .env file")
	}

	config.readHost()

	appAPIKey := os.Getenv("API_KEY")
	if appAPIKey != "" {
		config.App.APIKey = appAPIKey
	}

	appMasterKey := os.Getenv("MASTER_KEY")
	if appMasterKey != "" {
		config.App.MasterKey = appMasterKey
	}

	appName := os.Getenv("APP_NAME")
	if appName != "" {
		config.App.Name = appName
	}

	corsHost := os.Getenv("CORS_HOST")
	if corsHost != "" {
		config.App.CORSHost = corsHost
	}

	accessControl := os.Getenv("ACCESS_CONRTOL")
	if accessControl != "" {
		config.App.AccessControl = accessControl
	}

	if devMode, err := parseBool(os.Getenv("DEV_MODE")); err == nil {
		config.App.DevMode = devMode
	}

	dbImplName := os.Getenv("DB_IMPL_NAME")
	if dbImplName != "" {
		config.DB.ImplName = dbImplName
	}

	if config.DB.ImplName == "pq" && os.Getenv("DATABASE_URL") != "" {
		config.DB.Option = os.Getenv("DATABASE_URL")
	}

	if slave, err := parseBool(os.Getenv("SLAVE")); err == nil {
		config.App.Slave = slave
	}

	config.readTokenStore()
	config.readAssetStore()
	config.readAPNS()
	config.readGCM()
	config.readLog()
	config.readPlugins()
}

func (config *Configuration) readHost() {
	// Default to :3000 if both HOST and PORT is missing
	host := os.Getenv("HOST")
	if host != "" {
		config.HTTP.Host = host
	}
	if config.HTTP.Host == "" {
		port := os.Getenv("PORT")
		if port != "" {
			config.HTTP.Host = ":" + port
		}
	}
}

func (config *Configuration) readTokenStore() {
	tokenStore := os.Getenv("TOKEN_STORE")
	if tokenStore != "" {
		config.TokenStore.ImplName = tokenStore
	}
	tokenStorePath := os.Getenv("TOKEN_STORE_PATH")
	if tokenStorePath != "" {
		config.TokenStore.Path = tokenStorePath
	}

	tokenStorePrefix := os.Getenv("TOKEN_STORE_PREFIX")
	if tokenStorePrefix != "" {
		config.TokenStore.Prefix = tokenStorePrefix
	}

	if expiry, err := strconv.ParseInt(os.Getenv("TOKEN_STORE_EXPIRY"), 10, 64); err == nil {
		config.TokenStore.Expiry = expiry
	}

	tokenStoreSecret := os.Getenv("TOKEN_STORE_SECRET")
	if tokenStoreSecret != "" {
		config.TokenStore.Secret = tokenStoreSecret
	} else {
		config.TokenStore.Secret = config.App.MasterKey
	}
}

func (config *Configuration) readAssetStore() {
	assetStore := os.Getenv("ASSET_STORE")
	if assetStore != "" {
		config.AssetStore.ImplName = assetStore
	}

	if assetStorePublic, err := parseBool(os.Getenv("ASSET_STORE_PUBLIC")); err == nil {
		config.AssetStore.Public = assetStorePublic
	}

	// Local Storage related
	assetStorePath := os.Getenv("ASSET_STORE_PATH")
	if assetStorePath != "" {
		config.AssetStore.Path = assetStorePath
	}
	assetStorePrefix := os.Getenv("ASSET_STORE_URL_PREFIX")
	if assetStorePrefix != "" {
		config.AssetURLSigner.URLPrefix = assetStorePrefix
	}
	assetStoreSecret := os.Getenv("ASSET_STORE_SECRET")
	if assetStoreSecret != "" {
		config.AssetURLSigner.Secret = assetStoreSecret
	}

	// S3 related
	assetStoreAccessKey := os.Getenv("ASSET_STORE_ACCESS_KEY")
	if assetStoreAccessKey != "" {
		config.AssetStore.AccessToken = assetStoreAccessKey
	}
	assetStoreSecretKey := os.Getenv("ASSET_STORE_SECRET_KEY")
	if assetStoreSecretKey != "" {
		config.AssetStore.SecretToken = assetStoreSecretKey
	}
	assetStoreRegion := os.Getenv("ASSET_STORE_REGION")
	if assetStoreRegion != "" {
		config.AssetStore.Region = assetStoreRegion
	}
	assetStoreBucket := os.Getenv("ASSET_STORE_BUCKET")
	if assetStoreBucket != "" {
		config.AssetStore.Bucket = assetStoreBucket
	}

	// Cloud Asset related
	cloudAssetHost := os.Getenv("CLOUD_ASSET_HOST")
	if cloudAssetHost != "" {
		config.AssetStore.CloudAssetHost = cloudAssetHost
	}
	cloudAssetToken := os.Getenv("CLOUD_ASSET_TOKEN")
	if cloudAssetToken != "" {
		config.AssetStore.CloudAssetToken = cloudAssetToken
	}
	cloudAssetPublicPrefix := os.Getenv("CLOUD_ASSET_PUBLIC_PREFIX")
	if cloudAssetPublicPrefix != "" {
		config.AssetStore.CloudAssetPublicPrefix = cloudAssetPublicPrefix
	}
	cloudAssetPrivatePrefix := os.Getenv("CLOUD_ASSET_PRIVATE_PREFIX")
	if cloudAssetPrivatePrefix != "" {
		config.AssetStore.CloudAssetPrivatePrefix = cloudAssetPrivatePrefix
	}
}

func (config *Configuration) readAPNS() {
	if shouldEnableAPNS, err := parseBool(os.Getenv("APNS_ENABLE")); err == nil {
		config.APNS.Enable = shouldEnableAPNS
	}

	if !config.APNS.Enable {
		return
	}

	env := os.Getenv("APNS_ENV")
	if env != "" {
		config.APNS.Env = env
	}

	cert, key := os.Getenv("APNS_CERTIFICATE"), os.Getenv("APNS_PRIVATE_KEY")
	if cert != "" {
		config.APNS.Cert = cert
	}
	if key != "" {
		config.APNS.Key = key
	}

	certPath, keyPath := os.Getenv("APNS_CERTIFICATE_PATH"), os.Getenv("APNS_PRIVATE_KEY_PATH")
	if certPath != "" {
		config.APNS.CertPath = certPath
	}
	if keyPath != "" {
		config.APNS.KeyPath = keyPath
	}

}

func (config *Configuration) readGCM() {
	if shouldEnableGCM, err := parseBool(os.Getenv("GCM_ENABLE")); err == nil {
		config.GCM.Enable = shouldEnableGCM
	}

	gcmAPIKey := os.Getenv("GCM_APIKEY")
	if gcmAPIKey != "" {
		config.GCM.APIKey = gcmAPIKey
	}
}

func (config *Configuration) readLog() {
	logLevel := os.Getenv("LOG_LEVEL")
	if logLevel != "" {
		config.LOG.Level = logLevel
	}

	for _, environ := range os.Environ() {
		if !strings.HasPrefix(environ, "LOG_LEVEL_") {
			continue
		}

		components := strings.SplitN(environ, "=", 2)
		loggerName := strings.ToLower(strings.TrimPrefix(components[0], "LOG_LEVEL_"))
		loggerLevel := components[1]
		config.LOG.LoggersLevel[loggerName] = loggerLevel
	}

	sentry := os.Getenv("SENTRY_DSN")
	if sentry != "" {
		config.LogHook.SentryDSN = sentry
	}

	sentryLevel := os.Getenv("SENTRY_LEVEL")
	if sentryLevel != "" {
		config.LogHook.SentryLevel = sentryLevel
	}
}

func (config *Configuration) readPlugins() {
	plugin := os.Getenv("PLUGINS")
	if plugin == "" {
		return
	}

	plugins := strings.Split(plugin, ",")
	for _, p := range plugins {
		pluginConfig := &PluginConfig{}
		pluginConfig.Transport = os.Getenv(p + "_TRANSPORT")
		pluginConfig.Path = os.Getenv(p + "_PATH")
		args := os.Getenv(p + "_ARGS")
		if args != "" {
			pluginConfig.Args = strings.Split(args, ",")
		}
		config.Plugin[p] = pluginConfig
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubernetes

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/watch"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
)

// LogAggregator aggregates the logs for all the deployed pods.
type LogAggregator struct {
	output      io.Writer
	podSelector PodSelector
	colorPicker ColorPicker

	muted             int32
	startTime         time.Time
	trackedContainers trackedContainers
}

// NewLogAggregator creates a new LogAggregator for a given output.
func NewLogAggregator(out io.Writer, podSelector PodSelector, colorPicker ColorPicker) *LogAggregator {
	return &LogAggregator{
		output:      out,
		podSelector: podSelector,
		colorPicker: colorPicker,
		trackedContainers: trackedContainers{
			ids: map[string]bool{},
		},
	}
}

func (a *LogAggregator) Start(ctx context.Context, client corev1.CoreV1Interface) error {
	a.startTime = time.Now()

	watcher, err := client.Pods("").Watch(meta_v1.ListOptions{
		IncludeUninitialized: true,
	})
	if err != nil {
		return err
	}

	go func() {
		for {
			select {
			case <-ctx.Done():
				return
			case evt := <-watcher.ResultChan():
				if evt.Type != watch.Added && evt.Type != watch.Modified {
					continue
				}

				pod, ok := evt.Object.(*v1.Pod)
				if !ok {
					continue
				}

				if a.podSelector.Select(pod) {
					go a.streamLogs(client, pod)
				}
			}
		}
	}()

	return nil
}

// nolint: interfacer
func (a *LogAggregator) streamLogs(client corev1.CoreV1Interface, pod *v1.Pod) error {
	pods := client.Pods(pod.Namespace)
	if err := WaitForPodReady(pods, pod.Name); err != nil {
		return errors.Wrap(err, "waiting for pod ready")
	}

	for _, container := range pod.Status.ContainerStatuses {
		containerID := container.ContainerID
		if containerID == "" {
			continue
		}

		alreadyTracked := a.trackedContainers.add(containerID)
		if alreadyTracked {
			continue
		}

		logrus.Infof("Stream logs from pod: %s container: %s", pod.Name, container.Name)

		req := pods.GetLogs(pod.Name, &v1.PodLogOptions{
			Follow:    true,
			Container: container.Name,
			SinceTime: &meta_v1.Time{
				Time: a.startTime,
			},
		})

		rc, err := req.Stream()
		if err != nil {
			a.trackedContainers.remove(containerID)
			return errors.Wrap(err, "setting up container log stream")
		}

		color := a.colorPicker.Pick(pod)
		prefix := color.Sprint(prefix(pod, container))

		go func() {
			defer func() {
				a.trackedContainers.remove(containerID)
				rc.Close()
			}()

			if err := a.streamRequest(prefix, rc); err != nil {
				logrus.Errorf("streaming request %s", err)
			}
		}()
	}

	return nil
}

func prefix(pod *v1.Pod, container v1.ContainerStatus) string {
	if pod.Name != container.Name {
		return fmt.Sprintf("[%s %s]", pod.Name, container.Name)
	}
	return fmt.Sprintf("[%s]", container.Name)
}

func (a *LogAggregator) streamRequest(header string, rc io.Reader) error {
	r := bufio.NewReader(rc)
	for {
		// Read up to newline
		line, err := r.ReadBytes('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			return errors.Wrap(err, "reading bytes from log stream")
		}

		if a.IsMuted() {
			continue
		}

		if _, err := fmt.Fprintf(a.output, "%s %s", header, line); err != nil {
			return errors.Wrap(err, "writing to out")
		}
	}
	logrus.Infof("%s exited", header)
	return nil
}

// Mute mutes the logs.
func (a *LogAggregator) Mute() {
	atomic.StoreInt32(&a.muted, 1)
}

// Unmute unmutes the logs.
func (a *LogAggregator) Unmute() {
	atomic.StoreInt32(&a.muted, 0)
}

// IsMuted says if the logs are to be muted.
func (a *LogAggregator) IsMuted() bool {
	return atomic.LoadInt32(&a.muted) == 1
}

type trackedContainers struct {
	sync.Mutex
	ids map[string]bool
}

// add adds a containerID to be tracked. Return true if the container
// was already tracked.
func (t *trackedContainers) add(id string) bool {
	t.Lock()
	alreadyTracked := t.ids[id]
	t.ids[id] = true
	t.Unlock()

	return alreadyTracked
}

func (t *trackedContainers) remove(id string) {
	t.Lock()
	delete(t.ids, id)
	t.Unlock()
}

// PodSelector is used to choose which pods to log.
type PodSelector interface {
	Select(pod *v1.Pod) bool
}

// ImageList implements PodSelector based on a list of images names.
type ImageList struct {
	sync.RWMutex
	names map[string]bool
}

// NewImageList creates a new ImageList.
func NewImageList() *ImageList {
	return &ImageList{
		names: make(map[string]bool),
	}
}

// AddImage adds an image to the list.
func (l *ImageList) AddImage(image string) {
	l.Lock()
	l.names[image] = true
	l.Unlock()
}

// Select returns true if one of the pod's images is in the list.
func (l *ImageList) Select(pod *v1.Pod) bool {
	l.RLock()
	defer l.RUnlock()

	for _, container := range pod.Spec.Containers {
		if l.names[container.Image] {
			return true
		}
	}

	return false
}
<commit_msg>log: use since seconds instead of host time<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubernetes

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/watch"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
)

// LogAggregator aggregates the logs for all the deployed pods.
type LogAggregator struct {
	output      io.Writer
	podSelector PodSelector
	colorPicker ColorPicker

	muted             int32
	startTime         time.Time
	trackedContainers trackedContainers
}

// NewLogAggregator creates a new LogAggregator for a given output.
func NewLogAggregator(out io.Writer, podSelector PodSelector, colorPicker ColorPicker) *LogAggregator {
	return &LogAggregator{
		output:      out,
		podSelector: podSelector,
		colorPicker: colorPicker,
		trackedContainers: trackedContainers{
			ids: map[string]bool{},
		},
	}
}

func (a *LogAggregator) Start(ctx context.Context, client corev1.CoreV1Interface) error {
	a.startTime = time.Now()

	watcher, err := client.Pods("").Watch(meta_v1.ListOptions{
		IncludeUninitialized: true,
	})
	if err != nil {
		return err
	}

	go func() {
		for {
			select {
			case <-ctx.Done():
				return
			case evt := <-watcher.ResultChan():
				if evt.Type != watch.Added && evt.Type != watch.Modified {
					continue
				}

				pod, ok := evt.Object.(*v1.Pod)
				if !ok {
					continue
				}

				if a.podSelector.Select(pod) {
					go a.streamLogs(client, pod)
				}
			}
		}
	}()

	return nil
}

// nolint: interfacer
func (a *LogAggregator) streamLogs(client corev1.CoreV1Interface, pod *v1.Pod) error {
	pods := client.Pods(pod.Namespace)
	if err := WaitForPodReady(pods, pod.Name); err != nil {
		return errors.Wrap(err, "waiting for pod ready")
	}

	for _, container := range pod.Status.ContainerStatuses {
		containerID := container.ContainerID
		if containerID == "" {
			continue
		}

		alreadyTracked := a.trackedContainers.add(containerID)
		if alreadyTracked {
			continue
		}

		logrus.Infof("Stream logs from pod: %s container: %s", pod.Name, container.Name)

		sinceSeconds := int64(time.Since(a.startTime))
		req := pods.GetLogs(pod.Name, &v1.PodLogOptions{
			Follow:       true,
			Container:    container.Name,
			SinceSeconds: &sinceSeconds,
		})

		rc, err := req.Stream()
		if err != nil {
			a.trackedContainers.remove(containerID)
			return errors.Wrap(err, "setting up container log stream")
		}

		color := a.colorPicker.Pick(pod)
		prefix := color.Sprint(prefix(pod, container))

		go func() {
			defer func() {
				a.trackedContainers.remove(containerID)
				rc.Close()
			}()

			if err := a.streamRequest(prefix, rc); err != nil {
				logrus.Errorf("streaming request %s", err)
			}
		}()
	}

	return nil
}

func prefix(pod *v1.Pod, container v1.ContainerStatus) string {
	if pod.Name != container.Name {
		return fmt.Sprintf("[%s %s]", pod.Name, container.Name)
	}
	return fmt.Sprintf("[%s]", container.Name)
}

func (a *LogAggregator) streamRequest(header string, rc io.Reader) error {
	r := bufio.NewReader(rc)
	for {
		// Read up to newline
		line, err := r.ReadBytes('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			return errors.Wrap(err, "reading bytes from log stream")
		}

		if a.IsMuted() {
			continue
		}

		if _, err := fmt.Fprintf(a.output, "%s %s", header, line); err != nil {
			return errors.Wrap(err, "writing to out")
		}
	}
	logrus.Infof("%s exited", header)
	return nil
}

// Mute mutes the logs.
func (a *LogAggregator) Mute() {
	atomic.StoreInt32(&a.muted, 1)
}

// Unmute unmutes the logs.
func (a *LogAggregator) Unmute() {
	atomic.StoreInt32(&a.muted, 0)
}

// IsMuted says if the logs are to be muted.
func (a *LogAggregator) IsMuted() bool {
	return atomic.LoadInt32(&a.muted) == 1
}

type trackedContainers struct {
	sync.Mutex
	ids map[string]bool
}

// add adds a containerID to be tracked. Return true if the container
// was already tracked.
func (t *trackedContainers) add(id string) bool {
	t.Lock()
	alreadyTracked := t.ids[id]
	t.ids[id] = true
	t.Unlock()

	return alreadyTracked
}

func (t *trackedContainers) remove(id string) {
	t.Lock()
	delete(t.ids, id)
	t.Unlock()
}

// PodSelector is used to choose which pods to log.
type PodSelector interface {
	Select(pod *v1.Pod) bool
}

// ImageList implements PodSelector based on a list of images names.
type ImageList struct {
	sync.RWMutex
	names map[string]bool
}

// NewImageList creates a new ImageList.
func NewImageList() *ImageList {
	return &ImageList{
		names: make(map[string]bool),
	}
}

// AddImage adds an image to the list.
func (l *ImageList) AddImage(image string) {
	l.Lock()
	l.names[image] = true
	l.Unlock()
}

// Select returns true if one of the pod's images is in the list.
func (l *ImageList) Select(pod *v1.Pod) bool {
	l.RLock()
	defer l.RUnlock()

	for _, container := range pod.Spec.Containers {
		if l.names[container.Image] {
			return true
		}
	}

	return false
}
<|endoftext|>