<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package typed

import (
	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
	"sigs.k8s.io/structured-merge-diff/v4/schema"
	"sigs.k8s.io/structured-merge-diff/v4/value"
)

type mergingWalker struct {
	lhs     value.Value
	rhs     value.Value
	schema  *schema.Schema
	typeRef schema.TypeRef

	// Current path that we are merging
	path fieldpath.Path

	// How to merge. Called after schema validation for all leaf fields.
	rule mergeRule

	// If set, called after non-leaf items have been merged. (`out` is
	// probably already set.)
	postItemHook mergeRule

	// output of the merge operation (nil if none)
	out *interface{}

	// internal housekeeping--don't set when constructing.
	inLeaf bool // Set to true if we're in a "big leaf"--atomic map/list

	// Allocate only as many walkers as needed for the depth by storing them here.
	spareWalkers *[]*mergingWalker

	allocator value.Allocator
}

// merge rules examine w.lhs and w.rhs (up to one of which may be nil) and
// optionally set w.out. If lhs and rhs are both set, they will be of
// comparable type.
type mergeRule func(w *mergingWalker)

var (
	ruleKeepRHS = mergeRule(func(w *mergingWalker) {
		if w.rhs != nil {
			v := w.rhs.Unstructured()
			w.out = &v
		} else if w.lhs != nil {
			v := w.lhs.Unstructured()
			w.out = &v
		}
	})
)

// merge sets w.out.
func (w *mergingWalker) merge(prefixFn func() string) (errs ValidationErrors) {
	if w.lhs == nil && w.rhs == nil {
		// check this condidition here instead of everywhere below.
		return errorf("at least one of lhs and rhs must be provided")
	}
	a, ok := w.schema.Resolve(w.typeRef)
	if !ok {
		return errorf("schema error: no type found matching: %v", *w.typeRef.NamedType)
	}

	alhs := deduceAtom(a, w.lhs)
	arhs := deduceAtom(a, w.rhs)
	if alhs.Equals(&arhs) {
		errs = append(errs, handleAtom(arhs, w.typeRef, w)...)
	} else {
		w2 := *w
		errs = append(errs, handleAtom(alhs, w.typeRef, &w2)...)
		errs = append(errs, handleAtom(arhs, w.typeRef, w)...)
	}

	if !w.inLeaf && w.postItemHook != nil {
		w.postItemHook(w)
	}
	return errs.WithLazyPrefix(prefixFn)
}

// doLeaf should be called on leaves before descending into children, if there
// will be a descent. It modifies w.inLeaf.
func (w *mergingWalker) doLeaf() {
	if w.inLeaf {
		// We're in a "big leaf", an atomic map or list. Ignore
		// subsequent leaves.
		return
	}
	w.inLeaf = true

	// We don't recurse into leaf fields for merging.
	w.rule(w)
}

func (w *mergingWalker) doScalar(t *schema.Scalar) (errs ValidationErrors) {
	errs = append(errs, validateScalar(t, w.lhs, "lhs: ")...)
	errs = append(errs, validateScalar(t, w.rhs, "rhs: ")...)
	if len(errs) > 0 {
		return errs
	}

	// All scalars are leaf fields.
	w.doLeaf()

	return nil
}

func (w *mergingWalker) prepareDescent(pe fieldpath.PathElement, tr schema.TypeRef) *mergingWalker {
	if w.spareWalkers == nil {
		// first descent.
		w.spareWalkers = &[]*mergingWalker{}
	}
	var w2 *mergingWalker
	if n := len(*w.spareWalkers); n > 0 {
		w2, *w.spareWalkers = (*w.spareWalkers)[n-1], (*w.spareWalkers)[:n-1]
	} else {
		w2 = &mergingWalker{}
	}
	*w2 = *w
	w2.typeRef = tr
	w2.path = append(w2.path, pe)
	w2.lhs = nil
	w2.rhs = nil
	w2.out = nil
	return w2
}

func (w *mergingWalker) finishDescent(w2 *mergingWalker) {
	// if the descent caused a realloc, ensure that we reuse the buffer
	// for the next sibling.
	w.path = w2.path[:len(w2.path)-1]
	*w.spareWalkers = append(*w.spareWalkers, w2)
}

func (w *mergingWalker) derefMap(prefix string, v value.Value) (value.Map, ValidationErrors) {
	if v == nil {
		return nil, nil
	}
	m, err := mapValue(w.allocator, v)
	if err != nil {
		return nil, errorf("%v: %v", prefix, err)
	}
	return m, nil
}

func (w *mergingWalker) visitListItems(t *schema.List, lhs, rhs value.List) (errs ValidationErrors) {
	rLen := 0
	if rhs != nil {
		rLen = rhs.Length()
	}
	lLen := 0
	if lhs != nil {
		lLen = lhs.Length()
	}
	outLen := lLen
	if outLen < rLen {
		outLen = rLen
	}
	out := make([]interface{}, 0, outLen)

	lhsOrder, observedLHS, lhsErrs := w.visitListOperand(t, lhs)
	errs = append(errs, lhsErrs...)
	rhsOrder, observedRHS, rhsErrs := w.visitListOperand(t, rhs)
	errs = append(errs, rhsErrs...)
	seen := fieldpath.MakePathElementSet(outLen)

	lLen, rLen = len(lhsOrder), len(rhsOrder)
	for lI, rI := 0, 0; lI < lLen || rI < rLen; {
		if lI < lLen && rI < rLen && lhsOrder[lI].Equals(rhsOrder[rI]) {
			// merge LHS & RHS items
			pe := lhsOrder[lI]
			lChild, _ := observedLHS.Get(pe)
			rChild, _ := observedRHS.Get(pe)
			mergeOut, errs := w.mergeListItem(t, pe, lChild, rChild)
			errs = append(errs, errs...)
			if mergeOut != nil {
				out = append(out, *mergeOut)
			}
			seen.Insert(pe)
			lI++
			rI++
			continue
		}
		if lI < lLen {
			pe := lhsOrder[lI]
			if ok := seen.Has(pe); ok {
				// Skip the LHS item because it has already appeared
				lI++
				continue
			}
			if _, ok := observedRHS.Get(pe); !ok {
				// Take the LHS item, without a matching RHS item to merge with
				lChild, _ := observedLHS.Get(pe)
				mergeOut, errs := w.mergeListItem(t, pe, lChild, nil)
				errs = append(errs, errs...)
				if mergeOut != nil {
					out = append(out, *mergeOut)
				}
				seen.Insert(pe)
				lI++
				continue
			}
		}
		if rI < rLen {
			// Take the RHS item, merge with matching LHS item if possible
			pe := rhsOrder[rI]
			rChild, _ := observedRHS.Get(pe)
			lChild, _ := observedLHS.Get(pe) // may be nil
			mergeOut, errs := w.mergeListItem(t, pe, lChild, rChild)
			errs = append(errs, errs...)
			if mergeOut != nil {
				out = append(out, *mergeOut)
			}
			seen.Insert(pe)
			rI++
		}
	}

	if len(out) > 0 {
		i := interface{}(out)
		w.out = &i
	}

	return errs
}

func (w *mergingWalker) visitListOperand(t *schema.List, list value.List) ([]fieldpath.PathElement, fieldpath.PathElementValueMap, ValidationErrors) {
	var errs ValidationErrors
	length := 0
	if list != nil {
		length = list.Length()
	}
	observed := fieldpath.MakePathElementValueMap(length)
	pes := make([]fieldpath.PathElement, 0, length)
	for i := 0; i < length; i++ {
		child := list.At(i)
		pe, err := listItemToPathElement(w.allocator, w.schema, t, i, child)
		if err != nil {
			errs = append(errs, errorf("element %v: %v", i, err.Error())...)
			// If we can't construct the path element, we can't
			// even report errors deeper in the schema, so bail on
			// this element.
			continue
		}
		if _, found := observed.Get(pe); found {
			errs = append(errs, errorf("duplicate entries for key %v", pe.String())...)
			continue
		}
		observed.Insert(pe, child)
		pes = append(pes, pe)
	}
	return pes, observed, errs
}

func (w *mergingWalker) mergeListItem(t *schema.List, pe fieldpath.PathElement, lChild, rChild value.Value) (out *interface{}, errs ValidationErrors) {
	w2 := w.prepareDescent(pe, t.ElementType)
	w2.lhs = lChild
	w2.rhs = rChild
	errs = append(errs, w2.merge(pe.String)...)
	if w2.out != nil {
		out = w2.out
	}
	w.finishDescent(w2)
	return
}

func (w *mergingWalker) derefList(prefix string, v value.Value) (value.List, ValidationErrors) {
	if v == nil {
		return nil, nil
	}
	l, err := listValue(w.allocator, v)
	if err != nil {
		return nil, errorf("%v: %v", prefix, err)
	}
	return l, nil
}

func (w *mergingWalker) doList(t *schema.List) (errs ValidationErrors) {
	lhs, _ := w.derefList("lhs: ", w.lhs)
	if lhs != nil {
		defer w.allocator.Free(lhs)
	}
	rhs, _ := w.derefList("rhs: ", w.rhs)
	if rhs != nil {
		defer w.allocator.Free(rhs)
	}

	// If both lhs and rhs are empty/null, treat it as a
	// leaf: this helps preserve the empty/null
	// distinction.
	emptyPromoteToLeaf := (lhs == nil || lhs.Length() == 0) && (rhs == nil || rhs.Length() == 0)

	if t.ElementRelationship == schema.Atomic || emptyPromoteToLeaf {
		w.doLeaf()
		return nil
	}

	if lhs == nil && rhs == nil {
		return nil
	}

	errs = w.visitListItems(t, lhs, rhs)

	return errs
}

func (w *mergingWalker) visitMapItem(t *schema.Map, out map[string]interface{}, key string, lhs, rhs value.Value) (errs ValidationErrors) {
	fieldType := t.ElementType
	if sf, ok := t.FindField(key); ok {
		fieldType = sf.Type
	}
	pe := fieldpath.PathElement{FieldName: &key}
	w2 := w.prepareDescent(pe, fieldType)
	w2.lhs = lhs
	w2.rhs = rhs
	errs = append(errs, w2.merge(pe.String)...)
	if w2.out != nil {
		out[key] = *w2.out
	}
	w.finishDescent(w2)
	return errs
}

func (w *mergingWalker) visitMapItems(t *schema.Map, lhs, rhs value.Map) (errs ValidationErrors) {
	out := map[string]interface{}{}

	value.MapZipUsing(w.allocator, lhs, rhs, value.Unordered, func(key string, lhsValue, rhsValue value.Value) bool {
		errs = append(errs, w.visitMapItem(t, out, key, lhsValue, rhsValue)...)
		return true
	})
	if len(out) > 0 {
		i := interface{}(out)
		w.out = &i
	}

	return errs
}

func (w *mergingWalker) doMap(t *schema.Map) (errs ValidationErrors) {
	lhs, _ := w.derefMap("lhs: ", w.lhs)
	if lhs != nil {
		defer w.allocator.Free(lhs)
	}
	rhs, _ := w.derefMap("rhs: ", w.rhs)
	if rhs != nil {
		defer w.allocator.Free(rhs)
	}
	// If both lhs and rhs are empty/null, treat it as a
	// leaf: this helps preserve the empty/null
	// distinction.
	emptyPromoteToLeaf := (lhs == nil || lhs.Empty()) && (rhs == nil || rhs.Empty())

	if t.ElementRelationship == schema.Atomic || emptyPromoteToLeaf {
		w.doLeaf()
		return nil
	}

	if lhs == nil && rhs == nil {
		return nil
	}

	errs = append(errs, w.visitMapItems(t, lhs, rhs)...)

	return errs
}
<commit_msg>visitListOperand -> indexListPathElements<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package typed

import (
	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
	"sigs.k8s.io/structured-merge-diff/v4/schema"
	"sigs.k8s.io/structured-merge-diff/v4/value"
)

type mergingWalker struct {
	lhs     value.Value
	rhs     value.Value
	schema  *schema.Schema
	typeRef schema.TypeRef

	// Current path that we are merging
	path fieldpath.Path

	// How to merge. Called after schema validation for all leaf fields.
	rule mergeRule

	// If set, called after non-leaf items have been merged. (`out` is
	// probably already set.)
	postItemHook mergeRule

	// output of the merge operation (nil if none)
	out *interface{}

	// internal housekeeping--don't set when constructing.
	inLeaf bool // Set to true if we're in a "big leaf"--atomic map/list

	// Allocate only as many walkers as needed for the depth by storing them here.
	spareWalkers *[]*mergingWalker

	allocator value.Allocator
}

// merge rules examine w.lhs and w.rhs (up to one of which may be nil) and
// optionally set w.out. If lhs and rhs are both set, they will be of
// comparable type.
type mergeRule func(w *mergingWalker)

var (
	ruleKeepRHS = mergeRule(func(w *mergingWalker) {
		if w.rhs != nil {
			v := w.rhs.Unstructured()
			w.out = &v
		} else if w.lhs != nil {
			v := w.lhs.Unstructured()
			w.out = &v
		}
	})
)

// merge sets w.out.
func (w *mergingWalker) merge(prefixFn func() string) (errs ValidationErrors) {
	if w.lhs == nil && w.rhs == nil {
		// check this condidition here instead of everywhere below.
		return errorf("at least one of lhs and rhs must be provided")
	}
	a, ok := w.schema.Resolve(w.typeRef)
	if !ok {
		return errorf("schema error: no type found matching: %v", *w.typeRef.NamedType)
	}

	alhs := deduceAtom(a, w.lhs)
	arhs := deduceAtom(a, w.rhs)
	if alhs.Equals(&arhs) {
		errs = append(errs, handleAtom(arhs, w.typeRef, w)...)
	} else {
		w2 := *w
		errs = append(errs, handleAtom(alhs, w.typeRef, &w2)...)
		errs = append(errs, handleAtom(arhs, w.typeRef, w)...)
	}

	if !w.inLeaf && w.postItemHook != nil {
		w.postItemHook(w)
	}
	return errs.WithLazyPrefix(prefixFn)
}

// doLeaf should be called on leaves before descending into children, if there
// will be a descent. It modifies w.inLeaf.
func (w *mergingWalker) doLeaf() {
	if w.inLeaf {
		// We're in a "big leaf", an atomic map or list. Ignore
		// subsequent leaves.
		return
	}
	w.inLeaf = true

	// We don't recurse into leaf fields for merging.
	w.rule(w)
}

func (w *mergingWalker) doScalar(t *schema.Scalar) (errs ValidationErrors) {
	errs = append(errs, validateScalar(t, w.lhs, "lhs: ")...)
	errs = append(errs, validateScalar(t, w.rhs, "rhs: ")...)
	if len(errs) > 0 {
		return errs
	}

	// All scalars are leaf fields.
	w.doLeaf()

	return nil
}

func (w *mergingWalker) prepareDescent(pe fieldpath.PathElement, tr schema.TypeRef) *mergingWalker {
	if w.spareWalkers == nil {
		// first descent.
		w.spareWalkers = &[]*mergingWalker{}
	}
	var w2 *mergingWalker
	if n := len(*w.spareWalkers); n > 0 {
		w2, *w.spareWalkers = (*w.spareWalkers)[n-1], (*w.spareWalkers)[:n-1]
	} else {
		w2 = &mergingWalker{}
	}
	*w2 = *w
	w2.typeRef = tr
	w2.path = append(w2.path, pe)
	w2.lhs = nil
	w2.rhs = nil
	w2.out = nil
	return w2
}

func (w *mergingWalker) finishDescent(w2 *mergingWalker) {
	// if the descent caused a realloc, ensure that we reuse the buffer
	// for the next sibling.
	w.path = w2.path[:len(w2.path)-1]
	*w.spareWalkers = append(*w.spareWalkers, w2)
}

func (w *mergingWalker) derefMap(prefix string, v value.Value) (value.Map, ValidationErrors) {
	if v == nil {
		return nil, nil
	}
	m, err := mapValue(w.allocator, v)
	if err != nil {
		return nil, errorf("%v: %v", prefix, err)
	}
	return m, nil
}

func (w *mergingWalker) visitListItems(t *schema.List, lhs, rhs value.List) (errs ValidationErrors) {
	rLen := 0
	if rhs != nil {
		rLen = rhs.Length()
	}
	lLen := 0
	if lhs != nil {
		lLen = lhs.Length()
	}
	outLen := lLen
	if outLen < rLen {
		outLen = rLen
	}
	out := make([]interface{}, 0, outLen)

	lhsOrder, observedLHS, lhsErrs := w.indexListPathElements(t, lhs)
	errs = append(errs, lhsErrs...)
	rhsOrder, observedRHS, rhsErrs := w.indexListPathElements(t, rhs)
	errs = append(errs, rhsErrs...)
	seen := fieldpath.MakePathElementSet(outLen)

	lLen, rLen = len(lhsOrder), len(rhsOrder)
	for lI, rI := 0, 0; lI < lLen || rI < rLen; {
		if lI < lLen && rI < rLen && lhsOrder[lI].Equals(rhsOrder[rI]) {
			// merge LHS & RHS items
			pe := lhsOrder[lI]
			lChild, _ := observedLHS.Get(pe)
			rChild, _ := observedRHS.Get(pe)
			mergeOut, errs := w.mergeListItem(t, pe, lChild, rChild)
			errs = append(errs, errs...)
			if mergeOut != nil {
				out = append(out, *mergeOut)
			}
			seen.Insert(pe)
			lI++
			rI++
			continue
		}
		if lI < lLen {
			pe := lhsOrder[lI]
			if ok := seen.Has(pe); ok {
				// Skip the LHS item because it has already appeared
				lI++
				continue
			}
			if _, ok := observedRHS.Get(pe); !ok {
				// Take the LHS item, without a matching RHS item to merge with
				lChild, _ := observedLHS.Get(pe)
				mergeOut, errs := w.mergeListItem(t, pe, lChild, nil)
				errs = append(errs, errs...)
				if mergeOut != nil {
					out = append(out, *mergeOut)
				}
				seen.Insert(pe)
				lI++
				continue
			}
		}
		if rI < rLen {
			// Take the RHS item, merge with matching LHS item if possible
			pe := rhsOrder[rI]
			rChild, _ := observedRHS.Get(pe)
			lChild, _ := observedLHS.Get(pe) // may be nil
			mergeOut, errs := w.mergeListItem(t, pe, lChild, rChild)
			errs = append(errs, errs...)
			if mergeOut != nil {
				out = append(out, *mergeOut)
			}
			seen.Insert(pe)
			rI++
		}
	}

	if len(out) > 0 {
		i := interface{}(out)
		w.out = &i
	}

	return errs
}

func (w *mergingWalker) indexListPathElements(t *schema.List, list value.List) ([]fieldpath.PathElement, fieldpath.PathElementValueMap, ValidationErrors) {
	var errs ValidationErrors
	length := 0
	if list != nil {
		length = list.Length()
	}
	observed := fieldpath.MakePathElementValueMap(length)
	pes := make([]fieldpath.PathElement, 0, length)
	for i := 0; i < length; i++ {
		child := list.At(i)
		pe, err := listItemToPathElement(w.allocator, w.schema, t, i, child)
		if err != nil {
			errs = append(errs, errorf("element %v: %v", i, err.Error())...)
			// If we can't construct the path element, we can't
			// even report errors deeper in the schema, so bail on
			// this element.
			continue
		}
		if _, found := observed.Get(pe); found {
			errs = append(errs, errorf("duplicate entries for key %v", pe.String())...)
			continue
		}
		observed.Insert(pe, child)
		pes = append(pes, pe)
	}
	return pes, observed, errs
}

func (w *mergingWalker) mergeListItem(t *schema.List, pe fieldpath.PathElement, lChild, rChild value.Value) (out *interface{}, errs ValidationErrors) {
	w2 := w.prepareDescent(pe, t.ElementType)
	w2.lhs = lChild
	w2.rhs = rChild
	errs = append(errs, w2.merge(pe.String)...)
	if w2.out != nil {
		out = w2.out
	}
	w.finishDescent(w2)
	return
}

func (w *mergingWalker) derefList(prefix string, v value.Value) (value.List, ValidationErrors) {
	if v == nil {
		return nil, nil
	}
	l, err := listValue(w.allocator, v)
	if err != nil {
		return nil, errorf("%v: %v", prefix, err)
	}
	return l, nil
}

func (w *mergingWalker) doList(t *schema.List) (errs ValidationErrors) {
	lhs, _ := w.derefList("lhs: ", w.lhs)
	if lhs != nil {
		defer w.allocator.Free(lhs)
	}
	rhs, _ := w.derefList("rhs: ", w.rhs)
	if rhs != nil {
		defer w.allocator.Free(rhs)
	}

	// If both lhs and rhs are empty/null, treat it as a
	// leaf: this helps preserve the empty/null
	// distinction.
	emptyPromoteToLeaf := (lhs == nil || lhs.Length() == 0) && (rhs == nil || rhs.Length() == 0)

	if t.ElementRelationship == schema.Atomic || emptyPromoteToLeaf {
		w.doLeaf()
		return nil
	}

	if lhs == nil && rhs == nil {
		return nil
	}

	errs = w.visitListItems(t, lhs, rhs)

	return errs
}

func (w *mergingWalker) visitMapItem(t *schema.Map, out map[string]interface{}, key string, lhs, rhs value.Value) (errs ValidationErrors) {
	fieldType := t.ElementType
	if sf, ok := t.FindField(key); ok {
		fieldType = sf.Type
	}
	pe := fieldpath.PathElement{FieldName: &key}
	w2 := w.prepareDescent(pe, fieldType)
	w2.lhs = lhs
	w2.rhs = rhs
	errs = append(errs, w2.merge(pe.String)...)
	if w2.out != nil {
		out[key] = *w2.out
	}
	w.finishDescent(w2)
	return errs
}

func (w *mergingWalker) visitMapItems(t *schema.Map, lhs, rhs value.Map) (errs ValidationErrors) {
	out := map[string]interface{}{}

	value.MapZipUsing(w.allocator, lhs, rhs, value.Unordered, func(key string, lhsValue, rhsValue value.Value) bool {
		errs = append(errs, w.visitMapItem(t, out, key, lhsValue, rhsValue)...)
		return true
	})
	if len(out) > 0 {
		i := interface{}(out)
		w.out = &i
	}

	return errs
}

func (w *mergingWalker) doMap(t *schema.Map) (errs ValidationErrors) {
	lhs, _ := w.derefMap("lhs: ", w.lhs)
	if lhs != nil {
		defer w.allocator.Free(lhs)
	}
	rhs, _ := w.derefMap("rhs: ", w.rhs)
	if rhs != nil {
		defer w.allocator.Free(rhs)
	}
	// If both lhs and rhs are empty/null, treat it as a
	// leaf: this helps preserve the empty/null
	// distinction.
	emptyPromoteToLeaf := (lhs == nil || lhs.Empty()) && (rhs == nil || rhs.Empty())

	if t.ElementRelationship == schema.Atomic || emptyPromoteToLeaf {
		w.doLeaf()
		return nil
	}

	if lhs == nil && rhs == nil {
		return nil
	}

	errs = append(errs, w.visitMapItems(t, lhs, rhs)...)

	return errs
}
<|endoftext|>


<commit_before>package types

import (
	"fmt"
	"time"
)

const KeelDefaultPort = 9300
const KeelPolicyLabel = "keel.observer/policy"

type Repository struct {
	Host string `json:"host,omitempty"`
	Name string `json:"name,omitempty"`
	Tag  string `json:"tag,omitempty"`
}

type Event struct {
	Repository Repository `json:"repository,omitempty"`
	CreatedAt  time.Time  `json:"createdAt,omitempty"`
	// optional field to identify trigger
	TriggerName string `json:"triggerName,omitempty"`
}

type Version struct {
	Major      int64
	Minor      int64
	Patch      int64
	PreRelease string
	Metadata   string
}

func (v Version) String() string {
	return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
}

// PolicyType - policy type
type PolicyType int

// ParsePolicy - parse policy type
func ParsePolicy(policy string) PolicyType {
	switch policy {
	case "all":
		return PolicyTypeAll
	case "major":
		return PolicyTypeMajor
	case "minor":
		return PolicyTypeMinor
	case "patch":
		return PolicyTypePatch
	default:
		return PolicyTypeUnknown
	}
}

func (t PolicyType) String() string {
	switch t {
	case PolicyTypeUnknown:
		return "unknown"
	case PolicyTypeAll:
		return "all"
	case PolicyTypeMajor:
		return "major"
	case PolicyTypeMinor:
		return "minor"
	case PolicyTypePatch:
		return "patch"
	case PolicyTypeForce:
		return "force"
	default:
		return ""
	}
}

// available policies
const (
	PolicyTypeUnknown = iota
	PolicyTypeAll
	PolicyTypeMajor
	PolicyTypeMinor
	PolicyTypePatch
	PolicyTypeForce // update always when a new image is available
)
<commit_msg>tidy up<commit_after>package types

import (
	"fmt"
	"time"
)

const KeelDefaultPort = 9300

// KeelPolicyLabel - keel update policies (version checking)
const KeelPolicyLabel = "keel.sh/policy"

// KeelTriggerLabel - trigger label is used to specify custom trigger types
// for example keel.sh/trigger=poll would signal poll trigger to start watching for repository
// changes
const KeelTriggerLabel = "keel.sh/trigger"

// KeelPollSchedule - optional variable to setup custom schedule for polling, defaults to @every 10m
const KeelPollSchedule = "keel.sh/pollSchedule"

// KeelPollDefaultSchedule - defaul polling schedule
const KeelPollDefaultSchedule = "@every 1m"

type Repository struct {
	Host string `json:"host,omitempty"`
	Name string `json:"name,omitempty"`
	Tag  string `json:"tag,omitempty"`
}

type Event struct {
	Repository Repository `json:"repository,omitempty"`
	CreatedAt  time.Time  `json:"createdAt,omitempty"`
	// optional field to identify trigger
	TriggerName string `json:"triggerName,omitempty"`
}

type Version struct {
	Major      int64
	Minor      int64
	Patch      int64
	PreRelease string
	Metadata   string
}

func (v Version) String() string {
	return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
}

// TriggerType - trigger types
type TriggerType int

// Available trigger types
const (
	TriggerTypeDefault TriggerType = iota // default policy is to wait for external triggers
	TriggerTypePoll                       // poll policy sets up watchers for the affected repositories
)

func (t TriggerType) String() string {
	switch t {
	case TriggerTypeDefault:
		return "default"
	case TriggerTypePoll:
		return "poll"
	default:
		return "unknown"
	}
}

func ParseTrigger(trigger string) TriggerType {
	switch trigger {
	case "poll":
		return TriggerTypePoll
	}
	return TriggerTypeDefault
}

// PolicyType - policy type
type PolicyType int

// ParsePolicy - parse policy type
func ParsePolicy(policy string) PolicyType {
	switch policy {
	case "all":
		return PolicyTypeAll
	case "major":
		return PolicyTypeMajor
	case "minor":
		return PolicyTypeMinor
	case "patch":
		return PolicyTypePatch
	case "force":
		return PolicyTypeForce
	default:
		return PolicyTypeNone
	}
}

func (t PolicyType) String() string {
	switch t {
	case PolicyTypeNone:
		return "none"
	case PolicyTypeAll:
		return "all"
	case PolicyTypeMajor:
		return "major"
	case PolicyTypeMinor:
		return "minor"
	case PolicyTypePatch:
		return "patch"
	case PolicyTypeForce:
		return "force"
	default:
		return ""
	}
}

// available policies
const (
	PolicyTypeNone = iota
	PolicyTypeAll
	PolicyTypeMajor
	PolicyTypeMinor
	PolicyTypePatch
	PolicyTypeForce // update always when a new image is available
)
<|endoftext|>


<commit_before>package cache

import (
	"fmt"
	"strings"

	"github.com/argoproj/gitops-engine/pkg/utils/kube"
	"github.com/argoproj/gitops-engine/pkg/utils/text"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	resourcehelper "k8s.io/kubectl/pkg/util/resource"

	"github.com/argoproj/argo-cd/v2/common"
	"github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
	"github.com/argoproj/argo-cd/v2/util/resource"
)

func populateNodeInfo(un *unstructured.Unstructured, res *ResourceInfo) {
	gvk := un.GroupVersionKind()
	revision := resource.GetRevision(un)
	if revision > 0 {
		res.Info = append(res.Info, v1alpha1.InfoItem{Name: "Revision", Value: fmt.Sprintf("Rev:%v", revision)})
	}
	switch gvk.Group {
	case "":
		switch gvk.Kind {
		case kube.PodKind:
			populatePodInfo(un, res)
			return
		case kube.ServiceKind:
			populateServiceInfo(un, res)
			return
		case "Node":
			populateHostNodeInfo(un, res)
			return
		}
	case "extensions", "networking.k8s.io":
		switch gvk.Kind {
		case kube.IngressKind:
			populateIngressInfo(un, res)
			return
		}
	case "networking.istio.io":
		switch gvk.Kind {
		case "VirtualService":
			populateIstioVirtualServiceInfo(un, res)
			return
		}
	}

	for k, v := range un.GetAnnotations() {
		if strings.HasPrefix(k, common.AnnotationKeyLinkPrefix) {
			if res.NetworkingInfo == nil {
				res.NetworkingInfo = &v1alpha1.ResourceNetworkingInfo{}
			}
			res.NetworkingInfo.ExternalURLs = append(res.NetworkingInfo.ExternalURLs, v)
		}
	}
}

func getIngress(un *unstructured.Unstructured) []v1.LoadBalancerIngress {
	ingress, ok, err := unstructured.NestedSlice(un.Object, "status", "loadBalancer", "ingress")
	if !ok || err != nil {
		return nil
	}
	res := make([]v1.LoadBalancerIngress, 0)
	for _, item := range ingress {
		if lbIngress, ok := item.(map[string]interface{}); ok {
			if hostname := lbIngress["hostname"]; hostname != nil {
				res = append(res, v1.LoadBalancerIngress{Hostname: fmt.Sprintf("%s", hostname)})
			} else if ip := lbIngress["ip"]; ip != nil {
				res = append(res, v1.LoadBalancerIngress{IP: fmt.Sprintf("%s", ip)})
			}
		}
	}
	return res
}

func populateServiceInfo(un *unstructured.Unstructured, res *ResourceInfo) {
	targetLabels, _, _ := unstructured.NestedStringMap(un.Object, "spec", "selector")
	ingress := make([]v1.LoadBalancerIngress, 0)
	if serviceType, ok, err := unstructured.NestedString(un.Object, "spec", "type"); ok && err == nil && serviceType == string(v1.ServiceTypeLoadBalancer) {
		ingress = getIngress(un)
	}
	res.NetworkingInfo = &v1alpha1.ResourceNetworkingInfo{TargetLabels: targetLabels, Ingress: ingress}
}

func populateIngressInfo(un *unstructured.Unstructured, res *ResourceInfo) {
	ingress := getIngress(un)
	targetsMap := make(map[v1alpha1.ResourceRef]bool)
	if backend, ok, err := unstructured.NestedMap(un.Object, "spec", "backend"); ok && err == nil {
		targetsMap[v1alpha1.ResourceRef{
			Group:     "",
			Kind:      kube.ServiceKind,
			Namespace: un.GetNamespace(),
			Name:      fmt.Sprintf("%s", backend["serviceName"]),
		}] = true
	}
	urlsSet := make(map[string]bool)
	if rules, ok, err := unstructured.NestedSlice(un.Object, "spec", "rules"); ok && err == nil {
		for i := range rules {
			rule, ok := rules[i].(map[string]interface{})
			if !ok {
				continue
			}
			host := rule["host"]
			if host == nil || host == "" {
				for i := range ingress {
					host = text.FirstNonEmpty(ingress[i].Hostname, ingress[i].IP)
					if host != "" {
						break
					}
				}
			}
			paths, ok, err := unstructured.NestedSlice(rule, "http", "paths")
			if !ok || err != nil {
				continue
			}
			for i := range paths {
				path, ok := paths[i].(map[string]interface{})
				if !ok {
					continue
				}

				if serviceName, ok, err := unstructured.NestedString(path, "backend", "serviceName"); ok && err == nil {
					targetsMap[v1alpha1.ResourceRef{
						Group:     "",
						Kind:      kube.ServiceKind,
						Namespace: un.GetNamespace(),
						Name:      serviceName,
					}] = true
				}

				if host == nil || host == "" {
					continue
				}
				stringPort := "http"
				if tls, ok, err := unstructured.NestedSlice(un.Object, "spec", "tls"); ok && err == nil {
					for i := range tls {
						tlsline, ok := tls[i].(map[string]interface{})
						secretName := tlsline["secretName"]
						if ok && secretName != nil {
							stringPort = "https"
						}
						tlshost := tlsline["host"]
						if tlshost == host {
							stringPort = "https"
						}
					}
				}

				externalURL := fmt.Sprintf("%s://%s", stringPort, host)

				subPath := ""
				if nestedPath, ok, err := unstructured.NestedString(path, "path"); ok && err == nil {
					subPath = strings.TrimSuffix(nestedPath, "*")
				}
				externalURL += subPath
				urlsSet[externalURL] = true
			}
		}
	}
	targets := make([]v1alpha1.ResourceRef, 0)
	for target := range targetsMap {
		targets = append(targets, target)
	}

	var urls []string
	if res.NetworkingInfo != nil {
		urls = res.NetworkingInfo.ExternalURLs
	}
	for url := range urlsSet {
		urls = append(urls, url)
	}
	res.NetworkingInfo = &v1alpha1.ResourceNetworkingInfo{TargetRefs: targets, Ingress: ingress, ExternalURLs: urls}
}

func populateIstioVirtualServiceInfo(un *unstructured.Unstructured, res *ResourceInfo) {
	targetsMap := make(map[v1alpha1.ResourceRef]bool)

	if rules, ok, err := unstructured.NestedSlice(un.Object, "spec", "http"); ok && err == nil {
		for i := range rules {
			rule, ok := rules[i].(map[string]interface{})
			if !ok {
				continue
			}
			routes, ok, err := unstructured.NestedSlice(rule, "route")
			if !ok || err != nil {
				continue
			}
			for i := range routes {
				route, ok := routes[i].(map[string]interface{})
				if !ok {
					continue
				}

				if hostName, ok, err := unstructured.NestedString(route, "destination", "host"); ok && err == nil {
					hostSplits := strings.Split(hostName, ".")
					serviceName := hostSplits[0]

					var namespace string
					if len(hostSplits) >= 2 {
						namespace = hostSplits[1]
					} else {
						namespace = un.GetNamespace()
					}

					targetsMap[v1alpha1.ResourceRef{
						Kind:      kube.ServiceKind,
						Name:      serviceName,
						Namespace: namespace,
					}] = true
				}
			}
		}
	}
	targets := make([]v1alpha1.ResourceRef, 0)
	for target := range targetsMap {
		targets = append(targets, target)
	}

	res.NetworkingInfo = &v1alpha1.ResourceNetworkingInfo{TargetRefs: targets}
}

func populatePodInfo(un *unstructured.Unstructured, res *ResourceInfo) {
	pod := v1.Pod{}
	err := runtime.DefaultUnstructuredConverter.FromUnstructured(un.Object, &pod)
	if err != nil {
		return
	}
	restarts := 0
	totalContainers := len(pod.Spec.Containers)
	readyContainers := 0

	reason := string(pod.Status.Phase)
	if pod.Status.Reason != "" {
		reason = pod.Status.Reason
	}

	imagesSet := make(map[string]bool)
	for _, container := range pod.Spec.InitContainers {
		imagesSet[container.Image] = true
	}
	for _, container := range pod.Spec.Containers {
		imagesSet[container.Image] = true
	}

	res.Images = nil
	for image := range imagesSet {
		res.Images = append(res.Images, image)
	}

	initializing := false
	for i := range pod.Status.InitContainerStatuses {
		container := pod.Status.InitContainerStatuses[i]
		restarts += int(container.RestartCount)
		switch {
		case container.State.Terminated != nil && container.State.Terminated.ExitCode == 0:
			continue
		case container.State.Terminated != nil:
			// initialization is failed
			if len(container.State.Terminated.Reason) == 0 {
				if container.State.Terminated.Signal != 0 {
					reason = fmt.Sprintf("Init:Signal:%d", container.State.Terminated.Signal)
				} else {
					reason = fmt.Sprintf("Init:ExitCode:%d", container.State.Terminated.ExitCode)
				}
			} else {
				reason = "Init:" + container.State.Terminated.Reason
			}
			initializing = true
		case container.State.Waiting != nil && len(container.State.Waiting.Reason) > 0 && container.State.Waiting.Reason != "PodInitializing":
			reason = "Init:" + container.State.Waiting.Reason
			initializing = true
		default:
			reason = fmt.Sprintf("Init:%d/%d", i, len(pod.Spec.InitContainers))
			initializing = true
		}
		break
	}
	if !initializing {
		restarts = 0
		hasRunning := false
		for i := len(pod.Status.ContainerStatuses) - 1; i >= 0; i-- {
			container := pod.Status.ContainerStatuses[i]

			restarts += int(container.RestartCount)
			if container.State.Waiting != nil && container.State.Waiting.Reason != "" {
				reason = container.State.Waiting.Reason
			} else if container.State.Terminated != nil && container.State.Terminated.Reason != "" {
				reason = container.State.Terminated.Reason
			} else if container.State.Terminated != nil && container.State.Terminated.Reason == "" {
				if container.State.Terminated.Signal != 0 {
					reason = fmt.Sprintf("Signal:%d", container.State.Terminated.Signal)
				} else {
					reason = fmt.Sprintf("ExitCode:%d", container.State.Terminated.ExitCode)
				}
			} else if container.Ready && container.State.Running != nil {
				hasRunning = true
				readyContainers++
			}
		}

		// change pod status back to "Running" if there is at least one container still reporting as "Running" status
		if reason == "Completed" && hasRunning {
			reason = "Running"
		}
	}

	// "NodeLost" = https://github.com/kubernetes/kubernetes/blob/cb8ad64243d48d9a3c26b11b2e0945c098457282/pkg/util/node/node.go#L46
	// But depending on the k8s.io/kubernetes package just for a constant
	// is not worth it.
	// See https://github.com/argoproj/argo-cd/issues/5173
	// and https://github.com/kubernetes/kubernetes/issues/90358#issuecomment-617859364
	if pod.DeletionTimestamp != nil && pod.Status.Reason == "NodeLost" {
		reason = "Unknown"
	} else if pod.DeletionTimestamp != nil {
		reason = "Terminating"
	}

	if reason != "" {
		res.Info = append(res.Info, v1alpha1.InfoItem{Name: "Status Reason", Value: reason})
	}

	req, _ := resourcehelper.PodRequestsAndLimits(&pod)
	res.PodInfo = &PodInfo{NodeName: pod.Spec.NodeName, ResourceRequests: req, Phase: pod.Status.Phase}

	res.Info = append(res.Info, v1alpha1.InfoItem{Name: "Node", Value: pod.Spec.NodeName})
	res.Info = append(res.Info, v1alpha1.InfoItem{Name: "Containers", Value: fmt.Sprintf("%d/%d", readyContainers, totalContainers)})
	if restarts > 0 {
		res.Info = append(res.Info, v1alpha1.InfoItem{Name: "Restart Count", Value: fmt.Sprintf("%d", restarts)})
	}
	res.NetworkingInfo = &v1alpha1.ResourceNetworkingInfo{Labels: un.GetLabels()}
}

func populateHostNodeInfo(un *unstructured.Unstructured, res *ResourceInfo) {
	node := v1.Node{}
	err := runtime.DefaultUnstructuredConverter.FromUnstructured(un.Object, &node)
	if err != nil {
		return
	}
	res.NodeInfo = &NodeInfo{
		Name:       node.Name,
		Capacity:   node.Status.Capacity,
		SystemInfo: node.Status.NodeInfo,
	}
}
<commit_msg>fix: Add https prefix to ingress URLs if hosts field is present (#6901)<commit_after>package cache

import (
	"fmt"
	"strings"

	"github.com/argoproj/gitops-engine/pkg/utils/kube"
	"github.com/argoproj/gitops-engine/pkg/utils/text"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	resourcehelper "k8s.io/kubectl/pkg/util/resource"

	"github.com/argoproj/argo-cd/v2/common"
	"github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
	"github.com/argoproj/argo-cd/v2/util/resource"
)

func populateNodeInfo(un *unstructured.Unstructured, res *ResourceInfo) {
	gvk := un.GroupVersionKind()
	revision := resource.GetRevision(un)
	if revision > 0 {
		res.Info = append(res.Info, v1alpha1.InfoItem{Name: "Revision", Value: fmt.Sprintf("Rev:%v", revision)})
	}
	switch gvk.Group {
	case "":
		switch gvk.Kind {
		case kube.PodKind:
			populatePodInfo(un, res)
			return
		case kube.ServiceKind:
			populateServiceInfo(un, res)
			return
		case "Node":
			populateHostNodeInfo(un, res)
			return
		}
	case "extensions", "networking.k8s.io":
		switch gvk.Kind {
		case kube.IngressKind:
			populateIngressInfo(un, res)
			return
		}
	case "networking.istio.io":
		switch gvk.Kind {
		case "VirtualService":
			populateIstioVirtualServiceInfo(un, res)
			return
		}
	}

	for k, v := range un.GetAnnotations() {
		if strings.HasPrefix(k, common.AnnotationKeyLinkPrefix) {
			if res.NetworkingInfo == nil {
				res.NetworkingInfo = &v1alpha1.ResourceNetworkingInfo{}
			}
			res.NetworkingInfo.ExternalURLs = append(res.NetworkingInfo.ExternalURLs, v)
		}
	}
}

func getIngress(un *unstructured.Unstructured) []v1.LoadBalancerIngress {
	ingress, ok, err := unstructured.NestedSlice(un.Object, "status", "loadBalancer", "ingress")
	if !ok || err != nil {
		return nil
	}
	res := make([]v1.LoadBalancerIngress, 0)
	for _, item := range ingress {
		if lbIngress, ok := item.(map[string]interface{}); ok {
			if hostname := lbIngress["hostname"]; hostname != nil {
				res = append(res, v1.LoadBalancerIngress{Hostname: fmt.Sprintf("%s", hostname)})
			} else if ip := lbIngress["ip"]; ip != nil {
				res = append(res, v1.LoadBalancerIngress{IP: fmt.Sprintf("%s", ip)})
			}
		}
	}
	return res
}

func populateServiceInfo(un *unstructured.Unstructured, res *ResourceInfo) {
	targetLabels, _, _ := unstructured.NestedStringMap(un.Object, "spec", "selector")
	ingress := make([]v1.LoadBalancerIngress, 0)
	if serviceType, ok, err := unstructured.NestedString(un.Object, "spec", "type"); ok && err == nil && serviceType == string(v1.ServiceTypeLoadBalancer) {
		ingress = getIngress(un)
	}
	res.NetworkingInfo = &v1alpha1.ResourceNetworkingInfo{TargetLabels: targetLabels, Ingress: ingress}
}

func populateIngressInfo(un *unstructured.Unstructured, res *ResourceInfo) {
	ingress := getIngress(un)
	targetsMap := make(map[v1alpha1.ResourceRef]bool)
	if backend, ok, err := unstructured.NestedMap(un.Object, "spec", "backend"); ok && err == nil {
		targetsMap[v1alpha1.ResourceRef{
			Group:     "",
			Kind:      kube.ServiceKind,
			Namespace: un.GetNamespace(),
			Name:      fmt.Sprintf("%s", backend["serviceName"]),
		}] = true
	}
	urlsSet := make(map[string]bool)
	if rules, ok, err := unstructured.NestedSlice(un.Object, "spec", "rules"); ok && err == nil {
		for i := range rules {
			rule, ok := rules[i].(map[string]interface{})
			if !ok {
				continue
			}
			host := rule["host"]
			if host == nil || host == "" {
				for i := range ingress {
					host = text.FirstNonEmpty(ingress[i].Hostname, ingress[i].IP)
					if host != "" {
						break
					}
				}
			}
			paths, ok, err := unstructured.NestedSlice(rule, "http", "paths")
			if !ok || err != nil {
				continue
			}
			for i := range paths {
				path, ok := paths[i].(map[string]interface{})
				if !ok {
					continue
				}

				if serviceName, ok, err := unstructured.NestedString(path, "backend", "serviceName"); ok && err == nil {
					targetsMap[v1alpha1.ResourceRef{
						Group:     "",
						Kind:      kube.ServiceKind,
						Namespace: un.GetNamespace(),
						Name:      serviceName,
					}] = true
				}

				if host == nil || host == "" {
					continue
				}
				stringPort := "http"
				if tls, ok, err := unstructured.NestedSlice(un.Object, "spec", "tls"); ok && err == nil {
					for i := range tls {
						tlsline, ok := tls[i].(map[string]interface{})
						secretName := tlsline["secretName"]
						if ok && secretName != nil {
							stringPort = "https"
						}
						tlshost := tlsline["host"]
						if tlshost == host {
							stringPort = "https"
							continue
						}
						if hosts := tlsline["hosts"]; hosts != nil {
							tlshosts, ok := tlsline["hosts"].(map[string]interface{})
							if ok {
								for j := range tlshosts {
									if tlshosts[j] == host {
										stringPort = "https"
									}
								}
							}
						}
					}
				}

				externalURL := fmt.Sprintf("%s://%s", stringPort, host)

				subPath := ""
				if nestedPath, ok, err := unstructured.NestedString(path, "path"); ok && err == nil {
					subPath = strings.TrimSuffix(nestedPath, "*")
				}
				externalURL += subPath
				urlsSet[externalURL] = true
			}
		}
	}
	targets := make([]v1alpha1.ResourceRef, 0)
	for target := range targetsMap {
		targets = append(targets, target)
	}

	var urls []string
	if res.NetworkingInfo != nil {
		urls = res.NetworkingInfo.ExternalURLs
	}
	for url := range urlsSet {
		urls = append(urls, url)
	}
	res.NetworkingInfo = &v1alpha1.ResourceNetworkingInfo{TargetRefs: targets, Ingress: ingress, ExternalURLs: urls}
}

func populateIstioVirtualServiceInfo(un *unstructured.Unstructured, res *ResourceInfo) {
	targetsMap := make(map[v1alpha1.ResourceRef]bool)

	if rules, ok, err := unstructured.NestedSlice(un.Object, "spec", "http"); ok && err == nil {
		for i := range rules {
			rule, ok := rules[i].(map[string]interface{})
			if !ok {
				continue
			}
			routes, ok, err := unstructured.NestedSlice(rule, "route")
			if !ok || err != nil {
				continue
			}
			for i := range routes {
				route, ok := routes[i].(map[string]interface{})
				if !ok {
					continue
				}

				if hostName, ok, err := unstructured.NestedString(route, "destination", "host"); ok && err == nil {
					hostSplits := strings.Split(hostName, ".")
					serviceName := hostSplits[0]

					var namespace string
					if len(hostSplits) >= 2 {
						namespace = hostSplits[1]
					} else {
						namespace = un.GetNamespace()
					}

					targetsMap[v1alpha1.ResourceRef{
						Kind:      kube.ServiceKind,
						Name:      serviceName,
						Namespace: namespace,
					}] = true
				}
			}
		}
	}
	targets := make([]v1alpha1.ResourceRef, 0)
	for target := range targetsMap {
		targets = append(targets, target)
	}

	res.NetworkingInfo = &v1alpha1.ResourceNetworkingInfo{TargetRefs: targets}
}

func populatePodInfo(un *unstructured.Unstructured, res *ResourceInfo) {
	pod := v1.Pod{}
	err := runtime.DefaultUnstructuredConverter.FromUnstructured(un.Object, &pod)
	if err != nil {
		return
	}
	restarts := 0
	totalContainers := len(pod.Spec.Containers)
	readyContainers := 0

	reason := string(pod.Status.Phase)
	if pod.Status.Reason != "" {
		reason = pod.Status.Reason
	}

	imagesSet := make(map[string]bool)
	for _, container := range pod.Spec.InitContainers {
		imagesSet[container.Image] = true
	}
	for _, container := range pod.Spec.Containers {
		imagesSet[container.Image] = true
	}

	res.Images = nil
	for image := range imagesSet {
		res.Images = append(res.Images, image)
	}

	initializing := false
	for i := range pod.Status.InitContainerStatuses {
		container := pod.Status.InitContainerStatuses[i]
		restarts += int(container.RestartCount)
		switch {
		case container.State.Terminated != nil && container.State.Terminated.ExitCode == 0:
			continue
		case container.State.Terminated != nil:
			// initialization is failed
			if len(container.State.Terminated.Reason) == 0 {
				if container.State.Terminated.Signal != 0 {
					reason = fmt.Sprintf("Init:Signal:%d", container.State.Terminated.Signal)
				} else {
					reason = fmt.Sprintf("Init:ExitCode:%d", container.State.Terminated.ExitCode)
				}
			} else {
				reason = "Init:" + container.State.Terminated.Reason
			}
			initializing = true
		case container.State.Waiting != nil && len(container.State.Waiting.Reason) > 0 && container.State.Waiting.Reason != "PodInitializing":
			reason = "Init:" + container.State.Waiting.Reason
			initializing = true
		default:
			reason = fmt.Sprintf("Init:%d/%d", i, len(pod.Spec.InitContainers))
			initializing = true
		}
		break
	}
	if !initializing {
		restarts = 0
		hasRunning := false
		for i := len(pod.Status.ContainerStatuses) - 1; i >= 0; i-- {
			container := pod.Status.ContainerStatuses[i]

			restarts += int(container.RestartCount)
			if container.State.Waiting != nil && container.State.Waiting.Reason != "" {
				reason = container.State.Waiting.Reason
			} else if container.State.Terminated != nil && container.State.Terminated.Reason != "" {
				reason = container.State.Terminated.Reason
			} else if container.State.Terminated != nil && container.State.Terminated.Reason == "" {
				if container.State.Terminated.Signal != 0 {
					reason = fmt.Sprintf("Signal:%d", container.State.Terminated.Signal)
				} else {
					reason = fmt.Sprintf("ExitCode:%d", container.State.Terminated.ExitCode)
				}
			} else if container.Ready && container.State.Running != nil {
				hasRunning = true
				readyContainers++
			}
		}

		// change pod status back to "Running" if there is at least one container still reporting as "Running" status
		if reason == "Completed" && hasRunning {
			reason = "Running"
		}
	}

	// "NodeLost" = https://github.com/kubernetes/kubernetes/blob/cb8ad64243d48d9a3c26b11b2e0945c098457282/pkg/util/node/node.go#L46
	// But depending on the k8s.io/kubernetes package just for a constant
	// is not worth it.
	// See https://github.com/argoproj/argo-cd/issues/5173
	// and https://github.com/kubernetes/kubernetes/issues/90358#issuecomment-617859364
	if pod.DeletionTimestamp != nil && pod.Status.Reason == "NodeLost" {
		reason = "Unknown"
	} else if pod.DeletionTimestamp != nil {
		reason = "Terminating"
	}

	if reason != "" {
		res.Info = append(res.Info, v1alpha1.InfoItem{Name: "Status Reason", Value: reason})
	}

	req, _ := resourcehelper.PodRequestsAndLimits(&pod)
	res.PodInfo = &PodInfo{NodeName: pod.Spec.NodeName, ResourceRequests: req, Phase: pod.Status.Phase}

	res.Info = append(res.Info, v1alpha1.InfoItem{Name: "Node", Value: pod.Spec.NodeName})
	res.Info = append(res.Info, v1alpha1.InfoItem{Name: "Containers", Value: fmt.Sprintf("%d/%d", readyContainers, totalContainers)})
	if restarts > 0 {
		res.Info = append(res.Info, v1alpha1.InfoItem{Name: "Restart Count", Value: fmt.Sprintf("%d", restarts)})
	}
	res.NetworkingInfo = &v1alpha1.ResourceNetworkingInfo{Labels: un.GetLabels()}
}

func populateHostNodeInfo(un *unstructured.Unstructured, res *ResourceInfo) {
	node := v1.Node{}
	err := runtime.DefaultUnstructuredConverter.FromUnstructured(un.Object, &node)
	if err != nil {
		return
	}
	res.NodeInfo = &NodeInfo{
		Name:       node.Name,
		Capacity:   node.Status.Capacity,
		SystemInfo: node.Status.NodeInfo,
	}
}
<|endoftext|>


<commit_before>package controller

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	"github.com/argoproj/argo-cd/common"
	appv1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
	appclientset "github.com/argoproj/argo-cd/pkg/client/clientset/versioned"
	appinformers "github.com/argoproj/argo-cd/pkg/client/informers/externalversions"
	"github.com/argoproj/argo-cd/reposerver"
	"github.com/argoproj/argo-cd/reposerver/repository"
	apireposerver "github.com/argoproj/argo-cd/server/repository"
	"github.com/argoproj/argo-cd/util"
	argoutil "github.com/argoproj/argo-cd/util/argo"
	log "github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/selection"
	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
)

// ApplicationController is the controller for application resources.
type ApplicationController struct {
	repoClientset        reposerver.Clientset
	kubeClientset        kubernetes.Interface
	applicationClientset appclientset.Interface
	appQueue             workqueue.RateLimitingInterface
	appInformer          cache.SharedIndexInformer
	appComparator        AppComparator
	statusRefreshTimeout time.Duration
	apiRepoService       apireposerver.RepositoryServiceServer
}

type ApplicationControllerConfig struct {
	InstanceID string
	Namespace  string
}

// NewApplicationController creates new instance of ApplicationController.
func NewApplicationController(
	kubeClientset kubernetes.Interface,
	applicationClientset appclientset.Interface,
	repoClientset reposerver.Clientset,
	apiRepoService apireposerver.RepositoryServiceServer,
	appComparator AppComparator,
	appResyncPeriod time.Duration,
	config *ApplicationControllerConfig,
) *ApplicationController {
	appQueue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
	return &ApplicationController{
		kubeClientset:        kubeClientset,
		applicationClientset: applicationClientset,
		repoClientset:        repoClientset,
		appQueue:             appQueue,
		apiRepoService:       apiRepoService,
		appComparator:        appComparator,
		appInformer:          newApplicationInformer(applicationClientset, appQueue, appResyncPeriod, config),
		statusRefreshTimeout: appResyncPeriod,
	}
}

// Run starts the Application CRD controller.
func (ctrl *ApplicationController) Run(ctx context.Context, appWorkers int) {
	defer runtime.HandleCrash()
	defer ctrl.appQueue.ShutDown()

	go ctrl.appInformer.Run(ctx.Done())

	if !cache.WaitForCacheSync(ctx.Done(), ctrl.appInformer.HasSynced) {
		log.Error("Timed out waiting for caches to sync")
		return
	}

	for i := 0; i < appWorkers; i++ {
		go wait.Until(ctrl.runWorker, time.Second, ctx.Done())
	}

	<-ctx.Done()
}

func (ctrl *ApplicationController) processNextItem() bool {
	appKey, shutdown := ctrl.appQueue.Get()
	if shutdown {
		return false
	}

	defer ctrl.appQueue.Done(appKey)

	obj, exists, err := ctrl.appInformer.GetIndexer().GetByKey(appKey.(string))
	if err != nil {
		log.Errorf("Failed to get application '%s' from informer index: %+v", appKey, err)
		return true
	}
	if !exists {
		// This happens after app was deleted, but the work queue still had an entry for it.
		return true
	}
	app, ok := obj.(*appv1.Application)
	if !ok {
		log.Warnf("Key '%s' in index is not an application", appKey)
		return true
	}

	if app.NeedRefreshAppStatus(ctrl.statusRefreshTimeout) {
		updatedApp := app.DeepCopy()
		status, err := ctrl.tryRefreshAppStatus(updatedApp)
		if err != nil {
			updatedApp.Status.ComparisonResult = appv1.ComparisonResult{
				Status:     appv1.ComparisonStatusError,
				Error:      fmt.Sprintf("Failed to get application status for application '%s': %v", app.Name, err),
				ComparedTo: app.Spec.Source,
				ComparedAt: metav1.Time{Time: time.Now().UTC()},
			}
		} else {
			updatedApp.Status = *status
		}
		ctrl.persistApp(updatedApp)
	}

	return true
}

func (ctrl *ApplicationController) tryRefreshAppStatus(app *appv1.Application) (*appv1.ApplicationStatus, error) {
	conn, client, err := ctrl.repoClientset.NewRepositoryClient()
	if err != nil {
		return nil, err
	}
	defer util.Close(conn)
	repo, err := ctrl.apiRepoService.Get(context.Background(), &apireposerver.RepoQuery{Repo: app.Spec.Source.RepoURL})
	if err != nil {
		// If we couldn't retrieve from the repo service, assume public repositories
		repo = &appv1.Repository{
			Repo:     app.Spec.Source.RepoURL,
			Username: "",
			Password: "",
		}
	}
	revision := app.Spec.Source.TargetRevision
	manifestInfo, err := client.GenerateManifest(context.Background(), &repository.ManifestRequest{
		Repo:        repo,
		Revision:    revision,
		Path:        app.Spec.Source.Path,
		Environment: app.Spec.Source.Environment,
		AppLabel:    app.Name,
	})
	if err != nil {
		log.Errorf("Failed to load application manifest %v", err)
		return nil, err
	}
	targetObjs := make([]*unstructured.Unstructured, len(manifestInfo.Manifests))
	for i, manifestStr := range manifestInfo.Manifests {
		var obj unstructured.Unstructured
		if err := json.Unmarshal([]byte(manifestStr), &obj); err != nil {
			if err != nil {
				return nil, err
			}
		}
		targetObjs[i] = &obj
	}

	server, namespace := argoutil.ResolveServerNamespace(app.Spec.Destination, manifestInfo)
	comparisonResult, err := ctrl.appComparator.CompareAppState(server, namespace, targetObjs, app)
	if err != nil {
		return nil, err
	}
	log.Infof("App %s comparison result: prev: %s. current: %s", app.Name, app.Status.ComparisonResult.Status, comparisonResult.Status)
	return &appv1.ApplicationStatus{
		ComparisonResult: *comparisonResult,
	}, nil
}

func (ctrl *ApplicationController) runWorker() {
	for ctrl.processNextItem() {
	}
}

func (ctrl *ApplicationController) persistApp(app *appv1.Application) {
	appClient := ctrl.applicationClientset.ArgoprojV1alpha1().Applications(app.ObjectMeta.Namespace)
	_, err := appClient.Update(app)
	if err != nil {
		log.Warnf("Error updating application: %v", err)
	}
	log.Info("Application update successful")
}

func newApplicationInformer(
	appClientset appclientset.Interface, appQueue workqueue.RateLimitingInterface, appResyncPeriod time.Duration, config *ApplicationControllerConfig) cache.SharedIndexInformer {

	appInformerFactory := appinformers.NewFilteredSharedInformerFactory(
		appClientset,
		appResyncPeriod,
		config.Namespace,
		func(options *metav1.ListOptions) {
			var instanceIDReq *labels.Requirement
			var err error
			if config.InstanceID != "" {
				instanceIDReq, err = labels.NewRequirement(common.LabelKeyApplicationControllerInstanceID, selection.Equals, []string{config.InstanceID})
			} else {
				instanceIDReq, err = labels.NewRequirement(common.LabelKeyApplicationControllerInstanceID, selection.DoesNotExist, nil)
			}
			if err != nil {
				panic(err)
			}

			options.FieldSelector = fields.Everything().String()
			labelSelector := labels.NewSelector().Add(*instanceIDReq)
			options.LabelSelector = labelSelector.String()
		},
	)
	informer := appInformerFactory.Argoproj().V1alpha1().Applications().Informer()
	informer.AddEventHandler(
		cache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				key, err := cache.MetaNamespaceKeyFunc(obj)
				if err == nil {
					appQueue.Add(key)
				}
			},
			UpdateFunc: func(old, new interface{}) {
				key, err := cache.MetaNamespaceKeyFunc(new)
				if err == nil {
					appQueue.Add(key)
				}
			},
			DeleteFunc: func(obj interface{}) {
				// IndexerInformer uses a delta queue, therefore for deletes we have to use this
				// key function.
				key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
				if err == nil {
					appQueue.Add(key)
				}
			},
		},
	)
	return informer
}
<commit_msg>Don't overwrite application status in tryRefreshAppStatus (#42)<commit_after>package controller

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	"github.com/argoproj/argo-cd/common"
	appv1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
	appclientset "github.com/argoproj/argo-cd/pkg/client/clientset/versioned"
	appinformers "github.com/argoproj/argo-cd/pkg/client/informers/externalversions"
	"github.com/argoproj/argo-cd/reposerver"
	"github.com/argoproj/argo-cd/reposerver/repository"
	apireposerver "github.com/argoproj/argo-cd/server/repository"
	"github.com/argoproj/argo-cd/util"
	argoutil "github.com/argoproj/argo-cd/util/argo"
	log "github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/selection"
	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
)

// ApplicationController is the controller for application resources.
type ApplicationController struct {
	repoClientset        reposerver.Clientset
	kubeClientset        kubernetes.Interface
	applicationClientset appclientset.Interface
	appQueue             workqueue.RateLimitingInterface
	appInformer          cache.SharedIndexInformer
	appComparator        AppComparator
	statusRefreshTimeout time.Duration
	apiRepoService       apireposerver.RepositoryServiceServer
}

type ApplicationControllerConfig struct {
	InstanceID string
	Namespace  string
}

// NewApplicationController creates new instance of ApplicationController.
func NewApplicationController(
	kubeClientset kubernetes.Interface,
	applicationClientset appclientset.Interface,
	repoClientset reposerver.Clientset,
	apiRepoService apireposerver.RepositoryServiceServer,
	appComparator AppComparator,
	appResyncPeriod time.Duration,
	config *ApplicationControllerConfig,
) *ApplicationController {
	appQueue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
	return &ApplicationController{
		kubeClientset:        kubeClientset,
		applicationClientset: applicationClientset,
		repoClientset:        repoClientset,
		appQueue:             appQueue,
		apiRepoService:       apiRepoService,
		appComparator:        appComparator,
		appInformer:          newApplicationInformer(applicationClientset, appQueue, appResyncPeriod, config),
		statusRefreshTimeout: appResyncPeriod,
	}
}

// Run starts the Application CRD controller.
func (ctrl *ApplicationController) Run(ctx context.Context, appWorkers int) {
	defer runtime.HandleCrash()
	defer ctrl.appQueue.ShutDown()

	go ctrl.appInformer.Run(ctx.Done())

	if !cache.WaitForCacheSync(ctx.Done(), ctrl.appInformer.HasSynced) {
		log.Error("Timed out waiting for caches to sync")
		return
	}

	for i := 0; i < appWorkers; i++ {
		go wait.Until(ctrl.runWorker, time.Second, ctx.Done())
	}

	<-ctx.Done()
}

func (ctrl *ApplicationController) processNextItem() bool {
	appKey, shutdown := ctrl.appQueue.Get()
	if shutdown {
		return false
	}

	defer ctrl.appQueue.Done(appKey)

	obj, exists, err := ctrl.appInformer.GetIndexer().GetByKey(appKey.(string))
	if err != nil {
		log.Errorf("Failed to get application '%s' from informer index: %+v", appKey, err)
		return true
	}
	if !exists {
		// This happens after app was deleted, but the work queue still had an entry for it.
		return true
	}
	app, ok := obj.(*appv1.Application)
	if !ok {
		log.Warnf("Key '%s' in index is not an application", appKey)
		return true
	}

	if app.NeedRefreshAppStatus(ctrl.statusRefreshTimeout) {
		updatedApp := app.DeepCopy()
		status, err := ctrl.tryRefreshAppStatus(updatedApp)
		if err != nil {
			updatedApp.Status.ComparisonResult = appv1.ComparisonResult{
				Status:     appv1.ComparisonStatusError,
				Error:      fmt.Sprintf("Failed to get application status for application '%s': %v", app.Name, err),
				ComparedTo: app.Spec.Source,
				ComparedAt: metav1.Time{Time: time.Now().UTC()},
			}
		} else {
			updatedApp.Status = *status
		}
		ctrl.persistApp(updatedApp)
	}

	return true
}

func (ctrl *ApplicationController) tryRefreshAppStatus(app *appv1.Application) (*appv1.ApplicationStatus, error) {
	conn, client, err := ctrl.repoClientset.NewRepositoryClient()
	if err != nil {
		return nil, err
	}
	defer util.Close(conn)
	repo, err := ctrl.apiRepoService.Get(context.Background(), &apireposerver.RepoQuery{Repo: app.Spec.Source.RepoURL})
	if err != nil {
		// If we couldn't retrieve from the repo service, assume public repositories
		repo = &appv1.Repository{
			Repo:     app.Spec.Source.RepoURL,
			Username: "",
			Password: "",
		}
	}
	revision := app.Spec.Source.TargetRevision
	manifestInfo, err := client.GenerateManifest(context.Background(), &repository.ManifestRequest{
		Repo:        repo,
		Revision:    revision,
		Path:        app.Spec.Source.Path,
		Environment: app.Spec.Source.Environment,
		AppLabel:    app.Name,
	})
	if err != nil {
		log.Errorf("Failed to load application manifest %v", err)
		return nil, err
	}
	targetObjs := make([]*unstructured.Unstructured, len(manifestInfo.Manifests))
	for i, manifestStr := range manifestInfo.Manifests {
		var obj unstructured.Unstructured
		if err := json.Unmarshal([]byte(manifestStr), &obj); err != nil {
			if err != nil {
				return nil, err
			}
		}
		targetObjs[i] = &obj
	}

	server, namespace := argoutil.ResolveServerNamespace(app.Spec.Destination, manifestInfo)
	comparisonResult, err := ctrl.appComparator.CompareAppState(server, namespace, targetObjs, app)
	if err != nil {
		return nil, err
	}
	log.Infof("App %s comparison result: prev: %s. current: %s", app.Name, app.Status.ComparisonResult.Status, comparisonResult.Status)
	newStatus := app.Status
	newStatus.ComparisonResult = *comparisonResult
	return &newStatus, nil
}

func (ctrl *ApplicationController) runWorker() {
	for ctrl.processNextItem() {
	}
}

func (ctrl *ApplicationController) persistApp(app *appv1.Application) {
	appClient := ctrl.applicationClientset.ArgoprojV1alpha1().Applications(app.ObjectMeta.Namespace)
	_, err := appClient.Update(app)
	if err != nil {
		log.Warnf("Error updating application: %v", err)
	}
	log.Info("Application update successful")
}

func newApplicationInformer(
	appClientset appclientset.Interface, appQueue workqueue.RateLimitingInterface, appResyncPeriod time.Duration, config *ApplicationControllerConfig) cache.SharedIndexInformer {

	appInformerFactory := appinformers.NewFilteredSharedInformerFactory(
		appClientset,
		appResyncPeriod,
		config.Namespace,
		func(options *metav1.ListOptions) {
			var instanceIDReq *labels.Requirement
			var err error
			if config.InstanceID != "" {
				instanceIDReq, err = labels.NewRequirement(common.LabelKeyApplicationControllerInstanceID, selection.Equals, []string{config.InstanceID})
			} else {
				instanceIDReq, err = labels.NewRequirement(common.LabelKeyApplicationControllerInstanceID, selection.DoesNotExist, nil)
			}
			if err != nil {
				panic(err)
			}

			options.FieldSelector = fields.Everything().String()
			labelSelector := labels.NewSelector().Add(*instanceIDReq)
			options.LabelSelector = labelSelector.String()
		},
	)
	informer := appInformerFactory.Argoproj().V1alpha1().Applications().Informer()
	informer.AddEventHandler(
		cache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				key, err := cache.MetaNamespaceKeyFunc(obj)
				if err == nil {
					appQueue.Add(key)
				}
			},
			UpdateFunc: func(old, new interface{}) {
				key, err := cache.MetaNamespaceKeyFunc(new)
				if err == nil {
					appQueue.Add(key)
				}
			},
			DeleteFunc: func(obj interface{}) {
				// IndexerInformer uses a delta queue, therefore for deletes we have to use this
				// key function.
				key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
				if err == nil {
					appQueue.Add(key)
				}
			},
		},
	)
	return informer
}
<|endoftext|>


<commit_before>package pb

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/keepalive"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
)

const (
	Max_Message_Size = 1 << 30 // 1 GB
)

var (
	// cache grpc connections
	grpcClients     = make(map[string]*grpc.ClientConn)
	grpcClientsLock sync.Mutex
)

func init() {
	http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 1024
	http.DefaultTransport.(*http.Transport).MaxIdleConns = 1024
}

func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
	var options []grpc.ServerOption
	options = append(options,
		grpc.KeepaliveParams(keepalive.ServerParameters{
			Time:             10 * time.Second, // wait time before ping if no activity
			Timeout:          20 * time.Second, // ping timeout
			MaxConnectionAge: 10 * time.Hour,
		}),
		grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
			MinTime:             60 * time.Second, // min time a client should wait before sending a ping
			PermitWithoutStream: false,
		}),
		grpc.MaxRecvMsgSize(Max_Message_Size),
		grpc.MaxSendMsgSize(Max_Message_Size),
	)
	for _, opt := range opts {
		if opt != nil {
			options = append(options, opt)
		}
	}
	return grpc.NewServer(options...)
}

func GrpcDial(ctx context.Context, address string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
	// opts = append(opts, grpc.WithBlock())
	// opts = append(opts, grpc.WithTimeout(time.Duration(5*time.Second)))
	var options []grpc.DialOption
	options = append(options,
		// grpc.WithInsecure(),
		grpc.WithDefaultCallOptions(
			grpc.MaxCallSendMsgSize(Max_Message_Size),
			grpc.MaxCallRecvMsgSize(Max_Message_Size),
		),
		grpc.WithKeepaliveParams(keepalive.ClientParameters{
			Time:                30 * time.Second, // client ping server if no activity for this long
			Timeout:             20 * time.Second,
			PermitWithoutStream: false,
		}))
	for _, opt := range opts {
		if opt != nil {
			options = append(options, opt)
		}
	}
	return grpc.DialContext(ctx, address, options...)
}

func getOrCreateConnection(address string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {

	grpcClientsLock.Lock()
	defer grpcClientsLock.Unlock()

	existingConnection, found := grpcClients[address]
	if found {
		return existingConnection, nil
	}

	grpcConnection, err := GrpcDial(context.Background(), address, opts...)
	if err != nil {
		return nil, fmt.Errorf("fail to dial %s: %v", address, err)
	}

	grpcClients[address] = grpcConnection

	return grpcConnection, nil
}

func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts ...grpc.DialOption) error {

	grpcConnection, err := getOrCreateConnection(address, opts...)
	if err != nil {
		return fmt.Errorf("getOrCreateConnection %s: %v", address, err)
	}
	return fn(grpcConnection)
}

func ParseServerToGrpcAddress(server string) (serverGrpcAddress string, err error) {
	return ParseServerAddress(server, 10000)
}
func ParseServersToGrpcAddresses(servers []string) (serverGrpcAddresses []string, err error) {
	for _, server := range servers {
		if serverGrpcAddress, parseErr := ParseServerToGrpcAddress(server); parseErr == nil {
			serverGrpcAddresses = append(serverGrpcAddresses, serverGrpcAddress)
		} else {
			return nil, parseErr
		}
	}
	return
}

func ParseServerAddress(server string, deltaPort int) (newServerAddress string, err error) {

	host, port, parseErr := hostAndPort(server)
	if parseErr != nil {
		return "", fmt.Errorf("server port parse error: %v", parseErr)
	}

	newPort := int(port) + deltaPort

	return fmt.Sprintf("%s:%d", host, newPort), nil
}

func hostAndPort(address string) (host string, port uint64, err error) {
	colonIndex := strings.LastIndex(address, ":")
	if colonIndex < 0 {
		return "", 0, fmt.Errorf("server should have hostname:port format: %v", address)
	}
	port, err = strconv.ParseUint(address[colonIndex+1:], 10, 64)
	if err != nil {
		return "", 0, fmt.Errorf("server port parse error: %v", err)
	}

	return address[:colonIndex], port, err
}

func ServerToGrpcAddress(server string) (serverGrpcAddress string) {

	host, port, parseErr := hostAndPort(server)
	if parseErr != nil {
		glog.Fatalf("server address %s parse error: %v", server, parseErr)
	}

	grpcPort := int(port) + 10000

	return fmt.Sprintf("%s:%d", host, grpcPort)
}

func GrpcAddressToServerAddress(grpcAddress string) (serverAddress string) {
	host, grpcPort, parseErr := hostAndPort(grpcAddress)
	if parseErr != nil {
		glog.Fatalf("server grpc address %s parse error: %v", grpcAddress, parseErr)
	}

	port := int(grpcPort) - 10000

	return fmt.Sprintf("%s:%d", host, port)
}

func WithMasterClient(master string, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) error {

	masterGrpcAddress, parseErr := ParseServerToGrpcAddress(master)
	if parseErr != nil {
		return fmt.Errorf("failed to parse master grpc %v: %v", master, parseErr)
	}

	return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
		client := master_pb.NewSeaweedClient(grpcConnection)
		return fn(client)
	}, masterGrpcAddress, grpcDialOption)

}

func WithBrokerGrpcClient(brokerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client messaging_pb.SeaweedMessagingClient) error) error {

	return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
		client := messaging_pb.NewSeaweedMessagingClient(grpcConnection)
		return fn(client)
	}, brokerGrpcAddress, grpcDialOption)

}

func WithFilerClient(filer string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {

	filerGrpcAddress, parseErr := ParseServerToGrpcAddress(filer)
	if parseErr != nil {
		return fmt.Errorf("failed to parse filer grpc %v: %v", filer, parseErr)
	}

	return WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, fn)

}

func WithGrpcFilerClient(filerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {

	return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
		return fn(client)
	}, filerGrpcAddress, grpcDialOption)

}

func WithOneOfGrpcFilerClients(filerGrpcAddresses []string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) (err error) {

	for _, filerGrpcAddress := range filerGrpcAddresses {
		err = WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
			client := filer_pb.NewSeaweedFilerClient(grpcConnection)
			return fn(client)
		}, filerGrpcAddress, grpcDialOption)
		if err == nil {
			return nil
		}
	}

	return err
}
<commit_msg>filer: re-create grpc connections if having transport error<commit_after>package pb

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/keepalive"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
)

const (
	Max_Message_Size = 1 << 30 // 1 GB
)

var (
	// cache grpc connections
	grpcClients     = make(map[string]*versionedGrpcClient)
	grpcClientsLock sync.Mutex
)

type versionedGrpcClient struct {
	*grpc.ClientConn
	version int
}

func init() {
	http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 1024
	http.DefaultTransport.(*http.Transport).MaxIdleConns = 1024
}

func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
	var options []grpc.ServerOption
	options = append(options,
		grpc.KeepaliveParams(keepalive.ServerParameters{
			Time:             10 * time.Second, // wait time before ping if no activity
			Timeout:          20 * time.Second, // ping timeout
			MaxConnectionAge: 10 * time.Hour,
		}),
		grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
			MinTime:             60 * time.Second, // min time a client should wait before sending a ping
			PermitWithoutStream: false,
		}),
		grpc.MaxRecvMsgSize(Max_Message_Size),
		grpc.MaxSendMsgSize(Max_Message_Size),
	)
	for _, opt := range opts {
		if opt != nil {
			options = append(options, opt)
		}
	}
	return grpc.NewServer(options...)
}

func GrpcDial(ctx context.Context, address string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
	// opts = append(opts, grpc.WithBlock())
	// opts = append(opts, grpc.WithTimeout(time.Duration(5*time.Second)))
	var options []grpc.DialOption
	options = append(options,
		// grpc.WithInsecure(),
		grpc.WithDefaultCallOptions(
			grpc.MaxCallSendMsgSize(Max_Message_Size),
			grpc.MaxCallRecvMsgSize(Max_Message_Size),
		),
		grpc.WithKeepaliveParams(keepalive.ClientParameters{
			Time:                30 * time.Second, // client ping server if no activity for this long
			Timeout:             20 * time.Second,
			PermitWithoutStream: false,
		}))
	for _, opt := range opts {
		if opt != nil {
			options = append(options, opt)
		}
	}
	return grpc.DialContext(ctx, address, options...)
}

func getOrCreateConnection(address string, opts ...grpc.DialOption) (*versionedGrpcClient, error) {

	grpcClientsLock.Lock()
	defer grpcClientsLock.Unlock()

	existingConnection, found := grpcClients[address]
	if found {
		return existingConnection, nil
	}

	grpcConnection, err := GrpcDial(context.Background(), address, opts...)
	if err != nil {
		return nil, fmt.Errorf("fail to dial %s: %v", address, err)
	}

	vgc := &versionedGrpcClient{
		grpcConnection,
		rand.Int(),
	}
	grpcClients[address] = vgc

	return vgc, nil
}

func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts ...grpc.DialOption) error {

	vgc, err := getOrCreateConnection(address, opts...)
	if err != nil {
		return fmt.Errorf("getOrCreateConnection %s: %v", address, err)
	}
	executionErr := fn(vgc.ClientConn)
	if executionErr != nil && strings.Contains(executionErr.Error(), "transport") {
		grpcClientsLock.Lock()
		if t, ok := grpcClients[address]; ok {
			if t.version == vgc.version {
				vgc.Close()
				delete(grpcClients, address)
			}
		}
		grpcClientsLock.Unlock()
	}

	return executionErr
}

func ParseServerToGrpcAddress(server string) (serverGrpcAddress string, err error) {
	return ParseServerAddress(server, 10000)
}
func ParseServersToGrpcAddresses(servers []string) (serverGrpcAddresses []string, err error) {
	for _, server := range servers {
		if serverGrpcAddress, parseErr := ParseServerToGrpcAddress(server); parseErr == nil {
			serverGrpcAddresses = append(serverGrpcAddresses, serverGrpcAddress)
		} else {
			return nil, parseErr
		}
	}
	return
}

func ParseServerAddress(server string, deltaPort int) (newServerAddress string, err error) {

	host, port, parseErr := hostAndPort(server)
	if parseErr != nil {
		return "", fmt.Errorf("server port parse error: %v", parseErr)
	}

	newPort := int(port) + deltaPort

	return fmt.Sprintf("%s:%d", host, newPort), nil
}

func hostAndPort(address string) (host string, port uint64, err error) {
	colonIndex := strings.LastIndex(address, ":")
	if colonIndex < 0 {
		return "", 0, fmt.Errorf("server should have hostname:port format: %v", address)
	}
	port, err = strconv.ParseUint(address[colonIndex+1:], 10, 64)
	if err != nil {
		return "", 0, fmt.Errorf("server port parse error: %v", err)
	}

	return address[:colonIndex], port, err
}

func ServerToGrpcAddress(server string) (serverGrpcAddress string) {

	host, port, parseErr := hostAndPort(server)
	if parseErr != nil {
		glog.Fatalf("server address %s parse error: %v", server, parseErr)
	}

	grpcPort := int(port) + 10000

	return fmt.Sprintf("%s:%d", host, grpcPort)
}

func GrpcAddressToServerAddress(grpcAddress string) (serverAddress string) {
	host, grpcPort, parseErr := hostAndPort(grpcAddress)
	if parseErr != nil {
		glog.Fatalf("server grpc address %s parse error: %v", grpcAddress, parseErr)
	}

	port := int(grpcPort) - 10000

	return fmt.Sprintf("%s:%d", host, port)
}

func WithMasterClient(master string, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) error {

	masterGrpcAddress, parseErr := ParseServerToGrpcAddress(master)
	if parseErr != nil {
		return fmt.Errorf("failed to parse master grpc %v: %v", master, parseErr)
	}

	return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
		client := master_pb.NewSeaweedClient(grpcConnection)
		return fn(client)
	}, masterGrpcAddress, grpcDialOption)

}

func WithBrokerGrpcClient(brokerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client messaging_pb.SeaweedMessagingClient) error) error {

	return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
		client := messaging_pb.NewSeaweedMessagingClient(grpcConnection)
		return fn(client)
	}, brokerGrpcAddress, grpcDialOption)

}

func WithFilerClient(filer string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {

	filerGrpcAddress, parseErr := ParseServerToGrpcAddress(filer)
	if parseErr != nil {
		return fmt.Errorf("failed to parse filer grpc %v: %v", filer, parseErr)
	}

	return WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, fn)

}

func WithGrpcFilerClient(filerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {

	return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
		return fn(client)
	}, filerGrpcAddress, grpcDialOption)

}

func WithOneOfGrpcFilerClients(filerGrpcAddresses []string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) (err error) {

	for _, filerGrpcAddress := range filerGrpcAddresses {
		err = WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
			client := filer_pb.NewSeaweedFilerClient(grpcConnection)
			return fn(client)
		}, filerGrpcAddress, grpcDialOption)
		if err == nil {
			return nil
		}
	}

	return err
}
<|endoftext|>


<commit_before>package controllers

import (
	"github.com/freeznet/tomato/cloud"
	"github.com/freeznet/tomato/errs"
	"github.com/freeznet/tomato/rest"
	"github.com/freeznet/tomato/types"
	"github.com/freeznet/tomato/utils"
	"time"
)

// CloudCodeController ...
type CloudCodeController struct {
	ClassesController
}

func formatJobSchedule(jobSchedule types.M) types.M  {
	if _, ok := jobSchedule["startAfter"]; !ok {
		jobSchedule["startAfter"] = utils.TimetoString(time.Now())
	}
	return jobSchedule
}

func (c *CloudCodeController) validateJobSchedule()  {
	jobs := cloud.GetJobs()
	if v, ok := c.JSONBody["jobName"]; ok {
		if _, ok := jobs[v.(string)]; !ok {
			c.HandleError(errs.E(errs.InternalServerError, "Cannot Schedule a job that is not deployed"), 0)
		}
	}
}

//Prepare ...
func (c *CloudCodeController) Prepare()  {
	c.ClassesController.Prepare()
	if c.Ctx.ResponseWriter.Started == false {
		c.EnforceMasterKeyAccess()
	}
}

// HandleFind ...
// @router /jobs [get]
func (c *CloudCodeController) GetJobs()  {
	response, err := rest.Find(c.Auth, "_JobSchedule", types.M{}, types.M{}, c.Info.ClientSDK)

	if err != nil {
		c.HandleError(err, 0)
		return
	}

	results := utils.A(response["results"])

	c.Data["json"] = results[0]
	c.ServeJSON()
}

// HandleGet ...
// @router /jobs/data [get]
func (c *CloudCodeController) GetJobsData() {
	jobs := cloud.GetJobs()
	jobNames := []string{}
	for n := range jobs {
		jobNames = append(jobNames, n)
	}

	response, err := rest.Find(c.Auth, "_JobSchedule", types.M{}, types.M{}, nil)

	if err != nil {
		c.HandleError(err, 0)
		return
	}

	results := utils.A(response["results"])
	resultsJobNames := []string{}
	for _, v := range results {
		result := utils.M(v)
		resultsJobNames = append(resultsJobNames, result["jobName"].(string))
	}

	c.Data["json"] = types.M{"in_use": resultsJobNames, "jobs": jobNames}
	c.ServeJSON()
}

// CreateJob ...
// @router /jobs [create]
func (c *CloudCodeController) CreateJob()  {
	c.validateJobSchedule()

	c.ClassName = "_JobSchedule"
	c.JSONBody = formatJobSchedule(c.JSONBody)
	c.ClassesController.HandleCreate()
}

// EditJob ...
// @router /jobs/:objectId [put]
func (c *CloudCodeController) EditJob()  {
	c.validateJobSchedule()

	c.ClassName = "_JobSchedule"
	c.ObjectID = c.Ctx.Input.Param(":objectId")
	c.JSONBody = formatJobSchedule(c.JSONBody)
	c.ClassesController.HandleUpdate()
}

// DeleteJob ...
// @router / [delete]
func (c *CloudCodeController) DeleteJob()  {
	c.ClassName = "_JobSchedule"
	c.ClassesController.HandleDelete()
}
<commit_msg>BKD-8-restores-the-job-scheduling-endpoints：函数注释修改——同步名称<commit_after>package controllers

import (
	"github.com/freeznet/tomato/cloud"
	"github.com/freeznet/tomato/errs"
	"github.com/freeznet/tomato/rest"
	"github.com/freeznet/tomato/types"
	"github.com/freeznet/tomato/utils"
	"time"
)

// CloudCodeController ...
type CloudCodeController struct {
	ClassesController
}

func formatJobSchedule(jobSchedule types.M) types.M  {
	if _, ok := jobSchedule["startAfter"]; !ok {
		jobSchedule["startAfter"] = utils.TimetoString(time.Now())
	}
	return jobSchedule
}

func (c *CloudCodeController) validateJobSchedule()  {
	jobs := cloud.GetJobs()
	if v, ok := c.JSONBody["jobName"]; ok {
		if _, ok := jobs[v.(string)]; !ok {
			c.HandleError(errs.E(errs.InternalServerError, "Cannot Schedule a job that is not deployed"), 0)
		}
	}
}

//Prepare ...
func (c *CloudCodeController) Prepare()  {
	c.ClassesController.Prepare()
	if c.Ctx.ResponseWriter.Started == false {
		c.EnforceMasterKeyAccess()
	}
}

// GetJobs ...
// @router /jobs [get]
func (c *CloudCodeController) GetJobs()  {
	response, err := rest.Find(c.Auth, "_JobSchedule", types.M{}, types.M{}, c.Info.ClientSDK)

	if err != nil {
		c.HandleError(err, 0)
		return
	}

	results := utils.A(response["results"])

	c.Data["json"] = results[0]
	c.ServeJSON()
}

// GetJobsData ...
// @router /jobs/data [get]
func (c *CloudCodeController) GetJobsData() {
	jobs := cloud.GetJobs()
	jobNames := []string{}
	for n := range jobs {
		jobNames = append(jobNames, n)
	}

	response, err := rest.Find(c.Auth, "_JobSchedule", types.M{}, types.M{}, nil)

	if err != nil {
		c.HandleError(err, 0)
		return
	}

	results := utils.A(response["results"])
	resultsJobNames := []string{}
	for _, v := range results {
		result := utils.M(v)
		resultsJobNames = append(resultsJobNames, result["jobName"].(string))
	}

	c.Data["json"] = types.M{"in_use": resultsJobNames, "jobs": jobNames}
	c.ServeJSON()
}

// CreateJob ...
// @router /jobs [create]
func (c *CloudCodeController) CreateJob()  {
	c.validateJobSchedule()

	c.ClassName = "_JobSchedule"
	c.JSONBody = formatJobSchedule(c.JSONBody)
	c.ClassesController.HandleCreate()
}

// EditJob ...
// @router /jobs/:objectId [put]
func (c *CloudCodeController) EditJob()  {
	c.validateJobSchedule()

	c.ClassName = "_JobSchedule"
	c.ObjectID = c.Ctx.Input.Param(":objectId")
	c.JSONBody = formatJobSchedule(c.JSONBody)
	c.ClassesController.HandleUpdate()
}

// DeleteJob ...
// @router / [delete]
func (c *CloudCodeController) DeleteJob()  {
	c.ClassName = "_JobSchedule"
	c.ClassesController.HandleDelete()
}
<|endoftext|>