<commit_before>package template

import (
	"bytes"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"path"
	"regexp"
	"strconv"
	"strings"
	"text/template/parse"
)

type FuncMap map[string]interface{}

type ScriptType int

const (
	_ ScriptType = iota
	ScriptTypeStandard
	ScriptTypeAsync
	ScriptTypeOnload
)

const (
	leftDelim       = "{{"
	rightDelim      = "}}"
	stylesTmplName  = "__styles"
	scriptsTmplName = "__scripts"
)

var stylesBoilerplate = `
  {{ range __getstyles }}
    <link rel="stylesheet" type="text/css" href="{{ asset . }}">
  {{ end }}
`

var scriptsBoilerplate = `
  {{ range __getscripts }}
    {{ if .IsAsync }}

    {{ else }}

    {{ end }}
  {{ end }}
`

var (
	commentRe   = regexp.MustCompile(`(?s:\{\{\\*(.*?)\*/\}\})`)
	keyRe       = regexp.MustCompile(`(?s:\s*([\w\-_])+:)`)
	stylesTree  = compileTree(stylesTmplName, stylesBoilerplate)
	scriptsTree = compileTree(scriptsTmplName, scriptsBoilerplate)
)

type script struct {
	Name string
	Type ScriptType
}

func (s *script) IsAsync() bool {
	return s.Type == ScriptTypeAsync
}

type Template struct {
	*template.Template
	funcMap FuncMap
	root    string
	scripts []*script
	styles  []string
}

func (t *Template) parseScripts(value string, st ScriptType) {
	for _, v := range strings.Split(value, ",") {
		name := strings.TrimSpace(v)
		t.scripts = append(t.scripts, &script{name, st})
	}
}

func (t *Template) parseComment(comment string, file string) error {
	lines := strings.Split(comment, "\n")
	extended := false
	for _, v := range lines {
		m := keyRe.FindStringSubmatchIndex(v)
		if m != nil && m[0] == 0 && len(m) == 4 {
			start := m[1] - m[3]
			end := start + m[2]
			key := strings.TrimSpace(v[start:end])
			value := strings.TrimSpace(v[m[1]:])
			if value != "" {
				switch strings.ToLower(key) {
				case "script", "scripts":
					t.parseScripts(value, ScriptTypeStandard)
				case "ascript", "ascripts":
					t.parseScripts(value, ScriptTypeAsync)
				case "css", "style", "styles":
					for _, v := range strings.Split(value, ",") {
						style := strings.TrimSpace(v)
						t.styles = append(t.styles, style)
					}
				case "extend", "extends":
					extendedFile := path.Join(path.Dir(file), value)
					err := t.load(extendedFile)
					if err != nil {
						return err
					}
					extended = true
				}
			}
		}
	}
	if !extended {
		t.root = file
	}
	return nil
}

func (t *Template) load(file string) error {
	b, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	s := string(b)
	matches := commentRe.FindStringSubmatch(s)
	comment := ""
	if matches != nil && len(matches) > 0 {
		comment = matches[1]
	}
	err = t.parseComment(comment, file)
	if err != nil {
		return err
	}
	if idx := strings.Index(s, ""); idx >= 0 {
		s = s[:idx] + "{{ template \"__styles\" }}" + s[idx:]
	}
	if idx := strings.Index(s, ""); idx >= 0 {
		s = s[:idx] + "{{ template \"__scripts\" }}" + s[idx:]
	}
	var tmpl *template.Template
	if t.Template == nil {
		t.Template = template.New(file)
		tmpl = t.Template
	} else {
		tmpl = t.Template.New(file)
	}
	tmpl.Funcs(templateFuncs).Funcs(template.FuncMap(t.funcMap))
	tmpl, err = tmpl.Parse(s)
	if err != nil {
		return err
	}
	return nil
}

func (t *Template) Funcs(funcs FuncMap) {
	if t.funcMap == nil {
		t.funcMap = make(FuncMap)
	}
	for k, v := range funcs {
		t.funcMap[k] = v
	}
}

func (t *Template) Parse(file string) error {
	err := t.load(file)
	if err != nil {
		return err
	}
	/* Add styles and scripts */
	_, err = t.AddParseTree(stylesTmplName, stylesTree)
	if err != nil {
		return err
	}
	_, err = t.AddParseTree(scriptsTmplName, scriptsTree)
	if err != nil {
		return err
	}
	return nil
}

func (t *Template) Execute(w io.Writer, data interface{}) error {
	var buf bytes.Buffer
	err := t.ExecuteTemplate(&buf, t.root, data)
	if err != nil {
		return err
	}
	if rw, ok := w.(http.ResponseWriter); ok {
		header := rw.Header()
		header.Set("Content-Type", "text/html; charset=utf-8")
		header.Set("Content-Length", strconv.Itoa(buf.Len()))
		rw.Write(buf.Bytes())
	}
	return nil
}

func (t *Template) MustExecute(w io.Writer, data interface{}) {
	err := t.Execute(w, data)
	if err != nil {
		log.Panicf("Error executing template: %s\n", err)
	}
}

func AddFunc(name string, f interface{}) {
	templateFuncs[name] = f
}

func New() *Template {
	t := &Template{}
	t.Funcs(FuncMap{
		"__getstyles":  func() []string { return t.styles },
		"__getscripts": func() []*script { return t.scripts },
	})
	return t
}

func Parse(file string) (*Template, error) {
	t := New()
	err := t.Parse(file)
	if err != nil {
		return nil, err
	}
	return t, nil
}

func MustParse(file string) *Template {
	t, err := Parse(file)
	if err != nil {
		log.Fatalf("Error loading template %s: %s\n", file, err)
	}
	return t
}

func compileTree(name, text string) *parse.Tree {
	funcs := map[string]interface{}{
		"__getstyles":  func() {},
		"__getscripts": func() {},
		"asset":        func() {},
	}
	treeMap, err := parse.Parse(name, text, leftDelim, rightDelim, funcs)
	if err != nil {
		panic(err)
	}
	return treeMap[name]
}
<commit_msg>Use text/template/parse to load the templates<commit_after>package template

import (
	"bytes"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"path"
	"regexp"
	"strconv"
	"strings"
	"text/template/parse"
)

type FuncMap map[string]interface{}

type ScriptType int

const (
	_ ScriptType = iota
	ScriptTypeStandard
	ScriptTypeAsync
	ScriptTypeOnload
)

const (
	leftDelim       = "{{"
	rightDelim      = "}}"
	stylesTmplName  = "__styles"
	scriptsTmplName = "__scripts"
)

var stylesBoilerplate = `
  {{ range __getstyles }}
    <link rel="stylesheet" type="text/css" href="{{ asset . }}">
  {{ end }}
`

var scriptsBoilerplate = `
  {{ range __getscripts }}
    {{ if .IsAsync }}

    {{ else }}

    {{ end }}
  {{ end }}
`

var (
	commentRe   = regexp.MustCompile(`(?s:\{\{\\*(.*?)\*/\}\})`)
	keyRe       = regexp.MustCompile(`(?s:\s*([\w\-_])+:)`)
	stylesTree  = compileTree(stylesTmplName, stylesBoilerplate)
	scriptsTree = compileTree(scriptsTmplName, scriptsBoilerplate)
)

type script struct {
	Name string
	Type ScriptType
}

func (s *script) IsAsync() bool {
	return s.Type == ScriptTypeAsync
}

type Template struct {
	*template.Template
	Trees   map[string]*parse.Tree
	funcMap FuncMap
	root    string
	scripts []*script
	styles  []string
}

func (t *Template) parseScripts(value string, st ScriptType) {
	for _, v := range strings.Split(value, ",") {
		name := strings.TrimSpace(v)
		t.scripts = append(t.scripts, &script{name, st})
	}
}

func (t *Template) parseComment(comment string, file string) error {
	lines := strings.Split(comment, "\n")
	extended := false
	for _, v := range lines {
		m := keyRe.FindStringSubmatchIndex(v)
		if m != nil && m[0] == 0 && len(m) == 4 {
			start := m[1] - m[3]
			end := start + m[2]
			key := strings.TrimSpace(v[start:end])
			value := strings.TrimSpace(v[m[1]:])
			if value != "" {
				switch strings.ToLower(key) {
				case "script", "scripts":
					t.parseScripts(value, ScriptTypeStandard)
				case "ascript", "ascripts":
					t.parseScripts(value, ScriptTypeAsync)
				case "css", "style", "styles":
					for _, v := range strings.Split(value, ",") {
						style := strings.TrimSpace(v)
						t.styles = append(t.styles, style)
					}
				case "extend", "extends":
					extendedFile := path.Join(path.Dir(file), value)
					err := t.load(extendedFile)
					if err != nil {
						return err
					}
					extended = true
				}
			}
		}
	}
	if !extended {
		t.root = file
	}
	return nil
}

func (t *Template) load(file string) error {
	b, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	s := string(b)
	matches := commentRe.FindStringSubmatch(s)
	comment := ""
	if matches != nil && len(matches) > 0 {
		comment = matches[1]
	}
	err = t.parseComment(comment, file)
	if err != nil {
		return err
	}
	if idx := strings.Index(s, ""); idx >= 0 {
		s = s[:idx] + "{{ template \"__styles\" }}" + s[idx:]
	}
	if idx := strings.Index(s, ""); idx >= 0 {
		s = s[:idx] + "{{ template \"__scripts\" }}" + s[idx:]
	}
	treeMap, err := parse.Parse(file, s, leftDelim, rightDelim, templateFuncs, t.funcMap)
	if err != nil {
		return err
	}
	for k, v := range treeMap {
		err := t.AddParseTree(k, v)
		if err != nil {
			return err
		}
	}
	return nil
}

func (t *Template) Funcs(funcs FuncMap) {
	if t.funcMap == nil {
		t.funcMap = make(FuncMap)
	}
	for k, v := range funcs {
		t.funcMap[k] = v
	}
	t.Template.Funcs(template.FuncMap(t.funcMap))
}

func (t *Template) Parse(file string) error {
	err := t.load(file)
	if err != nil {
		return err
	}
	/* Add styles and scripts */
	err = t.AddParseTree(stylesTmplName, stylesTree)
	if err != nil {
		return err
	}
	err = t.AddParseTree(scriptsTmplName, scriptsTree)
	if err != nil {
		return err
	}
	return nil
}

func (t *Template) AddParseTree(name string, tree *parse.Tree) error {
	_, err := t.Template.AddParseTree(name, tree)
	if err != nil {
		return err
	}
	t.Trees[name] = tree
	return nil
}

func (t *Template) Execute(w io.Writer, data interface{}) error {
	var buf bytes.Buffer
	err := t.ExecuteTemplate(&buf, t.root, data)
	if err != nil {
		return err
	}
	if rw, ok := w.(http.ResponseWriter); ok {
		header := rw.Header()
		header.Set("Content-Type", "text/html; charset=utf-8")
		header.Set("Content-Length", strconv.Itoa(buf.Len()))
		rw.Write(buf.Bytes())
	}
	return nil
}

func (t *Template) MustExecute(w io.Writer, data interface{}) {
	err := t.Execute(w, data)
	if err != nil {
		log.Panicf("Error executing template: %s\n", err)
	}
}

func AddFunc(name string, f interface{}) {
	templateFuncs[name] = f
}

func New() *Template {
	t := &Template{
		Template: template.New(""),
		Trees:    make(map[string]*parse.Tree),
	}
	// This is required so text/template calls t.init()
	// and initializes the common data structure
	t.Template.New("")
	funcs := FuncMap{
		"__getstyles":  func() []string { return t.styles },
		"__getscripts": func() []*script { return t.scripts },
	}
	t.Funcs(funcs)
	t.Template.Funcs(template.FuncMap(funcs)).Funcs(templateFuncs)
	return t
}

func Parse(file string) (*Template, error) {
	t := New()
	err := t.Parse(file)
	if err != nil {
		return nil, err
	}
	return t, nil
}

func MustParse(file string) *Template {
	t, err := Parse(file)
	if err != nil {
		log.Fatalf("Error loading template %s: %s\n", file, err)
	}
	return t
}

func compileTree(name, text string) *parse.Tree {
	funcs := map[string]interface{}{
		"__getstyles":  func() {},
		"__getscripts": func() {},
		"asset":        func() {},
	}
	treeMap, err := parse.Parse(name, text, leftDelim, rightDelim, funcs)
	if err != nil {
		panic(err)
	}
	return treeMap[name]
}
<|endoftext|>


<commit_before>// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package markdown

import (
	"bytes"
	"fmt"
	"strings"

	"code.gitea.io/gitea/modules/markup"
	"code.gitea.io/gitea/modules/markup/common"
	giteautil "code.gitea.io/gitea/modules/util"

	"github.com/yuin/goldmark/ast"
	east "github.com/yuin/goldmark/extension/ast"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/renderer"
	"github.com/yuin/goldmark/renderer/html"
	"github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"
)

var byteMailto = []byte("mailto:")

// GiteaASTTransformer is a default transformer of the goldmark tree.
type GiteaASTTransformer struct{}

// Transform transforms the given AST tree.
func (g *GiteaASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
	_ = ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
		if !entering {
			return ast.WalkContinue, nil
		}

		switch v := n.(type) {
		case *ast.Image:
			// Images need two things:
			//
			// 1. Their src needs to munged to be a real value
			// 2. If they're not wrapped with a link they need a link wrapper

			// Check if the destination is a real link
			link := v.Destination
			if len(link) > 0 && !markup.IsLink(link) {
				prefix := pc.Get(urlPrefixKey).(string)
				if pc.Get(isWikiKey).(bool) {
					prefix = giteautil.URLJoin(prefix, "wiki", "raw")
				}
				prefix = strings.Replace(prefix, "/src/", "/media/", 1)

				lnk := string(link)
				lnk = giteautil.URLJoin(prefix, lnk)
				lnk = strings.Replace(lnk, " ", "+", -1)
				link = []byte(lnk)
			}
			v.Destination = link

			parent := n.Parent()
			// Create a link around image only if parent is not already a link
			if _, ok := parent.(*ast.Link); !ok && parent != nil {
				wrap := ast.NewLink()
				wrap.Destination = link
				wrap.Title = v.Title
				parent.ReplaceChild(parent, n, wrap)
				wrap.AppendChild(wrap, n)
			}
		case *ast.Link:
			// Links need their href to munged to be a real value
			link := v.Destination
			if len(link) > 0 && !markup.IsLink(link) &&
				link[0] != '#' && !bytes.HasPrefix(link, byteMailto) {
				// special case: this is not a link, a hash link or a mailto:, so it's a
				// relative URL
				lnk := string(link)
				if pc.Get(isWikiKey).(bool) {
					lnk = giteautil.URLJoin("wiki", lnk)
				}
				link = []byte(giteautil.URLJoin(pc.Get(urlPrefixKey).(string), lnk))
			}
			v.Destination = link
		}
		return ast.WalkContinue, nil
	})
}

type prefixedIDs struct {
	values map[string]bool
}

// Generate generates a new element id.
func (p *prefixedIDs) Generate(value []byte, kind ast.NodeKind) []byte {
	dft := []byte("id")
	if kind == ast.KindHeading {
		dft = []byte("heading")
	}
	return p.GenerateWithDefault(value, dft)
}

// Generate generates a new element id.
func (p *prefixedIDs) GenerateWithDefault(value []byte, dft []byte) []byte {
	result := common.CleanValue(value)
	if len(result) == 0 {
		result = dft
	}
	if !bytes.HasPrefix(result, []byte("user-content-")) {
		result = append([]byte("user-content-"), result...)
	}
	if _, ok := p.values[util.BytesToReadOnlyString(result)]; !ok {
		p.values[util.BytesToReadOnlyString(result)] = true
		return result
	}
	for i := 1; ; i++ {
		newResult := fmt.Sprintf("%s-%d", result, i)
		if _, ok := p.values[newResult]; !ok {
			p.values[newResult] = true
			return []byte(newResult)
		}
	}
}

// Put puts a given element id to the used ids table.
func (p *prefixedIDs) Put(value []byte) {
	p.values[util.BytesToReadOnlyString(value)] = true
}

func newPrefixedIDs() *prefixedIDs {
	return &prefixedIDs{
		values: map[string]bool{},
	}
}

// NewTaskCheckBoxHTMLRenderer creates a TaskCheckBoxHTMLRenderer to render tasklists
// in the gitea form.
func NewTaskCheckBoxHTMLRenderer(opts ...html.Option) renderer.NodeRenderer {
	r := &TaskCheckBoxHTMLRenderer{
		Config: html.NewConfig(),
	}
	for _, opt := range opts {
		opt.SetHTMLOption(&r.Config)
	}
	return r
}

// TaskCheckBoxHTMLRenderer is a renderer.NodeRenderer implementation that
// renders checkboxes in list items.
// Overrides the default goldmark one to present the gitea format
type TaskCheckBoxHTMLRenderer struct {
	html.Config
}

// RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs.
func (r *TaskCheckBoxHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) {
	reg.Register(east.KindTaskCheckBox, r.renderTaskCheckBox)
}

func (r *TaskCheckBoxHTMLRenderer) renderTaskCheckBox(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
	if !entering {
		return ast.WalkContinue, nil
	}
	n := node.(*east.TaskCheckBox)

	end := ">"
	if r.XHTML {
		end = " />"
	}
	var err error
	if n.IsChecked {
		_, err = w.WriteString(`<span class="ui fitted disabled checkbox"><input type="checkbox" disabled="disabled"` + end + `<label` + end + `</span>`)
	} else {
		_, err = w.WriteString(`<span class="ui checked fitted disabled checkbox"><input type="checkbox" checked="" disabled="disabled"` + end + `<label` + end + `</span>`)
	}
	if err != nil {
		return ast.WalkStop, err
	}
	return ast.WalkContinue, nil
}
<commit_msg>Fix markdown anchor links (#9673)<commit_after>// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package markdown

import (
	"bytes"
	"fmt"
	"strings"

	"code.gitea.io/gitea/modules/markup"
	"code.gitea.io/gitea/modules/markup/common"
	giteautil "code.gitea.io/gitea/modules/util"

	"github.com/yuin/goldmark/ast"
	east "github.com/yuin/goldmark/extension/ast"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/renderer"
	"github.com/yuin/goldmark/renderer/html"
	"github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"
)

var byteMailto = []byte("mailto:")

// GiteaASTTransformer is a default transformer of the goldmark tree.
type GiteaASTTransformer struct{}

// Transform transforms the given AST tree.
func (g *GiteaASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
	_ = ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
		if !entering {
			return ast.WalkContinue, nil
		}

		switch v := n.(type) {
		case *ast.Image:
			// Images need two things:
			//
			// 1. Their src needs to munged to be a real value
			// 2. If they're not wrapped with a link they need a link wrapper

			// Check if the destination is a real link
			link := v.Destination
			if len(link) > 0 && !markup.IsLink(link) {
				prefix := pc.Get(urlPrefixKey).(string)
				if pc.Get(isWikiKey).(bool) {
					prefix = giteautil.URLJoin(prefix, "wiki", "raw")
				}
				prefix = strings.Replace(prefix, "/src/", "/media/", 1)

				lnk := string(link)
				lnk = giteautil.URLJoin(prefix, lnk)
				lnk = strings.Replace(lnk, " ", "+", -1)
				link = []byte(lnk)
			}
			v.Destination = link

			parent := n.Parent()
			// Create a link around image only if parent is not already a link
			if _, ok := parent.(*ast.Link); !ok && parent != nil {
				wrap := ast.NewLink()
				wrap.Destination = link
				wrap.Title = v.Title
				parent.ReplaceChild(parent, n, wrap)
				wrap.AppendChild(wrap, n)
			}
		case *ast.Link:
			// Links need their href to munged to be a real value
			link := v.Destination
			if len(link) > 0 && !markup.IsLink(link) &&
				link[0] != '#' && !bytes.HasPrefix(link, byteMailto) {
				// special case: this is not a link, a hash link or a mailto:, so it's a
				// relative URL
				lnk := string(link)
				if pc.Get(isWikiKey).(bool) {
					lnk = giteautil.URLJoin("wiki", lnk)
				}
				link = []byte(giteautil.URLJoin(pc.Get(urlPrefixKey).(string), lnk))
			}
			if len(link) > 0 && link[0] == '#' {
				link = []byte("#user-content-" + string(link)[1:])
			}
			v.Destination = link
		}
		return ast.WalkContinue, nil
	})
}

type prefixedIDs struct {
	values map[string]bool
}

// Generate generates a new element id.
func (p *prefixedIDs) Generate(value []byte, kind ast.NodeKind) []byte {
	dft := []byte("id")
	if kind == ast.KindHeading {
		dft = []byte("heading")
	}
	return p.GenerateWithDefault(value, dft)
}

// Generate generates a new element id.
func (p *prefixedIDs) GenerateWithDefault(value []byte, dft []byte) []byte {
	result := common.CleanValue(value)
	if len(result) == 0 {
		result = dft
	}
	if !bytes.HasPrefix(result, []byte("user-content-")) {
		result = append([]byte("user-content-"), result...)
	}
	if _, ok := p.values[util.BytesToReadOnlyString(result)]; !ok {
		p.values[util.BytesToReadOnlyString(result)] = true
		return result
	}
	for i := 1; ; i++ {
		newResult := fmt.Sprintf("%s-%d", result, i)
		if _, ok := p.values[newResult]; !ok {
			p.values[newResult] = true
			return []byte(newResult)
		}
	}
}

// Put puts a given element id to the used ids table.
func (p *prefixedIDs) Put(value []byte) {
	p.values[util.BytesToReadOnlyString(value)] = true
}

func newPrefixedIDs() *prefixedIDs {
	return &prefixedIDs{
		values: map[string]bool{},
	}
}

// NewTaskCheckBoxHTMLRenderer creates a TaskCheckBoxHTMLRenderer to render tasklists
// in the gitea form.
func NewTaskCheckBoxHTMLRenderer(opts ...html.Option) renderer.NodeRenderer {
	r := &TaskCheckBoxHTMLRenderer{
		Config: html.NewConfig(),
	}
	for _, opt := range opts {
		opt.SetHTMLOption(&r.Config)
	}
	return r
}

// TaskCheckBoxHTMLRenderer is a renderer.NodeRenderer implementation that
// renders checkboxes in list items.
// Overrides the default goldmark one to present the gitea format
type TaskCheckBoxHTMLRenderer struct {
	html.Config
}

// RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs.
func (r *TaskCheckBoxHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) {
	reg.Register(east.KindTaskCheckBox, r.renderTaskCheckBox)
}

func (r *TaskCheckBoxHTMLRenderer) renderTaskCheckBox(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
	if !entering {
		return ast.WalkContinue, nil
	}
	n := node.(*east.TaskCheckBox)

	end := ">"
	if r.XHTML {
		end = " />"
	}
	var err error
	if n.IsChecked {
		_, err = w.WriteString(`<span class="ui fitted disabled checkbox"><input type="checkbox" disabled="disabled"` + end + `<label` + end + `</span>`)
	} else {
		_, err = w.WriteString(`<span class="ui checked fitted disabled checkbox"><input type="checkbox" checked="" disabled="disabled"` + end + `<label` + end + `</span>`)
	}
	if err != nil {
		return ast.WalkStop, err
	}
	return ast.WalkContinue, nil
}
<|endoftext|>


<commit_before>package contractor

import (
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// renewThreshold calculates an appropriate renewThreshold for a given period.
func renewThreshold(period types.BlockHeight) types.BlockHeight {
	threshold := period / 4
	if threshold > 1000 {
		threshold = 1000
	}
	return threshold
}

// ProcessConsensusChange will be called by the consensus set every time there
// is a change in the blockchain. Updates will always be called in order.
func (c *Contractor) ProcessConsensusChange(cc modules.ConsensusChange) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if c.blockHeight != 0 || cc.AppliedBlocks[len(cc.AppliedBlocks)-1].ID() != types.GenesisBlock.ID() {
		c.blockHeight += types.BlockHeight(len(cc.AppliedBlocks))
		c.blockHeight -= types.BlockHeight(len(cc.RevertedBlocks))
	}

	// renew/replace contracts
	if c.blockHeight+renewThreshold(c.allowance.Period) > c.renewHeight {
		// set new renewHeight immediately; this prevents new renew/replace
		// goroutines from being spawned on every block until they succeed.
		// However, this means those goroutines must reset the renewHeight
		// manually if they fail.
		oldHeight := c.renewHeight
		c.renewHeight += c.allowance.Period

		// if newAllowance is set, use the new allowance to form new
		// contracts. Otherwise, renew existing contracts.
		if c.newAllowance.Hosts != 0 {
			go func() {
				err := c.formContracts(c.newAllowance)
				if err != nil {
					c.log.Println("WARN: failed to form contracts with new allowance:", err)
				}
				c.mu.Lock()
				defer c.mu.Unlock()
				if err == nil {
					// if contract formation succeeded, clear newAllowance
					c.allowance = c.newAllowance
					c.newAllowance = modules.Allowance{}
				} else {
					// otherwise, reset renewHeight so that we'll try again on the next block
					c.renewHeight = oldHeight
				}
			}()
		} else {
			go func() {
				newHeight := c.renewHeight + c.allowance.Period
				for _, contract := range c.contracts {
					if contract.FileContract.WindowStart == c.renewHeight {
						_, err := c.managedRenew(contract.ID, newHeight)
						if err != nil {
							c.log.Println("WARN: failed to renew contract", contract.ID, ":", err)
						}
					}
				}
				// TODO: reset renewHeight if too many rewewals failed.
			}()
		}
	}
}
<commit_msg>fix race condition in contractor update<commit_after>package contractor

import (
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// renewThreshold calculates an appropriate renewThreshold for a given period.
func renewThreshold(period types.BlockHeight) types.BlockHeight {
	threshold := period / 4
	if threshold > 1000 {
		threshold = 1000
	}
	return threshold
}

// ProcessConsensusChange will be called by the consensus set every time there
// is a change in the blockchain. Updates will always be called in order.
func (c *Contractor) ProcessConsensusChange(cc modules.ConsensusChange) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if c.blockHeight != 0 || cc.AppliedBlocks[len(cc.AppliedBlocks)-1].ID() != types.GenesisBlock.ID() {
		c.blockHeight += types.BlockHeight(len(cc.AppliedBlocks))
		c.blockHeight -= types.BlockHeight(len(cc.RevertedBlocks))
	}

	// renew/replace contracts
	if c.blockHeight+renewThreshold(c.allowance.Period) > c.renewHeight {
		// set new renewHeight immediately; this prevents new renew/replace
		// goroutines from being spawned on every block until they succeed.
		// However, this means those goroutines must reset the renewHeight
		// manually if they fail.
		oldHeight := c.renewHeight
		newHeight := oldHeight + c.allowance.Period
		c.renewHeight = newHeight

		// if newAllowance is set, use the new allowance to form new
		// contracts. Otherwise, renew existing contracts.
		if c.newAllowance.Hosts != 0 {
			go func() {
				err := c.formContracts(c.newAllowance)
				if err != nil {
					c.log.Println("WARN: failed to form contracts with new allowance:", err)
				}
				c.mu.Lock()
				defer c.mu.Unlock()
				if err == nil {
					// if contract formation succeeded, clear newAllowance
					c.allowance = c.newAllowance
					c.newAllowance = modules.Allowance{}
				} else {
					// otherwise, reset renewHeight so that we'll try again on the next block
					c.renewHeight = oldHeight
				}
			}()
		} else {
			go func() {
				for _, contract := range c.contracts {
					if contract.FileContract.WindowStart == c.renewHeight {
						_, err := c.managedRenew(contract.ID, newHeight)
						if err != nil {
							c.log.Println("WARN: failed to renew contract", contract.ID, ":", err)
						}
					}
				}
				// TODO: reset renewHeight if too many rewewals failed.
			}()
		}
	}
}
<|endoftext|>


<commit_before>package middleware

import (
	"fmt"
	"net/http"

	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"

	"gopkg.in/macaron.v1"
)

func RequestTracing(handler string) macaron.Handler {
	return func(res http.ResponseWriter, req *http.Request, c *macaron.Context) {
		rw := res.(macaron.ResponseWriter)

		tracer := opentracing.GlobalTracer()
		wireContext, _ := tracer.Extract(opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(req.Header))
		span := tracer.StartSpan(fmt.Sprintf("HTTP %s", handler), ext.RPCServerOption(wireContext))
		defer span.Finish()

		ctx := opentracing.ContextWithSpan(req.Context(), span)
		c.Req.Request = req.WithContext(ctx)

		c.Next()

		status := rw.Status()

		ext.HTTPStatusCode.Set(span, uint16(status))
		ext.HTTPUrl.Set(span, req.RequestURI)
		ext.HTTPMethod.Set(span, req.Method)
	}
}
<commit_msg>mark >=400 responses as error<commit_after>package middleware

import (
	"fmt"
	"net/http"

	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"

	"gopkg.in/macaron.v1"
)

func RequestTracing(handler string) macaron.Handler {
	return func(res http.ResponseWriter, req *http.Request, c *macaron.Context) {
		rw := res.(macaron.ResponseWriter)

		tracer := opentracing.GlobalTracer()
		wireContext, _ := tracer.Extract(opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(req.Header))
		span := tracer.StartSpan(fmt.Sprintf("HTTP %s", handler), ext.RPCServerOption(wireContext))
		defer span.Finish()

		ctx := opentracing.ContextWithSpan(req.Context(), span)
		c.Req.Request = req.WithContext(ctx)

		c.Next()

		status := rw.Status()

		ext.HTTPStatusCode.Set(span, uint16(status))
		ext.HTTPUrl.Set(span, req.RequestURI)
		ext.HTTPMethod.Set(span, req.Method)
		if status >= 400 {
			ext.Error.Set(span, true)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package components

import (
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi/loader"
)

// KubeProxyOptionsBuilder adds options for kube-proxy to the model
type KubeProxyOptionsBuilder struct {
	Context *OptionsContext
}

var _ loader.OptionsBuilder = &KubeProxyOptionsBuilder{}

func (b *KubeProxyOptionsBuilder) BuildOptions(o interface{}) error {
	clusterSpec := o.(*kops.ClusterSpec)
	if clusterSpec.KubeProxy == nil {
		clusterSpec.KubeProxy = &kops.KubeProxyConfig{}
	}

	config := clusterSpec.KubeProxy

	if config.LogLevel == 0 {
		// TODO: No way to set to 0?
		config.LogLevel = 2
	}

	// Any change here should be accompanied by a proportional change in CPU
	// requests of other per-node add-ons (e.g. fluentd).
	if config.CPURequest == "" {
		config.CPURequest = "100m"
	}

	image, err := Image("kube-proxy", clusterSpec, b.Context.AssetBuilder)
	if err != nil {
		return err
	}
	config.Image = image

	// We set the master URL during node configuration, because if we use the internal dns name,
	// then we get a circular dependency:
	// * kube-proxy uses DNS for resolution
	// * dns is set up by dns-controller
	// * dns-controller talks to the API using the kube-proxy configured kubernetes service

	if config.ClusterCIDR == "" {
		// If we're using the AmazonVPC networking, we should omit the ClusterCIDR
		// because pod IPs are real, routable IPs in the VPC, and they are not in a specific
		// CIDR range that allows us to distinguish them from other IPs.  Omitting the ClusterCIDR
		// causes kube-proxy never to SNAT when proxying clusterIPs, which is the behavior
		// we want for pods.
		// If we're not using the AmazonVPC networking, and the KubeControllerMananger has
		// a ClusterCIDR, use that because most networking plug ins draw pod IPs from this range.
		if clusterSpec.Networking.AmazonVPC == nil && clusterSpec.KubeControllerManager != nil {
			config.ClusterCIDR = clusterSpec.KubeControllerManager.ClusterCIDR
		}
	}

	// Set the kube-proxy hostname-override (actually the NodeName), to avoid #2915 et al
	cloudProvider := kops.CloudProviderID(clusterSpec.CloudProvider)
	if cloudProvider == kops.CloudProviderAWS {
		// Use the hostname from the AWS metadata service
		// if hostnameOverride is not set.
		if config.HostnameOverride == "" {
			config.HostnameOverride = "@aws"
		}
	}

	return nil
}
<commit_msg>add kube-proxy hostname override<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package components

import (
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi/loader"
)

// KubeProxyOptionsBuilder adds options for kube-proxy to the model
type KubeProxyOptionsBuilder struct {
	Context *OptionsContext
}

var _ loader.OptionsBuilder = &KubeProxyOptionsBuilder{}

func (b *KubeProxyOptionsBuilder) BuildOptions(o interface{}) error {
	clusterSpec := o.(*kops.ClusterSpec)
	if clusterSpec.KubeProxy == nil {
		clusterSpec.KubeProxy = &kops.KubeProxyConfig{}
	}

	config := clusterSpec.KubeProxy

	if config.LogLevel == 0 {
		// TODO: No way to set to 0?
		config.LogLevel = 2
	}

	// Any change here should be accompanied by a proportional change in CPU
	// requests of other per-node add-ons (e.g. fluentd).
	if config.CPURequest == "" {
		config.CPURequest = "100m"
	}

	image, err := Image("kube-proxy", clusterSpec, b.Context.AssetBuilder)
	if err != nil {
		return err
	}
	config.Image = image

	// We set the master URL during node configuration, because if we use the internal dns name,
	// then we get a circular dependency:
	// * kube-proxy uses DNS for resolution
	// * dns is set up by dns-controller
	// * dns-controller talks to the API using the kube-proxy configured kubernetes service

	if config.ClusterCIDR == "" {
		// If we're using the AmazonVPC networking, we should omit the ClusterCIDR
		// because pod IPs are real, routable IPs in the VPC, and they are not in a specific
		// CIDR range that allows us to distinguish them from other IPs.  Omitting the ClusterCIDR
		// causes kube-proxy never to SNAT when proxying clusterIPs, which is the behavior
		// we want for pods.
		// If we're not using the AmazonVPC networking, and the KubeControllerMananger has
		// a ClusterCIDR, use that because most networking plug ins draw pod IPs from this range.
		if clusterSpec.Networking.AmazonVPC == nil && clusterSpec.KubeControllerManager != nil {
			config.ClusterCIDR = clusterSpec.KubeControllerManager.ClusterCIDR
		}
	}

	// Set the kube-proxy hostname-override (actually the NodeName), to avoid #2915 et al
	cloudProvider := kops.CloudProviderID(clusterSpec.CloudProvider)
	if cloudProvider == kops.CloudProviderAWS {
		// Use the hostname from the AWS metadata service
		// if hostnameOverride is not set.
		if config.HostnameOverride == "" {
			config.HostnameOverride = "@aws"
		}
	}

	if cloudProvider == kops.CloudProviderDO {
		if config.HostnameOverride == "" {
			config.HostnameOverride = "@digitalocean"
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vindexes

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"io/ioutil"

	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/key"
	"vitess.io/vitess/go/vt/log"
)

var (
	_ MultiColumn   = (*RegionVindex)(nil)
)

func init() {
	Register("region_vindex", NewRegionVindex)
}

// RegionMap is used to store mapping of country to region
type RegionMap map[string]uint64

// RegionVindex defines a vindex that uses a lookup table.
// The table is expected to define the id column as unique. It's
// Unique and a Lookup.
type RegionVindex struct {
	name        string
	regionMap   RegionMap
	regionBytes int
}

// NewRegionVindex creates a RegionVindex vindex.
// The supplied map requires all the fields of "region_experimental".
// Additionally, it requires a region_map argument representing the path to a json file
// containing a map of country to region.
func NewRegionVindex(name string, m map[string]string) (Vindex, error) {
	rmPath := m["region_map"]
	rmap := make(map[string]uint64)
	data, err := ioutil.ReadFile(rmPath)
	if err != nil {
		return nil, err
	}
	log.Infof("Loaded Region map from: %s", rmPath)
	err = json.Unmarshal(data, &rmap)
	if err != nil {
		return nil, err
	}

	return &RegionVindex{
		name:      name,
		regionMap: rmap,
	}, nil
}

// String returns the name of the vindex.
func (rv *RegionVindex) String() string {
	return rv.name
}

// Cost returns the cost of this index as 1.
func (rv *RegionVindex) Cost() int {
	return 1
}

// IsUnique returns true since the Vindex is unique.
func (rv *RegionVindex) IsUnique() bool {
	return true;
}

// Map satisfies MultiColumn.
func (rv *RegionVindex) Map(vcursor VCursor, rowsColValues [][]sqltypes.Value) ([]key.Destination, error) {
	destinations := make([]key.Destination, 0, len(rowsColValues))
	for _, row := range rowsColValues {
		if len(row) != 2 {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		// Compute hash.
		hn, err := sqltypes.ToUint64(row[0])
		if err != nil {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		h := vhash(hn)

		log.Infof("Region map: %v", rv.regionMap)
		// Compute region prefix.
		log.Infof("Country: %v", row[1].ToString())
		rn, ok := rv.regionMap[row[1].ToString()]
		log.Infof("Found region %v, true/false: %v", rn, ok)
		if !ok {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		r := make([]byte, 2)
		binary.BigEndian.PutUint16(r, uint16(rn))

		// Concatenate and add to destinations.
		if rv.regionBytes == 1 {
			r = r[1:]
		}
		dest := append(r, h...)
		destinations = append(destinations, key.DestinationKeyspaceID(dest))
	}
	return destinations, nil
}

// Verify satisfies MultiColumn
func (rv *RegionVindex) Verify(vcursor VCursor, rowsColValues [][]sqltypes.Value, ksids [][]byte) ([]bool, error) {
	result := make([]bool, len(rowsColValues))
	destinations, _ := rv.Map(vcursor, rowsColValues)
	for i, dest := range destinations {
		destksid, ok := dest.(key.DestinationKeyspaceID)
		if !ok {
			continue
		}
		result[i] = bytes.Equal([]byte(destksid), ksids[i])
	}
	return result, nil
}

// NeedVCursor staisfies the Vindex interface.
func (rv *RegionVindex) NeedsVCursor() bool {
	return false
}<commit_msg>Updated formatting.<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vindexes

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"io/ioutil"

	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/key"
	"vitess.io/vitess/go/vt/log"
)

var (
	_ MultiColumn = (*RegionVindex)(nil)
)

func init() {
	Register("region_vindex", NewRegionVindex)
}

// RegionMap is used to store mapping of country to region
type RegionMap map[string]uint64

// RegionVindex defines a vindex that uses a lookup table.
// The table is expected to define the id column as unique. It's
// Unique and a Lookup.
type RegionVindex struct {
	name        string
	regionMap   RegionMap
	regionBytes int
}

// NewRegionVindex creates a RegionVindex vindex.
// The supplied map requires all the fields of "region_experimental".
// Additionally, it requires a region_map argument representing the path to a json file
// containing a map of country to region.
func NewRegionVindex(name string, m map[string]string) (Vindex, error) {
	rmPath := m["region_map"]
	rmap := make(map[string]uint64)
	data, err := ioutil.ReadFile(rmPath)
	if err != nil {
		return nil, err
	}
	log.Infof("Loaded Region map from: %s", rmPath)
	err = json.Unmarshal(data, &rmap)
	if err != nil {
		return nil, err
	}

	return &RegionVindex{
		name:      name,
		regionMap: rmap,
	}, nil
}

// String returns the name of the vindex.
func (rv *RegionVindex) String() string {
	return rv.name
}

// Cost returns the cost of this index as 1.
func (rv *RegionVindex) Cost() int {
	return 1
}

// IsUnique returns true since the Vindex is unique.
func (rv *RegionVindex) IsUnique() bool {
	return true
}

// Map satisfies MultiColumn.
func (rv *RegionVindex) Map(vcursor VCursor, rowsColValues [][]sqltypes.Value) ([]key.Destination, error) {
	destinations := make([]key.Destination, 0, len(rowsColValues))
	for _, row := range rowsColValues {
		if len(row) != 2 {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		// Compute hash.
		hn, err := sqltypes.ToUint64(row[0])
		if err != nil {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		h := vhash(hn)

		log.Infof("Region map: %v", rv.regionMap)
		// Compute region prefix.
		log.Infof("Country: %v", row[1].ToString())
		rn, ok := rv.regionMap[row[1].ToString()]
		log.Infof("Found region %v, true/false: %v", rn, ok)
		if !ok {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		r := make([]byte, 2)
		binary.BigEndian.PutUint16(r, uint16(rn))

		// Concatenate and add to destinations.
		if rv.regionBytes == 1 {
			r = r[1:]
		}
		dest := append(r, h...)
		destinations = append(destinations, key.DestinationKeyspaceID(dest))
	}
	return destinations, nil
}

// Verify satisfies MultiColumn
func (rv *RegionVindex) Verify(vcursor VCursor, rowsColValues [][]sqltypes.Value, ksids [][]byte) ([]bool, error) {
	result := make([]bool, len(rowsColValues))
	destinations, _ := rv.Map(vcursor, rowsColValues)
	for i, dest := range destinations {
		destksid, ok := dest.(key.DestinationKeyspaceID)
		if !ok {
			continue
		}
		result[i] = bytes.Equal([]byte(destksid), ksids[i])
	}
	return result, nil
}

// NeedVCursor staisfies the Vindex interface.
func (rv *RegionVindex) NeedsVCursor() bool {
	return false
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ingress

import (
	"context"
	"fmt"

	"knative.dev/net-kourier/pkg/config"

	kubeclient "k8s.io/client-go/kubernetes"

	"knative.dev/networking/pkg/apis/networking/v1alpha1"
	"knative.dev/networking/pkg/client/injection/reconciler/networking/v1alpha1/ingress"
	"knative.dev/networking/pkg/status"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/reconciler"

	"knative.dev/net-kourier/pkg/envoy"
	"knative.dev/net-kourier/pkg/generator"
	"knative.dev/net-kourier/pkg/knative"
)

type Reconciler struct {
	xdsServer         *envoy.XdsServer
	kubeClient        kubeclient.Interface
	caches            *generator.Caches
	statusManager     *status.Prober
	ingressTranslator *generator.IngressTranslator
	extAuthz          bool
}

var _ ingress.Interface = (*Reconciler)(nil)
var _ ingress.ReadOnlyInterface = (*Reconciler)(nil)
var _ ingress.Finalizer = (*Reconciler)(nil)
var _ ingress.ReadOnlyFinalizer = (*Reconciler)(nil)

func (r *Reconciler) ReconcileKind(ctx context.Context, ingress *v1alpha1.Ingress) reconciler.Event {
	before := ingress.DeepCopy()

	r.ObserveKind(ctx, ingress)

	if ready, err := r.statusManager.IsReady(context.TODO(), before); err == nil && ready {
		knative.MarkIngressReady(ingress)
	} else {
		ingress.Status.MarkLoadBalancerNotReady()
		if err != nil {
			return fmt.Errorf("failed to probe Ingress %s/%s: %w", ingress.GetNamespace(), ingress.GetName(), err)
		}
	}

	return nil
}

func (r *Reconciler) ObserveKind(ctx context.Context, ingress *v1alpha1.Ingress) reconciler.Event {
	ingress.SetDefaults(ctx)

	if err := r.updateIngress(ctx, ingress); err == generator.ErrDomainConflict {
		// If we had an error due to a duplicated domain, we must mark the ingress as failed with a
		// custom status. We don't want to return an error in this case as we want to update its status.
		logging.FromContext(ctx).Errorw(err.Error(), ingress.Name, ingress.Namespace)
		ingress.Status.MarkLoadBalancerFailed("DomainConflict", "Ingress rejected as its domain conflicts with another ingress")
	} else if err != nil {
		return fmt.Errorf("failed to update ingress: %w", err)
	}

	return nil
}

func (r *Reconciler) FinalizeKind(ctx context.Context, ing *v1alpha1.Ingress) reconciler.Event {
	return r.ObserveFinalizeKind(ctx, ing)
}
func (r *Reconciler) ObserveFinalizeKind(ctx context.Context, ing *v1alpha1.Ingress) reconciler.Event {
	logger := logging.FromContext(ctx)
	logger.Infof("Deleting Ingress %s namespace: %s", ing.Name, ing.Namespace)
	ingress := r.caches.GetIngress(ing.Name, ing.Namespace)

	// We need to check for ingress not being nil, because we can receive an event from an already
	// removed ingress, like for example, when the endpoints object for that ingress is updated/removed.
	if ingress != nil {
		r.statusManager.CancelIngressProbing(ingress)
	}

	if err := r.caches.DeleteIngressInfo(ctx, ing.Name, ing.Namespace, r.kubeClient); err != nil {
		return err
	}

	return r.updateEnvoyConfig()
}

func (r *Reconciler) updateIngress(ctx context.Context, ingress *v1alpha1.Ingress) error {
	logger := logging.FromContext(ctx)
	logger.Infof("Updating Ingress %s namespace: %s", ingress.Name, ingress.Namespace)

	if err := generator.UpdateInfoForIngress(
		ctx, r.caches, ingress, r.kubeClient, r.ingressTranslator, r.extAuthz,
	); err != nil {
		return err
	}

	if err := r.updateEnvoyConfig(); err != nil {
		return err
	}

	return nil
}

func (r *Reconciler) updateEnvoyConfig() error {
	currentSnapshot, err := r.xdsServer.GetSnapshot(config.EnvoyNodeID)
	if err != nil {
		return err
	}
	newSnapshot, err := r.caches.ToEnvoySnapshot()
	if err != nil {
		return err
	}

	// Let's warm the Clusters first, by sending the previous snapshot with the new cluster list, that includes
	// both new and old clusters.
	currentSnapshot.Clusters = newSnapshot.Clusters

	//Validate that the snapshot is consistent.
	if err := currentSnapshot.Consistent(); err != nil {
		return err
	}

	if err := r.xdsServer.SetSnapshot(&currentSnapshot, nodeID); err != nil {
		return err
	}

	// Now, send the full new snapshot.
	return r.xdsServer.SetSnapshot(&newSnapshot, nodeID)
}
<commit_msg>Check if ingress is ready before probing (#202)<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ingress

import (
	"context"
	"fmt"

	"knative.dev/net-kourier/pkg/config"

	kubeclient "k8s.io/client-go/kubernetes"

	"knative.dev/networking/pkg/apis/networking/v1alpha1"
	"knative.dev/networking/pkg/client/injection/reconciler/networking/v1alpha1/ingress"
	"knative.dev/networking/pkg/status"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/reconciler"

	"knative.dev/net-kourier/pkg/envoy"
	"knative.dev/net-kourier/pkg/generator"
	"knative.dev/net-kourier/pkg/knative"
)

type Reconciler struct {
	xdsServer         *envoy.XdsServer
	kubeClient        kubeclient.Interface
	caches            *generator.Caches
	statusManager     *status.Prober
	ingressTranslator *generator.IngressTranslator
	extAuthz          bool
}

var _ ingress.Interface = (*Reconciler)(nil)
var _ ingress.ReadOnlyInterface = (*Reconciler)(nil)
var _ ingress.Finalizer = (*Reconciler)(nil)
var _ ingress.ReadOnlyFinalizer = (*Reconciler)(nil)

func (r *Reconciler) ReconcileKind(ctx context.Context, ing *v1alpha1.Ingress) reconciler.Event {
	before := ing.DeepCopy()

	r.ObserveKind(ctx, ing)

	if !ing.IsReady() {
		ready, err := r.statusManager.IsReady(context.TODO(), before)
		if err != nil {
			return fmt.Errorf("failed to probe Ingress %s/%s: %w", ing.GetNamespace(), ing.GetName(), err)
		}
		if ready {
			knative.MarkIngressReady(ing)
		} else {
			ing.Status.MarkLoadBalancerNotReady()
		}
	}

	return nil
}

func (r *Reconciler) ObserveKind(ctx context.Context, ingress *v1alpha1.Ingress) reconciler.Event {
	ingress.SetDefaults(ctx)

	if err := r.updateIngress(ctx, ingress); err == generator.ErrDomainConflict {
		// If we had an error due to a duplicated domain, we must mark the ingress as failed with a
		// custom status. We don't want to return an error in this case as we want to update its status.
		logging.FromContext(ctx).Errorw(err.Error(), ingress.Name, ingress.Namespace)
		ingress.Status.MarkLoadBalancerFailed("DomainConflict", "Ingress rejected as its domain conflicts with another ingress")
	} else if err != nil {
		return fmt.Errorf("failed to update ingress: %w", err)
	}

	return nil
}

func (r *Reconciler) FinalizeKind(ctx context.Context, ing *v1alpha1.Ingress) reconciler.Event {
	return r.ObserveFinalizeKind(ctx, ing)
}
func (r *Reconciler) ObserveFinalizeKind(ctx context.Context, ing *v1alpha1.Ingress) reconciler.Event {
	logger := logging.FromContext(ctx)
	logger.Infof("Deleting Ingress %s namespace: %s", ing.Name, ing.Namespace)
	ingress := r.caches.GetIngress(ing.Name, ing.Namespace)

	// We need to check for ingress not being nil, because we can receive an event from an already
	// removed ingress, like for example, when the endpoints object for that ingress is updated/removed.
	if ingress != nil {
		r.statusManager.CancelIngressProbing(ingress)
	}

	if err := r.caches.DeleteIngressInfo(ctx, ing.Name, ing.Namespace, r.kubeClient); err != nil {
		return err
	}

	return r.updateEnvoyConfig()
}

func (r *Reconciler) updateIngress(ctx context.Context, ingress *v1alpha1.Ingress) error {
	logger := logging.FromContext(ctx)
	logger.Infof("Updating Ingress %s namespace: %s", ingress.Name, ingress.Namespace)

	if err := generator.UpdateInfoForIngress(
		ctx, r.caches, ingress, r.kubeClient, r.ingressTranslator, r.extAuthz,
	); err != nil {
		return err
	}

	if err := r.updateEnvoyConfig(); err != nil {
		return err
	}

	return nil
}

func (r *Reconciler) updateEnvoyConfig() error {
	currentSnapshot, err := r.xdsServer.GetSnapshot(config.EnvoyNodeID)
	if err != nil {
		return err
	}
	newSnapshot, err := r.caches.ToEnvoySnapshot()
	if err != nil {
		return err
	}

	// Let's warm the Clusters first, by sending the previous snapshot with the new cluster list, that includes
	// both new and old clusters.
	currentSnapshot.Clusters = newSnapshot.Clusters

	//Validate that the snapshot is consistent.
	if err := currentSnapshot.Consistent(); err != nil {
		return err
	}

	if err := r.xdsServer.SetSnapshot(&currentSnapshot, nodeID); err != nil {
		return err
	}

	// Now, send the full new snapshot.
	return r.xdsServer.SetSnapshot(&newSnapshot, nodeID)
}
<|endoftext|>


<commit_before>package main

import (
	_ "bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"encoding/json"
)

type TemplateModule struct {
	Owner    string
	Group    string
	Mode     string
	Dest     string
	RmtSrc   string
	Override string
}

var result map[string]interface{} = map[string]interface{}{}

func main() {
	// recover code
	// also does the printout of result
	defer func() {
		if r := recover(); r != nil {
			result["status"] = "error"
			result["msg"] = r
		}

		output, err := json.Marshal(result)
		if err != nil {
			panic(err)
		}
		fmt.Print(string(output))
	}()

	templateParams := TemplateModule{}

	// basically unmarshall but can take in a io.Reader
	dec := json.NewDecoder(os.Stdin)
	if err := dec.Decode(&templateParams); err != nil {
		panic(err.Error())
	}

	if templateParams.Dest == "" {
		panic("Required parameter 'dest' not found")
	}

	// Override param is passed in as a string and bools tend to default to false
	override := true
	str := strings.ToLower(templateParams.Override)
	if str == "false" {
		override = false
	} else if str != "" && str != "true" {
		panic("override param must be true or false")
	}

	// Creates all necessary nested directories
	if err := os.MkdirAll(templateParams.Dest, 0755); err != nil {
		panic(fmt.Sprintf("Error creating directories - %s", err.Error()))
	}

	if override {
		// Removes the last file/folder
		if err := os.RemoveAll(templateParams.Dest); err != nil {
			panic(fmt.Sprintf("Error removing endpoint for override - %s", err.Error()))
		}
	} else {
		// extends dest
		templateParams.Dest = filepath.Join(templateParams.Dest, filepath.Base(templateParams.RmtSrc))
	}

	// moves the file/folder to the destination
	if err := os.Rename(templateParams.RmtSrc, templateParams.Dest); err != nil {
		panic(fmt.Sprintf("Error moving file/folder - %s", err.Error()))
	}

	// using chown command os.Chown(...) only takes in ints for now GO 1.5
	var cmd *exec.Cmd
	cmdList := []string{"-R", templateParams.Owner + ":" + templateParams.Group, templateParams.Dest}
	cmd = exec.Command("/bin/chown", cmdList...)

	if output, err := cmd.CombinedOutput(); err != nil {
		panic(fmt.Sprintf("Error chown file/folder - %s", string(output)))
	}

	if templateParams.Mode != "" {
		i, err := strconv.ParseInt(templateParams.Mode, 8, 32)
		if err != nil {
			panic(fmt.Sprintf("Error retrieving mode - %s", err.Error()))
		}
		if i < 0 {
			panic("Error mode must be an unsigned integer")
		}

		if err := os.Chmod(templateParams.Dest, os.FileMode(i)); err != nil {
			panic(fmt.Sprintf("Error chmod file/folder - %s", err.Error()))
		}
	}

	result["status"] = "changed"
	result["msg"] = fmt.Sprintf("State of '%s' changed", templateParams.Dest)
}
<commit_msg>don't mind me just added some imports<commit_after>package main

import (
	_ "bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"encoding/json"
)

type TemplateModule struct {
	Owner    string
	Group    string
	Mode     string
	Dest     string
	RmtSrc   string
	Override string
}

var result map[string]interface{} = map[string]interface{}{}

func main() {
	// recover code
	// also does the printout of result
	defer func() {
		if r := recover(); r != nil {
			result["status"] = "error"
			result["msg"] = r
		}

		output, err := json.Marshal(result)
		if err != nil {
			panic(err)
		}
		fmt.Print(string(output))
	}()

	templateParams := TemplateModule{}

	// basically unmarshall but can take in a io.Reader
	dec := json.NewDecoder(os.Stdin)
	if err := dec.Decode(&templateParams); err != nil {
		panic(err.Error())
	}

	if templateParams.Dest == "" {
		panic("Required parameter 'dest' not found")
	}

	// Override param is passed in as a string and bools tend to default to false
	override := true
	str := strings.ToLower(templateParams.Override)
	if str == "false" {
		override = false
	} else if str != "" && str != "true" {
		panic("override param must be true or false")
	}

	// Creates all necessary nested directories
	if err := os.MkdirAll(templateParams.Dest, 0755); err != nil {
		panic(fmt.Sprintf("Error creating directories - %s", err.Error()))
	}

	if override {
		// Removes the last file/folder
		if err := os.RemoveAll(templateParams.Dest); err != nil {
			panic(fmt.Sprintf("Error removing endpoint for override - %s", err.Error()))
		}
	} else {
		// extends dest
		templateParams.Dest = filepath.Join(templateParams.Dest, filepath.Base(templateParams.RmtSrc))
	}

	// moves the file/folder to the destination
	if err := os.Rename(templateParams.RmtSrc, templateParams.Dest); err != nil {
		panic(fmt.Sprintf("Error moving file/folder - %s", err.Error()))
	}

	// using chown command os.Chown(...) only takes in ints for now GO 1.5
	var cmd *exec.Cmd
	cmdList := []string{"-R", templateParams.Owner + ":" + templateParams.Group, templateParams.Dest}
	cmd = exec.Command("/bin/chown", cmdList...)

	if output, err := cmd.CombinedOutput(); err != nil {
		panic(fmt.Sprintf("Error chown file/folder - %s", string(output)))
	}

	if templateParams.Mode != "" {
		i, err := strconv.ParseInt(templateParams.Mode, 8, 32)
		if err != nil {
			panic(fmt.Sprintf("Error retrieving mode - %s", err.Error()))
		}
		if i < 0 {
			panic("Error mode must be an unsigned integer")
		}

		if err := os.Chmod(templateParams.Dest, os.FileMode(i)); err != nil {
			panic(fmt.Sprintf("Error chmod file/folder - %s", err.Error()))
		}
	}

	result["status"] = "changed"
	result["msg"] = fmt.Sprintf("State of '%s' changed", templateParams.Dest)
}
<|endoftext|>


<commit_before>package gcplogs

import (
	"fmt"
	"sync/atomic"
	"time"

	"github.com/docker/docker/daemon/logger"

	"github.com/Sirupsen/logrus"
	"golang.org/x/net/context"
	"google.golang.org/cloud/compute/metadata"
	"google.golang.org/cloud/logging"
)

const (
	name = "gcplogs"

	projectOptKey = "gcp-project"
	logLabelsKey  = "labels"
	logEnvKey     = "env"
	logCmdKey     = "gcp-log-cmd"
)

var (
	// The number of logs the gcplogs driver has dropped.
	droppedLogs uint64

	onGCE = metadata.OnGCE()

	// instance metadata populated from the metadata server if available
	projectID    string
	zone         string
	instanceName string
	instanceID   string
)

func init() {
	if onGCE {
		// These will fail on instances if the metadata service is
		// down or the client is compiled with an API version that
		// has been removed. Since these are not vital, let's ignore
		// them and make their fields in the dockeLogEntry ,omitempty
		projectID, _ = metadata.ProjectID()
		zone, _ = metadata.Zone()
		instanceName, _ = metadata.InstanceName()
		instanceID, _ = metadata.InstanceID()
	}

	if err := logger.RegisterLogDriver(name, New); err != nil {
		logrus.Fatal(err)
	}

	if err := logger.RegisterLogOptValidator(name, ValidateLogOpts); err != nil {
		logrus.Fatal(err)
	}
}

type gcplogs struct {
	client    *logging.Client
	instance  *instanceInfo
	container *containerInfo
}

type dockerLogEntry struct {
	Instance  *instanceInfo  `json:"instance,omitempty"`
	Container *containerInfo `json:"container,omitempty"`
	Data      string         `json:"data,omitempty"`
}

type instanceInfo struct {
	Zone string `json:"zone,omitempty"`
	Name string `json:"name,omitempty"`
	ID   string `json:"id,omitempty"`
}

type containerInfo struct {
	Name      string            `json:"name,omitempty"`
	ID        string            `json:"id,omitempty"`
	ImageName string            `json:"imageName,omitempty"`
	ImageID   string            `json:"imageId,omitempty"`
	Created   time.Time         `json:"created,omitempty"`
	Command   string            `json:"command,omitempty"`
	Metadata  map[string]string `json:"metadata,omitempty"`
}

// New creates a new logger that logs to Google Cloud Logging using the application
// default credentials.
//
// See https://developers.google.com/identity/protocols/application-default-credentials
func New(ctx logger.Context) (logger.Logger, error) {

	var project string
	if projectID != "" {
		project = projectID
	}
	if projectID, found := ctx.Config[projectOptKey]; found {
		project = projectID
	}
	if project == "" {
		return nil, fmt.Errorf("No project was specified and couldn't read project from the meatadata server. Please specify a project")
	}

	c, err := logging.NewClient(context.Background(), project, "gcplogs-docker-driver")
	if err != nil {
		return nil, err
	}

	if err := c.Ping(); err != nil {
		return nil, fmt.Errorf("unable to connect or authenticate with Google Cloud Logging: %v", err)
	}

	l := &gcplogs{
		client: c,
		container: &containerInfo{
			Name:      ctx.ContainerName,
			ID:        ctx.ContainerID,
			ImageName: ctx.ContainerImageName,
			ImageID:   ctx.ContainerImageID,
			Created:   ctx.ContainerCreated,
			Metadata:  ctx.ExtraAttributes(nil),
		},
	}

	if ctx.Config[logCmdKey] == "true" {
		l.container.Command = ctx.Command()
	}

	if onGCE {
		l.instance = &instanceInfo{
			Zone: zone,
			Name: instanceName,
			ID:   instanceID,
		}
	}

	// The logger "overflows" at a rate of 10,000 logs per second and this
	// overflow func is called. We want to surface the error to the user
	// without overly spamming /var/log/docker.log so we log the first time
	// we overflow and every 1000th time after.
	c.Overflow = func(_ *logging.Client, _ logging.Entry) error {
		if i := atomic.AddUint64(&droppedLogs, 1); i%1000 == 1 {
			logrus.Errorf("gcplogs driver has dropped %v logs", i)
		}
		return nil
	}

	return l, nil
}

// ValidateLogOpts validates the opts passed to the gcplogs driver. Currently, the gcplogs
// driver doesn't take any arguments.
func ValidateLogOpts(cfg map[string]string) error {
	for k := range cfg {
		switch k {
		case projectOptKey, logLabelsKey, logEnvKey, logCmdKey:
		default:
			return fmt.Errorf("%q is not a valid option for the gcplogs driver", k)
		}
	}
	return nil
}

func (l *gcplogs) Log(m *logger.Message) error {
	return l.client.Log(logging.Entry{
		Time: m.Timestamp,
		Payload: &dockerLogEntry{
			Instance:  l.instance,
			Container: l.container,
			Data:      string(m.Line),
		},
	})
}

func (l *gcplogs) Close() error {
	return l.client.Flush()
}

func (l *gcplogs) Name() string {
	return name
}
<commit_msg>Do not call out to Google on init<commit_after>package gcplogs

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/docker/docker/daemon/logger"

	"github.com/Sirupsen/logrus"
	"golang.org/x/net/context"
	"google.golang.org/cloud/compute/metadata"
	"google.golang.org/cloud/logging"
)

const (
	name = "gcplogs"

	projectOptKey = "gcp-project"
	logLabelsKey  = "labels"
	logEnvKey     = "env"
	logCmdKey     = "gcp-log-cmd"
)

var (
	// The number of logs the gcplogs driver has dropped.
	droppedLogs uint64

	onGCE bool

	// instance metadata populated from the metadata server if available
	projectID    string
	zone         string
	instanceName string
	instanceID   string
)

func init() {

	if err := logger.RegisterLogDriver(name, New); err != nil {
		logrus.Fatal(err)
	}

	if err := logger.RegisterLogOptValidator(name, ValidateLogOpts); err != nil {
		logrus.Fatal(err)
	}
}

type gcplogs struct {
	client    *logging.Client
	instance  *instanceInfo
	container *containerInfo
}

type dockerLogEntry struct {
	Instance  *instanceInfo  `json:"instance,omitempty"`
	Container *containerInfo `json:"container,omitempty"`
	Data      string         `json:"data,omitempty"`
}

type instanceInfo struct {
	Zone string `json:"zone,omitempty"`
	Name string `json:"name,omitempty"`
	ID   string `json:"id,omitempty"`
}

type containerInfo struct {
	Name      string            `json:"name,omitempty"`
	ID        string            `json:"id,omitempty"`
	ImageName string            `json:"imageName,omitempty"`
	ImageID   string            `json:"imageId,omitempty"`
	Created   time.Time         `json:"created,omitempty"`
	Command   string            `json:"command,omitempty"`
	Metadata  map[string]string `json:"metadata,omitempty"`
}

var initGCPOnce sync.Once

func initGCP() {
	initGCPOnce.Do(func() {
		onGCE = metadata.OnGCE()
		if onGCE {
			// These will fail on instances if the metadata service is
			// down or the client is compiled with an API version that
			// has been removed. Since these are not vital, let's ignore
			// them and make their fields in the dockeLogEntry ,omitempty
			projectID, _ = metadata.ProjectID()
			zone, _ = metadata.Zone()
			instanceName, _ = metadata.InstanceName()
			instanceID, _ = metadata.InstanceID()
		}
	})
}

// New creates a new logger that logs to Google Cloud Logging using the application
// default credentials.
//
// See https://developers.google.com/identity/protocols/application-default-credentials
func New(ctx logger.Context) (logger.Logger, error) {
	initGCP()

	var project string
	if projectID != "" {
		project = projectID
	}
	if projectID, found := ctx.Config[projectOptKey]; found {
		project = projectID
	}
	if project == "" {
		return nil, fmt.Errorf("No project was specified and couldn't read project from the meatadata server. Please specify a project")
	}

	c, err := logging.NewClient(context.Background(), project, "gcplogs-docker-driver")
	if err != nil {
		return nil, err
	}

	if err := c.Ping(); err != nil {
		return nil, fmt.Errorf("unable to connect or authenticate with Google Cloud Logging: %v", err)
	}

	l := &gcplogs{
		client: c,
		container: &containerInfo{
			Name:      ctx.ContainerName,
			ID:        ctx.ContainerID,
			ImageName: ctx.ContainerImageName,
			ImageID:   ctx.ContainerImageID,
			Created:   ctx.ContainerCreated,
			Metadata:  ctx.ExtraAttributes(nil),
		},
	}

	if ctx.Config[logCmdKey] == "true" {
		l.container.Command = ctx.Command()
	}

	if onGCE {
		l.instance = &instanceInfo{
			Zone: zone,
			Name: instanceName,
			ID:   instanceID,
		}
	}

	// The logger "overflows" at a rate of 10,000 logs per second and this
	// overflow func is called. We want to surface the error to the user
	// without overly spamming /var/log/docker.log so we log the first time
	// we overflow and every 1000th time after.
	c.Overflow = func(_ *logging.Client, _ logging.Entry) error {
		if i := atomic.AddUint64(&droppedLogs, 1); i%1000 == 1 {
			logrus.Errorf("gcplogs driver has dropped %v logs", i)
		}
		return nil
	}

	return l, nil
}

// ValidateLogOpts validates the opts passed to the gcplogs driver. Currently, the gcplogs
// driver doesn't take any arguments.
func ValidateLogOpts(cfg map[string]string) error {
	for k := range cfg {
		switch k {
		case projectOptKey, logLabelsKey, logEnvKey, logCmdKey:
		default:
			return fmt.Errorf("%q is not a valid option for the gcplogs driver", k)
		}
	}
	return nil
}

func (l *gcplogs) Log(m *logger.Message) error {
	return l.client.Log(logging.Entry{
		Time: m.Timestamp,
		Payload: &dockerLogEntry{
			Instance:  l.instance,
			Container: l.container,
			Data:      string(m.Line),
		},
	})
}

func (l *gcplogs) Close() error {
	return l.client.Flush()
}

func (l *gcplogs) Name() string {
	return name
}
<|endoftext|>


<commit_before>package redblackbst

import (
	"bytes"
	"fmt"
	"reflect"
	"testing"
)

var tests = []struct {
	want graph
	prog Program
}{
	{
		prog: Program{
			Name: "simple put",
			Ops: []Op{
				{"Put", "b"},
				{"Put", "a"},
			},
		},
		want: makeGraph([]edge{
			{from: "b",
				to:    "a",
				dir:   "left",
				color: "red"},
		}),
	},
}

type Op struct {
	Op  string
	Key string
}

type Program struct {
	Name string
	Ops  []Op
}

func (p *Program) Run(tree *RedBlack) {
	for step, op := range p.Ops {
		switch op.Op {
		case "Put":
			tree.Put(K(op.Key), struct{}{})
		case "Delete":
			tree.Delete(K(op.Key))
		case "Print":
			printTreeStats(tree, fmt.Sprintf("%s-%d", p.Name, step))
		}
	}
}

func TestScenarios(t *testing.T) {

	for _, tt := range tests {
		tree := New()
		tt.prog.Run(tree)

		want := tt.want
		got := toGraph(tree)

		if !reflect.DeepEqual(want.nodes, got.nodes) {
			openDot(want.toDot())
			openDot(got.toDot())
			t.Logf("want=%#v", want.nodes)
			t.Logf("got =%#v", got.nodes)
			t.Fatalf("%q: failed,", tt.prog.Name)
		}
	}
}

type nd struct {
	name  string
	edges []edge
}

type edge struct {
	from  string
	to    string
	dir   string
	color string
}

type graph struct {
	name  string
	nodes map[string]nd
}

func toGraph(tree *RedBlack) graph {
	g := graph{
		name:  "got",
		nodes: make(map[string]nd),
	}

	lo, _, _ := tree.Min()
	hi, _, _ := tree.Max()
	nodes(tree.root, func(n *node) bool {

		h := nd{
			name: string(n.key.(K)),
		}

		if n.left != nil {
			var leftColor string
			if isRed(n.left) {
				leftColor = "red"
			} else {
				leftColor = "black"
			}

			h.edges = append(h.edges, edge{
				from:  h.name,
				to:    string(n.left.key.(K)),
				color: leftColor,
				dir:   "left",
			})
		}
		if n.right != nil {
			var rightColor string
			if isRed(n.right) {
				rightColor = "red"
			} else {
				rightColor = "black"
			}
			h.edges = append(h.edges, edge{
				from:  h.name,
				to:    string(n.right.key.(K)),
				color: rightColor,
				dir:   "right",
			})
		}

		g.nodes[h.name] = h
		return true
	}, lo, hi)

	return g
}

func makeGraph(edges []edge) graph {
	g := graph{
		name:  "want",
		nodes: make(map[string]nd),
	}
	for _, e := range edges {
		n, ok := g.nodes[e.from]
		if !ok {
			n = nd{
				name:  e.from,
				edges: []edge{e},
			}
		} else {
			n.edges = append(n.edges, e)
		}
		g.nodes[e.from] = n
		n, ok = g.nodes[e.to]
		if !ok {
			n = nd{
				name: e.to,
			}
			g.nodes[e.to] = n
		}

	}
	return g
}

func (g graph) toDot() *bytes.Buffer {
	w := bytes.NewBuffer(nil)

	fmt.Fprintf(w, "digraph %q {", g.name)

	for _, node := range g.nodes {
		fmt.Fprintf(w, "\t%q [shape = circle];\n", node.name)
	}

	for _, node := range g.nodes {
		for _, edge := range node.edges {
			fmt.Fprintf(w, "\t%q -> %q [label=%q, color=%s];\n", node.name, edge.to, edge.dir, edge.color)
		}
	}

	fmt.Fprintf(w, "}\n")

	return w
}
<commit_msg>add some tests for insertion rotation<commit_after>package redblackbst

import (
	"bytes"
	"fmt"
	"reflect"
	"testing"
)

var tests = []struct {
	want graph
	prog P
}{
	{
		prog: P{Name: "simple put", Ops: []Op{
			{"Put", "b"},
			{"Put", "a"},
		}},
		want: makeGraph([]edge{
			{from: "b",
				to:    "a",
				dir:   "left",
				color: "red"},
		}),
	},
	{
		prog: P{Name: "simple put 2", Ops: []Op{
			{"Put", "a"},
			{"Put", "b"},
		}},
		want: makeGraph([]edge{
			{from: "b",
				to:    "a",
				dir:   "left",
				color: "red"},
		}),
	},
	{
		prog: P{Name: "two blacks", Ops: []Op{
			{"Put", "a"},
			{"Put", "e"},
			{"Put", "s"},
		}},
		want: makeGraph([]edge{
			{from: "e",
				to:    "a",
				dir:   "left",
				color: "black"},
			{from: "e",
				to:    "s",
				dir:   "right",
				color: "black"},
		}),
	},

	{
		prog: P{Name: "two blacks, one red", Ops: []Op{
			{"Put", "a"},
			{"Put", "e"},
			{"Put", "s"},
			{"Put", "r"},
		}},
		want: makeGraph([]edge{
			{from: "e",
				to:    "a",
				dir:   "left",
				color: "black"},
			{from: "e",
				to:    "s",
				dir:   "right",
				color: "black"},
			{from: "s",
				to:    "r",
				dir:   "left",
				color: "red"},
		}),
	},

	{
		prog: P{Name: "two blacks, two red", Ops: []Op{
			{"Put", "a"},
			{"Put", "e"},
			{"Put", "s"},
			{"Put", "r"},
			{"Put", "c"},
		}},
		want: makeGraph([]edge{
			{from: "e",
				to:    "c",
				dir:   "left",
				color: "black"},
			{from: "e",
				to:    "s",
				dir:   "right",
				color: "black"},
			{from: "s",
				to:    "r",
				dir:   "left",
				color: "red"},
			{from: "c",
				to:    "a",
				dir:   "left",
				color: "red"},
		}),
	},
}

type Op struct {
	Op  string
	Key string
}

type P struct {
	Name string
	Ops  []Op
}

func (p *P) Run(tree *RedBlack) {
	for step, op := range p.Ops {
		switch op.Op {
		case "Put":
			tree.Put(K(op.Key), struct{}{})
		case "Delete":
			tree.Delete(K(op.Key))
		case "Print":
			printTreeStats(tree, fmt.Sprintf("%s-%d", p.Name, step))
		}
	}
}

func TestScenarios(t *testing.T) {

	for _, tt := range tests {
		tree := New()
		tt.prog.Run(tree)

		want := tt.want
		got := toGraph(tree)

		if !reflect.DeepEqual(want.nodes, got.nodes) {
			openDot(want.toDot())
			openDot(got.toDot())
			t.Logf("want=%#v", want.nodes)
			t.Logf("got =%#v", got.nodes)
			t.Fatalf("%q: failed,", tt.prog.Name)
		}
	}
}

type nd struct {
	name  string
	edges []edge
}

type edge struct {
	from  string
	to    string
	dir   string
	color string
}

type graph struct {
	name  string
	nodes map[string]nd
}

func toGraph(tree *RedBlack) graph {
	g := graph{
		name:  "got",
		nodes: make(map[string]nd),
	}

	lo, _, _ := tree.Min()
	hi, _, _ := tree.Max()
	nodes(tree.root, func(n *node) bool {

		h := nd{
			name: string(n.key.(K)),
		}

		if n.left != nil {
			var leftColor string
			if isRed(n.left) {
				leftColor = "red"
			} else {
				leftColor = "black"
			}

			h.edges = append(h.edges, edge{
				from:  h.name,
				to:    string(n.left.key.(K)),
				color: leftColor,
				dir:   "left",
			})
		}
		if n.right != nil {
			var rightColor string
			if isRed(n.right) {
				rightColor = "red"
			} else {
				rightColor = "black"
			}
			h.edges = append(h.edges, edge{
				from:  h.name,
				to:    string(n.right.key.(K)),
				color: rightColor,
				dir:   "right",
			})
		}

		g.nodes[h.name] = h
		return true
	}, lo, hi)

	return g
}

func makeGraph(edges []edge) graph {
	g := graph{
		name:  "want",
		nodes: make(map[string]nd),
	}
	for _, e := range edges {
		n, ok := g.nodes[e.from]
		if !ok {
			n = nd{
				name:  e.from,
				edges: []edge{e},
			}
		} else {
			n.edges = append(n.edges, e)
		}
		g.nodes[e.from] = n
		n, ok = g.nodes[e.to]
		if !ok {
			n = nd{
				name: e.to,
			}
			g.nodes[e.to] = n
		}

	}
	return g
}

func (g graph) toDot() *bytes.Buffer {
	w := bytes.NewBuffer(nil)

	fmt.Fprintf(w, "digraph %q {", g.name)

	for _, node := range g.nodes {
		fmt.Fprintf(w, "\t%q [shape = circle];\n", node.name)
	}

	for _, node := range g.nodes {
		for _, edge := range node.edges {
			fmt.Fprintf(w, "\t%q -> %q [label=%q, color=%s];\n", node.name, edge.to, edge.dir, edge.color)
		}
	}

	fmt.Fprintf(w, "}\n")

	return w
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
	"fmt"
	"time"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	clientset "k8s.io/client-go/kubernetes"
	corelisters "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
	schedulerapi "k8s.io/kubernetes/plugin/pkg/scheduler/api"
	"k8s.io/kubernetes/plugin/pkg/scheduler/core"
	"k8s.io/kubernetes/plugin/pkg/scheduler/metrics"
	"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"
	"k8s.io/kubernetes/plugin/pkg/scheduler/util"

	"github.com/golang/glog"
)

// Binder knows how to write a binding.
type Binder interface {
	Bind(binding *v1.Binding) error
}

// PodConditionUpdater updates the condition of a pod based on the passed
// PodCondition
type PodConditionUpdater interface {
	Update(pod *v1.Pod, podCondition *v1.PodCondition) error
}

// Scheduler watches for new unscheduled pods. It attempts to find
// nodes that they fit on and writes bindings back to the api server.
type Scheduler struct {
	config *Config
}

// StopEverything closes the scheduler config's StopEverything channel, to shut
// down the Scheduler.
func (sched *Scheduler) StopEverything() {
	close(sched.config.StopEverything)
}

// Configurator defines I/O, caching, and other functionality needed to
// construct a new scheduler. An implementation of this can be seen in
// factory.go.
type Configurator interface {
	GetPriorityFunctionConfigs(priorityKeys sets.String) ([]algorithm.PriorityConfig, error)
	GetPriorityMetadataProducer() (algorithm.MetadataProducer, error)
	GetPredicateMetadataProducer() (algorithm.MetadataProducer, error)
	GetPredicates(predicateKeys sets.String) (map[string]algorithm.FitPredicate, error)
	GetHardPodAffinitySymmetricWeight() int
	GetSchedulerName() string
	MakeDefaultErrorFunc(backoff *util.PodBackoff, podQueue *cache.FIFO) func(pod *v1.Pod, err error)

	// Probably doesn't need to be public.  But exposed for now in case.
	ResponsibleForPod(pod *v1.Pod) bool

	// Needs to be exposed for things like integration tests where we want to make fake nodes.
	GetNodeLister() corelisters.NodeLister
	GetClient() clientset.Interface
	GetScheduledPodLister() corelisters.PodLister

	Create() (*Config, error)
	CreateFromProvider(providerName string) (*Config, error)
	CreateFromConfig(policy schedulerapi.Policy) (*Config, error)
	CreateFromKeys(predicateKeys, priorityKeys sets.String, extenders []algorithm.SchedulerExtender) (*Config, error)
}

// Config is an implementation of the Scheduler's configured input data.
// TODO over time we should make this struct a hidden implementation detail of the scheduler.
type Config struct {
	// It is expected that changes made via SchedulerCache will be observed
	// by NodeLister and Algorithm.
	SchedulerCache schedulercache.Cache
	// Ecache is used for optimistically invalid affected cache items after
	// successfully binding a pod
	Ecache     *core.EquivalenceCache
	NodeLister algorithm.NodeLister
	Algorithm  algorithm.ScheduleAlgorithm
	Binder     Binder
	// PodConditionUpdater is used only in case of scheduling errors. If we succeed
	// with scheduling, PodScheduled condition will be updated in apiserver in /bind
	// handler so that binding and setting PodCondition it is atomic.
	PodConditionUpdater PodConditionUpdater

	// NextPod should be a function that blocks until the next pod
	// is available. We don't use a channel for this, because scheduling
	// a pod may take some amount of time and we don't want pods to get
	// stale while they sit in a channel.
	NextPod func() *v1.Pod

	// WaitForCacheSync waits for scheduler cache to populate.
	// It returns true if it was successful, false if the controller should shutdown.
	WaitForCacheSync func() bool

	// Error is called if there is an error. It is passed the pod in
	// question, and the error
	Error func(*v1.Pod, error)

	// Recorder is the EventRecorder to use
	Recorder record.EventRecorder

	// Close this to shut down the scheduler.
	StopEverything chan struct{}
}

// NewFromConfigurator returns a new scheduler that is created entirely by the Configurator.  Assumes Create() is implemented.
// Supports intermediate Config mutation for now if you provide modifier functions which will run after Config is created.
func NewFromConfigurator(c Configurator, modifiers ...func(c *Config)) (*Scheduler, error) {
	cfg, err := c.Create()
	if err != nil {
		return nil, err
	}
	// Mutate it if any functions were provided, changes might be required for certain types of tests (i.e. change the recorder).
	for _, modifier := range modifiers {
		modifier(cfg)
	}
	// From this point on the config is immutable to the outside.
	s := &Scheduler{
		config: cfg,
	}
	metrics.Register()
	return s, nil
}

// Run begins watching and scheduling. It waits for cache to be synced, then starts a goroutine and returns immediately.
func (sched *Scheduler) Run() {
	if !sched.config.WaitForCacheSync() {
		return
	}

	go wait.Until(sched.scheduleOne, 0, sched.config.StopEverything)
}

// Config return scheduler's config pointer. It is exposed for testing purposes.
func (sched *Scheduler) Config() *Config {
	return sched.config
}

// schedule implements the scheduling algorithm and returns the suggested host.
func (sched *Scheduler) schedule(pod *v1.Pod) (string, error) {
	host, err := sched.config.Algorithm.Schedule(pod, sched.config.NodeLister)
	if err != nil {
		glog.V(1).Infof("Failed to schedule pod: %v/%v", pod.Namespace, pod.Name)
		copied, cerr := api.Scheme.Copy(pod)
		if cerr != nil {
			runtime.HandleError(err)
			return "", err
		}
		pod = copied.(*v1.Pod)
		sched.config.Error(pod, err)
		sched.config.Recorder.Eventf(pod, v1.EventTypeWarning, "FailedScheduling", "%v", err)
		sched.config.PodConditionUpdater.Update(pod, &v1.PodCondition{
			Type:    v1.PodScheduled,
			Status:  v1.ConditionFalse,
			Reason:  v1.PodReasonUnschedulable,
			Message: err.Error(),
		})
		return "", err
	}
	return host, err
}

// assume signals to the cache that a pod is already in the cache, so that binding can be asnychronous.
// assume modifies `assumed`.
func (sched *Scheduler) assume(assumed *v1.Pod, host string) error {
	// Optimistically assume that the binding will succeed and send it to apiserver
	// in the background.
	// If the binding fails, scheduler will release resources allocated to assumed pod
	// immediately.
	assumed.Spec.NodeName = host
	if err := sched.config.SchedulerCache.AssumePod(assumed); err != nil {
		glog.Errorf("scheduler cache AssumePod failed: %v", err)
		// TODO: This means that a given pod is already in cache (which means it
		// is either assumed or already added). This is most probably result of a
		// BUG in retrying logic. As a temporary workaround (which doesn't fully
		// fix the problem, but should reduce its impact), we simply return here,
		// as binding doesn't make sense anyway.
		// This should be fixed properly though.
		return err
	}

	// Optimistically assume that the binding will succeed, so we need to invalidate affected
	// predicates in equivalence cache.
	// If the binding fails, these invalidated item will not break anything.
	if sched.config.Ecache != nil {
		sched.config.Ecache.InvalidateCachedPredicateItemForPodAdd(assumed, host)
	}
	return nil
}

// bind binds a pod to a given node defined in a binding object.  We expect this to run asynchronously, so we
// handle binding metrics internally.
func (sched *Scheduler) bind(assumed *v1.Pod, b *v1.Binding) error {
	bindingStart := time.Now()
	// If binding succeeded then PodScheduled condition will be updated in apiserver so that
	// it's atomic with setting host.
	err := sched.config.Binder.Bind(b)
	if err != nil {
		glog.V(1).Infof("Failed to bind pod: %v/%v", assumed.Namespace, assumed.Name)
		if err := sched.config.SchedulerCache.ForgetPod(assumed); err != nil {
			return fmt.Errorf("scheduler cache ForgetPod failed: %v", err)
		}
		sched.config.Error(assumed, err)
		sched.config.Recorder.Eventf(assumed, v1.EventTypeWarning, "FailedScheduling", "Binding rejected: %v", err)
		sched.config.PodConditionUpdater.Update(assumed, &v1.PodCondition{
			Type:   v1.PodScheduled,
			Status: v1.ConditionFalse,
			Reason: "BindingRejected",
		})
		return err
	}

	if err := sched.config.SchedulerCache.FinishBinding(assumed); err != nil {
		return fmt.Errorf("scheduler cache FinishBinding failed: %v", err)
	}

	metrics.BindingLatency.Observe(metrics.SinceInMicroseconds(bindingStart))
	sched.config.Recorder.Eventf(assumed, v1.EventTypeNormal, "Scheduled", "Successfully assigned %v to %v", assumed.Name, b.Target.Name)
	return nil
}

// scheduleOne does the entire scheduling workflow for a single pod.  It is serialized on the scheduling algorithm's host fitting.
func (sched *Scheduler) scheduleOne() {
	pod := sched.config.NextPod()
	if pod.DeletionTimestamp != nil {
		sched.config.Recorder.Eventf(pod, v1.EventTypeWarning, "FailedScheduling", "skip schedule deleting pod: %v/%v", pod.Namespace, pod.Name)
		glog.V(3).Infof("Skip schedule deleting pod: %v/%v", pod.Namespace, pod.Name)
		return
	}

	glog.V(3).Infof("Attempting to schedule pod: %v/%v", pod.Namespace, pod.Name)

	// Synchronously attempt to find a fit for the pod.
	start := time.Now()
	suggestedHost, err := sched.schedule(pod)
	metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start))
	if err != nil {
		return
	}

	// Tell the cache to assume that a pod now is running on a given node, even though it hasn't been bound yet.
	// This allows us to keep scheduling without waiting on binding to occur.
	assumedPod := *pod
	// assume modifies `assumedPod` by setting NodeName=suggestedHost
	err = sched.assume(&assumedPod, suggestedHost)
	if err != nil {
		return
	}

	// bind the pod to its host asynchronously (we can do this b/c of the assumption step above).
	go func() {
		err := sched.bind(&assumedPod, &v1.Binding{
			ObjectMeta: metav1.ObjectMeta{Namespace: assumedPod.Namespace, Name: assumedPod.Name, UID: assumedPod.UID},
			Target: v1.ObjectReference{
				Kind: "Node",
				Name: suggestedHost,
			},
		})
		metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start))
		if err != nil {
			glog.Errorf("Internal error binding pod: (%v)", err)
		}
	}()
}
<commit_msg>Handle errors more consistently in scheduler<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
	"time"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	clientset "k8s.io/client-go/kubernetes"
	corelisters "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
	schedulerapi "k8s.io/kubernetes/plugin/pkg/scheduler/api"
	"k8s.io/kubernetes/plugin/pkg/scheduler/core"
	"k8s.io/kubernetes/plugin/pkg/scheduler/metrics"
	"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"
	"k8s.io/kubernetes/plugin/pkg/scheduler/util"

	"github.com/golang/glog"
)

// Binder knows how to write a binding.
type Binder interface {
	Bind(binding *v1.Binding) error
}

// PodConditionUpdater updates the condition of a pod based on the passed
// PodCondition
type PodConditionUpdater interface {
	Update(pod *v1.Pod, podCondition *v1.PodCondition) error
}

// Scheduler watches for new unscheduled pods. It attempts to find
// nodes that they fit on and writes bindings back to the api server.
type Scheduler struct {
	config *Config
}

// StopEverything closes the scheduler config's StopEverything channel, to shut
// down the Scheduler.
func (sched *Scheduler) StopEverything() {
	close(sched.config.StopEverything)
}

// Configurator defines I/O, caching, and other functionality needed to
// construct a new scheduler. An implementation of this can be seen in
// factory.go.
type Configurator interface {
	GetPriorityFunctionConfigs(priorityKeys sets.String) ([]algorithm.PriorityConfig, error)
	GetPriorityMetadataProducer() (algorithm.MetadataProducer, error)
	GetPredicateMetadataProducer() (algorithm.MetadataProducer, error)
	GetPredicates(predicateKeys sets.String) (map[string]algorithm.FitPredicate, error)
	GetHardPodAffinitySymmetricWeight() int
	GetSchedulerName() string
	MakeDefaultErrorFunc(backoff *util.PodBackoff, podQueue *cache.FIFO) func(pod *v1.Pod, err error)

	// Probably doesn't need to be public.  But exposed for now in case.
	ResponsibleForPod(pod *v1.Pod) bool

	// Needs to be exposed for things like integration tests where we want to make fake nodes.
	GetNodeLister() corelisters.NodeLister
	GetClient() clientset.Interface
	GetScheduledPodLister() corelisters.PodLister

	Create() (*Config, error)
	CreateFromProvider(providerName string) (*Config, error)
	CreateFromConfig(policy schedulerapi.Policy) (*Config, error)
	CreateFromKeys(predicateKeys, priorityKeys sets.String, extenders []algorithm.SchedulerExtender) (*Config, error)
}

// Config is an implementation of the Scheduler's configured input data.
// TODO over time we should make this struct a hidden implementation detail of the scheduler.
type Config struct {
	// It is expected that changes made via SchedulerCache will be observed
	// by NodeLister and Algorithm.
	SchedulerCache schedulercache.Cache
	// Ecache is used for optimistically invalid affected cache items after
	// successfully binding a pod
	Ecache     *core.EquivalenceCache
	NodeLister algorithm.NodeLister
	Algorithm  algorithm.ScheduleAlgorithm
	Binder     Binder
	// PodConditionUpdater is used only in case of scheduling errors. If we succeed
	// with scheduling, PodScheduled condition will be updated in apiserver in /bind
	// handler so that binding and setting PodCondition it is atomic.
	PodConditionUpdater PodConditionUpdater

	// NextPod should be a function that blocks until the next pod
	// is available. We don't use a channel for this, because scheduling
	// a pod may take some amount of time and we don't want pods to get
	// stale while they sit in a channel.
	NextPod func() *v1.Pod

	// WaitForCacheSync waits for scheduler cache to populate.
	// It returns true if it was successful, false if the controller should shutdown.
	WaitForCacheSync func() bool

	// Error is called if there is an error. It is passed the pod in
	// question, and the error
	Error func(*v1.Pod, error)

	// Recorder is the EventRecorder to use
	Recorder record.EventRecorder

	// Close this to shut down the scheduler.
	StopEverything chan struct{}
}

// NewFromConfigurator returns a new scheduler that is created entirely by the Configurator.  Assumes Create() is implemented.
// Supports intermediate Config mutation for now if you provide modifier functions which will run after Config is created.
func NewFromConfigurator(c Configurator, modifiers ...func(c *Config)) (*Scheduler, error) {
	cfg, err := c.Create()
	if err != nil {
		return nil, err
	}
	// Mutate it if any functions were provided, changes might be required for certain types of tests (i.e. change the recorder).
	for _, modifier := range modifiers {
		modifier(cfg)
	}
	// From this point on the config is immutable to the outside.
	s := &Scheduler{
		config: cfg,
	}
	metrics.Register()
	return s, nil
}

// Run begins watching and scheduling. It waits for cache to be synced, then starts a goroutine and returns immediately.
func (sched *Scheduler) Run() {
	if !sched.config.WaitForCacheSync() {
		return
	}

	go wait.Until(sched.scheduleOne, 0, sched.config.StopEverything)
}

// Config return scheduler's config pointer. It is exposed for testing purposes.
func (sched *Scheduler) Config() *Config {
	return sched.config
}

// schedule implements the scheduling algorithm and returns the suggested host.
func (sched *Scheduler) schedule(pod *v1.Pod) (string, error) {
	host, err := sched.config.Algorithm.Schedule(pod, sched.config.NodeLister)
	if err != nil {
		glog.V(1).Infof("Failed to schedule pod: %v/%v", pod.Namespace, pod.Name)
		copied, cerr := api.Scheme.Copy(pod)
		if cerr != nil {
			runtime.HandleError(err)
			return "", err
		}
		pod = copied.(*v1.Pod)
		sched.config.Error(pod, err)
		sched.config.Recorder.Eventf(pod, v1.EventTypeWarning, "FailedScheduling", "%v", err)
		sched.config.PodConditionUpdater.Update(pod, &v1.PodCondition{
			Type:    v1.PodScheduled,
			Status:  v1.ConditionFalse,
			Reason:  v1.PodReasonUnschedulable,
			Message: err.Error(),
		})
		return "", err
	}
	return host, err
}

// assume signals to the cache that a pod is already in the cache, so that binding can be asnychronous.
// assume modifies `assumed`.
func (sched *Scheduler) assume(assumed *v1.Pod, host string) error {
	// Optimistically assume that the binding will succeed and send it to apiserver
	// in the background.
	// If the binding fails, scheduler will release resources allocated to assumed pod
	// immediately.
	assumed.Spec.NodeName = host
	if err := sched.config.SchedulerCache.AssumePod(assumed); err != nil {
		glog.Errorf("scheduler cache AssumePod failed: %v", err)

		// This is most probably result of a BUG in retrying logic.
		// We report an error here so that pod scheduling can be retried.
		// This relies on the fact that Error will check if the pod has been bound
		// to a node and if so will not add it back to the unscheduled pods queue
		// (otherwise this would cause an infinite loop).
		sched.config.Error(assumed, err)
		sched.config.Recorder.Eventf(assumed, v1.EventTypeWarning, "FailedScheduling", "AssumePod failed: %v", err)
		sched.config.PodConditionUpdater.Update(assumed, &v1.PodCondition{
			Type:    v1.PodScheduled,
			Status:  v1.ConditionFalse,
			Reason:  "SchedulerError",
			Message: err.Error(),
		})
		return err
	}

	// Optimistically assume that the binding will succeed, so we need to invalidate affected
	// predicates in equivalence cache.
	// If the binding fails, these invalidated item will not break anything.
	if sched.config.Ecache != nil {
		sched.config.Ecache.InvalidateCachedPredicateItemForPodAdd(assumed, host)
	}
	return nil
}

// bind binds a pod to a given node defined in a binding object.  We expect this to run asynchronously, so we
// handle binding metrics internally.
func (sched *Scheduler) bind(assumed *v1.Pod, b *v1.Binding) error {
	bindingStart := time.Now()
	// If binding succeeded then PodScheduled condition will be updated in apiserver so that
	// it's atomic with setting host.
	err := sched.config.Binder.Bind(b)
	if err := sched.config.SchedulerCache.FinishBinding(assumed); err != nil {
		glog.Errorf("scheduler cache FinishBinding failed: %v", err)
	}
	if err != nil {
		glog.V(1).Infof("Failed to bind pod: %v/%v", assumed.Namespace, assumed.Name)
		if err := sched.config.SchedulerCache.ForgetPod(assumed); err != nil {
			glog.Errorf("scheduler cache ForgetPod failed: %v", err)
		}
		sched.config.Error(assumed, err)
		sched.config.Recorder.Eventf(assumed, v1.EventTypeWarning, "FailedScheduling", "Binding rejected: %v", err)
		sched.config.PodConditionUpdater.Update(assumed, &v1.PodCondition{
			Type:   v1.PodScheduled,
			Status: v1.ConditionFalse,
			Reason: "BindingRejected",
		})
		return err
	}

	metrics.BindingLatency.Observe(metrics.SinceInMicroseconds(bindingStart))
	sched.config.Recorder.Eventf(assumed, v1.EventTypeNormal, "Scheduled", "Successfully assigned %v to %v", assumed.Name, b.Target.Name)
	return nil
}

// scheduleOne does the entire scheduling workflow for a single pod.  It is serialized on the scheduling algorithm's host fitting.
func (sched *Scheduler) scheduleOne() {
	pod := sched.config.NextPod()
	if pod.DeletionTimestamp != nil {
		sched.config.Recorder.Eventf(pod, v1.EventTypeWarning, "FailedScheduling", "skip schedule deleting pod: %v/%v", pod.Namespace, pod.Name)
		glog.V(3).Infof("Skip schedule deleting pod: %v/%v", pod.Namespace, pod.Name)
		return
	}

	glog.V(3).Infof("Attempting to schedule pod: %v/%v", pod.Namespace, pod.Name)

	// Synchronously attempt to find a fit for the pod.
	start := time.Now()
	suggestedHost, err := sched.schedule(pod)
	metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start))
	if err != nil {
		return
	}

	// Tell the cache to assume that a pod now is running on a given node, even though it hasn't been bound yet.
	// This allows us to keep scheduling without waiting on binding to occur.
	assumedPod := *pod
	// assume modifies `assumedPod` by setting NodeName=suggestedHost
	err = sched.assume(&assumedPod, suggestedHost)
	if err != nil {
		return
	}

	// bind the pod to its host asynchronously (we can do this b/c of the assumption step above).
	go func() {
		err := sched.bind(&assumedPod, &v1.Binding{
			ObjectMeta: metav1.ObjectMeta{Namespace: assumedPod.Namespace, Name: assumedPod.Name, UID: assumedPod.UID},
			Target: v1.ObjectReference{
				Kind: "Node",
				Name: suggestedHost,
			},
		})
		metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start))
		if err != nil {
			glog.Errorf("Internal error binding pod: (%v)", err)
		}
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"reflect"
	"time"

	"github.com/mndrix/tap-go"
	rspecs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/opencontainers/runtime-tools/specerror"
	"github.com/opencontainers/runtime-tools/validation/util"
	uuid "github.com/satori/go.uuid"
)

func main() {
	t := tap.New()
	t.Header(0)
	bundleDir, err := util.PrepareBundle()
	if err != nil {
		util.Fatal(err)
	}
	defer os.RemoveAll(bundleDir)

	targetErr := specerror.NewError(specerror.KillNonCreateRunHaveNoEffect, fmt.Errorf("attempting to send a signal to a container that is neither `created` nor `running` MUST have no effect on the container"), rspecs.Version)
	containerID := uuid.NewV4().String()
	g := util.GetDefaultGenerator()
	g.SetProcessArgs([]string{"true"})

	config := util.LifecycleConfig{
		Config:    g,
		BundleDir: bundleDir,
		Actions:   util.LifecycleActionCreate | util.LifecycleActionStart | util.LifecycleActionDelete,
		PreCreate: func(r *util.Runtime) error {
			r.SetID(containerID)
			return nil
		},
		PreDelete: func(r *util.Runtime) error {
			err := util.WaitingForStatus(*r, util.LifecycleStatusStopped, time.Second*5, time.Second*1)
			if err != nil {
				return err
			}
			currentState, err := r.State()
			if err != nil {
				return err
			}
			r.Kill("KILL")
			newState, err := r.State()
			if err != nil || !reflect.DeepEqual(newState, currentState) {
				return targetErr
			}
			return nil
		},
	}
	err = util.RuntimeLifecycleValidate(config)
	if err != nil && err != targetErr {
		util.Fatal(err)
	} else {
		util.SpecErrorOK(t, err == nil, targetErr, nil)
	}
	t.AutoPlan()
}
<commit_msg>validation/kill_no_effect: fix bug<commit_after>package main

import (
	"fmt"
	"os"
	"reflect"
	"time"

	"github.com/mndrix/tap-go"
	rspecs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/opencontainers/runtime-tools/specerror"
	"github.com/opencontainers/runtime-tools/validation/util"
	uuid "github.com/satori/go.uuid"
)

func main() {
	t := tap.New()
	t.Header(0)
	bundleDir, err := util.PrepareBundle()
	if err != nil {
		util.Fatal(err)
	}
	defer os.RemoveAll(bundleDir)

	targetErr := specerror.NewError(specerror.KillNonCreateRunHaveNoEffect, fmt.Errorf("attempting to send a signal to a container that is neither `created` nor `running` MUST have no effect on the container"), rspecs.Version)
	containerID := uuid.NewV4().String()
	g, err := util.GetDefaultGenerator()
	if err != nil {
		util.Fatal(err)
	}
	g.SetProcessArgs([]string{"true"})

	config := util.LifecycleConfig{
		Config:    g,
		BundleDir: bundleDir,
		Actions:   util.LifecycleActionCreate | util.LifecycleActionStart | util.LifecycleActionDelete,
		PreCreate: func(r *util.Runtime) error {
			r.SetID(containerID)
			return nil
		},
		PreDelete: func(r *util.Runtime) error {
			err := util.WaitingForStatus(*r, util.LifecycleStatusStopped, time.Second*5, time.Second*1)
			if err != nil {
				return err
			}
			currentState, err := r.State()
			if err != nil {
				return err
			}
			r.Kill("KILL")
			newState, err := r.State()
			if err != nil || !reflect.DeepEqual(newState, currentState) {
				return targetErr
			}
			return nil
		},
	}
	err = util.RuntimeLifecycleValidate(config)
	if err != nil && err != targetErr {
		util.Fatal(err)
	} else {
		util.SpecErrorOK(t, err == nil, targetErr, nil)
	}
	t.AutoPlan()
}
<|endoftext|>