<commit_before>//go:build js
// +build js

package reflect_test

import (
	"math"
	"reflect"
	"testing"
)

func TestAlignment(t *testing.T) {
	t.Skip()
}

func TestSliceOverflow(t *testing.T) {
	t.Skip()
}

func TestFuncLayout(t *testing.T) {
	t.Skip()
}

func TestArrayOfDirectIface(t *testing.T) {
	t.Skip()
}

func TestTypelinksSorted(t *testing.T) {
	t.Skip()
}

func TestGCBits(t *testing.T) {
	t.Skip()
}

func TestChanAlloc(t *testing.T) {
	t.Skip()
}

func TestNameBytesAreAligned(t *testing.T) {
	t.Skip()
}

func TestOffsetLock(t *testing.T) {
	t.Skip()
}

func TestSelectOnInvalid(t *testing.T) {
	reflect.Select([]reflect.SelectCase{
		{
			Dir:  reflect.SelectRecv,
			Chan: reflect.Value{},
		}, {
			Dir:  reflect.SelectSend,
			Chan: reflect.Value{},
			Send: reflect.ValueOf(1),
		}, {
			Dir: reflect.SelectDefault,
		},
	})
}

func TestStructOfDirectIface(t *testing.T) {
	t.Skip("reflect.Value.InterfaceData is not supported by GopherJS.")
}

func TestStructOfWithInterface(t *testing.T) {
	// TODO(nevkontakte) Most of this test actually passes, but there is something
	// about embedding fields with methods that can or can't be stored in an
	// interface value directly that GopherJS does differently from upstream. As
	// a result, GopherJS's implementation of StructOf() doesn't panic where
	// upstream does. It seems to be a result of our implementation not propagating
	// the kindDirectIface flag in struct types created by StructOf(), but at this
	// point I wasn't able to figure out what that flag actually means in the
	// GopherJS context or how it maps onto our own reflection implementation.
	t.Skip("GopherJS doesn't support storing types directly in interfaces.")
}

var deepEqualTests = []DeepEqualTest{
	// Equalities
	{nil, nil, true},
	{1, 1, true},
	{int32(1), int32(1), true},
	{0.5, 0.5, true},
	{float32(0.5), float32(0.5), true},
	{"hello", "hello", true},
	{make([]int, 10), make([]int, 10), true},
	{&[3]int{1, 2, 3}, &[3]int{1, 2, 3}, true},
	{Basic{1, 0.5}, Basic{1, 0.5}, true},
	{error(nil), error(nil), true},
	{map[int]string{1: "one", 2: "two"}, map[int]string{2: "two", 1: "one"}, true},
	{fn1, fn2, true},

	// Inequalities
	{1, 2, false},
	{int32(1), int32(2), false},
	{0.5, 0.6, false},
	{float32(0.5), float32(0.6), false},
	{"hello", "hey", false},
	{make([]int, 10), make([]int, 11), false},
	{&[3]int{1, 2, 3}, &[3]int{1, 2, 4}, false},
	{Basic{1, 0.5}, Basic{1, 0.6}, false},
	{Basic{1, 0}, Basic{2, 0}, false},
	{map[int]string{1: "one", 3: "two"}, map[int]string{2: "two", 1: "one"}, false},
	{map[int]string{1: "one", 2: "txo"}, map[int]string{2: "two", 1: "one"}, false},
	{map[int]string{1: "one"}, map[int]string{2: "two", 1: "one"}, false},
	{map[int]string{2: "two", 1: "one"}, map[int]string{1: "one"}, false},
	{nil, 1, false},
	{1, nil, false},
	{fn1, fn3, false},
	{fn3, fn3, false},
	{[][]int{{1}}, [][]int{{2}}, false},
	{math.NaN(), math.NaN(), false},
	{&[1]float64{math.NaN()}, &[1]float64{math.NaN()}, false},
	{&[1]float64{math.NaN()}, self{}, true},
	{[]float64{math.NaN()}, []float64{math.NaN()}, false},
	{[]float64{math.NaN()}, self{}, true},
	{map[float64]float64{math.NaN(): 1}, map[float64]float64{1: 2}, false},
	{map[float64]float64{math.NaN(): 1}, self{}, true},

	// Nil vs empty: not the same.
	{[]int{}, []int(nil), false},
	{[]int{}, []int{}, true},
	{[]int(nil), []int(nil), true},
	{map[int]int{}, map[int]int(nil), false},
	{map[int]int{}, map[int]int{}, true},
	{map[int]int(nil), map[int]int(nil), true},

	// Mismatched types
	{1, 1.0, false},
	{int32(1), int64(1), false},
	{0.5, "hello", false},
	{[]int{1, 2, 3}, [3]int{1, 2, 3}, false},
	{&[3]interface{}{1, 2, 4}, &[3]interface{}{1, 2, "s"}, false},
	{Basic{1, 0.5}, NotBasic{1, 0.5}, false},
	{map[uint]string{1: "one", 2: "two"}, map[int]string{2: "two", 1: "one"}, false},

	// Possible loops.
	{&loop1, &loop1, true},
	//{&loop1, &loop2, true}, // TODO: Fix.
	{&loopy1, &loopy1, true},
	//{&loopy1, &loopy2, true}, // TODO: Fix.
}

// TODO: Fix this. See https://github.com/gopherjs/gopherjs/issues/763.
func TestIssue22073(t *testing.T) {
	m := reflect.ValueOf(NonExportedFirst(0)).Method(0)

	if got := m.Type().NumOut(); got != 0 {
		t.Errorf("NumOut: got %v, want 0", got)
	}

	// TODO: Fix this. The call below fails with:
	//
	// 	var $call = function(fn, rcvr, args) { return fn.apply(rcvr, args); };
	// 	                                                 ^
	// 	TypeError: Cannot read property 'apply' of undefined

	// Shouldn't panic.
	//m.Call(nil)
}

func TestCallReturnsEmpty(t *testing.T) {
	t.Skip("test uses runtime.SetFinalizer, which is not supported by GopherJS")
}

func init() {
	// TODO: This is a failure in 1.11, try to determine the cause and fix.
	typeTests = append(typeTests[:31], typeTests[32:]...) // skip test case #31
}

func TestConvertNaNs(t *testing.T) {
	// This test is exactly the same as the upstream, except it uses a "quiet NaN"
	// value instead of "signalling NaN". JavaScript appears to coerce all NaNs
	// into quiet ones, but for the purpose of this test either is fine.

	const qnan uint32 = 0x7fc00001 // Originally: 0x7f800001.
	type myFloat32 float32
	x := V(myFloat32(math.Float32frombits(qnan)))
	y := x.Convert(reflect.TypeOf(float32(0)))
	z := y.Interface().(float32)
	if got := math.Float32bits(z); got != qnan {
		t.Errorf("quiet nan conversion got %x, want %x", got, qnan)
	}
}
<commit_msg>reflect: Skip memory allocation test in TestMapIterSet.<commit_after>//go:build js
// +build js

package reflect_test

import (
	"math"
	. "reflect"
	"testing"
)

func TestAlignment(t *testing.T) {
	t.Skip()
}

func TestSliceOverflow(t *testing.T) {
	t.Skip()
}

func TestFuncLayout(t *testing.T) {
	t.Skip()
}

func TestArrayOfDirectIface(t *testing.T) {
	t.Skip()
}

func TestTypelinksSorted(t *testing.T) {
	t.Skip()
}

func TestGCBits(t *testing.T) {
	t.Skip()
}

func TestChanAlloc(t *testing.T) {
	t.Skip()
}

func TestNameBytesAreAligned(t *testing.T) {
	t.Skip()
}

func TestOffsetLock(t *testing.T) {
	t.Skip()
}

func TestSelectOnInvalid(t *testing.T) {
	Select([]SelectCase{
		{
			Dir:  SelectRecv,
			Chan: Value{},
		}, {
			Dir:  SelectSend,
			Chan: Value{},
			Send: ValueOf(1),
		}, {
			Dir: SelectDefault,
		},
	})
}

func TestStructOfDirectIface(t *testing.T) {
	t.Skip("reflect.Value.InterfaceData is not supported by GopherJS.")
}

func TestStructOfWithInterface(t *testing.T) {
	// TODO(nevkontakte) Most of this test actually passes, but there is something
	// about embedding fields with methods that can or can't be stored in an
	// interface value directly that GopherJS does differently from upstream. As
	// a result, GopherJS's implementation of StructOf() doesn't panic where
	// upstream does. It seems to be a result of our implementation not propagating
	// the kindDirectIface flag in struct types created by StructOf(), but at this
	// point I wasn't able to figure out what that flag actually means in the
	// GopherJS context or how it maps onto our own reflection implementation.
	t.Skip("GopherJS doesn't support storing types directly in interfaces.")
}

var deepEqualTests = []DeepEqualTest{
	// Equalities
	{nil, nil, true},
	{1, 1, true},
	{int32(1), int32(1), true},
	{0.5, 0.5, true},
	{float32(0.5), float32(0.5), true},
	{"hello", "hello", true},
	{make([]int, 10), make([]int, 10), true},
	{&[3]int{1, 2, 3}, &[3]int{1, 2, 3}, true},
	{Basic{1, 0.5}, Basic{1, 0.5}, true},
	{error(nil), error(nil), true},
	{map[int]string{1: "one", 2: "two"}, map[int]string{2: "two", 1: "one"}, true},
	{fn1, fn2, true},

	// Inequalities
	{1, 2, false},
	{int32(1), int32(2), false},
	{0.5, 0.6, false},
	{float32(0.5), float32(0.6), false},
	{"hello", "hey", false},
	{make([]int, 10), make([]int, 11), false},
	{&[3]int{1, 2, 3}, &[3]int{1, 2, 4}, false},
	{Basic{1, 0.5}, Basic{1, 0.6}, false},
	{Basic{1, 0}, Basic{2, 0}, false},
	{map[int]string{1: "one", 3: "two"}, map[int]string{2: "two", 1: "one"}, false},
	{map[int]string{1: "one", 2: "txo"}, map[int]string{2: "two", 1: "one"}, false},
	{map[int]string{1: "one"}, map[int]string{2: "two", 1: "one"}, false},
	{map[int]string{2: "two", 1: "one"}, map[int]string{1: "one"}, false},
	{nil, 1, false},
	{1, nil, false},
	{fn1, fn3, false},
	{fn3, fn3, false},
	{[][]int{{1}}, [][]int{{2}}, false},
	{math.NaN(), math.NaN(), false},
	{&[1]float64{math.NaN()}, &[1]float64{math.NaN()}, false},
	{&[1]float64{math.NaN()}, self{}, true},
	{[]float64{math.NaN()}, []float64{math.NaN()}, false},
	{[]float64{math.NaN()}, self{}, true},
	{map[float64]float64{math.NaN(): 1}, map[float64]float64{1: 2}, false},
	{map[float64]float64{math.NaN(): 1}, self{}, true},

	// Nil vs empty: not the same.
	{[]int{}, []int(nil), false},
	{[]int{}, []int{}, true},
	{[]int(nil), []int(nil), true},
	{map[int]int{}, map[int]int(nil), false},
	{map[int]int{}, map[int]int{}, true},
	{map[int]int(nil), map[int]int(nil), true},

	// Mismatched types
	{1, 1.0, false},
	{int32(1), int64(1), false},
	{0.5, "hello", false},
	{[]int{1, 2, 3}, [3]int{1, 2, 3}, false},
	{&[3]interface{}{1, 2, 4}, &[3]interface{}{1, 2, "s"}, false},
	{Basic{1, 0.5}, NotBasic{1, 0.5}, false},
	{map[uint]string{1: "one", 2: "two"}, map[int]string{2: "two", 1: "one"}, false},

	// Possible loops.
	{&loop1, &loop1, true},
	//{&loop1, &loop2, true}, // TODO: Fix.
	{&loopy1, &loopy1, true},
	//{&loopy1, &loopy2, true}, // TODO: Fix.
}

// TODO: Fix this. See https://github.com/gopherjs/gopherjs/issues/763.
func TestIssue22073(t *testing.T) {
	m := ValueOf(NonExportedFirst(0)).Method(0)

	if got := m.Type().NumOut(); got != 0 {
		t.Errorf("NumOut: got %v, want 0", got)
	}

	// TODO: Fix this. The call below fails with:
	//
	// 	var $call = function(fn, rcvr, args) { return fn.apply(rcvr, args); };
	// 	                                                 ^
	// 	TypeError: Cannot read property 'apply' of undefined

	// Shouldn't panic.
	//m.Call(nil)
}

func TestCallReturnsEmpty(t *testing.T) {
	t.Skip("test uses runtime.SetFinalizer, which is not supported by GopherJS")
}

func init() {
	// TODO: This is a failure in 1.11, try to determine the cause and fix.
	typeTests = append(typeTests[:31], typeTests[32:]...) // skip test case #31
}

func TestConvertNaNs(t *testing.T) {
	// This test is exactly the same as the upstream, except it uses a "quiet NaN"
	// value instead of "signalling NaN". JavaScript appears to coerce all NaNs
	// into quiet ones, but for the purpose of this test either is fine.

	const qnan uint32 = 0x7fc00001 // Originally: 0x7f800001.
	type myFloat32 float32
	x := V(myFloat32(math.Float32frombits(qnan)))
	y := x.Convert(TypeOf(float32(0)))
	z := y.Interface().(float32)
	if got := math.Float32bits(z); got != qnan {
		t.Errorf("quiet nan conversion got %x, want %x", got, qnan)
	}
}

func TestMapIterSet(t *testing.T) {
	m := make(map[string]any, len(valueTests))
	for _, tt := range valueTests {
		m[tt.s] = tt.i
	}
	v := ValueOf(m)

	k := New(v.Type().Key()).Elem()
	e := New(v.Type().Elem()).Elem()

	iter := v.MapRange()
	for iter.Next() {
		k.SetIterKey(iter)
		e.SetIterValue(iter)
		want := m[k.String()]
		got := e.Interface()
		if got != want {
			t.Errorf("%q: want (%T) %v, got (%T) %v", k.String(), want, want, got, got)
		}
		if setkey, key := valueToString(k), valueToString(iter.Key()); setkey != key {
			t.Errorf("MapIter.Key() = %q, MapIter.SetKey() = %q", key, setkey)
		}
		if setval, val := valueToString(e), valueToString(iter.Value()); setval != val {
			t.Errorf("MapIter.Value() = %q, MapIter.SetValue() = %q", val, setval)
		}
	}

	// Upstream test also tests allocations made by the iterator. GopherJS doesn't
	// support runtime.ReadMemStats(), so we leave that part out.
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor:
// - Aaron Meihm ameihm@mozilla.com
package scribe

import (
	"fmt"
)

type Modifier struct {
	Sources []ModifierSource `json:"sources"`
	Concat  ConcatModifier   `json:"concat"`
}

func (m *Modifier) addMergeTarget(ms *ModifierSource) {
	if m.Concat.Operator != "" {
		m.Concat.setMergeTarget(ms)
	} else {
		return
	}
	debugPrint("addMergeTarget(): merge target at %p\n", ms)
}

type ModifierSource struct {
	Name   string `json:"name"`
	Select string `json:"select"`

	criteria modifierData
}

func (m *ModifierSource) selectCriteria(t *Test) error {
	debugPrint("selectCriteria(): modifier selecting criteria from \"%v\"\n", t.Name)
	// XXX Just support "all" for now, this could change to select specific
	// elements of the source criteria slice.
	if m.Select != "all" {
		return fmt.Errorf("invalid selection criteria in modifier source")
	}
	s := t.getSourceInterface()
	if s == nil {
		return fmt.Errorf("source has no valid interface")
	}
	m.criteria.testName = t.Name
	m.criteria.criteria = s.getCriteria()
	debugPrint("selectCriteria(): copied %v criteria elements\n", len(m.criteria.criteria))
	return nil
}

type modifierData struct {
	testName string
	criteria []EvaluationCriteria
}

type mergingModifier struct {
	targets  []*ModifierSource
	criteria []EvaluationCriteria
}

func (m *mergingModifier) setMergeTarget(ms *ModifierSource) {
	m.targets = append(m.targets, ms)
}

func (m *mergingModifier) mergeTargets() {
	m.criteria = make([]EvaluationCriteria, 0)
	for _, x := range m.targets {
		for _, y := range x.criteria.criteria {
			m.criteria = append(m.criteria, y)
		}
	}
}

type ConcatModifier struct {
	Operator string `json:"operator"`
	mergingModifier
}

func (c *ConcatModifier) prepare() error {
	c.mergeTargets()
	return nil
}

func (c *ConcatModifier) expandVariables(v []Variable) {
}

func (c *ConcatModifier) getCriteria() []EvaluationCriteria {
	ret := make([]EvaluationCriteria, 0)
	nc := EvaluationCriteria{}
	ncid := ""
	buf := ""
	for _, x := range c.criteria {
		if len(buf) == 0 {
			ncid = "concat:" + x.Identifier
			buf = x.TestValue
		} else {
			ncid = ncid + "," + x.Identifier
			buf = buf + c.Operator + x.TestValue
		}
	}
	nc.Identifier = ncid
	nc.TestValue = buf
	ret = append(ret, nc)
	return ret
}
<commit_msg>return no criteria for modifier is no source data is present<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor:
// - Aaron Meihm ameihm@mozilla.com
package scribe

import (
	"fmt"
)

type Modifier struct {
	Sources []ModifierSource `json:"sources"`
	Concat  ConcatModifier   `json:"concat"`
}

func (m *Modifier) addMergeTarget(ms *ModifierSource) {
	if m.Concat.Operator != "" {
		m.Concat.setMergeTarget(ms)
	} else {
		return
	}
	debugPrint("addMergeTarget(): merge target at %p\n", ms)
}

type ModifierSource struct {
	Name   string `json:"name"`
	Select string `json:"select"`

	criteria modifierData
}

func (m *ModifierSource) selectCriteria(t *Test) error {
	debugPrint("selectCriteria(): modifier selecting criteria from \"%v\"\n", t.Name)
	// XXX Just support "all" for now, this could change to select specific
	// elements of the source criteria slice.
	if m.Select != "all" {
		return fmt.Errorf("invalid selection criteria in modifier source")
	}
	s := t.getSourceInterface()
	if s == nil {
		return fmt.Errorf("source has no valid interface")
	}
	m.criteria.testName = t.Name
	m.criteria.criteria = s.getCriteria()
	debugPrint("selectCriteria(): copied %v criteria elements\n", len(m.criteria.criteria))
	return nil
}

type modifierData struct {
	testName string
	criteria []EvaluationCriteria
}

type mergingModifier struct {
	targets  []*ModifierSource
	criteria []EvaluationCriteria
}

func (m *mergingModifier) setMergeTarget(ms *ModifierSource) {
	m.targets = append(m.targets, ms)
}

func (m *mergingModifier) mergeTargets() {
	m.criteria = make([]EvaluationCriteria, 0)
	for _, x := range m.targets {
		for _, y := range x.criteria.criteria {
			m.criteria = append(m.criteria, y)
		}
	}
}

type ConcatModifier struct {
	Operator string `json:"operator"`
	mergingModifier
}

func (c *ConcatModifier) prepare() error {
	c.mergeTargets()
	return nil
}

func (c *ConcatModifier) expandVariables(v []Variable) {
}

func (c *ConcatModifier) getCriteria() []EvaluationCriteria {
	ret := make([]EvaluationCriteria, 0)
	if len(c.criteria) == 0 {
		return ret
	}
	nc := EvaluationCriteria{}
	ncid := ""
	buf := ""
	for _, x := range c.criteria {
		if len(buf) == 0 {
			ncid = "concat:" + x.Identifier
			buf = x.TestValue
		} else {
			ncid = ncid + "," + x.Identifier
			buf = buf + c.Operator + x.TestValue
		}
	}
	nc.Identifier = ncid
	nc.TestValue = buf
	ret = append(ret, nc)
	return ret
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
	"unicode"

	"github.com/lemmi/glubcms"
)

func delspace(r rune) rune {
	if unicode.In(r, unicode.Latin, unicode.Digit) {
		return r
	} else {
		return '_'
	}
}

func main() {
	author := flag.String("author", "Webmaster", "Set the autorname")
	title := flag.String("title", "New Page", "Set the title")
	dirname := flag.String("dirname", "", "Set the directory name")
	priority := flag.Int("priority", 0, "Set the priority")
	simulate := flag.Bool("n", false, "Only show the result")
	hidden := flag.Bool("hidden", false, "Hide the page")
	edit := flag.Bool("e", false, "Open vim to edit the files")
	flag.Parse()

	meta := glubcms.Meta{
		Author:   *author,
		Title:    *title,
		Date:     glubcms.GCTime(time.Now()),
		Priority: *priority,
		Hidden:   *hidden,
	}

	if *dirname == "" {
		*dirname = time.Time(meta.Date).Format("2006-01-02_")
		*dirname += strings.NewReplacer(
			"ä", "ae",
			"ö", "oe",
			"ü", "ue",
			"ß", "ss").Replace(
			strings.Map(delspace, strings.ToLower(*title)))
	}

	b, err := json.MarshalIndent(meta, "", "\t")

	if err != nil {
		panic(err)
	}

	if !*simulate {
		if err := os.Mkdir(*dirname, 0755); err != nil {
			panic(err)
		}

		metafile, err := os.Create(filepath.Join(*dirname, "meta.json"))
		if err != nil {
			//remove dir?
			panic(err)
		}
		defer metafile.Close()

		_, err = metafile.Write(b)
		if err != nil {
			panic(err)
		}
	}
	fmt.Println(*dirname)
	fmt.Println(string(b))

	if *edit {
		vimpath, err := exec.LookPath("vim")
		if err != nil {
			panic(err)
		}
		fmt.Printf("Fount vim in %q\n", vimpath)
		cmd := exec.Command(
			vimpath,
			"-O",
			filepath.Join(*dirname, "article.md"),
			filepath.Join(*dirname, "meta.json"))
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			fmt.Println(cmd.Run())
		}
	}
}
<commit_msg>add template path to images<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
	"unicode"

	"github.com/lemmi/glubcms"
)

func delspace(r rune) rune {
	if unicode.In(r, unicode.Latin, unicode.Digit) {
		return r
	} else {
		return '_'
	}
}

func main() {
	author := flag.String("author", "Webmaster", "Set the autorname")
	title := flag.String("title", "New Page", "Set the title")
	dirname := flag.String("dirname", "", "Set the directory name")
	priority := flag.Int("priority", 0, "Set the priority")
	simulate := flag.Bool("n", false, "Only show the result")
	hidden := flag.Bool("hidden", false, "Hide the page")
	edit := flag.Bool("e", false, "Open vim to edit the files")
	flag.Parse()

	meta := glubcms.Meta{
		Author:   *author,
		Title:    *title,
		Date:     glubcms.GCTime(time.Now()),
		Priority: *priority,
		Hidden:   *hidden,
	}

	if *dirname == "" {
		*dirname = time.Time(meta.Date).Format("2006-01-02_")
		*dirname += strings.NewReplacer(
			"ä", "ae",
			"ö", "oe",
			"ü", "ue",
			"ß", "ss").Replace(
			strings.Map(delspace, strings.ToLower(*title)))
	}

	b, err := json.MarshalIndent(meta, "", "\t")

	if err != nil {
		panic(err)
	}

	if !*simulate {
		if err := os.Mkdir(*dirname, 0755); err != nil {
			panic(err)
		}

		metafile, err := os.Create(filepath.Join(*dirname, "meta.json"))
		if err != nil {
			//remove dir?
			panic(err)
		}
		defer metafile.Close()

		_, err = metafile.Write(b)
		if err != nil {
			panic(err)
		}

		func(dirname string) {
			articlefile, err := os.Create(filepath.Join(dirname, "article.md"))
			if err != nil {
				panic(err)
			}
			defer articlefile.Close()
			if _, err := fmt.Fprintf(articlefile, "![title](/static/images/%s/0001.jpg)\n", dirname); err != nil {
				panic(err)
			}
			if _, err := fmt.Fprintf(articlefile, "*Foto: *\n"); err != nil {
				panic(err)
			}
		}(*dirname)
	}
	fmt.Println(*dirname)
	fmt.Println(string(b))

	if *edit {
		vimpath, err := exec.LookPath("vim")
		if err != nil {
			panic(err)
		}
		fmt.Printf("Fount vim in %q\n", vimpath)
		cmd := exec.Command(
			vimpath,
			"-O",
			filepath.Join(*dirname, "article.md"),
			filepath.Join(*dirname, "meta.json"))
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			fmt.Println(cmd.Run())
		}
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"archive/zip"
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"github.com/fatih/color"
	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
	"github.com/grafana/grafana/pkg/util/errutil"

	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
	"github.com/grafana/grafana/pkg/cmd/grafana-cli/models"
	"github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
)

func validateInput(c utils.CommandLine, pluginFolder string) error {
	arg := c.Args().First()
	if arg == "" {
		return errors.New("please specify plugin to install")
	}

	pluginsDir := c.PluginDirectory()
	if pluginsDir == "" {
		return errors.New("missing pluginsDir flag")
	}

	fileInfo, err := os.Stat(pluginsDir)
	if err != nil {
		if err = os.MkdirAll(pluginsDir, os.ModePerm); err != nil {
			return fmt.Errorf("pluginsDir (%s) is not a writable directory", pluginsDir)
		}
		return nil
	}

	if !fileInfo.IsDir() {
		return errors.New("path is not a directory")
	}

	return nil
}

func (cmd Command) installCommand(c utils.CommandLine) error {
	pluginFolder := c.PluginDirectory()
	if err := validateInput(c, pluginFolder); err != nil {
		return err
	}

	pluginToInstall := c.Args().First()
	version := c.Args().Get(1)

	return InstallPlugin(pluginToInstall, version, c, cmd.Client)
}

// InstallPlugin downloads the plugin code as a zip file from the Grafana.com API
// and then extracts the zip into the plugins directory.
func InstallPlugin(pluginName, version string, c utils.CommandLine, client utils.ApiClient) error {
	pluginFolder := c.PluginDirectory()
	downloadURL := c.PluginURL()
	isInternal := false

	var checksum string
	if downloadURL == "" {
		if strings.HasPrefix(pluginName, "grafana-") {
			// At this point the plugin download is going through grafana.com API and thus the name is validated.
			// Checking for grafana prefix is how it is done there so no 3rd party plugin should have that prefix.
			// You can supply custom plugin name and then set custom download url to 3rd party plugin but then that
			// is up to the user to know what she is doing.
			isInternal = true
		}
		plugin, err := client.GetPlugin(pluginName, c.RepoDirectory())
		if err != nil {
			return err
		}

		v, err := SelectVersion(&plugin, version)
		if err != nil {
			return err
		}

		if version == "" {
			version = v.Version
		}
		downloadURL = fmt.Sprintf("%s/%s/versions/%s/download",
			c.String("repo"),
			pluginName,
			version,
		)

		// Plugins which are downloaded just as sourcecode zipball from github do not have checksum
		if v.Arch != nil {
			archMeta, exists := v.Arch[osAndArchString()]
			if !exists {
				archMeta = v.Arch["any"]
			}
			checksum = archMeta.SHA256
		}
	}

	logger.Infof("installing %v @ %v\n", pluginName, version)
	logger.Infof("from: %v\n", downloadURL)
	logger.Infof("into: %v\n", pluginFolder)
	logger.Info("\n")

	// Create temp file for downloading zip file
	tmpFile, err := ioutil.TempFile("", "*.zip")
	if err != nil {
		return errutil.Wrap("failed to create temporary file", err)
	}
	defer func() {
		if err := os.Remove(tmpFile.Name()); err != nil {
			logger.Warn("Failed to remove temporary file", "file", tmpFile.Name(), "err", err)
		}
	}()

	err = client.DownloadFile(pluginName, tmpFile, downloadURL, checksum)
	if err != nil {
		if err := tmpFile.Close(); err != nil {
			logger.Warn("Failed to close file", "err", err)
		}
		return errutil.Wrap("failed to download plugin archive", err)
	}
	err = tmpFile.Close()
	if err != nil {
		return errutil.Wrap("failed to close tmp file", err)
	}

	err = extractFiles(tmpFile.Name(), pluginName, pluginFolder, isInternal)
	if err != nil {
		return errutil.Wrap("failed to extract plugin archive", err)
	}

	logger.Infof("%s Installed %s successfully \n", color.GreenString("✔"), pluginName)

	res, _ := services.ReadPlugin(pluginFolder, pluginName)
	for _, v := range res.Dependencies.Plugins {
		if err := InstallPlugin(v.ID, "", c, client); err != nil {
			return errutil.Wrapf(err, "failed to install plugin '%s'", v.ID)
		}

		logger.Infof("Installed dependency: %v ✔\n", v.ID)
	}

	return err
}

func osAndArchString() string {
	osString := strings.ToLower(runtime.GOOS)
	arch := runtime.GOARCH
	return osString + "-" + arch
}

func supportsCurrentArch(version *models.Version) bool {
	if version.Arch == nil {
		return true
	}
	for arch := range version.Arch {
		if arch == osAndArchString() || arch == "any" {
			return true
		}
	}
	return false
}

func latestSupportedVersion(plugin *models.Plugin) *models.Version {
	for _, v := range plugin.Versions {
		ver := v
		if supportsCurrentArch(&ver) {
			return &ver
		}
	}
	return nil
}

// SelectVersion returns latest version if none is specified or the specified version. If the version string is not
// matched to existing version it errors out. It also errors out if version that is matched is not available for current
// os and platform. It expects plugin.Versions to be sorted so the newest version is first.
func SelectVersion(plugin *models.Plugin, version string) (*models.Version, error) {
	var ver models.Version

	latestForArch := latestSupportedVersion(plugin)
	if latestForArch == nil {
		return nil, fmt.Errorf("plugin is not supported on your architecture and OS")
	}

	if version == "" {
		return latestForArch, nil
	}
	for _, v := range plugin.Versions {
		if v.Version == version {
			ver = v
			break
		}
	}

	if len(ver.Version) == 0 {
		return nil, fmt.Errorf("could not find the version you're looking for")
	}

	if !supportsCurrentArch(&ver) {
		return nil, fmt.Errorf(
			"the version you want is not supported on your architecture and OS, latest suitable version is %s",
			latestForArch.Version)
	}

	return &ver, nil
}

var reGitBuild = regexp.MustCompile("^[a-zA-Z0-9_.-]*/")

func removeGitBuildFromName(pluginName, filename string) string {
	return reGitBuild.ReplaceAllString(filename, pluginName+"/")
}

const permissionsDeniedMessage = "could not create %q, permission denied, make sure you have write access to plugin dir"

func extractFiles(archiveFile string, pluginName string, dstDir string, allowSymlinks bool) error {
	var err error
	dstDir, err = filepath.Abs(dstDir)
	if err != nil {
		return err
	}
	logger.Debugf("Extracting archive %q to %q...\n", archiveFile, dstDir)

	r, err := zip.OpenReader(archiveFile)
	if err != nil {
		return err
	}
	for _, zf := range r.File {
		if filepath.IsAbs(zf.Name) || strings.HasPrefix(zf.Name, ".."+string(filepath.Separator)) {
			return fmt.Errorf(
				"archive member %q tries to write outside of plugin directory: %q, this can be a security risk",
				zf.Name, dstDir)
		}

		dstPath := filepath.Clean(filepath.Join(dstDir, removeGitBuildFromName(pluginName, zf.Name)))

		if zf.FileInfo().IsDir() {
			// We can ignore gosec G304 here since it makes sense to give all users read access
			// nolint:gosec
			if err := os.MkdirAll(dstPath, 0755); err != nil {
				if os.IsPermission(err) {
					return fmt.Errorf(permissionsDeniedMessage, dstPath)
				}

				return err
			}

			continue
		}

		// Create needed directories to extract file
		// We can ignore gosec G304 here since it makes sense to give all users read access
		// nolint:gosec
		if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
			return errutil.Wrap("failed to create directory to extract plugin files", err)
		}

		if isSymlink(zf) {
			if !allowSymlinks {
				logger.Warnf("%v: plugin archive contains a symlink, which is not allowed. Skipping \n", zf.Name)
				continue
			}
			if err := extractSymlink(zf, dstPath); err != nil {
				logger.Errorf("Failed to extract symlink: %v \n", err)
				continue
			}
			continue
		}

		if err := extractFile(zf, dstPath); err != nil {
			return errutil.Wrap("failed to extract file", err)
		}
	}

	return nil
}

func isSymlink(file *zip.File) bool {
	return file.Mode()&os.ModeSymlink == os.ModeSymlink
}

func extractSymlink(file *zip.File, filePath string) error {
	// symlink target is the contents of the file
	src, err := file.Open()
	if err != nil {
		return errutil.Wrap("failed to extract file", err)
	}
	buf := new(bytes.Buffer)
	if _, err := io.Copy(buf, src); err != nil {
		return errutil.Wrap("failed to copy symlink contents", err)
	}
	if err := os.Symlink(strings.TrimSpace(buf.String()), filePath); err != nil {
		return errutil.Wrapf(err, "failed to make symbolic link for %v", filePath)
	}
	return nil
}

func extractFile(file *zip.File, filePath string) (err error) {
	fileMode := file.Mode()
	// This is entry point for backend plugins so we want to make them executable
	if strings.HasSuffix(filePath, "_linux_amd64") || strings.HasSuffix(filePath, "_darwin_amd64") {
		fileMode = os.FileMode(0755)
	}

	// We can ignore the gosec G304 warning on this one, since the variable part of the file path stems
	// from command line flag "pluginsDir", and the only possible damage would be writing to the wrong directory.
	// If the user shouldn't be writing to this directory, they shouldn't have the permission in the file system.
	// nolint:gosec
	dst, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileMode)
	if err != nil {
		if os.IsPermission(err) {
			return fmt.Errorf(permissionsDeniedMessage, filePath)
		}

		unwrappedError := errors.Unwrap(err)
		if unwrappedError != nil && strings.EqualFold(unwrappedError.Error(), "text file busy") {
			return fmt.Errorf("file %q is in use - please stop Grafana, install the plugin and restart Grafana", filePath)
		}

		return errutil.Wrap("failed to open file", err)
	}
	defer func() {
		err = dst.Close()
	}()

	src, err := file.Open()
	if err != nil {
		return errutil.Wrap("failed to extract file", err)
	}
	defer func() {
		err = src.Close()
	}()

	_, err = io.Copy(dst, src)
	return err
}
<commit_msg>Plugins: Remove pre-existing plugin installs when installing with grafana-cli (#31515)<commit_after>package commands

import (
	"archive/zip"
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"github.com/fatih/color"
	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
	"github.com/grafana/grafana/pkg/util/errutil"

	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
	"github.com/grafana/grafana/pkg/cmd/grafana-cli/models"
	"github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
)

func validateInput(c utils.CommandLine, pluginFolder string) error {
	arg := c.Args().First()
	if arg == "" {
		return errors.New("please specify plugin to install")
	}

	pluginsDir := c.PluginDirectory()
	if pluginsDir == "" {
		return errors.New("missing pluginsDir flag")
	}

	fileInfo, err := os.Stat(pluginsDir)
	if err != nil {
		if err = os.MkdirAll(pluginsDir, os.ModePerm); err != nil {
			return fmt.Errorf("pluginsDir (%s) is not a writable directory", pluginsDir)
		}
		return nil
	}

	if !fileInfo.IsDir() {
		return errors.New("path is not a directory")
	}

	return nil
}

func (cmd Command) installCommand(c utils.CommandLine) error {
	pluginFolder := c.PluginDirectory()
	if err := validateInput(c, pluginFolder); err != nil {
		return err
	}

	pluginToInstall := c.Args().First()
	version := c.Args().Get(1)

	return InstallPlugin(pluginToInstall, version, c, cmd.Client)
}

// InstallPlugin downloads the plugin code as a zip file from the Grafana.com API
// and then extracts the zip into the plugins directory.
func InstallPlugin(pluginName, version string, c utils.CommandLine, client utils.ApiClient) error {
	pluginFolder := c.PluginDirectory()
	downloadURL := c.PluginURL()
	isInternal := false

	var checksum string
	if downloadURL == "" {
		if strings.HasPrefix(pluginName, "grafana-") {
			// At this point the plugin download is going through grafana.com API and thus the name is validated.
			// Checking for grafana prefix is how it is done there so no 3rd party plugin should have that prefix.
			// You can supply custom plugin name and then set custom download url to 3rd party plugin but then that
			// is up to the user to know what she is doing.
			isInternal = true
		}
		plugin, err := client.GetPlugin(pluginName, c.RepoDirectory())
		if err != nil {
			return err
		}

		v, err := SelectVersion(&plugin, version)
		if err != nil {
			return err
		}

		if version == "" {
			version = v.Version
		}
		downloadURL = fmt.Sprintf("%s/%s/versions/%s/download",
			c.String("repo"),
			pluginName,
			version,
		)

		// Plugins which are downloaded just as sourcecode zipball from github do not have checksum
		if v.Arch != nil {
			archMeta, exists := v.Arch[osAndArchString()]
			if !exists {
				archMeta = v.Arch["any"]
			}
			checksum = archMeta.SHA256
		}
	}

	logger.Infof("installing %v @ %v\n", pluginName, version)
	logger.Infof("from: %v\n", downloadURL)
	logger.Infof("into: %v\n", pluginFolder)
	logger.Info("\n")

	// Create temp file for downloading zip file
	tmpFile, err := ioutil.TempFile("", "*.zip")
	if err != nil {
		return errutil.Wrap("failed to create temporary file", err)
	}
	defer func() {
		if err := os.Remove(tmpFile.Name()); err != nil {
			logger.Warn("Failed to remove temporary file", "file", tmpFile.Name(), "err", err)
		}
	}()

	err = client.DownloadFile(pluginName, tmpFile, downloadURL, checksum)
	if err != nil {
		if err := tmpFile.Close(); err != nil {
			logger.Warn("Failed to close file", "err", err)
		}
		return errutil.Wrap("failed to download plugin archive", err)
	}
	err = tmpFile.Close()
	if err != nil {
		return errutil.Wrap("failed to close tmp file", err)
	}

	err = extractFiles(tmpFile.Name(), pluginName, pluginFolder, isInternal)
	if err != nil {
		return errutil.Wrap("failed to extract plugin archive", err)
	}

	logger.Infof("%s Installed %s successfully \n", color.GreenString("✔"), pluginName)

	res, _ := services.ReadPlugin(pluginFolder, pluginName)
	for _, v := range res.Dependencies.Plugins {
		if err := InstallPlugin(v.ID, "", c, client); err != nil {
			return errutil.Wrapf(err, "failed to install plugin '%s'", v.ID)
		}

		logger.Infof("Installed dependency: %v ✔\n", v.ID)
	}

	return err
}

func osAndArchString() string {
	osString := strings.ToLower(runtime.GOOS)
	arch := runtime.GOARCH
	return osString + "-" + arch
}

func supportsCurrentArch(version *models.Version) bool {
	if version.Arch == nil {
		return true
	}
	for arch := range version.Arch {
		if arch == osAndArchString() || arch == "any" {
			return true
		}
	}
	return false
}

func latestSupportedVersion(plugin *models.Plugin) *models.Version {
	for _, v := range plugin.Versions {
		ver := v
		if supportsCurrentArch(&ver) {
			return &ver
		}
	}
	return nil
}

// SelectVersion returns latest version if none is specified or the specified version. If the version string is not
// matched to existing version it errors out. It also errors out if version that is matched is not available for current
// os and platform. It expects plugin.Versions to be sorted so the newest version is first.
func SelectVersion(plugin *models.Plugin, version string) (*models.Version, error) {
	var ver models.Version

	latestForArch := latestSupportedVersion(plugin)
	if latestForArch == nil {
		return nil, fmt.Errorf("plugin is not supported on your architecture and OS")
	}

	if version == "" {
		return latestForArch, nil
	}
	for _, v := range plugin.Versions {
		if v.Version == version {
			ver = v
			break
		}
	}

	if len(ver.Version) == 0 {
		return nil, fmt.Errorf("could not find the version you're looking for")
	}

	if !supportsCurrentArch(&ver) {
		return nil, fmt.Errorf(
			"the version you want is not supported on your architecture and OS, latest suitable version is %s",
			latestForArch.Version)
	}

	return &ver, nil
}

var reGitBuild = regexp.MustCompile("^[a-zA-Z0-9_.-]*/")

func removeGitBuildFromName(pluginName, filename string) string {
	return reGitBuild.ReplaceAllString(filename, pluginName+"/")
}

const permissionsDeniedMessage = "could not create %q, permission denied, make sure you have write access to plugin dir"

func extractFiles(archiveFile string, pluginName string, dstDir string, allowSymlinks bool) error {
	var err error
	dstDir, err = filepath.Abs(dstDir)
	if err != nil {
		return err
	}
	logger.Debugf("Extracting archive %q to %q...\n", archiveFile, dstDir)

	existingInstallDir := filepath.Join(dstDir, pluginName)
	if _, err := os.Stat(existingInstallDir); !os.IsNotExist(err) {
		err = os.RemoveAll(existingInstallDir)
		if err != nil {
			return err
		}

		logger.Infof("Removed existing installation of %s\n\n", pluginName)
	}

	r, err := zip.OpenReader(archiveFile)
	if err != nil {
		return err
	}
	for _, zf := range r.File {
		if filepath.IsAbs(zf.Name) || strings.HasPrefix(zf.Name, ".."+string(filepath.Separator)) {
			return fmt.Errorf(
				"archive member %q tries to write outside of plugin directory: %q, this can be a security risk",
				zf.Name, dstDir)
		}

		dstPath := filepath.Clean(filepath.Join(dstDir, removeGitBuildFromName(pluginName, zf.Name)))

		if zf.FileInfo().IsDir() {
			// We can ignore gosec G304 here since it makes sense to give all users read access
			// nolint:gosec
			if err := os.MkdirAll(dstPath, 0755); err != nil {
				if os.IsPermission(err) {
					return fmt.Errorf(permissionsDeniedMessage, dstPath)
				}

				return err
			}

			continue
		}

		// Create needed directories to extract file
		// We can ignore gosec G304 here since it makes sense to give all users read access
		// nolint:gosec
		if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
			return errutil.Wrap("failed to create directory to extract plugin files", err)
		}

		if isSymlink(zf) {
			if !allowSymlinks {
				logger.Warnf("%v: plugin archive contains a symlink, which is not allowed. Skipping \n", zf.Name)
				continue
			}
			if err := extractSymlink(zf, dstPath); err != nil {
				logger.Errorf("Failed to extract symlink: %v \n", err)
				continue
			}
			continue
		}

		if err := extractFile(zf, dstPath); err != nil {
			return errutil.Wrap("failed to extract file", err)
		}
	}

	return nil
}

func isSymlink(file *zip.File) bool {
	return file.Mode()&os.ModeSymlink == os.ModeSymlink
}

func extractSymlink(file *zip.File, filePath string) error {
	// symlink target is the contents of the file
	src, err := file.Open()
	if err != nil {
		return errutil.Wrap("failed to extract file", err)
	}
	buf := new(bytes.Buffer)
	if _, err := io.Copy(buf, src); err != nil {
		return errutil.Wrap("failed to copy symlink contents", err)
	}
	if err := os.Symlink(strings.TrimSpace(buf.String()), filePath); err != nil {
		return errutil.Wrapf(err, "failed to make symbolic link for %v", filePath)
	}
	return nil
}

func extractFile(file *zip.File, filePath string) (err error) {
	fileMode := file.Mode()
	// This is entry point for backend plugins so we want to make them executable
	if strings.HasSuffix(filePath, "_linux_amd64") || strings.HasSuffix(filePath, "_darwin_amd64") {
		fileMode = os.FileMode(0755)
	}

	// We can ignore the gosec G304 warning on this one, since the variable part of the file path stems
	// from command line flag "pluginsDir", and the only possible damage would be writing to the wrong directory.
	// If the user shouldn't be writing to this directory, they shouldn't have the permission in the file system.
	// nolint:gosec
	dst, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileMode)
	if err != nil {
		if os.IsPermission(err) {
			return fmt.Errorf(permissionsDeniedMessage, filePath)
		}

		unwrappedError := errors.Unwrap(err)
		if unwrappedError != nil && strings.EqualFold(unwrappedError.Error(), "text file busy") {
			return fmt.Errorf("file %q is in use - please stop Grafana, install the plugin and restart Grafana", filePath)
		}

		return errutil.Wrap("failed to open file", err)
	}
	defer func() {
		err = dst.Close()
	}()

	src, err := file.Open()
	if err != nil {
		return errutil.Wrap("failed to extract file", err)
	}
	defer func() {
		err = src.Close()
	}()

	_, err = io.Copy(dst, src)
	return err
}
<|endoftext|>


<commit_before>// Copyright 2018 Adam Shannon
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build darwin

package test

import (
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/adamdecaf/cert-manage/pkg/certutil"
	"github.com/adamdecaf/cert-manage/pkg/file"
	"github.com/adamdecaf/cert-manage/pkg/store"
)

func TestIntegration__date(t *testing.T) {
	cmd := Command("date", "-u", "-r", "0").Trim()
	cmd.EqualT(t, "Thu Jan  1 00:00:00 UTC 1970")

	ans := `Command:
  date -u -r 0
Output:
  Thu Jan  1 00:00:00 UTC 1970`
	if cmd.String() != ans {
		t.Errorf("cmd.String() = %q", cmd.String())
	}
}

func TestIntegration__unknown(t *testing.T) {
	cmd := CertManage("other").Trim()
	cmd.FailedT(t)
}

func TestIntegration__list(t *testing.T) {
	t.Skip("darwin support is wip")

	cmd := CertManage("list", "-count").Trim()
	cmd.CmpIntF(t, func(i int) bool { return i > 1 })
}

func TestIntegration__listFromFile(t *testing.T) {
	cmd := CertManage("list", "-file", "../testdata/lots.crt", "-count").Trim()
	cmd.CmpIntF(t, func(i int) bool { return i == 5 })
}

func TestIntegration__backup(t *testing.T) {
	cmd := CertManage("backup").Trim()
	cmd.EqualT(t, "Backup completed successfully")
}

func TestIntegration__add(t *testing.T) {
	if !inCI() {
		t.Skip("not mutating non-CI login keychain")
	}
	setupKeychain(t)

	where := "../testdata/example.crt"
	certs, err := certutil.FromFile(where)
	if err != nil {
		t.Fatal(err)
	}
	if len(certs) != 1 {
		t.Fatalf("got %d certs", len(certs))
	}
	fp := certutil.GetHexSHA256Fingerprint(*certs[0])

	// verify cert doesn't exist already
	if inPlatformStore(t, fp) {
		name := certutil.StringifyPKIXName(certs[0].Subject)
		t.Fatalf("cert already in our store, please remove, %s", name)
	}

	// add cert and verify
	CertManage("add", "-file", where).SuccessT(t)
	if !inPlatformStore(t, fp) {
		t.Errorf("didn't find added cert, fp=%q", fp)
	}
}

func inPlatformStore(t *testing.T, fp string) bool {
	t.Helper()

	// Grab platform certs and verify ours is added
	found, err := store.Platform().List()
	if err != nil {
		t.Fatal(err)
	}
	for i := range found {
		ffp := certutil.GetHexSHA256Fingerprint(*found[i])
		if fp == ffp {
			return true
		}
	}
	return false
}

// Create a 'login.keychain' if it doesn't exist, only in CI
func setupKeychain(t *testing.T) {
	if !inCI() {
		return
	}
	t.Helper()

	where := filepath.Join(file.HomeDir(), "/Library/Keychains/login.keychain")
	if !file.Exists(where) {
		err := exec.Command("security", "create-keychain").Run()
		if err != nil {
			t.Error(err)
		}
	}
}

// TODO(adam): Need to run -whitelist and -restore

// Firefox tests
// func TestIntegration__firefox(t *testing.T) {
// 	if !online(t) {
// 		t.Skip("offline, can't run firefox tests (no NSS setup)")
// 	}

// 	// Make a request using the Keychain to get it ready
// 	// travis needs this
// 	cmd := Command("timeout", strings.Split("15s firefox --headless https://google.com 2>&1 >> /var/log/firefox.log", " ")...)
// 	cmd.SuccessT(t)

// 	// Verify firefox has found certificates
// 	cmd = CertManage("list", "-app", "firefox", "-count").Trim()
// 	cmd.SuccessT(t)
// 	cmd.CmpIntF(t, func(i int) bool { return i > 1 })
// }
<commit_msg>store/darwin: create CI keychain with blank password<commit_after>// Copyright 2018 Adam Shannon
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build darwin

package test

import (
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/adamdecaf/cert-manage/pkg/certutil"
	"github.com/adamdecaf/cert-manage/pkg/file"
	"github.com/adamdecaf/cert-manage/pkg/store"
)

func TestIntegration__date(t *testing.T) {
	cmd := Command("date", "-u", "-r", "0").Trim()
	cmd.EqualT(t, "Thu Jan  1 00:00:00 UTC 1970")

	ans := `Command:
  date -u -r 0
Output:
  Thu Jan  1 00:00:00 UTC 1970`
	if cmd.String() != ans {
		t.Errorf("cmd.String() = %q", cmd.String())
	}
}

func TestIntegration__unknown(t *testing.T) {
	cmd := CertManage("other").Trim()
	cmd.FailedT(t)
}

func TestIntegration__list(t *testing.T) {
	t.Skip("darwin support is wip")

	cmd := CertManage("list", "-count").Trim()
	cmd.CmpIntF(t, func(i int) bool { return i > 1 })
}

func TestIntegration__listFromFile(t *testing.T) {
	cmd := CertManage("list", "-file", "../testdata/lots.crt", "-count").Trim()
	cmd.CmpIntF(t, func(i int) bool { return i == 5 })
}

func TestIntegration__backup(t *testing.T) {
	cmd := CertManage("backup").Trim()
	cmd.EqualT(t, "Backup completed successfully")
}

func TestIntegration__add(t *testing.T) {
	if !inCI() {
		t.Skip("not mutating non-CI login keychain")
	}
	setupKeychain(t)

	where := "../testdata/example.crt"
	certs, err := certutil.FromFile(where)
	if err != nil {
		t.Fatal(err)
	}
	if len(certs) != 1 {
		t.Fatalf("got %d certs", len(certs))
	}
	fp := certutil.GetHexSHA256Fingerprint(*certs[0])

	// verify cert doesn't exist already
	if inPlatformStore(t, fp) {
		name := certutil.StringifyPKIXName(certs[0].Subject)
		t.Fatalf("cert already in our store, please remove, %s", name)
	}

	// add cert and verify
	CertManage("add", "-file", where).SuccessT(t)
	if !inPlatformStore(t, fp) {
		t.Errorf("didn't find added cert, fp=%q", fp)
	}
}

func inPlatformStore(t *testing.T, fp string) bool {
	t.Helper()

	// Grab platform certs and verify ours is added
	found, err := store.Platform().List()
	if err != nil {
		t.Fatal(err)
	}
	for i := range found {
		ffp := certutil.GetHexSHA256Fingerprint(*found[i])
		if fp == ffp {
			return true
		}
	}
	return false
}

// Create a 'login.keychain' if it doesn't exist, only in CI
func setupKeychain(t *testing.T) {
	if !inCI() {
		return
	}
	t.Helper()

	where := filepath.Join(file.HomeDir(), "/Library/Keychains/login.keychain")
	if !file.Exists(where) {
		err := exec.Command("security", "create-keychain", "-p", `''`).Run()
		if err != nil {
			t.Error(err)
		}
	}
}

// TODO(adam): Need to run -whitelist and -restore

// Firefox tests
// func TestIntegration__firefox(t *testing.T) {
// 	if !online(t) {
// 		t.Skip("offline, can't run firefox tests (no NSS setup)")
// 	}

// 	// Make a request using the Keychain to get it ready
// 	// travis needs this
// 	cmd := Command("timeout", strings.Split("15s firefox --headless https://google.com 2>&1 >> /var/log/firefox.log", " ")...)
// 	cmd.SuccessT(t)

// 	// Verify firefox has found certificates
// 	cmd = CertManage("list", "-app", "firefox", "-count").Trim()
// 	cmd.SuccessT(t)
// 	cmd.CmpIntF(t, func(i int) bool { return i > 1 })
// }
<|endoftext|>


<commit_before>package nodepool

import (
	"context"
	"fmt"
	"reflect"
	"regexp"
	"sort"
	"strconv"
	"sync"
	"time"

	v32 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"

	v3 "github.com/rancher/rancher/pkg/generated/norman/management.cattle.io/v3"
	"github.com/rancher/rancher/pkg/ref"
	"github.com/rancher/rancher/pkg/types/config"
	"github.com/rancher/rke/services"
	"github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
)

var (
	nameRegexp       = regexp.MustCompile("^(.*?)([0-9]+)$")
	unReachableTaint = v1.Taint{
		Key:    "node.kubernetes.io/unreachable",
		Effect: "NoExecute",
	}
	falseValue = false
)

type Controller struct {
	NodePoolController v3.NodePoolController
	NodePoolLister     v3.NodePoolLister
	NodePools          v3.NodePoolInterface
	NodeLister         v3.NodeLister
	Nodes              v3.NodeInterface
	mutex              sync.RWMutex
	syncmap            map[string]bool
}

func Register(ctx context.Context, management *config.ManagementContext) {
	p := &Controller{
		NodePoolController: management.Management.NodePools("").Controller(),
		NodePoolLister:     management.Management.NodePools("").Controller().Lister(),
		NodePools:          management.Management.NodePools(""),
		NodeLister:         management.Management.Nodes("").Controller().Lister(),
		Nodes:              management.Management.Nodes(""),
		syncmap:            make(map[string]bool),
	}

	// Add handlers
	p.NodePools.AddLifecycle(ctx, "nodepool-provisioner", p)
	management.Management.Nodes("").AddHandler(ctx, "nodepool-provisioner", p.machineChanged)
}

func (c *Controller) Create(nodePool *v3.NodePool) (runtime.Object, error) {
	return nodePool, nil
}

func (c *Controller) Updated(nodePool *v3.NodePool) (runtime.Object, error) {
	obj, err := v32.NodePoolConditionUpdated.Do(nodePool, func() (runtime.Object, error) {
		return nodePool, c.reconcile(nodePool)
	})
	return obj.(*v3.NodePool), err
}

func (c *Controller) Remove(nodePool *v3.NodePool) (runtime.Object, error) {
	logrus.Infof("Deleting nodePool [%s]", nodePool.Name)

	allNodes, err := c.nodes(nodePool, false)
	if err != nil {
		return nodePool, err
	}

	for _, node := range allNodes {
		_, nodePoolName := ref.Parse(node.Spec.NodePoolName)
		if nodePoolName != nodePool.Name {
			continue
		}

		err := c.deleteNode(node, time.Duration(0))
		if err != nil {
			return nil, err
		}
	}

	return nodePool, nil
}

func (c *Controller) machineChanged(key string, machine *v3.Node) (runtime.Object, error) {
	if machine == nil {
		nps, err := c.NodePoolLister.List("", labels.Everything())
		if err != nil {
			return nil, err
		}
		for _, np := range nps {
			c.NodePoolController.Enqueue(np.Namespace, np.Name)
		}
	} else if machine.Spec.NodePoolName != "" {
		ns, name := ref.Parse(machine.Spec.NodePoolName)
		c.NodePoolController.Enqueue(ns, name)
	}

	return nil, nil
}

func (c *Controller) createNode(name string, nodePool *v3.NodePool, simulate bool) (*v3.Node, error) {
	newNode := &v3.Node{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "m-",
			Namespace:    nodePool.Namespace,
			Labels:       nodePool.Labels,
			Annotations:  nodePool.Annotations,
		},
		Spec: v32.NodeSpec{
			Etcd:              nodePool.Spec.Etcd,
			ControlPlane:      nodePool.Spec.ControlPlane,
			Worker:            nodePool.Spec.Worker,
			NodeTemplateName:  nodePool.Spec.NodeTemplateName,
			NodePoolName:      ref.Ref(nodePool),
			RequestedHostname: name,
		},
	}

	if simulate {
		return newNode, nil
	}

	return c.Nodes.Create(newNode)
}

func (c *Controller) deleteNode(node *v3.Node, duration time.Duration) error {
	f := metav1.DeletePropagationBackground

	if duration > time.Duration(0) {
		go func() {
			time.Sleep(duration)
			c.Nodes.DeleteNamespaced(node.Namespace, node.Name, &metav1.DeleteOptions{
				PropagationPolicy: &f,
			})
		}()
		return nil
	}

	return c.Nodes.DeleteNamespaced(node.Namespace, node.Name, &metav1.DeleteOptions{
		PropagationPolicy: &f,
	})
}

func (c *Controller) reconcile(nodePool *v3.NodePool) error {
	changed, err := c.createOrCheckNodes(nodePool, true)
	if err != nil {
		return err
	}

	if changed {
		_, err = c.createOrCheckNodes(nodePool, false)
	}

	return err
}

func parsePrefix(fullPrefix string) (prefix string, minLength, start int) {
	m := nameRegexp.FindStringSubmatch(fullPrefix)
	if len(m) == 0 {
		return fullPrefix, 1, 1
	}
	prefix = m[1]
	start, _ = strconv.Atoi(m[2])
	return prefix, len(m[2]), start
}

func (c *Controller) nodes(nodePool *v3.NodePool, simulate bool) ([]*v3.Node, error) {
	if simulate {
		return c.NodeLister.List(nodePool.Namespace, labels.Everything())
	}

	nodeList, err := c.Nodes.ListNamespaced(nodePool.Namespace, metav1.ListOptions{})
	if err != nil {
		return nil, err
	}

	var nodes []*v3.Node
	for i := range nodeList.Items {
		nodes = append(nodes, &nodeList.Items[i])
	}

	return nodes, nil
}

func (c *Controller) createOrCheckNodes(nodePool *v3.NodePool, simulate bool) (bool, error) {
	var (
		err                 error
		byName              = map[string]*v3.Node{}
		changed             = false
		nodes               []*v3.Node
		deleteNotReadyAfter = nodePool.Spec.DeleteNotReadyAfterSecs * time.Second
	)

	allNodes, err := c.nodes(nodePool, simulate)
	if err != nil {
		return false, err
	}

	for _, node := range allNodes {
		byName[node.Spec.RequestedHostname] = node

		_, nodePoolName := ref.Parse(node.Spec.NodePoolName)
		if nodePoolName != nodePool.Name {
			continue
		}

		if v32.NodeConditionProvisioned.IsFalse(node) || v32.NodeConditionInitialized.IsFalse(node) || v32.NodeConditionConfigSaved.IsFalse(node) {
			changed = true
			if !simulate {
				_ = c.deleteNode(node, 2*time.Minute)
			}
		}
		// remove unreachable node with the unreachable taint & status of Ready being Unknown
		q := getTaint(node.Spec.InternalNodeSpec.Taints, &unReachableTaint)
		if q != nil && deleteNotReadyAfter > 0 {
			changed = true
			if isNodeReadyUnknown(node) && !simulate {
				start := q.TimeAdded.Time
				if time.Since(start) > deleteNotReadyAfter {
					err = c.deleteNode(node, 0)
					if err != nil {
						return false, err
					}
				} else {
					c.mutex.Lock()
					nodeid := node.Namespace + ":" + node.Name
					if _, ok := c.syncmap[nodeid]; !ok {
						c.syncmap[nodeid] = true
						go c.requeue(deleteNotReadyAfter, nodePool, node)
					}
					c.mutex.Unlock()
				}
			}
		}
		nodes = append(nodes, node)
	}

	quantity := nodePool.Spec.Quantity
	if quantity < 0 {
		quantity = 0
	}

	prefix, minLength, start := parsePrefix(nodePool.Spec.HostnamePrefix)

	for i := start; len(nodes) < quantity; i++ {
		ia := strconv.Itoa(i)
		name := prefix + ia
		if len(ia) < minLength {
			name = fmt.Sprintf("%s%0"+strconv.Itoa(minLength)+"d", prefix, i)
		}

		if byName[name] != nil {
			continue
		}

		changed = true
		newNode, err := c.createNode(name, nodePool, simulate)
		if err != nil {
			return false, err
		}

		byName[newNode.Spec.RequestedHostname] = newNode
		nodes = append(nodes, newNode)
	}

	for len(nodes) > quantity {
		sort.Sort(byHostname(nodes))

		toDelete := nodes[len(nodes)-1]

		changed = true
		if !simulate {
			c.deleteNode(toDelete, 0)
		}

		nodes = nodes[:len(nodes)-1]
		delete(byName, toDelete.Spec.RequestedHostname)
	}

	for _, n := range nodes {
		if needRoleUpdate(n, nodePool) {
			changed = true
			_, err := c.updateNodeRoles(n, nodePool, simulate)
			if err != nil {
				return false, err
			}
		}
	}

	return changed, nil
}

func needRoleUpdate(node *v3.Node, nodePool *v3.NodePool) bool {
	if node.Status.NodeConfig == nil {
		return false
	}
	if len(node.Status.NodeConfig.Role) == 0 && !nodePool.Spec.Worker {
		return true
	}

	nodeRolesMap := map[string]bool{}
	nodeRolesMap[services.ETCDRole] = false
	nodeRolesMap[services.ControlRole] = false
	nodeRolesMap[services.WorkerRole] = false

	for _, role := range node.Status.NodeConfig.Role {
		switch r := role; r {
		case services.ETCDRole:
			nodeRolesMap[services.ETCDRole] = true
		case services.ControlRole:
			nodeRolesMap[services.ControlRole] = true
		case services.WorkerRole:
			nodeRolesMap[services.WorkerRole] = true
		}
	}
	poolRolesMap := map[string]bool{}
	poolRolesMap[services.ETCDRole] = nodePool.Spec.Etcd
	poolRolesMap[services.ControlRole] = nodePool.Spec.ControlPlane
	poolRolesMap[services.WorkerRole] = nodePool.Spec.Worker

	r := !reflect.DeepEqual(nodeRolesMap, poolRolesMap)
	if r {
		logrus.Debugf("updating machine [%s] roles: nodepoolRoles: {%+v} node roles: {%+v}", node.Name, poolRolesMap, nodeRolesMap)
	}
	return r
}

func (c *Controller) updateNodeRoles(existing *v3.Node, nodePool *v3.NodePool, simulate bool) (*v3.Node, error) {
	toUpdate := existing.DeepCopy()
	var newRoles []string

	if nodePool.Spec.ControlPlane {
		newRoles = append(newRoles, "controlplane")
	}
	if nodePool.Spec.Etcd {
		newRoles = append(newRoles, "etcd")
	}
	if nodePool.Spec.Worker {
		newRoles = append(newRoles, "worker")
	}

	if len(newRoles) == 0 {
		newRoles = []string{"worker"}
	}

	toUpdate.Status.NodeConfig.Role = newRoles
	if simulate {
		return toUpdate, nil
	}
	return c.Nodes.Update(toUpdate)
}

// requeue checks every 5 seconds if the node is still unreachable with one goroutine per node
func (c *Controller) requeue(timeout time.Duration, np *v3.NodePool, node *v3.Node) {

	t := getTaint(node.Spec.InternalNodeSpec.Taints, &unReachableTaint)
	for t != nil {
		time.Sleep(5 * time.Second)
		exist, err := c.NodeLister.Get(node.Namespace, node.Name)
		if err != nil {
			break
		}
		t = getTaint(exist.Spec.InternalNodeSpec.Taints, &unReachableTaint)
		if t != nil && time.Since(t.TimeAdded.Time) > timeout {
			logrus.Debugf("Enqueue nodepool controller: %s %s", np.Namespace, np.Name)
			c.NodePoolController.Enqueue(np.Namespace, np.Name)
			break
		}
	}
	c.mutex.Lock()
	delete(c.syncmap, node.Namespace+":"+node.Name)
	c.mutex.Unlock()
}

// getTaint returns the taint that matches the given request
func getTaint(taints []v1.Taint, taintToFind *v1.Taint) *v1.Taint {
	for _, taint := range taints {
		if taint.MatchTaint(taintToFind) {
			return &taint
		}
	}
	return nil
}

// IsNodeReady returns true if a node Ready condition is Unknown; false otherwise.
func isNodeReadyUnknown(node *v3.Node) bool {
	for _, c := range node.Status.InternalNodeStatus.Conditions {
		if c.Type == v1.NodeReady {
			return c.Status == v1.ConditionUnknown
		}
	}
	return false
}
<commit_msg>Match unreachable taint key without comparing effect<commit_after>package nodepool

import (
	"context"
	"fmt"
	"reflect"
	"regexp"
	"sort"
	"strconv"
	"sync"
	"time"

	v32 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"

	v3 "github.com/rancher/rancher/pkg/generated/norman/management.cattle.io/v3"
	"github.com/rancher/rancher/pkg/ref"
	"github.com/rancher/rancher/pkg/types/config"
	"github.com/rancher/rke/services"
	"github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
)

var (
	nameRegexp = regexp.MustCompile("^(.*?)([0-9]+)$")
)

type Controller struct {
	NodePoolController v3.NodePoolController
	NodePoolLister     v3.NodePoolLister
	NodePools          v3.NodePoolInterface
	NodeLister         v3.NodeLister
	Nodes              v3.NodeInterface
	mutex              sync.RWMutex
	syncmap            map[string]bool
}

func Register(ctx context.Context, management *config.ManagementContext) {
	p := &Controller{
		NodePoolController: management.Management.NodePools("").Controller(),
		NodePoolLister:     management.Management.NodePools("").Controller().Lister(),
		NodePools:          management.Management.NodePools(""),
		NodeLister:         management.Management.Nodes("").Controller().Lister(),
		Nodes:              management.Management.Nodes(""),
		syncmap:            make(map[string]bool),
	}

	// Add handlers
	p.NodePools.AddLifecycle(ctx, "nodepool-provisioner", p)
	management.Management.Nodes("").AddHandler(ctx, "nodepool-provisioner", p.machineChanged)
}

func (c *Controller) Create(nodePool *v3.NodePool) (runtime.Object, error) {
	return nodePool, nil
}

func (c *Controller) Updated(nodePool *v3.NodePool) (runtime.Object, error) {
	obj, err := v32.NodePoolConditionUpdated.Do(nodePool, func() (runtime.Object, error) {
		return nodePool, c.reconcile(nodePool)
	})
	return obj.(*v3.NodePool), err
}

func (c *Controller) Remove(nodePool *v3.NodePool) (runtime.Object, error) {
	logrus.Infof("Deleting nodePool [%s]", nodePool.Name)

	allNodes, err := c.nodes(nodePool, false)
	if err != nil {
		return nodePool, err
	}

	for _, node := range allNodes {
		_, nodePoolName := ref.Parse(node.Spec.NodePoolName)
		if nodePoolName != nodePool.Name {
			continue
		}

		err := c.deleteNode(node, time.Duration(0))
		if err != nil {
			return nil, err
		}
	}

	return nodePool, nil
}

func (c *Controller) machineChanged(key string, machine *v3.Node) (runtime.Object, error) {
	if machine == nil {
		nps, err := c.NodePoolLister.List("", labels.Everything())
		if err != nil {
			return nil, err
		}
		for _, np := range nps {
			c.NodePoolController.Enqueue(np.Namespace, np.Name)
		}
	} else if machine.Spec.NodePoolName != "" {
		ns, name := ref.Parse(machine.Spec.NodePoolName)
		c.NodePoolController.Enqueue(ns, name)
	}

	return nil, nil
}

func (c *Controller) createNode(name string, nodePool *v3.NodePool, simulate bool) (*v3.Node, error) {
	newNode := &v3.Node{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "m-",
			Namespace:    nodePool.Namespace,
			Labels:       nodePool.Labels,
			Annotations:  nodePool.Annotations,
		},
		Spec: v32.NodeSpec{
			Etcd:              nodePool.Spec.Etcd,
			ControlPlane:      nodePool.Spec.ControlPlane,
			Worker:            nodePool.Spec.Worker,
			NodeTemplateName:  nodePool.Spec.NodeTemplateName,
			NodePoolName:      ref.Ref(nodePool),
			RequestedHostname: name,
		},
	}

	if simulate {
		return newNode, nil
	}

	return c.Nodes.Create(newNode)
}

func (c *Controller) deleteNode(node *v3.Node, duration time.Duration) error {
	f := metav1.DeletePropagationBackground

	if duration > time.Duration(0) {
		go func() {
			time.Sleep(duration)
			c.Nodes.DeleteNamespaced(node.Namespace, node.Name, &metav1.DeleteOptions{
				PropagationPolicy: &f,
			})
		}()
		return nil
	}

	return c.Nodes.DeleteNamespaced(node.Namespace, node.Name, &metav1.DeleteOptions{
		PropagationPolicy: &f,
	})
}

func (c *Controller) reconcile(nodePool *v3.NodePool) error {
	changed, err := c.createOrCheckNodes(nodePool, true)
	if err != nil {
		return err
	}

	if changed {
		_, err = c.createOrCheckNodes(nodePool, false)
	}

	return err
}

func parsePrefix(fullPrefix string) (prefix string, minLength, start int) {
	m := nameRegexp.FindStringSubmatch(fullPrefix)
	if len(m) == 0 {
		return fullPrefix, 1, 1
	}
	prefix = m[1]
	start, _ = strconv.Atoi(m[2])
	return prefix, len(m[2]), start
}

func (c *Controller) nodes(nodePool *v3.NodePool, simulate bool) ([]*v3.Node, error) {
	if simulate {
		return c.NodeLister.List(nodePool.Namespace, labels.Everything())
	}

	nodeList, err := c.Nodes.ListNamespaced(nodePool.Namespace, metav1.ListOptions{})
	if err != nil {
		return nil, err
	}

	var nodes []*v3.Node
	for i := range nodeList.Items {
		nodes = append(nodes, &nodeList.Items[i])
	}

	return nodes, nil
}

func (c *Controller) createOrCheckNodes(nodePool *v3.NodePool, simulate bool) (bool, error) {
	var (
		err                 error
		byName              = map[string]*v3.Node{}
		changed             = false
		nodes               []*v3.Node
		deleteNotReadyAfter = nodePool.Spec.DeleteNotReadyAfterSecs * time.Second
	)

	allNodes, err := c.nodes(nodePool, simulate)
	if err != nil {
		return false, err
	}

	for _, node := range allNodes {
		byName[node.Spec.RequestedHostname] = node

		_, nodePoolName := ref.Parse(node.Spec.NodePoolName)
		if nodePoolName != nodePool.Name {
			continue
		}

		if v32.NodeConditionProvisioned.IsFalse(node) || v32.NodeConditionInitialized.IsFalse(node) || v32.NodeConditionConfigSaved.IsFalse(node) {
			changed = true
			if !simulate {
				_ = c.deleteNode(node, 2*time.Minute)
			}
		}
		// remove unreachable node with the unreachable taint & status of Ready being Unknown
		q := getUnreachableTaint(node.Spec.InternalNodeSpec.Taints)
		if q != nil && deleteNotReadyAfter > 0 {
			changed = true
			if isNodeReadyUnknown(node) && !simulate {
				start := q.TimeAdded.Time
				if time.Since(start) > deleteNotReadyAfter {
					err = c.deleteNode(node, 0)
					if err != nil {
						return false, err
					}
				} else {
					c.mutex.Lock()
					nodeid := node.Namespace + ":" + node.Name
					if _, ok := c.syncmap[nodeid]; !ok {
						c.syncmap[nodeid] = true
						go c.requeue(deleteNotReadyAfter, nodePool, node)
					}
					c.mutex.Unlock()
				}
			}
		}
		nodes = append(nodes, node)
	}

	quantity := nodePool.Spec.Quantity
	if quantity < 0 {
		quantity = 0
	}

	prefix, minLength, start := parsePrefix(nodePool.Spec.HostnamePrefix)

	for i := start; len(nodes) < quantity; i++ {
		ia := strconv.Itoa(i)
		name := prefix + ia
		if len(ia) < minLength {
			name = fmt.Sprintf("%s%0"+strconv.Itoa(minLength)+"d", prefix, i)
		}

		if byName[name] != nil {
			continue
		}

		changed = true
		newNode, err := c.createNode(name, nodePool, simulate)
		if err != nil {
			return false, err
		}

		byName[newNode.Spec.RequestedHostname] = newNode
		nodes = append(nodes, newNode)
	}

	for len(nodes) > quantity {
		sort.Sort(byHostname(nodes))

		toDelete := nodes[len(nodes)-1]

		changed = true
		if !simulate {
			c.deleteNode(toDelete, 0)
		}

		nodes = nodes[:len(nodes)-1]
		delete(byName, toDelete.Spec.RequestedHostname)
	}

	for _, n := range nodes {
		if needRoleUpdate(n, nodePool) {
			changed = true
			_, err := c.updateNodeRoles(n, nodePool, simulate)
			if err != nil {
				return false, err
			}
		}
	}

	return changed, nil
}

func needRoleUpdate(node *v3.Node, nodePool *v3.NodePool) bool {
	if node.Status.NodeConfig == nil {
		return false
	}
	if len(node.Status.NodeConfig.Role) == 0 && !nodePool.Spec.Worker {
		return true
	}

	nodeRolesMap := map[string]bool{}
	nodeRolesMap[services.ETCDRole] = false
	nodeRolesMap[services.ControlRole] = false
	nodeRolesMap[services.WorkerRole] = false

	for _, role := range node.Status.NodeConfig.Role {
		switch r := role; r {
		case services.ETCDRole:
			nodeRolesMap[services.ETCDRole] = true
		case services.ControlRole:
			nodeRolesMap[services.ControlRole] = true
		case services.WorkerRole:
			nodeRolesMap[services.WorkerRole] = true
		}
	}
	poolRolesMap := map[string]bool{}
	poolRolesMap[services.ETCDRole] = nodePool.Spec.Etcd
	poolRolesMap[services.ControlRole] = nodePool.Spec.ControlPlane
	poolRolesMap[services.WorkerRole] = nodePool.Spec.Worker

	r := !reflect.DeepEqual(nodeRolesMap, poolRolesMap)
	if r {
		logrus.Debugf("updating machine [%s] roles: nodepoolRoles: {%+v} node roles: {%+v}", node.Name, poolRolesMap, nodeRolesMap)
	}
	return r
}

func (c *Controller) updateNodeRoles(existing *v3.Node, nodePool *v3.NodePool, simulate bool) (*v3.Node, error) {
	toUpdate := existing.DeepCopy()
	var newRoles []string

	if nodePool.Spec.ControlPlane {
		newRoles = append(newRoles, "controlplane")
	}
	if nodePool.Spec.Etcd {
		newRoles = append(newRoles, "etcd")
	}
	if nodePool.Spec.Worker {
		newRoles = append(newRoles, "worker")
	}

	if len(newRoles) == 0 {
		newRoles = []string{"worker"}
	}

	toUpdate.Status.NodeConfig.Role = newRoles
	if simulate {
		return toUpdate, nil
	}
	return c.Nodes.Update(toUpdate)
}

// requeue checks every 5 seconds if the node is still unreachable with one goroutine per node
func (c *Controller) requeue(timeout time.Duration, np *v3.NodePool, node *v3.Node) {

	t := getUnreachableTaint(node.Spec.InternalNodeSpec.Taints)
	for t != nil {
		time.Sleep(5 * time.Second)
		exist, err := c.NodeLister.Get(node.Namespace, node.Name)
		if err != nil {
			break
		}
		t = getUnreachableTaint(exist.Spec.InternalNodeSpec.Taints)
		if t != nil && time.Since(t.TimeAdded.Time) > timeout {
			logrus.Debugf("Enqueue nodepool controller: %s %s", np.Namespace, np.Name)
			c.NodePoolController.Enqueue(np.Namespace, np.Name)
			break
		}
	}
	c.mutex.Lock()
	delete(c.syncmap, node.Namespace+":"+node.Name)
	c.mutex.Unlock()
}

func getUnreachableTaint(taints []v1.Taint) *v1.Taint {
	for _, taint := range taints {
		if taint.Key == v1.TaintNodeUnreachable {
			return &taint
		}
	}
	return nil
}

// IsNodeReady returns true if a node Ready condition is Unknown; false otherwise.
func isNodeReadyUnknown(node *v3.Node) bool {
	for _, c := range node.Status.InternalNodeStatus.Conditions {
		if c.Type == v1.NodeReady {
			return c.Status == v1.ConditionUnknown
		}
	}
	return false
}
<|endoftext|>


<commit_before>package manager

import (
	"context"
	"encoding/json"
	"net/http"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/grafana/grafana-azure-sdk-go/azsettings"
	"github.com/grafana/grafana-plugin-sdk-go/backend"
	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
	"github.com/stretchr/testify/require"
	"go.opentelemetry.io/otel/trace"
	"gopkg.in/ini.v1"

	"github.com/grafana/grafana/pkg/infra/tracing"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin"
	"github.com/grafana/grafana/pkg/plugins/backendplugin/provider"
	"github.com/grafana/grafana/pkg/plugins/config"
	"github.com/grafana/grafana/pkg/plugins/manager/client"
	"github.com/grafana/grafana/pkg/plugins/manager/loader"
	"github.com/grafana/grafana/pkg/plugins/manager/registry"
	"github.com/grafana/grafana/pkg/plugins/manager/signature"
	"github.com/grafana/grafana/pkg/plugins/manager/store"
	"github.com/grafana/grafana/pkg/services/featuremgmt"
	"github.com/grafana/grafana/pkg/services/licensing"
	"github.com/grafana/grafana/pkg/services/searchV2"
	"github.com/grafana/grafana/pkg/services/sqlstore"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/tsdb/azuremonitor"
	"github.com/grafana/grafana/pkg/tsdb/cloudmonitoring"
	"github.com/grafana/grafana/pkg/tsdb/cloudwatch"
	"github.com/grafana/grafana/pkg/tsdb/elasticsearch"
	"github.com/grafana/grafana/pkg/tsdb/grafanads"
	"github.com/grafana/grafana/pkg/tsdb/graphite"
	"github.com/grafana/grafana/pkg/tsdb/influxdb"
	"github.com/grafana/grafana/pkg/tsdb/loki"
	"github.com/grafana/grafana/pkg/tsdb/mssql"
	"github.com/grafana/grafana/pkg/tsdb/mysql"
	"github.com/grafana/grafana/pkg/tsdb/opentsdb"
	"github.com/grafana/grafana/pkg/tsdb/postgres"
	"github.com/grafana/grafana/pkg/tsdb/prometheus"
	"github.com/grafana/grafana/pkg/tsdb/tempo"
	"github.com/grafana/grafana/pkg/tsdb/testdatasource"
)

func TestIntegrationPluginManager(t *testing.T) {
	t.Helper()

	staticRootPath, err := filepath.Abs("../../../public/")
	require.NoError(t, err)

	bundledPluginsPath, err := filepath.Abs("../../../plugins-bundled/internal")
	require.NoError(t, err)

	features := featuremgmt.WithFeatures()

	// We use the raw config here as it forms the basis for the setting.Provider implementation
	// The plugin manager also relies directly on the setting.Cfg struct to provide Grafana specific
	// properties such as the loading paths
	raw, err := ini.Load([]byte(`
		app_mode = production

		[plugin.test-app]
		path=testdata/test-app

		[plugin.test-panel]
		not=included
		`),
	)
	require.NoError(t, err)

	cfg := &setting.Cfg{
		Raw:                raw,
		StaticRootPath:     staticRootPath,
		BundledPluginsPath: bundledPluginsPath,
		Azure:              &azsettings.AzureSettings{},
	}

	tracer := &fakeTracer{}

	license := &licensing.OSSLicensingService{
		Cfg: cfg,
	}

	hcp := httpclient.NewProvider()
	am := azuremonitor.ProvideService(cfg, hcp, tracer)
	cw := cloudwatch.ProvideService(cfg, hcp, features)
	cm := cloudmonitoring.ProvideService(hcp, tracer)
	es := elasticsearch.ProvideService(hcp)
	grap := graphite.ProvideService(hcp, tracer)
	idb := influxdb.ProvideService(hcp)
	lk := loki.ProvideService(hcp, features, tracer)
	otsdb := opentsdb.ProvideService(hcp)
	pr := prometheus.ProvideService(hcp, cfg, features, tracer)
	tmpo := tempo.ProvideService(hcp)
	td := testdatasource.ProvideService(cfg, features)
	pg := postgres.ProvideService(cfg)
	my := mysql.ProvideService(cfg, hcp)
	ms := mssql.ProvideService(cfg)
	sv2 := searchV2.ProvideService(cfg, sqlstore.InitTestDB(t), nil, nil, tracing.InitializeTracerForTest(), featuremgmt.WithFeatures(), nil, nil)
	graf := grafanads.ProvideService(cfg, sv2, nil)

	coreRegistry := coreplugin.ProvideCoreRegistry(am, cw, cm, es, grap, idb, lk, otsdb, pr, tmpo, td, pg, my, ms, graf)

	pCfg := config.ProvideConfig(setting.ProvideProvider(cfg), cfg)
	reg := registry.ProvideService()
	l := loader.ProvideService(pCfg, license, signature.NewUnsignedAuthorizer(pCfg), reg, provider.ProvideService(coreRegistry))
	ps, err := store.ProvideService(cfg, pCfg, reg, l)
	require.NoError(t, err)

	ctx := context.Background()
	verifyCorePluginCatalogue(t, ctx, ps)
	verifyBundledPlugins(t, ctx, ps)
	verifyPluginStaticRoutes(t, ctx, ps)
	verifyBackendProcesses(t, reg.Plugins(ctx))
	verifyPluginQuery(t, ctx, client.ProvideService(reg))
}

func verifyPluginQuery(t *testing.T, ctx context.Context, c plugins.Client) {
	now := time.Unix(1661420870, 0)
	req := &backend.QueryDataRequest{
		PluginContext: backend.PluginContext{
			PluginID: "testdata",
		},
		Queries: []backend.DataQuery{
			{
				RefID: "A",
				TimeRange: backend.TimeRange{
					From: now.Add(-5 * time.Minute),
					To:   now,
				},
				JSON: json.RawMessage(`{"scenarioId":"csv_metric_values","stringInput":"1,20,90,30,5,0"}`),
			},
		},
	}

	resp, err := c.QueryData(ctx, req)
	require.NoError(t, err)
	payload, err := resp.MarshalJSON()
	require.NoError(t, err)
	require.JSONEq(t, `{"results":{"A":{"frames":[{"schema":{"refId":"A","fields":[{"name":"time","type":"time","typeInfo":{"frame":"time.Time"}},{"name":"A-series","type":"number","typeInfo":{"frame":"int64","nullable":true}}]},"data":{"values":[[1661420570000,1661420630000,1661420690000,1661420750000,1661420810000,1661420870000],[1,20,90,30,5,0]]}}]}}}`, string(payload))
}

func verifyCorePluginCatalogue(t *testing.T, ctx context.Context, ps *store.Service) {
	t.Helper()

	expPanels := map[string]struct{}{
		"alertGroups":    {},
		"alertlist":      {},
		"annolist":       {},
		"barchart":       {},
		"bargauge":       {},
		"canvas":         {},
		"dashlist":       {},
		"debug":          {},
		"gauge":          {},
		"geomap":         {},
		"gettingstarted": {},
		"graph":          {},
		"heatmap":        {},
		"heatmap-old":    {},
		"histogram":      {},
		"icon":           {},
		"live":           {},
		"logs":           {},
		"candlestick":    {},
		"news":           {},
		"nodeGraph":      {},
		"traces":         {},
		"piechart":       {},
		"stat":           {},
		"state-timeline": {},
		"status-history": {},
		"table":          {},
		"table-old":      {},
		"text":           {},
		"timeseries":     {},
		"welcome":        {},
		"xychart":        {},
	}

	expDataSources := map[string]struct{}{
		"cloudwatch":                       {},
		"stackdriver":                      {},
		"grafana-azure-monitor-datasource": {},
		"elasticsearch":                    {},
		"graphite":                         {},
		"influxdb":                         {},
		"loki":                             {},
		"opentsdb":                         {},
		"prometheus":                       {},
		"tempo":                            {},
		"testdata":                         {},
		"postgres":                         {},
		"mysql":                            {},
		"mssql":                            {},
		"grafana":                          {},
		"alertmanager":                     {},
		"dashboard":                        {},
		"input":                            {},
		"jaeger":                           {},
		"mixed":                            {},
		"zipkin":                           {},
	}

	expApps := map[string]struct{}{
		"test-app": {},
	}

	panels := ps.Plugins(ctx, plugins.Panel)
	require.Equal(t, len(expPanels), len(panels))
	for _, p := range panels {
		p, exists := ps.Plugin(ctx, p.ID)
		require.NotEqual(t, plugins.PluginDTO{}, p)
		require.True(t, exists)
		require.Contains(t, expPanels, p.ID)
	}

	dataSources := ps.Plugins(ctx, plugins.DataSource)
	require.Equal(t, len(expDataSources), len(dataSources))
	for _, ds := range dataSources {
		p, exists := ps.Plugin(ctx, ds.ID)
		require.NotEqual(t, plugins.PluginDTO{}, p)
		require.True(t, exists)
		require.Contains(t, expDataSources, ds.ID)
	}

	apps := ps.Plugins(ctx, plugins.App)
	require.Equal(t, len(expApps), len(apps))
	for _, app := range apps {
		p, exists := ps.Plugin(ctx, app.ID)
		require.True(t, exists)
		require.NotNil(t, p)
		require.Contains(t, expApps, app.ID)
	}

	require.Equal(t, len(expPanels)+len(expDataSources)+len(expApps), len(ps.Plugins(ctx)))
}

func verifyBundledPlugins(t *testing.T, ctx context.Context, ps *store.Service) {
	t.Helper()

	dsPlugins := make(map[string]struct{})
	for _, p := range ps.Plugins(ctx, plugins.DataSource) {
		dsPlugins[p.ID] = struct{}{}
	}

	inputPlugin, exists := ps.Plugin(ctx, "input")
	require.True(t, exists)
	require.NotEqual(t, plugins.PluginDTO{}, inputPlugin)
	require.NotNil(t, dsPlugins["input"])

	pluginRoutes := make(map[string]*plugins.StaticRoute)
	for _, r := range ps.Routes() {
		pluginRoutes[r.PluginID] = r
	}

	for _, pluginID := range []string{"input"} {
		require.Contains(t, pluginRoutes, pluginID)
		require.True(t, strings.HasPrefix(pluginRoutes[pluginID].Directory, inputPlugin.PluginDir))
	}
}

func verifyPluginStaticRoutes(t *testing.T, ctx context.Context, ps *store.Service) {
	routes := make(map[string]*plugins.StaticRoute)
	for _, route := range ps.Routes() {
		routes[route.PluginID] = route
	}

	require.Len(t, routes, 2)

	inputPlugin, _ := ps.Plugin(ctx, "input")
	require.NotNil(t, routes["input"])
	require.Equal(t, routes["input"].Directory, inputPlugin.PluginDir)

	testAppPlugin, _ := ps.Plugin(ctx, "test-app")
	require.Contains(t, routes, "test-app")
	require.Equal(t, routes["test-app"].Directory, testAppPlugin.PluginDir)
}

func verifyBackendProcesses(t *testing.T, ps []*plugins.Plugin) {
	for _, p := range ps {
		if p.Backend {
			pc, exists := p.Client()
			require.True(t, exists)
			require.NotNil(t, pc)
		}
	}
}

type fakeTracer struct {
	tracing.Tracer
}

func (ft *fakeTracer) Run(context.Context) error {
	return nil
}

func (ft *fakeTracer) Start(ctx context.Context, _ string, _ ...trace.SpanStartOption) (context.Context, tracing.Span) {
	return ctx, nil
}

func (ft *fakeTracer) Inject(context.Context, http.Header, tracing.Span) {

}
<commit_msg>re-use fake trace + feature toggles (#56186)<commit_after>package manager

import (
	"context"
	"encoding/json"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/grafana/grafana-azure-sdk-go/azsettings"
	"github.com/grafana/grafana-plugin-sdk-go/backend"
	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
	"github.com/stretchr/testify/require"
	"gopkg.in/ini.v1"

	"github.com/grafana/grafana/pkg/infra/tracing"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin"
	"github.com/grafana/grafana/pkg/plugins/backendplugin/provider"
	"github.com/grafana/grafana/pkg/plugins/config"
	"github.com/grafana/grafana/pkg/plugins/manager/client"
	"github.com/grafana/grafana/pkg/plugins/manager/loader"
	"github.com/grafana/grafana/pkg/plugins/manager/registry"
	"github.com/grafana/grafana/pkg/plugins/manager/signature"
	"github.com/grafana/grafana/pkg/plugins/manager/store"
	"github.com/grafana/grafana/pkg/services/featuremgmt"
	"github.com/grafana/grafana/pkg/services/licensing"
	"github.com/grafana/grafana/pkg/services/searchV2"
	"github.com/grafana/grafana/pkg/services/sqlstore"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/tsdb/azuremonitor"
	"github.com/grafana/grafana/pkg/tsdb/cloudmonitoring"
	"github.com/grafana/grafana/pkg/tsdb/cloudwatch"
	"github.com/grafana/grafana/pkg/tsdb/elasticsearch"
	"github.com/grafana/grafana/pkg/tsdb/grafanads"
	"github.com/grafana/grafana/pkg/tsdb/graphite"
	"github.com/grafana/grafana/pkg/tsdb/influxdb"
	"github.com/grafana/grafana/pkg/tsdb/loki"
	"github.com/grafana/grafana/pkg/tsdb/mssql"
	"github.com/grafana/grafana/pkg/tsdb/mysql"
	"github.com/grafana/grafana/pkg/tsdb/opentsdb"
	"github.com/grafana/grafana/pkg/tsdb/postgres"
	"github.com/grafana/grafana/pkg/tsdb/prometheus"
	"github.com/grafana/grafana/pkg/tsdb/tempo"
	"github.com/grafana/grafana/pkg/tsdb/testdatasource"
)

func TestIntegrationPluginManager(t *testing.T) {
	t.Helper()

	staticRootPath, err := filepath.Abs("../../../public/")
	require.NoError(t, err)

	bundledPluginsPath, err := filepath.Abs("../../../plugins-bundled/internal")
	require.NoError(t, err)

	// We use the raw config here as it forms the basis for the setting.Provider implementation
	// The plugin manager also relies directly on the setting.Cfg struct to provide Grafana specific
	// properties such as the loading paths
	raw, err := ini.Load([]byte(`
		app_mode = production

		[plugin.test-app]
		path=testdata/test-app

		[plugin.test-panel]
		not=included
		`),
	)
	require.NoError(t, err)

	cfg := &setting.Cfg{
		Raw:                raw,
		StaticRootPath:     staticRootPath,
		BundledPluginsPath: bundledPluginsPath,
		Azure:              &azsettings.AzureSettings{},
	}

	tracer := tracing.InitializeTracerForTest()
	features := featuremgmt.WithFeatures()

	hcp := httpclient.NewProvider()
	am := azuremonitor.ProvideService(cfg, hcp, tracer)
	cw := cloudwatch.ProvideService(cfg, hcp, features)
	cm := cloudmonitoring.ProvideService(hcp, tracer)
	es := elasticsearch.ProvideService(hcp)
	grap := graphite.ProvideService(hcp, tracer)
	idb := influxdb.ProvideService(hcp)
	lk := loki.ProvideService(hcp, features, tracer)
	otsdb := opentsdb.ProvideService(hcp)
	pr := prometheus.ProvideService(hcp, cfg, features, tracer)
	tmpo := tempo.ProvideService(hcp)
	td := testdatasource.ProvideService(cfg, features)
	pg := postgres.ProvideService(cfg)
	my := mysql.ProvideService(cfg, hcp)
	ms := mssql.ProvideService(cfg)
	sv2 := searchV2.ProvideService(cfg, sqlstore.InitTestDB(t), nil, nil, tracer, features, nil, nil)
	graf := grafanads.ProvideService(cfg, sv2, nil)

	coreRegistry := coreplugin.ProvideCoreRegistry(am, cw, cm, es, grap, idb, lk, otsdb, pr, tmpo, td, pg, my, ms, graf)

	pCfg := config.ProvideConfig(setting.ProvideProvider(cfg), cfg)
	reg := registry.ProvideService()
	l := loader.ProvideService(pCfg, &licensing.OSSLicensingService{Cfg: cfg}, signature.NewUnsignedAuthorizer(pCfg), reg, provider.ProvideService(coreRegistry))
	ps, err := store.ProvideService(cfg, pCfg, reg, l)
	require.NoError(t, err)

	ctx := context.Background()
	verifyCorePluginCatalogue(t, ctx, ps)
	verifyBundledPlugins(t, ctx, ps)
	verifyPluginStaticRoutes(t, ctx, ps)
	verifyBackendProcesses(t, reg.Plugins(ctx))
	verifyPluginQuery(t, ctx, client.ProvideService(reg))
}

func verifyPluginQuery(t *testing.T, ctx context.Context, c plugins.Client) {
	now := time.Unix(1661420870, 0)
	req := &backend.QueryDataRequest{
		PluginContext: backend.PluginContext{
			PluginID: "testdata",
		},
		Queries: []backend.DataQuery{
			{
				RefID: "A",
				TimeRange: backend.TimeRange{
					From: now.Add(-5 * time.Minute),
					To:   now,
				},
				JSON: json.RawMessage(`{"scenarioId":"csv_metric_values","stringInput":"1,20,90,30,5,0"}`),
			},
		},
	}

	resp, err := c.QueryData(ctx, req)
	require.NoError(t, err)
	payload, err := resp.MarshalJSON()
	require.NoError(t, err)
	require.JSONEq(t, `{"results":{"A":{"frames":[{"schema":{"refId":"A","fields":[{"name":"time","type":"time","typeInfo":{"frame":"time.Time"}},{"name":"A-series","type":"number","typeInfo":{"frame":"int64","nullable":true}}]},"data":{"values":[[1661420570000,1661420630000,1661420690000,1661420750000,1661420810000,1661420870000],[1,20,90,30,5,0]]}}]}}}`, string(payload))
}

func verifyCorePluginCatalogue(t *testing.T, ctx context.Context, ps *store.Service) {
	t.Helper()

	expPanels := map[string]struct{}{
		"alertGroups":    {},
		"alertlist":      {},
		"annolist":       {},
		"barchart":       {},
		"bargauge":       {},
		"canvas":         {},
		"dashlist":       {},
		"debug":          {},
		"gauge":          {},
		"geomap":         {},
		"gettingstarted": {},
		"graph":          {},
		"heatmap":        {},
		"heatmap-old":    {},
		"histogram":      {},
		"icon":           {},
		"live":           {},
		"logs":           {},
		"candlestick":    {},
		"news":           {},
		"nodeGraph":      {},
		"traces":         {},
		"piechart":       {},
		"stat":           {},
		"state-timeline": {},
		"status-history": {},
		"table":          {},
		"table-old":      {},
		"text":           {},
		"timeseries":     {},
		"welcome":        {},
		"xychart":        {},
	}

	expDataSources := map[string]struct{}{
		"cloudwatch":                       {},
		"stackdriver":                      {},
		"grafana-azure-monitor-datasource": {},
		"elasticsearch":                    {},
		"graphite":                         {},
		"influxdb":                         {},
		"loki":                             {},
		"opentsdb":                         {},
		"prometheus":                       {},
		"tempo":                            {},
		"testdata":                         {},
		"postgres":                         {},
		"mysql":                            {},
		"mssql":                            {},
		"grafana":                          {},
		"alertmanager":                     {},
		"dashboard":                        {},
		"input":                            {},
		"jaeger":                           {},
		"mixed":                            {},
		"zipkin":                           {},
	}

	expApps := map[string]struct{}{
		"test-app": {},
	}

	panels := ps.Plugins(ctx, plugins.Panel)
	require.Equal(t, len(expPanels), len(panels))
	for _, p := range panels {
		p, exists := ps.Plugin(ctx, p.ID)
		require.NotEqual(t, plugins.PluginDTO{}, p)
		require.True(t, exists)
		require.Contains(t, expPanels, p.ID)
	}

	dataSources := ps.Plugins(ctx, plugins.DataSource)
	require.Equal(t, len(expDataSources), len(dataSources))
	for _, ds := range dataSources {
		p, exists := ps.Plugin(ctx, ds.ID)
		require.NotEqual(t, plugins.PluginDTO{}, p)
		require.True(t, exists)
		require.Contains(t, expDataSources, ds.ID)
	}

	apps := ps.Plugins(ctx, plugins.App)
	require.Equal(t, len(expApps), len(apps))
	for _, app := range apps {
		p, exists := ps.Plugin(ctx, app.ID)
		require.True(t, exists)
		require.NotNil(t, p)
		require.Contains(t, expApps, app.ID)
	}

	require.Equal(t, len(expPanels)+len(expDataSources)+len(expApps), len(ps.Plugins(ctx)))
}

func verifyBundledPlugins(t *testing.T, ctx context.Context, ps *store.Service) {
	t.Helper()

	dsPlugins := make(map[string]struct{})
	for _, p := range ps.Plugins(ctx, plugins.DataSource) {
		dsPlugins[p.ID] = struct{}{}
	}

	inputPlugin, exists := ps.Plugin(ctx, "input")
	require.True(t, exists)
	require.NotEqual(t, plugins.PluginDTO{}, inputPlugin)
	require.NotNil(t, dsPlugins["input"])

	pluginRoutes := make(map[string]*plugins.StaticRoute)
	for _, r := range ps.Routes() {
		pluginRoutes[r.PluginID] = r
	}

	for _, pluginID := range []string{"input"} {
		require.Contains(t, pluginRoutes, pluginID)
		require.True(t, strings.HasPrefix(pluginRoutes[pluginID].Directory, inputPlugin.PluginDir))
	}
}

func verifyPluginStaticRoutes(t *testing.T, ctx context.Context, ps *store.Service) {
	routes := make(map[string]*plugins.StaticRoute)
	for _, route := range ps.Routes() {
		routes[route.PluginID] = route
	}

	require.Len(t, routes, 2)

	inputPlugin, _ := ps.Plugin(ctx, "input")
	require.NotNil(t, routes["input"])
	require.Equal(t, routes["input"].Directory, inputPlugin.PluginDir)

	testAppPlugin, _ := ps.Plugin(ctx, "test-app")
	require.Contains(t, routes, "test-app")
	require.Equal(t, routes["test-app"].Directory, testAppPlugin.PluginDir)
}

func verifyBackendProcesses(t *testing.T, ps []*plugins.Plugin) {
	for _, p := range ps {
		if p.Backend {
			pc, exists := p.Client()
			require.True(t, exists)
			require.NotNil(t, pc)
		}
	}
}
<|endoftext|>


<commit_before>package ws_test

import (
	"context"
	"github.com/notegio/openrelay/channels"
	"github.com/notegio/openrelay/channels/ws"
	"github.com/gorilla/websocket"
	"testing"
	"time"
	// "log"
)

type TestConsumer struct {
	channel *ws.WebsocketChannel
}

func (consumer *TestConsumer) Consume(delivery channels.Delivery) {
	if delivery.Payload() == "quit" {
		consumer.channel.StopConsuming()
		delivery.Ack()
	}
	consumer.channel.Publish(delivery.Payload())
	delivery.Ack()
}

func TestGetChannels(t *testing.T) {
	clean := false
	channels, quit := ws.GetChannels(4321, nil, func() { clean = true })
	go func() {
		for channel := range channels {
			channel.AddConsumer(&TestConsumer{channel})
			channel.StartConsuming()
		}
	}()
	ctx, cancel := context.WithCancel(context.Background())
	c, resp, err := websocket.DefaultDialer.DialContext(ctx, "ws://localhost:4321/v2/", nil)
	defer cancel()
	if err != nil {
		content := []byte{}
		resp.Body.Read(content[:])
		t.Fatalf("%v - (%v) %v", err.Error(), resp.StatusCode, content)
	}
	if err := c.WriteMessage(websocket.BinaryMessage, []byte("ping")); err != nil {
		t.Errorf(err.Error())
	}
	mtype, p, err := c.ReadMessage()
	if err != nil {
		t.Errorf(err.Error())
	}
	if mtype != websocket.BinaryMessage {
		t.Errorf("Unexpected message type %v", mtype)
	}
	if string(p) != "ping" {
		t.Errorf("Unexpected message: %v", string(p))
	}
	c.Close()
	if err := quit(); err != nil {
		t.Errorf(err.Error())
	}
	time.Sleep(50 * time.Millisecond)
	if clean != true {
		t.Errorf("Should have cleaned up")
	}
}
<commit_msg>Fix changed method signature in tests<commit_after>package ws_test

import (
	"context"
	"github.com/notegio/openrelay/channels"
	"github.com/notegio/openrelay/channels/ws"
	"github.com/gorilla/websocket"
	"testing"
	"time"
	// "log"
)

type TestConsumer struct {
	channel *ws.WebsocketChannel
}

func (consumer *TestConsumer) Consume(delivery channels.Delivery) {
	if delivery.Payload() == "quit" {
		consumer.channel.StopConsuming()
		delivery.Ack()
	}
	consumer.channel.Publish(delivery.Payload())
	delivery.Ack()
}

func TestGetChannels(t *testing.T) {
	clean := false
	channels, quit := ws.GetChannels(4321, nil, func(channels.Publisher) { clean = true })
	go func() {
		for channel := range channels {
			channel.AddConsumer(&TestConsumer{channel})
			channel.StartConsuming()
		}
	}()
	ctx, cancel := context.WithCancel(context.Background())
	c, resp, err := websocket.DefaultDialer.DialContext(ctx, "ws://localhost:4321/v2/", nil)
	defer cancel()
	if err != nil {
		content := []byte{}
		resp.Body.Read(content[:])
		t.Fatalf("%v - (%v) %v", err.Error(), resp.StatusCode, content)
	}
	if err := c.WriteMessage(websocket.BinaryMessage, []byte("ping")); err != nil {
		t.Errorf(err.Error())
	}
	mtype, p, err := c.ReadMessage()
	if err != nil {
		t.Errorf(err.Error())
	}
	if mtype != websocket.BinaryMessage {
		t.Errorf("Unexpected message type %v", mtype)
	}
	if string(p) != "ping" {
		t.Errorf("Unexpected message: %v", string(p))
	}
	c.Close()
	if err := quit(); err != nil {
		t.Errorf(err.Error())
	}
	time.Sleep(50 * time.Millisecond)
	if clean != true {
		t.Errorf("Should have cleaned up")
	}
}
<|endoftext|>


<commit_before>package jade

import (
	// "fmt"
	"strings"
)

func (t *Tree) parse(treeSet map[string]*Tree) (next Node) {
	token := t.next()
	t.Root = t.newList(token.pos)

	for token.typ != itemEOF {

		switch token.typ {
		case itemError:
			t.errorf("%s", token.val)
		case itemDoctype:
			t.Root.append( t.newDoctype(token.pos, token.val) )
		case itemHtmlTag:
			t.Root.append( t.newLine(token.pos, token.val, token.typ, 0, 0) )

		case itemTag, itemDiv, itemInlineTag, itemAction, itemComment, itemBlank:
			nest := t.newNest(token.pos, token.val, token.typ, 0, 0)
			if token.typ < itemComment { t.parseAttr( nest ) }
			t.Root.append( nest )
			t.parseInside( nest )
		}

		token = t.next()
		// fmt.Printf("%s\t\t\t%s\n", itemToStr[token.typ], token.val)
	}
	// fmt.Println("dixi.")
	return nil
}

func (t *Tree) parseInside( outTag *NestNode ) int {
	indentCount := 0
	token := t.next()

	for token.typ != itemEOF {
		// fmt.Printf("-%d-%d---- %s\t\t\t%s\n", indentCount, outTag.Indent, itemToStr[token.typ], token.val)

		switch token.typ {
		case itemError:
			t.errorf("%s", token.val)

		case itemEndL:
			indentCount = 0
		case itemIdentSpace:
			indentCount ++
		case itemIdentTab:
			indentCount += tabSize
		case itemParentIdent:
			indentCount = outTag.Indent + 1  // for  "tag: tag: tag"

		case itemText, itemInlineText, itemInlineAction, itemTemplate:
			outTag.append( t.newLine(token.pos, token.val, token.typ, indentCount, outTag.Nesting + 1) )

		case itemHtmlTag:
			if indentCount > outTag.Indent {
				outTag.append( t.newLine(token.pos, token.val, token.typ, indentCount, outTag.Nesting + 1) )
			}else{
				t.backup(); return indentCount
			}

		case itemVoidTag, itemInlineVoidTag:
			if indentCount > outTag.Indent {
				nest := t.newNest(token.pos, token.val, token.typ, indentCount, outTag.Nesting + 1)
				t.parseAttr( nest )
				outTag.append( nest )
			}else{
				t.backup(); return indentCount
			}

		case itemTag, itemDiv, itemInlineTag:
			if indentCount > outTag.Indent {
				nest := t.newNest(token.pos, token.val, token.typ, indentCount, outTag.Nesting + 1)
				t.parseAttr( nest )
				outTag.append( nest )
				indentCount = t.parseInside( nest )
			}else{
				t.backup(); return indentCount
			}

		case itemAction, itemActionEnd:
			if indentCount > outTag.Indent {
				nest := t.newNest(token.pos, token.val, token.typ, indentCount, outTag.Nesting + 1)
				outTag.append( nest )
				indentCount = t.parseInside( nest )
				if strings.HasPrefix(nest.Tag, "if") {
					action := t.next()
					if action.val == "else" { nest.typ = itemAction }
					t.backup()
				}
			}else{
				t.backup(); return indentCount
			}

		case itemBlank, itemComment:
			if indentCount > outTag.Indent {
				nest := t.newNest(token.pos, token.val, token.typ, indentCount, outTag.Nesting + 1)
				if token.typ == itemComment { outTag.append( nest ) }
				indentCount = t.parseInside( nest )
			}else{
				t.backup(); return indentCount
			}

		}
		token = t.next()
	}
	t.backup(); return indentCount
}

func (t *Tree) parseAttr( currentTag *NestNode ) {
	for {
		attr := t.next()
		// fmt.Println(itemToStr[attr.typ], attr.val)
		switch attr.typ {
		case itemError:
			t.errorf("%s", attr.val)
		case itemId:
			if len(currentTag.id) > 0 { t.errorf("unexpected second id \"%s\" ", attr.val) }
			currentTag.id = attr.val
		case itemClass:
			currentTag.class = append( currentTag.class, attr.val )
		case itemAttr, itemAttrN, itemAttrName, itemAttrVoid:
			currentTag.append( t.newAttr(attr.pos, attr.val, attr.typ) )
		default:
			t.backup()
			currentTag.append( t.newAttr(attr.pos, ">", itemEndAttr) )
			return
		}
	}
}<commit_msg>fix itemActionEnd with, else<commit_after>package jade

import (
	// "fmt"
	"strings"
)

func (t *Tree) parse(treeSet map[string]*Tree) (next Node) {
	token := t.next()
	t.Root = t.newList(token.pos)

	for token.typ != itemEOF {

		switch token.typ {
		case itemError:
			t.errorf("%s", token.val)
		case itemDoctype:
			t.Root.append( t.newDoctype(token.pos, token.val) )
		case itemHtmlTag:
			t.Root.append( t.newLine(token.pos, token.val, token.typ, 0, 0) )

		case itemTag, itemDiv, itemInlineTag, itemAction, itemComment, itemBlank:
			nest := t.newNest(token.pos, token.val, token.typ, 0, 0)
			if token.typ < itemComment { t.parseAttr( nest ) }
			t.Root.append( nest )
			t.parseInside( nest )
		}

		token = t.next()
		// fmt.Printf("%s\t\t\t%s\n", itemToStr[token.typ], token.val)
	}
	// fmt.Println("dixi.")
	return nil
}

func (t *Tree) parseInside( outTag *NestNode ) int {
	indentCount := 0
	token := t.next()

	for token.typ != itemEOF {
		// fmt.Printf("-%d-%d---- %s\t\t\t%s\n", indentCount, outTag.Indent, itemToStr[token.typ], token.val)

		switch token.typ {
		case itemError:
			t.errorf("%s", token.val)

		case itemEndL:
			indentCount = 0
		case itemIdentSpace:
			indentCount ++
		case itemIdentTab:
			indentCount += tabSize
		case itemParentIdent:
			indentCount = outTag.Indent + 1  // for  "tag: tag: tag"

		case itemText, itemInlineText, itemInlineAction, itemTemplate:
			outTag.append( t.newLine(token.pos, token.val, token.typ, indentCount, outTag.Nesting + 1) )

		case itemHtmlTag:
			if indentCount > outTag.Indent {
				outTag.append( t.newLine(token.pos, token.val, token.typ, indentCount, outTag.Nesting + 1) )
			}else{
				t.backup(); return indentCount
			}

		case itemVoidTag, itemInlineVoidTag:
			if indentCount > outTag.Indent {
				nest := t.newNest(token.pos, token.val, token.typ, indentCount, outTag.Nesting + 1)
				t.parseAttr( nest )
				outTag.append( nest )
			}else{
				t.backup(); return indentCount
			}

		case itemTag, itemDiv, itemInlineTag:
			if indentCount > outTag.Indent {
				nest := t.newNest(token.pos, token.val, token.typ, indentCount, outTag.Nesting + 1)
				t.parseAttr( nest )
				outTag.append( nest )
				indentCount = t.parseInside( nest )
			}else{
				t.backup(); return indentCount
			}

		case itemAction, itemActionEnd:
			if indentCount > outTag.Indent {
				nest := t.newNest(token.pos, token.val, token.typ, indentCount, outTag.Nesting + 1)
				outTag.append( nest )
				indentCount = t.parseInside( nest )
				if strings.HasPrefix(nest.Tag, "if") || strings.HasPrefix(nest.Tag, "with") {
					action := t.next()
					if strings.HasPrefix(action.val, "else") { nest.typ = itemAction }
					t.backup()
				}
			}else{
				t.backup(); return indentCount
			}

		case itemBlank, itemComment:
			if indentCount > outTag.Indent {
				nest := t.newNest(token.pos, token.val, token.typ, indentCount, outTag.Nesting + 1)
				if token.typ == itemComment { outTag.append( nest ) }
				indentCount = t.parseInside( nest )
			}else{
				t.backup(); return indentCount
			}

		}
		token = t.next()
	}
	t.backup(); return indentCount
}

func (t *Tree) parseAttr( currentTag *NestNode ) {
	for {
		attr := t.next()
		// fmt.Println(itemToStr[attr.typ], attr.val)
		switch attr.typ {
		case itemError:
			t.errorf("%s", attr.val)
		case itemId:
			if len(currentTag.id) > 0 { t.errorf("unexpected second id \"%s\" ", attr.val) }
			currentTag.id = attr.val
		case itemClass:
			currentTag.class = append( currentTag.class, attr.val )
		case itemAttr, itemAttrN, itemAttrName, itemAttrVoid:
			currentTag.append( t.newAttr(attr.pos, attr.val, attr.typ) )
		default:
			t.backup()
			currentTag.append( t.newAttr(attr.pos, ">", itemEndAttr) )
			return
		}
	}
}<|endoftext|>