<commit_before>package edit

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"sort"
	"strings"

	"github.com/elves/elvish/eval"
	"github.com/elves/elvish/parse"
	"github.com/elves/elvish/util"
)

// completer takes the current Node (always a leaf in the AST) and an Editor and
// returns a compl. If the completer does not apply to the type of the current
// Node, it should return an error of ErrCompletionUnapplicable.
type completer func(parse.Node, *eval.Evaler) (*compl, error)

// compl is the result of a completer, meaning that any of the candidates can
// replace the text in the interval [begin, end).
type compl struct {
	begin int
	end   int
	cands []*candidate
}

var errCompletionUnapplicable = errors.New("completion unapplicable")

// completers is the list of all completers.
// TODO(xiaq): Make this list programmable.
var completers = []struct {
	name string
	completer
}{
	{"variable", complVariable},
	{"command name", complFormHead},
	{"argument", complArg},
}

func complVariable(n parse.Node, ev *eval.Evaler) (*compl, error) {
	begin, end := n.Begin(), n.End()

	primary, ok := n.(*parse.Primary)
	if !ok || primary.Type != parse.Variable {
		return nil, errCompletionUnapplicable
	}

	// XXX Repeats eval.ParseVariable.
	explode, qname := eval.ParseVariableSplice(primary.Value)
	nsPart, nameHead := eval.ParseVariableQName(qname)
	ns := nsPart
	if len(ns) > 0 {
		ns = ns[:len(ns)-1]
	}

	// Collect matching variables.
	var varnames []string
	iterateVariables(ev, ns, func(varname string) {
		if strings.HasPrefix(varname, nameHead) {
			varnames = append(varnames, nsPart+varname)
		}
	})
	// Collect namespace prefixes.
	// TODO Support non-module namespaces.
	for ns := range ev.Modules {
		if hasProperPrefix(ns+":", qname) {
			varnames = append(varnames, ns+":")
		}
	}
	sort.Strings(varnames)

	cands := make([]*candidate, len(varnames))
	// Build candidates.
	for i, varname := range varnames {
		cands[i] = newPlainCandidate("$" + explode + varname)
	}
	return &compl{begin, end, cands}, nil
}

func hasProperPrefix(s, p string) bool {
	return len(s) > len(p) && strings.HasPrefix(s, p)
}

func iterateVariables(ev *eval.Evaler, ns string, f func(string)) {
	switch ns {
	case "":
		for varname := range eval.Builtin() {
			f(varname)
		}
		for varname := range ev.Global {
			f(varname)
		}
		// TODO Include local names as well.
	case "E":
		for _, s := range os.Environ() {
			f(s[:strings.IndexByte(s, '=')])
		}
	default:
		// TODO Support non-module namespaces.
		for varname := range ev.Modules[ns] {
			f(varname)
		}
	}
}

func complFormHead(n parse.Node, ev *eval.Evaler) (*compl, error) {
	begin, end, head, q := findFormHeadContext(n)
	if begin == -1 {
		return nil, errCompletionUnapplicable
	}
	cands, err := complFormHeadInner(head, ev)
	if err != nil {
		return nil, err
	}
	quoteCandidates(cands, q)
	return &compl{begin, end, cands}, nil
}

func findFormHeadContext(n parse.Node) (int, int, string, parse.PrimaryType) {
	_, isChunk := n.(*parse.Chunk)
	_, isPipeline := n.(*parse.Pipeline)
	if isChunk || isPipeline {
		return n.Begin(), n.End(), "", parse.Bareword
	}

	if primary, ok := n.(*parse.Primary); ok {
		if compound, head := primaryInSimpleCompound(primary); compound != nil {
			if form, ok := compound.Parent().(*parse.Form); ok {
				if form.Head == compound {
					return compound.Begin(), compound.End(), head, primary.Type
				}
			}
		}
	}
	return -1, -1, "", 0
}

func complFormHeadInner(head string, ev *eval.Evaler) ([]*candidate, error) {
	if util.DontSearch(head) {
		return complFilenameInner(head, true)
	}

	var commands []string
	got := func(s string) {
		if strings.HasPrefix(s, head) {
			commands = append(commands, s)
		}
	}
	for special := range isBuiltinSpecial {
		got(special)
	}
	explode, ns, _ := eval.ParseVariable(head)
	if !explode {
		iterateVariables(ev, ns, func(varname string) {
			if strings.HasPrefix(varname, eval.FnPrefix) {
				got(eval.MakeVariableName(false, ns, varname[len(eval.FnPrefix):]))
			} else {
				got(eval.MakeVariableName(false, ns, varname) + "=")
			}
		})
	}
	ev.EachExternal(func(command string) {
		got(command)
		if strings.HasPrefix(head, "e:") {
			got("e:" + command)
		}
	})
	// TODO Support non-module namespaces.
	for ns := range ev.Modules {
		if head != ns+":" {
			got(ns + ":")
		}
	}
	sort.Strings(commands)

	cands := []*candidate{}
	for _, cmd := range commands {
		cands = append(cands, newPlainCandidate(cmd))
	}
	return cands, nil
}

// complArg completes arguments. It identifies the context and then delegates
// the actual completion work to a suitable completer.
func complArg(n parse.Node, ev *eval.Evaler) (*compl, error) {
	begin, end, current, q, form := findArgContext(n)
	if begin == -1 {
		return nil, errCompletionUnapplicable
	}

	// Find out head of the form and preceding arguments.
	// If Form.Head is not a simple compound, head will be "", just what we want.
	_, head, _ := simpleCompound(form.Head, nil)
	var args []string
	for _, compound := range form.Args {
		if compound.Begin() >= begin {
			break
		}
		ok, arg, _ := simpleCompound(compound, nil)
		if ok {
			// XXX Arguments that are not simple compounds are simply ignored.
			args = append(args, arg)
		}
	}

	words := make([]string, len(args)+2)
	words[0] = head
	words[len(words)-1] = current
	copy(words[1:len(words)-1], args[:])

	cands, err := completeArg(words, ev)
	if err != nil {
		return nil, err
	}
	quoteCandidates(cands, q)
	return &compl{begin, end, cands}, nil
}

func findArgContext(n parse.Node) (int, int, string, parse.PrimaryType, *parse.Form) {
	if sep, ok := n.(*parse.Sep); ok {
		if form, ok := sep.Parent().(*parse.Form); ok {
			return n.End(), n.End(), "", parse.Bareword, form
		}
	}
	if primary, ok := n.(*parse.Primary); ok {
		if compound, head := primaryInSimpleCompound(primary); compound != nil {
			if form, ok := compound.Parent().(*parse.Form); ok {
				if form.Head != compound {
					return compound.Begin(), compound.End(), head, primary.Type, form
				}
			}
		}
	}
	return -1, -1, "", 0, nil
}

// TODO: getStyle does redundant stats.
func complFilenameInner(head string, executableOnly bool) ([]*candidate, error) {
	dir, fileprefix := path.Split(head)
	if dir == "" {
		dir = "."
	}

	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, fmt.Errorf("cannot list directory %s: %v", dir, err)
	}

	cands := []*candidate{}
	lsColor := getLsColor()
	// Make candidates out of elements that match the file component.
	for _, info := range infos {
		name := info.Name()
		// Irrevelant file.
		if !strings.HasPrefix(name, fileprefix) {
			continue
		}
		// Hide dot files unless file starts with a dot.
		if !dotfile(fileprefix) && dotfile(name) {
			continue
		}
		// Only accept searchable directories and executable files if
		// executableOnly is true.
		if executableOnly && !(info.IsDir() || (info.Mode()&0111) != 0) {
			continue
		}

		// Full filename for source and getStyle.
		full := head + name[len(fileprefix):]

		suffix := " "
		if info.IsDir() {
			suffix = "/"
		} else if info.Mode()&os.ModeSymlink != 0 {
			stat, err := os.Stat(full)
			if err == nil && stat.IsDir() {
				// Symlink to directory.
				suffix = "/"
			}
		}

		cands = append(cands, &candidate{
			text: full, suffix: suffix,
			display: styled{name, stylesFromString(lsColor.getStyle(full))},
		})
	}

	return cands, nil
}

func quoteCandidates(cands []*candidate, q parse.PrimaryType) []*candidate {
	for _, cand := range cands {
		quoted, _ := parse.QuoteAs(cand.text, q)
		cand.text = quoted + cand.suffix
	}
	return cands
}

func dotfile(fname string) bool {
	return strings.HasPrefix(fname, ".")
}
<commit_msg>A trivial cleanup.<commit_after>package edit

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"sort"
	"strings"

	"github.com/elves/elvish/eval"
	"github.com/elves/elvish/parse"
	"github.com/elves/elvish/util"
)

// completer takes the current Node (always a leaf in the AST) and an Editor and
// returns a compl. If the completer does not apply to the type of the current
// Node, it should return an error of ErrCompletionUnapplicable.
type completer func(parse.Node, *eval.Evaler) (*compl, error)

// compl is the result of a completer, meaning that any of the candidates can
// replace the text in the interval [begin, end).
type compl struct {
	begin int
	end   int
	cands []*candidate
}

var errCompletionUnapplicable = errors.New("completion unapplicable")

// completers is the list of all completers.
// TODO(xiaq): Make this list programmable.
var completers = []struct {
	name string
	completer
}{
	{"variable", complVariable},
	{"command name", complFormHead},
	{"argument", complArg},
}

func complVariable(n parse.Node, ev *eval.Evaler) (*compl, error) {
	primary, ok := n.(*parse.Primary)
	if !ok || primary.Type != parse.Variable {
		return nil, errCompletionUnapplicable
	}

	// XXX Repeats eval.ParseVariable.
	explode, qname := eval.ParseVariableSplice(primary.Value)
	nsPart, nameHead := eval.ParseVariableQName(qname)
	ns := nsPart
	if len(ns) > 0 {
		ns = ns[:len(ns)-1]
	}

	// Collect matching variables.
	var varnames []string
	iterateVariables(ev, ns, func(varname string) {
		if strings.HasPrefix(varname, nameHead) {
			varnames = append(varnames, nsPart+varname)
		}
	})
	// Collect namespace prefixes.
	// TODO Support non-module namespaces.
	for ns := range ev.Modules {
		if hasProperPrefix(ns+":", qname) {
			varnames = append(varnames, ns+":")
		}
	}
	sort.Strings(varnames)

	cands := make([]*candidate, len(varnames))
	// Build candidates.
	for i, varname := range varnames {
		cands[i] = newPlainCandidate("$" + explode + varname)
	}
	return &compl{n.Begin(), n.End(), cands}, nil
}

func hasProperPrefix(s, p string) bool {
	return len(s) > len(p) && strings.HasPrefix(s, p)
}

func iterateVariables(ev *eval.Evaler, ns string, f func(string)) {
	switch ns {
	case "":
		for varname := range eval.Builtin() {
			f(varname)
		}
		for varname := range ev.Global {
			f(varname)
		}
		// TODO Include local names as well.
	case "E":
		for _, s := range os.Environ() {
			f(s[:strings.IndexByte(s, '=')])
		}
	default:
		// TODO Support non-module namespaces.
		for varname := range ev.Modules[ns] {
			f(varname)
		}
	}
}

func complFormHead(n parse.Node, ev *eval.Evaler) (*compl, error) {
	begin, end, head, q := findFormHeadContext(n)
	if begin == -1 {
		return nil, errCompletionUnapplicable
	}
	cands, err := complFormHeadInner(head, ev)
	if err != nil {
		return nil, err
	}
	quoteCandidates(cands, q)
	return &compl{begin, end, cands}, nil
}

func findFormHeadContext(n parse.Node) (int, int, string, parse.PrimaryType) {
	_, isChunk := n.(*parse.Chunk)
	_, isPipeline := n.(*parse.Pipeline)
	if isChunk || isPipeline {
		return n.Begin(), n.End(), "", parse.Bareword
	}

	if primary, ok := n.(*parse.Primary); ok {
		if compound, head := primaryInSimpleCompound(primary); compound != nil {
			if form, ok := compound.Parent().(*parse.Form); ok {
				if form.Head == compound {
					return compound.Begin(), compound.End(), head, primary.Type
				}
			}
		}
	}
	return -1, -1, "", 0
}

func complFormHeadInner(head string, ev *eval.Evaler) ([]*candidate, error) {
	if util.DontSearch(head) {
		return complFilenameInner(head, true)
	}

	var commands []string
	got := func(s string) {
		if strings.HasPrefix(s, head) {
			commands = append(commands, s)
		}
	}
	for special := range isBuiltinSpecial {
		got(special)
	}
	explode, ns, _ := eval.ParseVariable(head)
	if !explode {
		iterateVariables(ev, ns, func(varname string) {
			if strings.HasPrefix(varname, eval.FnPrefix) {
				got(eval.MakeVariableName(false, ns, varname[len(eval.FnPrefix):]))
			} else {
				got(eval.MakeVariableName(false, ns, varname) + "=")
			}
		})
	}
	ev.EachExternal(func(command string) {
		got(command)
		if strings.HasPrefix(head, "e:") {
			got("e:" + command)
		}
	})
	// TODO Support non-module namespaces.
	for ns := range ev.Modules {
		if head != ns+":" {
			got(ns + ":")
		}
	}
	sort.Strings(commands)

	cands := []*candidate{}
	for _, cmd := range commands {
		cands = append(cands, newPlainCandidate(cmd))
	}
	return cands, nil
}

// complArg completes arguments. It identifies the context and then delegates
// the actual completion work to a suitable completer.
func complArg(n parse.Node, ev *eval.Evaler) (*compl, error) {
	begin, end, current, q, form := findArgContext(n)
	if begin == -1 {
		return nil, errCompletionUnapplicable
	}

	// Find out head of the form and preceding arguments.
	// If Form.Head is not a simple compound, head will be "", just what we want.
	_, head, _ := simpleCompound(form.Head, nil)
	var args []string
	for _, compound := range form.Args {
		if compound.Begin() >= begin {
			break
		}
		ok, arg, _ := simpleCompound(compound, nil)
		if ok {
			// XXX Arguments that are not simple compounds are simply ignored.
			args = append(args, arg)
		}
	}

	words := make([]string, len(args)+2)
	words[0] = head
	words[len(words)-1] = current
	copy(words[1:len(words)-1], args[:])

	cands, err := completeArg(words, ev)
	if err != nil {
		return nil, err
	}
	quoteCandidates(cands, q)
	return &compl{begin, end, cands}, nil
}

func findArgContext(n parse.Node) (int, int, string, parse.PrimaryType, *parse.Form) {
	if sep, ok := n.(*parse.Sep); ok {
		if form, ok := sep.Parent().(*parse.Form); ok {
			return n.End(), n.End(), "", parse.Bareword, form
		}
	}
	if primary, ok := n.(*parse.Primary); ok {
		if compound, head := primaryInSimpleCompound(primary); compound != nil {
			if form, ok := compound.Parent().(*parse.Form); ok {
				if form.Head != compound {
					return compound.Begin(), compound.End(), head, primary.Type, form
				}
			}
		}
	}
	return -1, -1, "", 0, nil
}

// TODO: getStyle does redundant stats.
func complFilenameInner(head string, executableOnly bool) ([]*candidate, error) {
	dir, fileprefix := path.Split(head)
	if dir == "" {
		dir = "."
	}

	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, fmt.Errorf("cannot list directory %s: %v", dir, err)
	}

	cands := []*candidate{}
	lsColor := getLsColor()
	// Make candidates out of elements that match the file component.
	for _, info := range infos {
		name := info.Name()
		// Irrevelant file.
		if !strings.HasPrefix(name, fileprefix) {
			continue
		}
		// Hide dot files unless file starts with a dot.
		if !dotfile(fileprefix) && dotfile(name) {
			continue
		}
		// Only accept searchable directories and executable files if
		// executableOnly is true.
		if executableOnly && !(info.IsDir() || (info.Mode()&0111) != 0) {
			continue
		}

		// Full filename for source and getStyle.
		full := head + name[len(fileprefix):]

		suffix := " "
		if info.IsDir() {
			suffix = "/"
		} else if info.Mode()&os.ModeSymlink != 0 {
			stat, err := os.Stat(full)
			if err == nil && stat.IsDir() {
				// Symlink to directory.
				suffix = "/"
			}
		}

		cands = append(cands, &candidate{
			text: full, suffix: suffix,
			display: styled{name, stylesFromString(lsColor.getStyle(full))},
		})
	}

	return cands, nil
}

func quoteCandidates(cands []*candidate, q parse.PrimaryType) []*candidate {
	for _, cand := range cands {
		quoted, _ := parse.QuoteAs(cand.text, q)
		cand.text = quoted + cand.suffix
	}
	return cands
}

func dotfile(fname string) bool {
	return strings.HasPrefix(fname, ".")
}
<|endoftext|>


<commit_before>// +build integ
// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package telemetry

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"
	"testing"
	"time"

	promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
	"github.com/prometheus/common/model"
	"golang.org/x/sync/errgroup"
	kubeApiMeta "k8s.io/apimachinery/pkg/apis/meta/v1"

	"istio.io/istio/pkg/config/protocol"
	"istio.io/istio/pkg/test/env"
	"istio.io/istio/pkg/test/framework"
	"istio.io/istio/pkg/test/framework/components/echo"
	"istio.io/istio/pkg/test/framework/components/echo/echoboot"
	"istio.io/istio/pkg/test/framework/components/namespace"
	"istio.io/istio/pkg/test/framework/components/prometheus"
	"istio.io/istio/pkg/test/framework/resource"
	"istio.io/istio/pkg/test/scopes"
	"istio.io/istio/pkg/test/util/retry"
	"istio.io/istio/pkg/test/util/yml"
	"istio.io/pkg/log"
)

var (
	dashboards = []struct {
		configmap      string
		name           string
		excluded       []string
		requirePrimary bool
	}{
		{
			"istio-grafana-dashboards",
			"pilot-dashboard.json",
			[]string{
				"pilot_xds_push_errors",
				"pilot_total_xds_internal_errors",
				"pilot_xds_push_context_errors",
				`pilot_xds_pushes{type!~"lds|cds|rds|eds"}`,
				// We do not push credentials in this test
				`pilot_xds_pushes{type="sds"}`,
				"_timeout",
				"_rejects",
				// We do not simulate injection errors
				"sidecar_injection_failure_total",
				// In default install, we have no proxy
				"istio-proxy",
				// https://github.com/istio/istio/issues/22674 this causes flaky tests
				"galley_validation_passed",
				// cAdvisor does not expose this metrics, and we don't have kubelet in kind
				"container_fs_usage_bytes",
			},
			// Pilot is installed only on Primary cluster, hence validate for primary clusters only.
			true,
		},
		{
			"istio-services-grafana-dashboards",
			"istio-mesh-dashboard.json",
			[]string{
				"galley_",
				"istio_tcp_",
				"max(pilot_k8s_cfg_events{",
			},
			false,
		},
		{
			"istio-services-grafana-dashboards",
			"istio-service-dashboard.json",
			[]string{
				"istio_tcp_",
			},
			false,
		},
		{
			"istio-services-grafana-dashboards",
			"istio-workload-dashboard.json",
			[]string{
				"istio_tcp_",
			},
			false,
		},
		{
			"istio-grafana-dashboards",
			"istio-performance-dashboard.json",
			[]string{
				// TODO add these back: https://github.com/istio/istio/issues/20175
				`istio-telemetry`,
				`istio-policy`,
				// cAdvisor does not expose this metrics, and we don't have kubelet in kind
				"container_fs_usage_bytes",
			},
			true,
		},
		{
			"istio-services-grafana-dashboards",
			"istio-extension-dashboard.json",
			[]string{
				"avg(envoy_wasm_vm_v8_",
			},
			false,
		},
	}
)

func TestDashboard(t *testing.T) {
	framework.NewTest(t).
		Features("observability.telemetry.dashboard").
		Run(func(ctx framework.TestContext) {

			p := prometheus.NewOrFail(ctx, ctx, prometheus.Config{})
			setupDashboardTest(ctx)
			waitForMetrics(ctx, p)
			for _, d := range dashboards {
				d := d
				ctx.NewSubTest(d.name).RunParallel(func(t framework.TestContext) {
					for _, cl := range ctx.Clusters() {
						if !cl.IsPrimary() && d.requirePrimary {
							// Skip verification of dashboards that won't be present on non primary(remote) clusters.
							continue
						}
						t.Logf("Verifying %s for cluster %s", d.name, cl.Name())
						cm, err := cl.CoreV1().ConfigMaps(i.Settings().TelemetryNamespace).Get(
							context.TODO(), d.configmap, kubeApiMeta.GetOptions{})
						if err != nil {
							t.Fatalf("Failed to find dashboard %v: %v", d.configmap, err)
						}

						config, f := cm.Data[d.name]
						if !f {
							t.Fatalf("Failed to find expected dashboard: %v", d.name)
						}

						queries, err := extractQueries(config)
						if err != nil {
							t.Fatalf("Failed to extract queries: %v", err)
						}

						for _, query := range queries {
							if err := checkMetric(cl, p, query, d.excluded); err != nil {
								t.Errorf("Check query failed for cluster %s: %v", cl.Name(), err)
							}
						}
					}
				})
			}
		})
}

var (
	// Some templates use replacement variables. Instead, replace those with wildcard
	replacer = strings.NewReplacer(
		"$dstns", ".*",
		"$dstwl", ".*",
		"$service", ".*",
		"$srcns", ".*",
		"$srcwl", ".*",
		"$namespace", ".*",
		"$workload", ".*",
		"$dstsvc", ".*",
		"$adapter", ".*",
		// Just allow all mTLS settings rather than trying to send mtls and plaintext
		`connection_security_policy="unknown"`, `connection_security_policy=~".*"`,
		`connection_security_policy="mutual_tls"`, `connection_security_policy=~".*"`,
		`connection_security_policy!="mutual_tls"`, `connection_security_policy=~".*"`,
		// Test runs in istio-system
		`destination_workload_namespace!="istio-system"`, `destination_workload_namespace=~".*"`,
		`source_workload_namespace!="istio-system"`, `source_workload_namespace=~".*"`,
	)
)

func checkMetric(cl resource.Cluster, p prometheus.Instance, query string, excluded []string) error {
	query = replacer.Replace(query)
	value, _, err := p.APIForCluster(cl).QueryRange(context.Background(), query, promv1.Range{
		Start: time.Now().Add(-time.Minute),
		End:   time.Now(),
		Step:  time.Second,
	})
	if err != nil {
		return fmt.Errorf("failure executing query (%s): %v", query, err)
	}
	if value == nil {
		return fmt.Errorf("returned value should not be nil for '%s'", query)
	}
	numSamples := 0
	switch v := value.(type) {
	case model.Vector:
		numSamples = v.Len()
	case model.Matrix:
		numSamples = v.Len()
	case *model.Scalar:
		numSamples = 1
	default:
		return fmt.Errorf("unknown metric value type: %T", v)
	}
	if includeQuery(query, excluded) {
		if numSamples == 0 {
			return fmt.Errorf("expected a metric value for '%s', found no samples: %#v", query, value)
		}
	} else {
		if numSamples != 0 {
			scopes.Framework.Infof("Filtered out metric '%v', but got samples: %v", query, value)
		}
	}
	return nil
}

// nolint: interfacer
func waitForMetrics(t framework.TestContext, instance prometheus.Instance) {
	// These are sentinel metrics that will be used to evaluate if prometheus
	// scraping has occurred and data is available via promQL.
	// We will retry these queries, but not future ones, otherwise failures will take too long
	queries := []string{
		`istio_requests_total`,
		`istio_tcp_received_bytes_total`,
	}

	for _, cl := range t.Clusters() {
		for _, query := range queries {
			err := retry.UntilSuccess(func() error {
				return checkMetric(cl, instance, query, nil)
			})
			// Do not fail here - this is just to let the metrics sync. We will fail on the test if query fails
			if err != nil {
				t.Logf("Sentinel query %v failed: %v", query, err)
			}
		}
	}
}

const gatewayConfig = `
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
  name: echo-gateway
spec:
  selector:
    istio: ingressgateway
  servers:
  - port:
      number: 80
      name: http
      protocol: HTTP
    hosts:
    - "*"
  - port:
      number: 31400
      name: tcp
      protocol: TCP
    hosts:
    - "*"
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: echo
spec:
  hosts:
  - "*"
  gateways:
  - echo-gateway
  http:
  - match:
    - uri:
        exact: /echo-%s
    route:
    - destination:
        host: server
        port:
          number: 80
  tcp:
  - match:
    - port: 31400
    route:
    - destination:
        host: server
        port:
          number: 7777
`

func setupDashboardTest(t framework.TestContext) {
	ns := namespace.NewOrFail(t, t, namespace.Config{
		Prefix: "dashboard",
		Inject: true,
	})
	t.Config().ApplyYAMLOrFail(t, ns.Name(), fmt.Sprintf(gatewayConfig, ns.Name()))

	// Apply just the grafana dashboards
	cfg, err := ioutil.ReadFile(filepath.Join(env.IstioSrc, "samples/addons/grafana.yaml"))
	if err != nil {
		t.Fatal(err)
	}
	t.Config().ApplyYAMLOrFail(t, "istio-system", yml.SplitYamlByKind(string(cfg))["ConfigMap"])

	for _, cl := range t.Clusters() {
		var instance echo.Instance
		echoboot.
			NewBuilder(t).
			With(&instance, echo.Config{
				Service:   "server",
				Cluster:   cl,
				Namespace: ns,
				Subsets:   []echo.SubsetConfig{{}},
				Ports: []echo.Port{
					{
						Name:     "http",
						Protocol: protocol.HTTP,
						// We use a port > 1024 to not require root
						InstancePort: 8090,
					},
					{
						Name:         "tcp",
						Protocol:     protocol.TCP,
						InstancePort: 7777,
						ServicePort:  7777,
					},
				},
			}).
			BuildOrFail(t)
	}
	for _, ing := range ingr {
		// Send 200 http requests, 20 tcp requests across goroutines, generating a variety of error codes.
		// Spread out over 5s so rate() queries will behave correctly
		g, _ := errgroup.WithContext(context.Background())
		tcpAddr := ing.TCPAddress()
		ticker := time.NewTicker(time.Second * 5)
		for t := 0; t < 20; t++ {
			<-ticker.C
			g.Go(func() error {
				for i := 0; i < 10; i++ {
					_, err := ing.CallEcho(echo.CallOptions{
						Port: &echo.Port{
							Protocol: protocol.HTTP,
						},
						Path: fmt.Sprintf("/echo-%s?codes=418:10,520:15,200:75", ns.Name()),
						Headers: map[string][]string{
							"Host": {"server"},
						},
					})
					if err != nil {
						// Do not fail on errors since there may be initial startup errors
						// These calls are not under tests, the dashboards are, so we can be leniant here
						log.Warnf("requests failed: %v", err)
					}
				}
				_, err := ing.CallEcho(echo.CallOptions{
					Port: &echo.Port{
						Protocol:    protocol.HTTP,
						ServicePort: tcpAddr.Port,
					},
					Address: tcpAddr.IP.String(),
					Path:    fmt.Sprintf("/echo-%s", ns.Name()),
					Headers: map[string][]string{
						"Host": {"server"},
					},
				})
				if err != nil {
					// Do not fail on errors since there may be initial startup errors
					// These calls are not under tests, the dashboards are, so we can be leniant here
					log.Warnf("requests failed: %v", err)
				}
				return nil
			})
		}
		if err := g.Wait(); err != nil {
			t.Fatalf("ingress call failed: %v", err)
		}
	}
}

// extractQueries pulls all prometheus queries out of a grafana dashboard
// Rather than importing the entire grafana API just for this test, do some shoddy json parsing
// Equivalent to jq command: '.panels[].targets[]?.expr'
func extractQueries(dash string) ([]string, error) {
	var queries []string
	js := map[string]interface{}{}
	if err := json.Unmarshal([]byte(dash), &js); err != nil {
		return nil, err
	}
	panels, f := js["panels"]
	if !f {
		return nil, fmt.Errorf("failed to find panels in %v", dash)
	}
	panelsList, f := panels.([]interface{})
	if !f {
		return nil, fmt.Errorf("failed to find panelsList in type %T: %v", panels, panels)
	}
	for _, p := range panelsList {
		pm := p.(map[string]interface{})
		targets, f := pm["targets"]
		if !f {
			continue
		}
		targetsList, f := targets.([]interface{})
		if !f {
			return nil, fmt.Errorf("failed to find targetsList in type %T: %v", targets, targets)
		}
		for _, t := range targetsList {
			tm := t.(map[string]interface{})
			expr, f := tm["expr"]
			if !f {
				return nil, fmt.Errorf("failed to find expr in %v", t)
			}
			queries = append(queries, expr.(string))
		}
	}
	return queries, nil
}

func includeQuery(query string, excluded []string) bool {
	for _, f := range excluded {
		if strings.Contains(query, f) {
			return false
		}
	}
	return true
}
<commit_msg>Improve dashboard test (#28963)<commit_after>// +build integ
// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package telemetry

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"
	"testing"
	"time"

	promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
	"github.com/prometheus/common/model"
	"golang.org/x/sync/errgroup"
	kubeApiMeta "k8s.io/apimachinery/pkg/apis/meta/v1"

	"istio.io/istio/pkg/config/protocol"
	"istio.io/istio/pkg/test/env"
	"istio.io/istio/pkg/test/framework"
	"istio.io/istio/pkg/test/framework/components/echo"
	"istio.io/istio/pkg/test/framework/components/echo/echoboot"
	"istio.io/istio/pkg/test/framework/components/namespace"
	"istio.io/istio/pkg/test/framework/components/prometheus"
	"istio.io/istio/pkg/test/framework/resource"
	"istio.io/istio/pkg/test/scopes"
	"istio.io/istio/pkg/test/util/retry"
	"istio.io/istio/pkg/test/util/yml"
	"istio.io/pkg/log"
)

var (
	dashboards = []struct {
		configmap      string
		name           string
		excluded       []string
		requirePrimary bool
	}{
		{
			"istio-grafana-dashboards",
			"pilot-dashboard.json",
			[]string{
				"pilot_xds_push_errors",
				"pilot_total_xds_internal_errors",
				"pilot_xds_push_context_errors",
				`pilot_xds_pushes{type!~"lds|cds|rds|eds"}`,
				// We do not push credentials in this test
				`pilot_xds_pushes{type="sds"}`,
				"_timeout",
				"_rejects",
				// We do not simulate injection errors
				"sidecar_injection_failure_total",
				// In default install, we have no proxy
				"istio-proxy",
				// https://github.com/istio/istio/issues/22674 this causes flaky tests
				"galley_validation_passed",
				// cAdvisor does not expose this metrics, and we don't have kubelet in kind
				"container_fs_usage_bytes",
			},
			// Pilot is installed only on Primary cluster, hence validate for primary clusters only.
			true,
		},
		{
			"istio-services-grafana-dashboards",
			"istio-mesh-dashboard.json",
			[]string{
				"galley_",
				"istio_tcp_",
				"max(pilot_k8s_cfg_events{",
			},
			false,
		},
		{
			"istio-services-grafana-dashboards",
			"istio-service-dashboard.json",
			[]string{
				"istio_tcp_",
			},
			false,
		},
		{
			"istio-services-grafana-dashboards",
			"istio-workload-dashboard.json",
			[]string{
				"istio_tcp_",
			},
			false,
		},
		{
			"istio-grafana-dashboards",
			"istio-performance-dashboard.json",
			[]string{
				// TODO add these back: https://github.com/istio/istio/issues/20175
				`istio-telemetry`,
				`istio-policy`,
				// cAdvisor does not expose this metrics, and we don't have kubelet in kind
				"container_fs_usage_bytes",
			},
			true,
		},
		{
			"istio-services-grafana-dashboards",
			"istio-extension-dashboard.json",
			[]string{
				"avg(envoy_wasm_vm_v8_",
			},
			false,
		},
	}
)

func TestDashboard(t *testing.T) {
	framework.NewTest(t).
		Features("observability.telemetry.dashboard").
		Run(func(ctx framework.TestContext) {

			p := prometheus.NewOrFail(ctx, ctx, prometheus.Config{})
			setupDashboardTest(ctx)
			waitForMetrics(ctx, p)
			for _, d := range dashboards {
				d := d
				ctx.NewSubTest(d.name).RunParallel(func(t framework.TestContext) {
					for _, cl := range ctx.Clusters() {
						if !cl.IsPrimary() && d.requirePrimary {
							// Skip verification of dashboards that won't be present on non primary(remote) clusters.
							continue
						}
						t.Logf("Verifying %s for cluster %s", d.name, cl.Name())
						cm, err := cl.CoreV1().ConfigMaps(i.Settings().TelemetryNamespace).Get(
							context.TODO(), d.configmap, kubeApiMeta.GetOptions{})
						if err != nil {
							t.Fatalf("Failed to find dashboard %v: %v", d.configmap, err)
						}

						config, f := cm.Data[d.name]
						if !f {
							t.Fatalf("Failed to find expected dashboard: %v", d.name)
						}

						queries, err := extractQueries(config)
						if err != nil {
							t.Fatalf("Failed to extract queries: %v", err)
						}

						for _, query := range queries {
							if err := checkMetric(cl, p, query, d.excluded); err != nil {
								t.Errorf("Check query failed for cluster %s: %v", cl.Name(), err)
							}
						}
					}
				})
			}
		})
}

var (
	// Some templates use replacement variables. Instead, replace those with wildcard
	replacer = strings.NewReplacer(
		"$dstns", ".*",
		"$dstwl", ".*",
		"$service", ".*",
		"$srcns", ".*",
		"$srcwl", ".*",
		"$namespace", ".*",
		"$workload", ".*",
		"$dstsvc", ".*",
		"$adapter", ".*",
		// Just allow all mTLS settings rather than trying to send mtls and plaintext
		`connection_security_policy="unknown"`, `connection_security_policy=~".*"`,
		`connection_security_policy="mutual_tls"`, `connection_security_policy=~".*"`,
		`connection_security_policy!="mutual_tls"`, `connection_security_policy=~".*"`,
		// Test runs in istio-system
		`destination_workload_namespace!="istio-system"`, `destination_workload_namespace=~".*"`,
		`source_workload_namespace!="istio-system"`, `source_workload_namespace=~".*"`,
	)
)

func checkMetric(cl resource.Cluster, p prometheus.Instance, query string, excluded []string) error {
	query = replacer.Replace(query)
	value, _, err := p.APIForCluster(cl).QueryRange(context.Background(), query, promv1.Range{
		Start: time.Now().Add(-time.Minute),
		End:   time.Now(),
		Step:  time.Second,
	})
	if err != nil {
		return fmt.Errorf("failure executing query (%s): %v", query, err)
	}
	if value == nil {
		return fmt.Errorf("returned value should not be nil for '%s'", query)
	}
	numSamples := 0
	switch v := value.(type) {
	case model.Vector:
		numSamples = v.Len()
	case model.Matrix:
		numSamples = v.Len()
	case *model.Scalar:
		numSamples = 1
	default:
		return fmt.Errorf("unknown metric value type: %T", v)
	}
	if includeQuery(query, excluded) {
		if numSamples == 0 {
			return fmt.Errorf("expected a metric value for '%s', found no samples: %#v", query, value)
		}
	} else {
		if numSamples != 0 {
			scopes.Framework.Infof("Filtered out metric '%v', but got samples: %v", query, value)
		}
	}
	return nil
}

// nolint: interfacer
func waitForMetrics(t framework.TestContext, instance prometheus.Instance) {
	// These are sentinel metrics that will be used to evaluate if prometheus
	// scraping has occurred and data is available via promQL.
	// We will retry these queries, but not future ones, otherwise failures will take too long
	queries := []string{
		`istio_requests_total`,
		`istio_tcp_received_bytes_total`,
	}

	for _, cl := range t.Clusters() {
		for _, query := range queries {
			err := retry.UntilSuccess(func() error {
				return checkMetric(cl, instance, query, nil)
			})
			// Do not fail here - this is just to let the metrics sync. We will fail on the test if query fails
			if err != nil {
				t.Logf("Sentinel query %v failed: %v", query, err)
			}
		}
	}
}

const gatewayConfig = `
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
  name: echo-gateway
spec:
  selector:
    istio: ingressgateway
  servers:
  - port:
      number: 80
      name: http
      protocol: HTTP
    hosts:
    - "*"
  - port:
      number: 31400
      name: tcp
      protocol: TCP
    hosts:
    - "*"
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: echo
spec:
  hosts:
  - "*"
  gateways:
  - echo-gateway
  http:
  - match:
    - uri:
        exact: /echo-%s
    route:
    - destination:
        host: server
        port:
          number: 80
  tcp:
  - match:
    - port: 31400
    route:
    - destination:
        host: server
        port:
          number: 7777
`

func setupDashboardTest(t framework.TestContext) {
	ns := namespace.NewOrFail(t, t, namespace.Config{
		Prefix: "dashboard",
		Inject: true,
	})
	t.Config().ApplyYAMLOrFail(t, ns.Name(), fmt.Sprintf(gatewayConfig, ns.Name()))

	// Apply just the grafana dashboards
	cfg, err := ioutil.ReadFile(filepath.Join(env.IstioSrc, "samples/addons/grafana.yaml"))
	if err != nil {
		t.Fatal(err)
	}
	t.Config().ApplyYAMLOrFail(t, "istio-system", yml.SplitYamlByKind(string(cfg))["ConfigMap"])

	for _, cl := range t.Clusters() {
		var instance echo.Instance
		echoboot.
			NewBuilder(t).
			With(&instance, echo.Config{
				Service:   "server",
				Cluster:   cl,
				Namespace: ns,
				Subsets:   []echo.SubsetConfig{{}},
				Ports: []echo.Port{
					{
						Name:     "http",
						Protocol: protocol.HTTP,
						// We use a port > 1024 to not require root
						InstancePort: 8090,
					},
					{
						Name:         "tcp",
						Protocol:     protocol.TCP,
						InstancePort: 7777,
						ServicePort:  7777,
					},
				},
			}).
			BuildOrFail(t)
	}
	// Send 200 http requests, 20 tcp requests across goroutines, generating a variety of error codes.
	// Spread out over 20s so rate() queries will behave correctly
	g, _ := errgroup.WithContext(context.Background())
	ticker := time.NewTicker(time.Second)
	for t := 0; t < 20; t++ {
		<-ticker.C
		g.Go(func() error {
			for _, ing := range ingr {
				tcpAddr := ing.TCPAddress()
				for i := 0; i < 10; i++ {
					_, err := ing.CallEcho(echo.CallOptions{
						Port: &echo.Port{
							Protocol: protocol.HTTP,
						},
						Path: fmt.Sprintf("/echo-%s?codes=418:10,520:15,200:75", ns.Name()),
						Headers: map[string][]string{
							"Host": {"server"},
						},
					})
					if err != nil {
						// Do not fail on errors since there may be initial startup errors
						// These calls are not under tests, the dashboards are, so we can be leniant here
						log.Warnf("requests failed: %v", err)
					}
				}
				_, err := ing.CallEcho(echo.CallOptions{
					Port: &echo.Port{
						Protocol:    protocol.TCP,
						ServicePort: tcpAddr.Port,
					},
					Address: tcpAddr.IP.String(),
					Path:    fmt.Sprintf("/echo-%s", ns.Name()),
					Headers: map[string][]string{
						"Host": {"server"},
					},
				})
				if err != nil {
					// Do not fail on errors since there may be initial startup errors
					// These calls are not under tests, the dashboards are, so we can be leniant here
					log.Warnf("requests failed: %v", err)
				}
			}
			return nil
		})
	}
	if err := g.Wait(); err != nil {
		t.Fatalf("ingress call failed: %v", err)
	}
}

// extractQueries pulls all prometheus queries out of a grafana dashboard
// Rather than importing the entire grafana API just for this test, do some shoddy json parsing
// Equivalent to jq command: '.panels[].targets[]?.expr'
func extractQueries(dash string) ([]string, error) {
	var queries []string
	js := map[string]interface{}{}
	if err := json.Unmarshal([]byte(dash), &js); err != nil {
		return nil, err
	}
	panels, f := js["panels"]
	if !f {
		return nil, fmt.Errorf("failed to find panels in %v", dash)
	}
	panelsList, f := panels.([]interface{})
	if !f {
		return nil, fmt.Errorf("failed to find panelsList in type %T: %v", panels, panels)
	}
	for _, p := range panelsList {
		pm := p.(map[string]interface{})
		targets, f := pm["targets"]
		if !f {
			continue
		}
		targetsList, f := targets.([]interface{})
		if !f {
			return nil, fmt.Errorf("failed to find targetsList in type %T: %v", targets, targets)
		}
		for _, t := range targetsList {
			tm := t.(map[string]interface{})
			expr, f := tm["expr"]
			if !f {
				return nil, fmt.Errorf("failed to find expr in %v", t)
			}
			queries = append(queries, expr.(string))
		}
	}
	return queries, nil
}

func includeQuery(query string, excluded []string) bool {
	for _, f := range excluded {
		if strings.Contains(query, f) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"github.com/HouzuoGuo/laitos/misc"
	"github.com/HouzuoGuo/laitos/toolbox"
	"strconv"
	"sync"
	"time"
)

const (
	TimerCommandTimeoutSec = 10 // TimerCommandTimeoutSec is a hard coded timeout number constraining all commands run by timer.
)

/*
CommandTimer executes series of commands, one at a time, at regular interval. Execution results of recent commands are
memorised and can be retrieved at a later time. Beyond command execution results, arbitrary text messages may also be
memorised and retrieved together with command results. CommandTimer is a useful structure for implementing notification
kind of mechanism.
*/
type CommandTimer struct {
	// PreConfiguredCommands are toolbox commands pre-configured to run by user, they never deleted upon clearing.
	PreConfiguredCommands []string `json:"PreConfiguredCommands"`
	// IntervalSec is the number of seconds to sleep between execution of all commands.
	IntervalSec int `json:"IntervalSec"`
	// MaxResults is the maximum number of results to memorise from command execution and text messages.
	MaxResults int `json:"MaxResults"`
	// CommandProcessor is the one going to run all commands.
	CommandProcessor *CommandProcessor `json:"-"`

	/*
		transientCommands are new commands that are added on the fly and can be cleared by calling a function.
		During trigger, these commands are executed after the pre-configured commands.
	*/
	transientCommands []string
	results           *misc.RingBuffer // results are the most recent command results and test messages to retrieve.
	mutex             sync.Mutex       // mutex prevents concurrent access to internal structures.
	running           bool             // running becomes true when command processing loop is running
	stop              chan struct{}    // stop channel signals Run function to return soon.
}

// Initialise prepares internal states of a new CommandTimer.
func (timer *CommandTimer) Initialise() error {
	if timer.IntervalSec < 1 {
		return fmt.Errorf("CommandTimer.Initialise: IntervalSec must be greater than 0")
	}
	if timer.MaxResults < 1 {
		return fmt.Errorf("CommandTimer.Initialise: MaxResults must be greater than 0")
	}
	timer.results = misc.NewRingBuffer(int64(timer.MaxResults))
	timer.transientCommands = make([]string, 0, 10)
	timer.stop = make(chan struct{})
	return nil
}

/*
GetTransientCommands returns a copy of all transient commands memorises for execution. If there is none, it returns
an empty string array.
*/
func (timer *CommandTimer) GetTransientCommands() []string {
	timer.mutex.Lock()
	defer timer.mutex.Unlock()
	ret := make([]string, len(timer.transientCommands))
	copy(ret, timer.transientCommands)
	return ret
}

// AddTransientCommand places a new toolbox command toward the end of transient command list.
func (timer *CommandTimer) AddTransientCommand(cmd string) {
	timer.mutex.Lock()
	defer timer.mutex.Unlock()
	timer.transientCommands = append(timer.transientCommands, cmd)
}

// ClearTransientCommands removes all transient commands.
func (timer *CommandTimer) ClearTransientCommands() {
	timer.mutex.Lock()
	defer timer.mutex.Unlock()
	timer.transientCommands = make([]string, 0, 10)
}

// runAllCommands executes all pre-configured and transient commands one after another and store their results.
func (timer *CommandTimer) runAllCommands() {
	//	Access to the commands array is not protected by mutex since no other function modifies it
	if timer.PreConfiguredCommands != nil {
		for _, cmd := range timer.PreConfiguredCommands {
			timer.results.Push(timer.CommandProcessor.Process(toolbox.Command{
				TimeoutSec: TimerCommandTimeoutSec,
				Content:    cmd,
			}).CombinedOutput)
		}
	}
	// Make a copy of the latest transient commands to run
	timer.mutex.Lock()
	transientCommands := make([]string, len(timer.transientCommands))
	copy(transientCommands, timer.transientCommands)
	timer.mutex.Unlock()
	// Run transient commands one after another
	for _, cmd := range transientCommands {
		timer.results.Push(timer.CommandProcessor.Process(toolbox.Command{
			TimeoutSec: TimerCommandTimeoutSec,
			Content:    cmd,
		}).CombinedOutput)
	}
}

/*
Start runs an infinite loop to execute all commands one after another, then sleep for an interval.
The function blocks caller until Stop function is called.
If Start function is already running, calling it a second time will do nothing and return immediately.
*/
func (timer *CommandTimer) Start() {
	timer.mutex.Lock()
	if timer.running {
		misc.DefaultLogger.Warning("CommandTimer.Start", strconv.Itoa(timer.IntervalSec), nil, "starting an already started CommandTimer becomes a nop")
		timer.mutex.Unlock()
		return
	}
	timer.mutex.Unlock()
	misc.DefaultLogger.Info("CommandTimer.Start", strconv.Itoa(timer.IntervalSec), nil, "timer has started")
	for {
		timer.running = true
		select {
		case <-time.After(time.Duration(timer.IntervalSec) * time.Second):
			timer.runAllCommands()
		case <-timer.stop:
			return
		}
	}
}

/*
Stop informs the running command processing loop to terminate as early as possible. Blocks until the loop has
terminated. Calling the function while command processing loop is not running yields no effect.
*/
func (timer *CommandTimer) Stop() {
	timer.mutex.Lock()
	if timer.running {
		timer.stop <- struct{}{}
		timer.running = false
	}
	timer.mutex.Unlock()
}

// AddArbitraryTextToResult simply places an arbitrary text string into result.
func (timer *CommandTimer) AddArbitraryTextToResult(text string) {
	// RingBuffer supports concurrent push access, there is no need to protect it with timer's own mutex.
	timer.results.Push(text)
}

// GetResults returns the latest command execution results and text messages, then clears the result buffer.
func (timer *CommandTimer) GetResults() []string {
	timer.mutex.Lock()
	defer timer.mutex.Unlock()
	ret := timer.results.GetAll()
	timer.results.Clear()
	return ret
}
<commit_msg>allow empty entry of preconfigured commands in command timer, for convenience<commit_after>package common

import (
	"fmt"
	"github.com/HouzuoGuo/laitos/misc"
	"github.com/HouzuoGuo/laitos/toolbox"
	"strconv"
	"sync"
	"time"
)

const (
	TimerCommandTimeoutSec = 10 // TimerCommandTimeoutSec is a hard coded timeout number constraining all commands run by timer.
)

/*
CommandTimer executes series of commands, one at a time, at regular interval. Execution results of recent commands are
memorised and can be retrieved at a later time. Beyond command execution results, arbitrary text messages may also be
memorised and retrieved together with command results. CommandTimer is a useful structure for implementing notification
kind of mechanism.
*/
type CommandTimer struct {
	// PreConfiguredCommands are toolbox commands pre-configured to run by user, they never deleted upon clearing.
	PreConfiguredCommands []string `json:"PreConfiguredCommands"`
	// IntervalSec is the number of seconds to sleep between execution of all commands.
	IntervalSec int `json:"IntervalSec"`
	// MaxResults is the maximum number of results to memorise from command execution and text messages.
	MaxResults int `json:"MaxResults"`
	// CommandProcessor is the one going to run all commands.
	CommandProcessor *CommandProcessor `json:"-"`

	/*
		transientCommands are new commands that are added on the fly and can be cleared by calling a function.
		During trigger, these commands are executed after the pre-configured commands.
	*/
	transientCommands []string
	results           *misc.RingBuffer // results are the most recent command results and test messages to retrieve.
	mutex             sync.Mutex       // mutex prevents concurrent access to internal structures.
	running           bool             // running becomes true when command processing loop is running
	stop              chan struct{}    // stop channel signals Run function to return soon.
}

// Initialise prepares internal states of a new CommandTimer.
func (timer *CommandTimer) Initialise() error {
	if timer.IntervalSec < 1 {
		return fmt.Errorf("CommandTimer.Initialise: IntervalSec must be greater than 0")
	}
	if timer.MaxResults < 1 {
		return fmt.Errorf("CommandTimer.Initialise: MaxResults must be greater than 0")
	}
	if timer.PreConfiguredCommands == nil {
		timer.PreConfiguredCommands = []string{}
	}
	timer.results = misc.NewRingBuffer(int64(timer.MaxResults))
	timer.transientCommands = make([]string, 0, 10)
	timer.stop = make(chan struct{})
	return nil
}

/*
GetTransientCommands returns a copy of all transient commands memorises for execution. If there is none, it returns
an empty string array.
*/
func (timer *CommandTimer) GetTransientCommands() []string {
	timer.mutex.Lock()
	defer timer.mutex.Unlock()
	ret := make([]string, len(timer.transientCommands))
	copy(ret, timer.transientCommands)
	return ret
}

// AddTransientCommand places a new toolbox command toward the end of transient command list.
func (timer *CommandTimer) AddTransientCommand(cmd string) {
	timer.mutex.Lock()
	defer timer.mutex.Unlock()
	timer.transientCommands = append(timer.transientCommands, cmd)
}

// ClearTransientCommands removes all transient commands.
func (timer *CommandTimer) ClearTransientCommands() {
	timer.mutex.Lock()
	defer timer.mutex.Unlock()
	timer.transientCommands = make([]string, 0, 10)
}

// runAllCommands executes all pre-configured and transient commands one after another and store their results.
func (timer *CommandTimer) runAllCommands() {
	//	Access to the commands array is not protected by mutex since no other function modifies it
	if timer.PreConfiguredCommands != nil {
		for _, cmd := range timer.PreConfiguredCommands {
			timer.results.Push(timer.CommandProcessor.Process(toolbox.Command{
				TimeoutSec: TimerCommandTimeoutSec,
				Content:    cmd,
			}).CombinedOutput)
		}
	}
	// Make a copy of the latest transient commands to run
	timer.mutex.Lock()
	transientCommands := make([]string, len(timer.transientCommands))
	copy(transientCommands, timer.transientCommands)
	timer.mutex.Unlock()
	// Run transient commands one after another
	for _, cmd := range transientCommands {
		timer.results.Push(timer.CommandProcessor.Process(toolbox.Command{
			TimeoutSec: TimerCommandTimeoutSec,
			Content:    cmd,
		}).CombinedOutput)
	}
}

/*
Start runs an infinite loop to execute all commands one after another, then sleep for an interval.
The function blocks caller until Stop function is called.
If Start function is already running, calling it a second time will do nothing and return immediately.
*/
func (timer *CommandTimer) Start() {
	timer.mutex.Lock()
	if timer.running {
		misc.DefaultLogger.Warning("CommandTimer.Start", strconv.Itoa(timer.IntervalSec), nil, "starting an already started CommandTimer becomes a nop")
		timer.mutex.Unlock()
		return
	}
	timer.mutex.Unlock()
	misc.DefaultLogger.Info("CommandTimer.Start", strconv.Itoa(timer.IntervalSec), nil, "timer has started")
	for {
		timer.running = true
		select {
		case <-time.After(time.Duration(timer.IntervalSec) * time.Second):
			timer.runAllCommands()
		case <-timer.stop:
			return
		}
	}
}

/*
Stop informs the running command processing loop to terminate as early as possible. Blocks until the loop has
terminated. Calling the function while command processing loop is not running yields no effect.
*/
func (timer *CommandTimer) Stop() {
	timer.mutex.Lock()
	if timer.running {
		timer.stop <- struct{}{}
		timer.running = false
	}
	timer.mutex.Unlock()
}

// AddArbitraryTextToResult simply places an arbitrary text string into result.
func (timer *CommandTimer) AddArbitraryTextToResult(text string) {
	// RingBuffer supports concurrent push access, there is no need to protect it with timer's own mutex.
	timer.results.Push(text)
}

// GetResults returns the latest command execution results and text messages, then clears the result buffer.
func (timer *CommandTimer) GetResults() []string {
	timer.mutex.Lock()
	defer timer.mutex.Unlock()
	ret := timer.results.GetAll()
	timer.results.Clear()
	return ret
}
<|endoftext|>


<commit_before>package tunnel

import (
	"net/http"
	"os"
	"strconv"
	"time"

	"gopkg.in/inconshreveable/log15.v2"
)

var VV string

func SetVV(vv string) { VV = vv }

func writePid(file string) {
	if file != "" {
		_ = os.Remove(file)
		pid := os.Getpid()
		f, err := os.Create(file)
		if err != nil {
			log15.Crit("Can't create pidfile", "file", file, "err", err.Error())
			os.Exit(1)
		}
		_, err = f.WriteString(strconv.Itoa(pid) + "\n")
		if err != nil {
			log15.Crit("Can't write to pidfile", "file", file, "err", err.Error())
			os.Exit(1)
		}
		f.Close()
	}
}

// Set logging to use the file or syslog, one of the them must be "" else an error ensues
func makeLogger(pkg, file, facility string) log15.Logger {
	log := log15.New("pkg", pkg)
	if file != "" {
		if facility != "" {
			log.Crit("Can't log to syslog and logfile simultaneously")
			os.Exit(1)
		}
		log.Info("Switching logging", "file", file)
		h, err := log15.FileHandler(file, log15.TerminalFormat())
		if err != nil {
			log.Crit("Can't create log file", "file", file, "err", err.Error())
			os.Exit(1)
		}
		log15.Root().SetHandler(h)
		log.Info("Started logging here")
	} else if facility != "" {
		log.Info("Switching logging to syslog", "facility", facility)
		h, err := log15.SyslogHandler(facility, log15.TerminalFormat())
		if err != nil {
			log.Crit("Can't connect to syslog", "err", err.Error())
			os.Exit(1)
		}
		log15.Root().SetHandler(h)
		log.Info("Started logging here")
	} else {
		log.Info("WStunnel starting")
	}
	return log
}

func calcWsTimeout(tout int) time.Duration {
	var wsTimeout time.Duration
	if tout < 3 {
		wsTimeout = 3 * time.Second
	} else if tout > 600 {
		wsTimeout = 600 * time.Second
	} else {
		wsTimeout = time.Duration(tout) * time.Second
	}
	log15.Info("Websocket keep-alive timeout", "timeout", wsTimeout)
	return wsTimeout
}

// copy http headers over
func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}
<commit_msg>change syslog to LogFormatter, which isn't great either<commit_after>package tunnel

import (
	"net/http"
	"os"
	"strconv"
	"time"

	"gopkg.in/inconshreveable/log15.v2"
)

var VV string

func SetVV(vv string) { VV = vv }

func writePid(file string) {
	if file != "" {
		_ = os.Remove(file)
		pid := os.Getpid()
		f, err := os.Create(file)
		if err != nil {
			log15.Crit("Can't create pidfile", "file", file, "err", err.Error())
			os.Exit(1)
		}
		_, err = f.WriteString(strconv.Itoa(pid) + "\n")
		if err != nil {
			log15.Crit("Can't write to pidfile", "file", file, "err", err.Error())
			os.Exit(1)
		}
		f.Close()
	}
}

// Set logging to use the file or syslog, one of the them must be "" else an error ensues
func makeLogger(pkg, file, facility string) log15.Logger {
	log := log15.New("pkg", pkg)
	if file != "" {
		if facility != "" {
			log.Crit("Can't log to syslog and logfile simultaneously")
			os.Exit(1)
		}
		log.Info("Switching logging", "file", file)
		h, err := log15.FileHandler(file, log15.TerminalFormat())
		if err != nil {
			log.Crit("Can't create log file", "file", file, "err", err.Error())
			os.Exit(1)
		}
		log15.Root().SetHandler(h)
		log.Info("Started logging here")
	} else if facility != "" {
		log.Info("Switching logging to syslog", "facility", facility)
		h, err := log15.SyslogHandler(facility, log15.LogfmtFormat())
		if err != nil {
			log.Crit("Can't connect to syslog", "err", err.Error())
			os.Exit(1)
		}
		log15.Root().SetHandler(h)
		log.Info("Started logging here")
	} else {
		log.Info("WStunnel starting")
	}
	return log
}

func calcWsTimeout(tout int) time.Duration {
	var wsTimeout time.Duration
	if tout < 3 {
		wsTimeout = 3 * time.Second
	} else if tout > 600 {
		wsTimeout = 600 * time.Second
	} else {
		wsTimeout = time.Duration(tout) * time.Second
	}
	log15.Info("Websocket keep-alive timeout", "timeout", wsTimeout)
	return wsTimeout
}

// copy http headers over
func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}
<|endoftext|>


<commit_before>package k8s_test

import (
	"github.com/onsi/gomega/gbytes"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Container Limits", func() {
	const (
		TaskCPULimit    = "--set=concourse.web.defaultTaskCpuLimit=512"
		TaskMemoryLimit = "--set=concourse.web.defaultTaskMemoryLimit=1GB"
		COS             = "--set=worker.nodeSelector.nodeImage=cos"
		UBUNTU          = "--set=worker.nodeSelector.nodeImage=ubuntu"
	)

	BeforeEach(func() {
		setReleaseNameAndNamespace("cl")
	})

	onPks(func() {
		containerLimitsWork(TaskCPULimit, TaskMemoryLimit)
	})

	onGke(func() {
		containerLimitsWork(COS, TaskCPULimit, TaskMemoryLimit)
		containerLimitsFail(UBUNTU, TaskCPULimit, TaskMemoryLimit)
	})

	AfterEach(func() {
		cleanup(releaseName, namespace)
	})

})

func deployWithSelectors(selectorFlags ...string) {
	helmDeployTestFlags := []string{
		"--set=concourse.web.kubernetes.enabled=false",
		"--set=worker.replicas=1",
	}

	deployConcourseChart(releaseName, append(helmDeployTestFlags, selectorFlags...)...)
}

func containerLimitsWork(selectorFlags ...string) {
	Context("container limits work", func() {
		It("returns the configure default container limit", func() {
			deployWithSelectors(selectorFlags...)

			atc := waitAndLogin(namespace, releaseName+"-web")
			defer atc.Close()

			buildSession := fly.Start("execute", "-c", "tasks/tiny.yml")
			<-buildSession.Exited

			Expect(buildSession.ExitCode()).To(Equal(0))

			hijackSession := fly.Start(
				"hijack",
				"-b", "1",
				"--", "sh", "-c",
				"cat /sys/fs/cgroup/memory/memory.memsw.limit_in_bytes; cat /sys/fs/cgroup/cpu/cpu.shares",
			)
			<-hijackSession.Exited

			Expect(hijackSession.ExitCode()).To(Equal(0))
			Expect(hijackSession).To(gbytes.Say("1073741824\n512"))
		})
	})
}

func containerLimitsFail(selectorFlags ...string) {
	Context("container limits fail", func() {
		It("fails to set the memory limit", func() {
			deployWithSelectors(selectorFlags...)

			atc := waitAndLogin(namespace, releaseName+"-web")
			defer atc.Close()

			buildSession := fly.Start("execute", "-c", "tasks/tiny.yml")
			<-buildSession.Exited
			Expect(buildSession.ExitCode()).To(Equal(2))
			Expect(buildSession).To(gbytes.Say(
				"failed to write 1073741824 to memory.memsw.limit_in_bytes",
			))
			Expect(buildSession).To(gbytes.Say("permission denied"))
		})
	})
}
<commit_msg>topgun/k8s: more flexible regex<commit_after>package k8s_test

import (
	"github.com/onsi/gomega/gbytes"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Container Limits", func() {
	const (
		TaskCPULimit    = "--set=concourse.web.defaultTaskCpuLimit=512"
		TaskMemoryLimit = "--set=concourse.web.defaultTaskMemoryLimit=1GB"
		COS             = "--set=worker.nodeSelector.nodeImage=cos"
		UBUNTU          = "--set=worker.nodeSelector.nodeImage=ubuntu"
	)

	BeforeEach(func() {
		setReleaseNameAndNamespace("cl")
	})

	onPks(func() {
		containerLimitsWork(TaskCPULimit, TaskMemoryLimit)
	})

	onGke(func() {
		containerLimitsWork(COS, TaskCPULimit, TaskMemoryLimit)
		containerLimitsFail(UBUNTU, TaskCPULimit, TaskMemoryLimit)
	})

	AfterEach(func() {
		cleanup(releaseName, namespace)
	})

})

func deployWithSelectors(selectorFlags ...string) {
	helmDeployTestFlags := []string{
		"--set=concourse.web.kubernetes.enabled=false",
		"--set=worker.replicas=1",
	}

	deployConcourseChart(releaseName, append(helmDeployTestFlags, selectorFlags...)...)
}

func containerLimitsWork(selectorFlags ...string) {
	Context("container limits work", func() {
		It("returns the configure default container limit", func() {
			deployWithSelectors(selectorFlags...)

			atc := waitAndLogin(namespace, releaseName+"-web")
			defer atc.Close()

			buildSession := fly.Start("execute", "-c", "tasks/tiny.yml")
			<-buildSession.Exited

			Expect(buildSession.ExitCode()).To(Equal(0))

			hijackSession := fly.Start(
				"hijack",
				"-b", "1",
				"--", "sh", "-c",
				"cat /sys/fs/cgroup/memory/memory.memsw.limit_in_bytes; cat /sys/fs/cgroup/cpu/cpu.shares",
			)
			<-hijackSession.Exited

			Expect(hijackSession.ExitCode()).To(Equal(0))
			Expect(hijackSession).To(gbytes.Say("1073741824\n512"))
		})
	})
}

func containerLimitsFail(selectorFlags ...string) {
	Context("container limits fail", func() {
		It("fails to set the memory limit", func() {
			deployWithSelectors(selectorFlags...)

			atc := waitAndLogin(namespace, releaseName+"-web")
			defer atc.Close()

			buildSession := fly.Start("execute", "-c", "tasks/tiny.yml")
			<-buildSession.Exited
			Expect(buildSession.ExitCode()).To(Equal(2))
			Expect(buildSession).To(gbytes.Say(
				"memory.memsw.limit_in_bytes: permission denied",
			))
		})
	})
}
<|endoftext|>


<commit_before>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package tracksprocessor

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/bogem/id3v2"
	"github.com/bogem/nehm/applescript"
	"github.com/bogem/nehm/config"
	"github.com/bogem/nehm/track"
	"github.com/bogem/nehm/ui"
)

type TracksProcessor struct {
	DownloadFolder string // In this folder tracks will be downloaded
	ItunesPlaylist string // In this playlist tracks will be added
}

func NewConfiguredTracksProcessor() *TracksProcessor {
	return &TracksProcessor{
		DownloadFolder: config.Get("dlFolder"),
		ItunesPlaylist: config.Get("itunesPlaylist"),
	}
}

func (tp TracksProcessor) ProcessAll(tracks []track.Track) {
	if len(tracks) == 0 {
		ui.Term("there are no tracks to download", nil)
	}

	var errors []string
	// Start with last track
	for i := len(tracks) - 1; i >= 0; i-- {
		track := tracks[i]
		if err := tp.Process(track); err != nil {
			errors = append(errors, track.Fullname()+": "+err.Error())
			ui.Error("there was an error while downloading "+track.Fullname(), err)
		}
		ui.Println("")
	}

	if len(errors) > 0 {
		ui.Println(ui.RedString("There were errors while downloading tracks:"))
		for _, errText := range errors {
			ui.Println(ui.RedString("  " + errText))
		}
		ui.Println("")
	}

	ui.Success("Done!")
	ui.Quit()
}

func (tp TracksProcessor) Process(t track.Track) error {
	// Download track
	ui.Println("Downloading " + t.Artist() + " - " + t.Title())
	trackPath := filepath.Join(tp.DownloadFolder, t.Filename())
	if _, e := os.Create(trackPath); e != nil {
		return fmt.Errorf("couldn't create track file:", e)
	}
	if e := downloadTrack(t, trackPath); e != nil {
		return fmt.Errorf("couldn't download track:", e)
	}

	// err lets us to not prevent the processing of track further.
	var err error

	// Download artwork
	var artworkPath string
	var artworkBytes []byte
	artworkFile, e := ioutil.TempFile("", "")
	if e != nil {
		err = fmt.Errorf("couldn't create artwork file:", e)
	} else {
		artworkPath = artworkFile.Name()
		if e = downloadArtwork(t, artworkPath); e != nil {
			err = fmt.Errorf("couldn't download artwork file:", e)
		}
		if err == nil {
			artworkBytes, e = ioutil.ReadAll(artworkFile)
			if e != nil {
				err = fmt.Errorf("couldn't read artwork file:", e)
			}
		}
	}

	// Tag track
	if e := tag(t, trackPath, artworkBytes); e != nil {
		err = fmt.Errorf("coudln't tag file:", e)
	}

	// Delete artwork
	if e := artworkFile.Close(); e != nil {
		err = fmt.Errorf("couldn't close artwork file:", e)
	}
	if e := os.Remove(artworkPath); e != nil {
		err = fmt.Errorf("couldn't remove artwork file:", e)
	}

	// Add to iTunes
	if tp.ItunesPlaylist != "" {
		ui.Println("Adding to iTunes")
		if e := applescript.AddTrackToPlaylist(trackPath, tp.ItunesPlaylist); e != nil {
			err = fmt.Errorf("couldn't add track to playlist:", e)
		}
	}

	return err
}

func downloadTrack(t track.Track, path string) error {
	return runDownloadCmd(path, t.URL())
}

func downloadArtwork(t track.Track, path string) error {
	ui.Println("Downloading artwork")
	return runDownloadCmd(path, t.ArtworkURL())
}

func runDownloadCmd(path, url string) error {
	cmd := exec.Command("curl", "-#", "-o", path, "-L", url)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func tag(t track.Track, trackPath string, artwork []byte) error {
	tag, err := id3v2.Open(trackPath)
	if err != nil {
		return err
	}
	defer tag.Close()

	tag.SetArtist(t.Artist())
	tag.SetTitle(t.Title())
	tag.SetYear(t.Year())

	if artwork != nil {
		pic := id3v2.PictureFrame{
			Encoding:    id3v2.ENUTF8,
			MimeType:    "image/jpeg",
			PictureType: id3v2.PTFrontCover,
			Picture:     artwork,
		}
		tag.AddAttachedPicture(pic)
	}

	return tag.Save()
}
<commit_msg>Simplify tracksprocessor<commit_after>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package tracksprocessor

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/bogem/id3v2"
	"github.com/bogem/nehm/applescript"
	"github.com/bogem/nehm/config"
	"github.com/bogem/nehm/track"
	"github.com/bogem/nehm/ui"
)

type TracksProcessor struct {
	DownloadFolder string // In this folder tracks will be downloaded
	ItunesPlaylist string // In this playlist tracks will be added
}

func NewConfiguredTracksProcessor() *TracksProcessor {
	return &TracksProcessor{
		DownloadFolder: config.Get("dlFolder"),
		ItunesPlaylist: config.Get("itunesPlaylist"),
	}
}

func (tp TracksProcessor) ProcessAll(tracks []track.Track) {
	if len(tracks) == 0 {
		ui.Term("there are no tracks to download", nil)
	}

	var errors []string
	// Start with last track
	for i := len(tracks) - 1; i >= 0; i-- {
		track := tracks[i]
		if err := tp.Process(track); err != nil {
			errors = append(errors, track.Fullname()+": "+err.Error())
			ui.Error("there was an error while downloading "+track.Fullname(), err)
		}
		ui.Println("")
	}

	if len(errors) > 0 {
		ui.Println(ui.RedString("There were errors while downloading tracks:"))
		for _, errText := range errors {
			ui.Println(ui.RedString("  " + errText))
		}
		ui.Println("")
	}

	ui.Success("Done!")
	ui.Quit()
}

func (tp TracksProcessor) Process(t track.Track) error {
	// Download track
	ui.Println("Downloading " + t.Fullname())
	trackPath := filepath.Join(tp.DownloadFolder, t.Filename())
	if _, e := os.Create(trackPath); e != nil {
		return fmt.Errorf("couldn't create track file: %v", e)
	}
	if e := downloadTrack(t, trackPath); e != nil {
		return fmt.Errorf("couldn't download track: %v", e)
	}

	// err lets us to not prevent the processing of track further
	var err error

	// Download artwork
	artworkFile, e := ioutil.TempFile("", "nehm")
	if e != nil {
		err = fmt.Errorf("couldn't create artwork file: %v", e)
	} else {
		if e = downloadArtwork(t, artworkFile.Name()); e != nil {
			err = fmt.Errorf("couldn't download artwork file: %v", e)
		}

		// Defer deleting artwork
		defer artworkFile.Close()
		defer os.Remove(artworkFile.Name())
	}

	// Tag track
	if e := tag(t, trackPath, artworkFile); e != nil {
		err = fmt.Errorf("there was an error while taging track: %v", e)
	}

	// Add to iTunes
	if tp.ItunesPlaylist != "" {
		ui.Println("Adding to iTunes")
		if e := applescript.AddTrackToPlaylist(trackPath, tp.ItunesPlaylist); e != nil {
			err = fmt.Errorf("couldn't add track to playlist: %v", e)
		}
	}

	return err
}

func downloadTrack(t track.Track, path string) error {
	return runDownloadCmd(path, t.URL())
}

func downloadArtwork(t track.Track, path string) error {
	ui.Println("Downloading artwork")
	return runDownloadCmd(path, t.ArtworkURL())
}

func runDownloadCmd(path, url string) error {
	cmd := exec.Command("curl", "-#", "-o", path, "-L", url)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func tag(t track.Track, trackPath string, artwork *os.File) error {
	tag, e := id3v2.Open(trackPath)
	if e != nil {
		return e
	}
	defer tag.Close()

	tag.SetArtist(t.Artist())
	tag.SetTitle(t.Title())
	tag.SetYear(t.Year())

	var err error

	artworkBytes, e := ioutil.ReadAll(artwork)
	if e != nil {
		err = fmt.Errorf("couldn't read artwork file: %v", e)
	}
	if artworkBytes != nil {
		pic := id3v2.PictureFrame{
			Encoding:    id3v2.ENUTF8,
			MimeType:    "image/jpeg",
			PictureType: id3v2.PTFrontCover,
			Picture:     artworkBytes,
		}
		tag.AddAttachedPicture(pic)
	}

	if e := tag.Save(); e != nil {
		err = fmt.Errorf("couldn't save tag: %v", e)
	}

	return err
}
<|endoftext|>


<commit_before>package imgio

import (
	"fmt"
	"image"
	"image/color"
	"io/ioutil"
	"testing"

	"github.com/stretchr/testify/require"
)

func Test_Image_Write(t *testing.T) {
	img := &Image{
		img: image.NewRGBA(image.Rect(0, 0, 5, 5)),
		gen: &SimplePointsSequenceGenerator{
			rect:   image.Rect(0, 0, 5, 5),
			cursor: 0,
		},
		prw: SimplePointReadWriter{},
	}

	fmt.Fprint(img, "testing")

	r1, g1, b1, a1 := img.img.At(0, 0).RGBA()
	r2, g2, b2, a2 := img.img.At(1, 0).RGBA()

	require.Equal(t, byte('t'), byte(r1))
	require.Equal(t, byte('e'), byte(g1))
	require.Equal(t, byte('s'), byte(b1))
	require.Equal(t, byte('t'), byte(a1))
	require.Equal(t, byte('i'), byte(r2))
	require.Equal(t, byte('n'), byte(g2))
	require.Equal(t, byte('g'), byte(b2))
	require.Equal(t, byte(0), byte(a2))
}

func Test_Image_Read(t *testing.T) {
	img := &Image{
		img: image.NewRGBA(image.Rect(0, 0, 5, 5)),
		gen: &SimplePointsSequenceGenerator{
			rect:   image.Rect(0, 0, 5, 5),
			cursor: 0,
		},
		prw: SimplePointReadWriter{},
	}

	img.img.Set(0, 0, &color.RGBA{0, 't', 'e', 's'})
	img.img.Set(1, 0, &color.RGBA{0, 't', 0, 'i'})
	img.img.Set(2, 0, &color.RGBA{'n', 'g', 0, 0})

	b, err := ioutil.ReadAll(img)
	b = b[:12]
	require.Nil(t, err, "Cannot read all from img")
	require.Equal(t, []byte{0, 't', 'e', 's', 0, 't', 0, 'i', 'n', 'g', 0, 0}, b)
}
<commit_msg>Fix image write test<commit_after>package imgio

import (
	"image"
	"image/color"
	"io/ioutil"
	"testing"

	"github.com/stretchr/testify/require"
)

func Test_Image_Write(t *testing.T) {
	img := &Image{
		img: image.NewRGBA(image.Rect(0, 0, 5, 5)),
		gen: &SimplePointsSequenceGenerator{
			rect:   image.Rect(0, 0, 5, 5),
			cursor: 0,
		},
		prw: SimplePointReadWriter{},
	}

	n, err := img.Write([]byte("testing"))
	require.Equal(t, 7, n)
	require.Nil(t, err)

	r1, g1, b1, a1 := img.img.At(0, 0).RGBA()
	r2, g2, b2, a2 := img.img.At(1, 0).RGBA()

	require.Equal(t, byte('t'), byte(r1))
	require.Equal(t, byte('e'), byte(g1))
	require.Equal(t, byte('s'), byte(b1))
	require.Equal(t, byte('t'), byte(a1))
	require.Equal(t, byte('i'), byte(r2))
	require.Equal(t, byte('n'), byte(g2))
	require.Equal(t, byte('g'), byte(b2))
	require.Equal(t, byte(0), byte(a2))
}

func Test_Image_Read(t *testing.T) {
	img := &Image{
		img: image.NewRGBA(image.Rect(0, 0, 5, 5)),
		gen: &SimplePointsSequenceGenerator{
			rect:   image.Rect(0, 0, 5, 5),
			cursor: 0,
		},
		prw: SimplePointReadWriter{},
	}

	img.img.Set(0, 0, &color.RGBA{0, 't', 'e', 's'})
	img.img.Set(1, 0, &color.RGBA{0, 't', 0, 'i'})
	img.img.Set(2, 0, &color.RGBA{'n', 'g', 0, 0})

	b, err := ioutil.ReadAll(img)
	b = b[:12]
	require.Nil(t, err, "Cannot read all from img")
	require.Equal(t, []byte{0, 't', 'e', 's', 0, 't', 0, 'i', 'n', 'g', 0, 0}, b)
}
<|endoftext|>


<commit_before>// Copyright 2016 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten_test

import (
	"image"
	_ "image/png"
	"testing"

	. "github.com/hajimehoshi/ebiten"
)

var ebitenImageBin = ""

func openImage(path string) (image.Image, error) {
	file, err := readFile(path)
	if err != nil {
		return nil, err
	}

	img, _, err := image.Decode(file)
	if err != nil {
		return nil, err
	}
	return img, nil
}

func openEbitenImage(path string) (*Image, image.Image, error) {
	img, err := openImage(path)
	if err != nil {
		return nil, nil, err
	}

	eimg, err := NewImageFromImage(img, FilterNearest)
	if err != nil {
		return nil, nil, err
	}
	return eimg, img, nil
}

func TestImageSelf(t *testing.T) {
	img, _, err := openEbitenImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}
	if err := img.DrawImage(img, nil); err == nil {
		t.Fatalf("img.DrawImage(img, nil) doesn't return error; an error should be returned")
	}
}

func TestImageDispose(t *testing.T) {
	img, err := NewImage(16, 16, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}
	if err := img.Dispose(); err != nil {
		t.Errorf("img.Dipose() returns error: %v", err)
	}
}
<commit_msg>graphics: Revive image_test.go<commit_after>// Copyright 2016 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten_test

import (
	"errors"
	"image"
	"image/color"
	"image/draw"
	_ "image/png"
	"math"
	"os"
	"testing"

	. "github.com/hajimehoshi/ebiten"
)

func TestMain(m *testing.M) {
	code := 0
	// Run an Ebiten process so that (*Image).At is available.
	f := func(screen *Image) error {
		code = m.Run()
		return errors.New("regular termination")
	}
	Run(f, 320, 240, 1, "Test")
	os.Exit(code)
}

var ebitenImageBin = ""

func openImage(path string) (image.Image, error) {
	file, err := readFile(path)
	if err != nil {
		return nil, err
	}

	img, _, err := image.Decode(file)
	if err != nil {
		return nil, err
	}
	return img, nil
}

func openEbitenImage(path string) (*Image, image.Image, error) {
	img, err := openImage(path)
	if err != nil {
		return nil, nil, err
	}

	eimg, err := NewImageFromImage(img, FilterNearest)
	if err != nil {
		return nil, nil, err
	}
	return eimg, img, nil
}

func diff(x, y uint8) uint8 {
	if x <= y {
		return y - x
	}
	return x - y
}

func TestImagePixels(t *testing.T) {
	img0, img, err := openEbitenImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}

	if got := img0.Bounds().Size(); got != img.Bounds().Size() {
		t.Errorf("img size: got %d; want %d", got, img.Bounds().Size())
	}

	for j := 0; j < img0.Bounds().Size().Y; j++ {
		for i := 0; i < img0.Bounds().Size().X; i++ {
			got := img0.At(i, j)
			want := color.RGBAModel.Convert(img.At(i, j))
			if got != want {
				t.Errorf("img0 At(%d, %d): got %#v; want %#v", i, j, got, want)
			}
		}
	}
}

func TestImageComposition(t *testing.T) {
	img2Color := color.NRGBA{0x24, 0x3f, 0x6a, 0x88}
	img3Color := color.NRGBA{0x85, 0xa3, 0x08, 0xd3}

	// TODO: Rename this to img0
	img1, _, err := openEbitenImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}

	w, h := img1.Bounds().Size().X, img1.Bounds().Size().Y

	img2, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}

	img3, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}

	img2.Fill(img2Color)
	img3.Fill(img3Color)
	img_12_3, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}
	img2.DrawImage(img1, nil)
	img3.DrawImage(img2, nil)
	img_12_3.DrawImage(img3, nil)

	img2.Fill(img2Color)
	img3.Fill(img3Color)
	img_1_23, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}
	img3.DrawImage(img2, nil)
	img3.DrawImage(img1, nil)
	img_1_23.DrawImage(img3, nil)

	for j := 0; j < h; j++ {
		for i := 0; i < w; i++ {
			c1 := img_12_3.At(i, j).(color.RGBA)
			c2 := img_1_23.At(i, j).(color.RGBA)
			if 1 < diff(c1.R, c2.R) || 1 < diff(c1.G, c2.G) || 1 < diff(c1.B, c2.B) || 1 < diff(c1.A, c2.A) {
				t.Errorf("img_12_3.At(%d, %d) = %#v; img_1_23.At(%[1]d, %[2]d) = %#[4]v", i, j, c1, c2)
			}
			if c1.A == 0 {
				t.Fatalf("img_12_3.At(%d, %d).A = 0; nothing is rendered?", i, j)
			}
			if c2.A == 0 {
				t.Fatalf("img_1_23.At(%d, %d).A = 0; nothing is rendered?", i, j)
			}
		}
	}
}

func TestImageSelf(t *testing.T) {
	img, _, err := openEbitenImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}
	if err := img.DrawImage(img, nil); err == nil {
		t.Fatalf("img.DrawImage(img, nil) doesn't return error; an error should be returned")
	}
}

func TestImageDotByDotInversion(t *testing.T) {
	img0, _, err := openEbitenImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}
	w, h := img0.Size()
	img1, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}
	op := &DrawImageOptions{}
	op.GeoM.Rotate(math.Pi)
	op.GeoM.Translate(float64(w), float64(h))
	img1.DrawImage(img0, op)

	for j := 0; j < h; j++ {
		for i := 0; i < w; i++ {
			c0 := img0.At(i, j).(color.RGBA)
			c1 := img1.At(w-i-1, h-j-1).(color.RGBA)
			if c0 != c1 {
				t.Errorf("img0.At(%[1]d, %[2]d) should equal to img1.At(%[3]d, %[4]d) but not: %[5]v vs %[6]v", i, j, w-i-1, h-j-1, c0, c1)
			}
		}
	}
}

func TestReplacePixels(t *testing.T) {
	origImg, err := openImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}
	// Convert to RGBA
	img := image.NewRGBA(origImg.Bounds())
	draw.Draw(img, img.Bounds(), origImg, image.ZP, draw.Src)

	size := img.Bounds().Size()
	img0, err := NewImage(size.X, size.Y, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}

	img0.ReplacePixels(img.Pix)
	for j := 0; j < img0.Bounds().Size().Y; j++ {
		for i := 0; i < img0.Bounds().Size().X; i++ {
			got := img0.At(i, j)
			want := img.At(i, j)
			if got != want {
				t.Errorf("img0 At(%d, %d): got %#v; want %#v", i, j, got, want)
			}
		}
	}

	p := make([]uint8, 4*size.X*size.Y)
	for i, _ := range p {
		p[i] = 0x80
	}
	img0.ReplacePixels(p)
	for j := 0; j < img0.Bounds().Size().Y; j++ {
		for i := 0; i < img0.Bounds().Size().X; i++ {
			got := img0.At(i, j)
			want := color.RGBA{p[4*i], p[4*i+1], p[4*i+2], p[4*i+3]}
			if got != want {
				t.Errorf("img0 At(%d, %d): got %#v; want %#v", i, j, got, want)
			}
		}
	}
}

func TestImageDispose(t *testing.T) {
	img, err := NewImage(16, 16, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}
	if err := img.Dispose(); err != nil {
		t.Errorf("img.Dipose() returns error: %v", err)
	}
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func TestImageCompositeModeLighter(t *testing.T) {
	img0, _, err := openEbitenImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}

	w, h := img0.Size()
	img1, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}
	img1.Fill(color.RGBA{0x01, 0x02, 0x03, 0x04})
	op := &DrawImageOptions{}
	op.CompositeMode = CompositeModeLighter
	img1.DrawImage(img0, op)
	for j := 0; j < img1.Bounds().Size().Y; j++ {
		for i := 0; i < img1.Bounds().Size().X; i++ {
			got := img1.At(i, j).(color.RGBA)
			want := img0.At(i, j).(color.RGBA)
			want.R = uint8(min(0xff, int(want.R)+1))
			want.G = uint8(min(0xff, int(want.G)+2))
			want.B = uint8(min(0xff, int(want.B)+3))
			want.A = uint8(min(0xff, int(want.A)+4))
			if got != want {
				t.Errorf("img1 At(%d, %d): got %#v; want %#v", i, j, got, want)
			}
		}
	}
}

func TestNewImageFromEbitenImage(t *testing.T) {
	img, _, err := openEbitenImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}
	if _, err := NewImageFromImage(img, FilterNearest); err != nil {
		t.Errorf("NewImageFromImage returns error: %v", err)
	}
}
<|endoftext|>


<commit_before>package linux

import (
	"io/ioutil"
	"strconv"
	"strings"
	"time"
)

type Stat struct {
	CPUStatAll      CPUStat
	CPUStats        []CPUStat
	Interrupts      uint64
	ContextSwitches uint64
	BootTime        time.Time
	Processes       uint64
	ProcsRunning    uint64
	ProcsBlocked    uint64
}

type CPUStat struct {
	Id        string
	User      uint64
	Nice      uint64
	System    uint64
	Idle      uint64
	IOWait    uint64
	IRQ       uint64
	SoftIRQ   uint64
	Steal     uint64
	Guest     uint64
	GuestNice uint64
}

func parseCPUStat(line string) *CPUStat {
	fields := strings.Fields(line)
	s := CPUStat{}
	s.User, _ = strconv.ParseUint(fields[1], 10, 32)
	s.Nice, _ = strconv.ParseUint(fields[2], 10, 32)
	s.System, _ = strconv.ParseUint(fields[3], 10, 32)
	s.Idle, _ = strconv.ParseUint(fields[4], 10, 32)
	s.IOWait, _ = strconv.ParseUint(fields[5], 10, 32)
	s.IRQ, _ = strconv.ParseUint(fields[6], 10, 32)
	s.SoftIRQ, _ = strconv.ParseUint(fields[7], 10, 32)
	s.Steal, _ = strconv.ParseUint(fields[8], 10, 32)
	s.Guest, _ = strconv.ParseUint(fields[9], 10, 32)
	s.GuestNice, _ = strconv.ParseUint(fields[10], 10, 32)
	return &s
}

func ReadStat(path string) (*Stat, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	content := string(b)
	lines := strings.Split(content, "\n")

	var stat Stat = Stat{}

	for i, line := range lines {
		if strings.HasPrefix(line, "cpu") {
			if cpuStat := parseCPUStat(line); cpuStat != nil {
				if i == 0 {
					stat.CPUStatAll = *cpuStat
				} else {
					stat.CPUStats = append(stat.CPUStats, *cpuStat)
				}
			}
		} else if strings.HasPrefix(line, "intr") {
			fields := strings.Fields(line)
			stat.Interrupts, _ = strconv.ParseUint(fields[1], 10, 64)
		} else if strings.HasPrefix(line, "ctxt") {
			fields := strings.Fields(line)
			stat.ContextSwitches, _ = strconv.ParseUint(fields[1], 10, 64)
		} else if strings.HasPrefix(line, "btime") {
			fields := strings.Fields(line)
			seconds, _ := strconv.ParseInt(fields[1], 10, 64)
			stat.BootTime = time.Unix(seconds, 0)
		} else if strings.HasPrefix(line, "processes") {
			fields := strings.Fields(line)
			stat.Processes, _ = strconv.ParseUint(fields[1], 10, 64)
		} else if strings.HasPrefix(line, "procs_running") {
			fields := strings.Fields(line)
			stat.ProcsRunning, _ = strconv.ParseUint(fields[1], 10, 64)
		} else if strings.HasPrefix(line, "procs_blocked") {
			fields := strings.Fields(line)
			stat.ProcsBlocked, _ = strconv.ParseUint(fields[1], 10, 64)
		}
	}
	return &stat, nil
}
<commit_msg>refactor stat parser<commit_after>package linux

import (
	"io/ioutil"
	"strconv"
	"strings"
	"time"
)

type Stat struct {
	CPUStatAll      CPUStat
	CPUStats        []CPUStat
	Interrupts      uint64
	ContextSwitches uint64
	BootTime        time.Time
	Processes       uint64
	ProcsRunning    uint64
	ProcsBlocked    uint64
}

type CPUStat struct {
	Id        string
	User      uint64
	Nice      uint64
	System    uint64
	Idle      uint64
	IOWait    uint64
	IRQ       uint64
	SoftIRQ   uint64
	Steal     uint64
	Guest     uint64
	GuestNice uint64
}

func createCPUStat(fields []string) *CPUStat {
	s := CPUStat{}
	s.User, _ = strconv.ParseUint(fields[1], 10, 32)
	s.Nice, _ = strconv.ParseUint(fields[2], 10, 32)
	s.System, _ = strconv.ParseUint(fields[3], 10, 32)
	s.Idle, _ = strconv.ParseUint(fields[4], 10, 32)
	s.IOWait, _ = strconv.ParseUint(fields[5], 10, 32)
	s.IRQ, _ = strconv.ParseUint(fields[6], 10, 32)
	s.SoftIRQ, _ = strconv.ParseUint(fields[7], 10, 32)
	s.Steal, _ = strconv.ParseUint(fields[8], 10, 32)
	s.Guest, _ = strconv.ParseUint(fields[9], 10, 32)
	s.GuestNice, _ = strconv.ParseUint(fields[10], 10, 32)
	return &s
}

func ReadStat(path string) (*Stat, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	content := string(b)
	lines := strings.Split(content, "\n")

	var stat Stat = Stat{}

	for i, line := range lines {
		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}
		if fields[0] == "cpu" {
			if cpuStat := createCPUStat(fields); cpuStat != nil {
				if i == 0 {
					stat.CPUStatAll = *cpuStat
				} else {
					stat.CPUStats = append(stat.CPUStats, *cpuStat)
				}
			}
		} else if fields[0] == "intr" {
			stat.Interrupts, _ = strconv.ParseUint(fields[1], 10, 64)
		} else if fields[0] == "ctxt" {
			stat.ContextSwitches, _ = strconv.ParseUint(fields[1], 10, 64)
		} else if fields[0] == "btime" {
			seconds, _ := strconv.ParseInt(fields[1], 10, 64)
			stat.BootTime = time.Unix(seconds, 0)
		} else if fields[0] == "processes" {
			stat.Processes, _ = strconv.ParseUint(fields[1], 10, 64)
		} else if fields[0] == "procs_running" {
			stat.ProcsRunning, _ = strconv.ParseUint(fields[1], 10, 64)
		} else if fields[0] == "procs_blocked" {
			stat.ProcsBlocked, _ = strconv.ParseUint(fields[1], 10, 64)
		}
	}
	return &stat, nil
}
<|endoftext|>


<commit_before>package images

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"strings"
	"text/tabwriter"
	"time"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/service/ec2"
	"github.com/fatih/color"
)

type AwsImages struct {
	svc *ec2.EC2

	images []*ec2.Image
}

func NewAwsImages(region string) *AwsImages {
	return &AwsImages{
		svc: ec2.New(&aws.Config{Region: region}),
	}
}

func (a *AwsImages) Fetch() error {
	input := &ec2.DescribeImagesInput{
		Owners: stringSlice("self"),
	}

	resp, err := a.svc.DescribeImages(input)
	if err != nil {
		return err
	}

	a.images = resp.Images

	// sort from oldest to newest
	if len(a.images) > 1 {
		sort.Sort(a)
	}

	return nil
}

func (a *AwsImages) Print() {
	if len(a.images) == 0 {
		fmt.Println("no images found")
		return
	}

	color.Green("AWS: Region: %s (%d images)\n\n", a.svc.Config.Region, len(a.images))

	w := new(tabwriter.Writer)
	w.Init(os.Stdout, 10, 8, 0, '\t', 0)
	defer w.Flush()

	fmt.Fprintln(w, "    Name\tID\tState\tTags")

	for i, image := range a.images {
		tags := make([]string, len(image.Tags))
		for i, tag := range image.Tags {
			tags[i] = *tag.Key + ":" + *tag.Value
		}

		fmt.Fprintf(w, "[%d] %s\t%s\t%s\t%+v\n",
			i, *image.Name, *image.ImageID, *image.State, tags)
	}
}

func (a *AwsImages) Help(command string) string {
	switch command {
	case "modify":
		return `Usage: images modify --provider aws [options]

  Modify AMI properties.

Options:

  -create-tags                  Create or override tags
  -delete-tags                  Delete tags
`
	case "list":
	}

	return "no help found for command " + command
}

func (a *AwsImages) Modify(args []string) error {
	var (
		createTags string
		deleteTags string
		imageIds   string
		dryRun     bool
	)

	flagSet := flag.NewFlagSet("modify", flag.ContinueOnError)
	flagSet.StringVar(&createTags, "create-tags", "", "Create  or override tags")
	flagSet.StringVar(&deleteTags, "delete-tags", "", "Delete tags")
	flagSet.StringVar(&imageIds, "image-ids", "", "Images to be used with actions")
	flagSet.BoolVar(&dryRun, "dry-run", false, "Don't run command, but show the action")
	flagSet.Usage = func() {
		helpMsg := `Usage: images modify --provider aws [options]

  Modify AMI properties.

Options:

  -image-ids   "ami-123,..."   Images to be used with below actions

  -create-tags "key=val,..."   Create or override tags
  -delete-tags "key,..."       Delete tags
  -dry-run                     Don't run command, but show the action
`
		fmt.Fprintf(os.Stderr, helpMsg)
	}

	flagSet.SetOutput(ioutil.Discard) // don't print anything without my permission
	if err := flagSet.Parse(args); err != nil {
		return nil // we don't return error, the usage will be printed instead
	}

	if len(args) == 0 {
		flagSet.Usage()
		return errors.New("no flags are passed")
	}

	if imageIds == "" {
		return errors.New("no images are passed with [--image-ids]")
	}

	fmt.Printf("imageIds = %+v\n", imageIds)
	fmt.Printf("createTags = %+v\n", createTags)
	fmt.Printf("deleteTags = %+v\n", deleteTags)

	if createTags != "" && deleteTags != "" {
		return errors.New("not allowed to be used together: [--create-tags,--delete-tags]")
	}

	if createTags != "" {
		keyVals := make(map[string]string, 0)

		for _, keyVal := range strings.Split(createTags, ",") {
			keys := strings.Split(keyVal, "=")
			if len(keys) != 2 {
				return fmt.Errorf("malformed value passed to --create-tags: %v", keys)
			}
			keyVals[keys[0]] = keys[1]
		}

		return a.AddTags(keyVals, dryRun, strings.Split(imageIds, ",")...)
	}

	if deleteTags != "" {
	}

	return nil
}

// Add tags adds or overwrites all tags for the specified images
func (a *AwsImages) AddTags(tags map[string]string, dryRun bool, images ...string) error {
	ec2Tags := make([]*ec2.Tag, 0)
	for key, val := range tags {
		ec2Tags = append(ec2Tags, &ec2.Tag{
			Key:   aws.String(key),
			Value: aws.String(val),
		})
	}

	params := &ec2.CreateTagsInput{
		Resources: stringSlice(images...),
		Tags:      ec2Tags,
		DryRun:    aws.Boolean(dryRun),
	}

	_, err := a.svc.CreateTags(params)
	return err
}

//
// Sort interface
//

func (a *AwsImages) Len() int {
	return len(a.images)
}

func (a *AwsImages) Less(i, j int) bool {
	it, err := time.Parse(time.RFC3339, *a.images[i].CreationDate)
	if err != nil {
		log.Println("aws: sorting err: ", err)
	}

	jt, err := time.Parse(time.RFC3339, *a.images[j].CreationDate)
	if err != nil {
		log.Println("aws: sorting err: ", err)
	}

	return it.Before(jt)
}

func (a *AwsImages) Swap(i, j int) {
	a.images[i], a.images[j] = a.images[j], a.images[i]
}

//
// Utils
//

func stringSlice(vals ...string) []*string {
	a := make([]*string, len(vals))

	for i, v := range vals {
		a[i] = aws.String(v)
	}

	return a
}
<commit_msg>images/aws: implement delete tags<commit_after>package images

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"strings"
	"text/tabwriter"
	"time"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/service/ec2"
	"github.com/fatih/color"
)

type AwsImages struct {
	svc *ec2.EC2

	images []*ec2.Image
}

func NewAwsImages(region string) *AwsImages {
	return &AwsImages{
		svc: ec2.New(&aws.Config{Region: region}),
	}
}

func (a *AwsImages) Fetch() error {
	input := &ec2.DescribeImagesInput{
		Owners: stringSlice("self"),
	}

	resp, err := a.svc.DescribeImages(input)
	if err != nil {
		return err
	}

	a.images = resp.Images

	// sort from oldest to newest
	if len(a.images) > 1 {
		sort.Sort(a)
	}

	return nil
}

func (a *AwsImages) Print() {
	if len(a.images) == 0 {
		fmt.Println("no images found")
		return
	}

	color.Green("AWS: Region: %s (%d images)\n\n", a.svc.Config.Region, len(a.images))

	w := new(tabwriter.Writer)
	w.Init(os.Stdout, 10, 8, 0, '\t', 0)
	defer w.Flush()

	fmt.Fprintln(w, "    Name\tID\tState\tTags")

	for i, image := range a.images {
		tags := make([]string, len(image.Tags))
		for i, tag := range image.Tags {
			tags[i] = *tag.Key + ":" + *tag.Value
		}

		fmt.Fprintf(w, "[%d] %s\t%s\t%s\t%+v\n",
			i, *image.Name, *image.ImageID, *image.State, tags)
	}
}

func (a *AwsImages) Help(command string) string {
	switch command {
	case "modify":
		return `Usage: images modify --provider aws [options]

  Modify AMI properties.

Options:

  -create-tags                  Create or override tags
  -delete-tags                  Delete tags
`
	case "list":
	}

	return "no help found for command " + command
}

func (a *AwsImages) Modify(args []string) error {
	var (
		createTags string
		deleteTags string
		imageIds   string
		dryRun     bool
	)

	flagSet := flag.NewFlagSet("modify", flag.ContinueOnError)
	flagSet.StringVar(&createTags, "create-tags", "", "Create  or override tags")
	flagSet.StringVar(&deleteTags, "delete-tags", "", "Delete tags")
	flagSet.StringVar(&imageIds, "image-ids", "", "Images to be used with actions")
	flagSet.BoolVar(&dryRun, "dry-run", false, "Don't run command, but show the action")
	flagSet.Usage = func() {
		helpMsg := `Usage: images modify --provider aws [options]

  Modify AMI properties.

Options:

  -image-ids   "ami-123,..."   Images to be used with below actions

  -create-tags "key=val,..."   Create or override tags
  -delete-tags "key,..."       Delete tags
  -dry-run                     Don't run command, but show the action
`
		fmt.Fprintf(os.Stderr, helpMsg)
	}

	flagSet.SetOutput(ioutil.Discard) // don't print anything without my permission
	if err := flagSet.Parse(args); err != nil {
		return nil // we don't return error, the usage will be printed instead
	}

	if len(args) == 0 {
		flagSet.Usage()
		return errors.New("no flags are passed")
	}

	if imageIds == "" {
		return errors.New("no images are passed with [--image-ids]")
	}

	fmt.Printf("imageIds = %+v\n", imageIds)
	fmt.Printf("createTags = %+v\n", createTags)
	fmt.Printf("deleteTags = %+v\n", deleteTags)

	if createTags != "" && deleteTags != "" {
		return errors.New("not allowed to be used together: [--create-tags,--delete-tags]")
	}

	if createTags != "" {
		keyVals := make(map[string]string, 0)
		for _, keyVal := range strings.Split(createTags, ",") {
			keys := strings.Split(keyVal, "=")
			if len(keys) != 2 {
				return fmt.Errorf("malformed value passed to --create-tags: %v", keys)
			}
			keyVals[keys[0]] = keys[1]
		}

		return a.AddTags(keyVals, dryRun, strings.Split(imageIds, ",")...)
	}

	if deleteTags != "" {
		a.DeleteTags(deleteTags, dryRun, strings.Split(imageIds, ",")...)
	}

	return nil
}

// Add tags adds or overwrites all tags for the specified images.
func (a *AwsImages) AddTags(tags map[string]string, dryRun bool, images ...string) error {
	ec2Tags := make([]*ec2.Tag, 0)
	for key, val := range tags {
		ec2Tags = append(ec2Tags, &ec2.Tag{
			Key:   aws.String(key),
			Value: aws.String(val),
		})
	}

	params := &ec2.CreateTagsInput{
		Resources: stringSlice(images...),
		Tags:      ec2Tags,
		DryRun:    aws.Boolean(dryRun),
	}

	_, err := a.svc.CreateTags(params)
	return err
}

// DeleteTags deletes the given tags for the given images. Tags is in the form
// of "key1=val1,key2=val2,key3,key4="
func (a *AwsImages) DeleteTags(tags string, dryRun bool, images ...string) error {
	ec2Tags := make([]*ec2.Tag, 0)

	for _, keyVal := range strings.Split(tags, ",") {
		keys := strings.Split(keyVal, "=")
		ec2Tag := &ec2.Tag{
			Key: aws.String(keys[0]), // index 0 is always available
		}

		// means value is not omitted. We don't care if value is empty or not,
		// the AWS API takes care of it.
		if len(keys) == 2 {
			ec2Tag.Value = aws.String(keys[1])
		}

		ec2Tags = append(ec2Tags, ec2Tag)
	}

	params := &ec2.DeleteTagsInput{
		Resources: stringSlice(images...),
		Tags:      ec2Tags,
		DryRun:    aws.Boolean(dryRun),
	}

	_, err := a.svc.DeleteTags(params)
	return err
}

//
// Sort interface
//

func (a *AwsImages) Len() int {
	return len(a.images)
}

func (a *AwsImages) Less(i, j int) bool {
	it, err := time.Parse(time.RFC3339, *a.images[i].CreationDate)
	if err != nil {
		log.Println("aws: sorting err: ", err)
	}

	jt, err := time.Parse(time.RFC3339, *a.images[j].CreationDate)
	if err != nil {
		log.Println("aws: sorting err: ", err)
	}

	return it.Before(jt)
}

func (a *AwsImages) Swap(i, j int) {
	a.images[i], a.images[j] = a.images[j], a.images[i]
}

//
// Utils
//

func stringSlice(vals ...string) []*string {
	a := make([]*string, len(vals))

	for i, v := range vals {
		a[i] = aws.String(v)
	}

	return a
}
<|endoftext|>