<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package resolve

import (
	"context"
	"fmt"
	"reflect"

	"github.com/google/gapid/core/data/deep"
	"github.com/google/gapid/gapis/api"
	"github.com/google/gapid/gapis/capture"
	"github.com/google/gapid/gapis/database"
	"github.com/google/gapid/gapis/messages"
	"github.com/google/gapid/gapis/service"
	"github.com/google/gapid/gapis/service/path"
)

// Set creates a copy of the capture referenced by the request's path, but
// with the object, value or memory at p replaced with v. The path returned is
// identical to p, but with the base changed to refer to the new capture.
func Set(ctx context.Context, p *path.Any, v interface{}) (*path.Any, error) {
	obj, err := database.Build(ctx, &SetResolvable{p, service.NewValue(v)})
	if err != nil {
		return nil, err
	}
	return obj.(*path.Any), nil
}

// Resolve implements the database.Resolver interface.
func (r *SetResolvable) Resolve(ctx context.Context) (interface{}, error) {
	if c := path.FindCapture(r.Path.Node()); c != nil {
		ctx = capture.Put(ctx, c)
	}

	v, err := serviceToInternal(r.Value.Get())
	if err != nil {
		return nil, err
	}

	p, err := change(ctx, r.Path.Node(), v)
	if err != nil {
		return nil, err
	}
	return p.Path(), nil
}

func change(ctx context.Context, p path.Node, val interface{}) (path.Node, error) {
	switch p := p.(type) {
	case *path.Report:
		return nil, fmt.Errorf("Reports are immutable")

	case *path.ResourceData:
		meta, err := ResourceMeta(ctx, p.Id, p.After)
		if err != nil {
			return nil, err
		}

		cmdIdx := p.After.Indices[0]
		// If we change resource data, subcommands do not affect this, so change
		// the main comand.

		oldCmds, err := NCmds(ctx, p.After.Capture, cmdIdx+1)
		if err != nil {
			return nil, err
		}

		cmds := make([]api.Cmd, len(oldCmds))
		copy(cmds, oldCmds)

		replaceCommands := func(where uint64, with interface{}) {
			cmds[where] = with.(api.Cmd)
		}

		data, ok := val.(*api.ResourceData)
		if !ok {
			return nil, fmt.Errorf("Expected ResourceData, got %T", val)
		}

		if err := meta.Resource.SetResourceData(ctx, p.After, data, meta.IDMap, replaceCommands); err != nil {
			return nil, err
		}

		// Store the new command list
		c, err := changeCommands(ctx, p.After.Capture, cmds)
		if err != nil {
			return nil, err
		}

		return &path.ResourceData{
			Id: p.Id, // TODO: Shouldn't this change?
			After: &path.Command{
				Capture: c,
				Indices: p.After.Indices,
			},
		}, nil

	case *path.Command:
		cmdIdx := p.Indices[0]
		if len(p.Indices) > 1 {
			return nil, fmt.Errorf("Cannot modify subcommands") // TODO: Subcommands
		}

		// Resolve the command list
		oldCmds, err := NCmds(ctx, p.Capture, cmdIdx+1)
		if err != nil {
			return nil, err
		}

		// Validate the value
		if val == nil {
			return nil, fmt.Errorf("Command cannot be nil")
		}
		cmd, ok := val.(api.Cmd)
		if !ok {
			return nil, fmt.Errorf("Expected Cmd, got %T", val)
		}

		// Clone the command list
		cmds := make([]api.Cmd, len(oldCmds))
		copy(cmds, oldCmds)

		// Propagate extras if the new command omitted them
		oldCmd := oldCmds[cmdIdx]
		if len(cmd.Extras().All()) == 0 {
			cmd.Extras().Add(oldCmd.Extras().All()...)
		}
		cmds[cmdIdx] = cmd

		// Store the new command list
		c, err := changeCommands(ctx, p.Capture, cmds)
		if err != nil {
			return nil, err
		}

		return &path.Command{
			Capture: c,
			Indices: p.Indices,
		}, nil

	case *path.Commands:
		return nil, fmt.Errorf("Commands can not be changed directly")

	case *path.State:
		return nil, fmt.Errorf("State can not currently be mutated")

	case *path.Field, *path.Parameter, *path.ArrayIndex, *path.MapIndex:
		oldObj, err := ResolveInternal(ctx, p.Parent())
		if err != nil {
			return nil, err
		}

		obj, err := clone(reflect.ValueOf(oldObj))
		if err != nil {
			return nil, err
		}

		switch p := p.(type) {
		case *path.Parameter:
			// TODO: Deal with parameters belonging to sub-commands.
			cmd := obj.Interface().(api.Cmd)
			err := api.SetParameter(ctx, cmd, p.Name, val)
			switch err {
			case nil:
			case api.ErrParameterNotFound:
				return nil, &service.ErrInvalidPath{
					Reason: messages.ErrParameterDoesNotExist(cmd.CmdName(), p.Name),
					Path:   p.Path(),
				}
			default:
				return nil, err
			}

			parent, err := change(ctx, p.Parent(), obj.Interface())
			if err != nil {
				return nil, err
			}
			return parent.(*path.Command).Parameter(p.Name), nil

		case *path.Result:
			// TODO: Deal with parameters belonging to sub-commands.
			cmd := obj.Interface().(api.Cmd)
			err := api.SetResult(ctx, cmd, val)
			switch err {
			case nil:
			case api.ErrResultNotFound:
				return nil, &service.ErrInvalidPath{
					Reason: messages.ErrResultDoesNotExist(cmd.CmdName()),
					Path:   p.Path(),
				}
			default:
				return nil, err
			}

			parent, err := change(ctx, p.Parent(), obj.Interface())
			if err != nil {
				return nil, err
			}
			return parent.(*path.Command).Result(), nil

		case *path.Field:
			parent, err := setField(ctx, obj, reflect.ValueOf(val), p.Name, p)
			if err != nil {
				return nil, err
			}
			out := &path.Field{Name: p.Name}
			out.SetParent(parent)
			return out, nil

		case *path.ArrayIndex:
			a, ty := obj, obj.Type()
			switch a.Kind() {
			case reflect.Array, reflect.Slice:
				ty = ty.Elem()
			case reflect.String:
			default:
				return nil, &service.ErrInvalidPath{
					Reason: messages.ErrTypeNotArrayIndexable(typename(a.Type())),
					Path:   p.Path(),
				}
			}
			val, ok := convert(reflect.ValueOf(val), ty)
			if !ok {
				return nil, fmt.Errorf("Slice or array at %s has element of type %v, got type %v",
					p.Parent(), ty, val.Type())
			}
			if count := uint64(a.Len()); p.Index >= count {
				return nil, errPathOOB(p.Index, "Index", 0, count-1, p)
			}
			if err := assign(a.Index(int(p.Index)), val); err != nil {
				return nil, err
			}
			parent, err := change(ctx, p.Parent(), a.Interface())
			if err != nil {
				return nil, err
			}
			p = &path.ArrayIndex{Index: p.Index}
			p.SetParent(parent)
			return p, nil

		case *path.MapIndex:
			m := obj
			if m.Kind() != reflect.Map {
				return nil, &service.ErrInvalidPath{
					Reason: messages.ErrTypeNotMapIndexable(typename(m.Type())),
					Path:   p.Path(),
				}
			}
			key, ok := convert(reflect.ValueOf(p.KeyValue()), m.Type().Key())
			if !ok {
				return nil, &service.ErrInvalidPath{
					Reason: messages.ErrIncorrectMapKeyType(
						typename(reflect.TypeOf(p.KeyValue())), // got
						typename(m.Type().Key())),              // expected
					Path: p.Path(),
				}
			}
			val, ok := convert(reflect.ValueOf(val), m.Type().Elem())
			if !ok {
				return nil, fmt.Errorf("Map at %s has value of type %v, got type %v",
					p.Parent(), m.Type().Elem(), val.Type())
			}
			m.SetMapIndex(key, val)
			parent, err := change(ctx, p.Parent(), m.Interface())
			if err != nil {
				return nil, err
			}
			p = &path.MapIndex{Key: p.Key}
			p.SetParent(parent)
			return p, nil
		}
	}
	return nil, fmt.Errorf("Unknown path type %T", p)
}

func changeCommands(ctx context.Context, p *path.Capture, newCmds []api.Cmd) (*path.Capture, error) {
	old, err := capture.ResolveFromPath(ctx, p)
	if err != nil {
		return nil, err
	}
	c, err := capture.New(ctx, old.Name+"*", old.Header, newCmds)
	if err != nil {
		return nil, err
	}
	return c, nil
}

func setField(ctx context.Context, str, val reflect.Value, name string, p path.Node) (path.Node, error) {
	dst, err := field(ctx, str, name, p)
	if err != nil {
		return nil, err
	}
	if err := assign(dst, val); err != nil {
		return nil, err
	}
	return change(ctx, p.Parent(), str.Interface())
}

func clone(v reflect.Value) (reflect.Value, error) {
	var o reflect.Value
	switch v.Kind() {
	case reflect.Slice:
		o = reflect.MakeSlice(v.Type(), v.Len(), v.Len())
	case reflect.Map:
		o = reflect.MakeMap(v.Type())
	default:
		o = reflect.New(v.Type()).Elem()
	}
	return o, shallowCopy(o, v)
}

func shallowCopy(dst, src reflect.Value) error {
	switch dst.Kind() {
	case reflect.Ptr, reflect.Interface:
		if !src.IsNil() {
			o := reflect.New(src.Elem().Type())
			shallowCopy(o.Elem(), src.Elem())
			dst.Set(o)
		}

	case reflect.Slice, reflect.Array:
		reflect.Copy(dst, src)

	case reflect.Map:
		for _, k := range src.MapKeys() {
			val := src.MapIndex(k)
			dst.SetMapIndex(k, val)
		}

	default:
		dst.Set(src)
	}
	return nil
}

func assign(dst, src reflect.Value) error {
	if !dst.CanSet() {
		return fmt.Errorf("Value is unassignable")
	}

	return deep.Copy(dst.Addr().Interface(), src.Interface())
}
<commit_msg>gapis/resolve: Fix commands vanishing on edit.<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package resolve

import (
	"context"
	"fmt"
	"reflect"

	"github.com/google/gapid/core/data/deep"
	"github.com/google/gapid/gapis/api"
	"github.com/google/gapid/gapis/capture"
	"github.com/google/gapid/gapis/database"
	"github.com/google/gapid/gapis/messages"
	"github.com/google/gapid/gapis/service"
	"github.com/google/gapid/gapis/service/path"
)

// Set creates a copy of the capture referenced by the request's path, but
// with the object, value or memory at p replaced with v. The path returned is
// identical to p, but with the base changed to refer to the new capture.
func Set(ctx context.Context, p *path.Any, v interface{}) (*path.Any, error) {
	obj, err := database.Build(ctx, &SetResolvable{p, service.NewValue(v)})
	if err != nil {
		return nil, err
	}
	return obj.(*path.Any), nil
}

// Resolve implements the database.Resolver interface.
func (r *SetResolvable) Resolve(ctx context.Context) (interface{}, error) {
	if c := path.FindCapture(r.Path.Node()); c != nil {
		ctx = capture.Put(ctx, c)
	}

	v, err := serviceToInternal(r.Value.Get())
	if err != nil {
		return nil, err
	}

	p, err := change(ctx, r.Path.Node(), v)
	if err != nil {
		return nil, err
	}
	return p.Path(), nil
}

func change(ctx context.Context, p path.Node, val interface{}) (path.Node, error) {
	switch p := p.(type) {
	case *path.Report:
		return nil, fmt.Errorf("Reports are immutable")

	case *path.ResourceData:
		meta, err := ResourceMeta(ctx, p.Id, p.After)
		if err != nil {
			return nil, err
		}

		cmdIdx := p.After.Indices[0]
		// If we change resource data, subcommands do not affect this, so change
		// the main comand.

		oldCmds, err := NCmds(ctx, p.After.Capture, cmdIdx+1)
		if err != nil {
			return nil, err
		}

		cmds := make([]api.Cmd, len(oldCmds))
		copy(cmds, oldCmds)

		replaceCommands := func(where uint64, with interface{}) {
			cmds[where] = with.(api.Cmd)
		}

		data, ok := val.(*api.ResourceData)
		if !ok {
			return nil, fmt.Errorf("Expected ResourceData, got %T", val)
		}

		if err := meta.Resource.SetResourceData(ctx, p.After, data, meta.IDMap, replaceCommands); err != nil {
			return nil, err
		}

		// Store the new command list
		c, err := changeCommands(ctx, p.After.Capture, cmds)
		if err != nil {
			return nil, err
		}

		return &path.ResourceData{
			Id: p.Id, // TODO: Shouldn't this change?
			After: &path.Command{
				Capture: c,
				Indices: p.After.Indices,
			},
		}, nil

	case *path.Command:
		cmdIdx := p.Indices[0]
		if len(p.Indices) > 1 {
			return nil, fmt.Errorf("Cannot modify subcommands") // TODO: Subcommands
		}

		// Resolve the command list
		oldCmds, err := NCmds(ctx, p.Capture, cmdIdx+1)
		if err != nil {
			return nil, err
		}

		// Validate the value
		if val == nil {
			return nil, fmt.Errorf("Command cannot be nil")
		}
		cmd, ok := val.(api.Cmd)
		if !ok {
			return nil, fmt.Errorf("Expected Cmd, got %T", val)
		}

		// Clone the command list
		cmds := make([]api.Cmd, len(oldCmds))
		copy(cmds, oldCmds)

		// Propagate extras if the new command omitted them
		oldCmd := oldCmds[cmdIdx]
		if len(cmd.Extras().All()) == 0 {
			cmd.Extras().Add(oldCmd.Extras().All()...)
		}

		// Propagate caller (not exposed to client)
		cmd.SetCaller(oldCmd.Caller())

		// Replace the command
		cmds[cmdIdx] = cmd

		// Store the new command list
		c, err := changeCommands(ctx, p.Capture, cmds)
		if err != nil {
			return nil, err
		}

		return &path.Command{
			Capture: c,
			Indices: p.Indices,
		}, nil

	case *path.Commands:
		return nil, fmt.Errorf("Commands can not be changed directly")

	case *path.State:
		return nil, fmt.Errorf("State can not currently be mutated")

	case *path.Field, *path.Parameter, *path.ArrayIndex, *path.MapIndex:
		oldObj, err := ResolveInternal(ctx, p.Parent())
		if err != nil {
			return nil, err
		}

		obj, err := clone(reflect.ValueOf(oldObj))
		if err != nil {
			return nil, err
		}

		switch p := p.(type) {
		case *path.Parameter:
			// TODO: Deal with parameters belonging to sub-commands.
			cmd := obj.Interface().(api.Cmd)
			err := api.SetParameter(ctx, cmd, p.Name, val)
			switch err {
			case nil:
			case api.ErrParameterNotFound:
				return nil, &service.ErrInvalidPath{
					Reason: messages.ErrParameterDoesNotExist(cmd.CmdName(), p.Name),
					Path:   p.Path(),
				}
			default:
				return nil, err
			}

			parent, err := change(ctx, p.Parent(), obj.Interface())
			if err != nil {
				return nil, err
			}
			return parent.(*path.Command).Parameter(p.Name), nil

		case *path.Result:
			// TODO: Deal with parameters belonging to sub-commands.
			cmd := obj.Interface().(api.Cmd)
			err := api.SetResult(ctx, cmd, val)
			switch err {
			case nil:
			case api.ErrResultNotFound:
				return nil, &service.ErrInvalidPath{
					Reason: messages.ErrResultDoesNotExist(cmd.CmdName()),
					Path:   p.Path(),
				}
			default:
				return nil, err
			}

			parent, err := change(ctx, p.Parent(), obj.Interface())
			if err != nil {
				return nil, err
			}
			return parent.(*path.Command).Result(), nil

		case *path.Field:
			parent, err := setField(ctx, obj, reflect.ValueOf(val), p.Name, p)
			if err != nil {
				return nil, err
			}
			out := &path.Field{Name: p.Name}
			out.SetParent(parent)
			return out, nil

		case *path.ArrayIndex:
			a, ty := obj, obj.Type()
			switch a.Kind() {
			case reflect.Array, reflect.Slice:
				ty = ty.Elem()
			case reflect.String:
			default:
				return nil, &service.ErrInvalidPath{
					Reason: messages.ErrTypeNotArrayIndexable(typename(a.Type())),
					Path:   p.Path(),
				}
			}
			val, ok := convert(reflect.ValueOf(val), ty)
			if !ok {
				return nil, fmt.Errorf("Slice or array at %s has element of type %v, got type %v",
					p.Parent(), ty, val.Type())
			}
			if count := uint64(a.Len()); p.Index >= count {
				return nil, errPathOOB(p.Index, "Index", 0, count-1, p)
			}
			if err := assign(a.Index(int(p.Index)), val); err != nil {
				return nil, err
			}
			parent, err := change(ctx, p.Parent(), a.Interface())
			if err != nil {
				return nil, err
			}
			p = &path.ArrayIndex{Index: p.Index}
			p.SetParent(parent)
			return p, nil

		case *path.MapIndex:
			m := obj
			if m.Kind() != reflect.Map {
				return nil, &service.ErrInvalidPath{
					Reason: messages.ErrTypeNotMapIndexable(typename(m.Type())),
					Path:   p.Path(),
				}
			}
			key, ok := convert(reflect.ValueOf(p.KeyValue()), m.Type().Key())
			if !ok {
				return nil, &service.ErrInvalidPath{
					Reason: messages.ErrIncorrectMapKeyType(
						typename(reflect.TypeOf(p.KeyValue())), // got
						typename(m.Type().Key())),              // expected
					Path: p.Path(),
				}
			}
			val, ok := convert(reflect.ValueOf(val), m.Type().Elem())
			if !ok {
				return nil, fmt.Errorf("Map at %s has value of type %v, got type %v",
					p.Parent(), m.Type().Elem(), val.Type())
			}
			m.SetMapIndex(key, val)
			parent, err := change(ctx, p.Parent(), m.Interface())
			if err != nil {
				return nil, err
			}
			p = &path.MapIndex{Key: p.Key}
			p.SetParent(parent)
			return p, nil
		}
	}
	return nil, fmt.Errorf("Unknown path type %T", p)
}

func changeCommands(ctx context.Context, p *path.Capture, newCmds []api.Cmd) (*path.Capture, error) {
	old, err := capture.ResolveFromPath(ctx, p)
	if err != nil {
		return nil, err
	}
	c, err := capture.New(ctx, old.Name+"*", old.Header, newCmds)
	if err != nil {
		return nil, err
	}
	return c, nil
}

func setField(ctx context.Context, str, val reflect.Value, name string, p path.Node) (path.Node, error) {
	dst, err := field(ctx, str, name, p)
	if err != nil {
		return nil, err
	}
	if err := assign(dst, val); err != nil {
		return nil, err
	}
	return change(ctx, p.Parent(), str.Interface())
}

func clone(v reflect.Value) (reflect.Value, error) {
	var o reflect.Value
	switch v.Kind() {
	case reflect.Slice:
		o = reflect.MakeSlice(v.Type(), v.Len(), v.Len())
	case reflect.Map:
		o = reflect.MakeMap(v.Type())
	default:
		o = reflect.New(v.Type()).Elem()
	}
	return o, shallowCopy(o, v)
}

func shallowCopy(dst, src reflect.Value) error {
	switch dst.Kind() {
	case reflect.Ptr, reflect.Interface:
		if !src.IsNil() {
			o := reflect.New(src.Elem().Type())
			shallowCopy(o.Elem(), src.Elem())
			dst.Set(o)
		}

	case reflect.Slice, reflect.Array:
		reflect.Copy(dst, src)

	case reflect.Map:
		for _, k := range src.MapKeys() {
			val := src.MapIndex(k)
			dst.SetMapIndex(k, val)
		}

	default:
		dst.Set(src)
	}
	return nil
}

func assign(dst, src reflect.Value) error {
	if !dst.CanSet() {
		return fmt.Errorf("Value is unassignable")
	}

	return deep.Copy(dst.Addr().Interface(), src.Interface())
}
<|endoftext|>


<commit_before>// +build windows

package mpwindowsprocessstats

import (
	"flag"
	"fmt"
	"os"
	"regexp"
	"sort"
	"strconv"

	"github.com/StackExchange/wmi"
	mp "github.com/mackerelio/go-mackerel-plugin-helper"
	"github.com/mackerelio/golib/logging"
)

var logger = logging.GetLogger("metrics.plugin.windows-process-stats")

type Win32_PerfFormattedData_PerfProc_Process struct {
	ElapsedTime          uint64
	Name                 string
	IDProcess            uint32
	PercentProcessorTime uint64
	WorkingSet           uint64
}

type WindowsProcessStatsPlugin struct {
	Process string
	Prefix  string
}

func getProcesses(processName string) ([]Win32_PerfFormattedData_PerfProc_Process, error) {
	var procs []Win32_PerfFormattedData_PerfProc_Process

	q := wmi.CreateQuery(&procs, "WHERE Name like '"+processName+"%'")
	if err := wmi.Query(q, &procs); err != nil {
		return procs, err
	}

	sort.Slice(procs, func(i, j int) bool {
		return procs[i].IDProcess < procs[j].IDProcess
	})
	return procs, nil
}

// FetchMetrics interface for mackerelplugin
func (m WindowsProcessStatsPlugin) FetchMetrics() (map[string]interface{}, error) {
	procs, err := getProcesses(m.Process)
	if err != nil {
		return nil, err
	}
	stat := make(map[string]interface{})
	prefix := m.MetricKeyPrefix()
	var re = regexp.MustCompile(`#[0-9]+$`)
	for k, v := range procs {
		name := re.ReplaceAllString(v.Name, "")
		processName := name + "_" + strconv.Itoa(k)
		metricNameCPU := prefix + "-windows-process-stats.cpu." + processName + ".cpu"
		metricNameMemory := prefix + "-windows-process-stats.memory." + processName + ".working_set"
		stat[metricNameCPU] = v.PercentProcessorTime
		stat[metricNameMemory] = v.WorkingSet
	}

	return stat, nil
}

// GraphDefinition interface for mackerelplugin
func (m WindowsProcessStatsPlugin) GraphDefinition() map[string](mp.Graphs) {
	prefix := m.MetricKeyPrefix()
	return map[string](mp.Graphs){
		fmt.Sprintf("%s-windows-process-stats.cpu.#", prefix): mp.Graphs{
			Label: fmt.Sprintf("%s Windows Process Stats CPU", prefix),
			Unit:  "percentage",
			Metrics: []mp.Metrics{
				{Name: "cpu", Label: "cpu", Diff: false, Stacked: false},
			},
		},
		fmt.Sprintf("%s-windows-process-stats.memory.#", prefix): mp.Graphs{
			Label: fmt.Sprintf("%s Windows Process Stats Memory", prefix),
			Unit:  "bytes",
			Metrics: []mp.Metrics{
				{Name: "working_set", Label: "memory", Diff: false, Stacked: false},
			},
		},
	}
}

// MetricKeyPrefix interface for mackerelplugin
func (m WindowsProcessStatsPlugin) MetricKeyPrefix() string {
	if m.Prefix == "" {
		return m.Process
	}
	return m.Prefix
}

// Do the plugin
func Do() {
	optProcess := flag.String("process", "", "Process name")
	optMetricKeyPrefix := flag.String("metric-key-prefix", "", "Metric Key Prefix")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	if *optProcess == "" {
		logger.Warningf("Process name is required")
		flag.PrintDefaults()
		os.Exit(1)
	}

	var plugin WindowsProcessStatsPlugin
	plugin.Process = *optProcess
	plugin.Prefix = *optMetricKeyPrefix

	helper := mp.NewMackerelPlugin(plugin)
	helper.Tempfile = *optTempfile
	helper.Run()
}
<commit_msg>fit to field name of struct<commit_after>// +build windows

package mpwindowsprocessstats

import (
	"flag"
	"fmt"
	"os"
	"regexp"
	"sort"
	"strconv"

	"github.com/StackExchange/wmi"
	mp "github.com/mackerelio/go-mackerel-plugin-helper"
	"github.com/mackerelio/golib/logging"
)

var logger = logging.GetLogger("metrics.plugin.windows-process-stats")

type Win32_PerfFormattedData_PerfProc_Process struct {
	ElapsedTime          uint64
	Name                 string
	IDProcess            uint32
	PercentProcessorTime uint64
	WorkingSet           uint64
}

type WindowsProcessStatsPlugin struct {
	Process string
	Prefix  string
}

func getProcesses(processName string) ([]Win32_PerfFormattedData_PerfProc_Process, error) {
	var procs []Win32_PerfFormattedData_PerfProc_Process

	q := wmi.CreateQuery(&procs, "WHERE Name like '"+processName+"%'")
	if err := wmi.Query(q, &procs); err != nil {
		return procs, err
	}

	sort.Slice(procs, func(i, j int) bool {
		return procs[i].IDProcess < procs[j].IDProcess
	})
	return procs, nil
}

// FetchMetrics interface for mackerelplugin
func (m WindowsProcessStatsPlugin) FetchMetrics() (map[string]interface{}, error) {
	procs, err := getProcesses(m.Process)
	if err != nil {
		return nil, err
	}
	stat := make(map[string]interface{})
	prefix := m.MetricKeyPrefix()
	var re = regexp.MustCompile(`#[0-9]+$`)
	for k, v := range procs {
		name := re.ReplaceAllString(v.Name, "")
		processName := name + "_" + strconv.Itoa(k)
		metricNameCPU := prefix + "-windows-process-stats.cpu." + processName + ".percent_processor_time"
		metricNameMemory := prefix + "-windows-process-stats.memory." + processName + ".working_set"
		stat[metricNameCPU] = v.PercentProcessorTime
		stat[metricNameMemory] = v.WorkingSet
	}

	return stat, nil
}

// GraphDefinition interface for mackerelplugin
func (m WindowsProcessStatsPlugin) GraphDefinition() map[string](mp.Graphs) {
	prefix := m.MetricKeyPrefix()
	return map[string](mp.Graphs){
		fmt.Sprintf("%s-windows-process-stats.cpu.#", prefix): mp.Graphs{
			Label: fmt.Sprintf("%s Windows Process Stats CPU", prefix),
			Unit:  "percentage",
			Metrics: []mp.Metrics{
				{Name: "percent_processor_time", Label: "cpu", Diff: false, Stacked: false},
			},
		},
		fmt.Sprintf("%s-windows-process-stats.memory.#", prefix): mp.Graphs{
			Label: fmt.Sprintf("%s Windows Process Stats Memory", prefix),
			Unit:  "bytes",
			Metrics: []mp.Metrics{
				{Name: "working_set", Label: "memory", Diff: false, Stacked: false},
			},
		},
	}
}

// MetricKeyPrefix interface for mackerelplugin
func (m WindowsProcessStatsPlugin) MetricKeyPrefix() string {
	if m.Prefix == "" {
		return m.Process
	}
	return m.Prefix
}

// Do the plugin
func Do() {
	optProcess := flag.String("process", "", "Process name")
	optMetricKeyPrefix := flag.String("metric-key-prefix", "", "Metric Key Prefix")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	if *optProcess == "" {
		logger.Warningf("Process name is required")
		flag.PrintDefaults()
		os.Exit(1)
	}

	var plugin WindowsProcessStatsPlugin
	plugin.Process = *optProcess
	plugin.Prefix = *optMetricKeyPrefix

	helper := mp.NewMackerelPlugin(plugin)
	helper.Tempfile = *optTempfile
	helper.Run()
}
<|endoftext|>


<commit_before>// Package generate implements functions that generate random distributions of points. Each function
// adds noise with a 2-D Normal (Gaussian) distribution.
package generate

import (
	"math"
	"math/rand"
)

// Point generates points around a given point.
func Point(n int, p [2]float64, stdDev float64) [][2]float64 {
	output := make([][2]float64, n)
	for i := range output {
		output[i][0] = p[0] + rand.NormFloat64()*stdDev
		output[i][1] = p[1] + rand.NormFloat64()*stdDev
	}
	return output
}

// CircularArc generates points along a circular arc, i.e. part of a circle. The center and radius
// arguments specify the circle; from and to are angles in radians that specify what part of the
// circule to include.
func CircularArc(n int, center [2]float64, radius, from, to float64, stdDev float64) [][2]float64 {
	output := make([][2]float64, n)
	d := to - from
	for i := range output {
		theta := from + d*rand.Float64()
		output[i][0] = center[0] + radius*math.Sin(theta) + rand.NormFloat64()*stdDev
		output[i][1] = center[1] + radius*math.Cos(theta) + rand.NormFloat64()*stdDev
	}
	return output
}
<commit_msg>Fix typo<commit_after>// Package generate implements functions that generate random distributions of points. Each function
// adds noise with a 2-D Normal (Gaussian) distribution.
package generate

import (
	"math"
	"math/rand"
)

// Point generates points around a given point.
func Point(n int, p [2]float64, stdDev float64) [][2]float64 {
	output := make([][2]float64, n)
	for i := range output {
		output[i][0] = p[0] + rand.NormFloat64()*stdDev
		output[i][1] = p[1] + rand.NormFloat64()*stdDev
	}
	return output
}

// CircularArc generates points along a circular arc, i.e. part of a circle. The center and radius
// arguments specify the circle; from and to are angles in radians that specify what part of the
// circle to include.
func CircularArc(n int, center [2]float64, radius, from, to float64, stdDev float64) [][2]float64 {
	output := make([][2]float64, n)
	d := to - from
	for i := range output {
		theta := from + d*rand.Float64()
		output[i][0] = center[0] + radius*math.Sin(theta) + rand.NormFloat64()*stdDev
		output[i][1] = center[1] + radius*math.Cos(theta) + rand.NormFloat64()*stdDev
	}
	return output
}
<|endoftext|>


<commit_before>package generator

import (
	"embed"
	"io/fs"
	"path/filepath"
)

//go:embed templates
var _bindata embed.FS

// AssetNames returns the names of the assets.
func AssetNames() []string {
	names := make([]string, 0)
	_ = filepath.WalkDir("templates", func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		names = append(names, path)
		return nil
	})
	return names
}

// Asset loads and returns the asset for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func Asset(name string) ([]byte, error) {
	return _bindata.ReadFile(name)
}

// MustAsset is like Asset but panics when Asset would return an error.
// It simplifies safe initialization of global variables.
func MustAsset(name string) []byte {
	a, err := Asset(name)
	if err != nil {
		panic("asset: Asset(" + name + "): " + err.Error())
	}

	return a
}
<commit_msg>fixed AssetNames<commit_after>package generator

import (
	"embed"
	"io/fs"
)

//go:embed templates
var _bindata embed.FS

// AssetNames returns the names of the assets.
func AssetNames() []string {
	names := make([]string, 0)
	_ = fs.WalkDir(_bindata, "templates", func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		names = append(names, path)
		return nil
	})
	return names
}

// Asset loads and returns the asset for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func Asset(name string) ([]byte, error) {
	return _bindata.ReadFile(name)
}

// MustAsset is like Asset but panics when Asset would return an error.
// It simplifies safe initialization of global variables.
func MustAsset(name string) []byte {
	a, err := Asset(name)
	if err != nil {
		panic("asset: Asset(" + name + "): " + err.Error())
	}

	return a
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package generator

import (
	"bytes"
	"fmt"
	"go/format"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"k8s.io/gengo/namer"
	"k8s.io/gengo/types"

	"github.com/golang/glog"
)

func errs2strings(errors []error) []string {
	strs := make([]string, len(errors))
	for i := range errors {
		strs[i] = errors[i].Error()
	}
	return strs
}

// ExecutePackages runs the generators for every package in 'packages'. 'outDir'
// is the base directory in which to place all the generated packages; it
// should be a physical path on disk, not an import path. e.g.:
// /path/to/home/path/to/gopath/src/
// Each package has its import path already, this will be appended to 'outDir'.
func (c *Context) ExecutePackages(outDir string, packages Packages) error {
	var errors []error
	for _, p := range packages {
		if err := c.ExecutePackage(outDir, p); err != nil {
			errors = append(errors, err)
		}
	}
	if len(errors) > 0 {
		return fmt.Errorf("some packages had errors:\n%v\n", strings.Join(errs2strings(errors), "\n"))
	}
	return nil
}

type DefaultFileType struct {
	Format   func([]byte) ([]byte, error)
	Assemble func(io.Writer, *File)
}

func (ft DefaultFileType) AssembleFile(f *File, pathname string) error {
	glog.V(2).Infof("Assembling file %q", pathname)
	destFile, err := os.Create(pathname)
	if err != nil {
		return err
	}
	defer destFile.Close()

	b := &bytes.Buffer{}
	et := NewErrorTracker(b)
	ft.Assemble(et, f)
	if et.Error() != nil {
		return et.Error()
	}
	if formatted, err := ft.Format(b.Bytes()); err != nil {
		err = fmt.Errorf("unable to format file %q (%v).", pathname, err)
		// Write the file anyway, so they can see what's going wrong and fix the generator.
		if _, err2 := destFile.Write(b.Bytes()); err2 != nil {
			return err2
		}
		return err
	} else {
		_, err = destFile.Write(formatted)
		return err
	}
}

func (ft DefaultFileType) VerifyFile(f *File, pathname string) error {
	glog.V(2).Infof("Verifying file %q", pathname)
	friendlyName := filepath.Join(f.PackageName, f.Name)
	b := &bytes.Buffer{}
	et := NewErrorTracker(b)
	ft.Assemble(et, f)
	if et.Error() != nil {
		return et.Error()
	}
	formatted, err := ft.Format(b.Bytes())
	if err != nil {
		return fmt.Errorf("unable to format the output for %q: %v", friendlyName, err)
	}
	existing, err := ioutil.ReadFile(pathname)
	if err != nil {
		return fmt.Errorf("unable to read file %q for comparison: %v", friendlyName, err)
	}
	if bytes.Compare(formatted, existing) == 0 {
		return nil
	}
	// Be nice and find the first place where they differ
	i := 0
	for i < len(formatted) && i < len(existing) && formatted[i] == existing[i] {
		i++
	}
	eDiff, fDiff := existing[i:], formatted[i:]
	if len(eDiff) > 100 {
		eDiff = eDiff[:100]
	}
	if len(fDiff) > 100 {
		fDiff = fDiff[:100]
	}
	return fmt.Errorf("output for %q differs; first existing/expected diff: \n  %q\n  %q", friendlyName, string(eDiff), string(fDiff))
}

func assembleGolangFile(w io.Writer, f *File) {
	w.Write(f.Header)
	fmt.Fprintf(w, "package %v\n\n", f.PackageName)

	if len(f.Imports) > 0 {
		fmt.Fprint(w, "import (\n")
		// TODO: sort imports like goimports does.
		for i := range f.Imports {
			if strings.Contains(i, "\"") {
				// they included quotes, or are using the
				// `name "path/to/pkg"` format.
				fmt.Fprintf(w, "\t%s\n", i)
			} else {
				fmt.Fprintf(w, "\t%q\n", i)
			}
		}
		fmt.Fprint(w, ")\n\n")
	}

	if f.Vars.Len() > 0 {
		fmt.Fprint(w, "var (\n")
		w.Write(f.Vars.Bytes())
		fmt.Fprint(w, ")\n\n")
	}

	if f.Consts.Len() > 0 {
		fmt.Fprint(w, "const (\n")
		w.Write(f.Consts.Bytes())
		fmt.Fprint(w, ")\n\n")
	}

	w.Write(f.Body.Bytes())
}

func NewGolangFile() *DefaultFileType {
	return &DefaultFileType{
		Format:   format.Source,
		Assemble: assembleGolangFile,
	}
}

// format should be one line only, and not end with \n.
func addIndentHeaderComment(b *bytes.Buffer, format string, args ...interface{}) {
	if b.Len() > 0 {
		fmt.Fprintf(b, "\n// "+format+"\n", args...)
	} else {
		fmt.Fprintf(b, "// "+format+"\n", args...)
	}
}

func (c *Context) filteredBy(f func(*Context, *types.Type) bool) *Context {
	c2 := *c
	c2.Order = []*types.Type{}
	for _, t := range c.Order {
		if f(c, t) {
			c2.Order = append(c2.Order, t)
		}
	}
	return &c2
}

// make a new context; inheret c.Namers, but add on 'namers'. In case of a name
// collision, the namer in 'namers' wins.
func (c *Context) addNameSystems(namers namer.NameSystems) *Context {
	if namers == nil {
		return c
	}
	c2 := *c
	// Copy the existing name systems so we don't corrupt a parent context
	c2.Namers = namer.NameSystems{}
	for k, v := range c.Namers {
		c2.Namers[k] = v
	}

	for name, namer := range namers {
		c2.Namers[name] = namer
	}
	return &c2
}

// ExecutePackage executes a single package. 'outDir' is the base directory in
// which to place the package; it should be a physical path on disk, not an
// import path. e.g.: '/path/to/home/path/to/gopath/src/' The package knows its
// import path already, this will be appended to 'outDir'.
func (c *Context) ExecutePackage(outDir string, p Package) error {
	path := filepath.Join(outDir, p.Path())
	glog.V(2).Infof("Processing package %q, disk location %q", p.Name(), path)
	// Filter out any types the *package* doesn't care about.
	packageContext := c.filteredBy(p.Filter)
	os.MkdirAll(path, 0755)
	files := map[string]*File{}
	for _, g := range p.Generators(packageContext) {
		// Filter out types the *generator* doesn't care about.
		genContext := packageContext.filteredBy(g.Filter)
		// Now add any extra name systems defined by this generator
		genContext = genContext.addNameSystems(g.Namers(genContext))

		fileType := g.FileType()
		if len(fileType) == 0 {
			return fmt.Errorf("generator %q must specify a file type", g.Name())
		}
		f := files[g.Filename()]
		if f == nil {
			// This is the first generator to reference this file, so start it.
			f = &File{
				Name:        g.Filename(),
				FileType:    fileType,
				PackageName: p.Name(),
				Header:      p.Header(g.Filename()),
				Imports:     map[string]struct{}{},
			}
			files[f.Name] = f
		} else {
			if f.FileType != g.FileType() {
				return fmt.Errorf("file %q already has type %q, but generator %q wants to use type %q", f.Name, f.FileType, g.Name(), g.FileType())
			}
		}

		if vars := g.PackageVars(genContext); len(vars) > 0 {
			addIndentHeaderComment(&f.Vars, "Package-wide variables from generator %q.", g.Name())
			for _, v := range vars {
				if _, err := fmt.Fprintf(&f.Vars, "%s\n", v); err != nil {
					return err
				}
			}
		}
		if consts := g.PackageVars(genContext); len(consts) > 0 {
			addIndentHeaderComment(&f.Consts, "Package-wide consts from generator %q.", g.Name())
			for _, v := range consts {
				if _, err := fmt.Fprintf(&f.Consts, "%s\n", v); err != nil {
					return err
				}
			}
		}
		if err := genContext.executeBody(&f.Body, g); err != nil {
			return err
		}
		if imports := g.Imports(genContext); len(imports) > 0 {
			for _, i := range imports {
				f.Imports[i] = struct{}{}
			}
		}
	}

	var errors []error
	for _, f := range files {
		finalPath := filepath.Join(path, f.Name)
		assembler, ok := c.FileTypes[f.FileType]
		if !ok {
			return fmt.Errorf("the file type %q registered for file %q does not exist in the context", f.FileType, f.Name)
		}
		var err error
		if c.Verify {
			err = assembler.VerifyFile(f, finalPath)
		} else {
			err = assembler.AssembleFile(f, finalPath)
		}
		if err != nil {
			errors = append(errors, err)
		}
	}
	if len(errors) > 0 {
		return fmt.Errorf("errors in package %q:\n%v\n", p.Path(), strings.Join(errs2strings(errors), "\n"))
	}
	return nil
}

func (c *Context) executeBody(w io.Writer, generator Generator) error {
	et := NewErrorTracker(w)
	if err := generator.Init(c, et); err != nil {
		return err
	}
	for _, t := range c.Order {
		if err := generator.GenerateType(c, t, et); err != nil {
			return err
		}
	}
	if err := generator.Finalize(c, et); err != nil {
		return err
	}
	return et.Error()
}
<commit_msg>Fix bug in generating vars / consts<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package generator

import (
	"bytes"
	"fmt"
	"go/format"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"k8s.io/gengo/namer"
	"k8s.io/gengo/types"

	"github.com/golang/glog"
)

func errs2strings(errors []error) []string {
	strs := make([]string, len(errors))
	for i := range errors {
		strs[i] = errors[i].Error()
	}
	return strs
}

// ExecutePackages runs the generators for every package in 'packages'. 'outDir'
// is the base directory in which to place all the generated packages; it
// should be a physical path on disk, not an import path. e.g.:
// /path/to/home/path/to/gopath/src/
// Each package has its import path already, this will be appended to 'outDir'.
func (c *Context) ExecutePackages(outDir string, packages Packages) error {
	var errors []error
	for _, p := range packages {
		if err := c.ExecutePackage(outDir, p); err != nil {
			errors = append(errors, err)
		}
	}
	if len(errors) > 0 {
		return fmt.Errorf("some packages had errors:\n%v\n", strings.Join(errs2strings(errors), "\n"))
	}
	return nil
}

type DefaultFileType struct {
	Format   func([]byte) ([]byte, error)
	Assemble func(io.Writer, *File)
}

func (ft DefaultFileType) AssembleFile(f *File, pathname string) error {
	glog.V(2).Infof("Assembling file %q", pathname)
	destFile, err := os.Create(pathname)
	if err != nil {
		return err
	}
	defer destFile.Close()

	b := &bytes.Buffer{}
	et := NewErrorTracker(b)
	ft.Assemble(et, f)
	if et.Error() != nil {
		return et.Error()
	}
	if formatted, err := ft.Format(b.Bytes()); err != nil {
		err = fmt.Errorf("unable to format file %q (%v).", pathname, err)
		// Write the file anyway, so they can see what's going wrong and fix the generator.
		if _, err2 := destFile.Write(b.Bytes()); err2 != nil {
			return err2
		}
		return err
	} else {
		_, err = destFile.Write(formatted)
		return err
	}
}

func (ft DefaultFileType) VerifyFile(f *File, pathname string) error {
	glog.V(2).Infof("Verifying file %q", pathname)
	friendlyName := filepath.Join(f.PackageName, f.Name)
	b := &bytes.Buffer{}
	et := NewErrorTracker(b)
	ft.Assemble(et, f)
	if et.Error() != nil {
		return et.Error()
	}
	formatted, err := ft.Format(b.Bytes())
	if err != nil {
		return fmt.Errorf("unable to format the output for %q: %v", friendlyName, err)
	}
	existing, err := ioutil.ReadFile(pathname)
	if err != nil {
		return fmt.Errorf("unable to read file %q for comparison: %v", friendlyName, err)
	}
	if bytes.Compare(formatted, existing) == 0 {
		return nil
	}
	// Be nice and find the first place where they differ
	i := 0
	for i < len(formatted) && i < len(existing) && formatted[i] == existing[i] {
		i++
	}
	eDiff, fDiff := existing[i:], formatted[i:]
	if len(eDiff) > 100 {
		eDiff = eDiff[:100]
	}
	if len(fDiff) > 100 {
		fDiff = fDiff[:100]
	}
	return fmt.Errorf("output for %q differs; first existing/expected diff: \n  %q\n  %q", friendlyName, string(eDiff), string(fDiff))
}

func assembleGolangFile(w io.Writer, f *File) {
	w.Write(f.Header)
	fmt.Fprintf(w, "package %v\n\n", f.PackageName)

	if len(f.Imports) > 0 {
		fmt.Fprint(w, "import (\n")
		// TODO: sort imports like goimports does.
		for i := range f.Imports {
			if strings.Contains(i, "\"") {
				// they included quotes, or are using the
				// `name "path/to/pkg"` format.
				fmt.Fprintf(w, "\t%s\n", i)
			} else {
				fmt.Fprintf(w, "\t%q\n", i)
			}
		}
		fmt.Fprint(w, ")\n\n")
	}

	if f.Vars.Len() > 0 {
		fmt.Fprint(w, "var (\n")
		w.Write(f.Vars.Bytes())
		fmt.Fprint(w, ")\n\n")
	}

	if f.Consts.Len() > 0 {
		fmt.Fprint(w, "const (\n")
		w.Write(f.Consts.Bytes())
		fmt.Fprint(w, ")\n\n")
	}

	w.Write(f.Body.Bytes())
}

func NewGolangFile() *DefaultFileType {
	return &DefaultFileType{
		Format:   format.Source,
		Assemble: assembleGolangFile,
	}
}

// format should be one line only, and not end with \n.
func addIndentHeaderComment(b *bytes.Buffer, format string, args ...interface{}) {
	if b.Len() > 0 {
		fmt.Fprintf(b, "\n// "+format+"\n", args...)
	} else {
		fmt.Fprintf(b, "// "+format+"\n", args...)
	}
}

func (c *Context) filteredBy(f func(*Context, *types.Type) bool) *Context {
	c2 := *c
	c2.Order = []*types.Type{}
	for _, t := range c.Order {
		if f(c, t) {
			c2.Order = append(c2.Order, t)
		}
	}
	return &c2
}

// make a new context; inheret c.Namers, but add on 'namers'. In case of a name
// collision, the namer in 'namers' wins.
func (c *Context) addNameSystems(namers namer.NameSystems) *Context {
	if namers == nil {
		return c
	}
	c2 := *c
	// Copy the existing name systems so we don't corrupt a parent context
	c2.Namers = namer.NameSystems{}
	for k, v := range c.Namers {
		c2.Namers[k] = v
	}

	for name, namer := range namers {
		c2.Namers[name] = namer
	}
	return &c2
}

// ExecutePackage executes a single package. 'outDir' is the base directory in
// which to place the package; it should be a physical path on disk, not an
// import path. e.g.: '/path/to/home/path/to/gopath/src/' The package knows its
// import path already, this will be appended to 'outDir'.
func (c *Context) ExecutePackage(outDir string, p Package) error {
	path := filepath.Join(outDir, p.Path())
	glog.V(2).Infof("Processing package %q, disk location %q", p.Name(), path)
	// Filter out any types the *package* doesn't care about.
	packageContext := c.filteredBy(p.Filter)
	os.MkdirAll(path, 0755)
	files := map[string]*File{}
	for _, g := range p.Generators(packageContext) {
		// Filter out types the *generator* doesn't care about.
		genContext := packageContext.filteredBy(g.Filter)
		// Now add any extra name systems defined by this generator
		genContext = genContext.addNameSystems(g.Namers(genContext))

		fileType := g.FileType()
		if len(fileType) == 0 {
			return fmt.Errorf("generator %q must specify a file type", g.Name())
		}
		f := files[g.Filename()]
		if f == nil {
			// This is the first generator to reference this file, so start it.
			f = &File{
				Name:        g.Filename(),
				FileType:    fileType,
				PackageName: p.Name(),
				Header:      p.Header(g.Filename()),
				Imports:     map[string]struct{}{},
			}
			files[f.Name] = f
		} else {
			if f.FileType != g.FileType() {
				return fmt.Errorf("file %q already has type %q, but generator %q wants to use type %q", f.Name, f.FileType, g.Name(), g.FileType())
			}
		}

		if vars := g.PackageVars(genContext); len(vars) > 0 {
			addIndentHeaderComment(&f.Vars, "Package-wide variables from generator %q.", g.Name())
			for _, v := range vars {
				if _, err := fmt.Fprintf(&f.Vars, "%s\n", v); err != nil {
					return err
				}
			}
		}
		if consts := g.PackageConsts(genContext); len(consts) > 0 {
			addIndentHeaderComment(&f.Consts, "Package-wide consts from generator %q.", g.Name())
			for _, v := range consts {
				if _, err := fmt.Fprintf(&f.Consts, "%s\n", v); err != nil {
					return err
				}
			}
		}
		if err := genContext.executeBody(&f.Body, g); err != nil {
			return err
		}
		if imports := g.Imports(genContext); len(imports) > 0 {
			for _, i := range imports {
				f.Imports[i] = struct{}{}
			}
		}
	}

	var errors []error
	for _, f := range files {
		finalPath := filepath.Join(path, f.Name)
		assembler, ok := c.FileTypes[f.FileType]
		if !ok {
			return fmt.Errorf("the file type %q registered for file %q does not exist in the context", f.FileType, f.Name)
		}
		var err error
		if c.Verify {
			err = assembler.VerifyFile(f, finalPath)
		} else {
			err = assembler.AssembleFile(f, finalPath)
		}
		if err != nil {
			errors = append(errors, err)
		}
	}
	if len(errors) > 0 {
		return fmt.Errorf("errors in package %q:\n%v\n", p.Path(), strings.Join(errs2strings(errors), "\n"))
	}
	return nil
}

func (c *Context) executeBody(w io.Writer, generator Generator) error {
	et := NewErrorTracker(w)
	if err := generator.Init(c, et); err != nil {
		return err
	}
	for _, t := range c.Order {
		if err := generator.GenerateType(c, t, et); err != nil {
			return err
		}
	}
	if err := generator.Finalize(c, et); err != nil {
		return err
	}
	return et.Error()
}
<|endoftext|>


<commit_before>package system

import (
	"errors"
	"regexp"
	"sort"
	"sync"
)

// HandlerFunc ...
type HandlerFunc func(*Context)

//////////////////////////////////
// 		COMMAND ROUTER
/////////////////////////////////

// CommandRouter ...
type CommandRouter struct {
	sync.Mutex
	CurrentCategory string

	// Prefix is appended to the beginning of each command added with On
	Prefix string
	// Suffix is appended to the end of each command added with On
	Suffix string

	Routes     []*CommandRoute
	Subrouters []*SubCommandRouter
}

// NewCommandRouter ..,
func NewCommandRouter() *CommandRouter {
	return &CommandRouter{
		Routes:     []*CommandRoute{},
		Subrouters: []*SubCommandRouter{},
	}
}

// On adds a command router to the list of routes.
//		matcher: The regular expression to use when searching for this route.
//		handler: The handler function for this command route.
func (c *CommandRouter) On(matcher string, handler HandlerFunc) *CommandRoute {

	// Specify that the matched text must be at the beginning and end in a whitespace character
	// Or end of line.
	reg := "^" + c.Prefix + matcher + c.Suffix + `(\s|$)`

	route := &CommandRoute{
		Matcher:  regexp.MustCompile(reg),
		Handler:  handler,
		Name:     matcher,
		Category: c.CurrentCategory,
	}

	c.Lock()
	c.Routes = append(c.Routes, route)
	c.Unlock()

	return route
}

// SetCategory sets the routers current category
//		name: the name of the category to add new routes to by default
func (c *CommandRouter) SetCategory(name string) {
	c.Lock()
	c.CurrentCategory = name
	c.Unlock()
}

// OnReg allows you to supply a custom regular expression as the route matcher.
//		matcher: The regular expression to use when searching for this route
//		handler: The handler function for this command route.
func (c *CommandRouter) OnReg(matcher string, handler HandlerFunc) *CommandRoute {
	route := &CommandRoute{
		Matcher: regexp.MustCompile(matcher),
		Handler: handler,
		Name:    matcher,
	}

	c.Lock()
	c.Routes = append(c.Routes, route)
	c.Unlock()

	return route
}

// Off removes a CommandRoute from the list of routes and returns a pointer
// To the removed value.
//		name:	The regular expression to match against
func (c *CommandRouter) Off(name string) *CommandRoute {
	c.Lock()
	defer c.Unlock()

	for i, v := range c.Routes {
		if v.Matcher.MatchString(name) {
			c.Routes = append(c.Routes[:i], c.Routes[i+1:]...)
			return v
		}
	}

	return nil
}

// SetDisabled sets the specified command to disabled
func (c *CommandRouter) SetDisabled(name string, disabled bool) error {
	if route, _ := c.FindMatch(name); route != nil {
		route.Disabled = disabled
	}
	return errors.New("route not found")
}

// AddSubrouter adds a subrouter to the list of subrouters.
func (c *CommandRouter) AddSubrouter(subrouter *SubCommandRouter) *SubCommandRouter {

	// Set the default category to this routers current category.
	if subrouter.Category() == "" {
		subrouter.SetCategory(c.CurrentCategory)
	}

	c.Lock()
	c.Subrouters = append(c.Subrouters, subrouter)
	c.Unlock()

	return subrouter
}

// FindMatch returns the first match found
//		name: The name of the route to find
func (c *CommandRouter) findMatch(name string, skipDisabled bool) (*CommandRoute, []int) {

	for _, route := range c.Routes {
		if skipDisabled && route.Disabled == true {
			continue
		}
		if loc := route.Matcher.FindStringIndex(name); loc != nil {
			return route, loc
		}
	}

	for _, v := range c.Subrouters {
		if loc := v.Matcher.FindStringIndex(name); loc != nil {
			if match, loc2 := v.Router.findMatch(name[loc[1]:], skipDisabled); match != nil {
				return match, []int{loc[0], loc[1] + loc2[1]}
			}

			if skipDisabled && v.CommandRoute != nil && v.CommandRoute.Disabled == true {
				continue
			}

			// Return the subrouters command route if nothing is found
			return v.CommandRoute, loc
		}
	}

	return nil, nil
}

// FindMatch returns the first match that matches the given string
//		name: The name of the route to find
func (c *CommandRouter) FindMatch(name string) (*CommandRoute, []int) {
	return c.findMatch(name, false)
}

// FindEnabledMatch returns the first non-disabled route that matches the given string
//		name: The name of the route to find
func (c *CommandRouter) FindEnabledMatch(name string) (*CommandRoute, []int) {
	return c.findMatch(name, true)
}

// TODO Return an array of match locations

// FindMatches will return all commands matching the given string
//		name: The name of the route to find
// func (c *CommandRouter) FindMatches(name string) []*CommandRoute {
// 	matches := []*CommandRoute{}

// 	// Search routes
// 	for _, route := range c.Routes {
// 		if route.Matcher.MatchString(name) {
// 			matches = append(matches, route)
// 		}
// 	}

// 	// Search subrouters
// 	for _, v := range c.Subrouters {
// 		if v.Matcher.MatchString(name) {
// 			if route, _ := v.Router.FindMatch(name); route != nil {
// 				matches = append(matches, route)
// 			} else if v.CommandRoute != nil {
// 				matches = append(matches, v.CommandRoute)
// 			}
// 		}
// 	}

// 	return matches
// }

// GetAllRoutes returns all routes including the routes
// of this routers subrouters.
func (c *CommandRouter) GetAllRoutes() []*CommandRoute {

	var find func(router *CommandRouter) []*CommandRoute
	find = func(router *CommandRouter) []*CommandRoute {
		routes := []*CommandRoute{}

		for _, v := range router.Routes {
			routes = append(routes, v)
		}

		for _, v := range router.Subrouters {
			if v.CommandRoute != nil {
				routes = append(routes, v.CommandRoute)
			}
			routes = append(routes, find(v.Router)...)
		}

		return routes
	}

	return find(c)
}

//////////////////////////////////
// 		SUB COMMAND ROUTER
/////////////////////////////////

// SubCommandRouter is a subrouter for commands
type SubCommandRouter struct {
	Matcher *regexp.Regexp
	Router  *CommandRouter
	Name    string

	// CommandRoute is retrieved when there are no matching routes found under the subrouter,
	// But the subrouter was matched.
	CommandRoute *CommandRoute
}

// NewSubCommandRouter returns a pointer to a new SubCommandRouter
//		matcher: The regular expression to use when matching for commands.
//				 to match everything..
//
//		name: 	 The name to give the subrouter.
func NewSubCommandRouter(matcher string, name string) (*SubCommandRouter, error) {
	reg, err := regexp.Compile(matcher)
	if err != nil {
		return nil, err
	}

	router := NewCommandRouter()
	router.Prefix = " "
	// Set the prefix to be space separated by default.

	return &SubCommandRouter{
		Matcher:      reg,
		Router:       router,
		Name:         name,
		CommandRoute: nil,
	}, nil
}

// SetCategory sets the current category of the routers
func (s *SubCommandRouter) SetCategory(name string) {
	s.Router.SetCategory(name)
}

// Set sets the field values of the CommandRoute
// Accepts three fields:
//		1:	Name
//		2:  Description
//		3:  Category
func (s *SubCommandRouter) Set(values ...string) {
	if s.CommandRoute == nil {
		s.CommandRoute = &CommandRoute{}
	}

	switch {

	case len(values) > 2:
		if values[2] != "" {
			s.CommandRoute.Category = values[2]
		}
		fallthrough

	case len(values) > 1:
		if values[1] != "" {
			s.CommandRoute.Desc = values[1]
		}
		fallthrough

	case len(values) > 0:
		if values[0] != "" {
			s.CommandRoute.Name = values[0]
		}
	}
}

// Category returns the category of the subrouter
func (s *SubCommandRouter) Category() string {
	if s.Router != nil {
		return s.Router.CurrentCategory
	}
	return ""
}

//////////////////////////////////
// 		COMMAND ROUTE
/////////////////////////////////

// CommandRoute ...
type CommandRoute struct {
	Matcher  *regexp.Regexp
	Handler  HandlerFunc
	Name     string
	Desc     string
	Category string
	Disabled bool
}

// Set sets the field values of the CommandRoute
// Accepts three fields:
//		1:	Name
//		2:  Description
//		3:  Category
func (c *CommandRoute) Set(values ...string) {
	switch {

	case len(values) > 2:
		if values[2] != "" {
			c.Category = values[2]
		}
		fallthrough

	case len(values) > 1:
		if values[1] != "" {
			c.Desc = values[1]
		}
		fallthrough

	case len(values) > 0:
		if values[0] != "" {
			c.Name = values[0]
		}
	}

}

//////////////////////////////////
// 		SORTING BY CATEGORY
/////////////////////////////////

// CommandRoutesByCategory implements the sort.Sortable interface
// To allow CommandRouters to be sorted in alphabetical order based on their
// Category.
type CommandRoutesByCategory []*CommandRoute

func (c CommandRoutesByCategory) Swap(a, b int) {
	c[a], c[b] = c[b], c[a]
}

// Len implements the sorter.Sortable interface
func (c CommandRoutesByCategory) Len() int {
	return len(c)
}

// Less implements the sorter.Sortable interface
func (c CommandRoutesByCategory) Less(a, b int) bool {
	return c[a].Category < c[b].Category
}

// Group splits the CommandRouters into separate slices according to group
func (c CommandRoutesByCategory) Group() [][]*CommandRoute {
	var (
		groups       = [][]*CommandRoute{}
		lastCategory = "__undefined__"
		currentGroup = []*CommandRoute{}
	)

	sort.Sort(c)

	for _, v := range c {

		if v.Category != lastCategory {
			if len(currentGroup) > 0 {
				groups = append(groups, currentGroup)
				currentGroup = []*CommandRoute{}
			}
		}

		currentGroup = append(currentGroup, v)
	}

	if len(currentGroup) > 0 {
		groups = append(groups, currentGroup)
	}

	return groups
}
<commit_msg>Add diabled property to SubCommandrouter<commit_after>package system

import (
	"errors"
	"regexp"
	"sort"
	"sync"
)

// HandlerFunc ...
type HandlerFunc func(*Context)

//////////////////////////////////
// 		COMMAND ROUTER
/////////////////////////////////

// CommandRouter ...
type CommandRouter struct {
	sync.Mutex
	CurrentCategory string

	// Prefix is appended to the beginning of each command added with On
	Prefix string
	// Suffix is appended to the end of each command added with On
	Suffix string

	Routes     []*CommandRoute
	Subrouters []*SubCommandRouter
}

// NewCommandRouter ..,
func NewCommandRouter() *CommandRouter {
	return &CommandRouter{
		Routes:     []*CommandRoute{},
		Subrouters: []*SubCommandRouter{},
	}
}

// On adds a command router to the list of routes.
//		matcher: The regular expression to use when searching for this route.
//		handler: The handler function for this command route.
func (c *CommandRouter) On(matcher string, handler HandlerFunc) *CommandRoute {

	// Specify that the matched text must be at the beginning and end in a whitespace character
	// Or end of line.
	reg := "^" + c.Prefix + matcher + c.Suffix + `(\s|$)`

	route := &CommandRoute{
		Matcher:  regexp.MustCompile(reg),
		Handler:  handler,
		Name:     matcher,
		Category: c.CurrentCategory,
	}

	c.Lock()
	c.Routes = append(c.Routes, route)
	c.Unlock()

	return route
}

// SetCategory sets the routers current category
//		name: the name of the category to add new routes to by default
func (c *CommandRouter) SetCategory(name string) {
	c.Lock()
	c.CurrentCategory = name
	c.Unlock()
}

// OnReg allows you to supply a custom regular expression as the route matcher.
//		matcher: The regular expression to use when searching for this route
//		handler: The handler function for this command route.
func (c *CommandRouter) OnReg(matcher string, handler HandlerFunc) *CommandRoute {
	route := &CommandRoute{
		Matcher: regexp.MustCompile(matcher),
		Handler: handler,
		Name:    matcher,
	}

	c.Lock()
	c.Routes = append(c.Routes, route)
	c.Unlock()

	return route
}

// Off removes a CommandRoute from the list of routes and returns a pointer
// To the removed value.
//		name:	The regular expression to match against
func (c *CommandRouter) Off(name string) *CommandRoute {
	c.Lock()
	defer c.Unlock()

	for i, v := range c.Routes {
		if v.Matcher.MatchString(name) {
			c.Routes = append(c.Routes[:i], c.Routes[i+1:]...)
			return v
		}
	}

	return nil
}

// SetDisabled sets the specified command to disabled
func (c *CommandRouter) SetDisabled(name string, disabled bool) error {
	if route, _ := c.FindMatch(name); route != nil {
		route.Disabled = disabled
	}
	return errors.New("route not found")
}

// AddSubrouter adds a subrouter to the list of subrouters.
func (c *CommandRouter) AddSubrouter(subrouter *SubCommandRouter) *SubCommandRouter {

	// Set the default category to this routers current category.
	if subrouter.Category() == "" {
		subrouter.SetCategory(c.CurrentCategory)
	}

	c.Lock()
	c.Subrouters = append(c.Subrouters, subrouter)
	c.Unlock()

	return subrouter
}

// FindMatch returns the first match found
//		name: The name of the route to find
func (c *CommandRouter) findMatch(name string, skipDisabled bool) (*CommandRoute, []int) {

	for _, route := range c.Routes {
		if skipDisabled && route.Disabled == true {
			continue
		}
		if loc := route.Matcher.FindStringIndex(name); loc != nil {
			return route, loc
		}
	}

	for _, v := range c.Subrouters {
		if loc := v.Matcher.FindStringIndex(name); loc != nil {
			if match, loc2 := v.Router.findMatch(name[loc[1]:], skipDisabled); match != nil {
				return match, []int{loc[0], loc[1] + loc2[1]}
			}

			if skipDisabled && v.CommandRoute != nil && v.CommandRoute.Disabled == true {
				continue
			}

			// Return the subrouters command route if nothing is found
			return v.CommandRoute, loc
		}
	}

	return nil, nil
}

// FindMatch returns the first match that matches the given string
//		name: The name of the route to find
func (c *CommandRouter) FindMatch(name string) (*CommandRoute, []int) {
	return c.findMatch(name, false)
}

// FindEnabledMatch returns the first non-disabled route that matches the given string
//		name: The name of the route to find
func (c *CommandRouter) FindEnabledMatch(name string) (*CommandRoute, []int) {
	return c.findMatch(name, true)
}

// TODO Return an array of match locations

// FindMatches will return all commands matching the given string
//		name: The name of the route to find
// func (c *CommandRouter) FindMatches(name string) []*CommandRoute {
// 	matches := []*CommandRoute{}

// 	// Search routes
// 	for _, route := range c.Routes {
// 		if route.Matcher.MatchString(name) {
// 			matches = append(matches, route)
// 		}
// 	}

// 	// Search subrouters
// 	for _, v := range c.Subrouters {
// 		if v.Matcher.MatchString(name) {
// 			if route, _ := v.Router.FindMatch(name); route != nil {
// 				matches = append(matches, route)
// 			} else if v.CommandRoute != nil {
// 				matches = append(matches, v.CommandRoute)
// 			}
// 		}
// 	}

// 	return matches
// }

// GetAllRoutes returns all routes including the routes
// of this routers subrouters.
func (c *CommandRouter) GetAllRoutes() []*CommandRoute {

	var find func(router *CommandRouter) []*CommandRoute
	find = func(router *CommandRouter) []*CommandRoute {
		routes := []*CommandRoute{}

		for _, v := range router.Routes {
			routes = append(routes, v)
		}

		for _, v := range router.Subrouters {
			if v.CommandRoute != nil {
				routes = append(routes, v.CommandRoute)
			}
			routes = append(routes, find(v.Router)...)
		}

		return routes
	}

	return find(c)
}

//////////////////////////////////
// 		SUB COMMAND ROUTER
/////////////////////////////////

// SubCommandRouter is a subrouter for commands
type SubCommandRouter struct {
	Matcher  *regexp.Regexp
	Router   *CommandRouter
	Name     string
	Disabled bool

	// CommandRoute is retrieved when there are no matching routes found under the subrouter,
	// But the subrouter was matched.
	CommandRoute *CommandRoute
}

// NewSubCommandRouter returns a pointer to a new SubCommandRouter
//		matcher: The regular expression to use when matching for commands.
//				 to match everything..
//
//		name: 	 The name to give the subrouter.
func NewSubCommandRouter(matcher string, name string) (*SubCommandRouter, error) {
	reg, err := regexp.Compile(matcher)
	if err != nil {
		return nil, err
	}

	router := NewCommandRouter()
	router.Prefix = " "
	// Set the prefix to be space separated by default.

	return &SubCommandRouter{
		Matcher:      reg,
		Router:       router,
		Name:         name,
		CommandRoute: nil,
	}, nil
}

// SetCategory sets the current category of the routers
func (s *SubCommandRouter) SetCategory(name string) {
	s.Router.SetCategory(name)
}

// Set sets the field values of the CommandRoute
// Accepts three fields:
//		1:	Name
//		2:  Description
//		3:  Category
func (s *SubCommandRouter) Set(values ...string) {
	if s.CommandRoute == nil {
		s.CommandRoute = &CommandRoute{}
	}

	switch {

	case len(values) > 2:
		if values[2] != "" {
			s.CommandRoute.Category = values[2]
		}
		fallthrough

	case len(values) > 1:
		if values[1] != "" {
			s.CommandRoute.Desc = values[1]
		}
		fallthrough

	case len(values) > 0:
		if values[0] != "" {
			s.CommandRoute.Name = values[0]
		}
	}
}

// Category returns the category of the subrouter
func (s *SubCommandRouter) Category() string {
	if s.Router != nil {
		return s.Router.CurrentCategory
	}
	return ""
}

//////////////////////////////////
// 		COMMAND ROUTE
/////////////////////////////////

// CommandRoute ...
type CommandRoute struct {
	Matcher  *regexp.Regexp
	Handler  HandlerFunc
	Name     string
	Desc     string
	Category string
	Disabled bool
}

// Set sets the field values of the CommandRoute
// Accepts three fields:
//		1:	Name
//		2:  Description
//		3:  Category
func (c *CommandRoute) Set(values ...string) {
	switch {

	case len(values) > 2:
		if values[2] != "" {
			c.Category = values[2]
		}
		fallthrough

	case len(values) > 1:
		if values[1] != "" {
			c.Desc = values[1]
		}
		fallthrough

	case len(values) > 0:
		if values[0] != "" {
			c.Name = values[0]
		}
	}

}

//////////////////////////////////
// 		SORTING BY CATEGORY
/////////////////////////////////

// CommandRoutesByCategory implements the sort.Sortable interface
// To allow CommandRouters to be sorted in alphabetical order based on their
// Category.
type CommandRoutesByCategory []*CommandRoute

func (c CommandRoutesByCategory) Swap(a, b int) {
	c[a], c[b] = c[b], c[a]
}

// Len implements the sorter.Sortable interface
func (c CommandRoutesByCategory) Len() int {
	return len(c)
}

// Less implements the sorter.Sortable interface
func (c CommandRoutesByCategory) Less(a, b int) bool {
	return c[a].Category < c[b].Category
}

// Group splits the CommandRouters into separate slices according to group
func (c CommandRoutesByCategory) Group() [][]*CommandRoute {
	var (
		groups       = [][]*CommandRoute{}
		lastCategory = "__undefined__"
		currentGroup = []*CommandRoute{}
	)

	sort.Sort(c)

	for _, v := range c {

		if v.Category != lastCategory {
			if len(currentGroup) > 0 {
				groups = append(groups, currentGroup)
				currentGroup = []*CommandRoute{}
			}
		}

		currentGroup = append(currentGroup, v)
	}

	if len(currentGroup) > 0 {
		groups = append(groups, currentGroup)
	}

	return groups
}
<|endoftext|>


<commit_before>package geolite2v2

import (
	"archive/zip"
	"context"
	"errors"
	"log"
	"net"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/storage"
	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/annotation-service/iputils"
	"github.com/m-lab/annotation-service/loader"
	"github.com/m-lab/annotation-service/metrics"
)

const (
	mapMax = 200000
)

var (
	gLite2Prefix              = "GeoLite2-City"
	geoLite2BlocksFilenameIP4 = "GeoLite2-City-Blocks-IPv4.csv"  // Filename of ipv4 blocks file
	geoLite2BlocksFilenameIP6 = "GeoLite2-City-Blocks-IPv6.csv"  // Filename of ipv6 blocks file
	geoLite2LocationsFilename = "GeoLite2-City-Locations-en.csv" // Filename of locations file
)

// The GeoDataset struct bundles all the data needed to search and
// find data into one common structure
// It implements the api.Annotator interface.
type GeoDataset struct {
	Start         time.Time      // Date from which to start using this dataset
	IP4Nodes      []GeoIPNode    // The IPNode list containing IP4Nodes
	IP6Nodes      []GeoIPNode    // The IPNode list containing IP6Nodes
	LocationNodes []LocationNode // The location nodes corresponding to the IPNodes
}

// LoadG2 loads a dataset from a GCS object.
func LoadG2(file *storage.ObjectAttrs) (api.Annotator, error) {
	return LoadG2Dataset(file.Name, file.Bucket)
}

// LoadG2Dataset loads the dataset from the specified filename and bucket
func LoadG2Dataset(filename string, bucketname string) (*GeoDataset, error) {
	zip, err := loader.CreateZipReader(context.Background(), bucketname, filename)
	log.Println("Loading dataset from", filename)
	if err != nil {
		return nil, err
	}
	dataset, err := loadGeoLite2(zip)
	if err != nil {
		return nil, err
	}
	date, err := api.ExtractDateFromFilename(filename)
	if err != nil {
		log.Println("Error extracting date:", filename)
	} else {
		dataset.Start = date
	}
	return dataset, nil
}

// loadGeoLite2 composes the result of location and IPv4, IPv6 IPNode lists
func loadGeoLite2(zip *zip.Reader) (*GeoDataset, error) {
	locations, err := loader.FindFile(geoLite2LocationsFilename, zip)
	if err != nil {
		return nil, err
	}
	// geoidMap is just a temporary map that will be discarded once the blocks are parsed
	locationNode, geoidMap, err := LoadLocationsG2(locations)
	locations.Close()
	if err != nil {
		return nil, err
	}

	blocks4, err := loader.FindFile(geoLite2BlocksFilenameIP4, zip)
	if err != nil {
		return nil, err
	}
	ipNodes4, err := LoadIPListG2(blocks4, geoidMap)
	blocks4.Close()
	if err != nil {
		return nil, err
	}
	blocks6, err := loader.FindFile(geoLite2BlocksFilenameIP6, zip)
	if err != nil {
		return nil, err
	}
	ipNodes6, err := LoadIPListG2(blocks6, geoidMap)
	blocks6.Close()
	if err != nil {
		return nil, err
	}
	return &GeoDataset{IP4Nodes: ipNodes4, IP6Nodes: ipNodes6, LocationNodes: locationNode}, nil
}

// ConvertIPNodeToGeoData takes a parser.IPNode, plus a list of
// locationNodes. It will then use that data to fill in a GeoData struct.
func populateLocationData(ipNode iputils.IPNode, locationNodes []LocationNode, data *api.GeoData) {
	locNode := LocationNode{}
	geoIPNode := ipNode.(*GeoIPNode)

	if geoIPNode.LocationIndex >= 0 {
		locNode = locationNodes[geoIPNode.LocationIndex]
	}
	data.Geo = &api.GeolocationIP{
		ContinentCode: locNode.ContinentCode,
		CountryCode:   locNode.CountryCode,
		CountryCode3:  "", // missing from geoLite2 ?
		CountryName:   locNode.CountryName,
		Region:        locNode.RegionCode,
		MetroCode:     locNode.MetroCode,
		City:          locNode.CityName,
		AreaCode:      0, // new geoLite2 does not have area code.
		PostalCode:    geoIPNode.PostalCode,
		Latitude:      geoIPNode.Latitude,
		Longitude:     geoIPNode.Longitude,
	}
}

var lastLogTime = time.Time{}

// Annotate annotates the api.GeoData with the location informations
func (ds *GeoDataset) Annotate(ip string, data *api.GeoData) error {
	if data == nil {
		return errors.New("ErrNilGeoData") // TODO
	}
	if data.Geo != nil {
		return errors.New("ErrAlreadyPopulated") // TODO
	}
	parsedIP := net.ParseIP(ip)
	if parsedIP == nil {
		metrics.BadIPTotal.Inc()
		return errors.New("ErrInvalidIP") // TODO
	}
	ipNodes := ds.IP6Nodes
	if parsedIP.To4() != nil {
		ipNodes = ds.IP4Nodes
	}

	node, err := iputils.SearchBinary(ip,
		len(ipNodes),
		func(idx int) iputils.IPNode {
			return &ipNodes[idx]
		})

	if err != nil {
		// ErrNodeNotFound is super spammy - 10% of requests, so suppress those.
		if err != iputils.ErrNodeNotFound {
			// Horribly noisy now.
			if time.Since(lastLogTime) > time.Minute {
				log.Println(err, ip)
				lastLogTime = time.Now()
			}
		}
		//TODO metric here
		return err
	}

	populateLocationData(node, ds.LocationNodes, data)
	return nil
}

// AnnotatorDate returns the date that the dataset was published.
// TODO implement actual dataset time!!
func (ds *GeoDataset) AnnotatorDate() time.Time {
	return ds.Start
}

// IsEqualIPNodes returns nil if two nodes are equal
// Used by the search package
func IsEqualIPNodes(expectedIPNode, ipNode iputils.IPNode) error {
	expected, eok := expectedIPNode.(*GeoIPNode)
	node, nok := ipNode.(*GeoIPNode)
	if !eok || !nok {
		return errors.New("Illegal type of IPNode")
	}

	if !((node.IPAddressLow).Equal(expected.IPAddressLow)) {
		output := strings.Join([]string{"IPAddress Low inconsistent\ngot:", node.IPAddressLow.String(), " \nwanted:", expected.IPAddressLow.String()}, "")
		log.Println(output)
		return errors.New(output)
	}
	if !((node.IPAddressHigh).Equal(expected.IPAddressHigh)) {
		output := strings.Join([]string{"IPAddressHigh inconsistent\ngot:", node.IPAddressHigh.String(), " \nwanted:", expected.IPAddressHigh.String()}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.LocationIndex != expected.LocationIndex {
		output := strings.Join([]string{"LocationIndex inconsistent\ngot:", strconv.Itoa(node.LocationIndex), " \nwanted:", strconv.Itoa(expected.LocationIndex)}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.PostalCode != expected.PostalCode {
		output := strings.Join([]string{"PostalCode inconsistent\ngot:", node.PostalCode, " \nwanted:", expected.PostalCode}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.Latitude != expected.Latitude {
		output := strings.Join([]string{"Latitude inconsistent\ngot:", floatToString(node.Latitude), " \nwanted:", floatToString(expected.Latitude)}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.Longitude != expected.Longitude {
		output := strings.Join([]string{"Longitude inconsistent\ngot:", floatToString(node.Longitude), " \nwanted:", floatToString(expected.Longitude)}, "")
		log.Println(output)
		return errors.New(output)
	}
	return nil
}

func floatToString(num float64) string {
	return strconv.FormatFloat(num, 'f', 6, 64)
}
<commit_msg>implement SearchBinary<commit_after>package geolite2v2

import (
	"archive/zip"
	"context"
	"errors"
	"log"
	"net"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/storage"
	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/annotation-service/iputils"
	"github.com/m-lab/annotation-service/loader"
	"github.com/m-lab/annotation-service/metrics"
)

const (
	mapMax = 200000
)

var (
	gLite2Prefix              = "GeoLite2-City"
	geoLite2BlocksFilenameIP4 = "GeoLite2-City-Blocks-IPv4.csv"  // Filename of ipv4 blocks file
	geoLite2BlocksFilenameIP6 = "GeoLite2-City-Blocks-IPv6.csv"  // Filename of ipv6 blocks file
	geoLite2LocationsFilename = "GeoLite2-City-Locations-en.csv" // Filename of locations file
)

// The GeoDataset struct bundles all the data needed to search and
// find data into one common structure
// It implements the api.Annotator interface.
type GeoDataset struct {
	Start         time.Time      // Date from which to start using this dataset
	IP4Nodes      []GeoIPNode    // The IPNode list containing IP4Nodes
	IP6Nodes      []GeoIPNode    // The IPNode list containing IP6Nodes
	LocationNodes []LocationNode // The location nodes corresponding to the IPNodes
}

// LoadG2 loads a dataset from a GCS object.
func LoadG2(file *storage.ObjectAttrs) (api.Annotator, error) {
	return LoadG2Dataset(file.Name, file.Bucket)
}

// LoadG2Dataset loads the dataset from the specified filename and bucket
func LoadG2Dataset(filename string, bucketname string) (*GeoDataset, error) {
	zip, err := loader.CreateZipReader(context.Background(), bucketname, filename)
	log.Println("Loading dataset from", filename)
	if err != nil {
		return nil, err
	}
	dataset, err := loadGeoLite2(zip)
	if err != nil {
		return nil, err
	}
	date, err := api.ExtractDateFromFilename(filename)
	if err != nil {
		log.Println("Error extracting date:", filename)
	} else {
		dataset.Start = date
	}
	return dataset, nil
}

// loadGeoLite2 composes the result of location and IPv4, IPv6 IPNode lists
func loadGeoLite2(zip *zip.Reader) (*GeoDataset, error) {
	locations, err := loader.FindFile(geoLite2LocationsFilename, zip)
	if err != nil {
		return nil, err
	}
	// geoidMap is just a temporary map that will be discarded once the blocks are parsed
	locationNode, geoidMap, err := LoadLocationsG2(locations)
	locations.Close()
	if err != nil {
		return nil, err
	}

	blocks4, err := loader.FindFile(geoLite2BlocksFilenameIP4, zip)
	if err != nil {
		return nil, err
	}
	ipNodes4, err := LoadIPListG2(blocks4, geoidMap)
	blocks4.Close()
	if err != nil {
		return nil, err
	}
	blocks6, err := loader.FindFile(geoLite2BlocksFilenameIP6, zip)
	if err != nil {
		return nil, err
	}
	ipNodes6, err := LoadIPListG2(blocks6, geoidMap)
	blocks6.Close()
	if err != nil {
		return nil, err
	}
	return &GeoDataset{IP4Nodes: ipNodes4, IP6Nodes: ipNodes6, LocationNodes: locationNode}, nil
}

// ConvertIPNodeToGeoData takes a parser.IPNode, plus a list of
// locationNodes. It will then use that data to fill in a GeoData struct.
func populateLocationData(ipNode iputils.IPNode, locationNodes []LocationNode, data *api.GeoData) {
	locNode := LocationNode{}
	geoIPNode := ipNode.(*GeoIPNode)

	if geoIPNode.LocationIndex >= 0 {
		locNode = locationNodes[geoIPNode.LocationIndex]
	}
	data.Geo = &api.GeolocationIP{
		ContinentCode: locNode.ContinentCode,
		CountryCode:   locNode.CountryCode,
		CountryCode3:  "", // missing from geoLite2 ?
		CountryName:   locNode.CountryName,
		Region:        locNode.RegionCode,
		MetroCode:     locNode.MetroCode,
		City:          locNode.CityName,
		AreaCode:      0, // new geoLite2 does not have area code.
		PostalCode:    geoIPNode.PostalCode,
		Latitude:      geoIPNode.Latitude,
		Longitude:     geoIPNode.Longitude,
	}
}

// SearchBinary does a binary search for a list element.
func (ds *GeoDataset) SearchBinary(ipLookUp string) (p iputils.IPNode, e error) {
	parsedIP := net.ParseIP(ipLookUp)
	if parsedIP == nil {
		metrics.BadIPTotal.Inc()
		return nil, errors.New("ErrInvalidIP") // TODO
	}
	ipNodes := ds.IP6Nodes
	if parsedIP.To4() != nil {
		ipNodes = ds.IP4Nodes
	}

	node, err := iputils.SearchBinary(ipLookUp,
		len(ipNodes),
		func(idx int) iputils.IPNode {
			return &ipNodes[idx]
		})

	return node, err
}

var lastLogTime = time.Time{}

// Annotate annotates the api.GeoData with the location informations
func (ds *GeoDataset) Annotate(ip string, data *api.GeoData) error {
	if data == nil {
		return errors.New("ErrNilGeoData") // TODO
	}
	if data.Geo != nil {
		return errors.New("ErrAlreadyPopulated") // TODO
	}

	node, err := ds.SearchBinary(ip)

	if err != nil {
		// ErrNodeNotFound is super spammy - 10% of requests, so suppress those.
		if err != iputils.ErrNodeNotFound {
			// Horribly noisy now.
			if time.Since(lastLogTime) > time.Minute {
				log.Println(err, ip)
				lastLogTime = time.Now()
			}
		}
		//TODO metric here
		return err
	}

	populateLocationData(node, ds.LocationNodes, data)
	return nil
}

// AnnotatorDate returns the date that the dataset was published.
// TODO implement actual dataset time!!
func (ds *GeoDataset) AnnotatorDate() time.Time {
	return ds.Start
}

// IsEqualIPNodes returns nil if two nodes are equal
// Used by the search package
func IsEqualIPNodes(expectedIPNode, ipNode iputils.IPNode) error {
	expected, eok := expectedIPNode.(*GeoIPNode)
	node, nok := ipNode.(*GeoIPNode)
	if !eok || !nok {
		return errors.New("Illegal type of IPNode")
	}

	if !((node.IPAddressLow).Equal(expected.IPAddressLow)) {
		output := strings.Join([]string{"IPAddress Low inconsistent\ngot:", node.IPAddressLow.String(), " \nwanted:", expected.IPAddressLow.String()}, "")
		log.Println(output)
		return errors.New(output)
	}
	if !((node.IPAddressHigh).Equal(expected.IPAddressHigh)) {
		output := strings.Join([]string{"IPAddressHigh inconsistent\ngot:", node.IPAddressHigh.String(), " \nwanted:", expected.IPAddressHigh.String()}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.LocationIndex != expected.LocationIndex {
		output := strings.Join([]string{"LocationIndex inconsistent\ngot:", strconv.Itoa(node.LocationIndex), " \nwanted:", strconv.Itoa(expected.LocationIndex)}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.PostalCode != expected.PostalCode {
		output := strings.Join([]string{"PostalCode inconsistent\ngot:", node.PostalCode, " \nwanted:", expected.PostalCode}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.Latitude != expected.Latitude {
		output := strings.Join([]string{"Latitude inconsistent\ngot:", floatToString(node.Latitude), " \nwanted:", floatToString(expected.Latitude)}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.Longitude != expected.Longitude {
		output := strings.Join([]string{"Longitude inconsistent\ngot:", floatToString(node.Longitude), " \nwanted:", floatToString(expected.Longitude)}, "")
		log.Println(output)
		return errors.New(output)
	}
	return nil
}

func floatToString(num float64) string {
	return strconv.FormatFloat(num, 'f', 6, 64)
}
<|endoftext|>


<commit_before>package experiment

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/pkg/errors"
)

const (
	// CPUModelNameKey defines a key in the platform metrics map
	CPUModelNameKey = "cpu_model"
	// KernelVersionKey defines a key in the platform metrics map
	KernelVersionKey = "kernel_version"
	// CentOSVersionKey defines a key in the platform metrics map
	CentOSVersionKey = "centos_version"
	// CPUTopologyKey defines a key in the platform metrics map
	CPUTopologyKey = "cpu_topology"
	// DockerVersionKey defines a key in the platform metrics map
	DockerVersionKey = "docker_version"
	// SnapteldVersionKey defines a key in the platform metrics map
	SnapteldVersionKey = "snapteld_version"
	// PowerGovernorKey defines a key in the platform metrics map
	PowerGovernorKey = "power_governor"
	// IRQAffinityKey defines a key in the platform metrics map
	IRQAffinityKey = "irq_affinity"
	// EtcdVersionKey defines a key in the platform metrics map
	EtcdVersionKey = "etcd_version"
)

// GetPlatformMetrics returns map of strings with platform metrics.
// If metric could not be retreived value for the key is empty string.
func GetPlatformMetrics() (platformMetrics map[string]string) {
	platformMetrics = make(map[string]string)
	item, err := CPUModelName()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", CPUModelNameKey, err.Error()))
	}
	platformMetrics[CPUModelNameKey] = item

	item, err = KernelVersion()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", KernelVersionKey, err.Error()))
	}
	platformMetrics[KernelVersionKey] = item

	item, err = CentOSVersion()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", CentOSVersionKey, err.Error()))
	}
	platformMetrics[CentOSVersionKey] = item

	item, err = CPUTopology()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", CPUTopologyKey, err.Error()))
	}
	platformMetrics[CPUTopologyKey] = item

	item, err = DockerVersion()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", DockerVersionKey, err.Error()))
	}
	platformMetrics[DockerVersionKey] = item

	item, err = SnapteldVersion()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", SnapteldVersionKey, err.Error()))
	}
	platformMetrics[SnapteldVersionKey] = item

	item, err = PowerGovernor()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", PowerGovernorKey, err.Error()))
	}
	platformMetrics[PowerGovernorKey] = item

	item, err = IRQAffinity()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", IRQAffinityKey, err.Error()))
	}
	platformMetrics[IRQAffinityKey] = item

	item, err = EtcdVersion()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", EtcdVersionKey, err.Error()))
	}
	platformMetrics[EtcdVersionKey] = item

	return platformMetrics
}

// CPUModelName reads /proc/cpuinfo and returns line 'model name' line.
// Note that it returns only first occurrence of the model since mixed cpu models
// in > 2 CPUs are not supported
// In case of an error empty string is returned.
func CPUModelName() (string, error) {
	file, err := os.Open("/proc/cpuinfo")
	if err != nil {
		return "", errors.Wrapf(err, "Cannot open /proc/cpuinfo file.")
	}
	defer file.Close()

	procScanner := bufio.NewScanner(file)

	for procScanner.Scan() {
		line := procScanner.Text()
		chunks := strings.Split(line, ":")
		key := strings.TrimSpace(chunks[0])
		value := strings.TrimSpace(chunks[1])
		if key == "model name" {
			return value, nil
		}
	}
	// Return error from scanner or newly created one.
	err = procScanner.Err()
	if err == nil {
		err = errors.New("Did not find phrase 'model name' in /proc/cpuinfo")
	}
	return "", err
}

// KernelVersion return kernel version as stated in /proc/version
// In case of an error empty string is returned
func KernelVersion() (string, error) {
	return readContents("/proc/version")
}

// CentOSVersion returns OS version as stated in /etc/redhat-release
// In case of an error empty string is returned
func CentOSVersion() (string, error) {
	return readContents("/etc/redhat-release")
}

// CPUTopology returns CPU topology returned by 'lscpu -e' command.
// The whole output of the command is returned.
// In case of an error empty string is returned
func CPUTopology() (string, error) {
	cmd := exec.Command("lscpu", "-e")
	output, err := cmd.Output()
	if err != nil {
		return "", errors.Wrapf(err, "Failed to get output from lscpu -e")
	}
	return strings.TrimSpace(string(output)), nil
}

// DockerVersion returns docker version as returned by 'docker version' command.
// In case of an error empty string is returned
func DockerVersion() (string, error) {
	cmd := exec.Command("docker", "version")
	output, err := cmd.Output()
	if err != nil {
		return "", errors.Wrapf(err, "Failed to get output from docker version")
	}
	return strings.TrimSpace(string(output)), nil
}

// SnapteldVersion returns snapteld version as returned by 'snapteld -v' command.
// In case of an error empty string is returned
func SnapteldVersion() (string, error) {
	cmd := exec.Command("snapteld", "-v")
	output, err := cmd.Output()
	if err != nil {
		return "", errors.Wrapf(err, "Failed to get output from snapteld version")
	}
	return strings.TrimSpace(string(output)), nil

}

// PowerGovernor returns a comma separated list of CPU:power_policy.
// Example (snippet):
//    "performance,1:performance,10:performance,11:performance"
// In case of an error empty string is returned
func PowerGovernor() (string, error) {
	dir := "/sys/devices/system/cpu"
	files, err := ioutil.ReadDir(dir)

	if err != nil {
		return "", errors.Wrapf(err, "Failed to scan sysfs for CPU devices")
	}

	output := []string{}
	for _, file := range files {
		if file.IsDir() && strings.HasPrefix(file.Name(), "cpu") {
			cpufreq := path.Join(dir, file.Name(), "cpufreq/scaling_governor")

			// Just try to read it. Don't try to be smart here. Failure is OK.
			gov, err := readContents(cpufreq)
			if err != nil {
				return "", err
			}
			item := fmt.Sprintf("%s:%s", strings.TrimLeft(file.Name(), "cpu"), gov)
			output = append(output, item)
		}
	}

	return strings.Join(output, ","), nil
}

// IRQAffinity returns semicolon (;) separated list of pairs iface {comma separated
// list of pairs queue:affinity}
// Example:
//   enp0s31f6 {134:6};enp3s0 {129:5,130:4,131:3,132:2,133:2}
// In case of an error empty string is returned
func IRQAffinity() (string, error) {
	dir := "/sys/class/net"
	ifaces, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", errors.Wrapf(err, "Failed to scan sysfs for ifaces")
	}
	output := []string{}
	// Enumerate all network interfaces in the OS
	for _, iface := range ifaces {
		// for each network interface check for 'device' directory.
		// Note: local interfaces (lo) doesn't have this so on err is to skip.
		device := path.Join(dir, iface.Name(), "device/msi_irqs")
		info, err := os.Stat(device)
		if err == nil && info.IsDir() {
			queues, err := getIfaceQueuesAffinity(iface.Name())
			if err != nil {
				return "", errors.Wrapf(err, "Failed to get %s queues affinity", iface.Name())
			}
			item := fmt.Sprintf("%s {%s}", iface.Name(), queues)
			output = append(output, item)
		}
	}
	return strings.Join(output, ";"), nil
}

// EtcdVersion returns etcd version as returned by 'etcd --version'.
// In case of an error empty string is returned
func EtcdVersion() (string, error) {
	cmd := exec.Command("etcd", "--version")
	output, err := cmd.Output()
	if err != nil {
		return "", errors.Wrapf(err, "Failed to get etcd version")
	}
	return strings.TrimSpace(string(output)), nil
}

func readContents(name string) (string, error) {
	content, err := ioutil.ReadFile(name)
	if err != nil {
		return "", errors.Wrapf(err, "Failed to read %s", name)
	}
	return strings.TrimSpace(string(content)), nil
}

func getIfaceQueuesAffinity(iface string) (string, error) {
	ifaceDir := path.Join("/sys/class/net", iface, "device/msi_irqs")
	queues, err := ioutil.ReadDir(ifaceDir)
	if err != nil {
		return "", errors.Wrapf(err, "Failed to read %s directory", ifaceDir)
	}
	output := []string{}

	for _, queue := range queues {
		smpAffinityFile := path.Join("/proc/irq", queue.Name(), "smp_affinity_list")
		smpAffinity, err := readContents(smpAffinityFile)
		if err != nil {
			return "", err
		}
		item := fmt.Sprintf("%s:%s", queue.Name(), smpAffinity)
		output = append(output, item)
	}
	return strings.Join(output, ","), nil
}
<commit_msg>PowerGovernor function failed to collect metrics (#544)<commit_after>package experiment

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
	"regexp"

	"github.com/Sirupsen/logrus"
	"github.com/pkg/errors"
)

const (
	// CPUModelNameKey defines a key in the platform metrics map
	CPUModelNameKey = "cpu_model"
	// KernelVersionKey defines a key in the platform metrics map
	KernelVersionKey = "kernel_version"
	// CentOSVersionKey defines a key in the platform metrics map
	CentOSVersionKey = "centos_version"
	// CPUTopologyKey defines a key in the platform metrics map
	CPUTopologyKey = "cpu_topology"
	// DockerVersionKey defines a key in the platform metrics map
	DockerVersionKey = "docker_version"
	// SnapteldVersionKey defines a key in the platform metrics map
	SnapteldVersionKey = "snapteld_version"
	// PowerGovernorKey defines a key in the platform metrics map
	PowerGovernorKey = "power_governor"
	// IRQAffinityKey defines a key in the platform metrics map
	IRQAffinityKey = "irq_affinity"
	// EtcdVersionKey defines a key in the platform metrics map
	EtcdVersionKey = "etcd_version"
)

// GetPlatformMetrics returns map of strings with platform metrics.
// If metric could not be retreived value for the key is empty string.
func GetPlatformMetrics() (platformMetrics map[string]string) {
	platformMetrics = make(map[string]string)
	item, err := CPUModelName()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", CPUModelNameKey, err.Error()))
	}
	platformMetrics[CPUModelNameKey] = item

	item, err = KernelVersion()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", KernelVersionKey, err.Error()))
	}
	platformMetrics[KernelVersionKey] = item

	item, err = CentOSVersion()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", CentOSVersionKey, err.Error()))
	}
	platformMetrics[CentOSVersionKey] = item

	item, err = CPUTopology()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", CPUTopologyKey, err.Error()))
	}
	platformMetrics[CPUTopologyKey] = item

	item, err = DockerVersion()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", DockerVersionKey, err.Error()))
	}
	platformMetrics[DockerVersionKey] = item

	item, err = SnapteldVersion()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", SnapteldVersionKey, err.Error()))
	}
	platformMetrics[SnapteldVersionKey] = item

	item, err = PowerGovernor()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", PowerGovernorKey, err.Error()))
	}
	platformMetrics[PowerGovernorKey] = item

	item, err = IRQAffinity()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", IRQAffinityKey, err.Error()))
	}
	platformMetrics[IRQAffinityKey] = item

	item, err = EtcdVersion()
	if err != nil {
		logrus.Warn(fmt.Sprintf("GetPlatformMetrics: Failed to get %s metric. Skipping. Error: %s", EtcdVersionKey, err.Error()))
	}
	platformMetrics[EtcdVersionKey] = item

	return platformMetrics
}

// CPUModelName reads /proc/cpuinfo and returns line 'model name' line.
// Note that it returns only first occurrence of the model since mixed cpu models
// in > 2 CPUs are not supported
// In case of an error empty string is returned.
func CPUModelName() (string, error) {
	file, err := os.Open("/proc/cpuinfo")
	if err != nil {
		return "", errors.Wrapf(err, "Cannot open /proc/cpuinfo file.")
	}
	defer file.Close()

	procScanner := bufio.NewScanner(file)

	for procScanner.Scan() {
		line := procScanner.Text()
		chunks := strings.Split(line, ":")
		key := strings.TrimSpace(chunks[0])
		value := strings.TrimSpace(chunks[1])
		if key == "model name" {
			return value, nil
		}
	}
	// Return error from scanner or newly created one.
	err = procScanner.Err()
	if err == nil {
		err = errors.New("Did not find phrase 'model name' in /proc/cpuinfo")
	}
	return "", err
}

// KernelVersion return kernel version as stated in /proc/version
// In case of an error empty string is returned
func KernelVersion() (string, error) {
	return readContents("/proc/version")
}

// CentOSVersion returns OS version as stated in /etc/redhat-release
// In case of an error empty string is returned
func CentOSVersion() (string, error) {
	return readContents("/etc/redhat-release")
}

// CPUTopology returns CPU topology returned by 'lscpu -e' command.
// The whole output of the command is returned.
// In case of an error empty string is returned
func CPUTopology() (string, error) {
	cmd := exec.Command("lscpu", "-e")
	output, err := cmd.Output()
	if err != nil {
		return "", errors.Wrapf(err, "Failed to get output from lscpu -e")
	}
	return strings.TrimSpace(string(output)), nil
}

// DockerVersion returns docker version as returned by 'docker version' command.
// In case of an error empty string is returned
func DockerVersion() (string, error) {
	cmd := exec.Command("docker", "version")
	output, err := cmd.Output()
	if err != nil {
		return "", errors.Wrapf(err, "Failed to get output from docker version")
	}
	return strings.TrimSpace(string(output)), nil
}

// SnapteldVersion returns snapteld version as returned by 'snapteld -v' command.
// In case of an error empty string is returned
func SnapteldVersion() (string, error) {
	cmd := exec.Command("snapteld", "-v")
	output, err := cmd.Output()
	if err != nil {
		return "", errors.Wrapf(err, "Failed to get output from snapteld version")
	}
	return strings.TrimSpace(string(output)), nil

}

// PowerGovernor returns a comma separated list of CPU:power_policy.
// Example (snippet):
//    "performance,1:performance,10:performance,11:performance"
// In case of an error empty string is returned
func PowerGovernor() (string, error) {
	dir := "/sys/devices/system/cpu"
	files, err := ioutil.ReadDir(dir)

	if err != nil {
		return "", errors.Wrapf(err, "Failed to scan sysfs for CPU devices")
	}

	re := regexp.MustCompile("cpu[0-9]+")
	output := []string{}
	for _, file := range files {
		if file.IsDir() && re.MatchString(file.Name()) {
			cpufreq := path.Join(dir, file.Name(), "cpufreq/scaling_governor")

			// Just try to read it. Don't try to be smart here. Failure is OK.
			gov, err := readContents(cpufreq)
			if err != nil {
				return "", err
			}
			item := fmt.Sprintf("%s:%s", strings.TrimLeft(file.Name(), "cpu"), gov)
			output = append(output, item)
		}
	}

	return strings.Join(output, ","), nil
}

// IRQAffinity returns semicolon (;) separated list of pairs iface {comma separated
// list of pairs queue:affinity}
// Example:
//   enp0s31f6 {134:6};enp3s0 {129:5,130:4,131:3,132:2,133:2}
// In case of an error empty string is returned
func IRQAffinity() (string, error) {
	dir := "/sys/class/net"
	ifaces, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", errors.Wrapf(err, "Failed to scan sysfs for ifaces")
	}
	output := []string{}
	// Enumerate all network interfaces in the OS
	for _, iface := range ifaces {
		// for each network interface check for 'device' directory.
		// Note: local interfaces (lo) doesn't have this so on err is to skip.
		device := path.Join(dir, iface.Name(), "device/msi_irqs")
		info, err := os.Stat(device)
		if err == nil && info.IsDir() {
			queues, err := getIfaceQueuesAffinity(iface.Name())
			if err != nil {
				return "", errors.Wrapf(err, "Failed to get %s queues affinity", iface.Name())
			}
			item := fmt.Sprintf("%s {%s}", iface.Name(), queues)
			output = append(output, item)
		}
	}
	return strings.Join(output, ";"), nil
}

// EtcdVersion returns etcd version as returned by 'etcd --version'.
// In case of an error empty string is returned
func EtcdVersion() (string, error) {
	cmd := exec.Command("etcd", "--version")
	output, err := cmd.Output()
	if err != nil {
		return "", errors.Wrapf(err, "Failed to get etcd version")
	}
	return strings.TrimSpace(string(output)), nil
}

func readContents(name string) (string, error) {
	content, err := ioutil.ReadFile(name)
	if err != nil {
		return "", errors.Wrapf(err, "Failed to read %s", name)
	}
	return strings.TrimSpace(string(content)), nil
}

func getIfaceQueuesAffinity(iface string) (string, error) {
	ifaceDir := path.Join("/sys/class/net", iface, "device/msi_irqs")
	queues, err := ioutil.ReadDir(ifaceDir)
	if err != nil {
		return "", errors.Wrapf(err, "Failed to read %s directory", ifaceDir)
	}
	output := []string{}

	for _, queue := range queues {
		smpAffinityFile := path.Join("/proc/irq", queue.Name(), "smp_affinity_list")
		smpAffinity, err := readContents(smpAffinityFile)
		if err != nil {
			return "", err
		}
		item := fmt.Sprintf("%s:%s", queue.Name(), smpAffinity)
		output = append(output, item)
	}
	return strings.Join(output, ","), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"reflect"
	"testing"

	"k8s.io/client-go/pkg/api/v1"

	"github.com/coreos/prometheus-operator/pkg/client/monitoring/v1alpha1"
)

func TestStatefulSetLabelingAndAnnotations(t *testing.T) {
	labels := map[string]string{
		"testlabel": "testlabelvalue",
	}
	annotations := map[string]string{
		"testannotation": "testannotationvalue",
	}

	sset := makeStatefulSet(v1alpha1.Prometheus{
		ObjectMeta: v1.ObjectMeta{
			Labels:      labels,
			Annotations: annotations,
		},
	}, nil)

	if !reflect.DeepEqual(labels, sset.Labels) || !reflect.DeepEqual(annotations, sset.Annotations) {
		t.Fatal("Labels or Annotations are not properly being propagated to the StatefulSet")
	}
}
<commit_msg>prometheus: fix statefulset test after introducing config<commit_after>// Copyright 2016 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"reflect"
	"testing"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/coreos/prometheus-operator/pkg/client/monitoring/v1alpha1"
)

var (
	defaultTestConfig = &Config{
		ConfigReloaderImage: "quay.io/coreos/configmap-reload:latest",
	}
)

func TestStatefulSetLabelingAndAnnotations(t *testing.T) {
	labels := map[string]string{
		"testlabel": "testlabelvalue",
	}
	annotations := map[string]string{
		"testannotation": "testannotationvalue",
	}

	sset := makeStatefulSet(v1alpha1.Prometheus{
		ObjectMeta: metav1.ObjectMeta{
			Labels:      labels,
			Annotations: annotations,
		},
	}, nil, defaultTestConfig)

	if !reflect.DeepEqual(labels, sset.Labels) || !reflect.DeepEqual(annotations, sset.Annotations) {
		t.Fatal("Labels or Annotations are not properly being propagated to the StatefulSet")
	}
}
<|endoftext|>