<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package healthcheck

import (
	"fmt"
	"strings"

	"github.com/GoogleCloudPlatform/container-engine-accelerators/pkg/gpu/nvidia/util"
	"github.com/NVIDIA/gpu-monitoring-tools/bindings/go/nvml"
	"github.com/golang/glog"

	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
)

// GPUHealthChecker checks the health of nvidia GPUs. Note that with the current
// device naming pattern in device manager, GPUHealthChecker will not work with
// MIG devices.
type GPUHealthChecker struct {
	devices     map[string]pluginapi.Device
	nvmlDevices map[string]*nvml.Device
	health      chan pluginapi.Device
	eventSet    nvml.EventSet
	stop        chan bool
}

// NewGPUHealthChecker returns a GPUHealthChecker object for a given device name
func NewGPUHealthChecker(devices map[string]pluginapi.Device, health chan pluginapi.Device) *GPUHealthChecker {
	return &GPUHealthChecker{
		devices:     devices,
		nvmlDevices: make(map[string]*nvml.Device),
		health:      health,
		stop:        make(chan bool),
	}
}

// Start registers NVML events and starts listening to them
func (hc *GPUHealthChecker) Start() error {
	glog.Info("Starting GPU Health Checker")

	// Building mapping between device ID and their nvml represetation
	count, err := nvml.GetDeviceCount()
	if err != nil {
		return fmt.Errorf("failed to get device count: %s", err)
	}

	glog.Infof("Found %d GPU devices", count)
	for i := uint(0); i < count; i++ {
		device, err := nvml.NewDeviceLite(i)
		if err != nil {
			return fmt.Errorf("failed to read device with index %d: %v", i, err)
		}
		deviceName, err := util.DeviceNameFromPath(device.Path)
		if err != nil {
			glog.Errorf("Invalid GPU device path found: %s. Skipping this device", device.Path)
			continue
		}

		if _, ok := hc.devices[deviceName]; !ok {
			// we only monitor the devices passed in
			glog.Warningf("Ignoring device %s for health check.", deviceName)
			continue
		}

		glog.Infof("Found device %s for health monitoring. UUID: %s", deviceName, device.UUID)
		hc.nvmlDevices[deviceName] = device
	}

	hc.eventSet = nvml.NewEventSet()
	for _, d := range hc.nvmlDevices {
		gpu, _, _, err := nvml.ParseMigDeviceUUID(d.UUID)
		if err != nil {
			gpu = d.UUID
		}

		glog.Infof("Registering device %v. UUID: %s", d.Path, d.UUID)
		err = nvml.RegisterEventForDevice(hc.eventSet, nvml.XidCriticalError, gpu)
		if err != nil {
			if strings.HasSuffix(err.Error(), "Not Supported") {
				glog.Warningf("Warning: %s is too old to support healthchecking: %v. It will always be marked healthy.", d.Path, err)
				continue
			} else {
				return fmt.Errorf("failed to register device %s for NVML eventSet: %v", d.Path, err)
			}
		}
	}

	go func() {
		if err := hc.listenToEvents(); err != nil {
			glog.Errorf("GPUHealthChecker listenToEvents error: %v", err)
		}
	}()

	return nil
}

// listenToEvents listens to events from NVML to detect GPU critical errors
func (hc *GPUHealthChecker) listenToEvents() error {
	for {
		select {
		case <-hc.stop:
			close(hc.stop)
			return nil
		default:
		}

		e, err := nvml.WaitForEvent(hc.eventSet, 5000)
		if err != nil || e.Etype != nvml.XidCriticalError {
			glog.Infof("XidCriticalError: Xid=%d, All devices will go unhealthy.", e.Edata)
			continue
		}

		// Ignoring application errors. GPU should still be healthy
		// See https://docs.nvidia.com/deploy/xid-errors/index.html#topic_4
		if e.Edata == 31 || e.Edata == 43 || e.Edata == 45 {
			continue
		}

		if e.UUID == nil || len(*e.UUID) == 0 {
			// All devices are unhealthy
			glog.Errorf("XidCriticalError: Xid=%d, All devices will go unhealthy.", e.Edata)
			for id, d := range hc.devices {
				d.Health = pluginapi.Unhealthy
				hc.devices[id] = d
				hc.health <- d
			}
			continue
		}

		for _, d := range hc.devices {
			// Please see https://github.com/NVIDIA/gpu-monitoring-tools/blob/148415f505c96052cb3b7fdf443b34ac853139ec/bindings/go/nvml/nvml.h#L1424
			// for the rationale why gi and ci can be set as such when the UUID is a full GPU UUID and not a MIG device UUID.
			uuid := hc.nvmlDevices[d.ID].UUID
			gpu, gi, ci, err := nvml.ParseMigDeviceUUID(uuid)
			if err != nil {
				gpu = uuid
				gi = 0xFFFFFFFF
				ci = 0xFFFFFFFF
			}

			if gpu == *e.UUID && gi == *e.GpuInstanceId && ci == *e.ComputeInstanceId {
				glog.Errorf("XidCriticalError: Xid=%d on Device=%s, uuid=%s, the device will go unhealthy.", e.Edata, d.ID, uuid)
				d.Health = pluginapi.Unhealthy
				hc.devices[d.ID] = d
				hc.health <- d
			}
		}
	}
}

// Stop deletes the NVML events and stops the listening go routine
func (hc *GPUHealthChecker) Stop() {
	nvml.DeleteEventSet(hc.eventSet)
	hc.stop <- true
	<-hc.stop
}
<commit_msg>Only mark device unhealthy on Xid 48<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package healthcheck

import (
	"fmt"
	"strings"

	"github.com/GoogleCloudPlatform/container-engine-accelerators/pkg/gpu/nvidia/util"
	"github.com/NVIDIA/gpu-monitoring-tools/bindings/go/nvml"
	"github.com/golang/glog"

	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
)

// GPUHealthChecker checks the health of nvidia GPUs. Note that with the current
// device naming pattern in device manager, GPUHealthChecker will not work with
// MIG devices.
type GPUHealthChecker struct {
	devices     map[string]pluginapi.Device
	nvmlDevices map[string]*nvml.Device
	health      chan pluginapi.Device
	eventSet    nvml.EventSet
	stop        chan bool
}

// NewGPUHealthChecker returns a GPUHealthChecker object for a given device name
func NewGPUHealthChecker(devices map[string]pluginapi.Device, health chan pluginapi.Device) *GPUHealthChecker {
	return &GPUHealthChecker{
		devices:     devices,
		nvmlDevices: make(map[string]*nvml.Device),
		health:      health,
		stop:        make(chan bool),
	}
}

// Start registers NVML events and starts listening to them
func (hc *GPUHealthChecker) Start() error {
	glog.Info("Starting GPU Health Checker")

	// Building mapping between device ID and their nvml represetation
	count, err := nvml.GetDeviceCount()
	if err != nil {
		return fmt.Errorf("failed to get device count: %s", err)
	}

	glog.Infof("Found %d GPU devices", count)
	for i := uint(0); i < count; i++ {
		device, err := nvml.NewDeviceLite(i)
		if err != nil {
			return fmt.Errorf("failed to read device with index %d: %v", i, err)
		}
		deviceName, err := util.DeviceNameFromPath(device.Path)
		if err != nil {
			glog.Errorf("Invalid GPU device path found: %s. Skipping this device", device.Path)
			continue
		}

		if _, ok := hc.devices[deviceName]; !ok {
			// we only monitor the devices passed in
			glog.Warningf("Ignoring device %s for health check.", deviceName)
			continue
		}

		glog.Infof("Found device %s for health monitoring. UUID: %s", deviceName, device.UUID)
		hc.nvmlDevices[deviceName] = device
	}

	hc.eventSet = nvml.NewEventSet()
	for _, d := range hc.nvmlDevices {
		gpu, _, _, err := nvml.ParseMigDeviceUUID(d.UUID)
		if err != nil {
			gpu = d.UUID
		}

		glog.Infof("Registering device %v. UUID: %s", d.Path, d.UUID)
		err = nvml.RegisterEventForDevice(hc.eventSet, nvml.XidCriticalError, gpu)
		if err != nil {
			if strings.HasSuffix(err.Error(), "Not Supported") {
				glog.Warningf("Warning: %s is too old to support healthchecking: %v. It will always be marked healthy.", d.Path, err)
				continue
			} else {
				return fmt.Errorf("failed to register device %s for NVML eventSet: %v", d.Path, err)
			}
		}
	}

	go func() {
		if err := hc.listenToEvents(); err != nil {
			glog.Errorf("GPUHealthChecker listenToEvents error: %v", err)
		}
	}()

	return nil
}

// listenToEvents listens to events from NVML to detect GPU critical errors
func (hc *GPUHealthChecker) listenToEvents() error {
	for {
		select {
		case <-hc.stop:
			close(hc.stop)
			return nil
		default:
		}

		e, err := nvml.WaitForEvent(hc.eventSet, 5000)
		if err != nil || e.Etype != nvml.XidCriticalError {
			glog.Infof("XidCriticalError: Xid=%d, All devices will go unhealthy.", e.Edata)
			continue
		}

		// Only marking device unhealthy on Double Bit ECC Error
		// See https://docs.nvidia.com/deploy/xid-errors/index.html#topic_4
		if e.Edata != 48 {
			continue
		}

		if e.UUID == nil || len(*e.UUID) == 0 {
			// All devices are unhealthy
			glog.Errorf("XidCriticalError: Xid=%d, All devices will go unhealthy.", e.Edata)
			for id, d := range hc.devices {
				d.Health = pluginapi.Unhealthy
				hc.devices[id] = d
				hc.health <- d
			}
			continue
		}

		for _, d := range hc.devices {
			// Please see https://github.com/NVIDIA/gpu-monitoring-tools/blob/148415f505c96052cb3b7fdf443b34ac853139ec/bindings/go/nvml/nvml.h#L1424
			// for the rationale why gi and ci can be set as such when the UUID is a full GPU UUID and not a MIG device UUID.
			uuid := hc.nvmlDevices[d.ID].UUID
			gpu, gi, ci, err := nvml.ParseMigDeviceUUID(uuid)
			if err != nil {
				gpu = uuid
				gi = 0xFFFFFFFF
				ci = 0xFFFFFFFF
			}

			if gpu == *e.UUID && gi == *e.GpuInstanceId && ci == *e.ComputeInstanceId {
				glog.Errorf("XidCriticalError: Xid=%d on Device=%s, uuid=%s, the device will go unhealthy.", e.Edata, d.ID, uuid)
				d.Health = pluginapi.Unhealthy
				hc.devices[d.ID] = d
				hc.health <- d
			}
		}
	}
}

// Stop deletes the NVML events and stops the listening go routine
func (hc *GPUHealthChecker) Stop() {
	nvml.DeleteEventSet(hc.eventSet)
	hc.stop <- true
	<-hc.stop
}
<|endoftext|>


<commit_before>package controller

import (
	"github.com/fabric8-services/fabric8-wit/app"
	"github.com/fabric8-services/fabric8-wit/application"
	"github.com/fabric8-services/fabric8-wit/errors"
	"github.com/fabric8-services/fabric8-wit/iteration"
	"github.com/fabric8-services/fabric8-wit/jsonapi"
	"github.com/fabric8-services/fabric8-wit/log"
	"github.com/fabric8-services/fabric8-wit/login"
	"github.com/fabric8-services/fabric8-wit/rest"
	"github.com/fabric8-services/fabric8-wit/workitem"
	uuid "github.com/satori/go.uuid"

	"github.com/goadesign/goa"
)

// SpaceIterationsControllerConfiguration configuration for the SpaceIterationsController
type SpaceIterationsControllerConfiguration interface {
	GetCacheControlIterations() string
}

// SpaceIterationsController implements the space-iterations resource.
type SpaceIterationsController struct {
	*goa.Controller
	db     application.DB
	config SpaceIterationsControllerConfiguration
}

// NewSpaceIterationsController creates a space-iterations controller.
func NewSpaceIterationsController(service *goa.Service, db application.DB, config SpaceIterationsControllerConfiguration) *SpaceIterationsController {
	return &SpaceIterationsController{Controller: service.NewController("SpaceIterationsController"), db: db, config: config}
}

// Create runs the create action.
func (c *SpaceIterationsController) Create(ctx *app.CreateSpaceIterationsContext) error {
	currentUser, err := login.ContextIdentity(ctx)
	if err != nil {
		return jsonapi.JSONErrorResponse(ctx, goa.ErrUnauthorized(err.Error()))
	}
	// Validate Request
	if ctx.Payload.Data == nil {
		return jsonapi.JSONErrorResponse(ctx, errors.NewBadParameterError("data", nil).Expected("not nil"))
	}
	reqIter := ctx.Payload.Data
	if reqIter.Attributes.Name == nil {
		return jsonapi.JSONErrorResponse(ctx, errors.NewBadParameterError("data.attributes.name", nil).Expected("not nil"))
	}

	var responseData *app.Iteration
	err = application.Transactional(c.db, func(appl application.Application) error {
		s, err := appl.Spaces().Load(ctx, ctx.SpaceID)
		if err != nil {
			return goa.ErrNotFound(err.Error())
		}
		if !uuid.Equal(*currentUser, s.OwnerID) {
			log.Warn(ctx, map[string]interface{}{
				"space_id":     ctx.SpaceID,
				"space_owner":  s.OwnerID,
				"current_user": *currentUser,
			}, "user is not the space owner")
			return errors.NewForbiddenError("user is not the space owner")
		}
		// Put iteration under root iteration
		rootIteration, err := appl.Iterations().Root(ctx, ctx.SpaceID)
		if err != nil {
			return goa.ErrNotFound(err.Error())
		}
		childPath := append(rootIteration.Path, rootIteration.ID)

		userActive := false
		if reqIter.Attributes.UserActive != nil {
			userActive = *reqIter.Attributes.UserActive
		}
		newItr := iteration.Iteration{
			SpaceID:    ctx.SpaceID,
			Name:       *reqIter.Attributes.Name,
			StartAt:    reqIter.Attributes.StartAt,
			EndAt:      reqIter.Attributes.EndAt,
			Path:       childPath,
			UserActive: userActive,
		}
		if reqIter.Attributes.Description != nil {
			newItr.Description = reqIter.Attributes.Description
		}
		err = appl.Iterations().Create(ctx, &newItr)
		if err != nil {
			return err
		}
		// For create, count will always be zero hence no need to query
		// by passing empty map, updateIterationsWithCounts will be able to put zero values
		wiCounts := make(map[string]workitem.WICountsPerIteration)
		log.Info(ctx, map[string]interface{}{
			"iteration_id": newItr.ID,
			"wiCounts":     wiCounts,
		}, "wicounts for created iteration %s -> %v", newItr.ID.String(), wiCounts)

		if newItr.Path.IsEmpty() == false {
			allParentsUUIDs := newItr.Path
			iterations, error := appl.Iterations().LoadMultiple(ctx, allParentsUUIDs)
			if error != nil {
				return err
			}
			itrMap := make(iterationIDMap)
			for _, itr := range iterations {
				itrMap[itr.ID] = itr
			}
			responseData = ConvertIteration(ctx.Request, newItr, parentPathResolver(itrMap), updateIterationsWithCounts(wiCounts))
		} else {
			responseData = ConvertIteration(ctx.Request, newItr, updateIterationsWithCounts(wiCounts))
		}
		return nil
	})
	if err != nil {
		return jsonapi.JSONErrorResponse(ctx, err)
	}
	res := &app.IterationSingle{
		Data: responseData,
	}
	ctx.ResponseData.Header().Set("Location", rest.AbsoluteURL(ctx.Request, app.IterationHref(res.Data.ID)))
	return ctx.Created(res)
}

// List runs the list action.
func (c *SpaceIterationsController) List(ctx *app.ListSpaceIterationsContext) error {
	err := application.Transactional(c.db, func(appl application.Application) error {
		err := appl.Spaces().CheckExists(ctx, ctx.SpaceID)
		if err != nil {
			return err
		}
		iterations, err := appl.Iterations().List(ctx, ctx.SpaceID)
		if err != nil {
			return err
		}
		return ctx.ConditionalEntities(iterations, c.config.GetCacheControlIterations, func() error {
			itrMap := make(iterationIDMap)
			for _, itr := range iterations {
				itrMap[itr.ID] = itr
			}
			// fetch extra information(counts of WI in each iteration of the space) to be added in response
			wiCounts, err := appl.WorkItems().GetCountsPerIteration(ctx, ctx.SpaceID)
			log.Info(ctx, map[string]interface{}{
				"space_id": ctx.SpaceID.String(),
				"wiCounts": wiCounts,
			}, "Retrieving wicounts for spaceID %s -> %v", ctx.SpaceID.String(), wiCounts)

			if err != nil {
				return err
			}
			res := &app.IterationList{}
			res.Data = ConvertIterations(ctx.Request, iterations, updateIterationsWithCounts(wiCounts), parentPathResolver(itrMap))
			return ctx.OK(res)
		})
	})
	if err != nil {
		return jsonapi.JSONErrorResponse(ctx, err)
	}
	return nil
}
<commit_msg>Don't use IDs as part of log message Keys (#1971) <commit_after>package controller

import (
	"github.com/fabric8-services/fabric8-wit/app"
	"github.com/fabric8-services/fabric8-wit/application"
	"github.com/fabric8-services/fabric8-wit/errors"
	"github.com/fabric8-services/fabric8-wit/iteration"
	"github.com/fabric8-services/fabric8-wit/jsonapi"
	"github.com/fabric8-services/fabric8-wit/log"
	"github.com/fabric8-services/fabric8-wit/login"
	"github.com/fabric8-services/fabric8-wit/rest"
	"github.com/fabric8-services/fabric8-wit/workitem"
	uuid "github.com/satori/go.uuid"

	"github.com/goadesign/goa"
)

// SpaceIterationsControllerConfiguration configuration for the SpaceIterationsController
type SpaceIterationsControllerConfiguration interface {
	GetCacheControlIterations() string
}

// SpaceIterationsController implements the space-iterations resource.
type SpaceIterationsController struct {
	*goa.Controller
	db     application.DB
	config SpaceIterationsControllerConfiguration
}

// NewSpaceIterationsController creates a space-iterations controller.
func NewSpaceIterationsController(service *goa.Service, db application.DB, config SpaceIterationsControllerConfiguration) *SpaceIterationsController {
	return &SpaceIterationsController{Controller: service.NewController("SpaceIterationsController"), db: db, config: config}
}

// Create runs the create action.
func (c *SpaceIterationsController) Create(ctx *app.CreateSpaceIterationsContext) error {
	currentUser, err := login.ContextIdentity(ctx)
	if err != nil {
		return jsonapi.JSONErrorResponse(ctx, goa.ErrUnauthorized(err.Error()))
	}
	// Validate Request
	if ctx.Payload.Data == nil {
		return jsonapi.JSONErrorResponse(ctx, errors.NewBadParameterError("data", nil).Expected("not nil"))
	}
	reqIter := ctx.Payload.Data
	if reqIter.Attributes.Name == nil {
		return jsonapi.JSONErrorResponse(ctx, errors.NewBadParameterError("data.attributes.name", nil).Expected("not nil"))
	}

	var responseData *app.Iteration
	err = application.Transactional(c.db, func(appl application.Application) error {
		s, err := appl.Spaces().Load(ctx, ctx.SpaceID)
		if err != nil {
			return goa.ErrNotFound(err.Error())
		}
		if !uuid.Equal(*currentUser, s.OwnerID) {
			log.Warn(ctx, map[string]interface{}{
				"space_id":     ctx.SpaceID,
				"space_owner":  s.OwnerID,
				"current_user": *currentUser,
			}, "user is not the space owner")
			return errors.NewForbiddenError("user is not the space owner")
		}
		// Put iteration under root iteration
		rootIteration, err := appl.Iterations().Root(ctx, ctx.SpaceID)
		if err != nil {
			return goa.ErrNotFound(err.Error())
		}
		childPath := append(rootIteration.Path, rootIteration.ID)

		userActive := false
		if reqIter.Attributes.UserActive != nil {
			userActive = *reqIter.Attributes.UserActive
		}
		newItr := iteration.Iteration{
			SpaceID:    ctx.SpaceID,
			Name:       *reqIter.Attributes.Name,
			StartAt:    reqIter.Attributes.StartAt,
			EndAt:      reqIter.Attributes.EndAt,
			Path:       childPath,
			UserActive: userActive,
		}
		if reqIter.Attributes.Description != nil {
			newItr.Description = reqIter.Attributes.Description
		}
		err = appl.Iterations().Create(ctx, &newItr)
		if err != nil {
			return err
		}
		// For create, count will always be zero hence no need to query
		// by passing empty map, updateIterationsWithCounts will be able to put zero values
		wiCounts := make(map[string]workitem.WICountsPerIteration)
		log.Info(ctx, map[string]interface{}{
			"iteration_id": newItr.ID,
			"wiCounts":     wiCounts,
		}, "wicounts for created iteration %s -> %v", newItr.ID.String(), wiCounts)

		if newItr.Path.IsEmpty() == false {
			allParentsUUIDs := newItr.Path
			iterations, error := appl.Iterations().LoadMultiple(ctx, allParentsUUIDs)
			if error != nil {
				return err
			}
			itrMap := make(iterationIDMap)
			for _, itr := range iterations {
				itrMap[itr.ID] = itr
			}
			responseData = ConvertIteration(ctx.Request, newItr, parentPathResolver(itrMap), updateIterationsWithCounts(wiCounts))
		} else {
			responseData = ConvertIteration(ctx.Request, newItr, updateIterationsWithCounts(wiCounts))
		}
		return nil
	})
	if err != nil {
		return jsonapi.JSONErrorResponse(ctx, err)
	}
	res := &app.IterationSingle{
		Data: responseData,
	}
	ctx.ResponseData.Header().Set("Location", rest.AbsoluteURL(ctx.Request, app.IterationHref(res.Data.ID)))
	return ctx.Created(res)
}

// List runs the list action.
func (c *SpaceIterationsController) List(ctx *app.ListSpaceIterationsContext) error {
	err := application.Transactional(c.db, func(appl application.Application) error {
		err := appl.Spaces().CheckExists(ctx, ctx.SpaceID)
		if err != nil {
			return err
		}
		iterations, err := appl.Iterations().List(ctx, ctx.SpaceID)
		if err != nil {
			return err
		}
		return ctx.ConditionalEntities(iterations, c.config.GetCacheControlIterations, func() error {
			itrMap := make(iterationIDMap)
			for _, itr := range iterations {
				itrMap[itr.ID] = itr
			}
			// fetch extra information(counts of WI in each iteration of the space) to be added in response
			wiCounts, err := appl.WorkItems().GetCountsPerIteration(ctx, ctx.SpaceID)
			log.Debug(ctx, map[string]interface{}{
				"space_id": ctx.SpaceID.String(),
			}, "Retrieving wicounts for spaceID %s -> %v", ctx.SpaceID.String(), wiCounts)

			if err != nil {
				return err
			}
			res := &app.IterationList{}
			res.Data = ConvertIterations(ctx.Request, iterations, updateIterationsWithCounts(wiCounts), parentPathResolver(itrMap))
			return ctx.OK(res)
		})
	})
	if err != nil {
		return jsonapi.JSONErrorResponse(ctx, err)
	}
	return nil
}
<|endoftext|>


<commit_before>package err

import "fmt"

var Pkg = "packageName"

type Err struct {
	Pkg  string
	Info string
	Err  error
}

func (e *Err) Error() string {
	if e.Err == nil {
		return fmt.Sprintf("%s: %s", e.Pkg, e.Info)
	}
	return fmt.Sprintf("%s: %s\n%v", e.Pkg, e.Info, e.Err)
}

func makeErr(err error, info string) *Err {
	return &Err{
		Pkg:  Pkg,
		Info: info,
		Err:  err,
	}
}

func ce(err error, info string) (ret bool) {
	ret = true
	if err != nil {
		panic(makeErr(err, info))
	}
	ret = false
	return
}

func ct(err *error) {
	if p := recover(); p != nil {
		if e, ok := p.(error); ok {
			*err = e
		} else {
			panic(p)
		}
	}
}
<commit_msg>err: remove return value of ce<commit_after>package err

import "fmt"

var Pkg = "packageName"

type Err struct {
	Pkg  string
	Info string
	Err  error
}

func (e *Err) Error() string {
	if e.Err == nil {
		return fmt.Sprintf("%s: %s", e.Pkg, e.Info)
	}
	return fmt.Sprintf("%s: %s\n%v", e.Pkg, e.Info, e.Err)
}

func makeErr(err error, info string) *Err {
	return &Err{
		Pkg:  Pkg,
		Info: info,
		Err:  err,
	}
}

func ce(err error, info string) {
	if err != nil {
		panic(makeErr(err, info))
	}
}

func ct(err *error) {
	if p := recover(); p != nil {
		if e, ok := p.(error); ok {
			*err = e
		} else {
			panic(p)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright  OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package redactionprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/redactionprocessor"

import (
	"context"
	"fmt"
	"regexp"
	"sort"
	"strings"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/consumer"
	"go.opentelemetry.io/collector/pdata/pcommon"
	"go.opentelemetry.io/collector/pdata/ptrace"
	"go.uber.org/zap"
)

const attrValuesSeparator = ","

var _ component.TracesProcessor = (*redaction)(nil)

type redaction struct {
	// Attribute keys allowed in a span
	allowList map[string]string
	// Attribute values blocked in a span
	blockRegexList map[string]*regexp.Regexp
	// Redaction processor configuration
	config *Config
	// Logger
	logger *zap.Logger
	// Next trace consumer in line
	next consumer.Traces
}

// newRedaction creates a new instance of the redaction processor
func newRedaction(ctx context.Context, config *Config, logger *zap.Logger, next consumer.Traces) (*redaction, error) {
	allowList := makeAllowList(config)
	blockRegexList, err := makeBlockRegexList(ctx, config)
	if err != nil {
		// TODO: Placeholder for an error metric in the next PR
		return nil, fmt.Errorf("failed to process block list: %w", err)
	}

	return &redaction{
		allowList:      allowList,
		blockRegexList: blockRegexList,
		config:         config,
		logger:         logger,
		next:           next,
	}, nil
}

// processTraces implements ProcessMetricsFunc. It processes the incoming data
// and returns the data to be sent to the next component
func (s *redaction) processTraces(ctx context.Context, batch ptrace.Traces) (ptrace.Traces, error) {
	for i := 0; i < batch.ResourceSpans().Len(); i++ {
		rs := batch.ResourceSpans().At(i)
		s.processResourceSpan(ctx, rs)
	}
	return batch, nil
}

// processResourceSpan processes the RS and all of its spans and then returns the last
// view metric context. The context can be used for tests
func (s *redaction) processResourceSpan(ctx context.Context, rs ptrace.ResourceSpans) {
	rsAttrs := rs.Resource().Attributes()

	// Attributes can be part of a resource span
	s.processAttrs(ctx, &rsAttrs)

	for j := 0; j < rs.ScopeSpans().Len(); j++ {
		ils := rs.ScopeSpans().At(j)
		for k := 0; k < ils.Spans().Len(); k++ {
			span := ils.Spans().At(k)
			spanAttrs := span.Attributes()

			// Attributes can also be part of span
			s.processAttrs(ctx, &spanAttrs)
		}
	}
}

// processAttrs redacts the attributes of a resource span or a span
func (s *redaction) processAttrs(_ context.Context, attributes *pcommon.Map) {
	// TODO: Use the context for recording metrics
	var toDelete []string
	var toBlock []string

	// Identify attributes to redact and mask in the following sequence
	// 1. Make a list of attribute keys to redact
	// 2. Mask any blocked values for the other attributes
	// 3. Delete the attributes from 1
	//
	// This sequence satisfies these performance constraints:
	// - Only range through all attributes once
	// - Don't mask any values if the whole attribute is slated for deletion
	attributes.Range(func(k string, value pcommon.Value) bool {
		// Make a list of attribute keys to redact
		if !s.config.AllowAllKeys {
			if _, allowed := s.allowList[k]; !allowed {
				toDelete = append(toDelete, k)
				// Skip to the next attribute
				return true
			}
		}

		// Mask any blocked values for the other attributes
		strVal := value.StringVal()
		for _, compiledRE := range s.blockRegexList {
			match := compiledRE.MatchString(strVal)
			if match {
				toBlock = append(toBlock, k)

				maskedValue := compiledRE.ReplaceAllString(strVal, "****")
				value.SetStringVal(maskedValue)
			}
		}
		return true
	})

	// Delete the attributes on the redaction list
	for _, k := range toDelete {
		attributes.Remove(k)
	}
	// Add diagnostic information to the span
	s.summarizeRedactedSpan(toDelete, attributes)
	s.summarizeMaskedSpan(toBlock, attributes)
}

// ConsumeTraces implements the SpanProcessor interface
func (s *redaction) ConsumeTraces(ctx context.Context, batch ptrace.Traces) error {
	batch, err := s.processTraces(ctx, batch)
	if err != nil {
		return err
	}

	err = s.next.ConsumeTraces(ctx, batch)
	return err
}

// summarizeRedactedSpan adds diagnostic information about redacted attribute keys
func (s *redaction) summarizeRedactedSpan(deletedAttrs []string, attributes *pcommon.Map) {
	redactedCount := int64(len(deletedAttrs))
	if redactedCount == 0 {
		return
	}

	// Record summary as span attributes
	if s.config.Summary == debug {
		if existingVal, found := attributes.Get(redactedKeys); found && existingVal.StringVal() != "" {
			deletedAttrs = append(deletedAttrs, strings.Split(existingVal.StringVal(), attrValuesSeparator)...)
		}
		sort.Strings(deletedAttrs)
		attributes.UpsertString(redactedKeys, strings.Join(deletedAttrs, attrValuesSeparator))
	}
	if s.config.Summary == info || s.config.Summary == debug {
		if existingVal, found := attributes.Get(redactedKeyCount); found {
			redactedCount += existingVal.IntVal()
		}
		attributes.UpsertInt(redactedKeyCount, redactedCount)
	}
}

// summarizeMaskedSpan adds diagnostic information about masked attribute values
func (s *redaction) summarizeMaskedSpan(maskedAttrs []string, attributes *pcommon.Map) {
	maskedCount := int64(len(maskedAttrs))
	if maskedCount == 0 {
		return
	}

	// Records summary as span attributes
	if s.config.Summary == debug {
		if existingVal, found := attributes.Get(maskedValues); found && existingVal.StringVal() != "" {
			maskedAttrs = append(maskedAttrs, strings.Split(existingVal.StringVal(), attrValuesSeparator)...)
		}
		sort.Strings(maskedAttrs)
		attributes.UpsertString(maskedValues, strings.Join(maskedAttrs, attrValuesSeparator))
	}
	if s.config.Summary == info || s.config.Summary == debug {
		if existingVal, found := attributes.Get(maskedValueCount); found {
			maskedCount += existingVal.IntVal()
		}
		attributes.UpsertInt(maskedValueCount, maskedCount)
	}
}

const (
	debug            = "debug"
	info             = "info"
	redactedKeys     = "redaction.redacted.keys"
	redactedKeyCount = "redaction.redacted.count"
	maskedValues     = "redaction.masked.keys"
	maskedValueCount = "redaction.masked.count"
)

// makeAllowList sets up a lookup table of allowed span attribute keys
func makeAllowList(c *Config) map[string]string {
	// redactionKeys are additional span attributes created by the processor to
	// summarize the changes it made to a span. If the processor removes
	// 2 attributes from a span (e.g. `birth_date`, `mothers_maiden_name`),
	// then it will list them in the `redaction.redacted.keys` span attribute
	// and set the `redaction.redacted.count` attribute to 2
	//
	// If the processor finds and masks values matching a blocked regex in 2
	// span attributes (e.g. `notes`, `description`), then it will those
	// attribute keys in `redaction.masked.keys` and set the
	// `redaction.masked.count` to 2
	redactionKeys := []string{redactedKeys, redactedKeyCount, maskedValues, maskedValueCount}
	// allowList consists of the keys explicitly allowed by the configuration
	// as well as of the new span attributes that the processor creates to
	// summarize its changes
	allowList := make(map[string]string, len(c.AllowedKeys)+len(redactionKeys))
	for _, key := range c.AllowedKeys {
		allowList[key] = key
	}
	for _, key := range redactionKeys {
		allowList[key] = key
	}
	return allowList
}

// makeBlockRegexList precompiles all the blocked regex patterns
func makeBlockRegexList(_ context.Context, config *Config) (map[string]*regexp.Regexp, error) {
	blockRegexList := make(map[string]*regexp.Regexp, len(config.BlockedValues))
	for _, pattern := range config.BlockedValues {
		re, err := regexp.Compile(pattern)
		if err != nil {
			// TODO: Placeholder for an error metric in the next PR
			return nil, fmt.Errorf("error compiling regex in block list: %w", err)
		}
		blockRegexList[pattern] = re
	}
	return blockRegexList, nil
}

// Capabilities specifies what this processor does, such as whether it mutates data
func (s *redaction) Capabilities() consumer.Capabilities {
	return consumer.Capabilities{MutatesData: true}
}

// Start the redaction processor
func (s *redaction) Start(_ context.Context, _ component.Host) error {
	return nil
}

// Shutdown the redaction processor
func (s *redaction) Shutdown(context.Context) error {
	return nil
}
<commit_msg>[processor/redaction] Remove code duplication in adding meta attributes (#13916)<commit_after>// Copyright  OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package redactionprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/redactionprocessor"

import (
	"context"
	"fmt"
	"regexp"
	"sort"
	"strings"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/consumer"
	"go.opentelemetry.io/collector/pdata/pcommon"
	"go.opentelemetry.io/collector/pdata/ptrace"
	"go.uber.org/zap"
)

const attrValuesSeparator = ","

var _ component.TracesProcessor = (*redaction)(nil)

type redaction struct {
	// Attribute keys allowed in a span
	allowList map[string]string
	// Attribute values blocked in a span
	blockRegexList map[string]*regexp.Regexp
	// Redaction processor configuration
	config *Config
	// Logger
	logger *zap.Logger
	// Next trace consumer in line
	next consumer.Traces
}

// newRedaction creates a new instance of the redaction processor
func newRedaction(ctx context.Context, config *Config, logger *zap.Logger, next consumer.Traces) (*redaction, error) {
	allowList := makeAllowList(config)
	blockRegexList, err := makeBlockRegexList(ctx, config)
	if err != nil {
		// TODO: Placeholder for an error metric in the next PR
		return nil, fmt.Errorf("failed to process block list: %w", err)
	}

	return &redaction{
		allowList:      allowList,
		blockRegexList: blockRegexList,
		config:         config,
		logger:         logger,
		next:           next,
	}, nil
}

// processTraces implements ProcessMetricsFunc. It processes the incoming data
// and returns the data to be sent to the next component
func (s *redaction) processTraces(ctx context.Context, batch ptrace.Traces) (ptrace.Traces, error) {
	for i := 0; i < batch.ResourceSpans().Len(); i++ {
		rs := batch.ResourceSpans().At(i)
		s.processResourceSpan(ctx, rs)
	}
	return batch, nil
}

// processResourceSpan processes the RS and all of its spans and then returns the last
// view metric context. The context can be used for tests
func (s *redaction) processResourceSpan(ctx context.Context, rs ptrace.ResourceSpans) {
	rsAttrs := rs.Resource().Attributes()

	// Attributes can be part of a resource span
	s.processAttrs(ctx, &rsAttrs)

	for j := 0; j < rs.ScopeSpans().Len(); j++ {
		ils := rs.ScopeSpans().At(j)
		for k := 0; k < ils.Spans().Len(); k++ {
			span := ils.Spans().At(k)
			spanAttrs := span.Attributes()

			// Attributes can also be part of span
			s.processAttrs(ctx, &spanAttrs)
		}
	}
}

// processAttrs redacts the attributes of a resource span or a span
func (s *redaction) processAttrs(_ context.Context, attributes *pcommon.Map) {
	// TODO: Use the context for recording metrics
	var toDelete []string
	var toBlock []string

	// Identify attributes to redact and mask in the following sequence
	// 1. Make a list of attribute keys to redact
	// 2. Mask any blocked values for the other attributes
	// 3. Delete the attributes from 1
	//
	// This sequence satisfies these performance constraints:
	// - Only range through all attributes once
	// - Don't mask any values if the whole attribute is slated for deletion
	attributes.Range(func(k string, value pcommon.Value) bool {
		// Make a list of attribute keys to redact
		if !s.config.AllowAllKeys {
			if _, allowed := s.allowList[k]; !allowed {
				toDelete = append(toDelete, k)
				// Skip to the next attribute
				return true
			}
		}

		// Mask any blocked values for the other attributes
		strVal := value.StringVal()
		for _, compiledRE := range s.blockRegexList {
			match := compiledRE.MatchString(strVal)
			if match {
				toBlock = append(toBlock, k)

				maskedValue := compiledRE.ReplaceAllString(strVal, "****")
				value.SetStringVal(maskedValue)
			}
		}
		return true
	})

	// Delete the attributes on the redaction list
	for _, k := range toDelete {
		attributes.Remove(k)
	}
	// Add diagnostic information to the span
	s.addMetaAttrs(toDelete, attributes, redactedKeys, redactedKeyCount)
	s.addMetaAttrs(toBlock, attributes, maskedValues, maskedValueCount)
}

// ConsumeTraces implements the SpanProcessor interface
func (s *redaction) ConsumeTraces(ctx context.Context, batch ptrace.Traces) error {
	batch, err := s.processTraces(ctx, batch)
	if err != nil {
		return err
	}

	err = s.next.ConsumeTraces(ctx, batch)
	return err
}

// addMetaAttrs adds diagnostic information about redacted or masked attribute keys
func (s *redaction) addMetaAttrs(redactedAttrs []string, attributes *pcommon.Map, valuesAttr, countAttr string) {
	redactedCount := int64(len(redactedAttrs))
	if redactedCount == 0 {
		return
	}

	// Record summary as span attributes
	if s.config.Summary == debug {
		if existingVal, found := attributes.Get(valuesAttr); found && existingVal.StringVal() != "" {
			redactedAttrs = append(redactedAttrs, strings.Split(existingVal.StringVal(), attrValuesSeparator)...)
		}
		sort.Strings(redactedAttrs)
		attributes.UpsertString(valuesAttr, strings.Join(redactedAttrs, attrValuesSeparator))
	}
	if s.config.Summary == info || s.config.Summary == debug {
		if existingVal, found := attributes.Get(countAttr); found {
			redactedCount += existingVal.IntVal()
		}
		attributes.UpsertInt(countAttr, redactedCount)
	}
}

const (
	debug            = "debug"
	info             = "info"
	redactedKeys     = "redaction.redacted.keys"
	redactedKeyCount = "redaction.redacted.count"
	maskedValues     = "redaction.masked.keys"
	maskedValueCount = "redaction.masked.count"
)

// makeAllowList sets up a lookup table of allowed span attribute keys
func makeAllowList(c *Config) map[string]string {
	// redactionKeys are additional span attributes created by the processor to
	// summarize the changes it made to a span. If the processor removes
	// 2 attributes from a span (e.g. `birth_date`, `mothers_maiden_name`),
	// then it will list them in the `redaction.redacted.keys` span attribute
	// and set the `redaction.redacted.count` attribute to 2
	//
	// If the processor finds and masks values matching a blocked regex in 2
	// span attributes (e.g. `notes`, `description`), then it will those
	// attribute keys in `redaction.masked.keys` and set the
	// `redaction.masked.count` to 2
	redactionKeys := []string{redactedKeys, redactedKeyCount, maskedValues, maskedValueCount}
	// allowList consists of the keys explicitly allowed by the configuration
	// as well as of the new span attributes that the processor creates to
	// summarize its changes
	allowList := make(map[string]string, len(c.AllowedKeys)+len(redactionKeys))
	for _, key := range c.AllowedKeys {
		allowList[key] = key
	}
	for _, key := range redactionKeys {
		allowList[key] = key
	}
	return allowList
}

// makeBlockRegexList precompiles all the blocked regex patterns
func makeBlockRegexList(_ context.Context, config *Config) (map[string]*regexp.Regexp, error) {
	blockRegexList := make(map[string]*regexp.Regexp, len(config.BlockedValues))
	for _, pattern := range config.BlockedValues {
		re, err := regexp.Compile(pattern)
		if err != nil {
			// TODO: Placeholder for an error metric in the next PR
			return nil, fmt.Errorf("error compiling regex in block list: %w", err)
		}
		blockRegexList[pattern] = re
	}
	return blockRegexList, nil
}

// Capabilities specifies what this processor does, such as whether it mutates data
func (s *redaction) Capabilities() consumer.Capabilities {
	return consumer.Capabilities{MutatesData: true}
}

// Start the redaction processor
func (s *redaction) Start(_ context.Context, _ component.Host) error {
	return nil
}

// Shutdown the redaction processor
func (s *redaction) Shutdown(context.Context) error {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/coreos/go-etcd/etcd"
	"os"
	"os/exec"
	"strings"
)

var key = flag.String("key", "", "etcd key")

func main() {
	flag.Parse()
	if *key == "" {
		fmt.Fprintln(os.Stderr, "etcdenv [-key=key] [...]")
		flag.PrintDefaults()
		os.Exit(1)
	}
	client := etcd.NewClient()
	res, err := client.Get(*key)
	if err != nil {
		fmt.Fprintf(os.Stderr, "etcdenv:", err)
		os.Exit(1)
	}

	envs := []string{}
	for _, n := range res {
		key := strings.Split(n.Key, "/")
		k, v := strings.ToUpper(key[len(key)-1]), n.Value
		envs = append(envs, k + "=" + v)
	}
	if flag.NArg() == 0 {
		for _, env := range envs {
			line := fmt.Sprintf("%q", env)
			fmt.Println(line[1:len(line)-1])
		}
		os.Exit(0)
	}
	cmd := exec.Command(flag.Args()[0], flag.Args()[1:]...)
	cmd.Env = envs
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin
	err = cmd.Run()
	if err != nil {
		os.Exit(1)
	}
}
<commit_msg>Fixes error format<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/coreos/go-etcd/etcd"
	"os"
	"os/exec"
	"strings"
)

var key = flag.String("key", "", "etcd key")

func main() {
	flag.Parse()
	if *key == "" {
		fmt.Fprintln(os.Stderr, "etcdenv [-key=key] [...]")
		flag.PrintDefaults()
		os.Exit(1)
	}
	client := etcd.NewClient()
	res, err := client.Get(*key)
	if err != nil {
		fmt.Fprintf(os.Stderr, "etcdenv: %s\n", err)
		os.Exit(1)
	}

	envs := []string{}
	for _, n := range res {
		key := strings.Split(n.Key, "/")
		k, v := strings.ToUpper(key[len(key)-1]), n.Value
		envs = append(envs, k + "=" + v)
	}
	if flag.NArg() == 0 {
		for _, env := range envs {
			line := fmt.Sprintf("%q", env)
			fmt.Println(line[1:len(line)-1])
		}
		os.Exit(0)
	}
	cmd := exec.Command(flag.Args()[0], flag.Args()[1:]...)
	cmd.Env = envs
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin
	err = cmd.Run()
	if err != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "os"

type Symbol struct {
	name string
	arity int
	seen uint64
}

func (s *Symbol) PrintCyclic(id uint64) {
	if s.seen != id {
		s.seen = id
		fmt.Printf("node_%d_%p [label=\"%s/%d\"]\n", id, s, s.name, s.arity)
	}
}

type TermNode struct {
	symbol *Symbol
	varNode *VarNode
	term *TermNode
	child []*TermNode
	seen uint64
}

func (t *TermNode) PrintCyclicRoot(id uint64) {
	fmt.Printf("subgraph cluster_%d {\n", id)
	t.PrintCyclic(id)
	fmt.Println("}")
}

func (t *TermNode) PrintCyclic(id uint64) {
	if t.seen != id {
		t.seen = id
		if t.symbol != nil {
			fmt.Printf("node_%d_%p [label=\"%s\"]\n", id, t, t.symbol.name)
		} else if t.varNode != nil {
			fmt.Printf("node_%d_%p [label=\"%s\"]\n", id, t, t.varNode.symbol)
			fmt.Printf("node_%d_%p -> node_%d_%p [label=\"var\"]\n", id, t, id, t.varNode)
			t.varNode.PrintCyclic(id)
		}
		if t.term != nil {
			fmt.Printf("node_%d_%p -> node_%d_%p [label=\"term\"]\n", id, t, id, t.term)
			t.term.PrintCyclic(id)
		}
		for i, c := range t.child {
			fmt.Printf("node_%d_%p -> node_%d_%p [label=\"#%d\"]\n", id, t, id, c, i)
			c.PrintCyclic(id)
		}
	}
}

type VarNode struct {
	symbol string
	rep *VarNode
	term *TermNode
	varCount int
	termCount int
	seen uint64
}

func (v *VarNode) PrintCyclic(id uint64) {
	if v.seen != id {
		v.seen = id
		fmt.Printf("node_%d_%p [label=\"%s (%d, %d)\"]\n", id, v, v.symbol, v.varCount, v.termCount)
		if v.rep != nil {
			fmt.Printf("node_%d_%p -> node_%d_%p [label=\"rep\"]\n", id, v, id, v.rep)
			v.rep.PrintCyclic(id)
		}
		if v.term != nil {
			fmt.Printf("node_%d_%p -> node_%d_%p [label=\"term\"]\n", id, v, id, v.term)
			v.term.PrintCyclic(id)
		}
	}
}

type UnificationError struct {
	left, right *Symbol
}

var queue []*VarNode

func add(v *VarNode, t *TermNode) {
	if v.termCount == 1 {
		queue = append(queue, v)
	}
	t0 := v.term
	if t0 == nil {
		v.term, t.term = t, t
	} else {
		t0.term, t.term = t, t0.term
	}
	v.termCount++
}

func merge(v1, v2 *VarNode) {
	r1, r2 := v1.varCount, v2.varCount
	var bigV, v *VarNode
	if r1 >= r2 {
		bigV, v = v1, v2
	} else {
		bigV, v = v2, v1
	}
	k1, k2 := bigV.termCount, v.termCount
	if k1 <= 1 && k1 + k2 > 1 {
		queue = append(queue, bigV)
	}
	t0, t1 := v.term, bigV.term
	if t1 == nil {
		bigV.term = t0
	} else if t0 != nil {
		t1.term, t0.term = t0.term, t1.term
	}
	v.rep, v.term, v.varCount, v.termCount = bigV, nil, 0, 0
	bigV.varCount, bigV.termCount = r1 + r1, k1 + k2
}

func rep(v *VarNode) *VarNode {
	v0 := v.rep
	for v0 != v0.rep {
		v0 = v0.rep
	}
	for v.rep != v0 {
		v.rep, v = v0, v.rep
	}
	return v0
}

func commonFrontier(t_list []*TermNode) *UnificationError {
	sym := t_list[0].symbol
	for _, term := range t_list {
		if term.symbol != sym {
			return &UnificationError{
				left:  sym,
				right: term.symbol,
			}
		}
	}
	a := sym.arity
	t0_list := make([]*TermNode, len(t_list))
	s0Backing := make([]int, len(t_list))
	s1Backing := make([]int, len(t_list))
	for i := 0; i < a; i++ {
		for j := range t_list {
			t0_list[j] = t_list[j].child[i]
		}
		s0 := s0Backing[0:0]
		s1 := s1Backing[0:0]
		for j, term := range t0_list {
			if term.varNode != nil {
				s0 = append(s0, j)
			} else {
				s1 = append(s1, j)
			}
		}
		if len(s0) != 0 {
			j := s0[0]
			s0 := s0[1:]
			tmp := *t_list[0]
			*t_list[0] = *t_list[j]
			*t_list[j] = tmp
			v := rep(t0_list[j].varNode)
			for _, k := range s0 {
				v2 := rep(t0_list[k].varNode)
				if v != v2 {
					merge(v, v2)
				}
			}
			for _, k := range s1 {
				add(v, t0_list[k])
			}
		} else {
			return commonFrontier(t0_list)
		}
	}
	return nil
}

func unify(t_list []*TermNode) *UnificationError {
	queue = make([]*VarNode, 0)
	err := commonFrontier(t_list)
	if err != nil {
		return err
	}
	for len(queue) != 0 {
		v := queue[0]
		queue = queue[1:]
		k := v.termCount
		if k >= 2 {
			t := make([]*TermNode, k)
			t0 := v.term
			for i := 0; i < k; i++ {
				t[i], t0 = t0, t0.term
			}
			t[0].term = t[0]
			v.termCount = 1
			err = commonFrontier(t)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func main() {
	f := &Symbol{ "f", 2, 0 }
	x := &VarNode{ "x", nil, nil, 1, 0, 0 }
	x.rep = x
	z := &VarNode{ "z", nil, nil, 1, 0, 0 }
	z.rep = z
	x1 := &TermNode{ nil, x, nil, nil, 0 }
	x2 := &TermNode{ nil, x, nil, nil, 0 }
	z2 := &TermNode{ nil, z, nil, nil, 0 }
	expr1 := &TermNode{ f, nil, nil, []*TermNode{ x1, x1 }, 0 }
	expr2 := &TermNode{ f, nil, expr1, []*TermNode{ &TermNode{ f, nil, nil, []*TermNode{ z2, z2 }, 0 }, &TermNode{ f, nil, nil, []*TermNode{ z2, x2 }, 0 } }, 0 }
	expr1.term = expr2
	//fmt.Printf("Expr 1: %+v\n", expr1)
	//fmt.Printf("Expr 2: %+v\n", expr2)
	fmt.Println("digraph {")
	expr1.PrintCyclicRoot(1)
	err := unify([]*TermNode{expr1, expr2})
	if err != nil {
		fmt.Printf("Error: %+v\n", err)
		os.Exit(1)
	}
	expr1.PrintCyclicRoot(2)
	fmt.Println("}")
}
<commit_msg>Remove Symbol<commit_after>package main

import "fmt"
import "os"

const (
	SYMBOL_PRODUCT = uint64(iota)
	SYMBOL_NUMBER
)

var names = [...]string{
	"*",
	"N",
}

var arities = [...]uint{
	2,
	0,
}

type TermNode struct {
	symbol uint64
	varNode *VarNode
	term *TermNode
	child []*TermNode
	seen uint64
}

func (t *TermNode) PrintCyclicRoot(id uint64) {
	fmt.Printf("subgraph cluster_%d {\n", id)
	t.PrintCyclic(id)
	fmt.Println("}")
}

func (t *TermNode) PrintCyclic(id uint64) {
	if t.seen != id {
		t.seen = id
		if t.varNode != nil {
			fmt.Printf("node_%d_%p [label=\"%s\"]\n", id, t, t.varNode.symbol)
			fmt.Printf("node_%d_%p -> node_%d_%p [label=\"var\"]\n", id, t, id, t.varNode)
			t.varNode.PrintCyclic(id)
		} else {
			fmt.Printf("node_%d_%p [label=\"%s\"]\n", id, t, names[t.symbol])
		}
		if t.term != nil {
			fmt.Printf("node_%d_%p -> node_%d_%p [label=\"term\"]\n", id, t, id, t.term)
			t.term.PrintCyclic(id)
		}
		for i, c := range t.child {
			fmt.Printf("node_%d_%p -> node_%d_%p [label=\"#%d\"]\n", id, t, id, c, i)
			c.PrintCyclic(id)
		}
	}
}

type VarNode struct {
	symbol string
	rep *VarNode
	term *TermNode
	varCount int
	termCount int
	seen uint64
}

func (v *VarNode) PrintCyclic(id uint64) {
	if v.seen != id {
		v.seen = id
		fmt.Printf("node_%d_%p [label=\"%s (%d, %d)\"]\n", id, v, v.symbol, v.varCount, v.termCount)
		if v.rep != nil {
			fmt.Printf("node_%d_%p -> node_%d_%p [label=\"rep\"]\n", id, v, id, v.rep)
			v.rep.PrintCyclic(id)
		}
		if v.term != nil {
			fmt.Printf("node_%d_%p -> node_%d_%p [label=\"term\"]\n", id, v, id, v.term)
			v.term.PrintCyclic(id)
		}
	}
}

type UnificationError struct {
	left, right uint64
}

var queue []*VarNode

func add(v *VarNode, t *TermNode) {
	if v.termCount == 1 {
		queue = append(queue, v)
	}
	t0 := v.term
	if t0 == nil {
		v.term, t.term = t, t
	} else {
		t0.term, t.term = t, t0.term
	}
	v.termCount++
}

func merge(v1, v2 *VarNode) {
	r1, r2 := v1.varCount, v2.varCount
	var bigV, v *VarNode
	if r1 >= r2 {
		bigV, v = v1, v2
	} else {
		bigV, v = v2, v1
	}
	k1, k2 := bigV.termCount, v.termCount
	if k1 <= 1 && k1 + k2 > 1 {
		queue = append(queue, bigV)
	}
	t0, t1 := v.term, bigV.term
	if t1 == nil {
		bigV.term = t0
	} else if t0 != nil {
		t1.term, t0.term = t0.term, t1.term
	}
	v.rep, v.term, v.varCount, v.termCount = bigV, nil, 0, 0
	bigV.varCount, bigV.termCount = r1 + r1, k1 + k2
}

func rep(v *VarNode) *VarNode {
	v0 := v.rep
	for v0 != v0.rep {
		v0 = v0.rep
	}
	for v.rep != v0 {
		v.rep, v = v0, v.rep
	}
	return v0
}

func commonFrontier(t_list []*TermNode) *UnificationError {
	sym := t_list[0].symbol
	for _, term := range t_list {
		if term.symbol != sym {
			return &UnificationError{
				left:  sym,
				right: term.symbol,
			}
		}
	}
	a := arities[sym]
	t0_list := make([]*TermNode, len(t_list))
	s0Backing := make([]int, len(t_list))
	s1Backing := make([]int, len(t_list))
	for i := uint(0); i < a; i++ {
		for j := range t_list {
			t0_list[j] = t_list[j].child[i]
		}
		s0 := s0Backing[0:0]
		s1 := s1Backing[0:0]
		for j, term := range t0_list {
			if term.varNode != nil {
				s0 = append(s0, j)
			} else {
				s1 = append(s1, j)
			}
		}
		if len(s0) != 0 {
			j := s0[0]
			s0 := s0[1:]
			tmp := *t_list[0]
			*t_list[0] = *t_list[j]
			*t_list[j] = tmp
			v := rep(t0_list[j].varNode)
			for _, k := range s0 {
				v2 := rep(t0_list[k].varNode)
				if v != v2 {
					merge(v, v2)
				}
			}
			for _, k := range s1 {
				add(v, t0_list[k])
			}
		} else {
			return commonFrontier(t0_list)
		}
	}
	return nil
}

func unify(t_list []*TermNode) *UnificationError {
	queue = make([]*VarNode, 0)
	err := commonFrontier(t_list)
	if err != nil {
		return err
	}
	for len(queue) != 0 {
		v := queue[0]
		queue = queue[1:]
		k := v.termCount
		if k >= 2 {
			t := make([]*TermNode, k)
			t0 := v.term
			for i := 0; i < k; i++ {
				t[i], t0 = t0, t0.term
			}
			t[0].term = t[0]
			v.termCount = 1
			err = commonFrontier(t)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func main() {
	f := SYMBOL_PRODUCT
	x := &VarNode{ "x", nil, nil, 1, 0, 0 }
	x.rep = x
	z := &VarNode{ "z", nil, nil, 1, 0, 0 }
	z.rep = z
	x1 := &TermNode{ 0, x, nil, nil, 0 }
	x2 := &TermNode{ 0, x, nil, nil, 0 }
	z2 := &TermNode{ 0, z, nil, nil, 0 }
	expr1 := &TermNode{ f, nil, nil, []*TermNode{ x1, x1 }, 0 }
	expr2 := &TermNode{ f, nil, expr1, []*TermNode{ &TermNode{ f, nil, nil, []*TermNode{ z2, z2 }, 0 }, &TermNode{ f, nil, nil, []*TermNode{ z2, x2 }, 0 } }, 0 }
	expr1.term = expr2
	//fmt.Printf("Expr 1: %+v\n", expr1)
	//fmt.Printf("Expr 2: %+v\n", expr2)
	fmt.Println("digraph {")
	expr1.PrintCyclicRoot(1)
	err := unify([]*TermNode{expr1, expr2})
	if err != nil {
		fmt.Printf("Error: %+v\n", err)
		os.Exit(1)
	}
	expr1.PrintCyclicRoot(2)
	fmt.Println("}")
}
<|endoftext|>


<commit_before>package main

import "math"

type LabelType int

const (
	POSITIVE  LabelType = 1
	NEGATIVE  LabelType = -1
	UNLABELED LabelType = 0
)

type Example struct {
	Label       LabelType `json:"Label"`
	Fv          FeatureVector
	Url         string `json:"Url"`
	Title       string `json:"Title"`
	Description string `json:"Description"`
	Body        string `json:"Body"`
	Score       float64
	IsNew       bool
	StatusCode  int `json:StatusCode`
}

type Examples []*Example

func NewExample(url string, label LabelType) *Example {
	IsNew := false
	if label == UNLABELED {
		IsNew = true
	}
	return &Example{label, []string{}, url, "", "", "", 0.0, IsNew, 0}
}

func (example *Example) Annotate(label LabelType) {
	example.Label = label
}

func (example *Example) IsLabeled() bool {
	return example.Label != UNLABELED
}

func (slice Examples) Len() int {
	return len(slice)
}

func (slice Examples) Less(i, j int) bool {
	return math.Abs(slice[i].Score) < math.Abs(slice[j].Score)
}

func (slice Examples) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}
<commit_msg>ダブルクオートがなくてgo vetで怒られていた<commit_after>package main

import "math"

type LabelType int

const (
	POSITIVE  LabelType = 1
	NEGATIVE  LabelType = -1
	UNLABELED LabelType = 0
)

type Example struct {
	Label       LabelType `json:"Label"`
	Fv          FeatureVector
	Url         string `json:"Url"`
	Title       string `json:"Title"`
	Description string `json:"Description"`
	Body        string `json:"Body"`
	Score       float64
	IsNew       bool
	StatusCode  int `json:"StatusCode"`
}

type Examples []*Example

func NewExample(url string, label LabelType) *Example {
	IsNew := false
	if label == UNLABELED {
		IsNew = true
	}
	return &Example{label, []string{}, url, "", "", "", 0.0, IsNew, 0}
}

func (example *Example) Annotate(label LabelType) {
	example.Label = label
}

func (example *Example) IsLabeled() bool {
	return example.Label != UNLABELED
}

func (slice Examples) Len() int {
	return len(slice)
}

func (slice Examples) Less(i, j int) bool {
	return math.Abs(slice[i].Score) < math.Abs(slice[j].Score)
}

func (slice Examples) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}
<|endoftext|>


<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package ast

import (
	"fmt"
)

// this ExpressionVisitor will determine if the
// given expression can be satisfied by the
// specified dependencies, or if it instead
// has other dependencies
type ExpressionFunctionalDependencyChecker struct {
	Dependencies        ExpressionList
	AggregatesSatisfied bool
}

func NewExpressionFunctionalDependencyChecker(deps ExpressionList) *ExpressionFunctionalDependencyChecker {
	return &ExpressionFunctionalDependencyChecker{
		Dependencies:        deps,
		AggregatesSatisfied: true,
	}
}

func NewExpressionFunctionalDependencyCheckerFull(deps ExpressionList) *ExpressionFunctionalDependencyChecker {
	return &ExpressionFunctionalDependencyChecker{
		Dependencies:        deps,
		AggregatesSatisfied: false,
	}
}

func (this *ExpressionFunctionalDependencyChecker) Visit(expr Expression) (Expression, error) {
	// first let scalar literals pass, they do not depend on anything
	switch expr.(type) {
	case *LiteralNull:
		return nil, nil
	case *LiteralBool:
		return nil, nil
	case *LiteralNumber:
		return nil, nil
	case *LiteralString:
		return nil, nil
	case AggregateFunctionCallExpression:
		if this.AggregatesSatisfied {
			return nil, nil
		} else {
			// aggregates actually depend on the aggregate having been caclulated
			// this can't actually be represented by the dep list
			return nil, fmt.Errorf("The expression %v is not satisfied by these dependencies", expr)
		}
	}

	// next see if this expression is directly equivalent
	// to one of the provided dependencies
	e := this.Dependencies.EquivalentTo(expr)
	if e != nil {
		return nil, nil
	}

	// if this expression has no dependencies
	// then this expression was unsatisfied
	deps := expr.Dependencies()
	if len(deps) == 0 {
		return nil, fmt.Errorf("The expression %v is not satisfied by these dependencies", expr)
	}

	// otherwise, we need to satisfy ALL of its dependencies
	// for ANY/ALL we need a custom checker
	// for all others we continue to use *this
	checker := this.getAppropriateCheckerForDeps(expr)

	for _, dep := range deps {
		_, err := dep.Accept(checker)
		if err != nil {
			return nil, err
		}
	}

	return nil, nil
}

func (this *ExpressionFunctionalDependencyChecker) getAppropriateCheckerForDeps(expr Expression) *ExpressionFunctionalDependencyChecker {
	switch expr := expr.(type) {
	case CollectionOperatorExpression:
		// add the collection AS value
		// as a new satisifed dependency before checking
		newDeps := make(ExpressionList, len(this.Dependencies)+1)
		for i, dep := range this.Dependencies {
			newDeps[i] = dep
		}
		newDeps[len(this.Dependencies)] = NewProperty(expr.GetAs())
		colDepChecker := NewExpressionFunctionalDependencyChecker(newDeps)
		return colDepChecker
	default:
		return this
	}
}
<commit_msg>fix bug not regnozing [] or {} as constant expressions<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package ast

import (
	"fmt"
)

// this ExpressionVisitor will determine if the
// given expression can be satisfied by the
// specified dependencies, or if it instead
// has other dependencies
type ExpressionFunctionalDependencyChecker struct {
	Dependencies        ExpressionList
	AggregatesSatisfied bool
}

func NewExpressionFunctionalDependencyChecker(deps ExpressionList) *ExpressionFunctionalDependencyChecker {
	return &ExpressionFunctionalDependencyChecker{
		Dependencies:        deps,
		AggregatesSatisfied: true,
	}
}

func NewExpressionFunctionalDependencyCheckerFull(deps ExpressionList) *ExpressionFunctionalDependencyChecker {
	return &ExpressionFunctionalDependencyChecker{
		Dependencies:        deps,
		AggregatesSatisfied: false,
	}
}

func (this *ExpressionFunctionalDependencyChecker) Visit(expr Expression) (Expression, error) {
	// first let scalar literals pass, they do not depend on anything
	switch expr := expr.(type) {
	case *LiteralNull:
		return nil, nil
	case *LiteralBool:
		return nil, nil
	case *LiteralNumber:
		return nil, nil
	case *LiteralString:
		return nil, nil
	case *LiteralArray:
		//empty array is satisfied (non-empty handled later)
		if len(expr.Val) == 0 {
			return nil, nil
		}
	case *LiteralObject:
		// empty object is satisfied (non-empty handled later)
		if len(expr.Val) == 0 {
			return nil, nil
		}
	case AggregateFunctionCallExpression:
		if this.AggregatesSatisfied {
			return nil, nil
		} else {
			// aggregates actually depend on the aggregate having been caclulated
			// this can't actually be represented by the dep list
			return nil, fmt.Errorf("The expression %v is not satisfied by these dependencies", expr)
		}
	}

	// next see if this expression is directly equivalent
	// to one of the provided dependencies
	e := this.Dependencies.EquivalentTo(expr)
	if e != nil {
		return nil, nil
	}

	// if this expression has no dependencies
	// then this expression was unsatisfied
	deps := expr.Dependencies()
	if len(deps) == 0 {
		return nil, fmt.Errorf("The expression %v is not satisfied by these dependencies", expr)
	}

	// otherwise, we need to satisfy ALL of its dependencies
	// for ANY/ALL we need a custom checker
	// for all others we continue to use *this
	checker := this.getAppropriateCheckerForDeps(expr)

	for _, dep := range deps {
		_, err := dep.Accept(checker)
		if err != nil {
			return nil, err
		}
	}

	return nil, nil
}

func (this *ExpressionFunctionalDependencyChecker) getAppropriateCheckerForDeps(expr Expression) *ExpressionFunctionalDependencyChecker {
	switch expr := expr.(type) {
	case CollectionOperatorExpression:
		// add the collection AS value
		// as a new satisifed dependency before checking
		newDeps := make(ExpressionList, len(this.Dependencies)+1)
		for i, dep := range this.Dependencies {
			newDeps[i] = dep
		}
		newDeps[len(this.Dependencies)] = NewProperty(expr.GetAs())
		colDepChecker := NewExpressionFunctionalDependencyChecker(newDeps)
		return colDepChecker
	default:
		return this
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package l3plugin

import (
	"fmt"
	"net"

	govppapi "git.fd.io/govpp.git/api"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/logging/measure"
	"github.com/ligato/cn-infra/utils/safeclose"
	"github.com/ligato/vpp-agent/idxvpp"
	"github.com/ligato/vpp-agent/plugins/defaultplugins/ifplugin/bin_api/ip"
	"github.com/ligato/vpp-agent/plugins/defaultplugins/ifplugin/ifaceidx"
	"github.com/ligato/vpp-agent/plugins/defaultplugins/l3plugin/model/l3"
	"github.com/ligato/vpp-agent/plugins/defaultplugins/l3plugin/vppcalls"
	"github.com/ligato/vpp-agent/plugins/govppmux"
)

// ArpConfiguratorruns in the background in its own goroutine where it watches for any changes
// in the configuration of L3 arp entries as modelled by the proto file "../model/l3/l3.proto" and stored
// in ETCD under the key "/vnf-agent/{vnf-agent}/vpp/config/v1/arp". Updates received from the northbound API
// are compared with the VPP run-time configuration and differences are applied through the VPP binary API.
type ArpConfigurator struct {
	Log logging.Logger

	GoVppmux    govppmux.API
	ArpIndexes  idxvpp.NameToIdxRW
	ArpIndexSeq uint32
	SwIfIndexes ifaceidx.SwIfIndex
	vppChan     *govppapi.Channel

	Stopwatch *measure.Stopwatch
}

// Init initializes ARP configurator
func (plugin *ArpConfigurator) Init() (err error) {
	plugin.Log.Debug("Initializing ArpConfigurator")

	// Init VPP API channel
	plugin.vppChan, err = plugin.GoVppmux.NewAPIChannel()
	if err != nil {
		return err
	}

	err = plugin.checkMsgCompatibility()
	if err != nil {
		return err
	}

	return nil
}

// TransformArp converts raw entry data to ARP object
func TransformArp(arpInput *l3.ArpTable_ArpTableEntry, index ifaceidx.SwIfIndex, log logging.Logger) (*vppcalls.ArpEntry, error) {
	if arpInput == nil {
		log.Infof("ARP input is empty")
		return nil, nil
	}
	if arpInput.Interface == "" {
		log.Infof("ARP input does not contain interface")
		return nil, nil
	}
	if arpInput.IpAddress == "" {
		log.Infof("ARP input does not contain IP")
		return nil, nil
	}
	if arpInput.PhysAddress == "" {
		log.Infof("ARP input does not contain MAC")
		return nil, nil
	}

	ifName := arpInput.Interface
	ifIndex, _, exists := index.LookupIdx(ifName)
	if !exists {
		return nil, fmt.Errorf("ARP entry interface %v not found", ifName)
	}

	ipAddr := net.ParseIP(arpInput.IpAddress)
	macAddr, err := net.ParseMAC(arpInput.PhysAddress)
	if err != nil {
		return nil, err
	}
	arp := &vppcalls.ArpEntry{
		Interface:  ifIndex,
		IPAddress:  ipAddr,
		MacAddress: macAddr,
		Static:     arpInput.Static,
	}
	return arp, nil
}

// AddArp processes the NB config and propagates it to bin api call
func (plugin *ArpConfigurator) AddArp(entry *l3.ArpTable_ArpTableEntry) error {
	//plugin.Log.Infof("Creating new ARP entry %v -> %v (%v) for interface %v", entry.IpAddress, entry.PhysAddress, entry.Static, entry.Interface)
	plugin.Log.Infof("Creating ARP entry %+v", *entry)

	// Transform route data
	arp, err := TransformArp(entry, plugin.SwIfIndexes, plugin.Log)
	if err != nil {
		return err
	}
	if arp == nil {
		return nil
	}
	plugin.Log.Debugf("adding ARP: %+v", *arp)

	// Create and register new route
	err = vppcalls.VppAddArp(arp, plugin.vppChan, measure.GetTimeLog(ip.IPNeighborAddDel{}, plugin.Stopwatch))
	if err != nil {
		return err
	}
	arpID := arpIdentifier(arp.Interface, arp.IPAddress.String(), arp.MacAddress.String())
	plugin.ArpIndexes.RegisterName(arpID, plugin.ArpIndexSeq, nil)
	plugin.ArpIndexSeq++
	plugin.Log.Infof("ARP entry %v registered", arpID)

	return nil
}

// ChangeArp processes the NB config and propagates it to bin api call
func (plugin *ArpConfigurator) ChangeArp(entry *l3.ArpTable_ArpTableEntry, prevEntry *l3.ArpTable_ArpTableEntry) error {
	return fmt.Errorf("CHANGE ARP NOT IMPLEMENTED")
}

// DeleteArp processes the NB config and propagates it to bin api call
func (plugin *ArpConfigurator) DeleteArp(entry *l3.ArpTable_ArpTableEntry) error {
	plugin.Log.Infof("Deleting ARP entry %+v", *entry)

	// Transform route data
	arp, err := TransformArp(entry, plugin.SwIfIndexes, plugin.Log)
	if err != nil {
		return err
	}
	if arp == nil {
		return nil
	}
	plugin.Log.Debugf("deleting ARP: %+v", arp)

	// Delete and unregister new route
	err = vppcalls.VppDelArp(arp, plugin.vppChan, measure.GetTimeLog(ip.IPNeighborAddDel{}, plugin.Stopwatch))
	if err != nil {
		return err
	}
	arpID := arpIdentifier(arp.Interface, arp.IPAddress.String(), arp.MacAddress.String())
	_, _, found := plugin.ArpIndexes.UnregisterName(arpID)
	if found {
		plugin.Log.Infof("ARP entry %v unregistered", arpID)
	} else {
		plugin.Log.Warnf("Unregister failed, ARP entry %v not found", arpID)
	}

	return nil
}

// Close GOVPP channel
func (plugin *ArpConfigurator) Close() error {
	return safeclose.Close(plugin.vppChan)
}

// Creates unique identifier which serves as a name in name to index mapping
func arpIdentifier(iface uint32, ip, mac string) string {
	return fmt.Sprintf("arp-iface%v-%v-%v", iface, ip, mac)
}

func (plugin *ArpConfigurator) checkMsgCompatibility() error {
	msgs := []govppapi.Message{
		&ip.IPNeighborAddDel{},
		&ip.IPNeighborAddDelReply{},
	}
	err := plugin.vppChan.CheckMessageCompatibility(msgs...)
	if err != nil {
		plugin.Log.Error(err)
	}
	return err
}
<commit_msg>Fix typo in comment to satisfy lint<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package l3plugin

import (
	"fmt"
	"net"

	govppapi "git.fd.io/govpp.git/api"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/logging/measure"
	"github.com/ligato/cn-infra/utils/safeclose"
	"github.com/ligato/vpp-agent/idxvpp"
	"github.com/ligato/vpp-agent/plugins/defaultplugins/ifplugin/bin_api/ip"
	"github.com/ligato/vpp-agent/plugins/defaultplugins/ifplugin/ifaceidx"
	"github.com/ligato/vpp-agent/plugins/defaultplugins/l3plugin/model/l3"
	"github.com/ligato/vpp-agent/plugins/defaultplugins/l3plugin/vppcalls"
	"github.com/ligato/vpp-agent/plugins/govppmux"
)

// ArpConfigurator runs in the background in its own goroutine where it watches for any changes
// in the configuration of L3 arp entries as modelled by the proto file "../model/l3/l3.proto" and stored
// in ETCD under the key "/vnf-agent/{vnf-agent}/vpp/config/v1/arp". Updates received from the northbound API
// are compared with the VPP run-time configuration and differences are applied through the VPP binary API.
type ArpConfigurator struct {
	Log logging.Logger

	GoVppmux    govppmux.API
	ArpIndexes  idxvpp.NameToIdxRW
	ArpIndexSeq uint32
	SwIfIndexes ifaceidx.SwIfIndex
	vppChan     *govppapi.Channel

	Stopwatch *measure.Stopwatch
}

// Init initializes ARP configurator
func (plugin *ArpConfigurator) Init() (err error) {
	plugin.Log.Debug("Initializing ArpConfigurator")

	// Init VPP API channel
	plugin.vppChan, err = plugin.GoVppmux.NewAPIChannel()
	if err != nil {
		return err
	}

	err = plugin.checkMsgCompatibility()
	if err != nil {
		return err
	}

	return nil
}

// TransformArp converts raw entry data to ARP object
func TransformArp(arpInput *l3.ArpTable_ArpTableEntry, index ifaceidx.SwIfIndex, log logging.Logger) (*vppcalls.ArpEntry, error) {
	if arpInput == nil {
		log.Infof("ARP input is empty")
		return nil, nil
	}
	if arpInput.Interface == "" {
		log.Infof("ARP input does not contain interface")
		return nil, nil
	}
	if arpInput.IpAddress == "" {
		log.Infof("ARP input does not contain IP")
		return nil, nil
	}
	if arpInput.PhysAddress == "" {
		log.Infof("ARP input does not contain MAC")
		return nil, nil
	}

	ifName := arpInput.Interface
	ifIndex, _, exists := index.LookupIdx(ifName)
	if !exists {
		return nil, fmt.Errorf("ARP entry interface %v not found", ifName)
	}

	ipAddr := net.ParseIP(arpInput.IpAddress)
	macAddr, err := net.ParseMAC(arpInput.PhysAddress)
	if err != nil {
		return nil, err
	}
	arp := &vppcalls.ArpEntry{
		Interface:  ifIndex,
		IPAddress:  ipAddr,
		MacAddress: macAddr,
		Static:     arpInput.Static,
	}
	return arp, nil
}

// AddArp processes the NB config and propagates it to bin api call
func (plugin *ArpConfigurator) AddArp(entry *l3.ArpTable_ArpTableEntry) error {
	//plugin.Log.Infof("Creating new ARP entry %v -> %v (%v) for interface %v", entry.IpAddress, entry.PhysAddress, entry.Static, entry.Interface)
	plugin.Log.Infof("Creating ARP entry %+v", *entry)

	// Transform route data
	arp, err := TransformArp(entry, plugin.SwIfIndexes, plugin.Log)
	if err != nil {
		return err
	}
	if arp == nil {
		return nil
	}
	plugin.Log.Debugf("adding ARP: %+v", *arp)

	// Create and register new route
	err = vppcalls.VppAddArp(arp, plugin.vppChan, measure.GetTimeLog(ip.IPNeighborAddDel{}, plugin.Stopwatch))
	if err != nil {
		return err
	}
	arpID := arpIdentifier(arp.Interface, arp.IPAddress.String(), arp.MacAddress.String())
	plugin.ArpIndexes.RegisterName(arpID, plugin.ArpIndexSeq, nil)
	plugin.ArpIndexSeq++
	plugin.Log.Infof("ARP entry %v registered", arpID)

	return nil
}

// ChangeArp processes the NB config and propagates it to bin api call
func (plugin *ArpConfigurator) ChangeArp(entry *l3.ArpTable_ArpTableEntry, prevEntry *l3.ArpTable_ArpTableEntry) error {
	return fmt.Errorf("CHANGE ARP NOT IMPLEMENTED")
}

// DeleteArp processes the NB config and propagates it to bin api call
func (plugin *ArpConfigurator) DeleteArp(entry *l3.ArpTable_ArpTableEntry) error {
	plugin.Log.Infof("Deleting ARP entry %+v", *entry)

	// Transform route data
	arp, err := TransformArp(entry, plugin.SwIfIndexes, plugin.Log)
	if err != nil {
		return err
	}
	if arp == nil {
		return nil
	}
	plugin.Log.Debugf("deleting ARP: %+v", arp)

	// Delete and unregister new route
	err = vppcalls.VppDelArp(arp, plugin.vppChan, measure.GetTimeLog(ip.IPNeighborAddDel{}, plugin.Stopwatch))
	if err != nil {
		return err
	}
	arpID := arpIdentifier(arp.Interface, arp.IPAddress.String(), arp.MacAddress.String())
	_, _, found := plugin.ArpIndexes.UnregisterName(arpID)
	if found {
		plugin.Log.Infof("ARP entry %v unregistered", arpID)
	} else {
		plugin.Log.Warnf("Unregister failed, ARP entry %v not found", arpID)
	}

	return nil
}

// Close GOVPP channel
func (plugin *ArpConfigurator) Close() error {
	return safeclose.Close(plugin.vppChan)
}

// Creates unique identifier which serves as a name in name to index mapping
func arpIdentifier(iface uint32, ip, mac string) string {
	return fmt.Sprintf("arp-iface%v-%v-%v", iface, ip, mac)
}

func (plugin *ArpConfigurator) checkMsgCompatibility() error {
	msgs := []govppapi.Message{
		&ip.IPNeighborAddDel{},
		&ip.IPNeighborAddDelReply{},
	}
	err := plugin.vppChan.CheckMessageCompatibility(msgs...)
	if err != nil {
		plugin.Log.Error(err)
	}
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/urfave/cli"
	"log"
	"os"
)

const VERSION = "0.2.0"

func main() {
	s := Settings{}
	s.Hostnames = []string{}
	app := cli.NewApp()
	app.Name = "gloon"
	app.Usage = "Custom dns resolver with build in docker container support"
	app.UsageText = "gloon [options]"
	app.Version = VERSION
	app.Action = func(c *cli.Context) error {
		if c.StringSlice("hostname") != nil {
			s.Hostnames = c.StringSlice("hostname")
		}
		return appMain(&s)
	}
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:        "disable-forward",
			Usage:       "Disable request forwarding",
			Destination: &s.DisableForwarding,
		},
		cli.BoolFlag{
			Name:        "disable-docker",
			Usage:       "Disable docker support",
			Destination: &s.DisableDocker,
		},
		cli.StringFlag{
			Name:        "resolvconf, r",
			Value:       "/etc/resolv.conf",
			Usage:       "resolv.conf compatible `FILE` to use for request forwarding",
			Destination: &s.ResolvFile,
		},
		cli.StringFlag{
			Name:        "listen, l",
			Value:       ":53",
			Usage:       "Resolver listens on `ADDR`",
			Destination: &s.ResolverAddr,
		},
		cli.StringFlag{
			Name:        "api-addr, a",
			Value:       "",
			Usage:       "Api http server listens on `ADDR`. Default is no api server",
			Destination: &s.ApiAddr,
		},
		cli.StringFlag{
			Name:        "hostname-filter, f",
			Value:       "",
			Usage:       "Only docker containers with hostnames matching this `REGEX`  will have A records published",
			Destination: &s.HostnameFilter,
		},
		cli.StringFlag{
			Name:        "append-domain, d",
			Value:       "",
			Usage:       "Append `DOMAIN NAME` to all configured A records",
			Destination: &s.AppendDomain,
		},
		cli.StringFlag{
			Name:        "hostfile",
			Value:       "",
			Usage:       "Load up `FILE` at startup and add any records found. Wilcards are supported",
			Destination: &s.Hostfile,
		},
		cli.IntFlag{
			Name:        "reload-interval, i",
			Value:       0,
			Usage:       "Reload hostfile (where applicable) every `SEC` seconds. If unset, default is to try inotify or similiar where available",
			Destination: &s.HostfileReloadInterval,
		},
		cli.StringSliceFlag{
			Name:  "hostname, n",
			Usage: "Add A records in the form of `HOSTNAME=IP`  ",
		},
	}

	app.Run(os.Args)
}

func appMain(settings *Settings) (err error) {
	log.Printf("gloon %s starting...", VERSION)
	s, err := NewServer(settings.ResolverAddr, settings)
	if err != nil {
		log.Fatalf("Unable to create server: %s", err.Error())
	}
	if !settings.DisableDocker {
		dm, err := NewDockerMonitor(s.Records, settings)
		if err != nil {
			log.Printf("WARNING: unable to start docker monitor: %s. Docker hostname support will be disabled", err.Error())
		}
		go func() {
			dm.Run()
		}()
	}
	if settings.Hostfile != "" {
		hf := NewHostfile(settings.Hostfile, s.Records, settings.HostfileReloadInterval)
		go func() {
			hf.Run()
		}()
	}

	if settings.ApiAddr != "" {
		go func() {
			RunApiServer(settings, s.Records)
		}()
	}
	err = s.ListenAndServe()
	defer s.Shutdown()
	if err != nil {
		log.Fatalf("Unable to start server: %s", err.Error())
	}
	return
}
<commit_msg>use new rs class<commit_after>package main

import (
	"github.com/urfave/cli"
	"log"
	"os"
)

const VERSION = "0.2.0"

func main() {
	s := Settings{}
	s.Hostnames = []string{}
	app := cli.NewApp()
	app.Name = "gloon"
	app.Usage = "Custom dns resolver with build in docker container support"
	app.UsageText = "gloon [options]"
	app.Version = VERSION
	app.Action = func(c *cli.Context) error {
		if c.StringSlice("hostname") != nil {
			s.Hostnames = c.StringSlice("hostname")
		}
		return appMain(&s)
	}
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:        "disable-forward",
			Usage:       "Disable request forwarding",
			Destination: &s.DisableForwarding,
		},
		cli.BoolFlag{
			Name:        "disable-docker",
			Usage:       "Disable docker support",
			Destination: &s.DisableDocker,
		},
		cli.StringFlag{
			Name:        "resolvconf, r",
			Value:       "/etc/resolv.conf",
			Usage:       "resolv.conf compatible `FILE` to use for request forwarding",
			Destination: &s.ResolvFile,
		},
		cli.StringFlag{
			Name:        "listen, l",
			Value:       ":53",
			Usage:       "Resolver listens on `ADDR`",
			Destination: &s.ResolverAddr,
		},
		cli.StringFlag{
			Name:        "api-addr, a",
			Value:       "",
			Usage:       "Api http server listens on `ADDR`. Default is no api server",
			Destination: &s.ApiAddr,
		},
		cli.StringFlag{
			Name:        "hostname-filter, f",
			Value:       "",
			Usage:       "Only docker containers with hostnames matching this `REGEX`  will have A records published",
			Destination: &s.HostnameFilter,
		},
		cli.StringFlag{
			Name:        "append-domain, d",
			Value:       "",
			Usage:       "Append `DOMAIN NAME` to all configured A records",
			Destination: &s.AppendDomain,
		},
		cli.StringFlag{
			Name:        "hostfile",
			Value:       "",
			Usage:       "Load up `FILE` at startup and add any records found. Wilcards are supported",
			Destination: &s.Hostfile,
		},
		cli.IntFlag{
			Name:        "reload-interval, i",
			Value:       0,
			Usage:       "Reload hostfile (where applicable) every `SEC` seconds. If unset, default is to try inotify or similiar where available",
			Destination: &s.HostfileReloadInterval,
		},
		cli.StringSliceFlag{
			Name:  "hostname, n",
			Usage: "Add A records in the form of `HOSTNAME=IP`  ",
		},
	}

	app.Run(os.Args)
}

func appMain(settings *Settings) (err error) {
	log.Printf("gloon %s starting...", VERSION)
	s, err := NewServer(settings.ResolverAddr, settings)
	if err != nil {
		log.Fatalf("Unable to create server: %s", err.Error())
	}
	if !settings.DisableDocker {
		dm, err := NewDockerMonitor(s.RecordSet, settings)
		if err != nil {
			log.Printf("WARNING: unable to start docker monitor: %s. Docker hostname support will be disabled", err.Error())
		}
		go func() {
			dm.Run()
		}()
	}
	if settings.Hostfile != "" {
		hf := NewHostfile(settings.Hostfile, s.RecordSet, settings.HostfileReloadInterval)
		go func() {
			hf.Run()
		}()
	}

	if settings.ApiAddr != "" {
		go func() {
			RunApiServer(settings, s.RecordSet)
		}()
	}
	err = s.ListenAndServe()
	defer s.Shutdown()
	if err != nil {
		log.Fatalf("Unable to start server: %s", err.Error())
	}
	return
}
<|endoftext|>


<commit_before>package command

import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	hclog "github.com/hashicorp/go-hclog"
	plugin "github.com/hashicorp/go-plugin"
	"github.com/hashicorp/hcl"
	"github.com/hashicorp/hcl/hcl/ast"
	hcl2 "github.com/hashicorp/hcl2/hcl"
	"github.com/hashicorp/hcl2/hcldec"
	"github.com/hashicorp/nomad/plugins/base"
	"github.com/hashicorp/nomad/plugins/device"
	"github.com/hashicorp/nomad/plugins/shared"
	"github.com/hashicorp/nomad/plugins/shared/hclspec"
	"github.com/kr/pretty"
	"github.com/mitchellh/cli"
	"github.com/zclconf/go-cty/cty/msgpack"
)

func DeviceCommandFactory(meta Meta) cli.CommandFactory {
	return func() (cli.Command, error) {
		return &Device{Meta: meta}, nil
	}
}

type Device struct {
	Meta

	// dev is the plugin device
	dev device.DevicePlugin

	// spec is the returned and parsed spec.
	spec hcldec.Spec
}

func (c *Device) Help() string {
	helpText := `
Usage: nomad-plugin-launcher device <device-binary> <config_file>

  Device launches the given device binary and provides a REPL for interacting
  with it.

General Options:

` + generalOptionsUsage() + `

Device Options:

  -trace
    Enable trace level log output.
`

	return strings.TrimSpace(helpText)
}

func (c *Device) Synopsis() string {
	return "REPL for interacting with device plugins"
}

func (c *Device) Run(args []string) int {
	var trace bool
	cmdFlags := c.FlagSet("device")
	cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }
	cmdFlags.BoolVar(&trace, "trace", false, "")

	if err := cmdFlags.Parse(args); err != nil {
		c.logger.Error("failed to parse flags:", "error", err)
		return 1
	}
	if trace {
		c.logger.SetLevel(hclog.Trace)
	} else if c.verbose {
		c.logger.SetLevel(hclog.Debug)
	}

	args = cmdFlags.Args()
	numArgs := len(args)
	if numArgs < 1 {
		c.logger.Error("expected at least 1 args (device binary)", "args", args)
		return 1
	} else if numArgs > 2 {
		c.logger.Error("expected at most 2 args (device binary and config file)", "args", args)
		return 1
	}

	binary := args[0]
	var config []byte
	if numArgs == 2 {
		var err error
		config, err = ioutil.ReadFile(args[1])
		if err != nil {
			c.logger.Error("failed to read config file", "error", err)
			return 1
		}

		c.logger.Trace("read config", "config", string(config))
	}

	// Get the plugin
	dev, cleanup, err := c.getDevicePlugin(binary)
	if err != nil {
		c.logger.Error("failed to launch device plugin", "error", err)
		return 1
	}
	defer cleanup()
	c.dev = dev

	spec, err := c.getSpec()
	if err != nil {
		c.logger.Error("failed to get config spec", "error", err)
		return 1
	}
	c.spec = spec

	if err := c.setConfig(spec, config); err != nil {
		c.logger.Error("failed to set config", "error", err)
		return 1
	}

	if err := c.startRepl(); err != nil {
		c.logger.Error("error interacting with plugin", "error", err)
		return 1
	}

	return 0
}

func (c *Device) getDevicePlugin(binary string) (device.DevicePlugin, func(), error) {
	// Launch the plugin
	client := plugin.NewClient(&plugin.ClientConfig{
		HandshakeConfig: base.Handshake,
		Plugins: map[string]plugin.Plugin{
			base.PluginTypeBase:   &base.PluginBase{},
			base.PluginTypeDevice: &device.PluginDevice{},
		},
		Cmd:              exec.Command(binary),
		AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
		Logger:           c.logger,
	})

	// Connect via RPC
	rpcClient, err := client.Client()
	if err != nil {
		client.Kill()
		return nil, nil, err
	}

	// Request the plugin
	raw, err := rpcClient.Dispense(base.PluginTypeDevice)
	if err != nil {
		client.Kill()
		return nil, nil, err
	}

	// We should have a KV store now! This feels like a normal interface
	// implementation but is in fact over an RPC connection.
	dev := raw.(device.DevicePlugin)
	return dev, func() { client.Kill() }, nil
}

func (c *Device) getSpec() (hcldec.Spec, error) {
	// Get the schema so we can parse the config
	spec, err := c.dev.ConfigSchema()
	if err != nil {
		return nil, fmt.Errorf("failed to get config schema: %v", err)
	}

	c.logger.Trace("device spec", "spec", hclog.Fmt("% #v", pretty.Formatter(spec)))

	// Convert the schema
	schema, diag := hclspec.Convert(spec)
	if diag.HasErrors() {
		errStr := "failed to convert HCL schema: "
		for _, err := range diag.Errs() {
			errStr = fmt.Sprintf("%s\n* %s", errStr, err.Error())
		}
		return nil, errors.New(errStr)
	}

	return schema, nil
}

func (c *Device) setConfig(spec hcldec.Spec, config []byte) error {
	// Parse the config into hcl
	configVal, err := hclConfigToInterface(config)
	if err != nil {
		return err
	}

	c.logger.Trace("raw hcl config", "config", hclog.Fmt("% #v", pretty.Formatter(configVal)))

	ctx := &hcl2.EvalContext{
		Functions: shared.GetStdlibFuncs(),
	}

	val, diag := shared.ParseHclInterface(configVal, spec, ctx)
	if diag.HasErrors() {
		errStr := "failed to parse config"
		for _, err := range diag.Errs() {
			errStr = fmt.Sprintf("%s\n* %s", errStr, err.Error())
		}
		return errors.New(errStr)
	}
	c.logger.Trace("parsed hcl config", "config", hclog.Fmt("% #v", pretty.Formatter(val)))

	cdata, err := msgpack.Marshal(val, val.Type())
	if err != nil {
		return err
	}

	c.logger.Trace("msgpack config", "config", string(cdata))
	if err := c.dev.SetConfig(cdata); err != nil {
		return err
	}

	return nil
}

func hclConfigToInterface(config []byte) (interface{}, error) {
	if len(config) == 0 {
		return map[string]interface{}{}, nil
	}

	// Parse as we do in the jobspec parser
	root, err := hcl.Parse(string(config))
	if err != nil {
		return nil, fmt.Errorf("failed to hcl parse the config: %v", err)
	}

	// Top-level item should be a list
	list, ok := root.Node.(*ast.ObjectList)
	if !ok {
		return nil, fmt.Errorf("root should be an object")
	}

	var m map[string]interface{}
	if err := hcl.DecodeObject(&m, list.Items[0]); err != nil {
		return nil, fmt.Errorf("failed to decode object: %v", err)
	}

	return m["config"], nil
}

func (c *Device) startRepl() error {
	// Start the output goroutine
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	fingerprint := make(chan context.Context)
	stats := make(chan context.Context)
	reserve := make(chan []string)
	go c.replOutput(ctx, fingerprint, stats, reserve)

	c.Ui.Output("> Availabile commands are: exit(), fingerprint(), stop_fingerprint(), stats(), stop_stats(), reserve(id1, id2, ...)")
	var fingerprintCtx, statsCtx context.Context
	var fingerprintCancel, statsCancel context.CancelFunc
	for {
		in, err := c.Ui.Ask("> ")
		if err != nil {
			return err
		}

		switch {
		case in == "exit()":
			return nil
		case in == "fingerprint()":
			if fingerprintCtx != nil {
				continue
			}
			fingerprintCtx, fingerprintCancel = context.WithCancel(ctx)
			fingerprint <- fingerprintCtx
		case in == "stop_fingerprint()":
			if fingerprintCtx == nil {
				continue
			}
			fingerprintCancel()
			fingerprintCtx = nil
		case in == "stats()":
			if statsCtx != nil {
				continue
			}
			statsCtx, statsCancel = context.WithCancel(ctx)
			stats <- statsCtx
		case in == "stop_stats()":
			if statsCtx == nil {
				continue
			}
			statsCancel()
			statsCtx = nil
		case strings.HasPrefix(in, "reserve(") && strings.HasSuffix(in, ")"):
			listString := strings.TrimSuffix(strings.TrimPrefix(in, "reserve("), ")")
			ids := strings.Split(strings.TrimSpace(listString), ",")
			reserve <- ids
		default:
			c.Ui.Error(fmt.Sprintf("> Unknown command %q", in))
		}
	}

	return nil
}

func (c *Device) replOutput(ctx context.Context, startFingerprint, startStats <-chan context.Context, reserve <-chan []string) {
	var fingerprint <-chan *device.FingerprintResponse
	var stats <-chan *device.StatsResponse
	for {
		select {
		case <-ctx.Done():
			return
		case ctx := <-startFingerprint:
			var err error
			fingerprint, err = c.dev.Fingerprint(ctx)
			if err != nil {
				c.Ui.Error(fmt.Sprintf("fingerprint: %s", err))
				os.Exit(1)
			}
		case resp, ok := <-fingerprint:
			if !ok {
				c.Ui.Output("> fingerprint: fingerprint output closed")
				fingerprint = nil
				continue
			}

			if resp == nil {
				c.Ui.Warn("> fingerprint: received nil result")
				os.Exit(1)
			}

			c.Ui.Output(fmt.Sprintf("> fingerprint: % #v", pretty.Formatter(resp)))
		case ctx := <-startStats:
			var err error
			stats, err = c.dev.Stats(ctx)
			if err != nil {
				c.Ui.Error(fmt.Sprintf("stats: %s", err))
				os.Exit(1)
			}
		case resp, ok := <-stats:
			if !ok {
				c.Ui.Output("> stats: stats output closed")
				stats = nil
				continue
			}

			if resp == nil {
				c.Ui.Warn("> stats: received nil result")
				os.Exit(1)
			}

			c.Ui.Output(fmt.Sprintf("> stats: % #v", pretty.Formatter(resp)))
		case ids := <-reserve:
			resp, err := c.dev.Reserve(ids)
			if err != nil {
				c.Ui.Warn(fmt.Sprintf("> reserve(%s): %v", strings.Join(ids, ", "), err))
			} else {
				c.Ui.Output(fmt.Sprintf("> reserve(%s): % #v", strings.Join(ids, ", "), pretty.Formatter(resp)))
			}
		}
	}
}
<commit_msg>Fix device launcher ctx cleanup<commit_after>package command

import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	hclog "github.com/hashicorp/go-hclog"
	plugin "github.com/hashicorp/go-plugin"
	"github.com/hashicorp/hcl"
	"github.com/hashicorp/hcl/hcl/ast"
	hcl2 "github.com/hashicorp/hcl2/hcl"
	"github.com/hashicorp/hcl2/hcldec"
	"github.com/hashicorp/nomad/plugins/base"
	"github.com/hashicorp/nomad/plugins/device"
	"github.com/hashicorp/nomad/plugins/shared"
	"github.com/hashicorp/nomad/plugins/shared/hclspec"
	"github.com/kr/pretty"
	"github.com/mitchellh/cli"
	"github.com/zclconf/go-cty/cty/msgpack"
)

func DeviceCommandFactory(meta Meta) cli.CommandFactory {
	return func() (cli.Command, error) {
		return &Device{Meta: meta}, nil
	}
}

type Device struct {
	Meta

	// dev is the plugin device
	dev device.DevicePlugin

	// spec is the returned and parsed spec.
	spec hcldec.Spec
}

func (c *Device) Help() string {
	helpText := `
Usage: nomad-plugin-launcher device <device-binary> <config_file>

  Device launches the given device binary and provides a REPL for interacting
  with it.

General Options:

` + generalOptionsUsage() + `

Device Options:

  -trace
    Enable trace level log output.
`

	return strings.TrimSpace(helpText)
}

func (c *Device) Synopsis() string {
	return "REPL for interacting with device plugins"
}

func (c *Device) Run(args []string) int {
	var trace bool
	cmdFlags := c.FlagSet("device")
	cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }
	cmdFlags.BoolVar(&trace, "trace", false, "")

	if err := cmdFlags.Parse(args); err != nil {
		c.logger.Error("failed to parse flags:", "error", err)
		return 1
	}
	if trace {
		c.logger.SetLevel(hclog.Trace)
	} else if c.verbose {
		c.logger.SetLevel(hclog.Debug)
	}

	args = cmdFlags.Args()
	numArgs := len(args)
	if numArgs < 1 {
		c.logger.Error("expected at least 1 args (device binary)", "args", args)
		return 1
	} else if numArgs > 2 {
		c.logger.Error("expected at most 2 args (device binary and config file)", "args", args)
		return 1
	}

	binary := args[0]
	var config []byte
	if numArgs == 2 {
		var err error
		config, err = ioutil.ReadFile(args[1])
		if err != nil {
			c.logger.Error("failed to read config file", "error", err)
			return 1
		}

		c.logger.Trace("read config", "config", string(config))
	}

	// Get the plugin
	dev, cleanup, err := c.getDevicePlugin(binary)
	if err != nil {
		c.logger.Error("failed to launch device plugin", "error", err)
		return 1
	}
	defer cleanup()
	c.dev = dev

	spec, err := c.getSpec()
	if err != nil {
		c.logger.Error("failed to get config spec", "error", err)
		return 1
	}
	c.spec = spec

	if err := c.setConfig(spec, config); err != nil {
		c.logger.Error("failed to set config", "error", err)
		return 1
	}

	if err := c.startRepl(); err != nil {
		c.logger.Error("error interacting with plugin", "error", err)
		return 1
	}

	return 0
}

func (c *Device) getDevicePlugin(binary string) (device.DevicePlugin, func(), error) {
	// Launch the plugin
	client := plugin.NewClient(&plugin.ClientConfig{
		HandshakeConfig: base.Handshake,
		Plugins: map[string]plugin.Plugin{
			base.PluginTypeBase:   &base.PluginBase{},
			base.PluginTypeDevice: &device.PluginDevice{},
		},
		Cmd:              exec.Command(binary),
		AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
		Logger:           c.logger,
	})

	// Connect via RPC
	rpcClient, err := client.Client()
	if err != nil {
		client.Kill()
		return nil, nil, err
	}

	// Request the plugin
	raw, err := rpcClient.Dispense(base.PluginTypeDevice)
	if err != nil {
		client.Kill()
		return nil, nil, err
	}

	// We should have a KV store now! This feels like a normal interface
	// implementation but is in fact over an RPC connection.
	dev := raw.(device.DevicePlugin)
	return dev, func() { client.Kill() }, nil
}

func (c *Device) getSpec() (hcldec.Spec, error) {
	// Get the schema so we can parse the config
	spec, err := c.dev.ConfigSchema()
	if err != nil {
		return nil, fmt.Errorf("failed to get config schema: %v", err)
	}

	c.logger.Trace("device spec", "spec", hclog.Fmt("% #v", pretty.Formatter(spec)))

	// Convert the schema
	schema, diag := hclspec.Convert(spec)
	if diag.HasErrors() {
		errStr := "failed to convert HCL schema: "
		for _, err := range diag.Errs() {
			errStr = fmt.Sprintf("%s\n* %s", errStr, err.Error())
		}
		return nil, errors.New(errStr)
	}

	return schema, nil
}

func (c *Device) setConfig(spec hcldec.Spec, config []byte) error {
	// Parse the config into hcl
	configVal, err := hclConfigToInterface(config)
	if err != nil {
		return err
	}

	c.logger.Trace("raw hcl config", "config", hclog.Fmt("% #v", pretty.Formatter(configVal)))

	ctx := &hcl2.EvalContext{
		Functions: shared.GetStdlibFuncs(),
	}

	val, diag := shared.ParseHclInterface(configVal, spec, ctx)
	if diag.HasErrors() {
		errStr := "failed to parse config"
		for _, err := range diag.Errs() {
			errStr = fmt.Sprintf("%s\n* %s", errStr, err.Error())
		}
		return errors.New(errStr)
	}
	c.logger.Trace("parsed hcl config", "config", hclog.Fmt("% #v", pretty.Formatter(val)))

	cdata, err := msgpack.Marshal(val, val.Type())
	if err != nil {
		return err
	}

	c.logger.Trace("msgpack config", "config", string(cdata))
	if err := c.dev.SetConfig(cdata); err != nil {
		return err
	}

	return nil
}

func hclConfigToInterface(config []byte) (interface{}, error) {
	if len(config) == 0 {
		return map[string]interface{}{}, nil
	}

	// Parse as we do in the jobspec parser
	root, err := hcl.Parse(string(config))
	if err != nil {
		return nil, fmt.Errorf("failed to hcl parse the config: %v", err)
	}

	// Top-level item should be a list
	list, ok := root.Node.(*ast.ObjectList)
	if !ok {
		return nil, fmt.Errorf("root should be an object")
	}

	var m map[string]interface{}
	if err := hcl.DecodeObject(&m, list.Items[0]); err != nil {
		return nil, fmt.Errorf("failed to decode object: %v", err)
	}

	return m["config"], nil
}

func (c *Device) startRepl() error {
	// Start the output goroutine
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	fingerprint := make(chan context.Context)
	stats := make(chan context.Context)
	reserve := make(chan []string)
	go c.replOutput(ctx, fingerprint, stats, reserve)

	c.Ui.Output("> Availabile commands are: exit(), fingerprint(), stop_fingerprint(), stats(), stop_stats(), reserve(id1, id2, ...)")
	var fingerprintCtx, statsCtx context.Context
	var fingerprintCancel, statsCancel context.CancelFunc

	for {
		in, err := c.Ui.Ask("> ")
		if err != nil {
			if fingerprintCancel != nil {
				fingerprintCancel()
			}
			if statsCancel != nil {
				statsCancel()
			}
			return err
		}

		switch {
		case in == "exit()":
			if fingerprintCancel != nil {
				fingerprintCancel()
			}
			if statsCancel != nil {
				statsCancel()
			}
			return nil
		case in == "fingerprint()":
			if fingerprintCtx != nil {
				continue
			}
			fingerprintCtx, fingerprintCancel = context.WithCancel(ctx)
			fingerprint <- fingerprintCtx
		case in == "stop_fingerprint()":
			if fingerprintCtx == nil {
				continue
			}
			fingerprintCancel()
			fingerprintCtx = nil
		case in == "stats()":
			if statsCtx != nil {
				continue
			}
			statsCtx, statsCancel = context.WithCancel(ctx)
			stats <- statsCtx
		case in == "stop_stats()":
			if statsCtx == nil {
				continue
			}
			statsCancel()
			statsCtx = nil
		case strings.HasPrefix(in, "reserve(") && strings.HasSuffix(in, ")"):
			listString := strings.TrimSuffix(strings.TrimPrefix(in, "reserve("), ")")
			ids := strings.Split(strings.TrimSpace(listString), ",")
			reserve <- ids
		default:
			c.Ui.Error(fmt.Sprintf("> Unknown command %q", in))
		}
	}
}

func (c *Device) replOutput(ctx context.Context, startFingerprint, startStats <-chan context.Context, reserve <-chan []string) {
	var fingerprint <-chan *device.FingerprintResponse
	var stats <-chan *device.StatsResponse
	for {
		select {
		case <-ctx.Done():
			return
		case ctx := <-startFingerprint:
			var err error
			fingerprint, err = c.dev.Fingerprint(ctx)
			if err != nil {
				c.Ui.Error(fmt.Sprintf("fingerprint: %s", err))
				os.Exit(1)
			}
		case resp, ok := <-fingerprint:
			if !ok {
				c.Ui.Output("> fingerprint: fingerprint output closed")
				fingerprint = nil
				continue
			}

			if resp == nil {
				c.Ui.Warn("> fingerprint: received nil result")
				os.Exit(1)
			}

			c.Ui.Output(fmt.Sprintf("> fingerprint: % #v", pretty.Formatter(resp)))
		case ctx := <-startStats:
			var err error
			stats, err = c.dev.Stats(ctx)
			if err != nil {
				c.Ui.Error(fmt.Sprintf("stats: %s", err))
				os.Exit(1)
			}
		case resp, ok := <-stats:
			if !ok {
				c.Ui.Output("> stats: stats output closed")
				stats = nil
				continue
			}

			if resp == nil {
				c.Ui.Warn("> stats: received nil result")
				os.Exit(1)
			}

			c.Ui.Output(fmt.Sprintf("> stats: % #v", pretty.Formatter(resp)))
		case ids := <-reserve:
			resp, err := c.dev.Reserve(ids)
			if err != nil {
				c.Ui.Warn(fmt.Sprintf("> reserve(%s): %v", strings.Join(ids, ", "), err))
			} else {
				c.Ui.Output(fmt.Sprintf("> reserve(%s): % #v", strings.Join(ids, ", "), pretty.Formatter(resp)))
			}
		}
	}
}
<|endoftext|>