<commit_before>package instance

import (
	"fmt"
	"net"
	"reflect"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/scaleway/scaleway-cli/internal/human"
	"github.com/scaleway/scaleway-cli/internal/terminal"
	"github.com/scaleway/scaleway-sdk-go/api/instance/v1"
	"github.com/scaleway/scaleway-sdk-go/logger"
	"github.com/scaleway/scaleway-sdk-go/scw"
)

func init() {
	// register external types custom human marshaler func
	// for internal types use the human.Marshaler interface (see code at the end of file)

	// Image
	human.RegisterMarshalerFunc(instance.CreateImageResponse{}, marshallNestedField("Image"))

	// IP
	human.RegisterMarshalerFunc(instance.CreateIPResponse{}, marshallNestedField("IP"))

	// Placement Group
	human.RegisterMarshalerFunc(instance.CreatePlacementGroupResponse{}, marshallNestedField("PlacementGroup"))

	// Security Group
	human.RegisterMarshalerFunc(instance.CreateSecurityGroupResponse{}, marshallNestedField("SecurityGroup"))
	human.RegisterMarshalerFunc(instance.SecurityGroupPolicy(0), human.BindAttributesMarshalFunc(securityGroupPolicyAttribute))

	// Security Group Rule
	human.RegisterMarshalerFunc(instance.CreateSecurityGroupRuleResponse{}, marshallNestedField("Rule"))
	human.RegisterMarshalerFunc(instance.SecurityGroupRuleAction(0), human.BindAttributesMarshalFunc(securityGroupRuleActionAttribute))

	// Server
	human.RegisterMarshalerFunc(instance.CreateServerResponse{}, marshallNestedField("Server"))
	human.RegisterMarshalerFunc(instance.ServerState(0), serverStateMarshallerFunc)
	human.RegisterMarshalerFunc(instance.ServerLocation{}, serverLocationMarshallerFunc)
	human.RegisterMarshalerFunc([]*instance.Server{}, serversMarshallerFunc)
	human.RegisterMarshalerFunc(instance.GetServerResponse{}, getServerResponseMarshallerFunc)
	human.RegisterMarshalerFunc(instance.Bootscript{}, bootscriptMarshallerFunc)

	// Snapshot
	human.RegisterMarshalerFunc(instance.CreateSnapshotResponse{}, marshallNestedField("Snapshot"))

	// Volume
	human.RegisterMarshalerFunc(instance.CreateVolumeResponse{}, marshallNestedField("Volume"))
	human.RegisterMarshalerFunc(instance.VolumeState(0), human.BindAttributesMarshalFunc(volumeStateAttributes))
	human.RegisterMarshalerFunc(instance.VolumeSummary{}, volumeSummaryMarshallerFunc)
	human.RegisterMarshalerFunc(map[string]*instance.Volume{}, volumeMapMarshallerFunc)
}

// serverStateMarshallerFunc marshals a instance.ServerState.
var (
	serverStateAttributes = human.Attributes{
		instance.ServerStateRunning:        color.FgGreen,
		instance.ServerStateStopped:        color.Faint,
		instance.ServerStateStoppedInPlace: color.Faint,
		instance.ServerStateStarting:       color.FgBlue,
		instance.ServerStateStopping:       color.FgBlue,
		instance.ServerStateLocked:         color.FgRed,
	}

	volumeStateAttributes = human.Attributes{
		instance.VolumeStateError:     color.FgRed,
		instance.VolumeStateAvailable: color.FgGreen,
	}

	securityGroupPolicyAttribute = human.Attributes{
		instance.SecurityGroupPolicyDrop:   color.FgRed,
		instance.SecurityGroupPolicyAccept: color.FgGreen,
	}

	securityGroupRuleActionAttribute = human.Attributes{
		instance.SecurityGroupRuleActionDrop:   color.FgRed,
		instance.SecurityGroupRuleActionAccept: color.FgGreen,
	}
)

// serverLocationMarshallerFunc marshals a instance.ServerLocation.
func serverLocationMarshallerFunc(i interface{}, opt *human.MarshalOpt) (string, error) {
	location := i.(instance.ServerLocation)
	zone, err := scw.ParseZone(location.ZoneID)
	if err != nil {
		return "", err
	}
	zoneStr := fmt.Sprintf("%s", zone)
	return zoneStr, nil
}

// serverStateMarshallerFunc marshals a instance.ServerState.
func serverStateMarshallerFunc(i interface{}, opt *human.MarshalOpt) (string, error) {
	// The Scaleway console shows "archived" for a stopped server.
	if i.(instance.ServerState) == instance.ServerStateStopped {
		return terminal.Style("archived", color.Faint), nil
	}
	return human.BindAttributesMarshalFunc(serverStateAttributes)(i, opt)
}

// serversMarshallerFunc marshals a Server.
func serversMarshallerFunc(i interface{}, opt *human.MarshalOpt) (string, error) {
	// humanServerInList is the custom Server type used for list view.
	type humanServerInList struct {
		ID                string
		Name              string
		State             instance.ServerState
		Zone              scw.Zone
		PublicIP          net.IP
		PrivateIP         *string
		ImageName         string
		Tags              []string
		ModificationDate  time.Time
		CreationDate      time.Time
		ImageId           string
		Protected         bool
		Volumes           int
		SecurityGroupId   string
		SecurityGroupName string
		StateDetail       string
		Arch              instance.Arch
		PlacementGroup    *instance.PlacementGroup
	}

	servers := i.([]*instance.Server)
	humanServers := make([]*humanServerInList, 0)
	for _, server := range servers {
		var zone scw.Zone
		if server.Location != nil {
			zone_, err := scw.ParseZone(server.Location.ZoneID)
			if err != nil {
				return "", err
			}
			zone = zone_
		}
		publicIPAddress := net.IP(nil)
		if server.PublicIP != nil {
			publicIPAddress = server.PublicIP.Address
		}
		serverImageID := ""
		serverImageName := ""
		if server.Image != nil {
			serverImageID = server.Image.ID
			serverImageName = server.Image.Name
		}
		humanServers = append(humanServers, &humanServerInList{
			ID:                server.ID,
			Name:              server.Name,
			State:             server.State,
			Zone:              zone,
			ModificationDate:  server.ModificationDate,
			CreationDate:      server.CreationDate,
			ImageId:           serverImageID,
			ImageName:         serverImageName,
			Protected:         server.Protected,
			PublicIP:          publicIPAddress,
			PrivateIP:         server.PrivateIP,
			Volumes:           len(server.Volumes),
			SecurityGroupId:   server.SecurityGroup.ID,
			SecurityGroupName: server.SecurityGroup.Name,
			StateDetail:       server.StateDetail,
			Arch:              server.Arch,
			PlacementGroup:    server.PlacementGroup,
			Tags:              server.Tags,
		})
	}
	return human.Marshal(humanServers, opt)
}

// serversMarshallerFunc marshals a BootscriptID.
func bootscriptMarshallerFunc(i interface{}, opt *human.MarshalOpt) (string, error) {
	bootscript := i.(instance.Bootscript)
	return bootscript.Title, nil
}

// serversMarshallerFunc marshals a VolumeSummary.
func volumeSummaryMarshallerFunc(i interface{}, opt *human.MarshalOpt) (string, error) {
	volumeSummary := i.(instance.VolumeSummary)
	return human.Marshal(volumeSummary.ID, opt)
}

// volumeMapMarshallerFunc returns the length of the map.
func volumeMapMarshallerFunc(i interface{}, opt *human.MarshalOpt) (string, error) {
	volumes := i.(map[string]*instance.Volume)
	return fmt.Sprintf("%v", len(volumes)), nil
}

func getServerResponseMarshallerFunc(i interface{}, opt *human.MarshalOpt) (string, error) {
	serverResponse := i.(instance.GetServerResponse)

	// Sections
	opt.Sections = []*human.MarshalSection{
		{
			FieldName: "server",
			Title:     "Server",
		},
		{
			FieldName: "server.image",
			Title:     "Server Image",
		}, {
			FieldName: "server.allowed-actions",
		}, {
			FieldName: "volumes",
			Title:     "Volumes",
		},
	}

	customServer := &struct {
		Server  *instance.Server
		Volumes []*instance.Volume
	}{
		serverResponse.Server,
		orderVolumes(serverResponse.Server.Volumes),
	}

	str, err := human.Marshal(customServer, opt)
	if err != nil {
		return "", err
	}

	return str, nil
}

// orderVolumes return an ordered slice based on the volume map key "0", "1", "2",...
func orderVolumes(v map[string]*instance.Volume) []*instance.Volume {
	indexes := []string(nil)
	for index := range v {
		indexes = append(indexes, index)
	}
	sort.Strings(indexes)
	var orderedVolumes []*instance.Volume
	for _, index := range indexes {
		orderedVolumes = append(orderedVolumes, v[index])
	}
	return orderedVolumes
}

// marshallNestedField will marshal only the given field of a struct.
func marshallNestedField(nestedKey string) human.MarshalerFunc {
	return func(i interface{}, opt *human.MarshalOpt) (s string, err error) {
		if reflect.TypeOf(i).Kind() != reflect.Struct {
			return "", fmt.Errorf("%T must be a struct", i)
		}
		nestedValue := reflect.ValueOf(i).FieldByName(nestedKey)
		return human.Marshal(nestedValue.Interface(), opt)
	}
}

////
// Type implementing human.Marshaler interface
////

// MarshalHuman marshals a customSecurityGroupResponse.
func (sg *customSecurityGroupResponse) MarshalHuman() (out string, err error) {
	humanSecurityGroup := struct {
		ID                    string
		Name                  string
		Description           string
		EnableDefaultSecurity bool
		OrganizationID        string
		UseByDefault          bool
		CreationDate          time.Time
		ModificationDate      time.Time
	}{
		ID:                    sg.ID,
		Name:                  sg.Name,
		Description:           sg.Description,
		EnableDefaultSecurity: sg.EnableDefaultSecurity,
		OrganizationID:        sg.Organization,
		UseByDefault:          sg.OrganizationDefault,
		CreationDate:          sg.CreationDate,
		ModificationDate:      sg.ModificationDate,
	}

	securityGroupView, err := human.Marshal(humanSecurityGroup, nil)
	if err != nil {
		return "", err
	}
	securityGroupView = terminal.Style("Security Group:\n", color.Bold) + securityGroupView

	type humanRule struct {
		ID       string
		Protocol instance.SecurityGroupRuleProtocol
		Action   instance.SecurityGroupRuleAction
		IPRange  string
		Dest     string
	}

	toHumanRule := func(rule *instance.SecurityGroupRule) *humanRule {
		dest := "ALL"
		if rule.DestPortFrom != nil {
			dest = strconv.Itoa(int(*rule.DestPortFrom))
		}
		if rule.DestPortTo != nil {
			dest += "-" + strconv.Itoa(int(*rule.DestPortTo))
		}
		return &humanRule{
			ID:       rule.ID,
			Protocol: rule.Protocol,
			Action:   rule.Action,
			IPRange:  rule.IPRange.String(),
			Dest:     dest,
		}
	}

	inboundRules := []*humanRule(nil)
	outboundRules := []*humanRule(nil)
	for _, rule := range sg.Rules {
		switch rule.Direction {
		case instance.SecurityGroupRuleDirectionInbound:
			inboundRules = append(inboundRules, toHumanRule(rule))
		case instance.SecurityGroupRuleDirectionOutbound:
			outboundRules = append(outboundRules, toHumanRule(rule))
		default:
			logger.Warningf("invalid security group rule direction: %v", rule.Direction)
		}
	}

	// defaultInboundPolicy will already be colored in green or red by the marshaler.
	defaultInboundPolicy, err := human.Marshal(sg.InboundDefaultPolicy, nil)
	if err != nil {
		return "", err
	}

	// defaultOutboundPolicy will already be colored in green or red by the marshaler.
	defaultOutboundPolicy, err := human.Marshal(sg.OutboundDefaultPolicy, nil)
	if err != nil {
		return "", err
	}

	// b returns the given string in bold.
	// For inboundRulesView and outboundRulesView, this function must be called for every
	// concatenated part of the string because of the color package escaping at the end of
	// a color resulting in a non-bold format after the default{In|Out}boundPolicy.
	b := color.New(color.Bold).SprintFunc()

	inboundRulesContent, err := human.Marshal(inboundRules, nil)
	if err != nil {
		return "", err
	}
	inboundRulesView := b("Inbound Rules (default policy ") + b(defaultInboundPolicy) + b("):\n") + inboundRulesContent

	outboundRulesContent, err := human.Marshal(outboundRules, nil)
	if err != nil {
		return "", err
	}
	outboundRulesView := b("Outbound Rules (default policy ") + b(defaultOutboundPolicy) + b("):\n") + outboundRulesContent

	return strings.Join([]string{securityGroupView, inboundRulesView, outboundRulesView}, "\n\n"), nil
}
<commit_msg>feat(instance): improve security-group get output (#602)<commit_after>package instance

import (
	"fmt"
	"net"
	"reflect"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/scaleway/scaleway-cli/internal/human"
	"github.com/scaleway/scaleway-cli/internal/terminal"
	"github.com/scaleway/scaleway-sdk-go/api/instance/v1"
	"github.com/scaleway/scaleway-sdk-go/logger"
	"github.com/scaleway/scaleway-sdk-go/scw"
)

func init() {
	// register external types custom human marshaler func
	// for internal types use the human.Marshaler interface (see code at the end of file)

	// Image
	human.RegisterMarshalerFunc(instance.CreateImageResponse{}, marshallNestedField("Image"))

	// IP
	human.RegisterMarshalerFunc(instance.CreateIPResponse{}, marshallNestedField("IP"))

	// Placement Group
	human.RegisterMarshalerFunc(instance.CreatePlacementGroupResponse{}, marshallNestedField("PlacementGroup"))

	// Security Group
	human.RegisterMarshalerFunc(instance.CreateSecurityGroupResponse{}, marshallNestedField("SecurityGroup"))
	human.RegisterMarshalerFunc(instance.SecurityGroupPolicy(0), human.BindAttributesMarshalFunc(securityGroupPolicyAttribute))

	// Security Group Rule
	human.RegisterMarshalerFunc(instance.CreateSecurityGroupRuleResponse{}, marshallNestedField("Rule"))
	human.RegisterMarshalerFunc(instance.SecurityGroupRuleAction(0), human.BindAttributesMarshalFunc(securityGroupRuleActionAttribute))

	// Server
	human.RegisterMarshalerFunc(instance.CreateServerResponse{}, marshallNestedField("Server"))
	human.RegisterMarshalerFunc(instance.ServerState(0), serverStateMarshallerFunc)
	human.RegisterMarshalerFunc(instance.ServerLocation{}, serverLocationMarshallerFunc)
	human.RegisterMarshalerFunc([]*instance.Server{}, serversMarshallerFunc)
	human.RegisterMarshalerFunc(instance.GetServerResponse{}, getServerResponseMarshallerFunc)
	human.RegisterMarshalerFunc(instance.Bootscript{}, bootscriptMarshallerFunc)

	// Snapshot
	human.RegisterMarshalerFunc(instance.CreateSnapshotResponse{}, marshallNestedField("Snapshot"))

	// Volume
	human.RegisterMarshalerFunc(instance.CreateVolumeResponse{}, marshallNestedField("Volume"))
	human.RegisterMarshalerFunc(instance.VolumeState(0), human.BindAttributesMarshalFunc(volumeStateAttributes))
	human.RegisterMarshalerFunc(instance.VolumeSummary{}, volumeSummaryMarshallerFunc)
	human.RegisterMarshalerFunc(map[string]*instance.Volume{}, volumeMapMarshallerFunc)
}

// serverStateMarshallerFunc marshals a instance.ServerState.
var (
	serverStateAttributes = human.Attributes{
		instance.ServerStateRunning:        color.FgGreen,
		instance.ServerStateStopped:        color.Faint,
		instance.ServerStateStoppedInPlace: color.Faint,
		instance.ServerStateStarting:       color.FgBlue,
		instance.ServerStateStopping:       color.FgBlue,
		instance.ServerStateLocked:         color.FgRed,
	}

	volumeStateAttributes = human.Attributes{
		instance.VolumeStateError:     color.FgRed,
		instance.VolumeStateAvailable: color.FgGreen,
	}

	securityGroupPolicyAttribute = human.Attributes{
		instance.SecurityGroupPolicyDrop:   color.FgRed,
		instance.SecurityGroupPolicyAccept: color.FgGreen,
	}

	securityGroupRuleActionAttribute = human.Attributes{
		instance.SecurityGroupRuleActionDrop:   color.FgRed,
		instance.SecurityGroupRuleActionAccept: color.FgGreen,
	}
)

// serverLocationMarshallerFunc marshals a instance.ServerLocation.
func serverLocationMarshallerFunc(i interface{}, opt *human.MarshalOpt) (string, error) {
	location := i.(instance.ServerLocation)
	zone, err := scw.ParseZone(location.ZoneID)
	if err != nil {
		return "", err
	}
	zoneStr := fmt.Sprintf("%s", zone)
	return zoneStr, nil
}

// serverStateMarshallerFunc marshals a instance.ServerState.
func serverStateMarshallerFunc(i interface{}, opt *human.MarshalOpt) (string, error) {
	// The Scaleway console shows "archived" for a stopped server.
	if i.(instance.ServerState) == instance.ServerStateStopped {
		return terminal.Style("archived", color.Faint), nil
	}
	return human.BindAttributesMarshalFunc(serverStateAttributes)(i, opt)
}

// serversMarshallerFunc marshals a Server.
func serversMarshallerFunc(i interface{}, opt *human.MarshalOpt) (string, error) {
	// humanServerInList is the custom Server type used for list view.
	type humanServerInList struct {
		ID                string
		Name              string
		State             instance.ServerState
		Zone              scw.Zone
		PublicIP          net.IP
		PrivateIP         *string
		ImageName         string
		Tags              []string
		ModificationDate  time.Time
		CreationDate      time.Time
		ImageId           string
		Protected         bool
		Volumes           int
		SecurityGroupId   string
		SecurityGroupName string
		StateDetail       string
		Arch              instance.Arch
		PlacementGroup    *instance.PlacementGroup
	}

	servers := i.([]*instance.Server)
	humanServers := make([]*humanServerInList, 0)
	for _, server := range servers {
		var zone scw.Zone
		if server.Location != nil {
			zone_, err := scw.ParseZone(server.Location.ZoneID)
			if err != nil {
				return "", err
			}
			zone = zone_
		}
		publicIPAddress := net.IP(nil)
		if server.PublicIP != nil {
			publicIPAddress = server.PublicIP.Address
		}
		serverImageID := ""
		serverImageName := ""
		if server.Image != nil {
			serverImageID = server.Image.ID
			serverImageName = server.Image.Name
		}
		humanServers = append(humanServers, &humanServerInList{
			ID:                server.ID,
			Name:              server.Name,
			State:             server.State,
			Zone:              zone,
			ModificationDate:  server.ModificationDate,
			CreationDate:      server.CreationDate,
			ImageId:           serverImageID,
			ImageName:         serverImageName,
			Protected:         server.Protected,
			PublicIP:          publicIPAddress,
			PrivateIP:         server.PrivateIP,
			Volumes:           len(server.Volumes),
			SecurityGroupId:   server.SecurityGroup.ID,
			SecurityGroupName: server.SecurityGroup.Name,
			StateDetail:       server.StateDetail,
			Arch:              server.Arch,
			PlacementGroup:    server.PlacementGroup,
			Tags:              server.Tags,
		})
	}
	return human.Marshal(humanServers, opt)
}

// serversMarshallerFunc marshals a BootscriptID.
func bootscriptMarshallerFunc(i interface{}, opt *human.MarshalOpt) (string, error) {
	bootscript := i.(instance.Bootscript)
	return bootscript.Title, nil
}

// serversMarshallerFunc marshals a VolumeSummary.
func volumeSummaryMarshallerFunc(i interface{}, opt *human.MarshalOpt) (string, error) {
	volumeSummary := i.(instance.VolumeSummary)
	return human.Marshal(volumeSummary.ID, opt)
}

// volumeMapMarshallerFunc returns the length of the map.
func volumeMapMarshallerFunc(i interface{}, opt *human.MarshalOpt) (string, error) {
	volumes := i.(map[string]*instance.Volume)
	return fmt.Sprintf("%v", len(volumes)), nil
}

func getServerResponseMarshallerFunc(i interface{}, opt *human.MarshalOpt) (string, error) {
	serverResponse := i.(instance.GetServerResponse)

	// Sections
	opt.Sections = []*human.MarshalSection{
		{
			FieldName: "server",
			Title:     "Server",
		},
		{
			FieldName: "server.image",
			Title:     "Server Image",
		}, {
			FieldName: "server.allowed-actions",
		}, {
			FieldName: "volumes",
			Title:     "Volumes",
		},
	}

	customServer := &struct {
		Server  *instance.Server
		Volumes []*instance.Volume
	}{
		serverResponse.Server,
		orderVolumes(serverResponse.Server.Volumes),
	}

	str, err := human.Marshal(customServer, opt)
	if err != nil {
		return "", err
	}

	return str, nil
}

// orderVolumes return an ordered slice based on the volume map key "0", "1", "2",...
func orderVolumes(v map[string]*instance.Volume) []*instance.Volume {
	indexes := []string(nil)
	for index := range v {
		indexes = append(indexes, index)
	}
	sort.Strings(indexes)
	var orderedVolumes []*instance.Volume
	for _, index := range indexes {
		orderedVolumes = append(orderedVolumes, v[index])
	}
	return orderedVolumes
}

// marshallNestedField will marshal only the given field of a struct.
func marshallNestedField(nestedKey string) human.MarshalerFunc {
	return func(i interface{}, opt *human.MarshalOpt) (s string, err error) {
		if reflect.TypeOf(i).Kind() != reflect.Struct {
			return "", fmt.Errorf("%T must be a struct", i)
		}
		nestedValue := reflect.ValueOf(i).FieldByName(nestedKey)
		return human.Marshal(nestedValue.Interface(), opt)
	}
}

////
// Type implementing human.Marshaler interface
////

// MarshalHuman marshals a customSecurityGroupResponse.
func (sg *customSecurityGroupResponse) MarshalHuman() (out string, err error) {
	humanSecurityGroup := struct {
		ID                    string
		Name                  string
		Description           string
		EnableDefaultSecurity bool
		OrganizationID        string
		OrganizationDefault   bool
		CreationDate          time.Time
		ModificationDate      time.Time
		Stateful              bool
	}{
		ID:                    sg.ID,
		Name:                  sg.Name,
		Description:           sg.Description,
		EnableDefaultSecurity: sg.EnableDefaultSecurity,
		OrganizationID:        sg.Organization,
		OrganizationDefault:   sg.OrganizationDefault,
		CreationDate:          sg.CreationDate,
		ModificationDate:      sg.ModificationDate,
		Stateful:              sg.Stateful,
	}

	securityGroupView, err := human.Marshal(humanSecurityGroup, nil)
	if err != nil {
		return "", err
	}
	securityGroupView = terminal.Style("Security Group:\n", color.Bold) + securityGroupView

	type humanRule struct {
		ID       string
		Protocol instance.SecurityGroupRuleProtocol
		Action   instance.SecurityGroupRuleAction
		IPRange  string
		Dest     string
	}

	toHumanRule := func(rule *instance.SecurityGroupRule) *humanRule {
		dest := "ALL"
		if rule.DestPortFrom != nil {
			dest = strconv.Itoa(int(*rule.DestPortFrom))
		}
		if rule.DestPortTo != nil {
			dest += "-" + strconv.Itoa(int(*rule.DestPortTo))
		}
		return &humanRule{
			ID:       rule.ID,
			Protocol: rule.Protocol,
			Action:   rule.Action,
			IPRange:  rule.IPRange.String(),
			Dest:     dest,
		}
	}

	inboundRules := []*humanRule(nil)
	outboundRules := []*humanRule(nil)
	for _, rule := range sg.Rules {
		switch rule.Direction {
		case instance.SecurityGroupRuleDirectionInbound:
			inboundRules = append(inboundRules, toHumanRule(rule))
		case instance.SecurityGroupRuleDirectionOutbound:
			outboundRules = append(outboundRules, toHumanRule(rule))
		default:
			logger.Warningf("invalid security group rule direction: %v", rule.Direction)
		}
	}

	// defaultInboundPolicy will already be colored in green or red by the marshaler.
	defaultInboundPolicy, err := human.Marshal(sg.InboundDefaultPolicy, nil)
	if err != nil {
		return "", err
	}

	// defaultOutboundPolicy will already be colored in green or red by the marshaler.
	defaultOutboundPolicy, err := human.Marshal(sg.OutboundDefaultPolicy, nil)
	if err != nil {
		return "", err
	}

	// b returns the given string in bold.
	// For inboundRulesView and outboundRulesView, this function must be called for every
	// concatenated part of the string because of the color package escaping at the end of
	// a color resulting in a non-bold format after the default{In|Out}boundPolicy.
	b := color.New(color.Bold).SprintFunc()

	inboundRulesContent, err := human.Marshal(inboundRules, nil)
	if err != nil {
		return "", err
	}
	inboundRulesView := b("Inbound Rules (default policy ") + b(defaultInboundPolicy) + b("):\n") + inboundRulesContent

	outboundRulesContent, err := human.Marshal(outboundRules, nil)
	if err != nil {
		return "", err
	}
	outboundRulesView := b("Outbound Rules (default policy ") + b(defaultOutboundPolicy) + b("):\n") + outboundRulesContent

	return strings.Join([]string{securityGroupView, inboundRulesView, outboundRulesView}, "\n\n"), nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package source

import (
	"context"
	"errors"
	"fmt"
	"strings"

	"go.uber.org/zap"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/equality"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"knative.dev/eventing/pkg/reconciler/source"
	"knative.dev/pkg/apis"
	duckv1 "knative.dev/pkg/apis/duck/v1"

	"knative.dev/eventing-contrib/kafka/source/pkg/apis/sources/v1alpha1"
	"knative.dev/eventing-contrib/kafka/source/pkg/reconciler/source/resources"

	"k8s.io/client-go/kubernetes"
	appsv1listers "k8s.io/client-go/listers/apps/v1"
	"knative.dev/pkg/logging"
	pkgreconciler "knative.dev/pkg/reconciler"
	"knative.dev/pkg/resolver"

	"knative.dev/eventing-contrib/kafka/source/pkg/client/clientset/versioned"
	reconcilerkafkasource "knative.dev/eventing-contrib/kafka/source/pkg/client/injection/reconciler/sources/v1alpha1/kafkasource"
	listers "knative.dev/eventing-contrib/kafka/source/pkg/client/listers/sources/v1alpha1"
)

const (
	raImageEnvVar                = "KAFKA_RA_IMAGE"
	kafkaSourceDeploymentCreated = "KafkaSourceDeploymentCreated"
	kafkaSourceDeploymentUpdated = "KafkaSourceDeploymentUpdated"
	kafkaSourceDeploymentFailed  = "KafkaSourceDeploymentUpdated"
	component                    = "kafkasource"
)

// newDeploymentCreated makes a new reconciler event with event type Normal, and
// reason KafkaSourceDeploymentCreated.
func newDeploymentCreated(namespace, name string) pkgreconciler.Event {
	return pkgreconciler.NewEvent(corev1.EventTypeNormal, kafkaSourceDeploymentCreated, "KafkaSource created deployment: \"%s/%s\"", namespace, name)
}

// deploymentUpdated makes a new reconciler event with event type Normal, and
// reason KafkaSourceDeploymentUpdated.
func deploymentUpdated(namespace, name string) pkgreconciler.Event {
	return pkgreconciler.NewEvent(corev1.EventTypeNormal, kafkaSourceDeploymentUpdated, "KafkaSource updated deployment: \"%s/%s\"", namespace, name)
}

// newDeploymentFailed makes a new reconciler event with event type Warning, and
// reason KafkaSourceDeploymentFailed.
func newDeploymentFailed(namespace, name string, err error) pkgreconciler.Event {
	return pkgreconciler.NewEvent(corev1.EventTypeWarning, kafkaSourceDeploymentFailed, "KafkaSource failed to create deployment: \"%s/%s\", %w", namespace, name, err)
}

type Reconciler struct {
	// KubeClientSet allows us to talk to the k8s for core APIs
	KubeClientSet kubernetes.Interface

	receiveAdapterImage string

	kafkaLister      listers.KafkaSourceLister
	deploymentLister appsv1listers.DeploymentLister

	kafkaClientSet versioned.Interface
	loggingContext context.Context

	sinkResolver *resolver.URIResolver

	configs source.ConfigAccessor
}

// Check that our Reconciler implements Interface
var _ reconcilerkafkasource.Interface = (*Reconciler)(nil)

func (r *Reconciler) ReconcileKind(ctx context.Context, src *v1alpha1.KafkaSource) pkgreconciler.Event {
	src.Status.InitializeConditions()

	if src.Spec.Sink == nil {
		src.Status.MarkNoSink("SinkMissing", "")
		return fmt.Errorf("spec.sink missing")
	}

	dest := src.Spec.Sink.DeepCopy()
	if dest.Ref != nil {
		// To call URIFromDestination(), dest.Ref must have a Namespace. If there is
		// no Namespace defined in dest.Ref, we will use the Namespace of the source
		// as the Namespace of dest.Ref.
		if dest.Ref.Namespace == "" {
			dest.Ref.Namespace = src.GetNamespace()
		}
	}
	sinkURI, err := r.sinkResolver.URIFromDestinationV1(*dest, src)
	if err != nil {
		src.Status.MarkNoSink("NotFound", "")
		return fmt.Errorf("getting sink URI: %v", err)
	}
	src.Status.MarkSink(sinkURI)

	if val, ok := src.GetLabels()[v1alpha1.KafkaKeyTypeLabel]; ok {
		found := false
		for _, allowed := range v1alpha1.KafkaKeyTypeAllowed {
			if allowed == val {
				found = true
			}
		}
		if found {
			src.Status.MarkResourcesIncorrect("IncorrectKafkaKeyTypeLabel", "Invalid value for %s: %s. Allowed: %v", v1alpha1.KafkaKeyTypeLabel, val, v1alpha1.KafkaKeyTypeAllowed)
			logging.FromContext(ctx).Errorf("Invalid value for %s: %s. Allowed: %v", v1alpha1.KafkaKeyTypeLabel, val, v1alpha1.KafkaKeyTypeAllowed)
			return errors.New("IncorrectKafkaKeyTypeLabel")
		}
	}

	// TODO(mattmoor): create KafkaBinding for the receive adapter.

	ra, err := r.createReceiveAdapter(ctx, src, sinkURI)
	if err != nil {
		logging.FromContext(ctx).Error("Unable to create the receive adapter", zap.Error(err))
		return err
	}
	src.Status.MarkDeployed(ra)
	src.Status.CloudEventAttributes = r.createCloudEventAttributes(src)

	return nil
}

func checkResourcesStatus(src *v1alpha1.KafkaSource) error {
	for _, rsrc := range []struct {
		key   string
		field string
	}{{
		key:   "Request.CPU",
		field: src.Spec.Resources.Requests.ResourceCPU,
	}, {
		key:   "Request.Memory",
		field: src.Spec.Resources.Requests.ResourceMemory,
	}, {
		key:   "Limit.CPU",
		field: src.Spec.Resources.Limits.ResourceCPU,
	}, {
		key:   "Limit.Memory",
		field: src.Spec.Resources.Limits.ResourceMemory,
	}} {
		// In the event the field isn't specified, we assign a default in the receive_adapter
		if rsrc.field != "" {
			if _, err := resource.ParseQuantity(rsrc.field); err != nil {
				src.Status.MarkResourcesIncorrect("Incorrect Resource", "%s: %s, Error: %s", rsrc.key, rsrc.field, err)
				return err
			}
		}
	}
	src.Status.MarkResourcesCorrect()
	return nil
}

func (r *Reconciler) createReceiveAdapter(ctx context.Context, src *v1alpha1.KafkaSource, sinkURI *apis.URL) (*appsv1.Deployment, error) {
	if err := checkResourcesStatus(src); err != nil {
		return nil, err
	}

	raArgs := resources.ReceiveAdapterArgs{
		Image:          r.receiveAdapterImage,
		Source:         src,
		Labels:         resources.GetLabels(src.Name),
		SinkURI:        sinkURI.String(),
		AdditionalEnvs: r.configs.ToEnvVars(),
	}
	expected := resources.MakeReceiveAdapter(&raArgs)

	ra, err := r.KubeClientSet.AppsV1().Deployments(src.Namespace).Get(expected.Name, metav1.GetOptions{})
	if apierrors.IsNotFound(err) {
		ra, err = r.KubeClientSet.AppsV1().Deployments(src.Namespace).Create(expected)
		if err != nil {
			return nil, newDeploymentFailed(ra.Namespace, ra.Name, err)
		}
		return ra, newDeploymentCreated(ra.Namespace, ra.Name)
	} else if err != nil {
		logging.FromContext(ctx).Error("Unable to get an existing receive adapter", zap.Error(err))
		return nil, err
	} else if !metav1.IsControlledBy(ra, src) {
		return nil, fmt.Errorf("deployment %q is not owned by KafkaSource %q", ra.Name, src.Name)
	} else if podSpecChanged(ra.Spec.Template.Spec, expected.Spec.Template.Spec) {
		ra.Spec.Template.Spec = expected.Spec.Template.Spec
		if ra, err = r.KubeClientSet.AppsV1().Deployments(src.Namespace).Update(ra); err != nil {
			return ra, err
		}
		return ra, deploymentUpdated(ra.Namespace, ra.Name)
	} else {
		logging.FromContext(ctx).Debug("Reusing existing receive adapter", zap.Any("receiveAdapter", ra))
	}
	return ra, nil
}

func podSpecChanged(oldPodSpec corev1.PodSpec, newPodSpec corev1.PodSpec) bool {
	if !equality.Semantic.DeepDerivative(newPodSpec, oldPodSpec) {
		return true
	}
	if len(oldPodSpec.Containers) != len(newPodSpec.Containers) {
		return true
	}
	for i := range newPodSpec.Containers {
		if !equality.Semantic.DeepEqual(newPodSpec.Containers[i].Env, oldPodSpec.Containers[i].Env) {
			return true
		}
	}
	return false
}

func (r *Reconciler) createCloudEventAttributes(src *v1alpha1.KafkaSource) []duckv1.CloudEventAttributes {
	ceAttributes := make([]duckv1.CloudEventAttributes, 0, len(src.Spec.Topics))
	for i := range src.Spec.Topics {
		topics := strings.Split(src.Spec.Topics[i], ",")
		for _, topic := range topics {
			ceAttributes = append(ceAttributes, duckv1.CloudEventAttributes{
				Type:   v1alpha1.KafkaEventType,
				Source: v1alpha1.KafkaEventSource(src.Namespace, src.Name, topic),
			})
		}
	}
	return ceAttributes
}
<commit_msg>Super small issue that breaks the key type deserializer (#1275)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package source

import (
	"context"
	"errors"
	"fmt"
	"strings"

	"go.uber.org/zap"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/equality"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"knative.dev/eventing/pkg/reconciler/source"
	"knative.dev/pkg/apis"
	duckv1 "knative.dev/pkg/apis/duck/v1"

	"knative.dev/eventing-contrib/kafka/source/pkg/apis/sources/v1alpha1"
	"knative.dev/eventing-contrib/kafka/source/pkg/reconciler/source/resources"

	"k8s.io/client-go/kubernetes"
	appsv1listers "k8s.io/client-go/listers/apps/v1"
	"knative.dev/pkg/logging"
	pkgreconciler "knative.dev/pkg/reconciler"
	"knative.dev/pkg/resolver"

	"knative.dev/eventing-contrib/kafka/source/pkg/client/clientset/versioned"
	reconcilerkafkasource "knative.dev/eventing-contrib/kafka/source/pkg/client/injection/reconciler/sources/v1alpha1/kafkasource"
	listers "knative.dev/eventing-contrib/kafka/source/pkg/client/listers/sources/v1alpha1"
)

const (
	raImageEnvVar                = "KAFKA_RA_IMAGE"
	kafkaSourceDeploymentCreated = "KafkaSourceDeploymentCreated"
	kafkaSourceDeploymentUpdated = "KafkaSourceDeploymentUpdated"
	kafkaSourceDeploymentFailed  = "KafkaSourceDeploymentUpdated"
	component                    = "kafkasource"
)

// newDeploymentCreated makes a new reconciler event with event type Normal, and
// reason KafkaSourceDeploymentCreated.
func newDeploymentCreated(namespace, name string) pkgreconciler.Event {
	return pkgreconciler.NewEvent(corev1.EventTypeNormal, kafkaSourceDeploymentCreated, "KafkaSource created deployment: \"%s/%s\"", namespace, name)
}

// deploymentUpdated makes a new reconciler event with event type Normal, and
// reason KafkaSourceDeploymentUpdated.
func deploymentUpdated(namespace, name string) pkgreconciler.Event {
	return pkgreconciler.NewEvent(corev1.EventTypeNormal, kafkaSourceDeploymentUpdated, "KafkaSource updated deployment: \"%s/%s\"", namespace, name)
}

// newDeploymentFailed makes a new reconciler event with event type Warning, and
// reason KafkaSourceDeploymentFailed.
func newDeploymentFailed(namespace, name string, err error) pkgreconciler.Event {
	return pkgreconciler.NewEvent(corev1.EventTypeWarning, kafkaSourceDeploymentFailed, "KafkaSource failed to create deployment: \"%s/%s\", %w", namespace, name, err)
}

type Reconciler struct {
	// KubeClientSet allows us to talk to the k8s for core APIs
	KubeClientSet kubernetes.Interface

	receiveAdapterImage string

	kafkaLister      listers.KafkaSourceLister
	deploymentLister appsv1listers.DeploymentLister

	kafkaClientSet versioned.Interface
	loggingContext context.Context

	sinkResolver *resolver.URIResolver

	configs source.ConfigAccessor
}

// Check that our Reconciler implements Interface
var _ reconcilerkafkasource.Interface = (*Reconciler)(nil)

func (r *Reconciler) ReconcileKind(ctx context.Context, src *v1alpha1.KafkaSource) pkgreconciler.Event {
	src.Status.InitializeConditions()

	if src.Spec.Sink == nil {
		src.Status.MarkNoSink("SinkMissing", "")
		return fmt.Errorf("spec.sink missing")
	}

	dest := src.Spec.Sink.DeepCopy()
	if dest.Ref != nil {
		// To call URIFromDestination(), dest.Ref must have a Namespace. If there is
		// no Namespace defined in dest.Ref, we will use the Namespace of the source
		// as the Namespace of dest.Ref.
		if dest.Ref.Namespace == "" {
			dest.Ref.Namespace = src.GetNamespace()
		}
	}
	sinkURI, err := r.sinkResolver.URIFromDestinationV1(*dest, src)
	if err != nil {
		src.Status.MarkNoSink("NotFound", "")
		return fmt.Errorf("getting sink URI: %v", err)
	}
	src.Status.MarkSink(sinkURI)

	if val, ok := src.GetLabels()[v1alpha1.KafkaKeyTypeLabel]; ok {
		found := false
		for _, allowed := range v1alpha1.KafkaKeyTypeAllowed {
			if allowed == val {
				found = true
			}
		}
		if !found {
			src.Status.MarkResourcesIncorrect("IncorrectKafkaKeyTypeLabel", "Invalid value for %s: %s. Allowed: %v", v1alpha1.KafkaKeyTypeLabel, val, v1alpha1.KafkaKeyTypeAllowed)
			logging.FromContext(ctx).Errorf("Invalid value for %s: %s. Allowed: %v", v1alpha1.KafkaKeyTypeLabel, val, v1alpha1.KafkaKeyTypeAllowed)
			return errors.New("IncorrectKafkaKeyTypeLabel")
		}
	}

	// TODO(mattmoor): create KafkaBinding for the receive adapter.

	ra, err := r.createReceiveAdapter(ctx, src, sinkURI)
	if err != nil {
		logging.FromContext(ctx).Error("Unable to create the receive adapter", zap.Error(err))
		return err
	}
	src.Status.MarkDeployed(ra)
	src.Status.CloudEventAttributes = r.createCloudEventAttributes(src)

	return nil
}

func checkResourcesStatus(src *v1alpha1.KafkaSource) error {
	for _, rsrc := range []struct {
		key   string
		field string
	}{{
		key:   "Request.CPU",
		field: src.Spec.Resources.Requests.ResourceCPU,
	}, {
		key:   "Request.Memory",
		field: src.Spec.Resources.Requests.ResourceMemory,
	}, {
		key:   "Limit.CPU",
		field: src.Spec.Resources.Limits.ResourceCPU,
	}, {
		key:   "Limit.Memory",
		field: src.Spec.Resources.Limits.ResourceMemory,
	}} {
		// In the event the field isn't specified, we assign a default in the receive_adapter
		if rsrc.field != "" {
			if _, err := resource.ParseQuantity(rsrc.field); err != nil {
				src.Status.MarkResourcesIncorrect("Incorrect Resource", "%s: %s, Error: %s", rsrc.key, rsrc.field, err)
				return err
			}
		}
	}
	src.Status.MarkResourcesCorrect()
	return nil
}

func (r *Reconciler) createReceiveAdapter(ctx context.Context, src *v1alpha1.KafkaSource, sinkURI *apis.URL) (*appsv1.Deployment, error) {
	if err := checkResourcesStatus(src); err != nil {
		return nil, err
	}

	raArgs := resources.ReceiveAdapterArgs{
		Image:          r.receiveAdapterImage,
		Source:         src,
		Labels:         resources.GetLabels(src.Name),
		SinkURI:        sinkURI.String(),
		AdditionalEnvs: r.configs.ToEnvVars(),
	}
	expected := resources.MakeReceiveAdapter(&raArgs)

	ra, err := r.KubeClientSet.AppsV1().Deployments(src.Namespace).Get(expected.Name, metav1.GetOptions{})
	if apierrors.IsNotFound(err) {
		ra, err = r.KubeClientSet.AppsV1().Deployments(src.Namespace).Create(expected)
		if err != nil {
			return nil, newDeploymentFailed(ra.Namespace, ra.Name, err)
		}
		return ra, newDeploymentCreated(ra.Namespace, ra.Name)
	} else if err != nil {
		logging.FromContext(ctx).Error("Unable to get an existing receive adapter", zap.Error(err))
		return nil, err
	} else if !metav1.IsControlledBy(ra, src) {
		return nil, fmt.Errorf("deployment %q is not owned by KafkaSource %q", ra.Name, src.Name)
	} else if podSpecChanged(ra.Spec.Template.Spec, expected.Spec.Template.Spec) {
		ra.Spec.Template.Spec = expected.Spec.Template.Spec
		if ra, err = r.KubeClientSet.AppsV1().Deployments(src.Namespace).Update(ra); err != nil {
			return ra, err
		}
		return ra, deploymentUpdated(ra.Namespace, ra.Name)
	} else {
		logging.FromContext(ctx).Debug("Reusing existing receive adapter", zap.Any("receiveAdapter", ra))
	}
	return ra, nil
}

func podSpecChanged(oldPodSpec corev1.PodSpec, newPodSpec corev1.PodSpec) bool {
	if !equality.Semantic.DeepDerivative(newPodSpec, oldPodSpec) {
		return true
	}
	if len(oldPodSpec.Containers) != len(newPodSpec.Containers) {
		return true
	}
	for i := range newPodSpec.Containers {
		if !equality.Semantic.DeepEqual(newPodSpec.Containers[i].Env, oldPodSpec.Containers[i].Env) {
			return true
		}
	}
	return false
}

func (r *Reconciler) createCloudEventAttributes(src *v1alpha1.KafkaSource) []duckv1.CloudEventAttributes {
	ceAttributes := make([]duckv1.CloudEventAttributes, 0, len(src.Spec.Topics))
	for i := range src.Spec.Topics {
		topics := strings.Split(src.Spec.Topics[i], ",")
		for _, topic := range topics {
			ceAttributes = append(ceAttributes, duckv1.CloudEventAttributes{
				Type:   v1alpha1.KafkaEventType,
				Source: v1alpha1.KafkaEventSource(src.Namespace, src.Name, topic),
			})
		}
	}
	return ceAttributes
}
<|endoftext|>


<commit_before>package pdxcmd

import (
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
	"testing"
)

func datafile() string {
	d := EasyDate(2011, 1, 19)
	dir := MinutesDataDir(d)

	fi, err := ioutil.ReadFile(dir + "/statements.csv")
	if err != nil {
		panic(err)
	}
	text := string(fi)
	return text
}

func TestFindData(t *testing.T) {
	d := EasyDate(2011, 1, 19)
	dir := MinutesDataDir(d)

	fi, err := ioutil.ReadFile(dir + "/statements.csv")
	if err != nil {
		t.Error(err)
		panic(err)
	}
	text := string(fi)

	if len(text) < 500 {
		t.Errorf(text)
	}

	lines := strings.Split(text, "\n")
	test_index := 12
	if len(lines) < 100 {
		t.Errorf(lines[test_index])
	}

	split := strings.Split(lines[test_index], "::")
	if len(split) != 3 {
		t.Errorf("%#v\n", split)
	}

}

func TestTextParse(t *testing.T) {
	text := datafile()
	d := EasyDate(2011, 1, 19)

	c := make(chan []string)
	go ParseDoc(text, c)
	for s := range c {
		stmt := ParseTripleStmt(s, d)
		fmt.Println(stmt)
	}
	//t.Errorf("allgood")

}

func TestESLoad(t *testing.T) {
	text := datafile()
	d := EasyDate(2011, 1, 19)
	//testIndex := "pdxcmd_test"

	var stmts []Statement
	//stmts := make([]Statement, 0)
	c := make(chan []string)
	go ParseDoc(text, c)
	for s := range c {
		stmt := ParseTripleStmt(s, d)
		fmt.Println(stmt)
		stmts = append(stmts, stmt)
	}

	fmt.Printf("%#v\n", c)

	esc := ESConnect("localhost")
	blkindxr := esc.NewBulkIndexerErrors(10, 60)
	blkindxr.Start()
	for _, s := range stmts {
		blkindxr.Index("wat", "stmt", strconv.Itoa(s.Index), "", &s.Date, s, true)
		//fmt.Printf("%#v %#v\n", i, s)
	}
	blkindxr.Stop()
}

func TestESLoadFunc(t *testing.T) {
	text := datafile()
	d := EasyDate(2011, 1, 19)
	//testIndex := "pdxcmd_test"

	var stmts []Statement
	//stmts := make([]Statement, 0)
	c := make(chan []string)
	go ParseDoc(text, c)
	for s := range c {
		stmt := ParseTripleStmt(s, d)
		fmt.Println(stmt)
		stmts = append(stmts, stmt)
	}

	fmt.Printf("%#v\n", c)

	esc := ESConnect("localhost")
	LoadStatments(&stmts, esc, "watcat", "pdxcmd")

}
<commit_msg>Cleaning up loading tests<commit_after>package pdxcmd

import (
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
	"testing"
)

func datafile() string {
	d := EasyDate(2011, 1, 19)
	dir := MinutesDataDir(d)

	fi, err := ioutil.ReadFile(dir + "/statements.csv")
	if err != nil {
		panic(err)
	}
	text := string(fi)
	return text
}

func TestFindData(t *testing.T) {
	d := EasyDate(2011, 1, 19)
	dir := MinutesDataDir(d)

	fi, err := ioutil.ReadFile(dir + "/statements.csv")
	if err != nil {
		t.Error(err)
		panic(err)
	}
	text := string(fi)

	if len(text) < 500 {
		t.Errorf(text)
	}

	lines := strings.Split(text, "\n")
	test_index := 12
	if len(lines) < 100 {
		t.Errorf(lines[test_index])
	}

	split := strings.Split(lines[test_index], "::")
	if len(split) != 3 {
		t.Errorf("%#v\n", split)
	}

}

func TestTextParse(t *testing.T) {
	text := datafile()
	d := EasyDate(2011, 1, 19)

	c := make(chan []string)
	go ParseDoc(text, c)
	for s := range c {
		stmt := ParseTripleStmt(s, d)
		fmt.Println(stmt)
	}
	//t.Errorf("allgood")

}

func TestESLoad(t *testing.T) {
	text := datafile()
	d := EasyDate(2011, 1, 19)
	//testIndex := "pdxcmd_test"

	var stmts []Statement
	//stmts := make([]Statement, 0)
	c := make(chan []string)
	go ParseDoc(text, c)
	for s := range c {
		stmt := ParseTripleStmt(s, d)
		//fmt.Println(stmt)
		stmts = append(stmts, stmt)
	}

	//fmt.Printf("%#v\n", c)

	esc := ESConnect("localhost")
	blkindxr := esc.NewBulkIndexerErrors(10, 60)
	blkindxr.Start()
	for _, s := range stmts {
		blkindxr.Index("wat", "stmt", strconv.Itoa(s.Index), "", &s.Date, s, true)
		//fmt.Printf("%#v %#v\n", i, s)
	}
	blkindxr.Stop()
}

func TestESLoadFunc(t *testing.T) {
	text := datafile()
	d := EasyDate(2011, 1, 19)
	//testIndex := "pdxcmd_test"

	var stmts []Statement
	//stmts := make([]Statement, 0)
	c := make(chan []string)
	go ParseDoc(text, c)
	for s := range c {
		stmt := ParseTripleStmt(s, d)
		fmt.Println(stmt)
		stmts = append(stmts, stmt)
	}

	//fmt.Printf("%#v\n", c)

	esc := ESConnect("localhost")
	LoadStatments(&stmts, esc, "watcat", "pdxcmd")
}
<|endoftext|>


<commit_before>package css // import "github.com/tdewolff/parse/css"

import (
	"bytes"
	"testing"

	"github.com/stretchr/testify/assert"
)

////////////////////////////////////////////////////////////////

func assertParse(t *testing.T, input, expected string) {
	stylesheet, err := NewParser(NewTokenizer(&ReaderMockup{bytes.NewBufferString(input)})).Parse()
	assert.Nil(t, err, "parser must not return error")

	b := &bytes.Buffer{}
	stylesheet.Serialize(b)
	assert.Equal(t, expected, b.String(), "parsed string must match expected result in "+input)
}

func TestParser(t *testing.T) {
	assertParse(t, " <!-- x : y ; --> ", "<!--x:y;-->")
	assertParse(t, "color: red;", "color:red;")
	assertParse(t, "color : red;", "color:red;")
	assertParse(t, "color: red; border: 0;", "color:red;border:0;")
	assertParse(t, "color: red !important;", "color:red !important;")
	assertParse(t, "color: red ! important;", "color:red !important;")
	assertParse(t, "white-space: -moz-pre-wrap;", "white-space:-moz-pre-wrap;")
	assertParse(t, "display: -moz-inline-stack;", "display:-moz-inline-stack;")
	assertParse(t, "x: 10px / 1em;", "x:10px/1em;")
	assertParse(t, "x: 1em/1.5em \"Times New Roman\", Times, serif;", "x:1em/1.5em \"Times New Roman\",Times,serif;")
	assertParse(t, "x: hsla(100,50%, 75%, 0.5);", "x:hsla(100,50%,75%,0.5);")
	assertParse(t, "x: hsl(100,50%, 75%);", "x:hsl(100,50%,75%);")
	assertParse(t, "x: rgba(255, 238 , 221, 0.3);", "x:rgba(255,238,221,0.3);")
	assertParse(t, "x: 50vmax;", "x:50vmax;")
	assertParse(t, "color: linear-gradient(to right, black, white);", "color:linear-gradient(to right,black,white);")
	assertParse(t, "color: calc(100%/2 - 1em);", "color:calc(100%/2 - 1em);")
	assertParse(t, "color: calc(100%/2--1em);", "color:calc(100%/2 - -1em);")
	assertParse(t, "@media print, screen { }", "@media print,screen;")
	assertParse(t, "@media { @viewport {} }", "@media{@viewport;}")
	assertParse(t, "@keyframes 'diagonal-slide' {  from { left: 0; top: 0; } to { left: 100px; top: 100px; } }", "@keyframes 'diagonal-slide'{from{left:0;top:0;}to{left:100px;top:100px;}}")
	assertParse(t, "@keyframes movingbox{0%{left:90%;}50%{left:10%;}100%{left:90%;}}", "@keyframes movingbox{0%{left:90%;}50%{left:10%;}100%{left:90%;}}")
	assertParse(t, ".foo { color: #fff;}", ".foo{color:#fff;}")
	assertParse(t, ".foo { *color: #fff;}", ".foo{color:#fff;}")
	assertParse(t, ".foo { _color: #fff;}", ".foo{_color:#fff;}")
	assertParse(t, "a { color: red; border: 0; }", "a{color:red;border:0;}")
	assertParse(t, "a { color: red; ; } ; ;", "a{color:red;}")
	assertParse(t, "a { color: red; border: 0; } b { padding: 0; }", "a{color:red;border:0;}b{padding:0;}")

	// extraordinary
	assertParse(t, "color: red;;", "color:red;")
	assertParse(t, "@import;;", "@import;")
	assertParse(t, "@import;@import;", "@import;@import;")
	assertParse(t, ".a .b#c, .d<.e { x:y; }", ".a .b#c,.d<.e{x:y;}")
	assertParse(t, ".a[b~=c]d { x:y; }", ".a[b~=c]d{x:y;}")
	assertParse(t, "{x:y;}", "{x:y;}")
	assertParse(t, "a{}", "a{}")
	assertParse(t, "a,{x:y;}", "a{x:y;}")
	assertParse(t, "a,.b/*comment*/ {x:y;}", "a,.b{x:y;}")
	assertParse(t, "a,.b/*comment*/.c {x:y;}", "a,.b.c{x:y;}")
	assertParse(t, "a{x:; z:q;}", "a{x:;z:q;}")
	assertParse(t, "@import { @media f; x:y; }", "@import{@media f;x:y;}")
	assertParse(t, "a:not([controls]){x:y;}", "a:not([controls]){x:y;}")
	assertParse(t, "color:#c0c0c0", "color:#c0c0c0;")
	assertParse(t, "background:URL(x.png);", "background:URL(x.png);")
	assertParse(t, "@document regexp('https:.*') { p { color: red; } }", "@document regexp('https:.*'){p{color:red;}}")
	assertParse(t, "@media all and ( max-width:400px ) { }", "@media all and (max-width:400px);")
	assertParse(t, "@media (max-width:400px) { }", "@media (max-width:400px);")
	assertParse(t, "@media (max-width:400px)", "@media (max-width:400px);")
	assertParse(t, "a[x={]{x:y;}", "a[x={]{x:y;}")
	assertParse(t, "a[x=,]{x:y;}", "a[x=,]{x:y;}")
	assertParse(t, "a[x=+]{x:y;}", "a[x=+]{x:y;}")
	assertParse(t, ".cla .ss > #id { x:y; }", ".cla .ss>#id{x:y;}")
	assertParse(t, ".cla /*a*/ /*b*/ .ss{}", ".cla .ss{}")
	assertParse(t, "filter: progid : DXImageTransform.Microsoft.BasicImage(rotation=1);", "filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=1);")
	assertParse(t, "a{x:f(a(),b);}", "a{x:f(a(),b);}")

	// issues
	assertParse(t, "@media print {.class{width:5px;}}", "@media print{.class{width:5px;}}") // #6
	assertParse(t, ".class{width:calc((50% + 2em)/2 + 14px);}}", ".class{width:calc((50% + 2em)/2 + 14px);}}") // #7
}
<commit_msg>Parse function like before<commit_after>package css // import "github.com/tdewolff/parse/css"

import (
	"bytes"
	"testing"

	"github.com/stretchr/testify/assert"
)

////////////////////////////////////////////////////////////////

func assertParse(t *testing.T, input, expected string) {
	stylesheet, err := Parse(&ReaderMockup{bytes.NewBufferString(input)})
	assert.Nil(t, err, "parser must not return error")

	b := &bytes.Buffer{}
	stylesheet.Serialize(b)
	assert.Equal(t, expected, b.String(), "parsed string must match expected result in "+input)
}

func TestParser(t *testing.T) {
	assertParse(t, " <!-- x : y ; --> ", "<!--x:y;-->")
	assertParse(t, "color: red;", "color:red;")
	assertParse(t, "color : red;", "color:red;")
	assertParse(t, "color: red; border: 0;", "color:red;border:0;")
	assertParse(t, "color: red !important;", "color:red !important;")
	assertParse(t, "color: red ! important;", "color:red !important;")
	assertParse(t, "white-space: -moz-pre-wrap;", "white-space:-moz-pre-wrap;")
	assertParse(t, "display: -moz-inline-stack;", "display:-moz-inline-stack;")
	assertParse(t, "x: 10px / 1em;", "x:10px/1em;")
	assertParse(t, "x: 1em/1.5em \"Times New Roman\", Times, serif;", "x:1em/1.5em \"Times New Roman\",Times,serif;")
	assertParse(t, "x: hsla(100,50%, 75%, 0.5);", "x:hsla(100,50%,75%,0.5);")
	assertParse(t, "x: hsl(100,50%, 75%);", "x:hsl(100,50%,75%);")
	assertParse(t, "x: rgba(255, 238 , 221, 0.3);", "x:rgba(255,238,221,0.3);")
	assertParse(t, "x: 50vmax;", "x:50vmax;")
	assertParse(t, "color: linear-gradient(to right, black, white);", "color:linear-gradient(to right,black,white);")
	assertParse(t, "color: calc(100%/2 - 1em);", "color:calc(100%/2 - 1em);")
	assertParse(t, "color: calc(100%/2--1em);", "color:calc(100%/2 - -1em);")
	assertParse(t, "@media print, screen { }", "@media print,screen;")
	assertParse(t, "@media { @viewport {} }", "@media{@viewport;}")
	assertParse(t, "@keyframes 'diagonal-slide' {  from { left: 0; top: 0; } to { left: 100px; top: 100px; } }", "@keyframes 'diagonal-slide'{from{left:0;top:0;}to{left:100px;top:100px;}}")
	assertParse(t, "@keyframes movingbox{0%{left:90%;}50%{left:10%;}100%{left:90%;}}", "@keyframes movingbox{0%{left:90%;}50%{left:10%;}100%{left:90%;}}")
	assertParse(t, ".foo { color: #fff;}", ".foo{color:#fff;}")
	assertParse(t, ".foo { *color: #fff;}", ".foo{color:#fff;}")
	assertParse(t, ".foo { _color: #fff;}", ".foo{_color:#fff;}")
	assertParse(t, "a { color: red; border: 0; }", "a{color:red;border:0;}")
	assertParse(t, "a { color: red; ; } ; ;", "a{color:red;}")
	assertParse(t, "a { color: red; border: 0; } b { padding: 0; }", "a{color:red;border:0;}b{padding:0;}")

	// extraordinary
	assertParse(t, "color: red;;", "color:red;")
	assertParse(t, "@import;;", "@import;")
	assertParse(t, "@import;@import;", "@import;@import;")
	assertParse(t, ".a .b#c, .d<.e { x:y; }", ".a .b#c,.d<.e{x:y;}")
	assertParse(t, ".a[b~=c]d { x:y; }", ".a[b~=c]d{x:y;}")
	assertParse(t, "{x:y;}", "{x:y;}")
	assertParse(t, "a{}", "a{}")
	assertParse(t, "a,{x:y;}", "a{x:y;}")
	assertParse(t, "a,.b/*comment*/ {x:y;}", "a,.b{x:y;}")
	assertParse(t, "a,.b/*comment*/.c {x:y;}", "a,.b.c{x:y;}")
	assertParse(t, "a{x:; z:q;}", "a{x:;z:q;}")
	assertParse(t, "@import { @media f; x:y; }", "@import{@media f;x:y;}")
	assertParse(t, "a:not([controls]){x:y;}", "a:not([controls]){x:y;}")
	assertParse(t, "color:#c0c0c0", "color:#c0c0c0;")
	assertParse(t, "background:URL(x.png);", "background:URL(x.png);")
	assertParse(t, "@document regexp('https:.*') { p { color: red; } }", "@document regexp('https:.*'){p{color:red;}}")
	assertParse(t, "@media all and ( max-width:400px ) { }", "@media all and (max-width:400px);")
	assertParse(t, "@media (max-width:400px) { }", "@media (max-width:400px);")
	assertParse(t, "@media (max-width:400px)", "@media (max-width:400px);")
	assertParse(t, "a[x={]{x:y;}", "a[x={]{x:y;}")
	assertParse(t, "a[x=,]{x:y;}", "a[x=,]{x:y;}")
	assertParse(t, "a[x=+]{x:y;}", "a[x=+]{x:y;}")
	assertParse(t, ".cla .ss > #id { x:y; }", ".cla .ss>#id{x:y;}")
	assertParse(t, ".cla /*a*/ /*b*/ .ss{}", ".cla .ss{}")
	assertParse(t, "filter: progid : DXImageTransform.Microsoft.BasicImage(rotation=1);", "filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=1);")
	assertParse(t, "a{x:f(a(),b);}", "a{x:f(a(),b);}")

	// issues
	assertParse(t, "@media print {.class{width:5px;}}", "@media print{.class{width:5px;}}") // #6
	assertParse(t, ".class{width:calc((50% + 2em)/2 + 14px);}}", ".class{width:calc((50% + 2em)/2 + 14px);}}") // #7
}
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import (
	"encoding/json"
	"net/url"
)

type Url url.URL

func (u *Url) UnmarshalJSON(data []byte) error {
	var tu string
	if err := json.Unmarshal(data, &tu); err != nil {
		return err
	}

	pu, err := url.Parse(tu)
	*u = Url(*pu)
	return err
}

func (u Url) MarshalJSON() ([]byte, error) {
	return []byte(`"` + u.String() + `"`), nil
}

func (u Url) String() string {
	tu := url.URL(u)
	return (&tu).String()
}
<commit_msg>config/types: fix url parsing<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import (
	"encoding/json"
	"net/url"
)

type Url url.URL

func (u *Url) UnmarshalJSON(data []byte) error {
	var tu string
	if err := json.Unmarshal(data, &tu); err != nil {
		return err
	}

	pu, err := url.Parse(tu)
	if err != nil {
		return err
	}

	*u = Url(*pu)
	return nil
}

func (u Url) MarshalJSON() ([]byte, error) {
	return []byte(`"` + u.String() + `"`), nil
}

func (u Url) String() string {
	tu := url.URL(u)
	return (&tu).String()
}
<|endoftext|>


<commit_before>package connector

import (
	"fmt"
	"strings"
	"sync"

	"github.com/bcicen/ctop/connector/collector"
	"github.com/bcicen/ctop/container"
	api "github.com/fsouza/go-dockerclient"
)

type Docker struct {
	client       *api.Client
	containers   map[string]*container.Container
	needsRefresh chan string // container IDs requiring refresh
	lock         sync.RWMutex
}

func NewDocker() Connector {
	// init docker client
	client, err := api.NewClientFromEnv()
	if err != nil {
		panic(err)
	}
	cm := &Docker{
		client:       client,
		containers:   make(map[string]*container.Container),
		needsRefresh: make(chan string, 60),
		lock:         sync.RWMutex{},
	}
	go cm.Loop()
	cm.refreshAll()
	go cm.watchEvents()
	return cm
}

// Docker events watcher
func (cm *Docker) watchEvents() {
	log.Info("docker event listener starting")
	events := make(chan *api.APIEvents)
	cm.client.AddEventListener(events)

	for e := range events {
		if e.Type != "container" {
			continue
		}
		switch e.Action {
		case "start", "die", "pause", "unpause":
			log.Debugf("handling docker event: action=%s id=%s", e.Action, e.ID)
			cm.needsRefresh <- e.ID
		case "destroy":
			log.Debugf("handling docker event: action=%s id=%s", e.Action, e.ID)
			cm.delByID(e.ID)
		}
	}
}

func portsFormat(ports map[api.Port][]api.PortBinding) string {
	var exposed []string
	var published []string

	for k, v := range ports {
		if len(v) == 0 {
			exposed = append(exposed, string(k))
			continue
		}
		for _, binding := range v {
			s := fmt.Sprintf("%s -> %s:%s", k, binding.HostIP, binding.HostPort)
			published = append(published, s)
		}
	}

	return strings.Join(append(exposed, published...), "\n")
}

func (cm *Docker) refresh(c *container.Container) {
	insp := cm.inspect(c.Id)
	// remove container if no longer exists
	if insp == nil {
		cm.delByID(c.Id)
		return
	}
	c.SetMeta("name", shortName(insp.Name))
	c.SetMeta("image", insp.Config.Image)
	c.SetMeta("ports", portsFormat(insp.NetworkSettings.Ports))
	c.SetMeta("created", insp.Created.Format("Mon Jan 2 15:04:05 2006"))
	c.SetState(insp.State.Status)
}

func (cm *Docker) inspect(id string) *api.Container {
	c, err := cm.client.InspectContainer(id)
	if err != nil {
		if _, ok := err.(*api.NoSuchContainer); ok == false {
			log.Errorf(err.Error())
		}
	}
	return c
}

// Mark all container IDs for refresh
func (cm *Docker) refreshAll() {
	opts := api.ListContainersOptions{All: true}
	allContainers, err := cm.client.ListContainers(opts)
	if err != nil {
		panic(err)
	}

	for _, i := range allContainers {
		c := cm.MustGet(i.ID)
		c.SetMeta("name", shortName(i.Names[0]))
		c.SetState(i.State)
		cm.needsRefresh <- c.Id
	}
}

func (cm *Docker) Loop() {
	for id := range cm.needsRefresh {
		c := cm.MustGet(id)
		cm.refresh(c)
	}
}

// Get a single container, creating one anew if not existing
func (cm *Docker) MustGet(id string) *container.Container {
	c, ok := cm.Get(id)
	// append container struct for new containers
	if !ok {
		// create collector
		collector := collector.NewDocker(cm.client, id)
		// create container
		c = container.New(id, collector)
		cm.lock.Lock()
		cm.containers[id] = c
		cm.lock.Unlock()
	}
	return c
}

// Get a single container, by ID
func (cm *Docker) Get(id string) (*container.Container, bool) {
	cm.lock.Lock()
	c, ok := cm.containers[id]
	cm.lock.Unlock()
	return c, ok
}

// Remove containers by ID
func (cm *Docker) delByID(id string) {
	cm.lock.Lock()
	delete(cm.containers, id)
	cm.lock.Unlock()
	log.Infof("removed dead container: %s", id)
}

// Return array of all containers, sorted by field
func (cm *Docker) All() (containers container.Containers) {
	cm.lock.Lock()
	for _, c := range cm.containers {
		containers = append(containers, c)
	}
	containers.Sort()
	containers.Filter()
	cm.lock.Unlock()
	return containers
}

// use primary container name
func shortName(name string) string {
	return strings.Replace(name, "/", "", 1)
}
<commit_msg>reverse host and container port in metadata<commit_after>package connector

import (
	"fmt"
	"strings"
	"sync"

	"github.com/bcicen/ctop/connector/collector"
	"github.com/bcicen/ctop/container"
	api "github.com/fsouza/go-dockerclient"
)

type Docker struct {
	client       *api.Client
	containers   map[string]*container.Container
	needsRefresh chan string // container IDs requiring refresh
	lock         sync.RWMutex
}

func NewDocker() Connector {
	// init docker client
	client, err := api.NewClientFromEnv()
	if err != nil {
		panic(err)
	}
	cm := &Docker{
		client:       client,
		containers:   make(map[string]*container.Container),
		needsRefresh: make(chan string, 60),
		lock:         sync.RWMutex{},
	}
	go cm.Loop()
	cm.refreshAll()
	go cm.watchEvents()
	return cm
}

// Docker events watcher
func (cm *Docker) watchEvents() {
	log.Info("docker event listener starting")
	events := make(chan *api.APIEvents)
	cm.client.AddEventListener(events)

	for e := range events {
		if e.Type != "container" {
			continue
		}
		switch e.Action {
		case "start", "die", "pause", "unpause":
			log.Debugf("handling docker event: action=%s id=%s", e.Action, e.ID)
			cm.needsRefresh <- e.ID
		case "destroy":
			log.Debugf("handling docker event: action=%s id=%s", e.Action, e.ID)
			cm.delByID(e.ID)
		}
	}
}

func portsFormat(ports map[api.Port][]api.PortBinding) string {
	var exposed []string
	var published []string

	for k, v := range ports {
		if len(v) == 0 {
			exposed = append(exposed, string(k))
			continue
		}
		for _, binding := range v {
			s := fmt.Sprintf("%s:%s -> %s", binding.HostIP, binding.HostPort, k)
			published = append(published, s)
		}
	}

	return strings.Join(append(exposed, published...), "\n")
}

func (cm *Docker) refresh(c *container.Container) {
	insp := cm.inspect(c.Id)
	// remove container if no longer exists
	if insp == nil {
		cm.delByID(c.Id)
		return
	}
	c.SetMeta("name", shortName(insp.Name))
	c.SetMeta("image", insp.Config.Image)
	c.SetMeta("ports", portsFormat(insp.NetworkSettings.Ports))
	c.SetMeta("created", insp.Created.Format("Mon Jan 2 15:04:05 2006"))
	c.SetState(insp.State.Status)
}

func (cm *Docker) inspect(id string) *api.Container {
	c, err := cm.client.InspectContainer(id)
	if err != nil {
		if _, ok := err.(*api.NoSuchContainer); ok == false {
			log.Errorf(err.Error())
		}
	}
	return c
}

// Mark all container IDs for refresh
func (cm *Docker) refreshAll() {
	opts := api.ListContainersOptions{All: true}
	allContainers, err := cm.client.ListContainers(opts)
	if err != nil {
		panic(err)
	}

	for _, i := range allContainers {
		c := cm.MustGet(i.ID)
		c.SetMeta("name", shortName(i.Names[0]))
		c.SetState(i.State)
		cm.needsRefresh <- c.Id
	}
}

func (cm *Docker) Loop() {
	for id := range cm.needsRefresh {
		c := cm.MustGet(id)
		cm.refresh(c)
	}
}

// Get a single container, creating one anew if not existing
func (cm *Docker) MustGet(id string) *container.Container {
	c, ok := cm.Get(id)
	// append container struct for new containers
	if !ok {
		// create collector
		collector := collector.NewDocker(cm.client, id)
		// create container
		c = container.New(id, collector)
		cm.lock.Lock()
		cm.containers[id] = c
		cm.lock.Unlock()
	}
	return c
}

// Get a single container, by ID
func (cm *Docker) Get(id string) (*container.Container, bool) {
	cm.lock.Lock()
	c, ok := cm.containers[id]
	cm.lock.Unlock()
	return c, ok
}

// Remove containers by ID
func (cm *Docker) delByID(id string) {
	cm.lock.Lock()
	delete(cm.containers, id)
	cm.lock.Unlock()
	log.Infof("removed dead container: %s", id)
}

// Return array of all containers, sorted by field
func (cm *Docker) All() (containers container.Containers) {
	cm.lock.Lock()
	for _, c := range cm.containers {
		containers = append(containers, c)
	}
	containers.Sort()
	containers.Filter()
	cm.lock.Unlock()
	return containers
}

// use primary container name
func shortName(name string) string {
	return strings.Replace(name, "/", "", 1)
}
<|endoftext|>


<commit_before>// Package consul implements the KVDB interface based on consul.
// Code from docker/libkv was leveraged to build parts of this module.
package consul

import (
	"encoding/json"
	"net/http"
	"strings"
	"time"

	api "github.com/hashicorp/consul/api"

	"github.com/portworx/kvdb"
)

const (
	// Name is the name of this kvdb implementation.
	Name    = "consul-kv"
	defHost = "127.0.0.1:8500"
)

func init() {
	if err := kvdb.Register(Name, New); err != nil {
		panic(err.Error())
	}
}

type consulKV struct {
	client *api.Client
	config *api.Config
	domain string
}

type consulLock struct {
	lock    *api.Lock
	renewCh chan struct{}
}

// New constructs a new kvdb.Kvdb.
func New(
	domain string,
	machines []string,
) (kvdb.Kvdb, error) {

	if len(machines) == 0 {
		machines = []string{defHost}
	} else {
		if strings.HasPrefix(machines[0], "http://") {
			machines[0] = strings.TrimPrefix(machines[0], "http://")
		} else if strings.HasPrefix(machines[0], "https://") {
			machines[0] = strings.TrimPrefix(machines[0], "https://")
		}
	}

	if domain != "" && !strings.HasSuffix(domain, "/") {
		domain = domain + "/"
	}

	// Create Consul client
	config := api.DefaultConfig()
	config.HttpClient = http.DefaultClient
	config.Address = machines[0]
	config.Scheme = "http"

	// Creates a new client
	client, err := api.NewClient(config)
	if err != nil {
		return nil, err
	}
	return &consulKV{
		client,
		config,
		domain,
	}, nil
}

func (kv *consulKV) String() string {
	return Name
}

func (kv *consulKV) Get(key string) (*kvdb.KVPair, error) {
	options := &api.QueryOptions{
		AllowStale:        false,
		RequireConsistent: true,
	}

	key = kv.domain + key
	pair, meta, err := kv.client.KV().Get(key, options)
	if err != nil {
		return nil, err
	}

	if pair == nil {
		return nil, kvdb.ErrNotFound
	}

	return kv.pairToKv("get", pair, meta), nil
}

func (kv *consulKV) GetVal(key string, val interface{}) (*kvdb.KVPair, error) {
	kvp, err := kv.Get(key)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(kvp.Value, val)
	return kvp, err
}

func (kv *consulKV) Put(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}

	pair := &api.KVPair{
		Key:   key,
		Value: b,
	}

	_, err = kv.client.KV().Put(pair, nil)
	if err != nil {
		return nil, err
	}

	return kv.pairToKv("put", pair, nil), nil
}

func (kv *consulKV) Create(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	_, err := kv.Get(key)
	if err == nil {
		return nil, kvdb.ErrExist
	}

	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}

	pair := &api.KVPair{
		Key:   key,
		Value: b,
	}

	_, err = kv.client.KV().Put(pair, nil)
	if err != nil {
		return nil, err
	}

	return kv.pairToKv("create", pair, nil), nil
}

func (kv *consulKV) Update(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	_, err := kv.Get(key)
	if err != nil {
		return nil, err
	}

	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}

	pair := &api.KVPair{
		Key:   key,
		Value: b,
	}

	_, err = kv.client.KV().Put(pair, nil)
	if err != nil {
		return nil, err
	}

	return kv.pairToKv("update", pair, nil), nil
}

func (kv *consulKV) Enumerate(prefix string) (kvdb.KVPairs, error) {
	prefix = kv.domain + prefix

	pairs, _, err := kv.client.KV().List(prefix, nil)
	if err != nil {
		return nil, err
	}

	return kv.pairToKvs("enumerate", pairs, nil), nil
}

func (kv *consulKV) Delete(key string) (*kvdb.KVPair, error) {
	pair, err := kv.Get(key)
	if err != nil {
		return nil, err
	}

	key = kv.domain + key

	_, err = kv.client.KV().Delete(key, nil)
	if err != nil {
		return nil, err
	}

	return pair, nil
}

func (kv *consulKV) DeleteTree(key string) error {
	key = kv.domain + key

	_, err := kv.client.KV().DeleteTree(key, nil)
	if err != nil {
		return err
	}

	return nil
}

func (kv *consulKV) Keys(prefix, key string) ([]string, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *consulKV) CompareAndSet(kvp *kvdb.KVPair, flags kvdb.KVFlags, prevValue []byte) (*kvdb.KVPair, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *consulKV) CompareAndDelete(kvp *kvdb.KVPair, flags kvdb.KVFlags) (*kvdb.KVPair, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *consulKV) WatchKey(key string, waitIndex uint64, opaque interface{}, cb kvdb.WatchCB) error {
	return kvdb.ErrNotSupported
}

func (kv *consulKV) WatchTree(prefix string, waitIndex uint64, opaque interface{}, cb kvdb.WatchCB) error {
	return kvdb.ErrNotSupported
}

func (kv *consulKV) Lock(key string, ttl uint64) (*kvdb.KVPair, error) {
	l, err := kv.getLock(key, ttl)
	if err != nil {
		return nil, err
	}

	_, err = l.lock.Lock(nil)
	if err != nil {
		return nil, err
	}

	pair := &kvdb.KVPair{
		Key:  key,
		Lock: l,
	}

	return pair, nil
}

func (kv *consulKV) Unlock(kvp *kvdb.KVPair) error {
	l := kvp.Lock.(*consulLock)

	return l.lock.Unlock()
}

func (kv *consulKV) TxNew() (kvdb.Tx, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *consulKV) createKv(pair *api.KVPair) *kvdb.KVPair {
	kvp := &kvdb.KVPair{
		Key:   pair.Key,
		Value: []byte(pair.Value),
	}

	kvp.Key = strings.TrimPrefix(kvp.Key, kv.domain)
	return kvp
}

func (kv *consulKV) pairToKv(action string, pair *api.KVPair, meta *api.QueryMeta) *kvdb.KVPair {
	kvp := kv.createKv(pair)
	switch action {
	case "create":
		kvp.Action = kvdb.KVCreate
	case "set", "update", "put":
		kvp.Action = kvdb.KVSet
	case "delete":
		kvp.Action = kvdb.KVDelete
	case "get":
		kvp.Action = kvdb.KVGet
	default:
		kvp.Action = kvdb.KVUknown
	}

	if meta != nil {
		kvp.KVDBIndex = meta.LastIndex
	}

	return kvp
}

func (kv *consulKV) pairToKvs(action string, pair []*api.KVPair, meta *api.QueryMeta) kvdb.KVPairs {
	kvs := make([]*kvdb.KVPair, len(pair))
	for i := range pair {
		kvs[i] = kv.pairToKv(action, pair[i], meta)
		if meta != nil {
			kvs[i].KVDBIndex = meta.LastIndex
		}
	}
	return kvs
}

func (kv *consulKV) toBytes(val interface{}) ([]byte, error) {
	var (
		b   []byte
		err error
	)

	switch val.(type) {
	case string:
		b = []byte(val.(string))
	case []byte:
		b = val.([]byte)
	default:
		b, err = json.Marshal(val)
		if err != nil {
			return nil, err
		}
	}

	return b, nil
}

func (kv *consulKV) getLock(key string, ttl uint64) (*consulLock, error) {
	key = kv.domain + key

	lockOpts := &api.LockOptions{
		Key: key,
	}

	lock := &consulLock{}

	if ttl != 0 {
		TTL := time.Duration(0)
		entry := &api.SessionEntry{
			Behavior:  api.SessionBehaviorRelease, // Release the lock when the session expires
			TTL:       (TTL / 2).String(),         // Consul multiplies the TTL by 2x
			LockDelay: 1 * time.Millisecond,       // Virtually disable lock delay
		}

		// Create the key session
		session, _, err := kv.client.Session().Create(entry, nil)
		if err != nil {
			return nil, err
		}

		// Place the session on lock
		lockOpts.Session = session

		// Renew the session ttl lock periodically
		go func() {
			// TODO: do something with the error
			_ = kv.client.Session().RenewPeriodic(entry.TTL, session, nil, nil)
		}()
	}

	l, err := kv.client.LockOpts(lockOpts)
	if err != nil {
		return nil, err
	}

	lock.lock = l
	return lock, nil
}
<commit_msg>general cleanup for consul<commit_after>// Package consul implements the KVDB interface based on consul.
// Code from docker/libkv was leveraged to build parts of this module.
package consul

import (
	"encoding/json"
	"net/http"
	"path"
	"strings"
	"time"

	"github.com/hashicorp/consul/api"
	"github.com/portworx/kvdb"
)

const (
	// Name is the name of this kvdb implementation.
	Name = "consul-kv"
)

var (
	defaultMachines = []string{"127.0.0.1:8500"}
)

func init() {
	if err := kvdb.Register(Name, New); err != nil {
		panic(err.Error())
	}
}

type consulKV struct {
	client *api.Client
	config *api.Config
	domain string
}

type consulLock struct {
	lock    *api.Lock
	renewCh chan struct{}
}

// New constructs a new kvdb.Kvdb.
func New(
	domain string,
	machines []string,
) (kvdb.Kvdb, error) {
	if len(machines) == 0 {
		machines = defaultMachines
	} else {
		if strings.HasPrefix(machines[0], "http://") {
			machines[0] = strings.TrimPrefix(machines[0], "http://")
		} else if strings.HasPrefix(machines[0], "https://") {
			machines[0] = strings.TrimPrefix(machines[0], "https://")
		}
	}
	config := api.DefaultConfig()
	config.HttpClient = http.DefaultClient
	config.Address = machines[0]
	config.Scheme = "http"

	client, err := api.NewClient(config)
	if err != nil {
		return nil, err
	}
	return &consulKV{
		client,
		config,
		domain,
	}, nil
}

func (kv *consulKV) String() string {
	return Name
}

func (kv *consulKV) Get(key string) (*kvdb.KVPair, error) {
	options := &api.QueryOptions{
		AllowStale:        false,
		RequireConsistent: true,
	}
	key = path.Join(kv.domain, key)
	pair, meta, err := kv.client.KV().Get(key, options)
	if err != nil {
		return nil, err
	}
	if pair == nil {
		return nil, kvdb.ErrNotFound
	}
	return kv.pairToKv("get", pair, meta), nil
}

func (kv *consulKV) GetVal(key string, val interface{}) (*kvdb.KVPair, error) {
	kvp, err := kv.Get(key)
	if err != nil {
		return nil, err
	}
	return kvp, json.Unmarshal(kvp.Value, val)
}

func (kv *consulKV) Put(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	key = path.Join(kv.domain, key)
	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	pair := &api.KVPair{
		Key:   key,
		Value: b,
	}
	if _, err := kv.client.KV().Put(pair, nil); err != nil {
		return nil, err
	}
	return kv.pairToKv("put", pair, nil), nil
}

func (kv *consulKV) Create(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	if _, err := kv.Get(key); err == nil {
		return nil, kvdb.ErrExist
	}
	key = path.Join(kv.domain, key)
	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	pair := &api.KVPair{
		Key:   key,
		Value: b,
	}
	if _, err := kv.client.KV().Put(pair, nil); err != nil {
		return nil, err
	}
	return kv.pairToKv("create", pair, nil), nil
}

func (kv *consulKV) Update(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	if _, err := kv.Get(key); err != nil {
		return nil, err
	}
	key = path.Join(kv.domain, key)
	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	pair := &api.KVPair{
		Key:   key,
		Value: b,
	}
	if _, err := kv.client.KV().Put(pair, nil); err != nil {
		return nil, err
	}
	return kv.pairToKv("update", pair, nil), nil
}

func (kv *consulKV) Enumerate(prefix string) (kvdb.KVPairs, error) {
	prefix = path.Join(kv.domain, prefix)
	pairs, _, err := kv.client.KV().List(prefix, nil)
	if err != nil {
		return nil, err
	}
	return kv.pairToKvs("enumerate", pairs, nil), nil
}

func (kv *consulKV) Delete(key string) (*kvdb.KVPair, error) {
	pair, err := kv.Get(key)
	if err != nil {
		return nil, err
	}
	key = path.Join(kv.domain, key)
	if _, err := kv.client.KV().Delete(key, nil); err != nil {
		return nil, err
	}
	return pair, nil
}

func (kv *consulKV) DeleteTree(key string) error {
	key = kv.domain + key
	if _, err := kv.client.KV().DeleteTree(key, nil); err != nil {
		return err
	}
	return nil
}

func (kv *consulKV) Keys(prefix, key string) ([]string, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *consulKV) CompareAndSet(kvp *kvdb.KVPair, flags kvdb.KVFlags, prevValue []byte) (*kvdb.KVPair, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *consulKV) CompareAndDelete(kvp *kvdb.KVPair, flags kvdb.KVFlags) (*kvdb.KVPair, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *consulKV) WatchKey(key string, waitIndex uint64, opaque interface{}, cb kvdb.WatchCB) error {
	return kvdb.ErrNotSupported
}

func (kv *consulKV) WatchTree(prefix string, waitIndex uint64, opaque interface{}, cb kvdb.WatchCB) error {
	return kvdb.ErrNotSupported
}

func (kv *consulKV) Lock(key string, ttl uint64) (*kvdb.KVPair, error) {
	l, err := kv.getLock(key, ttl)
	if err != nil {
		return nil, err
	}
	if _, err := l.lock.Lock(nil); err != nil {
		return nil, err
	}
	return &kvdb.KVPair{
		Key:  key,
		Lock: l,
	}, nil
}

func (kv *consulKV) Unlock(kvp *kvdb.KVPair) error {
	return kvp.Lock.(*consulLock).lock.Unlock()
}

func (kv *consulKV) TxNew() (kvdb.Tx, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *consulKV) createKv(pair *api.KVPair) *kvdb.KVPair {
	return &kvdb.KVPair{
		Key:   strings.TrimPrefix(pair.Key, kv.domain),
		Value: []byte(pair.Value),
	}
}

func (kv *consulKV) pairToKv(action string, pair *api.KVPair, meta *api.QueryMeta) *kvdb.KVPair {
	kvp := kv.createKv(pair)
	switch action {
	case "create":
		kvp.Action = kvdb.KVCreate
	case "set", "update", "put":
		kvp.Action = kvdb.KVSet
	case "delete":
		kvp.Action = kvdb.KVDelete
	case "get":
		kvp.Action = kvdb.KVGet
	default:
		kvp.Action = kvdb.KVUknown
	}
	if meta != nil {
		kvp.KVDBIndex = meta.LastIndex
	}
	return kvp
}

func (kv *consulKV) pairToKvs(action string, pair []*api.KVPair, meta *api.QueryMeta) kvdb.KVPairs {
	kvs := make([]*kvdb.KVPair, len(pair))
	for i := range pair {
		kvs[i] = kv.pairToKv(action, pair[i], meta)
		if meta != nil {
			kvs[i].KVDBIndex = meta.LastIndex
		}
	}
	return kvs
}

func (kv *consulKV) toBytes(val interface{}) ([]byte, error) {
	var b []byte
	var err error
	switch val.(type) {
	case string:
		b = []byte(val.(string))
	case []byte:
		b = val.([]byte)
	default:
		b, err = json.Marshal(val)
		if err != nil {
			return nil, err
		}
	}
	return b, nil
}

func (kv *consulKV) getLock(key string, ttl uint64) (*consulLock, error) {
	key = path.Join(kv.domain, key)
	lockOpts := &api.LockOptions{
		Key: key,
	}
	lock := &consulLock{}
	if ttl != 0 {
		TTL := time.Duration(0)
		entry := &api.SessionEntry{
			Behavior:  api.SessionBehaviorRelease, // Release the lock when the session expires
			TTL:       (TTL / 2).String(),         // Consul multiplies the TTL by 2x
			LockDelay: 1 * time.Millisecond,       // Virtually disable lock delay
		}

		// Create the key session
		session, _, err := kv.client.Session().Create(entry, nil)
		if err != nil {
			return nil, err
		}

		// Place the session on lock
		lockOpts.Session = session

		// Renew the session ttl lock periodically
		go func() {
			// TODO: do something with the error
			_ = kv.client.Session().RenewPeriodic(entry.TTL, session, nil, nil)
		}()
	}

	l, err := kv.client.LockOpts(lockOpts)
	if err != nil {
		return nil, err
	}
	lock.lock = l
	return lock, nil
}
<|endoftext|>