<commit_before>package k8sutil

import (
	"strings"
	"testing"

	"github.com/zalando/postgres-operator/pkg/util/constants"

	v1 "k8s.io/api/core/v1"
)

func newsService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.Service {
	svc := &v1.Service{
		Spec: v1.ServiceSpec{
			Type:                     svcT,
			LoadBalancerSourceRanges: lbSr,
		},
	}
	svc.Annotations = ann
	return svc
}

func TestServiceAnnotations(t *testing.T) {
	tests := []struct {
		about   string
		current *v1.Service
		new     *v1.Service
		reason  string
		match   bool
	}{
		{
			about: "two equal services",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeClusterIP,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeClusterIP,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match: true,
		},
		{
			about: "services differ on service type",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeClusterIP,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's type "LoadBalancer" doesn't match the current one "ClusterIP"`,
		},
		{
			about: "services differ on lb source ranges",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"185.249.56.0/22"}),
			match:  false,
			reason: `new service's LoadBalancerSourceRange doesn't match the current one`,
		},
		{
			about: "new service doesn't have lb source ranges",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{}),
			match:  false,
			reason: `new service's LoadBalancerSourceRange doesn't match the current one`,
		},
		{
			about: "services differ on DNS annotation",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "new_clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's annotations doesn't match the current one: 'external-dns.alpha.kubernetes.io/hostname' changed from 'clstr.acid.zalan.do' to 'new_clstr.acid.zalan.do'.`,
		},
		{
			about: "services differ on AWS ELB annotation",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: "1800",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's annotations doesn't match the current one: 'service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout' changed from '3600' to '1800'.`,
		},
		{
			about: "service changes existing annotation",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "bar",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "baz",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's annotations doesn't match the current one: 'foo' changed from 'bar' to 'baz'.`,
		},
		{
			about: "service changes multiple existing annotations",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "bar",
					"bar":                              "foo",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "baz",
					"bar":                              "fooz",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match: false,
			// Test just the prefix to avoid flakiness and map sorting
			reason: `new service's annotations doesn't match the current one:`,
		},
		{
			about: "service adds a new custom annotation",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "bar",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's annotations doesn't match the current one: Added 'foo' with value 'bar'.`,
		},
		{
			about: "service removes a custom annotation",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "bar",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's annotations doesn't match the current one: Removed 'foo'.`,
		},
		{
			about: "service removes a custom annotation and adds a new one",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "bar",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"bar":                              "foo",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's annotations doesn't match the current one: Removed 'foo'. Added 'bar' with value 'foo'.`,
		},
		{
			about: "service removes a custom annotation, adds a new one and change another",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "bar",
					"zalan":                            "do",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"bar":                              "foo",
					"zalan":                            "do.com",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's annotations doesn't match the current one: Removed 'foo'. Added 'bar' with value 'foo'. 'zalan' changed from 'do' to 'do.com'`,
		},
		{
			about: "service add annotations",
			current: newsService(
				map[string]string{},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match: false,
			// Test just the prefix to avoid flakiness and map sorting
			reason: `new service's annotations doesn't match the current one: Added `,
		},
	}
	for _, tt := range tests {
		t.Run(tt.about, func(t *testing.T) {
			match, reason := SameService(tt.current, tt.new)
			if match && !tt.match {
				t.Errorf("expected services to do not match: '%q' and '%q'", tt.current, tt.new)
				return
			}
			if !match && tt.match {
				t.Errorf("expected services to be the same: '%q' and '%q'", tt.current, tt.new)
				return
			}
			if !match && !tt.match {
				if !strings.HasPrefix(reason, tt.reason) {
					t.Errorf("expected reason '%s', found '%s'", tt.reason, reason)
					return
				}
			}
		})
	}
}
<commit_msg>Fix test flakiness on TestSameService (#833)<commit_after>package k8sutil

import (
	"strings"
	"testing"

	"github.com/zalando/postgres-operator/pkg/util/constants"

	v1 "k8s.io/api/core/v1"
)

func newsService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1.Service {
	svc := &v1.Service{
		Spec: v1.ServiceSpec{
			Type:                     svcT,
			LoadBalancerSourceRanges: lbSr,
		},
	}
	svc.Annotations = ann
	return svc
}

func TestSameService(t *testing.T) {
	tests := []struct {
		about   string
		current *v1.Service
		new     *v1.Service
		reason  string
		match   bool
	}{
		{
			about: "two equal services",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeClusterIP,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeClusterIP,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match: true,
		},
		{
			about: "services differ on service type",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeClusterIP,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's type "LoadBalancer" doesn't match the current one "ClusterIP"`,
		},
		{
			about: "services differ on lb source ranges",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"185.249.56.0/22"}),
			match:  false,
			reason: `new service's LoadBalancerSourceRange doesn't match the current one`,
		},
		{
			about: "new service doesn't have lb source ranges",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{}),
			match:  false,
			reason: `new service's LoadBalancerSourceRange doesn't match the current one`,
		},
		{
			about: "services differ on DNS annotation",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "new_clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's annotations doesn't match the current one: 'external-dns.alpha.kubernetes.io/hostname' changed from 'clstr.acid.zalan.do' to 'new_clstr.acid.zalan.do'.`,
		},
		{
			about: "services differ on AWS ELB annotation",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: "1800",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's annotations doesn't match the current one: 'service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout' changed from '3600' to '1800'.`,
		},
		{
			about: "service changes existing annotation",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "bar",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "baz",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's annotations doesn't match the current one: 'foo' changed from 'bar' to 'baz'.`,
		},
		{
			about: "service changes multiple existing annotations",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "bar",
					"bar":                              "foo",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "baz",
					"bar":                              "fooz",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match: false,
			// Test just the prefix to avoid flakiness and map sorting
			reason: `new service's annotations doesn't match the current one:`,
		},
		{
			about: "service adds a new custom annotation",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "bar",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's annotations doesn't match the current one: Added 'foo' with value 'bar'.`,
		},
		{
			about: "service removes a custom annotation",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "bar",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's annotations doesn't match the current one: Removed 'foo'.`,
		},
		{
			about: "service removes a custom annotation and adds a new one",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "bar",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"bar":                              "foo",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match:  false,
			reason: `new service's annotations doesn't match the current one: Removed 'foo'. Added 'bar' with value 'foo'.`,
		},
		{
			about: "service removes a custom annotation, adds a new one and change another",
			current: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"foo":                              "bar",
					"zalan":                            "do",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
					"bar":                              "foo",
					"zalan":                            "do.com",
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match: false,
			// Test just the prefix to avoid flakiness and map sorting
			reason: `new service's annotations doesn't match the current one: Removed 'foo'.`,
		},
		{
			about: "service add annotations",
			current: newsService(
				map[string]string{},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			new: newsService(
				map[string]string{
					constants.ZalandoDNSNameAnnotation: "clstr.acid.zalan.do",
					constants.ElbTimeoutAnnotationName: constants.ElbTimeoutAnnotationValue,
				},
				v1.ServiceTypeLoadBalancer,
				[]string{"128.141.0.0/16", "137.138.0.0/16"}),
			match: false,
			// Test just the prefix to avoid flakiness and map sorting
			reason: `new service's annotations doesn't match the current one: Added `,
		},
	}
	for _, tt := range tests {
		t.Run(tt.about, func(t *testing.T) {
			match, reason := SameService(tt.current, tt.new)
			if match && !tt.match {
				t.Errorf("expected services to do not match: '%q' and '%q'", tt.current, tt.new)
				return
			}
			if !match && tt.match {
				t.Errorf("expected services to be the same: '%q' and '%q'", tt.current, tt.new)
				return
			}
			if !match && !tt.match {
				if !strings.HasPrefix(reason, tt.reason) {
					t.Errorf("expected reason prefix '%s', found '%s'", tt.reason, reason)
					return
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cinder

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/golang/glog"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	clientset "k8s.io/client-go/kubernetes"
	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
	"k8s.io/kubernetes/pkg/volume"
	"k8s.io/utils/exec"
)

type CinderDiskUtil struct{}

// Attaches a disk specified by a volume.CinderPersistenDisk to the current kubelet.
// Mounts the disk to its global path.
func (util *CinderDiskUtil) AttachDisk(b *cinderVolumeMounter, globalPDPath string) error {
	options := []string{}
	if b.readOnly {
		options = append(options, "ro")
	}
	cloud, err := b.plugin.getCloudProvider()
	if err != nil {
		return err
	}
	instanceid, err := cloud.InstanceID()
	if err != nil {
		return err
	}
	diskid, err := cloud.AttachDisk(instanceid, b.pdName)
	if err != nil {
		return err
	}

	var devicePath string
	numTries := 0
	for {
		devicePath = cloud.GetDevicePath(diskid)
		probeAttachedVolume()

		_, err := os.Stat(devicePath)
		if err == nil {
			break
		}
		if err != nil && !os.IsNotExist(err) {
			return err
		}
		numTries++
		if numTries == 10 {
			return errors.New("Could not attach disk: Timeout after 60s")
		}
		time.Sleep(time.Second * 6)
	}
	notmnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
	if err != nil {
		if os.IsNotExist(err) {
			if err := os.MkdirAll(globalPDPath, 0750); err != nil {
				return err
			}
			notmnt = true
		} else {
			return err
		}
	}
	if notmnt {
		err = b.blockDeviceMounter.FormatAndMount(devicePath, globalPDPath, b.fsType, options)
		if err != nil {
			os.Remove(globalPDPath)
			return err
		}
		glog.V(2).Infof("Safe mount successful: %q\n", devicePath)
	}
	return nil
}

// Unmounts the device and detaches the disk from the kubelet's host machine.
func (util *CinderDiskUtil) DetachDisk(cd *cinderVolumeUnmounter) error {
	globalPDPath := makeGlobalPDName(cd.plugin.host, cd.pdName)
	if err := cd.mounter.Unmount(globalPDPath); err != nil {
		return err
	}
	if err := os.Remove(globalPDPath); err != nil {
		return err
	}
	glog.V(2).Infof("Successfully unmounted main device: %s\n", globalPDPath)

	cloud, err := cd.plugin.getCloudProvider()
	if err != nil {
		return err
	}
	instanceid, err := cloud.InstanceID()
	if err != nil {
		return err
	}
	if err = cloud.DetachDisk(instanceid, cd.pdName); err != nil {
		return err
	}
	glog.V(2).Infof("Successfully detached cinder volume %s", cd.pdName)
	return nil
}

func (util *CinderDiskUtil) DeleteVolume(cd *cinderVolumeDeleter) error {
	cloud, err := cd.plugin.getCloudProvider()
	if err != nil {
		return err
	}

	if err = cloud.DeleteVolume(cd.pdName); err != nil {
		// OpenStack cloud provider returns volume.tryAgainError when necessary,
		// no handling needed here.
		glog.V(2).Infof("Error deleting cinder volume %s: %v", cd.pdName, err)
		return err
	}
	glog.V(2).Infof("Successfully deleted cinder volume %s", cd.pdName)
	return nil
}

func getZonesFromNodes(kubeClient clientset.Interface) (sets.String, error) {
	// TODO: caching, currently it is overkill because it calls this function
	// only when it creates dynamic PV
	zones := make(sets.String)
	nodes, err := kubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		glog.V(2).Infof("Error listing nodes")
		return zones, err
	}
	for _, node := range nodes.Items {
		if zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain]; ok {
			zones.Insert(zone)
		}
	}
	glog.V(4).Infof("zones found: %v", zones)
	return zones, nil
}

func (util *CinderDiskUtil) CreateVolume(c *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, volumeLabels map[string]string, fstype string, err error) {
	cloud, err := c.plugin.getCloudProvider()
	if err != nil {
		return "", 0, nil, "", err
	}

	capacity := c.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	volSizeBytes := capacity.Value()
	// Cinder works with gigabytes, convert to GiB with rounding up
	volSizeGB := int(volume.RoundUpSize(volSizeBytes, 1024*1024*1024))
	name := volume.GenerateVolumeName(c.options.ClusterName, c.options.PVName, 255) // Cinder volume name can have up to 255 characters
	vtype := ""
	availability := ""
	// Apply ProvisionerParameters (case-insensitive). We leave validation of
	// the values to the cloud provider.
	for k, v := range c.options.Parameters {
		switch strings.ToLower(k) {
		case "type":
			vtype = v
		case "availability":
			availability = v
		case volume.VolumeParameterFSType:
			fstype = v
		default:
			return "", 0, nil, "", fmt.Errorf("invalid option %q for volume plugin %s", k, c.plugin.GetPluginName())
		}
	}
	// TODO: implement PVC.Selector parsing
	if c.options.PVC.Spec.Selector != nil {
		return "", 0, nil, "", fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on Cinder")
	}

	if availability == "" {
		// No zone specified, choose one randomly in the same region
		zones, err := getZonesFromNodes(c.plugin.host.GetKubeClient())
		if err != nil {
			glog.V(2).Infof("error getting zone information: %v", err)
			return "", 0, nil, "", err
		}
		// if we did not get any zones, lets leave it blank and gophercloud will
		// use zone "nova" as default
		if len(zones) > 0 {
			availability = volume.ChooseZoneForVolume(zones, c.options.PVC.Name)
		}
	}

	volumeID, volumeAZ, IgnoreVolumeAZ, errr := cloud.CreateVolume(name, volSizeGB, vtype, availability, c.options.CloudTags)
	if errr != nil {
		glog.V(2).Infof("Error creating cinder volume: %v", errr)
		return "", 0, nil, "", errr
	}
	glog.V(2).Infof("Successfully created cinder volume %s", volumeID)

	// these are needed that pod is spawning to same AZ
	volumeLabels = make(map[string]string)
	if IgnoreVolumeAZ == false {
		volumeLabels[kubeletapis.LabelZoneFailureDomain] = volumeAZ
	}
	return volumeID, volSizeGB, volumeLabels, fstype, nil
}

func probeAttachedVolume() error {
	// rescan scsi bus
	scsiHostRescan()

	executor := exec.New()

        // udevadm settle waits for udevd to process the device creation
        // events for all hardware devices, thus ensuring that any device
        // nodes have been created successfully before proceeding.
	argsSettle := []string{"settle", "--timeout=1"}
	cmdSettle := executor.Command("udevadm", argsSettle...)
	_, errSettle := cmdSettle.CombinedOutput()
	if errSettle != nil {
		glog.Errorf("error running udevadm settle %v\n", errSettle)
		return errSettle
	}

	args := []string{"trigger"}
	cmd := executor.Command("udevadm", args...)
	_, err := cmd.CombinedOutput()
	if err != nil {
		glog.Errorf("error running udevadm trigger %v\n", err)
		return err
	}
	glog.V(4).Infof("Successfully probed all attachments")
	return nil
}

func scsiHostRescan() {
	scsi_path := "/sys/class/scsi_host/"
	if dirs, err := ioutil.ReadDir(scsi_path); err == nil {
		for _, f := range dirs {
			name := scsi_path + f.Name() + "/scan"
			data := []byte("- - -")
			ioutil.WriteFile(name, data, 0666)
		}
	}
}
<commit_msg>modify some wording<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cinder

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/golang/glog"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	clientset "k8s.io/client-go/kubernetes"
	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
	"k8s.io/kubernetes/pkg/volume"
	"k8s.io/utils/exec"
)

type CinderDiskUtil struct{}

// Attaches a disk specified by a volume.CinderPersistenDisk to the current kubelet.
// Mounts the disk to its global path.
func (util *CinderDiskUtil) AttachDisk(b *cinderVolumeMounter, globalPDPath string) error {
	options := []string{}
	if b.readOnly {
		options = append(options, "ro")
	}
	cloud, err := b.plugin.getCloudProvider()
	if err != nil {
		return err
	}
	instanceid, err := cloud.InstanceID()
	if err != nil {
		return err
	}
	diskid, err := cloud.AttachDisk(instanceid, b.pdName)
	if err != nil {
		return err
	}

	var devicePath string
	numTries := 0
	for {
		devicePath = cloud.GetDevicePath(diskid)
		probeAttachedVolume()

		_, err := os.Stat(devicePath)
		if err == nil {
			break
		}
		if err != nil && !os.IsNotExist(err) {
			return err
		}
		numTries++
		if numTries == 10 {
			return errors.New("Could not attach disk: Timeout after 60s")
		}
		time.Sleep(time.Second * 6)
	}
	notmnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
	if err != nil {
		if os.IsNotExist(err) {
			if err := os.MkdirAll(globalPDPath, 0750); err != nil {
				return err
			}
			notmnt = true
		} else {
			return err
		}
	}
	if notmnt {
		err = b.blockDeviceMounter.FormatAndMount(devicePath, globalPDPath, b.fsType, options)
		if err != nil {
			os.Remove(globalPDPath)
			return err
		}
		glog.V(2).Infof("Safe mount successful: %q\n", devicePath)
	}
	return nil
}

// Unmounts the device and detaches the disk from the kubelet's host machine.
func (util *CinderDiskUtil) DetachDisk(cd *cinderVolumeUnmounter) error {
	globalPDPath := makeGlobalPDName(cd.plugin.host, cd.pdName)
	if err := cd.mounter.Unmount(globalPDPath); err != nil {
		return err
	}
	if err := os.Remove(globalPDPath); err != nil {
		return err
	}
	glog.V(2).Infof("Successfully unmounted main device: %s\n", globalPDPath)

	cloud, err := cd.plugin.getCloudProvider()
	if err != nil {
		return err
	}
	instanceid, err := cloud.InstanceID()
	if err != nil {
		return err
	}
	if err = cloud.DetachDisk(instanceid, cd.pdName); err != nil {
		return err
	}
	glog.V(2).Infof("Successfully detached cinder volume %s", cd.pdName)
	return nil
}

func (util *CinderDiskUtil) DeleteVolume(cd *cinderVolumeDeleter) error {
	cloud, err := cd.plugin.getCloudProvider()
	if err != nil {
		return err
	}

	if err = cloud.DeleteVolume(cd.pdName); err != nil {
		// OpenStack cloud provider returns volume.tryAgainError when necessary,
		// no handling needed here.
		glog.V(2).Infof("Error deleting cinder volume %s: %v", cd.pdName, err)
		return err
	}
	glog.V(2).Infof("Successfully deleted cinder volume %s", cd.pdName)
	return nil
}

func getZonesFromNodes(kubeClient clientset.Interface) (sets.String, error) {
	// TODO: caching, currently it is overkill because it calls this function
	// only when it creates dynamic PV
	zones := make(sets.String)
	nodes, err := kubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		glog.V(2).Infof("Error listing nodes")
		return zones, err
	}
	for _, node := range nodes.Items {
		if zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain]; ok {
			zones.Insert(zone)
		}
	}
	glog.V(4).Infof("zones found: %v", zones)
	return zones, nil
}

func (util *CinderDiskUtil) CreateVolume(c *cinderVolumeProvisioner) (volumeID string, volumeSizeGB int, volumeLabels map[string]string, fstype string, err error) {
	cloud, err := c.plugin.getCloudProvider()
	if err != nil {
		return "", 0, nil, "", err
	}

	capacity := c.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	volSizeBytes := capacity.Value()
	// Cinder works with gigabytes, convert to GiB with rounding up
	volSizeGB := int(volume.RoundUpSize(volSizeBytes, 1024*1024*1024))
	name := volume.GenerateVolumeName(c.options.ClusterName, c.options.PVName, 255) // Cinder volume name can have up to 255 characters
	vtype := ""
	availability := ""
	// Apply ProvisionerParameters (case-insensitive). We leave validation of
	// the values to the cloud provider.
	for k, v := range c.options.Parameters {
		switch strings.ToLower(k) {
		case "type":
			vtype = v
		case "availability":
			availability = v
		case volume.VolumeParameterFSType:
			fstype = v
		default:
			return "", 0, nil, "", fmt.Errorf("invalid option %q for volume plugin %s", k, c.plugin.GetPluginName())
		}
	}
	// TODO: implement PVC.Selector parsing
	if c.options.PVC.Spec.Selector != nil {
		return "", 0, nil, "", fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on Cinder")
	}

	if availability == "" {
		// No zone specified, choose one randomly in the same region
		zones, err := getZonesFromNodes(c.plugin.host.GetKubeClient())
		if err != nil {
			glog.V(2).Infof("error getting zone information: %v", err)
			return "", 0, nil, "", err
		}
		// if we did not get any zones, lets leave it blank and gophercloud will
		// use zone "nova" as default
		if len(zones) > 0 {
			availability = volume.ChooseZoneForVolume(zones, c.options.PVC.Name)
		}
	}

	volumeID, volumeAZ, IgnoreVolumeAZ, errr := cloud.CreateVolume(name, volSizeGB, vtype, availability, c.options.CloudTags)
	if errr != nil {
		glog.V(2).Infof("Error creating cinder volume: %v", errr)
		return "", 0, nil, "", errr
	}
	glog.V(2).Infof("Successfully created cinder volume %s", volumeID)

	// these are needed that pod is spawning to same AZ
	volumeLabels = make(map[string]string)
	if IgnoreVolumeAZ == false {
		volumeLabels[kubeletapis.LabelZoneFailureDomain] = volumeAZ
	}
	return volumeID, volSizeGB, volumeLabels, fstype, nil
}

func probeAttachedVolume() error {
	// rescan scsi bus
	scsiHostRescan()

	executor := exec.New()

	// udevadm settle waits for udevd to process the device creation
	// events for all hardware devices, thus ensuring that any device
	// nodes have been created successfully before proceeding.
	argsSettle := []string{"settle", "--timeout=1"}
	cmdSettle := executor.Command("udevadm", argsSettle...)
	_, errSettle := cmdSettle.CombinedOutput()
	if errSettle != nil {
		glog.Errorf("error running udevadm settle %v\n", errSettle)
		return errSettle
	}

	args := []string{"trigger"}
	cmd := executor.Command("udevadm", args...)
	_, err := cmd.CombinedOutput()
	if err != nil {
		glog.Errorf("error running udevadm trigger %v\n", err)
		return err
	}
	glog.V(4).Infof("Successfully probed all attachments")
	return nil
}

func scsiHostRescan() {
	scsi_path := "/sys/class/scsi_host/"
	if dirs, err := ioutil.ReadDir(scsi_path); err == nil {
		for _, f := range dirs {
			name := scsi_path + f.Name() + "/scan"
			data := []byte("- - -")
			ioutil.WriteFile(name, data, 0666)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package base

import (
	"encoding/json"
	"io/ioutil"
	"net/http"

	"github.com/juju/errors"

	"github.com/juju/juju/apiserver/params"
)

// HTTPCaller exposes direct HTTP request functionality for the API.
// This is significant for upload and download of files, which the
// websockets-based RPC does not support.
type HTTPCaller interface {
	// NewHTTPRequest returns a new API-relative HTTP request.  Callers
	// should finish the request (setting headers, body) before passing
	// the request to SendHTTPRequest.
	NewHTTPRequest(method, path string) (*http.Request, error)
	// SendHTTPRequest returns the HTTP response from the API server.
	// The caller is then responsible for handling the response.
	SendHTTPRequest(req *http.Request) (*http.Response, error)
}

// HandleHTTPFailure returns the failure serialized in the response
// body.  This function should only be called if the status code is not
// http.StatusOkay.
func HandleHTTPFailure(resp *http.Response) error {
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return errors.Annotate(err, "while reading HTTP response")
	}

	var failure params.Error
	if resp.Header.Get("Content-Type") == "application/json" {
		if err := json.Unmarshal(body, &failure); err != nil {
			return errors.Annotate(err, "while unserializing the error")
		}
	} else {
		switch resp.StatusCode {
		case http.StatusNotFound:
			fallthrough
		case http.StatusMethodNotAllowed:
			failure.Code = params.CodeNotImplemented
		default:
			// Leave Code empty.
		}

		failure.Message = string(body)
	}
	return &failure
}
<commit_msg>HandleHTTPFailure -> CheckHTTPResponse.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package base

import (
	"encoding/json"
	"io/ioutil"
	"net/http"

	"github.com/juju/errors"

	"github.com/juju/juju/apiserver/params"
)

// HTTPCaller exposes direct HTTP request functionality for the API.
// This is significant for upload and download of files, which the
// websockets-based RPC does not support.
type HTTPCaller interface {
	// NewHTTPRequest returns a new API-relative HTTP request.  Callers
	// should finish the request (setting headers, body) before passing
	// the request to SendHTTPRequest.
	NewHTTPRequest(method, path string) (*http.Request, error)
	// SendHTTPRequest returns the HTTP response from the API server.
	// The caller is then responsible for handling the response.
	SendHTTPRequest(req *http.Request) (*http.Response, error)
}

// HandleHTTPFailure returns the failure serialized in the response
// body.  If there is no failure (an OK status code), it simply returns
// nil.
func CheckHTTPResponse(resp *http.Response) error {
	if resp.StatusCode == http.StatusOK {
		return nil
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return errors.Annotate(err, "while reading HTTP response")
	}

	var failure params.Error
	if resp.Header.Get("Content-Type") == "application/json" {
		if err := json.Unmarshal(body, &failure); err != nil {
			return errors.Annotate(err, "while unserializing the error")
		}
	} else {
		switch resp.StatusCode {
		case http.StatusNotFound:
			fallthrough
		case http.StatusMethodNotAllowed:
			failure.Code = params.CodeNotImplemented
		default:
			// Leave Code empty.
		}

		failure.Message = string(body)
	}
	return &failure
}
<|endoftext|>


<commit_before>package assets

import (
	"embed"
	"io/fs"
	"strings"

	classifier "github.com/google/licenseclassifier/v2"
)

//go:embed */*/*
var licenseFS embed.FS

// DefaultClassifier returns a classifier loaded with the contents of the
// assets directory.
func DefaultClassifier() (*classifier.Classifier, error) {
	c := classifier.NewClassifier(.8)

	err := fs.WalkDir(licenseFS, ".", func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}

		b, err := licenseFS.ReadFile(path)
		if err != nil {
			return err
		}
		splits := strings.Split(path, "/")
		category, name, variant := splits[0], splits[1], splits[2]
		c.AddContent(category, name, variant, b)
		return nil
	})

	if err != nil {
		return nil, err
	}
	return c, nil

}

// ReadLicenseFile locates and reads the license archive file.  Absolute paths are used unmodified.  Relative paths are expected to be in the licenses directory of the licenseclassifier package.
func ReadLicenseFile(filename string) ([]byte, error) {
	return licenseFS.ReadFile(filename)
}

// ReadLicenseDir reads directory containing the license files.
func ReadLicenseDir() ([]fs.DirEntry, error) {
	return licenseFS.ReadDir(".")
}
<commit_msg>Update embed.go<commit_after>// Copyright 2022 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package assets

import (
	"embed"
	"io/fs"
	"strings"

	classifier "github.com/google/licenseclassifier/v2"
)

//go:embed */*/*
var licenseFS embed.FS

// DefaultClassifier returns a classifier loaded with the contents of the
// assets directory.
func DefaultClassifier() (*classifier.Classifier, error) {
	c := classifier.NewClassifier(.8)

	err := fs.WalkDir(licenseFS, ".", func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}

		b, err := licenseFS.ReadFile(path)
		if err != nil {
			return err
		}
		splits := strings.Split(path, "/")
		category, name, variant := splits[0], splits[1], splits[2]
		c.AddContent(category, name, variant, b)
		return nil
	})

	if err != nil {
		return nil, err
	}
	return c, nil

}

// ReadLicenseFile locates and reads the license archive file.  Absolute paths are used unmodified.  Relative paths are expected to be in the licenses directory of the licenseclassifier package.
func ReadLicenseFile(filename string) ([]byte, error) {
	return licenseFS.ReadFile(filename)
}

// ReadLicenseDir reads directory containing the license files.
func ReadLicenseDir() ([]fs.DirEntry, error) {
	return licenseFS.ReadDir(".")
}
<|endoftext|>


<commit_before>package toystore

import (
	"fmt"
	"log"

	"github.com/rlayte/toystore/data"
)

type Toystore struct {
	// Number of nodes to replicate each data item.
	ReplicationLevel int

	// Number of successful writes required.
	W int

	// Number of successful reads required.
	R int

	// Port number to serve client requests.
	Port int

	// Port number to serve RPC requests between nodes.
	RPCPort int

	// Host address to bind to.
	Host string

	// Concrete Store implementation to persist data.
	Data Store

	// Hash ring for nodes in the cluster.
	Ring *Ring

	// Concrete Members implementation to represent the current cluster state.
	Members Members

	// Store of hinted data meant for other nodes.
	Hints *HintedHandoff

	// Concrete PeerClient implementation to make calls to other nodes.
	client PeerClient
}

// rpcAddress returns a string for the RPC address.
func (t *Toystore) rpcAddress() string {
	return fmt.Sprintf("%s:%d", t.Host, t.RPCPort)
}

// Get finds the key on the correct node in the cluster and returns
// the value and an existence bool.
// If the key is on the current node then it coordinates the operation.
// Otherwise it sends the coordination request to the correct node.
func (t *Toystore) Get(key string) (interface{}, bool) {
	lookup := t.Ring.KeyAddress([]byte(key))
	address, _, _ := lookup()
	var data *data.Data
	var ok bool

	if t.isCoordinator(address) {
		data, ok = t.CoordinateGet(key)
	} else {
		data, ok = t.client.CoordinateGet(string(address), key)
	}

	if ok {
		return data.Value, ok
	} else {
		return nil, ok
	}
}

// Put finds the key on the correct node in the cluster, sets
// the value and returns a status bool.
// If the key is owned by current node then it coordinates the operation.
// Otherwise it sends the coordination request to the correct node.
func (t *Toystore) Put(key string, value interface{}) (ok bool) {
	lookup := t.Ring.KeyAddress([]byte(key))
	address, _, _ := lookup()

	if t.isCoordinator(address) {
		ok = t.CoordinatePut(data.New(key, value))
	} else {
		ok = t.client.CoordinatePut(string(address), data.New(key, value))
	}
	return
}

// GetString returns a string of the value for the specified key/value pair.
func (t *Toystore) GetString(key string) (string, bool) {
	d, ok := t.Get(key)
	return d.(string), ok
}

// Merge updates the data object only if its Timestamp is later than the
// current value.
// If the key doesn't exist it adds it.
// Requires Store implementation to be thread safe.
func (t *Toystore) Merge(data *data.Data) bool {
	current, ok := t.Data.Get(data.Key)

	if !ok || data.IsLater(current) {
		t.Data.Put(data)
		return true
	}

	return false
}

// Transfer sends a list of keys to another node in the cluster.
func (t *Toystore) Transfer(address string) {
	// TODO: should use Transfer RPC
	keys := t.Data.Keys()
	for _, key := range keys {
		val, ok := t.Data.Get(key)
		if !ok {
			panic("I was told this key existed but it doesn't...")
		}
		log.Printf("Forward %s/%s\n", key, fmt.Sprint(val.Value))
		lookup := t.Ring.KeyAddress([]byte(key))
		address, _, _ := lookup()

		if !t.isCoordinator(address) {
			t.client.CoordinatePut(string(address), val)
		}
	}
}

// AddMember adds a new node to the hash ring.
// If the new node is adjacent to the current node then it transfers
// any keys in its range that should be owned by the new node.
func (t *Toystore) AddMember(member Member) {
	log.Printf("%s adding member %s", t.Host, member.Name())
	t.Ring.AddString(member.Address())
	localAddress := t.rpcAddress()
	adjacent := t.Ring.Adjacent([]byte(localAddress), member.Meta())

	if adjacent {
		t.Transfer(member.Address())
	}
}

// RemoveMember removes a member from the hash ring.
func (t *Toystore) RemoveMember(member Member) {
	if member.Address() != t.rpcAddress() {
		log.Printf("%s removing member %s", t.Host, member.Name())
		// TODO: should fail nodes rather than removing them.
		t.Ring.RemoveString(member.Address())
	}
}

// New creates a new Toystore instance using the config variables.
// It starts the RPC server and gossip protocols to handle node
// communication between the cluster.
func New(config Config) *Toystore {
	t := &Toystore{
		ReplicationLevel: config.ReplicationLevel,
		W:                config.W,
		R:                config.R,
		Host:             config.Host,
		Port:             config.ClientPort,
		RPCPort:          config.RPCPort,
		Ring:             NewRingHead(),
		Data:             config.Store,

		client: NewRpcClient(),
	}

	// Start new gossip protocol
	t.Members = NewMemberlist(t, config.SeedAddress)

	// Start hinted handoff scan
	t.Hints = NewHintedHandoff(config, t.client)

	// Setup new hash ring
	t.Ring.ReplicationDepth = t.ReplicationLevel
	t.Ring.AddString(t.rpcAddress())

	// Start RPC server
	NewRpcHandler(t)

	return t
}
<commit_msg>Add package description<commit_after>// Package toystore is a simple implementation of a Dynamo database -
// http://www.allthingsdistributed.com/files/amazon-dynamo-sosp2007.pdf
//
// For more information see the project's readme -
// https://github.com/rlayte/toystore
package toystore

import (
	"fmt"
	"log"

	"github.com/rlayte/toystore/data"
)

type Toystore struct {
	// Number of nodes to replicate each data item.
	ReplicationLevel int

	// Number of successful writes required.
	W int

	// Number of successful reads required.
	R int

	// Port number to serve client requests.
	Port int

	// Port number to serve RPC requests between nodes.
	RPCPort int

	// Host address to bind to.
	Host string

	// Concrete Store implementation to persist data.
	Data Store

	// Hash ring for nodes in the cluster.
	Ring *Ring

	// Concrete Members implementation to represent the current cluster state.
	Members Members

	// Store of hinted data meant for other nodes.
	Hints *HintedHandoff

	// Concrete PeerClient implementation to make calls to other nodes.
	client PeerClient
}

// rpcAddress returns a string for the RPC address.
func (t *Toystore) rpcAddress() string {
	return fmt.Sprintf("%s:%d", t.Host, t.RPCPort)
}

// Get finds the key on the correct node in the cluster and returns
// the value and an existence bool.
// If the key is on the current node then it coordinates the operation.
// Otherwise it sends the coordination request to the correct node.
func (t *Toystore) Get(key string) (interface{}, bool) {
	lookup := t.Ring.KeyAddress([]byte(key))
	address, _, _ := lookup()
	var data *data.Data
	var ok bool

	if t.isCoordinator(address) {
		data, ok = t.CoordinateGet(key)
	} else {
		data, ok = t.client.CoordinateGet(string(address), key)
	}

	if ok {
		return data.Value, ok
	} else {
		return nil, ok
	}
}

// Put finds the key on the correct node in the cluster, sets
// the value and returns a status bool.
// If the key is owned by current node then it coordinates the operation.
// Otherwise it sends the coordination request to the correct node.
func (t *Toystore) Put(key string, value interface{}) (ok bool) {
	lookup := t.Ring.KeyAddress([]byte(key))
	address, _, _ := lookup()

	if t.isCoordinator(address) {
		ok = t.CoordinatePut(data.New(key, value))
	} else {
		ok = t.client.CoordinatePut(string(address), data.New(key, value))
	}
	return
}

// GetString returns a string of the value for the specified key/value pair.
func (t *Toystore) GetString(key string) (string, bool) {
	d, ok := t.Get(key)
	return d.(string), ok
}

// Merge updates the data object only if its Timestamp is later than the
// current value.
// If the key doesn't exist it adds it.
// Requires Store implementation to be thread safe.
func (t *Toystore) Merge(data *data.Data) bool {
	current, ok := t.Data.Get(data.Key)

	if !ok || data.IsLater(current) {
		t.Data.Put(data)
		return true
	}

	return false
}

// Transfer sends a list of keys to another node in the cluster.
func (t *Toystore) Transfer(address string) {
	// TODO: should use Transfer RPC
	keys := t.Data.Keys()
	for _, key := range keys {
		val, ok := t.Data.Get(key)
		if !ok {
			panic("I was told this key existed but it doesn't...")
		}
		log.Printf("Forward %s/%s\n", key, fmt.Sprint(val.Value))
		lookup := t.Ring.KeyAddress([]byte(key))
		address, _, _ := lookup()

		if !t.isCoordinator(address) {
			t.client.CoordinatePut(string(address), val)
		}
	}
}

// AddMember adds a new node to the hash ring.
// If the new node is adjacent to the current node then it transfers
// any keys in its range that should be owned by the new node.
func (t *Toystore) AddMember(member Member) {
	log.Printf("%s adding member %s", t.Host, member.Name())
	t.Ring.AddString(member.Address())
	localAddress := t.rpcAddress()
	adjacent := t.Ring.Adjacent([]byte(localAddress), member.Meta())

	if adjacent {
		t.Transfer(member.Address())
	}
}

// RemoveMember removes a member from the hash ring.
func (t *Toystore) RemoveMember(member Member) {
	if member.Address() != t.rpcAddress() {
		log.Printf("%s removing member %s", t.Host, member.Name())
		// TODO: should fail nodes rather than removing them.
		t.Ring.RemoveString(member.Address())
	}
}

// New creates a new Toystore instance using the config variables.
// It starts the RPC server and gossip protocols to handle node
// communication between the cluster.
func New(config Config) *Toystore {
	t := &Toystore{
		ReplicationLevel: config.ReplicationLevel,
		W:                config.W,
		R:                config.R,
		Host:             config.Host,
		Port:             config.ClientPort,
		RPCPort:          config.RPCPort,
		Ring:             NewRingHead(),
		Data:             config.Store,

		client: NewRpcClient(),
	}

	// Start new gossip protocol
	t.Members = NewMemberlist(t, config.SeedAddress)

	// Start hinted handoff scan
	t.Hints = NewHintedHandoff(config, t.client)

	// Setup new hash ring
	t.Ring.ReplicationDepth = t.ReplicationLevel
	t.Ring.AddString(t.rpcAddress())

	// Start RPC server
	NewRpcHandler(t)

	return t
}
<|endoftext|>


<commit_before>package migrations_test

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"testing"

	"github.com/OpenBazaar/openbazaar-go/repo/migrations"
	"github.com/OpenBazaar/openbazaar-go/schema"
)

const preMigration017Config = `{
	"DataSharing": {
		"AcceptStoreRequests": false,
		"PushTo": [
			"QmY8puEnVx66uEet64gAf4VZRo7oUyMCwG6KdB9KM92EGQ",
			"QmPPg2qeF3n2KvTRXRZLaTwHCw8JxzF4uZK93RfMoDvf2o"
		]
	},
	"OtherConfigProperty1": [1, 2, 3],
	"OtherConfigProperty2": "abc123"
}`

const postMigration017Config = `{
	"DataSharing": {
		"AcceptStoreRequests": false,
		"PushTo": [
			"QmY8puEnVx66uEet64gAf4VZRo7oUyMCwG6KdB9KM92EGQ",
			"QmPPg2qeF3n2KvTRXRZLaTwHCw8JxzF4uZK93RfMoDvf2o"
		]
	},
	"OtherConfigProperty1": [1, 2, 3],
	"OtherConfigProperty2": "abc123"
}`

func migration017AssertAPI(t *testing.T, actual interface{}, expected string) {
	actualSlice := actual.([]interface{})
	if len(actualSlice) != 1 || actualSlice[0] != expected {
		t.Fatalf("incorrect api endpoint.\n\twanted: %s\n\tgot: %s\n", expected, actual)
	}
}

func TestMigration017(t *testing.T) {
	var testRepo, err = schema.NewCustomSchemaManager(schema.SchemaContext{
		DataPath:        schema.GenerateTempPath(),
		TestModeEnabled: true,
	})
	if err != nil {
		t.Fatal(err)
	}

	if err = testRepo.BuildSchemaDirectories(); err != nil {
		t.Fatal(err)
	}
	defer testRepo.DestroySchemaDirectories()

	var (
		configPath  = testRepo.DataPathJoin("config")
		repoverPath = testRepo.DataPathJoin("repover")
	)
	if err = ioutil.WriteFile(configPath, []byte(preMigration017Config), os.ModePerm); err != nil {
		t.Fatal(err)
	}

	if err = ioutil.WriteFile(repoverPath, []byte("15"), os.ModePerm); err != nil {
		t.Fatal(err)
	}

	var m migrations.Migration017
	err = m.Up(testRepo.DataPath(), "", true)
	if err != nil {
		t.Fatal(err)
	}

	configBytes, err := ioutil.ReadFile(configPath)
	if err != nil {
		t.Fatal(err)
	}

	config := map[string]interface{}{}
	if err = json.Unmarshal(configBytes, &config); err != nil {
		t.Fatal(err)
	}

	ds := config["DataSharing"].(map[string]interface{})

	if accept, ok := ds["AcceptStoreRequests"].(bool); !ok || accept {
		t.Fatal("Incorrect config value for AcceptStoreRequests")
	}

	pushTo := ds["PushTo"].([]interface{})
	for i, peer := range pushTo {
		if peer != migrations.Migration017PushToAfter[i] {
			t.Fatal("Unexpected peer", peer, "wanted:", migrations.Migration017PushToAfter[i])
		}
	}
	assertCorrectRepoVer(t, repoverPath, "18")

	err = m.Down(testRepo.DataPath(), "", true)
	if err != nil {
		t.Fatal(err)
	}

	configBytes, err = ioutil.ReadFile(configPath)
	if err != nil {
		t.Fatal(err)
	}

	config = map[string]interface{}{}
	if err = json.Unmarshal(configBytes, &config); err != nil {
		t.Fatal(err)
	}

	ds = config["DataSharing"].(map[string]interface{})

	if accept, ok := ds["AcceptStoreRequests"].(bool); !ok || accept {
		t.Fatal("Incorrect config value for AcceptStoreRequests")
	}

	pushTo = ds["PushTo"].([]interface{})
	for i, peer := range pushTo {
		if peer != migrations.Migration017PushToBefore[i] {
			t.Fatal("Unexpected peer", peer, "wanted:", migrations.Migration017PushToBefore[i])
		}
	}

	assertCorrectRepoVer(t, repoverPath, "17")
}
<commit_msg>TWEAK: Improve migration17 test.<commit_after>package migrations_test

import (
	"io/ioutil"
	"os"
	"regexp"
	"testing"

	"github.com/OpenBazaar/openbazaar-go/repo/migrations"
	"github.com/OpenBazaar/openbazaar-go/schema"
)

const preMigration017Config = `{
	"DataSharing": {
		"AcceptStoreRequests": false,
		"PushTo": [
			"QmY8puEnVx66uEet64gAf4VZRo7oUyMCwG6KdB9KM92EGQ",
			"QmPPg2qeF3n2KvTRXRZLaTwHCw8JxzF4uZK93RfMoDvf2o"
		]
	},
	"OtherConfigProperty1": [1, 2, 3],
	"OtherConfigProperty2": "abc123"
}`

const postMigration017Config = `{
	"DataSharing": {
		"AcceptStoreRequests": false,
		"PushTo": [
			"QmbwN82MVyBukT7WTdaQDppaACo62oUfma8dUa5R9nBFHm",
			"QmY8puEnVx66uEet64gAf4VZRo7oUyMCwG6KdB9KM92EGQ",
			"QmPPg2qeF3n2KvTRXRZLaTwHCw8JxzF4uZK93RfMoDvf2o"
		]
	},
	"OtherConfigProperty1": [1, 2, 3],
	"OtherConfigProperty2": "abc123"
}`

func migration017AssertAPI(t *testing.T, actual interface{}, expected string) {
	actualSlice := actual.([]interface{})
	if len(actualSlice) != 1 || actualSlice[0] != expected {
		t.Fatalf("incorrect api endpoint.\n\twanted: %s\n\tgot: %s\n", expected, actual)
	}
}

func TestMigration017(t *testing.T) {
	var testRepo, err = schema.NewCustomSchemaManager(schema.SchemaContext{
		DataPath:        schema.GenerateTempPath(),
		TestModeEnabled: true,
	})
	if err != nil {
		t.Fatal(err)
	}

	if err = testRepo.BuildSchemaDirectories(); err != nil {
		t.Fatal(err)
	}
	defer testRepo.DestroySchemaDirectories()

	var (
		configPath  = testRepo.DataPathJoin("config")
		repoverPath = testRepo.DataPathJoin("repover")
	)
	if err = ioutil.WriteFile(configPath, []byte(preMigration017Config), os.ModePerm); err != nil {
		t.Fatal(err)
	}

	if err = ioutil.WriteFile(repoverPath, []byte("15"), os.ModePerm); err != nil {
		t.Fatal(err)
	}

	var m migrations.Migration017
	err = m.Up(testRepo.DataPath(), "", true)
	if err != nil {
		t.Fatal(err)
	}

	configBytes, err := ioutil.ReadFile(configPath)
	if err != nil {
		t.Fatal(err)
	}

	var re = regexp.MustCompile(`\s`)
	if re.ReplaceAllString(string(configBytes), "") != re.ReplaceAllString(string(postMigration017Config), "") {
		t.Logf("actual: %s", re.ReplaceAllString(string(configBytes), ""))
		t.Fatal("incorrect post-migration config")
	}

	assertCorrectRepoVer(t, repoverPath, "18")

	err = m.Down(testRepo.DataPath(), "", true)
	if err != nil {
		t.Fatal(err)
	}

	configBytes, err = ioutil.ReadFile(configPath)
	if err != nil {
		t.Fatal(err)
	}

	if re.ReplaceAllString(string(configBytes), "") != re.ReplaceAllString(string(preMigration017Config), "") {
		t.Logf("actual: %s", re.ReplaceAllString(string(configBytes), ""))
		t.Fatal("incorrect post-migration config")
	}

	assertCorrectRepoVer(t, repoverPath, "17")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/user"
)

type Config struct {
	WebhookUrl string `json:"webhook_url"`
}

func ReadConfig() (*Config, error) {
	homeDir := ""
	usr, err := user.Current()
	if err == nil {
		homeDir = usr.HomeDir
	}

	for _, path := range []string{"/etc/slackcat.conf", homeDir + "/.slackcat.conf", "./slackcat.conf"} {
		file, err := os.Open(path)
		if os.IsNotExist(err) {
			continue
		}
		if err != nil {
			return nil, err
		}

		json.NewDecoder(file)
		conf := Config{}
		err = json.NewDecoder(file).Decode(&conf)
		if err != nil {
			return nil, err
		}
		return &conf, nil
	}
	return &Config{}, nil
}

type SlackMsg struct {
	Username string `json:"username"`
	Text     string `json:"text"`
	Parse    string `json:"parse"`
}

func (m SlackMsg) Encode() (string, error) {
	b, err := json.Marshal(m)
	if err != nil {
		return "", err
	}
	return string(b), nil
}

func (m SlackMsg) Post(WebhookURL string) error {
	encoded, err := m.Encode()
	if err != nil {
		return err
	}

	resp, err := http.PostForm(WebhookURL, url.Values{"payload": {encoded}})
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusOK {
		return errors.New("Not OK")
	}
	return nil
}

func username() string {
	username := "<unknown>"
	usr, err := user.Current()
	if err == nil {
		username = usr.Username
	}

	hostname := "<unknown>"
	host, err := os.Hostname()
	if err == nil {
		hostname = host
	}
	return fmt.Sprintf("%s@%s", username, hostname)
}

func main() {

	cfg, err := ReadConfig()
	if err != nil {
		log.Fatalf("Coult not read config: %v", err)
	}
	bytes, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		log.Fatalf("Cannot read STDIN: %v", err)
	}

	msg := SlackMsg{
		Parse:    "full",
		Username: username(),
		Text:     string(bytes),
	}

	err = msg.Post(cfg.WebhookUrl)
	if err != nil {
		log.Fatalf("Post failed: %v", err)
	}
}
<commit_msg>Post a message for every line of stdin<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/user"
)

type Config struct {
	WebhookUrl string `json:"webhook_url"`
}

func ReadConfig() (*Config, error) {
	homeDir := ""
	usr, err := user.Current()
	if err == nil {
		homeDir = usr.HomeDir
	}

	for _, path := range []string{"/etc/slackcat.conf", homeDir + "/.slackcat.conf", "./slackcat.conf"} {
		file, err := os.Open(path)
		if os.IsNotExist(err) {
			continue
		}
		if err != nil {
			return nil, err
		}

		json.NewDecoder(file)
		conf := Config{}
		err = json.NewDecoder(file).Decode(&conf)
		if err != nil {
			return nil, err
		}
		return &conf, nil
	}
	return &Config{}, nil
}

type SlackMsg struct {
	Username string `json:"username"`
	Text     string `json:"text"`
	Parse    string `json:"parse"`
}

func (m SlackMsg) Encode() (string, error) {
	b, err := json.Marshal(m)
	if err != nil {
		return "", err
	}
	return string(b), nil
}

func (m SlackMsg) Post(WebhookURL string) error {
	encoded, err := m.Encode()
	if err != nil {
		return err
	}

	resp, err := http.PostForm(WebhookURL, url.Values{"payload": {encoded}})
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusOK {
		return errors.New("Not OK")
	}
	return nil
}

func username() string {
	username := "<unknown>"
	usr, err := user.Current()
	if err == nil {
		username = usr.Username
	}

	hostname := "<unknown>"
	host, err := os.Hostname()
	if err == nil {
		hostname = host
	}
	return fmt.Sprintf("%s@%s", username, hostname)
}

func main() {

	cfg, err := ReadConfig()
	if err != nil {
		log.Fatalf("Coult not read config: %v", err)
	}

	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		msg := SlackMsg{
			Parse:    "full",
			Username: username(),
			Text:     scanner.Text(),
		}

		err = msg.Post(cfg.WebhookUrl)
		if err != nil {
			log.Fatalf("Post failed: %v", err)
		}
	}
	if err := scanner.Err(); err != nil {
		log.Fatalf("Error reading: %v", err)
	}
}
<|endoftext|>


<commit_before>// snowball stemmer
//
// Example:
//  stemmer = snowball.New("en")
//  fmt.Println(stemmer.stem("running")) // Will print "run"
package snowball

import (
	"fmt"
	"runtime"
	"unsafe"
)

// #include "libstemmer.h"
import "C"
type Stemmer struct {
	lang string
	stemmer *C.struct_sb_stemmer
}

func free(stemmer *Stemmer) {
	if stemmer.stemmer != nil {
		C.sb_stemmer_delete(stemmer.stemmer)
		stemmer.stemmer = nil
	}
}

// New creates a new stemmer for lang
func New(lang string) (*Stemmer, error) {
	stemmer := &Stemmer{
		lang,
		C.sb_stemmer_new(C.CString(lang), nil),
	}

	if stemmer.stemmer == nil {
		return nil, fmt.Errorf("can't create stemmer for lang %s", lang)
	}

	runtime.SetFinalizer(stemmer, free)

	return stemmer, nil
}

// Lang return the stemmer language
func (stemmer *Stemmer) Lang() string {
	return stemmer.lang
}

// Stem returns them stem of word (e.g. running -> run)
func (stemmer *Stemmer) Stem(word string) string {
	ptr := unsafe.Pointer(C.CString(word))
	w := (*C.sb_symbol)(ptr)
	res := unsafe.Pointer(C.sb_stemmer_stem(stemmer.stemmer, w, C.int(len(word))))
	size := C.sb_stemmer_length(stemmer.stemmer)

	buf := C.GoBytes(res, size)
	return string(buf)
}

// List returns the list of languages supported by snowball
func List() []string {
	names := []string{}

	cp := uintptr(unsafe.Pointer(C.sb_stemmer_list()))
	size := unsafe.Sizeof(uintptr(0))

	for {
		name := C.GoString(*(**C.char)(unsafe.Pointer(cp)))
		if len(name) == 0 {
			break
		}
		names = append(names, name)
		cp += size
	}
	return names
}
<commit_msg>fix doc<commit_after>// snowball stemmer
//
// Example:
//  stemmer = snowball.New("english")
//  fmt.Println(stemmer.stem("running")) // Will print "run"
package snowball

import (
	"fmt"
	"runtime"
	"unsafe"
)

// #include "libstemmer.h"
import "C"
type Stemmer struct {
	lang string
	stemmer *C.struct_sb_stemmer
}

func free(stemmer *Stemmer) {
	if stemmer.stemmer != nil {
		C.sb_stemmer_delete(stemmer.stemmer)
		stemmer.stemmer = nil
	}
}

// New creates a new stemmer for lang
func New(lang string) (*Stemmer, error) {
	stemmer := &Stemmer{
		lang,
		C.sb_stemmer_new(C.CString(lang), nil),
	}

	if stemmer.stemmer == nil {
		return nil, fmt.Errorf("can't create stemmer for lang %s", lang)
	}

	runtime.SetFinalizer(stemmer, free)

	return stemmer, nil
}

// Lang return the stemmer language
func (stemmer *Stemmer) Lang() string {
	return stemmer.lang
}

// Stem returns them stem of word (e.g. running -> run)
func (stemmer *Stemmer) Stem(word string) string {
	ptr := unsafe.Pointer(C.CString(word))
	w := (*C.sb_symbol)(ptr)
	res := unsafe.Pointer(C.sb_stemmer_stem(stemmer.stemmer, w, C.int(len(word))))
	size := C.sb_stemmer_length(stemmer.stemmer)

	buf := C.GoBytes(res, size)
	return string(buf)
}

// List returns the list of languages supported by snowball
func List() []string {
	names := []string{}

	cp := uintptr(unsafe.Pointer(C.sb_stemmer_list()))
	size := unsafe.Sizeof(uintptr(0))

	for {
		name := C.GoString(*(**C.char)(unsafe.Pointer(cp)))
		if len(name) == 0 {
			break
		}
		names = append(names, name)
		cp += size
	}
	return names
}
<|endoftext|>


<commit_before>package shared

/*
Sortable allows sorting Objectinfos by path.
*/
type Sortable []*ObjectInfo

func (s Sortable) Len() int {
	return len(s)
}

func (s Sortable) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s Sortable) Less(i, j int) bool {
	// path are sorted alphabetically all by themselves! :D
	return s[i].Path < s[j].Path
}
<commit_msg>added sortable []string<commit_after>package shared

/*
Sortable allows sorting Objectinfos by path.
*/
type Sortable []*ObjectInfo

func (s Sortable) Len() int {
	return len(s)
}

func (s Sortable) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s Sortable) Less(i, j int) bool {
	// path are sorted alphabetically all by themselves! :D
	return s[i].Path < s[j].Path
}

/*
SortableString is a string slice that can be sorted by length.
*/
type SortableString []*string

func (s SortableString) Len() int {
	return len(s)
}

func (s SortableString) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s SortableString) Less(i, j int) bool {
	// path are sorted alphabetically all by themselves! :D
	return *s[i] < *s[j]
}
<|endoftext|>


<commit_before>package main

import (
	"strings"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

const (
	exporterNamespace = "varnish"
)

// prometheusExporter

type prometheusExporter struct {
	sync.RWMutex

	up      prometheus.Gauge
	version prometheus.Gauge
}

func NewPrometheusExporter() *prometheusExporter {
	return &prometheusExporter{
		// @todo varnishstat never fails, even if varnish is not running.
		// Figure our a reliable way to detect if varnish is running and use that for up.
		up: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: exporterNamespace,
			Name:      "up",
			Help:      "Was the last scrape of varnish successful.",
		}),
	}
}

func (pe *prometheusExporter) Initialize() error {
	pe.version = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace:   exporterNamespace,
		Name:        "version",
		Help:        "Varnish version information",
		ConstLabels: VarnishVersion.Labels(),
	})
	pe.version.Set(1)
	return nil
}

// Implements prometheus.Collector
func (pe *prometheusExporter) Describe(ch chan<- *prometheus.Desc) {
	start := time.Now()

	ch <- pe.up.Desc()
	ch <- pe.version.Desc()

	if StartParams.Verbose {
		logInfo("prometheus.Collector.Describe  %s", time.Now().Sub(start))
	}
}

// Implements prometheus.Collector
func (pe *prometheusExporter) Collect(ch chan<- prometheus.Metric) {
	start := time.Now()

	pe.Lock()
	defer pe.Unlock()

	if err := scrapeVarnish(ch); err == nil {
		pe.up.Set(1)
	} else {
		logError(err.Error())
		pe.up.Set(0)
	}

	ch <- pe.up
	ch <- pe.version

	if StartParams.Verbose {
		logInfo("prometheus.Collector.Collect   %s", time.Now().Sub(start))
	}
}

// utils

type group struct {
	name     string
	prefixes []string
}

var (
	groups = []group{
		// @note varnish 3.x does not seem to mark 'MAIN.' prefixes
		group{name: "backend", prefixes: []string{
			"vbe.",
			// varnish 4.x
			"main.backend_",
			"main.s_fetch",
			// varnish 3.x
			"backend_",
			"main.s_fetch",
		}},
		group{name: "mempool", prefixes: []string{
			"mempool.",
		}},
		group{name: "lck", prefixes: []string{
			"lck.",
		}},
		group{name: "sma", prefixes: []string{
			"sma.",
		}},
		group{name: "mgt", prefixes: []string{
			"mgt.",
		}},
		// must be last so above groups have a opportunity to override
		group{name: "main", prefixes: []string{
			"main.",
		}},
	}
)

const (
	fq_main_fetch = exporterNamespace + "_main_fetch"
)

// https://prometheus.io/docs/practices/naming/
func computePrometheusInfo(vName, vGroup, vIdentifier, vDescription string) (name, description string, labelKeys, labelValues []string) {
	// name and description
	{
		fq := strings.ToLower(vName)
		// Remove unique identifiers from name to group similar metrics by labeling
		if len(vIdentifier) > 0 {
			fq = strings.Replace(fq, "."+strings.ToLower(vIdentifier), "", -1)
		}
		// Make sure our group is prefixed only once
		fq = prometheusTrimGroupPrefix(fq)
		// Build fq name
		name = exporterNamespace + "_" + vGroup + "_" + strings.Replace(fq, ".", "_", -1)
		description = vDescription
	}
	// labels: can alter final name and description
	{
		if len(vIdentifier) > 0 {
			if isVBE := startsWith(vName, "VBE.", caseSensitive); isVBE {
				// @todo this is quick and dirty, do regexp?
				if VarnishVersion.Major == 4 {
					// <uuid>.<name>
					if len(vIdentifier) > 37 && vIdentifier[8] == '-' && vIdentifier[36] == '.' {
						labelKeys, labelValues = append(labelKeys, "id"), append(labelValues, vIdentifier[0:36])
						labelKeys, labelValues = append(labelKeys, "backend"), append(labelValues, vIdentifier[37:])
					}
				} else if VarnishVersion.Major == 3 {
					// <name>(<ip>,<something>,<port>)
					iStart, iEnd := strings.Index(vIdentifier, "("), strings.Index(vIdentifier, ")")
					if iStart > 0 && iEnd > 1 && iStart < iEnd {
						labelKeys, labelValues = append(labelKeys, "id"), append(labelValues, vIdentifier[iStart+1:iEnd])
						labelKeys, labelValues = append(labelKeys, "backend"), append(labelValues, vIdentifier[0:iStart])
					}
				}
			}
			if len(labelKeys) == 0 {
				labelKeys, labelValues = append(labelKeys, "id"), append(labelValues, vIdentifier)
			}
		}
		// name grouping bt moving part of the fq name as a label
		if startsWith(name, fq_main_fetch+"_", caseSensitive) {
			// If name is manipulated to be the same for multiple metrics
			// the description needs to match as well.
			labelKeys, labelValues = append(labelKeys, "code"), append(labelValues, name[len(fq_main_fetch)+1:])
			name, description = fq_main_fetch, "Number of backend fetches"
		}
	}
	return name, description, labelKeys, labelValues
}

func prometheusTrimGroupPrefix(name string) string {
	nameLower := strings.ToLower(name)
	for _, group := range groups {
		for _, prefix := range group.prefixes {
			if startsWith(nameLower, prefix, caseSensitive) {
				return name[len(prefix):]
			}
		}
	}
	return name
}

// Always returns at least one main label
func prometheusGroup(vName string) string {
	vNameLower := strings.ToLower(vName)
	for _, group := range groups {
		if startsWithAny(vNameLower, group.prefixes, caseSensitive) {
			return group.name
		}
	}
	return "main"
}
<commit_msg>Group s_fetch as main_fetch with 'total' label. Change label key from 'code' to 'type'.<commit_after>package main

import (
	"strings"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

const (
	exporterNamespace = "varnish"
)

// prometheusExporter

type prometheusExporter struct {
	sync.RWMutex

	up      prometheus.Gauge
	version prometheus.Gauge
}

func NewPrometheusExporter() *prometheusExporter {
	return &prometheusExporter{
		// @todo varnishstat never fails, even if varnish is not running.
		// Figure our a reliable way to detect if varnish is running and use that for up.
		up: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: exporterNamespace,
			Name:      "up",
			Help:      "Was the last scrape of varnish successful.",
		}),
	}
}

func (pe *prometheusExporter) Initialize() error {
	pe.version = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace:   exporterNamespace,
		Name:        "version",
		Help:        "Varnish version information",
		ConstLabels: VarnishVersion.Labels(),
	})
	pe.version.Set(1)
	return nil
}

// Implements prometheus.Collector
func (pe *prometheusExporter) Describe(ch chan<- *prometheus.Desc) {
	start := time.Now()

	ch <- pe.up.Desc()
	ch <- pe.version.Desc()

	if StartParams.Verbose {
		logInfo("prometheus.Collector.Describe  %s", time.Now().Sub(start))
	}
}

// Implements prometheus.Collector
func (pe *prometheusExporter) Collect(ch chan<- prometheus.Metric) {
	start := time.Now()

	pe.Lock()
	defer pe.Unlock()

	if err := scrapeVarnish(ch); err == nil {
		pe.up.Set(1)
	} else {
		logError(err.Error())
		pe.up.Set(0)
	}

	ch <- pe.up
	ch <- pe.version

	if StartParams.Verbose {
		logInfo("prometheus.Collector.Collect   %s", time.Now().Sub(start))
	}
}

// utils

type group struct {
	name     string
	prefixes []string
}

var (
	groups = []group{
		// @note varnish 3.x does not seem to mark 'MAIN.' prefixes
		group{name: "backend", prefixes: []string{
			"vbe.",
			// varnish 4.x
			"main.backend_",
			// varnish 3.x
			"backend_",
		}},
		group{name: "mempool", prefixes: []string{
			"mempool.",
		}},
		group{name: "lck", prefixes: []string{
			"lck.",
		}},
		group{name: "sma", prefixes: []string{
			"sma.",
		}},
		group{name: "mgt", prefixes: []string{
			"mgt.",
		}},
		// must be last so above groups have a opportunity to override
		group{name: "main", prefixes: []string{
			"main.",
		}},
	}
)

const (
	fq_main_fetch       = exporterNamespace + "_main_fetch"
	fq_main_fetch_total = exporterNamespace + "_main_s_fetch"
)

// https://prometheus.io/docs/practices/naming/
func computePrometheusInfo(vName, vGroup, vIdentifier, vDescription string) (name, description string, labelKeys, labelValues []string) {
	// name and description
	{
		fq := strings.ToLower(vName)
		// Remove unique identifiers from name to group similar metrics by labeling
		if len(vIdentifier) > 0 {
			fq = strings.Replace(fq, "."+strings.ToLower(vIdentifier), "", -1)
		}
		// Make sure our group is prefixed only once
		fq = prometheusTrimGroupPrefix(fq)
		// Build fq name
		name = exporterNamespace + "_" + vGroup + "_" + strings.Replace(fq, ".", "_", -1)
		description = vDescription
	}
	// labels: can alter final name and description
	{
		if len(vIdentifier) > 0 {
			if isVBE := startsWith(vName, "VBE.", caseSensitive); isVBE {
				// @todo this is quick and dirty, do regexp?
				if VarnishVersion.Major == 4 {
					// <uuid>.<name>
					if len(vIdentifier) > 37 && vIdentifier[8] == '-' && vIdentifier[36] == '.' {
						labelKeys, labelValues = append(labelKeys, "id"), append(labelValues, vIdentifier[0:36])
						labelKeys, labelValues = append(labelKeys, "backend"), append(labelValues, vIdentifier[37:])
					}
				} else if VarnishVersion.Major == 3 {
					// <name>(<ip>,<something>,<port>)
					iStart, iEnd := strings.Index(vIdentifier, "("), strings.Index(vIdentifier, ")")
					if iStart > 0 && iEnd > 1 && iStart < iEnd {
						labelKeys, labelValues = append(labelKeys, "id"), append(labelValues, vIdentifier[iStart+1:iEnd])
						labelKeys, labelValues = append(labelKeys, "backend"), append(labelValues, vIdentifier[0:iStart])
					}
				}
			}
			if len(labelKeys) == 0 {
				labelKeys, labelValues = append(labelKeys, "id"), append(labelValues, vIdentifier)
			}
		}
		// name grouping bt moving part of the fq name as a label
		if name == fq_main_fetch_total {
			labelKeys, labelValues = append(labelKeys, "type"), append(labelValues, "total")
			name, description = fq_main_fetch, "Number of fetches"
		} else if startsWith(name, fq_main_fetch+"_", caseSensitive) {
			// If name is manipulated to be the same for multiple metrics
			// the description needs to match as well.
			labelKeys, labelValues = append(labelKeys, "type"), append(labelValues, name[len(fq_main_fetch)+1:])
			name, description = fq_main_fetch, "Number of fetches"
		}

	}
	return name, description, labelKeys, labelValues
}

func prometheusTrimGroupPrefix(name string) string {
	nameLower := strings.ToLower(name)
	for _, group := range groups {
		for _, prefix := range group.prefixes {
			if startsWith(nameLower, prefix, caseSensitive) {
				return name[len(prefix):]
			}
		}
	}
	return name
}

// Always returns at least one main label
func prometheusGroup(vName string) string {
	vNameLower := strings.ToLower(vName)
	for _, group := range groups {
		if startsWithAny(vNameLower, group.prefixes, caseSensitive) {
			return group.name
		}
	}
	return "main"
}
<|endoftext|>


<commit_before>package hoverfly

import (
	log "github.com/Sirupsen/logrus"
	"github.com/rusenask/goproxy"

	"bufio"
	"fmt"
	"net"
	"net/http"
	"regexp"
)

// VirtualizeMode - default mode when Hoverfly looks for captured requests to respond
const VirtualizeMode = "virtualize"

// SynthesizeMode - all requests are sent to middleware to create response
const SynthesizeMode = "synthesize"

// ModifyMode - middleware is applied to outgoing and incoming traffic
const ModifyMode = "modify"

// CaptureMode - requests are captured and stored in cache
const CaptureMode = "capture"

// orPanic - wrapper for logging errors
func orPanic(err error) {
	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Panic("Got error.")
	}
}

// StartHoverflyProxy - starts given proxy
func StartHoverflyProxy(cfg *Configuration, proxy *goproxy.ProxyHttpServer) {
	log.WithFields(log.Fields{
		"destination": cfg.Destination,
		"port":        cfg.ProxyPort,
		"mode":        cfg.GetMode(),
	}).Info("current proxy configuration")

	// creating TCP listener
	listener, err := net.Listen("tcp", fmt.Sprintf(":%s", cfg.ProxyPort))
	if err != nil {
		panic(err)
	}

	sl, err := NewStoppableListener(listener)
	if err != nil {
		panic(err)
	}
	cfg.SL = sl
	server := http.Server{}

	cfg.ProxyControlWG.Add(1)

	go func() {
		defer cfg.ProxyControlWG.Done()
		log.Info("serving proxy")
		server.Handler = proxy
		log.Warn(server.Serve(sl))
	}()
}

// GetNewHoverfly returns a configured ProxyHttpServer and DBClient
func GetNewHoverfly(cfg *Configuration, cache Cache) (*goproxy.ProxyHttpServer, DBClient) {

	counter := NewModeCounter()
	// getting connections
	d := DBClient{
		Cache:   cache,
		HTTP:    &http.Client{},
		Cfg:     cfg,
		Counter: counter,
		Hooks:   make(ActionTypeHooks),
	}

	// creating proxy
	proxy := goproxy.NewProxyHttpServer()

	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(d.Cfg.Destination))).
		HandleConnect(goproxy.AlwaysMitm)

	// enable curl -p for all hosts on port 80
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(d.Cfg.Destination))).
		HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) {
			defer func() {
				if e := recover(); e != nil {
					ctx.Logf("error connecting to remote: %v", e)
					client.Write([]byte("HTTP/1.1 500 Cannot reach destination\r\n\r\n"))
				}
				client.Close()
			}()
			clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client))
			remote, err := net.Dial("tcp", req.URL.Host)
			orPanic(err)
			remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote))
			for {
				req, err := http.ReadRequest(clientBuf.Reader)
				orPanic(err)
				orPanic(req.Write(remoteBuf))
				orPanic(remoteBuf.Flush())
				resp, err := http.ReadResponse(remoteBuf.Reader, req)

				orPanic(err)
				orPanic(resp.Write(clientBuf.Writer))
				orPanic(clientBuf.Flush())
			}
		})

	// processing connections
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(cfg.Destination))).DoFunc(
		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
			return d.processRequest(r)
		})

	if cfg.Verbose {
		proxy.OnRequest().DoFunc(
			func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
				log.WithFields(log.Fields{
					"destination": r.Host,
					"path":        r.URL.Path,
					"query":       r.URL.RawQuery,
					"method":      r.Method,
					"remoteAddr":  r.RemoteAddr,
					"mode":        cfg.GetMode(),
				}).Debug("got request..")
				return r, nil
			})
	}

	// intercepts response
	proxy.OnResponse(goproxy.ReqHostMatches(regexp.MustCompile(cfg.Destination))).DoFunc(
		func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
			d.Counter.Count(d.Cfg.GetMode())
			return resp
		})

	proxy.Verbose = d.Cfg.Verbose
	// proxy starting message
	log.WithFields(log.Fields{
		"Destination": d.Cfg.Destination,
		"ProxyPort":   d.Cfg.ProxyPort,
		"Mode":        d.Cfg.GetMode(),
	}).Info("Proxy prepared...")

	return proxy, d
}

func hoverflyError(req *http.Request, err error, msg string, statusCode int) *http.Response {
	return goproxy.NewResponse(req,
		goproxy.ContentTypeText, statusCode,
		fmt.Sprintf("Hoverfly Error! %s. Got error: %s \n", msg, err.Error()))
}

// processRequest - processes incoming requests and based on proxy state (record/playback)
// returns HTTP response.
func (d *DBClient) processRequest(req *http.Request) (*http.Request, *http.Response) {

	mode := d.Cfg.GetMode()

	if mode == CaptureMode {
		newResponse, err := d.captureRequest(req)

		if err != nil {
			return req, hoverflyError(req, err, "Could not capture request", http.StatusServiceUnavailable)
		}
		log.WithFields(log.Fields{
			"mode":        mode,
			"middleware":  d.Cfg.Middleware,
			"path":        req.URL.Path,
			"rawQuery":    req.URL.RawQuery,
			"method":      req.Method,
			"destination": req.Host,
		}).Info("request and response captured")

		return req, newResponse

	} else if mode == SynthesizeMode {
		response, err := SynthesizeResponse(req, d.Cfg.Middleware)

		if err != nil {
			return req, hoverflyError(req, err, "Could not create synthetic response!", http.StatusServiceUnavailable)
		}

		log.WithFields(log.Fields{
			"mode":        mode,
			"middleware":  d.Cfg.Middleware,
			"path":        req.URL.Path,
			"rawQuery":    req.URL.RawQuery,
			"method":      req.Method,
			"destination": req.Host,
		}).Info("synthetic response created successfuly")

		return req, response

	} else if mode == ModifyMode {
		response, err := d.modifyRequestResponse(req, d.Cfg.Middleware)

		if err != nil {
			log.WithFields(log.Fields{
				"error":      err.Error(),
				"middleware": d.Cfg.Middleware,
			}).Error("Got error when performing request modification")
			return req, hoverflyError(
				req,
				err,
				fmt.Sprintf("Middleware (%s) failed or something else happened!", d.Cfg.Middleware),
				http.StatusServiceUnavailable)
		}
		// returning modified response
		return req, response
	}

	newResponse := d.getResponse(req)
	return req, newResponse

}
<commit_msg>cleanup<commit_after>package hoverfly

import (
	log "github.com/Sirupsen/logrus"
	"github.com/rusenask/goproxy"

	"bufio"
	"fmt"
	"net"
	"net/http"
	"regexp"
)

// VirtualizeMode - default mode when Hoverfly looks for captured requests to respond
const VirtualizeMode = "virtualize"

// SynthesizeMode - all requests are sent to middleware to create response
const SynthesizeMode = "synthesize"

// ModifyMode - middleware is applied to outgoing and incoming traffic
const ModifyMode = "modify"

// CaptureMode - requests are captured and stored in cache
const CaptureMode = "capture"

// orPanic - wrapper for logging errors
func orPanic(err error) {
	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Panic("Got error.")
	}
}

// GetNewHoverfly returns a configured ProxyHttpServer and DBClient
func GetNewHoverfly(cfg *Configuration, cache Cache) DBClient {
	counter := NewModeCounter()
	// getting connections
	d := DBClient{
		Cache:   cache,
		HTTP:    &http.Client{},
		Cfg:     cfg,
		Counter: counter,
		Hooks:   make(ActionTypeHooks),
	}

	d.UpdateProxy()

	return d
}

// UpdateProxy - applies hooks
func (d *DBClient) UpdateProxy() {
	// creating proxy
	proxy := goproxy.NewProxyHttpServer()

	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(d.Cfg.Destination))).
		HandleConnect(goproxy.AlwaysMitm)

	// enable curl -p for all hosts on port 80
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(d.Cfg.Destination))).
		HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) {
			defer func() {
				if e := recover(); e != nil {
					ctx.Logf("error connecting to remote: %v", e)
					client.Write([]byte("HTTP/1.1 500 Cannot reach destination\r\n\r\n"))
				}
				client.Close()
			}()
			clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client))
			remote, err := net.Dial("tcp", req.URL.Host)
			orPanic(err)
			remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote))
			for {
				req, err := http.ReadRequest(clientBuf.Reader)
				orPanic(err)
				orPanic(req.Write(remoteBuf))
				orPanic(remoteBuf.Flush())
				resp, err := http.ReadResponse(remoteBuf.Reader, req)

				orPanic(err)
				orPanic(resp.Write(clientBuf.Writer))
				orPanic(clientBuf.Flush())
			}
		})

	// processing connections
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(d.Cfg.Destination))).DoFunc(
		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
			return d.processRequest(r)
		})

	if d.Cfg.Verbose {
		proxy.OnRequest().DoFunc(
			func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
				log.WithFields(log.Fields{
					"destination": r.Host,
					"path":        r.URL.Path,
					"query":       r.URL.RawQuery,
					"method":      r.Method,
					"remoteAddr":  r.RemoteAddr,
					"mode":        d.Cfg.GetMode(),
				}).Debug("got request..")
				return r, nil
			})
	}

	// intercepts response
	proxy.OnResponse(goproxy.ReqHostMatches(regexp.MustCompile(d.Cfg.Destination))).DoFunc(
		func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
			d.Counter.Count(d.Cfg.GetMode())
			return resp
		})

	proxy.Verbose = d.Cfg.Verbose
	// proxy starting message
	log.WithFields(log.Fields{
		"Destination": d.Cfg.Destination,
		"ProxyPort":   d.Cfg.ProxyPort,
		"Mode":        d.Cfg.GetMode(),
	}).Info("Proxy prepared...")

	d.Proxy = proxy
	return
}

func hoverflyError(req *http.Request, err error, msg string, statusCode int) *http.Response {
	return goproxy.NewResponse(req,
		goproxy.ContentTypeText, statusCode,
		fmt.Sprintf("Hoverfly Error! %s. Got error: %s \n", msg, err.Error()))
}

// processRequest - processes incoming requests and based on proxy state (record/playback)
// returns HTTP response.
func (d *DBClient) processRequest(req *http.Request) (*http.Request, *http.Response) {

	mode := d.Cfg.GetMode()

	if mode == CaptureMode {
		newResponse, err := d.captureRequest(req)

		if err != nil {
			return req, hoverflyError(req, err, "Could not capture request", http.StatusServiceUnavailable)
		}
		log.WithFields(log.Fields{
			"mode":        mode,
			"middleware":  d.Cfg.Middleware,
			"path":        req.URL.Path,
			"rawQuery":    req.URL.RawQuery,
			"method":      req.Method,
			"destination": req.Host,
		}).Info("request and response captured")

		return req, newResponse

	} else if mode == SynthesizeMode {
		response, err := SynthesizeResponse(req, d.Cfg.Middleware)

		if err != nil {
			return req, hoverflyError(req, err, "Could not create synthetic response!", http.StatusServiceUnavailable)
		}

		log.WithFields(log.Fields{
			"mode":        mode,
			"middleware":  d.Cfg.Middleware,
			"path":        req.URL.Path,
			"rawQuery":    req.URL.RawQuery,
			"method":      req.Method,
			"destination": req.Host,
		}).Info("synthetic response created successfuly")

		return req, response

	} else if mode == ModifyMode {
		response, err := d.modifyRequestResponse(req, d.Cfg.Middleware)

		if err != nil {
			log.WithFields(log.Fields{
				"error":      err.Error(),
				"middleware": d.Cfg.Middleware,
			}).Error("Got error when performing request modification")
			return req, hoverflyError(
				req,
				err,
				fmt.Sprintf("Middleware (%s) failed or something else happened!", d.Cfg.Middleware),
				http.StatusServiceUnavailable)
		}
		// returning modified response
		return req, response
	}

	newResponse := d.getResponse(req)
	return req, newResponse

}
<|endoftext|>


<commit_before>// +build !vanilla

package mains

import (
	"strings"
	"unicode/utf8"

	"github.com/yuin/gopher-lua"
)

func utf8char(L *lua.LState) int {
	var buffer strings.Builder
	for i, n := 1, L.GetTop(); i <= n; i++ {
		number, ok := L.Get(i).(lua.LNumber)
		if !ok {
			return lerror(L, "NaN")
		}
		buffer.WriteRune(rune(number))
	}
	L.Push(lua.LString(buffer.String()))
	return 1
}

func utf8codes(L *lua.LState) int {
	lstr, ok := L.Get(-1).(lua.LString)
	if !ok {
		return lerror(L, "invalid utf8")
	}

	p := strings.NewReader(string(lstr))
	pos := 1

	f := func(LL *lua.LState) int {
		r, siz, err := p.ReadRune()
		if err != nil {
			return 0
		}
		LL.Push(lua.LNumber(pos))
		LL.Push(lua.LNumber(r))
		pos += siz
		return 2
	}
	L.Push(L.NewFunction(f))
	L.Push(lstr)
	L.Push(lua.LNumber(1))
	return 3
}

func utf8len(L *lua.LState) int {
	_s, ok := L.Get(1).(lua.LString)
	if !ok {
		return lerror(L, "utf8.len(s,i,j): s is not string")
	}
	s := string(_s)
	_len := len(s)

	j := _len + 1
	i := 1
	top := L.GetTop()
	if top >= 2 {
		if _i, ok := L.Get(2).(lua.LNumber); ok {
			i = int(_i)
		} else {
			return lerror(L, "utf8.len(s,i,j): i is not a number")
		}
		if top >= 3 {
			if _j, ok := L.Get(3).(lua.LNumber); ok {
				j = int(_j)
			} else {
				return lerror(L, "utf8.len(s,i,j): j is not a number)")
			}
		}
	}
	if j < 0 {
		j += _len + 1
	} else if j > 0 {
		j--
	}
	if i < 0 {
		i += _len + 1
	} else if i > 0 {
		i--
	}
	if !utf8.RuneStart(s[i]) {
		return lerror(L, "utf8.len: not start byte")
	}
	s = s[i:]
	j -= i
	length := 0
	for pos := range s {
		if pos > j {
			break
		}
		length++
	}
	L.Push(lua.LNumber(length))
	return 1
}

func utf8offset(L *lua.LState) int {
	_s, ok := L.Get(1).(lua.LString)
	if !ok {
		return lerror(L, "utf8.offset: not string")
	}
	s := string(_s)
	_n, ok := L.Get(2).(lua.LNumber)
	if !ok {
		return lerror(L, "utf8.offset: not a number")
	}
	n := int(_n)

	_i, ok := L.Get(3).(lua.LNumber)
	i := 1
	if ok {
		i = int(_i)
	}
	if i == 0 {
		i = 1
	} else if i < 0 {
		i += len(s) + 1
	}
	for i > 0 && i < len(s) && !utf8.RuneStart(s[i-1]) {
		i++
	}
	s = s[i-1:]

	if n < 0 {
		n = utf8.RuneCountInString(s) + n
	} else if n > 0 {
		n--
	}
	for pos := range s {
		if n <= 0 {
			L.Push(lua.LNumber(pos + 1 + i - 1))
			return 1
		}
		n--
	}
	L.Push(lua.LNumber(len(s)))
	return 1
}

func setupUtf8Table(L *lua.LState) {
	table := L.NewTable()
	L.SetField(table, "codes", L.NewFunction(utf8codes))
	L.SetField(table, "charpattern", lua.LString("[\000-\x7F\xC2-\xF4][\x80-\xBF]*"))
	L.SetField(table, "char", L.NewFunction(utf8char))
	L.SetField(table, "len", L.NewFunction(utf8len))
	L.SetField(table, "offset", L.NewFunction(utf8offset))
	L.SetGlobal("utf8", table)
}
<commit_msg>utf8.len(lua): fix result for utf8 sequnse stating invalid bytes<commit_after>// +build !vanilla

package mains

import (
	"strings"
	"unicode/utf8"

	"github.com/yuin/gopher-lua"
)

func utf8char(L *lua.LState) int {
	var buffer strings.Builder
	for i, n := 1, L.GetTop(); i <= n; i++ {
		number, ok := L.Get(i).(lua.LNumber)
		if !ok {
			return lerror(L, "NaN")
		}
		buffer.WriteRune(rune(number))
	}
	L.Push(lua.LString(buffer.String()))
	return 1
}

func utf8codes(L *lua.LState) int {
	lstr, ok := L.Get(-1).(lua.LString)
	if !ok {
		return lerror(L, "invalid utf8")
	}

	p := strings.NewReader(string(lstr))
	pos := 1

	f := func(LL *lua.LState) int {
		r, siz, err := p.ReadRune()
		if err != nil {
			return 0
		}
		LL.Push(lua.LNumber(pos))
		LL.Push(lua.LNumber(r))
		pos += siz
		return 2
	}
	L.Push(L.NewFunction(f))
	L.Push(lstr)
	L.Push(lua.LNumber(1))
	return 3
}

func runeCountLua(b []byte) (int, int) {
	if len(b) <= 0 {
		return 0, 0
	}
	if 0x80 <= b[0] && b[0] <= 0xBF && 0xC2 <= b[0] && b[0] <= 0xDF {
		if len(b) < 2 || b[1] < 0x80 || b[1] > 0xBF {
			return -1, 1
		}
	} else if 0xE0 <= b[0] && b[0] <= 0xEF {
		if len(b) < 2 || b[1] < 0x80 || b[1] > 0xBF {
			return -1, 1
		} else if len(b) < 3 || b[2] < 0x80 || b[2] > 0xBF {
			return -1, 2
		}
	} else if 0xF0 <= b[0] && b[0] <= 0xF7 {
		if len(b) < 2 || b[1] < 0x80 || b[1] > 0xBF {
			return -1, 1
		} else if len(b) < 3 || b[2] < 0x80 || b[2] > 0xBF {
			return -1, 2
		} else if len(b) < 4 || b[3] < 0x80 || b[3] > 0xBF {
			return -1, 3
		}
	} else if b[0] >= 0x80 {
		return -1, 0
	}
	return utf8.RuneCount(b), 0
}

func utf8len(L *lua.LState) int {
	_s, ok := L.Get(1).(lua.LString)
	if !ok {
		return lerror(L, "utf8.len(s,i,j): s is not string")
	}
	s := string(_s)
	_len := len(s)

	j := _len + 1
	i := 1
	top := L.GetTop()
	if top >= 2 {
		if _i, ok := L.Get(2).(lua.LNumber); ok {
			i = int(_i)
		} else {
			return lerror(L, "utf8.len(s,i,j): i is not a number")
		}
		if top >= 3 {
			if _j, ok := L.Get(3).(lua.LNumber); ok {
				j = int(_j)
			} else {
				return lerror(L, "utf8.len(s,i,j): j is not a number)")
			}
		}
	}
	if j <= 0 {
		j += _len
	}
	if i <= 0 {
		i += _len
	}

	pos, errpos := runeCountLua([]byte(s[(i - 1):(j - 1)]))
	if pos < 0 {
		L.Push(lua.LNil)
		L.Push(lua.LNumber(errpos + i))
		return 2
	}
	L.Push(lua.LNumber(pos))
	return 1
}

func utf8offset(L *lua.LState) int {
	_s, ok := L.Get(1).(lua.LString)
	if !ok {
		return lerror(L, "utf8.offset: not string")
	}
	s := string(_s)
	_n, ok := L.Get(2).(lua.LNumber)
	if !ok {
		return lerror(L, "utf8.offset: not a number")
	}
	n := int(_n)

	_i, ok := L.Get(3).(lua.LNumber)
	i := 1
	if ok {
		i = int(_i)
	}
	if i == 0 {
		i = 1
	} else if i < 0 {
		i += len(s) + 1
	}
	for i > 0 && i < len(s) && !utf8.RuneStart(s[i-1]) {
		i++
	}
	s = s[i-1:]

	if n < 0 {
		n = utf8.RuneCountInString(s) + n
	} else if n > 0 {
		n--
	}
	for pos := range s {
		if n <= 0 {
			L.Push(lua.LNumber(pos + 1 + i - 1))
			return 1
		}
		n--
	}
	L.Push(lua.LNumber(len(s)))
	return 1
}

func setupUtf8Table(L *lua.LState) {
	table := L.NewTable()
	L.SetField(table, "codes", L.NewFunction(utf8codes))
	L.SetField(table, "charpattern", lua.LString("[\000-\x7F\xC2-\xF4][\x80-\xBF]*"))
	L.SetField(table, "char", L.NewFunction(utf8char))
	L.SetField(table, "len", L.NewFunction(utf8len))
	L.SetField(table, "offset", L.NewFunction(utf8offset))
	L.SetGlobal("utf8", table)
}
<|endoftext|>