<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"fmt"
	"testing"
	"time"

	v1 "k8s.io/api/networking/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/retry"
	"k8s.io/ingress-gce/pkg/annotations"
	backendconfig "k8s.io/ingress-gce/pkg/apis/backendconfig/v1"
	"k8s.io/ingress-gce/pkg/e2e"
	"k8s.io/ingress-gce/pkg/e2e/adapter"
	"k8s.io/ingress-gce/pkg/fuzz"
	"k8s.io/ingress-gce/pkg/fuzz/features"
	"k8s.io/ingress-gce/pkg/utils"
)

const (
	transitionPollTimeout  = 10 * time.Minute
	transitionPollInterval = 30 * time.Second
)

type affinityTransition struct {
	affinity string
	ttl      int64
}

func TestAffinity(t *testing.T) {
	t.Parallel()

	for _, tc := range []struct {
		desc       string
		ttl        int64
		expect     string
		transition affinityTransition
		beConfig   *backendconfig.BackendConfig
	}{
		{
			desc:       "http with no affinity.",
			beConfig:   fuzz.NewBackendConfigBuilder("", "backendconfig-1").Build(),
			expect:     "NONE",
			transition: affinityTransition{"CLIENT_IP", 10},
		},
		{
			desc: "http with cookie based affinity.",
			beConfig: fuzz.NewBackendConfigBuilder("", "backendconfig-1").
				SetSessionAffinity("GENERATED_COOKIE").
				Build(),
			expect:     "GENERATED_COOKIE",
			transition: affinityTransition{"NONE", 20},
		},
		{
			desc: "http with cookie based affinity and 60s ttl.",
			beConfig: fuzz.NewBackendConfigBuilder("", "backendconfig-1").
				SetSessionAffinity("GENERATED_COOKIE").
				SetAffinityCookieTtlSec(60).
				Build(),
			expect:     "GENERATED_COOKIE",
			ttl:        60,
			transition: affinityTransition{"CLIENT_IP", 30},
		},
	} {
		tc := tc // Capture tc as we are running this in parallel.
		Framework.RunWithSandbox(tc.desc, t, func(t *testing.T, s *e2e.Sandbox) {
			t.Parallel()

			ctx := context.Background()

			backendConfigAnnotation := map[string]string{
				annotations.BetaBackendConfigKey: `{"default":"backendconfig-1"}`,
			}

			bcCRUD := adapter.BackendConfigCRUD{C: Framework.BackendConfigClient}
			tc.beConfig.Namespace = s.Namespace
			if _, err := bcCRUD.Create(tc.beConfig); err != nil {
				t.Fatalf("error creating BackendConfig: %v", err)
			}
			t.Logf("BackendConfig created (%s/%s) ", s.Namespace, tc.beConfig.Name)

			_, err := e2e.CreateEchoService(s, "service-1", backendConfigAnnotation)
			if err != nil {
				t.Fatalf("error creating echo service: %v", err)
			}
			t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1")

			ing := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", "").
				AddPath("test.com", "/", "service-1", v1.ServiceBackendPort{Number: 80}).
				Build()
			crud := adapter.IngressCRUD{C: Framework.Clientset}
			if _, err := crud.Create(ing); err != nil {
				t.Fatalf("error creating Ingress spec: %v", err)
			}
			t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name)

			ing, err = e2e.WaitForIngress(s, ing, nil, nil)
			if err != nil {
				t.Fatalf("error waiting for Ingress to stabilize: %v", err)
			}
			t.Logf("GCLB resources created (%s/%s)", s.Namespace, ing.Name)

			vip := ing.Status.LoadBalancer.Ingress[0].IP
			t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip)

			params := &fuzz.GCLBForVIPParams{VIP: vip, Validators: fuzz.FeatureValidators(features.All)}
			gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, params)
			if err != nil {
				t.Fatalf("Error getting GCP resources for LB with IP = %q: %v", vip, err)
			}

			// Check conformity
			if err := verifyAffinity(t, gclb, s.Namespace, "service-1", tc.expect, tc.ttl); err != nil {
				t.Error(err)
			}

			// Test modifications
			if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
				bc, err := bcCRUD.Get(tc.beConfig.Namespace, tc.beConfig.Name)
				if err != nil {
					return err
				}
				if bc.Spec.SessionAffinity == nil {
					bc.Spec.SessionAffinity = &backendconfig.SessionAffinityConfig{}
				}
				bc.Spec.SessionAffinity.AffinityType = tc.transition.affinity
				bc.Spec.SessionAffinity.AffinityCookieTtlSec = &tc.transition.ttl
				_, err = bcCRUD.Update(bc)
				return err
			}); err != nil {
				t.Errorf("Failed to update BackendConfig affinity settings for %s: %v", t.Name(), err)
			}

			if err := wait.Poll(transitionPollInterval, transitionPollTimeout, func() (bool, error) {
				gclb, err = fuzz.GCLBForVIP(context.Background(), Framework.Cloud, params)
				if err != nil {
					t.Logf("error getting GCP resources for LB with IP = %q: %v", vip, err)
					return false, nil
				}
				if err := verifyAffinity(t, gclb, s.Namespace, "service-1", tc.transition.affinity, tc.transition.ttl); err != nil {
					return false, nil
				}
				return true, nil
			}); err != nil {
				t.Errorf("error waiting for BackendConfig affinity transition propagation to GCLB: %v", err)
			}

			// Wait for GCLB resources to be deleted.
			if err := crud.Delete(s.Namespace, ing.Name); err != nil {
				t.Errorf("Delete(%q) = %v, want nil", ing.Name, err)
			}

			deleteOptions := &fuzz.GCLBDeleteOptions{
				SkipDefaultBackend: true,
			}
			t.Logf("Waiting for GCLB resources to be deleted (%s/%s)", s.Namespace, ing.Name)
			if err := e2e.WaitForGCLBDeletion(ctx, Framework.Cloud, gclb, deleteOptions); err != nil {
				t.Errorf("e2e.WaitForGCLBDeletion(...) = %v, want nil", err)
			}
			t.Logf("GCLB resources deleted (%s/%s)", s.Namespace, ing.Name)
		})
	}
}

// Session affinity test for L7-ILB
// Named without "Affinity" to avoid running this test accidentally
func TestILBSA(t *testing.T) {
	t.Parallel()

	for _, tc := range []struct {
		desc       string
		ttl        int64
		expect     string
		transition affinityTransition
		beConfig   *backendconfig.BackendConfig
	}{
		{
			desc:       "http with no affinity.",
			beConfig:   fuzz.NewBackendConfigBuilder("", "backendconfig-1").Build(),
			expect:     "NONE",
			transition: affinityTransition{"CLIENT_IP", 0},
		},
		{
			desc: "http with cookie based affinity.",
			beConfig: fuzz.NewBackendConfigBuilder("", "backendconfig-1").
				SetSessionAffinity("GENERATED_COOKIE").
				Build(),
			expect:     "GENERATED_COOKIE",
			transition: affinityTransition{"NONE", 0},
		},
		{
			desc: "http with cookie based affinity and 60s ttl.",
			beConfig: fuzz.NewBackendConfigBuilder("", "backendconfig-1").
				SetSessionAffinity("GENERATED_COOKIE").
				SetAffinityCookieTtlSec(60).
				Build(),
			expect:     "GENERATED_COOKIE",
			ttl:        60,
			transition: affinityTransition{"CLIENT_IP", 0},
		},
	} {
		tc := tc // Capture tc as we are running this in parallel.
		Framework.RunWithSandbox(tc.desc, t, func(t *testing.T, s *e2e.Sandbox) {
			t.Parallel()

			ctx := context.Background()

			svcAnnotation := map[string]string{
				annotations.BetaBackendConfigKey: `{"default":"backendconfig-1"}`,
				annotations.NEGAnnotationKey:     negVal.String(),
			}

			bcCRUD := adapter.BackendConfigCRUD{C: Framework.BackendConfigClient}
			tc.beConfig.Namespace = s.Namespace
			if _, err := bcCRUD.Create(tc.beConfig); err != nil {
				t.Fatalf("error creating BackendConfig: %v", err)
			}
			t.Logf("BackendConfig created (%s/%s) ", s.Namespace, tc.beConfig.Name)

			_, err := e2e.CreateEchoService(s, "service-1", svcAnnotation)
			if err != nil {
				t.Fatalf("error creating echo service: %v", err)
			}
			t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1")

			ing := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", "").
				AddPath("test.com", "/", "service-1", v1.ServiceBackendPort{Number: 80}).
				ConfigureForILB().
				Build()
			crud := adapter.IngressCRUD{C: Framework.Clientset}
			if _, err := crud.Create(ing); err != nil {
				t.Fatalf("error creating Ingress spec: %v", err)
			}
			t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name)

			ing, err = e2e.WaitForIngress(s, ing, nil, nil)
			if err != nil {
				t.Fatalf("error waiting for Ingress to stabilize: %v", err)
			}
			t.Logf("GCLB resources created (%s/%s)", s.Namespace, ing.Name)

			vip := ing.Status.LoadBalancer.Ingress[0].IP
			t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip)

			params := &fuzz.GCLBForVIPParams{VIP: vip, Validators: fuzz.FeatureValidators(features.All), Region: Framework.Region}
			gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, params)
			if err != nil {
				t.Fatalf("Error getting GCP resources for LB with IP = %q: %v", vip, err)
			}

			// Check conformity
			if err := verifyAffinity(t, gclb, s.Namespace, "service-1", tc.expect, tc.ttl); err != nil {
				t.Error(err)
			}

			// Test modifications
			if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
				bc, err := bcCRUD.Get(tc.beConfig.Namespace, tc.beConfig.Name)
				if err != nil {
					return err
				}
				if bc.Spec.SessionAffinity == nil {
					bc.Spec.SessionAffinity = &backendconfig.SessionAffinityConfig{}
				}
				bc.Spec.SessionAffinity.AffinityType = tc.transition.affinity
				bc.Spec.SessionAffinity.AffinityCookieTtlSec = &tc.transition.ttl
				_, err = bcCRUD.Update(bc)
				return err
			}); err != nil {
				t.Errorf("Failed to update BackendConfig affinity settings for %s: %v", t.Name(), err)
			}

			if err := wait.Poll(transitionPollInterval, transitionPollTimeout, func() (bool, error) {
				gclb, err = fuzz.GCLBForVIP(context.Background(), Framework.Cloud, params)
				if err != nil {
					t.Logf("error getting GCP resources for LB with IP = %q: %v", vip, err)
					return false, nil
				}
				if err := verifyAffinity(t, gclb, s.Namespace, "service-1", tc.transition.affinity, tc.transition.ttl); err != nil {
					return false, nil
				}
				return true, nil
			}); err != nil {
				t.Errorf("error waiting for BackendConfig affinity transition propagation to GCLB: %v", err)
			}

			// Wait for GCLB resources to be deleted.
			if err := crud.Delete(s.Namespace, ing.Name); err != nil {
				t.Errorf("Delete(%q) = %v, want nil", ing.Name, err)
			}

			deleteOptions := &fuzz.GCLBDeleteOptions{
				SkipDefaultBackend: true,
			}
			t.Logf("Waiting for GCLB resources to be deleted (%s/%s)", s.Namespace, ing.Name)
			if err := e2e.WaitForGCLBDeletion(ctx, Framework.Cloud, gclb, deleteOptions); err != nil {
				t.Errorf("e2e.WaitForGCLBDeletion(...) = %v, want nil", err)
			}
			t.Logf("GCLB resources deleted (%s/%s)", s.Namespace, ing.Name)
		})
	}
}

func verifyAffinity(t *testing.T, gclb *fuzz.GCLB, svcNamespace, svcName string, expect string, ttl int64) error {
	for _, bs := range gclb.BackendService {
		desc := utils.DescriptionFromString(bs.Beta.Description)
		if desc.ServiceName != fmt.Sprintf("%s/%s", svcNamespace, svcName) {
			continue
		}
		if bs.GA.SessionAffinity != expect {
			return fmt.Errorf("verifyAffinity(..., %q, %q, ...) = %s, want %s (SessionAffinity)",
				svcNamespace, svcName, bs.GA.SessionAffinity, expect)
		}
		if bs.GA.AffinityCookieTtlSec != ttl {
			return fmt.Errorf("verifyAffinity(..., %q, %q, ...) = %v, want %v (AffinityCookieTtlSec)",
				svcNamespace, svcName, bs.GA.AffinityCookieTtlSec, ttl)
		}
	}
	return nil
}
<commit_msg>Add backend service name into affinity check error<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"fmt"
	"testing"
	"time"

	v1 "k8s.io/api/networking/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/retry"
	"k8s.io/ingress-gce/pkg/annotations"
	backendconfig "k8s.io/ingress-gce/pkg/apis/backendconfig/v1"
	"k8s.io/ingress-gce/pkg/e2e"
	"k8s.io/ingress-gce/pkg/e2e/adapter"
	"k8s.io/ingress-gce/pkg/fuzz"
	"k8s.io/ingress-gce/pkg/fuzz/features"
	"k8s.io/ingress-gce/pkg/utils"
)

const (
	transitionPollTimeout  = 10 * time.Minute
	transitionPollInterval = 30 * time.Second
)

type affinityTransition struct {
	affinity string
	ttl      int64
}

func TestAffinity(t *testing.T) {
	t.Parallel()

	for _, tc := range []struct {
		desc       string
		ttl        int64
		expect     string
		transition affinityTransition
		beConfig   *backendconfig.BackendConfig
	}{
		{
			desc:       "http with no affinity.",
			beConfig:   fuzz.NewBackendConfigBuilder("", "backendconfig-1").Build(),
			expect:     "NONE",
			transition: affinityTransition{"CLIENT_IP", 10},
		},
		{
			desc: "http with cookie based affinity.",
			beConfig: fuzz.NewBackendConfigBuilder("", "backendconfig-1").
				SetSessionAffinity("GENERATED_COOKIE").
				Build(),
			expect:     "GENERATED_COOKIE",
			transition: affinityTransition{"NONE", 20},
		},
		{
			desc: "http with cookie based affinity and 60s ttl.",
			beConfig: fuzz.NewBackendConfigBuilder("", "backendconfig-1").
				SetSessionAffinity("GENERATED_COOKIE").
				SetAffinityCookieTtlSec(60).
				Build(),
			expect:     "GENERATED_COOKIE",
			ttl:        60,
			transition: affinityTransition{"CLIENT_IP", 30},
		},
	} {
		tc := tc // Capture tc as we are running this in parallel.
		Framework.RunWithSandbox(tc.desc, t, func(t *testing.T, s *e2e.Sandbox) {
			t.Parallel()

			ctx := context.Background()

			backendConfigAnnotation := map[string]string{
				annotations.BetaBackendConfigKey: `{"default":"backendconfig-1"}`,
			}

			bcCRUD := adapter.BackendConfigCRUD{C: Framework.BackendConfigClient}
			tc.beConfig.Namespace = s.Namespace
			if _, err := bcCRUD.Create(tc.beConfig); err != nil {
				t.Fatalf("error creating BackendConfig: %v", err)
			}
			t.Logf("BackendConfig created (%s/%s) ", s.Namespace, tc.beConfig.Name)

			_, err := e2e.CreateEchoService(s, "service-1", backendConfigAnnotation)
			if err != nil {
				t.Fatalf("error creating echo service: %v", err)
			}
			t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1")

			ing := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", "").
				AddPath("test.com", "/", "service-1", v1.ServiceBackendPort{Number: 80}).
				Build()
			crud := adapter.IngressCRUD{C: Framework.Clientset}
			if _, err := crud.Create(ing); err != nil {
				t.Fatalf("error creating Ingress spec: %v", err)
			}
			t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name)

			ing, err = e2e.WaitForIngress(s, ing, nil, nil)
			if err != nil {
				t.Fatalf("error waiting for Ingress to stabilize: %v", err)
			}
			t.Logf("GCLB resources created (%s/%s)", s.Namespace, ing.Name)

			vip := ing.Status.LoadBalancer.Ingress[0].IP
			t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip)

			params := &fuzz.GCLBForVIPParams{VIP: vip, Validators: fuzz.FeatureValidators(features.All)}
			gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, params)
			if err != nil {
				t.Fatalf("Error getting GCP resources for LB with IP = %q: %v", vip, err)
			}

			// Check conformity
			if err := verifyAffinity(t, gclb, s.Namespace, "service-1", tc.expect, tc.ttl); err != nil {
				t.Error(err)
			}

			// Test modifications
			if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
				bc, err := bcCRUD.Get(tc.beConfig.Namespace, tc.beConfig.Name)
				if err != nil {
					return err
				}
				if bc.Spec.SessionAffinity == nil {
					bc.Spec.SessionAffinity = &backendconfig.SessionAffinityConfig{}
				}
				bc.Spec.SessionAffinity.AffinityType = tc.transition.affinity
				bc.Spec.SessionAffinity.AffinityCookieTtlSec = &tc.transition.ttl
				_, err = bcCRUD.Update(bc)
				return err
			}); err != nil {
				t.Errorf("Failed to update BackendConfig affinity settings for %s: %v", t.Name(), err)
			}

			if err := wait.Poll(transitionPollInterval, transitionPollTimeout, func() (bool, error) {
				gclb, err = fuzz.GCLBForVIP(context.Background(), Framework.Cloud, params)
				if err != nil {
					t.Logf("error getting GCP resources for LB with IP = %q: %v", vip, err)
					return false, nil
				}
				if err := verifyAffinity(t, gclb, s.Namespace, "service-1", tc.transition.affinity, tc.transition.ttl); err != nil {
					return false, nil
				}
				return true, nil
			}); err != nil {
				t.Errorf("error waiting for BackendConfig affinity transition propagation to GCLB: %v", err)
			}

			// Wait for GCLB resources to be deleted.
			if err := crud.Delete(s.Namespace, ing.Name); err != nil {
				t.Errorf("Delete(%q) = %v, want nil", ing.Name, err)
			}

			deleteOptions := &fuzz.GCLBDeleteOptions{
				SkipDefaultBackend: true,
			}
			t.Logf("Waiting for GCLB resources to be deleted (%s/%s)", s.Namespace, ing.Name)
			if err := e2e.WaitForGCLBDeletion(ctx, Framework.Cloud, gclb, deleteOptions); err != nil {
				t.Errorf("e2e.WaitForGCLBDeletion(...) = %v, want nil", err)
			}
			t.Logf("GCLB resources deleted (%s/%s)", s.Namespace, ing.Name)
		})
	}
}

// Session affinity test for L7-ILB
// Named without "Affinity" to avoid running this test accidentally
func TestILBSA(t *testing.T) {
	t.Parallel()

	for _, tc := range []struct {
		desc       string
		ttl        int64
		expect     string
		transition affinityTransition
		beConfig   *backendconfig.BackendConfig
	}{
		{
			desc:       "http with no affinity.",
			beConfig:   fuzz.NewBackendConfigBuilder("", "backendconfig-1").Build(),
			expect:     "NONE",
			transition: affinityTransition{"CLIENT_IP", 0},
		},
		{
			desc: "http with cookie based affinity.",
			beConfig: fuzz.NewBackendConfigBuilder("", "backendconfig-1").
				SetSessionAffinity("GENERATED_COOKIE").
				Build(),
			expect:     "GENERATED_COOKIE",
			transition: affinityTransition{"NONE", 0},
		},
		{
			desc: "http with cookie based affinity and 60s ttl.",
			beConfig: fuzz.NewBackendConfigBuilder("", "backendconfig-1").
				SetSessionAffinity("GENERATED_COOKIE").
				SetAffinityCookieTtlSec(60).
				Build(),
			expect:     "GENERATED_COOKIE",
			ttl:        60,
			transition: affinityTransition{"CLIENT_IP", 0},
		},
	} {
		tc := tc // Capture tc as we are running this in parallel.
		Framework.RunWithSandbox(tc.desc, t, func(t *testing.T, s *e2e.Sandbox) {
			t.Parallel()

			ctx := context.Background()

			svcAnnotation := map[string]string{
				annotations.BetaBackendConfigKey: `{"default":"backendconfig-1"}`,
				annotations.NEGAnnotationKey:     negVal.String(),
			}

			bcCRUD := adapter.BackendConfigCRUD{C: Framework.BackendConfigClient}
			tc.beConfig.Namespace = s.Namespace
			if _, err := bcCRUD.Create(tc.beConfig); err != nil {
				t.Fatalf("error creating BackendConfig: %v", err)
			}
			t.Logf("BackendConfig created (%s/%s) ", s.Namespace, tc.beConfig.Name)

			_, err := e2e.CreateEchoService(s, "service-1", svcAnnotation)
			if err != nil {
				t.Fatalf("error creating echo service: %v", err)
			}
			t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1")

			ing := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", "").
				AddPath("test.com", "/", "service-1", v1.ServiceBackendPort{Number: 80}).
				ConfigureForILB().
				Build()
			crud := adapter.IngressCRUD{C: Framework.Clientset}
			if _, err := crud.Create(ing); err != nil {
				t.Fatalf("error creating Ingress spec: %v", err)
			}
			t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name)

			ing, err = e2e.WaitForIngress(s, ing, nil, nil)
			if err != nil {
				t.Fatalf("error waiting for Ingress to stabilize: %v", err)
			}
			t.Logf("GCLB resources created (%s/%s)", s.Namespace, ing.Name)

			vip := ing.Status.LoadBalancer.Ingress[0].IP
			t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip)

			params := &fuzz.GCLBForVIPParams{VIP: vip, Validators: fuzz.FeatureValidators(features.All), Region: Framework.Region}
			gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, params)
			if err != nil {
				t.Fatalf("Error getting GCP resources for LB with IP = %q: %v", vip, err)
			}

			// Check conformity
			if err := verifyAffinity(t, gclb, s.Namespace, "service-1", tc.expect, tc.ttl); err != nil {
				t.Error(err)
			}

			// Test modifications
			if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
				bc, err := bcCRUD.Get(tc.beConfig.Namespace, tc.beConfig.Name)
				if err != nil {
					return err
				}
				if bc.Spec.SessionAffinity == nil {
					bc.Spec.SessionAffinity = &backendconfig.SessionAffinityConfig{}
				}
				bc.Spec.SessionAffinity.AffinityType = tc.transition.affinity
				bc.Spec.SessionAffinity.AffinityCookieTtlSec = &tc.transition.ttl
				_, err = bcCRUD.Update(bc)
				return err
			}); err != nil {
				t.Errorf("Failed to update BackendConfig affinity settings for %s: %v", t.Name(), err)
			}

			if err := wait.Poll(transitionPollInterval, transitionPollTimeout, func() (bool, error) {
				gclb, err = fuzz.GCLBForVIP(context.Background(), Framework.Cloud, params)
				if err != nil {
					t.Logf("error getting GCP resources for LB with IP = %q: %v", vip, err)
					return false, nil
				}
				if err := verifyAffinity(t, gclb, s.Namespace, "service-1", tc.transition.affinity, tc.transition.ttl); err != nil {
					return false, nil
				}
				return true, nil
			}); err != nil {
				t.Errorf("error waiting for BackendConfig affinity transition propagation to GCLB: %v", err)
			}

			// Wait for GCLB resources to be deleted.
			if err := crud.Delete(s.Namespace, ing.Name); err != nil {
				t.Errorf("Delete(%q) = %v, want nil", ing.Name, err)
			}

			deleteOptions := &fuzz.GCLBDeleteOptions{
				SkipDefaultBackend: true,
			}
			t.Logf("Waiting for GCLB resources to be deleted (%s/%s)", s.Namespace, ing.Name)
			if err := e2e.WaitForGCLBDeletion(ctx, Framework.Cloud, gclb, deleteOptions); err != nil {
				t.Errorf("e2e.WaitForGCLBDeletion(...) = %v, want nil", err)
			}
			t.Logf("GCLB resources deleted (%s/%s)", s.Namespace, ing.Name)
		})
	}
}

func verifyAffinity(t *testing.T, gclb *fuzz.GCLB, svcNamespace, svcName string, expect string, ttl int64) error {
	for _, bs := range gclb.BackendService {
		desc := utils.DescriptionFromString(bs.Beta.Description)
		if desc.ServiceName != fmt.Sprintf("%s/%s", svcNamespace, svcName) {
			continue
		}
		if bs.GA.SessionAffinity != expect {
			return fmt.Errorf("Backend Service %s verifyAffinity(..., %q, %q, ...) = %s, want %s (SessionAffinity)", bs.GA.Name,
				svcNamespace, svcName, bs.GA.SessionAffinity, expect)
		}
		if bs.GA.AffinityCookieTtlSec != ttl {
			return fmt.Errorf("Backend Service %s verifyAffinity(..., %q, %q, ...) = %v, want %v (AffinityCookieTtlSec)", bs.GA.Name,
				svcNamespace, svcName, bs.GA.AffinityCookieTtlSec, ttl)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"time"

	"github.com/mesos/mesos-go"
	"github.com/mesos/mesos-go/encoding"
	"github.com/mesos/mesos-go/httpcli"
	"github.com/mesos/mesos-go/scheduler"
	"github.com/mesos/mesos-go/scheduler/calls"
)

func main() {
	cfg := config{
		user:       "foobar",
		name:       "example",
		url:        "http://:5050/api/v1/scheduler",
		codec:      codec{Codec: &encoding.ProtobufCodec},
		timeout:    time.Second,
		checkpoint: true,
	}

	fs := flag.NewFlagSet("example-scheduler", flag.ExitOnError)
	fs.StringVar(&cfg.user, "user", cfg.user, "Framework user to register with the Mesos master")
	fs.StringVar(&cfg.name, "name", cfg.name, "Framework name to register with the Mesos master")
	fs.Var(&cfg.codec, "codec", "Codec to encode/decode scheduler API communications [protobuf, json]")
	fs.StringVar(&cfg.url, "url", cfg.url, "Mesos scheduler API URL")
	fs.DurationVar(&cfg.timeout, "timeout", cfg.timeout, "Mesos scheduler API connection timeout")
	fs.BoolVar(&cfg.checkpoint, "checkpoint", cfg.checkpoint, "Enable/disable framework checkpointing")
	fs.StringVar(&cfg.principal, "principal", cfg.principal, "Framework principal with which to authenticate")
	fs.StringVar(&cfg.hostname, "hostname", cfg.hostname, "Framework hostname that is advertised to the master")
	fs.Var(&cfg.labels, "label", "Framework label, may be specified multiple times")
	fs.Parse(os.Args[1:])

	if err := run(&cfg); err != nil {
		log.Fatal(err)
	}
}

type internalState struct {
	tasksLaunched int
	totalTasks    int
	frameworkID   string
	role          string
	executor      *mesos.ExecutorInfo
	cli           *httpcli.Client
}

func run(cfg *config) error {
	var state internalState
	state.cli = httpcli.New(
		httpcli.URL(cfg.url),
		httpcli.Codec(cfg.codec.Codec),
		httpcli.Do(httpcli.With(httpcli.Timeout(cfg.timeout))),
	)

	frameworkInfo := &mesos.FrameworkInfo{
		User:       cfg.user,
		Name:       cfg.name,
		Checkpoint: &cfg.checkpoint,
	}
	if cfg.principal != "" {
		frameworkInfo.Principal = &cfg.principal
	}
	if cfg.hostname != "" {
		frameworkInfo.Hostname = &cfg.hostname
	}
	if len(cfg.labels) > 0 {
		log.Println("using labels:", cfg.labels)
		frameworkInfo.Labels = &mesos.Labels{Labels: cfg.labels}
	}
	subscribe := calls.Subscribe(true, frameworkInfo)
	registrationTokens := backoffBucket(1*time.Second, 15*time.Second, nil)
	for {
		events, conn, err := state.cli.Do(subscribe, httpcli.Close(true))
		if err == nil {
			err = eventLoop(&state, events, conn)
		}
		if err != nil {
			log.Println(err)
		} else {
			log.Println("disconnected")
		}
		if state.frameworkID != "" {
			subscribe.Subscribe.FrameworkInfo.ID = &mesos.FrameworkID{Value: state.frameworkID}
		}
		<-registrationTokens
		log.Println("reconnecting..")
	}
}

// returns the framework ID received by mesos (if any); callers should check for a
// framework ID regardless of whether error != nil.
func eventLoop(state *internalState, events encoding.Decoder, conn io.Closer) (err error) {
	defer func() {
		if conn != nil {
			conn.Close()
		}
	}()

	state.frameworkID = ""
	callOptions := scheduler.CallOptions{} // should be applied to every outgoing call

	for err == nil {
		var e scheduler.Event
		if err = events.Decode(&e); err != nil {
			if err == io.EOF {
				err = nil
			}
			continue
		}

		switch e.GetType().Enum() {
		case scheduler.Event_FAILURE.Enum():
			log.Println("received a FAILURE event")
			if eid := e.GetFailure().GetExecutorID(); eid != nil {
				// executor failed..
				msg := "executor '" + eid.Value + "' terminated"
				if e.Failure.AgentID != nil {
					msg += "on agent '" + e.Failure.AgentID.Value + "'"
				}
				if e.Failure.Status != nil {
					msg += ", with status=" + strconv.Itoa(int(*e.Failure.Status))
				}
				log.Println(msg)
			} else if e.GetFailure().GetAgentID() != nil {
				// agent failed..
				log.Println("agent '" + e.Failure.AgentID.Value + "' terminated")
			}

		case scheduler.Event_OFFERS.Enum():
			log.Println("received an OFFERS event")
			resourceOffers(state, callOptions.Copy(), e.GetOffers().GetOffers())

		case scheduler.Event_UPDATE.Enum():
			statusUpdate(e.GetUpdate().GetStatus())

		case scheduler.Event_ERROR.Enum():
			// it's recommended that we abort and re-try subscribing; setting
			// err here will cause the event loop to terminate and the connection
			// will be reset.
			err = fmt.Errorf("ERROR: " + e.GetError().GetMessage())

		case scheduler.Event_SUBSCRIBED.Enum():
			log.Println("received a SUBSCRIBED event")
			if state.frameworkID == "" {
				state.frameworkID = e.GetSubscribed().GetFrameworkID().GetValue()
				if state.frameworkID == "" {
					// sanity check
					panic("mesos gave us an empty frameworkID")
				}
				callOptions = append(callOptions, calls.Framework(state.frameworkID))
			}
			// else, ignore subsequently received events like this on the same connection
		default:
			// handle unknown event
		}

		log.Printf("%+v\n", e)
	}
	return err
}

const (
	CPUS_PER_TASK = 1
	MEM_PER_TASK  = 128
)

var wantsTaskResources = mesos.Resources{
	*mesos.BuildResource().Name("cpus").Scalar(CPUS_PER_TASK).Resource,
	*mesos.BuildResource().Name("mem").Scalar(MEM_PER_TASK).Resource,
}

func resourceOffers(state *internalState, callOptions scheduler.CallOptions, offers []mesos.Offer) {
	for i := range offers {
		log.Println("received offer id '" + offers[i].ID.Value + "'")
		var (
			remaining = mesos.Resources(offers[i].Resources)
			tasks     = []mesos.TaskInfo{}
		)
		for state.tasksLaunched < state.totalTasks && remaining.Flatten("", nil).ContainsAll(wantsTaskResources) {
			state.tasksLaunched++
			taskID := state.tasksLaunched

			log.Println("launching task " + strconv.Itoa(taskID) + " using offer " + offers[i].ID.Value)

			task := mesos.TaskInfo{TaskID: mesos.TaskID{Value: strconv.Itoa(taskID)}}
			task.Name = "Task " + task.TaskID.Value
			task.AgentID = offers[i].AgentID
			task.Executor = state.executor
			task.Resources = remaining.Find(wantsTaskResources.Flatten(state.role, nil))

			remaining.Subtract(task.Resources...)
			tasks = append(tasks, task)
		}

		// build Accept call to launch all of the tasks we've assembled
		accept := calls.Accept(
			calls.OfferWithOperations(
				offers[i].ID,
				calls.Launch(tasks...),
			),
		).With(callOptions...)

		// send Accept call to mesos
		_, con, err := state.cli.Do(accept)
		if err != nil {
			log.Println("failed to launch tasks: %+v", err)
		} else {
			con.Close() // no data for these calls
		}
	}
}

func statusUpdate(s mesos.TaskStatus) {
	// TODO..
}

type config struct {
	id         string
	user       string
	name       string
	url        string
	codec      codec
	timeout    time.Duration
	checkpoint bool
	principal  string
	hostname   string
	labels     Labels
}
<commit_msg>flesh out statusUpdate func<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"time"

	"github.com/mesos/mesos-go"
	"github.com/mesos/mesos-go/encoding"
	"github.com/mesos/mesos-go/httpcli"
	"github.com/mesos/mesos-go/scheduler"
	"github.com/mesos/mesos-go/scheduler/calls"
)

func main() {
	cfg := config{
		user:       "foobar",
		name:       "example",
		url:        "http://:5050/api/v1/scheduler",
		codec:      codec{Codec: &encoding.ProtobufCodec},
		timeout:    time.Second,
		checkpoint: true,
	}

	fs := flag.NewFlagSet("example-scheduler", flag.ExitOnError)
	fs.StringVar(&cfg.user, "user", cfg.user, "Framework user to register with the Mesos master")
	fs.StringVar(&cfg.name, "name", cfg.name, "Framework name to register with the Mesos master")
	fs.Var(&cfg.codec, "codec", "Codec to encode/decode scheduler API communications [protobuf, json]")
	fs.StringVar(&cfg.url, "url", cfg.url, "Mesos scheduler API URL")
	fs.DurationVar(&cfg.timeout, "timeout", cfg.timeout, "Mesos scheduler API connection timeout")
	fs.BoolVar(&cfg.checkpoint, "checkpoint", cfg.checkpoint, "Enable/disable framework checkpointing")
	fs.StringVar(&cfg.principal, "principal", cfg.principal, "Framework principal with which to authenticate")
	fs.StringVar(&cfg.hostname, "hostname", cfg.hostname, "Framework hostname that is advertised to the master")
	fs.Var(&cfg.labels, "label", "Framework label, may be specified multiple times")
	fs.Parse(os.Args[1:])

	if err := run(&cfg); err != nil {
		log.Fatal(err)
	}
}

type internalState struct {
	tasksLaunched int
	tasksFinished int
	totalTasks    int
	frameworkID   string
	role          string
	executor      *mesos.ExecutorInfo
	cli           *httpcli.Client
}

func run(cfg *config) error {
	var state internalState
	state.cli = httpcli.New(
		httpcli.URL(cfg.url),
		httpcli.Codec(cfg.codec.Codec),
		httpcli.Do(httpcli.With(httpcli.Timeout(cfg.timeout))),
	)

	frameworkInfo := &mesos.FrameworkInfo{
		User:       cfg.user,
		Name:       cfg.name,
		Checkpoint: &cfg.checkpoint,
	}
	if cfg.principal != "" {
		frameworkInfo.Principal = &cfg.principal
	}
	if cfg.hostname != "" {
		frameworkInfo.Hostname = &cfg.hostname
	}
	if len(cfg.labels) > 0 {
		log.Println("using labels:", cfg.labels)
		frameworkInfo.Labels = &mesos.Labels{Labels: cfg.labels}
	}
	subscribe := calls.Subscribe(true, frameworkInfo)
	registrationTokens := backoffBucket(1*time.Second, 15*time.Second, nil)
	for {
		events, conn, err := state.cli.Do(subscribe, httpcli.Close(true))
		if err == nil {
			err = eventLoop(&state, events, conn)
		}
		if err != nil {
			log.Println(err)
		} else {
			log.Println("disconnected")
		}
		if state.frameworkID != "" {
			subscribe.Subscribe.FrameworkInfo.ID = &mesos.FrameworkID{Value: state.frameworkID}
		}
		<-registrationTokens
		log.Println("reconnecting..")
	}
}

// returns the framework ID received by mesos (if any); callers should check for a
// framework ID regardless of whether error != nil.
func eventLoop(state *internalState, events encoding.Decoder, conn io.Closer) (err error) {
	defer func() {
		if conn != nil {
			conn.Close()
		}
	}()

	state.frameworkID = ""
	callOptions := scheduler.CallOptions{} // should be applied to every outgoing call

	for err == nil {
		var e scheduler.Event
		if err = events.Decode(&e); err != nil {
			if err == io.EOF {
				err = nil
			}
			continue
		}

		switch e.GetType().Enum() {
		case scheduler.Event_FAILURE.Enum():
			log.Println("received a FAILURE event")
			if eid := e.GetFailure().GetExecutorID(); eid != nil {
				// executor failed..
				msg := "executor '" + eid.Value + "' terminated"
				if e.Failure.AgentID != nil {
					msg += "on agent '" + e.Failure.AgentID.Value + "'"
				}
				if e.Failure.Status != nil {
					msg += ", with status=" + strconv.Itoa(int(*e.Failure.Status))
				}
				log.Println(msg)
			} else if e.GetFailure().GetAgentID() != nil {
				// agent failed..
				log.Println("agent '" + e.Failure.AgentID.Value + "' terminated")
			}

		case scheduler.Event_OFFERS.Enum():
			log.Println("received an OFFERS event")
			resourceOffers(state, callOptions.Copy(), e.GetOffers().GetOffers())

		case scheduler.Event_UPDATE.Enum():
			statusUpdate(state, callOptions.Copy(), e.GetUpdate().GetStatus())

		case scheduler.Event_ERROR.Enum():
			// it's recommended that we abort and re-try subscribing; setting
			// err here will cause the event loop to terminate and the connection
			// will be reset.
			err = fmt.Errorf("ERROR: " + e.GetError().GetMessage())

		case scheduler.Event_SUBSCRIBED.Enum():
			log.Println("received a SUBSCRIBED event")
			if state.frameworkID == "" {
				state.frameworkID = e.GetSubscribed().GetFrameworkID().GetValue()
				if state.frameworkID == "" {
					// sanity check
					panic("mesos gave us an empty frameworkID")
				}
				callOptions = append(callOptions, calls.Framework(state.frameworkID))
			}
			// else, ignore subsequently received events like this on the same connection
		default:
			// handle unknown event
		}

		log.Printf("%+v\n", e)
	}
	return err
}

const (
	CPUS_PER_TASK = 1
	MEM_PER_TASK  = 128
)

var wantsTaskResources = mesos.Resources{
	*mesos.BuildResource().Name("cpus").Scalar(CPUS_PER_TASK).Resource,
	*mesos.BuildResource().Name("mem").Scalar(MEM_PER_TASK).Resource,
}

func resourceOffers(state *internalState, callOptions scheduler.CallOptions, offers []mesos.Offer) {
	for i := range offers {
		log.Println("received offer id '" + offers[i].ID.Value + "'")
		var (
			remaining = mesos.Resources(offers[i].Resources)
			tasks     = []mesos.TaskInfo{}
		)
		for state.tasksLaunched < state.totalTasks && remaining.Flatten("", nil).ContainsAll(wantsTaskResources) {
			state.tasksLaunched++
			taskID := state.tasksLaunched

			log.Println("launching task " + strconv.Itoa(taskID) + " using offer " + offers[i].ID.Value)

			task := mesos.TaskInfo{TaskID: mesos.TaskID{Value: strconv.Itoa(taskID)}}
			task.Name = "Task " + task.TaskID.Value
			task.AgentID = offers[i].AgentID
			task.Executor = state.executor
			task.Resources = remaining.Find(wantsTaskResources.Flatten(state.role, nil))

			remaining.Subtract(task.Resources...)
			tasks = append(tasks, task)
		}

		// build Accept call to launch all of the tasks we've assembled
		accept := calls.Accept(
			calls.OfferWithOperations(
				offers[i].ID,
				calls.OpLaunch(tasks...),
			),
		).With(callOptions...)

		// send Accept call to mesos
		_, con, err := state.cli.Do(accept)
		if err != nil {
			log.Println("failed to launch tasks: %+v", err)
		} else {
			con.Close() // no data for these calls
		}
	}
}

func statusUpdate(state *internalState, callOptions scheduler.CallOptions, s mesos.TaskStatus) {
	msg := "Task " + s.TaskID.Value + " is in state " + s.GetState().String()
	if m := s.GetMessage(); m != "" {
		msg += " with message '" + m + "'"
	}
	log.Println(msg)

	if uuid := s.GetUUID(); len(uuid) > 0 {
		ack := calls.Acknowledge(
			s.GetAgentID().GetValue(),
			s.TaskID.Value,
			uuid,
		).With(callOptions...)

		// send Accept call to mesos
		_, con, err := state.cli.Do(ack)
		if err != nil {
			log.Println("failed to ack status update for task: %+v", err)
			return
		} else {
			con.Close() // no data for these calls
		}
	}
	switch st := s.GetState(); st {
	case mesos.TASK_FINISHED:
		state.tasksFinished++
	case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED:
		log.Println("Exiting because task " + s.GetTaskID().Value +
			" is in an unexpected state " + st.String() +
			" with reason " + s.GetReason().String() +
			" from source " + s.GetSource().String() +
			" with message '" + s.GetMessage() + "'")
		os.Exit(1)
	}

	if state.tasksFinished == state.totalTasks {
		log.Println("mission accomplished, terminating")
		os.Exit(0)
	}
}

type config struct {
	id         string
	user       string
	name       string
	url        string
	codec      codec
	timeout    time.Duration
	checkpoint bool
	principal  string
	hostname   string
	labels     Labels
}
<|endoftext|>


<commit_before>// Copyright (c) 2018, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"testing"
	"time"
)

func TestEndToEnd(t *testing.T) {
	url := config.Repos[0].URL
	client := http.Client{Timeout: 2 * time.Second}
	if _, err := client.Get(url); err != nil {
		t.Skipf("skipping since %s is unreachable: %v", url, err)
	}

	dir, err := ioutil.TempDir("", "fdroidcl")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	// Build fdroidcl in the temporary directory.
	fdroidcl := filepath.Join(dir, "fdroidcl")
	if out, err := exec.Command("go", "build",
		"-ldflags=-X main.testBasedir="+dir,
		"-o", fdroidcl).CombinedOutput(); err != nil {
		t.Fatalf("%s", out)
	}

	mustSucceed := func(t *testing.T, want string, args ...string) {
		mustRun(t, true, want, fdroidcl, args...)
	}
	mustFail := func(t *testing.T, want string, args ...string) {
		mustRun(t, false, want, fdroidcl, args...)
	}

	t.Run("Help", func(t *testing.T) {
		mustFail(t, `Usage: fdroidcl`, "-h")
	})
	t.Run("UnknownCommand", func(t *testing.T) {
		mustFail(t, `Unrecognised command`, "unknown")
	})
	t.Run("Version", func(t *testing.T) {
		mustSucceed(t, `^v`, "version")
	})

	t.Run("SearchBeforeUpdate", func(t *testing.T) {
		mustFail(t, `could not open index`, "search")
	})
	t.Run("UpdateFirst", func(t *testing.T) {
		mustSucceed(t, `done`, "update")
	})
	t.Run("UpdateCached", func(t *testing.T) {
		mustSucceed(t, `not modified`, "update")
	})

	t.Run("SearchNoArgs", func(t *testing.T) {
		mustSucceed(t, `F-Droid`, "search")
	})
	t.Run("SearchWithArgs", func(t *testing.T) {
		mustSucceed(t, `F-Droid`, "search", "fdroid.fdroid")
	})
	t.Run("SearchWithArgsNone", func(t *testing.T) {
		mustSucceed(t, `^$`, "search", "nomatches")
	})
	t.Run("SearchOnlyPackageNames", func(t *testing.T) {
		mustSucceed(t, `^[^ ]*$`, "search", "-q", "fdroid.fdroid")
	})
}

func mustRun(t *testing.T, success bool, wantRe, name string, args ...string) {
	cmd := exec.Command(name, args...)
	out, err := cmd.CombinedOutput()
	if success && err != nil {
		t.Fatalf("unexpected error: %v\n%s", err, out)
	} else if !success && err == nil {
		t.Fatalf("expected error, got none\n%s", out)
	}
	if !regexp.MustCompile(wantRe).Match(out) {
		t.Fatalf("output does not match %#q:\n%s", wantRe, out)
	}
}
<commit_msg>cmd/fdroidcl: install, upgrade, uninstall tests<commit_after>// Copyright (c) 2018, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"testing"
	"time"
)

// chosenApp is the app that will be installed and uninstalled on a connected
// device. This one was chosen because it's tiny, requires no permissions, and
// should be compatible with every device.
//
// It also stores no data, so it is fine to uninstall it and the user won't lose
// any data.
const chosenApp = "org.vi_server.red_screen"

func TestEndToEnd(t *testing.T) {
	url := config.Repos[0].URL
	client := http.Client{Timeout: 2 * time.Second}
	if _, err := client.Get(url); err != nil {
		t.Skipf("skipping since %s is unreachable: %v", url, err)
	}

	dir, err := ioutil.TempDir("", "fdroidcl")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	// Build fdroidcl in the temporary directory.
	fdroidcl := filepath.Join(dir, "fdroidcl")
	if out, err := exec.Command("go", "build",
		"-ldflags=-X main.testBasedir="+dir,
		"-o", fdroidcl).CombinedOutput(); err != nil {
		t.Fatalf("%s", out)
	}

	mustSucceed := func(t *testing.T, want string, args ...string) {
		mustRun(t, true, want, fdroidcl, args...)
	}
	mustFail := func(t *testing.T, want string, args ...string) {
		mustRun(t, false, want, fdroidcl, args...)
	}

	t.Run("Help", func(t *testing.T) {
		mustFail(t, `Usage: fdroidcl`, "-h")
	})
	t.Run("UnknownCommand", func(t *testing.T) {
		mustFail(t, `Unrecognised command`, "unknown")
	})
	t.Run("Version", func(t *testing.T) {
		mustSucceed(t, `^v`, "version")
	})

	t.Run("SearchBeforeUpdate", func(t *testing.T) {
		mustFail(t, `could not open index`, "search")
	})
	t.Run("UpdateFirst", func(t *testing.T) {
		mustSucceed(t, `done`, "update")
	})
	t.Run("UpdateCached", func(t *testing.T) {
		mustSucceed(t, `not modified`, "update")
	})

	t.Run("SearchNoArgs", func(t *testing.T) {
		mustSucceed(t, `F-Droid`, "search")
	})
	t.Run("SearchWithArgs", func(t *testing.T) {
		mustSucceed(t, `F-Droid`, "search", "fdroid.fdroid")
	})
	t.Run("SearchWithArgsNone", func(t *testing.T) {
		mustSucceed(t, `^$`, "search", "nomatches")
	})
	t.Run("SearchOnlyPackageNames", func(t *testing.T) {
		mustSucceed(t, `^[^ ]*$`, "search", "-q", "fdroid.fdroid")
	})

	t.Run("ShowOne", func(t *testing.T) {
		mustSucceed(t, `fdroid/fdroidclient`, "show", "org.fdroid.fdroid")
	})
	t.Run("ShowMany", func(t *testing.T) {
		mustSucceed(t, `fdroid/fdroidclient.*fdroid/privileged-extension`,
			"show", "org.fdroid.fdroid", "org.fdroid.fdroid.privileged")
	})

	t.Run("ListCategories", func(t *testing.T) {
		mustSucceed(t, `Development`, "list", "categories")
	})

	out, err := exec.Command(fdroidcl, "devices").CombinedOutput()
	if err != nil {
		t.Fatal(err)
	}
	switch bytes.Count(out, []byte("\n")) {
	case 0:
		t.Log("skipping the device tests as none was found via ADB")
	case 1:
		// continue below
	default:
		t.Log("skipping the device tests as too many were found via ADB")
	}

	// try to uninstall the app first
	exec.Command(fdroidcl, "uninstall", chosenApp).Run()
	t.Run("UninstallMissing", func(t *testing.T) {
		mustFail(t, `not installed$`, "uninstall", chosenApp)
	})
	t.Run("InstallVersioned", func(t *testing.T) {
		mustSucceed(t, `Installing `+regexp.QuoteMeta(chosenApp),
			"install", chosenApp+":1")
	})
	t.Run("Upgrade", func(t *testing.T) {
		mustSucceed(t, `Upgrading `+regexp.QuoteMeta(chosenApp),
			"upgrade", chosenApp)
	})
	t.Run("UpgradeAlreadyInstalled", func(t *testing.T) {
		mustFail(t, `is up to date$`, "upgrade", chosenApp)
	})
	t.Run("UninstallExisting", func(t *testing.T) {
		mustSucceed(t, `Uninstalling `+regexp.QuoteMeta(chosenApp),
			"uninstall", chosenApp)
	})
}

func mustRun(t *testing.T, success bool, wantRe, name string, args ...string) {
	cmd := exec.Command(name, args...)
	out, err := cmd.CombinedOutput()
	if success && err != nil {
		t.Fatalf("unexpected error: %v\n%s", err, out)
	} else if !success && err == nil {
		t.Fatalf("expected error, got none\n%s", out)
	}
	// Let '.' match newlines, and treat the output as a single line.
	wantRe = "(?sm)" + wantRe
	if !regexp.MustCompile(wantRe).Match(out) {
		t.Fatalf("output does not match %#q:\n%s", wantRe, out)
	}
}
<|endoftext|>


<commit_before>// Extensions for Protocol Buffers to create more go like structures.
//
// Copyright (c) 2015, Vastech SA (PTY) LTD. All rights reserved.
// http://github.com/gogo/protobuf/gogoproto
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package main

import (
	"github.com/gogo/protobuf/vanity"
	"github.com/gogo/protobuf/vanity/command"

	_ "github.com/relab/gorums/plugins/gorums"
)

func main() {
	req := command.Read()
	files := req.GetProtoFile()
	files = vanity.FilterFiles(files, vanity.NotInPackageGoogleProtobuf)

	vanity.ForEachFile(files, vanity.TurnOnMarshalerAll)
	vanity.ForEachFile(files, vanity.TurnOnSizerAll)
	vanity.ForEachFile(files, vanity.TurnOnUnmarshalerAll)

	vanity.ForEachFieldInFilesExcludingExtensions(vanity.OnlyProto2(files), vanity.TurnOffNullableForNativeTypesWithoutDefaultsOnly)
	vanity.ForEachFile(files, vanity.TurnOffGoUnrecognizedAll)

	vanity.ForEachFile(files, vanity.TurnOffGoEnumPrefixAll)
	vanity.ForEachFile(files, vanity.TurnOffGoEnumStringerAll)
	vanity.ForEachFile(files, vanity.TurnOnEnumStringerAll)

	vanity.ForEachFile(files, vanity.TurnOnEqualAll)
	//vanity.ForEachFile(files, vanity.TurnOnGoStringAll) // disabled
	vanity.ForEachFile(files, vanity.TurnOffGoStringerAll)
	vanity.ForEachFile(files, vanity.TurnOnStringerAll)

	// extra: not in gogoslick
	vanity.ForEachFile(files, vanity.TurnOnVerboseEqualAll)

	resp := command.Generate(req)
	command.Write(resp)
}
<commit_msg>cmd/protoc-gen-gorums: fix due to gogoprotobuf update<commit_after>// Extensions for Protocol Buffers to create more go like structures.
//
// Copyright (c) 2015, Vastech SA (PTY) LTD. All rights reserved.
// http://github.com/gogo/protobuf/gogoproto
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package main

import (
	"github.com/gogo/protobuf/vanity"
	"github.com/gogo/protobuf/vanity/command"

	_ "github.com/relab/gorums/plugins/gorums"
)

func main() {
	req := command.Read()
	files := req.GetProtoFile()
	files = vanity.FilterFiles(files, vanity.NotGoogleProtobufDescriptorProto)

	vanity.ForEachFile(files, vanity.TurnOnMarshalerAll)
	vanity.ForEachFile(files, vanity.TurnOnSizerAll)
	vanity.ForEachFile(files, vanity.TurnOnUnmarshalerAll)

	vanity.ForEachFieldInFilesExcludingExtensions(vanity.OnlyProto2(files), vanity.TurnOffNullableForNativeTypesWithoutDefaultsOnly)
	vanity.ForEachFile(files, vanity.TurnOffGoUnrecognizedAll)

	vanity.ForEachFile(files, vanity.TurnOffGoEnumPrefixAll)
	vanity.ForEachFile(files, vanity.TurnOffGoEnumStringerAll)
	vanity.ForEachFile(files, vanity.TurnOnEnumStringerAll)

	vanity.ForEachFile(files, vanity.TurnOnEqualAll)
	//vanity.ForEachFile(files, vanity.TurnOnGoStringAll) // disabled
	vanity.ForEachFile(files, vanity.TurnOffGoStringerAll)
	vanity.ForEachFile(files, vanity.TurnOnStringerAll)

	// extra: not in gogoslick
	vanity.ForEachFile(files, vanity.TurnOnVerboseEqualAll)

	resp := command.Generate(req)
	command.Write(resp)
}
<|endoftext|>


<commit_before>// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package main

import (
	"crypto/tls"
	"encoding/binary"
	"fmt"
	io "io"
	"log"
	"net"
	"time"

	"github.com/syncthing/syncthing/lib/protocol"
)

type replicator interface {
	send(key string, addrs []DatabaseAddress, seen int64)
}

// a replicationSender tries to connect to the remote address and provide
// them with a feed of replication updates.
type replicationSender struct {
	dst        string
	cert       tls.Certificate // our certificate
	allowedIDs []protocol.DeviceID
	outbox     chan ReplicationRecord
	stop       chan struct{}
}

func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
	return &replicationSender{
		dst:        dst,
		cert:       cert,
		allowedIDs: allowedIDs,
		outbox:     make(chan ReplicationRecord, replicationOutboxSize),
		stop:       make(chan struct{}),
	}
}

func (s *replicationSender) Serve() {
	// Sleep a little at startup. Peers often restart at the same time, and
	// this avoid the service failing and entering backoff state
	// unnecessarily, while also reducing the reconnect rate to something
	// reasonable by default.
	time.Sleep(2 * time.Second)

	tlsCfg := &tls.Config{
		Certificates:       []tls.Certificate{s.cert},
		MinVersion:         tls.VersionTLS12,
		InsecureSkipVerify: true,
	}

	// Dial the TLS connection.
	conn, err := tls.Dial("tcp", s.dst, tlsCfg)
	if err != nil {
		log.Println("Replication connect:", err)
		return
	}
	defer func() {
		conn.SetWriteDeadline(time.Now().Add(time.Second))
		conn.Close()
	}()

	// Get the other side device ID.
	remoteID, err := deviceID(conn)
	if err != nil {
		log.Println("Replication connect:", err)
		return
	}

	// Verify it's in the set of allowed device IDs.
	if !deviceIDIn(remoteID, s.allowedIDs) {
		log.Println("Replication connect: unexpected device ID:", remoteID)
		return
	}

	// Send records.
	buf := make([]byte, 1024)
	for {
		select {
		case rec := <-s.outbox:
			// Buffer must hold record plus four bytes for size
			size := rec.Size()
			if len(buf) < size+4 {
				buf = make([]byte, size+4)
			}

			// Record comes after the four bytes size
			n, err := rec.MarshalTo(buf[4:])
			if err != nil {
				// odd to get an error here, but we haven't sent anything
				// yet so it's not fatal
				replicationSendsTotal.WithLabelValues("error").Inc()
				log.Println("Replication marshal:", err)
				continue
			}
			binary.BigEndian.PutUint32(buf, uint32(n))

			// Send
			conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
			if _, err := conn.Write(buf[:4+n]); err != nil {
				replicationSendsTotal.WithLabelValues("error").Inc()
				log.Println("Replication write:", err)
				return
			}
			replicationSendsTotal.WithLabelValues("success").Inc()

		case <-s.stop:
			return
		}
	}
}

func (s *replicationSender) Stop() {
	close(s.stop)
}

func (s *replicationSender) String() string {
	return fmt.Sprintf("replicationSender(%q)", s.dst)
}

func (s *replicationSender) send(key string, ps []DatabaseAddress, seen int64) {
	item := ReplicationRecord{
		Key:       key,
		Addresses: ps,
	}

	// The send should never block. The inbox is suitably buffered for at
	// least a few seconds of stalls, which shouldn't happen in practice.
	select {
	case s.outbox <- item:
	default:
		replicationSendsTotal.WithLabelValues("drop").Inc()
	}
}

// a replicationMultiplexer sends to multiple replicators
type replicationMultiplexer []replicator

func (m replicationMultiplexer) send(key string, ps []DatabaseAddress, seen int64) {
	for _, s := range m {
		// each send is nonblocking
		s.send(key, ps, seen)
	}
}

// replicationListener accepts incoming connections and reads replication
// items from them. Incoming items are applied to the KV store.
type replicationListener struct {
	addr       string
	cert       tls.Certificate
	allowedIDs []protocol.DeviceID
	db         database
	stop       chan struct{}
}

func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
	return &replicationListener{
		addr:       addr,
		cert:       cert,
		allowedIDs: allowedIDs,
		db:         db,
		stop:       make(chan struct{}),
	}
}

func (l *replicationListener) Serve() {
	tlsCfg := &tls.Config{
		Certificates:       []tls.Certificate{l.cert},
		ClientAuth:         tls.RequestClientCert,
		MinVersion:         tls.VersionTLS12,
		InsecureSkipVerify: true,
	}

	lst, err := tls.Listen("tcp", l.addr, tlsCfg)
	if err != nil {
		log.Println("Replication listen:", err)
		return
	}
	defer lst.Close()

	for {
		select {
		case <-l.stop:
			return
		default:
		}

		// Accept a connection
		conn, err := lst.Accept()
		if err != nil {
			log.Println("Replication accept:", err)
			return
		}

		// Figure out the other side device ID
		remoteID, err := deviceID(conn.(*tls.Conn))
		if err != nil {
			log.Println("Replication accept:", err)
			conn.SetWriteDeadline(time.Now().Add(time.Second))
			conn.Close()
			continue
		}

		// Verify it is in the set of allowed device IDs
		if !deviceIDIn(remoteID, l.allowedIDs) {
			log.Println("Replication accept: unexpected device ID:", remoteID)
			conn.SetWriteDeadline(time.Now().Add(time.Second))
			conn.Close()
			continue
		}

		go l.handle(conn)
	}
}

func (l *replicationListener) Stop() {
	close(l.stop)
}

func (l *replicationListener) String() string {
	return fmt.Sprintf("replicationListener(%q)", l.addr)
}

func (l *replicationListener) handle(conn net.Conn) {
	defer func() {
		conn.SetWriteDeadline(time.Now().Add(time.Second))
		conn.Close()
	}()

	buf := make([]byte, 1024)

	for {
		select {
		case <-l.stop:
			return
		default:
		}

		conn.SetReadDeadline(time.Now().Add(time.Minute))

		// First four bytes are the size
		if _, err := io.ReadFull(conn, buf[:4]); err != nil {
			log.Println("Replication read size:", err)
			replicationRecvsTotal.WithLabelValues("error").Inc()
			return
		}

		// Read the rest of the record
		size := int(binary.BigEndian.Uint32(buf[:4]))
		if len(buf) < size {
			buf = make([]byte, size)
		}
		if _, err := io.ReadFull(conn, buf[:size]); err != nil {
			log.Println("Replication read record:", err)
			replicationRecvsTotal.WithLabelValues("error").Inc()
			return
		}

		// Unmarshal
		var rec ReplicationRecord
		if err := rec.Unmarshal(buf[:size]); err != nil {
			log.Println("Replication unmarshal:", err)
			replicationRecvsTotal.WithLabelValues("error").Inc()
			continue
		}

		// Store
		l.db.merge(rec.Key, rec.Addresses, rec.Seen)
		replicationRecvsTotal.WithLabelValues("success").Inc()
	}
}

func deviceID(conn *tls.Conn) (protocol.DeviceID, error) {
	// Handshake may not be complete on the server side yet, which we need
	// to get the client certificate.
	if !conn.ConnectionState().HandshakeComplete {
		if err := conn.Handshake(); err != nil {
			return protocol.DeviceID{}, err
		}
	}

	// We expect exactly one certificate.
	certs := conn.ConnectionState().PeerCertificates
	if len(certs) != 1 {
		return protocol.DeviceID{}, fmt.Errorf("unexpected number of certificates (%d != 1)", len(certs))
	}

	return protocol.NewDeviceID(certs[0].Raw), nil
}

func deviceIDIn(id protocol.DeviceID, ids []protocol.DeviceID) bool {
	for _, candidate := range ids {
		if id == candidate {
			return true
		}
	}
	return false
}
<commit_msg>cmd/stdiscosrv: Add replication heartbeats (fixes #5117) (#5120)<commit_after>// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package main

import (
	"crypto/tls"
	"encoding/binary"
	"fmt"
	io "io"
	"log"
	"net"
	"time"

	"github.com/syncthing/syncthing/lib/protocol"
)

const replicationReadTimeout = time.Minute
const replicationHeartbeatInterval = time.Second * 30

type replicator interface {
	send(key string, addrs []DatabaseAddress, seen int64)
}

// a replicationSender tries to connect to the remote address and provide
// them with a feed of replication updates.
type replicationSender struct {
	dst        string
	cert       tls.Certificate // our certificate
	allowedIDs []protocol.DeviceID
	outbox     chan ReplicationRecord
	stop       chan struct{}
}

func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
	return &replicationSender{
		dst:        dst,
		cert:       cert,
		allowedIDs: allowedIDs,
		outbox:     make(chan ReplicationRecord, replicationOutboxSize),
		stop:       make(chan struct{}),
	}
}

func (s *replicationSender) Serve() {
	// Sleep a little at startup. Peers often restart at the same time, and
	// this avoid the service failing and entering backoff state
	// unnecessarily, while also reducing the reconnect rate to something
	// reasonable by default.
	time.Sleep(2 * time.Second)

	tlsCfg := &tls.Config{
		Certificates:       []tls.Certificate{s.cert},
		MinVersion:         tls.VersionTLS12,
		InsecureSkipVerify: true,
	}

	// Dial the TLS connection.
	conn, err := tls.Dial("tcp", s.dst, tlsCfg)
	if err != nil {
		log.Println("Replication connect:", err)
		return
	}
	defer func() {
		conn.SetWriteDeadline(time.Now().Add(time.Second))
		conn.Close()
	}()

	// Get the other side device ID.
	remoteID, err := deviceID(conn)
	if err != nil {
		log.Println("Replication connect:", err)
		return
	}

	// Verify it's in the set of allowed device IDs.
	if !deviceIDIn(remoteID, s.allowedIDs) {
		log.Println("Replication connect: unexpected device ID:", remoteID)
		return
	}

	heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
	defer heartBeatTicker.Stop()

	// Send records.
	buf := make([]byte, 1024)
	for {
		select {
		case <-heartBeatTicker.C:
			if len(s.outbox) > 0 {
				// No need to send heartbeats if there are events/prevrious
				// heartbeats to send, they will keep the connection alive.
				continue
			}
			// Empty replication message is the heartbeat:
			s.outbox <- ReplicationRecord{}

		case rec := <-s.outbox:
			// Buffer must hold record plus four bytes for size
			size := rec.Size()
			if len(buf) < size+4 {
				buf = make([]byte, size+4)
			}

			// Record comes after the four bytes size
			n, err := rec.MarshalTo(buf[4:])
			if err != nil {
				// odd to get an error here, but we haven't sent anything
				// yet so it's not fatal
				replicationSendsTotal.WithLabelValues("error").Inc()
				log.Println("Replication marshal:", err)
				continue
			}
			binary.BigEndian.PutUint32(buf, uint32(n))

			// Send
			conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
			if _, err := conn.Write(buf[:4+n]); err != nil {
				replicationSendsTotal.WithLabelValues("error").Inc()
				log.Println("Replication write:", err)
				// Yes, we are loosing the replication event here.
				return
			}
			replicationSendsTotal.WithLabelValues("success").Inc()

		case <-s.stop:
			return
		}
	}
}

func (s *replicationSender) Stop() {
	close(s.stop)
}

func (s *replicationSender) String() string {
	return fmt.Sprintf("replicationSender(%q)", s.dst)
}

func (s *replicationSender) send(key string, ps []DatabaseAddress, seen int64) {
	item := ReplicationRecord{
		Key:       key,
		Addresses: ps,
	}

	// The send should never block. The inbox is suitably buffered for at
	// least a few seconds of stalls, which shouldn't happen in practice.
	select {
	case s.outbox <- item:
	default:
		replicationSendsTotal.WithLabelValues("drop").Inc()
	}
}

// a replicationMultiplexer sends to multiple replicators
type replicationMultiplexer []replicator

func (m replicationMultiplexer) send(key string, ps []DatabaseAddress, seen int64) {
	for _, s := range m {
		// each send is nonblocking
		s.send(key, ps, seen)
	}
}

// replicationListener accepts incoming connections and reads replication
// items from them. Incoming items are applied to the KV store.
type replicationListener struct {
	addr       string
	cert       tls.Certificate
	allowedIDs []protocol.DeviceID
	db         database
	stop       chan struct{}
}

func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
	return &replicationListener{
		addr:       addr,
		cert:       cert,
		allowedIDs: allowedIDs,
		db:         db,
		stop:       make(chan struct{}),
	}
}

func (l *replicationListener) Serve() {
	tlsCfg := &tls.Config{
		Certificates:       []tls.Certificate{l.cert},
		ClientAuth:         tls.RequestClientCert,
		MinVersion:         tls.VersionTLS12,
		InsecureSkipVerify: true,
	}

	lst, err := tls.Listen("tcp", l.addr, tlsCfg)
	if err != nil {
		log.Println("Replication listen:", err)
		return
	}
	defer lst.Close()

	for {
		select {
		case <-l.stop:
			return
		default:
		}

		// Accept a connection
		conn, err := lst.Accept()
		if err != nil {
			log.Println("Replication accept:", err)
			return
		}

		// Figure out the other side device ID
		remoteID, err := deviceID(conn.(*tls.Conn))
		if err != nil {
			log.Println("Replication accept:", err)
			conn.SetWriteDeadline(time.Now().Add(time.Second))
			conn.Close()
			continue
		}

		// Verify it is in the set of allowed device IDs
		if !deviceIDIn(remoteID, l.allowedIDs) {
			log.Println("Replication accept: unexpected device ID:", remoteID)
			conn.SetWriteDeadline(time.Now().Add(time.Second))
			conn.Close()
			continue
		}

		go l.handle(conn)
	}
}

func (l *replicationListener) Stop() {
	close(l.stop)
}

func (l *replicationListener) String() string {
	return fmt.Sprintf("replicationListener(%q)", l.addr)
}

func (l *replicationListener) handle(conn net.Conn) {
	defer func() {
		conn.SetWriteDeadline(time.Now().Add(time.Second))
		conn.Close()
	}()

	buf := make([]byte, 1024)

	for {
		select {
		case <-l.stop:
			return
		default:
		}

		conn.SetReadDeadline(time.Now().Add(replicationReadTimeout))

		// First four bytes are the size
		if _, err := io.ReadFull(conn, buf[:4]); err != nil {
			log.Println("Replication read size:", err)
			replicationRecvsTotal.WithLabelValues("error").Inc()
			return
		}

		// Read the rest of the record
		size := int(binary.BigEndian.Uint32(buf[:4]))
		if len(buf) < size {
			buf = make([]byte, size)
		}

		if size == 0 {
			// Heartbeat, ignore
			continue
		}

		if _, err := io.ReadFull(conn, buf[:size]); err != nil {
			log.Println("Replication read record:", err)
			replicationRecvsTotal.WithLabelValues("error").Inc()
			return
		}

		// Unmarshal
		var rec ReplicationRecord
		if err := rec.Unmarshal(buf[:size]); err != nil {
			log.Println("Replication unmarshal:", err)
			replicationRecvsTotal.WithLabelValues("error").Inc()
			continue
		}

		// Store
		l.db.merge(rec.Key, rec.Addresses, rec.Seen)
		replicationRecvsTotal.WithLabelValues("success").Inc()
	}
}

func deviceID(conn *tls.Conn) (protocol.DeviceID, error) {
	// Handshake may not be complete on the server side yet, which we need
	// to get the client certificate.
	if !conn.ConnectionState().HandshakeComplete {
		if err := conn.Handshake(); err != nil {
			return protocol.DeviceID{}, err
		}
	}

	// We expect exactly one certificate.
	certs := conn.ConnectionState().PeerCertificates
	if len(certs) != 1 {
		return protocol.DeviceID{}, fmt.Errorf("unexpected number of certificates (%d != 1)", len(certs))
	}

	return protocol.NewDeviceID(certs[0].Raw), nil
}

func deviceIDIn(id protocol.DeviceID, ids []protocol.DeviceID) bool {
	for _, candidate := range ids {
		if id == candidate {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This program manages a zoekt indexing deployment:
// * recycling logs
// * periodically fetching new data.
// * periodically reindexing all git repos.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"math"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/google/zoekt"
	"github.com/google/zoekt/gitindex"
)

const day = time.Hour * 24

func loggedRun(cmd *exec.Cmd) {
	out := &bytes.Buffer{}
	errOut := &bytes.Buffer{}
	cmd.Stdout = out
	cmd.Stderr = errOut

	if err := cmd.Run(); err != nil {
		log.Printf("command %s failed: %v\nOUT: %s\nERR: %s",
			cmd.Args, err, out.String(), errOut.String())
	} else {
		log.Printf("ran successfully %s", cmd.Args)
	}
}

func refresh(repoDir, indexDir, indexConfigFile string, indexFlags []string, fetchInterval time.Duration, cpuFraction float64) {
	// Start with indexing something, so we can start the webserver.
	runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags, cpuFraction)

	t := time.NewTicker(fetchInterval)
	for {
		repos, err := gitindex.FindGitRepos(repoDir)
		if err != nil {
			log.Println(err)
			continue
		}
		if len(repos) == 0 {
			log.Printf("no repos found under %s", repoDir)
		}
		for _, dir := range repos {
			cmd := exec.Command("git", "--git-dir", dir, "fetch", "origin")
			// Prevent prompting
			cmd.Stdin = &bytes.Buffer{}
			loggedRun(cmd)
		}

		runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags, cpuFraction)
		<-t.C
	}
}

func repoIndexCommand(indexDir, repoDir string, configs []RepoHostConfig) {
	for _, cfg := range configs {
		cmd := exec.Command("zoekt-repo-index",
			"-parallelism=1",
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-base_url", cfg.BaseURL,
			"-rev_prefix", cfg.RevPrefix,
			"-max_sub_projects=5",
			"-manifest_repo_url", cfg.ManifestRepoURL,
			"-manifest_rev_prefix", cfg.ManifestRevPrefix)

		cmd.Args = append(cmd.Args, cfg.BranchXMLs...)
		loggedRun(cmd)
	}
}

func repositoryOnRepoHost(repoBaseDir, dir string, repoHosts []RepoHostConfig) bool {
	for _, rh := range repoHosts {
		u, _ := url.Parse(rh.BaseURL)

		if strings.HasPrefix(dir, filepath.Join(repoBaseDir, u.Host)) {
			return true
		}
	}
	return false
}

func runIndexCommand(indexDir, repoDir, indexConfigFile string, indexFlags []string, cpuFraction float64) {
	var indexConfig *IndexConfig
	if indexConfigFile != "" {
		var err error
		indexConfig, err = readIndexConfig(indexConfigFile)
		if err != nil {
			log.Printf("index config: %v", err)
		}

		repoIndexCommand(indexDir, repoDir, indexConfig.RepoHosts)
	}

	repos, err := gitindex.FindGitRepos(repoDir)
	if err != nil {
		log.Println("FindGitRepos", err)
		return
	}

	cpuCount := int(math.Round(float64(runtime.NumCPU()) * cpuFraction))
	if cpuCount < 1 {
		cpuCount = 1
	}
	for _, dir := range repos {
		if indexConfig != nil {
			// Don't want to index the subrepos of a repo
			// host separately.
			if repositoryOnRepoHost(repoDir, dir, indexConfig.RepoHosts) {
				continue
			}

			// TODO(hanwen): we should have similar
			// functionality for avoiding to index a
			// submodule separately too.
		}

		args := []string{
			"-require_ctags",
			fmt.Sprintf("-parallelism=%d", cpuCount),
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-incremental",
		}
		args = append(args, indexFlags...)
		args = append(args, dir)
		cmd := exec.Command("zoekt-git-index", args...)
		loggedRun(cmd)
	}
}

// deleteLogs deletes old logs.
func deleteLogs(logDir string, maxAge time.Duration) {
	tick := time.NewTicker(maxAge / 100)
	for {
		fs, err := filepath.Glob(filepath.Join(logDir, "*"))
		if err != nil {
			log.Fatalf("filepath.Glob(%s): %v", logDir, err)
		}

		threshold := time.Now().Add(-maxAge)
		for _, fn := range fs {
			if fi, err := os.Lstat(fn); err == nil && fi.ModTime().Before(threshold) {
				os.Remove(fn)
			}
		}
		<-tick.C
	}
}

// Delete the shard if its corresponding git repo can't be found.
func deleteIfStale(repoDir string, fn string) error {
	f, err := os.Open(fn)
	if err != nil {
		return nil
	}
	defer f.Close()

	ifile, err := zoekt.NewIndexFile(f)
	if err != nil {
		return nil
	}
	defer ifile.Close()

	repo, _, err := zoekt.ReadMetadata(ifile)
	if err != nil {
		return nil
	}

	_, err = os.Stat(gitindex.Path(repoDir, repo.Name))
	if os.IsNotExist(err) {
		log.Printf("deleting stale shard %s for %q", fn, repo.Name)
		return os.Remove(fn)
	}

	return err
}

func deleteStaleIndexes(indexDir, repoDir string, watchInterval time.Duration) {
	t := time.NewTicker(watchInterval)

	expr := indexDir + "/*"
	for {
		fs, err := filepath.Glob(expr)
		if err != nil {
			log.Printf("Glob(%q): %v", expr, err)
		}

		for _, f := range fs {
			if err := deleteIfStale(repoDir, f); err != nil {
				log.Printf("deleteIfStale(%q): %v", f, err)
			}
		}
		<-t.C
	}
}

func main() {
	maxLogAge := flag.Duration("max_log_age", 3*day, "recycle index logs after this much time")
	fetchInterval := flag.Duration("fetch_interval", time.Hour, "run fetches this often")
	dataDir := flag.String("data_dir",
		filepath.Join(os.Getenv("HOME"), "zoekt-serving"), "directory holding all data.")
	indexDir := flag.String("index_dir", "", "directory holding index shards. Defaults to $data_dir/index/")
	mirrorConfig := flag.String("mirror_config",
		"", "JSON file holding mirror configuration.")
	indexConfig := flag.String("index_config",
		"", "JSON file holding index configuration.")
	mirrorInterval := flag.Duration("mirror_duration", 24*time.Hour, "find and clone new repos at this frequency.")
	cpuFraction := flag.Float64("cpu_fraction", 0.25,
		"use this fraction of the cores for indexing.")
	indexFlagsStr := flag.String("git_index_flags", "", "space separated list of flags passed through to zoekt-git-index (e.g. -git_index_flags='-symbols=false -submodules=false'")
	flag.Parse()

	if *cpuFraction <= 0.0 || *cpuFraction > 1.0 {
		log.Fatal("cpu_fraction must be between 0.0 and 1.0")
	}
	if *dataDir == "" {
		log.Fatal("must set --data_dir")
	}

	var indexFlags []string
	if *indexFlagsStr != "" {
		indexFlags = strings.Split(*indexFlagsStr, " ")
	}

	// Automatically prepend our own path at the front, to minimize
	// required configuration.
	if l, err := os.Readlink("/proc/self/exe"); err == nil {
		os.Setenv("PATH", filepath.Dir(l)+":"+os.Getenv("PATH"))
	}

	logDir := filepath.Join(*dataDir, "logs")
	if *indexDir == "" {
		*indexDir = filepath.Join(*dataDir, "index")
	}
	repoDir := filepath.Join(*dataDir, "repos")
	for _, s := range []string{logDir, *indexDir, repoDir} {
		if _, err := os.Stat(s); err == nil {
			continue
		}

		if err := os.MkdirAll(s, 0755); err != nil {
			log.Fatalf("MkdirAll %s: %v", s, err)
		}
	}

	if strings.HasPrefix(*mirrorConfig, "https://") || strings.HasPrefix(*mirrorConfig, "http://") {
		_, err := readConfigURL(*mirrorConfig)
		if err != nil {
			log.Fatalf("readConfigURL(%s): %v", *mirrorConfig, err)
		} else {
			go periodicMirrorURL(repoDir, *mirrorConfig, *mirrorInterval)
		}
	} else {
		_, err := readConfigFile(*mirrorConfig)
		if err != nil {
			log.Fatalf("readConfig(%s): %v", *mirrorConfig, err)
		} else {
			go periodicMirrorFile(repoDir, *mirrorConfig, *mirrorInterval)
		}
	}

	go deleteLogs(logDir, *maxLogAge)
	go deleteStaleIndexes(*indexDir, repoDir, *fetchInterval)

	refresh(repoDir, *indexDir, *indexConfig, indexFlags, *fetchInterval, *cpuFraction)
}
<commit_msg>cmd/zoekt-indexserver: add context to 'delete stale shard' message<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This program manages a zoekt indexing deployment:
// * recycling logs
// * periodically fetching new data.
// * periodically reindexing all git repos.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"math"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/google/zoekt"
	"github.com/google/zoekt/gitindex"
)

const day = time.Hour * 24

func loggedRun(cmd *exec.Cmd) {
	out := &bytes.Buffer{}
	errOut := &bytes.Buffer{}
	cmd.Stdout = out
	cmd.Stderr = errOut

	if err := cmd.Run(); err != nil {
		log.Printf("command %s failed: %v\nOUT: %s\nERR: %s",
			cmd.Args, err, out.String(), errOut.String())
	} else {
		log.Printf("ran successfully %s", cmd.Args)
	}
}

func refresh(repoDir, indexDir, indexConfigFile string, indexFlags []string, fetchInterval time.Duration, cpuFraction float64) {
	// Start with indexing something, so we can start the webserver.
	runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags, cpuFraction)

	t := time.NewTicker(fetchInterval)
	for {
		repos, err := gitindex.FindGitRepos(repoDir)
		if err != nil {
			log.Println(err)
			continue
		}
		if len(repos) == 0 {
			log.Printf("no repos found under %s", repoDir)
		}
		for _, dir := range repos {
			cmd := exec.Command("git", "--git-dir", dir, "fetch", "origin")
			// Prevent prompting
			cmd.Stdin = &bytes.Buffer{}
			loggedRun(cmd)
		}

		runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags, cpuFraction)
		<-t.C
	}
}

func repoIndexCommand(indexDir, repoDir string, configs []RepoHostConfig) {
	for _, cfg := range configs {
		cmd := exec.Command("zoekt-repo-index",
			"-parallelism=1",
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-base_url", cfg.BaseURL,
			"-rev_prefix", cfg.RevPrefix,
			"-max_sub_projects=5",
			"-manifest_repo_url", cfg.ManifestRepoURL,
			"-manifest_rev_prefix", cfg.ManifestRevPrefix)

		cmd.Args = append(cmd.Args, cfg.BranchXMLs...)
		loggedRun(cmd)
	}
}

func repositoryOnRepoHost(repoBaseDir, dir string, repoHosts []RepoHostConfig) bool {
	for _, rh := range repoHosts {
		u, _ := url.Parse(rh.BaseURL)

		if strings.HasPrefix(dir, filepath.Join(repoBaseDir, u.Host)) {
			return true
		}
	}
	return false
}

func runIndexCommand(indexDir, repoDir, indexConfigFile string, indexFlags []string, cpuFraction float64) {
	var indexConfig *IndexConfig
	if indexConfigFile != "" {
		var err error
		indexConfig, err = readIndexConfig(indexConfigFile)
		if err != nil {
			log.Printf("index config: %v", err)
		}

		repoIndexCommand(indexDir, repoDir, indexConfig.RepoHosts)
	}

	repos, err := gitindex.FindGitRepos(repoDir)
	if err != nil {
		log.Println("FindGitRepos", err)
		return
	}

	cpuCount := int(math.Round(float64(runtime.NumCPU()) * cpuFraction))
	if cpuCount < 1 {
		cpuCount = 1
	}
	for _, dir := range repos {
		if indexConfig != nil {
			// Don't want to index the subrepos of a repo
			// host separately.
			if repositoryOnRepoHost(repoDir, dir, indexConfig.RepoHosts) {
				continue
			}

			// TODO(hanwen): we should have similar
			// functionality for avoiding to index a
			// submodule separately too.
		}

		args := []string{
			"-require_ctags",
			fmt.Sprintf("-parallelism=%d", cpuCount),
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-incremental",
		}
		args = append(args, indexFlags...)
		args = append(args, dir)
		cmd := exec.Command("zoekt-git-index", args...)
		loggedRun(cmd)
	}
}

// deleteLogs deletes old logs.
func deleteLogs(logDir string, maxAge time.Duration) {
	tick := time.NewTicker(maxAge / 100)
	for {
		fs, err := filepath.Glob(filepath.Join(logDir, "*"))
		if err != nil {
			log.Fatalf("filepath.Glob(%s): %v", logDir, err)
		}

		threshold := time.Now().Add(-maxAge)
		for _, fn := range fs {
			if fi, err := os.Lstat(fn); err == nil && fi.ModTime().Before(threshold) {
				os.Remove(fn)
			}
		}
		<-tick.C
	}
}

// Delete the shard if its corresponding git repo can't be found.
func deleteIfStale(repoDir string, fn string) error {
	f, err := os.Open(fn)
	if err != nil {
		return nil
	}
	defer f.Close()

	ifile, err := zoekt.NewIndexFile(f)
	if err != nil {
		return nil
	}
	defer ifile.Close()

	repo, _, err := zoekt.ReadMetadata(ifile)
	if err != nil {
		return nil
	}

	repoDir = gitindex.Path(repoDir, repo.Name)
	_, err = os.Stat(repoDir)
	if os.IsNotExist(err) {
		log.Printf("deleting stale shard %s; dir %q not found", fn, repoDir)
		return os.Remove(fn)
	}

	return err
}

func deleteStaleIndexes(indexDir, repoDir string, watchInterval time.Duration) {
	t := time.NewTicker(watchInterval)

	expr := indexDir + "/*"
	for {
		fs, err := filepath.Glob(expr)
		if err != nil {
			log.Printf("Glob(%q): %v", expr, err)
		}

		for _, f := range fs {
			if err := deleteIfStale(repoDir, f); err != nil {
				log.Printf("deleteIfStale(%q): %v", f, err)
			}
		}
		<-t.C
	}
}

func main() {
	maxLogAge := flag.Duration("max_log_age", 3*day, "recycle index logs after this much time")
	fetchInterval := flag.Duration("fetch_interval", time.Hour, "run fetches this often")
	dataDir := flag.String("data_dir",
		filepath.Join(os.Getenv("HOME"), "zoekt-serving"), "directory holding all data.")
	indexDir := flag.String("index_dir", "", "directory holding index shards. Defaults to $data_dir/index/")
	mirrorConfig := flag.String("mirror_config",
		"", "JSON file holding mirror configuration.")
	indexConfig := flag.String("index_config",
		"", "JSON file holding index configuration.")
	mirrorInterval := flag.Duration("mirror_duration", 24*time.Hour, "find and clone new repos at this frequency.")
	cpuFraction := flag.Float64("cpu_fraction", 0.25,
		"use this fraction of the cores for indexing.")
	indexFlagsStr := flag.String("git_index_flags", "", "space separated list of flags passed through to zoekt-git-index (e.g. -git_index_flags='-symbols=false -submodules=false'")
	flag.Parse()

	if *cpuFraction <= 0.0 || *cpuFraction > 1.0 {
		log.Fatal("cpu_fraction must be between 0.0 and 1.0")
	}
	if *dataDir == "" {
		log.Fatal("must set --data_dir")
	}

	var indexFlags []string
	if *indexFlagsStr != "" {
		indexFlags = strings.Split(*indexFlagsStr, " ")
	}

	// Automatically prepend our own path at the front, to minimize
	// required configuration.
	if l, err := os.Readlink("/proc/self/exe"); err == nil {
		os.Setenv("PATH", filepath.Dir(l)+":"+os.Getenv("PATH"))
	}

	logDir := filepath.Join(*dataDir, "logs")
	if *indexDir == "" {
		*indexDir = filepath.Join(*dataDir, "index")
	}
	repoDir := filepath.Join(*dataDir, "repos")
	for _, s := range []string{logDir, *indexDir, repoDir} {
		if _, err := os.Stat(s); err == nil {
			continue
		}

		if err := os.MkdirAll(s, 0755); err != nil {
			log.Fatalf("MkdirAll %s: %v", s, err)
		}
	}

	if strings.HasPrefix(*mirrorConfig, "https://") || strings.HasPrefix(*mirrorConfig, "http://") {
		_, err := readConfigURL(*mirrorConfig)
		if err != nil {
			log.Fatalf("readConfigURL(%s): %v", *mirrorConfig, err)
		} else {
			go periodicMirrorURL(repoDir, *mirrorConfig, *mirrorInterval)
		}
	} else {
		_, err := readConfigFile(*mirrorConfig)
		if err != nil {
			log.Fatalf("readConfig(%s): %v", *mirrorConfig, err)
		} else {
			go periodicMirrorFile(repoDir, *mirrorConfig, *mirrorInterval)
		}
	}

	go deleteLogs(logDir, *maxLogAge)
	go deleteStaleIndexes(*indexDir, repoDir, *fetchInterval)

	refresh(repoDir, *indexDir, *indexConfig, indexFlags, *fetchInterval, *cpuFraction)
}
<|endoftext|>


<commit_before>package lang

import (
	"testing"
)

func saneQueryTest(query, expected string, t *testing.T, env *LangEnv) {
	val := Eval(query, env)
	if val.Val == nil || val.Err != nil {
		t.Errorf("Expected %s to not be nil. Err: %s", query, val.Err)
	} else {
		if val.Val.Str() != expected {
			t.Errorf("Expected %s to be %s, but was %s", query, expected, val.Val.Str())
		}
	}
}

func malformedQueryTest(query string, t *testing.T, env *LangEnv) {
	val := Eval(query, env)
	if val.Val != nil {
		t.Errorf("Expected value of %s to be nil, was %s", val.Val.Str())
	} else if val.Err == nil {
		t.Errorf("Expected %s to return a non-nil error")
	}
}

func TestBasicLang(t *testing.T) {
	env := new(LangEnv)
	env.Init()

	saneQueryTest("(+ 1 2)", "3", t, env)
	saneQueryTest("(+ 1 2 3 4 5)", "15", t, env)
	saneQueryTest("(+ \"Hello\" \",\" \"World!\")", "\"Hello,World!\"", t, env)
	saneQueryTest("(- 1 2)", "-1", t, env)
	saneQueryTest("(* 1 2)", "2", t, env)
	saneQueryTest("(* 1 2 3 4 5)", "120", t, env)
	saneQueryTest("(/ 1 2)", "0", t, env)

	saneQueryTest("(+ 1.1 2.1)", "3.2", t, env)
	saneQueryTest("(- 1.3 2.1)", "-0.8", t, env)
	saneQueryTest("(* 1.3 2)", "2.6", t, env)
	saneQueryTest("(/ 1.3 2)", "0.65", t, env)

	saneQueryTest("(- 1 (/ 0.5509 0.5698))", "0.033169533169533194", t, env)
	saneQueryTest("(- 1 (/ 6 3))", "-1", t, env)

	saneQueryTest("(defvar x 2.0)", "2", t, env)
	saneQueryTest("(+ x 2.0)", "4", t, env)
	saneQueryTest("(defvar y 1.9)", "1.9", t, env)
	saneQueryTest("(* x y)", "3.8", t, env)
	saneQueryTest("(defvar i 5.0)", "5", t, env)
	saneQueryTest("(defvar i 6.0)", "6", t, env)
	malformedQueryTest("(+ i j)", t, env)
	saneQueryTest("(defvar j 3.1)", "3.1", t, env)
	saneQueryTest("(+ i j)", "9.1", t, env)

	saneQueryTest("(> 3 2)", "true", t, env)
	saneQueryTest("(> 2 3)", "false", t, env)
	saneQueryTest("(< 3 2)", "false", t, env)
	saneQueryTest("(< 2 3)", "true", t, env)
	saneQueryTest("(>= 3 2)", "true", t, env)
	saneQueryTest("(>= 3 3)", "true", t, env)
	saneQueryTest("(<= 3 3)", "true", t, env)
	saneQueryTest("(<= 3 2)", "false", t, env)

	saneQueryTest("(> \"a\" \"b\")", "false", t, env)
	saneQueryTest("(> \"b\" \"a\")", "true", t, env)
	saneQueryTest("(< \"a\" \"b\")", "true", t, env)
	saneQueryTest("(< \"b\" \"a\")", "false", t, env)

	malformedQueryTest(")(", t, env)
	malformedQueryTest(")", t, env)
	malformedQueryTest("(", t, env)
	malformedQueryTest("]]]", t, env)

	malformedQueryTest("(/ 1 0)", t, env)
}
<commit_msg>Added smoke tests<commit_after>package lang

import (
	"fmt"
	"math/rand"
	"testing"
	"time"
)

func saneQueryTest(query string, t *testing.T, env *LangEnv) {
	val := Eval(query, env)
	if val.Val == nil || val.Err != nil {
		t.Errorf("Expected %s to not be nil. Err: %s", query, val.Err)
	}
}

func checkExprResultTest(query, expected string, t *testing.T, env *LangEnv) {
	val := Eval(query, env)
	if val.Val == nil || val.Err != nil {
		t.Errorf("Expected %s to not be nil. Err: %s", query, val.Err)
	} else {
		if val.Val.Str() != expected {
			t.Errorf("Expected %s to be %s, but was %s", query, expected, val.Val.Str())
		}
	}
}

func malformedQueryTest(query string, t *testing.T, env *LangEnv) {
	val := Eval(query, env)
	if val.Val != nil {
		t.Errorf("Expected value of %s to be nil, was %s", val.Val.Str())
	} else if val.Err == nil {
		t.Errorf("Expected %s to return a non-nil error")
	}
}

// Generates an s-expression out of the basic {+, -, *, /} operators.
func genSimpleOperatorsExpr(terminationProb float32, r *rand.Rand, depth int) string {
	p := r.Float32()
	// fmt.Printf("Got %f, terminationProb was %f, depth: %d\n", p, terminationProb, depth)
	if p < terminationProb || depth > 1000 {
		return fmt.Sprintf("%f", r.Float32())
	}
	operands := []string{"+", "-", "*", "/"}
	return fmt.Sprintf("(%s %s %s)", operands[r.Intn(len(operands))], genSimpleOperatorsExpr(terminationProb, r, depth + 1), genSimpleOperatorsExpr(terminationProb, r, depth + 1))
}

// This will run a lot of random smoke tests with simple operators.
func runRandomSmokeTests(t *testing.T, env *LangEnv) {
	seed := time.Now().Unix()
	// fmt.Printf("Using the seed %d. Use it to reproduce test failures.\n", seed)
	r := rand.New(rand.NewSource(seed))
	for i := 0; i < 100; i++ {
		expr := genSimpleOperatorsExpr(0.5, r, 0)
		// fmt.Printf("Generated Test: %s\n", expr)
		saneQueryTest(expr, t, env)
	}
}

func TestBasicLang(t *testing.T) {
	env := new(LangEnv)
	env.Init()

	checkExprResultTest("(+ 1 2)", "3", t, env)
	checkExprResultTest("(+ 1 2 3 4 5)", "15", t, env)
	checkExprResultTest("(+ \"Hello\" \",\" \"World!\")", "\"Hello,World!\"", t, env)
	checkExprResultTest("(- 1 2)", "-1", t, env)
	checkExprResultTest("(* 1 2)", "2", t, env)
	checkExprResultTest("(* 1 2 3 4 5)", "120", t, env)
	checkExprResultTest("(/ 1 2)", "0", t, env)

	checkExprResultTest("(+ 1.1 2.1)", "3.2", t, env)
	checkExprResultTest("(- 1.3 2.1)", "-0.8", t, env)
	checkExprResultTest("(* 1.3 2)", "2.6", t, env)
	checkExprResultTest("(/ 1.3 2)", "0.65", t, env)

	checkExprResultTest("(- 1 (/ 0.5509 0.5698))", "0.033169533169533194", t, env)
	checkExprResultTest("(- 1 (/ 6 3))", "-1", t, env)

	checkExprResultTest("(defvar x 2.0)", "2", t, env)
	checkExprResultTest("(+ x 2.0)", "4", t, env)
	checkExprResultTest("(defvar y 1.9)", "1.9", t, env)
	checkExprResultTest("(* x y)", "3.8", t, env)
	checkExprResultTest("(defvar i 5.0)", "5", t, env)
	checkExprResultTest("(defvar i 6.0)", "6", t, env)
	malformedQueryTest("(+ i j)", t, env)
	checkExprResultTest("(defvar j 3.1)", "3.1", t, env)
	checkExprResultTest("(+ i j)", "9.1", t, env)

	checkExprResultTest("(> 3 2)", "true", t, env)
	checkExprResultTest("(> 2 3)", "false", t, env)
	checkExprResultTest("(< 3 2)", "false", t, env)
	checkExprResultTest("(< 2 3)", "true", t, env)
	checkExprResultTest("(>= 3 2)", "true", t, env)
	checkExprResultTest("(>= 3 3)", "true", t, env)
	checkExprResultTest("(<= 3 3)", "true", t, env)
	checkExprResultTest("(<= 3 2)", "false", t, env)

	checkExprResultTest("(> \"a\" \"b\")", "false", t, env)
	checkExprResultTest("(> \"b\" \"a\")", "true", t, env)
	checkExprResultTest("(< \"a\" \"b\")", "true", t, env)
	checkExprResultTest("(< \"b\" \"a\")", "false", t, env)

	malformedQueryTest(")(", t, env)
	malformedQueryTest(")", t, env)
	malformedQueryTest("(", t, env)
	malformedQueryTest("]]]", t, env)

	malformedQueryTest("(/ 1 0)", t, env)

	runRandomSmokeTests(t, env)
}
<|endoftext|>