<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"reflect"
	"testing"
	"time"

	apps "k8s.io/api/apps/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/diff"
	kubeinformers "k8s.io/client-go/informers"
	k8sfake "k8s.io/client-go/kubernetes/fake"
	core "k8s.io/client-go/testing"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"

	samplecontroller "k8s.io/sample-controller/pkg/apis/samplecontroller/v1alpha1"
	"k8s.io/sample-controller/pkg/client/clientset/versioned/fake"
	informers "k8s.io/sample-controller/pkg/client/informers/externalversions"
)

var (
	alwaysReady        = func() bool { return true }
	noResyncPeriodFunc = func() time.Duration { return 0 }
)

type fixture struct {
	t *testing.T

	client     *fake.Clientset
	kubeclient *k8sfake.Clientset
	// Objects to put in the store.
	fooLister        []*samplecontroller.Foo
	deploymentLister []*apps.Deployment
	// Actions expected to happen on the client.
	kubeactions []core.Action
	actions     []core.Action
	// Objects from here preloaded into NewSimpleFake.
	kubeobjects []runtime.Object
	objects     []runtime.Object
}

func newFixture(t *testing.T) *fixture {
	f := &fixture{}
	f.t = t
	f.objects = []runtime.Object{}
	f.kubeobjects = []runtime.Object{}
	return f
}

func newFoo(name string, replicas *int32) *samplecontroller.Foo {
	return &samplecontroller.Foo{
		TypeMeta: metav1.TypeMeta{APIVersion: samplecontroller.SchemeGroupVersion.String()},
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: metav1.NamespaceDefault,
		},
		Spec: samplecontroller.FooSpec{
			DeploymentName: fmt.Sprintf("%s-deployment", name),
			Replicas:       replicas,
		},
	}
}

func (f *fixture) newController() (*Controller, informers.SharedInformerFactory, kubeinformers.SharedInformerFactory) {
	f.client = fake.NewSimpleClientset(f.objects...)
	f.kubeclient = k8sfake.NewSimpleClientset(f.kubeobjects...)

	i := informers.NewSharedInformerFactory(f.client, noResyncPeriodFunc())
	k8sI := kubeinformers.NewSharedInformerFactory(f.kubeclient, noResyncPeriodFunc())

	c := NewController(f.kubeclient, f.client,
		k8sI.Apps().V1().Deployments(), i.Samplecontroller().V1alpha1().Foos())

	c.foosSynced = alwaysReady
	c.deploymentsSynced = alwaysReady
	c.recorder = &record.FakeRecorder{}

	for _, f := range f.fooLister {
		i.Samplecontroller().V1alpha1().Foos().Informer().GetIndexer().Add(f)
	}

	for _, d := range f.deploymentLister {
		k8sI.Apps().V1().Deployments().Informer().GetIndexer().Add(d)
	}

	return c, i, k8sI
}

func (f *fixture) run(fooName string) {
	f.runController(fooName, true, false)
}

func (f *fixture) runExpectError(fooName string) {
	f.runController(fooName, true, true)
}

func (f *fixture) runController(fooName string, startInformers bool, expectError bool) {
	c, i, k8sI := f.newController()
	if startInformers {
		stopCh := make(chan struct{})
		defer close(stopCh)
		i.Start(stopCh)
		k8sI.Start(stopCh)
	}

	err := c.syncHandler(fooName)
	if !expectError && err != nil {
		f.t.Errorf("error syncing foo: %v", err)
	} else if expectError && err == nil {
		f.t.Error("expected error syncing foo, got nil")
	}

	actions := filterInformerActions(f.client.Actions())
	for i, action := range actions {
		if len(f.actions) < i+1 {
			f.t.Errorf("%d unexpected actions: %+v", len(actions)-len(f.actions), actions[i:])
			break
		}

		expectedAction := f.actions[i]
		checkAction(expectedAction, action, f.t)
	}

	if len(f.actions) > len(actions) {
		f.t.Errorf("%d additional expected actions:%+v", len(f.actions)-len(actions), f.actions[len(actions):])
	}

	k8sActions := filterInformerActions(f.kubeclient.Actions())
	for i, action := range k8sActions {
		if len(f.kubeactions) < i+1 {
			f.t.Errorf("%d unexpected actions: %+v", len(k8sActions)-len(f.kubeactions), k8sActions[i:])
			break
		}

		expectedAction := f.kubeactions[i]
		checkAction(expectedAction, action, f.t)
	}

	if len(f.kubeactions) > len(k8sActions) {
		f.t.Errorf("%d additional expected actions:%+v", len(f.kubeactions)-len(k8sActions), f.kubeactions[len(k8sActions):])
	}
}

// checkAction verifies that expected and actual actions are equal and both have
// same attached resources
func checkAction(expected, actual core.Action, t *testing.T) {
	if !(expected.Matches(actual.GetVerb(), actual.GetResource().Resource) && actual.GetSubresource() == expected.GetSubresource()) {
		t.Errorf("Expected\n\t%#v\ngot\n\t%#v", expected, actual)
		return
	}

	if reflect.TypeOf(actual) != reflect.TypeOf(expected) {
		t.Errorf("Action has wrong type. Expected: %t. Got: %t", expected, actual)
		return
	}

	switch a := actual.(type) {
	case core.CreateAction:
		e, _ := expected.(core.CreateAction)
		expObject := e.GetObject()
		object := a.GetObject()

		if !reflect.DeepEqual(expObject, object) {
			t.Errorf("Action %s %s has wrong object\nDiff:\n %s",
				a.GetVerb(), a.GetResource().Resource, diff.ObjectGoPrintDiff(expObject, object))
		}
	case core.UpdateAction:
		e, _ := expected.(core.UpdateAction)
		expObject := e.GetObject()
		object := a.GetObject()

		if !reflect.DeepEqual(expObject, object) {
			t.Errorf("Action %s %s has wrong object\nDiff:\n %s",
				a.GetVerb(), a.GetResource().Resource, diff.ObjectGoPrintDiff(expObject, object))
		}
	case core.PatchAction:
		e, _ := expected.(core.PatchAction)
		expPatch := e.GetPatch()
		patch := a.GetPatch()

		if !reflect.DeepEqual(expPatch, expPatch) {
			t.Errorf("Action %s %s has wrong patch\nDiff:\n %s",
				a.GetVerb(), a.GetResource().Resource, diff.ObjectGoPrintDiff(expPatch, patch))
		}
	}
}

// filterInformerActions filters list and watch actions for testing resources.
// Since list and watch don't change resource state we can filter it to lower
// nose level in our tests.
func filterInformerActions(actions []core.Action) []core.Action {
	ret := []core.Action{}
	for _, action := range actions {
		if len(action.GetNamespace()) == 0 &&
			(action.Matches("list", "foos") ||
				action.Matches("watch", "foos") ||
				action.Matches("list", "deployments") ||
				action.Matches("watch", "deployments")) {
			continue
		}
		ret = append(ret, action)
	}

	return ret
}

func (f *fixture) expectCreateDeploymentAction(d *apps.Deployment) {
	f.kubeactions = append(f.kubeactions, core.NewCreateAction(schema.GroupVersionResource{Resource: "deployments"}, d.Namespace, d))
}

func (f *fixture) expectUpdateDeploymentAction(d *apps.Deployment) {
	f.kubeactions = append(f.kubeactions, core.NewUpdateAction(schema.GroupVersionResource{Resource: "deployments"}, d.Namespace, d))
}

func (f *fixture) expectUpdateFooStatusAction(foo *samplecontroller.Foo) {
	action := core.NewUpdateAction(schema.GroupVersionResource{Resource: "foos"}, foo.Namespace, foo)
	// TODO: Until #38113 is merged, we can't use Subresource
	//action.Subresource = "status"
	f.actions = append(f.actions, action)
}

func getKey(foo *samplecontroller.Foo, t *testing.T) string {
	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(foo)
	if err != nil {
		t.Errorf("Unexpected error getting key for foo %v: %v", foo.Name, err)
		return ""
	}
	return key
}

func TestCreatesDeployment(t *testing.T) {
	f := newFixture(t)
	foo := newFoo("test", int32Ptr(1))

	f.fooLister = append(f.fooLister, foo)
	f.objects = append(f.objects, foo)

	expDeployment := newDeployment(foo)
	f.expectCreateDeploymentAction(expDeployment)
	f.expectUpdateFooStatusAction(foo)

	f.run(getKey(foo, t))
}

func TestDoNothing(t *testing.T) {
	f := newFixture(t)
	foo := newFoo("test", int32Ptr(1))
	d := newDeployment(foo)

	f.fooLister = append(f.fooLister, foo)
	f.objects = append(f.objects, foo)
	f.deploymentLister = append(f.deploymentLister, d)
	f.kubeobjects = append(f.kubeobjects, d)

	f.expectUpdateFooStatusAction(foo)
	f.run(getKey(foo, t))
}

func TestUpdateDeployment(t *testing.T) {
	f := newFixture(t)
	foo := newFoo("test", int32Ptr(1))
	d := newDeployment(foo)

	// Update replicas
	foo.Spec.Replicas = int32Ptr(2)
	expDeployment := newDeployment(foo)

	f.fooLister = append(f.fooLister, foo)
	f.objects = append(f.objects, foo)
	f.deploymentLister = append(f.deploymentLister, d)
	f.kubeobjects = append(f.kubeobjects, d)

	f.expectUpdateFooStatusAction(foo)
	f.expectUpdateDeploymentAction(expDeployment)
	f.run(getKey(foo, t))
}

func TestNotControlledByUs(t *testing.T) {
	f := newFixture(t)
	foo := newFoo("test", int32Ptr(1))
	d := newDeployment(foo)

	d.ObjectMeta.OwnerReferences = []metav1.OwnerReference{}

	f.fooLister = append(f.fooLister, foo)
	f.objects = append(f.objects, foo)
	f.deploymentLister = append(f.deploymentLister, d)
	f.kubeobjects = append(f.kubeobjects, d)

	f.runExpectError(getKey(foo, t))
}

func int32Ptr(i int32) *int32 { return &i }
<commit_msg>fix patch compare in test<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"reflect"
	"testing"
	"time"

	apps "k8s.io/api/apps/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/diff"
	kubeinformers "k8s.io/client-go/informers"
	k8sfake "k8s.io/client-go/kubernetes/fake"
	core "k8s.io/client-go/testing"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"

	samplecontroller "k8s.io/sample-controller/pkg/apis/samplecontroller/v1alpha1"
	"k8s.io/sample-controller/pkg/client/clientset/versioned/fake"
	informers "k8s.io/sample-controller/pkg/client/informers/externalversions"
)

var (
	alwaysReady        = func() bool { return true }
	noResyncPeriodFunc = func() time.Duration { return 0 }
)

type fixture struct {
	t *testing.T

	client     *fake.Clientset
	kubeclient *k8sfake.Clientset
	// Objects to put in the store.
	fooLister        []*samplecontroller.Foo
	deploymentLister []*apps.Deployment
	// Actions expected to happen on the client.
	kubeactions []core.Action
	actions     []core.Action
	// Objects from here preloaded into NewSimpleFake.
	kubeobjects []runtime.Object
	objects     []runtime.Object
}

func newFixture(t *testing.T) *fixture {
	f := &fixture{}
	f.t = t
	f.objects = []runtime.Object{}
	f.kubeobjects = []runtime.Object{}
	return f
}

func newFoo(name string, replicas *int32) *samplecontroller.Foo {
	return &samplecontroller.Foo{
		TypeMeta: metav1.TypeMeta{APIVersion: samplecontroller.SchemeGroupVersion.String()},
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: metav1.NamespaceDefault,
		},
		Spec: samplecontroller.FooSpec{
			DeploymentName: fmt.Sprintf("%s-deployment", name),
			Replicas:       replicas,
		},
	}
}

func (f *fixture) newController() (*Controller, informers.SharedInformerFactory, kubeinformers.SharedInformerFactory) {
	f.client = fake.NewSimpleClientset(f.objects...)
	f.kubeclient = k8sfake.NewSimpleClientset(f.kubeobjects...)

	i := informers.NewSharedInformerFactory(f.client, noResyncPeriodFunc())
	k8sI := kubeinformers.NewSharedInformerFactory(f.kubeclient, noResyncPeriodFunc())

	c := NewController(f.kubeclient, f.client,
		k8sI.Apps().V1().Deployments(), i.Samplecontroller().V1alpha1().Foos())

	c.foosSynced = alwaysReady
	c.deploymentsSynced = alwaysReady
	c.recorder = &record.FakeRecorder{}

	for _, f := range f.fooLister {
		i.Samplecontroller().V1alpha1().Foos().Informer().GetIndexer().Add(f)
	}

	for _, d := range f.deploymentLister {
		k8sI.Apps().V1().Deployments().Informer().GetIndexer().Add(d)
	}

	return c, i, k8sI
}

func (f *fixture) run(fooName string) {
	f.runController(fooName, true, false)
}

func (f *fixture) runExpectError(fooName string) {
	f.runController(fooName, true, true)
}

func (f *fixture) runController(fooName string, startInformers bool, expectError bool) {
	c, i, k8sI := f.newController()
	if startInformers {
		stopCh := make(chan struct{})
		defer close(stopCh)
		i.Start(stopCh)
		k8sI.Start(stopCh)
	}

	err := c.syncHandler(fooName)
	if !expectError && err != nil {
		f.t.Errorf("error syncing foo: %v", err)
	} else if expectError && err == nil {
		f.t.Error("expected error syncing foo, got nil")
	}

	actions := filterInformerActions(f.client.Actions())
	for i, action := range actions {
		if len(f.actions) < i+1 {
			f.t.Errorf("%d unexpected actions: %+v", len(actions)-len(f.actions), actions[i:])
			break
		}

		expectedAction := f.actions[i]
		checkAction(expectedAction, action, f.t)
	}

	if len(f.actions) > len(actions) {
		f.t.Errorf("%d additional expected actions:%+v", len(f.actions)-len(actions), f.actions[len(actions):])
	}

	k8sActions := filterInformerActions(f.kubeclient.Actions())
	for i, action := range k8sActions {
		if len(f.kubeactions) < i+1 {
			f.t.Errorf("%d unexpected actions: %+v", len(k8sActions)-len(f.kubeactions), k8sActions[i:])
			break
		}

		expectedAction := f.kubeactions[i]
		checkAction(expectedAction, action, f.t)
	}

	if len(f.kubeactions) > len(k8sActions) {
		f.t.Errorf("%d additional expected actions:%+v", len(f.kubeactions)-len(k8sActions), f.kubeactions[len(k8sActions):])
	}
}

// checkAction verifies that expected and actual actions are equal and both have
// same attached resources
func checkAction(expected, actual core.Action, t *testing.T) {
	if !(expected.Matches(actual.GetVerb(), actual.GetResource().Resource) && actual.GetSubresource() == expected.GetSubresource()) {
		t.Errorf("Expected\n\t%#v\ngot\n\t%#v", expected, actual)
		return
	}

	if reflect.TypeOf(actual) != reflect.TypeOf(expected) {
		t.Errorf("Action has wrong type. Expected: %t. Got: %t", expected, actual)
		return
	}

	switch a := actual.(type) {
	case core.CreateAction:
		e, _ := expected.(core.CreateAction)
		expObject := e.GetObject()
		object := a.GetObject()

		if !reflect.DeepEqual(expObject, object) {
			t.Errorf("Action %s %s has wrong object\nDiff:\n %s",
				a.GetVerb(), a.GetResource().Resource, diff.ObjectGoPrintDiff(expObject, object))
		}
	case core.UpdateAction:
		e, _ := expected.(core.UpdateAction)
		expObject := e.GetObject()
		object := a.GetObject()

		if !reflect.DeepEqual(expObject, object) {
			t.Errorf("Action %s %s has wrong object\nDiff:\n %s",
				a.GetVerb(), a.GetResource().Resource, diff.ObjectGoPrintDiff(expObject, object))
		}
	case core.PatchAction:
		e, _ := expected.(core.PatchAction)
		expPatch := e.GetPatch()
		patch := a.GetPatch()

		if !reflect.DeepEqual(expPatch, patch) {
			t.Errorf("Action %s %s has wrong patch\nDiff:\n %s",
				a.GetVerb(), a.GetResource().Resource, diff.ObjectGoPrintDiff(expPatch, patch))
		}
	}
}

// filterInformerActions filters list and watch actions for testing resources.
// Since list and watch don't change resource state we can filter it to lower
// nose level in our tests.
func filterInformerActions(actions []core.Action) []core.Action {
	ret := []core.Action{}
	for _, action := range actions {
		if len(action.GetNamespace()) == 0 &&
			(action.Matches("list", "foos") ||
				action.Matches("watch", "foos") ||
				action.Matches("list", "deployments") ||
				action.Matches("watch", "deployments")) {
			continue
		}
		ret = append(ret, action)
	}

	return ret
}

func (f *fixture) expectCreateDeploymentAction(d *apps.Deployment) {
	f.kubeactions = append(f.kubeactions, core.NewCreateAction(schema.GroupVersionResource{Resource: "deployments"}, d.Namespace, d))
}

func (f *fixture) expectUpdateDeploymentAction(d *apps.Deployment) {
	f.kubeactions = append(f.kubeactions, core.NewUpdateAction(schema.GroupVersionResource{Resource: "deployments"}, d.Namespace, d))
}

func (f *fixture) expectUpdateFooStatusAction(foo *samplecontroller.Foo) {
	action := core.NewUpdateAction(schema.GroupVersionResource{Resource: "foos"}, foo.Namespace, foo)
	// TODO: Until #38113 is merged, we can't use Subresource
	//action.Subresource = "status"
	f.actions = append(f.actions, action)
}

func getKey(foo *samplecontroller.Foo, t *testing.T) string {
	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(foo)
	if err != nil {
		t.Errorf("Unexpected error getting key for foo %v: %v", foo.Name, err)
		return ""
	}
	return key
}

func TestCreatesDeployment(t *testing.T) {
	f := newFixture(t)
	foo := newFoo("test", int32Ptr(1))

	f.fooLister = append(f.fooLister, foo)
	f.objects = append(f.objects, foo)

	expDeployment := newDeployment(foo)
	f.expectCreateDeploymentAction(expDeployment)
	f.expectUpdateFooStatusAction(foo)

	f.run(getKey(foo, t))
}

func TestDoNothing(t *testing.T) {
	f := newFixture(t)
	foo := newFoo("test", int32Ptr(1))
	d := newDeployment(foo)

	f.fooLister = append(f.fooLister, foo)
	f.objects = append(f.objects, foo)
	f.deploymentLister = append(f.deploymentLister, d)
	f.kubeobjects = append(f.kubeobjects, d)

	f.expectUpdateFooStatusAction(foo)
	f.run(getKey(foo, t))
}

func TestUpdateDeployment(t *testing.T) {
	f := newFixture(t)
	foo := newFoo("test", int32Ptr(1))
	d := newDeployment(foo)

	// Update replicas
	foo.Spec.Replicas = int32Ptr(2)
	expDeployment := newDeployment(foo)

	f.fooLister = append(f.fooLister, foo)
	f.objects = append(f.objects, foo)
	f.deploymentLister = append(f.deploymentLister, d)
	f.kubeobjects = append(f.kubeobjects, d)

	f.expectUpdateFooStatusAction(foo)
	f.expectUpdateDeploymentAction(expDeployment)
	f.run(getKey(foo, t))
}

func TestNotControlledByUs(t *testing.T) {
	f := newFixture(t)
	foo := newFoo("test", int32Ptr(1))
	d := newDeployment(foo)

	d.ObjectMeta.OwnerReferences = []metav1.OwnerReference{}

	f.fooLister = append(f.fooLister, foo)
	f.objects = append(f.objects, foo)
	f.deploymentLister = append(f.deploymentLister, d)
	f.kubeobjects = append(f.kubeobjects, d)

	f.runExpectError(getKey(foo, t))
}

func int32Ptr(i int32) *int32 { return &i }
<|endoftext|>


<commit_before>package main_test

import (
	"encoding/hex"
	"io/ioutil"
	"os"
	"strings"

	"github.com/cloudfoundry-incubator/cli-plugin-repo/web"

	"net/url"

	"crypto/sha1"
	"fmt"
	"net/http"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"gopkg.in/yaml.v2"
)

var _ = Describe("Database", func() {
	It("correctly parses the current repo-index.yml", func() {
		var plugins web.PluginsJson

		b, err := ioutil.ReadFile("repo-index.yml")
		Expect(err).NotTo(HaveOccurred())

		err = yaml.Unmarshal(b, &plugins)
		Expect(err).NotTo(HaveOccurred())
	})

	Describe("validations", func() {
		var plugins web.PluginsJson

		BeforeEach(func() {
			b, err := ioutil.ReadFile("repo-index.yml")
			Expect(err).NotTo(HaveOccurred())

			err = yaml.Unmarshal(b, &plugins)
			Expect(err).NotTo(HaveOccurred())
		})

		It("has every binary link over https", func() {
			for _, plugin := range plugins.Plugins {
				for _, binary := range plugin.Binaries {
					url, err := url.Parse(binary.Url)
					Expect(err).NotTo(HaveOccurred())

					Expect(url.Scheme).To(Equal("https"))
				}
			}
		})

		It("has every version parseable by semver", func() {
			for _, plugin := range plugins.Plugins {
				Expect(plugin.Version).To(MatchRegexp(`^\d+\.\d+\.\d+$`), fmt.Sprintf("Plugin '%s' has a non-semver version", plugin.Name))
			}
		})

		It("validates the platforms for every binary", func() {
			for _, plugin := range plugins.Plugins {
				for _, binary := range plugin.Binaries {
					Expect(web.ValidPlatforms).To(
						ContainElement(binary.Platform),
						fmt.Sprintf(
							"Plugin '%s' contains a platform '%s' that is invalid. Please use one of the following: '%s'",
							plugin.Name,
							binary.Platform,
							strings.Join(web.ValidPlatforms, ", "),
						))
				}
			}
		})

		It("requires HTTPS for all downloads", func() {
			for _, plugin := range plugins.Plugins {
				for _, binary := range plugin.Binaries {
					Expect(binary.Url).To(
						MatchRegexp("^https|ftps"),
						fmt.Sprintf(
							"Plugin '%s' links to a Binary's URL '%s' that cannot be downloaded over SSL (begins with https/ftps). Please provide a secure download link to your binaries. If you are unsure how to provide one, try out GitHub Releases: https://help.github.com/articles/creating-releases",
							plugin.Name,
							binary.Url,
						))
				}
			}
		})

		It("every binary download had a matching sha1", func() {
			if os.Getenv("BINARY_VALIDATION") != "true" {
				Skip("Skipping SHA1 binary checking. To enable, set the BINARY_VALIDATION env variable to 'true'")
			}

			fmt.Println("\nRunning Binary Validations, this could take 10+ minutes")

			for _, plugin := range plugins.Plugins {
				for _, binary := range plugin.Binaries {
					resp, err := http.Get(binary.Url)
					Expect(err).NotTo(HaveOccurred())

					defer resp.Body.Close()
					b, err := ioutil.ReadAll(resp.Body)
					Expect(err).NotTo(HaveOccurred())

					s := sha1.Sum(b)
					if hex.EncodeToString(s[:]) != binary.Checksum {
						fmt.Printf("response code: #%d\n", resp.StatusCode)
						fmt.Printf("response body: #%s\n", string(b))
					}
					Expect(hex.EncodeToString(s[:])).To(Equal(binary.Checksum), fmt.Sprintf("Plugin '%s' has an invalid checksum for platform '%s'", plugin.Name, binary.Platform))
				}
			}
		})
	})
})
<commit_msg>add a retry in case we get a 5xx error from github<commit_after>package main_test

import (
	"encoding/hex"
	"io/ioutil"
	"os"
	"strings"

	"github.com/cloudfoundry-incubator/cli-plugin-repo/web"

	"net/url"

	"crypto/sha1"
	"fmt"
	"net/http"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"gopkg.in/yaml.v2"
)

var _ = Describe("Database", func() {
	It("correctly parses the current repo-index.yml", func() {
		var plugins web.PluginsJson

		b, err := ioutil.ReadFile("repo-index.yml")
		Expect(err).NotTo(HaveOccurred())

		err = yaml.Unmarshal(b, &plugins)
		Expect(err).NotTo(HaveOccurred())
	})

	Describe("validations", func() {
		var plugins web.PluginsJson

		BeforeEach(func() {
			b, err := ioutil.ReadFile("repo-index.yml")
			Expect(err).NotTo(HaveOccurred())

			err = yaml.Unmarshal(b, &plugins)
			Expect(err).NotTo(HaveOccurred())
		})

		It("has every binary link over https", func() {
			for _, plugin := range plugins.Plugins {
				for _, binary := range plugin.Binaries {
					url, err := url.Parse(binary.Url)
					Expect(err).NotTo(HaveOccurred())

					Expect(url.Scheme).To(Equal("https"))
				}
			}
		})

		It("has every version parseable by semver", func() {
			for _, plugin := range plugins.Plugins {
				Expect(plugin.Version).To(MatchRegexp(`^\d+\.\d+\.\d+$`), fmt.Sprintf("Plugin '%s' has a non-semver version", plugin.Name))
			}
		})

		It("validates the platforms for every binary", func() {
			for _, plugin := range plugins.Plugins {
				for _, binary := range plugin.Binaries {
					Expect(web.ValidPlatforms).To(
						ContainElement(binary.Platform),
						fmt.Sprintf(
							"Plugin '%s' contains a platform '%s' that is invalid. Please use one of the following: '%s'",
							plugin.Name,
							binary.Platform,
							strings.Join(web.ValidPlatforms, ", "),
						))
				}
			}
		})

		It("requires HTTPS for all downloads", func() {
			for _, plugin := range plugins.Plugins {
				for _, binary := range plugin.Binaries {
					Expect(binary.Url).To(
						MatchRegexp("^https|ftps"),
						fmt.Sprintf(
							"Plugin '%s' links to a Binary's URL '%s' that cannot be downloaded over SSL (begins with https/ftps). Please provide a secure download link to your binaries. If you are unsure how to provide one, try out GitHub Releases: https://help.github.com/articles/creating-releases",
							plugin.Name,
							binary.Url,
						))
				}
			}
		})

		It("every binary download had a matching sha1", func() {
			if os.Getenv("BINARY_VALIDATION") != "true" {
				Skip("Skipping SHA1 binary checking. To enable, set the BINARY_VALIDATION env variable to 'true'")
			}

			fmt.Println("\nRunning Binary Validations, this could take 10+ minutes")

			for _, plugin := range plugins.Plugins {
				for _, binary := range plugin.Binaries {
					var err error
					resp, err := http.Get(binary.Url)
					Expect(err).NotTo(HaveOccurred())

					// If there's a network error, retry exactly once for this plugin binary.
					switch resp.StatusCode {
					case http.StatusInternalServerError,
						http.StatusBadGateway,
						http.StatusServiceUnavailable,
						http.StatusGatewayTimeout:
						resp.Body.Close()
						resp, err = http.Get(binary.Url)
						Expect(err).NotTo(HaveOccurred())
					}

					defer resp.Body.Close()
					b, err := ioutil.ReadAll(resp.Body)
					Expect(err).NotTo(HaveOccurred())

					s := sha1.Sum(b)
					Expect(hex.EncodeToString(s[:])).To(Equal(binary.Checksum), fmt.Sprintf("Plugin '%s' has an invalid checksum for platform '%s'\nResponse Status Code: %d\nResponse Body: %s", plugin.Name, binary.Platform, resp.StatusCode, string(b)))
				}
			}
		})
	})
})
<|endoftext|>


<commit_before>package api

import (
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"

	"github.com/NebulousLabs/Sia/modules"
)

// A Server is essentially a collection of modules and an API server to talk
// to them all.
type Server struct {
	cs       modules.ConsensusSet
	explorer modules.Explorer
	gateway  modules.Gateway
	host     modules.Host
	miner    modules.Miner
	renter   modules.Renter
	tpool    modules.TransactionPool
	wallet   modules.Wallet

	apiServer         *http.Server
	listener          net.Listener
	requiredUserAgent string

	// wg is used to block Close() from returning until Serve() has finished. A
	// WaitGroup is used instead of a chan struct{} so that Close() can be called
	// without necessarily calling Serve() first.
	wg sync.WaitGroup
}

// NewServer creates a new API server from the provided modules.
func NewServer(APIaddr string, requiredUserAgent string, cs modules.ConsensusSet, e modules.Explorer, g modules.Gateway, h modules.Host, m modules.Miner, r modules.Renter, tp modules.TransactionPool, w modules.Wallet) (*Server, error) {
	l, err := net.Listen("tcp", APIaddr)
	if err != nil {
		return nil, err
	}

	srv := &Server{
		cs:       cs,
		explorer: e,
		gateway:  g,
		host:     h,
		miner:    m,
		renter:   r,
		tpool:    tp,
		wallet:   w,

		listener:          l,
		requiredUserAgent: requiredUserAgent,
	}

	// Register API handlers
	srv.initAPI()

	return srv, nil
}

// Serve listens for and handles API calls. It is a blocking function.
func (srv *Server) Serve() error {
	// Block the Close() method until Serve() has finished.
	srv.wg.Add(1)
	defer srv.wg.Done()

	// stop the server if a kill signal is caught
	sigChan := make(chan os.Signal)
	signal.Notify(sigChan, os.Interrupt, os.Kill)
	defer signal.Reset(os.Interrupt, os.Kill)
	go func() {
		<-sigChan
		fmt.Println("\rCaught stop signal, quitting...")
		srv.listener.Close()
	}()

	// The server will run until an error is encountered or the listener is
	// closed, via either the Close method or the signal handling above.
	// Closing the listener will result in the benign error handled below.
	err := srv.apiServer.Serve(srv.listener)
	if err != nil && !strings.HasSuffix(err.Error(), "use of closed network connection") {
		return err
	}
	return nil
}

// Close closes the Server's listener, causing the HTTP server to shut down.
func (srv *Server) Close() error {
	var errStrs []string

	// Close the listener, which will cause Server.Serve() to return.
	if err := srv.listener.Close(); err != nil {
		errStrs = append(errStrs, fmt.Sprintf("listener err: %v", err))
	}

	// Wait for Server.Serve() to exit. We wait so that it's guaranteed that the
	// server has completely closed after Close() returns. This is particularly
	// useful during testing so that we don't exit a test before Serve() finishes.
	srv.wg.Wait()

	// Safely close each module.
	if srv.host != nil {
		if err := srv.host.Close(); err != nil {
			errStrs = append(errStrs, fmt.Sprintf("host err: %v", err))
		}
	}
	// TODO: close renter (which should close hostdb as well)
	if srv.explorer != nil {
		if err := srv.explorer.Close(); err != nil {
			errStrs = append(errStrs, fmt.Sprintf("explorer err: %v", err))
		}
	}
	// TODO: close miner
	if srv.wallet != nil {
		// TODO: close wallet and lock the wallet in the wallet's Close method.
		if srv.wallet.Unlocked() {
			if err := srv.wallet.Lock(); err != nil {
				errStrs = append(errStrs, fmt.Sprintf("wallet err: %v", err))
			}
		}
	}
	// TODO: close transaction pool
	if srv.cs != nil {
		if err := srv.cs.Close(); err != nil {
			errStrs = append(errStrs, fmt.Sprintf("consensus err: %v", err))
		}
	}
	if srv.gateway != nil {
		if err := srv.gateway.Close(); err != nil {
			errStrs = append(errStrs, fmt.Sprintf("gateway err: %v", err))
		}
	}

	if len(errStrs) > 0 {
		return errors.New(strings.Join(errStrs, "\n"))
	}
	return nil
}
<commit_msg>Use a buffered channel for receiving signals<commit_after>package api

import (
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"

	"github.com/NebulousLabs/Sia/modules"
)

// A Server is essentially a collection of modules and an API server to talk
// to them all.
type Server struct {
	cs       modules.ConsensusSet
	explorer modules.Explorer
	gateway  modules.Gateway
	host     modules.Host
	miner    modules.Miner
	renter   modules.Renter
	tpool    modules.TransactionPool
	wallet   modules.Wallet

	apiServer         *http.Server
	listener          net.Listener
	requiredUserAgent string

	// wg is used to block Close() from returning until Serve() has finished. A
	// WaitGroup is used instead of a chan struct{} so that Close() can be called
	// without necessarily calling Serve() first.
	wg sync.WaitGroup
}

// NewServer creates a new API server from the provided modules.
func NewServer(APIaddr string, requiredUserAgent string, cs modules.ConsensusSet, e modules.Explorer, g modules.Gateway, h modules.Host, m modules.Miner, r modules.Renter, tp modules.TransactionPool, w modules.Wallet) (*Server, error) {
	l, err := net.Listen("tcp", APIaddr)
	if err != nil {
		return nil, err
	}

	srv := &Server{
		cs:       cs,
		explorer: e,
		gateway:  g,
		host:     h,
		miner:    m,
		renter:   r,
		tpool:    tp,
		wallet:   w,

		listener:          l,
		requiredUserAgent: requiredUserAgent,
	}

	// Register API handlers
	srv.initAPI()

	return srv, nil
}

// Serve listens for and handles API calls. It is a blocking function.
func (srv *Server) Serve() error {
	// Block the Close() method until Serve() has finished.
	srv.wg.Add(1)
	defer srv.wg.Done()

	// stop the server if a kill signal is caught
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, os.Kill)
	defer signal.Stop(sigChan)
	stop := make(chan struct{})
	defer close(stop)
	go func() {
		select {
		case <-sigChan:
			fmt.Println("\rCaught stop signal, quitting...")
			srv.listener.Close()
		case <-stop:
			// Don't leave a dangling goroutine.
		}
	}()

	// The server will run until an error is encountered or the listener is
	// closed, via either the Close method or the signal handling above.
	// Closing the listener will result in the benign error handled below.
	err := srv.apiServer.Serve(srv.listener)
	if err != nil && !strings.HasSuffix(err.Error(), "use of closed network connection") {
		return err
	}
	return nil
}

// Close closes the Server's listener, causing the HTTP server to shut down.
func (srv *Server) Close() error {
	var errStrs []string

	// Close the listener, which will cause Server.Serve() to return.
	if err := srv.listener.Close(); err != nil {
		errStrs = append(errStrs, fmt.Sprintf("listener err: %v", err))
	}

	// Wait for Server.Serve() to exit. We wait so that it's guaranteed that the
	// server has completely closed after Close() returns. This is particularly
	// useful during testing so that we don't exit a test before Serve() finishes.
	srv.wg.Wait()

	// Safely close each module.
	if srv.host != nil {
		if err := srv.host.Close(); err != nil {
			errStrs = append(errStrs, fmt.Sprintf("host err: %v", err))
		}
	}
	// TODO: close renter (which should close hostdb as well)
	if srv.explorer != nil {
		if err := srv.explorer.Close(); err != nil {
			errStrs = append(errStrs, fmt.Sprintf("explorer err: %v", err))
		}
	}
	// TODO: close miner
	if srv.wallet != nil {
		// TODO: close wallet and lock the wallet in the wallet's Close method.
		if srv.wallet.Unlocked() {
			if err := srv.wallet.Lock(); err != nil {
				errStrs = append(errStrs, fmt.Sprintf("wallet err: %v", err))
			}
		}
	}
	// TODO: close transaction pool
	if srv.cs != nil {
		if err := srv.cs.Close(); err != nil {
			errStrs = append(errStrs, fmt.Sprintf("consensus err: %v", err))
		}
	}
	if srv.gateway != nil {
		if err := srv.gateway.Close(); err != nil {
			errStrs = append(errStrs, fmt.Sprintf("gateway err: %v", err))
		}
	}

	if len(errStrs) > 0 {
		return errors.New(strings.Join(errStrs, "\n"))
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package api

import (
	"fmt"
	"github.com/bmizerany/pat"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/app"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"net"
	"net/http"
)

type TsuruHandler struct {
	path   string
	method string
	h      http.Handler
}

func fatal(err error) {
	log.Fatal(err.Error())
}

var tsuruHandlerList []TsuruHandler

//RegisterHandler inserts a handler on a list of handlers
func RegisterHandler(h TsuruHandler) {
	tsuruHandlerList = append(tsuruHandlerList, h)
}

// RunServer starts Tsuru API server. The dry parameter indicates whether the
// server should run in dry mode, not starting the HTTP listener (for testing
// purposes).
func RunServer(dry bool) {
	log.Init()
	connString, err := config.GetString("database:url")
	if err != nil {
		connString = db.DefaultDatabaseURL
	}
	dbName, err := config.GetString("database:name")
	if err != nil {
		dbName = db.DefaultDatabaseName
	}
	fmt.Printf("Using the database %q from the server %q.\n\n", dbName, connString)

	for _, handler := range tsuruHandlerList {
		m.Add(handler.method, handler.path, handler.h)
	}

	m := pat.New()

	m.Get("/schema/app", authorizationRequiredHandler(appSchema))
	m.Get("/schema/service", authorizationRequiredHandler(serviceSchema))
	m.Get("/schema/services", authorizationRequiredHandler(servicesSchema))

	m.Get("/services/instances", authorizationRequiredHandler(serviceInstances))
	m.Get("/services/instances/:name", authorizationRequiredHandler(serviceInstance))
	m.Del("/services/instances/:name", authorizationRequiredHandler(removeServiceInstance))
	m.Post("/services/instances", authorizationRequiredHandler(createServiceInstance))
	m.Put("/services/instances/:instance/:app", authorizationRequiredHandler(bindServiceInstance))
	m.Del("/services/instances/:instance/:app", authorizationRequiredHandler(unbindServiceInstance))
	m.Get("/services/instances/:instance/status", authorizationRequiredHandler(serviceInstanceStatus))

	m.Get("/services", authorizationRequiredHandler(serviceList))
	m.Post("/services", authorizationRequiredHandler(serviceCreate))
	m.Put("/services", authorizationRequiredHandler(serviceUpdate))
	m.Del("/services/:name", authorizationRequiredHandler(serviceDelete))
	m.Get("/services/:name", authorizationRequiredHandler(serviceInfo))
	m.Get("/services/:name/plans", authorizationRequiredHandler(servicePlans))
	m.Get("/services/:name/doc", authorizationRequiredHandler(serviceDoc))
	m.Put("/services/:name/doc", authorizationRequiredHandler(serviceAddDoc))
	m.Put("/services/:service/:team", authorizationRequiredHandler(grantServiceAccess))
	m.Del("/services/:service/:team", authorizationRequiredHandler(revokeServiceAccess))

	m.Del("/apps/:app", authorizationRequiredHandler(appDelete))
	m.Get("/apps/:app", authorizationRequiredHandler(appInfo))
	m.Post("/apps/:app/cname", authorizationRequiredHandler(setCName))
	m.Del("/apps/:app/cname", authorizationRequiredHandler(unsetCName))
	m.Post("/apps/:app/run", authorizationRequiredHandler(runCommand))
	m.Get("/apps/:app/restart", authorizationRequiredHandler(restart))
	m.Get("/apps/:app/start", authorizationRequiredHandler(start))
	m.Get("/apps/:app/env", authorizationRequiredHandler(getEnv))
	m.Post("/apps/:app/env", authorizationRequiredHandler(setEnv))
	m.Del("/apps/:app/env", authorizationRequiredHandler(unsetEnv))
	m.Get("/apps", authorizationRequiredHandler(appList))
	m.Post("/apps", authorizationRequiredHandler(createApp))
	m.Put("/apps/:app/units", authorizationRequiredHandler(addUnits))
	m.Del("/apps/:app/units", authorizationRequiredHandler(removeUnits))
	m.Put("/apps/:app/:team", authorizationRequiredHandler(grantAppAccess))
	m.Del("/apps/:app/:team", authorizationRequiredHandler(revokeAppAccess))
	m.Get("/apps/:app/log", authorizationRequiredHandler(appLog))
	m.Post("/apps/:app/log", authorizationRequiredHandler(addLog))

	m.Get("/deploys", adminRequiredHandler(deploysList))

	m.Get("/platforms", authorizationRequiredHandler(platformList))

	// These handlers don't use :app on purpose. Using :app means that only
	// the token generate for the given app is valid, but these handlers
	// use a token generated for Gandalf.
	m.Get("/apps/:appname/available", authorizationRequiredHandler(appIsAvailable))
	m.Post("/apps/:appname/repository/clone", authorizationRequiredHandler(deploy))
	m.Post("/apps/:appname/deploy", authorizationRequiredHandler(deploy))

	if registrationEnabled, _ := config.GetBool("auth:user-registration"); registrationEnabled {
		m.Post("/users", Handler(createUser))
	}

	m.Post("/users/:email/password", Handler(resetPassword))
	m.Post("/users/:email/tokens", Handler(login))
	m.Del("/users/tokens", authorizationRequiredHandler(logout))
	m.Put("/users/password", authorizationRequiredHandler(changePassword))
	m.Del("/users", authorizationRequiredHandler(removeUser))
	m.Get("/users/:email/keys", authorizationRequiredHandler(listKeys))
	m.Post("/users/keys", authorizationRequiredHandler(addKeyToUser))
	m.Del("/users/keys", authorizationRequiredHandler(removeKeyFromUser))

	m.Post("/tokens", adminRequiredHandler(generateAppToken))

	m.Del("/logs", adminRequiredHandler(logRemove))

	m.Get("/teams", authorizationRequiredHandler(teamList))
	m.Post("/teams", authorizationRequiredHandler(createTeam))
	m.Get("/teams/:name", authorizationRequiredHandler(getTeam))
	m.Del("/teams/:name", authorizationRequiredHandler(removeTeam))
	m.Put("/teams/:team/:user", authorizationRequiredHandler(addUserToTeam))
	m.Del("/teams/:team/:user", authorizationRequiredHandler(removeUserFromTeam))

	m.Get("/healers", authorizationRequiredHandler(healers))
	m.Get("/healers/:healer", authorizationRequiredHandler(healer))

	m.Put("/swap", authorizationRequiredHandler(swap))

	m.Get("/healthcheck/", http.HandlerFunc(healthcheck))

	if !dry {
		provisioner, err := getProvisioner()
		if err != nil {
			fmt.Printf("Warning: configuration didn't declare a provisioner, using default provisioner.\n")
		}
		app.Provisioner, err = provision.Get(provisioner)
		if err != nil {
			fatal(err)
		}
		fmt.Printf("Using %q provisioner.\n\n", provisioner)

		listen, err := config.GetString("listen")
		if err != nil {
			fatal(err)
		}
		tls, _ := config.GetBool("use-tls")
		if tls {
			certFile, err := config.GetString("tls:cert-file")
			if err != nil {
				fatal(err)
			}
			keyFile, err := config.GetString("tls:key-file")
			if err != nil {
				fatal(err)
			}
			fmt.Printf("tsuru HTTP/TLS server listening at %s...\n", listen)
			fatal(http.ListenAndServeTLS(listen, certFile, keyFile, m))
		} else {
			listener, err := net.Listen("tcp", listen)
			if err != nil {
				fatal(err)
			}
			fmt.Printf("tsuru HTTP server listening at %s...\n", listen)
			http.Handle("/", m)
			fatal(http.Serve(listener, nil))
		}
	}
}
<commit_msg>fixed RegisterHandler<commit_after>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package api

import (
	"fmt"
	"github.com/bmizerany/pat"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/app"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"net"
	"net/http"
)

type TsuruHandler struct {
	method string
	path   string
	h      http.Handler
}

func fatal(err error) {
	log.Fatal(err.Error())
}

var tsuruHandlerList []TsuruHandler

//RegisterHandler inserts a handler on a list of handlers
func RegisterHandler(path string, method string, h http.Handler) {
	var th TsuruHandler
	th.path = path
	th.method = method
	th.h = h
	tsuruHandlerList = append(tsuruHandlerList, th)
}

// RunServer starts Tsuru API server. The dry parameter indicates whether the
// server should run in dry mode, not starting the HTTP listener (for testing
// purposes).
func RunServer(dry bool) {
	log.Init()
	connString, err := config.GetString("database:url")
	if err != nil {
		connString = db.DefaultDatabaseURL
	}
	dbName, err := config.GetString("database:name")
	if err != nil {
		dbName = db.DefaultDatabaseName
	}
	fmt.Printf("Using the database %q from the server %q.\n\n", dbName, connString)

	m := pat.New()

	for _, handler := range tsuruHandlerList {
		m.Add(handler.method, handler.path, handler.h)
	}

	m.Get("/schema/app", authorizationRequiredHandler(appSchema))
	m.Get("/schema/service", authorizationRequiredHandler(serviceSchema))
	m.Get("/schema/services", authorizationRequiredHandler(servicesSchema))

	m.Get("/services/instances", authorizationRequiredHandler(serviceInstances))
	m.Get("/services/instances/:name", authorizationRequiredHandler(serviceInstance))
	m.Del("/services/instances/:name", authorizationRequiredHandler(removeServiceInstance))
	m.Post("/services/instances", authorizationRequiredHandler(createServiceInstance))
	m.Put("/services/instances/:instance/:app", authorizationRequiredHandler(bindServiceInstance))
	m.Del("/services/instances/:instance/:app", authorizationRequiredHandler(unbindServiceInstance))
	m.Get("/services/instances/:instance/status", authorizationRequiredHandler(serviceInstanceStatus))

	m.Get("/services", authorizationRequiredHandler(serviceList))
	m.Post("/services", authorizationRequiredHandler(serviceCreate))
	m.Put("/services", authorizationRequiredHandler(serviceUpdate))
	m.Del("/services/:name", authorizationRequiredHandler(serviceDelete))
	m.Get("/services/:name", authorizationRequiredHandler(serviceInfo))
	m.Get("/services/:name/plans", authorizationRequiredHandler(servicePlans))
	m.Get("/services/:name/doc", authorizationRequiredHandler(serviceDoc))
	m.Put("/services/:name/doc", authorizationRequiredHandler(serviceAddDoc))
	m.Put("/services/:service/:team", authorizationRequiredHandler(grantServiceAccess))
	m.Del("/services/:service/:team", authorizationRequiredHandler(revokeServiceAccess))

	m.Del("/apps/:app", authorizationRequiredHandler(appDelete))
	m.Get("/apps/:app", authorizationRequiredHandler(appInfo))
	m.Post("/apps/:app/cname", authorizationRequiredHandler(setCName))
	m.Del("/apps/:app/cname", authorizationRequiredHandler(unsetCName))
	m.Post("/apps/:app/run", authorizationRequiredHandler(runCommand))
	m.Get("/apps/:app/restart", authorizationRequiredHandler(restart))
	m.Get("/apps/:app/start", authorizationRequiredHandler(start))
	m.Get("/apps/:app/env", authorizationRequiredHandler(getEnv))
	m.Post("/apps/:app/env", authorizationRequiredHandler(setEnv))
	m.Del("/apps/:app/env", authorizationRequiredHandler(unsetEnv))
	m.Get("/apps", authorizationRequiredHandler(appList))
	m.Post("/apps", authorizationRequiredHandler(createApp))
	m.Put("/apps/:app/units", authorizationRequiredHandler(addUnits))
	m.Del("/apps/:app/units", authorizationRequiredHandler(removeUnits))
	m.Put("/apps/:app/:team", authorizationRequiredHandler(grantAppAccess))
	m.Del("/apps/:app/:team", authorizationRequiredHandler(revokeAppAccess))
	m.Get("/apps/:app/log", authorizationRequiredHandler(appLog))
	m.Post("/apps/:app/log", authorizationRequiredHandler(addLog))

	m.Get("/deploys", adminRequiredHandler(deploysList))

	m.Get("/platforms", authorizationRequiredHandler(platformList))

	// These handlers don't use :app on purpose. Using :app means that only
	// the token generate for the given app is valid, but these handlers
	// use a token generated for Gandalf.
	m.Get("/apps/:appname/available", authorizationRequiredHandler(appIsAvailable))
	m.Post("/apps/:appname/repository/clone", authorizationRequiredHandler(deploy))
	m.Post("/apps/:appname/deploy", authorizationRequiredHandler(deploy))

	if registrationEnabled, _ := config.GetBool("auth:user-registration"); registrationEnabled {
		m.Post("/users", Handler(createUser))
	}

	m.Post("/users/:email/password", Handler(resetPassword))
	m.Post("/users/:email/tokens", Handler(login))
	m.Del("/users/tokens", authorizationRequiredHandler(logout))
	m.Put("/users/password", authorizationRequiredHandler(changePassword))
	m.Del("/users", authorizationRequiredHandler(removeUser))
	m.Get("/users/:email/keys", authorizationRequiredHandler(listKeys))
	m.Post("/users/keys", authorizationRequiredHandler(addKeyToUser))
	m.Del("/users/keys", authorizationRequiredHandler(removeKeyFromUser))

	m.Post("/tokens", adminRequiredHandler(generateAppToken))

	m.Del("/logs", adminRequiredHandler(logRemove))

	m.Get("/teams", authorizationRequiredHandler(teamList))
	m.Post("/teams", authorizationRequiredHandler(createTeam))
	m.Get("/teams/:name", authorizationRequiredHandler(getTeam))
	m.Del("/teams/:name", authorizationRequiredHandler(removeTeam))
	m.Put("/teams/:team/:user", authorizationRequiredHandler(addUserToTeam))
	m.Del("/teams/:team/:user", authorizationRequiredHandler(removeUserFromTeam))

	m.Get("/healers", authorizationRequiredHandler(healers))
	m.Get("/healers/:healer", authorizationRequiredHandler(healer))

	m.Put("/swap", authorizationRequiredHandler(swap))

	m.Get("/healthcheck/", http.HandlerFunc(healthcheck))

	if !dry {
		provisioner, err := getProvisioner()
		if err != nil {
			fmt.Printf("Warning: configuration didn't declare a provisioner, using default provisioner.\n")
		}
		app.Provisioner, err = provision.Get(provisioner)
		if err != nil {
			fatal(err)
		}
		fmt.Printf("Using %q provisioner.\n\n", provisioner)

		listen, err := config.GetString("listen")
		if err != nil {
			fatal(err)
		}
		tls, _ := config.GetBool("use-tls")
		if tls {
			certFile, err := config.GetString("tls:cert-file")
			if err != nil {
				fatal(err)
			}
			keyFile, err := config.GetString("tls:key-file")
			if err != nil {
				fatal(err)
			}
			fmt.Printf("tsuru HTTP/TLS server listening at %s...\n", listen)
			fatal(http.ListenAndServeTLS(listen, certFile, keyFile, m))
		} else {
			listener, err := net.Listen("tcp", listen)
			if err != nil {
				fatal(err)
			}
			fmt.Printf("tsuru HTTP server listening at %s...\n", listen)
			http.Handle("/", m)
			fatal(http.Serve(listener, nil))
		}
	}
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
)

// A Server is essentially a collection of modules and an API server to talk
// to them all.
type Server struct {
	cs       modules.ConsensusSet
	explorer modules.Explorer
	gateway  modules.Gateway
	host     modules.Host
	miner    modules.Miner
	renter   modules.Renter
	tpool    modules.TransactionPool
	wallet   modules.Wallet

	apiServer         *http.Server
	listener          net.Listener
	requiredUserAgent string

	// wg is used to block Close() from returning until Serve() has finished. A
	// WaitGroup is used instead of a chan struct{} so that Close() can be called
	// without necessarily calling Serve() first.
	wg sync.WaitGroup
}

// NewServer creates a new API server from the provided modules.
func NewServer(APIaddr string, requiredUserAgent string, cs modules.ConsensusSet, e modules.Explorer, g modules.Gateway, h modules.Host, m modules.Miner, r modules.Renter, tp modules.TransactionPool, w modules.Wallet) (*Server, error) {
	l, err := net.Listen("tcp", APIaddr)
	if err != nil {
		return nil, err
	}

	srv := &Server{
		cs:       cs,
		explorer: e,
		gateway:  g,
		host:     h,
		miner:    m,
		renter:   r,
		tpool:    tp,
		wallet:   w,

		listener:          l,
		requiredUserAgent: requiredUserAgent,
	}

	// Register API handlers
	srv.initAPI()

	return srv, nil
}

// Serve listens for and handles API calls. It is a blocking function.
func (srv *Server) Serve() error {
	// Block the Close() method until Serve() has finished.
	srv.wg.Add(1)
	defer srv.wg.Done()

	// stop the server if a kill signal is caught
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, os.Kill)
	defer signal.Stop(sigChan)
	stop := make(chan struct{})
	defer close(stop)
	go func() {
		select {
		case <-sigChan:
			fmt.Println("\rCaught stop signal, quitting...")
			srv.Close()
		case <-stop:
			// Don't leave a dangling goroutine.
		}
	}()

	// The server will run until an error is encountered or the listener is
	// closed, via either the Close method or the signal handling above.
	// Closing the listener will result in the benign error handled below.
	err := srv.apiServer.Serve(srv.listener)
	if err != nil && !strings.HasSuffix(err.Error(), "use of closed network connection") {
		return err
	}
	return nil
}

// Close closes the Server's listener, causing the HTTP server to shut down.
func (srv *Server) Close() error {
	var errs []error

	// Close the listener, which will cause Server.Serve() to return.
	if err := srv.listener.Close(); err != nil {
		errs = append(errs, fmt.Errorf("listener.Close failed: %v", err))
	}

	// Wait for Server.Serve() to exit. We wait so that it's guaranteed that the
	// server has completely closed after Close() returns. This is particularly
	// useful during testing so that we don't exit a test before Serve() finishes.
	srv.wg.Wait()

	// Safely close each module.
	if srv.host != nil {
		if err := srv.host.Close(); err != nil {
			errs = append(errs, fmt.Errorf("host.Close failed: %v", err))
		}
	}
	// TODO: close renter (which should close hostdb as well)
	if srv.explorer != nil {
		if err := srv.explorer.Close(); err != nil {
			errs = append(errs, fmt.Errorf("explorer.Close failed: %v", err))
		}
	}
	if srv.miner != nil {
		if err := srv.miner.Close(); err != nil {
			errs = append(errs, fmt.Errorf("miner.Close failed: %v", err))
		}
	}
	if srv.wallet != nil {
		// TODO: close wallet and lock the wallet in the wallet's Close method.
		if srv.wallet.Unlocked() {
			if err := srv.wallet.Lock(); err != nil {
				errs = append(errs, fmt.Errorf("wallet.Lock failed: %v", err))
			}
		}
	}
	// TODO: close transaction pool
	if srv.cs != nil {
		if err := srv.cs.Close(); err != nil {
			errs = append(errs, fmt.Errorf("consensusset.Close failed: %v", err))
		}
	}
	if srv.gateway != nil {
		if err := srv.gateway.Close(); err != nil {
			errs = append(errs, fmt.Errorf("gateway.Close failed: %v", err))
		}
	}

	return build.JoinErrors(errs, "\n")
}
<commit_msg>DRY server shutdown<commit_after>package api

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
)

// A Server is essentially a collection of modules and an API server to talk
// to them all.
type Server struct {
	cs       modules.ConsensusSet
	explorer modules.Explorer
	gateway  modules.Gateway
	host     modules.Host
	miner    modules.Miner
	renter   modules.Renter
	tpool    modules.TransactionPool
	wallet   modules.Wallet

	apiServer         *http.Server
	listener          net.Listener
	requiredUserAgent string

	// wg is used to block Close() from returning until Serve() has finished. A
	// WaitGroup is used instead of a chan struct{} so that Close() can be called
	// without necessarily calling Serve() first.
	wg sync.WaitGroup
}

// NewServer creates a new API server from the provided modules.
func NewServer(APIaddr string, requiredUserAgent string, cs modules.ConsensusSet, e modules.Explorer, g modules.Gateway, h modules.Host, m modules.Miner, r modules.Renter, tp modules.TransactionPool, w modules.Wallet) (*Server, error) {
	l, err := net.Listen("tcp", APIaddr)
	if err != nil {
		return nil, err
	}

	srv := &Server{
		cs:       cs,
		explorer: e,
		gateway:  g,
		host:     h,
		miner:    m,
		renter:   r,
		tpool:    tp,
		wallet:   w,

		listener:          l,
		requiredUserAgent: requiredUserAgent,
	}

	// Register API handlers
	srv.initAPI()

	return srv, nil
}

// Serve listens for and handles API calls. It is a blocking function.
func (srv *Server) Serve() error {
	// Block the Close() method until Serve() has finished.
	srv.wg.Add(1)
	defer srv.wg.Done()

	// stop the server if a kill signal is caught
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, os.Kill)
	defer signal.Stop(sigChan)
	stop := make(chan struct{})
	defer close(stop)
	go func() {
		select {
		case <-sigChan:
			fmt.Println("\rCaught stop signal, quitting...")
			srv.Close()
		case <-stop:
			// Don't leave a dangling goroutine.
		}
	}()

	// The server will run until an error is encountered or the listener is
	// closed, via either the Close method or the signal handling above.
	// Closing the listener will result in the benign error handled below.
	err := srv.apiServer.Serve(srv.listener)
	if err != nil && !strings.HasSuffix(err.Error(), "use of closed network connection") {
		return err
	}
	return nil
}

// Close closes the Server's listener, causing the HTTP server to shut down.
func (srv *Server) Close() error {
	var errs []error

	// Close the listener, which will cause Server.Serve() to return.
	if err := srv.listener.Close(); err != nil {
		errs = append(errs, fmt.Errorf("listener.Close failed: %v", err))
	}

	// Wait for Server.Serve() to exit. We wait so that it's guaranteed that the
	// server has completely closed after Close() returns. This is particularly
	// useful during testing so that we don't exit a test before Serve() finishes.
	srv.wg.Wait()

	// Safely close each module.
	// TODO: close renter
	// TODO: close transaction pool

	// wallet has special closing mechanics
	if srv.wallet != nil {
		// TODO: close wallet and lock the wallet in the wallet's Close method.
		if srv.wallet.Unlocked() {
			if err := srv.wallet.Lock(); err != nil {
				errs = append(errs, fmt.Errorf("wallet.Lock failed: %v", err))
			}
		}
	}

	mods := []struct {
		name string
		c    io.Closer
	}{
		{"host", srv.host},
		{"explorer", srv.explorer},
		{"miner", srv.miner},
		{"consensus", srv.cs},
		{"gateway", srv.gateway},
	}
	for _, mod := range mods {
		if mod.c != nil {
			if err := mod.c.Close(); err != nil {
				errs = append(errs, fmt.Errorf("%v.Close failed: %v", mod.name, err))
			}
		}
	}

	return build.JoinErrors(errs, "\n")
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	_ "github.com/bmizerany/pq"
	"github.com/gorilla/mux"
	"log"
	"net/http"
)

var db *sql.DB

func main() {
	log.Print("starting ChicagoWorksforYou.com API server")

	// open database
	db, err := sql.Open("postgres", "dbname=cwfy sslmode=disable")
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}
	defer db.Close()

        log.Print("db is", db)

	router := mux.NewRouter()
	router.HandleFunc("/health_check", HealthCheckHandler)
	router.HandleFunc("/services.json", ServicesHandler)
	http.ListenAndServe(":4000", router)
}

// type ServicesCount struct {
//      Count        int
//      Service_code string
//      Service_name string
// }

func ServicesHandler(response http.ResponseWriter, request *http.Request) {
	// return counts of requests, grouped by service name

        // var services []ServicesCount

        log.Print("db is now", db)

        rows, err := db.Query("SELECT COUNT(*), service_code, service_name FROM service_requests WHERE duplicate IS NULL GROUP BY service_code,service_name;")

	if err != nil {
		log.Fatal("error fetching data for ServicesHandler", err)
	}

	for rows.Next() {
	        var count int
	        var service_code, service_name string

		if err := rows.Scan(&count, &service_code, &service_name); err != nil {
		        log.Fatal("error reading row", err)
		}
		log.Print(count, service_code, service_name)
	}

        // jsn, _ := json.Marshal(services)
        // response.Write()
}

func HealthCheckHandler(response http.ResponseWriter, request *http.Request) {
	response.Header().Add("Content-type", "application/json")
	health_check := map[string]string{}
	health_check["database"] = "dbconn" // FIXME: meaningful db information
	health_check["sr_count"] = "123"    // FIXME: meaningful count
	jsn, _ := json.Marshal(health_check)
	response.Write(jsn)
}
<commit_msg>working API endpoint for fetching counts for each service<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	_ "github.com/bmizerany/pq"
	"github.com/gorilla/mux"
	"log"
	"net/http"
)

func main() {
	log.Print("starting ChicagoWorksforYou.com API server")

	router := mux.NewRouter()
	router.HandleFunc("/health_check", HealthCheckHandler)
	router.HandleFunc("/services.json", ServicesHandler)
	http.ListenAndServe(":4000", router)
}

func ServicesHandler(response http.ResponseWriter, request *http.Request) {
	// return counts of requests, grouped by service name
	//
	// Sample output:
	//
	// [
	//   {
	//     "Count": 1139,
	//     "Service_code": "4fd3b167e750846744000005",
	//     "Service_name": "Graffiti Removal"
	//   },
	//   {
	//     "Count": 25,
	//     "Service_code": "4fd6e4ece750840569000019",
	//     "Service_name": "Restaurant Complaint"
	//   },
	//
	//  ... snip ...
	//
	// ]

	type ServicesCount struct {
		Count        int
		Service_code string
		Service_name string
	}

	var services []ServicesCount

	// open database
	db, err := sql.Open("postgres", "dbname=cwfy sslmode=disable")
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}
	defer db.Close()

	rows, err := db.Query("SELECT COUNT(*), service_code, service_name FROM service_requests WHERE duplicate IS NULL GROUP BY service_code,service_name;")

	if err != nil {
		log.Fatal("error fetching data for ServicesHandler", err)
	}

	for rows.Next() {
		var count int
		var service_code, service_name string

		if err := rows.Scan(&count, &service_code, &service_name); err != nil {
			log.Fatal("error reading row", err)
		}

		row := ServicesCount{Count: count, Service_code: service_code, Service_name: service_name}
		services = append(services, row)
	}

	jsn, _ := json.MarshalIndent(services, "", "  ")
	response.Write(jsn)
}

func HealthCheckHandler(response http.ResponseWriter, request *http.Request) {
	response.Header().Add("Content-type", "application/json")
	health_check := map[string]string{}
	health_check["database"] = "dbconn" // FIXME: meaningful db information
	health_check["sr_count"] = "123"    // FIXME: meaningful count
	jsn, _ := json.Marshal(health_check)
	response.Write(jsn)
}
<|endoftext|>


<commit_before>package body

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"sync"

	"github.com/AlexanderChen1989/xrest"

	"golang.org/x/net/context"
)

type bodyPlug struct {
	pool *sync.Pool
	next xrest.Handler
}

type readCloser struct {
	io.ReadCloser
	bp  *bodyPlug
	buf *bytes.Buffer
}

func newBodyPlug() *bodyPlug {
	return &bodyPlug{
		pool: &sync.Pool{
			New: func() interface{} {
				return bytes.NewBuffer(nil)
			},
		},
	}
}

var DefaultPlug = newBodyPlug()

var DecodeJSON = DefaultPlug.DecodeJSON

func (rc *readCloser) Close() error {
	rc.bp.pool.Put(rc.buf)

	return rc.ReadCloser.Close()
}

var ErrPlugNotPlugged = errors.New("DecodeJSON not plugged.")

func (bp *bodyPlug) DecodeJSON(ctx context.Context, r *http.Request, v interface{}) error {
	// fetch a buf from pool
	data, ok := ctx.Value(&ctxBodyKey).([]byte)

	if !ok {
		return ErrPlugNotPlugged
	}

	return json.Unmarshal(data, v)
}

var ctxBodyKey uint8

func (bp *bodyPlug) Plug(h xrest.Handler) xrest.Handler {
	bp.next = h
	return bp
}

func FetchBodyFromCtx(ctx context.Context) ([]byte, bool) {
	body, ok := ctx.Value(&ctxBodyKey).([]byte)
	return body, ok
}

func (bp *bodyPlug) ServeHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	if _, ok := FetchBodyFromCtx(ctx); !ok {
		buf := bp.pool.Get().(*bytes.Buffer)
		buf.Reset()
		if _, err := io.Copy(buf, r.Body); err != nil {
			bp.pool.Put(buf)
		}
		ctx = context.WithValue(ctx, &ctxBodyKey, buf.Bytes())
		// reconstruct http.Request.Body
		rc := &readCloser{
			ReadCloser: r.Body,
			bp:         bp,
			buf:        buf,
		}
		r.Body = rc
	}

	bp.next.ServeHTTP(ctx, w, r)
}
<commit_msg>update<commit_after>package body

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"sync"

	"github.com/AlexanderChen1989/xrest"

	"golang.org/x/net/context"
)

type bodyPlug struct {
	pool *sync.Pool
	next xrest.Handler
}

type readCloser struct {
	io.ReadCloser
	bp  *bodyPlug
	buf *bytes.Buffer
}

func newBodyPlug() *bodyPlug {
	return &bodyPlug{
		pool: &sync.Pool{
			New: func() interface{} {
				return bytes.NewBuffer(nil)
			},
		},
	}
}

var DefaultPlug = newBodyPlug()

var DecodeJSON = DefaultPlug.DecodeJSON

func (rc *readCloser) Close() error {
	rc.bp.pool.Put(rc.buf)

	return rc.ReadCloser.Close()
}

var ErrPlugNotPlugged = errors.New("DecodeJSON not plugged.")

func (bp *bodyPlug) Body(ctx context.Context) ([]byte, error) {
	data, ok := ctx.Value(&ctxBodyKey).([]byte)

	if !ok {
		return nil, ErrPlugNotPlugged
	}

	return data, nil
}

func (bp *bodyPlug) DecodeJSON(ctx context.Context, r *http.Request, v interface{}) error {
	// fetch a buf from pool
	data, ok := ctx.Value(&ctxBodyKey).([]byte)

	if !ok {
		return ErrPlugNotPlugged
	}

	return json.Unmarshal(data, v)
}

var ctxBodyKey uint8

func (bp *bodyPlug) Plug(h xrest.Handler) xrest.Handler {
	bp.next = h
	return bp
}

func FetchBodyFromCtx(ctx context.Context) ([]byte, bool) {
	body, ok := ctx.Value(&ctxBodyKey).([]byte)
	return body, ok
}

func (bp *bodyPlug) ServeHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	if _, ok := FetchBodyFromCtx(ctx); !ok {
		buf := bp.pool.Get().(*bytes.Buffer)
		buf.Reset()
		if _, err := io.Copy(buf, r.Body); err != nil {
			bp.pool.Put(buf)
		}
		ctx = context.WithValue(ctx, &ctxBodyKey, buf.Bytes())
		// reconstruct http.Request.Body
		rc := &readCloser{
			ReadCloser: r.Body,
			bp:         bp,
			buf:        buf,
		}
		r.Body = rc
	}

	bp.next.ServeHTTP(ctx, w, r)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha256"
	"crypto/tls"
	"encoding/base64"
	"fmt"
	"github.com/antonholmquist/jason"
	"github.com/julienschmidt/httprouter"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"
)

// Client methods (one per "slave", communicates with the server)

type Client struct {
	Id        string
	Hostname  string
	AuthToken string
	mux       sync.RWMutex
}

// Start client
func (s *Client) Start() bool {
	log.Printf("Starting client from seed %s with tags %v", conf.Seed, conf.tags)

	// Ping server to register
	s.PingServer()

	// Get auth token from server
	s.AuthServer()

	// Start webserver
	go func() {
		router := httprouter.New()
		router.GET("/ping", Ping)

		log.Printf("%v", http.ListenAndServe(fmt.Sprintf(":%d", clientPort), router))
	}()

	// Register with server
	go func() {
		c := time.Tick(time.Duration(CLIENT_PING_INTERVAL) * time.Second)
		for _ = range c {
			s.PingServer()

			// Should we reload auth?
			var reloadAuth bool = false
			s.mux.RLock()
			if len(s.AuthToken) < 1 {
				reloadAuth = true
			}
			s.mux.RUnlock()
			if reloadAuth {
				s.AuthServer()
			}
		}
	}()

	// Long poll commands
	go func() {
		for {
			s.PollCmds()
		}
	}()

	return true
}

// Fetch commands
func (s *Client) PollCmds() {
	bytes, err := s._get(fmt.Sprintf("client/%s/cmds", url.QueryEscape(s.Id)))
	if err == nil {
		obj, jerr := jason.NewObjectFromBytes(bytes)
		if jerr == nil {
			cmds, _ := obj.GetObjectArray("cmds")
			for _, cmd := range cmds {
				id, _ := cmd.GetString("Id")
				command, _ := cmd.GetString("Command")
				signature, _ := cmd.GetString("Signature")
				timeout, _ := cmd.GetInt64("Timeout")
				cmd := newCmd(command, int(timeout))
				cmd.Id = id
				cmd.Signature = signature
				cmd.Execute(s)
			}
		}
	} else {
		// In case of fast error back off a bit
		time.Sleep(1 * time.Second)
	}
}

// Auth server, token is used for verifying commands
func (s *Client) AuthServer() {
	b, e := s._req("POST", fmt.Sprintf("client/%s/auth", url.QueryEscape(s.Id)), nil)
	if e == nil {
		obj, jerr := jason.NewObjectFromBytes(b)
		if jerr == nil {
			token, et := obj.GetString("token")
			if et != nil || len(token) < 1 {
				return
			}
			s.mux.Lock()
			s.AuthToken = token
			s.mux.Unlock()
			log.Printf("Client authenticated with server")
		}
	}
}

// Ping server
func (s *Client) PingServer() {
	s._get(fmt.Sprintf("client/%s/ping?tags=%s&hostname=%s", url.QueryEscape(s.Id), url.QueryEscape(strings.Join(conf.Tags(), ",")), url.QueryEscape(s.Hostname)))
}

// Get
func (s *Client) _get(uri string) ([]byte, error) {
	return s._req("GET", uri, nil)
}

// Generic request method with retry handling
func (s *Client) _req(method string, uri string, data []byte) ([]byte, error) {
	var bytes []byte = nil
	var err error = nil
	for i := 0; i < 10; i++ {
		bytes, err = s._reqUnsafe(method, uri, data)
		if err == nil {
			return bytes, err
		}

		// Sleep a bit before the retry and apply ~25ms jitter
		var sleep float64 = 25 + float64(rand.Intn(50)) + (math.Pow(float64(i), 2) * 10000)
		time.Sleep(time.Duration(sleep) * time.Millisecond)
	}
	if err != nil {
		log.Printf("Failed request after retries to %s with error: %s", uri, err)
	}
	return bytes, err
}

// Generic request method
func (s *Client) _reqUnsafe(method string, uri string, data []byte) ([]byte, error) {
	// Transport
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: true,
		}, // Ignore certificate as this is self generated and invalid
	}
	// For some reasons connections were not closed, this helps
	defer tr.CloseIdleConnections()

	// Client
	client := &http.Client{
		Transport: tr,
	}

	// Sanitize urls
	uri = fmt.Sprintf("/%s", strings.TrimLeft(uri, "/"))

	// Append random string to uri
	var randStr, _ = secureRandomString(32)
	if !strings.Contains(uri, "?") {
		uri = fmt.Sprintf("%s?_rand=%s", uri, randStr)
	} else {
		uri = fmt.Sprintf("%s&_rand=%s", uri, randStr)
	}
	url := fmt.Sprintf("%s%s", strings.TrimRight(seedUri, "/"), uri)

	// Req
	// @todo support data
	req, reqErr := http.NewRequest(method, url, nil)
	if reqErr != nil {
		return nil, reqErr
	}

	// Signed token
	hasher := sha256.New()
	hasher.Write([]byte(uri))
	hasher.Write([]byte(conf.SecureToken))
	signedToken := base64.URLEncoding.EncodeToString(hasher.Sum(nil))

	// Auth token
	req.Header.Add("X-Auth", signedToken)

	// Execute
	resp, respErr := client.Do(req)
	if respErr != nil {
		return nil, respErr
	}

	// Read body
	defer resp.Body.Close()
	body, bodyErr := ioutil.ReadAll(resp.Body)
	if bodyErr != nil {
		return nil, bodyErr
	}
	return body, nil
}

// Create new client
func newClient() *Client {
	return &Client{
		Id:       hostname,
		Hostname: hostname,
	}
}
<commit_msg>Support body data<commit_after>package main

import (
	"bytes"
	"crypto/sha256"
	"crypto/tls"
	"encoding/base64"
	"fmt"
	"github.com/antonholmquist/jason"
	"github.com/julienschmidt/httprouter"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"
)

// Client methods (one per "slave", communicates with the server)

type Client struct {
	Id        string
	Hostname  string
	AuthToken string
	mux       sync.RWMutex
}

// Start client
func (s *Client) Start() bool {
	log.Printf("Starting client from seed %s with tags %v", conf.Seed, conf.tags)

	// Ping server to register
	s.PingServer()

	// Get auth token from server
	s.AuthServer()

	// Start webserver
	go func() {
		router := httprouter.New()
		router.GET("/ping", Ping)

		log.Printf("%v", http.ListenAndServe(fmt.Sprintf(":%d", clientPort), router))
	}()

	// Register with server
	go func() {
		c := time.Tick(time.Duration(CLIENT_PING_INTERVAL) * time.Second)
		for _ = range c {
			s.PingServer()

			// Should we reload auth?
			var reloadAuth bool = false
			s.mux.RLock()
			if len(s.AuthToken) < 1 {
				reloadAuth = true
			}
			s.mux.RUnlock()
			if reloadAuth {
				s.AuthServer()
			}
		}
	}()

	// Long poll commands
	go func() {
		for {
			s.PollCmds()
		}
	}()

	return true
}

// Fetch commands
func (s *Client) PollCmds() {
	bytes, err := s._get(fmt.Sprintf("client/%s/cmds", url.QueryEscape(s.Id)))
	if err == nil {
		obj, jerr := jason.NewObjectFromBytes(bytes)
		if jerr == nil {
			cmds, _ := obj.GetObjectArray("cmds")
			for _, cmd := range cmds {
				id, _ := cmd.GetString("Id")
				command, _ := cmd.GetString("Command")
				signature, _ := cmd.GetString("Signature")
				timeout, _ := cmd.GetInt64("Timeout")
				cmd := newCmd(command, int(timeout))
				cmd.Id = id
				cmd.Signature = signature
				cmd.Execute(s)
			}
		}
	} else {
		// In case of fast error back off a bit
		time.Sleep(1 * time.Second)
	}
}

// Auth server, token is used for verifying commands
func (s *Client) AuthServer() {
	b, e := s._req("POST", fmt.Sprintf("client/%s/auth", url.QueryEscape(s.Id)), nil)
	if e == nil {
		obj, jerr := jason.NewObjectFromBytes(b)
		if jerr == nil {
			token, et := obj.GetString("token")
			if et != nil || len(token) < 1 {
				return
			}
			s.mux.Lock()
			s.AuthToken = token
			s.mux.Unlock()
			log.Printf("Client authenticated with server")
		}
	}
}

// Ping server
func (s *Client) PingServer() {
	s._get(fmt.Sprintf("client/%s/ping?tags=%s&hostname=%s", url.QueryEscape(s.Id), url.QueryEscape(strings.Join(conf.Tags(), ",")), url.QueryEscape(s.Hostname)))
}

// Get
func (s *Client) _get(uri string) ([]byte, error) {
	return s._req("GET", uri, nil)
}

// Generic request method with retry handling
func (s *Client) _req(method string, uri string, data []byte) ([]byte, error) {
	var bytes []byte = nil
	var err error = nil
	for i := 0; i < 10; i++ {
		bytes, err = s._reqUnsafe(method, uri, data)
		if err == nil {
			return bytes, err
		}

		// Sleep a bit before the retry and apply ~25ms jitter
		var sleep float64 = 25 + float64(rand.Intn(50)) + (math.Pow(float64(i), 2) * 10000)
		time.Sleep(time.Duration(sleep) * time.Millisecond)
	}
	if err != nil {
		log.Printf("Failed request after retries to %s with error: %s", uri, err)
	}
	return bytes, err
}

// Generic request method
func (s *Client) _reqUnsafe(method string, uri string, data []byte) ([]byte, error) {
	// Transport
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: true,
		}, // Ignore certificate as this is self generated and invalid
	}
	// For some reasons connections were not closed, this helps
	defer tr.CloseIdleConnections()

	// Client
	client := &http.Client{
		Transport: tr,
	}

	// Sanitize urls
	uri = fmt.Sprintf("/%s", strings.TrimLeft(uri, "/"))

	// Append random string to uri
	var randStr, _ = secureRandomString(32)
	if !strings.Contains(uri, "?") {
		uri = fmt.Sprintf("%s?_rand=%s", uri, randStr)
	} else {
		uri = fmt.Sprintf("%s&_rand=%s", uri, randStr)
	}
	url := fmt.Sprintf("%s%s", strings.TrimRight(seedUri, "/"), uri)

	// Req
	var buf *bytes.Buffer
	if data != nil && len(data) > 0 {
		buf = bytes.NewBuffer(data)
	} else {
		buf = bytes.NewBuffer(make([]byte, 0))
	}
	req, reqErr := http.NewRequest(method, url, buf)
	if reqErr != nil {
		return nil, reqErr
	}

	// Signed token
	hasher := sha256.New()
	hasher.Write([]byte(uri))
	hasher.Write([]byte(conf.SecureToken))
	signedToken := base64.URLEncoding.EncodeToString(hasher.Sum(nil))

	// Auth token
	req.Header.Add("X-Auth", signedToken)

	// Execute
	resp, respErr := client.Do(req)
	if respErr != nil {
		return nil, respErr
	}

	// Read body
	defer resp.Body.Close()
	body, bodyErr := ioutil.ReadAll(resp.Body)
	if bodyErr != nil {
		return nil, bodyErr
	}
	return body, nil
}

// Create new client
func newClient() *Client {
	return &Client{
		Id:       hostname,
		Hostname: hostname,
	}
}
<|endoftext|>


<commit_before>/*
                       WARNING WARNING WARNING

                Attention all potential contributors

   This testfile is not in the best state. We've been slowly transitioning
   from the built in "testing" package to using Ginkgo. As you can see, we've
   changed the format, but a lot of the setup, test body, descriptions, etc
   are either hardcoded, completely lacking, or misleading.

   For example:

   Describe("Testing with ginkgo"...)      // This is not a great description
   It("TestDoesSoemthing"...)              // This is a horrible description

   Describe("create-user command"...       // Describe the actual object under test
   It("creates a user when provided ..."   // this is more descriptive

   For good examples of writing Ginkgo tests for the cli, refer to

   src/github.com/cloudfoundry/cli/cf/commands/application/delete_app_test.go
   src/github.com/cloudfoundry/cli/cf/terminal/ui_test.go
   src/github.com/cloudfoundry/loggregator_consumer/consumer_test.go
*/

package servicebroker_test

import (
	. "github.com/cloudfoundry/cli/cf/commands/servicebroker"
	testapi "github.com/cloudfoundry/cli/testhelpers/api"
	testcmd "github.com/cloudfoundry/cli/testhelpers/commands"
	testconfig "github.com/cloudfoundry/cli/testhelpers/configuration"
	. "github.com/cloudfoundry/cli/testhelpers/matchers"
	testreq "github.com/cloudfoundry/cli/testhelpers/requirements"
	testterm "github.com/cloudfoundry/cli/testhelpers/terminal"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Testing with ginkgo", func() {
	It("TestCreateServiceBrokerFailsWithUsage", func() {
		requirementsFactory := &testreq.FakeReqFactory{LoginSuccess: true}
		serviceBrokerRepo := &testapi.FakeServiceBrokerRepo{}

		ui := callCreateServiceBroker([]string{}, requirementsFactory, serviceBrokerRepo)
		Expect(ui.FailedWithUsage).To(BeTrue())

		ui = callCreateServiceBroker([]string{"1arg"}, requirementsFactory, serviceBrokerRepo)
		Expect(ui.FailedWithUsage).To(BeTrue())

		ui = callCreateServiceBroker([]string{"1arg", "2arg"}, requirementsFactory, serviceBrokerRepo)
		Expect(ui.FailedWithUsage).To(BeTrue())

		ui = callCreateServiceBroker([]string{"1arg", "2arg", "3arg"}, requirementsFactory, serviceBrokerRepo)
		Expect(ui.FailedWithUsage).To(BeTrue())

		ui = callCreateServiceBroker([]string{"1arg", "2arg", "3arg", "4arg"}, requirementsFactory, serviceBrokerRepo)
		Expect(ui.FailedWithUsage).To(BeFalse())
	})
	It("TestCreateServiceBrokerRequirements", func() {

		requirementsFactory := &testreq.FakeReqFactory{}
		serviceBrokerRepo := &testapi.FakeServiceBrokerRepo{}
		args := []string{"1arg", "2arg", "3arg", "4arg"}

		requirementsFactory.LoginSuccess = false
		callCreateServiceBroker(args, requirementsFactory, serviceBrokerRepo)
		Expect(testcmd.CommandDidPassRequirements).To(BeFalse())

		requirementsFactory.LoginSuccess = true
		callCreateServiceBroker(args, requirementsFactory, serviceBrokerRepo)
		Expect(testcmd.CommandDidPassRequirements).To(BeTrue())
	})
	It("TestCreateServiceBroker", func() {

		requirementsFactory := &testreq.FakeReqFactory{LoginSuccess: true}
		serviceBrokerRepo := &testapi.FakeServiceBrokerRepo{}
		args := []string{"my-broker", "my username", "my password", "http://example.com"}
		ui := callCreateServiceBroker(args, requirementsFactory, serviceBrokerRepo)

		Expect(ui.Outputs).To(ContainSubstrings(
			[]string{"Creating service broker", "my-broker", "my-user"},
			[]string{"OK"},
		))

		Expect(serviceBrokerRepo.CreateName).To(Equal("my-broker"))
		Expect(serviceBrokerRepo.CreateUrl).To(Equal("http://example.com"))
		Expect(serviceBrokerRepo.CreateUsername).To(Equal("my username"))
		Expect(serviceBrokerRepo.CreatePassword).To(Equal("my password"))
	})
})

func callCreateServiceBroker(args []string, requirementsFactory *testreq.FakeReqFactory, serviceBrokerRepo *testapi.FakeServiceBrokerRepo) (ui *testterm.FakeUI) {
	ui = &testterm.FakeUI{}
	config := testconfig.NewRepositoryWithDefaults()
	cmd := NewCreateServiceBroker(ui, config, serviceBrokerRepo)
	testcmd.RunCommand(cmd, args, requirementsFactory)
	return
}
<commit_msg>Cleanup create-service-broker tests<commit_after>package servicebroker_test

import (
	"github.com/cloudfoundry/cli/cf/configuration"
	testapi "github.com/cloudfoundry/cli/testhelpers/api"
	testcmd "github.com/cloudfoundry/cli/testhelpers/commands"
	testconfig "github.com/cloudfoundry/cli/testhelpers/configuration"
	testreq "github.com/cloudfoundry/cli/testhelpers/requirements"
	testterm "github.com/cloudfoundry/cli/testhelpers/terminal"

	. "github.com/cloudfoundry/cli/cf/commands/servicebroker"
	. "github.com/cloudfoundry/cli/testhelpers/matchers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("create-service-broker command", func() {
	var (
		ui                  *testterm.FakeUI
		requirementsFactory *testreq.FakeReqFactory
		configRepo          configuration.ReadWriter
		serviceBrokerRepo   *testapi.FakeServiceBrokerRepo
	)

	BeforeEach(func() {
		configRepo = testconfig.NewRepositoryWithDefaults()

		ui = &testterm.FakeUI{}
		requirementsFactory = &testreq.FakeReqFactory{}
		serviceBrokerRepo = &testapi.FakeServiceBrokerRepo{}
	})

	runCommand := func(args ...string) {
		testcmd.RunCommand(NewCreateServiceBroker(ui, configRepo, serviceBrokerRepo), args, requirementsFactory)
	}

	Describe("requirements", func() {
		It("fails with usage when called without exactly four args", func() {
			requirementsFactory.LoginSuccess = true
			runCommand("whoops", "not-enough", "args")
			Expect(ui.FailedWithUsage).To(BeTrue())
		})

		It("fails when not logged in", func() {
			runCommand("Just", "Enough", "Args", "Provided")
			Expect(testcmd.CommandDidPassRequirements).To(BeFalse())
		})
	})

	Context("when logged in", func() {
		BeforeEach(func() {
			requirementsFactory.LoginSuccess = true
		})

		It("creates a service broker, obviously", func() {
			runCommand("my-broker", "my-username", "my-password", "http://example.com")

			Expect(ui.Outputs).To(ContainSubstrings(
				[]string{"Creating service broker", "my-broker", "my-user"},
				[]string{"OK"},
			))

			Expect(serviceBrokerRepo.CreateName).To(Equal("my-broker"))
			Expect(serviceBrokerRepo.CreateUrl).To(Equal("http://example.com"))
			Expect(serviceBrokerRepo.CreateUsername).To(Equal("my-username"))
			Expect(serviceBrokerRepo.CreatePassword).To(Equal("my-password"))
		})
	})
})
<|endoftext|>


<commit_before>package cf

import (
	"os"
	"path/filepath"
)

const (
	Version = "6.0.1-BUILT_FROM_SOURCE"
	Usage   = "A command line tool to interact with Cloud Foundry"
)

func Name() string {
	return filepath.Base(os.Args[0])
}
<commit_msg>Bump version number<commit_after>package cf

import (
	"os"
	"path/filepath"
)

const (
	Version = "6.0.2-BUILT_FROM_SOURCE"
	Usage   = "A command line tool to interact with Cloud Foundry"
)

func Name() string {
	return filepath.Base(os.Args[0])
}
<|endoftext|>


<commit_before>package hdfs

import (
	hdfs "github.com/dklassen/hdfs/protocol/hadoop_hdfs"
	"github.com/dklassen/hdfs/rpc"
	"github.com/golang/protobuf/proto"
	"io"
	"io/ioutil"
	"os"
	"os/user"
)

// A Client represents a connection to an HDFS cluster
type Client struct {
	namenode *rpc.NamenodeConnection
	defaults *hdfs.FsServerDefaultsProto
}

// New returns a connected Client, or an error if it can't connect. The user
// will be the user the code is running under.
func New(address string) (*Client, error) {
	currentUser, err := user.Current()
	if err != nil {
		return nil, err
	}

	return NewForUser(address, currentUser.Username)
}

// NewForUser returns a connected Client with the user specified, or an error if
// it can't connect.
func NewForUser(address string, user string) (*Client, error) {
	namenode, err := rpc.NewNamenodeConnection(address, user)
	if err != nil {
		return nil, err
	}

	return &Client{namenode: namenode}, nil
}

// ReadFile reads the file named by filename and returns the contents.
func (c *Client) ReadFile(filename string) ([]byte, error) {
	f, err := c.Open(filename)
	if err != nil {
		return nil, err
	}

	return ioutil.ReadAll(f)
}

// CopyToLocal copies the HDFS file specified by src to the local file at dst.
// If dst already exists, it will be overwritten.
func (c *Client) CopyToLocal(src string, dst string) error {
	remote, err := c.Open(src)
	if err != nil {
		return err
	}

	local, err := os.Create(dst)
	if err != nil {
		return err
	}

	_, err = io.Copy(local, remote)
	return err
}

// CreateEmptyFile creates a empty file named by filename, with the permissions
// 0644.
func (c *Client) CreateEmptyFile(filename string) error {
	_, err := c.getFileInfo(filename)
	if err == nil {
		return &os.PathError{"create", filename, os.ErrExist}
	} else if !os.IsNotExist(err) {
		return &os.PathError{"create", filename, err}
	}

	defaults, err := c.fetchDefaults()
	if err != nil {
		return err
	}

	createReq := &hdfs.CreateRequestProto{
		Src:          proto.String(filename),
		Masked:       &hdfs.FsPermissionProto{Perm: proto.Uint32(uint32(0644))},
		ClientName:   proto.String(rpc.ClientName),
		CreateFlag:   proto.Uint32(1),
		CreateParent: proto.Bool(false),
		Replication:  proto.Uint32(defaults.GetReplication()),
		BlockSize:    proto.Uint64(defaults.GetBlockSize()),
	}
	createResp := &hdfs.CreateResponseProto{}

	err = c.namenode.Execute("create", createReq, createResp)
	if err != nil {
		if nnErr, ok := err.(*rpc.NamenodeError); ok {
			err = interpretException(nnErr.Exception, err)
		}

		return &os.PathError{"create", filename, err}
	}

	completeReq := &hdfs.CompleteRequestProto{
		Src:        proto.String(filename),
		ClientName: proto.String(rpc.ClientName),
	}
	completeResp := &hdfs.CompleteResponseProto{}

	err = c.namenode.Execute("complete", completeReq, completeResp)
	if err != nil {
		return &os.PathError{"create", filename, err}
	}

	return nil
}

func (c *Client) fetchDefaults() (*hdfs.FsServerDefaultsProto, error) {
	if c.defaults != nil {
		return c.defaults, nil
	}

	req := &hdfs.GetServerDefaultsRequestProto{}
	resp := &hdfs.GetServerDefaultsResponseProto{}

	err := c.namenode.Execute("getServerDefaults", req, resp)
	if err != nil {
		return nil, err
	}

	c.defaults = resp.GetServerDefaults()
	return c.defaults, nil
}
<commit_msg>Won't work in a kerberos environment but in unprotected environments we want to protend to be another user<commit_after>package hdfs

import (
	hdfs "github.com/dklassen/hdfs/protocol/hadoop_hdfs"
	"github.com/dklassen/hdfs/rpc"
	"github.com/golang/protobuf/proto"
	"io"
	"io/ioutil"
	"os"
	"os/user"
)

// A Client represents a connection to an HDFS cluster
type Client struct {
	namenode *rpc.NamenodeConnection
	defaults *hdfs.FsServerDefaultsProto
}

// New returns a connected Client, or an error if it can't connect. The user
// will be the user the code is running under.
func New(address string) (*Client, error) {
  currentUser := os.Getenv("HADOOP_USER_NAME")
  if currentUser == "" {
	  localUser, err := user.Current()
    if err != nil {
      return nil, err
    }
    currentUser = localUser.Username
  }
	return NewForUser(address, currentUser)
}

// NewForUser returns a connected Client with the user specified, or an error if
// it can't connect.
func NewForUser(address string, user string) (*Client, error) {
	namenode, err := rpc.NewNamenodeConnection(address, user)
	if err != nil {
		return nil, err
	}

	return &Client{namenode: namenode}, nil
}

// ReadFile reads the file named by filename and returns the contents.
func (c *Client) ReadFile(filename string) ([]byte, error) {
	f, err := c.Open(filename)
	if err != nil {
		return nil, err
	}

	return ioutil.ReadAll(f)
}

// CopyToLocal copies the HDFS file specified by src to the local file at dst.
// If dst already exists, it will be overwritten.
func (c *Client) CopyToLocal(src string, dst string) error {
	remote, err := c.Open(src)
	if err != nil {
		return err
	}

	local, err := os.Create(dst)
	if err != nil {
		return err
	}

	_, err = io.Copy(local, remote)
	return err
}

// CreateEmptyFile creates a empty file named by filename, with the permissions
// 0644.
func (c *Client) CreateEmptyFile(filename string) error {
	_, err := c.getFileInfo(filename)
	if err == nil {
		return &os.PathError{"create", filename, os.ErrExist}
	} else if !os.IsNotExist(err) {
		return &os.PathError{"create", filename, err}
	}

	defaults, err := c.fetchDefaults()
	if err != nil {
		return err
	}

	createReq := &hdfs.CreateRequestProto{
		Src:          proto.String(filename),
		Masked:       &hdfs.FsPermissionProto{Perm: proto.Uint32(uint32(0644))},
		ClientName:   proto.String(rpc.ClientName),
		CreateFlag:   proto.Uint32(1),
		CreateParent: proto.Bool(false),
		Replication:  proto.Uint32(defaults.GetReplication()),
		BlockSize:    proto.Uint64(defaults.GetBlockSize()),
	}
	createResp := &hdfs.CreateResponseProto{}

	err = c.namenode.Execute("create", createReq, createResp)
	if err != nil {
		if nnErr, ok := err.(*rpc.NamenodeError); ok {
			err = interpretException(nnErr.Exception, err)
		}

		return &os.PathError{"create", filename, err}
	}

	completeReq := &hdfs.CompleteRequestProto{
		Src:        proto.String(filename),
		ClientName: proto.String(rpc.ClientName),
	}
	completeResp := &hdfs.CompleteResponseProto{}

	err = c.namenode.Execute("complete", completeReq, completeResp)
	if err != nil {
		return &os.PathError{"create", filename, err}
	}

	return nil
}

func (c *Client) fetchDefaults() (*hdfs.FsServerDefaultsProto, error) {
	if c.defaults != nil {
		return c.defaults, nil
	}

	req := &hdfs.GetServerDefaultsRequestProto{}
	resp := &hdfs.GetServerDefaultsResponseProto{}

	err := c.namenode.Execute("getServerDefaults", req, resp)
	if err != nil {
		return nil, err
	}

	c.defaults = resp.GetServerDefaults()
	return c.defaults, nil
}
<|endoftext|>


<commit_before>package paranoidhttp

import (
	"context"
	"errors"
	"fmt"
	"log"
	"net"
	"net/http"
	"regexp"
	"time"
)

// DefaultClient is the default Client whose setting is the same as http.DefaultClient.
var DefaultClient *http.Client

func mustParseCIDR(addr string) *net.IPNet {
	_, ipnet, err := net.ParseCIDR(addr)
	if err != nil {
		log.Fatalf("%s must be parsed", addr)
	}
	return ipnet
}

var (
	netPrivateClassA = mustParseCIDR("10.0.0.0/8")
	netPrivateClassB = mustParseCIDR("172.16.0.0/12")
	netPrivateClassC = mustParseCIDR("192.168.0.0/16")
	netTestNet       = mustParseCIDR("192.0.2.0/24")
	net6To4Relay     = mustParseCIDR("192.88.99.0/24")
)

func init() {
	DefaultClient, _, _ = NewClient()
}

func safeAddr(hostport string) (string, error) {
	host, port, err := net.SplitHostPort(hostport)
	if err != nil {
		return "", err
	}

	ip := net.ParseIP(host)
	if ip != nil {
		if ip.To4() != nil && isBadIPv4(ip) {
			return "", fmt.Errorf("bad ip is detected: %v", ip)
		}
		return net.JoinHostPort(ip.String(), port), nil
	}

	if isBadHost(host) {
		return "", fmt.Errorf("bad host is detected: %v", host)
	}

	ips, err := net.LookupIP(host) // TODO timeout
	if err != nil || len(ips) <= 0 {
		return "", err
	}
	for _, ip := range ips {
		if ip.To4() != nil && isBadIPv4(ip) {
			return "", fmt.Errorf("bad ip is detected: %v", ip)
		}
	}
	return net.JoinHostPort(ips[0].String(), port), nil
}

// NewDialer returns a dialer function which only allows IPv4 connections.
//
// This is used to create a new paranoid http.Client,
// because I'm not sure about a paranoid behavior for IPv6 connections :(
func NewDialer(dialer *net.Dialer) func(ctx context.Context, network, addr string) (net.Conn, error) {
	return func(ctx context.Context, network, hostport string) (net.Conn, error) {
		switch network {
		case "tcp", "tcp4":
			addr, err := safeAddr(hostport)
			if err != nil {
				return nil, err
			}
			return dialer.DialContext(ctx, "tcp4", addr)
		default:
			return nil, errors.New("does not support any networks except tcp4")
		}
	}
}

// NewClient returns a new http.Client configured to be paranoid for attackers.
//
// This also returns http.Tranport and net.Dialer so that you can customize those behavior.
func NewClient() (*http.Client, *http.Transport, *net.Dialer) {
	dialer := &net.Dialer{
		Timeout:   30 * time.Second,
		KeepAlive: 30 * time.Second,
	}
	transport := &http.Transport{
		Proxy:               http.ProxyFromEnvironment,
		DialContext:         NewDialer(dialer),
		TLSHandshakeTimeout: 10 * time.Second,
	}
	return &http.Client{
		Timeout:   30 * time.Second,
		Transport: transport,
	}, transport, dialer
}

var regLocalhost = regexp.MustCompile("(?i)^localhost$")
var regHasSpace = regexp.MustCompile("(?i)\\s+")

func isBadHost(host string) bool {
	if regLocalhost.MatchString(host) {
		return true
	}
	if regHasSpace.MatchString(host) {
		return true
	}

	return false
}

func isBadIPv4(ip net.IP) bool {
	if ip.To4() == nil {
		panic("cannot be called for IPv6")
	}

	if ip.Equal(net.IPv4bcast) || !ip.IsGlobalUnicast() ||
		netPrivateClassA.Contains(ip) || netPrivateClassB.Contains(ip) || netPrivateClassC.Contains(ip) ||
		netTestNet.Contains(ip) || net6To4Relay.Contains(ip) {
		return true
	}

	return false
}
<commit_msg>use net.DefaultResolver in safeAddr<commit_after>package paranoidhttp

import (
	"context"
	"errors"
	"fmt"
	"log"
	"net"
	"net/http"
	"regexp"
	"time"
)

// DefaultClient is the default Client whose setting is the same as http.DefaultClient.
var DefaultClient *http.Client

func mustParseCIDR(addr string) *net.IPNet {
	_, ipnet, err := net.ParseCIDR(addr)
	if err != nil {
		log.Fatalf("%s must be parsed", addr)
	}
	return ipnet
}

var (
	netPrivateClassA = mustParseCIDR("10.0.0.0/8")
	netPrivateClassB = mustParseCIDR("172.16.0.0/12")
	netPrivateClassC = mustParseCIDR("192.168.0.0/16")
	netTestNet       = mustParseCIDR("192.0.2.0/24")
	net6To4Relay     = mustParseCIDR("192.88.99.0/24")
)

func init() {
	DefaultClient, _, _ = NewClient()
}

func safeAddr(ctx context.Context, hostport string) (string, error) {
	host, port, err := net.SplitHostPort(hostport)
	if err != nil {
		return "", err
	}

	ip := net.ParseIP(host)
	if ip != nil {
		if ip.To4() != nil && isBadIPv4(ip) {
			return "", fmt.Errorf("bad ip is detected: %v", ip)
		}
		return net.JoinHostPort(ip.String(), port), nil
	}

	if isBadHost(host) {
		return "", fmt.Errorf("bad host is detected: %v", host)
	}

	addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
	if err != nil || len(addrs) <= 0 {
		return "", err
	}
	for _, addr := range addrs {
		if addr.IP.To4() != nil && isBadIPv4(addr.IP) {
			return "", fmt.Errorf("bad ip is detected: %v", addr.IP)
		}
	}
	return net.JoinHostPort(addrs[0].IP.String(), port), nil
}

// NewDialer returns a dialer function which only allows IPv4 connections.
//
// This is used to create a new paranoid http.Client,
// because I'm not sure about a paranoid behavior for IPv6 connections :(
func NewDialer(dialer *net.Dialer) func(ctx context.Context, network, addr string) (net.Conn, error) {
	return func(ctx context.Context, network, hostport string) (net.Conn, error) {
		switch network {
		case "tcp", "tcp4":
			addr, err := safeAddr(ctx, hostport)
			if err != nil {
				return nil, err
			}
			return dialer.DialContext(ctx, "tcp4", addr)
		default:
			return nil, errors.New("does not support any networks except tcp4")
		}
	}
}

// NewClient returns a new http.Client configured to be paranoid for attackers.
//
// This also returns http.Tranport and net.Dialer so that you can customize those behavior.
func NewClient() (*http.Client, *http.Transport, *net.Dialer) {
	dialer := &net.Dialer{
		Timeout:   30 * time.Second,
		KeepAlive: 30 * time.Second,
	}
	transport := &http.Transport{
		Proxy:               http.ProxyFromEnvironment,
		DialContext:         NewDialer(dialer),
		TLSHandshakeTimeout: 10 * time.Second,
	}
	return &http.Client{
		Timeout:   30 * time.Second,
		Transport: transport,
	}, transport, dialer
}

var regLocalhost = regexp.MustCompile("(?i)^localhost$")
var regHasSpace = regexp.MustCompile("(?i)\\s+")

func isBadHost(host string) bool {
	if regLocalhost.MatchString(host) {
		return true
	}
	if regHasSpace.MatchString(host) {
		return true
	}

	return false
}

func isBadIPv4(ip net.IP) bool {
	if ip.To4() == nil {
		panic("cannot be called for IPv6")
	}

	if ip.Equal(net.IPv4bcast) || !ip.IsGlobalUnicast() ||
		netPrivateClassA.Contains(ip) || netPrivateClassB.Contains(ip) || netPrivateClassC.Contains(ip) ||
		netTestNet.Contains(ip) || net6To4Relay.Contains(ip) {
		return true
	}

	return false
}
<|endoftext|>


<commit_before>package main

import (
	irc "github.com/fluffle/goirc/client"
	"fmt"
	"os"
	"bufio"
	"strings"
)

func main() {
	// create new IRC connection
	c := irc.New("GoTest", "gotest", "GoBot")
	c.Debug = true
	c.AddHandler("connected",
		func(conn *irc.Conn, line *irc.Line) { conn.Join("#go-nuts") })

	// Set up a handler to notify of disconnect events.
	quit := make(chan bool)
	c.AddHandler("disconnected",
		func(conn *irc.Conn, line *irc.Line) { quit <- true })

	// set up a goroutine to read commands from stdin
	in := make(chan string, 4)
	reallyquit := false
	go func() {
		con := bufio.NewReader(os.Stdin)
		for {
			s, err := con.ReadString('\n')
			if err != nil {
				// wha?, maybe ctrl-D...
				close(in)
				break
			}
			// no point in sending empty lines down the channel
			if len(s) > 2 {
				in <- s[0 : len(s)-1]
			}
		}
	}()

	// set up a goroutine to do parsey things with the stuff from stdin
	go func() {
		for cmd := range in {
			if cmd[0] == ':' {
				switch idx := strings.Index(cmd, " "); {
				case cmd[1] == 'd':
					fmt.Printf(c.String())
				case cmd[1] == 'f':
					if len(cmd) > 2 && cmd[2] == 'e' {
						// enable flooding
						c.Flood = true
					} else if len(cmd) > 2 && cmd[2] == 'd' {
						// disable flooding
						c.Flood = false
					}
					for i := 0; i < 20; i++ {
						c.Privmsg("#", "flood test!")
					}
				case idx == -1:
					continue
				case cmd[1] == 'q':
					reallyquit = true
					c.Quit(cmd[idx+1 : len(cmd)])
				case cmd[1] == 'j':
					c.Join(cmd[idx+1 : len(cmd)])
				case cmd[1] == 'p':
					c.Part(cmd[idx+1 : len(cmd)])
				}
			} else {
				c.Raw(cmd)
			}
		}
	}()

	for !reallyquit {
		// connect to server
		if err := c.Connect("irc.freenode.net"); err != nil {
			fmt.Printf("Connection error: %s\n", err)
			return
		}

		// wait on quit channel
		<-quit
	}
}
<commit_msg>Missed removal of conn.Debug.<commit_after>package main

import (
	irc "github.com/fluffle/goirc/client"
	"fmt"
	"os"
	"bufio"
	"strings"
)

func main() {
	// create new IRC connection
	c := irc.New("GoTest", "gotest", "GoBot")
	c.AddHandler("connected",
		func(conn *irc.Conn, line *irc.Line) { conn.Join("#go-nuts") })

	// Set up a handler to notify of disconnect events.
	quit := make(chan bool)
	c.AddHandler("disconnected",
		func(conn *irc.Conn, line *irc.Line) { quit <- true })

	// set up a goroutine to read commands from stdin
	in := make(chan string, 4)
	reallyquit := false
	go func() {
		con := bufio.NewReader(os.Stdin)
		for {
			s, err := con.ReadString('\n')
			if err != nil {
				// wha?, maybe ctrl-D...
				close(in)
				break
			}
			// no point in sending empty lines down the channel
			if len(s) > 2 {
				in <- s[0 : len(s)-1]
			}
		}
	}()

	// set up a goroutine to do parsey things with the stuff from stdin
	go func() {
		for cmd := range in {
			if cmd[0] == ':' {
				switch idx := strings.Index(cmd, " "); {
				case cmd[1] == 'd':
					fmt.Printf(c.String())
				case cmd[1] == 'f':
					if len(cmd) > 2 && cmd[2] == 'e' {
						// enable flooding
						c.Flood = true
					} else if len(cmd) > 2 && cmd[2] == 'd' {
						// disable flooding
						c.Flood = false
					}
					for i := 0; i < 20; i++ {
						c.Privmsg("#", "flood test!")
					}
				case idx == -1:
					continue
				case cmd[1] == 'q':
					reallyquit = true
					c.Quit(cmd[idx+1 : len(cmd)])
				case cmd[1] == 'j':
					c.Join(cmd[idx+1 : len(cmd)])
				case cmd[1] == 'p':
					c.Part(cmd[idx+1 : len(cmd)])
				}
			} else {
				c.Raw(cmd)
			}
		}
	}()

	for !reallyquit {
		// connect to server
		if err := c.Connect("irc.freenode.net"); err != nil {
			fmt.Printf("Connection error: %s\n", err)
			return
		}

		// wait on quit channel
		<-quit
	}
}
<|endoftext|>