<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"time"

	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kubernetes/pkg/api/v1"
	storage "k8s.io/kubernetes/pkg/apis/storage/v1beta1"
	storageutil "k8s.io/kubernetes/pkg/apis/storage/v1beta1/util"
	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	// Requested size of the volume
	requestedSize = "1500Mi"
	// Expected size of the volume is 2GiB, for "openstack", "gce", "aws", "gke", as they allocate volumes in 1GiB chunks
	expectedSize = "2Gi"
	// vsphere provider does not allocate volumes in 1GiB chunks, so setting expected size equal to requestedSize
	vsphereExpectedSize = "1500Mi"
)

func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVolumeClaim) {
	err := framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
	Expect(err).NotTo(HaveOccurred())

	By("checking the claim")
	// Get new copy of the claim
	claim, err = client.Core().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
	Expect(err).NotTo(HaveOccurred())

	// Get the bound PV
	pv, err := client.Core().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
	Expect(err).NotTo(HaveOccurred())

	// Check sizes
	expectedCapacity := resource.MustParse(expectedSize)
	if framework.ProviderIs("vsphere") {
		expectedCapacity = resource.MustParse(vsphereExpectedSize)
	}
	pvCapacity := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)]
	Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()), "pvCapacity is not equal to expectedCapacity")

	requestedCapacity := resource.MustParse(requestedSize)
	claimCapacity := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	Expect(claimCapacity.Value()).To(Equal(requestedCapacity.Value()), "claimCapacity is not equal to requestedCapacity")

	// Check PV properties
	Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(v1.PersistentVolumeReclaimDelete))
	expectedAccessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}
	Expect(pv.Spec.AccessModes).To(Equal(expectedAccessModes))
	Expect(pv.Spec.ClaimRef.Name).To(Equal(claim.ObjectMeta.Name))
	Expect(pv.Spec.ClaimRef.Namespace).To(Equal(claim.ObjectMeta.Namespace))

	// We start two pods:
	// - The first writes 'hello word' to the /mnt/test (= the volume).
	// - The second one runs grep 'hello world' on /mnt/test.
	// If both succeed, Kubernetes actually allocated something that is
	// persistent across pods.
	By("checking the created volume is writable")
	runInPodWithVolume(client, claim.Namespace, claim.Name, "echo 'hello world' > /mnt/test/data")

	By("checking the created volume is readable and retains data")
	runInPodWithVolume(client, claim.Namespace, claim.Name, "grep 'hello world' /mnt/test/data")

	By("deleting the claim")
	framework.ExpectNoError(client.Core().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil))

	// Wait for the PV to get deleted. Technically, the first few delete
	// attempts may fail, as the volume is still attached to a node because
	// kubelet is slowly cleaning up a pod, however it should succeed in a
	// couple of minutes. Wait 20 minutes to recover from random cloud hiccups.
	framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(client, pv.Name, 5*time.Second, 20*time.Minute))
}

var _ = framework.KubeDescribe("Dynamic provisioning", func() {
	f := framework.NewDefaultFramework("volume-provisioning")

	// filled in BeforeEach
	var c clientset.Interface
	var ns string

	BeforeEach(func() {
		c = f.ClientSet
		ns = f.Namespace.Name
	})

	framework.KubeDescribe("DynamicProvisioner", func() {
		It("should create and delete persistent volumes [Slow] [Volume]", func() {
			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")

			By("creating a StorageClass")
			class := newStorageClass()
			_, err := c.Storage().StorageClasses().Create(class)
			defer c.Storage().StorageClasses().Delete(class.Name, nil)
			Expect(err).NotTo(HaveOccurred())

			By("creating a claim with a dynamic provisioning annotation")
			claim := newClaim(ns, false)
			defer func() {
				c.Core().PersistentVolumeClaims(ns).Delete(claim.Name, nil)
			}()
			claim, err = c.Core().PersistentVolumeClaims(ns).Create(claim)
			Expect(err).NotTo(HaveOccurred())

			testDynamicProvisioning(c, claim)
		})
	})

	framework.KubeDescribe("DynamicProvisioner Alpha", func() {
		It("should create and delete alpha persistent volumes [Slow] [Volume]", func() {
			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")

			By("creating a claim with an alpha dynamic provisioning annotation")
			claim := newClaim(ns, true)
			defer func() {
				c.Core().PersistentVolumeClaims(ns).Delete(claim.Name, nil)
			}()
			claim, err := c.Core().PersistentVolumeClaims(ns).Create(claim)
			Expect(err).NotTo(HaveOccurred())

			testDynamicProvisioning(c, claim)
		})
	})
})

func newClaim(ns string, alpha bool) *v1.PersistentVolumeClaim {
	claim := v1.PersistentVolumeClaim{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "pvc-",
			Namespace:    ns,
		},
		Spec: v1.PersistentVolumeClaimSpec{
			AccessModes: []v1.PersistentVolumeAccessMode{
				v1.ReadWriteOnce,
			},
			Resources: v1.ResourceRequirements{
				Requests: v1.ResourceList{
					v1.ResourceName(v1.ResourceStorage): resource.MustParse(requestedSize),
				},
			},
		},
	}

	if alpha {
		claim.Annotations = map[string]string{
			storageutil.AlphaStorageClassAnnotation: "",
		}
	} else {
		claim.Annotations = map[string]string{
			storageutil.StorageClassAnnotation: "fast",
		}

	}

	return &claim
}

// runInPodWithVolume runs a command in a pod with given claim mounted to /mnt directory.
func runInPodWithVolume(c clientset.Interface, ns, claimName, command string) {
	pod := &v1.Pod{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Pod",
			APIVersion: "v1",
		},
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "pvc-volume-tester-",
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:    "volume-tester",
					Image:   "gcr.io/google_containers/busybox:1.24",
					Command: []string{"/bin/sh"},
					Args:    []string{"-c", command},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "my-volume",
							MountPath: "/mnt/test",
						},
					},
				},
			},
			RestartPolicy: v1.RestartPolicyNever,
			Volumes: []v1.Volume{
				{
					Name: "my-volume",
					VolumeSource: v1.VolumeSource{
						PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
							ClaimName: claimName,
							ReadOnly:  false,
						},
					},
				},
			},
		},
	}
	pod, err := c.Core().Pods(ns).Create(pod)
	defer func() {
		framework.ExpectNoError(c.Core().Pods(ns).Delete(pod.Name, nil))
	}()
	framework.ExpectNoError(err, "Failed to create pod: %v", err)
	framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace))
}

func newStorageClass() *storage.StorageClass {
	var pluginName string

	switch {
	case framework.ProviderIs("gke"), framework.ProviderIs("gce"):
		pluginName = "kubernetes.io/gce-pd"
	case framework.ProviderIs("aws"):
		pluginName = "kubernetes.io/aws-ebs"
	case framework.ProviderIs("openstack"):
		pluginName = "kubernetes.io/cinder"
	case framework.ProviderIs("vsphere"):
		pluginName = "kubernetes.io/vsphere-volume"
	}

	return &storage.StorageClass{
		TypeMeta: metav1.TypeMeta{
			Kind: "StorageClass",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name: "fast",
		},
		Provisioner: pluginName,
	}
}
<commit_msg>Add e2e test for external provisioners<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"time"

	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apiserver/pkg/authentication/serviceaccount"
	"k8s.io/kubernetes/pkg/api/v1"
	rbacv1beta1 "k8s.io/kubernetes/pkg/apis/rbac/v1beta1"
	storage "k8s.io/kubernetes/pkg/apis/storage/v1beta1"
	storageutil "k8s.io/kubernetes/pkg/apis/storage/v1beta1/util"
	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	// Requested size of the volume
	requestedSize = "1500Mi"
	// Plugin name of the external provisioner
	externalPluginName = "example.com/nfs"
)

func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVolumeClaim, expectedSize string) {
	err := framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
	Expect(err).NotTo(HaveOccurred())

	By("checking the claim")
	// Get new copy of the claim
	claim, err = client.Core().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
	Expect(err).NotTo(HaveOccurred())

	// Get the bound PV
	pv, err := client.Core().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
	Expect(err).NotTo(HaveOccurred())

	// Check sizes
	expectedCapacity := resource.MustParse(expectedSize)
	pvCapacity := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)]
	Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()), "pvCapacity is not equal to expectedCapacity")

	requestedCapacity := resource.MustParse(requestedSize)
	claimCapacity := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	Expect(claimCapacity.Value()).To(Equal(requestedCapacity.Value()), "claimCapacity is not equal to requestedCapacity")

	// Check PV properties
	Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(v1.PersistentVolumeReclaimDelete))
	expectedAccessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}
	Expect(pv.Spec.AccessModes).To(Equal(expectedAccessModes))
	Expect(pv.Spec.ClaimRef.Name).To(Equal(claim.ObjectMeta.Name))
	Expect(pv.Spec.ClaimRef.Namespace).To(Equal(claim.ObjectMeta.Namespace))

	// We start two pods:
	// - The first writes 'hello word' to the /mnt/test (= the volume).
	// - The second one runs grep 'hello world' on /mnt/test.
	// If both succeed, Kubernetes actually allocated something that is
	// persistent across pods.
	By("checking the created volume is writable")
	runInPodWithVolume(client, claim.Namespace, claim.Name, "echo 'hello world' > /mnt/test/data")

	By("checking the created volume is readable and retains data")
	runInPodWithVolume(client, claim.Namespace, claim.Name, "grep 'hello world' /mnt/test/data")

	By("deleting the claim")
	framework.ExpectNoError(client.Core().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil))

	// Wait for the PV to get deleted. Technically, the first few delete
	// attempts may fail, as the volume is still attached to a node because
	// kubelet is slowly cleaning up a pod, however it should succeed in a
	// couple of minutes. Wait 20 minutes to recover from random cloud hiccups.
	framework.ExpectNoError(framework.WaitForPersistentVolumeDeleted(client, pv.Name, 5*time.Second, 20*time.Minute))
}

var _ = framework.KubeDescribe("Dynamic provisioning", func() {
	f := framework.NewDefaultFramework("volume-provisioning")

	// filled in BeforeEach
	var c clientset.Interface
	var ns string

	BeforeEach(func() {
		c = f.ClientSet
		ns = f.Namespace.Name
	})

	framework.KubeDescribe("DynamicProvisioner", func() {
		It("should create and delete persistent volumes [Slow] [Volume]", func() {
			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")

			By("creating a StorageClass")
			class := newStorageClass("", "internal")
			_, err := c.Storage().StorageClasses().Create(class)
			defer c.Storage().StorageClasses().Delete(class.Name, nil)
			Expect(err).NotTo(HaveOccurred())

			By("creating a claim with a dynamic provisioning annotation")
			claim := newClaim(ns, "internal", false)
			defer func() {
				c.Core().PersistentVolumeClaims(ns).Delete(claim.Name, nil)
			}()
			claim, err = c.Core().PersistentVolumeClaims(ns).Create(claim)
			Expect(err).NotTo(HaveOccurred())

			if framework.ProviderIs("vsphere") {
				// vsphere provider does not allocate volumes in 1GiB chunks, so setting expected size
				// equal to requestedSize
				testDynamicProvisioning(c, claim, requestedSize)
			} else {
				// Expected size of the volume is 2GiB, because the other three supported cloud
				// providers allocate volumes in 1GiB chunks.
				testDynamicProvisioning(c, claim, "2Gi")
			}
		})
	})

	framework.KubeDescribe("DynamicProvisioner Alpha", func() {
		It("should create and delete alpha persistent volumes [Slow] [Volume]", func() {
			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")

			By("creating a claim with an alpha dynamic provisioning annotation")
			claim := newClaim(ns, "", true)
			defer func() {
				c.Core().PersistentVolumeClaims(ns).Delete(claim.Name, nil)
			}()
			claim, err := c.Core().PersistentVolumeClaims(ns).Create(claim)
			Expect(err).NotTo(HaveOccurred())

			if framework.ProviderIs("vsphere") {
				testDynamicProvisioning(c, claim, requestedSize)
			} else {
				testDynamicProvisioning(c, claim, "2Gi")
			}
		})
	})

	framework.KubeDescribe("DynamicProvisioner External", func() {
		It("should let an external dynamic provisioner create and delete persistent volumes [Slow]", func() {
			// external dynamic provisioner pods need additional permissions provided by the
			// persistent-volume-provisioner role
			framework.BindClusterRole(c.Rbac(), "system:persistent-volume-provisioner", ns,
				rbacv1beta1.Subject{Kind: rbacv1beta1.ServiceAccountKind, Namespace: ns, Name: "default"})

			err := framework.WaitForAuthorizationUpdate(c.AuthorizationV1beta1(),
				serviceaccount.MakeUsername(ns, "default"),
				"", "get", schema.GroupResource{Group: "storage.k8s.io", Resource: "storageclasses"}, true)
			framework.ExpectNoError(err, "Failed to update authorization: %v", err)

			By("creating an external dynamic provisioner pod")
			pod := startExternalProvisioner(c, ns)
			defer c.Core().Pods(ns).Delete(pod.Name, nil)

			By("creating a StorageClass")
			class := newStorageClass(externalPluginName, "external")
			_, err = c.Storage().StorageClasses().Create(class)
			defer c.Storage().StorageClasses().Delete(class.Name, nil)
			Expect(err).NotTo(HaveOccurred())

			By("creating a claim with a dynamic provisioning annotation")
			claim := newClaim(ns, "external", false)
			defer func() {
				c.Core().PersistentVolumeClaims(ns).Delete(claim.Name, nil)
			}()
			claim, err = c.Core().PersistentVolumeClaims(ns).Create(claim)
			Expect(err).NotTo(HaveOccurred())

			// Expected size of the externally provisioned volume depends on the external
			// provisioner: for nfs-provisioner used here, it's equal to requested
			testDynamicProvisioning(c, claim, requestedSize)
		})
	})
})

func newClaim(ns, suffix string, alpha bool) *v1.PersistentVolumeClaim {
	claim := v1.PersistentVolumeClaim{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "pvc-",
			Namespace:    ns,
		},
		Spec: v1.PersistentVolumeClaimSpec{
			AccessModes: []v1.PersistentVolumeAccessMode{
				v1.ReadWriteOnce,
			},
			Resources: v1.ResourceRequirements{
				Requests: v1.ResourceList{
					v1.ResourceName(v1.ResourceStorage): resource.MustParse(requestedSize),
				},
			},
		},
	}

	if alpha {
		claim.Annotations = map[string]string{
			storageutil.AlphaStorageClassAnnotation: "",
		}
	} else {
		claim.Annotations = map[string]string{
			storageutil.StorageClassAnnotation: "myclass-" + suffix,
		}

	}

	return &claim
}

// runInPodWithVolume runs a command in a pod with given claim mounted to /mnt directory.
func runInPodWithVolume(c clientset.Interface, ns, claimName, command string) {
	pod := &v1.Pod{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Pod",
			APIVersion: "v1",
		},
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "pvc-volume-tester-",
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:    "volume-tester",
					Image:   "gcr.io/google_containers/busybox:1.24",
					Command: []string{"/bin/sh"},
					Args:    []string{"-c", command},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "my-volume",
							MountPath: "/mnt/test",
						},
					},
				},
			},
			RestartPolicy: v1.RestartPolicyNever,
			Volumes: []v1.Volume{
				{
					Name: "my-volume",
					VolumeSource: v1.VolumeSource{
						PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
							ClaimName: claimName,
							ReadOnly:  false,
						},
					},
				},
			},
		},
	}
	pod, err := c.Core().Pods(ns).Create(pod)
	defer func() {
		framework.ExpectNoError(c.Core().Pods(ns).Delete(pod.Name, nil))
	}()
	framework.ExpectNoError(err, "Failed to create pod: %v", err)
	framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace))
}

func newStorageClass(pluginName, suffix string) *storage.StorageClass {
	if pluginName == "" {
		switch {
		case framework.ProviderIs("gke"), framework.ProviderIs("gce"):
			pluginName = "kubernetes.io/gce-pd"
		case framework.ProviderIs("aws"):
			pluginName = "kubernetes.io/aws-ebs"
		case framework.ProviderIs("openstack"):
			pluginName = "kubernetes.io/cinder"
		case framework.ProviderIs("vsphere"):
			pluginName = "kubernetes.io/vsphere-volume"
		}
	}

	return &storage.StorageClass{
		TypeMeta: metav1.TypeMeta{
			Kind: "StorageClass",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name: "myclass-" + suffix,
		},
		Provisioner: pluginName,
	}
}

func startExternalProvisioner(c clientset.Interface, ns string) *v1.Pod {
	podClient := c.Core().Pods(ns)

	provisionerPod := &v1.Pod{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Pod",
			APIVersion: "v1",
		},
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "external-provisioner-",
		},

		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:  "nfs-provisioner",
					Image: "quay.io/kubernetes_incubator/nfs-provisioner:v1.0.3",
					SecurityContext: &v1.SecurityContext{
						Capabilities: &v1.Capabilities{
							Add: []v1.Capability{"DAC_READ_SEARCH"},
						},
					},
					Args: []string{
						"-provisioner=" + externalPluginName,
						"-grace-period=0",
					},
					Ports: []v1.ContainerPort{
						{Name: "nfs", ContainerPort: 2049},
						{Name: "mountd", ContainerPort: 20048},
						{Name: "rpcbind", ContainerPort: 111},
						{Name: "rpcbind-udp", ContainerPort: 111, Protocol: v1.ProtocolUDP},
					},
					Env: []v1.EnvVar{
						{
							Name: "POD_IP",
							ValueFrom: &v1.EnvVarSource{
								FieldRef: &v1.ObjectFieldSelector{
									FieldPath: "status.podIP",
								},
							},
						},
					},
					ImagePullPolicy: v1.PullIfNotPresent,
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "export-volume",
							MountPath: "/export",
						},
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: "export-volume",
					VolumeSource: v1.VolumeSource{
						EmptyDir: &v1.EmptyDirVolumeSource{},
					},
				},
			},
		},
	}
	provisionerPod, err := podClient.Create(provisionerPod)
	framework.ExpectNoError(err, "Failed to create %s pod: %v", provisionerPod.Name, err)

	framework.ExpectNoError(framework.WaitForPodRunningInNamespace(c, provisionerPod))

	By("locating the provisioner pod")
	pod, err := podClient.Get(provisionerPod.Name, metav1.GetOptions{})
	framework.ExpectNoError(err, "Cannot locate the provisioner pod %v: %v", provisionerPod.Name, err)

	return pod
}
<|endoftext|>


<commit_before>package db_test

import (
	"fmt"
	"time"

	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/db"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("ResourceCacheLifecycle", func() {

	var resourceCacheLifecycle db.ResourceCacheLifecycle

	BeforeEach(func() {
		resourceCacheLifecycle = db.NewResourceCacheLifecycle(dbConn)
	})

	Describe("CleanUpInvalidCaches", func() {
		Context("the resource cache is used by a build", func() {
			resourceCacheForOneOffBuild := func() (db.UsedResourceCache, db.Build) {
				build, err := defaultTeam.CreateOneOffBuild()
				Expect(err).ToNot(HaveOccurred())
				return createResourceCacheWithUser(db.ForBuild(build.ID())), build
			}

			resourceCacheForJobBuild := func() (db.UsedResourceCache, db.Build) {
				build, err := defaultJob.CreateBuild()
				Expect(err).ToNot(HaveOccurred())
				return createResourceCacheWithUser(db.ForBuild(build.ID())), build
			}

			Context("when its a one off build", func() {
				It("doesn't delete the resource cache", func() {
					_, _ = resourceCacheForOneOffBuild()

					err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
					Expect(err).ToNot(HaveOccurred())
					Expect(countResourceCaches()).ToNot(BeZero())
				})

				Context("when the build goes away", func() {
					BeforeEach(func() {
						_, build := resourceCacheForOneOffBuild()

						_, err := build.Delete()
						Expect(err).ToNot(HaveOccurred())

						Expect(countResourceCaches()).ToNot(BeZero())

						err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
						Expect(err).ToNot(HaveOccurred())
					})

					It("deletes the resource cache", func() {
						Expect(countResourceCaches()).To(BeZero())
					})
				})

				Context("when the cache is for a saved image resource version for a finished build", func() {
					setBuildStatus := func(a db.BuildStatus) {
						resourceCache, build := resourceCacheForOneOffBuild()

						err := build.SaveImageResourceVersion(resourceCache)
						Expect(err).ToNot(HaveOccurred())

						err = build.SetInterceptible(false)
						Expect(err).ToNot(HaveOccurred())

						err = build.Finish(a)
						Expect(err).ToNot(HaveOccurred())

						err = resourceCacheLifecycle.CleanUsesForFinishedBuilds(logger)
						Expect(err).ToNot(HaveOccurred())
					}

					Context("when the build has succeeded", func() {
						It("does not remove the image resource cache", func() {
							setBuildStatus(db.BuildStatusSucceeded)
							Expect(countResourceCaches()).ToNot(BeZero())

							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
							Expect(err).ToNot(HaveOccurred())

							Expect(countResourceCaches()).ToNot(BeZero())
						})
					})

					Context("when build has not succeeded", func() {
						It("does not removes the image resource cache", func() {
							setBuildStatus(db.BuildStatusFailed)
							Expect(countResourceCaches()).ToNot(BeZero())

							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
							Expect(err).ToNot(HaveOccurred())

							Expect(countResourceCaches()).ToNot(BeZero())
						})
					})
				})
			})

			Context("when its a build of a job in a pipeline", func() {
				Context("when the cache is for a saved image resource version for a finished build", func() {
					setBuildStatus := func(a db.BuildStatus) (db.UsedResourceCache, db.Build) {
						resourceCache, build := resourceCacheForJobBuild()
						Expect(build.JobID()).ToNot(BeZero())

						err := build.SaveImageResourceVersion(resourceCache)
						Expect(err).ToNot(HaveOccurred())

						err = build.SetInterceptible(false)
						Expect(err).ToNot(HaveOccurred())

						err = build.Finish(a)
						Expect(err).ToNot(HaveOccurred())

						err = resourceCacheLifecycle.CleanUsesForFinishedBuilds(logger)
						Expect(err).ToNot(HaveOccurred())
						return resourceCache, build
					}

					Context("when the build has succeeded", func() {
						It("does not remove the resource cache for the most recent build", func() {
							setBuildStatus(db.BuildStatusSucceeded)
							Expect(countResourceCaches()).To(Equal(1))

							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
							Expect(err).ToNot(HaveOccurred())

							Expect(countResourceCaches()).To(Equal(1))
						})

						It("removes resource caches for previous finished builds", func() {
							resourceCache1, _ := setBuildStatus(db.BuildStatusFailed)
							resourceCache2, _ := setBuildStatus(db.BuildStatusSucceeded)
							Expect(resourceCache1.ID()).ToNot(Equal(resourceCache2.ID()))

							Expect(countResourceCaches()).To(Equal(2))

							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
							Expect(err).ToNot(HaveOccurred())

							Expect(countResourceCaches()).To(Equal(1))
						})
						It("does not remove the resource caches from other jobs", func() {
							By("creating a second pipeline")
							secondPipeline, _, err := defaultTeam.SavePipeline(atc.PipelineRef{Name: "second-pipeline"}, atc.Config{
								Jobs: atc.JobConfigs{
									{
										Name: "some-job",
									},
								},
								Resources: atc.ResourceConfigs{
									{
										Name: "some-resource",
										Type: "some-base-resource-type",
										Source: atc.Source{
											"some": "source",
										},
									},
								},
								ResourceTypes: atc.ResourceTypes{
									{
										Name: "some-type",
										Type: "some-base-resource-type",
										Source: atc.Source{
											"some-type": "source",
										},
									},
								},
							}, db.ConfigVersion(0), false)
							Expect(err).NotTo(HaveOccurred())

							By("creating an image resource cache tied to the job in the second pipeline")
							job, _, err := secondPipeline.Job("some-job")
							Expect(err).ToNot(HaveOccurred())
							build, err := job.CreateBuild()
							Expect(err).ToNot(HaveOccurred())
							resourceCache := createResourceCacheWithUser(db.ForBuild(build.ID()))

							err = build.SaveImageResourceVersion(resourceCache)
							Expect(err).ToNot(HaveOccurred())

							err = build.SetInterceptible(false)
							Expect(err).ToNot(HaveOccurred())

							By("creating an image resource cached in the default pipeline")
							setBuildStatus(db.BuildStatusSucceeded)

							Expect(countResourceCaches()).To(Equal(2))
						})
					})

					Context("when build has not succeeded", func() {
						It("does not remove the image resource cache", func() {
							setBuildStatus(db.BuildStatusFailed)
							Expect(countResourceCaches()).ToNot(BeZero())

							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
							Expect(err).ToNot(HaveOccurred())

							Expect(countResourceCaches()).ToNot(BeZero())
						})
					})
				})
			})
		})

		Context("the resource cache is used by a container", func() {
			var (
				container      db.CreatingContainer
				containerOwner db.ContainerOwner
			)

			BeforeEach(func() {
				var err error
				resourceConfig, err := resourceConfigFactory.FindOrCreateResourceConfig(
					"some-base-resource-type",
					atc.Source{
						"some": "source",
					},
					atc.VersionedResourceTypes{},
				)
				Expect(err).ToNot(HaveOccurred())

				containerOwner = db.NewResourceConfigCheckSessionContainerOwner(
					resourceConfig.ID(),
					resourceConfig.OriginBaseResourceType().ID,
					db.ContainerOwnerExpiries{},
				)

				container, err = defaultWorker.CreateContainer(containerOwner, db.ContainerMetadata{})
				Expect(err).ToNot(HaveOccurred())

				_ = createResourceCacheWithUser(db.ForContainer(container.ID()))
			})

			Context("and the container still exists", func() {
				BeforeEach(func() {
					err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
					Expect(err).ToNot(HaveOccurred())
				})

				It("doesn't delete the resource cache", func() {
					Expect(countResourceCaches()).ToNot(BeZero())
				})
			})

			Context("and the container has been deleted", func() {
				BeforeEach(func() {
					createdContainer, err := container.Created()
					Expect(err).ToNot(HaveOccurred())

					destroyingContainer, err := createdContainer.Destroying()
					Expect(err).ToNot(HaveOccurred())

					_, err = destroyingContainer.Destroy()
					Expect(err).ToNot(HaveOccurred())

					err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
					Expect(err).ToNot(HaveOccurred())
				})

				It("deletes the resource cache", func() {
					Expect(countResourceCaches()).To(BeZero())
				})
			})
		})

		Context("when the cache is for a custom resource type", func() {
			It("does not remove the cache if the type is still configured", func() {
				_, err := resourceConfigFactory.FindOrCreateResourceConfig(
					"some-type",
					atc.Source{
						"some": "source",
					},
					atc.VersionedResourceTypes{
						atc.VersionedResourceType{
							ResourceType: atc.ResourceType{
								Name: "some-type",
								Type: "some-base-resource-type",
								Source: atc.Source{
									"some": "source",
								},
							},
							Version: atc.Version{"showme": "whatyougot"},
						},
					},
				)
				Expect(err).ToNot(HaveOccurred())

				Expect(countResourceCaches()).ToNot(BeZero())
				err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
				Expect(err).ToNot(HaveOccurred())

				Expect(countResourceCaches()).ToNot(BeZero())
			})

			It("removes the cache if the type is no longer configured", func() {
				_, err := resourceConfigFactory.FindOrCreateResourceConfig(
					"some-type",
					atc.Source{
						"some": "source",
					},
					atc.VersionedResourceTypes{
						atc.VersionedResourceType{
							ResourceType: atc.ResourceType{
								Name: "some-type",
								Type: "some-base-resource-type",
								Source: atc.Source{
									"some": "source",
								},
							},
							Version: atc.Version{"showme": "whatyougot"},
						},
					},
				)
				Expect(err).ToNot(HaveOccurred())

				err = resourceConfigCheckSessionLifecycle.CleanInactiveResourceConfigCheckSessions()
				Expect(err).ToNot(HaveOccurred())

				err = resourceConfigFactory.CleanUnreferencedConfigs(0)
				Expect(err).ToNot(HaveOccurred())

				Expect(countResourceCaches()).ToNot(BeZero())

				err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
				Expect(err).ToNot(HaveOccurred())

				Expect(countResourceCaches()).To(BeZero())
			})
		})

		Context("when the cache is for a resource version used as an input for the next build of a job", func() {
			It("does not remove the cache", func() {
				err := defaultPipeline.Unpause()
				Expect(err).ToNot(HaveOccurred())

				resourceConfigScope, err := defaultResource.SetResourceConfig(
					atc.Source{"some": "source"},
					atc.VersionedResourceTypes{},
				)
				Expect(err).ToNot(HaveOccurred())

				containerOwner := db.NewResourceConfigCheckSessionContainerOwner(
					resourceConfigScope.ResourceConfig().ID(),
					resourceConfigScope.ResourceConfig().OriginBaseResourceType().ID,
					db.ContainerOwnerExpiries{},
				)

				container, err := defaultWorker.CreateContainer(containerOwner, db.ContainerMetadata{})
				Expect(err).ToNot(HaveOccurred())

				_ = createResourceCacheWithUser(db.ForContainer(container.ID()))

				err = resourceConfigScope.SaveVersions(nil, []atc.Version{{"some": "version"}})
				Expect(err).ToNot(HaveOccurred())

				resourceConfigVersion, found, err := resourceConfigScope.FindVersion(atc.Version{"some": "version"})
				Expect(found).To(BeTrue())
				Expect(err).ToNot(HaveOccurred())

				err = defaultJob.SaveNextInputMapping(db.InputMapping{
					"some-resource": db.InputResult{
						Input: &db.AlgorithmInput{
							AlgorithmVersion: db.AlgorithmVersion{
								Version:    db.ResourceVersion(convertToMD5(atc.Version(resourceConfigVersion.Version()))),
								ResourceID: defaultResource.ID(),
							},
						},
						PassedBuildIDs: []int{},
					},
				}, true)
				Expect(err).ToNot(HaveOccurred())

				createdContainer, err := container.Created()
				Expect(err).ToNot(HaveOccurred())

				destroyingContainer, err := createdContainer.Destroying()
				Expect(err).ToNot(HaveOccurred())

				_, err = destroyingContainer.Destroy()
				Expect(err).ToNot(HaveOccurred())

				Expect(countResourceCaches()).ToNot(BeZero())

				err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
				Expect(err).ToNot(HaveOccurred())

				Expect(countResourceCaches()).ToNot(BeZero())
			})
		})
	})
})

func countResourceCaches() int {
	var result int
	err := psql.Select("count(*)").
		From("resource_caches").
		RunWith(dbConn).
		QueryRow().
		Scan(&result)
	Expect(err).ToNot(HaveOccurred())

	return result

}

func createResourceCacheWithUser(resourceCacheUser db.ResourceCacheUser) db.UsedResourceCache {
	usedResourceCache, err := resourceCacheFactory.FindOrCreateResourceCache(
		resourceCacheUser,
		"some-base-resource-type",
		atc.Version{"some": "version"},
		atc.Source{
			"some": "source",
		},
		atc.Params{"some": fmt.Sprintf("param-%d", time.Now().UnixNano())},
		atc.VersionedResourceTypes{},
	)
	Expect(err).ToNot(HaveOccurred())

	return usedResourceCache
}
<commit_msg>restructure resource cache lifestyle tests<commit_after>package db_test

import (
	"fmt"
	"time"

	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/db/dbtest"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("ResourceCacheLifecycle", func() {

	var resourceCacheLifecycle db.ResourceCacheLifecycle

	BeforeEach(func() {
		resourceCacheLifecycle = db.NewResourceCacheLifecycle(dbConn)
	})

	Describe("CleanUpInvalidCaches", func() {
		Context("the resource cache is used by a build", func() {
			resourceCacheForOneOffBuild := func() (db.UsedResourceCache, db.Build) {
				build, err := defaultTeam.CreateOneOffBuild()
				Expect(err).ToNot(HaveOccurred())
				return createResourceCacheWithUser(db.ForBuild(build.ID())), build
			}

			resourceCacheForJobBuild := func() (db.UsedResourceCache, db.Build) {
				build, err := defaultJob.CreateBuild()
				Expect(err).ToNot(HaveOccurred())
				return createResourceCacheWithUser(db.ForBuild(build.ID())), build
			}

			Context("when its a one off build", func() {
				It("doesn't delete the resource cache", func() {
					_, _ = resourceCacheForOneOffBuild()

					err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
					Expect(err).ToNot(HaveOccurred())
					Expect(countResourceCaches()).ToNot(BeZero())
				})

				Context("when the build goes away", func() {
					BeforeEach(func() {
						_, build := resourceCacheForOneOffBuild()

						_, err := build.Delete()
						Expect(err).ToNot(HaveOccurred())

						Expect(countResourceCaches()).ToNot(BeZero())

						err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
						Expect(err).ToNot(HaveOccurred())
					})

					It("deletes the resource cache", func() {
						Expect(countResourceCaches()).To(BeZero())
					})
				})

				Context("when the cache is for a saved image resource version for a finished build", func() {
					setBuildStatus := func(a db.BuildStatus) {
						resourceCache, build := resourceCacheForOneOffBuild()

						err := build.SaveImageResourceVersion(resourceCache)
						Expect(err).ToNot(HaveOccurred())

						err = build.SetInterceptible(false)
						Expect(err).ToNot(HaveOccurred())

						err = build.Finish(a)
						Expect(err).ToNot(HaveOccurred())

						err = resourceCacheLifecycle.CleanUsesForFinishedBuilds(logger)
						Expect(err).ToNot(HaveOccurred())
					}

					Context("when the build has succeeded", func() {
						It("does not remove the image resource cache", func() {
							setBuildStatus(db.BuildStatusSucceeded)
							Expect(countResourceCaches()).ToNot(BeZero())

							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
							Expect(err).ToNot(HaveOccurred())

							Expect(countResourceCaches()).ToNot(BeZero())
						})
					})

					Context("when build has not succeeded", func() {
						It("does not removes the image resource cache", func() {
							setBuildStatus(db.BuildStatusFailed)
							Expect(countResourceCaches()).ToNot(BeZero())

							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
							Expect(err).ToNot(HaveOccurred())

							Expect(countResourceCaches()).ToNot(BeZero())
						})
					})
				})
			})

			Context("when its a build of a job in a pipeline", func() {
				Context("when the cache is for a saved image resource version for a finished build", func() {
					setBuildStatus := func(a db.BuildStatus) (db.UsedResourceCache, db.Build) {
						resourceCache, build := resourceCacheForJobBuild()
						Expect(build.JobID()).ToNot(BeZero())

						err := build.SaveImageResourceVersion(resourceCache)
						Expect(err).ToNot(HaveOccurred())

						err = build.SetInterceptible(false)
						Expect(err).ToNot(HaveOccurred())

						err = build.Finish(a)
						Expect(err).ToNot(HaveOccurred())

						err = resourceCacheLifecycle.CleanUsesForFinishedBuilds(logger)
						Expect(err).ToNot(HaveOccurred())
						return resourceCache, build
					}

					Context("when the build has succeeded", func() {
						It("does not remove the resource cache for the most recent build", func() {
							setBuildStatus(db.BuildStatusSucceeded)
							Expect(countResourceCaches()).To(Equal(1))

							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
							Expect(err).ToNot(HaveOccurred())

							Expect(countResourceCaches()).To(Equal(1))
						})

						It("removes resource caches for previous finished builds", func() {
							resourceCache1, _ := setBuildStatus(db.BuildStatusFailed)
							resourceCache2, _ := setBuildStatus(db.BuildStatusSucceeded)
							Expect(resourceCache1.ID()).ToNot(Equal(resourceCache2.ID()))

							Expect(countResourceCaches()).To(Equal(2))

							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
							Expect(err).ToNot(HaveOccurred())

							Expect(countResourceCaches()).To(Equal(1))
						})
						It("does not remove the resource caches from other jobs", func() {
							By("creating a second pipeline")
							secondPipeline, _, err := defaultTeam.SavePipeline(atc.PipelineRef{Name: "second-pipeline"}, atc.Config{
								Jobs: atc.JobConfigs{
									{
										Name: "some-job",
									},
								},
								Resources: atc.ResourceConfigs{
									{
										Name: "some-resource",
										Type: "some-base-resource-type",
										Source: atc.Source{
											"some": "source",
										},
									},
								},
								ResourceTypes: atc.ResourceTypes{
									{
										Name: "some-type",
										Type: "some-base-resource-type",
										Source: atc.Source{
											"some-type": "source",
										},
									},
								},
							}, db.ConfigVersion(0), false)
							Expect(err).NotTo(HaveOccurred())

							By("creating an image resource cache tied to the job in the second pipeline")
							job, _, err := secondPipeline.Job("some-job")
							Expect(err).ToNot(HaveOccurred())
							build, err := job.CreateBuild()
							Expect(err).ToNot(HaveOccurred())
							resourceCache := createResourceCacheWithUser(db.ForBuild(build.ID()))

							err = build.SaveImageResourceVersion(resourceCache)
							Expect(err).ToNot(HaveOccurred())

							err = build.SetInterceptible(false)
							Expect(err).ToNot(HaveOccurred())

							By("creating an image resource cached in the default pipeline")
							setBuildStatus(db.BuildStatusSucceeded)

							Expect(countResourceCaches()).To(Equal(2))
						})
					})

					Context("when build has not succeeded", func() {
						It("does not remove the image resource cache", func() {
							setBuildStatus(db.BuildStatusFailed)
							Expect(countResourceCaches()).ToNot(BeZero())

							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
							Expect(err).ToNot(HaveOccurred())

							Expect(countResourceCaches()).ToNot(BeZero())
						})
					})
				})
			})
		})

		Context("the resource cache is used by a container", func() {
			var (
				container      db.CreatingContainer
				containerOwner db.ContainerOwner
			)

			BeforeEach(func() {
				var err error
				resourceConfig, err := resourceConfigFactory.FindOrCreateResourceConfig(
					"some-base-resource-type",
					atc.Source{
						"some": "source",
					},
					atc.VersionedResourceTypes{},
				)
				Expect(err).ToNot(HaveOccurred())

				containerOwner = db.NewResourceConfigCheckSessionContainerOwner(
					resourceConfig.ID(),
					resourceConfig.OriginBaseResourceType().ID,
					db.ContainerOwnerExpiries{},
				)

				container, err = defaultWorker.CreateContainer(containerOwner, db.ContainerMetadata{})
				Expect(err).ToNot(HaveOccurred())

				_ = createResourceCacheWithUser(db.ForContainer(container.ID()))
			})

			Context("and the container still exists", func() {
				BeforeEach(func() {
					err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
					Expect(err).ToNot(HaveOccurred())
				})

				It("doesn't delete the resource cache", func() {
					Expect(countResourceCaches()).ToNot(BeZero())
				})
			})

			Context("and the container has been deleted", func() {
				BeforeEach(func() {
					createdContainer, err := container.Created()
					Expect(err).ToNot(HaveOccurred())

					destroyingContainer, err := createdContainer.Destroying()
					Expect(err).ToNot(HaveOccurred())

					_, err = destroyingContainer.Destroy()
					Expect(err).ToNot(HaveOccurred())

					err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
					Expect(err).ToNot(HaveOccurred())
				})

				It("deletes the resource cache", func() {
					Expect(countResourceCaches()).To(BeZero())
				})
			})
		})

		Context("when the cache is for a custom resource type", func() {
			It("does not remove the cache if the type is still configured", func() {
				_, err := resourceConfigFactory.FindOrCreateResourceConfig(
					"some-type",
					atc.Source{
						"some": "source",
					},
					atc.VersionedResourceTypes{
						atc.VersionedResourceType{
							ResourceType: atc.ResourceType{
								Name: "some-type",
								Type: "some-base-resource-type",
								Source: atc.Source{
									"some": "source",
								},
							},
							Version: atc.Version{"showme": "whatyougot"},
						},
					},
				)
				Expect(err).ToNot(HaveOccurred())

				Expect(countResourceCaches()).ToNot(BeZero())
				err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
				Expect(err).ToNot(HaveOccurred())

				Expect(countResourceCaches()).ToNot(BeZero())
			})

			It("removes the cache if the type is no longer configured", func() {
				_, err := resourceConfigFactory.FindOrCreateResourceConfig(
					"some-type",
					atc.Source{
						"some": "source",
					},
					atc.VersionedResourceTypes{
						atc.VersionedResourceType{
							ResourceType: atc.ResourceType{
								Name: "some-type",
								Type: "some-base-resource-type",
								Source: atc.Source{
									"some": "source",
								},
							},
							Version: atc.Version{"showme": "whatyougot"},
						},
					},
				)
				Expect(err).ToNot(HaveOccurred())

				err = resourceConfigCheckSessionLifecycle.CleanInactiveResourceConfigCheckSessions()
				Expect(err).ToNot(HaveOccurred())

				err = resourceConfigFactory.CleanUnreferencedConfigs(0)
				Expect(err).ToNot(HaveOccurred())

				Expect(countResourceCaches()).ToNot(BeZero())

				err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
				Expect(err).ToNot(HaveOccurred())

				Expect(countResourceCaches()).To(BeZero())
			})
		})

		Context("when the cache is for a resource version used as an input for the next build of a job", func() {
			It("does not remove the cache", func() {
				scenario := dbtest.Setup(
					builder.WithPipeline(atc.Config{
						Resources: atc.ResourceConfigs{
							{
								Name: "some-resource",
								Type: "some-base-resource-type",
								Source: atc.Source{
									"some": "source",
								},
							},
						},
					}),
					builder.WithResourceVersions("some-resource", atc.Version{"some": "version"}),
				)

				rc, found, err := resourceConfigFactory.FindResourceConfigByID(scenario.Resource("some-resource").ResourceConfigID())
				Expect(found).To(BeTrue())
				Expect(err).ToNot(HaveOccurred())

				resourceConfigScope, err := rc.FindOrCreateScope(scenario.Resource("some-resource"))
				Expect(err).ToNot(HaveOccurred())

				containerOwner := db.NewResourceConfigCheckSessionContainerOwner(
					resourceConfigScope.ResourceConfig().ID(),
					resourceConfigScope.ResourceConfig().OriginBaseResourceType().ID,
					db.ContainerOwnerExpiries{},
				)

				container, err := defaultWorker.CreateContainer(containerOwner, db.ContainerMetadata{})
				Expect(err).ToNot(HaveOccurred())

				_ = createResourceCacheWithUser(db.ForContainer(container.ID()))

				resourceConfigVersion, found, err := resourceConfigScope.FindVersion(atc.Version{"some": "version"})
				Expect(found).To(BeTrue())
				Expect(err).ToNot(HaveOccurred())

				err = defaultJob.SaveNextInputMapping(db.InputMapping{
					"some-resource": db.InputResult{
						Input: &db.AlgorithmInput{
							AlgorithmVersion: db.AlgorithmVersion{
								Version:    db.ResourceVersion(convertToMD5(atc.Version(resourceConfigVersion.Version()))),
								ResourceID: scenario.Resource("some-resource").ID(),
							},
						},
						PassedBuildIDs: []int{},
					},
				}, true)
				Expect(err).ToNot(HaveOccurred())

				createdContainer, err := container.Created()
				Expect(err).ToNot(HaveOccurred())

				destroyingContainer, err := createdContainer.Destroying()
				Expect(err).ToNot(HaveOccurred())

				_, err = destroyingContainer.Destroy()
				Expect(err).ToNot(HaveOccurred())

				Expect(countResourceCaches()).ToNot(BeZero())

				err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
				Expect(err).ToNot(HaveOccurred())

				Expect(countResourceCaches()).ToNot(BeZero())
			})
		})
	})
})

func countResourceCaches() int {
	var result int
	err := psql.Select("count(*)").
		From("resource_caches").
		RunWith(dbConn).
		QueryRow().
		Scan(&result)
	Expect(err).ToNot(HaveOccurred())

	return result

}

func createResourceCacheWithUser(resourceCacheUser db.ResourceCacheUser) db.UsedResourceCache {
	usedResourceCache, err := resourceCacheFactory.FindOrCreateResourceCache(
		resourceCacheUser,
		"some-base-resource-type",
		atc.Version{"some": "version"},
		atc.Source{
			"some": "source",
		},
		atc.Params{"some": fmt.Sprintf("param-%d", time.Now().UnixNano())},
		atc.VersionedResourceTypes{},
	)
	Expect(err).ToNot(HaveOccurred())

	return usedResourceCache
}
<|endoftext|>


<commit_before>//go:build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"os"
	"os/exec"
	"strings"
	"testing"
)

// TestDockerFlags makes sure the --docker-env and --docker-opt parameters are respected
func TestDockerFlags(t *testing.T) {
	if NoneDriver() {
		t.Skip("skipping: none driver does not support ssh or bundle docker")
	}
	if ContainerRuntime() != "docker" {
		t.Skipf("skipping: only runs with docker container runtime, currently testing %s", ContainerRuntime())
	}
	MaybeParallel(t)

	profile := UniqueProfileName("docker-flags")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
	defer CleanupWithLogs(t, profile, cancel)

	// Use the most verbose logging for the simplest test. If it fails, something is very wrong.
	args := append([]string{"start", "-p", profile, "--cache-images=false", "--memory=2048", "--install-addons=false", "--wait=false", "--docker-env=FOO=BAR", "--docker-env=BAZ=BAT", "--docker-opt=debug", "--docker-opt=icc=true", "--alsologtostderr", "-v=5"}, StartArgs()...)
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed to start minikube with args: %q : %v", rr.Command(), err)
	}

	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "sudo systemctl show docker --property=Environment --no-pager"))
	if err != nil {
		t.Errorf("failed to 'systemctl show docker' inside minikube. args %q: %v", rr.Command(), err)
	}

	for _, envVar := range []string{"FOO=BAR", "BAZ=BAT"} {
		if !strings.Contains(rr.Stdout.String(), envVar) {
			t.Errorf("expected env key/value %q to be passed to minikube's docker and be included in: *%q*.", envVar, rr.Stdout)
		}
	}

	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "sudo systemctl show docker --property=ExecStart --no-pager"))
	if err != nil {
		t.Errorf("failed on the second 'systemctl show docker' inside minikube. args %q: %v", rr.Command(), err)
	}
	for _, opt := range []string{"--debug", "--icc=true"} {
		if !strings.Contains(rr.Stdout.String(), opt) {
			t.Fatalf("expected %q output to have include *%s* . output: %q", rr.Command(), opt, rr.Stdout)
		}
	}
}

// TestForceSystemdFlag tests the --force-systemd flag, as one would expect.
func TestForceSystemdFlag(t *testing.T) {
	if NoneDriver() {
		t.Skip("skipping: none driver does not support ssh or bundle docker")
	}
	MaybeParallel(t)

	profile := UniqueProfileName("force-systemd-flag")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
	defer CleanupWithLogs(t, profile, cancel)

	// Use the most verbose logging for the simplest test. If it fails, something is very wrong.
	args := append([]string{"start", "-p", profile, "--memory=2048", "--force-systemd", "--alsologtostderr", "-v=5"}, StartArgs()...)
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed to start minikube with args: %q : %v", rr.Command(), err)
	}

	containerRuntime := ContainerRuntime()
	switch containerRuntime {
	case "docker":
		validateDockerSystemd(ctx, t, profile)
	case "containerd":
		validateContainerdSystemd(ctx, t, profile)
	case "crio":
		validateCrioSystemd(ctx, t, profile)
	}

}

// validateDockerSystemd makes sure the --force-systemd flag worked with the docker container runtime
func validateDockerSystemd(ctx context.Context, t *testing.T, profile string) {
	rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "docker info --format {{.CgroupDriver}}"))
	if err != nil {
		t.Errorf("failed to get docker cgroup driver. args %q: %v", rr.Command(), err)
	}
	if !strings.Contains(rr.Output(), "systemd") {
		t.Fatalf("expected systemd cgroup driver, got: %v", rr.Output())
	}
}

// validateContainerdSystemd makes sure the --force-systemd flag worked with the containerd container runtime
func validateContainerdSystemd(ctx context.Context, t *testing.T, profile string) {
	rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "cat /etc/containerd/config.toml"))
	if err != nil {
		t.Errorf("failed to get containerd cgroup driver. args %q: %v", rr.Command(), err)
	}
	if !strings.Contains(rr.Output(), "SystemdCgroup = true") {
		t.Fatalf("expected systemd cgroup driver, got: %v", rr.Output())
	}
}

// validateCrioSystemd makes sure the --force-systemd flag worked with the cri-o container runtime
func validateCrioSystemd(ctx context.Context, t *testing.T, profile string) {
	rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "cat /etc/crio/crio.conf.d/02-crio.conf"))
	if err != nil {
		t.Errorf("failed to get cri-o cgroup driver. args %q: %v", rr.Command(), err)
	}
	if strings.Contains(rr.Output(), "cgroup_manager = systemd") {
		t.Fatalf("expected systemd cgroup driver, got: %v", rr.Output())
	}
}

// TestForceSystemdEnv makes sure the MINIKUBE_FORCE_SYSTEMD environment variable works just as well as the --force-systemd flag
func TestForceSystemdEnv(t *testing.T) {
	if NoneDriver() {
		t.Skip("skipping: none driver does not support ssh or bundle docker")
	}
	MaybeParallel(t)

	profile := UniqueProfileName("force-systemd-env")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
	defer CleanupWithLogs(t, profile, cancel)

	args := append([]string{"start", "-p", profile, "--memory=2048", "--alsologtostderr", "-v=5"}, StartArgs()...)
	cmd := exec.CommandContext(ctx, Target(), args...)
	cmd.Env = append(os.Environ(), "MINIKUBE_FORCE_SYSTEMD=true")
	rr, err := Run(t, cmd)
	if err != nil {
		t.Errorf("failed to start minikube with args: %q : %v", rr.Command(), err)
	}
	containerRuntime := ContainerRuntime()
	switch containerRuntime {
	case "docker":
		validateDockerSystemd(ctx, t, profile)
	case "containerd":
		validateContainerdSystemd(ctx, t, profile)
	}
}
<commit_msg>add quotes<commit_after>//go:build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"os"
	"os/exec"
	"strings"
	"testing"
)

// TestDockerFlags makes sure the --docker-env and --docker-opt parameters are respected
func TestDockerFlags(t *testing.T) {
	if NoneDriver() {
		t.Skip("skipping: none driver does not support ssh or bundle docker")
	}
	if ContainerRuntime() != "docker" {
		t.Skipf("skipping: only runs with docker container runtime, currently testing %s", ContainerRuntime())
	}
	MaybeParallel(t)

	profile := UniqueProfileName("docker-flags")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
	defer CleanupWithLogs(t, profile, cancel)

	// Use the most verbose logging for the simplest test. If it fails, something is very wrong.
	args := append([]string{"start", "-p", profile, "--cache-images=false", "--memory=2048", "--install-addons=false", "--wait=false", "--docker-env=FOO=BAR", "--docker-env=BAZ=BAT", "--docker-opt=debug", "--docker-opt=icc=true", "--alsologtostderr", "-v=5"}, StartArgs()...)
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed to start minikube with args: %q : %v", rr.Command(), err)
	}

	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "sudo systemctl show docker --property=Environment --no-pager"))
	if err != nil {
		t.Errorf("failed to 'systemctl show docker' inside minikube. args %q: %v", rr.Command(), err)
	}

	for _, envVar := range []string{"FOO=BAR", "BAZ=BAT"} {
		if !strings.Contains(rr.Stdout.String(), envVar) {
			t.Errorf("expected env key/value %q to be passed to minikube's docker and be included in: *%q*.", envVar, rr.Stdout)
		}
	}

	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "sudo systemctl show docker --property=ExecStart --no-pager"))
	if err != nil {
		t.Errorf("failed on the second 'systemctl show docker' inside minikube. args %q: %v", rr.Command(), err)
	}
	for _, opt := range []string{"--debug", "--icc=true"} {
		if !strings.Contains(rr.Stdout.String(), opt) {
			t.Fatalf("expected %q output to have include *%s* . output: %q", rr.Command(), opt, rr.Stdout)
		}
	}
}

// TestForceSystemdFlag tests the --force-systemd flag, as one would expect.
func TestForceSystemdFlag(t *testing.T) {
	if NoneDriver() {
		t.Skip("skipping: none driver does not support ssh or bundle docker")
	}
	MaybeParallel(t)

	profile := UniqueProfileName("force-systemd-flag")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
	defer CleanupWithLogs(t, profile, cancel)

	// Use the most verbose logging for the simplest test. If it fails, something is very wrong.
	args := append([]string{"start", "-p", profile, "--memory=2048", "--force-systemd", "--alsologtostderr", "-v=5"}, StartArgs()...)
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed to start minikube with args: %q : %v", rr.Command(), err)
	}

	containerRuntime := ContainerRuntime()
	switch containerRuntime {
	case "docker":
		validateDockerSystemd(ctx, t, profile)
	case "containerd":
		validateContainerdSystemd(ctx, t, profile)
	case "crio":
		validateCrioSystemd(ctx, t, profile)
	}

}

// validateDockerSystemd makes sure the --force-systemd flag worked with the docker container runtime
func validateDockerSystemd(ctx context.Context, t *testing.T, profile string) {
	rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "docker info --format {{.CgroupDriver}}"))
	if err != nil {
		t.Errorf("failed to get docker cgroup driver. args %q: %v", rr.Command(), err)
	}
	if !strings.Contains(rr.Output(), "systemd") {
		t.Fatalf("expected systemd cgroup driver, got: %v", rr.Output())
	}
}

// validateContainerdSystemd makes sure the --force-systemd flag worked with the containerd container runtime
func validateContainerdSystemd(ctx context.Context, t *testing.T, profile string) {
	rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "cat /etc/containerd/config.toml"))
	if err != nil {
		t.Errorf("failed to get containerd cgroup driver. args %q: %v", rr.Command(), err)
	}
	if !strings.Contains(rr.Output(), "SystemdCgroup = true") {
		t.Fatalf("expected systemd cgroup driver, got: %v", rr.Output())
	}
}

// validateCrioSystemd makes sure the --force-systemd flag worked with the cri-o container runtime
func validateCrioSystemd(ctx context.Context, t *testing.T, profile string) {
	rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "cat /etc/crio/crio.conf.d/02-crio.conf"))
	if err != nil {
		t.Errorf("failed to get cri-o cgroup driver. args %q: %v", rr.Command(), err)
	}
	if strings.Contains(rr.Output(), "cgroup_manager = \"systemd\"") {
		t.Fatalf("expected systemd cgroup driver, got: %v", rr.Output())
	}
}

// TestForceSystemdEnv makes sure the MINIKUBE_FORCE_SYSTEMD environment variable works just as well as the --force-systemd flag
func TestForceSystemdEnv(t *testing.T) {
	if NoneDriver() {
		t.Skip("skipping: none driver does not support ssh or bundle docker")
	}
	MaybeParallel(t)

	profile := UniqueProfileName("force-systemd-env")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
	defer CleanupWithLogs(t, profile, cancel)

	args := append([]string{"start", "-p", profile, "--memory=2048", "--alsologtostderr", "-v=5"}, StartArgs()...)
	cmd := exec.CommandContext(ctx, Target(), args...)
	cmd.Env = append(os.Environ(), "MINIKUBE_FORCE_SYSTEMD=true")
	rr, err := Run(t, cmd)
	if err != nil {
		t.Errorf("failed to start minikube with args: %q : %v", rr.Command(), err)
	}
	containerRuntime := ContainerRuntime()
	switch containerRuntime {
	case "docker":
		validateDockerSystemd(ctx, t, profile)
	case "containerd":
		validateContainerdSystemd(ctx, t, profile)
	}
}
<|endoftext|>


<commit_before>package amipublisher

import (
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/log/prefixlogger"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"sync"
	"time"
)

const expiresAtFormat = "2006-01-02:15:04:05"

func expireResources(accountNames []string, logger log.Logger) error {
	waitGroup := &sync.WaitGroup{}
	for _, accountName := range accountNames {
		awsSession, err := createSession(accountName)
		if err != nil {
			return err
		}
		aRegionName := "us-east-1"
		aAwsService := createService(awsSession, aRegionName)
		regions, err := listRegions(aAwsService)
		if err != nil {
			return err
		}
		for _, region := range regions {
			awsService := aAwsService
			if region != aRegionName {
				awsService = createService(awsSession, region)
			}
			logger := prefixlogger.New(accountName+": "+region+": ", logger)
			waitGroup.Add(1)
			go expireRegionResources(awsService, waitGroup, logger)
		}

	}
	waitGroup.Wait()
	return nil
}

func expireRegionResources(awsService *ec2.EC2, waitGroup *sync.WaitGroup,
	logger log.Logger) {
	defer waitGroup.Done()
	filters := make([]*ec2.Filter, 1)
	values := make([]string, 1)
	values[0] = "ExpiresAt"
	filters[0] = &ec2.Filter{
		Name:   aws.String("tag-key"),
		Values: aws.StringSlice(values),
	}
	images, err := awsService.DescribeImages(&ec2.DescribeImagesInput{
		Filters: filters,
	})
	if err == nil {
		for _, image := range images.Images {
			expireImage(awsService, image, logger)
		}
	}
	snapshots, err := awsService.DescribeSnapshots(&ec2.DescribeSnapshotsInput{
		Filters: filters,
	})
	if err == nil {
		for _, snapshot := range snapshots.Snapshots {
			expireSnapshot(awsService, snapshot, logger)
		}
	}
}

func expireImage(awsService *ec2.EC2, image *ec2.Image, logger log.Logger) {
	if hasExpired(image.Tags) {
		err := deregisterAmi(awsService, aws.StringValue(image.ImageId))
		if err != nil {
			logger.Printf("error deleting: %s: %s\n", *image.ImageId, err)
		} else {
			logger.Printf("deleted: %s\n", *image.ImageId)
		}
	}
}

func expireSnapshot(awsService *ec2.EC2, snapshot *ec2.Snapshot,
	logger log.Logger) {
	if hasExpired(snapshot.Tags) {
		err := deleteSnapshot(awsService, aws.StringValue(snapshot.SnapshotId))
		if err != nil {
			logger.Printf("error deleting: %s: %s\n", *snapshot.SnapshotId, err)
		} else {
			logger.Printf("deleted: %s\n", *snapshot.SnapshotId)
		}
	}
}

func hasExpired(tags []*ec2.Tag) bool {
	for _, tag := range tags {
		if *tag.Key != "ExpiresAt" {
			continue
		}
		expirationTime, err := time.Parse(expiresAtFormat, *tag.Value)
		if err != nil {
			continue
		}
		return time.Now().After(expirationTime)
	}
	return false
}
<commit_msg>Use common "now" time in imagepublishers.amipublisher.ExpireResources().<commit_after>package amipublisher

import (
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/log/prefixlogger"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"sync"
	"time"
)

const expiresAtFormat = "2006-01-02:15:04:05"

func expireResources(accountNames []string, logger log.Logger) error {
	waitGroup := &sync.WaitGroup{}
	currentTime := time.Now() // Need a common "now" time.
	for _, accountName := range accountNames {
		awsSession, err := createSession(accountName)
		if err != nil {
			return err
		}
		aRegionName := "us-east-1"
		aAwsService := createService(awsSession, aRegionName)
		regions, err := listRegions(aAwsService)
		if err != nil {
			return err
		}
		for _, region := range regions {
			awsService := aAwsService
			if region != aRegionName {
				awsService = createService(awsSession, region)
			}
			logger := prefixlogger.New(accountName+": "+region+": ", logger)
			waitGroup.Add(1)
			go expireRegionResources(awsService, waitGroup, currentTime,
				logger)
		}

	}
	waitGroup.Wait()
	return nil
}

func expireRegionResources(awsService *ec2.EC2, waitGroup *sync.WaitGroup,
	currentTime time.Time, logger log.Logger) {
	defer waitGroup.Done()
	filters := make([]*ec2.Filter, 1)
	values := make([]string, 1)
	values[0] = "ExpiresAt"
	filters[0] = &ec2.Filter{
		Name:   aws.String("tag-key"),
		Values: aws.StringSlice(values),
	}
	images, err := awsService.DescribeImages(&ec2.DescribeImagesInput{
		Filters: filters,
	})
	if err == nil {
		for _, image := range images.Images {
			expireImage(awsService, image, currentTime, logger)
		}
	}
	snapshots, err := awsService.DescribeSnapshots(&ec2.DescribeSnapshotsInput{
		Filters: filters,
	})
	if err == nil {
		for _, snapshot := range snapshots.Snapshots {
			expireSnapshot(awsService, snapshot, currentTime, logger)
		}
	}
}

func expireImage(awsService *ec2.EC2, image *ec2.Image, currentTime time.Time,
	logger log.Logger) {
	if hasExpired(image.Tags, currentTime) {
		err := deregisterAmi(awsService, aws.StringValue(image.ImageId))
		if err != nil {
			logger.Printf("error deleting: %s: %s\n", *image.ImageId, err)
		} else {
			logger.Printf("deleted: %s\n", *image.ImageId)
		}
	}
}

func expireSnapshot(awsService *ec2.EC2, snapshot *ec2.Snapshot,
	currentTime time.Time, logger log.Logger) {
	if hasExpired(snapshot.Tags, currentTime) {
		err := deleteSnapshot(awsService, aws.StringValue(snapshot.SnapshotId))
		if err != nil {
			logger.Printf("error deleting: %s: %s\n", *snapshot.SnapshotId, err)
		} else {
			logger.Printf("deleted: %s\n", *snapshot.SnapshotId)
		}
	}
}

func hasExpired(tags []*ec2.Tag, currentTime time.Time) bool {
	for _, tag := range tags {
		if *tag.Key != "ExpiresAt" {
			continue
		}
		expirationTime, err := time.Parse(expiresAtFormat, *tag.Value)
		if err != nil {
			continue
		}
		return currentTime.After(expirationTime)
	}
	return false
}
<|endoftext|>


<commit_before>package vm

import (
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"strings"

	"github.com/goby-lang/goby/vm/errors"
)

const (
	invalidSplatArgument    = "Splat arguments must be a string, got: %s on argument #%d"
	couldNotCompleteRequest = "Could not complete request, %s"
	non200Response          = "Non-200 response, %s (%d)"
)

var (
	httpRequestClass  *RClass
	httpResponseClass *RClass
	httpClientClass   *RClass
)

// Class methods --------------------------------------------------------
var builtinHTTPClassMethods = []*BuiltinMethodObject{
	{
		// Sends a GET request to the target and returns the HTTP response as a string. Will error on non-200 responses, for more control over http requests look at the `start` method.
		Name: "get",
		Fn: func(receiver Object, sourceLine int, t *Thread, args []Object, blockFrame *normalCallFrame) Object {
			arg0, ok := args[0].(*StringObject)
			if !ok {
				return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, errors.WrongArgumentTypeFormatNum, 0, "String", args[0].Class().Name)
			}

			uri, err := url.Parse(arg0.value)

			if len(args) > 1 {
				var arr []string

				for i, v := range args[1:] {
					argn, ok := v.(*StringObject)
					if !ok {
						return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, invalidSplatArgument, v.Class().Name, i)
					}
					arr = append(arr, argn.value)
				}

				uri.Path = path.Join(arr...)
			}

			resp, err := http.Get(uri.String())
			if err != nil {
				return t.vm.InitErrorObject(errors.HTTPError, sourceLine, couldNotCompleteRequest, err)
			}
			if resp.StatusCode != http.StatusOK {
				return t.vm.InitErrorObject(errors.HTTPError, sourceLine, non200Response, resp.Status, resp.StatusCode)
			}

			content, err := ioutil.ReadAll(resp.Body)
			resp.Body.Close()

			if err != nil {
				return t.vm.InitErrorObject(errors.InternalError, sourceLine, err.Error())
			}

			return t.vm.InitStringObject(string(content))

		},
	}, {
		// Sends a POST request to the target with type header and body. Returns the HTTP response as a string. Will error on non-200 responses, for more control over http requests look at the `start` method.
		Name: "post",
		Fn: func(receiver Object, sourceLine int, t *Thread, args []Object, blockFrame *normalCallFrame) Object {
			if len(args) != 3 {
				return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, errors.WrongNumberOfArgument, 3, len(args))
			}

			arg0, ok := args[0].(*StringObject)
			if !ok {
				return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, errors.WrongArgumentTypeFormatNum, 0, "String", args[0].Class().Name)
			}
			host := arg0.value

			arg1, ok := args[1].(*StringObject)
			if !ok {
				return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, errors.WrongArgumentTypeFormatNum, 1, "String", args[0].Class().Name)
			}
			contentType := arg1.value

			arg2, ok := args[2].(*StringObject)
			if !ok {
				return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, errors.WrongArgumentTypeFormatNum, 2, "String", args[0].Class().Name)
			}
			body := arg2.value

			resp, err := http.Post(host, contentType, strings.NewReader(body))
			if err != nil {
				return t.vm.InitErrorObject(errors.HTTPError, sourceLine, couldNotCompleteRequest, err)
			}
			if resp.StatusCode != http.StatusOK {
				return t.vm.InitErrorObject(errors.HTTPError, sourceLine, non200Response, resp.Status, resp.StatusCode)
			}

			content, err := ioutil.ReadAll(resp.Body)
			resp.Body.Close()

			if err != nil {
				return t.vm.InitErrorObject(errors.InternalError, sourceLine, err.Error())
			}

			return t.vm.InitStringObject(string(content))

		},
	}, {
		// Sends a HEAD request to the target with type header and body. Returns the HTTP headers as a map[string]string. Will error on non-200 responses, for more control over http requests look at the `start` method.
		Name: "head",
		Fn: func(receiver Object, sourceLine int, t *Thread, args []Object, blockFrame *normalCallFrame) Object {
			arg0, ok := args[0].(*StringObject)
			if !ok {
				return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, errors.WrongArgumentTypeFormatNum, 0, "String", args[0].Class().Name)
			}

			uri, err := url.Parse(arg0.value)

			if len(args) > 1 {
				var arr []string

				for i, v := range args[1:] {
					argn, ok := v.(*StringObject)
					if !ok {
						return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, invalidSplatArgument, v.Class().Name, i)
					}
					arr = append(arr, argn.value)
				}

				uri.Path = path.Join(arr...)
			}

			resp, err := http.Head(uri.String())
			if err != nil {
				return t.vm.InitErrorObject(errors.HTTPError, sourceLine, couldNotCompleteRequest, err)
			}
			if resp.StatusCode != http.StatusOK {
				return t.vm.InitErrorObject(errors.HTTPError, sourceLine, non200Response, resp.Status, resp.StatusCode)
			}

			ret := t.vm.InitHashObject(map[string]Object{})

			for k, v := range resp.Header {
				ret.Pairs[k] = t.vm.InitStringObject(strings.Join(v, " "))
			}

			return ret

		},
	}, {
		// Starts an HTTP client. This method requires a block which takes a Net::HTTP::Client object. The return value of this method is the last evaluated value of the provided block.
		Name: "start",
		Fn: func(receiver Object, sourceLine int, t *Thread, args []Object, blockFrame *normalCallFrame) Object {
			if len(args) != 0 {
				return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, errors.WrongNumberOfArgument, 0, len(args))
			}

			gobyClient := httpClientClass.initializeInstance()

			result := t.builtinMethodYield(blockFrame, gobyClient)

			if err, ok := result.Target.(*Error); ok {
				return err //an Error object
			}

			return result.Target

		},
	},
}

// Internal functions ===================================================

// Functions for initialization -----------------------------------------

func initHTTPClass(vm *VM) {
	net := vm.loadConstant("Net", true)
	http := vm.initializeClass("HTTP")
	http.setBuiltinMethods(builtinHTTPClassMethods, true)
	initRequestClass(vm, http)
	initResponseClass(vm, http)
	initClientClass(vm, http)

	net.setClassConstant(http)

	// Use Goby code to extend request and response classes.
	vm.mainThread.execGobyLib("net/http/response.gb")
	vm.mainThread.execGobyLib("net/http/request.gb")
}

func initRequestClass(vm *VM, hc *RClass) *RClass {
	requestClass := vm.initializeClass("Request")
	hc.setClassConstant(requestClass)
	builtinHTTPRequestInstanceMethods := []*BuiltinMethodObject{}

	requestClass.setBuiltinMethods(builtinHTTPRequestInstanceMethods, false)

	httpRequestClass = requestClass
	return requestClass
}

func initResponseClass(vm *VM, hc *RClass) *RClass {
	responseClass := vm.initializeClass("Response")
	hc.setClassConstant(responseClass)
	builtinHTTPResponseInstanceMethods := []*BuiltinMethodObject{}

	responseClass.setBuiltinMethods(builtinHTTPResponseInstanceMethods, false)

	httpResponseClass = responseClass
	return responseClass
}
<commit_msg>Fix unhandled err<commit_after>package vm

import (
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"strings"

	"github.com/goby-lang/goby/vm/errors"
)

const (
	invalidSplatArgument    = "Splat arguments must be a string, got: %s on argument #%d"
	couldNotCompleteRequest = "Could not complete request, %s"
	non200Response          = "Non-200 response, %s (%d)"
)

var (
	httpRequestClass  *RClass
	httpResponseClass *RClass
	httpClientClass   *RClass
)

// Class methods --------------------------------------------------------
var builtinHTTPClassMethods = []*BuiltinMethodObject{
	{
		// Sends a GET request to the target and returns the HTTP response as a string. Will error on non-200 responses, for more control over http requests look at the `start` method.
		Name: "get",
		Fn: func(receiver Object, sourceLine int, t *Thread, args []Object, blockFrame *normalCallFrame) Object {
			arg0, ok := args[0].(*StringObject)
			if !ok {
				return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, errors.WrongArgumentTypeFormatNum, 0, "String", args[0].Class().Name)
			}

			uri, err := url.Parse(arg0.value)

			if len(args) > 1 {
				var arr []string

				for i, v := range args[1:] {
					argn, ok := v.(*StringObject)
					if !ok {
						return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, invalidSplatArgument, v.Class().Name, i)
					}
					arr = append(arr, argn.value)
				}

				uri.Path = path.Join(arr...)
			}

			resp, err := http.Get(uri.String())
			if err != nil {
				return t.vm.InitErrorObject(errors.HTTPError, sourceLine, couldNotCompleteRequest, err)
			}
			if resp.StatusCode != http.StatusOK {
				return t.vm.InitErrorObject(errors.HTTPError, sourceLine, non200Response, resp.Status, resp.StatusCode)
			}

			content, err := ioutil.ReadAll(resp.Body)
			resp.Body.Close()

			if err != nil {
				return t.vm.InitErrorObject(errors.InternalError, sourceLine, err.Error())
			}

			return t.vm.InitStringObject(string(content))

		},
	}, {
		// Sends a POST request to the target with type header and body. Returns the HTTP response as a string. Will error on non-200 responses, for more control over http requests look at the `start` method.
		Name: "post",
		Fn: func(receiver Object, sourceLine int, t *Thread, args []Object, blockFrame *normalCallFrame) Object {
			if len(args) != 3 {
				return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, errors.WrongNumberOfArgument, 3, len(args))
			}

			arg0, ok := args[0].(*StringObject)
			if !ok {
				return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, errors.WrongArgumentTypeFormatNum, 0, "String", args[0].Class().Name)
			}
			host := arg0.value

			arg1, ok := args[1].(*StringObject)
			if !ok {
				return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, errors.WrongArgumentTypeFormatNum, 1, "String", args[0].Class().Name)
			}
			contentType := arg1.value

			arg2, ok := args[2].(*StringObject)
			if !ok {
				return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, errors.WrongArgumentTypeFormatNum, 2, "String", args[0].Class().Name)
			}
			body := arg2.value

			resp, err := http.Post(host, contentType, strings.NewReader(body))
			if err != nil {
				return t.vm.InitErrorObject(errors.HTTPError, sourceLine, couldNotCompleteRequest, err)
			}
			if resp.StatusCode != http.StatusOK {
				return t.vm.InitErrorObject(errors.HTTPError, sourceLine, non200Response, resp.Status, resp.StatusCode)
			}

			content, err := ioutil.ReadAll(resp.Body)
			resp.Body.Close()

			if err != nil {
				return t.vm.InitErrorObject(errors.InternalError, sourceLine, err.Error())
			}

			return t.vm.InitStringObject(string(content))

		},
	}, {
		// Sends a HEAD request to the target with type header and body. Returns the HTTP headers as a map[string]string. Will error on non-200 responses, for more control over http requests look at the `start` method.
		Name: "head",
		Fn: func(receiver Object, sourceLine int, t *Thread, args []Object, blockFrame *normalCallFrame) Object {
			arg0, ok := args[0].(*StringObject)
			if !ok {
				return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, errors.WrongArgumentTypeFormatNum, 0, "String", args[0].Class().Name)
			}

			uri, err := url.Parse(arg0.value)
			if err != nil {
				return t.vm.InitErrorObject(errors.HTTPError, sourceLine, couldNotCompleteRequest, err)
			}

			if len(args) > 1 {
				var arr []string

				for i, v := range args[1:] {
					argn, ok := v.(*StringObject)
					if !ok {
						return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, invalidSplatArgument, v.Class().Name, i)
					}
					arr = append(arr, argn.value)
				}

				uri.Path = path.Join(arr...)
			}

			resp, err := http.Head(uri.String())
			if err != nil {
				return t.vm.InitErrorObject(errors.HTTPError, sourceLine, couldNotCompleteRequest, err)
			}
			if resp.StatusCode != http.StatusOK {
				return t.vm.InitErrorObject(errors.HTTPError, sourceLine, non200Response, resp.Status, resp.StatusCode)
			}

			ret := t.vm.InitHashObject(map[string]Object{})

			for k, v := range resp.Header {
				ret.Pairs[k] = t.vm.InitStringObject(strings.Join(v, " "))
			}

			return ret

		},
	}, {
		// Starts an HTTP client. This method requires a block which takes a Net::HTTP::Client object. The return value of this method is the last evaluated value of the provided block.
		Name: "start",
		Fn: func(receiver Object, sourceLine int, t *Thread, args []Object, blockFrame *normalCallFrame) Object {
			if len(args) != 0 {
				return t.vm.InitErrorObject(errors.ArgumentError, sourceLine, errors.WrongNumberOfArgument, 0, len(args))
			}

			gobyClient := httpClientClass.initializeInstance()

			result := t.builtinMethodYield(blockFrame, gobyClient)

			if err, ok := result.Target.(*Error); ok {
				return err //an Error object
			}

			return result.Target

		},
	},
}

// Internal functions ===================================================

// Functions for initialization -----------------------------------------

func initHTTPClass(vm *VM) {
	net := vm.loadConstant("Net", true)
	http := vm.initializeClass("HTTP")
	http.setBuiltinMethods(builtinHTTPClassMethods, true)
	initRequestClass(vm, http)
	initResponseClass(vm, http)
	initClientClass(vm, http)

	net.setClassConstant(http)

	// Use Goby code to extend request and response classes.
	vm.mainThread.execGobyLib("net/http/response.gb")
	vm.mainThread.execGobyLib("net/http/request.gb")
}

func initRequestClass(vm *VM, hc *RClass) *RClass {
	requestClass := vm.initializeClass("Request")
	hc.setClassConstant(requestClass)
	builtinHTTPRequestInstanceMethods := []*BuiltinMethodObject{}

	requestClass.setBuiltinMethods(builtinHTTPRequestInstanceMethods, false)

	httpRequestClass = requestClass
	return requestClass
}

func initResponseClass(vm *VM, hc *RClass) *RClass {
	responseClass := vm.initializeClass("Response")
	hc.setClassConstant(responseClass)
	builtinHTTPResponseInstanceMethods := []*BuiltinMethodObject{}

	responseClass.setBuiltinMethods(builtinHTTPResponseInstanceMethods, false)

	httpResponseClass = responseClass
	return responseClass
}
<|endoftext|>


<commit_before>// Package method is a package
package method

import (
	"github.com/gorilla/rpc/v2/json2"
	"github.com/rvelhote/bitcoind-status/bitcoind/rpc"
)

/*
 * The MIT License (MIT)
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
type AddedNodeInfo struct {
	// AddedNode is the node ip address or name (as provided to addnode)
	AddedNode string `json:"addednode"`
	// Connected is a boolean that specifies if we are currently connected to the node
	Connected bool `json:"connected"`
	// Addresses is a list of addresses of the added node. Only present if connected == true
	Addresses []AddedNodeAddress `json:"addresses"`
}

type AddedNodeAddress struct {
	// Address is the bitcoin server IP and port we're connected to
	Address string `json:"address"`
	// Connected specified wether the connection is inbound or outbound
	Connected string `json:"connected"`
}

// GetAddedNodeInfo Returns information about the given added node, or all added nodes.
func GetAddedNodeInfo(client *rpc.RPCClient) ([]AddedNodeInfo, error) {
	response, err := client.Post("getaddednodeinfo", PeerInfoArgs{})

	if err != nil {
		return []AddedNodeInfo{}, err
	}

	defer response.Body.Close()

	var result []AddedNodeInfo
	err = json2.DecodeClientResponse(response.Body, &result)

	if err != nil {
		return []AddedNodeInfo{}, err
	}

	return result, nil
}
<commit_msg>Stray import ;)<commit_after>// Package method is a package
package method

/*
 * The MIT License (MIT)
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
import (
	"github.com/gorilla/rpc/v2/json2"
	"github.com/rvelhote/bitcoind-status/bitcoind/rpc"
)

type AddedNodeInfo struct {
	// AddedNode is the node ip address or name (as provided to addnode)
	AddedNode string `json:"addednode"`
	// Connected is a boolean that specifies if we are currently connected to the node
	Connected bool `json:"connected"`
	// Addresses is a list of addresses of the added node. Only present if connected == true
	Addresses []AddedNodeAddress `json:"addresses"`
}

type AddedNodeAddress struct {
	// Address is the bitcoin server IP and port we're connected to
	Address string `json:"address"`
	// Connected specified wether the connection is inbound or outbound
	Connected string `json:"connected"`
}

// GetAddedNodeInfo Returns information about the given added node, or all added nodes.
func GetAddedNodeInfo(client *rpc.RPCClient) ([]AddedNodeInfo, error) {
	response, err := client.Post("getaddednodeinfo", PeerInfoArgs{})

	if err != nil {
		return []AddedNodeInfo{}, err
	}

	defer response.Body.Close()

	var result []AddedNodeInfo
	err = json2.DecodeClientResponse(response.Body, &result)

	if err != nil {
		return []AddedNodeInfo{}, err
	}

	return result, nil
}
<|endoftext|>


<commit_before>package watcher

import (
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"gopkg.in/fsnotify.v1"
)

// Op describes a set of file operations. Wraps fsnotify.
type Op fsnotify.Op

// FileEvent wraps information about the file events
// from fsnotify.
type FileEvent struct {
	// Absolute path of file.
	Path string
	// Name of the file.
	Name string
	// The file extension, ex. html, js
	Ext string
	// The operation that triggered the event
	Op
}

// Watcher watches files for changes
type Watcher struct {
	fsw *fsnotify.Watcher

	files map[string]struct{}

	ignorers []func(string) bool
	done     chan struct{}

	isClosed bool
}

func (w *Watcher) wait() {
	defer func() {
		close(w.done)
	}()
	for {
		select {
		case <-w.done:
			w.fsw.Close()
			return
		default:
		}
	}
}

// Close the watcher.
func (w *Watcher) Close() {
	if w.isClosed {
		return
	}
	log.Println("CLOSING WATCHER")
	w.isClosed = true
	w.done <- struct{}{}
}

// New creates a Watcher.
func New(root string, ignorers ...func(string) bool) (*Watcher, error) {
	w := Watcher{
		done: make(chan struct{}),
	}
	fsw, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}

	w.fsw = fsw
	w.ignorers = append(w.ignorers, IgnoreDotfiles)

	for _, ign := range ignorers {
		w.ignorers = append(w.ignorers, ign)
	}

	err = w.addFiles(root)
	if err != nil {
		return nil, err
	}

	// Wait for the close signal.
	go w.wait()
	return &w, nil
}

// AddFiles starts to recurse from the root and add files to
// the watch list.
func (w *Watcher) addFiles(root string) error {
	root = os.ExpandEnv(root)
	errc := w.walkFS(root)

	if err := <-errc; err != nil && err != filepath.SkipDir {
		return err
	}
	return nil
}

// Watch watches stuff.
func (w *Watcher) Watch() <-chan *FileEvent {
	fchan := make(chan *FileEvent, 5)
	go func() {
		defer close(fchan)
		defer func() {
			log.Println("EXITING GOROUTINE")
		}()

		for {
			select {
			case ev, ok := <-w.fsw.Events:
				// If the fsnotify event chan is closed
				// there's no reason for this goroutine to
				// keep running.
				if !ok {
					log.Println("EXITING WATCH CHAN")
					return
				}
				fchan <- parseEvent(ev)
			case err, ok := <-w.fsw.Errors:
				// If the channel is closed done has
				// already been shutdown.
				if !ok {
					log.Println("EXITING WATCH CHAN")
					return
				}
				log.Println(err)
				w.done <- struct{}{}
				return
			}
		}
	}()

	return fchan
}

// ignore loops through our ignorers to see if we should ignore
// the path.
func (w *Watcher) ignore(path string) bool {
	for _, i := range w.ignorers {
		if i(path) {
			return true
		}
	}
	return false
}

// walkFS walks the filesystem.
func (w *Watcher) walkFS(root string) <-chan error {
	errc := make(chan error, 1)
	go func() {
		var wg sync.WaitGroup
		err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}

			// If it's an directory and it matches our ignore
			// clause, then skip looking at the whole directory
			if w.ignore(filepath.Base(info.Name())) && info.IsDir() {
				return filepath.SkipDir
			}

			// If the file isn't regular and not a directory, move on.
			if !info.Mode().IsRegular() && !info.IsDir() {
				return nil
			}

			// If a file matches a ignore clause or is a directory move on.
			if !info.IsDir() {
				return nil
			}

			log.Println(path)
			wg.Add(1)
			go func() {
				defer wg.Done()
				log.Println("ADDING:", info.Name())
				w.fsw.Add(path)
			}()
			return nil
		})

		go func() {
			wg.Wait()
		}()
		errc <- err

	}()
	return errc
}

// parseEvent parses the event wrapping it into a filevent
// making it easier to work with.
func parseEvent(ev fsnotify.Event) *FileEvent {
	spl := strings.Split(ev.String(), " ")
	// fmt.Println(spl, len(spl))

	fi := &FileEvent{}

	if len(spl) > 0 {
		path := spl[0]
		// op := Op(ev.Op)

		fi.Ext = filepath.Ext(path)
		fi.Path = path
		fi.Name = filepath.Base(path)
	}
	return fi
}
<commit_msg>fix the path, ext and name fields of FileEvent<commit_after>package watcher

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"gopkg.in/fsnotify.v1"
)

// Op describes a set of file operations. Wraps fsnotify.
type Op fsnotify.Op

// FileEvent wraps information about the file events
// from fsnotify.
type FileEvent struct {
	// Absolute path of file.
	Path string
	// Name of the file.
	Name string
	// The file extension, ex. html, js
	Ext string
	// The operation that triggered the event
	Op
}

// Watcher watches files for changes
type Watcher struct {
	fsw *fsnotify.Watcher

	files map[string]struct{}

	ignorers []func(string) bool
	done     chan struct{}

	isClosed bool
}

func (w *Watcher) wait() {
	defer func() {
		close(w.done)
	}()
	for {
		select {
		case <-w.done:
			w.fsw.Close()
			return
		default:
		}
	}
}

// Close the watcher.
func (w *Watcher) Close() {
	if w.isClosed {
		return
	}
	log.Println("CLOSING WATCHER")
	w.isClosed = true
	w.done <- struct{}{}
}

// New creates a Watcher.
func New(root string, ignorers ...func(string) bool) (*Watcher, error) {
	w := Watcher{
		done: make(chan struct{}),
	}
	fsw, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}

	w.fsw = fsw
	w.ignorers = append(w.ignorers, IgnoreDotfiles)

	for _, ign := range ignorers {
		w.ignorers = append(w.ignorers, ign)
	}

	err = w.addFiles(root)
	if err != nil {
		return nil, err
	}

	// Wait for the close signal.
	go w.wait()
	return &w, nil
}

// AddFiles starts to recurse from the root and add files to
// the watch list.
func (w *Watcher) addFiles(root string) error {
	root = os.ExpandEnv(root)
	errc := w.walkFS(root)

	if err := <-errc; err != nil && err != filepath.SkipDir {
		return err
	}
	return nil
}

// Watch watches stuff.
func (w *Watcher) Watch() <-chan *FileEvent {
	fchan := make(chan *FileEvent, 5)
	go func() {
		defer close(fchan)
		defer func() {
			log.Println("EXITING GOROUTINE")
		}()

		for {
			select {
			case ev, ok := <-w.fsw.Events:
				// If the fsnotify event chan is closed
				// there's no reason for this goroutine to
				// keep running.
				if !ok {
					log.Println("EXITING WATCH CHAN")
					return
				}
				fchan <- parseEvent(ev)
			case err, ok := <-w.fsw.Errors:
				// If the channel is closed done has
				// already been shutdown.
				if !ok {
					log.Println("EXITING WATCH CHAN")
					return
				}
				log.Println(err)
				w.done <- struct{}{}
				return
			}
		}
	}()

	return fchan
}

// ignore loops through our ignorers to see if we should ignore
// the path.
func (w *Watcher) ignore(path string) bool {
	for _, i := range w.ignorers {
		if i(path) {
			return true
		}
	}
	return false
}

// walkFS walks the filesystem.
func (w *Watcher) walkFS(root string) <-chan error {
	errc := make(chan error, 1)
	go func() {
		var wg sync.WaitGroup
		err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}

			// If it's an directory and it matches our ignore
			// clause, then skip looking at the whole directory
			if w.ignore(filepath.Base(info.Name())) && info.IsDir() {
				return filepath.SkipDir
			}

			// If the file isn't regular and not a directory, move on.
			if !info.Mode().IsRegular() && !info.IsDir() {
				return nil
			}

			// If a file matches a ignore clause or is a directory move on.
			if !info.IsDir() {
				return nil
			}

			log.Println(path)
			wg.Add(1)
			go func() {
				defer wg.Done()
				log.Println("ADDING:", info.Name())
				w.fsw.Add(path)
			}()
			return nil
		})

		go func() {
			wg.Wait()
		}()
		errc <- err

	}()
	return errc
}

// parseEvent parses the event wrapping it into a filevent
// making it easier to work with.
func parseEvent(ev fsnotify.Event) *FileEvent {
	spl := strings.Split(ev.String(), ": ")
	// fmt.Println(spl, len(spl))

	fi := &FileEvent{}

	if len(spl) > 0 {
		path := spl[0]
		// op := Op(ev.Op)

		path = strings.Trim(path, "\"")

		fmt.Println(path)
		fi.Ext = filepath.Ext(path)
		fi.Name = filepath.Base(path)
		fi.Path = path
		fi.Op = Op(ev.Op)
	}
	return fi
}
<|endoftext|>


<commit_before>package lazytest

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"gopkg.in/fsnotify.v1"
)

type fileWatcher struct {
	extensions []string
	exclude    []string
	watcher    *fsnotify.Watcher
}

type Mod struct {
	Package  string
	FilePath string
	Function string
	Line     int
}

func (w *fileWatcher) handleDir(path string) error {
	if !w.isIncluded(path) {
		return filepath.SkipDir
	}

	if len(path) > 1 && strings.HasPrefix(filepath.Base(path), ".") {
		return filepath.SkipDir
	}

	return w.watcher.Add(path)
}

func (w *fileWatcher) handleEvent(e fsnotify.Event, eventChannel chan Mod) {
	if e.Op|fsnotify.Rename == e.Op || e.Op|fsnotify.Chmod == e.Op {
		return
	}

	eventChannel <- Mod{FilePath: e.Name}
	// TODO: remove old watches on delete, add new watches on create, do both on rename
}

func (w *fileWatcher) isIncluded(path string) bool {
	include := len(w.extensions) == 0

	if !include {
		ext := filepath.Ext(path)
		for _, e := range w.extensions {
			if ext == e {
				include = true
			}
		}
	}

	for _, e := range w.exclude {
		if filepath.HasPrefix(path, e) {
			include = false
		}
	}

	return include
}

func (w *fileWatcher) listenForEvents(eventChannel chan Mod) {
	for {
		select {
		case e := <-w.watcher.Events:
			w.handleEvent(e, eventChannel)

		case err := <-w.watcher.Errors:
			log(fmt.Sprintf("Watcher error %v", err))
		}
	}
}

func (w *fileWatcher) walk(path string, info os.FileInfo, err error) error {
	if info.IsDir() {
		return w.handleDir(path)
	}

	if w.isIncluded(path) {
		return w.watcher.Add(path)
	}

	return err
}

func Watch(root string, extensions []string, exclude []string) (chan Mod,
	error) {

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}

	absolutePath, err := filepath.Abs(root)
	if err != nil {
		return nil, err
	}

	w := &fileWatcher{
		extensions: extensions,
		exclude:    exclude,
		watcher:    watcher,
	}

	events := make(chan Mod, 50)
	go w.listenForEvents(events)
	return events, filepath.Walk(absolutePath, w.walk)
}
<commit_msg>Trimmed a few lines from watcher.go<commit_after>package lazytest

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"gopkg.in/fsnotify.v1"
)

type fileWatcher struct {
	extensions []string
	exclude    []string
	watcher    *fsnotify.Watcher
}

type Mod struct {
	Package  string
	FilePath string
	Function string
	Line     int
}

func (w *fileWatcher) handleDir(path string) error {
	if !w.isIncluded(path) {
		return filepath.SkipDir
	}

	if len(path) > 1 && strings.HasPrefix(filepath.Base(path), ".") {
		return filepath.SkipDir
	}

	return w.watcher.Add(path)
}

func (w *fileWatcher) handleEvent(e fsnotify.Event, eventChannel chan Mod) {
	if e.Op|fsnotify.Rename == e.Op || e.Op|fsnotify.Chmod == e.Op {
		return
	}

	eventChannel <- Mod{FilePath: e.Name}
	// TODO: remove old watches on delete, add new watches on create, do both on rename
}

func (w *fileWatcher) isIncluded(path string) bool {
	include := len(w.extensions) == 0

	for _, e := range w.extensions {
		if filepath.Ext(path) == e {
			include = true
		}
	}

	for _, e := range w.exclude {
		if filepath.HasPrefix(path, e) {
		   return false
		}
	}

	return include
}

func (w *fileWatcher) listenForEvents(eventChannel chan Mod) {
	for {
		select {
		case e := <-w.watcher.Events:
			w.handleEvent(e, eventChannel)

		case err := <-w.watcher.Errors:
			log(fmt.Sprintf("Watcher error %v", err))
		}
	}
}

func (w *fileWatcher) walk(path string, info os.FileInfo, err error) error {
	if info.IsDir() {
		return w.handleDir(path)
	}

	if w.isIncluded(path) {
		return w.watcher.Add(path)
	}

	return err
}

func Watch(root string, extensions []string, exclude []string) (chan Mod,
	error) {

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}

	absolutePath, err := filepath.Abs(root)
	if err != nil {
		return nil, err
	}

	w := &fileWatcher{
		extensions: extensions,
		exclude:    exclude,
		watcher:    watcher,
	}

	events := make(chan Mod, 50)
	go w.listenForEvents(events)
	return events, filepath.Walk(absolutePath, w.walk)
}
<|endoftext|>