<commit_before>package storage

import (
	"context"
	"time"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"kubevirt.io/client-go/kubecli"

	"kubevirt.io/kubevirt/pkg/virtctl/guestfs"
	"kubevirt.io/kubevirt/tests/clientcmd"
	"kubevirt.io/kubevirt/tests/exec"
	"kubevirt.io/kubevirt/tests/libstorage"
	"kubevirt.io/kubevirt/tests/util"
)

const libguestsTools = "libguestfs-tools-"

type fakeAttacher struct {
	done chan bool
}

// fakeCreateAttacher simulates the attacher to the pod console. It has to block until the test terminates.
func (f *fakeAttacher) fakeCreateAttacher(client *guestfs.K8sClient, p *corev1.Pod, command string) error {
	<-f.done
	return nil
}

func (f *fakeAttacher) closeChannel() {
	f.done <- true
}

var _ = SIGDescribe("[rfe_id:6364][[Serial]Guestfs", func() {
	var (
		virtClient kubecli.KubevirtClient
		pvcClaim   string
		setGroup   bool
		testGroup  string
	)
	execCommandLibguestfsPod := func(podName string, c []string) (string, string, error) {
		pod, err := virtClient.CoreV1().Pods(util.NamespaceTestDefault).Get(context.Background(), podName, metav1.GetOptions{})
		Expect(err).ToNot(HaveOccurred())
		return exec.ExecuteCommandOnPodWithResults(virtClient, pod, "libguestfs", c)
	}

	createPVCFilesystem := func(name string) {
		quantity, _ := resource.ParseQuantity("500Mi")
		_, err := virtClient.CoreV1().PersistentVolumeClaims(util.NamespaceTestDefault).Create(context.Background(), &corev1.PersistentVolumeClaim{
			ObjectMeta: metav1.ObjectMeta{Name: name},
			Spec: corev1.PersistentVolumeClaimSpec{
				AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
				Resources: corev1.ResourceRequirements{
					Requests: corev1.ResourceList{
						"storage": quantity,
					},
				},
			},
		}, metav1.CreateOptions{})
		Expect(err).ToNot(HaveOccurred())

	}

	createFakeAttacher := func() *fakeAttacher {
		f := &fakeAttacher{}
		f.done = make(chan bool, 1)
		guestfs.SetAttacher(f.fakeCreateAttacher)
		return f
	}

	runGuestfsOnPVC := func(pvcClaim string, options ...string) {
		podName := libguestsTools + pvcClaim
		o := append([]string{"guestfs", pvcClaim, "--namespace", util.NamespaceTestDefault}, options...)
		if setGroup {
			o = append(o, "--fsGroup", testGroup)
		}
		guestfsCmd := clientcmd.NewVirtctlCommand(o...)
		go func() {
			defer GinkgoRecover()
			Expect(guestfsCmd.Execute()).ToNot(HaveOccurred())
		}()
		// Waiting until the libguestfs pod is running
		Eventually(func() bool {
			pod, _ := virtClient.CoreV1().Pods(util.NamespaceTestDefault).Get(context.Background(), podName, metav1.GetOptions{})
			ready := false
			for _, status := range pod.Status.ContainerStatuses {
				if status.State.Running != nil {
					return true
				}
			}
			return ready

		}, 90*time.Second, 2*time.Second).Should(BeTrue())
		// Verify that the appliance has been extracted before running any tests by checking the done file
		Eventually(func() bool {
			_, _, err := execCommandLibguestfsPod(podName, []string{"ls", "/usr/local/lib/guestfs/appliance/done"})
			if err != nil {
				return false
			}
			return true
		}, 30*time.Second, 2*time.Second).Should(BeTrue())

	}

	verifyCanRunOnFSPVC := func(podName string) {
		stdout, stderr, err := execCommandLibguestfsPod(podName, []string{"qemu-img", "create", "/disk/disk.img", "500M"})
		Expect(stderr).To(Equal(""))
		Expect(stdout).To(ContainSubstring("Formatting"))
		Expect(err).ToNot(HaveOccurred())
		stdout, stderr, err = execCommandLibguestfsPod(podName, []string{"guestfish", "-a", "/disk/disk.img", "run"})
		Expect(stderr).To(BeEmpty())
		Expect(stdout).To(BeEmpty())
		Expect(err).ToNot(HaveOccurred())

	}

	Context("Run libguestfs on PVCs", func() {
		BeforeEach(func() {
			var err error
			virtClient, err = kubecli.GetKubevirtClient()
			Expect(err).ToNot(HaveOccurred())
			// TODO: Always setGroup to true until we have the ability to control how virtctl guestfs is run
			setGroup = true
			testGroup = "2000"
		})

		AfterEach(func() {
			err := virtClient.CoreV1().PersistentVolumeClaims(util.NamespaceTestDefault).Delete(context.Background(), pvcClaim, metav1.DeleteOptions{})
			Expect(err).ToNot(HaveOccurred())
		})

		// libguestfs-test-tool verifies the setup to run libguestfs-tools
		It("Should successfully run libguestfs-test-tool", func() {
			f := createFakeAttacher()
			defer f.closeChannel()
			pvcClaim = "pvc-verify"
			createPVCFilesystem(pvcClaim)
			runGuestfsOnPVC(pvcClaim)
			output, _, err := execCommandLibguestfsPod(libguestsTools+pvcClaim, []string{"libguestfs-test-tool"})
			Expect(err).ToNot(HaveOccurred())
			Expect(output).To(ContainSubstring("===== TEST FINISHED OK ====="))

		})

		It("[posneg:positive][test_id:6480]Should successfully run guestfs command on a filesystem-based PVC", func() {
			f := createFakeAttacher()
			defer f.closeChannel()
			pvcClaim = "pvc-fs"
			podName := libguestsTools + pvcClaim
			createPVCFilesystem(pvcClaim)
			runGuestfsOnPVC(pvcClaim)
			verifyCanRunOnFSPVC(podName)
		})

		It("[posneg:negative][test_id:6480]Should fail to run the guestfs command on a PVC in use", func() {
			f := createFakeAttacher()
			defer f.closeChannel()
			pvcClaim = "pvc-fail-to-run-twice"
			createPVCFilesystem(pvcClaim)
			runGuestfsOnPVC(pvcClaim)
			guestfsCmd := clientcmd.NewVirtctlCommand("guestfs",
				pvcClaim,
				"--namespace", util.NamespaceTestDefault)
			Expect(guestfsCmd.Execute()).To(HaveOccurred())
		})

		It("[posneg:positive][test_id:6479]Should successfully run guestfs command on a block-based PVC", func() {
			f := createFakeAttacher()
			defer f.closeChannel()

			pvcClaim = "pvc-block"
			podName := libguestsTools + pvcClaim
			libstorage.CreateBlockPVC(pvcClaim, "500Mi")
			runGuestfsOnPVC(pvcClaim)
			stdout, stderr, err := execCommandLibguestfsPod(podName, []string{"guestfish", "-a", "/dev/vda", "run"})
			Expect(stderr).To(Equal(""))
			Expect(stdout).To(Equal(""))
			Expect(err).ToNot(HaveOccurred())

		})
		It("Should successfully run guestfs command on a filesystem-based PVC setting the uid", func() {
			f := createFakeAttacher()
			defer f.closeChannel()
			pvcClaim = "pvc-fs-with-different-uid"
			podName := libguestsTools + pvcClaim
			createPVCFilesystem(pvcClaim)
			runGuestfsOnPVC(pvcClaim, "--uid", "1002")
			verifyCanRunOnFSPVC(podName)
		})
	})
	Context("Run libguestfs on PVCs with root", func() {
		BeforeEach(func() {
			var err error
			virtClient, err = kubecli.GetKubevirtClient()
			Expect(err).ToNot(HaveOccurred())
			setGroup = false
		})

		AfterEach(func() {
			err := virtClient.CoreV1().PersistentVolumeClaims(util.NamespaceTestDefault).Delete(context.Background(), pvcClaim, metav1.DeleteOptions{})
			Expect(err).ToNot(HaveOccurred())
		})
		It("Should successfully run guestfs command on a filesystem-based PVC with root", func() {
			f := createFakeAttacher()
			defer f.closeChannel()
			pvcClaim = "pvc-fs-with-different-uid"
			podName := libguestsTools + pvcClaim
			createPVCFilesystem(pvcClaim)
			runGuestfsOnPVC(pvcClaim, "--root")
			verifyCanRunOnFSPVC(podName)
		})
	})

})
<commit_msg>Use a unique pvc/pod name for guestfs test<commit_after>package storage

import (
	"context"
	"time"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"kubevirt.io/client-go/kubecli"

	"kubevirt.io/kubevirt/pkg/virtctl/guestfs"
	"kubevirt.io/kubevirt/tests/clientcmd"
	"kubevirt.io/kubevirt/tests/exec"
	"kubevirt.io/kubevirt/tests/libstorage"
	"kubevirt.io/kubevirt/tests/util"
)

const libguestsTools = "libguestfs-tools-"

type fakeAttacher struct {
	done chan bool
}

// fakeCreateAttacher simulates the attacher to the pod console. It has to block until the test terminates.
func (f *fakeAttacher) fakeCreateAttacher(client *guestfs.K8sClient, p *corev1.Pod, command string) error {
	<-f.done
	return nil
}

func (f *fakeAttacher) closeChannel() {
	f.done <- true
}

var _ = SIGDescribe("[rfe_id:6364][[Serial]Guestfs", func() {
	var (
		virtClient kubecli.KubevirtClient
		pvcClaim   string
		setGroup   bool
		testGroup  string
	)
	execCommandLibguestfsPod := func(podName string, c []string) (string, string, error) {
		pod, err := virtClient.CoreV1().Pods(util.NamespaceTestDefault).Get(context.Background(), podName, metav1.GetOptions{})
		Expect(err).ToNot(HaveOccurred())
		return exec.ExecuteCommandOnPodWithResults(virtClient, pod, "libguestfs", c)
	}

	createPVCFilesystem := func(name string) {
		quantity, _ := resource.ParseQuantity("500Mi")
		_, err := virtClient.CoreV1().PersistentVolumeClaims(util.NamespaceTestDefault).Create(context.Background(), &corev1.PersistentVolumeClaim{
			ObjectMeta: metav1.ObjectMeta{Name: name},
			Spec: corev1.PersistentVolumeClaimSpec{
				AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
				Resources: corev1.ResourceRequirements{
					Requests: corev1.ResourceList{
						"storage": quantity,
					},
				},
			},
		}, metav1.CreateOptions{})
		Expect(err).ToNot(HaveOccurred())

	}

	createFakeAttacher := func() *fakeAttacher {
		f := &fakeAttacher{}
		f.done = make(chan bool, 1)
		guestfs.SetAttacher(f.fakeCreateAttacher)
		return f
	}

	runGuestfsOnPVC := func(pvcClaim string, options ...string) {
		podName := libguestsTools + pvcClaim
		o := append([]string{"guestfs", pvcClaim, "--namespace", util.NamespaceTestDefault}, options...)
		if setGroup {
			o = append(o, "--fsGroup", testGroup)
		}
		guestfsCmd := clientcmd.NewVirtctlCommand(o...)
		go func() {
			defer GinkgoRecover()
			Expect(guestfsCmd.Execute()).ToNot(HaveOccurred())
		}()
		// Waiting until the libguestfs pod is running
		Eventually(func() bool {
			pod, _ := virtClient.CoreV1().Pods(util.NamespaceTestDefault).Get(context.Background(), podName, metav1.GetOptions{})
			ready := false
			for _, status := range pod.Status.ContainerStatuses {
				if status.State.Running != nil {
					return true
				}
			}
			return ready

		}, 90*time.Second, 2*time.Second).Should(BeTrue())
		// Verify that the appliance has been extracted before running any tests by checking the done file
		Eventually(func() bool {
			_, _, err := execCommandLibguestfsPod(podName, []string{"ls", "/usr/local/lib/guestfs/appliance/done"})
			if err != nil {
				return false
			}
			return true
		}, 30*time.Second, 2*time.Second).Should(BeTrue())

	}

	verifyCanRunOnFSPVC := func(podName string) {
		stdout, stderr, err := execCommandLibguestfsPod(podName, []string{"qemu-img", "create", "/disk/disk.img", "500M"})
		Expect(stderr).To(Equal(""))
		Expect(stdout).To(ContainSubstring("Formatting"))
		Expect(err).ToNot(HaveOccurred())
		stdout, stderr, err = execCommandLibguestfsPod(podName, []string{"guestfish", "-a", "/disk/disk.img", "run"})
		Expect(stderr).To(BeEmpty())
		Expect(stdout).To(BeEmpty())
		Expect(err).ToNot(HaveOccurred())

	}

	Context("Run libguestfs on PVCs", func() {
		BeforeEach(func() {
			var err error
			virtClient, err = kubecli.GetKubevirtClient()
			Expect(err).ToNot(HaveOccurred())
			// TODO: Always setGroup to true until we have the ability to control how virtctl guestfs is run
			setGroup = true
			testGroup = "2000"
		})

		AfterEach(func() {
			err := virtClient.CoreV1().PersistentVolumeClaims(util.NamespaceTestDefault).Delete(context.Background(), pvcClaim, metav1.DeleteOptions{})
			Expect(err).ToNot(HaveOccurred())
		})

		// libguestfs-test-tool verifies the setup to run libguestfs-tools
		It("Should successfully run libguestfs-test-tool", func() {
			f := createFakeAttacher()
			defer f.closeChannel()
			pvcClaim = "pvc-verify"
			createPVCFilesystem(pvcClaim)
			runGuestfsOnPVC(pvcClaim)
			output, _, err := execCommandLibguestfsPod(libguestsTools+pvcClaim, []string{"libguestfs-test-tool"})
			Expect(err).ToNot(HaveOccurred())
			Expect(output).To(ContainSubstring("===== TEST FINISHED OK ====="))

		})

		It("[posneg:positive][test_id:6480]Should successfully run guestfs command on a filesystem-based PVC", func() {
			f := createFakeAttacher()
			defer f.closeChannel()
			pvcClaim = "pvc-fs"
			podName := libguestsTools + pvcClaim
			createPVCFilesystem(pvcClaim)
			runGuestfsOnPVC(pvcClaim)
			verifyCanRunOnFSPVC(podName)
		})

		It("[posneg:negative][test_id:6480]Should fail to run the guestfs command on a PVC in use", func() {
			f := createFakeAttacher()
			defer f.closeChannel()
			pvcClaim = "pvc-fail-to-run-twice"
			createPVCFilesystem(pvcClaim)
			runGuestfsOnPVC(pvcClaim)
			guestfsCmd := clientcmd.NewVirtctlCommand("guestfs",
				pvcClaim,
				"--namespace", util.NamespaceTestDefault)
			Expect(guestfsCmd.Execute()).To(HaveOccurred())
		})

		It("[posneg:positive][test_id:6479]Should successfully run guestfs command on a block-based PVC", func() {
			f := createFakeAttacher()
			defer f.closeChannel()

			pvcClaim = "pvc-block"
			podName := libguestsTools + pvcClaim
			libstorage.CreateBlockPVC(pvcClaim, "500Mi")
			runGuestfsOnPVC(pvcClaim)
			stdout, stderr, err := execCommandLibguestfsPod(podName, []string{"guestfish", "-a", "/dev/vda", "run"})
			Expect(stderr).To(Equal(""))
			Expect(stdout).To(Equal(""))
			Expect(err).ToNot(HaveOccurred())

		})
		It("Should successfully run guestfs command on a filesystem-based PVC setting the uid", func() {
			f := createFakeAttacher()
			defer f.closeChannel()
			pvcClaim = "pvc-fs-with-different-uid"
			podName := libguestsTools + pvcClaim
			createPVCFilesystem(pvcClaim)
			runGuestfsOnPVC(pvcClaim, "--uid", "1002")
			verifyCanRunOnFSPVC(podName)
		})
	})
	Context("Run libguestfs on PVCs with root", func() {
		BeforeEach(func() {
			var err error
			virtClient, err = kubecli.GetKubevirtClient()
			Expect(err).ToNot(HaveOccurred())
			setGroup = false
		})

		AfterEach(func() {
			err := virtClient.CoreV1().PersistentVolumeClaims(util.NamespaceTestDefault).Delete(context.Background(), pvcClaim, metav1.DeleteOptions{})
			Expect(err).ToNot(HaveOccurred())
		})
		It("Should successfully run guestfs command on a filesystem-based PVC with root", func() {
			f := createFakeAttacher()
			defer f.closeChannel()
			pvcClaim = "pvc-fs-with-root"
			podName := libguestsTools + pvcClaim
			createPVCFilesystem(pvcClaim)
			runGuestfsOnPVC(pvcClaim, "--root")
			verifyCanRunOnFSPVC(podName)
		})
	})

})
<|endoftext|>


<commit_before>package service

import (
	"os"

	"github.com/garlsecurity/go-securepass/securepass"
	"gopkg.in/ini.v1"
)

var (
	// Service holds the global settings
	Service *securepass.SecurePass
)

// LoadConfiguration reads configuration from files
func LoadConfiguration(conffiles []string) error {
	cfg, _ := ini.Load([]byte("[default]\nAPP_ID=\nAPP_SECRET=\n"))
	for _, filename := range conffiles {
		if fp, err := os.Open(filename); err == nil {
			fp.Close()
			cfg.Append(filename)
		}
	}
	section, _ := cfg.GetSection("default")
	Service = &securepass.SecurePass{Endpoint: securepass.DefaultRemote}
	if err := section.MapTo(Service); err != nil {
		return err
	}
	return nil
}
<commit_msg>Handle multi-section configuration files<commit_after>package service

import (
	"io"
	"os"

	"github.com/garlsecurity/go-securepass/securepass"
	"gopkg.in/ini.v1"
)

var (
	// Service holds the global settings
	Service *securepass.SecurePass
	// NSSSettings holds the NSS settings
	NSSSettings *securepass.NSSConfig
	// SSHSettings holds the SSH settings
	SSHSettings *securepass.SSHConfig
)

// LoadConfiguration reads configuration from files
func LoadConfiguration(conffiles []string) error {
	cfg, _ := ini.Load([]byte("[default]\nAPP_ID=\nAPP_SECRET=\n[nss]\n\n[ssh]\n"))
	for _, filename := range conffiles {
		if fp, err := os.Open(filename); err == nil {
			fp.Close()
			cfg.Append(filename)
		}
	}
	defaultSection, _ := cfg.GetSection("default")
	nssSection, _ := cfg.GetSection("nss")
	sshSection, _ := cfg.GetSection("ssh")
	Service = &securepass.SecurePass{Endpoint: securepass.DefaultRemote}
	NSSSettings = &securepass.NSSConfig{
		DefaultGid:   100,
		DefaultHome:  "/home",
		DefaultShell: "/bin/bash",
	}
	SSHSettings = &securepass.SSHConfig{}
	if err := defaultSection.MapTo(Service); err != nil {
		return err
	}
	if err := nssSection.MapTo(NSSSettings); err != nil {
		return err
	}
	if err := sshSection.MapTo(SSHSettings); err != nil {
		return err
	}
	return nil
}

// WriteConfiguration saves configuration to file
func WriteConfiguration(writer io.Writer, appid, endpoint, appsecret, realm, root string) (int64, error) {
	globalConfig := securepass.GlobalConfig{
		*Service,
		*NSSSettings,
		*SSHSettings,
	}

	cfg := ini.Empty()
	if err := ini.ReflectFrom(cfg, &globalConfig); err != nil {
		return 0, err
	}
	return cfg.WriteTo(writer)
}
<|endoftext|>


<commit_before>package provision

import (
	"bytes"
	"errors"
	"fmt"
	"text/template"

	"github.com/docker/machine/libmachine/auth"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/engine"
	"github.com/docker/machine/libmachine/log"
	"github.com/docker/machine/libmachine/mcnutils"
	"github.com/docker/machine/libmachine/provision/pkgaction"
	"github.com/docker/machine/libmachine/provision/serviceaction"
	"github.com/docker/machine/libmachine/ssh"
	"github.com/docker/machine/libmachine/swarm"
)

var (
	ErrUnknownYumOsRelease = errors.New("unknown OS for Yum repository")

	packageListTemplate = `[docker]
name=Docker Stable Repository
baseurl=https://yum.dockerproject.org/repo/main/{{.OsRelease}}/{{.OsReleaseVersion}}
priority=1
enabled=1
gpgkey=https://yum.dockerproject.org/gpg
`
	engineConfigTemplate = `[Service]
ExecStart=/usr/bin/docker -d -H tcp://0.0.0.0:{{.DockerPort}} -H unix:///var/run/docker.sock --storage-driver {{.EngineOptions.StorageDriver}} --tlsverify --tlscacert {{.AuthOptions.CaCertRemotePath}} --tlscert {{.AuthOptions.ServerCertRemotePath}} --tlskey {{.AuthOptions.ServerKeyRemotePath}} {{ range .EngineOptions.Labels }}--label {{.}} {{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}} {{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}} {{ end }}
MountFlags=slave
LimitNOFILE=1048576
LimitNPROC=1048576
LimitCORE=infinity
Environment={{range .EngineOptions.Env}}{{ printf "%q" . }} {{end}}
`
)

type PackageListInfo struct {
	OsRelease        string
	OsReleaseVersion string
}

func init() {
	Register("RedHat", &RegisteredProvisioner{
		New: NewRedHatProvisioner,
	})
}

func NewRedHatProvisioner(d drivers.Driver) Provisioner {
	return &RedHatProvisioner{
		GenericProvisioner: GenericProvisioner{
			DockerOptionsDir:  "/etc/docker",
			DaemonOptionsFile: "/etc/systemd/system/docker.service",
			OsReleaseId:       "rhel",
			Packages: []string{
				"curl",
			},
			Driver: d,
		},
	}
}

type RedHatProvisioner struct {
	GenericProvisioner
}

func (provisioner *RedHatProvisioner) SSHCommand(args string) (string, error) {
	client, err := drivers.GetSSHClientFromDriver(provisioner.Driver)
	if err != nil {
		return "", err
	}

	// redhat needs "-t" for tty allocation on ssh therefore we check for the
	// external client and add as needed
	switch c := client.(type) {
	case ssh.ExternalClient:
		c.BaseArgs = append(c.BaseArgs, "-t")
		client = c
	case ssh.NativeClient:
		return c.OutputWithPty(args)
	}

	return client.Output(args)
}

func (provisioner *RedHatProvisioner) SetHostname(hostname string) error {
	// we have to have SetHostname here as well to use the RedHat provisioner
	// SSHCommand to add the tty allocation
	if _, err := provisioner.SSHCommand(fmt.Sprintf(
		"sudo hostname %s && echo %q | sudo tee /etc/hostname",
		hostname,
		hostname,
	)); err != nil {
		return err
	}

	// ubuntu/debian use 127.0.1.1 for non "localhost" loopback hostnames: https://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_hostname_resolution
	if _, err := provisioner.SSHCommand(fmt.Sprintf(
		"if grep -xq 127.0.1.1.* /etc/hosts; then sudo sed -i 's/^127.0.1.1.*/127.0.1.1 %s/g' /etc/hosts; else echo '127.0.1.1 %s' | sudo tee -a /etc/hosts; fi",
		hostname,
		hostname,
	)); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) Service(name string, action serviceaction.ServiceAction) error {
	reloadDaemon := false
	switch action {
	case serviceaction.Start, serviceaction.Restart:
		reloadDaemon = true
	}

	// systemd needs reloaded when config changes on disk; we cannot
	// be sure exactly when it changes from the provisioner so
	// we call a reload on every restart to be safe
	if reloadDaemon {
		if _, err := provisioner.SSHCommand("sudo systemctl daemon-reload"); err != nil {
			return err
		}
	}

	command := fmt.Sprintf("sudo systemctl %s %s", action.String(), name)

	if _, err := provisioner.SSHCommand(command); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) Package(name string, action pkgaction.PackageAction) error {
	var packageAction string

	switch action {
	case pkgaction.Install:
		packageAction = "install"
	case pkgaction.Remove:
		packageAction = "remove"
	case pkgaction.Upgrade:
		packageAction = "upgrade"
	}

	command := fmt.Sprintf("sudo -E yum %s -y %s", packageAction, name)

	if _, err := provisioner.SSHCommand(command); err != nil {
		return err
	}

	return nil
}

func installDocker(provisioner *RedHatProvisioner) error {
	if err := provisioner.installOfficialDocker(); err != nil {
		return err
	}

	if err := provisioner.Service("docker", serviceaction.Restart); err != nil {
		return err
	}

	if err := provisioner.Service("docker", serviceaction.Enable); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) installOfficialDocker() error {
	log.Debug("installing docker")

	if err := provisioner.ConfigurePackageList(); err != nil {
		return err
	}

	if _, err := provisioner.SSHCommand("sudo yum install -y docker-engine"); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) dockerDaemonResponding() bool {
	if _, err := provisioner.SSHCommand("sudo docker version"); err != nil {
		log.Warnf("Error getting SSH command to check if the daemon is up: %s", err)
		return false
	}

	// The daemon is up if the command worked.  Carry on.
	return true
}

func (provisioner *RedHatProvisioner) Provision(swarmOptions swarm.SwarmOptions, authOptions auth.AuthOptions, engineOptions engine.EngineOptions) error {
	provisioner.SwarmOptions = swarmOptions
	provisioner.AuthOptions = authOptions
	provisioner.EngineOptions = engineOptions

	// set default storage driver for redhat
	if provisioner.EngineOptions.StorageDriver == "" {
		provisioner.EngineOptions.StorageDriver = "devicemapper"
	}

	if err := provisioner.SetHostname(provisioner.Driver.GetMachineName()); err != nil {
		return err
	}

	for _, pkg := range provisioner.Packages {
		log.Debugf("installing base package: name=%s", pkg)
		if err := provisioner.Package(pkg, pkgaction.Install); err != nil {
			return err
		}
	}

	// update OS -- this is needed for libdevicemapper and the docker install
	if _, err := provisioner.SSHCommand("sudo yum -y update"); err != nil {
		return err
	}

	// install docker
	if err := installDocker(provisioner); err != nil {
		return err
	}

	if err := mcnutils.WaitFor(provisioner.dockerDaemonResponding); err != nil {
		return err
	}

	if err := makeDockerOptionsDir(provisioner); err != nil {
		return err
	}

	provisioner.AuthOptions = setRemoteAuthOptions(provisioner)

	if err := ConfigureAuth(provisioner); err != nil {
		return err
	}

	if err := configureSwarm(provisioner, swarmOptions, provisioner.AuthOptions); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) {
	var (
		engineCfg  bytes.Buffer
		configPath = provisioner.DaemonOptionsFile
	)

	driverNameLabel := fmt.Sprintf("provider=%s", provisioner.Driver.DriverName())
	provisioner.EngineOptions.Labels = append(provisioner.EngineOptions.Labels, driverNameLabel)

	// systemd / redhat will not load options if they are on newlines
	// instead, it just continues with a different set of options; yeah...
	t, err := template.New("engineConfig").Parse(engineConfigTemplate)
	if err != nil {
		return nil, err
	}

	engineConfigContext := EngineConfigContext{
		DockerPort:       dockerPort,
		AuthOptions:      provisioner.AuthOptions,
		EngineOptions:    provisioner.EngineOptions,
		DockerOptionsDir: provisioner.DockerOptionsDir,
	}

	t.Execute(&engineCfg, engineConfigContext)

	daemonOptsDir := configPath
	return &DockerOptions{
		EngineOptions:     engineCfg.String(),
		EngineOptionsPath: daemonOptsDir,
	}, nil
}

func generateYumRepoList(provisioner Provisioner) (*bytes.Buffer, error) {
	packageListInfo := &PackageListInfo{}

	releaseInfo, err := provisioner.GetOsReleaseInfo()
	if err != nil {
		return nil, err
	}

	switch releaseInfo.Id {
	case "rhel", "centos":
		// rhel and centos both use the "centos" repo
		packageListInfo.OsRelease = "centos"
		packageListInfo.OsReleaseVersion = "7"
	case "fedora":
		packageListInfo.OsRelease = "fedora"
		packageListInfo.OsReleaseVersion = "22"
	default:
		return nil, ErrUnknownYumOsRelease
	}

	t, err := template.New("packageList").Parse(packageListTemplate)
	if err != nil {
		return nil, err
	}

	var buf bytes.Buffer

	if err := t.Execute(&buf, packageListInfo); err != nil {
		return nil, err
	}

	return &buf, nil
}

func (provisioner *RedHatProvisioner) ConfigurePackageList() error {
	buf, err := generateYumRepoList(provisioner)
	if err != nil {
		return err
	}

	// we cannot use %q here as it combines the newlines in the formatting
	// on transport causing yum to not use the repo
	packageCmd := fmt.Sprintf("echo \"%s\" | sudo tee /etc/yum.repos.d/docker.repo", buf.String())
	if _, err := provisioner.SSHCommand(packageCmd); err != nil {
		return err
	}

	return nil
}
<commit_msg>force tty allocation for ssh with multiple '-tt'.<commit_after>package provision

import (
	"bytes"
	"errors"
	"fmt"
	"text/template"

	"github.com/docker/machine/libmachine/auth"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/engine"
	"github.com/docker/machine/libmachine/log"
	"github.com/docker/machine/libmachine/mcnutils"
	"github.com/docker/machine/libmachine/provision/pkgaction"
	"github.com/docker/machine/libmachine/provision/serviceaction"
	"github.com/docker/machine/libmachine/ssh"
	"github.com/docker/machine/libmachine/swarm"
)

var (
	ErrUnknownYumOsRelease = errors.New("unknown OS for Yum repository")

	packageListTemplate = `[docker]
name=Docker Stable Repository
baseurl=https://yum.dockerproject.org/repo/main/{{.OsRelease}}/{{.OsReleaseVersion}}
priority=1
enabled=1
gpgkey=https://yum.dockerproject.org/gpg
`
	engineConfigTemplate = `[Service]
ExecStart=/usr/bin/docker -d -H tcp://0.0.0.0:{{.DockerPort}} -H unix:///var/run/docker.sock --storage-driver {{.EngineOptions.StorageDriver}} --tlsverify --tlscacert {{.AuthOptions.CaCertRemotePath}} --tlscert {{.AuthOptions.ServerCertRemotePath}} --tlskey {{.AuthOptions.ServerKeyRemotePath}} {{ range .EngineOptions.Labels }}--label {{.}} {{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}} {{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}} {{ end }}
MountFlags=slave
LimitNOFILE=1048576
LimitNPROC=1048576
LimitCORE=infinity
Environment={{range .EngineOptions.Env}}{{ printf "%q" . }} {{end}}
`
)

type PackageListInfo struct {
	OsRelease        string
	OsReleaseVersion string
}

func init() {
	Register("RedHat", &RegisteredProvisioner{
		New: NewRedHatProvisioner,
	})
}

func NewRedHatProvisioner(d drivers.Driver) Provisioner {
	return &RedHatProvisioner{
		GenericProvisioner: GenericProvisioner{
			DockerOptionsDir:  "/etc/docker",
			DaemonOptionsFile: "/etc/systemd/system/docker.service",
			OsReleaseId:       "rhel",
			Packages: []string{
				"curl",
			},
			Driver: d,
		},
	}
}

type RedHatProvisioner struct {
	GenericProvisioner
}

func (provisioner *RedHatProvisioner) SSHCommand(args string) (string, error) {
	client, err := drivers.GetSSHClientFromDriver(provisioner.Driver)
	if err != nil {
		return "", err
	}

	// redhat needs "-t" for tty allocation on ssh therefore we check for the
	// external client and add as needed.
	// Note: CentOS 7.0 needs multiple "-tt" to force tty allocation when ssh has
	// no local tty.
	switch c := client.(type) {
	case ssh.ExternalClient:
		c.BaseArgs = append(c.BaseArgs, "-tt")
		client = c
	case ssh.NativeClient:
		return c.OutputWithPty(args)
	}

	return client.Output(args)
}

func (provisioner *RedHatProvisioner) SetHostname(hostname string) error {
	// we have to have SetHostname here as well to use the RedHat provisioner
	// SSHCommand to add the tty allocation
	if _, err := provisioner.SSHCommand(fmt.Sprintf(
		"sudo hostname %s && echo %q | sudo tee /etc/hostname",
		hostname,
		hostname,
	)); err != nil {
		return err
	}

	// ubuntu/debian use 127.0.1.1 for non "localhost" loopback hostnames: https://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_hostname_resolution
	if _, err := provisioner.SSHCommand(fmt.Sprintf(
		"if grep -xq 127.0.1.1.* /etc/hosts; then sudo sed -i 's/^127.0.1.1.*/127.0.1.1 %s/g' /etc/hosts; else echo '127.0.1.1 %s' | sudo tee -a /etc/hosts; fi",
		hostname,
		hostname,
	)); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) Service(name string, action serviceaction.ServiceAction) error {
	reloadDaemon := false
	switch action {
	case serviceaction.Start, serviceaction.Restart:
		reloadDaemon = true
	}

	// systemd needs reloaded when config changes on disk; we cannot
	// be sure exactly when it changes from the provisioner so
	// we call a reload on every restart to be safe
	if reloadDaemon {
		if _, err := provisioner.SSHCommand("sudo systemctl daemon-reload"); err != nil {
			return err
		}
	}

	command := fmt.Sprintf("sudo systemctl %s %s", action.String(), name)

	if _, err := provisioner.SSHCommand(command); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) Package(name string, action pkgaction.PackageAction) error {
	var packageAction string

	switch action {
	case pkgaction.Install:
		packageAction = "install"
	case pkgaction.Remove:
		packageAction = "remove"
	case pkgaction.Upgrade:
		packageAction = "upgrade"
	}

	command := fmt.Sprintf("sudo -E yum %s -y %s", packageAction, name)

	if _, err := provisioner.SSHCommand(command); err != nil {
		return err
	}

	return nil
}

func installDocker(provisioner *RedHatProvisioner) error {
	if err := provisioner.installOfficialDocker(); err != nil {
		return err
	}

	if err := provisioner.Service("docker", serviceaction.Restart); err != nil {
		return err
	}

	if err := provisioner.Service("docker", serviceaction.Enable); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) installOfficialDocker() error {
	log.Debug("installing docker")

	if err := provisioner.ConfigurePackageList(); err != nil {
		return err
	}

	if _, err := provisioner.SSHCommand("sudo yum install -y docker-engine"); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) dockerDaemonResponding() bool {
	if _, err := provisioner.SSHCommand("sudo docker version"); err != nil {
		log.Warnf("Error getting SSH command to check if the daemon is up: %s", err)
		return false
	}

	// The daemon is up if the command worked.  Carry on.
	return true
}

func (provisioner *RedHatProvisioner) Provision(swarmOptions swarm.SwarmOptions, authOptions auth.AuthOptions, engineOptions engine.EngineOptions) error {
	provisioner.SwarmOptions = swarmOptions
	provisioner.AuthOptions = authOptions
	provisioner.EngineOptions = engineOptions

	// set default storage driver for redhat
	if provisioner.EngineOptions.StorageDriver == "" {
		provisioner.EngineOptions.StorageDriver = "devicemapper"
	}

	if err := provisioner.SetHostname(provisioner.Driver.GetMachineName()); err != nil {
		return err
	}

	for _, pkg := range provisioner.Packages {
		log.Debugf("installing base package: name=%s", pkg)
		if err := provisioner.Package(pkg, pkgaction.Install); err != nil {
			return err
		}
	}

	// update OS -- this is needed for libdevicemapper and the docker install
	if _, err := provisioner.SSHCommand("sudo yum -y update"); err != nil {
		return err
	}

	// install docker
	if err := installDocker(provisioner); err != nil {
		return err
	}

	if err := mcnutils.WaitFor(provisioner.dockerDaemonResponding); err != nil {
		return err
	}

	if err := makeDockerOptionsDir(provisioner); err != nil {
		return err
	}

	provisioner.AuthOptions = setRemoteAuthOptions(provisioner)

	if err := ConfigureAuth(provisioner); err != nil {
		return err
	}

	if err := configureSwarm(provisioner, swarmOptions, provisioner.AuthOptions); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) {
	var (
		engineCfg  bytes.Buffer
		configPath = provisioner.DaemonOptionsFile
	)

	driverNameLabel := fmt.Sprintf("provider=%s", provisioner.Driver.DriverName())
	provisioner.EngineOptions.Labels = append(provisioner.EngineOptions.Labels, driverNameLabel)

	// systemd / redhat will not load options if they are on newlines
	// instead, it just continues with a different set of options; yeah...
	t, err := template.New("engineConfig").Parse(engineConfigTemplate)
	if err != nil {
		return nil, err
	}

	engineConfigContext := EngineConfigContext{
		DockerPort:       dockerPort,
		AuthOptions:      provisioner.AuthOptions,
		EngineOptions:    provisioner.EngineOptions,
		DockerOptionsDir: provisioner.DockerOptionsDir,
	}

	t.Execute(&engineCfg, engineConfigContext)

	daemonOptsDir := configPath
	return &DockerOptions{
		EngineOptions:     engineCfg.String(),
		EngineOptionsPath: daemonOptsDir,
	}, nil
}

func generateYumRepoList(provisioner Provisioner) (*bytes.Buffer, error) {
	packageListInfo := &PackageListInfo{}

	releaseInfo, err := provisioner.GetOsReleaseInfo()
	if err != nil {
		return nil, err
	}

	switch releaseInfo.Id {
	case "rhel", "centos":
		// rhel and centos both use the "centos" repo
		packageListInfo.OsRelease = "centos"
		packageListInfo.OsReleaseVersion = "7"
	case "fedora":
		packageListInfo.OsRelease = "fedora"
		packageListInfo.OsReleaseVersion = "22"
	default:
		return nil, ErrUnknownYumOsRelease
	}

	t, err := template.New("packageList").Parse(packageListTemplate)
	if err != nil {
		return nil, err
	}

	var buf bytes.Buffer

	if err := t.Execute(&buf, packageListInfo); err != nil {
		return nil, err
	}

	return &buf, nil
}

func (provisioner *RedHatProvisioner) ConfigurePackageList() error {
	buf, err := generateYumRepoList(provisioner)
	if err != nil {
		return err
	}

	// we cannot use %q here as it combines the newlines in the formatting
	// on transport causing yum to not use the repo
	packageCmd := fmt.Sprintf("echo \"%s\" | sudo tee /etc/yum.repos.d/docker.repo", buf.String())
	if _, err := provisioner.SSHCommand(packageCmd); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package interpolate

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"os/exec"
	"strings"
	"testing"
)

func BenchmarkConvolveArray200Filter21(b *testing.B) {
	out, xs := make([]float64, 200), make([]float64, 200)
	k := NewTophatKernel(21)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		k.ConvolveAt(xs, Extension, out)
	}
}

func BenchmarkNewSavGolKernel21(b *testing.B) {
	for i := 0; i < b.N; i++ {
		NewSavGolKernel(4, 21)
	}
}

func almostEq(xs, ys []float64) bool {
	if len(xs) != len(ys) { return false }
	eps := 1e-3
	for i := range xs {
		if !(xs[i] + eps > ys[i] && xs[i] - eps < ys[i]) {
			return false
		}
	}
	return true
}

func TestSavGolKernel(t *testing.T) {
	table := []struct{
		order, width int
		cs []float64
	} {
		{2, 5, []float64{-0.086, 0.343, 0.486, 0.343, -0.086}},
		{2, 11, []float64{-0.084, 0.021, 0.103, 0.161, 0.196,
			0.207, 0.196, 0.161, 0.103, 0.021, -0.084}},
		{4, 9, []float64{0.035, -0.128, 0.070, 0.315,
			0.417, 0.315, 0.070, -0.128, 0.035}},
		{4, 11, []float64{0.042, -0.105, -0.023, 0.140, 0.280,
			0.333, 0.280, 0.140, -0.023, -0.105, 0.042}},
	}
	for i, test := range table {
		k := NewSavGolKernel(test.order, test.width)
		if !almostEq(k.cs, test.cs) {
			t.Errorf("%d) Expected %.3f for coefficients. Got %.3f.",
				i+1, test.cs, k.cs)
		}
	}
}

func pythonArray(xs []float64) string {
	tokens := make([]string, len(xs))
	for i := range xs { tokens[i] = fmt.Sprintf("%g", xs[i]) }
	return fmt.Sprintf("[%s]", strings.Join(tokens, ", "))
}

func pyplotPlotString(xs, ys []float64, format, label string) string {
	return fmt.Sprintf(
		"plt.plot(%s, %s, '%s', label='%s', lw=3)",
		pythonArray(xs), pythonArray(ys), format, label,
	)
}

func pyplotFigString(i int) string {
	return `plt.figure(figsize=(8,7))
plt.rc('font',size=20)
plt.rc('xtick.major',pad=5); plt.rc('xtick.minor',pad=5)
plt.rc('ytick.major',pad=5); plt.rc('ytick.minor',pad=5)
plt.grid()`
}

func gaussian(x0, sigma, A, x float64) float64 {
	dx := x - x0
	return A * math.Exp(-dx*dx / (2*sigma*sigma))
}

func linspace(low, high float64, n int) []float64 {
	xs := make([]float64, n)
	dx := (high - low) / float64(n - 1)
	for i := range xs { xs[i] = low + dx*float64(i) }
	return xs
}

func rawSavGolFunc(x float64) float64 {
	return gaussian(2, 1, 1.5, x) + gaussian(4, 0.5, 1.5, x) +
		gaussian(5.5, 0.125, 1.5, x) + gaussian(0.5, 0.125, 1.5, x)
}

func TestPyplotSavGol(t *testing.T) {
	f, err := ioutil.TempFile(".", "savgol_pyplot")
	if err != nil { t.Fatal(err.Error()) }

	xs := linspace(0, 6, 200)
	rawYs := make([]float64, 200)
	noiseYs := make([]float64, 200)

	for i, x := range xs {
		rawYs[i] = rawSavGolFunc(x)
		noiseYs[i] = rawYs[i] + rand.Float64() - 0.5
	}

	window := 41
	windowSize := float64(window) / float64(len(xs)) * (xs[len(xs)-1]-xs[0])
	sigma := windowSize / 5

	tk := NewTophatKernel(window)
	gk:= NewGaussianKernel(window, sigma, xs[1]-xs[0])
	sgk := NewSavGolKernel(4, window)

	fig0 := pyplotFigString(0)
	plot1 := pyplotPlotString(xs, rawYs, "m", "Underlying Function")
	plot2 := pyplotPlotString(xs, noiseYs, "k", "Noisy Function")
	b := Extension
	plot3 := pyplotPlotString(xs, tk.Convolve(noiseYs, b), "r", "Tophat")
	plot4 := pyplotPlotString(xs, gk.Convolve(noiseYs, b), "g", "Gaussian")
	plot5 := pyplotPlotString(xs, sgk.Convolve(noiseYs, b),"b","Savitzky Golay")
	loc := "lower left"
	fileBody := fmt.Sprintf(`import numpy as np
import matplotlib.pyplot as plt

%s
%s
%s
%s
%s
%s

plt.legend(fontsize=12, loc='%s')
plt.show()`, fig0, plot1, plot2, plot3, plot4, plot5, loc)

	f.Write([]byte(fileBody))
	c := exec.Command("python", f.Name())
	rm := exec.Command("rm", f.Name())
	defer rm.Run()
	var out bytes.Buffer
	c.Stdout = &out

	err = c.Run()
	if err != nil { t.Fatal(err.Error()) }
}
<commit_msg>Allowed called to pyplotPlotString to omit labels.<commit_after>package interpolate

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"os/exec"
	"strings"
	"testing"
)

func BenchmarkConvolveArray200Filter21(b *testing.B) {
	out, xs := make([]float64, 200), make([]float64, 200)
	k := NewTophatKernel(21)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		k.ConvolveAt(xs, Extension, out)
	}
}

func BenchmarkNewSavGolKernel21(b *testing.B) {
	for i := 0; i < b.N; i++ {
		NewSavGolKernel(4, 21)
	}
}

func almostEq(xs, ys []float64) bool {
	if len(xs) != len(ys) { return false }
	eps := 1e-3
	for i := range xs {
		if !(xs[i] + eps > ys[i] && xs[i] - eps < ys[i]) {
			return false
		}
	}
	return true
}

func TestSavGolKernel(t *testing.T) {
	table := []struct{
		order, width int
		cs []float64
	} {
		{2, 5, []float64{-0.086, 0.343, 0.486, 0.343, -0.086}},
		{2, 11, []float64{-0.084, 0.021, 0.103, 0.161, 0.196,
			0.207, 0.196, 0.161, 0.103, 0.021, -0.084}},
		{4, 9, []float64{0.035, -0.128, 0.070, 0.315,
			0.417, 0.315, 0.070, -0.128, 0.035}},
		{4, 11, []float64{0.042, -0.105, -0.023, 0.140, 0.280,
			0.333, 0.280, 0.140, -0.023, -0.105, 0.042}},
	}
	for i, test := range table {
		k := NewSavGolKernel(test.order, test.width)
		if !almostEq(k.cs, test.cs) {
			t.Errorf("%d) Expected %.3f for coefficients. Got %.3f.",
				i+1, test.cs, k.cs)
		}
	}
}

func pythonArray(xs []float64) string {
	tokens := make([]string, len(xs))
	for i := range xs { tokens[i] = fmt.Sprintf("%g", xs[i]) }
	return fmt.Sprintf("[%s]", strings.Join(tokens, ", "))
}

func pyplotPlotString(xs, ys []float64, format, label string) string {
	if label == "" {
		return fmt.Sprintf(
			"plt.plot(%s, %s, '%s', lw=3)",
			pythonArray(xs), pythonArray(ys), format,
		)
	} else {
		return fmt.Sprintf(
			"plt.plot(%s, %s, '%s', label='%s', lw=3)",
			pythonArray(xs), pythonArray(ys), format, label,
		)
	}
}

func pyplotFigString(i int) string {
	return `plt.figure(figsize=(8,7))
plt.rc('font',size=20)
plt.rc('xtick.major',pad=5); plt.rc('xtick.minor',pad=5)
plt.rc('ytick.major',pad=5); plt.rc('ytick.minor',pad=5)
plt.grid()`
}

func gaussian(x0, sigma, A, x float64) float64 {
	dx := x - x0
	return A * math.Exp(-dx*dx / (2*sigma*sigma))
}

func linspace(low, high float64, n int) []float64 {
	xs := make([]float64, n)
	dx := (high - low) / float64(n - 1)
	for i := range xs { xs[i] = low + dx*float64(i) }
	return xs
}

func rawSavGolFunc(x float64) float64 {
	return gaussian(2, 1, 1.5, x) + gaussian(4, 0.5, 1.5, x) +
		gaussian(5.5, 0.125, 1.5, x) + gaussian(0.5, 0.125, 1.5, x)
}

func TestPyplotSavGol(t *testing.T) {
	f, err := ioutil.TempFile(".", "savgol_pyplot")
	if err != nil { t.Fatal(err.Error()) }

	xs := linspace(0, 6, 200)
	rawYs := make([]float64, 200)
	noiseYs := make([]float64, 200)

	for i, x := range xs {
		rawYs[i] = rawSavGolFunc(x)
		noiseYs[i] = rawYs[i] + rand.Float64() - 0.5
	}

	window := 41
	windowSize := float64(window) / float64(len(xs)) * (xs[len(xs)-1]-xs[0])
	sigma := windowSize / 5

	tk := NewTophatKernel(window)
	gk:= NewGaussianKernel(window, sigma, xs[1]-xs[0])
	sgk := NewSavGolKernel(4, window)

	fig0 := pyplotFigString(0)
	plot1 := pyplotPlotString(xs, rawYs, "m", "Underlying Function")
	plot2 := pyplotPlotString(xs, noiseYs, "k", "Noisy Function")
	b := Extension
	plot3 := pyplotPlotString(xs, tk.Convolve(noiseYs, b), "r", "Tophat")
	plot4 := pyplotPlotString(xs, gk.Convolve(noiseYs, b), "g", "Gaussian")
	plot5 := pyplotPlotString(xs, sgk.Convolve(noiseYs, b),"b","Savitzky Golay")
	loc := "lower left"
	fileBody := fmt.Sprintf(`import numpy as np
import matplotlib.pyplot as plt

%s
%s
%s
%s
%s
%s

plt.legend(fontsize=12, loc='%s')
plt.show()`, fig0, plot1, plot2, plot3, plot4, plot5, loc)

	f.Write([]byte(fileBody))
	c := exec.Command("python", f.Name())
	rm := exec.Command("rm", f.Name())
	defer rm.Run()
	var out bytes.Buffer
	c.Stdout = &out

	err = c.Run()
	if err != nil { t.Fatal(err.Error()) }
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package swim

import (
	"math"
	"sync"

	log "github.com/uber-common/bark"
	"github.com/uber/ringpop-go/logging"
)

var log10 = math.Log(10)

// A pChange is a change with a p count representing the number of times the
// change has been propagated to other nodes.
type pChange struct {
	Change
	p int
}

// A disseminator propagates changes to other nodes.
type disseminator struct {
	node    *Node
	changes map[string]*pChange
	maxP    int
	pFactor int

	sync.RWMutex

	logger log.Logger
}

// newDisseminator returns a new Disseminator instance.
func newDisseminator(n *Node) *disseminator {
	d := &disseminator{
		node:    n,
		changes: make(map[string]*pChange),
		maxP:    1,
		pFactor: 15,
		logger:  logging.Logger("disseminator").WithField("local", n.Address()),
	}

	return d
}

func (d *disseminator) AdjustMaxPropagations() {
	if !d.node.Ready() {
		return
	}

	d.Lock()

	numPingable := d.node.memberlist.NumPingableMembers()
	prevMaxP := d.maxP

	newMaxP := d.pFactor * int(math.Ceil(math.Log(float64(numPingable+1))/log10))

	if newMaxP != prevMaxP {
		d.maxP = newMaxP

		d.node.emit(MaxPAdjustedEvent{prevMaxP, newMaxP})

		d.logger.WithFields(log.Fields{
			"newMax":            newMaxP,
			"prevMax":           prevMaxP,
			"propagationFactor": d.pFactor,
			"numPingable":       numPingable,
		}).Debug("adjusted max propagation count")
	}

	d.Unlock()
}

// HasChanges reports whether disseminator has changes to disseminate.
func (d *disseminator) HasChanges() bool {
	d.Lock()
	r := len(d.changes) > 0
	d.Unlock()
	return r
}

func (d *disseminator) FullSync() (changes []Change) {
	d.Lock()

	for _, member := range d.node.memberlist.GetMembers() {
		changes = append(changes, Change{
			Address:           member.Address,
			Incarnation:       member.Incarnation,
			Source:            d.node.Address(),
			SourceIncarnation: d.node.Incarnation(),
			Status:            member.Status,
		})
	}

	d.Unlock()

	return changes
}

func (d *disseminator) IssueAsSender() []Change {
	return d.issueChanges(nil)
}

func (d *disseminator) IssueAsReceiver(senderAddress string,
	senderIncarnation int64, senderChecksum uint32) ([]Change, bool) {

	filter := func(c *pChange) bool {
		return senderIncarnation == c.SourceIncarnation &&
			senderAddress == c.Source
	}

	changes := d.issueChanges(filter)

	if len(changes) > 0 {
		return changes, false
	} else if d.node.memberlist.Checksum() != senderChecksum {
		d.node.emit(FullSyncEvent{senderAddress, senderChecksum})

		d.logger.WithFields(log.Fields{
			"localChecksum":  d.node.memberlist.Checksum(),
			"remote":         senderAddress,
			"remoteChecksum": senderChecksum,
		}).Info("full sync")

		return d.FullSync(), true
	}

	return []Change{}, false
}

func (d *disseminator) issueChanges(filter func(*pChange) bool) (changes []Change) {
	d.Lock()

	// To make JSON output [] instead of null on empty change list
	changes = make([]Change, 0)
	for _, change := range d.changes {
		if filter != nil && filter(change) {
			d.node.emit(ChangeFilteredEvent{change.Change})
			continue
		}

		change.p++

		if change.p > d.maxP {
			delete(d.changes, change.Address)
			continue
		}

		changes = append(changes, change.Change)
	}

	d.Unlock()

	d.node.emit(ChangesCalculatedEvent{changes})

	return changes
}

func (d *disseminator) ClearChanges() {
	d.Lock()
	d.changes = make(map[string]*pChange)
	d.Unlock()
}

func (d *disseminator) RecordChange(change Change) {
	d.Lock()
	d.changes[change.Address] = &pChange{change, 0}
	d.Unlock()
}
<commit_msg>address feedback<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package swim

import (
	"math"
	"sync"

	log "github.com/uber-common/bark"
	"github.com/uber/ringpop-go/logging"
)

var log10 = math.Log(10)

// A pChange is a change with a p count representing the number of times the
// change has been propagated to other nodes.
type pChange struct {
	Change
	p int
}

// A disseminator propagates changes to other nodes.
type disseminator struct {
	node    *Node
	changes map[string]*pChange
	maxP    int
	pFactor int

	sync.RWMutex

	logger log.Logger
}

// newDisseminator returns a new Disseminator instance.
func newDisseminator(n *Node) *disseminator {
	d := &disseminator{
		node:    n,
		changes: make(map[string]*pChange),
		maxP:    1,
		pFactor: 15,
		logger:  logging.Logger("disseminator").WithField("local", n.Address()),
	}

	return d
}

func (d *disseminator) AdjustMaxPropagations() {
	if !d.node.Ready() {
		return
	}

	d.Lock()

	numPingable := d.node.memberlist.NumPingableMembers()
	prevMaxP := d.maxP

	newMaxP := d.pFactor * int(math.Ceil(math.Log(float64(numPingable+1))/log10))

	if newMaxP != prevMaxP {
		d.maxP = newMaxP

		d.node.emit(MaxPAdjustedEvent{prevMaxP, newMaxP})

		d.logger.WithFields(log.Fields{
			"newMax":            newMaxP,
			"prevMax":           prevMaxP,
			"propagationFactor": d.pFactor,
			"numPingable":       numPingable,
		}).Debug("adjusted max propagation count")
	}

	d.Unlock()
}

// HasChanges reports whether disseminator has changes to disseminate.
func (d *disseminator) HasChanges() bool {
	d.RLock()
	result := len(d.changes) > 0
	d.RUnlock()
	return result
}

func (d *disseminator) FullSync() (changes []Change) {
	d.Lock()

	for _, member := range d.node.memberlist.GetMembers() {
		changes = append(changes, Change{
			Address:           member.Address,
			Incarnation:       member.Incarnation,
			Source:            d.node.Address(),
			SourceIncarnation: d.node.Incarnation(),
			Status:            member.Status,
		})
	}

	d.Unlock()

	return changes
}

func (d *disseminator) IssueAsSender() []Change {
	return d.issueChanges(nil)
}

func (d *disseminator) IssueAsReceiver(senderAddress string,
	senderIncarnation int64, senderChecksum uint32) ([]Change, bool) {

	filter := func(c *pChange) bool {
		return senderIncarnation == c.SourceIncarnation &&
			senderAddress == c.Source
	}

	changes := d.issueChanges(filter)

	if len(changes) > 0 {
		return changes, false
	} else if d.node.memberlist.Checksum() != senderChecksum {
		d.node.emit(FullSyncEvent{senderAddress, senderChecksum})

		d.logger.WithFields(log.Fields{
			"localChecksum":  d.node.memberlist.Checksum(),
			"remote":         senderAddress,
			"remoteChecksum": senderChecksum,
		}).Info("full sync")

		return d.FullSync(), true
	}

	return []Change{}, false
}

func (d *disseminator) issueChanges(filter func(*pChange) bool) (changes []Change) {
	d.Lock()

	// To make JSON output [] instead of null on empty change list
	changes = make([]Change, 0)
	for _, change := range d.changes {
		if filter != nil && filter(change) {
			d.node.emit(ChangeFilteredEvent{change.Change})
			continue
		}

		change.p++

		if change.p > d.maxP {
			delete(d.changes, change.Address)
			continue
		}

		changes = append(changes, change.Change)
	}

	d.Unlock()

	d.node.emit(ChangesCalculatedEvent{changes})

	return changes
}

func (d *disseminator) ClearChanges() {
	d.Lock()
	d.changes = make(map[string]*pChange)
	d.Unlock()
}

func (d *disseminator) RecordChange(change Change) {
	d.Lock()
	d.changes[change.Address] = &pChange{change, 0}
	d.Unlock()
}
<|endoftext|>


<commit_before>package vulcand

import (
	"context"
	"encoding/json"
	"fmt"
	"sync"
	"time"

	etcd "github.com/coreos/etcd/client"
	"github.com/mailgun/log"
	"github.com/pkg/errors"
)

const (
	localEtcdProxy = "http://127.0.0.1:2379"
	frontendFmt    = "%s/frontends/%s.%s/frontend"
	middlewareFmt  = "%s/frontends/%s.%s/middlewares/%s"
	backendFmt     = "%s/backends/%s/backend"
	serverFmt      = "%s/backends/%s/servers/%s"

	defaultRegistrationTTL = 30 * time.Second
)

type Config struct {
	Etcd   *etcd.Config
	Chroot string
	TTL    time.Duration
}

type Registry struct {
	cfg           Config
	etcdKeysAPI   etcd.KeysAPI
	backendSpec   *backendSpec
	frontendSpecs []*frontendSpec
	ctx           context.Context
	cancelFunc    context.CancelFunc
	wg            sync.WaitGroup
}

func NewRegistry(cfg Config, appName, ip string, port int) (*Registry, error) {
	backendSpec, err := newBackendSpec(appName, ip, port)
	if err != nil {
		return nil, errors.Wrap(err, "failed to create backend")
	}

	if cfg.TTL <= 0 {
		cfg.TTL = defaultRegistrationTTL
	}

	etcdCfg := cfg.Etcd
	if etcdCfg == nil {
		etcdCfg = &etcd.Config{Endpoints: []string{localEtcdProxy}}
	}

	etcdClt, err := etcd.New(*etcdCfg)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create Etcd client, cfg=%v", etcdCfg)
	}
	ctx, cancelFunc := context.WithCancel(context.Background())
	go func() {
		for {
			err := etcdClt.AutoSync(ctx, 10*time.Second)
			if err == context.DeadlineExceeded || err == context.Canceled {
				break
			}
			log.Errorf("Etcd auto sync failed: err=(%v)", err)
		}
	}()
	etcdKeysAPI := etcd.NewKeysAPI(etcdClt)
	c := Registry{
		cfg:         cfg,
		backendSpec: backendSpec,
		etcdKeysAPI: etcdKeysAPI,
		ctx:         ctx,
		cancelFunc:  cancelFunc,
	}
	return &c, nil
}

func (r *Registry) AddFrontend(host, path string, methods []string, middlewares []Middleware) {
	r.frontendSpecs = append(r.frontendSpecs, newFrontendSpec(r.backendSpec.AppName, host, path, methods, middlewares))
}

func (r *Registry) Start() error {
	if err := r.registerBackend(r.backendSpec); err != nil {
		return errors.Wrapf(err, "failed to register backend, %s", r.backendSpec.ID)
	}
	r.wg.Add(1)
	go r.heartbeat()

	for _, fes := range r.frontendSpecs {
		if err := r.registerFrontend(fes); err != nil {
			r.cancelFunc()
			return errors.Wrapf(err, "failed to register frontend, %s", fes.ID)
		}
	}
	return nil
}

func (r *Registry) Stop() {
	r.cancelFunc()
	r.wg.Wait()
}

func (r *Registry) heartbeat() {
	defer r.wg.Done()
	backendServerKey := fmt.Sprintf(serverFmt, r.cfg.Chroot, r.backendSpec.AppName, r.backendSpec.ID)
	backendServerVal := r.backendSpec.serverSpec()
	tick := time.NewTicker(r.cfg.TTL * 3 / 4)
	for {
		select {
		case <-tick.C:
			_, err := r.etcdKeysAPI.Set(r.ctx, backendServerKey, "",
				&etcd.SetOptions{PrevExist: etcd.PrevExist, Refresh: true, TTL: r.cfg.TTL})
			if err != nil {
				log.Errorf("Server TTL refresh failed: err=(%v)", err)
				_, err := r.etcdKeysAPI.Set(r.ctx, backendServerKey, backendServerVal,
					&etcd.SetOptions{TTL: r.cfg.TTL})
				if err != nil {
					log.Errorf("Server create failed: err=(%v)", err)
				}
			}
		case <-r.ctx.Done():
			_, err := r.etcdKeysAPI.Delete(context.Background(), backendServerKey, nil)
			log.Infof("Heartbeat stopped: err=(%v)", err)
			return
		}
	}
}

func (r *Registry) registerBackend(bes *backendSpec) error {
	betKey := fmt.Sprintf(backendFmt, r.cfg.Chroot, bes.AppName)
	betVal := bes.typeSpec()
	_, err := r.etcdKeysAPI.Set(r.ctx, betKey, betVal, nil)
	if err != nil {
		return errors.Wrapf(err, "failed to set backend type, %s", betKey)
	}
	besKey := fmt.Sprintf(serverFmt, r.cfg.Chroot, bes.AppName, bes.ID)
	besVar := bes.serverSpec()
	_, err = r.etcdKeysAPI.Set(r.ctx, besKey, besVar, &etcd.SetOptions{TTL: r.cfg.TTL})
	return errors.Wrapf(err, "failed to set backend spec, %s", besKey)
}

func (r *Registry) registerFrontend(fes *frontendSpec) error {
	fesKey := fmt.Sprintf(frontendFmt, r.cfg.Chroot, fes.Host, fes.ID)
	fesVal := fes.spec()
	_, err := r.etcdKeysAPI.Set(r.ctx, fesKey, fesVal, nil)
	if err != nil {
		return errors.Wrapf(err, "failed to set frontend spec, %s", fesKey)
	}
	for i, mw := range fes.Middlewares {
		mw.Priority = i
		mwKey := fmt.Sprintf(middlewareFmt, r.cfg.Chroot, fes.Host, fes.ID, mw.ID)
		mwVal, err := json.Marshal(mw)
		if err != nil {
			return errors.Wrapf(err, "failed to JSON middleware, %v", mw)
		}
		_, err = r.etcdKeysAPI.Set(r.ctx, mwKey, string(mwVal), nil)
		if err != nil {
			return errors.Wrapf(err, "failed to set middleware, %s", mwKey)
		}
	}
	return nil
}
<commit_msg>migrated vulcand registration to etcd v3<commit_after>package vulcand

import (
	"context"
	"encoding/json"
	"fmt"
	"sync"
	"time"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/mailgun/log"
	"github.com/pkg/errors"
)

const (
	localEtcdProxy = "127.0.0.1:2379"
	frontendFmt    = "%s/frontends/%s.%s/frontend"
	middlewareFmt  = "%s/frontends/%s.%s/middlewares/%s"
	backendFmt     = "%s/backends/%s/backend"
	serverFmt      = "%s/backends/%s/servers/%s"

	defaultRegistrationTTL = 30 * time.Second
)

type Config struct {
	Etcd   *etcd.Config
	Chroot string
	TTL    time.Duration
}

type Registry struct {
	cfg           Config
	client        *etcd.Client
	backendSpec   *backendSpec
	frontendSpecs []*frontendSpec
	ctx           context.Context
	cancelFunc    context.CancelFunc
	wg            sync.WaitGroup
	leaseID       etcd.LeaseID
}

func NewRegistry(cfg Config, appName, ip string, port int) (*Registry, error) {
	backendSpec, err := newBackendSpec(appName, ip, port)
	if err != nil {
		return nil, errors.Wrap(err, "failed to create backend")
	}

	if cfg.TTL <= 0 {
		cfg.TTL = defaultRegistrationTTL
	}

	if cfg.Etcd == nil {
		cfg.Etcd = &etcd.Config{Endpoints: []string{localEtcdProxy}}
	}

	client, err := etcd.New(*cfg.Etcd)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create Etcd client, cfg=%v", *cfg.Etcd)
	}
	ctx, cancelFunc := context.WithCancel(context.Background())

	// Grant a new lease for this client instance
	resp, err := client.Grant(ctx, int64(cfg.TTL.Seconds()))
	if err != nil {
		return nil, errors.Wrapf(err, "failed to grant a new lease, cfg=%v", *cfg.Etcd)
	}

	// Keep the lease alive for as long as we live
	_, err = client.KeepAlive(ctx, resp.ID)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to start keep alive, cfg=%v", *cfg.Etcd)
	}

	c := Registry{
		cfg:         cfg,
		backendSpec: backendSpec,
		client:      client,
		leaseID:     resp.ID,
		ctx:         ctx,
		cancelFunc:  cancelFunc,
	}
	return &c, nil
}

func (r *Registry) AddFrontend(host, path string, methods []string, middlewares []Middleware) {
	r.frontendSpecs = append(r.frontendSpecs, newFrontendSpec(r.backendSpec.AppName, host, path, methods, middlewares))
}

func (r *Registry) Start() error {
	if err := r.registerBackend(r.backendSpec); err != nil {
		return errors.Wrapf(err, "failed to register backend, %s", r.backendSpec.ID)
	}

	// Write our backend spec config
	key := fmt.Sprintf(serverFmt, r.cfg.Chroot, r.backendSpec.AppName, r.backendSpec.ID)
	_, err := r.client.Put(r.ctx, key, r.backendSpec.serverSpec(), etcd.WithLease(r.leaseID))
	if err != nil {
		return errors.Wrap(err, "failed to write backend spec")
	}

	go func() {
		for {
			select {
			case <-r.ctx.Done():
				_, err := r.client.Revoke(context.Background(), r.leaseID)
				log.Infof("lease revoked err=(%v)", err)
				return
			}
		}
	}()

	for _, fes := range r.frontendSpecs {
		if err := r.registerFrontend(fes); err != nil {
			r.cancelFunc()
			return errors.Wrapf(err, "failed to register frontend, %s", fes.ID)
		}
	}
	return nil
}

func (r *Registry) Stop() {
	r.cancelFunc()
	r.wg.Wait()
}

func (r *Registry) registerBackend(bes *backendSpec) error {
	betKey := fmt.Sprintf(backendFmt, r.cfg.Chroot, bes.AppName)
	betVal := bes.typeSpec()
	_, err := r.client.Put(r.ctx, betKey, betVal)
	if err != nil {
		return errors.Wrapf(err, "failed to set backend type, %s", betKey)
	}
	besKey := fmt.Sprintf(serverFmt, r.cfg.Chroot, bes.AppName, bes.ID)
	besVar := bes.serverSpec()
	_, err = r.client.Put(r.ctx, besKey, besVar, etcd.WithLease(r.leaseID))
	return errors.Wrapf(err, "failed to set backend spec, %s", besKey)
}

func (r *Registry) registerFrontend(fes *frontendSpec) error {
	fesKey := fmt.Sprintf(frontendFmt, r.cfg.Chroot, fes.Host, fes.ID)
	fesVal := fes.spec()
	_, err := r.client.Put(r.ctx, fesKey, fesVal, nil)
	if err != nil {
		return errors.Wrapf(err, "failed to set frontend spec, %s", fesKey)
	}
	for i, mw := range fes.Middlewares {
		mw.Priority = i
		mwKey := fmt.Sprintf(middlewareFmt, r.cfg.Chroot, fes.Host, fes.ID, mw.ID)
		mwVal, err := json.Marshal(mw)
		if err != nil {
			return errors.Wrapf(err, "failed to JSON middleware, %v", mw)
		}
		_, err = r.client.Put(r.ctx, mwKey, string(mwVal))
		if err != nil {
			return errors.Wrapf(err, "failed to set middleware, %s", mwKey)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package db

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/url"
	"sort"
	"strconv"
	"strings"

	"github.com/bosssauce/ponzu/content"
	"github.com/bosssauce/ponzu/management/editor"
	"github.com/bosssauce/ponzu/management/manager"

	"github.com/boltdb/bolt"
	"github.com/gorilla/schema"
	uuid "github.com/satori/go.uuid"
)

// SetContent inserts or updates values in the database.
// The `target` argument is a string made up of namespace:id (string:int)
func SetContent(target string, data url.Values) (int, error) {
	t := strings.Split(target, ":")
	ns, id := t[0], t[1]

	// check if content id == -1 (indicating new post).
	// if so, run an insert which will assign the next auto incremented int.
	// this is done because boltdb begins its bucket auto increment value at 0,
	// which is the zero-value of an int in the Item struct field for ID.
	// this is a problem when the original first post (with auto ID = 0) gets
	// overwritten by any new post, originally having no ID, defauting to 0.
	if id == "-1" {
		return insert(ns, data)
	}

	return update(ns, id, data)
}

func update(ns, id string, data url.Values) (int, error) {
	var specifier string // i.e. _pending, _sorted, etc.
	if strings.Contains(ns, "_") {
		spec := strings.Split(ns, "_")
		ns = spec[0]
		specifier = "_" + spec[1]
	}

	cid, err := strconv.Atoi(id)
	if err != nil {
		return 0, err
	}

	err = store.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(ns + specifier))
		if err != nil {
			return err
		}

		j, err := postToJSON(ns, data)
		if err != nil {
			return err
		}

		err = b.Put([]byte(fmt.Sprintf("%d", cid)), j)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return 0, nil
	}

	if specifier == "" {
		go SortContent(ns)
	}

	return cid, nil
}

func insert(ns string, data url.Values) (int, error) {
	var effectedID int
	var specifier string // i.e. _pending, _sorted, etc.
	if strings.Contains(ns, "_") {
		spec := strings.Split(ns, "_")
		ns = spec[0]
		specifier = "_" + spec[1]
	}

	err := store.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(ns + specifier))
		if err != nil {
			return err
		}

		// get the next available ID and convert to string
		// also set effectedID to int of ID
		id, err := b.NextSequence()
		if err != nil {
			return err
		}
		cid := strconv.FormatUint(id, 10)
		effectedID, err = strconv.Atoi(cid)
		if err != nil {
			return err
		}
		data.Set("id", cid)

		// add UUID to data for use in embedded Item
		uid := uuid.NewV4()
		data.Set("uuid", uid.String())

		j, err := postToJSON(ns, data)
		if err != nil {
			return err
		}

		err = b.Put([]byte(cid), j)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return 0, err
	}

	if specifier == "" {
		go SortContent(ns)
	}

	return effectedID, nil
}

// DeleteContent removes an item from the database. Deleting a non-existent item
// will return a nil error.
func DeleteContent(target string) error {
	t := strings.Split(target, ":")
	ns, id := t[0], t[1]

	err := store.Update(func(tx *bolt.Tx) error {
		tx.Bucket([]byte(ns)).Delete([]byte(id))
		return nil
	})
	if err != nil {
		return err
	}

	// exception to typical "run in goroutine" pattern:
	// we want to have an updated admin view as soon as this is deleted, so
	// in some cases, the delete and redirect is faster than the sort,
	// thus still showing a deleted post in the admin view.
	SortContent(ns)

	return nil
}

// Content retrives one item from the database. Non-existent values will return an empty []byte
// The `target` argument is a string made up of namespace:id (string:int)
func Content(target string) ([]byte, error) {
	t := strings.Split(target, ":")
	ns, id := t[0], t[1]

	val := &bytes.Buffer{}
	err := store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(ns))
		_, err := val.Write(b.Get([]byte(id)))
		if err != nil {
			log.Println(err)
			return err
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	return val.Bytes(), nil
}

// ContentAll retrives all items from the database within the provided namespace
func ContentAll(namespace string) [][]byte {
	var posts [][]byte
	store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(namespace))

		if b == nil {
			return nil
		}

		numKeys := b.Stats().KeyN
		posts = make([][]byte, 0, numKeys)

		b.ForEach(func(k, v []byte) error {
			posts = append(posts, v)

			return nil
		})

		return nil
	})

	return posts
}

// QueryOptions holds options for a query
type QueryOptions struct {
	Count  int
	Offset int
	Order  string
}

// Query retrieves a set of content from the db based on options
// and returns the total number of content in the namespace and the content
func Query(namespace string, opts QueryOptions) (int, [][]byte) {
	var posts [][]byte
	var total int

	// correct bad input rather than return nil or error
	// similar to default case for opts.Order switch below
	if opts.Count < 0 {
		opts.Count = 0
	}

	if opts.Offset < 0 {
		opts.Offset = 0
	}

	store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(namespace))
		if b == nil {
			return nil
		}

		c := b.Cursor()
		n := b.Stats().KeyN
		total = n

		// return nil if no content
		if n == 0 {
			return nil
		}

		var start, end int
		switch opts.Count {
		case -1:
			start = 0
			end = n

		default:
			start = opts.Count * opts.Offset
			end = start + opts.Count
		}

		// bounds check on posts given the start & end count
		if start > n {
			start = n - opts.Count
		}
		if end > n {
			end = n
		}

		i := 0   // count of num posts added
		cur := 0 // count of num cursor moves
		switch opts.Order {
		case "asc":
			for k, v := c.Last(); k != nil; k, v = c.Prev() {
				if cur < start {
					cur++
					continue
				}

				if cur >= end {
					break
				}

				posts = append(posts, v)
				i++
				cur++
			}

		case "desc", "":
			for k, v := c.First(); k != nil; k, v = c.Next() {
				if cur < start {
					cur++
					continue
				}

				if cur >= end {
					break
				}

				posts = append(posts, v)
				i++
				cur++
			}

		default:
			// results for DESC order
			for k, v := c.First(); k != nil; k, v = c.Next() {
				if cur < start {
					cur++
					continue
				}

				if cur >= end {
					break
				}

				posts = append(posts, v)
				i++
				cur++
			}
		}

		return nil
	})

	return total, posts
}

// SortContent sorts all content of the type supplied as the namespace by time,
// in descending order, from most recent to least recent
// Should be called from a goroutine after SetContent is successful
func SortContent(namespace string) {
	// only sort main content types i.e. Post
	if strings.Contains(namespace, "_") {
		return
	}

	all := ContentAll(namespace)

	var posts sortablePosts
	// decode each (json) into type to then sort
	for i := range all {
		j := all[i]
		post := content.Types[namespace]()

		err := json.Unmarshal(j, &post)
		if err != nil {
			log.Println("Error decoding json while sorting", namespace, ":", err)
			return
		}

		posts = append(posts, post.(editor.Sortable))
	}

	// sort posts
	sort.Sort(posts)

	// store in <namespace>_sorted bucket, first delete existing
	err := store.Update(func(tx *bolt.Tx) error {
		bname := []byte(namespace + "_sorted")
		err := tx.DeleteBucket(bname)
		if err != nil {
			return err
		}

		b, err := tx.CreateBucketIfNotExists(bname)
		if err != nil {
			return err
		}

		// encode to json and store as 'i:post.Time()':post
		for i := range posts {
			j, err := json.Marshal(posts[i])
			if err != nil {
				return err
			}

			cid := fmt.Sprintf("%d:%d", i, posts[i].Time())
			err = b.Put([]byte(cid), j)
			if err != nil {
				return err
			}
		}

		return nil
	})
	if err != nil {
		log.Println("Error while updating db with sorted", namespace, err)
	}

}

type sortablePosts []editor.Sortable

func (s sortablePosts) Len() int {
	return len(s)
}

func (s sortablePosts) Less(i, j int) bool {
	return s[i].Time() > s[j].Time()
}

func (s sortablePosts) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func postToJSON(ns string, data url.Values) ([]byte, error) {
	// find the content type and decode values into it
	ns = strings.TrimSuffix(ns, "_external")
	t, ok := content.Types[ns]
	if !ok {
		return nil, fmt.Errorf(content.ErrTypeNotRegistered, ns)
	}
	post := t()

	dec := schema.NewDecoder()
	dec.SetAliasTag("json")     // allows simpler struct tagging when creating a content type
	dec.IgnoreUnknownKeys(true) // will skip over form values submitted, but not in struct
	err := dec.Decode(post, data)
	if err != nil {
		return nil, err
	}

	slug, err := manager.Slug(post.(content.Identifiable))
	if err != nil {
		return nil, err
	}
	post.(content.Sluggable).SetSlug(slug)

	// marshall content struct to json for db storage
	j, err := json.Marshal(post)
	if err != nil {
		return nil, err
	}

	return j, nil
}
<commit_msg>removing old unnecessary text processing code<commit_after>package db

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/url"
	"sort"
	"strconv"
	"strings"

	"github.com/bosssauce/ponzu/content"
	"github.com/bosssauce/ponzu/management/editor"
	"github.com/bosssauce/ponzu/management/manager"

	"github.com/boltdb/bolt"
	"github.com/gorilla/schema"
	uuid "github.com/satori/go.uuid"
)

// SetContent inserts or updates values in the database.
// The `target` argument is a string made up of namespace:id (string:int)
func SetContent(target string, data url.Values) (int, error) {
	t := strings.Split(target, ":")
	ns, id := t[0], t[1]

	// check if content id == -1 (indicating new post).
	// if so, run an insert which will assign the next auto incremented int.
	// this is done because boltdb begins its bucket auto increment value at 0,
	// which is the zero-value of an int in the Item struct field for ID.
	// this is a problem when the original first post (with auto ID = 0) gets
	// overwritten by any new post, originally having no ID, defauting to 0.
	if id == "-1" {
		return insert(ns, data)
	}

	return update(ns, id, data)
}

func update(ns, id string, data url.Values) (int, error) {
	var specifier string // i.e. _pending, _sorted, etc.
	if strings.Contains(ns, "_") {
		spec := strings.Split(ns, "_")
		ns = spec[0]
		specifier = "_" + spec[1]
	}

	cid, err := strconv.Atoi(id)
	if err != nil {
		return 0, err
	}

	err = store.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(ns + specifier))
		if err != nil {
			return err
		}

		j, err := postToJSON(ns, data)
		if err != nil {
			return err
		}

		err = b.Put([]byte(fmt.Sprintf("%d", cid)), j)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return 0, nil
	}

	if specifier == "" {
		go SortContent(ns)
	}

	return cid, nil
}

func insert(ns string, data url.Values) (int, error) {
	var effectedID int
	var specifier string // i.e. _pending, _sorted, etc.
	if strings.Contains(ns, "_") {
		spec := strings.Split(ns, "_")
		ns = spec[0]
		specifier = "_" + spec[1]
	}

	err := store.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(ns + specifier))
		if err != nil {
			return err
		}

		// get the next available ID and convert to string
		// also set effectedID to int of ID
		id, err := b.NextSequence()
		if err != nil {
			return err
		}
		cid := strconv.FormatUint(id, 10)
		effectedID, err = strconv.Atoi(cid)
		if err != nil {
			return err
		}
		data.Set("id", cid)

		// add UUID to data for use in embedded Item
		uid := uuid.NewV4()
		data.Set("uuid", uid.String())

		j, err := postToJSON(ns, data)
		if err != nil {
			return err
		}

		err = b.Put([]byte(cid), j)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return 0, err
	}

	if specifier == "" {
		go SortContent(ns)
	}

	return effectedID, nil
}

// DeleteContent removes an item from the database. Deleting a non-existent item
// will return a nil error.
func DeleteContent(target string) error {
	t := strings.Split(target, ":")
	ns, id := t[0], t[1]

	err := store.Update(func(tx *bolt.Tx) error {
		tx.Bucket([]byte(ns)).Delete([]byte(id))
		return nil
	})
	if err != nil {
		return err
	}

	// exception to typical "run in goroutine" pattern:
	// we want to have an updated admin view as soon as this is deleted, so
	// in some cases, the delete and redirect is faster than the sort,
	// thus still showing a deleted post in the admin view.
	SortContent(ns)

	return nil
}

// Content retrives one item from the database. Non-existent values will return an empty []byte
// The `target` argument is a string made up of namespace:id (string:int)
func Content(target string) ([]byte, error) {
	t := strings.Split(target, ":")
	ns, id := t[0], t[1]

	val := &bytes.Buffer{}
	err := store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(ns))
		_, err := val.Write(b.Get([]byte(id)))
		if err != nil {
			log.Println(err)
			return err
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	return val.Bytes(), nil
}

// ContentAll retrives all items from the database within the provided namespace
func ContentAll(namespace string) [][]byte {
	var posts [][]byte
	store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(namespace))

		if b == nil {
			return nil
		}

		numKeys := b.Stats().KeyN
		posts = make([][]byte, 0, numKeys)

		b.ForEach(func(k, v []byte) error {
			posts = append(posts, v)

			return nil
		})

		return nil
	})

	return posts
}

// QueryOptions holds options for a query
type QueryOptions struct {
	Count  int
	Offset int
	Order  string
}

// Query retrieves a set of content from the db based on options
// and returns the total number of content in the namespace and the content
func Query(namespace string, opts QueryOptions) (int, [][]byte) {
	var posts [][]byte
	var total int

	// correct bad input rather than return nil or error
	// similar to default case for opts.Order switch below
	if opts.Count < 0 {
		opts.Count = 0
	}

	if opts.Offset < 0 {
		opts.Offset = 0
	}

	store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(namespace))
		if b == nil {
			return nil
		}

		c := b.Cursor()
		n := b.Stats().KeyN
		total = n

		// return nil if no content
		if n == 0 {
			return nil
		}

		var start, end int
		switch opts.Count {
		case -1:
			start = 0
			end = n

		default:
			start = opts.Count * opts.Offset
			end = start + opts.Count
		}

		// bounds check on posts given the start & end count
		if start > n {
			start = n - opts.Count
		}
		if end > n {
			end = n
		}

		i := 0   // count of num posts added
		cur := 0 // count of num cursor moves
		switch opts.Order {
		case "asc":
			for k, v := c.Last(); k != nil; k, v = c.Prev() {
				if cur < start {
					cur++
					continue
				}

				if cur >= end {
					break
				}

				posts = append(posts, v)
				i++
				cur++
			}

		case "desc", "":
			for k, v := c.First(); k != nil; k, v = c.Next() {
				if cur < start {
					cur++
					continue
				}

				if cur >= end {
					break
				}

				posts = append(posts, v)
				i++
				cur++
			}

		default:
			// results for DESC order
			for k, v := c.First(); k != nil; k, v = c.Next() {
				if cur < start {
					cur++
					continue
				}

				if cur >= end {
					break
				}

				posts = append(posts, v)
				i++
				cur++
			}
		}

		return nil
	})

	return total, posts
}

// SortContent sorts all content of the type supplied as the namespace by time,
// in descending order, from most recent to least recent
// Should be called from a goroutine after SetContent is successful
func SortContent(namespace string) {
	// only sort main content types i.e. Post
	if strings.Contains(namespace, "_") {
		return
	}

	all := ContentAll(namespace)

	var posts sortablePosts
	// decode each (json) into type to then sort
	for i := range all {
		j := all[i]
		post := content.Types[namespace]()

		err := json.Unmarshal(j, &post)
		if err != nil {
			log.Println("Error decoding json while sorting", namespace, ":", err)
			return
		}

		posts = append(posts, post.(editor.Sortable))
	}

	// sort posts
	sort.Sort(posts)

	// store in <namespace>_sorted bucket, first delete existing
	err := store.Update(func(tx *bolt.Tx) error {
		bname := []byte(namespace + "_sorted")
		err := tx.DeleteBucket(bname)
		if err != nil {
			return err
		}

		b, err := tx.CreateBucketIfNotExists(bname)
		if err != nil {
			return err
		}

		// encode to json and store as 'i:post.Time()':post
		for i := range posts {
			j, err := json.Marshal(posts[i])
			if err != nil {
				return err
			}

			cid := fmt.Sprintf("%d:%d", i, posts[i].Time())
			err = b.Put([]byte(cid), j)
			if err != nil {
				return err
			}
		}

		return nil
	})
	if err != nil {
		log.Println("Error while updating db with sorted", namespace, err)
	}

}

type sortablePosts []editor.Sortable

func (s sortablePosts) Len() int {
	return len(s)
}

func (s sortablePosts) Less(i, j int) bool {
	return s[i].Time() > s[j].Time()
}

func (s sortablePosts) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func postToJSON(ns string, data url.Values) ([]byte, error) {
	// find the content type and decode values into it
	t, ok := content.Types[ns]
	if !ok {
		return nil, fmt.Errorf(content.ErrTypeNotRegistered, ns)
	}
	post := t()

	dec := schema.NewDecoder()
	dec.SetAliasTag("json")     // allows simpler struct tagging when creating a content type
	dec.IgnoreUnknownKeys(true) // will skip over form values submitted, but not in struct
	err := dec.Decode(post, data)
	if err != nil {
		return nil, err
	}

	slug, err := manager.Slug(post.(content.Identifiable))
	if err != nil {
		return nil, err
	}
	post.(content.Sluggable).SetSlug(slug)

	// marshall content struct to json for db storage
	j, err := json.Marshal(post)
	if err != nil {
		return nil, err
	}

	return j, nil
}
<|endoftext|>


<commit_before>package gitlab

import (
	"fmt"
	"github.com/stretchr/testify/require"
	"net/http"
	"testing"
	"time"
)

func TestSystemHooksService_ListHooks(t *testing.T) {
	mux, server, client := setup(t)
	defer teardown(server)

	mux.HandleFunc("/api/v4/hooks", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, http.MethodGet)
		fmt.Fprint(w, `[{"id":1,"url":"https://gitlab.example.com/hook"}]`)
	})

	hooks, _, err := client.SystemHooks.ListHooks()
	require.NoError(t, err)

	want := []*Hook{{ID: 1, URL: "https://gitlab.example.com/hook"}}
	require.Equal(t, want, hooks)
}

func TestSystemHooksService_AddHook(t *testing.T) {
	mux, server, client := setup(t)
	defer teardown(server)

	mux.HandleFunc("/api/v4/hooks", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, http.MethodPost)
		fmt.Fprint(w, `{"id": 1, "url": "https://gitlab.example.com/hook"}`)
	})

	opt := &AddHookOptions{
		URL: String("https://gitlab.example.com/hook"),
	}

	h, _, err := client.SystemHooks.AddHook(opt)
	require.NoError(t, err)

	want := &Hook{ID: 1, URL: "https://gitlab.example.com/hook", CreatedAt: (*time.Time)(nil)}
	require.Equal(t, want, h)
}

func TestSystemHooksService_TestHook(t *testing.T) {
	mux, server, client := setup(t)
	defer teardown(server)

	mux.HandleFunc("/api/v4/hooks/1", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, http.MethodGet)
		fmt.Fprint(w, `{"project_id" : 1,"owner_email" : "example@gitlabhq.com","owner_name" : "Someone",
				"name" : "Ruby","path" : "ruby","event_name" : "project_create"}`)
	})

	he, _, err := client.SystemHooks.TestHook(1)

	require.NoError(t, err)
	want := &HookEvent{
		EventName:  "project_create",
		Name:       "Ruby",
		Path:       "ruby",
		ProjectID:  1,
		OwnerName:  "Someone",
		OwnerEmail: "example@gitlabhq.com",
	}
	require.Equal(t, want, he)
}

func TestSystemHooksService_DeleteHook(t *testing.T) {
	mux, server, client := setup(t)
	defer teardown(server)

	mux.HandleFunc("/api/v4/hooks/1", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, http.MethodDelete)
	})

	_, err := client.SystemHooks.DeleteHook(1)
	require.NoError(t, err)
}

func TestHook_String(t *testing.T) {
	h := &Hook{
		ID:        1,
		URL:       "https://github.com/",
		CreatedAt: nil,
	}
	hs := h.String()
	want := "gitlab.Hook{ID:1, URL:\"https://github.com/\"}"
	require.Equal(t,want , hs)
}

func TestHookEvent_String(t *testing.T) {
	he := &HookEvent{
		EventName:  "event",
		Name:       "name",
		Path:       "path",
		ProjectID:  1,
		OwnerName:  "owner",
		OwnerEmail: "owner_email",
	}

	hes := he.String()
	want := "gitlab.HookEvent{EventName:\"event\", Name:\"name\", Path:\"path\", ProjectID:1, OwnerName:\"owner\", OwnerEmail:\"owner_email\"}"
	require.Equal(t, want, hes)
}
<commit_msg>Update system_hooks_test.go<commit_after>package gitlab

import (
	"fmt"
	"github.com/stretchr/testify/require"
	"net/http"
	"testing"
	"time"
)

func TestSystemHooksService_ListHooks(t *testing.T) {
	mux, server, client := setup(t)
	defer teardown(server)

	mux.HandleFunc("/api/v4/hooks", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, http.MethodGet)
		fmt.Fprint(w, `[{"id":1,"url":"https://gitlab.example.com/hook"}]`)
	})

	hooks, _, err := client.SystemHooks.ListHooks()
	require.NoError(t, err)

	want := []*Hook{{ID: 1, URL: "https://gitlab.example.com/hook"}}
	require.Equal(t, want, hooks)
}

func TestSystemHooksService_AddHook(t *testing.T) {
	mux, server, client := setup(t)
	defer teardown(server)

	mux.HandleFunc("/api/v4/hooks", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, http.MethodPost)
		fmt.Fprint(w, `{"id": 1, "url": "https://gitlab.example.com/hook"}`)
	})

	opt := &AddHookOptions{
		URL: String("https://gitlab.example.com/hook"),
	}

	h, _, err := client.SystemHooks.AddHook(opt)
	require.NoError(t, err)

	want := &Hook{ID: 1, URL: "https://gitlab.example.com/hook", CreatedAt: (*time.Time)(nil)}
	require.Equal(t, want, h)
}

func TestSystemHooksService_TestHook(t *testing.T) {
	mux, server, client := setup(t)
	defer teardown(server)

	mux.HandleFunc("/api/v4/hooks/1", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, http.MethodGet)
		fmt.Fprint(w, `{"project_id" : 1,"owner_email" : "example@gitlabhq.com","owner_name" : "Someone",
				"name" : "Ruby","path" : "ruby","event_name" : "project_create"}`)
	})

	he, _, err := client.SystemHooks.TestHook(1)
	require.NoError(t, err)

	want := &HookEvent{
		EventName:  "project_create",
		Name:       "Ruby",
		Path:       "ruby",
		ProjectID:  1,
		OwnerName:  "Someone",
		OwnerEmail: "example@gitlabhq.com",
	}
	require.Equal(t, want, he)
}

func TestSystemHooksService_DeleteHook(t *testing.T) {
	mux, server, client := setup(t)
	defer teardown(server)

	mux.HandleFunc("/api/v4/hooks/1", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, http.MethodDelete)
	})

	_, err := client.SystemHooks.DeleteHook(1)
	require.NoError(t, err)
}

func TestHook_String(t *testing.T) {
	h := &Hook{
		ID:        1,
		URL:       "https://github.com/",
		CreatedAt: nil,
	}
	hs := h.String()
	want := "gitlab.Hook{ID:1, URL:\"https://github.com/\"}"
	require.Equal(t,want , hs)
}

func TestHookEvent_String(t *testing.T) {
	he := &HookEvent{
		EventName:  "event",
		Name:       "name",
		Path:       "path",
		ProjectID:  1,
		OwnerName:  "owner",
		OwnerEmail: "owner_email",
	}

	hes := he.String()
	want := "gitlab.HookEvent{EventName:\"event\", Name:\"name\", Path:\"path\", ProjectID:1, OwnerName:\"owner\", OwnerEmail:\"owner_email\"}"
	require.Equal(t, want, hes)
}
<|endoftext|>


<commit_before>package xstrings

//Prepend pre-pends the given item to the string.
func Prepend(item string, s string) string {
	return item + s
}
<commit_msg>Fix documentation for prepend<commit_after>package xstrings

// Prepend returns a new string
// with item at the head of the string.
func Prepend(item string, s string) string {
	return item + s
}
<|endoftext|>


<commit_before>package xmmsclient

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"net"
	"sync/atomic"
)

type context struct {
	result     chan XmmsValue
	sequenceNr uint32
}

type message struct {
	objectId  uint32
	commandId uint32
	args      XmmsValue
	context   context
}

type reply struct {
	sequenceNr uint32
	value      XmmsValue
}

type Client struct {
	conn       io.ReadWriteCloser
	sequenceNr uint32
	clientId   XmmsValue // TODO: uint32

	inbound  chan reply
	outbound chan message
	registry chan context
}

func parseHeader(header *bytes.Buffer) (objId uint32, cmdId uint32, sequenceNr uint32, length uint32, err error) {
	err = binary.Read(header, binary.BigEndian, &objId)
	if err != nil {
		return
	}

	err = binary.Read(header, binary.BigEndian, &cmdId)
	if err != nil {
		return
	}

	err = binary.Read(header, binary.BigEndian, &sequenceNr)
	if err != nil {
		return
	}

	err = binary.Read(header, binary.BigEndian, &length)
	if err != nil {
		return
	}
	return
}

func writeHeader(w io.ReadWriteCloser, msg message, length uint32) (err error) {
	err = binary.Write(w, binary.BigEndian, msg.objectId)
	if err != nil {
		return
	}

	err = binary.Write(w, binary.BigEndian, msg.commandId)
	if err != nil {
		return
	}

	err = binary.Write(w, binary.BigEndian, msg.context.sequenceNr)
	if err != nil {
		return
	}

	err = binary.Write(w, binary.BigEndian, length)
	if err != nil {
		return
	}

	return
}

func (c *Client) nextSequenceNr() (sequenceNr uint32) {
	sequenceNr = atomic.AddUint32(&c.sequenceNr, 1)
	return
}

func (c *Client) reader() {
	var header = make([]byte, 16)
	for {
		read, err := io.ReadFull(c.conn, header)
		if err != nil {
			// TODO: Post to error channel
			fmt.Println("error reading socket")
			continue
		}

		if read != len(header) {
			fmt.Println("nothing to read")
			continue
		}

		_, _, sequenceNr, length, err := parseHeader(bytes.NewBuffer(header))

		payload := make([]byte, length)
		c.conn.Read(payload)
		value, _ := DeserializeXmmsValue(bytes.NewBuffer(payload))
		c.inbound <- reply{sequenceNr, value}
	}
}

func (c *Client) writer() {
	for {
		select {
		case msg := <-c.outbound:
			var payload bytes.Buffer

			c.registry <- msg.context

			err := SerializeXmmsValue(msg.args, &payload)
			if err != nil {
				continue
			}

			err = writeHeader(c.conn, msg, uint32(len(payload.Bytes())))
			if err != nil {
				continue
			}

			payload.WriteTo(c.conn)
			if err != nil {
				continue
			}
		}
	}
}

func (c *Client) router() {
	var registry = make(map[uint32](context))

	for {
		select {
		case ctx := <-c.registry:
			registry[ctx.sequenceNr] = ctx
		case reply := <-c.inbound:
			registry[reply.sequenceNr].result <- reply.value
		}
	}
}

// TODO: Needs a better home, should be generated.
func (c *Client) MainHello(client_name string) XmmsValue {
	context := context{
		result:     make(chan XmmsValue),
		sequenceNr: c.nextSequenceNr()}
	c.outbound <- message{
		objectId:  ObjectMain,
		commandId: CommandMainHello,
		context:   context,
		args:      NewXmmsList(XmmsInt(IpcVersion), XmmsString(client_name))}
	return <-context.result
}

// TODO: Needs a better home, should be generated.
func (c *Client) MainListPlugins() XmmsValue {
	context := context{
		result:     make(chan XmmsValue),
		sequenceNr: c.nextSequenceNr()}
	c.outbound <- message{
		args:      NewXmmsList(XmmsInt(0)),
		objectId:  ObjectMain,
		commandId: CommandMainListPlugins,
		context:   context}
	return <-context.result
}

// TODO: Probably something else that creates a new Client rather than Dial.
func Dial(url string, name string) (client Client, err error) {
	addr, err := net.ResolveTCPAddr("tcp", url)
	if err != nil {
		return
	}

	conn, err := net.DialTCP("tcp", nil, addr)
	if err != nil {
		return
	}

	client = Client{
		conn:     conn,
		inbound:  make(chan reply),
		outbound: make(chan message),
		registry: make(chan context),
	}

	go client.router()
	go client.reader()
	go client.writer()

	client.clientId = client.MainHello(name)

	return
}
<commit_msg>Don't leak contexts.<commit_after>package xmmsclient

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"net"
	"sync/atomic"
)

type context struct {
	result     chan XmmsValue
	sequenceNr uint32
}

type message struct {
	objectId  uint32
	commandId uint32
	args      XmmsValue
	context   context
}

type reply struct {
	sequenceNr uint32
	value      XmmsValue
}

type Client struct {
	conn       io.ReadWriteCloser
	sequenceNr uint32
	clientId   XmmsValue // TODO: uint32

	inbound  chan reply
	outbound chan message
	registry chan context
}

func parseHeader(header *bytes.Buffer) (objId uint32, cmdId uint32, sequenceNr uint32, length uint32, err error) {
	err = binary.Read(header, binary.BigEndian, &objId)
	if err != nil {
		return
	}

	err = binary.Read(header, binary.BigEndian, &cmdId)
	if err != nil {
		return
	}

	err = binary.Read(header, binary.BigEndian, &sequenceNr)
	if err != nil {
		return
	}

	err = binary.Read(header, binary.BigEndian, &length)
	if err != nil {
		return
	}
	return
}

func writeHeader(w io.ReadWriteCloser, msg message, length uint32) (err error) {
	err = binary.Write(w, binary.BigEndian, msg.objectId)
	if err != nil {
		return
	}

	err = binary.Write(w, binary.BigEndian, msg.commandId)
	if err != nil {
		return
	}

	err = binary.Write(w, binary.BigEndian, msg.context.sequenceNr)
	if err != nil {
		return
	}

	err = binary.Write(w, binary.BigEndian, length)
	if err != nil {
		return
	}

	return
}

func (c *Client) nextSequenceNr() (sequenceNr uint32) {
	sequenceNr = atomic.AddUint32(&c.sequenceNr, 1)
	return
}

func (c *Client) reader() {
	var header = make([]byte, 16)
	for {
		read, err := io.ReadFull(c.conn, header)
		if err != nil {
			// TODO: Post to error channel
			fmt.Println("error reading socket")
			continue
		}

		if read != len(header) {
			fmt.Println("nothing to read")
			continue
		}

		_, _, sequenceNr, length, err := parseHeader(bytes.NewBuffer(header))

		payload := make([]byte, length)
		c.conn.Read(payload)
		value, _ := DeserializeXmmsValue(bytes.NewBuffer(payload))
		c.inbound <- reply{sequenceNr, value}
	}
}

func (c *Client) writer() {
	for {
		select {
		case msg := <-c.outbound:
			var payload bytes.Buffer

			c.registry <- msg.context

			err := SerializeXmmsValue(msg.args, &payload)
			if err != nil {
				continue
			}

			err = writeHeader(c.conn, msg, uint32(len(payload.Bytes())))
			if err != nil {
				continue
			}

			payload.WriteTo(c.conn)
			if err != nil {
				continue
			}
		}
	}
}

func (c *Client) router() {
	var registry = make(map[uint32](context))

	for {
		select {
		case ctx := <-c.registry:
			registry[ctx.sequenceNr] = ctx
		case reply := <-c.inbound:
			registry[reply.sequenceNr].result <- reply.value
			delete(registry, reply.sequenceNr)
		}
	}
}

// TODO: Needs a better home, should be generated.
func (c *Client) MainHello(client_name string) XmmsValue {
	context := context{
		result:     make(chan XmmsValue),
		sequenceNr: c.nextSequenceNr()}
	c.outbound <- message{
		objectId:  ObjectMain,
		commandId: CommandMainHello,
		context:   context,
		args:      NewXmmsList(XmmsInt(IpcVersion), XmmsString(client_name))}
	return <-context.result
}

// TODO: Needs a better home, should be generated.
func (c *Client) MainListPlugins() XmmsValue {
	context := context{
		result:     make(chan XmmsValue),
		sequenceNr: c.nextSequenceNr()}
	c.outbound <- message{
		args:      NewXmmsList(XmmsInt(0)),
		objectId:  ObjectMain,
		commandId: CommandMainListPlugins,
		context:   context}
	return <-context.result
}

// TODO: Probably something else that creates a new Client rather than Dial.
func Dial(url string, name string) (client Client, err error) {
	addr, err := net.ResolveTCPAddr("tcp", url)
	if err != nil {
		return
	}

	conn, err := net.DialTCP("tcp", nil, addr)
	if err != nil {
		return
	}

	client = Client{
		conn:     conn,
		inbound:  make(chan reply),
		outbound: make(chan message),
		registry: make(chan context),
	}

	go client.router()
	go client.reader()
	go client.writer()

	client.clientId = client.MainHello(name)

	return
}
<|endoftext|>


<commit_before>package main

import (
	"os/exec"
	"os"
	"fmt"
	"strings"
	"path/filepath"
	"runtime"
	"io/ioutil"
	"archive/zip"
	"bytes"
)

func main(){
	gopath:=os.Getenv("GOPATH")
	mustDeleteFile(filepath.Join(gopath,"tmp","file"))
	mustDeleteFile(filepath.Join(gopath,"tmp","fileZip"))
	mustRunCmd([]string{"go","test","-v","github.com/bronze1man/yaml2json/y2jLib"},nil)
	type buildInfo struct{
		goos string
		goarch string
	}
	buildInfoList := []buildInfo{
		{"darwin","386"},
		{"darwin","amd64"},
		{"linux","386"},
		{"linux","amd64"},
		{"linux","arm"},
		{"windows","386"},
		{"windows","amd64"},
		{"freebsd","386"},
		{"freebsd","amd64"},
		{"freebsd","arm"},
		{"netbsd","386"},
		{"netbsd","amd64"},
		{"netbsd","arm"},
		{"openbsd","386"},
		{"openbsd","amd64"},
		{"plan9","386"},
	}
	for _,info:=range buildInfoList{
		mustRunCmd([]string{"go","install","-ldflags","-s -w","-gcflags=-trimpath="+gopath,"github.com/bronze1man/yaml2json"},map[string]string{
			"GOOS":info.goos,
			"GOARCH":info.goarch,
		})
		var outputPath string
		if info.goarch == runtime.GOARCH && info.goos == runtime.GOOS{
			outputPath = filepath.Join(gopath,"bin","yaml2json")
		}else{
			outputPath = filepath.Join(gopath,"bin",info.goos+"_"+info.goarch,"yaml2json")
		}
		if info.goos=="windows"{
			outputPath+=".exe"
		}
		filePath:="yaml2json_"+info.goos+"_"+info.goarch
		if info.goos=="windows"{
			filePath+=".exe"
		}
		mustCopyFile(outputPath,filepath.Join(gopath,"tmp","file",filePath))
		filePath2:="yaml2json"
		if info.goos=="windows"{
			filePath2+=".exe"
		}
		mustCopyFile(outputPath,filepath.Join(gopath,"tmp","fileZip",info.goos+"_"+info.goarch,filePath2))
	}
	mustZipDir(filepath.Join(gopath,"tmp","fileZip"),filepath.Join(gopath,"tmp","file","yaml2json_all.zip"))
}

func mustCopyFile(fromPath string,toPath string){
	content,err:=ioutil.ReadFile(fromPath)
	if err!=nil{
		panic(err)
	}
	err=os.MkdirAll(filepath.Dir(toPath),0777)
	if err!=nil{
		panic(err)
	}
	err=ioutil.WriteFile(toPath,content,0777)
	if err!=nil{
		panic(err)
	}
}

func mustDeleteFile(path string){
	err:=os.RemoveAll(path)
	if err!=nil{
		if os.IsNotExist(err){
			return
		}
		panic(err)
	}
}

func mustZipDir(dir string,targetPath string){
	dir,err := filepath.Abs(dir)
	if err!=nil{
		panic(err)
	}
	buf:=&bytes.Buffer{}
	zipObj:=zip.NewWriter(buf)
	err=filepath.Walk(dir,func(path string, info os.FileInfo, err error) error{
		if err!=nil{
			panic(err)
		}
		if info.IsDir(){
			return nil
		}
		relPath,err:=filepath.Rel(dir,path)
		if err!=nil{
			panic(err)
		}
		w,err:=zipObj.Create(relPath)
		if err!=nil{
			panic(err)
		}
		content,err:=ioutil.ReadFile(path)
		if err!=nil{
			panic(err)
		}
		_,err=w.Write(content)
		if err!=nil{
			panic(err)
		}
		return nil
	})
	if err!=nil{
		panic(err)
	}
	err = zipObj.Close()
	if err!=nil{
		panic(err)
	}
	err=ioutil.WriteFile(targetPath,buf.Bytes(),0777)
	if err!=nil{
		panic(err)
	}
}

func mustRunCmd(cmdList []string,env map[string]string) {
	fmt.Println(">",strings.Join(cmdList," "))
	cmd:=exec.Command(cmdList[0],cmdList[1:]...)
	cmd.Env = os.Environ()
	for k,v:=range env{
		cmd.Env = append(cmd.Env,k+"="+v)
	}
	cmd.Stderr = os.Stdout
	cmd.Stdout = os.Stdout
	err:=cmd.Run()
	if err!=nil{
		panic(err)
	}
}<commit_msg>added option for linux.ppc64le<commit_after>package main

import (
	"os/exec"
	"os"
	"fmt"
	"strings"
	"path/filepath"
	"runtime"
	"io/ioutil"
	"archive/zip"
	"bytes"
)

func main(){
	gopath:=os.Getenv("GOPATH")
	mustDeleteFile(filepath.Join(gopath,"tmp","file"))
	mustDeleteFile(filepath.Join(gopath,"tmp","fileZip"))
	mustRunCmd([]string{"go","test","-v","github.com/bronze1man/yaml2json/y2jLib"},nil)
	type buildInfo struct{
		goos string
		goarch string
	}
	buildInfoList := []buildInfo{
		{"darwin","386"},
		{"darwin","amd64"},
		{"linux","386"},
		{"linux","amd64"},
		{"linux","arm"},
		{"windows","386"},
		{"windows","amd64"},
		{"freebsd","386"},
		{"freebsd","amd64"},
		{"freebsd","arm"},
		{"netbsd","386"},
		{"netbsd","amd64"},
		{"netbsd","arm"},
		{"openbsd","386"},
		{"openbsd","amd64"},
		{"plan9","386"},
		{"linux","ppc64le"},
	}
	for _,info:=range buildInfoList{
		mustRunCmd([]string{"go","install","-ldflags","-s -w","-gcflags=-trimpath="+gopath,"github.com/bronze1man/yaml2json"},map[string]string{
			"GOOS":info.goos,
			"GOARCH":info.goarch,
		})
		var outputPath string
		if info.goarch == runtime.GOARCH && info.goos == runtime.GOOS{
			outputPath = filepath.Join(gopath,"bin","yaml2json")
		}else{
			outputPath = filepath.Join(gopath,"bin",info.goos+"_"+info.goarch,"yaml2json")
		}
		if info.goos=="windows"{
			outputPath+=".exe"
		}
		filePath:="yaml2json_"+info.goos+"_"+info.goarch
		if info.goos=="windows"{
			filePath+=".exe"
		}
		mustCopyFile(outputPath,filepath.Join(gopath,"tmp","file",filePath))
		filePath2:="yaml2json"
		if info.goos=="windows"{
			filePath2+=".exe"
		}
		mustCopyFile(outputPath,filepath.Join(gopath,"tmp","fileZip",info.goos+"_"+info.goarch,filePath2))
	}
	mustZipDir(filepath.Join(gopath,"tmp","fileZip"),filepath.Join(gopath,"tmp","file","yaml2json_all.zip"))
}

func mustCopyFile(fromPath string,toPath string){
	content,err:=ioutil.ReadFile(fromPath)
	if err!=nil{
		panic(err)
	}
	err=os.MkdirAll(filepath.Dir(toPath),0777)
	if err!=nil{
		panic(err)
	}
	err=ioutil.WriteFile(toPath,content,0777)
	if err!=nil{
		panic(err)
	}
}

func mustDeleteFile(path string){
	err:=os.RemoveAll(path)
	if err!=nil{
		if os.IsNotExist(err){
			return
		}
		panic(err)
	}
}

func mustZipDir(dir string,targetPath string){
	dir,err := filepath.Abs(dir)
	if err!=nil{
		panic(err)
	}
	buf:=&bytes.Buffer{}
	zipObj:=zip.NewWriter(buf)
	err=filepath.Walk(dir,func(path string, info os.FileInfo, err error) error{
		if err!=nil{
			panic(err)
		}
		if info.IsDir(){
			return nil
		}
		relPath,err:=filepath.Rel(dir,path)
		if err!=nil{
			panic(err)
		}
		w,err:=zipObj.Create(relPath)
		if err!=nil{
			panic(err)
		}
		content,err:=ioutil.ReadFile(path)
		if err!=nil{
			panic(err)
		}
		_,err=w.Write(content)
		if err!=nil{
			panic(err)
		}
		return nil
	})
	if err!=nil{
		panic(err)
	}
	err = zipObj.Close()
	if err!=nil{
		panic(err)
	}
	err=ioutil.WriteFile(targetPath,buf.Bytes(),0777)
	if err!=nil{
		panic(err)
	}
}

func mustRunCmd(cmdList []string,env map[string]string) {
	fmt.Println(">",strings.Join(cmdList," "))
	cmd:=exec.Command(cmdList[0],cmdList[1:]...)
	cmd.Env = os.Environ()
	for k,v:=range env{
		cmd.Env = append(cmd.Env,k+"="+v)
	}
	cmd.Stderr = os.Stdout
	cmd.Stdout = os.Stdout
	err:=cmd.Run()
	if err!=nil{
		panic(err)
	}
}<|endoftext|>