<commit_before>/*
* This file is part of the KubeVirt project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright 2022 Red Hat, Inc.
*
 */

package admitters

import (
	"encoding/json"
	"fmt"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"

	"github.com/golang/mock/gomock"
	admissionv1 "k8s.io/api/admission/v1"
	k8sv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/rand"
	"k8s.io/client-go/tools/cache"
	"k8s.io/utils/pointer"

	"kubevirt.io/api/clone"
	clonev1lpha1 "kubevirt.io/api/clone/v1alpha1"
	"kubevirt.io/api/core"
	v1 "kubevirt.io/api/core/v1"
	"kubevirt.io/client-go/kubecli"

	"kubevirt.io/kubevirt/pkg/testutils"
	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
	"kubevirt.io/kubevirt/tests/util"
)

var _ = Describe("Validating VirtualMachineClone Admitter", func() {
	var ctrl *gomock.Controller
	var virtClient *kubecli.MockKubevirtClient
	var admitter *VirtualMachineCloneAdmitter
	var vmClone *clonev1lpha1.VirtualMachineClone
	var config *virtconfig.ClusterConfig
	var kvInformer cache.SharedIndexInformer
	var vmInterface *kubecli.MockVirtualMachineInterface
	var vm *v1.VirtualMachine

	enableFeatureGate := func(featureGate string) {
		testutils.UpdateFakeKubeVirtClusterConfig(kvInformer, &v1.KubeVirt{
			Spec: v1.KubeVirtSpec{
				Configuration: v1.KubeVirtConfiguration{
					DeveloperConfiguration: &v1.DeveloperConfiguration{
						FeatureGates: []string{featureGate},
					},
				},
			},
		})
	}

	disableFeatureGates := func() {
		testutils.UpdateFakeKubeVirtClusterConfig(kvInformer, &v1.KubeVirt{
			Spec: v1.KubeVirtSpec{
				Configuration: v1.KubeVirtConfiguration{
					DeveloperConfiguration: &v1.DeveloperConfiguration{
						FeatureGates: make([]string, 0),
					},
				},
			},
		})
	}

	newValidVM := func(namespace, name string) *v1.VirtualMachine {
		return &v1.VirtualMachine{
			ObjectMeta: metav1.ObjectMeta{
				Namespace: namespace,
				Name:      name,
			},
			Spec: v1.VirtualMachineSpec{
				Template: &v1.VirtualMachineInstanceTemplateSpec{
					Spec: v1.VirtualMachineInstanceSpec{
						Volumes: []v1.Volume{
							{
								Name: "dvVol",
								VolumeSource: v1.VolumeSource{
									DataVolume: &v1.DataVolumeSource{},
								},
							},
							{
								Name: "pvcVol",
								VolumeSource: v1.VolumeSource{
									PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{},
								},
							},
						},
					},
				},
			},
			Status: v1.VirtualMachineStatus{
				VolumeSnapshotStatuses: []v1.VolumeSnapshotStatus{
					{
						Name:    "dvVol",
						Enabled: true,
					},
					{
						Name:    "pvcVol",
						Enabled: true,
					},
				},
			},
		}
	}

	BeforeEach(func() {
		ctrl = gomock.NewController(GinkgoT())
		virtClient = kubecli.NewMockKubevirtClient(ctrl)
		config, _, kvInformer = testutils.NewFakeClusterConfigUsingKVConfig(&v1.KubeVirtConfiguration{})
		vmInterface = kubecli.NewMockVirtualMachineInterface(ctrl)
		virtClient.EXPECT().VirtualMachine(gomock.Any()).Return(vmInterface).AnyTimes()

		admitter = &VirtualMachineCloneAdmitter{Config: config, Client: virtClient}
		vmClone = newValidClone()
		vm = newValidVM(vmClone.Namespace, vmClone.Spec.Source.Name)
		vmInterface.EXPECT().Get(vmClone.Spec.Source.Name, gomock.Any()).Return(vm, nil).AnyTimes()
		vmInterface.EXPECT().Get(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("does-not-exist")).AnyTimes()
		enableFeatureGate("Snapshot")
	})

	AfterEach(func() {
		disableFeatureGates()
	})

	It("should allow legal clone", func() {
		admitter.admitAndExpect(vmClone, true)
	})

	DescribeTable("should reject clone with source that lacks information", func(getSource func() *k8sv1.TypedLocalObjectReference) {
		vmClone.Spec.Source = getSource()
		admitter.admitAndExpect(vmClone, false)
	},
		Entry("Source without Name", func() *k8sv1.TypedLocalObjectReference {
			source := newValidObjReference()
			source.Name = ""
			return source
		}),
		Entry("Source without Kind", func() *k8sv1.TypedLocalObjectReference {
			source := newValidObjReference()
			source.Kind = ""
			return source
		}),
		Entry("Source with nil APIGroup", func() *k8sv1.TypedLocalObjectReference {
			source := newValidObjReference()
			source.APIGroup = nil
			return source
		}),
		Entry("Source with empty APIGroup", func() *k8sv1.TypedLocalObjectReference {
			source := newValidObjReference()
			source.APIGroup = pointer.String("")
			return source
		}),
		Entry("Source with bad kind", func() *k8sv1.TypedLocalObjectReference {
			source := newValidObjReference()
			source.Kind = "Foobar"
			return source
		}),
	)

	It("Should reject unknown source type", func() {
		vmClone.Spec.Source.Kind = rand.String(5)
		admitter.admitAndExpect(vmClone, false)
	})

	It("Should reject unknown target type", func() {
		vmClone.Spec.Target.Kind = rand.String(5)
		admitter.admitAndExpect(vmClone, false)
	})

	It("Should reject a source VM that does not exist", func() {
		vmClone.Spec.Source.Name = "vm-that-doesnt-exist"
		admitter.admitAndExpect(vmClone, false)
	})

	It("Should reject if snapshot feature gate is not enabled", func() {
		disableFeatureGates()
		admitter.admitAndExpect(vmClone, false)
	})

	DescribeTable("Should reject a source volume not Snapshot-able", func(index int) {
		vm.Status.VolumeSnapshotStatuses[index].Enabled = false
		admitter.admitAndExpect(vmClone, false)
	},
		Entry("DataVolume", 0),
		Entry("PersistentVolumeClaim", 1),
	)

	Context("Annotations and labels filters", func() {
		testFilter := func(filter string, expectAllowed bool) {
			vmClone.Spec.LabelFilters = []string{filter}
			vmClone.Spec.AnnotationFilters = []string{filter}
			admitter.admitAndExpect(vmClone, expectAllowed)
		}

		DescribeTable("Should reject", func(filter string) {
			testFilter(filter, false)

		},
			Entry("negation character alone", "!"),
			Entry("negation in the middle", "mykey/!something"),
			Entry("negation in the end", "mykey/something!"),
			Entry("wildcard in the beginning", "*mykey/something"),
			Entry("wildcard in the middle", "mykey/*something"),
		)

		DescribeTable("Should allow", func(filter string) {
			testFilter(filter, true)
		},
			Entry("regular filter", "mykey/something"),
			Entry("wildcard only", "*"),
			Entry("wildcard in the end", "mykey/something*"),
			Entry("negation in the beginning", "!mykey/something"),
		)
	})

})

func createCloneAdmissionReview(vmClone *clonev1lpha1.VirtualMachineClone) *admissionv1.AdmissionReview {
	policyBytes, _ := json.Marshal(vmClone)

	ar := &admissionv1.AdmissionReview{
		Request: &admissionv1.AdmissionRequest{
			Operation: admissionv1.Create,
			Resource: metav1.GroupVersionResource{
				Group:    clonev1lpha1.VirtualMachineCloneKind.Group,
				Resource: clone.ResourceVMClonePlural,
			},
			Object: runtime.RawExtension{
				Raw: policyBytes,
			},
		},
	}

	return ar
}

func (admitter *VirtualMachineCloneAdmitter) admitAndExpect(clone *clonev1lpha1.VirtualMachineClone, expectAllowed bool) {
	ar := createCloneAdmissionReview(clone)
	resp := admitter.Admit(ar)
	Expect(resp.Allowed).To(Equal(expectAllowed))
}

func newValidClone() *clonev1lpha1.VirtualMachineClone {
	vmClone := kubecli.NewMinimalCloneWithNS("testclone", util.NamespaceTestDefault)
	vmClone.Spec.Source = newValidObjReference()
	vmClone.Spec.Target = newValidObjReference()
	vmClone.Spec.Target.Name = "clone-target-vm"

	return vmClone
}

func newValidObjReference() *k8sv1.TypedLocalObjectReference {
	return &k8sv1.TypedLocalObjectReference{
		APIGroup: pointer.String(core.GroupName),
		Kind:     "VirtualMachine",
		Name:     "clone-source-vm",
	}
}
<commit_msg>Clone admitter unit test: ensure snapshot is allowed as clone source<commit_after>/*
* This file is part of the KubeVirt project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright 2022 Red Hat, Inc.
*
 */

package admitters

import (
	"encoding/json"
	"fmt"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
	"k8s.io/client-go/testing"
	"kubevirt.io/api/snapshot/v1alpha1"
	"kubevirt.io/client-go/generated/kubevirt/clientset/versioned/fake"

	"github.com/golang/mock/gomock"
	admissionv1 "k8s.io/api/admission/v1"
	k8sv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/rand"
	"k8s.io/client-go/tools/cache"
	"k8s.io/utils/pointer"

	"kubevirt.io/api/clone"
	clonev1lpha1 "kubevirt.io/api/clone/v1alpha1"
	"kubevirt.io/api/core"
	v1 "kubevirt.io/api/core/v1"
	"kubevirt.io/client-go/kubecli"

	"kubevirt.io/kubevirt/pkg/testutils"
	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
	"kubevirt.io/kubevirt/tests/util"
)

var _ = Describe("Validating VirtualMachineClone Admitter", func() {
	var ctrl *gomock.Controller
	var virtClient *kubecli.MockKubevirtClient
	var kubevirtClient *fake.Clientset
	var admitter *VirtualMachineCloneAdmitter
	var vmClone *clonev1lpha1.VirtualMachineClone
	var config *virtconfig.ClusterConfig
	var kvInformer cache.SharedIndexInformer
	var vmInterface *kubecli.MockVirtualMachineInterface
	var vm *v1.VirtualMachine

	enableFeatureGate := func(featureGate string) {
		testutils.UpdateFakeKubeVirtClusterConfig(kvInformer, &v1.KubeVirt{
			Spec: v1.KubeVirtSpec{
				Configuration: v1.KubeVirtConfiguration{
					DeveloperConfiguration: &v1.DeveloperConfiguration{
						FeatureGates: []string{featureGate},
					},
				},
			},
		})
	}

	disableFeatureGates := func() {
		testutils.UpdateFakeKubeVirtClusterConfig(kvInformer, &v1.KubeVirt{
			Spec: v1.KubeVirtSpec{
				Configuration: v1.KubeVirtConfiguration{
					DeveloperConfiguration: &v1.DeveloperConfiguration{
						FeatureGates: make([]string, 0),
					},
				},
			},
		})
	}

	newValidVM := func(namespace, name string) *v1.VirtualMachine {
		return &v1.VirtualMachine{
			ObjectMeta: metav1.ObjectMeta{
				Namespace: namespace,
				Name:      name,
			},
			Spec: v1.VirtualMachineSpec{
				Template: &v1.VirtualMachineInstanceTemplateSpec{
					Spec: v1.VirtualMachineInstanceSpec{
						Volumes: []v1.Volume{
							{
								Name: "dvVol",
								VolumeSource: v1.VolumeSource{
									DataVolume: &v1.DataVolumeSource{},
								},
							},
							{
								Name: "pvcVol",
								VolumeSource: v1.VolumeSource{
									PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{},
								},
							},
							{
								Name: "containerDiskVol",
								VolumeSource: v1.VolumeSource{
									ContainerDisk: &v1.ContainerDiskSource{},
								},
							},
						},
					},
				},
			},
			Status: v1.VirtualMachineStatus{
				VolumeSnapshotStatuses: []v1.VolumeSnapshotStatus{
					{
						Name:    "dvVol",
						Enabled: true,
					},
					{
						Name:    "pvcVol",
						Enabled: true,
					},
					{
						Name:    "containerDiskVol",
						Enabled: false,
					},
				},
			},
		}
	}

	BeforeEach(func() {
		ctrl = gomock.NewController(GinkgoT())
		virtClient = kubecli.NewMockKubevirtClient(ctrl)
		config, _, kvInformer = testutils.NewFakeClusterConfigUsingKVConfig(&v1.KubeVirtConfiguration{})
		vmInterface = kubecli.NewMockVirtualMachineInterface(ctrl)
		kubevirtClient = fake.NewSimpleClientset()
		virtClient.
			EXPECT().
			VirtualMachine(util.NamespaceTestDefault).
			Return(vmInterface).
			AnyTimes()
		virtClient.
			EXPECT().
			VirtualMachineSnapshot(util.NamespaceTestDefault).
			Return(kubevirtClient.SnapshotV1alpha1().VirtualMachineSnapshots(util.NamespaceTestDefault)).
			AnyTimes()
		virtClient.
			EXPECT().
			VirtualMachineSnapshotContent(util.NamespaceTestDefault).
			Return(kubevirtClient.SnapshotV1alpha1().VirtualMachineSnapshotContents(util.NamespaceTestDefault)).
			AnyTimes()

		admitter = &VirtualMachineCloneAdmitter{Config: config, Client: virtClient}
		vmClone = newValidClone()
		vm = newValidVM(vmClone.Namespace, vmClone.Spec.Source.Name)
		vmInterface.EXPECT().Get(vmClone.Spec.Source.Name, gomock.Any()).Return(vm, nil).AnyTimes()
		vmInterface.EXPECT().Get(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("does-not-exist")).AnyTimes()

		kubevirtClient.Fake.PrependReactor("*", "*", func(action testing.Action) (handled bool, obj runtime.Object, err error) {
			Expect(action).To(BeNil())
			return true, nil, nil
		})
		kubevirtClient.Fake.PrependReactor("get", "virtualmachinesnapshots", func(action testing.Action) (handled bool, obj runtime.Object, err error) {
			snapshot := &v1alpha1.VirtualMachineSnapshot{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-snapshot",
					Namespace: util.NamespaceTestDefault,
				},
				Status: &v1alpha1.VirtualMachineSnapshotStatus{
					VirtualMachineSnapshotContentName: pointer.String("snapshot-contents"),
				},
			}
			return true, snapshot, nil
		})
		kubevirtClient.Fake.PrependReactor("get", "virtualmachinesnapshotcontents", func(action testing.Action) (handled bool, obj runtime.Object, err error) {
			var volumeBackups []v1alpha1.VolumeBackup
			for _, volume := range vm.Spec.Template.Spec.Volumes {
				volumeBackups = append(volumeBackups, v1alpha1.VolumeBackup{
					VolumeName: volume.Name,
				})
			}

			contents := &v1alpha1.VirtualMachineSnapshotContent{
				Spec: v1alpha1.VirtualMachineSnapshotContentSpec{
					VirtualMachineSnapshotName: pointer.String("test-vm"),
					Source: v1alpha1.SourceSpec{
						VirtualMachine: &v1alpha1.VirtualMachine{
							Spec: vm.Spec,
						},
					},
					VolumeBackups: volumeBackups,
				},
			}
			return true, contents, nil
		})

		enableFeatureGate("Snapshot")
	})

	AfterEach(func() {
		disableFeatureGates()
	})

	It("should allow legal clone", func() {
		admitter.admitAndExpect(vmClone, true)
	})

	DescribeTable("should reject clone with source that lacks information", func(getSource func() *k8sv1.TypedLocalObjectReference) {
		vmClone.Spec.Source = getSource()
		admitter.admitAndExpect(vmClone, false)
	},
		Entry("Source without Name", func() *k8sv1.TypedLocalObjectReference {
			source := newValidObjReference()
			source.Name = ""
			return source
		}),
		Entry("Source without Kind", func() *k8sv1.TypedLocalObjectReference {
			source := newValidObjReference()
			source.Kind = ""
			return source
		}),
		Entry("Source with nil APIGroup", func() *k8sv1.TypedLocalObjectReference {
			source := newValidObjReference()
			source.APIGroup = nil
			return source
		}),
		Entry("Source with empty APIGroup", func() *k8sv1.TypedLocalObjectReference {
			source := newValidObjReference()
			source.APIGroup = pointer.String("")
			return source
		}),
		Entry("Source with bad kind", func() *k8sv1.TypedLocalObjectReference {
			source := newValidObjReference()
			source.Kind = "Foobar"
			return source
		}),
	)

	Context("source types", func() {

		DescribeTable("should allow legal types", func(kind string) {
			vmClone.Spec.Source.Kind = kind
			admitter.admitAndExpect(vmClone, true)
		},
			Entry("VM", "VirtualMachine"),
			Entry("Snapshot", "VirtualMachineSnapshot"),
		)

		It("Should reject unknown source type", func() {
			vmClone.Spec.Source.Kind = rand.String(5)
			admitter.admitAndExpect(vmClone, false)
		})
	})

	It("Should reject unknown target type", func() {
		vmClone.Spec.Target.Kind = rand.String(5)
		admitter.admitAndExpect(vmClone, false)
	})

	It("Should reject a source VM that does not exist", func() {
		vmClone.Spec.Source.Name = "vm-that-doesnt-exist"
		admitter.admitAndExpect(vmClone, false)
	})

	It("Should reject if snapshot feature gate is not enabled", func() {
		disableFeatureGates()
		admitter.admitAndExpect(vmClone, false)
	})

	DescribeTable("Should reject a source volume not Snapshot-able", func(index int) {
		vm.Status.VolumeSnapshotStatuses[index].Enabled = false
		admitter.admitAndExpect(vmClone, false)
	},
		Entry("DataVolume", 0),
		Entry("PersistentVolumeClaim", 1),
	)

	Context("volume snapshots", func() {
		It("should allow non-PVC/DV volumes that have disabled volume snapshot status", func() {
			volumeName := "ephemeral-volume"
			vm.Spec.Template.Spec.Volumes = []v1.Volume{
				{
					Name:         volumeName,
					VolumeSource: v1.VolumeSource{ContainerDisk: &v1.ContainerDiskSource{}},
				},
			}
			vm.Status.VolumeSnapshotStatuses = []v1.VolumeSnapshotStatus{
				{
					Name:    volumeName,
					Enabled: false,
				},
			}

			admitter.admitAndExpect(vmClone, true)
		})

		It("should reject PVC/DV volumes with disabled volume snapshot status", func() {
			for i := range vm.Status.VolumeSnapshotStatuses {
				vm.Status.VolumeSnapshotStatuses[i].Enabled = false
			}
			admitter.admitAndExpect(vmClone, false)
		})

		It("should reject if vmsnapshot contents don't include a volume's backup", func() {
			vmClone.Spec.Source.Kind = "VirtualMachineSnapshot"

			kubevirtClient.Fake.PrependReactor("get", "virtualmachinesnapshotcontents", func(action testing.Action) (handled bool, obj runtime.Object, err error) {
				contents := &v1alpha1.VirtualMachineSnapshotContent{
					Spec: v1alpha1.VirtualMachineSnapshotContentSpec{
						VirtualMachineSnapshotName: pointer.String("test-vm"),
						Source: v1alpha1.SourceSpec{
							VirtualMachine: &v1alpha1.VirtualMachine{
								Spec: vm.Spec,
							},
						},
						VolumeBackups: nil,
					},
				}
				return true, contents, nil
			})

			admitter.admitAndExpect(vmClone, false)
		})
	})

	Context("Annotations and labels filters", func() {
		testFilter := func(filter string, expectAllowed bool) {
			vmClone.Spec.LabelFilters = []string{filter}
			vmClone.Spec.AnnotationFilters = []string{filter}
			admitter.admitAndExpect(vmClone, expectAllowed)
		}

		DescribeTable("Should reject", func(filter string) {
			testFilter(filter, false)

		},
			Entry("negation character alone", "!"),
			Entry("negation in the middle", "mykey/!something"),
			Entry("negation in the end", "mykey/something!"),
			Entry("wildcard in the beginning", "*mykey/something"),
			Entry("wildcard in the middle", "mykey/*something"),
		)

		DescribeTable("Should allow", func(filter string) {
			testFilter(filter, true)
		},
			Entry("regular filter", "mykey/something"),
			Entry("wildcard only", "*"),
			Entry("wildcard in the end", "mykey/something*"),
			Entry("negation in the beginning", "!mykey/something"),
		)
	})

})

func createCloneAdmissionReview(vmClone *clonev1lpha1.VirtualMachineClone) *admissionv1.AdmissionReview {
	policyBytes, _ := json.Marshal(vmClone)

	ar := &admissionv1.AdmissionReview{
		Request: &admissionv1.AdmissionRequest{
			Operation: admissionv1.Create,
			Resource: metav1.GroupVersionResource{
				Group:    clonev1lpha1.VirtualMachineCloneKind.Group,
				Resource: clone.ResourceVMClonePlural,
			},
			Object: runtime.RawExtension{
				Raw: policyBytes,
			},
		},
	}

	return ar
}

func (admitter *VirtualMachineCloneAdmitter) admitAndExpect(clone *clonev1lpha1.VirtualMachineClone, expectAllowed bool) {
	ar := createCloneAdmissionReview(clone)
	resp := admitter.Admit(ar)
	Expect(resp.Allowed).To(Equal(expectAllowed))
}

func newValidClone() *clonev1lpha1.VirtualMachineClone {
	vmClone := kubecli.NewMinimalCloneWithNS("testclone", util.NamespaceTestDefault)
	vmClone.Spec.Source = newValidObjReference()
	vmClone.Spec.Target = newValidObjReference()
	vmClone.Spec.Target.Name = "clone-target-vm"

	return vmClone
}

func newValidObjReference() *k8sv1.TypedLocalObjectReference {
	return &k8sv1.TypedLocalObjectReference{
		APIGroup: pointer.String(core.GroupName),
		Kind:     "VirtualMachine",
		Name:     "clone-source-vm",
	}
}
<|endoftext|>


<commit_before>package extra

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"

	"github.com/jmoiron/sqlx"

	"github.com/elpinal/coco3/extra/ast"
	"github.com/elpinal/coco3/extra/parser" // Only for ParseError.
	"github.com/elpinal/coco3/extra/typed"
	"github.com/elpinal/coco3/extra/types"
)

type Env struct {
	cmds map[string]typed.Command
	Option
}

type Option struct {
	DB *sqlx.DB
}

func New(opt Option) Env {
	return Env{
		Option: opt,
		cmds: map[string]typed.Command{
			"exec":    execCommand,
			"execenv": execenvCommand, // exec with env
			"cd":      cdCommand,
			"exit":    exitCommand,
			"free":    freeCommand,
			"history": historyCommand,

			"remove": removeCommand,

			"ls":   lsCommand,
			"man":  manCommand,
			"make": makeCommand,

			"git":   gitCommand,
			"cargo": cargoCommand,
			"go":    goCommand,
			"stack": stackCommand,
			"lein":  leinCommand,
			"ocaml": ocamlCommand,

			"vim":    vimCommand,
			"emacs":  emacsCommand,
			"screen": screenCommand,

			"cnp":  cnpCommand,
			"gvmn": gvmnCommand,
			"vvmn": vvmnCommand,
		},
	}
}

func WithoutDefault() Env {
	return Env{cmds: make(map[string]typed.Command)}
}

func (e *Env) Bind(name string, c typed.Command) {
	e.cmds[name] = c
}

func (e *Env) Eval(command *ast.Command) (err error) {
	if command == nil {
		return nil
	}
	tc, found := e.cmds[command.Name.Lit]
	if !found {
		return &parser.ParseError{
			Msg:    fmt.Sprintf("no such typed command: %q", command.Name.Lit),
			Line:   command.Name.Line,
			Column: command.Name.Column,
		}
	}
	if len(command.Args) != len(tc.Params) {
		return &parser.ParseError{
			Msg:    fmt.Sprintf("the length of args (%d) != the one of params (%d)", len(command.Args), len(tc.Params)),
			Line:   command.Name.Line,
			Column: command.Name.Column,
		}
	}
	for i, arg := range command.Args {
		if arg.Type() != tc.Params[i] {
			return &parser.ParseError{
				Msg:    fmt.Sprintf("type mismatch: (%v) (type of %v) does not match with (%v) (expected type)", arg.Type(), arg, tc.Params[i]),
				Line:   command.Name.Line,
				Column: command.Name.Column,
			}
		}
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	defer close(c)
	defer signal.Stop(c)

	defer func() {
		r := recover()
		if r == nil {
			return
		}
		var ok bool
		// may overwrite error of tc.Fn.
		err, ok = r.(error)
		if !ok {
			panic(r)
		}
	}()

	return tc.Fn(command.Args, e.DB)
}

func toSlice(list ast.List) ([]string, error) {
	ret := make([]string, 0, list.Length())
	for {
		switch x := list.(type) {
		case *ast.Cons:
			ret = append(ret, x.Head)
			list = x.Tail
		case *ast.Empty:
			return ret, nil
		default:
			return nil, fmt.Errorf("unexpected list type: %T", x)
		}
	}
}

var execCommand = typed.Command{
	Params: []types.Type{types.String, types.StringList},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, "exec")
		}
		cmd := stdCmd(args[0].(*ast.String).Lit, cmdArgs...)
		return cmd.Run()
	},
}

var execenvCommand = typed.Command{
	Params: []types.Type{types.StringList, types.String, types.StringList},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		cmdArgs, err := toSlice(args[2].(ast.List))
		if err != nil {
			return errors.Wrap(err, "execenv")
		}
		cmd := stdCmd(args[1].(*ast.String).Lit, cmdArgs...)
		env, err := toSlice(args[0].(ast.List))
		if err != nil {
			return errors.Wrap(err, "execenv")
		}
		for _, e := range env {
			if !strings.Contains(e, "=") {
				return errors.New(`execenv: each item of the first argument must be the form "key=value"`)
			}
		}
		cmd.Env = append(os.Environ(), env...)
		return cmd.Run()
	},
}

var cdCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		return os.Chdir(args[0].(*ast.String).Lit)
	},
}

var exitCommand = typed.Command{
	Params: []types.Type{types.Int},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		n, err := strconv.Atoi(args[0].(*ast.Int).Lit)
		if err != nil {
			return err
		}
		os.Exit(n)
		return nil
	},
}

var freeCommand = typed.Command{
	Params: []types.Type{types.String, types.StringList},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, "free")
		}
		name := args[0].(*ast.String).Lit
		cmd := exec.Cmd{Path: name, Args: append([]string{name}, cmdArgs...)}
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Stdin = os.Stdin
		return cmd.Run()
	},
}

func commandArgs(name string) func([]ast.Expr, *sqlx.DB) error {
	return func(args []ast.Expr, _ *sqlx.DB) error {
		list, err := toSlice(args[0].(ast.List))
		if err != nil {
			return errors.Wrap(err, name)
		}
		return stdCmd(name, list...).Run()
	}
}

func commandsInCommand(name string) func([]ast.Expr, *sqlx.DB) error {
	return func(args []ast.Expr, _ *sqlx.DB) error {
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, name)
		}
		var cmd *exec.Cmd
		switch lit := args[0].(*ast.Ident).Lit; lit {
		case "command":
			cmd = stdCmd(name, cmdArgs...)
		default:
			cmd = stdCmd(name, append([]string{lit}, cmdArgs...)...)
		}
		return cmd.Run()
	}
}

func goCommand1() func([]ast.Expr, *sqlx.DB) error {
	return func(args []ast.Expr, _ *sqlx.DB) error {
		name := "go"
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, name)
		}
		var cmd *exec.Cmd
		switch lit := args[0].(*ast.Ident).Lit; lit {
		case "command":
			cmd = stdCmd(name, cmdArgs...)
		case "testall":
			// I can't be confident in using such
			// a subcommand-specific way.  Another suggestion might
			// be like `go test all`, where 'all' is a postfix
			// operator of './...'.
			cmd = stdCmd(name, append([]string{"test"}, append(cmdArgs, "./...")...)...)
		default:
			cmd = stdCmd(name, append([]string{lit}, cmdArgs...)...)
		}
		return cmd.Run()
	}
}

var gitCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("git"),
}

var cargoCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("cargo"),
}

var goCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     goCommand1(),
}

var stackCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("stack"),
}

var leinCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("lein"),
}

func stdCmd(name string, args ...string) *exec.Cmd {
	cmd := exec.Command(name, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	return cmd
}

func stdExec(name string, args ...string) func([]ast.Expr, *sqlx.DB) error {
	return func(_ []ast.Expr, _ *sqlx.DB) error {
		return stdCmd(name, args...).Run()
	}
}

var vimCommand = typed.Command{
	Params: []types.Type{},
	Fn:     stdExec("vim"),
}

var emacsCommand = typed.Command{
	Params: []types.Type{},
	Fn:     stdExec("emacs"),
}

func withEnv(s string, cmd *exec.Cmd) *exec.Cmd {
	if cmd.Env == nil {
		cmd.Env = os.Environ()
	}
	cmd.Env = append(cmd.Env, s)
	return cmd
}

var screenCommand = typed.Command{
	Params: []types.Type{},
	Fn: func(_ []ast.Expr, _ *sqlx.DB) error {
		return withEnv("LANG=en_US.UTF-8", stdCmd("screen")).Run()
	},
}

type execution struct {
	Time time.Time
	Line string
}

var historyCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(e []ast.Expr, db *sqlx.DB) error {
		var jsonFormat bool
		var enc *json.Encoder
		switch format := e[0].(*ast.String).Lit; format {
		case "json":
			jsonFormat = true
		case "lines":
		default:
			return fmt.Errorf("history: format %q is not supported", format)
		}

		buf := bufio.NewWriter(os.Stdout)
		if jsonFormat {
			enc = json.NewEncoder(buf)
		}
		rows, err := db.Queryx("select * from command_info")
		if err != nil {
			return err
		}
		data := execution{}
		for rows.Next() {
			err := rows.StructScan(&data)
			if err != nil {
				return err
			}
			if jsonFormat {
				err := enc.Encode(data)
				if err != nil {
					return err
				}
			} else {
				buf.WriteString(data.Time.Format("Mon, 02 Jan 2006 15:04:05"))
				buf.Write([]byte("  "))
				buf.WriteString(data.Line)
				buf.WriteByte('\n')
			}
		}
		return buf.Flush()
	},
}

var lsCommand = typed.Command{
	Params: []types.Type{},
	Fn:     stdExec("ls", "--show-control-chars", "--color=auto"),
}

var manCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(e []ast.Expr, _ *sqlx.DB) error {
		lit := e[0].(*ast.String).Lit
		return stdCmd("man", lit).Run()
	},
}

var removeCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn:     remove,
}

func remove(exprs []ast.Expr, _ *sqlx.DB) error {
	s := exprs[0].(*ast.String).Lit
	fmt.Printf("remove %s?\n", s)
	for {
		fmt.Println("type y to continue")
		var ans string
		fmt.Scanf("%s", &ans)
		switch ans {
		case "i":
			fi, err := os.Stat(s)
			if err != nil {
				return err
			}
			fmt.Println("size", fi.Size())
			fmt.Println("is directory?:", fi.IsDir())
		case "y":
			return os.Remove(s)
		default:
			return nil
		}
	}
}

var cnpCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(e []ast.Expr, _ *sqlx.DB) error {
		lit := e[0].(*ast.String).Lit
		return stdCmd("create-new-project", lit).Run()
	},
}

var gvmnCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("gvmn"),
}

var vvmnCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("vvmn"),
}

var makeCommand = typed.Command{
	Params: []types.Type{types.StringList},
	Fn:     commandArgs("make"),
}

var ocamlCommand = typed.Command{
	Params: []types.Type{types.StringList},
	Fn:     commandArgs("ocaml"),
}
<commit_msg>Update the format of a message<commit_after>package extra

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"

	"github.com/jmoiron/sqlx"

	"github.com/elpinal/coco3/extra/ast"
	"github.com/elpinal/coco3/extra/parser" // Only for ParseError.
	"github.com/elpinal/coco3/extra/typed"
	"github.com/elpinal/coco3/extra/types"
)

type Env struct {
	cmds map[string]typed.Command
	Option
}

type Option struct {
	DB *sqlx.DB
}

func New(opt Option) Env {
	return Env{
		Option: opt,
		cmds: map[string]typed.Command{
			"exec":    execCommand,
			"execenv": execenvCommand, // exec with env
			"cd":      cdCommand,
			"exit":    exitCommand,
			"free":    freeCommand,
			"history": historyCommand,

			"remove": removeCommand,

			"ls":   lsCommand,
			"man":  manCommand,
			"make": makeCommand,

			"git":   gitCommand,
			"cargo": cargoCommand,
			"go":    goCommand,
			"stack": stackCommand,
			"lein":  leinCommand,
			"ocaml": ocamlCommand,

			"vim":    vimCommand,
			"emacs":  emacsCommand,
			"screen": screenCommand,

			"cnp":  cnpCommand,
			"gvmn": gvmnCommand,
			"vvmn": vvmnCommand,
		},
	}
}

func WithoutDefault() Env {
	return Env{cmds: make(map[string]typed.Command)}
}

func (e *Env) Bind(name string, c typed.Command) {
	e.cmds[name] = c
}

func (e *Env) Eval(command *ast.Command) (err error) {
	if command == nil {
		return nil
	}
	tc, found := e.cmds[command.Name.Lit]
	if !found {
		return &parser.ParseError{
			Msg:    fmt.Sprintf("no such typed command: %q", command.Name.Lit),
			Line:   command.Name.Line,
			Column: command.Name.Column,
		}
	}
	if len(command.Args) != len(tc.Params) {
		return &parser.ParseError{
			Msg:    fmt.Sprintf("the length of args (%d) != the one of params (%d)", len(command.Args), len(tc.Params)),
			Line:   command.Name.Line,
			Column: command.Name.Column,
		}
	}
	for i, arg := range command.Args {
		if arg.Type() != tc.Params[i] {
			return &parser.ParseError{
				Msg:    fmt.Sprintf("type mismatch: (%v) (type of %v) does not match with (%v) (expected type)", arg.Type(), arg, tc.Params[i]),
				Line:   command.Name.Line,
				Column: command.Name.Column,
			}
		}
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	defer close(c)
	defer signal.Stop(c)

	defer func() {
		r := recover()
		if r == nil {
			return
		}
		var ok bool
		// may overwrite error of tc.Fn.
		err, ok = r.(error)
		if !ok {
			panic(r)
		}
	}()

	return tc.Fn(command.Args, e.DB)
}

func toSlice(list ast.List) ([]string, error) {
	ret := make([]string, 0, list.Length())
	for {
		switch x := list.(type) {
		case *ast.Cons:
			ret = append(ret, x.Head)
			list = x.Tail
		case *ast.Empty:
			return ret, nil
		default:
			return nil, fmt.Errorf("unexpected list type: %T", x)
		}
	}
}

var execCommand = typed.Command{
	Params: []types.Type{types.String, types.StringList},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, "exec")
		}
		cmd := stdCmd(args[0].(*ast.String).Lit, cmdArgs...)
		return cmd.Run()
	},
}

var execenvCommand = typed.Command{
	Params: []types.Type{types.StringList, types.String, types.StringList},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		cmdArgs, err := toSlice(args[2].(ast.List))
		if err != nil {
			return errors.Wrap(err, "execenv")
		}
		cmd := stdCmd(args[1].(*ast.String).Lit, cmdArgs...)
		env, err := toSlice(args[0].(ast.List))
		if err != nil {
			return errors.Wrap(err, "execenv")
		}
		for _, e := range env {
			if !strings.Contains(e, "=") {
				return errors.New(`execenv: each item of the first argument must be the form "key=value"`)
			}
		}
		cmd.Env = append(os.Environ(), env...)
		return cmd.Run()
	},
}

var cdCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		return os.Chdir(args[0].(*ast.String).Lit)
	},
}

var exitCommand = typed.Command{
	Params: []types.Type{types.Int},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		n, err := strconv.Atoi(args[0].(*ast.Int).Lit)
		if err != nil {
			return err
		}
		os.Exit(n)
		return nil
	},
}

var freeCommand = typed.Command{
	Params: []types.Type{types.String, types.StringList},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, "free")
		}
		name := args[0].(*ast.String).Lit
		cmd := exec.Cmd{Path: name, Args: append([]string{name}, cmdArgs...)}
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Stdin = os.Stdin
		return cmd.Run()
	},
}

func commandArgs(name string) func([]ast.Expr, *sqlx.DB) error {
	return func(args []ast.Expr, _ *sqlx.DB) error {
		list, err := toSlice(args[0].(ast.List))
		if err != nil {
			return errors.Wrap(err, name)
		}
		return stdCmd(name, list...).Run()
	}
}

func commandsInCommand(name string) func([]ast.Expr, *sqlx.DB) error {
	return func(args []ast.Expr, _ *sqlx.DB) error {
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, name)
		}
		var cmd *exec.Cmd
		switch lit := args[0].(*ast.Ident).Lit; lit {
		case "command":
			cmd = stdCmd(name, cmdArgs...)
		default:
			cmd = stdCmd(name, append([]string{lit}, cmdArgs...)...)
		}
		return cmd.Run()
	}
}

func goCommand1() func([]ast.Expr, *sqlx.DB) error {
	return func(args []ast.Expr, _ *sqlx.DB) error {
		name := "go"
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, name)
		}
		var cmd *exec.Cmd
		switch lit := args[0].(*ast.Ident).Lit; lit {
		case "command":
			cmd = stdCmd(name, cmdArgs...)
		case "testall":
			// I can't be confident in using such
			// a subcommand-specific way.  Another suggestion might
			// be like `go test all`, where 'all' is a postfix
			// operator of './...'.
			cmd = stdCmd(name, append([]string{"test"}, append(cmdArgs, "./...")...)...)
		default:
			cmd = stdCmd(name, append([]string{lit}, cmdArgs...)...)
		}
		return cmd.Run()
	}
}

var gitCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("git"),
}

var cargoCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("cargo"),
}

var goCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     goCommand1(),
}

var stackCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("stack"),
}

var leinCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("lein"),
}

func stdCmd(name string, args ...string) *exec.Cmd {
	cmd := exec.Command(name, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	return cmd
}

func stdExec(name string, args ...string) func([]ast.Expr, *sqlx.DB) error {
	return func(_ []ast.Expr, _ *sqlx.DB) error {
		return stdCmd(name, args...).Run()
	}
}

var vimCommand = typed.Command{
	Params: []types.Type{},
	Fn:     stdExec("vim"),
}

var emacsCommand = typed.Command{
	Params: []types.Type{},
	Fn:     stdExec("emacs"),
}

func withEnv(s string, cmd *exec.Cmd) *exec.Cmd {
	if cmd.Env == nil {
		cmd.Env = os.Environ()
	}
	cmd.Env = append(cmd.Env, s)
	return cmd
}

var screenCommand = typed.Command{
	Params: []types.Type{},
	Fn: func(_ []ast.Expr, _ *sqlx.DB) error {
		return withEnv("LANG=en_US.UTF-8", stdCmd("screen")).Run()
	},
}

type execution struct {
	Time time.Time
	Line string
}

var historyCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(e []ast.Expr, db *sqlx.DB) error {
		var jsonFormat bool
		var enc *json.Encoder
		switch format := e[0].(*ast.String).Lit; format {
		case "json":
			jsonFormat = true
		case "lines":
		default:
			return fmt.Errorf("history: format %q is not supported", format)
		}

		buf := bufio.NewWriter(os.Stdout)
		if jsonFormat {
			enc = json.NewEncoder(buf)
		}
		rows, err := db.Queryx("select * from command_info")
		if err != nil {
			return err
		}
		data := execution{}
		for rows.Next() {
			err := rows.StructScan(&data)
			if err != nil {
				return err
			}
			if jsonFormat {
				err := enc.Encode(data)
				if err != nil {
					return err
				}
			} else {
				buf.WriteString(data.Time.Format("Mon, 02 Jan 2006 15:04:05"))
				buf.Write([]byte("  "))
				buf.WriteString(data.Line)
				buf.WriteByte('\n')
			}
		}
		return buf.Flush()
	},
}

var lsCommand = typed.Command{
	Params: []types.Type{},
	Fn:     stdExec("ls", "--show-control-chars", "--color=auto"),
}

var manCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(e []ast.Expr, _ *sqlx.DB) error {
		lit := e[0].(*ast.String).Lit
		return stdCmd("man", lit).Run()
	},
}

var removeCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn:     remove,
}

func remove(exprs []ast.Expr, _ *sqlx.DB) error {
	s := exprs[0].(*ast.String).Lit
	fmt.Printf("remove %s?\n", s)
	for {
		fmt.Println("type y to continue")
		var ans string
		fmt.Scanf("%s", &ans)
		switch ans {
		case "i":
			fi, err := os.Stat(s)
			if err != nil {
				return err
			}
			fmt.Println("size:", fi.Size())
			fmt.Println("is directory?:", fi.IsDir())
		case "y":
			return os.Remove(s)
		default:
			return nil
		}
	}
}

var cnpCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(e []ast.Expr, _ *sqlx.DB) error {
		lit := e[0].(*ast.String).Lit
		return stdCmd("create-new-project", lit).Run()
	},
}

var gvmnCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("gvmn"),
}

var vvmnCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("vvmn"),
}

var makeCommand = typed.Command{
	Params: []types.Type{types.StringList},
	Fn:     commandArgs("make"),
}

var ocamlCommand = typed.Command{
	Params: []types.Type{types.StringList},
	Fn:     commandArgs("ocaml"),
}
<|endoftext|>


<commit_before>package skl

import "bytes"

// NodeType identifies various AST nodes
type NodeType int

const (
	UseStatementNode NodeType = iota
)

// Node is an interface for AST nodes
type Node interface {
	NodeType() NodeType
	String() string
}

// ExprType identifies various expressions
type ExprType int

// Expr represents AST expressions
type Expr interface {
	Node
	ExprType() ExprType
}

// Statement is the interface for all SKL statements
type Statement interface {
	Node
	RequiredPermissions() []string
}

// UseStatement represents the USE statement
type UseStatement struct {
	Name string
}

// String returns a string representation
func (s *UseStatement) String() string {
	var buf bytes.Buffer
	buf.WriteString("USE ")
	buf.WriteString(s.Name)
	return buf.String()
}

// NodeType returns an NodeType id
func (s *UseStatement) NodeType() NodeType { return UseStatementNode }

// RequiredPermissions returns the required permissions in order to use this command
func (s *UseStatement) RequiredPermissions() []string { return []string{} }
<commit_msg>Remove pointer receivers from UseStatement<commit_after>package skl

import "bytes"

// NodeType identifies various AST nodes
type NodeType int

const (
	UseStatementType NodeType = iota
)

// Node is an interface for AST nodes
type Node interface {
	NodeType() NodeType
	String() string
}

// ExprType identifies various expressions
type ExprType int

// Expr represents AST expressions
type Expr interface {
	Node
	ExprType() ExprType
}

// Statement is the interface for all SKL statements
type Statement interface {
	Node
	RequiredPermissions() []string
}

// UseStatement represents the USE statement
type UseStatement struct {
	Name string
}

// String returns a string representation
func (s UseStatement) String() string {
	var buf bytes.Buffer
	buf.WriteString("USE ")
	buf.WriteString(s.Name)
	return buf.String()
}

// NodeType returns an NodeType id
func (s UseStatement) NodeType() NodeType { return UseStatementType }

// RequiredPermissions returns the required permissions in order to use this command
func (s UseStatement) RequiredPermissions() []string { return []string{} }
<|endoftext|>


<commit_before>// Copyright 2012 Junqing Tan <ivan@mysqlab.net> and The Go Authors
// Use of this source code is governed by a BSD-style
// Part of source code is from Go fcgi package

package fcgiclient

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"sync"
	"errors"
	"io"
	"net"
  "strings"
  "strconv"
  "net/http"
)

const FCGI_LISTENSOCK_FILENO uint8 = 0
const FCGI_HEADER_LEN uint8 = 8
const VERSION_1 uint8 = 1
const FCGI_NULL_REQUEST_ID uint8 = 0
const FCGI_KEEP_CONN uint8 = 1
const doubleCRLF = "\r\n\r\n"

const (
	FCGI_BEGIN_REQUEST uint8 = iota + 1
	FCGI_ABORT_REQUEST
	FCGI_END_REQUEST
	FCGI_PARAMS
	FCGI_STDIN
	FCGI_STDOUT
	FCGI_STDERR
	FCGI_DATA
	FCGI_GET_VALUES
	FCGI_GET_VALUES_RESULT
	FCGI_UNKNOWN_TYPE
	FCGI_MAXTYPE = FCGI_UNKNOWN_TYPE
)

const (
	FCGI_RESPONDER uint8 = iota + 1
	FCGI_AUTHORIZER
	FCGI_FILTER
)

const (
	FCGI_REQUEST_COMPLETE uint8 = iota
	FCGI_CANT_MPX_CONN
	FCGI_OVERLOADED
	FCGI_UNKNOWN_ROLE
)

const (
	FCGI_MAX_CONNS  string = "MAX_CONNS"
	FCGI_MAX_REQS   string = "MAX_REQS"
	FCGI_MPXS_CONNS string = "MPXS_CONNS"
)

const (
	maxWrite = 65500 // 65530 may work, but for compatibility
	maxPad   = 255
)

type header struct {
	Version       uint8
	Type          uint8
	Id            uint16
	ContentLength uint16
	PaddingLength uint8
	Reserved      uint8
}

// for padding so we don't have to allocate all the time
// not synchronized because we don't care what the contents are
var pad [maxPad]byte

func (h *header) init(recType uint8, reqId uint16, contentLength int) {
	h.Version = 1
	h.Type = recType
	h.Id = reqId
	h.ContentLength = uint16(contentLength)
	h.PaddingLength = uint8(-contentLength & 7)
}

type record struct {
	h   header
	rbuf []byte
}

func (rec *record) read(r io.Reader) (buf []byte, err error) {
	if err = binary.Read(r, binary.BigEndian, &rec.h); err != nil {
		return
	}
	if rec.h.Version != 1 {
    err = errors.New("fcgi: invalid header version")
		return
	}
  if rec.h.Type == FCGI_END_REQUEST {
    err = io.EOF
    return
  }
	n := int(rec.h.ContentLength) + int(rec.h.PaddingLength)
	if len(rec.rbuf) < n {
	  rec.rbuf = make([]byte, n)
	}
	if n, err = io.ReadFull(r, rec.rbuf[:n]); err != nil {
		return
	}
	buf = rec.rbuf[:int(rec.h.ContentLength)]

	return
}

type FCGIClient struct {
	mutex     sync.Mutex
	rwc       io.ReadWriteCloser
	h         header
	buf 	  bytes.Buffer
	keepAlive bool
}

func New(t string, a string) (fcgi *FCGIClient, err error) {
	var conn net.Conn

	conn, err = net.Dial(t, a)
  if err != nil {
    return
  }

	fcgi = &FCGIClient{
		rwc:       conn,
		keepAlive: false,
	}

	return
}

func (this *FCGIClient) Close() {
  this.rwc.Close()
}

func (this *FCGIClient) writeRecord(recType uint8, reqId uint16, content []byte) ( err error) {
	this.mutex.Lock()
	defer this.mutex.Unlock()
	this.buf.Reset()
	this.h.init(recType, reqId, len(content))
	if err := binary.Write(&this.buf, binary.BigEndian, this.h); err != nil {
		return err
	}
	if _, err := this.buf.Write(content); err != nil {
		return err
	}
	if _, err := this.buf.Write(pad[:this.h.PaddingLength]); err != nil {
		return err
	}
	_, err = this.rwc.Write(this.buf.Bytes())
	return err
}

func (this *FCGIClient) writeBeginRequest(reqId uint16, role uint16, flags uint8) error {
	b := [8]byte{byte(role >> 8), byte(role), flags}
	return this.writeRecord(FCGI_BEGIN_REQUEST, reqId, b[:])
}

func (this *FCGIClient) writeEndRequest(reqId uint16, appStatus int, protocolStatus uint8) error {
	b := make([]byte, 8)
	binary.BigEndian.PutUint32(b, uint32(appStatus))
	b[4] = protocolStatus
	return this.writeRecord(FCGI_END_REQUEST, reqId, b)
}

func (this *FCGIClient) writePairs(recType uint8, reqId uint16, pairs map[string]string) error {
	w  := newWriter(this, recType, reqId)
	b  := make([]byte, 8)
	nn := 0
	for k, v := range pairs {
    m := 8 + len(k) + len(v)
    if m > maxWrite {
      // param data size exceed 65535 bytes"
      vl := maxWrite - 8 - len(k)
      v = v[:vl]
    }
    n := encodeSize(b, uint32(len(k)))
    n += encodeSize(b[n:], uint32(len(v)))
    m = n + len(k) + len(v)
    if (nn + m) > maxWrite {
      w.Flush()
      nn = 0
    }
    nn += m
    if _, err := w.Write(b[:n]); err != nil {
      return err
    }
    if _, err := w.WriteString(k); err != nil {
      return err
    }
    if _, err := w.WriteString(v); err != nil {
      return err
    }
  }
	w.Close()
	return nil
}

func readSize(s []byte) (uint32, int) {
	if len(s) == 0 {
		return 0, 0
	}
	size, n := uint32(s[0]), 1
	if size&(1<<7) != 0 {
		if len(s) < 4 {
			return 0, 0
		}
		n = 4
		size = binary.BigEndian.Uint32(s)
		size &^= 1 << 31
	}
	return size, n
}

func readString(s []byte, size uint32) string {
	if size > uint32(len(s)) {
		return ""
	}
	return string(s[:size])
}

func encodeSize(b []byte, size uint32) int {
	if size > 127 {
		size |= 1 << 31
		binary.BigEndian.PutUint32(b, size)
		return 4
	}
	b[0] = byte(size)
	return 1
}

// bufWriter encapsulates bufio.Writer but also closes the underlying stream when
// Closed.
type bufWriter struct {
	closer io.Closer
	*bufio.Writer
}

func (w *bufWriter) Close() error {
	if err := w.Writer.Flush(); err != nil {
		w.closer.Close()
		return err
	}
	return w.closer.Close()
}

func newWriter(c *FCGIClient, recType uint8, reqId uint16) *bufWriter {
	s := &streamWriter{c: c, recType: recType, reqId: reqId}
	w := bufio.NewWriterSize(s, maxWrite)
	return &bufWriter{s, w}
}

// streamWriter abstracts out the separation of a stream into discrete records.
// It only writes maxWrite bytes at a time.
type streamWriter struct {
	c       *FCGIClient
	recType uint8
	reqId   uint16
}

func (w *streamWriter) Write(p []byte) (int, error) {
	nn := 0
	for len(p) > 0 {
		n := len(p)
		if n > maxWrite {
			n = maxWrite
		}
		if err := w.c.writeRecord(w.recType, w.reqId, p[:n]); err != nil {
			return nn, err
		}
		nn += n
		p = p[n:]
	}
	return nn, nil
}

func (w *streamWriter) Close() error {
	// send empty record to close the stream
	return w.c.writeRecord(w.recType, w.reqId, nil)
}

// data(post) example: "key1=val1&key2=val2"
// do not return content when pasv is ture, only pass it to writer
func (this *FCGIClient) Request(resp http.ResponseWriter, env map[string]string, data []byte, pasv bool) (ret []byte, err error) {

	var reqId uint16 = 1

	// set correct stdin length (required for php)
	env["CONTENT_LENGTH"] = strconv.Itoa(len(data))
	if len(data) > 0 {
	  env["REQUEST_METHOD"] = "POST"
	}

	err = this.writeBeginRequest(reqId, uint16(FCGI_RESPONDER), 0)
	if err != nil {
		return
	}

	err = this.writePairs(FCGI_PARAMS, reqId, env)
	if err != nil {
		return
	}

	for {
	  n := len(data)
	  if n > maxWrite {
	    n = maxWrite
	  }

	  err = this.writeRecord(FCGI_STDIN, reqId, data[:n])
	  if err != nil {
	  	return
	  }
	  if n <= 0 {
	    break
	  }
	  data = data[n:]
	}

  afterheader := false
	rec := &record{}
	for {
    buf, err := rec.read(this.rwc)
  	if err != nil {
  		break
  	}

    if afterheader {
      if resp != nil {
        resp.Write(buf)
      }
      if !pasv {
        ret = append(ret, buf...)
      }
    } else {
      ret = append(ret, buf...)
      // TODO: ensure binary-safed SplitN
      z := strings.SplitN(string(ret), doubleCRLF, 2)
      switch (len(z)) {
        case 2:
          if resp != nil {
            lines := strings.Split(z[0], "\n")
            for line := range lines  {
              v := strings.SplitN(lines[line], ":",2)
              if len(v) == 2 {
                resp.Header().Set(strings.TrimSpace(v[0]), strings.TrimSpace(v[1]))
              }
            }
            resp.Write([]byte(z[1]))
          }
          if pasv {
            ret = ret[:0]
          }else{
            ret = []byte(z[1])
          }
          afterheader = true
        default:
          // wait until doubleCRLF
          continue
      }
    }
  }

	return
}
<commit_msg>force use x-www-form-urlencoded as content-type when post<commit_after>// Copyright 2012 Junqing Tan <ivan@mysqlab.net> and The Go Authors
// Use of this source code is governed by a BSD-style
// Part of source code is from Go fcgi package

package fcgiclient

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"sync"
	"errors"
	"io"
	"net"
  "strings"
  "strconv"
  "net/http"
)

const FCGI_LISTENSOCK_FILENO uint8 = 0
const FCGI_HEADER_LEN uint8 = 8
const VERSION_1 uint8 = 1
const FCGI_NULL_REQUEST_ID uint8 = 0
const FCGI_KEEP_CONN uint8 = 1
const doubleCRLF = "\r\n\r\n"

const (
	FCGI_BEGIN_REQUEST uint8 = iota + 1
	FCGI_ABORT_REQUEST
	FCGI_END_REQUEST
	FCGI_PARAMS
	FCGI_STDIN
	FCGI_STDOUT
	FCGI_STDERR
	FCGI_DATA
	FCGI_GET_VALUES
	FCGI_GET_VALUES_RESULT
	FCGI_UNKNOWN_TYPE
	FCGI_MAXTYPE = FCGI_UNKNOWN_TYPE
)

const (
	FCGI_RESPONDER uint8 = iota + 1
	FCGI_AUTHORIZER
	FCGI_FILTER
)

const (
	FCGI_REQUEST_COMPLETE uint8 = iota
	FCGI_CANT_MPX_CONN
	FCGI_OVERLOADED
	FCGI_UNKNOWN_ROLE
)

const (
	FCGI_MAX_CONNS  string = "MAX_CONNS"
	FCGI_MAX_REQS   string = "MAX_REQS"
	FCGI_MPXS_CONNS string = "MPXS_CONNS"
)

const (
	maxWrite = 65500 // 65530 may work, but for compatibility
	maxPad   = 255
)

type header struct {
	Version       uint8
	Type          uint8
	Id            uint16
	ContentLength uint16
	PaddingLength uint8
	Reserved      uint8
}

// for padding so we don't have to allocate all the time
// not synchronized because we don't care what the contents are
var pad [maxPad]byte

func (h *header) init(recType uint8, reqId uint16, contentLength int) {
	h.Version = 1
	h.Type = recType
	h.Id = reqId
	h.ContentLength = uint16(contentLength)
	h.PaddingLength = uint8(-contentLength & 7)
}

type record struct {
	h   header
	rbuf []byte
}

func (rec *record) read(r io.Reader) (buf []byte, err error) {
	if err = binary.Read(r, binary.BigEndian, &rec.h); err != nil {
		return
	}
	if rec.h.Version != 1 {
    err = errors.New("fcgi: invalid header version")
		return
	}
  if rec.h.Type == FCGI_END_REQUEST {
    err = io.EOF
    return
  }
	n := int(rec.h.ContentLength) + int(rec.h.PaddingLength)
	if len(rec.rbuf) < n {
	  rec.rbuf = make([]byte, n)
	}
	if n, err = io.ReadFull(r, rec.rbuf[:n]); err != nil {
		return
	}
	buf = rec.rbuf[:int(rec.h.ContentLength)]

	return
}

type FCGIClient struct {
	mutex     sync.Mutex
	rwc       io.ReadWriteCloser
	h         header
	buf 	  bytes.Buffer
	keepAlive bool
}

func New(t string, a string) (fcgi *FCGIClient, err error) {
	var conn net.Conn

	conn, err = net.Dial(t, a)
  if err != nil {
    return
  }

	fcgi = &FCGIClient{
		rwc:       conn,
		keepAlive: false,
	}

	return
}

func (this *FCGIClient) Close() {
  this.rwc.Close()
}

func (this *FCGIClient) writeRecord(recType uint8, reqId uint16, content []byte) ( err error) {
	this.mutex.Lock()
	defer this.mutex.Unlock()
	this.buf.Reset()
	this.h.init(recType, reqId, len(content))
	if err := binary.Write(&this.buf, binary.BigEndian, this.h); err != nil {
		return err
	}
	if _, err := this.buf.Write(content); err != nil {
		return err
	}
	if _, err := this.buf.Write(pad[:this.h.PaddingLength]); err != nil {
		return err
	}
	_, err = this.rwc.Write(this.buf.Bytes())
	return err
}

func (this *FCGIClient) writeBeginRequest(reqId uint16, role uint16, flags uint8) error {
	b := [8]byte{byte(role >> 8), byte(role), flags}
	return this.writeRecord(FCGI_BEGIN_REQUEST, reqId, b[:])
}

func (this *FCGIClient) writeEndRequest(reqId uint16, appStatus int, protocolStatus uint8) error {
	b := make([]byte, 8)
	binary.BigEndian.PutUint32(b, uint32(appStatus))
	b[4] = protocolStatus
	return this.writeRecord(FCGI_END_REQUEST, reqId, b)
}

func (this *FCGIClient) writePairs(recType uint8, reqId uint16, pairs map[string]string) error {
	w  := newWriter(this, recType, reqId)
	b  := make([]byte, 8)
	nn := 0
	for k, v := range pairs {
    m := 8 + len(k) + len(v)
    if m > maxWrite {
      // param data size exceed 65535 bytes"
      vl := maxWrite - 8 - len(k)
      v = v[:vl]
    }
    n := encodeSize(b, uint32(len(k)))
    n += encodeSize(b[n:], uint32(len(v)))
    m = n + len(k) + len(v)
    if (nn + m) > maxWrite {
      w.Flush()
      nn = 0
    }
    nn += m
    if _, err := w.Write(b[:n]); err != nil {
      return err
    }
    if _, err := w.WriteString(k); err != nil {
      return err
    }
    if _, err := w.WriteString(v); err != nil {
      return err
    }
  }
	w.Close()
	return nil
}

func readSize(s []byte) (uint32, int) {
	if len(s) == 0 {
		return 0, 0
	}
	size, n := uint32(s[0]), 1
	if size&(1<<7) != 0 {
		if len(s) < 4 {
			return 0, 0
		}
		n = 4
		size = binary.BigEndian.Uint32(s)
		size &^= 1 << 31
	}
	return size, n
}

func readString(s []byte, size uint32) string {
	if size > uint32(len(s)) {
		return ""
	}
	return string(s[:size])
}

func encodeSize(b []byte, size uint32) int {
	if size > 127 {
		size |= 1 << 31
		binary.BigEndian.PutUint32(b, size)
		return 4
	}
	b[0] = byte(size)
	return 1
}

// bufWriter encapsulates bufio.Writer but also closes the underlying stream when
// Closed.
type bufWriter struct {
	closer io.Closer
	*bufio.Writer
}

func (w *bufWriter) Close() error {
	if err := w.Writer.Flush(); err != nil {
		w.closer.Close()
		return err
	}
	return w.closer.Close()
}

func newWriter(c *FCGIClient, recType uint8, reqId uint16) *bufWriter {
	s := &streamWriter{c: c, recType: recType, reqId: reqId}
	w := bufio.NewWriterSize(s, maxWrite)
	return &bufWriter{s, w}
}

// streamWriter abstracts out the separation of a stream into discrete records.
// It only writes maxWrite bytes at a time.
type streamWriter struct {
	c       *FCGIClient
	recType uint8
	reqId   uint16
}

func (w *streamWriter) Write(p []byte) (int, error) {
	nn := 0
	for len(p) > 0 {
		n := len(p)
		if n > maxWrite {
			n = maxWrite
		}
		if err := w.c.writeRecord(w.recType, w.reqId, p[:n]); err != nil {
			return nn, err
		}
		nn += n
		p = p[n:]
	}
	return nn, nil
}

func (w *streamWriter) Close() error {
	// send empty record to close the stream
	return w.c.writeRecord(w.recType, w.reqId, nil)
}

// data(post) example: "key1=val1&key2=val2"
// do not return content when pasv is ture, only pass it to writer
func (this *FCGIClient) Request(resp http.ResponseWriter, env map[string]string, data []byte, pasv bool) (ret []byte, err error) {

	var reqId uint16 = 1

	// set correct stdin length (required for php)
	env["CONTENT_LENGTH"] = strconv.Itoa(len(data))
	if len(data) > 0 {
    // TODO: organize post data automaticly
	  env["REQUEST_METHOD"] = "POST"
    env["CONTENT_TYPE"]   = "application/x-www-form-urlencoded"
	}

	err = this.writeBeginRequest(reqId, uint16(FCGI_RESPONDER), 0)
	if err != nil {
		return
	}

	err = this.writePairs(FCGI_PARAMS, reqId, env)
	if err != nil {
		return
	}

	for {
	  n := len(data)
	  if n > maxWrite {
	    n = maxWrite
	  }

	  err = this.writeRecord(FCGI_STDIN, reqId, data[:n])
	  if err != nil {
	  	return
	  }
	  if n <= 0 {
	    break
	  }
	  data = data[n:]
	}

  afterheader := false
	rec := &record{}
	for {
    buf, err := rec.read(this.rwc)
  	if err != nil {
  		break
  	}

    if afterheader {
      if resp != nil {
        resp.Write(buf)
      }
      if !pasv {
        ret = append(ret, buf...)
      }
    } else {
      ret = append(ret, buf...)
      // TODO: ensure binary-safed SplitN
      z := strings.SplitN(string(ret), doubleCRLF, 2)
      switch (len(z)) {
        case 2:
          if resp != nil {
            lines := strings.Split(z[0], "\n")
            for line := range lines  {
              v := strings.SplitN(lines[line], ":",2)
              if len(v) == 2 {
                resp.Header().Set(strings.TrimSpace(v[0]), strings.TrimSpace(v[1]))
              }
            }
            resp.Write([]byte(z[1]))
          }
          if pasv {
            ret = ret[:0]
          }else{
            ret = []byte(z[1])
          }
          afterheader = true
        default:
          // wait until doubleCRLF
          continue
      }
    }
  }

	return
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2016 Magnus Bäck <magnus@noun.se>

package testcase

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"

	"github.com/magnusbaeck/logstash-filter-verifier/logging"
	"github.com/magnusbaeck/logstash-filter-verifier/logstash"
)

// TestCase contains the configuration of a Logstash filter test case.
// Most of the fields are supplied by the user via a JSON file.
type TestCase struct {
	// File is the absolute path to the file from which this
	// test case was read.
	File string `json:"-"`

	// Codec names the Logstash codec that should be used when
	// events are read. This is normally "plain" or "json".
	Codec string `json:"codec"`

	// IgnoredFields contains a list of fields that will be
	// deleted from the events that Logstash returns before
	// they're compared to the events in ExpectedEevents.
	//
	// This can be used for skipping fields that Logstash
	// populates with unpredictable contents (hostnames or
	// timestamps) that can't be hard-wired into the test case
	// file.
	//
	// It's also useful for the @version field that Logstash
	// always adds with a constant value so that one doesn't have
	// to include that field in every event in ExpectedEvents.
	IgnoredFields []string `json:"ignore"`

	// InputFields contains a mapping of fields that should be
	// added to input events, like "type" or "tags". The map
	// values may be scalar values or arrays of scalar
	// values. This is often important since filters typically are
	// configured based on the event's type or its tags.
	InputFields logstash.FieldSet `json:"fields"`

	// InputLines contains the lines of input that should be fed
	// to the Logstash process.
	InputLines []string `json:"input"`

	// ExpectedEvents contains a slice of expected events to be
	// compared to the actual events produced by the Logstash
	// process.
	ExpectedEvents []logstash.Event `json:"expected"`
}

// ComparisonError indicates that there was a mismatch when the
// results of a test case was compared against the test case
// definition.
type ComparisonError struct {
	ActualCount   int
	ExpectedCount int
	Mismatches    []MismatchedEvent
}

// MismatchedEvent holds a single tuple of actual and expected events
// for a particular index in the list of events for a test case.
type MismatchedEvent struct {
	Actual   logstash.Event
	Expected logstash.Event
	Index    int
}

var (
	log = logging.MustGetLogger()

	defaultIgnoredFields = []string{"@version"}
)

// New reads a test case configuration from a reader and returns a
// TestCase. Defaults to a "plain" codec and ignoring the @version
// field. If the configuration being read lists additional fields to
// ignore those will be ignored in addition to @version.
func New(reader io.Reader) (*TestCase, error) {
	tc := TestCase{
		Codec: "plain",
	}
	buf, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(buf, &tc); err != nil {
		return nil, err
	}
	if err = tc.InputFields.IsValid(); err != nil {
		return nil, err
	}
	for _, f := range defaultIgnoredFields {
		tc.IgnoredFields = append(tc.IgnoredFields, f)
	}
	sort.Strings(tc.IgnoredFields)
	return &tc, nil
}

// NewFromFile reads a test case configuration from an on-disk file.
func NewFromFile(path string) (*TestCase, error) {
	abspath, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}

	log.Debug("Reading test case file: %s (%s)", path, abspath)
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer func() {
		_ = f.Close()
	}()

	tc, err := New(f)
	if err != nil {
		return nil, fmt.Errorf("Error reading/unmarshalling %s: %s", path, err)
	}
	tc.File = abspath
	return tc, nil
}

// Compare compares a slice of events against the expected events of
// this test case. Each event is written pretty-printed to a temporary
// file and the two files are passed to "diff -u". If quiet is true,
// the progress messages normally written to stderr will be emitted
// and the output of the diff program will be discarded.
func (tc *TestCase) Compare(events []logstash.Event, quiet bool, diffCommand []string) error {
	result := ComparisonError{
		ActualCount:   len(events),
		ExpectedCount: len(tc.ExpectedEvents),
		Mismatches:    []MismatchedEvent{},
	}

	// Don't even attempt to do a deep comparison of the event
	// lists unless their lengths are equal.
	if result.ActualCount != result.ExpectedCount {
		return result
	}

	// Create a directory structure for the JSON file being
	// compared that makes it easy for the user to identify
	// the failing test case in the diff output:
	// $TMP/<random>/<test case file>/<event number>/<actual|expected>
	tempdir, err := ioutil.TempDir("", "")
	if err != nil {
		return nil
	}
	defer func() {
		if err := os.RemoveAll(tempdir); err != nil {
			log.Error("Problem deleting temporary directory: %s", err.Error())
		}
	}()

	for i, actualEvent := range events {
		if !quiet {
			fmt.Fprintf(os.Stderr, "Comparing message %d of %s...\n", i+1, filepath.Base(tc.File))
		}

		for _, ignored := range tc.IgnoredFields {
			delete(actualEvent, ignored)
		}

		resultDir := filepath.Join(tempdir, filepath.Base(tc.File), strconv.Itoa(i))
		actualFilePath := filepath.Join(resultDir, "actual")
		if err = marshalToFile(actualEvent, actualFilePath); err != nil {
			return err
		}
		expectedFilePath := filepath.Join(resultDir, "expected")
		if err = marshalToFile(tc.ExpectedEvents[i], expectedFilePath); err != nil {
			return err
		}

		equal, err := runDiffCommand(diffCommand, expectedFilePath, actualFilePath, quiet)
		if err != nil {
			return err
		}
		if !equal {
			result.Mismatches = append(result.Mismatches, MismatchedEvent{actualEvent, tc.ExpectedEvents[i], i})
		}
	}
	if len(result.Mismatches) == 0 {
		return nil
	}
	return result
}

// marshalToFile pretty-prints a logstash.Event and writes it to a
// file, creating the file's parent directories as necessary.
func marshalToFile(event logstash.Event, filename string) error {
	buf, err := json.MarshalIndent(event, "", "  ")
	if err != nil {
		return fmt.Errorf("Failed to marshal %+v as JSON: %s", event, err.Error())
	}
	if err = os.MkdirAll(filepath.Dir(filename), 0700); err != nil {
		return err
	}
	if err = ioutil.WriteFile(filename, []byte(string(buf)+"\n"), 0600); err != nil {
		return err
	}
	return nil
}

// runDiffCommand passes two files to the supplied command (executable
// path and optional arguments) and returns whether the files were
// equal, i.e. whether the diff command returned a zero exit
// status. The returned error value will be set if there was a problem
// running the command. If quiet is true, the output of the diff
// command will be discarded. Otherwise the child process will inherit
// stdout and stderr from the parent.
func runDiffCommand(command []string, file1, file2 string, quiet bool) (bool, error) {
	fullCommand := append(command, file1)
	fullCommand = append(fullCommand, file2)
	c := exec.Command(fullCommand[0], fullCommand[1:]...)
	if !quiet {
		c.Stdout = os.Stdout
		c.Stderr = os.Stderr
	}
	log.Info("Starting %q with args %q.", c.Path, c.Args[1:])
	if err := c.Start(); err != nil {
		return false, err
	}
	if err := c.Wait(); err != nil {
		log.Info("Child with pid %d failed: %s", c.Process.Pid, err.Error())
		return false, nil
	}
	return true, nil
}

func (e ComparisonError) Error() string {
	if e.ActualCount != e.ExpectedCount {
		return fmt.Sprintf("Expected %d event(s), got %d instead.", e.ExpectedCount, e.ActualCount)
	}
	if len(e.Mismatches) > 0 {
		return fmt.Sprintf("%d message(s) did not match the expectations.", len(e.Mismatches))
	}
	return "No error"

}
<commit_msg>Use one-based numbering of temp dirs for result files.<commit_after>// Copyright (c) 2015-2016 Magnus Bäck <magnus@noun.se>

package testcase

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"

	"github.com/magnusbaeck/logstash-filter-verifier/logging"
	"github.com/magnusbaeck/logstash-filter-verifier/logstash"
)

// TestCase contains the configuration of a Logstash filter test case.
// Most of the fields are supplied by the user via a JSON file.
type TestCase struct {
	// File is the absolute path to the file from which this
	// test case was read.
	File string `json:"-"`

	// Codec names the Logstash codec that should be used when
	// events are read. This is normally "plain" or "json".
	Codec string `json:"codec"`

	// IgnoredFields contains a list of fields that will be
	// deleted from the events that Logstash returns before
	// they're compared to the events in ExpectedEevents.
	//
	// This can be used for skipping fields that Logstash
	// populates with unpredictable contents (hostnames or
	// timestamps) that can't be hard-wired into the test case
	// file.
	//
	// It's also useful for the @version field that Logstash
	// always adds with a constant value so that one doesn't have
	// to include that field in every event in ExpectedEvents.
	IgnoredFields []string `json:"ignore"`

	// InputFields contains a mapping of fields that should be
	// added to input events, like "type" or "tags". The map
	// values may be scalar values or arrays of scalar
	// values. This is often important since filters typically are
	// configured based on the event's type or its tags.
	InputFields logstash.FieldSet `json:"fields"`

	// InputLines contains the lines of input that should be fed
	// to the Logstash process.
	InputLines []string `json:"input"`

	// ExpectedEvents contains a slice of expected events to be
	// compared to the actual events produced by the Logstash
	// process.
	ExpectedEvents []logstash.Event `json:"expected"`
}

// ComparisonError indicates that there was a mismatch when the
// results of a test case was compared against the test case
// definition.
type ComparisonError struct {
	ActualCount   int
	ExpectedCount int
	Mismatches    []MismatchedEvent
}

// MismatchedEvent holds a single tuple of actual and expected events
// for a particular index in the list of events for a test case.
type MismatchedEvent struct {
	Actual   logstash.Event
	Expected logstash.Event
	Index    int
}

var (
	log = logging.MustGetLogger()

	defaultIgnoredFields = []string{"@version"}
)

// New reads a test case configuration from a reader and returns a
// TestCase. Defaults to a "plain" codec and ignoring the @version
// field. If the configuration being read lists additional fields to
// ignore those will be ignored in addition to @version.
func New(reader io.Reader) (*TestCase, error) {
	tc := TestCase{
		Codec: "plain",
	}
	buf, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(buf, &tc); err != nil {
		return nil, err
	}
	if err = tc.InputFields.IsValid(); err != nil {
		return nil, err
	}
	for _, f := range defaultIgnoredFields {
		tc.IgnoredFields = append(tc.IgnoredFields, f)
	}
	sort.Strings(tc.IgnoredFields)
	return &tc, nil
}

// NewFromFile reads a test case configuration from an on-disk file.
func NewFromFile(path string) (*TestCase, error) {
	abspath, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}

	log.Debug("Reading test case file: %s (%s)", path, abspath)
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer func() {
		_ = f.Close()
	}()

	tc, err := New(f)
	if err != nil {
		return nil, fmt.Errorf("Error reading/unmarshalling %s: %s", path, err)
	}
	tc.File = abspath
	return tc, nil
}

// Compare compares a slice of events against the expected events of
// this test case. Each event is written pretty-printed to a temporary
// file and the two files are passed to "diff -u". If quiet is true,
// the progress messages normally written to stderr will be emitted
// and the output of the diff program will be discarded.
func (tc *TestCase) Compare(events []logstash.Event, quiet bool, diffCommand []string) error {
	result := ComparisonError{
		ActualCount:   len(events),
		ExpectedCount: len(tc.ExpectedEvents),
		Mismatches:    []MismatchedEvent{},
	}

	// Don't even attempt to do a deep comparison of the event
	// lists unless their lengths are equal.
	if result.ActualCount != result.ExpectedCount {
		return result
	}

	tempdir, err := ioutil.TempDir("", "")
	if err != nil {
		return nil
	}
	defer func() {
		if err := os.RemoveAll(tempdir); err != nil {
			log.Error("Problem deleting temporary directory: %s", err.Error())
		}
	}()

	for i, actualEvent := range events {
		if !quiet {
			fmt.Fprintf(os.Stderr, "Comparing message %d of %s...\n", i+1, filepath.Base(tc.File))
		}

		for _, ignored := range tc.IgnoredFields {
			delete(actualEvent, ignored)
		}

		// Create a directory structure for the JSON file being
		// compared that makes it easy for the user to identify
		// the failing test case in the diff output:
		// $TMP/<random>/<test case file>/<event #>/<actual|expected>
		resultDir := filepath.Join(tempdir, filepath.Base(tc.File), strconv.Itoa(i+1))
		actualFilePath := filepath.Join(resultDir, "actual")
		if err = marshalToFile(actualEvent, actualFilePath); err != nil {
			return err
		}
		expectedFilePath := filepath.Join(resultDir, "expected")
		if err = marshalToFile(tc.ExpectedEvents[i], expectedFilePath); err != nil {
			return err
		}

		equal, err := runDiffCommand(diffCommand, expectedFilePath, actualFilePath, quiet)
		if err != nil {
			return err
		}
		if !equal {
			result.Mismatches = append(result.Mismatches, MismatchedEvent{actualEvent, tc.ExpectedEvents[i], i})
		}
	}
	if len(result.Mismatches) == 0 {
		return nil
	}
	return result
}

// marshalToFile pretty-prints a logstash.Event and writes it to a
// file, creating the file's parent directories as necessary.
func marshalToFile(event logstash.Event, filename string) error {
	buf, err := json.MarshalIndent(event, "", "  ")
	if err != nil {
		return fmt.Errorf("Failed to marshal %+v as JSON: %s", event, err.Error())
	}
	if err = os.MkdirAll(filepath.Dir(filename), 0700); err != nil {
		return err
	}
	if err = ioutil.WriteFile(filename, []byte(string(buf)+"\n"), 0600); err != nil {
		return err
	}
	return nil
}

// runDiffCommand passes two files to the supplied command (executable
// path and optional arguments) and returns whether the files were
// equal, i.e. whether the diff command returned a zero exit
// status. The returned error value will be set if there was a problem
// running the command. If quiet is true, the output of the diff
// command will be discarded. Otherwise the child process will inherit
// stdout and stderr from the parent.
func runDiffCommand(command []string, file1, file2 string, quiet bool) (bool, error) {
	fullCommand := append(command, file1)
	fullCommand = append(fullCommand, file2)
	c := exec.Command(fullCommand[0], fullCommand[1:]...)
	if !quiet {
		c.Stdout = os.Stdout
		c.Stderr = os.Stderr
	}
	log.Info("Starting %q with args %q.", c.Path, c.Args[1:])
	if err := c.Start(); err != nil {
		return false, err
	}
	if err := c.Wait(); err != nil {
		log.Info("Child with pid %d failed: %s", c.Process.Pid, err.Error())
		return false, nil
	}
	return true, nil
}

func (e ComparisonError) Error() string {
	if e.ActualCount != e.ExpectedCount {
		return fmt.Sprintf("Expected %d event(s), got %d instead.", e.ExpectedCount, e.ActualCount)
	}
	if len(e.Mismatches) > 0 {
		return fmt.Sprintf("%d message(s) did not match the expectations.", len(e.Mismatches))
	}
	return "No error"

}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package log4go

import (
	"os"
	"fmt"
	"net"
	"json"
)

const (
	EOM = "\n"
)

// This log writer sends output to a socket
type SocketLogWriter struct {
	sock net.Conn
}

// This is the SocketLogWriter's output method
func (slw *SocketLogWriter) LogWrite(rec *LogRecord) (int, os.Error) {
	if !slw.Good() {
		return -1, os.NewError("Socket was not opened successfully")
	}

	// Marshall into JSON
	js, err := json.Marshal(rec)
	if err != nil { return 0, err }

	// Write to socket
	n, err := slw.sock.Write(js)
	if err != nil {
		return n, err
	}

	// Send End Of Message
	m, err := slw.sock.Write([]byte(EOM))

	return n+m, err
}

func (slw *SocketLogWriter) Good() bool {
	return slw.sock != nil
}

func (slw *SocketLogWriter) Close() {
	slw.sock.Close()
	slw.sock = nil
}

func NewSocketLogWriter(proto, hostport string) *SocketLogWriter {
	s, err := net.Dial(proto, "", hostport)
	slw := new(SocketLogWriter)

	if err != nil || s == nil {
		fmt.Fprintf(os.Stderr, "NewSocketLogWriter: %s\n", err)
	}

	slw.sock = s
	return slw
}
<commit_msg>Fixed network logging via UDP<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package log4go

import (
	"os"
	"fmt"
	"net"
	"json"
)

// This log writer sends output to a socket
type SocketLogWriter struct {
	sock net.Conn
}

// This is the SocketLogWriter's output method
func (slw *SocketLogWriter) LogWrite(rec *LogRecord) (int, os.Error) {
	if !slw.Good() {
		return -1, os.NewError("Socket was not opened successfully")
	}

	// Marshall into JSON
	js, err := json.Marshal(rec)
	if err != nil { return 0, err }

	// Write to socket
	return slw.sock.Write(js)
}

func (slw *SocketLogWriter) Good() bool {
	return slw.sock != nil
}

func (slw *SocketLogWriter) Close() {
	if slw.sock != nil && slw.sock.RemoteAddr().Network() == "tcp" {
		slw.sock.Close()
	}
	slw.sock = nil
}

func NewSocketLogWriter(proto, hostport string) *SocketLogWriter {
	s, err := net.Dial(proto, "", hostport)
	slw := new(SocketLogWriter)

	if err != nil || s == nil {
		fmt.Fprintf(os.Stderr, "NewSocketLogWriter: %s\n", err)
	}

	slw.sock = s
	return slw
}
<|endoftext|>


<commit_before>package some

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/laher/someutils"
	"github.com/laher/uggo"
	"io"
	"regexp"
	"strings"
)

func init() {
	someutils.RegisterPipable(func() someutils.CliPipable { return new(SomeTr) })
}

type SomeTr struct {
	IsDelete     bool
	IsSqueeze    bool
	IsComplement bool // currently unused as I just don't get it.
	set1         string
	set2         string
	translations map[*regexp.Regexp]string
}

func (tr *SomeTr) Name() string {
	return "tr"
}

func (tr *SomeTr) ParseFlags(call []string, errWriter io.Writer) (error, int) {
	flagSet := uggo.NewFlagSetDefault("tr", "[OPTION]... SET1 [SET2]", someutils.VERSION)
	flagSet.SetOutput(errWriter)
	flagSet.AliasedBoolVar(&tr.IsDelete, []string{"d", "delete"}, false, "Delete characters in SET1, do not translate")
	flagSet.AliasedBoolVar(&tr.IsSqueeze, []string{"s", "squeeze-repeats"}, false, "replace each input sequence of a repeated character that is listed in SET1 with a single occurence of that character")
	//Don't get the Complement thing. Just don't understand it right now.
	flagSet.AliasedBoolVar(&tr.IsComplement, []string{"c", "complement"}, false, "use the complement of SET1")
	err, code := flagSet.ParsePlus(call[1:])
	if err != nil {
		return err, code
	}
	sets := flagSet.Args()
	if len(sets) > 0 {
		tr.set1 = sets[0]
	} else {
		return errors.New("Not enough args supplied"), 1
	}
	if len(sets) > 1 {
		tr.set2 = sets[1]
	} else if !tr.IsDelete {
		return errors.New("Not enough args supplied"), 1
	}
	err = tr.Preprocess()
	if err != nil {
		return err, 1
	}
	return nil, 0
}

func (tr *SomeTr) Preprocess() error {
	tr.translations = map[*regexp.Regexp]string{}
	set1 := tr.set1

	var set1Part, set2Part string
	var set2len int
	var err error
	if tr.IsDelete {
		// processSet1 only
		for len(set1) > 0 {
			set1Part, set1, _ = nextPartSet1(set1)
			reg, err := tr.toRegexp(set1Part)
			if err != nil {
				return err
			}
			//replacement is empty-string
			tr.translations[reg] = ""
		}
	} else {
		// process both sets together
		set2 := tr.set2
		for len(set1) > 0 {
			set1Part, set1, set2len = nextPartSet1(set1)
			var set2New string
			set2Part, set2New, err = nextPartSet2(set2, set2len)
			if len(set2New) > 0 { //incase set2 is shorter than set1, behave like BSD tr (rather than SystemV, which truncates set1 instead)
				set2 = set2New
			}
			if err != nil {
				return err
			}
			reg, err := tr.toRegexp(set1Part)
			if err != nil {
				return err
			}
			tr.translations[reg] = set2Part
		}
	}
	return nil
}

func (tr *SomeTr) toRegexp(set1Part string) (*regexp.Regexp, error) {
	maybeSqueeze := ""
	maybeComplement := ""
	if tr.IsSqueeze {
		maybeSqueeze = "+"
	}
	if tr.IsComplement {
		maybeComplement = "^"
	}
	regString := "^["+maybeComplement+set1Part+"]"+maybeSqueeze
	//fmt.Println(regString)
	reg, err := regexp.Compile(regString)
	return reg, err
}

func nextPartSet1(set1 string) (string, string, int) {
	if strings.HasPrefix(set1, "[") {
		//find matching
		if strings.Contains(set1, "]") {
			return set1[:strings.Index(set1, "]")+1], set1[strings.Index(set1, "]")+1:], 1

		} else {
			return set1[:1], set1[1:], 1
		}
	} else if len(set1)>2 && set1[1]=='-' {
		return set1[:3], set1[3:], 1
	} else {
		return set1[:1], set1[1:], 1
	}
}

func nextPartSet2(set2 string, set2len int) (string, string, error) {
	if len(set2) < set2len {
		return "", "", errors.New(fmt.Sprintf("Error out of range (%d - %s)", set2len, set2))
	}
	return set2[:set2len], set2[set2len:], nil
}

// Invoke actually carries out the command
func (tr *SomeTr) Invoke(invocation *someutils.Invocation) (error, int) {
	invocation.ErrPipe.Drain()
	invocation.AutoHandleSignals()
	tr.Preprocess()
	fu := func(inPipe io.Reader, outPipe2 io.Writer, errPipe io.Writer, line []byte) error {
		//println("tr processing line")
		//outline := line
		var buffer bytes.Buffer
		remainder := string(line)
		for len(remainder) > 0 {
			trimLeft := 1
			nextPart := remainder[:trimLeft]
			for reg, v := range tr.translations {
				//fmt.Printf("Translation '%v'=>'%s' on '%s'\n", reg, v, remainder)
				match := reg.MatchString(remainder)
				if match {
					toReplace := reg.FindString(remainder)
					replacement := reg.ReplaceAllString(toReplace, v)
					//fmt.Printf("Replace %s=>%s\n", toReplace, replacement)
					nextPart = replacement
					//if squeezing has taken place, remove more leading chars accordingly
					trimLeft = len(toReplace)
					if !tr.IsComplement {
						break
					}
				} else if tr.IsComplement {
					// this is a double-negative - non-match of negative-regex.
					// This implies that set1 matches the current input character.
					// So, keep it as-is and break out of the loop.
					trimLeft = 1
					nextPart = remainder[:trimLeft]
					break
				}
			}

			remainder = remainder[trimLeft:]
			buffer.WriteString(nextPart)
		}
		out := buffer.String()
		_, err := fmt.Fprintln(outPipe2, out)
		return err
	}
	err := someutils.LineProcessor(invocation.MainPipe.In, invocation.MainPipe.Out, invocation.ErrPipe.Out, fu)
	if err != nil {
		return err, 1
	}
	return nil, 0
}

func NewTr() *SomeTr {
	return new(SomeTr)
}

// Factory for normal `tr`
func Tr(set1, set2 string) someutils.CliPipable {
	tr := NewTr()
	tr.set1 = set1
	tr.set2 = set2
	return (tr)
}

// Factory for `tr -d` (delete set1)
func TrD(set1 string) someutils.CliPipable {
	tr := NewTr()
	tr.IsDelete = true
	tr.set1 = set1
	return (tr)
}

// Factory for `tr -c` (complement)
func TrC(set1 string, set2 string) someutils.CliPipable {
	tr := NewTr()
	tr.IsComplement = true
	tr.set1 = set1
	tr.set2 = set2
	return (tr)
}

//Cli helper for tr
func TrCli(call []string) (error, int) {

	util := new(SomeTr)
	return someutils.StdInvoke((util), call)
}
<commit_msg>Adding 'complement' support for tr<commit_after>package some

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/laher/someutils"
	"github.com/laher/uggo"
	"io"
	"regexp"
	"strings"
)

func init() {
	someutils.RegisterPipable(func() someutils.CliPipable { return new(SomeTr) })
}

// SomeTr mimics the functionality of the `tr` function from coreutils.
type SomeTr struct {
	IsDelete     bool
	IsSqueeze    bool
	IsComplement bool // currently unused as I just don't get it.
	set1         string
	set2         string
	translations map[*regexp.Regexp]string
}

// returns the name of the command
func (tr *SomeTr) Name() string {
	return "tr"
}

// ParseFlags parses commandline options.
func (tr *SomeTr) ParseFlags(call []string, errWriter io.Writer) (error, int) {
	flagSet := uggo.NewFlagSetDefault("tr", "[OPTION]... SET1 [SET2]", someutils.VERSION)
	flagSet.SetOutput(errWriter)
	flagSet.AliasedBoolVar(&tr.IsDelete, []string{"d", "delete"}, false, "Delete characters in SET1, do not translate")
	flagSet.AliasedBoolVar(&tr.IsSqueeze, []string{"s", "squeeze-repeats"}, false, "replace each input sequence of a repeated character that is listed in SET1 with a single occurence of that character")
	//Don't get the Complement thing. Just don't understand it right now.
	flagSet.AliasedBoolVar(&tr.IsComplement, []string{"c", "complement"}, false, "use the complement of SET1")
	err, code := flagSet.ParsePlus(call[1:])
	if err != nil {
		return err, code
	}
	sets := flagSet.Args()
	if len(sets) > 0 {
		tr.set1 = sets[0]
	} else {
		return errors.New("Not enough args supplied"), 1
	}
	if len(sets) > 1 {
		tr.set2 = sets[1]
	} else if !tr.IsDelete {
		return errors.New("Not enough args supplied"), 1
	}
	err = tr.Preprocess()
	if err != nil {
		return err, 1
	}
	return nil, 0
}

// Parses SET1 and SET2 into a map of translations
func (tr *SomeTr) Preprocess() error {
	tr.translations = map[*regexp.Regexp]string{}
	set1 := tr.set1

	var set1Part, set2Part string
	var set2len int
	var err error
	if tr.IsDelete {
		// processSet1 only
		for len(set1) > 0 {
			set1Part, set1, _ = nextPartSet1(set1)
			reg, err := tr.toRegexp(set1Part)
			if err != nil {
				return err
			}
			//replacement is empty-string
			tr.translations[reg] = ""
		}
	} else {
		// process both sets together
		set2 := tr.set2
		for len(set1) > 0 {
			set1Part, set1, set2len = nextPartSet1(set1)
			var set2New string
			set2Part, set2New, err = nextPartSet2(set2, set2len)
			if len(set2New) > 0 { //incase set2 is shorter than set1, behave like BSD tr (rather than SystemV, which truncates set1 instead)
				set2 = set2New
			}
			if err != nil {
				return err
			}
			reg, err := tr.toRegexp(set1Part)
			if err != nil {
				return err
			}
			tr.translations[reg] = set2Part
		}
	}
	return nil
}

func (tr *SomeTr) toRegexp(set1Part string) (*regexp.Regexp, error) {
	maybeSqueeze := ""
	maybeComplement := ""
	if tr.IsSqueeze {
		maybeSqueeze = "+"
	}
	if tr.IsComplement {
		maybeComplement = "^"
	}
	regString := "^["+maybeComplement+set1Part+"]"+maybeSqueeze
	//fmt.Println(regString)
	reg, err := regexp.Compile(regString)
	return reg, err
}

// Parser for SET1. Supports single-chars, ranges, and regex-like character groups
func nextPartSet1(set1 string) (string, string, int) {
	if strings.HasPrefix(set1, "[") {
		//find matching
		if strings.Contains(set1, "]") {
			return set1[:strings.Index(set1, "]")+1], set1[strings.Index(set1, "]")+1:], 1

		} else {
			return set1[:1], set1[1:], 1
		}
	} else if len(set1)>2 && set1[1]=='-' {
		return set1[:3], set1[3:], 1
	} else {
		return set1[:1], set1[1:], 1
	}
}

// Parser for SET2. Supports single and multiple chars
func nextPartSet2(set2 string, set2len int) (string, string, error) {
	if len(set2) < set2len {
		return "", "", errors.New(fmt.Sprintf("Error out of range (%d - %s)", set2len, set2))
	}
	return set2[:set2len], set2[set2len:], nil
}

// Invoke actually carries out the command
func (tr *SomeTr) Invoke(invocation *someutils.Invocation) (error, int) {
	invocation.ErrPipe.Drain()
	invocation.AutoHandleSignals()
	tr.Preprocess()
	fu := func(inPipe io.Reader, outPipe2 io.Writer, errPipe io.Writer, line []byte) error {
		//println("tr processing line")
		//outline := line
		var buffer bytes.Buffer
		remainder := string(line)
		for len(remainder) > 0 {
			trimLeft := 1
			nextPart := remainder[:trimLeft]
			for reg, v := range tr.translations {
				//fmt.Printf("Translation '%v'=>'%s' on '%s'\n", reg, v, remainder)
				match := reg.MatchString(remainder)
				if match {
					toReplace := reg.FindString(remainder)
					replacement := reg.ReplaceAllString(toReplace, v)
					//fmt.Printf("Replace %s=>%s\n", toReplace, replacement)
					nextPart = replacement
					//if squeezing has taken place, remove more leading chars accordingly
					trimLeft = len(toReplace)
					if !tr.IsComplement {
						break
					}
				} else if tr.IsComplement {
					// this is a double-negative - non-match of negative-regex.
					// This implies that set1 matches the current input character.
					// So, keep it as-is and break out of the loop.
					trimLeft = 1
					nextPart = remainder[:trimLeft]
					break
				}
			}

			remainder = remainder[trimLeft:]
			buffer.WriteString(nextPart)
		}
		out := buffer.String()
		_, err := fmt.Fprintln(outPipe2, out)
		return err
	}
	err := someutils.LineProcessor(invocation.MainPipe.In, invocation.MainPipe.Out, invocation.ErrPipe.Out, fu)
	if err != nil {
		return err, 1
	}
	return nil, 0
}

func NewTr() *SomeTr {
	return new(SomeTr)
}

// Factory for normal `tr`
func Tr(set1, set2 string) someutils.CliPipable {
	tr := NewTr()
	tr.set1 = set1
	tr.set2 = set2
	return (tr)
}

// Factory for `tr -d` (delete set1)
func TrD(set1 string) someutils.CliPipable {
	tr := NewTr()
	tr.IsDelete = true
	tr.set1 = set1
	return (tr)
}

// Factory for `tr -c` (complement)
func TrC(set1 string, set2 string) someutils.CliPipable {
	tr := NewTr()
	tr.IsComplement = true
	tr.set1 = set1
	tr.set2 = set2
	return (tr)
}

//Cli helper for tr
func TrCli(call []string) (error, int) {

	util := new(SomeTr)
	return someutils.StdInvoke((util), call)
}
<|endoftext|>


<commit_before>package blinkstick

import (
	"fmt"
	"image/color"

	"github.com/yesnault/hid"
)

// Version of Blinkstick
// One Line for this, used by release.sh script
// Keep "const Version on one line"
const Version = "0.1.4"

// vendorID blinkstick
const vendorID = 0x20a0

// productID blinkstick
const productID = 0x41e5

// usbDevice ...
type usbDevice struct {
	DeviceInfo *hid.DeviceInfo
	Device     *hid.Device
}

// Blinkstick represents a blinkstick device
type Blinkstick interface {
	List() []Blinkstick
	SetColor(color.Color) error
	Blink(color.Color, int, int) error
	GetDeviceInfo() *hid.DeviceInfo
	ListFilter(hid *hid.DeviceInfo) (bool, Blinkstick)
	getUSBDevice() *usbDevice
}

// SetColor set color
func (usbDevice *usbDevice) setColor(c color.Color, index byte) error {
	if usbDevice.Device == nil {
		if err := usbDevice.Open(); err != nil {
			return err
		}
	}
	r, g, b, _ := c.RGBA()
	d := *usbDevice.Device
	return d.WriteFeature([]byte{0x05, 0x00, index, byte(r >> 8), byte(g >> 8), byte(b >> 8)})
}

// Open open a device
func (usbDevice *usbDevice) Open() error {
	device, err := usbDevice.DeviceInfo.Open()
	if err != nil {
		return fmt.Errorf("Error while opening device: %s", err)
	}
	usbDevice.Device = &device
	return nil
}

// ListFilter is used to filter device on List
type ListFilter func(*hid.DeviceInfo) (bool, Blinkstick)

// List gets all blinkstick device
func List(opts ...ListFilter) []Blinkstick {
	out := []Blinkstick{}

	if len(opts) == 0 {
		opts = append(opts, Nano{}.ListFilter)
		opts = append(opts, Strip{}.ListFilter)
	}

	for di := range hid.Devices() {
		if di.VendorId == vendorID && di.ProductId == productID {
			//			fmt.Printf("di: %+v", di)
			for _, o := range opts {
				if toKeep, blinkstick := o(di); toKeep {
					out = append(out, blinkstick)
				}
			}

		}
	}
	return out
}
<commit_msg>[auto] bump version to v0.1.5<commit_after>package blinkstick

import (
	"fmt"
	"image/color"

	"github.com/yesnault/hid"
)

// Version of Blinkstick
// One Line for this, used by release.sh script
// Keep "const Version on one line"
const Version = "0.1.5"

// vendorID blinkstick
const vendorID = 0x20a0

// productID blinkstick
const productID = 0x41e5

// usbDevice ...
type usbDevice struct {
	DeviceInfo *hid.DeviceInfo
	Device     *hid.Device
}

// Blinkstick represents a blinkstick device
type Blinkstick interface {
	List() []Blinkstick
	SetColor(color.Color) error
	Blink(color.Color, int, int) error
	GetDeviceInfo() *hid.DeviceInfo
	ListFilter(hid *hid.DeviceInfo) (bool, Blinkstick)
	getUSBDevice() *usbDevice
}

// SetColor set color
func (usbDevice *usbDevice) setColor(c color.Color, index byte) error {
	if usbDevice.Device == nil {
		if err := usbDevice.Open(); err != nil {
			return err
		}
	}
	r, g, b, _ := c.RGBA()
	d := *usbDevice.Device
	return d.WriteFeature([]byte{0x05, 0x00, index, byte(r >> 8), byte(g >> 8), byte(b >> 8)})
}

// Open open a device
func (usbDevice *usbDevice) Open() error {
	device, err := usbDevice.DeviceInfo.Open()
	if err != nil {
		return fmt.Errorf("Error while opening device: %s", err)
	}
	usbDevice.Device = &device
	return nil
}

// ListFilter is used to filter device on List
type ListFilter func(*hid.DeviceInfo) (bool, Blinkstick)

// List gets all blinkstick device
func List(opts ...ListFilter) []Blinkstick {
	out := []Blinkstick{}

	if len(opts) == 0 {
		opts = append(opts, Nano{}.ListFilter)
		opts = append(opts, Strip{}.ListFilter)
	}

	for di := range hid.Devices() {
		if di.VendorId == vendorID && di.ProductId == productID {
			//			fmt.Printf("di: %+v", di)
			for _, o := range opts {
				if toKeep, blinkstick := o(di); toKeep {
					out = append(out, blinkstick)
				}
			}

		}
	}
	return out
}
<|endoftext|>


<commit_before>package common

import (
	"time"
)

const (
	DefaultPort          = 7894
	ProductVersion       = "dev" // protocol version!
	ProductName          = "GoshawkDB"
	HeartbeatInterval    = 2 * time.Second
	ConnectionBufferSize = 131072
)

var (
	VersionZero = MakeTxnId([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0})
)
<commit_msg>Set protocol version. Ref T52<commit_after>package common

import (
	"time"
)

const (
	DefaultPort          = 7894
	ProductVersion       = "0.3" // protocol version!
	ProductName          = "GoshawkDB"
	HeartbeatInterval    = 2 * time.Second
	ConnectionBufferSize = 131072
)

var (
	VersionZero = MakeTxnId([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0})
)
<|endoftext|>


<commit_before>package cfutil

import (
	"errors"
	"fmt"
	consul "github.com/hashicorp/consul/api"
	"net/url"
	"os"
)

// Services() returns the list of services available from the
// Consul cluster
func Services() (map[string]*consul.AgentService, error) {
	client, err := NewConsulClient()
	if err != nil {
		return nil, err
	}
	return client.Agent().Services()
}

func DiscoverServiceUrl(serviceName, tags string) (string, error) {
	client, err := NewConsulClient()
	if err != nil {
		return "", err
	}
	services, _, err := client.Catalog().Service(serviceName, tags, nil)
	if err != nil {
		return "", fmt.Errorf("Service `%s` not found: %s", serviceName, err)
	}
	if len(services) > 0 {
		return createUrlFromServiceCatalog(services[0])
	}
	return "", fmt.Errorf("Service `%s` not found", serviceName)

}

func createUrlFromServiceCatalog(catalog *consul.CatalogService) (string, error) {
	var serviceUrl url.URL
	if catalog.ServicePort == 443 {
		serviceUrl.Scheme = "https"
		serviceUrl.Host = catalog.ServiceAddress
	} else {
		serviceUrl.Scheme = "http"
		serviceUrl.Host = fmt.Sprintf("%s:%d", catalog.ServiceAddress, catalog.ServicePort)
	}
	return serviceUrl.String(), nil
}

// Use ServiceRegister() to register your app in the Consul cluster
// Optionally you can provide a health endpoint on your URL and
// a number of tags to make your service more discoverable
func ServiceRegister(name string, path string, tags ...string) error {
	appEnv, _ := Current()
	client, err := NewConsulClient()
	if err != nil {
		return err
	}
	schema, port := schemaAndPortForServices()
	err = client.Agent().ServiceRegister(&consul.AgentServiceRegistration{
		Name:    name,
		Address: appEnv.ApplicationURIs[0],
		Port:    port,
		Tags:    tags,
		Check: &consul.AgentServiceCheck{
			HTTP:     fmt.Sprintf(schema + "://" + appEnv.ApplicationURIs[0] + path),
			Interval: "60s",
		},
	})
	if err != nil {
		return err
	}
	return nil
}

// NewConsulClient() returns a new consul client which you can use to
// access the Consul cluster HTTP API. It uses `CONSUL_MASTER` and
// `CONSUL_TOKEN` environment variables to set up the HTTP API connection.
func NewConsulClient() (*consul.Client, error) {
	dialScheme, dialHost, err := consulDialstring("consul")
	if err != nil {
		return nil, err
	}
	client, consulErr := consul.NewClient(&consul.Config{
		Address: dialHost,
		Scheme:  dialScheme,
		Token:   os.Getenv("CONSUL_TOKEN"),
	})
	if consulErr != nil {
		return nil, consulErr
	}
	return client, nil
}

func consulDialstring(serviceName string) (string, string, error) {
	consulMaster := ""
	if consulMaster = os.Getenv("CONSUL_MASTER"); consulMaster != "" {
		parsed, err := url.Parse(consulMaster)
		if err == nil {
			return parsed.Scheme, parsed.Host, nil
		}
	}
	return "", "", errors.New("CONSUL_MASTER not found or invalid url")
}

func schemaAndPortForServices() (string, int) {
	if ForceHTTP() {
		return "http", 80
	}
	return "https", 443
}
<commit_msg>Stick to Go conventions<commit_after>package cfutil

import (
	"errors"
	"fmt"
	consul "github.com/hashicorp/consul/api"
	"net/url"
	"os"
)

// Services() returns the list of services available from the
// Consul cluster
func Services() (map[string]*consul.AgentService, error) {
	client, err := NewConsulClient()
	if err != nil {
		return nil, err
	}
	return client.Agent().Services()
}

func DiscoverServiceURL(serviceName, tags string) (string, error) {
	client, err := NewConsulClient()
	if err != nil {
		return "", err
	}
	services, _, err := client.Catalog().Service(serviceName, tags, nil)
	if err != nil {
		return "", fmt.Errorf("Service `%s` not found: %s", serviceName, err)
	}
	if len(services) > 0 {
		return createURLFromServiceCatalog(services[0])
	}
	return "", fmt.Errorf("Service `%s` not found", serviceName)

}

func createURLFromServiceCatalog(catalog *consul.CatalogService) (string, error) {
	var serviceURL url.URL
	if catalog.ServicePort == 443 {
		serviceURL.Scheme = "https"
		serviceURL.Host = catalog.ServiceAddress
	} else {
		serviceURL.Scheme = "http"
		serviceURL.Host = fmt.Sprintf("%s:%d", catalog.ServiceAddress, catalog.ServicePort)
	}
	return serviceURL.String(), nil
}

// Use ServiceRegister() to register your app in the Consul cluster
// Optionally you can provide a health endpoint on your URL and
// a number of tags to make your service more discoverable
func ServiceRegister(name string, path string, tags ...string) error {
	appEnv, _ := Current()
	client, err := NewConsulClient()
	if err != nil {
		return err
	}
	schema, port := schemaAndPortForServices()
	err = client.Agent().ServiceRegister(&consul.AgentServiceRegistration{
		Name:    name,
		Address: appEnv.ApplicationURIs[0],
		Port:    port,
		Tags:    tags,
		Check: &consul.AgentServiceCheck{
			HTTP:     fmt.Sprintf(schema + "://" + appEnv.ApplicationURIs[0] + path),
			Interval: "60s",
		},
	})
	if err != nil {
		return err
	}
	return nil
}

// NewConsulClient() returns a new consul client which you can use to
// access the Consul cluster HTTP API. It uses `CONSUL_MASTER` and
// `CONSUL_TOKEN` environment variables to set up the HTTP API connection.
func NewConsulClient() (*consul.Client, error) {
	dialScheme, dialHost, err := consulDialstring("consul")
	if err != nil {
		return nil, err
	}
	client, consulErr := consul.NewClient(&consul.Config{
		Address: dialHost,
		Scheme:  dialScheme,
		Token:   os.Getenv("CONSUL_TOKEN"),
	})
	if consulErr != nil {
		return nil, consulErr
	}
	return client, nil
}

func consulDialstring(serviceName string) (string, string, error) {
	consulMaster := ""
	if consulMaster = os.Getenv("CONSUL_MASTER"); consulMaster != "" {
		parsed, err := url.Parse(consulMaster)
		if err == nil {
			return parsed.Scheme, parsed.Host, nil
		}
	}
	return "", "", errors.New("CONSUL_MASTER not found or invalid url")
}

func schemaAndPortForServices() (string, int) {
	if ForceHTTP() {
		return "http", 80
	}
	return "https", 443
}
<|endoftext|>