<commit_before>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
)

const (
	nautilusImage       = "kubernetes/update-demo:nautilus"
	kittenImage         = "kubernetes/update-demo:kitten"
	updateDemoSelector  = "name=update-demo"
	updateDemoContainer = "update-demo"
	validateTimeout     = 60 * time.Second
	kubectlProxyPort    = 8011
)

var _ = Describe("kubectl", func() {

	// Constants.
	var (
		updateDemoRoot = filepath.Join(testContext.repoRoot, "examples/update-demo")
		nautilusPath   = filepath.Join(updateDemoRoot, "nautilus-rc.yaml")
		kittenPath     = filepath.Join(updateDemoRoot, "kitten-rc.yaml")
	)

	var cmd *exec.Cmd

	BeforeEach(func() {
		cmd = kubectlCmd("proxy", fmt.Sprintf("--port=%d", kubectlProxyPort))
		if err := cmd.Start(); err != nil {
			Failf("Unable to start kubectl proxy: %v", err)
		}
	})

	AfterEach(func() {
		if cmd.Process == nil {
			Logf("No process associated with the kubectl proxy")
			return
		}
		// Kill the proxy
		errChan := make(chan error, 1)
		go func() {
			if err := cmd.Process.Signal(os.Interrupt); err != nil {
				errChan <- err
			}
			_, err := cmd.Process.Wait()
			errChan <- err
		}()
		select {
		case <-time.After(10 * time.Second):
			Fail("Timed out waiting for kubectl proxy process to exit")
		case err := <-errChan:
			if err != nil {
				Failf("Unable to kill kubectl proxy: %v", err)
			}
		}
	})

	It("should create and stop a replication controller", func() {
		defer cleanup(nautilusPath)

		By("creating a replication controller")
		runKubectl("create", "-f", nautilusPath)
		validateController(nautilusImage, 2)
	})

	It("should scale a replication controller", func() {
		defer cleanup(nautilusPath)

		By("creating a replication controller")
		runKubectl("create", "-f", nautilusPath)
		validateController(nautilusImage, 2)
		By("scaling down the replication controller")
		runKubectl("resize", "rc", "update-demo-nautilus", "--replicas=1")
		validateController(nautilusImage, 1)
		By("scaling up the replication controller")
		runKubectl("resize", "rc", "update-demo-nautilus", "--replicas=2")
		validateController(nautilusImage, 2)
	})

	It("should do a rolling update of a replication controller", func() {
		// Cleanup all resources in case we fail somewhere in the middle
		defer cleanup(updateDemoRoot)

		By("creating the initial replication controller")
		runKubectl("create", "-f", nautilusPath)
		validateController(nautilusImage, 2)
		By("rollingupdate to new replication controller")
		runKubectl("rollingupdate", "update-demo-nautilus", "--update-period=1s", "-f", kittenPath)
		validateController(kittenImage, 2)
	})

})

func cleanup(filePath string) {
	By("using stop to clean up resources")
	runKubectl("stop", "-f", filePath)

	resources := runKubectl("get", "pods,rc", "-l", updateDemoSelector, "--no-headers")
	if resources != "" {
		Failf("Resources left running after stop:\n%s", resources)
	}
}

func validateController(image string, replicas int) {

	getPodsTemplate := "--template={{range.items}}{{.id}} {{end}}"

	// NB: kubectl adds the "exists" function to the standard template functions.
	// This lets us check to see if the "running" entry exists for each of the containers
	// we care about. Exists will never return an error and it's safe to check a chain of
	// things, any one of which may not exist. In the below template, all of info,
	// containername, and running might be nil, so the normal index function isn't very
	// helpful.
	// This template is unit-tested in kubectl, so if you change it, update the unit test.
	//
	// You can read about the syntax here: http://golang.org/pkg/text/template/
	getContainerStateTemplate := fmt.Sprintf(`--template={{and (exists . "currentState" "info" "%s" "state" "running")}}`, updateDemoContainer)

	getImageTemplate := fmt.Sprintf(`--template={{(index .currentState.info "%s").image}}`, updateDemoContainer)

	By(fmt.Sprintf("waiting for all containers in %s pods to come up.", updateDemoSelector))
	for start := time.Now(); time.Since(start) < validateTimeout; time.Sleep(5 * time.Second) {
		getPodsOutput := runKubectl("get", "pods", "-o", "template", getPodsTemplate, "-l", updateDemoSelector)
		pods := strings.Fields(getPodsOutput)
		if numPods := len(pods); numPods != replicas {
			By(fmt.Sprintf("Replicas for %s: expected=%d actual=%d", updateDemoSelector, replicas, numPods))
			continue
		}
		var runningPods []string
		for _, podID := range pods {
			running := runKubectl("get", "pods", podID, "-o", "template", getContainerStateTemplate)
			if running == "false" {
				Logf("%s is created but not running", podID)
				continue
			}

			currentImage := runKubectl("get", "pods", podID, "-o", "template", getImageTemplate)
			if currentImage != image {
				Logf("%s is created but running wrong image; expected: %s, actual: %s", podID, image, currentImage)
				continue
			}

			data, err := getData(podID)
			if err != nil {
				Logf("%s is running right image but fetching data failed: %v", podID, err)
				continue
			}
			if strings.Contains(data.image, image) {
				Logf("%s is running right image but fetched data has the wrong info: %s", podID, data)
				continue
			}

			Logf("%s is verified up and running", podID)
			runningPods = append(runningPods, podID)
		}
		if len(runningPods) == replicas {
			return
		}
	}
	Failf("Timed out waiting for %s pods to reach valid state", updateDemoSelector)
}

type updateDemoData struct {
	image string `json:"image"`
}

func getData(podID string) (*updateDemoData, error) {
	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1beta1/proxy/pods/%s/data.json", kubectlProxyPort, podID))
	if err != nil || resp.StatusCode != 200 {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	Logf("got data: %s", body)
	var data updateDemoData
	err = json.Unmarshal(body, &data)
	return &data, err
}

func kubectlCmd(args ...string) *exec.Cmd {
	defaultArgs := []string{"--auth-path=" + testContext.authConfig}
	if testContext.certDir != "" {
		defaultArgs = append(defaultArgs,
			fmt.Sprintf("--certificate-authority=%s", filepath.Join(testContext.certDir, "ca.crt")),
			fmt.Sprintf("--client-certificate=%s", filepath.Join(testContext.certDir, "kubecfg.crt")),
			fmt.Sprintf("--client-key=%s", filepath.Join(testContext.certDir, "kubecfg.key")))
	}
	kubectlArgs := append(defaultArgs, args...)
	// TODO: Remove this once gcloud writes a proper entry in the kubeconfig file.
	if testContext.provider == "gke" {
		kubectlArgs = append(kubectlArgs, "--server="+testContext.host)
	}
	cmd := exec.Command("kubectl", kubectlArgs...)
	Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args, " "))
	return cmd
}

func runKubectl(args ...string) string {
	var stdout, stderr bytes.Buffer
	cmd := kubectlCmd(args...)
	cmd.Stdout, cmd.Stderr = &stdout, &stderr

	if err := cmd.Run(); err != nil {
		Failf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr)
		return ""
	}
	Logf(stdout.String())
	// TODO: trimspace should be unnecessary after switching to use kubectl binary directly
	return strings.TrimSpace(stdout.String())
}
<commit_msg>Increase the timeout to 10 minutes to give docker plenty of time to pull the image. We've seen increasing pull times lately causing the e2e tests to flake.<commit_after>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
)

const (
	nautilusImage       = "kubernetes/update-demo:nautilus"
	kittenImage         = "kubernetes/update-demo:kitten"
	updateDemoSelector  = "name=update-demo"
	updateDemoContainer = "update-demo"
	validateTimeout     = 10 * time.Minute // TODO: Make this 30 seconds once #4566 is resolved.
	kubectlProxyPort    = 8011
)

var _ = Describe("kubectl", func() {

	// Constants.
	var (
		updateDemoRoot = filepath.Join(testContext.repoRoot, "examples/update-demo")
		nautilusPath   = filepath.Join(updateDemoRoot, "nautilus-rc.yaml")
		kittenPath     = filepath.Join(updateDemoRoot, "kitten-rc.yaml")
	)

	var cmd *exec.Cmd

	BeforeEach(func() {
		cmd = kubectlCmd("proxy", fmt.Sprintf("--port=%d", kubectlProxyPort))
		if err := cmd.Start(); err != nil {
			Failf("Unable to start kubectl proxy: %v", err)
		}
	})

	AfterEach(func() {
		if cmd.Process == nil {
			Logf("No process associated with the kubectl proxy")
			return
		}
		// Kill the proxy
		errChan := make(chan error, 1)
		go func() {
			if err := cmd.Process.Signal(os.Interrupt); err != nil {
				errChan <- err
			}
			_, err := cmd.Process.Wait()
			errChan <- err
		}()
		select {
		case <-time.After(10 * time.Second):
			Fail("Timed out waiting for kubectl proxy process to exit")
		case err := <-errChan:
			if err != nil {
				Failf("Unable to kill kubectl proxy: %v", err)
			}
		}
	})

	It("should create and stop a replication controller", func() {
		defer cleanup(nautilusPath)

		By("creating a replication controller")
		runKubectl("create", "-f", nautilusPath)
		validateController(nautilusImage, 2)
	})

	It("should scale a replication controller", func() {
		defer cleanup(nautilusPath)

		By("creating a replication controller")
		runKubectl("create", "-f", nautilusPath)
		validateController(nautilusImage, 2)
		By("scaling down the replication controller")
		runKubectl("resize", "rc", "update-demo-nautilus", "--replicas=1")
		validateController(nautilusImage, 1)
		By("scaling up the replication controller")
		runKubectl("resize", "rc", "update-demo-nautilus", "--replicas=2")
		validateController(nautilusImage, 2)
	})

	It("should do a rolling update of a replication controller", func() {
		// Cleanup all resources in case we fail somewhere in the middle
		defer cleanup(updateDemoRoot)

		By("creating the initial replication controller")
		runKubectl("create", "-f", nautilusPath)
		validateController(nautilusImage, 2)
		By("rollingupdate to new replication controller")
		runKubectl("rollingupdate", "update-demo-nautilus", "--update-period=1s", "-f", kittenPath)
		validateController(kittenImage, 2)
	})

})

func cleanup(filePath string) {
	By("using stop to clean up resources")
	runKubectl("stop", "-f", filePath)

	resources := runKubectl("get", "pods,rc", "-l", updateDemoSelector, "--no-headers")
	if resources != "" {
		Failf("Resources left running after stop:\n%s", resources)
	}
}

func validateController(image string, replicas int) {

	getPodsTemplate := "--template={{range.items}}{{.id}} {{end}}"

	// NB: kubectl adds the "exists" function to the standard template functions.
	// This lets us check to see if the "running" entry exists for each of the containers
	// we care about. Exists will never return an error and it's safe to check a chain of
	// things, any one of which may not exist. In the below template, all of info,
	// containername, and running might be nil, so the normal index function isn't very
	// helpful.
	// This template is unit-tested in kubectl, so if you change it, update the unit test.
	//
	// You can read about the syntax here: http://golang.org/pkg/text/template/
	getContainerStateTemplate := fmt.Sprintf(`--template={{and (exists . "currentState" "info" "%s" "state" "running")}}`, updateDemoContainer)

	getImageTemplate := fmt.Sprintf(`--template={{(index .currentState.info "%s").image}}`, updateDemoContainer)

	By(fmt.Sprintf("waiting for all containers in %s pods to come up.", updateDemoSelector))
	for start := time.Now(); time.Since(start) < validateTimeout; time.Sleep(5 * time.Second) {
		getPodsOutput := runKubectl("get", "pods", "-o", "template", getPodsTemplate, "-l", updateDemoSelector)
		pods := strings.Fields(getPodsOutput)
		if numPods := len(pods); numPods != replicas {
			By(fmt.Sprintf("Replicas for %s: expected=%d actual=%d", updateDemoSelector, replicas, numPods))
			continue
		}
		var runningPods []string
		for _, podID := range pods {
			running := runKubectl("get", "pods", podID, "-o", "template", getContainerStateTemplate)
			if running == "false" {
				Logf("%s is created but not running", podID)
				continue
			}

			currentImage := runKubectl("get", "pods", podID, "-o", "template", getImageTemplate)
			if currentImage != image {
				Logf("%s is created but running wrong image; expected: %s, actual: %s", podID, image, currentImage)
				continue
			}

			data, err := getData(podID)
			if err != nil {
				Logf("%s is running right image but fetching data failed: %v", podID, err)
				continue
			}
			if strings.Contains(data.image, image) {
				Logf("%s is running right image but fetched data has the wrong info: %s", podID, data)
				continue
			}

			Logf("%s is verified up and running", podID)
			runningPods = append(runningPods, podID)
		}
		if len(runningPods) == replicas {
			return
		}
	}
	Failf("Timed out waiting for %s pods to reach valid state", updateDemoSelector)
}

type updateDemoData struct {
	image string `json:"image"`
}

func getData(podID string) (*updateDemoData, error) {
	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1beta1/proxy/pods/%s/data.json", kubectlProxyPort, podID))
	if err != nil || resp.StatusCode != 200 {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	Logf("got data: %s", body)
	var data updateDemoData
	err = json.Unmarshal(body, &data)
	return &data, err
}

func kubectlCmd(args ...string) *exec.Cmd {
	defaultArgs := []string{"--auth-path=" + testContext.authConfig}
	if testContext.certDir != "" {
		defaultArgs = append(defaultArgs,
			fmt.Sprintf("--certificate-authority=%s", filepath.Join(testContext.certDir, "ca.crt")),
			fmt.Sprintf("--client-certificate=%s", filepath.Join(testContext.certDir, "kubecfg.crt")),
			fmt.Sprintf("--client-key=%s", filepath.Join(testContext.certDir, "kubecfg.key")))
	}
	kubectlArgs := append(defaultArgs, args...)
	// TODO: Remove this once gcloud writes a proper entry in the kubeconfig file.
	if testContext.provider == "gke" {
		kubectlArgs = append(kubectlArgs, "--server="+testContext.host)
	}
	cmd := exec.Command("kubectl", kubectlArgs...)
	Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args, " "))
	return cmd
}

func runKubectl(args ...string) string {
	var stdout, stderr bytes.Buffer
	cmd := kubectlCmd(args...)
	cmd.Stdout, cmd.Stderr = &stdout, &stderr

	if err := cmd.Run(); err != nil {
		Failf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr)
		return ""
	}
	Logf(stdout.String())
	// TODO: trimspace should be unnecessary after switching to use kubectl binary directly
	return strings.TrimSpace(stdout.String())
}
<|endoftext|>


<commit_before>package test

import (
	"fmt"
	"golog"
	"io/ioutil"
	"path/filepath"
	"testing"
	"time"

	// TODO: remove this and replace with something else when whale no longer needs to use steno loggers
	"butler/mixer"
	"steno/dummy"
	tf "tritium/transform"
	tp "tritium/proto"
	"tritium/packager"
	"tritium/whale"
)

var baseDir string

const (
	ENTRY_FILE                  = "main.ts"
	INPUT_FILE					= "input.html"
	OUTPUT_FILE					= "output.html"
)

func TestExports(t *testing.T) {
	var ok bool
	baseDir, ok = relativeDirectory(filepath.Join("engine_tests", "tritium", "export_test"))
	if !ok {
		t.Fatalf("Can't find the files to test.")
	}

	mixerNames, _, err := mixer.ParseMultiMixerLock(baseDir)
	if err != nil {
		t.Fatalf(err.Error())
	}

	logger := golog.NewLogger("")
	mixers := make([]*tp.Mixer, 0)
	for _, name := range mixerNames {
		pkgr := packager.New(filepath.Join(baseDir, "mixers", name), "lib", true, logger, placeholder)
		pkgr.Build()
		mixers = append(mixers, pkgr.Mixer)
	}

	// combine mixers...
	_, mixer, _, _, err := packager.GetPkgdMixers(mixers, false)
	if err != nil {
		t.Fatalf(err.Error())
	}

	transform, err := tf.CompileString(getFileString(ENTRY_FILE, t), "", "", "", mixer.Package, make([]string, 0))
	if err != nil {
		t.Fatalf("Building the transform failed.")
	}

	env := make(map[string]string)

	debugger := &dummy.DummyDebugger{}
	eng := whale.NewEngine(debugger)
	d, _ := time.ParseDuration("1m")

	exhaust := eng.Run(transform, nil, getFileString(INPUT_FILE, t), env, time.Now().Add(d), "", "", "", false)

	if exhaust.Output != getFileString(OUTPUT_FILE, t) {
		t.Errorf("Got Unexpected output:\n" + exhaust.Output)
	}

	print(".")
}

func getFileString(filename string, t *testing.T) string {
	output, err := ioutil.ReadFile(filepath.Join(baseDir, filename))
	if err != nil {
		t.Fatalf(err.Error())
	}
	return string(output)
}

// shouldn't download or attempt to get a precompiled mixer for this test
func placeholder(name, version string) (mxr *tp.Mixer, err error) {
	return nil, fmt.Errorf("Error.")
}

<commit_msg>fixing this test<commit_after>package test

import (
	"fmt"
	"golog"
	"io/ioutil"
	"path/filepath"
	"testing"
	"time"

	// TODO: remove this and replace with something else when whale no longer needs to use steno loggers
	"butler/mixer"
	"steno/dummy"
	tf "tritium/transform"
	tp "tritium/proto"
	"tritium/packager"
	"tritium/whale"
)

var baseDir string

const (
	ENTRY_FILE                  = "main.ts"
	INPUT_FILE					= "input.html"
	OUTPUT_FILE					= "output.html"
)

func TestExports(t *testing.T) {
	var ok bool
	baseDir, ok = relativeDirectory(filepath.Join("engine_tests", "tritium", "exports"))
	if !ok {
		t.Fatalf("Can't find the files to test.")
	}
	mixerNames, _, err := mixer.ParseMultiMixerLock(baseDir)
	if err != nil {
		t.Fatalf(err.Error())
	}

	logger := golog.NewLogger("")
	mixers := make([]*tp.Mixer, 0)
	for _, name := range mixerNames {
		pkgr := packager.New(filepath.Join(baseDir, "mixers", name), "lib", true, logger, placeholder)
		pkgr.Build()
		mixers = append(mixers, pkgr.Mixer)
	}

	// combine mixers...
	_, mixer, _, _, err := packager.GetPkgdMixers(mixers, false)
	if err != nil {
		t.Fatalf(err.Error())
	}

	transform, err := tf.CompileString(getFileString(ENTRY_FILE, t), "", "", "", mixer.Package, make([]string, 0))
	if err != nil {
		t.Fatalf("Building the transform failed.")
	}

	env := make(map[string]string)

	debugger := &dummy.DummyDebugger{}
	eng := whale.NewEngine(debugger)
	d, _ := time.ParseDuration("1m")

	exhaust := eng.Run(transform, nil, getFileString(INPUT_FILE, t), env, time.Now().Add(d), "", "", "", false)

	if exhaust.Output != getFileString(OUTPUT_FILE, t) {
		t.Errorf("Got Unexpected output:\n" + exhaust.Output)
	}

	print(".")
}

func getFileString(filename string, t *testing.T) string {
	output, err := ioutil.ReadFile(filepath.Join(baseDir, filename))
	if err != nil {
		t.Fatalf(err.Error())
	}
	return string(output)
}

// shouldn't download or attempt to get a precompiled mixer for this test
func placeholder(name, version string) (mxr *tp.Mixer, err error) {
	return nil, fmt.Errorf("Error.")
}

<|endoftext|>


<commit_before>package test

import (
	"bytes"
	"fmt"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"testing"
)

// Ideas
// - run command w/ matcher
// - verify arbitrary fs layer
// - debug, replay commands/output on failure

type Cmd struct {
	// Embed a `Once` so we can call `exec` only once
	sync.Once

	// Represents the arguments to `os/exec.Command`
	command string
	args    []string

	// Holder of output, always converted to a string for
	// easier comparison and print
	output string

	// Holds the error from an execution
	err error
}

func Command(cmd string, args ...string) *Cmd {
	return &Cmd{
		command: cmd,
		args:    args,
	}
}

// Quick wrapper around platform/arch specific call to cert-manage
// which is located at ../bin/cert-manage-GOOS-GOARCH
func CertManage(args ...string) *Cmd {
	render := func(tpl string) string {
		return fmt.Sprintf(tpl, runtime.GOARCH)
	}

	switch runtime.GOOS {
	case "darwin":
		return Command(render("../bin/cert-manage-osx-%s"), args...)
	case "linux":
		return Command(render("../bin/cert-manage-linux-%s"), args...)
	case "windows":
		return Command(render("../bin/cert-manage-%s.exe"), args...)
	}
	return nil
}

func (c *Cmd) exec() {
	c.Do(func() {
		cmd := exec.Command(c.command, c.args...)

		// Set output collectors
		var stdout bytes.Buffer
		cmd.Stdout = &stdout
		var stderr bytes.Buffer
		cmd.Stderr = &stderr

		err := cmd.Run()
		if err == nil {
			c.output = stdout.String()
		} else {
			c.output = stderr.String()
		}
		c.err = err
	})
	return
}

func (c *Cmd) Trim() *Cmd {
	c.exec()
	if c.output != "" {
		c.output = strings.TrimSpace(c.output)
	}
	return c
}

func (c *Cmd) CmpInt(t *testing.T, n1 int) {
	t.Helper()
	c.exec()

	c.CmpIntF(t, func(n2 int) bool { return n1 == n2 })
}

func (c *Cmd) CmpIntF(t *testing.T, f func(int) bool) {
	t.Helper()
	c.exec()

	n, err := strconv.Atoi(c.output)
	if err != nil {
		t.Errorf("ERROR: converting '%s' to integer failed, err=%v", c.output, err)
	}
	if !f(n) {
		t.Errorf("ERROR: got %d", n)
	}
}

func (c *Cmd) FailedT(t *testing.T) {
	t.Helper()
	c.exec()

	if c.err == nil {
		t.Errorf("Expected failure, but seeig none.\n Output: %s", c.output)
	}
}

func (c *Cmd) EqualT(t *testing.T, ans string) {
	t.Helper()
	c.exec()
	if c.output != ans {
		t.Errorf("ERROR: Output did not match expected answer!\n Output: %s\n Answer: %s", c.output, ans)
	}
}

func (c *Cmd) String() string {
	c.exec()

	buf := bytes.NewBufferString(fmt.Sprintf("Command:\n  %s %s\nOutput:\n  %s", c.command, strings.Join(c.args, " "), c.output))
	if c.err != nil {
		buf.WriteString(fmt.Sprintf("\nError:\n  %v", c.err))
	}
	return buf.String()
}

func (c *Cmd) Success() bool {
	c.exec()
	return c.err == nil
}

func (c *Cmd) SuccessT(t *testing.T) {
	t.Helper()
	if !c.Success() {
		t.Errorf("Expected no error, got err=%v\n Output: %s", c.err, c.output)
	}
}

func (c *Cmd) PendingT(t *testing.T, reason string) {
	t.Helper()
	c.exec()

	if c.err == nil {
		t.Errorf("Expected failing test, got success. Pending because %s", reason)
	} else {
		t.Skip(reason)
	}
}
<commit_msg>test: quote EqualT mis-match output<commit_after>package test

import (
	"bytes"
	"fmt"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"testing"
)

// Ideas
// - run command w/ matcher
// - verify arbitrary fs layer
// - debug, replay commands/output on failure

type Cmd struct {
	// Embed a `Once` so we can call `exec` only once
	sync.Once

	// Represents the arguments to `os/exec.Command`
	command string
	args    []string

	// Holder of output, always converted to a string for
	// easier comparison and print
	output string

	// Holds the error from an execution
	err error
}

func Command(cmd string, args ...string) *Cmd {
	return &Cmd{
		command: cmd,
		args:    args,
	}
}

// Quick wrapper around platform/arch specific call to cert-manage
// which is located at ../bin/cert-manage-GOOS-GOARCH
func CertManage(args ...string) *Cmd {
	render := func(tpl string) string {
		return fmt.Sprintf(tpl, runtime.GOARCH)
	}

	switch runtime.GOOS {
	case "darwin":
		return Command(render("../bin/cert-manage-osx-%s"), args...)
	case "linux":
		return Command(render("../bin/cert-manage-linux-%s"), args...)
	case "windows":
		return Command(render("../bin/cert-manage-%s.exe"), args...)
	}
	return nil
}

func (c *Cmd) exec() {
	c.Do(func() {
		cmd := exec.Command(c.command, c.args...)

		// Set output collectors
		var stdout bytes.Buffer
		cmd.Stdout = &stdout
		var stderr bytes.Buffer
		cmd.Stderr = &stderr

		err := cmd.Run()
		if err == nil {
			c.output = stdout.String()
		} else {
			c.output = stderr.String()
		}
		c.err = err
	})
	return
}

func (c *Cmd) Trim() *Cmd {
	c.exec()
	if c.output != "" {
		c.output = strings.TrimSpace(c.output)
	}
	return c
}

func (c *Cmd) CmpInt(t *testing.T, n1 int) {
	t.Helper()
	c.exec()

	c.CmpIntF(t, func(n2 int) bool { return n1 == n2 })
}

func (c *Cmd) CmpIntF(t *testing.T, f func(int) bool) {
	t.Helper()
	c.exec()

	n, err := strconv.Atoi(c.output)
	if err != nil {
		t.Errorf("ERROR: converting '%s' to integer failed, err=%v", c.output, err)
	}
	if !f(n) {
		t.Errorf("ERROR: got %d", n)
	}
}

func (c *Cmd) FailedT(t *testing.T) {
	t.Helper()
	c.exec()

	if c.err == nil {
		t.Errorf("Expected failure, but seeig none.\n Output: %s", c.output)
	}
}

func (c *Cmd) EqualT(t *testing.T, ans string) {
	t.Helper()
	c.exec()
	if c.output != ans {
		t.Errorf("ERROR: Output did not match expected answer!\n Output: %q\n Answer: %q", c.output, ans)
	}
}

func (c *Cmd) String() string {
	c.exec()

	buf := bytes.NewBufferString(fmt.Sprintf("Command:\n  %s %s\nOutput:\n  %s", c.command, strings.Join(c.args, " "), c.output))
	if c.err != nil {
		buf.WriteString(fmt.Sprintf("\nError:\n  %v", c.err))
	}
	return buf.String()
}

func (c *Cmd) Success() bool {
	c.exec()
	return c.err == nil
}

func (c *Cmd) SuccessT(t *testing.T) {
	t.Helper()
	if !c.Success() {
		t.Errorf("Expected no error, got err=%v\n Output: %s", c.err, c.output)
	}
}

func (c *Cmd) PendingT(t *testing.T, reason string) {
	t.Helper()
	c.exec()

	if c.err == nil {
		t.Errorf("Expected failing test, got success. Pending because %s", reason)
	} else {
		t.Skip(reason)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestSomething(t *testing.T) {
	assert := assert.New(t)

	// assert equality
	assert.Equal(123, 123, "they should be equal")

	// assert inequality
	assert.NotEqual(123, 456, "they should not be equal")
}
<commit_msg>init tests and fix builds<commit_after>package test

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestSomething(t *testing.T) {
	ass := assert.New(t)

	// assert equality
	ass.Equal(123, 123, "they should be equal")

	// assert inequality
	ass.NotEqual(123, 456, "they should not be equal")
}

func TestAverage(t *testing.T) {

}
<|endoftext|>


<commit_before>/*
Copyright 2015 Dariusz Górecki <darek.krk@gmail.com>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package matcher

import (
	"fmt"
	"regexp"
	"strings"
	"sync"

	"github.com/canni/paperboymq/amq"
)

// Direct matcher matches if both message routing key and binding key are equal.
var Direct = amq.MatchFunc(func(msg amq.Message, binding amq.Binding) bool {
	return msg.RoutingKey() == binding.BindingKey()
})

// Fanout matcher always matches
var Fanout = amq.MatchFunc(func(msg amq.Message, binding amq.Binding) bool {
	return true
})

var patternCache = struct {
	cache map[string]*regexp.Regexp
	sync.RWMutex
}{
	cache: make(map[string]*regexp.Regexp),
}

// Topic matcher matches when routing key matches binding pattern, see AMQP
// specification for detailed information. There is no need to rewrite all
// rules here.
var Topic = amq.MatchFunc(func(msg amq.Message, binding amq.Binding) bool {
	patternCache.RLock()
	defer patternCache.RUnlock()

	if matcher, found := patternCache.cache[binding.BindingKey()]; found {
		return matcher.MatchString(msg.RoutingKey())
	} else {
		pattern := strings.Replace(binding.BindingKey(), ".", `\.`, -1)
		pattern = strings.Replace(pattern, "*", `[0-9A-z]+`, -1)
		pattern = strings.Replace(pattern, `\.#\.`, `(\.|\.[0-9A-z\.]*\.)`, -1)
		pattern = strings.Replace(pattern, `\.#`, `(\.[0-9A-z\.]*)?`, -1)
		pattern = strings.Replace(pattern, `#\.`, `([0-9A-z\.]*\.)?`, -1)
		pattern = strings.Replace(pattern, "#", `[0-9A-z\.]*`, -1)

		matcher := regexp.MustCompile(fmt.Sprintf("^%s$", pattern))

		go func() {
			patternCache.Lock()
			defer patternCache.Unlock()

			patternCache.cache[binding.BindingKey()] = matcher
		}()

		return matcher.MatchString(msg.RoutingKey())
	}
})
<commit_msg>Add desc<commit_after>/*
Copyright 2015 Dariusz Górecki <darek.krk@gmail.com>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package matcher providers implementations for most basic AMQ match strategies.
package matcher

import (
	"fmt"
	"regexp"
	"strings"
	"sync"

	"github.com/canni/paperboymq/amq"
)

// Direct matcher matches if both message routing key and binding key are equal.
var Direct = amq.MatchFunc(func(msg amq.Message, binding amq.Binding) bool {
	return msg.RoutingKey() == binding.BindingKey()
})

// Fanout matcher always matches
var Fanout = amq.MatchFunc(func(msg amq.Message, binding amq.Binding) bool {
	return true
})

var patternCache = struct {
	cache map[string]*regexp.Regexp
	sync.RWMutex
}{
	cache: make(map[string]*regexp.Regexp),
}

// Topic matcher matches when routing key matches binding pattern, see AMQP
// specification for detailed information. There is no need to rewrite all
// rules here.
var Topic = amq.MatchFunc(func(msg amq.Message, binding amq.Binding) bool {
	patternCache.RLock()
	defer patternCache.RUnlock()

	if matcher, found := patternCache.cache[binding.BindingKey()]; found {
		return matcher.MatchString(msg.RoutingKey())
	} else {
		pattern := strings.Replace(binding.BindingKey(), ".", `\.`, -1)
		pattern = strings.Replace(pattern, "*", `[0-9A-z]+`, -1)
		pattern = strings.Replace(pattern, `\.#\.`, `(\.|\.[0-9A-z\.]*\.)`, -1)
		pattern = strings.Replace(pattern, `\.#`, `(\.[0-9A-z\.]*)?`, -1)
		pattern = strings.Replace(pattern, `#\.`, `([0-9A-z\.]*\.)?`, -1)
		pattern = strings.Replace(pattern, "#", `[0-9A-z\.]*`, -1)

		matcher := regexp.MustCompile(fmt.Sprintf("^%s$", pattern))

		go func() {
			patternCache.Lock()
			defer patternCache.Unlock()

			patternCache.cache[binding.BindingKey()] = matcher
		}()

		return matcher.MatchString(msg.RoutingKey())
	}
})
<|endoftext|>


<commit_before>package query

import (
	"database/sql"
	"fmt"
	ggproto "github.com/gogo/protobuf/proto"
	pb "github.com/mediachain/concat/proto"
	"strings"
)

// A RowSelector extracts the next value from an sql result set
type RowSelector interface {
	Scan(src RowScanner) (interface{}, error)
}

// Common Scan interface between sql.Row and sql.Rows
type RowScanner interface {
	Scan(res ...interface{}) error
}

type QueryCompileError string

func (e QueryCompileError) Error() string {
	return string(e)
}

// CompileQuery compiles a query to sql.
// Returns the compiled sql query and a selector for extracting
// values from an sql result set
// Note: The row selector should be used in single-threaded context
func CompileQuery(q *Query) (string, RowSelector, error) {
	var sqlq string
	switch {
	case isStatementQuery(q):
		sqlq = "SELECT %s FROM Statement"
	case isEnvelopeQuery(q):
		sqlq = "SELECT %s FROM Envelope"
	default:
		sqlq = "SELECT %s FROM Statement JOIN Envelope ON Statement.id = Envelope.id"
	}

	cols, err := compileQueryColumns(q)
	if err != nil {
		return "", nil, err
	}
	sqlq = fmt.Sprintf(sqlq, cols)

	crit, err := compileQueryCriteria(q)
	if err != nil {
		return "", nil, err
	}
	if crit != "" {
		sqlq = fmt.Sprintf("%s WHERE %s", sqlq, crit)
	}

	if q.limit > 0 {
		sqlq = fmt.Sprintf("%s LIMIT %d", sqlq, q.limit)
	}

	rsel, err := compileQueryRowSelector(q)
	if err != nil {
		return "", nil, err
	}

	return sqlq, rsel, nil
}

func compileQueryColumns(q *Query) (string, error) {
	switch sel := q.selector.(type) {
	case SimpleSelector:
		return selectorColumn(sel, selectorColumnSimple), nil

	case CompoundSelector:
		if len(sel) == 1 {
			return selectorColumn(sel[0], selectorColumnSimple), nil
		}

		cols := make([]string, len(sel))
		for x := 0; x < len(sel); x++ {
			cols[x] = selectorColumn(sel[x], selectorColumnCompound)
		}
		return strings.Join(cols, ", "), nil

	case *FunctionSelector:
		col := selectorColumn(sel.sel, selectorColumnFun)
		return fmt.Sprintf("%s(%s)", sel.op, col), nil

	default:
		return "", QueryCompileError(fmt.Sprintf("Unexpected selector type: %T", sel))
	}
}

func selectorColumn(sel SimpleSelector, rename map[string]string) string {
	col, ok := rename[string(sel)]
	if !ok {
		col = string(sel)
	}
	return col
}

var selectorColumnSimple = map[string]string{
	"*":         "data",
	"body":      "data",
	"namespace": "DISTINCT namespace",
	"publisher": "DISTINCT publisher",
	"source":    "DISTINCT source"}

var selectorColumnCompound = map[string]string{
	"*":    "data",
	"body": "data"}

var selectorColumnFun = map[string]string{
	"*":         "*",
	"body":      "data",
	"namespace": "DISTINCT namespace",
	"publisher": "DISTINCT publisher",
	"source":    "DISTINCT source"}

func compileQueryCriteria(q *Query) (string, error) {
	nscrit := compileNamespaceCriteria(q.namespace)
	if q.criteria == nil {
		return nscrit, nil
	}

	scrit, err := compileSelectorCriteria(q.criteria)
	if err != nil {
		return "", err
	}

	if nscrit != "" {
		return fmt.Sprintf("%s AND %s", nscrit, scrit), nil
	}
	return scrit, nil
}

func compileNamespaceCriteria(ns string) string {
	switch {
	case ns == "*":
		return ""
	case ns[len(ns)-1] == '*':
		pre := ns[:len(ns)-2]
		return fmt.Sprintf("namespace LIKE '%s%%'", pre)
	default:
		return fmt.Sprintf("namespace = '%s'", ns)
	}
}

func compileSelectorCriteria(c QueryCriteria) (string, error) {
	switch c := c.(type) {
	case *ValueCriteria:
		return fmt.Sprintf("%s %s '%s'", c.sel, c.op, c.val), nil

	case *TimeCriteria:
		return fmt.Sprintf("timestamp %s %d", c.op, c.ts), nil

	case *CompoundCriteria:
		left, err := compileSelectorCriteria(c.left)
		if err != nil {
			return "", err
		}

		right, err := compileSelectorCriteria(c.right)
		if err != nil {
			return "", err
		}

		return fmt.Sprintf("(%s %s %s)", left, c.op, right), nil

	case *NegatedCriteria:
		expr, err := compileSelectorCriteria(c.e)
		if err != nil {
			return "", err
		}

		return fmt.Sprintf("NOT %s", expr), nil

	default:
		return "", QueryCompileError(fmt.Sprintf("Unexpected criteria type: %T", c))
	}
}

func compileQueryRowSelector(q *Query) (RowSelector, error) {
	switch sel := q.selector.(type) {
	case SimpleSelector:
		makef, ok := makeSimpleRowSelector[string(sel)]
		if !ok {
			return nil, QueryCompileError(fmt.Sprintf("Unexpected selector: %s", sel))
		}

		return makef(string(sel)), nil

	case CompoundSelector:
		return nil, QueryCompileError("Implement me!")

	case *FunctionSelector:
		return nil, QueryCompileError("Implement me!")

	default:
		return nil, QueryCompileError(fmt.Sprintf("Unexpected selector type: %T", sel))
	}
}

type MakeSimpleRowSelector func(sel string) SimpleRowSelector

type SimpleRowSelector interface {
	RowSelector
	sel() string
	ptr() interface{}
	value() (interface{}, error)
}

type RowSelectBase struct {
	id string
}

func (rs *RowSelectBase) sel() string {
	return rs.id
}

type RowSelectStatement struct {
	RowSelectBase
	val sql.RawBytes
}

func (rs *RowSelectStatement) ptr() interface{} {
	return &rs.val
}

func (rs *RowSelectStatement) value() (interface{}, error) {
	stmt := new(pb.Statement)
	err := ggproto.Unmarshal(rs.val, stmt)
	if err != nil {
		return nil, err
	}
	return stmt, nil
}

func (rs *RowSelectStatement) Scan(src RowScanner) (interface{}, error) {
	err := src.Scan(&rs.val)
	if err != nil {
		return nil, err
	}
	return rs.value()
}

type RowSelectBody struct {
	RowSelectStatement
}

func (rs *RowSelectBody) value() (interface{}, error) {
	stmt, err := rs.RowSelectStatement.value()
	if err != nil {
		return nil, err
	}

	return stmt.(*pb.Statement).Body, nil
}

func (rs *RowSelectBody) Scan(src RowScanner) (interface{}, error) {
	err := src.Scan(&rs.val)
	if err != nil {
		return nil, err
	}
	return rs.value()
}

type RowSelectString struct {
	RowSelectBase
	val string
}

func (rs *RowSelectString) ptr() interface{} {
	return &rs.val
}

func (rs *RowSelectString) value() (interface{}, error) {
	return rs.val, nil
}

func (rs *RowSelectString) Scan(src RowScanner) (interface{}, error) {
	err := src.Scan(&rs.val)
	if err != nil {
		return nil, err
	}
	return rs.val, nil
}

type RowSelectInt64 struct {
	RowSelectBase
	val int64
}

func (rs *RowSelectInt64) ptr() interface{} {
	return &rs.val
}

func (rs *RowSelectInt64) value() (interface{}, error) {
	return rs.val, nil
}

func (rs *RowSelectInt64) Scan(src RowScanner) (interface{}, error) {
	err := src.Scan(&rs.val)
	if err != nil {
		return nil, err
	}
	return rs.val, nil
}

func makeRowSelectStatement(sel string) SimpleRowSelector {
	return &RowSelectStatement{RowSelectBase: RowSelectBase{id: sel}}
}

func makeRowSelectBody(sel string) SimpleRowSelector {
	return &RowSelectBody{RowSelectStatement: RowSelectStatement{RowSelectBase: RowSelectBase{id: sel}}}
}

func makeRowSelectString(sel string) SimpleRowSelector {
	return &RowSelectString{RowSelectBase: RowSelectBase{id: sel}}
}

func makeRowSelectInt64(sel string) SimpleRowSelector {
	return &RowSelectInt64{RowSelectBase: RowSelectBase{id: sel}}
}

var makeSimpleRowSelector = map[string]MakeSimpleRowSelector{
	"*":         makeRowSelectStatement,
	"body":      makeRowSelectBody,
	"id":        makeRowSelectString,
	"namespace": makeRowSelectString,
	"publisher": makeRowSelectString,
	"source":    makeRowSelectString,
	"timestamp": makeRowSelectInt64}

func isStatementQuery(q *Query) bool {
	// namespace = * and only has statement selector (*, id, body) and id criteria
	// id acts as statement column
	return q.namespace == "*" &&
		isStatementSelector(q.selector) &&
		(q.criteria == nil || isStatementCriteria(q.criteria))
}

func isEnvelopeQuery(q *Query) bool {
	// namespace != * or envelope selector or any criteria
	// id acts as envelope column
	return q.namespace != "*" ||
		q.criteria != nil ||
		isEnvelopeSelector(q.selector)
}

var statementSelectorp = map[string]bool{
	"*":    true,
	"body": true,
	"id":   true}

var envelopeSelectorp = map[string]bool{
	"id":        true,
	"publisher": true,
	"namespace": true,
	"source":    true,
	"timestamp": true}

func selectorp(sel QuerySelector, tbl map[string]bool) bool {
	switch sel := sel.(type) {
	case SimpleSelector:
		return tbl[string(sel)]

	case CompoundSelector:
		for _, ssel := range sel {
			if tbl[string(ssel)] {
				return true
			}
		}
		return false

	case *FunctionSelector:
		return tbl[string(sel.sel)]

	default:
		return false
	}
}

func isStatementSelector(sel QuerySelector) bool {
	return selectorp(sel, statementSelectorp)
}

func isEnvelopeSelector(sel QuerySelector) bool {
	return selectorp(sel, envelopeSelectorp)
}

func isStatementCriteria(c QueryCriteria) bool {
	switch c := c.(type) {
	case *ValueCriteria:
		return c.sel == "id"

	case *TimeCriteria:
		return false

	case *CompoundCriteria:
		return isStatementCriteria(c.left) && isStatementCriteria(c.right)

	case *NegatedCriteria:
		return isStatementCriteria(c.e)

	default:
		return false
	}
}
<commit_msg>query compile: drop RowSelectBase, not needed<commit_after>package query

import (
	"database/sql"
	"fmt"
	ggproto "github.com/gogo/protobuf/proto"
	pb "github.com/mediachain/concat/proto"
	"strings"
)

// A RowSelector extracts the next value from an sql result set
type RowSelector interface {
	Scan(src RowScanner) (interface{}, error)
}

// Common Scan interface between sql.Row and sql.Rows
type RowScanner interface {
	Scan(res ...interface{}) error
}

type QueryCompileError string

func (e QueryCompileError) Error() string {
	return string(e)
}

// CompileQuery compiles a query to sql.
// Returns the compiled sql query and a selector for extracting
// values from an sql result set
// Note: The row selector should be used in single-threaded context
func CompileQuery(q *Query) (string, RowSelector, error) {
	var sqlq string
	switch {
	case isStatementQuery(q):
		sqlq = "SELECT %s FROM Statement"
	case isEnvelopeQuery(q):
		sqlq = "SELECT %s FROM Envelope"
	default:
		sqlq = "SELECT %s FROM Statement JOIN Envelope ON Statement.id = Envelope.id"
	}

	cols, err := compileQueryColumns(q)
	if err != nil {
		return "", nil, err
	}
	sqlq = fmt.Sprintf(sqlq, cols)

	crit, err := compileQueryCriteria(q)
	if err != nil {
		return "", nil, err
	}
	if crit != "" {
		sqlq = fmt.Sprintf("%s WHERE %s", sqlq, crit)
	}

	if q.limit > 0 {
		sqlq = fmt.Sprintf("%s LIMIT %d", sqlq, q.limit)
	}

	rsel, err := compileQueryRowSelector(q)
	if err != nil {
		return "", nil, err
	}

	return sqlq, rsel, nil
}

func compileQueryColumns(q *Query) (string, error) {
	switch sel := q.selector.(type) {
	case SimpleSelector:
		return selectorColumn(sel, selectorColumnSimple), nil

	case CompoundSelector:
		if len(sel) == 1 {
			return selectorColumn(sel[0], selectorColumnSimple), nil
		}

		cols := make([]string, len(sel))
		for x := 0; x < len(sel); x++ {
			cols[x] = selectorColumn(sel[x], selectorColumnCompound)
		}
		return strings.Join(cols, ", "), nil

	case *FunctionSelector:
		col := selectorColumn(sel.sel, selectorColumnFun)
		return fmt.Sprintf("%s(%s)", sel.op, col), nil

	default:
		return "", QueryCompileError(fmt.Sprintf("Unexpected selector type: %T", sel))
	}
}

func selectorColumn(sel SimpleSelector, rename map[string]string) string {
	col, ok := rename[string(sel)]
	if !ok {
		col = string(sel)
	}
	return col
}

var selectorColumnSimple = map[string]string{
	"*":         "data",
	"body":      "data",
	"namespace": "DISTINCT namespace",
	"publisher": "DISTINCT publisher",
	"source":    "DISTINCT source"}

var selectorColumnCompound = map[string]string{
	"*":    "data",
	"body": "data"}

var selectorColumnFun = map[string]string{
	"*":         "*",
	"body":      "data",
	"namespace": "DISTINCT namespace",
	"publisher": "DISTINCT publisher",
	"source":    "DISTINCT source"}

func compileQueryCriteria(q *Query) (string, error) {
	nscrit := compileNamespaceCriteria(q.namespace)
	if q.criteria == nil {
		return nscrit, nil
	}

	scrit, err := compileSelectorCriteria(q.criteria)
	if err != nil {
		return "", err
	}

	if nscrit != "" {
		return fmt.Sprintf("%s AND %s", nscrit, scrit), nil
	}
	return scrit, nil
}

func compileNamespaceCriteria(ns string) string {
	switch {
	case ns == "*":
		return ""
	case ns[len(ns)-1] == '*':
		pre := ns[:len(ns)-2]
		return fmt.Sprintf("namespace LIKE '%s%%'", pre)
	default:
		return fmt.Sprintf("namespace = '%s'", ns)
	}
}

func compileSelectorCriteria(c QueryCriteria) (string, error) {
	switch c := c.(type) {
	case *ValueCriteria:
		return fmt.Sprintf("%s %s '%s'", c.sel, c.op, c.val), nil

	case *TimeCriteria:
		return fmt.Sprintf("timestamp %s %d", c.op, c.ts), nil

	case *CompoundCriteria:
		left, err := compileSelectorCriteria(c.left)
		if err != nil {
			return "", err
		}

		right, err := compileSelectorCriteria(c.right)
		if err != nil {
			return "", err
		}

		return fmt.Sprintf("(%s %s %s)", left, c.op, right), nil

	case *NegatedCriteria:
		expr, err := compileSelectorCriteria(c.e)
		if err != nil {
			return "", err
		}

		return fmt.Sprintf("NOT %s", expr), nil

	default:
		return "", QueryCompileError(fmt.Sprintf("Unexpected criteria type: %T", c))
	}
}

func compileQueryRowSelector(q *Query) (RowSelector, error) {
	switch sel := q.selector.(type) {
	case SimpleSelector:
		makef, ok := makeSimpleRowSelector[string(sel)]
		if !ok {
			return nil, QueryCompileError(fmt.Sprintf("Unexpected selector: %s", sel))
		}

		return makef(), nil

	case CompoundSelector:
		return nil, QueryCompileError("Implement me!")

	case *FunctionSelector:
		return nil, QueryCompileError("Implement me!")

	default:
		return nil, QueryCompileError(fmt.Sprintf("Unexpected selector type: %T", sel))
	}
}

type MakeSimpleRowSelector func() SimpleRowSelector

type SimpleRowSelector interface {
	RowSelector
	ptr() interface{}
	value() (interface{}, error)
}

type RowSelectStatement struct {
	val sql.RawBytes
}

func (rs *RowSelectStatement) ptr() interface{} {
	return &rs.val
}

func (rs *RowSelectStatement) value() (interface{}, error) {
	stmt := new(pb.Statement)
	err := ggproto.Unmarshal(rs.val, stmt)
	if err != nil {
		return nil, err
	}
	return stmt, nil
}

func (rs *RowSelectStatement) Scan(src RowScanner) (interface{}, error) {
	err := src.Scan(&rs.val)
	if err != nil {
		return nil, err
	}
	return rs.value()
}

type RowSelectBody struct {
	RowSelectStatement
}

func (rs *RowSelectBody) value() (interface{}, error) {
	stmt, err := rs.RowSelectStatement.value()
	if err != nil {
		return nil, err
	}

	return stmt.(*pb.Statement).Body, nil
}

func (rs *RowSelectBody) Scan(src RowScanner) (interface{}, error) {
	err := src.Scan(&rs.val)
	if err != nil {
		return nil, err
	}
	return rs.value()
}

type RowSelectString struct {
	val string
}

func (rs *RowSelectString) ptr() interface{} {
	return &rs.val
}

func (rs *RowSelectString) value() (interface{}, error) {
	return rs.val, nil
}

func (rs *RowSelectString) Scan(src RowScanner) (interface{}, error) {
	err := src.Scan(&rs.val)
	if err != nil {
		return nil, err
	}
	return rs.val, nil
}

type RowSelectInt64 struct {
	val int64
}

func (rs *RowSelectInt64) ptr() interface{} {
	return &rs.val
}

func (rs *RowSelectInt64) value() (interface{}, error) {
	return rs.val, nil
}

func (rs *RowSelectInt64) Scan(src RowScanner) (interface{}, error) {
	err := src.Scan(&rs.val)
	if err != nil {
		return nil, err
	}
	return rs.val, nil
}

func makeRowSelectStatement() SimpleRowSelector {
	return &RowSelectStatement{}
}

func makeRowSelectBody() SimpleRowSelector {
	return &RowSelectBody{}
}

func makeRowSelectString() SimpleRowSelector {
	return &RowSelectString{}
}

func makeRowSelectInt64() SimpleRowSelector {
	return &RowSelectInt64{}
}

var makeSimpleRowSelector = map[string]MakeSimpleRowSelector{
	"*":         makeRowSelectStatement,
	"body":      makeRowSelectBody,
	"id":        makeRowSelectString,
	"namespace": makeRowSelectString,
	"publisher": makeRowSelectString,
	"source":    makeRowSelectString,
	"timestamp": makeRowSelectInt64}

func isStatementQuery(q *Query) bool {
	// namespace = * and only has statement selector (*, id, body) and id criteria
	// id acts as statement column
	return q.namespace == "*" &&
		isStatementSelector(q.selector) &&
		(q.criteria == nil || isStatementCriteria(q.criteria))
}

func isEnvelopeQuery(q *Query) bool {
	// namespace != * or envelope selector or any criteria
	// id acts as envelope column
	return q.namespace != "*" ||
		q.criteria != nil ||
		isEnvelopeSelector(q.selector)
}

var statementSelectorp = map[string]bool{
	"*":    true,
	"body": true,
	"id":   true}

var envelopeSelectorp = map[string]bool{
	"id":        true,
	"publisher": true,
	"namespace": true,
	"source":    true,
	"timestamp": true}

func selectorp(sel QuerySelector, tbl map[string]bool) bool {
	switch sel := sel.(type) {
	case SimpleSelector:
		return tbl[string(sel)]

	case CompoundSelector:
		for _, ssel := range sel {
			if tbl[string(ssel)] {
				return true
			}
		}
		return false

	case *FunctionSelector:
		return tbl[string(sel.sel)]

	default:
		return false
	}
}

func isStatementSelector(sel QuerySelector) bool {
	return selectorp(sel, statementSelectorp)
}

func isEnvelopeSelector(sel QuerySelector) bool {
	return selectorp(sel, envelopeSelectorp)
}

func isStatementCriteria(c QueryCriteria) bool {
	switch c := c.(type) {
	case *ValueCriteria:
		return c.sel == "id"

	case *TimeCriteria:
		return false

	case *CompoundCriteria:
		return isStatementCriteria(c.left) && isStatementCriteria(c.right)

	case *NegatedCriteria:
		return isStatementCriteria(c.e)

	default:
		return false
	}
}
<|endoftext|>


<commit_before>package md2man

import (
	"bytes"
	"testing"
)

func TestTitleBlock(t *testing.T) {
	r := &roffRenderer{}
	buf := bytes.NewBuffer(nil)

	title := []byte("% stuff\n% more stuff\n% even more stuff")

	r.TitleBlock(buf, title)
	expected := ".TH \"stuff\" \"more stuff\" \"even more stuff\" \n.nh\n.ad l\n"
	actual := buf.String()
	if expected != actual {
		t.Fatalf("expected:\n%s\nactual:\n%s", expected, actual)
	}
}

func TestBlockCode(t *testing.T) {
	r := &roffRenderer{}
	buf := bytes.NewBuffer(nil)

	code := []byte("$ echo hello world\nhello world\n")
	r.BlockCode(buf, code, "")

	expected := `
.PP
.RS

.nf
$ echo hello world
hello world

.fi
.RE
`
	result := buf.String()
	if expected != result {
		t.Fatalf("got incorrect output:\nexpected:\n%v\n\ngot:\n%v", expected, result)
	}
}

func TestTableCell(t *testing.T) {
	r := &roffRenderer{}
	buf := bytes.NewBuffer(nil)
	cell := []byte{}

	r.TableCell(buf, cell, 0)
	expected := " "
	if buf.String() != expected {
		t.Fatal("expected %q, got %q", expected, buf.String())
	}

	r.TableCell(buf, cell, 0)
	expected += "\t "
	if buf.String() != expected {
		t.Fatalf("expected %q, got %q", expected, buf.String())
	}

	cell = []byte("*")
	r.TableCell(buf, cell, 0)
	expected += "\t*"
	if buf.String() != expected {
		t.Fatalf("expected %q, got %q", expected, buf.String())
	}

	cell = []byte("this is a test with some really long string")
	r.TableCell(buf, cell, 0)
	expected += "\tT{\nthis is a test with some really long string\nT}"
	if buf.String() != expected {
		t.Fatalf("expected %q, got %q", expected, buf.String())
	}

	cell = []byte("some short string")
	r.TableCell(buf, cell, 0)
	expected += "\tsome short string"
	if buf.String() != expected {
		t.Fatalf("expected %q, got %q", expected, buf.String())
	}

	cell = []byte{}
	r.TableCell(buf, cell, 0)
	expected += "\t "
	if buf.String() != expected {
		t.Fatalf("expected %q, got %q", expected, buf.String())
	}

	cell = []byte("*")
	r.TableCell(buf, cell, 0)
	expected += "\t*"
	if buf.String() != expected {
		t.Fatalf("expected %q, got %q", expected, buf.String())
	}
}
<commit_msg>Fix falal -> fatalf<commit_after>package md2man

import (
	"bytes"
	"testing"
)

func TestTitleBlock(t *testing.T) {
	r := &roffRenderer{}
	buf := bytes.NewBuffer(nil)

	title := []byte("% stuff\n% more stuff\n% even more stuff")

	r.TitleBlock(buf, title)
	expected := ".TH \"stuff\" \"more stuff\" \"even more stuff\" \n.nh\n.ad l\n"
	actual := buf.String()
	if expected != actual {
		t.Fatalf("expected:\n%s\nactual:\n%s", expected, actual)
	}
}

func TestBlockCode(t *testing.T) {
	r := &roffRenderer{}
	buf := bytes.NewBuffer(nil)

	code := []byte("$ echo hello world\nhello world\n")
	r.BlockCode(buf, code, "")

	expected := `
.PP
.RS

.nf
$ echo hello world
hello world

.fi
.RE
`
	result := buf.String()
	if expected != result {
		t.Fatalf("got incorrect output:\nexpected:\n%v\n\ngot:\n%v", expected, result)
	}
}

func TestTableCell(t *testing.T) {
	r := &roffRenderer{}
	buf := bytes.NewBuffer(nil)
	cell := []byte{}

	r.TableCell(buf, cell, 0)
	expected := " "
	if buf.String() != expected {
		t.Fatalf("expected %q, got %q", expected, buf.String())
	}

	r.TableCell(buf, cell, 0)
	expected += "\t "
	if buf.String() != expected {
		t.Fatalf("expected %q, got %q", expected, buf.String())
	}

	cell = []byte("*")
	r.TableCell(buf, cell, 0)
	expected += "\t*"
	if buf.String() != expected {
		t.Fatalf("expected %q, got %q", expected, buf.String())
	}

	cell = []byte("this is a test with some really long string")
	r.TableCell(buf, cell, 0)
	expected += "\tT{\nthis is a test with some really long string\nT}"
	if buf.String() != expected {
		t.Fatalf("expected %q, got %q", expected, buf.String())
	}

	cell = []byte("some short string")
	r.TableCell(buf, cell, 0)
	expected += "\tsome short string"
	if buf.String() != expected {
		t.Fatalf("expected %q, got %q", expected, buf.String())
	}

	cell = []byte{}
	r.TableCell(buf, cell, 0)
	expected += "\t "
	if buf.String() != expected {
		t.Fatalf("expected %q, got %q", expected, buf.String())
	}

	cell = []byte("*")
	r.TableCell(buf, cell, 0)
	expected += "\t*"
	if buf.String() != expected {
		t.Fatalf("expected %q, got %q", expected, buf.String())
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Datacratic. All rights reserved.

package metric

import (
	"sync"
	"sync/atomic"
)

type Aggregate struct {
	Counts map[string]int64
	Commit func(counts map[string]int64) bool

	once sync.Once
	mu   sync.RWMutex
}

func (a *Aggregate) NewWriter(s *Summary) Writer {
	a.once.Do(func() {
		a.Counts = make(map[string]int64)
	})

	return &aggregateWriter{a: a}
}

type aggregateWriter struct {
	a *Aggregate
}

func (w *aggregateWriter) Write(name string, value float64) error {
	return ErrIgnored
}

func (w *aggregateWriter) WriteScaled(name string, value float64) (err error) {
	n := int64(value)

	w.a.mu.RLock()
	count, ok := w.a.Counts[name]
	w.a.mu.RUnlock()

	if ok {
		atomic.AddInt64(&count, n)
	} else {
		w.a.mu.Lock()
		w.a.Counts[name] = n
		w.a.mu.Unlock()
	}

	return ErrIgnored
}

func (w *aggregateWriter) WriteString(name, text string) (err error) {
	return ErrIgnored
}

func (w *aggregateWriter) Close() {
	w.a.mu.Lock()

	if w.a.Commit != nil {
		done := w.a.Commit(w.a.Counts)
		if done {
			w.a.Counts = make(map[string]int64)
		}
	}

	w.a.mu.Unlock()
}
<commit_msg>fixed aggregation<commit_after>// Copyright (c) 2015 Datacratic. All rights reserved.

package metric

import "sync"

type Aggregate struct {
	Counts map[string]int64
	Commit func(counts map[string]int64) bool

	once sync.Once
	mu   sync.Mutex
}

func (a *Aggregate) NewWriter(s *Summary) Writer {
	a.once.Do(func() {
		a.Counts = make(map[string]int64)
	})

	return &aggregateWriter{a: a}
}

type aggregateWriter struct {
	a *Aggregate
}

func (w *aggregateWriter) Write(name string, value float64) error {
	return ErrIgnored
}

func (w *aggregateWriter) WriteScaled(name string, value float64) (err error) {
	n := int64(value)

	w.a.mu.Lock()
	w.a.Counts[name] = w.a.Counts[name] + n
	w.a.mu.Unlock()

	return ErrIgnored
}

func (w *aggregateWriter) WriteString(name, text string) (err error) {
	return ErrIgnored
}

func (w *aggregateWriter) Close() {
	w.a.mu.Lock()

	if w.a.Commit != nil {
		done := w.a.Commit(w.a.Counts)
		if done {
			w.a.Counts = make(map[string]int64)
		}
	}

	w.a.mu.Unlock()
}
<|endoftext|>


<commit_before>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package go_kafka_client

import (
	"encoding/json"
	"fmt"
	"github.com/rcrowley/go-metrics"
	"github.com/stealthly/go-avro"
	avroline "github.com/stealthly/go_kafka_client/avro"
	"regexp"
	"strings"
	"time"
)

type MetricsEmitter interface {
	//    SetRegistry(metrics.Registry)
	ReportingInterval() time.Duration
	Emit([]byte)
	//    Start()
}

type KafkaMetricsEmitterConfig struct {
	Registry metrics.Registry

	ReportingInterval time.Duration

	// Topic to emit logs to.
	Topic string

	// Confluent Avro schema registry URL.
	SchemaRegistryUrl string

	// Producer config that will be used by this emitter. Note that ValueEncoder WILL BE replaced by KafkaAvroEncoder.
	ProducerConfig *ProducerConfig
}

func NewKafkaMetricsEmitterConfig() *KafkaMetricsEmitterConfig {
	return new(KafkaMetricsEmitterConfig)
}

type KafkaMetricsEmitter struct {
	config   *KafkaMetricsEmitterConfig
	producer Producer
}

func NewKafkaMetricsEmitter(config *KafkaMetricsEmitterConfig) *KafkaMetricsEmitter {
	config.ProducerConfig.ValueEncoder = NewKafkaAvroEncoder(config.SchemaRegistryUrl)
	return &KafkaMetricsEmitter{
		config:   config,
		producer: NewSaramaProducer(config.ProducerConfig),
	}
}

func (m *KafkaMetricsEmitter) ReportingInterval() time.Duration {
	return m.config.ReportingInterval
}

func (m *KafkaMetricsEmitter) Emit(bytes []byte) {
	metrics := make(map[string]interface{})
	if err := json.Unmarshal(bytes, &metrics); err != nil {
		panic(err)
	}

	schema, lookupNames := m.parseSchema(metrics)
	record := avro.NewGenericRecord(schema)
	m.fillRecord(record, schema.(*avro.RecordSchema), metrics, lookupNames)

	m.producer.Input() <- &ProducerMessage{Topic: m.config.Topic, Value: record}
}

func (m *KafkaMetricsEmitter) parseSchema(metrics map[string]interface{}) (avro.Schema, map[string]string) {
	lookupNames := make(map[string]string)
	schema := &avro.RecordSchema{
		Name:      "Metrics",
		Namespace: "ly.stealth",
		Fields:    make([]*avro.SchemaField, 0),
	}
	for name, v := range metrics {
		lookupName := m.getLookupName(name)
		lookupNames[lookupName] = name

		field := &avro.SchemaField{
			Name:    lookupName,
			Default: nil,
			Type: &avro.UnionSchema{
				Types: []avro.Schema{&avro.NullSchema{}, m.parseRecordField(name, v.(map[string]interface{}), lookupNames)},
			},
		}
		schema.Fields = append(schema.Fields, field)
	}
	schema.Fields = append(schema.Fields, &avro.SchemaField{
		Name: "logLine",
		Type: avroline.NewLogLine().Schema(),
	})

	return schema, lookupNames
}

func (m *KafkaMetricsEmitter) getLookupName(name string) string {
	lookupName := name
	pattern := regexp.MustCompile("[0-9]+")
	if pattern.MatchString(name) {
		lookupName = fmt.Sprintf("metric%s", name)
	}
	lookupName = strings.Replace(lookupName, ".", "", -1)
	lookupName = strings.Replace(lookupName, "%", "", -1)
	lookupName = strings.Replace(lookupName, "-", "", -1)

	return lookupName
}

func (m *KafkaMetricsEmitter) parseRecordField(name string, v map[string]interface{}, lookupNames map[string]string) *avro.RecordSchema {
	lookupRecordName := m.getLookupName(name)
	lookupNames[lookupRecordName] = name
	record := &avro.RecordSchema{
		Name:      lookupRecordName,
		Namespace: "ly.stealth",
		Fields:    make([]*avro.SchemaField, 0),
	}

	for name, value := range v {
		lookupName := m.getLookupName(name)
		lookupNames[lookupName] = name

		var valueSchema avro.Schema
		switch value.(type) {
		case string:
			valueSchema = &avro.StringSchema{}
		case float64:
			valueSchema = &avro.DoubleSchema{}
		}

		field := &avro.SchemaField{
			Name:    lookupName,
			Default: nil,
			Type: &avro.UnionSchema{
				Types: []avro.Schema{&avro.NullSchema{}, valueSchema},
			},
		}
		record.Fields = append(record.Fields, field)
	}

	return record
}

func (m *KafkaMetricsEmitter) fillRecord(record *avro.GenericRecord, schema *avro.RecordSchema, metrics map[string]interface{}, lookupNames map[string]string) {
	for _, field := range schema.Fields {
		if field.Name == "logLine" {
			//TODO probably something more?
			logLine := avro.NewGenericRecord(field.Type)
			logLine.Set("source", "metrics")
			logLine.Set("logtypeid", MetricsLogTypeId)
			record.Set("logLine", logLine)
		} else {
			name := lookupNames[field.Name]
			switch fieldSchema := field.Type.(*avro.UnionSchema).Types[1].(type) {
			case *avro.RecordSchema:
				{
					fieldRecord := avro.NewGenericRecord(fieldSchema)
					m.fillRecord(fieldRecord, fieldSchema, metrics[name].(map[string]interface{}), lookupNames)
					record.Set(field.Name, fieldRecord)
				}
			default:
				{
					record.Set(field.Name, metrics[field.Name])
				}
			}
		}
	}
}

type EmptyMetricsEmitter struct{}

func NewEmptyMetricsEmitter() *EmptyMetricsEmitter {
	return new(EmptyMetricsEmitter)
}

func (*EmptyMetricsEmitter) Emit([]byte) {}
func (*EmptyMetricsEmitter) ReportingInterval() time.Duration {
	return time.Minute
}
<commit_msg>Added godoc for metrics emitters<commit_after>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package go_kafka_client

import (
	"encoding/json"
	"fmt"
	"github.com/rcrowley/go-metrics"
	"github.com/stealthly/go-avro"
	avroline "github.com/stealthly/go_kafka_client/avro"
	"regexp"
	"strings"
	"time"
)

// MetricsEmitter is an interface that handles structured metrics data.
type MetricsEmitter interface {
	// ReportingInterval returns an interval to report metrics.
	ReportingInterval() time.Duration

	// Emit sends a single metrics entry to some destination.
	Emit([]byte)
}

// KafkaMetricsEmitterConfig provides multiple configuration entries for KafkaMetricsEmitter
type KafkaMetricsEmitterConfig struct {
	// Registry is a metrics registry to report.
	Registry metrics.Registry

	// ReportingInterval is an interval to report metrics.
	ReportingInterval time.Duration

	// Topic to emit logs to.
	Topic string

	// Confluent Avro schema registry URL.
	SchemaRegistryUrl string

	// Producer config that will be used by this emitter. Note that ValueEncoder WILL BE replaced by KafkaAvroEncoder.
	ProducerConfig *ProducerConfig
}

// NewKafkaMetricsEmitterConfig creates a new KafkaMetricsEmitterConfig with ReportingInterval set to 1 second.
func NewKafkaMetricsEmitterConfig() *KafkaMetricsEmitterConfig {
	return &KafkaMetricsEmitterConfig{
		ReportingInterval: 1 * time.Second,
	}
}

// KafkaMetricsEmitter implements MetricsEmitter and sends all metrics data to a Kafka topic encoded as Avro.
type KafkaMetricsEmitter struct {
	config   *KafkaMetricsEmitterConfig
	producer Producer
}

// NewKafkaMetricsEmitter creates a new KafkaMetricsEmitter with a provided KafkaMetricsEmitterConfig.
func NewKafkaMetricsEmitter(config *KafkaMetricsEmitterConfig) *KafkaMetricsEmitter {
	config.ProducerConfig.ValueEncoder = NewKafkaAvroEncoder(config.SchemaRegistryUrl)
	return &KafkaMetricsEmitter{
		config:   config,
		producer: NewSaramaProducer(config.ProducerConfig),
	}
}

// ReportingInterval returns an interval to report metrics.
func (m *KafkaMetricsEmitter) ReportingInterval() time.Duration {
	return m.config.ReportingInterval
}

// Emit sends a single metrics entry to a Kafka topic encoded as Avro.
func (m *KafkaMetricsEmitter) Emit(bytes []byte) {
	metrics := make(map[string]interface{})
	if err := json.Unmarshal(bytes, &metrics); err != nil {
		panic(err)
	}

	schema, lookupNames := m.parseSchema(metrics)
	record := avro.NewGenericRecord(schema)
	m.fillRecord(record, schema.(*avro.RecordSchema), metrics, lookupNames)

	m.producer.Input() <- &ProducerMessage{Topic: m.config.Topic, Value: record}
}

func (m *KafkaMetricsEmitter) parseSchema(metrics map[string]interface{}) (avro.Schema, map[string]string) {
	lookupNames := make(map[string]string)
	schema := &avro.RecordSchema{
		Name:      "Metrics",
		Namespace: "ly.stealth",
		Fields:    make([]*avro.SchemaField, 0),
	}
	for name, v := range metrics {
		lookupName := m.getLookupName(name)
		lookupNames[lookupName] = name

		field := &avro.SchemaField{
			Name:    lookupName,
			Default: nil,
			Type: &avro.UnionSchema{
				Types: []avro.Schema{&avro.NullSchema{}, m.parseRecordField(name, v.(map[string]interface{}), lookupNames)},
			},
		}
		schema.Fields = append(schema.Fields, field)
	}
	schema.Fields = append(schema.Fields, &avro.SchemaField{
		Name: "logLine",
		Type: avroline.NewLogLine().Schema(),
	})

	return schema, lookupNames
}

func (m *KafkaMetricsEmitter) getLookupName(name string) string {
	lookupName := name
	pattern := regexp.MustCompile("[0-9]+")
	if pattern.MatchString(name) {
		lookupName = fmt.Sprintf("metric%s", name)
	}
	lookupName = strings.Replace(lookupName, ".", "", -1)
	lookupName = strings.Replace(lookupName, "%", "", -1)
	lookupName = strings.Replace(lookupName, "-", "", -1)

	return lookupName
}

func (m *KafkaMetricsEmitter) parseRecordField(name string, v map[string]interface{}, lookupNames map[string]string) *avro.RecordSchema {
	lookupRecordName := m.getLookupName(name)
	lookupNames[lookupRecordName] = name
	record := &avro.RecordSchema{
		Name:      lookupRecordName,
		Namespace: "ly.stealth",
		Fields:    make([]*avro.SchemaField, 0),
	}

	for name, value := range v {
		lookupName := m.getLookupName(name)
		lookupNames[lookupName] = name

		var valueSchema avro.Schema
		switch value.(type) {
		case string:
			valueSchema = &avro.StringSchema{}
		case float64:
			valueSchema = &avro.DoubleSchema{}
		}

		field := &avro.SchemaField{
			Name:    lookupName,
			Default: nil,
			Type: &avro.UnionSchema{
				Types: []avro.Schema{&avro.NullSchema{}, valueSchema},
			},
		}
		record.Fields = append(record.Fields, field)
	}

	return record
}

func (m *KafkaMetricsEmitter) fillRecord(record *avro.GenericRecord, schema *avro.RecordSchema, metrics map[string]interface{}, lookupNames map[string]string) {
	for _, field := range schema.Fields {
		if field.Name == "logLine" {
			//TODO probably something more?
			logLine := avro.NewGenericRecord(field.Type)
			logLine.Set("source", "metrics")
			logLine.Set("logtypeid", MetricsLogTypeId)
			record.Set("logLine", logLine)
		} else {
			name := lookupNames[field.Name]
			switch fieldSchema := field.Type.(*avro.UnionSchema).Types[1].(type) {
			case *avro.RecordSchema:
				{
					fieldRecord := avro.NewGenericRecord(fieldSchema)
					m.fillRecord(fieldRecord, fieldSchema, metrics[name].(map[string]interface{}), lookupNames)
					record.Set(field.Name, fieldRecord)
				}
			default:
				{
					record.Set(field.Name, metrics[field.Name])
				}
			}
		}
	}
}

// EmptyMetricsEmitter implements MetricsEmitter and ignores all incoming data. Used not to break anyone.
type EmptyMetricsEmitter struct{}

// NewEmptyMetricsEmitter creates a new EmptyMetricsEmitter.
func NewEmptyMetricsEmitter() *EmptyMetricsEmitter {
	return new(EmptyMetricsEmitter)
}

// Does nothing. Ignores given message.
func (*EmptyMetricsEmitter) Emit([]byte) {}

// Returns a reporting interval of 1 minute.
func (*EmptyMetricsEmitter) ReportingInterval() time.Duration {
	return time.Minute
}
<|endoftext|>


<commit_before>package rest

import (
	"errors"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/julienschmidt/httprouter"

	"github.com/intelsdi-x/pulse/core"
	"github.com/intelsdi-x/pulse/core/perror"
	"github.com/intelsdi-x/pulse/mgmt/rest/rbody"
)

var (
	ErrMissingPluginName = errors.New("missing plugin name")
)

type plugin struct {
	name    string
	version int
}

func (p *plugin) Name() string {
	return p.name
}

func (p *plugin) Version() int {
	return p.version
}

func (s *Server) loadPlugin(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	// Disabling in expectation of merging #184

	// loadRequest := make(map[string]string)
	// errCode, err := marshalBody(&loadRequest, r.Body)
	// if errCode != 0 && err != nil {
	// 	replyError(errCode, w, err)
	// 	return
	// }
	// loadErr := s.mm.Load(loadRequest["path"])
	// if loadErr != nil {
	// 	// restLogger.WithFields(log.Fields{
	// 	// 	"method": r.Method,
	// 	// 	"url":    r.URL.Path,
	// 	// }).WithFields(loadErr.Fields()).Warning(err.Error())
	// 	replyError(500, w, loadErr)
	// 	return
	// }
	// replySuccess(200, "", w, nil)
	mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
	if err != nil {
		log.Fatal(err)
		respond(500, rbody.FromError(err), w)
		return
	}
	if strings.HasPrefix(mediaType, "multipart/") {
		lp := &rbody.PluginsLoaded{}
		lp.LoadedPlugins = make([]rbody.LoadedPlugin, 0)
		mr := multipart.NewReader(r.Body, params["boundary"])
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				respond(201, lp, w)
				return
			}
			if err != nil {
				log.Fatal(err)
				respond(500, rbody.FromError(err), w)
				return
			}
			b, err := ioutil.ReadAll(p)
			if err != nil {
				log.Fatal(err)
				respond(500, rbody.FromError(err), w)
				return
			}
			autoPaths := s.mm.GetAutodiscoverPaths()
			var f *os.File
			if len(autoPaths) > 0 {
				// write to first autoPath
				f, err = os.Create(path.Join(autoPaths[0], p.FileName()))
			} else {
				// write to temp location
				f, err = ioutil.TempFile("", p.FileName())
			}
			if err != nil {
				log.Fatal(err)
				respond(500, rbody.FromError(err), w)
				return
			}
			n, err := f.Write(b)
			log.Debugf("wrote %v to %v", n, f.Name())
			if err != nil {
				log.Fatal(err)
				respond(500, rbody.FromError(err), w)
				return
			}
			err = f.Chmod(0700)
			if err != nil {
				log.Fatal(err)
				respond(500, rbody.FromError(err), w)
				return
			}
			// Close before load
			f.Close()
			pl, err := s.mm.Load(f.Name())
			if err != nil {
				log.Fatal(err)
				respond(500, rbody.FromError(err), w)
				return
			}
			lp.LoadedPlugins = append(lp.LoadedPlugins, *catalogedPluginToLoaded(pl))
		}
	}
}

func (s *Server) unloadPlugin(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	plName := p.ByName("name")
	plVersion, iErr := strconv.ParseInt(p.ByName("version"), 10, 0)
	f := map[string]interface{}{
		"plugin-name":    plName,
		"plugin-version": plVersion,
	}

	if iErr != nil {
		pe := perror.New(ErrMissingPluginName)
		pe.SetFields(f)
		respond(500, rbody.FromPulseError(pe), w)
		return
	}

	if plName == "" {
		pe := perror.New(errors.New("missing plugin name"))
		pe.SetFields(f)
		respond(500, rbody.FromPulseError(pe), w)
		return
	}
	up, pe := s.mm.Unload(&plugin{name: plName, version: int(plVersion)})
	if pe != nil {
		pe.SetFields(f)
		respond(500, rbody.FromPulseError(pe), w)
		return
	}
	pr := &rbody.PluginUnloaded{
		Name:    up.Name(),
		Version: up.Version(),
		Type:    up.TypeName(),
	}
	respond(200, pr, w)
}

func (s *Server) getPlugins(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	var detail bool
	// make this a function because DRY
	for k, _ := range r.URL.Query() {
		if k == "details" {
			detail = true
		}
	}

	plugins := new(rbody.PluginListReturned)

	// Cache the catalog here to avoid multiple reads
	plCatalog := s.mm.PluginCatalog()
	plugins.LoadedPlugins = make([]rbody.LoadedPlugin, len(plCatalog))
	for i, p := range s.mm.PluginCatalog() {
		plugins.LoadedPlugins[i] = *catalogedPluginToLoaded(p)
	}

	if detail {
		aPlugins := s.mm.AvailablePlugins()
		plugins.AvailablePlugins = make([]rbody.AvailablePlugin, len(aPlugins))
		for i, p := range aPlugins {
			plugins.AvailablePlugins[i] = rbody.AvailablePlugin{
				Name:             p.Name(),
				Version:          p.Version(),
				Type:             p.TypeName(),
				HitCount:         p.HitCount(),
				LastHitTimestamp: p.LastHit().Unix(),
				ID:               p.ID(),
			}
		}
	}

	respond(200, plugins, w)
}

func catalogedPluginToLoaded(c core.CatalogedPlugin) *rbody.LoadedPlugin {
	return &rbody.LoadedPlugin{
		Name:            c.Name(),
		Version:         c.Version(),
		Type:            c.TypeName(),
		Status:          c.Status(),
		LoadedTimestamp: c.LoadedTimestamp().Unix(),
	}
}

func (s *Server) getPluginsByName(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
}

func (s *Server) getPlugin(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
}
<commit_msg>Remove log statements<commit_after>package rest

import (
	"errors"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/julienschmidt/httprouter"

	"github.com/intelsdi-x/pulse/core"
	"github.com/intelsdi-x/pulse/core/perror"
	"github.com/intelsdi-x/pulse/mgmt/rest/rbody"
)

var (
	ErrMissingPluginName = errors.New("missing plugin name")
)

type plugin struct {
	name    string
	version int
}

func (p *plugin) Name() string {
	return p.name
}

func (p *plugin) Version() int {
	return p.version
}

func (s *Server) loadPlugin(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	// Disabling in expectation of merging #184

	// loadRequest := make(map[string]string)
	// errCode, err := marshalBody(&loadRequest, r.Body)
	// if errCode != 0 && err != nil {
	// 	replyError(errCode, w, err)
	// 	return
	// }
	// loadErr := s.mm.Load(loadRequest["path"])
	// if loadErr != nil {
	// 	// restLogger.WithFields(log.Fields{
	// 	// 	"method": r.Method,
	// 	// 	"url":    r.URL.Path,
	// 	// }).WithFields(loadErr.Fields()).Warning(err.Error())
	// 	replyError(500, w, loadErr)
	// 	return
	// }
	// replySuccess(200, "", w, nil)
	mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
	if err != nil {
		respond(500, rbody.FromError(err), w)
		return
	}
	if strings.HasPrefix(mediaType, "multipart/") {
		lp := &rbody.PluginsLoaded{}
		lp.LoadedPlugins = make([]rbody.LoadedPlugin, 0)
		mr := multipart.NewReader(r.Body, params["boundary"])
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				respond(201, lp, w)
				return
			}
			if err != nil {
				respond(500, rbody.FromError(err), w)
				return
			}
			b, err := ioutil.ReadAll(p)
			if err != nil {
				respond(500, rbody.FromError(err), w)
				return
			}
			autoPaths := s.mm.GetAutodiscoverPaths()
			var f *os.File
			if len(autoPaths) > 0 {
				// write to first autoPath
				f, err = os.Create(path.Join(autoPaths[0], p.FileName()))
			} else {
				// write to temp location
				f, err = ioutil.TempFile("", p.FileName())
			}
			if err != nil {
				respond(500, rbody.FromError(err), w)
				return
			}
			n, err := f.Write(b)
			log.Debugf("wrote %v to %v", n, f.Name())
			if err != nil {
				respond(500, rbody.FromError(err), w)
				return
			}
			err = f.Chmod(0700)
			if err != nil {
				respond(500, rbody.FromError(err), w)
				return
			}
			// Close before load
			f.Close()
			pl, err := s.mm.Load(f.Name())
			if err != nil {
				respond(500, rbody.FromError(err), w)
				return
			}
			lp.LoadedPlugins = append(lp.LoadedPlugins, *catalogedPluginToLoaded(pl))
		}
	}
}

func (s *Server) unloadPlugin(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	plName := p.ByName("name")
	plVersion, iErr := strconv.ParseInt(p.ByName("version"), 10, 0)
	f := map[string]interface{}{
		"plugin-name":    plName,
		"plugin-version": plVersion,
	}

	if iErr != nil {
		pe := perror.New(ErrMissingPluginName)
		pe.SetFields(f)
		respond(500, rbody.FromPulseError(pe), w)
		return
	}

	if plName == "" {
		pe := perror.New(errors.New("missing plugin name"))
		pe.SetFields(f)
		respond(500, rbody.FromPulseError(pe), w)
		return
	}
	up, pe := s.mm.Unload(&plugin{name: plName, version: int(plVersion)})
	if pe != nil {
		pe.SetFields(f)
		respond(500, rbody.FromPulseError(pe), w)
		return
	}
	pr := &rbody.PluginUnloaded{
		Name:    up.Name(),
		Version: up.Version(),
		Type:    up.TypeName(),
	}
	respond(200, pr, w)
}

func (s *Server) getPlugins(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	var detail bool
	// make this a function because DRY
	for k, _ := range r.URL.Query() {
		if k == "details" {
			detail = true
		}
	}

	plugins := new(rbody.PluginListReturned)

	// Cache the catalog here to avoid multiple reads
	plCatalog := s.mm.PluginCatalog()
	plugins.LoadedPlugins = make([]rbody.LoadedPlugin, len(plCatalog))
	for i, p := range s.mm.PluginCatalog() {
		plugins.LoadedPlugins[i] = *catalogedPluginToLoaded(p)
	}

	if detail {
		aPlugins := s.mm.AvailablePlugins()
		plugins.AvailablePlugins = make([]rbody.AvailablePlugin, len(aPlugins))
		for i, p := range aPlugins {
			plugins.AvailablePlugins[i] = rbody.AvailablePlugin{
				Name:             p.Name(),
				Version:          p.Version(),
				Type:             p.TypeName(),
				HitCount:         p.HitCount(),
				LastHitTimestamp: p.LastHit().Unix(),
				ID:               p.ID(),
			}
		}
	}

	respond(200, plugins, w)
}

func catalogedPluginToLoaded(c core.CatalogedPlugin) *rbody.LoadedPlugin {
	return &rbody.LoadedPlugin{
		Name:            c.Name(),
		Version:         c.Version(),
		Type:            c.TypeName(),
		Status:          c.Status(),
		LoadedTimestamp: c.LoadedTimestamp().Unix(),
	}
}

func (s *Server) getPluginsByName(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
}

func (s *Server) getPlugin(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package migrate

import (
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc32"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"

	raftpb "github.com/coreos/etcd/raft/raftpb"
)

type Snapshot4 struct {
	State     []byte `json:"state"`
	LastIndex uint64 `json:"lastIndex"`
	LastTerm  uint64 `json:"lastTerm"`

	Peers []struct {
		Name             string `json:"name"`
		ConnectionString string `json:"connectionString"`
	} `json:"peers"`
}

type Store4 struct {
	Root           *node
	CurrentIndex   uint64
	CurrentVersion int
}

type node struct {
	Path string

	CreatedIndex  uint64
	ModifiedIndex uint64

	Parent *node `json:"-"` // should not encode this field! avoid circular dependency.

	ExpireTime time.Time
	ACL        string
	Value      string           // for key-value pair
	Children   map[string]*node // for directory
}

func deepCopyNode(n *node, parent *node) *node {
	out := &node{
		Path:          n.Path,
		CreatedIndex:  n.CreatedIndex,
		ModifiedIndex: n.ModifiedIndex,
		Parent:        parent,
		ExpireTime:    n.ExpireTime,
		ACL:           n.ACL,
		Value:         n.Value,
		Children:      make(map[string]*node),
	}
	for k, v := range n.Children {
		out.Children[k] = deepCopyNode(v, out)
	}

	return out
}

func replacePathNames(n *node, s1, s2 string) {
	n.Path = path.Clean(strings.Replace(n.Path, s1, s2, 1))
	for _, c := range n.Children {
		replacePathNames(c, s1, s2)
	}
}

func pullNodesFromEtcd(n *node) map[string]uint64 {
	out := make(map[string]uint64)
	machines := n.Children["machines"]
	for name, c := range machines.Children {
		q, err := url.ParseQuery(c.Value)
		if err != nil {
			log.Fatal("Couldn't parse old query string value")
		}
		etcdurl := q.Get("etcd")
		rafturl := q.Get("raft")

		m := generateNodeMember(name, rafturl, etcdurl)
		out[m.Name] = uint64(m.ID)
	}
	return out
}

func fixEtcd(etcdref *node) *node {
	n := &node{
		Path:          "/0",
		CreatedIndex:  etcdref.CreatedIndex,
		ModifiedIndex: etcdref.ModifiedIndex,
		ExpireTime:    etcdref.ExpireTime,
		ACL:           etcdref.ACL,
		Children:      make(map[string]*node),
	}

	var machines *node
	if machineOrig, ok := etcdref.Children["machines"]; ok {
		machines = deepCopyNode(machineOrig, n)
	}
	if machines == nil {
		return n
	}
	n.Children["members"] = &node{
		Path:          "/0/members",
		CreatedIndex:  machines.CreatedIndex,
		ModifiedIndex: machines.ModifiedIndex,
		ExpireTime:    machines.ExpireTime,
		ACL:           machines.ACL,
		Children:      make(map[string]*node),
	}
	for name, c := range machines.Children {
		q, err := url.ParseQuery(c.Value)
		if err != nil {
			log.Fatal("Couldn't parse old query string value")
		}
		etcdurl := q.Get("etcd")
		rafturl := q.Get("raft")

		m := generateNodeMember(name, rafturl, etcdurl)
		attrBytes, err := json.Marshal(m.attributes)
		if err != nil {
			log.Fatal("Couldn't marshal attributes")
		}
		raftBytes, err := json.Marshal(m.raftAttributes)
		if err != nil {
			log.Fatal("Couldn't marshal raft attributes")
		}
		newNode := &node{
			Path:          path.Join("/0/members", m.ID.String()),
			CreatedIndex:  c.CreatedIndex,
			ModifiedIndex: c.ModifiedIndex,
			ExpireTime:    c.ExpireTime,
			ACL:           c.ACL,
			Children: map[string]*node{
				"attributes": &node{
					Path:          path.Join("/0/members", m.ID.String(), "attributes"),
					CreatedIndex:  c.CreatedIndex,
					ModifiedIndex: c.ModifiedIndex,
					ExpireTime:    c.ExpireTime,
					ACL:           c.ACL,
					Value:         string(attrBytes),
				},
				"raftAttributes": &node{
					Path:          path.Join("/0/members", m.ID.String(), "raftAttributes"),
					CreatedIndex:  c.CreatedIndex,
					ModifiedIndex: c.ModifiedIndex,
					ExpireTime:    c.ExpireTime,
					ACL:           c.ACL,
					Value:         string(raftBytes),
				},
			},
		}
		n.Children["members"].Children[m.ID.String()] = newNode
	}
	return n
}

func mangleRoot(n *node) *node {
	newRoot := &node{
		Path:          "/",
		CreatedIndex:  n.CreatedIndex,
		ModifiedIndex: n.ModifiedIndex,
		ExpireTime:    n.ExpireTime,
		ACL:           n.ACL,
		Children:      make(map[string]*node),
	}
	newRoot.Children["1"] = n
	etcd := n.Children["_etcd"]
	replacePathNames(n, "/", "/1/")
	newZero := fixEtcd(etcd)
	newRoot.Children["0"] = newZero
	return newRoot
}

func (s *Snapshot4) GetNodesFromStore() map[string]uint64 {
	st := &Store4{}
	if err := json.Unmarshal(s.State, st); err != nil {
		log.Fatal("Couldn't unmarshal snapshot")
	}
	etcd := st.Root.Children["_etcd"]
	return pullNodesFromEtcd(etcd)
}

func (s *Snapshot4) Snapshot2() *raftpb.Snapshot {
	st := &Store4{}
	if err := json.Unmarshal(s.State, st); err != nil {
		log.Fatal("Couldn't unmarshal snapshot")
	}
	st.Root = mangleRoot(st.Root)

	newState, err := json.Marshal(st)
	if err != nil {
		log.Fatal("Couldn't re-marshal new snapshot")
	}

	nodes := s.GetNodesFromStore()
	nodeList := make([]uint64, 0)
	for _, v := range nodes {
		nodeList = append(nodeList, v)
	}

	snap2 := raftpb.Snapshot{
		Data: newState,
		Metadata: raftpb.SnapshotMetadata{
			Index: s.LastIndex,
			Term:  s.LastTerm + termOffset4to2,
			ConfState: raftpb.ConfState{
				Nodes: nodeList,
			},
		},
	}

	return &snap2
}

func DecodeLatestSnapshot4FromDir(snapdir string) (*Snapshot4, error) {
	fname, err := FindLatestFile(snapdir)
	if err != nil {
		return nil, err
	}

	if fname == "" {
		return nil, nil
	}

	snappath := path.Join(snapdir, fname)
	log.Printf("Decoding snapshot from %s", snappath)

	return DecodeSnapshot4FromFile(snappath)
}

// FindLatestFile identifies the "latest" filename in a given directory
// by sorting all the files and choosing the highest value.
func FindLatestFile(dirpath string) (string, error) {
	dir, err := os.OpenFile(dirpath, os.O_RDONLY, 0)
	if err != nil {
		if os.IsNotExist(err) {
			err = nil
		}
		return "", err
	}
	defer dir.Close()

	fnames, err := dir.Readdirnames(-1)
	if err != nil {
		return "", err
	}

	if len(fnames) == 0 {
		return "", nil
	}

	names, err := NewSnapshotFileNames(fnames)
	if err != nil {
		return "", err
	}

	return names[len(names)-1].FileName, nil
}

func DecodeSnapshot4FromFile(path string) (*Snapshot4, error) {
	// Read snapshot data.
	f, err := os.OpenFile(path, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return DecodeSnapshot4(f)
}

func DecodeSnapshot4(f *os.File) (*Snapshot4, error) {
	// Verify checksum
	var checksum uint32
	n, err := fmt.Fscanf(f, "%08x\n", &checksum)
	if err != nil {
		return nil, err
	} else if n != 1 {
		return nil, errors.New("miss heading checksum")
	}

	// Load remaining snapshot contents.
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	// Generate checksum.
	byteChecksum := crc32.ChecksumIEEE(b)
	if uint32(checksum) != byteChecksum {
		return nil, errors.New("bad checksum")
	}

	// Decode snapshot.
	snapshot := new(Snapshot4)
	if err = json.Unmarshal(b, snapshot); err != nil {
		return nil, err
	}
	return snapshot, nil
}

func NewSnapshotFileNames(names []string) ([]SnapshotFileName, error) {

	s := make([]SnapshotFileName, 0)
	for _, n := range names {
		trimmed := strings.TrimSuffix(n, ".ss")
		if trimmed == n {
			return nil, fmt.Errorf("file %q does not have .ss extension", n)
		}

		parts := strings.SplitN(trimmed, "_", 2)
		if len(parts) != 2 {
			return nil, fmt.Errorf("unrecognized file name format %q", n)
		}

		fn := SnapshotFileName{FileName: n}

		var err error
		fn.Term, err = strconv.ParseUint(parts[0], 10, 64)
		if err != nil {
			return nil, fmt.Errorf("unable to parse term from filename %q: %v", n, err)
		}

		fn.Index, err = strconv.ParseUint(parts[1], 10, 64)
		if err != nil {
			return nil, fmt.Errorf("unable to parse index from filename %q: %v", n, err)
		}

		s = append(s, fn)
	}

	sortable := SnapshotFileNames(s)
	sort.Sort(&sortable)
	return s, nil
}

type SnapshotFileNames []SnapshotFileName
type SnapshotFileName struct {
	FileName string
	Term     uint64
	Index    uint64
}

func (n *SnapshotFileNames) Less(i, j int) bool {
	iTerm, iIndex := (*n)[i].Term, (*n)[i].Index
	jTerm, jIndex := (*n)[j].Term, (*n)[j].Index
	return iTerm < jTerm || (iTerm == jTerm && iIndex < jIndex)
}

func (n *SnapshotFileNames) Swap(i, j int) {
	(*n)[i], (*n)[j] = (*n)[j], (*n)[i]
}

func (n *SnapshotFileNames) Len() int {
	return len([]SnapshotFileName(*n))
}
<commit_msg>fix parent references<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package migrate

import (
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc32"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"

	raftpb "github.com/coreos/etcd/raft/raftpb"
)

type Snapshot4 struct {
	State     []byte `json:"state"`
	LastIndex uint64 `json:"lastIndex"`
	LastTerm  uint64 `json:"lastTerm"`

	Peers []struct {
		Name             string `json:"name"`
		ConnectionString string `json:"connectionString"`
	} `json:"peers"`
}

type Store4 struct {
	Root           *node
	CurrentIndex   uint64
	CurrentVersion int
}

type node struct {
	Path string

	CreatedIndex  uint64
	ModifiedIndex uint64

	Parent *node `json:"-"` // should not encode this field! avoid circular dependency.

	ExpireTime time.Time
	ACL        string
	Value      string           // for key-value pair
	Children   map[string]*node // for directory
}

func deepCopyNode(n *node, parent *node) *node {
	out := &node{
		Path:          n.Path,
		CreatedIndex:  n.CreatedIndex,
		ModifiedIndex: n.ModifiedIndex,
		Parent:        parent,
		ExpireTime:    n.ExpireTime,
		ACL:           n.ACL,
		Value:         n.Value,
		Children:      make(map[string]*node),
	}
	for k, v := range n.Children {
		out.Children[k] = deepCopyNode(v, out)
	}

	return out
}

func replacePathNames(n *node, s1, s2 string) {
	n.Path = path.Clean(strings.Replace(n.Path, s1, s2, 1))
	for _, c := range n.Children {
		replacePathNames(c, s1, s2)
	}
}

func pullNodesFromEtcd(n *node) map[string]uint64 {
	out := make(map[string]uint64)
	machines := n.Children["machines"]
	for name, c := range machines.Children {
		q, err := url.ParseQuery(c.Value)
		if err != nil {
			log.Fatal("Couldn't parse old query string value")
		}
		etcdurl := q.Get("etcd")
		rafturl := q.Get("raft")

		m := generateNodeMember(name, rafturl, etcdurl)
		out[m.Name] = uint64(m.ID)
	}
	return out
}

func fixEtcd(etcdref *node) *node {
	n := &node{
		Path:          "/0",
		CreatedIndex:  etcdref.CreatedIndex,
		ModifiedIndex: etcdref.ModifiedIndex,
		ExpireTime:    etcdref.ExpireTime,
		ACL:           etcdref.ACL,
		Children:      make(map[string]*node),
	}

	var machines *node
	if machineOrig, ok := etcdref.Children["machines"]; ok {
		machines = deepCopyNode(machineOrig, n)
	}
	if machines == nil {
		return n
	}
	n.Children["members"] = &node{
		Path:          "/0/members",
		CreatedIndex:  machines.CreatedIndex,
		ModifiedIndex: machines.ModifiedIndex,
		ExpireTime:    machines.ExpireTime,
		ACL:           machines.ACL,
		Children:      make(map[string]*node),
		Parent:        n,
	}
	for name, c := range machines.Children {
		q, err := url.ParseQuery(c.Value)
		if err != nil {
			log.Fatal("Couldn't parse old query string value")
		}
		etcdurl := q.Get("etcd")
		rafturl := q.Get("raft")

		m := generateNodeMember(name, rafturl, etcdurl)
		attrBytes, err := json.Marshal(m.attributes)
		if err != nil {
			log.Fatal("Couldn't marshal attributes")
		}
		raftBytes, err := json.Marshal(m.raftAttributes)
		if err != nil {
			log.Fatal("Couldn't marshal raft attributes")
		}
		newNode := &node{
			Path:          path.Join("/0/members", m.ID.String()),
			CreatedIndex:  c.CreatedIndex,
			ModifiedIndex: c.ModifiedIndex,
			ExpireTime:    c.ExpireTime,
			ACL:           c.ACL,
			Children:      make(map[string]*node),
			Parent:        n.Children["members"],
		}
		attrs := &node{
			Path:          path.Join("/0/members", m.ID.String(), "attributes"),
			CreatedIndex:  c.CreatedIndex,
			ModifiedIndex: c.ModifiedIndex,
			ExpireTime:    c.ExpireTime,
			ACL:           c.ACL,
			Value:         string(attrBytes),
			Parent:        newNode,
		}
		newNode.Children["attributes"] = attrs
		raftAttrs := &node{
			Path:          path.Join("/0/members", m.ID.String(), "raftAttributes"),
			CreatedIndex:  c.CreatedIndex,
			ModifiedIndex: c.ModifiedIndex,
			ExpireTime:    c.ExpireTime,
			ACL:           c.ACL,
			Value:         string(raftBytes),
			Parent:        newNode,
		}
		newNode.Children["raftAttributes"] = raftAttrs
		n.Children["members"].Children[m.ID.String()] = newNode
	}
	return n
}

func mangleRoot(n *node) *node {
	newRoot := &node{
		Path:          "/",
		CreatedIndex:  n.CreatedIndex,
		ModifiedIndex: n.ModifiedIndex,
		ExpireTime:    n.ExpireTime,
		ACL:           n.ACL,
		Children:      make(map[string]*node),
	}
	newRoot.Children["1"] = n
	etcd := n.Children["_etcd"]
	replacePathNames(n, "/", "/1/")
	newZero := fixEtcd(etcd)
	newZero.Parent = newRoot
	newRoot.Children["0"] = newZero
	return newRoot
}

func (s *Snapshot4) GetNodesFromStore() map[string]uint64 {
	st := &Store4{}
	if err := json.Unmarshal(s.State, st); err != nil {
		log.Fatal("Couldn't unmarshal snapshot")
	}
	etcd := st.Root.Children["_etcd"]
	return pullNodesFromEtcd(etcd)
}

func (s *Snapshot4) Snapshot2() *raftpb.Snapshot {
	st := &Store4{}
	if err := json.Unmarshal(s.State, st); err != nil {
		log.Fatal("Couldn't unmarshal snapshot")
	}
	st.Root = mangleRoot(st.Root)

	newState, err := json.Marshal(st)
	if err != nil {
		log.Fatal("Couldn't re-marshal new snapshot")
	}

	nodes := s.GetNodesFromStore()
	nodeList := make([]uint64, 0)
	for _, v := range nodes {
		nodeList = append(nodeList, v)
	}

	snap2 := raftpb.Snapshot{
		Data: newState,
		Metadata: raftpb.SnapshotMetadata{
			Index: s.LastIndex,
			Term:  s.LastTerm + termOffset4to2,
			ConfState: raftpb.ConfState{
				Nodes: nodeList,
			},
		},
	}

	return &snap2
}

func DecodeLatestSnapshot4FromDir(snapdir string) (*Snapshot4, error) {
	fname, err := FindLatestFile(snapdir)
	if err != nil {
		return nil, err
	}

	if fname == "" {
		return nil, nil
	}

	snappath := path.Join(snapdir, fname)
	log.Printf("Decoding snapshot from %s", snappath)

	return DecodeSnapshot4FromFile(snappath)
}

// FindLatestFile identifies the "latest" filename in a given directory
// by sorting all the files and choosing the highest value.
func FindLatestFile(dirpath string) (string, error) {
	dir, err := os.OpenFile(dirpath, os.O_RDONLY, 0)
	if err != nil {
		if os.IsNotExist(err) {
			err = nil
		}
		return "", err
	}
	defer dir.Close()

	fnames, err := dir.Readdirnames(-1)
	if err != nil {
		return "", err
	}

	if len(fnames) == 0 {
		return "", nil
	}

	names, err := NewSnapshotFileNames(fnames)
	if err != nil {
		return "", err
	}

	return names[len(names)-1].FileName, nil
}

func DecodeSnapshot4FromFile(path string) (*Snapshot4, error) {
	// Read snapshot data.
	f, err := os.OpenFile(path, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return DecodeSnapshot4(f)
}

func DecodeSnapshot4(f *os.File) (*Snapshot4, error) {
	// Verify checksum
	var checksum uint32
	n, err := fmt.Fscanf(f, "%08x\n", &checksum)
	if err != nil {
		return nil, err
	} else if n != 1 {
		return nil, errors.New("miss heading checksum")
	}

	// Load remaining snapshot contents.
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	// Generate checksum.
	byteChecksum := crc32.ChecksumIEEE(b)
	if uint32(checksum) != byteChecksum {
		return nil, errors.New("bad checksum")
	}

	// Decode snapshot.
	snapshot := new(Snapshot4)
	if err = json.Unmarshal(b, snapshot); err != nil {
		return nil, err
	}
	return snapshot, nil
}

func NewSnapshotFileNames(names []string) ([]SnapshotFileName, error) {

	s := make([]SnapshotFileName, 0)
	for _, n := range names {
		trimmed := strings.TrimSuffix(n, ".ss")
		if trimmed == n {
			return nil, fmt.Errorf("file %q does not have .ss extension", n)
		}

		parts := strings.SplitN(trimmed, "_", 2)
		if len(parts) != 2 {
			return nil, fmt.Errorf("unrecognized file name format %q", n)
		}

		fn := SnapshotFileName{FileName: n}

		var err error
		fn.Term, err = strconv.ParseUint(parts[0], 10, 64)
		if err != nil {
			return nil, fmt.Errorf("unable to parse term from filename %q: %v", n, err)
		}

		fn.Index, err = strconv.ParseUint(parts[1], 10, 64)
		if err != nil {
			return nil, fmt.Errorf("unable to parse index from filename %q: %v", n, err)
		}

		s = append(s, fn)
	}

	sortable := SnapshotFileNames(s)
	sort.Sort(&sortable)
	return s, nil
}

type SnapshotFileNames []SnapshotFileName
type SnapshotFileName struct {
	FileName string
	Term     uint64
	Index    uint64
}

func (n *SnapshotFileNames) Less(i, j int) bool {
	iTerm, iIndex := (*n)[i].Term, (*n)[i].Index
	jTerm, jIndex := (*n)[j].Term, (*n)[j].Index
	return iTerm < jTerm || (iTerm == jTerm && iIndex < jIndex)
}

func (n *SnapshotFileNames) Swap(i, j int) {
	(*n)[i], (*n)[j] = (*n)[j], (*n)[i]
}

func (n *SnapshotFileNames) Len() int {
	return len([]SnapshotFileName(*n))
}
<|endoftext|>