<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"math"
	"os"
	"strconv"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/controller/framework"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// This test suite can take a long time to run, so by default it is added to
// the ginkgo.skip list (see driver.go).
// To run this suite you must explicitly ask for it by setting the
// -t/--test flag or ginkgo.focus flag.
var _ = Describe("Density", func() {
	var c *client.Client
	var minionCount int
	var RCName string
	var ns string
	var uuid string

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		expectNoError(err)
		minions, err := c.Nodes().List(labels.Everything(), fields.Everything())
		expectNoError(err)
		minionCount = len(minions.Items)
		Expect(minionCount).NotTo(BeZero())
		nsForTesting, err := createTestingNS("density", c)
		ns = nsForTesting.Name
		expectNoError(err)
		uuid = string(util.NewUUID())

		expectNoError(os.Mkdir(fmt.Sprintf(testContext.OutputDir+"/%s", uuid), 0777))
		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "before"))
	})

	AfterEach(func() {
		// Remove any remaining pods from this test if the
		// replication controller still exists and the replica count
		// isn't 0.  This means the controller wasn't cleaned up
		// during the test so clean it up here
		rc, err := c.ReplicationControllers(ns).Get(RCName)
		if err == nil && rc.Spec.Replicas != 0 {
			By("Cleaning up the replication controller")
			err := DeleteRC(c, ns, RCName)
			expectNoError(err)
		}

		By(fmt.Sprintf("Destroying namespace for this suite %v", ns))
		if err := c.Namespaces().Delete(ns); err != nil {
			Failf("Couldn't delete ns %s", err)
		}

		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "after"))

		// Verify latency metrics
		// TODO: We should reset metrics before the test. Currently previous tests influence latency metrics.
		highLatencyRequests, err := HighLatencyRequests(c, 2*time.Second, util.NewStringSet("events"))
		expectNoError(err)
		Expect(highLatencyRequests).NotTo(BeNumerically(">", 0))
	})

	// Tests with "Skipped" substring in their name will be skipped when running
	// e2e test suite without --ginkgo.focus & --ginkgo.skip flags.
	type Density struct {
		skip          bool
		podsPerMinion int
		/* Controls how often the apiserver is polled for pods */
		interval int
	}

	densityTests := []Density{
		// This test should always run, even if larger densities are skipped.
		{podsPerMinion: 3, skip: false, interval: 10},
		{podsPerMinion: 30, skip: false, interval: 10},
		// More than 30 pods per node is outside our v1.0 goals.
		// We might want to enable those tests in the future.
		{podsPerMinion: 50, skip: true, interval: 10},
		{podsPerMinion: 100, skip: true, interval: 1},
	}

	for _, testArg := range densityTests {
		name := fmt.Sprintf("should allow starting %d pods per node", testArg.podsPerMinion)
		if testArg.podsPerMinion <= 30 {
			name = "[Performance suite] " + name
		}
		if testArg.skip {
			name = "[Skipped] " + name
		}
		itArg := testArg
		It(name, func() {
			totalPods := itArg.podsPerMinion * minionCount
			RCName = "density" + strconv.Itoa(totalPods) + "-" + uuid
			fileHndl, err := os.Create(fmt.Sprintf(testContext.OutputDir+"/%s/pod_states.csv", uuid))
			expectNoError(err)
			defer fileHndl.Close()

			config := RCConfig{Client: c,
				Image:         "gcr.io/google_containers/pause:go",
				Name:          RCName,
				Namespace:     ns,
				PollInterval:  itArg.interval,
				PodStatusFile: fileHndl,
				Replicas:      totalPods,
			}

			// Create a listener for events.
			events := make([](*api.Event), 0)
			_, controller := framework.NewInformer(
				&cache.ListWatch{
					ListFunc: func() (runtime.Object, error) {
						return c.Events(ns).List(labels.Everything(), fields.Everything())
					},
					WatchFunc: func(rv string) (watch.Interface, error) {
						return c.Events(ns).Watch(labels.Everything(), fields.Everything(), rv)
					},
				},
				&api.Event{},
				0,
				framework.ResourceEventHandlerFuncs{
					AddFunc: func(obj interface{}) {
						events = append(events, obj.(*api.Event))
					},
				},
			)
			stop := make(chan struct{})
			go controller.Run(stop)

			// Start the replication controller.
			startTime := time.Now()
			expectNoError(RunRC(config))
			e2eStartupTime := time.Now().Sub(startTime)
			Logf("E2E startup time for %d pods: %v", totalPods, e2eStartupTime)

			By("Waiting for all events to be recorded")
			last := -1
			current := len(events)
			timeout := 10 * time.Minute
			for start := time.Now(); last < current && time.Since(start) < timeout; time.Sleep(10 * time.Second) {
				last = current
				current = len(events)
			}
			close(stop)

			if current != last {
				Logf("Warning: Not all events were recorded after waiting %.2f minutes", timeout.Minutes())
			}
			Logf("Found %d events", current)

			// Tune the threshold for allowed failures.
			badEvents := BadEvents(events)
			Expect(badEvents).NotTo(BeNumerically(">", int(math.Floor(0.01*float64(totalPods)))))
		})
	}
})
<commit_msg>Print latency metrics before the density test<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"math"
	"os"
	"strconv"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/controller/framework"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// This test suite can take a long time to run, so by default it is added to
// the ginkgo.skip list (see driver.go).
// To run this suite you must explicitly ask for it by setting the
// -t/--test flag or ginkgo.focus flag.
var _ = Describe("Density", func() {
	var c *client.Client
	var minionCount int
	var RCName string
	var ns string
	var uuid string

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		expectNoError(err)
		minions, err := c.Nodes().List(labels.Everything(), fields.Everything())
		expectNoError(err)
		minionCount = len(minions.Items)
		Expect(minionCount).NotTo(BeZero())
		nsForTesting, err := createTestingNS("density", c)
		ns = nsForTesting.Name
		expectNoError(err)
		uuid = string(util.NewUUID())

		// Print latency metrics before the test.
		// TODO: Remove this once we reset metrics before the test.
		_, err = HighLatencyRequests(c, 2*time.Second, util.NewStringSet("events"))
		expectNoError(err)

		expectNoError(os.Mkdir(fmt.Sprintf(testContext.OutputDir+"/%s", uuid), 0777))
		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "before"))
	})

	AfterEach(func() {
		// Remove any remaining pods from this test if the
		// replication controller still exists and the replica count
		// isn't 0.  This means the controller wasn't cleaned up
		// during the test so clean it up here
		rc, err := c.ReplicationControllers(ns).Get(RCName)
		if err == nil && rc.Spec.Replicas != 0 {
			By("Cleaning up the replication controller")
			err := DeleteRC(c, ns, RCName)
			expectNoError(err)
		}

		By(fmt.Sprintf("Destroying namespace for this suite %v", ns))
		if err := c.Namespaces().Delete(ns); err != nil {
			Failf("Couldn't delete ns %s", err)
		}

		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "after"))

		// Verify latency metrics
		// TODO: We should reset metrics before the test. Currently previous tests influence latency metrics.
		highLatencyRequests, err := HighLatencyRequests(c, 2*time.Second, util.NewStringSet("events"))
		expectNoError(err)
		Expect(highLatencyRequests).NotTo(BeNumerically(">", 0))
	})

	// Tests with "Skipped" substring in their name will be skipped when running
	// e2e test suite without --ginkgo.focus & --ginkgo.skip flags.
	type Density struct {
		skip          bool
		podsPerMinion int
		/* Controls how often the apiserver is polled for pods */
		interval int
	}

	densityTests := []Density{
		// This test should always run, even if larger densities are skipped.
		{podsPerMinion: 3, skip: false, interval: 10},
		{podsPerMinion: 30, skip: false, interval: 10},
		// More than 30 pods per node is outside our v1.0 goals.
		// We might want to enable those tests in the future.
		{podsPerMinion: 50, skip: true, interval: 10},
		{podsPerMinion: 100, skip: true, interval: 1},
	}

	for _, testArg := range densityTests {
		name := fmt.Sprintf("should allow starting %d pods per node", testArg.podsPerMinion)
		if testArg.podsPerMinion <= 30 {
			name = "[Performance suite] " + name
		}
		if testArg.skip {
			name = "[Skipped] " + name
		}
		itArg := testArg
		It(name, func() {
			totalPods := itArg.podsPerMinion * minionCount
			RCName = "density" + strconv.Itoa(totalPods) + "-" + uuid
			fileHndl, err := os.Create(fmt.Sprintf(testContext.OutputDir+"/%s/pod_states.csv", uuid))
			expectNoError(err)
			defer fileHndl.Close()

			config := RCConfig{Client: c,
				Image:         "gcr.io/google_containers/pause:go",
				Name:          RCName,
				Namespace:     ns,
				PollInterval:  itArg.interval,
				PodStatusFile: fileHndl,
				Replicas:      totalPods,
			}

			// Create a listener for events.
			events := make([](*api.Event), 0)
			_, controller := framework.NewInformer(
				&cache.ListWatch{
					ListFunc: func() (runtime.Object, error) {
						return c.Events(ns).List(labels.Everything(), fields.Everything())
					},
					WatchFunc: func(rv string) (watch.Interface, error) {
						return c.Events(ns).Watch(labels.Everything(), fields.Everything(), rv)
					},
				},
				&api.Event{},
				0,
				framework.ResourceEventHandlerFuncs{
					AddFunc: func(obj interface{}) {
						events = append(events, obj.(*api.Event))
					},
				},
			)
			stop := make(chan struct{})
			go controller.Run(stop)

			// Start the replication controller.
			startTime := time.Now()
			expectNoError(RunRC(config))
			e2eStartupTime := time.Now().Sub(startTime)
			Logf("E2E startup time for %d pods: %v", totalPods, e2eStartupTime)

			By("Waiting for all events to be recorded")
			last := -1
			current := len(events)
			timeout := 10 * time.Minute
			for start := time.Now(); last < current && time.Since(start) < timeout; time.Sleep(10 * time.Second) {
				last = current
				current = len(events)
			}
			close(stop)

			if current != last {
				Logf("Warning: Not all events were recorded after waiting %.2f minutes", timeout.Minutes())
			}
			Logf("Found %d events", current)

			// Tune the threshold for allowed failures.
			badEvents := BadEvents(events)
			Expect(badEvents).NotTo(BeNumerically(">", int(math.Floor(0.01*float64(totalPods)))))
		})
	}
})
<|endoftext|>


<commit_before>package test

import (
	"bytes"
	"fmt"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"testing"
)

// Ideas
// - run command w/ matcher
// - verify arbitrary fs layer
// - debug, replay commands/output on failure

type Cmd struct {
	*Cmd

	// Embed a `Once` so we can call `exec` only once
	sync.Once

	// Represents the arguments to `os/exec.Command`
	command string
	args    []string

	// Holder of output, always converted to a string for
	// easier comparison and print
	output string

	// Holds the error from an execution
	err error
}

func Command(cmd string, args ...string) *Cmd {
	return &Cmd{
		command: cmd,
		args:    args,
	}
}

// Quick wrapper around platform/arch specific call to cert-manage
// which is located at ../bin/cert-manage-GOOS-GOARCH
func CertManage(args ...string) *Cmd {
	render := func(tpl string) string {
		return fmt.Sprintf(tpl, runtime.GOARCH)
	}

	switch runtime.GOOS {
	case "darwin":
		return Command(render("../bin/cert-manage-osx-%s"), args...)
	case "linux":
		return Command(render("../bin/cert-manage-linux-%s"), args...)
	case "windows":
		return Command(render("../bin/cert-manage-%s.exe"), args...)
	}
	return nil
}

func (c *Cmd) exec() {
	c.Do(func() {
		cmd := exec.Command(c.command, c.args...)

		// Set output collectors
		var stdout bytes.Buffer
		cmd.Stdout = &stdout
		var stderr bytes.Buffer
		cmd.Stderr = &stderr

		err := cmd.Run()
		if err == nil {
			c.output = stdout.String()
		} else {
			c.output = stderr.String()
		}
		c.err = err
	})
	return
}

func (c *Cmd) Trim() *Cmd {
	c.exec()
	if c.output != "" {
		c.output = strings.TrimSpace(c.output)
	}
	return c
}

func (c *Cmd) CmpInt(t *testing.T, n1 int) {
	t.Helper()
	c.exec()

	c.CmpIntF(t, func(n2 int) bool { return n1 == n2 })
}

func (c *Cmd) CmpIntF(t *testing.T, f func(int) bool) {
	t.Helper()
	c.exec()

	n, err := strconv.Atoi(c.output)
	if err != nil {
		t.Errorf("ERROR: converting '%s' to integer failed, err=%v", c.output, err)
	}
	if !f(n) {
		t.Errorf("ERROR: got %d", n)
	}
}

func (c *Cmd) FailedT(t *testing.T) {
	t.Helper()
	c.exec()

	if c.err == nil {
		t.Errorf("Expected failure, but seeig none.\n Output: %s", c.output)
	}
}

func (c *Cmd) EqualT(t *testing.T, ans string) {
	t.Helper()
	c.exec()
	if c.output != ans {
		t.Errorf("ERROR: Output did not match expected answer!\n Output: %s\n Answer: %s", c.output, ans)
	}
}

func (c *Cmd) String() string {
	c.exec()

	buf := bytes.NewBufferString(fmt.Sprintf("Command:\n  %s %s\nOutput:\n  %s", c.command, strings.Join(c.args, " "), c.output))
	if c.err != nil {
		buf.WriteString(fmt.Sprintf("\nError:\n  %v", c.err))
	}
	return buf.String()
}

func (c *Cmd) Success() bool {
	c.exec()
	return c.err == nil
}

func (c *Cmd) SuccessT(t *testing.T) {
	if !c.Success() {
		t.Errorf("Expected no error, got err=%v\n Output: %s", c.err, c.output)
	}
}

func (c *Cmd) PendingT(t *testing.T, reason string) {
	t.Helper()
	c.exec()

	if c.err == nil {
		t.Errorf("Expected failing test, got success. Pending because %s", reason)
	} else {
		t.Skip(reason)
	}
}
<commit_msg>test: remove Cmd self-embed<commit_after>package test

import (
	"bytes"
	"fmt"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"testing"
)

// Ideas
// - run command w/ matcher
// - verify arbitrary fs layer
// - debug, replay commands/output on failure

type Cmd struct {
	// Embed a `Once` so we can call `exec` only once
	sync.Once

	// Represents the arguments to `os/exec.Command`
	command string
	args    []string

	// Holder of output, always converted to a string for
	// easier comparison and print
	output string

	// Holds the error from an execution
	err error
}

func Command(cmd string, args ...string) *Cmd {
	return &Cmd{
		command: cmd,
		args:    args,
	}
}

// Quick wrapper around platform/arch specific call to cert-manage
// which is located at ../bin/cert-manage-GOOS-GOARCH
func CertManage(args ...string) *Cmd {
	render := func(tpl string) string {
		return fmt.Sprintf(tpl, runtime.GOARCH)
	}

	switch runtime.GOOS {
	case "darwin":
		return Command(render("../bin/cert-manage-osx-%s"), args...)
	case "linux":
		return Command(render("../bin/cert-manage-linux-%s"), args...)
	case "windows":
		return Command(render("../bin/cert-manage-%s.exe"), args...)
	}
	return nil
}

func (c *Cmd) exec() {
	c.Do(func() {
		cmd := exec.Command(c.command, c.args...)

		// Set output collectors
		var stdout bytes.Buffer
		cmd.Stdout = &stdout
		var stderr bytes.Buffer
		cmd.Stderr = &stderr

		err := cmd.Run()
		if err == nil {
			c.output = stdout.String()
		} else {
			c.output = stderr.String()
		}
		c.err = err
	})
	return
}

func (c *Cmd) Trim() *Cmd {
	c.exec()
	if c.output != "" {
		c.output = strings.TrimSpace(c.output)
	}
	return c
}

func (c *Cmd) CmpInt(t *testing.T, n1 int) {
	t.Helper()
	c.exec()

	c.CmpIntF(t, func(n2 int) bool { return n1 == n2 })
}

func (c *Cmd) CmpIntF(t *testing.T, f func(int) bool) {
	t.Helper()
	c.exec()

	n, err := strconv.Atoi(c.output)
	if err != nil {
		t.Errorf("ERROR: converting '%s' to integer failed, err=%v", c.output, err)
	}
	if !f(n) {
		t.Errorf("ERROR: got %d", n)
	}
}

func (c *Cmd) FailedT(t *testing.T) {
	t.Helper()
	c.exec()

	if c.err == nil {
		t.Errorf("Expected failure, but seeig none.\n Output: %s", c.output)
	}
}

func (c *Cmd) EqualT(t *testing.T, ans string) {
	t.Helper()
	c.exec()
	if c.output != ans {
		t.Errorf("ERROR: Output did not match expected answer!\n Output: %s\n Answer: %s", c.output, ans)
	}
}

func (c *Cmd) String() string {
	c.exec()

	buf := bytes.NewBufferString(fmt.Sprintf("Command:\n  %s %s\nOutput:\n  %s", c.command, strings.Join(c.args, " "), c.output))
	if c.err != nil {
		buf.WriteString(fmt.Sprintf("\nError:\n  %v", c.err))
	}
	return buf.String()
}

func (c *Cmd) Success() bool {
	c.exec()
	return c.err == nil
}

func (c *Cmd) SuccessT(t *testing.T) {
	if !c.Success() {
		t.Errorf("Expected no error, got err=%v\n Output: %s", c.err, c.output)
	}
}

func (c *Cmd) PendingT(t *testing.T, reason string) {
	t.Helper()
	c.exec()

	if c.err == nil {
		t.Errorf("Expected failing test, got success. Pending because %s", reason)
	} else {
		t.Skip(reason)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
)

func TestCountWords(t *testing.T) {
	s := "Hello these are some words. Count them for me, will you?"
	count := countWords(s)
	if count != 11 {
		t.Errorf("Expected %d, got %d", 11, count)
	}
}

func TestCountSentences(t *testing.T) {
	s := "Hello these are some words. Count them for me, will you? Let's continue; with more text. And then end it!"
	count := countSentences(s)
	if count != 5 {
		t.Errorf("Expected %d, got %d", 5, count)
	}
}

func TestCountSyllables(t *testing.T) {
	s := "Ambulance"
	count := countSyllables(s)
	if count != 3 {
		t.Errorf("Expected %d, got %d", 3, count)
	}

	s = "Ape"
	count = countSyllables(s)
	if count != 1 {
		t.Errorf("Expected %d, got %d", 1, count)
	}

	s = "Booze"
	count = countSyllables(s)
	if count != 1 {
		t.Errorf("Expected %d, got %d", 1, count)
	}
}
<commit_msg>add tests for isVowel and getWords<commit_after>package main

import (
	"strings"
	"testing"
)

func TestGetWords(t *testing.T) {
	s := "Hello world"
	words := getWords(s)
	if len(words) != 2 || words[0] != "Hello" || words[1] != "world" {
		t.Errorf("Invalid result %s", words)
	}
}

func TestCountWords(t *testing.T) {
	s := "Hello these are some words. Count them for me, will you?"
	count := countWords(s)
	if count != 11 {
		t.Errorf("Expected %d, got %d", 11, count)
	}
}

func TestCountSentences(t *testing.T) {
	s := "Hello these are some words. Count them for me, will you? Let's continue; with more text. And then end it!"
	count := countSentences(s)
	if count != 5 {
		t.Errorf("Expected %d, got %d", 5, count)
	}
}

func TestCountSyllables(t *testing.T) {
	s := "Ambulance"
	count := countSyllables(s)
	if count != 3 {
		t.Errorf("Expected %d, got %d", 3, count)
	}

	s = "Ape"
	count = countSyllables(s)
	if count != 1 {
		t.Errorf("Expected %d, got %d", 1, count)
	}

	s = "Booze"
	count = countSyllables(s)
	if count != 1 {
		t.Errorf("Expected %d, got %d", 1, count)
	}
}

func TestIsVowel(t *testing.T) {
	vowels := []string{"a", "o", "i", "u", "e"}
	for _, c := range vowels {
		if !isVowel(c) {
			t.Errorf("%s should return true", c)
		}
	}

	nonVowels := strings.Split("zxcvbnmqwrtypsdfghjkl", "")
	for _, c := range nonVowels {
		if isVowel(c) {
			t.Errorf("%s should return false", c)
		}
	}
}
<|endoftext|>


<commit_before>package textmagic

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"strconv"

	"github.com/MJKWoolnough/memio"
)

var apiUrlPrefix = "https://www.textmagic.com/app/api?"

const (
	cmdAccount       = "account"
	cmdCheckNumber   = "check_number"
	cmdDeleteReply   = "delete_reply"
	cmdMessageStatus = "message_status"
	cmdReceive       = "receive"
	cmdSend          = "send"
)

type TextMagic struct {
	username, password string
}

func New(username, password string) TextMagic {
	return TextMagic{username, password}
}

func (t TextMagic) sendAPI(cmd string, params url.Values, data interface{}) error {
	params.Add("username", t.username)
	params.Add("password", t.password)
	params.Add("cmd", cmd)
	r, err := http.Get(apiUrlPrefix + params.Encode())
	if err != nil {
		return RequestError{cmd, err}
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return StatusError{cmd, r.StatusCode}
	}
	jsonData := make([]byte, r.ContentLength) // avoid allocation using io.Pipe?
	var apiError APIError
	err = json.NewDecoder(io.TeeReader(r.Body, memio.Create(&jsonData))).Decode(&apiError)
	if err != nil {
		return JSONError{cmd, err}
	}
	if apiError.Code != 0 {
		apiError.Cmd = cmd
		return apiError
	}
	json.Unmarshal(jsonData, data)
	return nil
}

type balance struct {
	Balance float32 `json:"balance"`
}

func (t TextMagic) Account() (float32, error) {
	var b balance
	if err := t.sendAPI(cmdAccount, url.Values{}, &b); err != nil {
		return 0, err
	}
	return b.Balance, nil
}

type Status struct {
	Text      string  `json:"text"`
	Status    string  `json:"status"`
	Created   int64   `json:"created_time"`
	Reply     string  `json:"reply_number"`
	Cost      float32 `json:"credits_cost"`
	Completed int64   `json:"completed_time"`
}

func (t TextMagic) MessageStatus(ids ...uint) (map[uint]Status, error) {
	statuses := make(map[uint]Status)
	for _, tIds := range splitSlice(ids) {
		messageIds := joinUints(tIds...)
		strStatuses := make(map[string]Status)
		err := t.sendAPI(cmdMessageStatus, url.Values{"ids": {messageIds}}, strStatuses)
		if err != nil {
			return statuses, err
		}
		for messageID, status := range strStatuses {
			id, err := strconv.Atoi(messageID)
			if err != nil {
				continue
			}
			statuses[uint(id)] = status
		}
	}
	return statuses, nil
}

type Number struct {
	Price   float32 `json:"price"`
	Country string  `json:"country"`
}

func (t TextMagic) CheckNumber(numbers ...uint) (map[uint]Number, error) {
	ns := make(map[string]Number)
	if err := t.sendAPI(cmdCheckNumber, url.Values{"phone": {joinUints(numbers...)}}, ns); err != nil {
		return nil, err
	}
	toRet := make(map[uint]Number)
	for n, data := range ns {
		number, err := strconv.Atoi(n)
		if err != nil {
			continue
		}
		toRet[uint(number)] = data
	}
	return toRet, nil
}

const joinSep = ','

func joinUints(u ...uint) string {
	toStr := make([]byte, 0, 10*len(u))
	var digits [21]byte
	for n, num := range u {
		if n > 0 {
			toStr = append(toStr, joinSep)
		}
		if num == 0 {
			toStr = append(toStr, '0')
			continue
		}
		pos := 21
		for ; num > 0; num /= 10 {
			pos--
			digits[pos] = '0' + byte(num%10)
		}
		toStr = append(toStr, digits[pos:]...)
	}
	return string(toStr)
}

const maxInSlice = 100

func splitSlice(slice []uint) [][]uint {
	toRet := make([][]uint, 0, len(slice)/maxInSlice+1)
	for len(slice) > 100 {
		toRet = append(toRet, slice[:100])
		slice = slice[100:]
	}
	if len(slice) > 0 {
		toRet = append(toRet, slice)
	}
	return toRet
}

// Errors

type APIError struct {
	Cmd     string
	Code    int    `json:"error_code"`
	Message string `json:"error_message"`
}

func (a APIError) Error() string {
	return "command " + a.Cmd + " returned the following API error: " + a.Message
}

type RequestError struct {
	Cmd string
	Err error
}

func (r RequestError) Error() string {
	return "command " + r.Cmd + " returned the following error while makeing the API call: " + r.Err.Error()
}

type StatusError struct {
	Cmd        string
	StatusCode int
}

func (s StatusError) Error() string {
	return "command " + s.Cmd + " returned a non-200 OK response: " + http.StatusText(s.StatusCode)
}

type JSONError struct {
	Cmd string
	Err error
}

func (j JSONError) Error() string {
	return "command " + j.Cmd + " returned malformed JSON: " + j.Err.Error()
}
<commit_msg>Added DeleteReply<commit_after>package textmagic

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"strconv"

	"github.com/MJKWoolnough/memio"
)

var apiUrlPrefix = "https://www.textmagic.com/app/api?"

const (
	cmdAccount       = "account"
	cmdCheckNumber   = "check_number"
	cmdDeleteReply   = "delete_reply"
	cmdMessageStatus = "message_status"
	cmdReceive       = "receive"
	cmdSend          = "send"
)

type TextMagic struct {
	username, password string
}

func New(username, password string) TextMagic {
	return TextMagic{username, password}
}

func (t TextMagic) sendAPI(cmd string, params url.Values, data interface{}) error {
	params.Add("username", t.username)
	params.Add("password", t.password)
	params.Add("cmd", cmd)
	r, err := http.Get(apiUrlPrefix + params.Encode())
	if err != nil {
		return RequestError{cmd, err}
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return StatusError{cmd, r.StatusCode}
	}
	jsonData := make([]byte, r.ContentLength) // avoid allocation using io.Pipe?
	var apiError APIError
	err = json.NewDecoder(io.TeeReader(r.Body, memio.Create(&jsonData))).Decode(&apiError)
	if err != nil {
		return JSONError{cmd, err}
	}
	if apiError.Code != 0 {
		apiError.Cmd = cmd
		return apiError
	}
	json.Unmarshal(jsonData, data)
	return nil
}

type balance struct {
	Balance float32 `json:"balance"`
}

func (t TextMagic) Account() (float32, error) {
	var b balance
	if err := t.sendAPI(cmdAccount, url.Values{}, &b); err != nil {
		return 0, err
	}
	return b.Balance, nil
}

type Status struct {
	Text      string  `json:"text"`
	Status    string  `json:"status"`
	Created   int64   `json:"created_time"`
	Reply     string  `json:"reply_number"`
	Cost      float32 `json:"credits_cost"`
	Completed int64   `json:"completed_time"`
}

func (t TextMagic) MessageStatus(ids ...uint) (map[uint]Status, error) {
	statuses := make(map[uint]Status)
	for _, tIds := range splitSlice(ids) {
		messageIds := joinUints(tIds...)
		strStatuses := make(map[string]Status)
		err := t.sendAPI(cmdMessageStatus, url.Values{"ids": {messageIds}}, strStatuses)
		if err != nil {
			return statuses, err
		}
		for messageID, status := range strStatuses {
			id, err := strconv.Atoi(messageID)
			if err != nil {
				continue
			}
			statuses[uint(id)] = status
		}
	}
	return statuses, nil
}

type Number struct {
	Price   float32 `json:"price"`
	Country string  `json:"country"`
}

func (t TextMagic) CheckNumber(numbers ...uint) (map[uint]Number, error) {
	ns := make(map[string]Number)
	if err := t.sendAPI(cmdCheckNumber, url.Values{"phone": {joinUints(numbers...)}}, ns); err != nil {
		return nil, err
	}
	toRet := make(map[uint]Number)
	for n, data := range ns {
		number, err := strconv.Atoi(n)
		if err != nil {
			continue
		}
		toRet[uint(number)] = data
	}
	return toRet, nil
}

type deleted struct {
	Deleted []uint `json:"deleted"`
}

func (t TextMagic) DeleteReply(ids ...uint) ([]uint, error) {
	toRet := make([]uint, 0, len(ids))
	for _, tIds := range splitSlice(ids) {
		var d deleted
		if err := t.sendAPI(cmdDeleteReply, url.Values{"deleted": {joinUints(tIds...)}}, &d); err != nil {
			return toRet, err
		}
		toRet = append(toRet, d.Deleted...)
	}
	return toRet, nil
}

const joinSep = ','

func joinUints(u ...uint) string {
	toStr := make([]byte, 0, 10*len(u))
	var digits [21]byte
	for n, num := range u {
		if n > 0 {
			toStr = append(toStr, joinSep)
		}
		if num == 0 {
			toStr = append(toStr, '0')
			continue
		}
		pos := 21
		for ; num > 0; num /= 10 {
			pos--
			digits[pos] = '0' + byte(num%10)
		}
		toStr = append(toStr, digits[pos:]...)
	}
	return string(toStr)
}

const maxInSlice = 100

func splitSlice(slice []uint) [][]uint {
	toRet := make([][]uint, 0, len(slice)/maxInSlice+1)
	for len(slice) > 100 {
		toRet = append(toRet, slice[:100])
		slice = slice[100:]
	}
	if len(slice) > 0 {
		toRet = append(toRet, slice)
	}
	return toRet
}

// Errors

type APIError struct {
	Cmd     string
	Code    int    `json:"error_code"`
	Message string `json:"error_message"`
}

func (a APIError) Error() string {
	return "command " + a.Cmd + " returned the following API error: " + a.Message
}

type RequestError struct {
	Cmd string
	Err error
}

func (r RequestError) Error() string {
	return "command " + r.Cmd + " returned the following error while makeing the API call: " + r.Err.Error()
}

type StatusError struct {
	Cmd        string
	StatusCode int
}

func (s StatusError) Error() string {
	return "command " + s.Cmd + " returned a non-200 OK response: " + http.StatusText(s.StatusCode)
}

type JSONError struct {
	Cmd string
	Err error
}

func (j JSONError) Error() string {
	return "command " + j.Cmd + " returned malformed JSON: " + j.Err.Error()
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/coreos/go-etcd/etcd"
	"github.com/krolaw/dhcp4"
)

// DHCPService is the DHCP server instance
type DHCPService struct {
	ip             net.IP
	domain         string
	guestPool      *net.IPNet
	leaseDuration  time.Duration
	defaultOptions dhcp4.Options // FIXME: make different options per pool?
	etcdClient     *etcd.Client
}

func dhcpSetup(cfg *Config, etc *etcd.Client) chan error {
	etc.CreateDir("dhcp", 0)
	exit := make(chan error, 1)
	go func() {
		exit <- dhcp4.ListenAndServeIf(cfg.DHCPNIC(), &DHCPService{
			ip:            cfg.DHCPIP(),
			leaseDuration: cfg.DHCPLeaseDuration(),
			etcdClient:    etc,
			guestPool:     cfg.DHCPSubnet(),
			domain:        cfg.Domain(),
			defaultOptions: dhcp4.Options{
				dhcp4.OptionSubnetMask:       net.IP(cfg.Subnet().Mask),
				dhcp4.OptionRouter:           cfg.Gateway(),
				dhcp4.OptionDomainNameServer: cfg.DHCPIP(),
			},
		})
	}()
	return exit
}

// ServeDHCP is called by dhcp4.ListenAndServe when the service is started
func (d *DHCPService) ServeDHCP(packet dhcp4.Packet, msgType dhcp4.MessageType, reqOptions dhcp4.Options) (response dhcp4.Packet) {
	switch msgType {
	case dhcp4.Discover:
		// RFC 2131 4.3.1
		// FIXME: send to StatHat and/or increment a counter
		mac := packet.CHAddr()
		fmt.Printf("DHCP Discover from %s\n", mac.String())
		ip := d.getIPFromMAC(mac, packet, reqOptions)
		if ip != nil {
			options := d.getOptionsFromMAC(mac)
			fmt.Printf("DHCP Discover from %s (we return %s)\n", mac.String(), ip.String())
			// for x, y := range reqOptions {
			// 	fmt.Printf("\tR[%v] %v %s\n", x, y, y)
			// }
			// for x, y := range options {
			// 	fmt.Printf("\tO[%v] %v %s\n", x, y, y)
			// }
			return dhcp4.ReplyPacket(packet, dhcp4.Offer, d.ip.To4(), ip.To4(), d.leaseDuration, options.SelectOrderOrAll(reqOptions[dhcp4.OptionParameterRequestList]))
		}
		return nil
	case dhcp4.Request:
		// RFC 2131 4.3.2
		// FIXME: send to StatHat and/or increment a counter
		mac := packet.CHAddr()
		fmt.Printf("DHCP Request from %s...\n", mac.String())
		state := "NEW"
		requestedIP := net.IP(reqOptions[dhcp4.OptionRequestedIPAddress])
		if len(requestedIP) == 0 { // empty
			state = "RENEWAL"
			requestedIP = packet.CIAddr()
		}
		if len(requestedIP) == 4 { // valid and IPv4
			fmt.Printf("DHCP Request (%s) from %s wanting %s\n", state, mac.String(), requestedIP.String())
			ip := d.getIPFromMAC(mac, packet, reqOptions)
			if ip.Equal(requestedIP) {
				options := d.getOptionsFromMAC(mac)
				fmt.Printf("DHCP Request (%s) from %s wanting %s (we agree)\n", state, mac.String(), requestedIP.String())
				// for x, y := range reqOptions {
				// 	fmt.Printf("\tR[%v] %v %s\n", x, y, y)
				// }
				// for x, y := range options {
				// 	fmt.Printf("\tO[%v] %v %s\n", x, y, y)
				// }
				return dhcp4.ReplyPacket(packet, dhcp4.ACK, d.ip.To4(), requestedIP.To4(), d.leaseDuration, options.SelectOrderOrAll(reqOptions[dhcp4.OptionParameterRequestList]))
			}
		}
		if len(requestedIP) == 0 { // no IP provided at all... why? FIXME
			fmt.Printf("DHCP Request (%s) from %s (empty IP, so we're just ignoring this request)\n", state, mac.String())
			return nil
		}
		fmt.Printf("DHCP Request (%s) from %s (we disagree about %s)\n", state, mac.String(), requestedIP)
		return dhcp4.ReplyPacket(packet, dhcp4.NAK, d.ip.To4(), nil, 0, nil)
	case dhcp4.Decline:
		// RFC 2131 4.3.3
		// FIXME: release from DB?  tick a flag?  increment a counter?  send to StatHat?
		mac := packet.CHAddr()
		fmt.Printf("DHCP Decline from %s\n", mac.String())
	case dhcp4.Release:
		// RFC 2131 4.3.4
		// FIXME: release from DB?  tick a flag?  increment a counter?  send to StatHat?
		mac := packet.CHAddr()
		fmt.Printf("DHCP Release from %s\n", mac.String())
	case dhcp4.Inform:
		// RFC 2131 4.3.5
		// FIXME: release from DB?  tick a flag?  increment a counter?  send to StatHat?
		// FIXME: we should reply with valuable info, but not assign an IP to this client, per RFC 2131 for DHCPINFORM
		// NOTE: the client's IP is supposed to only be in the ciaddr field, not the requested IP field, per RFC 2131 4.4.3
		mac := packet.CHAddr()
		fmt.Printf("DHCP Inform from %s\n", mac.String())
	}
	return nil
}

func (d *DHCPService) getIPFromMAC(mac net.HardwareAddr, packet dhcp4.Packet, reqOptions dhcp4.Options) net.IP {
	response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/ip", false, false)
	if response != nil && response.Node != nil {
		ip := net.ParseIP(response.Node.Value)
		if ip != nil {
			d.etcdClient.Set("dhcp/"+ip.String(), mac.String(), uint64(d.leaseDuration.Seconds()+0.5))
			d.etcdClient.Set("dhcp/"+mac.String()+"/ip", ip.String(), uint64(d.leaseDuration.Seconds()+0.5))
			d.maintainDNSRecords(mac, ip, packet, reqOptions)
			return ip
		}
	}

	// TODO: determine whether or not this MAC should be permitted to get an IP at all (blacklist? whitelist?)

	// locate an unused IP address (can this be more efficient?  yes!  FIXME)
	var ip net.IP
	for testIP := dhcp4.IPAdd(d.guestPool.IP, 1); d.guestPool.Contains(testIP); testIP = dhcp4.IPAdd(testIP, 1) {
		fmt.Println(testIP.String())
		response, _ := d.etcdClient.Get("dhcp/"+testIP.String(), false, false)
		if response == nil || response.Node == nil { // this means that the IP is not already occupied
			ip = testIP
			break
		}
	}

	if ip != nil { // if nil then we're out of IP addresses!
		d.etcdClient.CreateDir("dhcp/"+mac.String(), 0)
		d.etcdClient.Set("dhcp/"+mac.String()+"/ip", ip.String(), uint64(d.leaseDuration.Seconds()+0.5))
		d.etcdClient.Set("dhcp/"+ip.String(), mac.String(), uint64(d.leaseDuration.Seconds()+0.5))

		d.maintainDNSRecords(mac, ip, packet, reqOptions)

		return ip
	}

	return nil
}

func (d *DHCPService) maintainDNSRecords(mac net.HardwareAddr, ip net.IP, packet dhcp4.Packet, reqOptions dhcp4.Options) {
	options := d.getOptionsFromMAC(mac)
	if domain, ok := options[dhcp4.OptionDomainName]; ok {
		// FIXME:  danger!  we're mixing systems here...  if we keep this up, we will have spaghetti!
		name := ""
		if val, ok := options[dhcp4.OptionHostName]; ok {
			name = string(val)
		} else if val, ok := reqOptions[dhcp4.OptionHostName]; ok {
			name = string(val)
		}
		if name != "" {
			host := strings.ToLower(strings.Join([]string{name, string(domain)}, "."))
			ipHash := fmt.Sprintf("%x", sha1.Sum([]byte(ip.String())))     // hash the IP address so we can have a unique key name (no other reason for this, honestly)
			pathParts := strings.Split(strings.TrimSuffix(host, "."), ".") // breakup the name
			queryPath := strings.Join(reverseSlice(pathParts), "/")        // reverse and join them with a slash delimiter
			fmt.Printf("Wanting to register against %s/%s\n", queryPath, name)
			d.etcdClient.Set("dns/"+queryPath+"/@a/val/"+ipHash, ip.String(), uint64(d.leaseDuration.Seconds()+0.5))
			hostHash := fmt.Sprintf("%x", sha1.Sum([]byte(host))) // hash the hostname so we can have a unique key name (no other reason for this, honestly)
			slashedIP := strings.Replace(ip.To4().String(), ".", "/", -1)
			d.etcdClient.Set("dns/arpa/in-addr/"+slashedIP+"/@ptr/val/"+hostHash, host, uint64(d.leaseDuration.Seconds()+0.5))
		} else {
			fmt.Println(">> No host name")
		}
	} else {
		fmt.Println(">> No domain name")
	}
}

func (d *DHCPService) getOptionsFromMAC(mac net.HardwareAddr) dhcp4.Options {
	options := dhcp4.Options{}

	for i := range d.defaultOptions {
		options[i] = d.defaultOptions[i]
		fmt.Printf("OPTION:[%d][%+v]\n", i, d.defaultOptions[i])
	}

	{ // Subnet Mask
		response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/mask", false, false)
		if response != nil && response.Node != nil {
			if response.Node.Value == "" {
				delete(options, dhcp4.OptionSubnetMask)
			} else {
				options[dhcp4.OptionSubnetMask] = []byte(response.Node.Value)
			}
		}
	}

	{ // Gateway/Router
		response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/gw", false, false)
		if response != nil && response.Node != nil {
			if response.Node.Value == "" {
				delete(options, dhcp4.OptionRouter)
			} else {
				options[dhcp4.OptionRouter] = []byte(response.Node.Value)
			}
		}
	}

	{ // Name Server
		response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/ns", false, false)
		if response != nil && response.Node != nil {
			if response.Node.Value == "" {
				delete(options, dhcp4.OptionDomainNameServer)
			} else {
				options[dhcp4.OptionDomainNameServer] = []byte(response.Node.Value)
			}
		}
	}

	{ // Host Name
		response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/name", false, false)
		if response != nil && response.Node != nil {
			if response.Node.Value == "" {
				delete(options, dhcp4.OptionHostName)
			} else {
				options[dhcp4.OptionHostName] = []byte(response.Node.Value)
			}
		}
	}

	{ // Domain Name
		response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/domain", false, false)
		if response != nil && response.Node != nil {
			if response.Node.Value != "" {
				options[dhcp4.OptionDomainName] = []byte(response.Node.Value)
			}
		}
		if len(options[dhcp4.OptionDomainName]) == 0 {
			if d.domain != "" {
				options[dhcp4.OptionDomainName] = []byte(d.domain)
			} else {
				delete(options, dhcp4.OptionDomainName)
			}
		}
	}

	{ // Broadcast Address
		response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/broadcast", false, false)
		if response != nil && response.Node != nil {
			if response.Node.Value == "" {
				delete(options, dhcp4.OptionBroadcastAddress)
			} else {
				options[dhcp4.OptionBroadcastAddress] = []byte(response.Node.Value)
			}
		}
	}

	{ // NTP Server
		response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/ntp", false, false)
		if response != nil && response.Node != nil {
			if response.Node.Value == "" {
				delete(options, dhcp4.OptionNetworkTimeProtocolServers)
			} else {
				options[dhcp4.OptionNetworkTimeProtocolServers] = []byte(response.Node.Value)
			}
		}
	}

	return options
}
<commit_msg>Significant refactoring of ServeDCHP and etcd access functions<commit_after>package main

import (
	"crypto/sha1"
	"errors"
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/coreos/go-etcd/etcd"
	"github.com/krolaw/dhcp4"
)

// DHCPService is the DHCP server instance
type DHCPService struct {
	ip             net.IP
	domain         string
	guestPool      *net.IPNet
	leaseDuration  time.Duration
	defaultOptions dhcp4.Options // FIXME: make different options per pool?
	etcdClient     *etcd.Client
}

type dhcpLease struct {
	mac      net.HardwareAddr
	ip       net.IP
	duration time.Duration
}

func dhcpSetup(cfg *Config, etc *etcd.Client) chan error {
	etc.CreateDir("dhcp", 0)
	exit := make(chan error, 1)
	go func() {
		exit <- dhcp4.ListenAndServeIf(cfg.DHCPNIC(), &DHCPService{
			ip:            cfg.DHCPIP(),
			leaseDuration: cfg.DHCPLeaseDuration(),
			etcdClient:    etc,
			guestPool:     cfg.DHCPSubnet(),
			domain:        cfg.Domain(),
			defaultOptions: dhcp4.Options{
				dhcp4.OptionSubnetMask:       net.IP(cfg.Subnet().Mask),
				dhcp4.OptionRouter:           cfg.Gateway(),
				dhcp4.OptionDomainNameServer: cfg.DHCPIP(),
			},
		})
	}()
	return exit
}

// ServeDHCP is called by dhcp4.ListenAndServe when the service is started
func (d *DHCPService) ServeDHCP(packet dhcp4.Packet, msgType dhcp4.MessageType, reqOptions dhcp4.Options) (response dhcp4.Packet) {
	switch msgType {
	case dhcp4.Discover:
		// RFC 2131 4.3.1
		// FIXME: send to StatHat and/or increment a counter
		mac := packet.CHAddr()
		if !d.isMACPermitted(mac) {
			fmt.Printf("DHCP Discover from %s\n is not permitted", mac.String())
			return nil
		}
		fmt.Printf("DHCP Discover from %s\n", mac.String())
		// Existing Lease
		lease, err := d.getLease(mac)
		if err == nil {
			options := d.getOptionsFromMAC(mac)
			fmt.Printf("DHCP Discover from %s (we offer %s from current lease)\n", mac.String(), lease.ip.String())
			// for x, y := range reqOptions {
			// 	fmt.Printf("\tR[%v] %v %s\n", x, y, y)
			// }
			// for x, y := range options {
			// 	fmt.Printf("\tO[%v] %v %s\n", x, y, y)
			// }
			return dhcp4.ReplyPacket(packet, dhcp4.Offer, d.ip.To4(), lease.ip.To4(), d.leaseDuration, options.SelectOrderOrAll(reqOptions[dhcp4.OptionParameterRequestList]))
		}
		// New Lease
		ip := d.getIPFromPool()
		if ip != nil {
			options := d.getOptionsFromMAC(mac)
			fmt.Printf("DHCP Discover from %s (we offer %s from pool)\n", mac.String(), ip.String())
			// for x, y := range reqOptions {
			// 	fmt.Printf("\tR[%v] %v %s\n", x, y, y)
			// }
			// for x, y := range options {
			// 	fmt.Printf("\tO[%v] %v %s\n", x, y, y)
			// }
			return dhcp4.ReplyPacket(packet, dhcp4.Offer, d.ip.To4(), ip.To4(), d.leaseDuration, options.SelectOrderOrAll(reqOptions[dhcp4.OptionParameterRequestList]))
		}

		fmt.Printf("DHCP Discover from %s (no offer due to no addresses available in pool)\n", mac.String())
		// FIXME: Send to StatHat and/or increment a counter
		// TODO: Send an email?

		return nil
	case dhcp4.Request:
		// RFC 2131 4.3.2
		// FIXME: send to StatHat and/or increment a counter
		mac := packet.CHAddr()
		// Check MAC blacklist
		if !d.isMACPermitted(mac) {
			fmt.Printf("DHCP Request from %s\n is not permitted", mac.String())
			return nil
		}
		// Check IP presence
		state, requestedIP := d.getRequestState(packet, reqOptions)
		fmt.Printf("DHCP Request (%s) from %s...\n", state, mac.String())
		if len(requestedIP) == 0 { // no IP provided at all... why? FIXME
			fmt.Printf("DHCP Request (%s) from %s (empty IP, so we're just ignoring this request)\n", state, mac.String())
			return nil
		}
		// Check IPv4
		if len(requestedIP) != 4 {
			fmt.Printf("DHCP Request (%s) from %s wanting %s (IPv6 address requested, so we're just ignoring this request)\n", state, mac.String(), requestedIP.String())
			return nil
		}
		// Check IP subnet
		if !d.guestPool.Contains(requestedIP) {
			fmt.Printf("DHCP Request (%s) from %s wanting %s (we reject due to wrong subnet)\n", state, mac.String(), requestedIP.String())
			return dhcp4.ReplyPacket(packet, dhcp4.NAK, d.ip.To4(), nil, 0, nil)
		}
		// Check Target Server
		targetServerIP := packet.SIAddr()
		if len(targetServerIP) == 4 {
			fmt.Printf("DHCP Request (%s) from %s wanting %s is in response to a DHCP offer from %s\n", state, mac.String(), requestedIP.String(), targetServerIP.String())
			if d.ip.Equal(targetServerIP) {
				return nil
			}
		}
		// Process Request
		fmt.Printf("DHCP Request (%s) from %s wanting %s...\n", state, mac.String(), requestedIP.String())
		lease, err := d.getLease(mac)
		if err == nil {
			// Existing Lease
			lease.duration = d.leaseDuration
			if lease.ip.Equal(requestedIP) {
				err = d.renewLease(lease)
			} else {
				fmt.Printf("DHCP Request (%s) from %s wanting %s (we reject due to lease mismatch, should be %s)\n", state, mac.String(), requestedIP.String(), lease.ip.String())
				return dhcp4.ReplyPacket(packet, dhcp4.NAK, d.ip.To4(), nil, 0, nil)
			}
		} else {
			// New lease
			lease = dhcpLease{
				mac:      mac,
				ip:       requestedIP,
				duration: d.leaseDuration,
			}
			err = d.createLease(lease)
		}

		if err != nil {
			d.maintainDNSRecords(lease.mac, lease.ip, packet, reqOptions) // TODO: Move this?
			options := d.getOptionsFromMAC(mac)
			fmt.Printf("DHCP Request (%s) from %s wanting %s (we agree)\n", state, mac.String(), requestedIP.String())
			return dhcp4.ReplyPacket(packet, dhcp4.ACK, d.ip.To4(), requestedIP.To4(), d.leaseDuration, options.SelectOrderOrAll(reqOptions[dhcp4.OptionParameterRequestList]))
		}

		fmt.Printf("DHCP Request (%s) from %s wanting %s (we reject due to address collision)\n", state, mac.String(), requestedIP.String())
		return dhcp4.ReplyPacket(packet, dhcp4.NAK, d.ip.To4(), nil, 0, nil)
	case dhcp4.Decline:
		// RFC 2131 4.3.3
		// FIXME: release from DB?  tick a flag?  increment a counter?  send to StatHat?
		mac := packet.CHAddr()
		fmt.Printf("DHCP Decline from %s\n", mac.String())
	case dhcp4.Release:
		// RFC 2131 4.3.4
		// FIXME: release from DB?  tick a flag?  increment a counter?  send to StatHat?
		mac := packet.CHAddr()
		fmt.Printf("DHCP Release from %s\n", mac.String())
	case dhcp4.Inform:
		// RFC 2131 4.3.5
		// FIXME: release from DB?  tick a flag?  increment a counter?  send to StatHat?
		// FIXME: we should reply with valuable info, but not assign an IP to this client, per RFC 2131 for DHCPINFORM
		// NOTE: the client's IP is supposed to only be in the ciaddr field, not the requested IP field, per RFC 2131 4.4.3
		mac := packet.CHAddr()
		fmt.Printf("DHCP Inform from %s\n", mac.String())
	}
	return nil
}

func (d *DHCPService) isMACPermitted(mac net.HardwareAddr) bool {
	// TODO: determine whether or not this MAC should be permitted to get an IP at all (blacklist? whitelist?)
	return true
}

func (d *DHCPService) getRequestState(packet dhcp4.Packet, reqOptions dhcp4.Options) (string, net.IP) {
	state := "NEW"
	requestedIP := net.IP(reqOptions[dhcp4.OptionRequestedIPAddress])
	if len(requestedIP) == 0 { // empty
		state = "RENEWAL"
		requestedIP = packet.CIAddr()
	}
	return state, requestedIP
}

func (d *DHCPService) getIPFromPool() net.IP {
	// locate an unused IP address (can this be more efficient?  yes!  FIXME)
	// TODO: Create a channel and spawn a goproc with something like this function to feed it; then have the server pull addresses from that channel
	for ip := dhcp4.IPAdd(d.guestPool.IP, 1); d.guestPool.Contains(ip); ip = dhcp4.IPAdd(ip, 1) {
		fmt.Println(ip.String())
		response, _ := d.etcdClient.Get("dhcp/"+ip.String(), false, false)
		if response == nil || response.Node == nil { // this means that the IP is not already occupied
			return ip
		}
	}
	return nil
}

func (d *DHCPService) getLease(mac net.HardwareAddr) (dhcpLease, error) {
	lease := dhcpLease{}
	response, err := d.etcdClient.Get("dhcp/"+mac.String()+"/ip", false, false)
	if err == nil {
		if response != nil && response.Node != nil {
			lease.mac = mac
			lease.ip = net.ParseIP(response.Node.Value)
			lease.duration = time.Duration(response.Node.TTL)
		} else {
			err = errors.New("Not Found")
		}
	}
	return lease, err
}

func (d *DHCPService) renewLease(lease dhcpLease) error {
	duration := uint64(lease.duration.Seconds() + 0.5) // Half second jitter to hide network delay
	_, err := d.etcdClient.CompareAndSwap("dhcp/"+lease.ip.String(), lease.mac.String(), duration, lease.mac.String(), 0)
	if err == nil {
		return d.writeLease(lease)
	}
	return err
}

func (d *DHCPService) createLease(lease dhcpLease) error {
	duration := uint64(lease.duration.Seconds() + 0.5)
	_, err := d.etcdClient.Create("dhcp/"+lease.ip.String(), lease.mac.String(), duration)
	if err == nil {
		return d.writeLease(lease)
	}
	return err
}

func (d *DHCPService) writeLease(lease dhcpLease) error {
	duration := uint64(lease.duration.Seconds() + 0.5) // Half second jitter to hide network delay
	// FIXME: Decide what to do if either of these calls returns an error
	d.etcdClient.CreateDir("dhcp/"+lease.mac.String(), 0)
	d.etcdClient.Set("dhcp/"+lease.mac.String()+"/ip", lease.ip.String(), duration)
	return nil
}

func (d *DHCPService) maintainDNSRecords(mac net.HardwareAddr, ip net.IP, packet dhcp4.Packet, reqOptions dhcp4.Options) {
	options := d.getOptionsFromMAC(mac)
	if domain, ok := options[dhcp4.OptionDomainName]; ok {
		// FIXME:  danger!  we're mixing systems here...  if we keep this up, we will have spaghetti!
		name := ""
		if val, ok := options[dhcp4.OptionHostName]; ok {
			name = string(val)
		} else if val, ok := reqOptions[dhcp4.OptionHostName]; ok {
			name = string(val)
		}
		if name != "" {
			host := strings.ToLower(strings.Join([]string{name, string(domain)}, "."))
			ipHash := fmt.Sprintf("%x", sha1.Sum([]byte(ip.String())))     // hash the IP address so we can have a unique key name (no other reason for this, honestly)
			pathParts := strings.Split(strings.TrimSuffix(host, "."), ".") // breakup the name
			queryPath := strings.Join(reverseSlice(pathParts), "/")        // reverse and join them with a slash delimiter
			fmt.Printf("Wanting to register against %s/%s\n", queryPath, name)
			d.etcdClient.Set("dns/"+queryPath+"/@a/val/"+ipHash, ip.String(), uint64(d.leaseDuration.Seconds()+0.5))
			hostHash := fmt.Sprintf("%x", sha1.Sum([]byte(host))) // hash the hostname so we can have a unique key name (no other reason for this, honestly)
			slashedIP := strings.Replace(ip.To4().String(), ".", "/", -1)
			d.etcdClient.Set("dns/arpa/in-addr/"+slashedIP+"/@ptr/val/"+hostHash, host, uint64(d.leaseDuration.Seconds()+0.5))
		} else {
			fmt.Println(">> No host name")
		}
	} else {
		fmt.Println(">> No domain name")
	}
}

func (d *DHCPService) getOptionsFromMAC(mac net.HardwareAddr) dhcp4.Options {
	options := dhcp4.Options{}

	for i := range d.defaultOptions {
		options[i] = d.defaultOptions[i]
		fmt.Printf("OPTION:[%d][%+v]\n", i, d.defaultOptions[i])
	}

	{ // Subnet Mask
		response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/mask", false, false)
		if response != nil && response.Node != nil {
			if response.Node.Value == "" {
				delete(options, dhcp4.OptionSubnetMask)
			} else {
				options[dhcp4.OptionSubnetMask] = []byte(response.Node.Value)
			}
		}
	}

	{ // Gateway/Router
		response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/gw", false, false)
		if response != nil && response.Node != nil {
			if response.Node.Value == "" {
				delete(options, dhcp4.OptionRouter)
			} else {
				options[dhcp4.OptionRouter] = []byte(response.Node.Value)
			}
		}
	}

	{ // Name Server
		response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/ns", false, false)
		if response != nil && response.Node != nil {
			if response.Node.Value == "" {
				delete(options, dhcp4.OptionDomainNameServer)
			} else {
				options[dhcp4.OptionDomainNameServer] = []byte(response.Node.Value)
			}
		}
	}

	{ // Host Name
		response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/name", false, false)
		if response != nil && response.Node != nil {
			if response.Node.Value == "" {
				delete(options, dhcp4.OptionHostName)
			} else {
				options[dhcp4.OptionHostName] = []byte(response.Node.Value)
			}
		}
	}

	{ // Domain Name
		response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/domain", false, false)
		if response != nil && response.Node != nil {
			if response.Node.Value != "" {
				options[dhcp4.OptionDomainName] = []byte(response.Node.Value)
			}
		}
		if len(options[dhcp4.OptionDomainName]) == 0 {
			if d.domain != "" {
				options[dhcp4.OptionDomainName] = []byte(d.domain)
			} else {
				delete(options, dhcp4.OptionDomainName)
			}
		}
	}

	{ // Broadcast Address
		response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/broadcast", false, false)
		if response != nil && response.Node != nil {
			if response.Node.Value == "" {
				delete(options, dhcp4.OptionBroadcastAddress)
			} else {
				options[dhcp4.OptionBroadcastAddress] = []byte(response.Node.Value)
			}
		}
	}

	{ // NTP Server
		response, _ := d.etcdClient.Get("dhcp/"+mac.String()+"/ntp", false, false)
		if response != nil && response.Node != nil {
			if response.Node.Value == "" {
				delete(options, dhcp4.OptionNetworkTimeProtocolServers)
			} else {
				options[dhcp4.OptionNetworkTimeProtocolServers] = []byte(response.Node.Value)
			}
		}
	}

	return options
}
<|endoftext|>


<commit_before>package dnsr

import (
	"fmt"
	"strings"
	"sync"
	"time"

	lru "github.com/hashicorp/golang-lru"
	"github.com/miekg/dns"
)

type Resolver struct {
	cache  *lru.Cache
	client *dns.Client
}

func New(size int) *Resolver {
	if size < 0 {
		size = 10000
	}
	cache, _ := lru.New(size)
	r := &Resolver{
		client: &dns.Client{},
		cache:  cache,
	}
	r.cacheRoot()
	return r
}

func (r *Resolver) Resolve(qname string, qtype uint16) <-chan dns.RR {
	c := make(chan dns.RR, 20)
	go func() {
		q := &dns.Question{qname, qtype, dns.ClassINET}
		defer fmt.Printf(";; QUESTION:\n%s\n\n\n", q.String())
		defer close(c)
		qname = toLowerFQDN(qname)
		if rrs := r.cacheGet(qname, qtype); rrs != nil {
			inject(c, rrs...)
			return
		}
		pname, ok := parent(qname)
		if !ok {
			return
		}
		for nrr := range r.Resolve(pname, dns.TypeNS) {
			ns, ok := nrr.(*dns.NS)
			if !ok {
				continue
			}
		outer:
			for arr := range r.Resolve(ns.Ns, dns.TypeA) {
				a, ok := arr.(*dns.A)
				if !ok {
					continue
				}
				addr := a.A.String() + ":53"
				qmsg := &dns.Msg{}
				qmsg.SetQuestion(qname, qtype)
				qmsg.MsgHdr.RecursionDesired = false
				fmt.Printf("; Querying DNS server %s for %s\n", addr, qname)
				rmsg, _, err := r.client.Exchange(qmsg, addr)
				if err != nil {
					fmt.Printf("; ERROR querying DNS server %s for %s: %s\n", addr, qname, err.Error())
					continue // FIXME: handle errors better from flaky/failing NS servers
				}
				if rmsg.Rcode == dns.RcodeNameError {
					r.cacheAdd(qname, qtype) // FIXME: cache NXDOMAIN responses responsibly
				}
				r.cacheSave(rmsg.Answer...)
				r.cacheSave(rmsg.Ns...)
				r.cacheSave(rmsg.Extra...)
				if rrs := r.cacheGet(qname, qtype); rrs != nil {
					inject(c, rrs...)
					return
				}
				break outer
			}
		}
		for _, crr := range r.cacheGet(qname, dns.TypeCNAME) {
			cn, ok := crr.(*dns.CNAME)
			if !ok {
				continue
			}
			for rr := range r.Resolve(cn.Target, qtype) {
				r.cacheAdd(qname, qtype, rr)
				c <- rr
			}
		}
	}()
	return c
}

func inject(c chan<- dns.RR, rrs ...dns.RR) {
	for _, rr := range rrs {
		c <- rr
	}
}

func parent(name string) (string, bool) {
	labels := dns.SplitDomainName(name)
	if labels == nil {
		return "", false
	}
	return toLowerFQDN(strings.Join(labels[1:], ".")), true
}

func toLowerFQDN(name string) string {
	return dns.Fqdn(strings.ToLower(name))
}

type key struct {
	qname string
	qtype uint16
}

type entry struct {
	m   sync.RWMutex
	exp time.Time
	rrs map[dns.RR]struct{}
}

func (r *Resolver) cacheRoot() {
	for t := range dns.ParseZone(strings.NewReader(root), "", "") {
		if t.Error == nil {
			r.cacheSave(t.RR)
		}
	}
}

// cacheGet returns a randomly ordered slice of DNS records.
func (r *Resolver) cacheGet(qname string, qtype uint16) []dns.RR {
	e := r.getEntry(qname, qtype)
	if e == nil {
		return nil
	}
	e.m.RLock()
	defer e.m.RUnlock()
	rrs := make([]dns.RR, len(e.rrs))
	for rr, _ := range e.rrs {
		rrs = append(rrs, rr)
	}
	return rrs
}

// cacheSave saves 1 or more DNS records to the resolver cache.
func (r *Resolver) cacheSave(rrs ...dns.RR) {
	for _, rr := range rrs {
		h := rr.Header()
		r.cacheAdd(toLowerFQDN(h.Name), h.Rrtype, rr)
	}
}

// cacheAdd adds 0 or more DNS records to the resolver cache for a specific
// domain name and record type. This ensures the cache entry exists, even
// if empty, for NXDOMAIN responses.
func (r *Resolver) cacheAdd(qname string, qtype uint16, rrs ...dns.RR) {
	now := time.Now()
	e := r.getEntry(qname, qtype)
	if e == nil {
		e = &entry{
			exp: now.Add(24 * time.Hour),
			rrs: make(map[dns.RR]struct{}, 0),
		}
		r.cache.Add(key{qname, qtype}, e)
	}
	e.m.Lock()
	defer e.m.Unlock()
	for _, rr := range rrs {
		// fmt.Printf("%s\n", rr.String())
		e.rrs[rr] = struct{}{}
		exp := now.Add(time.Duration(rr.Header().Ttl) * time.Second)
		if exp.Before(e.exp) {
			e.exp = exp
		}
	}
}

func (r *Resolver) getEntry(qname string, qtype uint16) *entry {
	c, ok := r.cache.Get(key{qname, qtype})
	if !ok {
		return nil
	}
	e := c.(*entry)
	if time.Now().After(e.exp) {
		return nil
	}
	fmt.Printf("; CACHE HIT: %s\n", qname)
	return e
}
<commit_msg>less logging<commit_after>package dnsr

import (
	"fmt"
	"strings"
	"sync"
	"time"

	lru "github.com/hashicorp/golang-lru"
	"github.com/miekg/dns"
)

type Resolver struct {
	cache  *lru.Cache
	client *dns.Client
}

func New(size int) *Resolver {
	if size < 0 {
		size = 10000
	}
	cache, _ := lru.New(size)
	r := &Resolver{
		client: &dns.Client{},
		cache:  cache,
	}
	r.cacheRoot()
	return r
}

func (r *Resolver) Resolve(qname string, qtype uint16) <-chan dns.RR {
	c := make(chan dns.RR, 20)
	go func() {
		q := &dns.Question{qname, qtype, dns.ClassINET}
		defer fmt.Printf(";; QUESTION:\n%s\n\n\n", q.String())
		defer close(c)
		qname = toLowerFQDN(qname)
		if rrs := r.cacheGet(qname, qtype); rrs != nil {
			inject(c, rrs...)
			return
		}
		pname, ok := parent(qname)
		if !ok {
			return
		}
		for nrr := range r.Resolve(pname, dns.TypeNS) {
			ns, ok := nrr.(*dns.NS)
			if !ok {
				continue
			}
		outer:
			for arr := range r.Resolve(ns.Ns, dns.TypeA) {
				a, ok := arr.(*dns.A)
				if !ok {
					continue
				}
				addr := a.A.String() + ":53"
				qmsg := &dns.Msg{}
				qmsg.SetQuestion(qname, qtype)
				qmsg.MsgHdr.RecursionDesired = false
				rmsg, _, err := r.client.Exchange(qmsg, addr)
				if err != nil {
					fmt.Printf("; ERROR querying DNS server %s for %s: %s\n", addr, qname, err.Error())
					continue // FIXME: handle errors better from flaky/failing NS servers
				}
				if rmsg.Rcode == dns.RcodeNameError {
					r.cacheAdd(qname, qtype) // FIXME: cache NXDOMAIN responses responsibly
				}
				r.cacheSave(rmsg.Answer...)
				r.cacheSave(rmsg.Ns...)
				r.cacheSave(rmsg.Extra...)
				if rrs := r.cacheGet(qname, qtype); rrs != nil {
					inject(c, rrs...)
					return
				}
				break outer
			}
		}
		for _, crr := range r.cacheGet(qname, dns.TypeCNAME) {
			cn, ok := crr.(*dns.CNAME)
			if !ok {
				continue
			}
			for rr := range r.Resolve(cn.Target, qtype) {
				r.cacheAdd(qname, qtype, rr)
				c <- rr
			}
		}
	}()
	return c
}

func inject(c chan<- dns.RR, rrs ...dns.RR) {
	for _, rr := range rrs {
		c <- rr
	}
}

func parent(name string) (string, bool) {
	labels := dns.SplitDomainName(name)
	if labels == nil {
		return "", false
	}
	return toLowerFQDN(strings.Join(labels[1:], ".")), true
}

func toLowerFQDN(name string) string {
	return dns.Fqdn(strings.ToLower(name))
}

type key struct {
	qname string
	qtype uint16
}

type entry struct {
	m   sync.RWMutex
	exp time.Time
	rrs map[dns.RR]struct{}
}

func (r *Resolver) cacheRoot() {
	for t := range dns.ParseZone(strings.NewReader(root), "", "") {
		if t.Error == nil {
			r.cacheSave(t.RR)
		}
	}
}

// cacheGet returns a randomly ordered slice of DNS records.
func (r *Resolver) cacheGet(qname string, qtype uint16) []dns.RR {
	e := r.getEntry(qname, qtype)
	if e == nil {
		return nil
	}
	e.m.RLock()
	defer e.m.RUnlock()
	rrs := make([]dns.RR, len(e.rrs))
	for rr, _ := range e.rrs {
		rrs = append(rrs, rr)
	}
	return rrs
}

// cacheSave saves 1 or more DNS records to the resolver cache.
func (r *Resolver) cacheSave(rrs ...dns.RR) {
	for _, rr := range rrs {
		h := rr.Header()
		r.cacheAdd(toLowerFQDN(h.Name), h.Rrtype, rr)
	}
}

// cacheAdd adds 0 or more DNS records to the resolver cache for a specific
// domain name and record type. This ensures the cache entry exists, even
// if empty, for NXDOMAIN responses.
func (r *Resolver) cacheAdd(qname string, qtype uint16, rrs ...dns.RR) {
	now := time.Now()
	e := r.getEntry(qname, qtype)
	if e == nil {
		e = &entry{
			exp: now.Add(24 * time.Hour),
			rrs: make(map[dns.RR]struct{}, 0),
		}
		r.cache.Add(key{qname, qtype}, e)
	}
	e.m.Lock()
	defer e.m.Unlock()
	for _, rr := range rrs {
		// fmt.Printf("%s\n", rr.String())
		e.rrs[rr] = struct{}{}
		exp := now.Add(time.Duration(rr.Header().Ttl) * time.Second)
		if exp.Before(e.exp) {
			e.exp = exp
		}
	}
}

func (r *Resolver) getEntry(qname string, qtype uint16) *entry {
	c, ok := r.cache.Get(key{qname, qtype})
	if !ok {
		return nil
	}
	e := c.(*entry)
	if time.Now().After(e.exp) {
		return nil
	}
	fmt.Printf("; CACHE HIT: %s\n", qname)
	return e
}
<|endoftext|>


<commit_before>package git

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

// Edit opens a file in the users editor and returns the title and body. It
// store a temporary file in your .git directory or /tmp if accessed outside of
// a git repo.
func Edit(filePrefix, message string) (string, string, error) {
	var (
		dir string
		err error
	)
	if InsideGitRepo() {
		dir, err = GitDir()
		if err != nil {
			return "", "", err
		}
	} else {
		dir = "/tmp"
	}
	filePath := filepath.Join(dir, fmt.Sprintf("%s_EDITMSG", filePrefix))
	editorPath, err := editorPath()
	if err != nil {
		return "", "", err
	}
	defer os.Remove(filePath)

	// Write generated/template message to file
	if _, err := os.Stat(filePath); os.IsNotExist(err) && message != "" {
		err = ioutil.WriteFile(filePath, []byte(message), 0644)
		if err != nil {
			return "", "", err
		}
	}

	cmd := editorCMD(editorPath, filePath)
	err = cmd.Run()
	if err != nil {
		return "", "", err
	}

	contents, err := ioutil.ReadFile(filePath)
	if err != nil {
		return "", "", err
	}

	return parseTitleBody(strings.TrimSpace(string(contents)))
}

func editorPath() (string, error) {
	cmd := New("var", "GIT_EDITOR")
	cmd.Stdout = nil
	e, err := cmd.Output()
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(e)), nil
}

func editorCMD(editorPath, filePath string) *exec.Cmd {
	parts := strings.Split(editorPath, " ")
	r := regexp.MustCompile("[nmg]?vi[m]?$")
	args := make([]string, 0, 3)
	if r.MatchString(editorPath) {
		args = append(args, "--cmd", "set ft=gitcommit tw=0 wrap lbr")
	}
	args = append(args, parts[1:]...)
	args = append(args, filePath)
	cmd := exec.Command(parts[0], args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd
}

func parseTitleBody(message string) (string, string, error) {
	// Grab all the lines that don't start with the comment char
	cc := CommentChar()
	r := regexp.MustCompile(`(?m:^)[^` + cc + `].*(?m:$)`)
	cr := regexp.MustCompile(`(?m:^)\s*` + cc)
	parts := r.FindAllString(message, -1)
	noComments := make([]string, 0)
	for _, p := range parts {
		if !cr.MatchString(p) {
			noComments = append(noComments, p)
		}
	}
	msg := strings.Join(noComments, "\n")
	if strings.TrimSpace(msg) == "" {
		return "", "", nil
	}

	r = regexp.MustCompile(`\n\s*\n`)
	msg = strings.Replace(msg, "\\#", "#", -1)
	parts = r.Split(msg, 2)

	if strings.Contains(parts[0], "\n") {
		return "\n", parts[0], nil
	}
	if len(parts) < 2 {
		return parts[0], "", nil
	}
	return parts[0], parts[1], nil
}
<commit_msg>refactor git/edit to prepare for issue notes<commit_after>package git

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

// Edit opens a file in the users editor and returns the title and body.
func Edit(filePrefix, message string) (string, string, error) {
	contents, err := EditFile(filePrefix, message)
	if err != nil {
		return "", "", err
	}

	return parseTitleBody(strings.TrimSpace(string(contents)))
}

// EditFile opens a file in the users editor and returns the contents. It
// stores a temporary file in your .git directory or /tmp if accessed outside of
// a git repo.
func EditFile(filePrefix, message string) (string, error) {
	var (
		dir string
		err error
	)
	if InsideGitRepo() {
		dir, err = GitDir()
		if err != nil {
			return "", err
		}
	} else {
		dir = "/tmp"
	}
	filePath := filepath.Join(dir, fmt.Sprintf("%s_EDITMSG", filePrefix))
	editorPath, err := editorPath()
	if err != nil {
		return "", err
	}
	defer os.Remove(filePath)

	// Write generated/template message to file
	if _, err := os.Stat(filePath); os.IsNotExist(err) && message != "" {
		err = ioutil.WriteFile(filePath, []byte(message), 0644)
		if err != nil {
			return "", err
		}
	}

	cmd := editorCMD(editorPath, filePath)
	err = cmd.Run()
	if err != nil {
		return "", err
	}

	contents, err := ioutil.ReadFile(filePath)
	if err != nil {
		return "", err
	}

	return removeComments(string(contents))
}

func editorPath() (string, error) {
	cmd := New("var", "GIT_EDITOR")
	cmd.Stdout = nil
	e, err := cmd.Output()
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(e)), nil
}

func editorCMD(editorPath, filePath string) *exec.Cmd {
	parts := strings.Split(editorPath, " ")
	r := regexp.MustCompile("[nmg]?vi[m]?$")
	args := make([]string, 0, 3)
	if r.MatchString(editorPath) {
		args = append(args, "--cmd", "set ft=gitcommit tw=0 wrap lbr")
	}
	args = append(args, parts[1:]...)
	args = append(args, filePath)
	cmd := exec.Command(parts[0], args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd
}

func removeComments(message string) (string, error) {
	// Grab all the lines that don't start with the comment char
	cc := CommentChar()
	r := regexp.MustCompile(`(?m:^)[^` + cc + `].*(?m:$)`)
	cr := regexp.MustCompile(`(?m:^)\s*` + cc)
	parts := r.FindAllString(message, -1)
	noComments := make([]string, 0)
	for _, p := range parts {
		if !cr.MatchString(p) {
			noComments = append(noComments, p)
		}
	}
	return strings.TrimSpace(strings.Join(noComments, "\n")), nil
}

func parseTitleBody(message string) (string, string, error) {
	msg, err := removeComments(message)
	if err != nil {
		return "", "", err
	}

	if msg == "" {
		return "", "", nil
	}

	r := regexp.MustCompile(`\n\s*\n`)
	msg = strings.Replace(msg, "\\#", "#", -1)
	parts := r.Split(msg, 2)

	if strings.Contains(parts[0], "\n") {
		return "\n", parts[0], nil
	}
	if len(parts) < 2 {
		return parts[0], "", nil
	}
	return parts[0], parts[1], nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package internal

// This file has code for accessing metadata.
//
// References:
//	https://cloud.google.com/compute/docs/metadata

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

const (
	metadataHost = "metadata"
	metadataPath = "/computeMetadata/v1/"
)

var (
	metadataRequestHeaders = http.Header{
		"Metadata-Flavor": []string{"Google"},
	}
)

// TODO(dsymonds): Do we need to support default values, like Python?
func mustGetMetadata(key string) []byte {
	b, err := getMetadata(key)
	if err != nil {
		log.Fatalf("Metadata fetch failed: %v", err)
	}
	return b
}

func getMetadata(key string) ([]byte, error) {
	// TODO(dsymonds): May need to use url.Parse to support keys with query args.
	req := &http.Request{
		Method: "GET",
		URL: &url.URL{
			Scheme: "http",
			Host:   metadataHost,
			Path:   metadataPath + key,
		},
		Header: metadataRequestHeaders,
		Host:   metadataHost,
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("metadata server returned HTTP %d", resp.StatusCode)
	}
	return ioutil.ReadAll(resp.Body)
}
<commit_msg>[internal] Panic instead of log.Fatal when metadata fetch fails<commit_after>// Copyright 2014 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package internal

// This file has code for accessing metadata.
//
// References:
//	https://cloud.google.com/compute/docs/metadata

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

const (
	metadataHost = "metadata"
	metadataPath = "/computeMetadata/v1/"
)

var (
	metadataRequestHeaders = http.Header{
		"Metadata-Flavor": []string{"Google"},
	}
)

// TODO(dsymonds): Do we need to support default values, like Python?
func mustGetMetadata(key string) []byte {
	b, err := getMetadata(key)
	if err != nil {
		panic(fmt.Sprintf("Metadata fetch failed: %v", err))
	}
	return b
}

func getMetadata(key string) ([]byte, error) {
	// TODO(dsymonds): May need to use url.Parse to support keys with query args.
	req := &http.Request{
		Method: "GET",
		URL: &url.URL{
			Scheme: "http",
			Host:   metadataHost,
			Path:   metadataPath + key,
		},
		Header: metadataRequestHeaders,
		Host:   metadataHost,
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("metadata server returned HTTP %d", resp.StatusCode)
	}
	return ioutil.ReadAll(resp.Body)
}
<|endoftext|>


<commit_before>package yum

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/gonuts/logger"
)

// global registry of known backends
var g_backends = make(map[string]func(repo *Repository) (Backend, error))

// NewBackend returns a new backend of type "backend"
func NewBackend(backend string, repo *Repository) (Backend, error) {
	factory, ok := g_backends[backend]
	if !ok {
		return nil, fmt.Errorf("yum: no such backend [%s]", backend)
	}
	return factory(repo)
}

// Backend queries a YUM DB repository
type Backend interface {

	// YumDataType returns the ID for the data type as used in the repomd.xml file
	YumDataType() string

	// Download the DB from server
	GetLatestDB(url string) error

	// Check whether the DB is there
	HasDB() bool

	// Load loads the DB
	LoadDB() error

	// FindLatestMatchingName locats a package by name, returns the latest available version.
	FindLatestMatchingName(name, version, release string) (*Package, error)

	// FindLatestMatchingRequire locates a package providing a given functionality.
	FindLatestMatchingRequire(requirement string) (*Package, error)

	// GetPackages returns all the packages known by a YUM repository
	GetPackages() []*Package
}

// Repository represents a YUM repository with all associated metadata.
type Repository struct {
	msg            *logger.Logger
	Name           string
	RepoUrl        string
	RepoMdUrl      string
	LocalRepoMdXml string
	CacheDir       string
	Backends       []string
	Backend        Backend
}

// NewRepository create a new Repository with name and from url.
func NewRepository(name, url, cachedir string, backends []string, setupBackend, checkForUpdates bool) (*Repository, error) {

	repo := Repository{
		msg:            logger.NewLogger("yum", logger.INFO, os.Stdout),
		Name:           name,
		RepoUrl:        url,
		RepoMdUrl:      url + "/repodata/repomd.xml",
		LocalRepoMdXml: filepath.Join(cachedir, "repomd.xml"),
		CacheDir:       cachedir,
		Backends:       make([]string, len(backends)),
	}
	copy(repo.Backends, backends)

	err := os.MkdirAll(cachedir, 0644)
	if err != nil {
		return nil, err
	}

	// load appropriate backend if requested
	if setupBackend {
		if checkForUpdates {
			err = repo.setupBackendFromRemote()
			if err != nil {
				return nil, err
			}
		} else {
			err = repo.setupBackendFromLocal()
			if err != nil {
				return nil, err
			}
		}
	}
	return &repo, err
}

// FindLatestMatchingName locats a package by name, returns the latest available version.
func (repo *Repository) FindLatestMatchingName(name, version, release string) (*Package, error) {
	return repo.Backend.FindLatestMatchingName(name, version, release)
}

// FindLatestMatchingRequire locates a package providing a given functionality.
func (repo *Repository) FindLatestMatchingRequire(requirement string) (*Package, error) {
	return repo.Backend.FindLatestMatchingRequire(requirement)
}

// GetPackages returns all the packages known by a YUM repository
func (repo *Repository) GetPackages() []*Package {
	return repo.Backend.GetPackages()
}

// setupBackendFromRemote checks which backend should be used and updates the DB files.
func (repo *Repository) setupBackendFromRemote() error {
	var err error
	var backend Backend
	// get repo metadata with list of available files
	remotedata, err := repo.remoteMetadata()
	if err != nil {
		return err
	}

	remotemd, err := repo.checkRepoMD(remotedata)
	if err != nil {
		return err
	}

	localdata, err := repo.localMetadata()
	if err != nil {
		return err
	}

	localmd, err := repo.checkRepoMD(localdata)
	if err != nil {
		return err
	}

	for _, bname := range repo.Backends {
		repo.msg.Infof("checking availability of backend [%s]\n", bname)
		ba, err := NewBackend(bname, repo)
		if err != nil {
			continue
		}

		rrepomd, ok := remotemd[ba.YumDataType()]
		if !ok {
			repo.msg.Warnf("remote repository does not provide [%s] DB\n", bname)
			continue
		}

		// a priori a match
		backend = ba
		repo.Backend = backend

		lrepomd, ok := localmd[ba.YumDataType()]
		if !ok {
			// doesn't matter, we download the DB in any case
		}

		if !repo.Backend.HasDB() || rrepomd.Timestamp.After(lrepomd.Timestamp) {
			// we need to update the DB
			url := repo.RepoUrl + "/" + rrepomd.Location
			repo.msg.Infof("updating the RPM database for %s\n", bname)
			err = repo.Backend.GetLatestDB(url)
			if err != nil {
				repo.msg.Warnf("problem updating RPM database for backend [%s]: %v\n", bname, err)
				err = nil
				backend = nil
				repo.Backend = nil
				continue
			}
			// save metadata to local repomd file
			err = ioutil.WriteFile(repo.LocalRepoMdXml, remotedata, 0644)
			if err != nil {
				repo.msg.Warnf("problem updating local repomd.xml file for backend [%s]: %v\n", bname, err)
				err = nil
				backend = nil
				repo.Backend = nil
				continue
			}
		}

		// load data necessary for the backend
		err = repo.Backend.LoadDB()
		if err != nil {
			repo.msg.Warnf("problem loading data for backend [%s]: %v\n", bname, err)
			err = nil
			backend = nil
			repo.Backend = nil
			continue
		}

		// stop at first one found
		break
	}

	if backend == nil {
		repo.msg.Errorf("No valid backend found\n")
		return fmt.Errorf("No valid backend found")
	}

	repo.msg.Infof("repository [%s] - chosen backend [%T]\n", repo.Name, repo.Backend)
	return err
}

func (repo *Repository) setupBackendFromLocal() error {
	var err error
	data, err := repo.localMetadata()
	if err != nil {
		return err
	}

	md, err := repo.checkRepoMD(data)
	if err != nil {
		return err
	}

	var backend Backend
	for _, bname := range repo.Backends {
		repo.msg.Infof("checking availability of backend [%s]\n", bname)
		ba, err := NewBackend(bname, repo)
		if err != nil {
			continue
		}
		_ /*repomd*/, ok := md[ba.YumDataType()]
		if !ok {
			repo.msg.Warnf("local repository does not provide [%s] DB\n", bname)
			continue
		}

		// a priori a match
		backend = ba
		repo.Backend = backend

		// loading data necessary for the backend
		err = repo.Backend.LoadDB()
		if err != nil {
			repo.msg.Warnf("problem loading data for backend [%s]: %v\n", bname, err)
			err = nil
			backend = nil
			repo.Backend = nil
			continue
		}

		// stop at first one found.
		break
	}

	if backend == nil {
		repo.msg.Errorf("No valid backend found\n")
		return fmt.Errorf("No valid backend found")
	}

	repo.msg.Infof("repository [%s] - chosen backend [%T]\n", repo.Name, repo.Backend)
	return err
}

// remoteMetadata retrieves the repo metadata file content
func (repo *Repository) remoteMetadata() ([]byte, error) {
	resp, err := http.Get(repo.RepoMdUrl)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	buf := new(bytes.Buffer)
	_, err = io.Copy(buf, resp.Body)
	if err != nil && err != io.EOF {
		return nil, err
	}
	return buf.Bytes(), err
}

// localMetadata retrieves the repo metadata from the repomd file
func (repo *Repository) localMetadata() ([]byte, error) {
	if !path_exists(repo.LocalRepoMdXml) {
		return nil, nil
	}
	f, err := os.Open(repo.LocalRepoMdXml)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	buf := new(bytes.Buffer)
	_, err = io.Copy(buf, f)
	if err != nil && err != io.EOF {
		return nil, err
	}
	return buf.Bytes(), err
}

// checkRepoMD parses the Repository metadata XML content
func (repo *Repository) checkRepoMD(data []byte) (map[string]RepoMD, error) {

	if len(data) <= 0 {
		return nil, nil
	}

	type xmlTree struct {
		XMLName xml.Name `xml:"repomd"`
		Data    []struct {
			Type     string `xml:"type,attr"`
			Checksum string `xml:"checksum"`
			Location struct {
				Href string `xml:"href,attr"`
			} `xml:"location"`
			Timestamp float64 `xml:"timestamp"`
		}
	}

	var tree xmlTree
	err := xml.Unmarshal(data, &tree)
	if err != nil {
		return nil, err
	}

	db := make(map[string]RepoMD)
	for _, data := range tree.Data {
		sec := int64(math.Floor(data.Timestamp))
		nsec := int64((data.Timestamp - float64(sec)) * 1e9)
		db[data.Type] = RepoMD{
			Checksum:  data.Checksum,
			Timestamp: time.Unix(sec, nsec),
			Location:  data.Location.Href,
		}
		repo.msg.Infof(">>> %s: %v\n", data.Type, db[data.Type])
	}
	return db, err
}

type RepoMD struct {
	Checksum  string
	Timestamp time.Time
	Location  string
}

// EOF
<commit_msg>yum: debug printouts<commit_after>package yum

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/gonuts/logger"
)

// global registry of known backends
var g_backends = make(map[string]func(repo *Repository) (Backend, error))

// NewBackend returns a new backend of type "backend"
func NewBackend(backend string, repo *Repository) (Backend, error) {
	factory, ok := g_backends[backend]
	if !ok {
		return nil, fmt.Errorf("yum: no such backend [%s]", backend)
	}
	return factory(repo)
}

// Backend queries a YUM DB repository
type Backend interface {

	// YumDataType returns the ID for the data type as used in the repomd.xml file
	YumDataType() string

	// Download the DB from server
	GetLatestDB(url string) error

	// Check whether the DB is there
	HasDB() bool

	// Load loads the DB
	LoadDB() error

	// FindLatestMatchingName locats a package by name, returns the latest available version.
	FindLatestMatchingName(name, version, release string) (*Package, error)

	// FindLatestMatchingRequire locates a package providing a given functionality.
	FindLatestMatchingRequire(requirement string) (*Package, error)

	// GetPackages returns all the packages known by a YUM repository
	GetPackages() []*Package
}

// Repository represents a YUM repository with all associated metadata.
type Repository struct {
	msg            *logger.Logger
	Name           string
	RepoUrl        string
	RepoMdUrl      string
	LocalRepoMdXml string
	CacheDir       string
	Backends       []string
	Backend        Backend
}

// NewRepository create a new Repository with name and from url.
func NewRepository(name, url, cachedir string, backends []string, setupBackend, checkForUpdates bool) (*Repository, error) {

	repo := Repository{
		msg:            logger.NewLogger("yum", logger.INFO, os.Stdout),
		Name:           name,
		RepoUrl:        url,
		RepoMdUrl:      url + "/repodata/repomd.xml",
		LocalRepoMdXml: filepath.Join(cachedir, "repomd.xml"),
		CacheDir:       cachedir,
		Backends:       make([]string, len(backends)),
	}
	copy(repo.Backends, backends)

	err := os.MkdirAll(cachedir, 0644)
	if err != nil {
		return nil, err
	}

	// load appropriate backend if requested
	if setupBackend {
		if checkForUpdates {
			err = repo.setupBackendFromRemote()
			if err != nil {
				return nil, err
			}
		} else {
			err = repo.setupBackendFromLocal()
			if err != nil {
				return nil, err
			}
		}
	}
	return &repo, err
}

// FindLatestMatchingName locats a package by name, returns the latest available version.
func (repo *Repository) FindLatestMatchingName(name, version, release string) (*Package, error) {
	return repo.Backend.FindLatestMatchingName(name, version, release)
}

// FindLatestMatchingRequire locates a package providing a given functionality.
func (repo *Repository) FindLatestMatchingRequire(requirement string) (*Package, error) {
	return repo.Backend.FindLatestMatchingRequire(requirement)
}

// GetPackages returns all the packages known by a YUM repository
func (repo *Repository) GetPackages() []*Package {
	return repo.Backend.GetPackages()
}

// setupBackendFromRemote checks which backend should be used and updates the DB files.
func (repo *Repository) setupBackendFromRemote() error {
	repo.msg.Infof("setupBackendFromRemote...\n")
	var err error
	var backend Backend
	// get repo metadata with list of available files
	remotedata, err := repo.remoteMetadata()
	if err != nil {
		return err
	}

	remotemd, err := repo.checkRepoMD(remotedata)
	if err != nil {
		return err
	}

	localdata, err := repo.localMetadata()
	if err != nil {
		return err
	}

	localmd, err := repo.checkRepoMD(localdata)
	if err != nil {
		return err
	}

	for _, bname := range repo.Backends {
		repo.msg.Infof("checking availability of backend [%s]\n", bname)
		ba, err := NewBackend(bname, repo)
		if err != nil {
			continue
		}

		rrepomd, ok := remotemd[ba.YumDataType()]
		if !ok {
			repo.msg.Warnf("remote repository does not provide [%s] DB\n", bname)
			continue
		}

		// a priori a match
		backend = ba
		repo.Backend = backend

		lrepomd, ok := localmd[ba.YumDataType()]
		if !ok {
			// doesn't matter, we download the DB in any case
		}

		if !repo.Backend.HasDB() || rrepomd.Timestamp.After(lrepomd.Timestamp) {
			// we need to update the DB
			url := repo.RepoUrl + "/" + rrepomd.Location
			repo.msg.Infof("updating the RPM database for %s\n", bname)
			err = repo.Backend.GetLatestDB(url)
			if err != nil {
				repo.msg.Warnf("problem updating RPM database for backend [%s]: %v\n", bname, err)
				err = nil
				backend = nil
				repo.Backend = nil
				continue
			}
			// save metadata to local repomd file
			err = ioutil.WriteFile(repo.LocalRepoMdXml, remotedata, 0644)
			if err != nil {
				repo.msg.Warnf("problem updating local repomd.xml file for backend [%s]: %v\n", bname, err)
				err = nil
				backend = nil
				repo.Backend = nil
				continue
			}
		}

		// load data necessary for the backend
		err = repo.Backend.LoadDB()
		if err != nil {
			repo.msg.Warnf("problem loading data for backend [%s]: %v\n", bname, err)
			err = nil
			backend = nil
			repo.Backend = nil
			continue
		}

		// stop at first one found
		break
	}

	if backend == nil {
		repo.msg.Errorf("No valid backend found\n")
		return fmt.Errorf("No valid backend found")
	}

	repo.msg.Infof("repository [%s] - chosen backend [%T]\n", repo.Name, repo.Backend)
	return err
}

func (repo *Repository) setupBackendFromLocal() error {
	repo.msg.Infof("setupBackendFromLocal...\n")
	var err error
	data, err := repo.localMetadata()
	if err != nil {
		return err
	}

	md, err := repo.checkRepoMD(data)
	if err != nil {
		return err
	}

	var backend Backend
	for _, bname := range repo.Backends {
		repo.msg.Infof("checking availability of backend [%s]\n", bname)
		ba, err := NewBackend(bname, repo)
		if err != nil {
			continue
		}
		_ /*repomd*/, ok := md[ba.YumDataType()]
		if !ok {
			repo.msg.Warnf("local repository does not provide [%s] DB\n", bname)
			continue
		}

		// a priori a match
		backend = ba
		repo.Backend = backend

		// loading data necessary for the backend
		err = repo.Backend.LoadDB()
		if err != nil {
			repo.msg.Warnf("problem loading data for backend [%s]: %v\n", bname, err)
			err = nil
			backend = nil
			repo.Backend = nil
			continue
		}

		// stop at first one found.
		break
	}

	if backend == nil {
		repo.msg.Errorf("No valid backend found\n")
		return fmt.Errorf("No valid backend found")
	}

	repo.msg.Infof("repository [%s] - chosen backend [%T]\n", repo.Name, repo.Backend)
	return err
}

// remoteMetadata retrieves the repo metadata file content
func (repo *Repository) remoteMetadata() ([]byte, error) {
	resp, err := http.Get(repo.RepoMdUrl)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	buf := new(bytes.Buffer)
	_, err = io.Copy(buf, resp.Body)
	if err != nil && err != io.EOF {
		return nil, err
	}
	return buf.Bytes(), err
}

// localMetadata retrieves the repo metadata from the repomd file
func (repo *Repository) localMetadata() ([]byte, error) {
	if !path_exists(repo.LocalRepoMdXml) {
		return nil, nil
	}
	f, err := os.Open(repo.LocalRepoMdXml)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	buf := new(bytes.Buffer)
	_, err = io.Copy(buf, f)
	if err != nil && err != io.EOF {
		return nil, err
	}
	return buf.Bytes(), err
}

// checkRepoMD parses the Repository metadata XML content
func (repo *Repository) checkRepoMD(data []byte) (map[string]RepoMD, error) {

	if len(data) <= 0 {
		return nil, nil
	}

	type xmlTree struct {
		XMLName xml.Name `xml:"repomd"`
		Data    []struct {
			Type     string `xml:"type,attr"`
			Checksum string `xml:"checksum"`
			Location struct {
				Href string `xml:"href,attr"`
			} `xml:"location"`
			Timestamp float64 `xml:"timestamp"`
		}
	}

	var tree xmlTree
	err := xml.Unmarshal(data, &tree)
	if err != nil {
		return nil, err
	}

	db := make(map[string]RepoMD)
	for _, data := range tree.Data {
		sec := int64(math.Floor(data.Timestamp))
		nsec := int64((data.Timestamp - float64(sec)) * 1e9)
		db[data.Type] = RepoMD{
			Checksum:  data.Checksum,
			Timestamp: time.Unix(sec, nsec),
			Location:  data.Location.Href,
		}
		repo.msg.Infof(">>> %s: %v\n", data.Type, db[data.Type])
	}
	return db, err
}

type RepoMD struct {
	Checksum  string
	Timestamp time.Time
	Location  string
}

// EOF
<|endoftext|>


<commit_before>package lib

import (
	mesos "github.com/boldfield/go-mesos"
	"log"
)

type FrameworkMap map[string]*mesos.Framework

type Mesos struct {
	Host        string
	Cluster     *mesos.Cluster
	_client     *mesos.Client
	initialized bool
}

type MesosSlave struct {
	Slave *mesos.Slave
}

func (m *Mesos) Client() *mesos.Client {
	if m._client != nil {
		return m._client
	}

	config := mesos.NewDefaultConfig()
	config.DiscoveryURL = m.Host
	m._client = mesos.NewClient(config)
	return m._client
}

// Loads the entire cluster, including information about slaves
func (m *Mesos) LoadCluster(c *mesos.Client) error {
	if err := m.LoadClusterInfo(c); err != nil {
		return err
	}

	if err := m.Cluster.LoadSlaveStates(c); err != nil {
		log.Printf("An error was encountered loading slave states: %s", err)
		return err
	}

	if err := m.Cluster.LoadSlaveStats(c); err != nil {
		log.Printf("An error was encountered loading slave states: %s", err)
		return err
	}
	return nil
}

// Loads information about the cluster from the master, does not check slaves
// TODO: (michaelb) what should this be called?
func (m *Mesos) LoadClusterInfo(c *mesos.Client) error {
	if cluster, err := mesos.DiscoverCluster(c); err != nil {
		log.Println(err)
		return err
	} else {
		m.Cluster = cluster
	}
	return nil
}

func (m *Mesos) Framework(framework string) FrameworkMap {
	return m.Cluster.GetFramework(framework)
}

func (m *Mesos) LoadSlaveState(host string, c *mesos.Client) (*MesosSlave, error) {
	slave := &mesos.Slave{HostName: host}
	err := slave.LoadState(c)
	return &MesosSlave{Slave: slave}, err
}

func (m *Mesos) LoadSlaveStats(host string, c *mesos.Client) (*MesosSlave, error) {
	slave := &mesos.Slave{HostName: host}
	err := slave.LoadStats(c)
	return &MesosSlave{Slave: slave}, err
}

func (s *MesosSlave) Framework(framework string) FrameworkMap {
	return s.Slave.GetFramework(framework)
}
<commit_msg>remove TODO<commit_after>package lib

import (
	mesos "github.com/boldfield/go-mesos"
	"log"
)

type FrameworkMap map[string]*mesos.Framework

type Mesos struct {
	Host        string
	Cluster     *mesos.Cluster
	_client     *mesos.Client
	initialized bool
}

type MesosSlave struct {
	Slave *mesos.Slave
}

func (m *Mesos) Client() *mesos.Client {
	if m._client != nil {
		return m._client
	}

	config := mesos.NewDefaultConfig()
	config.DiscoveryURL = m.Host
	m._client = mesos.NewClient(config)
	return m._client
}

// Loads the entire cluster, including information about slaves
func (m *Mesos) LoadCluster(c *mesos.Client) error {
	if err := m.LoadClusterInfo(c); err != nil {
		return err
	}

	if err := m.Cluster.LoadSlaveStates(c); err != nil {
		log.Printf("An error was encountered loading slave states: %s", err)
		return err
	}

	if err := m.Cluster.LoadSlaveStats(c); err != nil {
		log.Printf("An error was encountered loading slave states: %s", err)
		return err
	}
	return nil
}

// Loads information about the cluster from the master, does not check slaves
func (m *Mesos) LoadClusterInfo(c *mesos.Client) error {
	if cluster, err := mesos.DiscoverCluster(c); err != nil {
		log.Println(err)
		return err
	} else {
		m.Cluster = cluster
	}
	return nil
}

func (m *Mesos) Framework(framework string) FrameworkMap {
	return m.Cluster.GetFramework(framework)
}

func (m *Mesos) LoadSlaveState(host string, c *mesos.Client) (*MesosSlave, error) {
	slave := &mesos.Slave{HostName: host}
	err := slave.LoadState(c)
	return &MesosSlave{Slave: slave}, err
}

func (m *Mesos) LoadSlaveStats(host string, c *mesos.Client) (*MesosSlave, error) {
	slave := &mesos.Slave{HostName: host}
	err := slave.LoadStats(c)
	return &MesosSlave{Slave: slave}, err
}

func (s *MesosSlave) Framework(framework string) FrameworkMap {
	return s.Slave.GetFramework(framework)
}
<|endoftext|>


<commit_before>package lib

import (
	"fmt"
	"log"
	"net"

	"github.com/bruceadowns/syslogparser"
	"github.com/bruceadowns/syslogparser/mako"
	"github.com/bruceadowns/syslogparser/rfc3164"
	"github.com/bruceadowns/syslogparser/rfc3164raw"
	"github.com/bruceadowns/syslogparser/rfc5424"
	"github.com/bruceadowns/syslogparser/rfc5424raw"
	"github.com/bruceadowns/syslogparser/syslogmako"
)

// Packet holds the incoming traffic info
type Packet struct {
	Address  net.Addr
	Message  []byte
	LogEvent *LogEvent
}

var remoteTypeCache = make(map[string]string)

func (p *Packet) String() string {
	return fmt.Sprintf("Address: %s '%s'", p.Address, p.Message)
}

// IsValid returns T/F
func (p *Packet) IsValid() bool {
	if len(p.Address.String()) == 0 {
		log.Print("Address is empty")
		return false
	}

	if len(p.Message) == 0 {
		log.Print("Message is empty")
		return false
	}

	return true
}

func populate(p syslogparser.LogParser) (res *LogEvent) {
	if p == nil {
		return
	}

	logParts := p.Dump()

	app := logParts["app_name"]
	if len(app) == 0 {
		app = logParts["service_name"]
	}

	pid := logParts["service_version"]
	if len(pid) == 0 {
		pid = logParts["proc_id"]
		if len(pid) == 0 {
			pid = logParts["tag"]
		}
	}

	version := logParts["@version"]
	if len(version) == 0 {
		version = logParts["version"]
	}

	message := logParts["message"]
	if len(message) == 0 {
		message = logParts["content"]
	}

	timestamp := logParts["@timestamp"]
	if len(timestamp) == 0 {
		timestamp = logParts["timestamp"]
	}

	res = &LogEvent{
		DataCenter: logParts["service_environment"],
		Cluster:    logParts["service_pipeline"],
		Host:       logParts["hostname"],
		Service:    app,
		Instance:   pid,
		Version:    version,
		Level:      logParts["level"],
		ThreadName: logParts["thread_name"],
		LoggerName: logParts["logger_name"],
		Message:    message,
		Timestamp:  timestamp,
		//ThrownStackTrace
	}

	return
}

// Mill determines message type and parses into a LogEvent
func (p *Packet) Mill() (res *LogEvent) {
	log.Printf("%s", p)

	var parser syslogparser.LogParser

	for {
		// check cache for known type

		{
			parser = mako.NewParser(p.Message, p.Address.String())
			if err := parser.Parse(); err == nil {
				remoteTypeCache[p.Address.String()] = "mako"
				log.Printf("mako")
				break
			}
		}

		{
			parser = syslogmako.NewParser(p.Message)
			if err := parser.Parse(); err == nil {
				remoteTypeCache[p.Address.String()] = "syslogmako"
				log.Printf("syslogmako")
				break
			}
		}

		{
			parser = rfc5424raw.NewParser(p.Message)
			if err := parser.Parse(); err == nil {
				remoteTypeCache[p.Address.String()] = "rfc5424raw"
				log.Printf("rfc5424raw")
				break
			}
		}

		{
			parser = rfc3164raw.NewParser(p.Message)
			if err := parser.Parse(); err == nil {
				remoteTypeCache[p.Address.String()] = "rfc3164raw"
				log.Printf("rfc3164raw")
				break
			}
		}

		{
			parser = rfc3164.NewParser(p.Message)
			if err := parser.Parse(); err == nil {
				remoteTypeCache[p.Address.String()] = "rfc3164"
				log.Printf("rfc3164")
				break
			}
		}

		{
			parser = rfc5424.NewParser(p.Message)
			if err := parser.Parse(); err == nil {
				remoteTypeCache[p.Address.String()] = "rfc5424"
				log.Printf("rfc5424")
				break
			}
		}

		log.Printf("none")
		parser = nil
		break
	}

	res = populate(parser)
	if res == nil {
		log.Printf("Message from %s not parsed: %s", p.Address, p.Message)
	} else {
		log.Print(res)
	}

	return
}
<commit_msg>Added stack trace json field<commit_after>package lib

import (
	"fmt"
	"log"
	"net"
	"strings"

	"github.com/bruceadowns/syslogparser"
	"github.com/bruceadowns/syslogparser/mako"
	"github.com/bruceadowns/syslogparser/rfc3164"
	"github.com/bruceadowns/syslogparser/rfc3164raw"
	"github.com/bruceadowns/syslogparser/rfc5424"
	"github.com/bruceadowns/syslogparser/rfc5424raw"
	"github.com/bruceadowns/syslogparser/syslogmako"
)

// Packet holds the incoming traffic info
type Packet struct {
	Address  net.Addr
	Message  []byte
	LogEvent *LogEvent
}

var remoteTypeCache = make(map[string]string)

func (p *Packet) String() string {
	return fmt.Sprintf("Address: %s '%s'", p.Address, p.Message)
}

// IsValid returns T/F
func (p *Packet) IsValid() bool {
	if len(p.Address.String()) == 0 {
		log.Print("Address is empty")
		return false
	}

	if len(p.Message) == 0 {
		log.Print("Message is empty")
		return false
	}

	return true
}

func populate(p syslogparser.LogParser) (res *LogEvent) {
	if p == nil {
		return
	}

	logParts := p.Dump()

	app := logParts["app_name"]
	if len(app) == 0 {
		app = logParts["service_name"]
	}

	pid := logParts["service_version"]
	if len(pid) == 0 {
		pid = logParts["proc_id"]
		if len(pid) == 0 {
			pid = logParts["tag"]
		}
	}

	version := logParts["@version"]
	if len(version) == 0 {
		version = logParts["version"]
	}

	message := logParts["message"]
	if len(message) == 0 {
		message = logParts["content"]
	}

	timestamp := logParts["@timestamp"]
	if len(timestamp) == 0 {
		timestamp = logParts["timestamp"]
	}

	stackTrace := []string{}
	fullStackTrace := logParts["stack_trace"]
	if len(fullStackTrace) > 0 {
		stackTrace = strings.Split(fullStackTrace, "\n")
	}

	res = &LogEvent{
		DataCenter:       logParts["service_environment"],
		Cluster:          logParts["service_pipeline"],
		Host:             logParts["hostname"],
		Service:          app,
		Instance:         pid,
		Version:          version,
		Level:            logParts["level"],
		ThreadName:       logParts["thread_name"],
		LoggerName:       logParts["logger_name"],
		Message:          message,
		Timestamp:        timestamp,
		ThrownStackTrace: stackTrace,
	}

	return
}

// Mill determines message type and parses into a LogEvent
func (p *Packet) Mill() (res *LogEvent) {
	log.Printf("%s", p)

	var parser syslogparser.LogParser

	for {
		// check cache for known type

		{
			parser = mako.NewParser(p.Message, p.Address.String())
			if err := parser.Parse(); err == nil {
				remoteTypeCache[p.Address.String()] = "mako"
				log.Printf("mako")
				break
			}
		}

		{
			parser = syslogmako.NewParser(p.Message)
			if err := parser.Parse(); err == nil {
				remoteTypeCache[p.Address.String()] = "syslogmako"
				log.Printf("syslogmako")
				break
			}
		}

		{
			parser = rfc5424raw.NewParser(p.Message)
			if err := parser.Parse(); err == nil {
				remoteTypeCache[p.Address.String()] = "rfc5424raw"
				log.Printf("rfc5424raw")
				break
			}
		}

		{
			parser = rfc3164raw.NewParser(p.Message)
			if err := parser.Parse(); err == nil {
				remoteTypeCache[p.Address.String()] = "rfc3164raw"
				log.Printf("rfc3164raw")
				break
			}
		}

		{
			parser = rfc3164.NewParser(p.Message)
			if err := parser.Parse(); err == nil {
				remoteTypeCache[p.Address.String()] = "rfc3164"
				log.Printf("rfc3164")
				break
			}
		}

		{
			parser = rfc5424.NewParser(p.Message)
			if err := parser.Parse(); err == nil {
				remoteTypeCache[p.Address.String()] = "rfc5424"
				log.Printf("rfc5424")
				break
			}
		}

		log.Printf("none")
		parser = nil
		break
	}

	res = populate(parser)
	if res == nil {
		log.Printf("Message from %s not parsed: %s", p.Address, p.Message)
	} else {
		log.Print(res)
	}

	return
}
<|endoftext|>