<commit_before>// go-camo daemon (go-camod)
package main

import (
	"errors"
	"fmt"
	"github.com/cactus/go-camo/camoproxy/encoding"
	flags "github.com/jessevdk/go-flags"
	"log"
	"net/url"
	"os"
	"strings"
)

type Command struct {
	Name    string
}

func (c *Command) Usage() string {
	return fmt.Sprintf("[%s-OPTIONS] URL", c.Name)
}

func (c *Command) Execute(args []string) error {
	// clear log prefix -- not needed for tool
	log.SetFlags(0)

	if opts.HmacKey == "" {
		return errors.New("Empty HMAC")
	}

	if len(args) == 0 {
		return errors.New("No url argument provided")
	}

	oUrl := args[0]
	if oUrl == "" {
		return errors.New("No url argument provided")
	}

	hmacKeyBytes := []byte(opts.HmacKey)

	switch c.Name {
	case "encode":
		outUrl := encoding.EncodeUrl(&hmacKeyBytes, oUrl)
		fmt.Println(opts.Prefix + outUrl)
		return nil
	case "decode":
		u, err := url.Parse(oUrl)
		if err != nil {
			return err
		}
		comp := strings.SplitN(u.Path, "/", 3)
		decUrl, valid := encoding.DecodeUrl(&hmacKeyBytes, comp[1], comp[2])
		if !valid {
			return errors.New("hmac is invalid")
		}
		log.Println(decUrl)
		return nil
	}
	return errors.New("unknown command")
}

var opts struct {
	HmacKey string `short:"k" long:"key" description:"HMAC key"`
	Prefix  string `short:"p" long:"prefix" default:"" description:"Optional url prefix used by encode output"`
}

func main() {
	parser := flags.NewParser(&opts, flags.Default)
	parser.AddCommand("encode", "Encode a url and print result",
					  "Encode a url and print result", &Command{Name:"encode"})
	parser.AddCommand("decode", "Decode a url and print result",
					  "Decode a url and print result", &Command{Name:"decode"})

	// parse said flags
	_, err := parser.Parse()
	if err != nil {
		if e, ok := err.(*flags.Error); ok {
			if e.Type == flags.ErrHelp {
				os.Exit(0)
			}
		}
		os.Exit(1)
	}
}
<commit_msg>gofmt<commit_after>// go-camo daemon (go-camod)
package main

import (
	"errors"
	"fmt"
	"github.com/cactus/go-camo/camoproxy/encoding"
	flags "github.com/jessevdk/go-flags"
	"log"
	"net/url"
	"os"
	"strings"
)

type Command struct {
	Name string
}

func (c *Command) Usage() string {
	return fmt.Sprintf("[%s-OPTIONS] URL", c.Name)
}

func (c *Command) Execute(args []string) error {
	// clear log prefix -- not needed for tool
	log.SetFlags(0)

	if opts.HmacKey == "" {
		return errors.New("Empty HMAC")
	}

	if len(args) == 0 {
		return errors.New("No url argument provided")
	}

	oUrl := args[0]
	if oUrl == "" {
		return errors.New("No url argument provided")
	}

	hmacKeyBytes := []byte(opts.HmacKey)

	switch c.Name {
	case "encode":
		outUrl := encoding.EncodeUrl(&hmacKeyBytes, oUrl)
		fmt.Println(opts.Prefix + outUrl)
		return nil
	case "decode":
		u, err := url.Parse(oUrl)
		if err != nil {
			return err
		}
		comp := strings.SplitN(u.Path, "/", 3)
		decUrl, valid := encoding.DecodeUrl(&hmacKeyBytes, comp[1], comp[2])
		if !valid {
			return errors.New("hmac is invalid")
		}
		log.Println(decUrl)
		return nil
	}
	return errors.New("unknown command")
}

var opts struct {
	HmacKey string `short:"k" long:"key" description:"HMAC key"`
	Prefix  string `short:"p" long:"prefix" default:"" description:"Optional url prefix used by encode output"`
}

func main() {
	parser := flags.NewParser(&opts, flags.Default)
	parser.AddCommand("encode", "Encode a url and print result",
		"Encode a url and print result", &Command{Name: "encode"})
	parser.AddCommand("decode", "Decode a url and print result",
		"Decode a url and print result", &Command{Name: "decode"})

	// parse said flags
	_, err := parser.Parse()
	if err != nil {
		if e, ok := err.(*flags.Error); ok {
			if e.Type == flags.ErrHelp {
				os.Exit(0)
			}
		}
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package util

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestHash(t *testing.T) {
	// Given
	var key Key = "123"

	// When
	result := key.Hash()

	// Then
	assert.Equal(t, 1916298011, result)
}
<commit_msg>fix Key unit test<commit_after>package util

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestHash(t *testing.T) {
	// Given
	var key Key = []byte("123")

	// When
	result := key.Hash()

	// Then
	assert.Equal(t, 1916298011, result)
}
<|endoftext|>


<commit_before>package util

import (
	"time"
)

func Uts() int64 {
	return time.Now().UnixNano() / 1000000
}
<commit_msg>Just a shortcut of a unix nano call.<commit_after>package util

import (
	"time"
)

func Uts() int64 {
	return time.Now().UnixNano() / 1000000
}

func UnixNanoTs() int64 {
	return time.Now().UnixNano()
}
<|endoftext|>


<commit_before>package agent

import (
	"bytes"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/hashicorp/consul/testutil"
)

func TestEventFire(t *testing.T) {
	httpTest(t, func(srv *HTTPServer) {
		body := bytes.NewBuffer([]byte("test"))
		url := "/v1/event/fire/test?node=Node&service=foo&tag=bar"
		req, err := http.NewRequest("PUT", url, body)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		resp := httptest.NewRecorder()
		obj, err := srv.EventFire(resp, req)
		if err != nil {
			t.Fatalf("err: %v", err)
		}

		event, ok := obj.(*UserEvent)
		if !ok {
			t.Fatalf("bad: %#v", obj)
		}

		if event.ID == "" {
			t.Fatalf("bad: %#v", event)
		}
		if event.Name != "test" {
			t.Fatalf("bad: %#v", event)
		}
		if string(event.Payload) != "test" {
			t.Fatalf("bad: %#v", event)
		}
		if event.NodeFilter != "Node" {
			t.Fatalf("bad: %#v", event)
		}
		if event.ServiceFilter != "foo" {
			t.Fatalf("bad: %#v", event)
		}
		if event.TagFilter != "bar" {
			t.Fatalf("bad: %#v", event)
		}
	})
}

func TestEventList(t *testing.T) {
	httpTest(t, func(srv *HTTPServer) {
		p := &UserEvent{Name: "test"}
		if err := srv.agent.UserEvent("", p); err != nil {
			t.Fatalf("err: %v", err)
		}

		testutil.WaitForResult(func() (bool, error) {
			req, err := http.NewRequest("GET", "/v1/event/list", nil)
			if err != nil {
				return false, err
			}
			resp := httptest.NewRecorder()
			obj, err := srv.EventList(resp, req)
			if err != nil {
				return false, err
			}

			list, ok := obj.([]*UserEvent)
			if !ok {
				return false, fmt.Errorf("bad: %#v", obj)
			}
			if len(list) != 1 || list[0].Name != "test" {
				return false, fmt.Errorf("bad: %#v", list)
			}
			header := resp.Header().Get("X-Consul-Index")
			if header == "" || header == "0" {
				return false, fmt.Errorf("bad: %#v", header)
			}
			return true, nil
		}, func(err error) {
			t.Fatalf("err: %v", err)
		})
	})
}

func TestEventList_Filter(t *testing.T) {
	httpTest(t, func(srv *HTTPServer) {
		p := &UserEvent{Name: "test"}
		if err := srv.agent.UserEvent("", p); err != nil {
			t.Fatalf("err: %v", err)
		}

		p = &UserEvent{Name: "foo"}
		if err := srv.agent.UserEvent("", p); err != nil {
			t.Fatalf("err: %v", err)
		}

		testutil.WaitForResult(func() (bool, error) {
			req, err := http.NewRequest("GET", "/v1/event/list?name=foo", nil)
			if err != nil {
				return false, err
			}
			resp := httptest.NewRecorder()
			obj, err := srv.EventList(resp, req)
			if err != nil {
				return false, err
			}

			list, ok := obj.([]*UserEvent)
			if !ok {
				return false, fmt.Errorf("bad: %#v", obj)
			}
			if len(list) != 1 || list[0].Name != "foo" {
				return false, fmt.Errorf("bad: %#v", list)
			}
			header := resp.Header().Get("X-Consul-Index")
			if header == "" || header == "0" {
				return false, fmt.Errorf("bad: %#v", header)
			}
			return true, nil
		}, func(err error) {
			t.Fatalf("err: %v", err)
		})
	})
}

func TestEventList_Blocking(t *testing.T) {
	httpTest(t, func(srv *HTTPServer) {
		p := &UserEvent{Name: "test"}
		if err := srv.agent.UserEvent("", p); err != nil {
			t.Fatalf("err: %v", err)
		}

		var index string
		testutil.WaitForResult(func() (bool, error) {
			req, err := http.NewRequest("GET", "/v1/event/list", nil)
			if err != nil {
				return false, err
			}
			resp := httptest.NewRecorder()
			_, err = srv.EventList(resp, req)
			if err != nil {
				return false, err
			}
			header := resp.Header().Get("X-Consul-Index")
			if header == "" || header == "0" {
				return false, fmt.Errorf("bad: %#v", header)
			}
			index = header
			return true, nil
		}, func(err error) {
			t.Fatalf("err: %v", err)
		})

		go func() {
			time.Sleep(50 * time.Millisecond)
			p := &UserEvent{Name: "second"}
			if err := srv.agent.UserEvent("", p); err != nil {
				t.Fatalf("err: %v", err)
			}
		}()

		testutil.WaitForResult(func() (bool, error) {
			url := "/v1/event/list?index=" + index
			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				return false, err
			}
			resp := httptest.NewRecorder()
			obj, err := srv.EventList(resp, req)
			if err != nil {
				return false, err
			}

			list, ok := obj.([]*UserEvent)
			if !ok {
				return false, fmt.Errorf("bad: %#v", obj)
			}
			if len(list) != 2 || list[1].Name != "second" {
				return false, fmt.Errorf("bad: %#v", list)
			}
			return true, nil
		}, func(err error) {
			t.Fatalf("err: %v", err)
		})
	})
}

func TestUUIDToUint64(t *testing.T) {
	inp := "cb9a81ad-fff6-52ac-92a7-5f70687805ec"

	// Output value was computed using python
	if uuidToUint64(inp) != 6430540886266763072 {
		t.Fatalf("bad")
	}
}
<commit_msg>agent: Test event order preservation for watches<commit_after>package agent

import (
	"bytes"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/hashicorp/consul/testutil"
)

func TestEventFire(t *testing.T) {
	httpTest(t, func(srv *HTTPServer) {
		body := bytes.NewBuffer([]byte("test"))
		url := "/v1/event/fire/test?node=Node&service=foo&tag=bar"
		req, err := http.NewRequest("PUT", url, body)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
		resp := httptest.NewRecorder()
		obj, err := srv.EventFire(resp, req)
		if err != nil {
			t.Fatalf("err: %v", err)
		}

		event, ok := obj.(*UserEvent)
		if !ok {
			t.Fatalf("bad: %#v", obj)
		}

		if event.ID == "" {
			t.Fatalf("bad: %#v", event)
		}
		if event.Name != "test" {
			t.Fatalf("bad: %#v", event)
		}
		if string(event.Payload) != "test" {
			t.Fatalf("bad: %#v", event)
		}
		if event.NodeFilter != "Node" {
			t.Fatalf("bad: %#v", event)
		}
		if event.ServiceFilter != "foo" {
			t.Fatalf("bad: %#v", event)
		}
		if event.TagFilter != "bar" {
			t.Fatalf("bad: %#v", event)
		}
	})
}

func TestEventList(t *testing.T) {
	httpTest(t, func(srv *HTTPServer) {
		p := &UserEvent{Name: "test"}
		if err := srv.agent.UserEvent("", p); err != nil {
			t.Fatalf("err: %v", err)
		}

		testutil.WaitForResult(func() (bool, error) {
			req, err := http.NewRequest("GET", "/v1/event/list", nil)
			if err != nil {
				return false, err
			}
			resp := httptest.NewRecorder()
			obj, err := srv.EventList(resp, req)
			if err != nil {
				return false, err
			}

			list, ok := obj.([]*UserEvent)
			if !ok {
				return false, fmt.Errorf("bad: %#v", obj)
			}
			if len(list) != 1 || list[0].Name != "test" {
				return false, fmt.Errorf("bad: %#v", list)
			}
			header := resp.Header().Get("X-Consul-Index")
			if header == "" || header == "0" {
				return false, fmt.Errorf("bad: %#v", header)
			}
			return true, nil
		}, func(err error) {
			t.Fatalf("err: %v", err)
		})
	})
}

func TestEventList_Filter(t *testing.T) {
	httpTest(t, func(srv *HTTPServer) {
		p := &UserEvent{Name: "test"}
		if err := srv.agent.UserEvent("", p); err != nil {
			t.Fatalf("err: %v", err)
		}

		p = &UserEvent{Name: "foo"}
		if err := srv.agent.UserEvent("", p); err != nil {
			t.Fatalf("err: %v", err)
		}

		testutil.WaitForResult(func() (bool, error) {
			req, err := http.NewRequest("GET", "/v1/event/list?name=foo", nil)
			if err != nil {
				return false, err
			}
			resp := httptest.NewRecorder()
			obj, err := srv.EventList(resp, req)
			if err != nil {
				return false, err
			}

			list, ok := obj.([]*UserEvent)
			if !ok {
				return false, fmt.Errorf("bad: %#v", obj)
			}
			if len(list) != 1 || list[0].Name != "foo" {
				return false, fmt.Errorf("bad: %#v", list)
			}
			header := resp.Header().Get("X-Consul-Index")
			if header == "" || header == "0" {
				return false, fmt.Errorf("bad: %#v", header)
			}
			return true, nil
		}, func(err error) {
			t.Fatalf("err: %v", err)
		})
	})
}

func TestEventList_Blocking(t *testing.T) {
	httpTest(t, func(srv *HTTPServer) {
		p := &UserEvent{Name: "foo"}
		if err := srv.agent.UserEvent("", p); err != nil {
			t.Fatalf("err: %v", err)
		}

		var index string
		testutil.WaitForResult(func() (bool, error) {
			req, err := http.NewRequest("GET", "/v1/event/list?name=foo", nil)
			if err != nil {
				return false, err
			}
			resp := httptest.NewRecorder()
			_, err = srv.EventList(resp, req)
			if err != nil {
				return false, err
			}
			header := resp.Header().Get("X-Consul-Index")
			if header == "" || header == "0" {
				return false, fmt.Errorf("bad: %#v", header)
			}
			index = header
			return true, nil
		}, func(err error) {
			t.Fatalf("err: %v", err)
		})

		go func() {
			time.Sleep(50 * time.Millisecond)
			p := &UserEvent{Name: "bar"}
			if err := srv.agent.UserEvent("", p); err != nil {
				t.Fatalf("err: %v", err)
			}
		}()

		testutil.WaitForResult(func() (bool, error) {
			url := "/v1/event/list?name=bar&index=" + index
			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				return false, err
			}
			resp := httptest.NewRecorder()
			obj, err := srv.EventList(resp, req)
			if err != nil {
				return false, err
			}
			header := resp.Header().Get("X-Consul-Index")
			if header == "" || header == "0" {
				return false, fmt.Errorf("bad: %#v", header)
			}
			list, ok := obj.([]*UserEvent)
			if !ok {
				return false, fmt.Errorf("bad: %#v", obj)
			}
			if len(list) != 1 || list[0].Name != "bar" {
				return false, fmt.Errorf("bad: %#v", list)
			}
			index = header
			return true, nil
		}, func(err error) {
			t.Fatalf("err: %v", err)
		})

		// Test again to make sure that the event order is preserved
		// when name filtering on a list of > 1 matching event.
		p = &UserEvent{Name: "bar"}
		if err := srv.agent.UserEvent("", p); err != nil {
			t.Fatalf("err: %v", err)
		}

		testutil.WaitForResult(func() (bool, error) {
			url := "/v1/event/list?name=bar&index=" + index
			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				return false, err
			}
			resp := httptest.NewRecorder()
			obj, err := srv.EventList(resp, req)
			if err != nil {
				return false, err
			}
			header := resp.Header().Get("X-Consul-Index")
			if header == "" || header == "0" {
				return false, fmt.Errorf("bad: %#v", header)
			}
			list, ok := obj.([]*UserEvent)
			if !ok {
				return false, fmt.Errorf("bad: %#v", obj)
			}
			if len(list) != 2 || list[1].Name != "bar" || list[1].ID != p.ID {
				return false, fmt.Errorf("bad: %#v", list)
			}
			index = header
			return true, nil
		}, func(err error) {
			t.Fatalf("err: %v", err)
		})
	})
}

func TestUUIDToUint64(t *testing.T) {
	inp := "cb9a81ad-fff6-52ac-92a7-5f70687805ec"

	// Output value was computed using python
	if uuidToUint64(inp) != 6430540886266763072 {
		t.Fatalf("bad")
	}
}
<|endoftext|>


<commit_before>package supervisor

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/pborman/uuid"
	"github.com/starkandwayne/goutils/log"

	"github.com/starkandwayne/shield/agent"
)

type UpdateOp int

const (
	STOPPED UpdateOp = iota
	FAILED
	OUTPUT
	RESTORE_KEY
	PURGE_ARCHIVE
)

type WorkerUpdate struct {
	Task        uuid.UUID
	Archive     uuid.UUID
	TaskSuccess bool
	Op          UpdateOp
	StoppedAt   time.Time
	Output      string
}

type WorkerRequest struct {
	Operation      string `json:"operation"`
	TargetPlugin   string `json:"target_plugin"`
	TargetEndpoint string `json:"target_endpoint"`
	StorePlugin    string `json:"store_plugin"`
	StoreEndpoint  string `json:"store_endpoint"`
	RestoreKey     string `json:"restore_key"`
}

func worker(id uint, privateKeyFile string, work chan Task, updates chan WorkerUpdate) {
	config, err := agent.ConfigureSSHClient(privateKeyFile)
	if err != nil {
		log.Errorf("worker %d unable to read user key %s: %s; bailing out.\n",
			id, privateKeyFile, err)
		return
	}

	for t := range work {
		client := agent.NewClient(config)

		remote := t.Agent
		if remote == "" {
			updates <- WorkerUpdate{Task: t.UUID, Op: OUTPUT,
				Output: fmt.Sprintf("TASK FAILED!!  no remote agent specified for task %s\n", t.UUID)}
			updates <- WorkerUpdate{Task: t.UUID, Op: FAILED}
			continue
		}

		err = client.Dial(remote)
		if err != nil {
			updates <- WorkerUpdate{Task: t.UUID, Op: OUTPUT,
				Output: fmt.Sprintf("TASK FAILED!!  shield worker %d unable to connect to %s (%s)\n", id, remote, err)}
			updates <- WorkerUpdate{Task: t.UUID, Op: FAILED}
			continue
		}
		defer client.Close()

		// start a command and stream output
		final := make(chan string)
		partial := make(chan string)

		go func(out chan string, up chan WorkerUpdate, t Task, in chan string) {
			var buffer []string
			for {
				s, ok := <-in
				if !ok {
					break
				}

				switch s[0:2] {
				case "O:":
					buffer = append(buffer, s[2:])
				case "E:":
					up <- WorkerUpdate{
						Task:   t.UUID,
						Op:     OUTPUT,
						Output: s[2:] + "\n",
					}
				}
			}
			out <- strings.Join(buffer, "")
			close(out)
		}(final, updates, t, partial)

		command, err := json.Marshal(WorkerRequest{
			Operation:      t.Op.String(),
			TargetPlugin:   t.TargetPlugin,
			TargetEndpoint: t.TargetEndpoint,
			StorePlugin:    t.StorePlugin,
			StoreEndpoint:  t.StoreEndpoint,
			RestoreKey:     t.RestoreKey,
		})
		if err != nil {
			updates <- WorkerUpdate{Task: t.UUID, Op: OUTPUT,
				Output: fmt.Sprintf("TASK FAILED!! shield worker %d was unable to json encode the request bound for remote agent %s (%s)", id, remote, err),
			}
			updates <- WorkerUpdate{Task: t.UUID, Op: FAILED}
			continue
		}
		// exec the command
		var jobFailed bool
		err = client.Run(partial, string(command))
		if err != nil {
			updates <- WorkerUpdate{Task: t.UUID, Op: OUTPUT,
				Output: fmt.Sprintf("TASK FAILED!!  shield worker %d failed to execute the command against the remote agent %s (%s)\n", id, remote, err)}
			jobFailed = true
		}

		out := <-final
		if t.Op == BACKUP {
			// parse JSON from standard output and get the restore key
			// (this might fail, we might not get a key, etc.)
			v := struct {
				Key string
			}{}

			buf := bytes.NewBufferString(out)
			dec := json.NewDecoder(buf)
			err := dec.Decode(&v)

			if err != nil {
				updates <- WorkerUpdate{Task: t.UUID, Op: OUTPUT,
					Output: fmt.Sprintf("WORKER FAILED!!  shield worker %d failed to parse JSON response from remote agent %s (%s)\n", id, remote, err)}

			} else {
				if v.Key != "" {
					updates <- WorkerUpdate{
						Task:        t.UUID,
						Op:          RESTORE_KEY,
						TaskSuccess: !jobFailed,
						Output:      v.Key,
					}
				} else {
					updates <- WorkerUpdate{Task: t.UUID, Op: OUTPUT,
						Output: fmt.Sprintf("TASK FAILED!! No restore key detected in worker %d. Cowardly refusing to create an archive record", id)}
					jobFailed = true
				}
			}
		}

		if t.Op == PURGE && !jobFailed {
			updates <- WorkerUpdate{
				Task:    t.UUID,
				Op:      PURGE_ARCHIVE,
				Archive: t.ArchiveUUID,
			}
		}

		// signal to the supervisor that we finished
		if jobFailed {
			updates <- WorkerUpdate{Task: t.UUID, Op: FAILED, StoppedAt: time.Now()}
		} else {
			updates <- WorkerUpdate{
				Task:      t.UUID,
				Op:        STOPPED,
				StoppedAt: time.Now(),
			}
		}
	}
}

func (s *Supervisor) SpawnWorker() {
	s.nextWorker += 1
	go worker(s.nextWorker, s.PrivateKeyFile, s.workers, s.updates)
}
<commit_msg>Force task failures on failed json parsing of store_key<commit_after>package supervisor

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/pborman/uuid"
	"github.com/starkandwayne/goutils/log"

	"github.com/starkandwayne/shield/agent"
)

type UpdateOp int

const (
	STOPPED UpdateOp = iota
	FAILED
	OUTPUT
	RESTORE_KEY
	PURGE_ARCHIVE
)

type WorkerUpdate struct {
	Task        uuid.UUID
	Archive     uuid.UUID
	TaskSuccess bool
	Op          UpdateOp
	StoppedAt   time.Time
	Output      string
}

type WorkerRequest struct {
	Operation      string `json:"operation"`
	TargetPlugin   string `json:"target_plugin"`
	TargetEndpoint string `json:"target_endpoint"`
	StorePlugin    string `json:"store_plugin"`
	StoreEndpoint  string `json:"store_endpoint"`
	RestoreKey     string `json:"restore_key"`
}

func worker(id uint, privateKeyFile string, work chan Task, updates chan WorkerUpdate) {
	config, err := agent.ConfigureSSHClient(privateKeyFile)
	if err != nil {
		log.Errorf("worker %d unable to read user key %s: %s; bailing out.\n",
			id, privateKeyFile, err)
		return
	}

	for t := range work {
		client := agent.NewClient(config)

		remote := t.Agent
		if remote == "" {
			updates <- WorkerUpdate{Task: t.UUID, Op: OUTPUT,
				Output: fmt.Sprintf("TASK FAILED!!  no remote agent specified for task %s\n", t.UUID)}
			updates <- WorkerUpdate{Task: t.UUID, Op: FAILED}
			continue
		}

		err = client.Dial(remote)
		if err != nil {
			updates <- WorkerUpdate{Task: t.UUID, Op: OUTPUT,
				Output: fmt.Sprintf("TASK FAILED!!  shield worker %d unable to connect to %s (%s)\n", id, remote, err)}
			updates <- WorkerUpdate{Task: t.UUID, Op: FAILED}
			continue
		}
		defer client.Close()

		// start a command and stream output
		final := make(chan string)
		partial := make(chan string)

		go func(out chan string, up chan WorkerUpdate, t Task, in chan string) {
			var buffer []string
			for {
				s, ok := <-in
				if !ok {
					break
				}

				switch s[0:2] {
				case "O:":
					buffer = append(buffer, s[2:])
				case "E:":
					up <- WorkerUpdate{
						Task:   t.UUID,
						Op:     OUTPUT,
						Output: s[2:] + "\n",
					}
				}
			}
			out <- strings.Join(buffer, "")
			close(out)
		}(final, updates, t, partial)

		command, err := json.Marshal(WorkerRequest{
			Operation:      t.Op.String(),
			TargetPlugin:   t.TargetPlugin,
			TargetEndpoint: t.TargetEndpoint,
			StorePlugin:    t.StorePlugin,
			StoreEndpoint:  t.StoreEndpoint,
			RestoreKey:     t.RestoreKey,
		})
		if err != nil {
			updates <- WorkerUpdate{Task: t.UUID, Op: OUTPUT,
				Output: fmt.Sprintf("TASK FAILED!! shield worker %d was unable to json encode the request bound for remote agent %s (%s)", id, remote, err),
			}
			updates <- WorkerUpdate{Task: t.UUID, Op: FAILED}
			continue
		}
		// exec the command
		var jobFailed bool
		err = client.Run(partial, string(command))
		if err != nil {
			updates <- WorkerUpdate{Task: t.UUID, Op: OUTPUT,
				Output: fmt.Sprintf("TASK FAILED!!  shield worker %d failed to execute the command against the remote agent %s (%s)\n", id, remote, err)}
			jobFailed = true
		}

		out := <-final
		if t.Op == BACKUP {
			// parse JSON from standard output and get the restore key
			// (this might fail, we might not get a key, etc.)
			v := struct {
				Key string
			}{}

			buf := bytes.NewBufferString(out)
			dec := json.NewDecoder(buf)
			err := dec.Decode(&v)

			if err != nil {
				jobFailed = true
				updates <- WorkerUpdate{Task: t.UUID, Op: OUTPUT,
					Output: fmt.Sprintf("WORKER FAILED!!  shield worker %d failed to parse JSON response from remote agent %s (%s)\n", id, remote, err)}

			} else {
				if v.Key != "" {
					updates <- WorkerUpdate{
						Task:        t.UUID,
						Op:          RESTORE_KEY,
						TaskSuccess: !jobFailed,
						Output:      v.Key,
					}
				} else {
					jobFailed = true
					updates <- WorkerUpdate{Task: t.UUID, Op: OUTPUT,
						Output: fmt.Sprintf("TASK FAILED!! No restore key detected in worker %d. Cowardly refusing to create an archive record", id)}
				}
			}
		}

		if t.Op == PURGE && !jobFailed {
			updates <- WorkerUpdate{
				Task:    t.UUID,
				Op:      PURGE_ARCHIVE,
				Archive: t.ArchiveUUID,
			}
		}

		// signal to the supervisor that we finished
		if jobFailed {
			updates <- WorkerUpdate{Task: t.UUID, Op: FAILED, StoppedAt: time.Now()}
		} else {
			updates <- WorkerUpdate{
				Task:      t.UUID,
				Op:        STOPPED,
				StoppedAt: time.Now(),
			}
		}
	}
}

func (s *Supervisor) SpawnWorker() {
	s.nextWorker += 1
	go worker(s.nextWorker, s.PrivateKeyFile, s.workers, s.updates)
}
<|endoftext|>


<commit_before>/**********************************************************************************
* Copyright (c) 2009-2017 Misakai Ltd.
* This program is free software: you can redistribute it and/or modify it under the
* terms of the GNU Affero General Public License as published by the  Free Software
* Foundation, either version 3 of the License, or(at your option) any later version.
*
* This program is distributed  in the hope that it  will be useful, but WITHOUT ANY
* WARRANTY;  without even  the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE.  See the GNU Affero General Public License  for  more details.
*
* You should have  received a copy  of the  GNU Affero General Public License along
* with this program. If not, see<http://www.gnu.org/licenses/>.
************************************************************************************/

package broker

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/emitter-io/emitter/internal/message"
	"github.com/emitter-io/emitter/internal/security"
	"github.com/weaveworks/mesh"
)

const (
	idSystem = uint32(0)
	idQuery  = uint32(3939663052)
)

// Surveyee handles the surveys.
type Surveyee interface {
	OnSurvey(queryType string, request []byte) (response []byte, ok bool)
}

// QueryManager represents a request-response manager.
type QueryManager struct {
	service  *Service    // The service to use.
	luid     security.ID // The locally unique id of the manager.
	next     uint32      // The next available query identifier.
	awaiters *sync.Map   // The map of the awaiters.
	handlers []Surveyee  // The handlers array.
}

// newQueryManager creates a new request-response manager.
func newQueryManager(s *Service) *QueryManager {
	return &QueryManager{
		service:  s,
		luid:     security.NewID(),
		next:     0,
		awaiters: new(sync.Map),
		handlers: make([]Surveyee, 0),
	}
}

// Start subscribes the manager to the query channel.
func (c *QueryManager) Start() {
	ssid := message.Ssid{idSystem, idQuery}
	if ok := c.service.onSubscribe(ssid, c); ok {
		c.service.cluster.NotifySubscribe(c.luid, ssid)
	}
}

// HandleFunc adds a handler for a query.
func (c *QueryManager) HandleFunc(surveyees ...Surveyee) {
	for _, h := range surveyees {
		c.handlers = append(c.handlers, h)
	}
}

// ID returns the unique identifier of the subsriber.
func (c *QueryManager) ID() string {
	return c.luid.String()
}

// Type returns the type of the subscriber
func (c *QueryManager) Type() message.SubscriberType {
	return message.SubscriberDirect
}

// Send occurs when we have received a message.
func (c *QueryManager) Send(m *message.Message) error {
	ssid := m.Ssid()
	if len(ssid) != 3 {
		return errors.New("Invalid query received")
	}

	switch string(m.Channel) {
	case "response":
		// We received a response, find the awaiter and forward a message to it
		return c.onResponse(ssid[2], m.Payload)

	default:
		// We received a request, need to handle that by calling the appropriate handler
		return c.onRequest(ssid, string(m.Channel), m.Payload)
	}
}

// onRequest handles an incoming request
func (c *QueryManager) onResponse(id uint32, payload []byte) error {
	if awaiter, ok := c.awaiters.Load(id); ok {
		awaiter.(*queryAwaiter).receive <- payload
	}
	return nil
}

// onRequest handles an incoming request
func (c *QueryManager) onRequest(ssid message.Ssid, channel string, payload []byte) error {
	// Get the query and reply node
	ch := strings.Split(channel, "/")
	query := ch[0]
	reply, err := strconv.ParseInt(ch[1], 10, 64)
	if err != nil {
		return err
	}

	// Do not answer our own requests
	replyAddr := mesh.PeerName(reply)
	if c.service.cluster.ID() == uint64(replyAddr) {
		return nil
	}

	// Get the peer to reply to
	peer := c.service.cluster.FindPeer(replyAddr)
	if !peer.IsActive() {
		return errors.New("unable to reply to a request, peer is not active")
	}

	// Go through all the handlers and execute the first matching one
	for _, surveyee := range c.handlers {
		if response, ok := surveyee.OnSurvey(query, payload); ok {
			return peer.Send(message.New(ssid, []byte("response"), response))
		}
	}

	return errors.New("no query handler found for " + channel)
}

// Query issues a cluster-wide request.
func (c *QueryManager) Query(query string, payload []byte) (message.Awaiter, error) {

	// Create an awaiter
	// TODO: replace the max with the total number of cluster nodes
	awaiter := &queryAwaiter{
		id:      atomic.AddUint32(&c.next, 1),
		receive: make(chan []byte),
		maximum: c.service.NumPeers(),
		manager: c,
	}

	// Store an awaiter
	c.awaiters.Store(awaiter.id, awaiter)

	// Prepare a channel with the reply-to address
	channel := fmt.Sprintf("%v/%v", query, c.service.LocalName())

	// Publish the query as a message
	c.service.publish(message.New(
		message.Ssid{idSystem, idQuery, awaiter.id},
		[]byte(channel),
		payload,
	), "")
	return awaiter, nil
}

// queryAwaiter represents an asynchronously awaiting response channel.
type queryAwaiter struct {
	id      uint32        // The identifier of the query.
	maximum int           // The maximum number of responses to wait for.
	receive chan []byte   // The receive channel to use.
	manager *QueryManager // The query manager used.
}

// Gather awaits for the responses to be received, blocking until we're done.
func (a *queryAwaiter) Gather(timeout time.Duration) (r [][]byte) {
	defer func() { a.manager.awaiters.Delete(a.id) }()
	r = make([][]byte, 0, 4)
	t := time.After(timeout)
	c := a.maximum

	// If there's no peers, no need to receive anything
	if c == 0 {
		return
	}

	for {
		select {
		case msg := <-a.receive:
			r = append(r, msg)
			c-- // Decrement the counter
			if c == 0 {
				return // We got all the responses we needed
			}

		case <-t:
			return // We timed out
		}
	}
}
<commit_msg>fix issue cluster block on method `onResponse`<commit_after>/**********************************************************************************
* Copyright (c) 2009-2017 Misakai Ltd.
* This program is free software: you can redistribute it and/or modify it under the
* terms of the GNU Affero General Public License as published by the  Free Software
* Foundation, either version 3 of the License, or(at your option) any later version.
*
* This program is distributed  in the hope that it  will be useful, but WITHOUT ANY
* WARRANTY;  without even  the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE.  See the GNU Affero General Public License  for  more details.
*
* You should have  received a copy  of the  GNU Affero General Public License along
* with this program. If not, see<http://www.gnu.org/licenses/>.
************************************************************************************/

package broker

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/emitter-io/emitter/internal/message"
	"github.com/emitter-io/emitter/internal/security"
	"github.com/weaveworks/mesh"
)

const (
	idSystem = uint32(0)
	idQuery  = uint32(3939663052)
)

// Surveyee handles the surveys.
type Surveyee interface {
	OnSurvey(queryType string, request []byte) (response []byte, ok bool)
}

// QueryManager represents a request-response manager.
type QueryManager struct {
	service  *Service    // The service to use.
	luid     security.ID // The locally unique id of the manager.
	next     uint32      // The next available query identifier.
	awaiters *sync.Map   // The map of the awaiters.
	handlers []Surveyee  // The handlers array.
}

// newQueryManager creates a new request-response manager.
func newQueryManager(s *Service) *QueryManager {
	return &QueryManager{
		service:  s,
		luid:     security.NewID(),
		next:     0,
		awaiters: new(sync.Map),
		handlers: make([]Surveyee, 0),
	}
}

// Start subscribes the manager to the query channel.
func (c *QueryManager) Start() {
	ssid := message.Ssid{idSystem, idQuery}
	if ok := c.service.onSubscribe(ssid, c); ok {
		c.service.cluster.NotifySubscribe(c.luid, ssid)
	}
}

// HandleFunc adds a handler for a query.
func (c *QueryManager) HandleFunc(surveyees ...Surveyee) {
	for _, h := range surveyees {
		c.handlers = append(c.handlers, h)
	}
}

// ID returns the unique identifier of the subsriber.
func (c *QueryManager) ID() string {
	return c.luid.String()
}

// Type returns the type of the subscriber
func (c *QueryManager) Type() message.SubscriberType {
	return message.SubscriberDirect
}

// Send occurs when we have received a message.
func (c *QueryManager) Send(m *message.Message) error {
	ssid := m.Ssid()
	if len(ssid) != 3 {
		return errors.New("Invalid query received")
	}

	switch string(m.Channel) {
	case "response":
		// We received a response, find the awaiter and forward a message to it
		return c.onResponse(ssid[2], m.Payload)

	default:
		// We received a request, need to handle that by calling the appropriate handler
		return c.onRequest(ssid, string(m.Channel), m.Payload)
	}
}

// onRequest handles an incoming request
func (c *QueryManager) onResponse(id uint32, payload []byte) error {
	if awaiter, ok := c.awaiters.Load(id); ok {
		awaiter.(*queryAwaiter).receive <- payload
	}
	return nil
}

// onRequest handles an incoming request
func (c *QueryManager) onRequest(ssid message.Ssid, channel string, payload []byte) error {
	// Get the query and reply node
	ch := strings.Split(channel, "/")
	query := ch[0]
	reply, err := strconv.ParseInt(ch[1], 10, 64)
	if err != nil {
		return err
	}

	// Do not answer our own requests
	replyAddr := mesh.PeerName(reply)
	if c.service.cluster.ID() == uint64(replyAddr) {
		return nil
	}

	// Get the peer to reply to
	peer := c.service.cluster.FindPeer(replyAddr)
	if !peer.IsActive() {
		return errors.New("unable to reply to a request, peer is not active")
	}

	// Go through all the handlers and execute the first matching one
	for _, surveyee := range c.handlers {
		if response, ok := surveyee.OnSurvey(query, payload); ok {
			return peer.Send(message.New(ssid, []byte("response"), response))
		}
	}

	return errors.New("no query handler found for " + channel)
}

// Query issues a cluster-wide request.
func (c *QueryManager) Query(query string, payload []byte) (message.Awaiter, error) {

	// Create an awaiter
	// TODO: replace the max with the total number of cluster nodes
	numPeers := c.service.NumPeers()
	awaiter := &queryAwaiter{
		id:      atomic.AddUint32(&c.next, 1),
		receive: make(chan []byte, numPeers),
		maximum: numPeers,
		manager: c,
	}

	// Store an awaiter
	c.awaiters.Store(awaiter.id, awaiter)

	// Prepare a channel with the reply-to address
	channel := fmt.Sprintf("%v/%v", query, c.service.LocalName())

	// Publish the query as a message
	c.service.publish(message.New(
		message.Ssid{idSystem, idQuery, awaiter.id},
		[]byte(channel),
		payload,
	), "")
	return awaiter, nil
}

// queryAwaiter represents an asynchronously awaiting response channel.
type queryAwaiter struct {
	id      uint32        // The identifier of the query.
	maximum int           // The maximum number of responses to wait for.
	receive chan []byte   // The receive channel to use.
	manager *QueryManager // The query manager used.
}

// Gather awaits for the responses to be received, blocking until we're done.
func (a *queryAwaiter) Gather(timeout time.Duration) (r [][]byte) {
	defer func() { a.manager.awaiters.Delete(a.id) }()
	r = make([][]byte, 0, 4)
	t := time.After(timeout)
	c := a.maximum

	// If there's no peers, no need to receive anything
	if c == 0 {
		return
	}

	for {
		select {
		case msg := <-a.receive:
			r = append(r, msg)
			c-- // Decrement the counter
			if c == 0 {
				return // We got all the responses we needed
			}

		case <-t:
			return // We timed out
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cloudup

import (
	"context"
	"fmt"
	"net"
	"os"
	"strings"

	"k8s.io/klog/v2"
	"k8s.io/kops/dns-controller/pkg/dns"
	"k8s.io/kops/dnsprovider/pkg/dnsprovider"
	"k8s.io/kops/dnsprovider/pkg/dnsprovider/rrstype"
	"k8s.io/kops/pkg/apis/kops"
	kopsdns "k8s.io/kops/pkg/dns"
	"k8s.io/kops/pkg/featureflag"
	"k8s.io/kops/pkg/model"
	"k8s.io/kops/pkg/model/iam"
	"k8s.io/kops/upup/pkg/fi"
)

const (
	// PlaceholderIP is from TEST-NET-3
	// https://en.wikipedia.org/wiki/Reserved_IP_addresses
	PlaceholderIP  = "203.0.113.123"
	PlaceholderTTL = 10
	// DigitalOcean's DNS servers require a certain minimum TTL (it's 30), keeping 60 here.
	PlaceholderTTLDigitialOcean = 60
)

func findZone(cluster *kops.Cluster, cloud fi.Cloud) (dnsprovider.Zone, error) {
	dns, err := cloud.DNS()
	if err != nil {
		return nil, fmt.Errorf("error building DNS provider: %v", err)
	}

	zonesProvider, ok := dns.Zones()
	if !ok {
		return nil, fmt.Errorf("error getting DNS zones provider")
	}

	zones, err := zonesProvider.List()
	if err != nil {
		return nil, fmt.Errorf("error listing DNS zones: %v", err)
	}

	var matches []dnsprovider.Zone
	findName := strings.TrimSuffix(cluster.Spec.DNSZone, ".")
	for _, zone := range zones {
		id := zone.ID()
		name := strings.TrimSuffix(zone.Name(), ".")
		if id == cluster.Spec.DNSZone || name == findName {
			matches = append(matches, zone)
		}
	}
	if len(matches) == 0 {
		return nil, fmt.Errorf("cannot find DNS Zone %q.  Please pre-create the zone and set up NS records so that it resolves", cluster.Spec.DNSZone)
	}

	if len(matches) > 1 {
		klog.Infof("Found multiple DNS Zones matching %q, please set the cluster's spec.dnsZone to the desired Zone ID:", cluster.Spec.DNSZone)
		for _, zone := range zones {
			id := zone.ID()
			klog.Infof("\t%s", id)
		}
		return nil, fmt.Errorf("found multiple DNS Zones matching %q", cluster.Spec.DNSZone)
	}

	zone := matches[0]
	return zone, nil
}

func validateDNS(cluster *kops.Cluster, cloud fi.Cloud) error {
	kopsModelContext := &model.KopsModelContext{
		IAMModelContext: iam.IAMModelContext{Cluster: cluster},
		// We are not initializing a lot of the fields here; revisit once UsePrivateDNS is "real"
	}

	if kopsModelContext.UsePrivateDNS() {
		klog.Infof("Private DNS: skipping DNS validation")
		return nil
	}

	zone, err := findZone(cluster, cloud)
	if err != nil {
		return err
	}
	dnsName := strings.TrimSuffix(zone.Name(), ".")

	klog.V(2).Infof("Doing DNS lookup to verify NS records for %q", dnsName)
	ns, err := net.LookupNS(dnsName)
	if err != nil {
		return fmt.Errorf("error doing DNS lookup for NS records for %q: %v", dnsName, err)
	}

	if len(ns) == 0 {
		if os.Getenv("DNS_IGNORE_NS_CHECK") == "" {
			return fmt.Errorf("NS records not found for %q - please make sure they are correctly configured", dnsName)
		}
		klog.Warningf("Ignoring failed NS record check because DNS_IGNORE_NS_CHECK is set")
	} else {
		var hosts []string
		for _, n := range ns {
			hosts = append(hosts, n.Host)
		}
		klog.V(2).Infof("Found NS records for %q: %v", dnsName, hosts)
	}

	return nil
}

func precreateDNS(ctx context.Context, cluster *kops.Cluster, cloud fi.Cloud) error {
	// TODO: Move to update
	if !featureflag.DNSPreCreate.Enabled() {
		klog.V(4).Infof("Skipping DNS record pre-creation because feature flag not enabled")
		return nil
	}

	// We precreate some DNS names (where they don't exist), with a dummy IP address
	// This avoids hitting negative TTL on DNS lookups, which tend to be very long
	// If we get the names wrong here, it doesn't really matter (extra DNS name, slower boot)

	dnsHostnames := buildPrecreateDNSHostnames(cluster)

	{
		var filtered []string
		for _, name := range dnsHostnames {
			if !kopsdns.IsGossipHostname(name) {
				filtered = append(filtered, name)
			}
		}
		dnsHostnames = filtered
	}

	if len(dnsHostnames) == 0 {
		klog.Infof("No DNS records to pre-create")
		return nil
	}

	klog.Infof("Checking DNS records")

	zone, err := findZone(cluster, cloud)
	if err != nil {
		return err
	}

	rrs, ok := zone.ResourceRecordSets()
	if !ok {
		return fmt.Errorf("error getting DNS resource records for %q", zone.Name())
	}

	recordsMap := make(map[string]dnsprovider.ResourceRecordSet)
	// TODO: We should change the filter to be a suffix match instead
	//records, err := rrs.List("", "")
	records, err := rrs.List()
	if err != nil {
		return fmt.Errorf("error listing DNS resource records for %q: %v", zone.Name(), err)
	}

	for _, record := range records {
		name := dns.EnsureDotSuffix(record.Name())
		key := string(record.Type()) + "::" + name
		recordsMap[key] = record
	}

	changeset := rrs.StartChangeset()
	// TODO: Add ChangeSet.IsEmpty() method
	var created []string

	for _, dnsHostname := range dnsHostnames {
		dnsHostname = dns.EnsureDotSuffix(dnsHostname)
		found := false
		dnsRecord := recordsMap["A::"+dnsHostname]
		if dnsRecord != nil {
			rrdatas := dnsRecord.Rrdatas()
			if len(rrdatas) > 0 {
				klog.V(4).Infof("Found DNS record %s => %s; won't create", dnsHostname, rrdatas)
				found = true
			} else {
				// This is probably an alias target; leave it alone...
				klog.V(4).Infof("Found DNS record %s, but no records", dnsHostname)
				found = true
			}
		}

		if found {
			continue
		}

		klog.V(2).Infof("Pre-creating DNS record %s => %s", dnsHostname, PlaceholderIP)

		if cloud.ProviderID() == kops.CloudProviderDO {
			changeset.Add(rrs.New(dnsHostname, []string{PlaceholderIP}, PlaceholderTTLDigitialOcean, rrstype.A))
		} else {
			changeset.Add(rrs.New(dnsHostname, []string{PlaceholderIP}, PlaceholderTTL, rrstype.A))
		}

		created = append(created, dnsHostname)
	}

	if len(created) != 0 {
		klog.Infof("Pre-creating DNS records")

		err := changeset.Apply(ctx)
		if err != nil {
			return fmt.Errorf("error pre-creating DNS records: %v", err)
		}
		klog.V(2).Infof("Pre-created DNS names: %v", created)
	}

	return nil
}

// buildPrecreateDNSHostnames returns the hostnames we should precreate
func buildPrecreateDNSHostnames(cluster *kops.Cluster) []string {
	dnsInternalSuffix := ".internal." + cluster.ObjectMeta.Name

	var dnsHostnames []string

	if cluster.Spec.MasterPublicName != "" {
		dnsHostnames = append(dnsHostnames, cluster.Spec.MasterPublicName)
	} else {
		klog.Warningf("cannot pre-create MasterPublicName - not set")
	}

	if cluster.Spec.MasterInternalName != "" {
		dnsHostnames = append(dnsHostnames, cluster.Spec.MasterInternalName)
	} else {
		klog.Warningf("cannot pre-create MasterInternalName - not set")
	}

	for _, etcdCluster := range cluster.Spec.EtcdClusters {
		if etcdCluster.Provider == kops.EtcdProviderTypeManager {
			continue
		}
		etcClusterName := "etcd-" + etcdCluster.Name
		if etcdCluster.Name == "main" {
			// Special case
			etcClusterName = "etcd"
		}
		for _, etcdClusterMember := range etcdCluster.Members {
			name := etcClusterName + "-" + etcdClusterMember.Name + dnsInternalSuffix
			dnsHostnames = append(dnsHostnames, name)
		}
	}

	return dnsHostnames
}
<commit_msg>Precreate the kops-controller DNS name<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cloudup

import (
	"context"
	"fmt"
	"net"
	"os"
	"strings"

	"k8s.io/klog/v2"
	"k8s.io/kops/dns-controller/pkg/dns"
	"k8s.io/kops/dnsprovider/pkg/dnsprovider"
	"k8s.io/kops/dnsprovider/pkg/dnsprovider/rrstype"
	"k8s.io/kops/pkg/apis/kops"
	apimodel "k8s.io/kops/pkg/apis/kops/model"
	kopsdns "k8s.io/kops/pkg/dns"
	"k8s.io/kops/pkg/featureflag"
	"k8s.io/kops/pkg/model"
	"k8s.io/kops/pkg/model/iam"
	"k8s.io/kops/upup/pkg/fi"
)

const (
	// PlaceholderIP is from TEST-NET-3
	// https://en.wikipedia.org/wiki/Reserved_IP_addresses
	PlaceholderIP  = "203.0.113.123"
	PlaceholderTTL = 10
	// DigitalOcean's DNS servers require a certain minimum TTL (it's 30), keeping 60 here.
	PlaceholderTTLDigitialOcean = 60
)

func findZone(cluster *kops.Cluster, cloud fi.Cloud) (dnsprovider.Zone, error) {
	dns, err := cloud.DNS()
	if err != nil {
		return nil, fmt.Errorf("error building DNS provider: %v", err)
	}

	zonesProvider, ok := dns.Zones()
	if !ok {
		return nil, fmt.Errorf("error getting DNS zones provider")
	}

	zones, err := zonesProvider.List()
	if err != nil {
		return nil, fmt.Errorf("error listing DNS zones: %v", err)
	}

	var matches []dnsprovider.Zone
	findName := strings.TrimSuffix(cluster.Spec.DNSZone, ".")
	for _, zone := range zones {
		id := zone.ID()
		name := strings.TrimSuffix(zone.Name(), ".")
		if id == cluster.Spec.DNSZone || name == findName {
			matches = append(matches, zone)
		}
	}
	if len(matches) == 0 {
		return nil, fmt.Errorf("cannot find DNS Zone %q.  Please pre-create the zone and set up NS records so that it resolves", cluster.Spec.DNSZone)
	}

	if len(matches) > 1 {
		klog.Infof("Found multiple DNS Zones matching %q, please set the cluster's spec.dnsZone to the desired Zone ID:", cluster.Spec.DNSZone)
		for _, zone := range zones {
			id := zone.ID()
			klog.Infof("\t%s", id)
		}
		return nil, fmt.Errorf("found multiple DNS Zones matching %q", cluster.Spec.DNSZone)
	}

	zone := matches[0]
	return zone, nil
}

func validateDNS(cluster *kops.Cluster, cloud fi.Cloud) error {
	kopsModelContext := &model.KopsModelContext{
		IAMModelContext: iam.IAMModelContext{Cluster: cluster},
		// We are not initializing a lot of the fields here; revisit once UsePrivateDNS is "real"
	}

	if kopsModelContext.UsePrivateDNS() {
		klog.Infof("Private DNS: skipping DNS validation")
		return nil
	}

	zone, err := findZone(cluster, cloud)
	if err != nil {
		return err
	}
	dnsName := strings.TrimSuffix(zone.Name(), ".")

	klog.V(2).Infof("Doing DNS lookup to verify NS records for %q", dnsName)
	ns, err := net.LookupNS(dnsName)
	if err != nil {
		return fmt.Errorf("error doing DNS lookup for NS records for %q: %v", dnsName, err)
	}

	if len(ns) == 0 {
		if os.Getenv("DNS_IGNORE_NS_CHECK") == "" {
			return fmt.Errorf("NS records not found for %q - please make sure they are correctly configured", dnsName)
		}
		klog.Warningf("Ignoring failed NS record check because DNS_IGNORE_NS_CHECK is set")
	} else {
		var hosts []string
		for _, n := range ns {
			hosts = append(hosts, n.Host)
		}
		klog.V(2).Infof("Found NS records for %q: %v", dnsName, hosts)
	}

	return nil
}

func precreateDNS(ctx context.Context, cluster *kops.Cluster, cloud fi.Cloud) error {
	// TODO: Move to update
	if !featureflag.DNSPreCreate.Enabled() {
		klog.V(4).Infof("Skipping DNS record pre-creation because feature flag not enabled")
		return nil
	}

	// We precreate some DNS names (where they don't exist), with a dummy IP address
	// This avoids hitting negative TTL on DNS lookups, which tend to be very long
	// If we get the names wrong here, it doesn't really matter (extra DNS name, slower boot)

	dnsHostnames := buildPrecreateDNSHostnames(cluster)

	{
		var filtered []string
		for _, name := range dnsHostnames {
			if !kopsdns.IsGossipHostname(name) {
				filtered = append(filtered, name)
			}
		}
		dnsHostnames = filtered
	}

	if len(dnsHostnames) == 0 {
		klog.Infof("No DNS records to pre-create")
		return nil
	}

	klog.Infof("Checking DNS records")

	zone, err := findZone(cluster, cloud)
	if err != nil {
		return err
	}

	rrs, ok := zone.ResourceRecordSets()
	if !ok {
		return fmt.Errorf("error getting DNS resource records for %q", zone.Name())
	}

	recordsMap := make(map[string]dnsprovider.ResourceRecordSet)
	// TODO: We should change the filter to be a suffix match instead
	//records, err := rrs.List("", "")
	records, err := rrs.List()
	if err != nil {
		return fmt.Errorf("error listing DNS resource records for %q: %v", zone.Name(), err)
	}

	for _, record := range records {
		name := dns.EnsureDotSuffix(record.Name())
		key := string(record.Type()) + "::" + name
		recordsMap[key] = record
	}

	changeset := rrs.StartChangeset()
	// TODO: Add ChangeSet.IsEmpty() method
	var created []string

	for _, dnsHostname := range dnsHostnames {
		dnsHostname = dns.EnsureDotSuffix(dnsHostname)
		found := false
		dnsRecord := recordsMap["A::"+dnsHostname]
		if dnsRecord != nil {
			rrdatas := dnsRecord.Rrdatas()
			if len(rrdatas) > 0 {
				klog.V(4).Infof("Found DNS record %s => %s; won't create", dnsHostname, rrdatas)
				found = true
			} else {
				// This is probably an alias target; leave it alone...
				klog.V(4).Infof("Found DNS record %s, but no records", dnsHostname)
				found = true
			}
		}

		if found {
			continue
		}

		klog.V(2).Infof("Pre-creating DNS record %s => %s", dnsHostname, PlaceholderIP)

		if cloud.ProviderID() == kops.CloudProviderDO {
			changeset.Add(rrs.New(dnsHostname, []string{PlaceholderIP}, PlaceholderTTLDigitialOcean, rrstype.A))
		} else {
			changeset.Add(rrs.New(dnsHostname, []string{PlaceholderIP}, PlaceholderTTL, rrstype.A))
		}

		created = append(created, dnsHostname)
	}

	if len(created) != 0 {
		klog.Infof("Pre-creating DNS records")

		err := changeset.Apply(ctx)
		if err != nil {
			return fmt.Errorf("error pre-creating DNS records: %v", err)
		}
		klog.V(2).Infof("Pre-created DNS names: %v", created)
	}

	return nil
}

// buildPrecreateDNSHostnames returns the hostnames we should precreate
func buildPrecreateDNSHostnames(cluster *kops.Cluster) []string {
	dnsInternalSuffix := ".internal." + cluster.ObjectMeta.Name

	var dnsHostnames []string

	if cluster.Spec.MasterPublicName != "" {
		dnsHostnames = append(dnsHostnames, cluster.Spec.MasterPublicName)
	} else {
		klog.Warningf("cannot pre-create MasterPublicName - not set")
	}

	if cluster.Spec.MasterInternalName != "" {
		dnsHostnames = append(dnsHostnames, cluster.Spec.MasterInternalName)
	} else {
		klog.Warningf("cannot pre-create MasterInternalName - not set")
	}

	for _, etcdCluster := range cluster.Spec.EtcdClusters {
		if etcdCluster.Provider == kops.EtcdProviderTypeManager {
			continue
		}
		etcClusterName := "etcd-" + etcdCluster.Name
		if etcdCluster.Name == "main" {
			// Special case
			etcClusterName = "etcd"
		}
		for _, etcdClusterMember := range etcdCluster.Members {
			name := etcClusterName + "-" + etcdClusterMember.Name + dnsInternalSuffix
			dnsHostnames = append(dnsHostnames, name)
		}
	}

	if apimodel.UseKopsControllerForNodeBootstrap(cluster) {
		name := "kops-controller.internal." + cluster.ObjectMeta.Name
		dnsHostnames = append(dnsHostnames, name)
	}

	return dnsHostnames
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// +build go1.3

package lxd

import (
	"os/exec"
	"strconv"

	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils/packaging/commands"
	"github.com/juju/utils/packaging/manager"
	"github.com/juju/utils/series"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/testing"
)

type InitialiserSuite struct {
	testing.BaseSuite
	calledCmds []string
}

var _ = gc.Suite(&InitialiserSuite{})

// getMockRunCommandWithRetry is a helper function which returns a function
// with an identical signature to manager.RunCommandWithRetry which saves each
// command it recieves in a slice and always returns no output, error code 0
// and a nil error.
func getMockRunCommandWithRetry(calledCmds *[]string) func(string) (string, int, error) {
	return func(cmd string) (string, int, error) {
		*calledCmds = append(*calledCmds, cmd)
		return "", 0, nil
	}
}

func (s *InitialiserSuite) SetUpTest(c *gc.C) {
	s.BaseSuite.SetUpTest(c)
	s.calledCmds = []string{}
	s.PatchValue(&manager.RunCommandWithRetry, getMockRunCommandWithRetry(&s.calledCmds))
	s.PatchValue(&configureZFS, func() {})
	s.PatchValue(&configureLXDBridge, func() error { return nil })
}

func (s *InitialiserSuite) TestLTSSeriesPackages(c *gc.C) {
	// Momentarily, the only series with a dedicated cloud archive is precise,
	// which we will use for the following test:
	paccmder, err := commands.NewPackageCommander("trusty")
	c.Assert(err, jc.ErrorIsNil)

	s.PatchValue(&series.HostSeries, func() string { return "trusty" })
	container := NewContainerInitialiser("trusty")

	err = container.Initialise()
	c.Assert(err, jc.ErrorIsNil)

	c.Assert(s.calledCmds, gc.DeepEquals, []string{
		paccmder.InstallCmd("--target-release", "trusty-backports", "lxd"),
	})
}

func (s *InitialiserSuite) TestNoSeriesPackages(c *gc.C) {
	// Here we want to test for any other series whilst avoiding the
	// possibility of hitting a cloud archive-requiring release.
	// As such, we simply pass an empty series.
	paccmder, err := commands.NewPackageCommander("xenial")
	c.Assert(err, jc.ErrorIsNil)

	container := NewContainerInitialiser("")

	err = container.Initialise()
	c.Assert(err, jc.ErrorIsNil)

	c.Assert(s.calledCmds, gc.DeepEquals, []string{
		paccmder.InstallCmd("lxd"),
	})
}

func (s *InitialiserSuite) TestEditLXDBridgeFile(c *gc.C) {
	input := `# WARNING: Don't modify this file by hand, it is generated by debconf!
# To update those values, please run "dpkg-reconfigure lxd"

# Whether to setup a new bridge
USE_LXD_BRIDGE="true"
EXISTING_BRIDGE=""

# Bridge name
LXD_BRIDGE="lxdbr0"

# dnsmasq configuration path
LXD_CONFILE=""

# dnsmasq domain
LXD_DOMAIN="lxd"

# IPv4
LXD_IPV4_ADDR="10.0.4.1"
LXD_IPV4_NETMASK="255.255.255.0"
LXD_IPV4_NETWORK="10.0.4.1/24"
LXD_IPV4_DHCP_RANGE="10.0.4.2,10.0.4.100"
LXD_IPV4_DHCP_MAX="50"
LXD_IPV4_NAT="true"

# IPv6
LXD_IPV6_ADDR="2001:470:b2b5:9999::1"
LXD_IPV6_MASK="64"
LXD_IPV6_NETWORK="2001:470:b2b5:9999::1/64"
LXD_IPV6_NAT="true"

# Proxy server
LXD_IPV6_PROXY="true"
`
	expected := `# WARNING: Don't modify this file by hand, it is generated by debconf!
# To update those values, please run "dpkg-reconfigure lxd"

# Whether to setup a new bridge
USE_LXD_BRIDGE="true"
EXISTING_BRIDGE=""

# Bridge name
LXD_BRIDGE="lxdbr0"

# dnsmasq configuration path
LXD_CONFILE=""

# dnsmasq domain
LXD_DOMAIN="lxd"

# IPv4
LXD_IPV4_ADDR="10.0.19.1"
LXD_IPV4_NETMASK="255.255.255.0"
LXD_IPV4_NETWORK="10.0.19.1/24"
LXD_IPV4_DHCP_RANGE="10.0.19.2,10.0.19.254"
LXD_IPV4_DHCP_MAX="253"
LXD_IPV4_NAT="true"

# IPv6
LXD_IPV6_ADDR="2001:470:b2b5:9999::1"
LXD_IPV6_MASK="64"
LXD_IPV6_NETWORK="2001:470:b2b5:9999::1/64"
LXD_IPV6_NAT="true"

# Proxy server
LXD_IPV6_PROXY="false"

`
	result := editLXDBridgeFile(input, "19")
	c.Assert(result, jc.DeepEquals, expected)
}

func (s *InitialiserSuite) TestDetectSubnet(c *gc.C) {
	input := `1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 1c:6f:65:d5:56:98 brd ff:ff:ff:ff:ff:ff
    inet 192.168.0.69/24 brd 192.168.0.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fd5d:e5bb:c5f9::c0c/128 scope global dynamic
       valid_lft 83178sec preferred_lft 83178sec
    inet6 fd5d:e5bb:c5f9:0:1e6f:65ff:fed5:5698/64 scope global noprefixroute dynamic
       valid_lft 6967sec preferred_lft 1567sec
    inet6 fe80::1e6f:65ff:fed5:5698/64 scope link
       valid_lft forever preferred_lft forever
3: virbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 52:54:00:e4:70:2f brd ff:ff:ff:ff:ff:ff
    inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0
       valid_lft forever preferred_lft forever
4: virbr0-nic: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast master virbr0 state DOWN group default qlen 500
    link/ether 52:54:00:e4:70:2f brd ff:ff:ff:ff:ff:ff
5: virbr1: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
    link/ether 52:54:00:fe:04:e6 brd ff:ff:ff:ff:ff:ff
    inet 192.168.100.1/24 brd 192.168.100.255 scope global virbr1
       valid_lft forever preferred_lft forever
6: virbr1-nic: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast master virbr1 state DOWN group default qlen 500
    link/ether 52:54:00:fe:04:e6 brd ff:ff:ff:ff:ff:ff
7: lxcbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether fe:d3:9d:e4:ba:90 brd ff:ff:ff:ff:ff:ff
    inet 10.0.3.1/24 scope global lxcbr0
       valid_lft forever preferred_lft forever
    inet6 fe80::a00f:35ff:fe81:f7ed/64 scope link
       valid_lft forever preferred_lft forever
25: vethOG10XO@if24: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master lxcbr0 state UP group default qlen 1000
    link/ether fe:d3:9d:e4:ba:90 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet6 fe80::fcd3:9dff:fee4:ba90/64 scope link
       valid_lft forever preferred_lft forever
37: vnet0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master virbr0 state UNKNOWN group default qlen 500
    link/ether fe:54:00:6e:2d:7d brd ff:ff:ff:ff:ff:ff
    inet6 fe80::fc54:ff:fe6e:2d7d/64 scope link
       valid_lft forever preferred_lft forever
38: vnet1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master virbr0 state UNKNOWN group default qlen 500
    link/ether fe:54:00:3e:80:18 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::fc54:ff:fe3e:8018/64 scope link
       valid_lft forever preferred_lft forever
39: vnet2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master virbr0 state UNKNOWN group default qlen 500
    link/ether fe:54:00:ee:c7:95 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::fc54:ff:feee:c795/64 scope link
       valid_lft forever preferred_lft forever
40: vnet3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master virbr0 state UNKNOWN group default qlen 500
    link/ether fe:54:00:30:92:16 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::fc54:ff:fe30:9216/64 scope link
       valid_lft forever preferred_lft forever
`

	result, err := detectSubnet(input)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(result, jc.DeepEquals, "4")
}

func (s *InitialiserSuite) TestDetectSubnetLocal(c *gc.C) {
	output, err := exec.Command("ip", "addr", "show").CombinedOutput()
	c.Assert(err, jc.ErrorIsNil)

	subnet, err := detectSubnet(string(output))
	c.Assert(err, jc.ErrorIsNil)

	subnetInt, err := strconv.Atoi(subnet)
	c.Assert(err, jc.ErrorIsNil)

	c.Assert(subnetInt, jc.GreaterThan, 0)
	c.Assert(subnetInt, jc.LessThan, 255)

}
<commit_msg>lxd container type: skip local subnet test if no `ip` tool present<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// +build go1.3

package lxd

import (
	"os/exec"
	"strconv"

	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils/packaging/commands"
	"github.com/juju/utils/packaging/manager"
	"github.com/juju/utils/series"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/testing"
)

type InitialiserSuite struct {
	testing.BaseSuite
	calledCmds []string
}

var _ = gc.Suite(&InitialiserSuite{})

// getMockRunCommandWithRetry is a helper function which returns a function
// with an identical signature to manager.RunCommandWithRetry which saves each
// command it recieves in a slice and always returns no output, error code 0
// and a nil error.
func getMockRunCommandWithRetry(calledCmds *[]string) func(string) (string, int, error) {
	return func(cmd string) (string, int, error) {
		*calledCmds = append(*calledCmds, cmd)
		return "", 0, nil
	}
}

func (s *InitialiserSuite) SetUpTest(c *gc.C) {
	s.BaseSuite.SetUpTest(c)
	s.calledCmds = []string{}
	s.PatchValue(&manager.RunCommandWithRetry, getMockRunCommandWithRetry(&s.calledCmds))
	s.PatchValue(&configureZFS, func() {})
	s.PatchValue(&configureLXDBridge, func() error { return nil })
}

func (s *InitialiserSuite) TestLTSSeriesPackages(c *gc.C) {
	// Momentarily, the only series with a dedicated cloud archive is precise,
	// which we will use for the following test:
	paccmder, err := commands.NewPackageCommander("trusty")
	c.Assert(err, jc.ErrorIsNil)

	s.PatchValue(&series.HostSeries, func() string { return "trusty" })
	container := NewContainerInitialiser("trusty")

	err = container.Initialise()
	c.Assert(err, jc.ErrorIsNil)

	c.Assert(s.calledCmds, gc.DeepEquals, []string{
		paccmder.InstallCmd("--target-release", "trusty-backports", "lxd"),
	})
}

func (s *InitialiserSuite) TestNoSeriesPackages(c *gc.C) {
	// Here we want to test for any other series whilst avoiding the
	// possibility of hitting a cloud archive-requiring release.
	// As such, we simply pass an empty series.
	paccmder, err := commands.NewPackageCommander("xenial")
	c.Assert(err, jc.ErrorIsNil)

	container := NewContainerInitialiser("")

	err = container.Initialise()
	c.Assert(err, jc.ErrorIsNil)

	c.Assert(s.calledCmds, gc.DeepEquals, []string{
		paccmder.InstallCmd("lxd"),
	})
}

func (s *InitialiserSuite) TestEditLXDBridgeFile(c *gc.C) {
	input := `# WARNING: Don't modify this file by hand, it is generated by debconf!
# To update those values, please run "dpkg-reconfigure lxd"

# Whether to setup a new bridge
USE_LXD_BRIDGE="true"
EXISTING_BRIDGE=""

# Bridge name
LXD_BRIDGE="lxdbr0"

# dnsmasq configuration path
LXD_CONFILE=""

# dnsmasq domain
LXD_DOMAIN="lxd"

# IPv4
LXD_IPV4_ADDR="10.0.4.1"
LXD_IPV4_NETMASK="255.255.255.0"
LXD_IPV4_NETWORK="10.0.4.1/24"
LXD_IPV4_DHCP_RANGE="10.0.4.2,10.0.4.100"
LXD_IPV4_DHCP_MAX="50"
LXD_IPV4_NAT="true"

# IPv6
LXD_IPV6_ADDR="2001:470:b2b5:9999::1"
LXD_IPV6_MASK="64"
LXD_IPV6_NETWORK="2001:470:b2b5:9999::1/64"
LXD_IPV6_NAT="true"

# Proxy server
LXD_IPV6_PROXY="true"
`
	expected := `# WARNING: Don't modify this file by hand, it is generated by debconf!
# To update those values, please run "dpkg-reconfigure lxd"

# Whether to setup a new bridge
USE_LXD_BRIDGE="true"
EXISTING_BRIDGE=""

# Bridge name
LXD_BRIDGE="lxdbr0"

# dnsmasq configuration path
LXD_CONFILE=""

# dnsmasq domain
LXD_DOMAIN="lxd"

# IPv4
LXD_IPV4_ADDR="10.0.19.1"
LXD_IPV4_NETMASK="255.255.255.0"
LXD_IPV4_NETWORK="10.0.19.1/24"
LXD_IPV4_DHCP_RANGE="10.0.19.2,10.0.19.254"
LXD_IPV4_DHCP_MAX="253"
LXD_IPV4_NAT="true"

# IPv6
LXD_IPV6_ADDR="2001:470:b2b5:9999::1"
LXD_IPV6_MASK="64"
LXD_IPV6_NETWORK="2001:470:b2b5:9999::1/64"
LXD_IPV6_NAT="true"

# Proxy server
LXD_IPV6_PROXY="false"

`
	result := editLXDBridgeFile(input, "19")
	c.Assert(result, jc.DeepEquals, expected)
}

func (s *InitialiserSuite) TestDetectSubnet(c *gc.C) {
	input := `1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 1c:6f:65:d5:56:98 brd ff:ff:ff:ff:ff:ff
    inet 192.168.0.69/24 brd 192.168.0.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fd5d:e5bb:c5f9::c0c/128 scope global dynamic
       valid_lft 83178sec preferred_lft 83178sec
    inet6 fd5d:e5bb:c5f9:0:1e6f:65ff:fed5:5698/64 scope global noprefixroute dynamic
       valid_lft 6967sec preferred_lft 1567sec
    inet6 fe80::1e6f:65ff:fed5:5698/64 scope link
       valid_lft forever preferred_lft forever
3: virbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 52:54:00:e4:70:2f brd ff:ff:ff:ff:ff:ff
    inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0
       valid_lft forever preferred_lft forever
4: virbr0-nic: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast master virbr0 state DOWN group default qlen 500
    link/ether 52:54:00:e4:70:2f brd ff:ff:ff:ff:ff:ff
5: virbr1: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
    link/ether 52:54:00:fe:04:e6 brd ff:ff:ff:ff:ff:ff
    inet 192.168.100.1/24 brd 192.168.100.255 scope global virbr1
       valid_lft forever preferred_lft forever
6: virbr1-nic: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast master virbr1 state DOWN group default qlen 500
    link/ether 52:54:00:fe:04:e6 brd ff:ff:ff:ff:ff:ff
7: lxcbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether fe:d3:9d:e4:ba:90 brd ff:ff:ff:ff:ff:ff
    inet 10.0.3.1/24 scope global lxcbr0
       valid_lft forever preferred_lft forever
    inet6 fe80::a00f:35ff:fe81:f7ed/64 scope link
       valid_lft forever preferred_lft forever
25: vethOG10XO@if24: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master lxcbr0 state UP group default qlen 1000
    link/ether fe:d3:9d:e4:ba:90 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet6 fe80::fcd3:9dff:fee4:ba90/64 scope link
       valid_lft forever preferred_lft forever
37: vnet0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master virbr0 state UNKNOWN group default qlen 500
    link/ether fe:54:00:6e:2d:7d brd ff:ff:ff:ff:ff:ff
    inet6 fe80::fc54:ff:fe6e:2d7d/64 scope link
       valid_lft forever preferred_lft forever
38: vnet1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master virbr0 state UNKNOWN group default qlen 500
    link/ether fe:54:00:3e:80:18 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::fc54:ff:fe3e:8018/64 scope link
       valid_lft forever preferred_lft forever
39: vnet2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master virbr0 state UNKNOWN group default qlen 500
    link/ether fe:54:00:ee:c7:95 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::fc54:ff:feee:c795/64 scope link
       valid_lft forever preferred_lft forever
40: vnet3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master virbr0 state UNKNOWN group default qlen 500
    link/ether fe:54:00:30:92:16 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::fc54:ff:fe30:9216/64 scope link
       valid_lft forever preferred_lft forever
`

	result, err := detectSubnet(input)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(result, jc.DeepEquals, "4")
}

func (s *InitialiserSuite) TestDetectSubnetLocal(c *gc.C) {
	_, err := exec.LookPath("ip")
	if err != nil {
		c.Skip("skipping local detect subnet test, ip tool not found")
	}

	output, err := exec.Command("ip", "addr", "show").CombinedOutput()
	c.Assert(err, jc.ErrorIsNil)

	subnet, err := detectSubnet(string(output))
	c.Assert(err, jc.ErrorIsNil)

	subnetInt, err := strconv.Atoi(subnet)
	c.Assert(err, jc.ErrorIsNil)

	c.Assert(subnetInt, jc.GreaterThan, 0)
	c.Assert(subnetInt, jc.LessThan, 255)

}
<|endoftext|>


<commit_before>// Copyright 2015, Truveris Inc. All Rights Reserved.
// Use of this source code is governed by the ISC license in the LICENSE file.

package main

import (
	"encoding/json"
	"errors"
	"net/http"
	"net/url"
	"path"
	"strconv"
	"strings"
)

var (
	// These are the known domains to check for, where special formatting of
	// the passed URL is required so connected minions can most effectively
	// embed and manipulate the desired content.
	imgurHostNames = []string{
		"i.imgur.com",
		"www.imgur.com",
		"imgur.com",
	}

	supportedFormatsAndTypes = map[string][]string{
		"img": {
			"image/bmp",
			"image/cis-cod",
			"image/gif",
			"image/ief",
			"image/jpeg",
			"image/webp",
			"image/pict",
			"image/pipeg",
			"image/png",
			"image/svg+xml",
			"image/tiff",
			"image/vnd.microsoft.icon",
			"image/x-cmu-raster",
			"image/x-cmx",
			"image/x-icon",
			"image/x-portable-anymap",
			"image/x-portable-bitmap",
			"image/x-portable-graymap",
			"image/x-portable-pixmap",
			"image/x-rgb",
			"image/x-xbitmap",
			"image/x-xpixmap",
			"image/x-xwindowdump",
		},
		"audio": {
			"audio/aac",
			"audio/aiff",
			"audio/amr",
			"audio/basic",
			"audio/midi",
			"audio/mp3",
			"audio/mp4",
			"audio/mpeg",
			"audio/mpeg3",
			"audio/ogg",
			"audio/vorbis",
			"audio/wav",
			"audio/webm",
			"audio/x-m4a",
			"audio/x-ms-wma",
			"audio/vnd.rn-realaudio",
			"audio/vnd.wave",
		},
		"video": {
			"video/avi",
			"video/divx",
			"video/flc",
			"video/mp4",
			"video/mpeg",
			"video/ogg",
			"video/quicktime",
			"video/sd-video",
			"video/webm",
			"video/x-dv",
			"video/x-m4v",
			"video/x-mpeg",
			"video/x-ms-asf",
			"video/x-ms-wmv",
		},
		"web": {
			"text/",
		},
	}

	// ygor should fallback to checking the file extensions for potential
	// matches if the content-type doesn't appear to be supported. The server
	// may simply be providing the wrong content-type in the header.
	supportedFormatsAndExtensions = map[string][]string{
		"img": {
			".apng",
			".bmp",
			".dib",
			".gif",
			".jfi",
			".jfif",
			".jif",
			".jpe",
			".jpeg",
			".jpg",
			".png",
			".webp",
		},
		"audio": {
			".mp3",
			".wav",
			".wave",
		},
		"video": {
			".m4a",
			".m4b",
			".m4p",
			".m4r",
			".m4v",
			".mp4",
			".oga",
			".ogg",
			".ogm",
			".ogv",
			".ogx",
			".opus",
			".spx",
			".webm",
		},
	}
)

// MediaObj represents the relevant data that will eventually be passed to
// the connected minions. It is used to generate the information that connected
// minions would use to properly embed the desired content.
//
// It also provides several functions that can be used to more easily work with
// the data, so that command modules aren't filled with a lot of excessive
// code.
type MediaObj struct {
	// 'Src' is formatted over time and is what will eventually be passed to
	// the connected minions.
	Src  string `json:"src"`
	url  string
	host string
	// 'Format' tells the connected minions how to embed the desired content
	// using 'Src'.
	Format      string `json:"format"`
	mediaType   string
	// End represents where in the desired content's timeline to stop playing.
	End string `json:"end"`
	// Muted represents whether or not the desired content should be muted.
	Muted             bool `json:"muted"`
	Loop              bool `json:"loop"`
	track             string
	acceptableFormats []string
}

// SetAcceptableFormats takes in a string array of acceptable media types,
// which will be checked against during SetSrc. If the determined media type is
// not acceptable, the url will be rejected.
func (mObj *MediaObj) SetAcceptableFormats(formats []string) {
	mObj.acceptableFormats = formats
}

// checkFormatIsAcceptable checks to make sure that the determined media
// type is acceptable. If the MediaObj's acceptableFormats attribute is not
// set, it is assumed that the media type is acceptable.
func (mObj *MediaObj) checkFormatIsAcceptable() error {
	if len(mObj.acceptableFormats) == 0 {
		// if acceptableFormats is not set, all media types are acceptable
		return nil
	}

	for _, acceptableFormat := range mObj.acceptableFormats {
		if mObj.Format == acceptableFormat {
			// The determined media type is acceptable.
			return nil
		}
	}

	// If it made it here, the determined media type must not be acceptable.
	errMsg := "error: content-type (" + mObj.mediaType + ") not supported " +
		"by this command"
	return errors.New(errMsg)
}

// SetSrc takes in a string that represents a URL. This function determines if
// the URL is a valid URL, formats imgur links to use .webm instead of .gif(v),
// and determines the Format that the URL represents.
//
// The MediaObj's 'Src' attribute will either be set to the passed URL, or the
// formatted imgur URL (if it was an imgur link).
//
// The MediaObj's 'Src' attribute can be retrieved using the MediaObj's
// 'GetSrc()' function.
//
// The URL that was originally passed, is saved as the MediaObj's 'url'
// attribute, and can be retrieved with the MediaObj's 'GetURL()' function.
func (mObj *MediaObj) SetSrc(link string) error {
	uri, linkErr := url.ParseRequestURI(link)
	if linkErr != nil {
		errorMsg := "error: not a valid URL"
		return errors.New(errorMsg)
	}
	// Strip any query or fragment attached to the URL
	mObj.Src = uri.String()
	mObj.url = link
	mObj.host = uri.Host

	// Check that the URL returns a status code of 200.
	res, err := http.Head(mObj.Src)
	if err != nil {
		errMsg := "error: " + err.Error()
		return errors.New(errMsg)
	}
	statusCode := strconv.Itoa(res.StatusCode)
	if statusCode != "200" {
		errMsg := "error: response status code is " + statusCode
		return errors.New(errMsg)
	}

	headErr := mObj.setFormat(res.Header)
	if headErr != nil {
		return headErr
	}

	// If it's an imgur link, and the content-type contains "image/gif", modify
	// the MediaObj so minions embed the far more efficient webm version.
	if mObj.isImgur() {
		isGIF := strings.Contains(strings.ToLower(mObj.mediaType), "image/gif")
		hasGIFVExt := mObj.GetExt() == ".gifv"
		if isGIF || hasGIFVExt {
			mObj.replaceSrcExt(".webm")
			mObj.Format = "video"
			mObj.mediaType = "video/webm"
		}
	}

	merr := mObj.checkFormatIsAcceptable()
	if merr != nil {
		return merr
	}

	return nil
}

// GetSrc returns the MediaObj's 'Src' attribute (this is what should get
// passed to the connected minions).
func (mObj *MediaObj) GetSrc() string {
	return mObj.Src
}

// GetURL returns the URL that was originally passed to the 'SetSrc()'
// function.
func (mObj *MediaObj) GetURL() string {
	return mObj.url
}

// setFormat sets the 'Format' attribute of the MediaObj. This tells the
// connected minions what kind of content they should be trying to embed.
func (mObj *MediaObj) setFormat(header map[string][]string) error {
	// Is the media type in the contentType an image|audio|video type that
	// Chromium supports?
	if contentType, ok := header["Content-Type"]; ok {
		// Check for standard, supported media types.
		for format, formatMediaTypes := range supportedFormatsAndTypes {
			for _, mediaType := range formatMediaTypes {
				for _, cType := range contentType {
					if strings.Contains(cType, mediaType) {
						mObj.Format = format
						mObj.mediaType = mediaType
						return nil
					}
				}
			}
		}

		// Fallback to known supported file extensions if content-type isn't
		// recognized as supported.
		ext := mObj.GetExt()
		for format, formatExtensions := range supportedFormatsAndExtensions {
			for _, extension := range formatExtensions {
				if extension == ext {
					mObj.Format = format
					mObj.mediaType = ext
					return nil
				}
			}
		}

		// If the media type isn't supported, return an error.
		errMsg := "error: unsupported content-type " +
			"(" + strings.Join(contentType, ", ") + ")"
		return errors.New(errMsg)
	}

	// It will only get here if it didn't have a content-type in the header.
	errMsg := "error: no content-type found"
	return errors.New(errMsg)
}

// GetFormat returnes the MediaObj's 'Format' attribute. The 'Format'
// tells the connected minions what kind of content they should be trying to
// embed when using the MediaObj's 'Src' attribute.
func (mObj *MediaObj) GetFormat() string {
	return mObj.Format
}

// IsOfFormat determines if the MediaObj's Format is contained in the
// passed string array.
func (mObj *MediaObj) IsOfFormat(formats []string) bool {
	format := mObj.GetFormat()
	for _, mt := range formats {
		if format == mt {
			return true
		}
	}
	return false
}

// GetExt is a convenience function to get the extension of theMediaObj's
// current Src.
func (mObj *MediaObj) GetExt() string {
	return strings.ToLower(path.Ext(mObj.Src))
}

// isImgur attempts to determine if the desired content is hosted on imgur.
func (mObj *MediaObj) isImgur() bool {
	for _, d := range imgurHostNames {
		if mObj.host == d {
			return true
		}
	}
	return false
}

// replaceSrcExt is a convenience function to replace the extension of the
// MediaObj's current Src.
func (mObj *MediaObj) replaceSrcExt(newExt string) {
	mObj.Src = mObj.Src[0:len(mObj.Src)-len(mObj.GetExt())] + newExt
}

// Serialize generates and returns the JSON string out of the MediaObj. This
// JSON string is what should be sent to the connected minions.
func (mObj *MediaObj) Serialize() string {
	serializedJSON, _ := json.Marshal(struct {
		MediaObj *MediaObj `json:"mediaObj"`
		Status   string    `json:"status"`
		Track    string    `json:"track"`
	}{
		Status:   "media",
		Track:    mObj.track,
		MediaObj: mObj,
	})
	return string(serializedJSON)
}

// NewMediaObj is a convenience function meant to clean up the code of modules.
// It builds the MediaObj.
func NewMediaObj(mediaItem map[string]string, track string, muted bool, loop bool, acceptableFormats []string) (*MediaObj, error) {
	// Parse the mediaItem map into a MediaObj.
	mObj := new(MediaObj)
	mObj.End = mediaItem["end"]
	mObj.Muted = muted
	mObj.Loop = loop
	mObj.track = track
	mObj.SetAcceptableFormats(acceptableFormats)

	setSrcErr := mObj.SetSrc(mediaItem["url"])
	if setSrcErr != nil {
		return nil, setSrcErr
	}

	return mObj, nil
}
<commit_msg>now grabs youtube video id from url if present, and instructs minions to embed a youtube video with that id<commit_after>// Copyright 2015, Truveris Inc. All Rights Reserved.
// Use of this source code is governed by the ISC license in the LICENSE file.

package main

import (
	"encoding/json"
	"errors"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"strconv"
	"strings"
)

var (
	// These are the known domains to check for, where special formatting of
	// the passed URL is required so connected minions can most effectively
	// embed and manipulate the desired content.
	imgurHostNames = []string{
		"i.imgur.com",
		"www.imgur.com",
		"imgur.com",
	}
	youtubeHostNames = []string{
		"www.youtube.com",
		"www.youtu.be",
		"youtube.com",
		"youtu.be",
	}

	supportedFormatsAndTypes = map[string][]string{
		"img": {
			"image/bmp",
			"image/cis-cod",
			"image/gif",
			"image/ief",
			"image/jpeg",
			"image/webp",
			"image/pict",
			"image/pipeg",
			"image/png",
			"image/svg+xml",
			"image/tiff",
			"image/vnd.microsoft.icon",
			"image/x-cmu-raster",
			"image/x-cmx",
			"image/x-icon",
			"image/x-portable-anymap",
			"image/x-portable-bitmap",
			"image/x-portable-graymap",
			"image/x-portable-pixmap",
			"image/x-rgb",
			"image/x-xbitmap",
			"image/x-xpixmap",
			"image/x-xwindowdump",
		},
		"audio": {
			"audio/aac",
			"audio/aiff",
			"audio/amr",
			"audio/basic",
			"audio/midi",
			"audio/mp3",
			"audio/mp4",
			"audio/mpeg",
			"audio/mpeg3",
			"audio/ogg",
			"audio/vorbis",
			"audio/wav",
			"audio/webm",
			"audio/x-m4a",
			"audio/x-ms-wma",
			"audio/vnd.rn-realaudio",
			"audio/vnd.wave",
		},
		"video": {
			"video/avi",
			"video/divx",
			"video/flc",
			"video/mp4",
			"video/mpeg",
			"video/ogg",
			"video/quicktime",
			"video/sd-video",
			"video/webm",
			"video/x-dv",
			"video/x-m4v",
			"video/x-mpeg",
			"video/x-ms-asf",
			"video/x-ms-wmv",
		},
		"web": {
			"text/",
		},
	}

	// ygor should fallback to checking the file extensions for potential
	// matches if the content-type doesn't appear to be supported. The server
	// may simply be providing the wrong content-type in the header.
	supportedFormatsAndExtensions = map[string][]string{
		"img": {
			".apng",
			".bmp",
			".dib",
			".gif",
			".jfi",
			".jfif",
			".jif",
			".jpe",
			".jpeg",
			".jpg",
			".png",
			".webp",
		},
		"audio": {
			".mp3",
			".wav",
			".wave",
		},
		"video": {
			".m4a",
			".m4b",
			".m4p",
			".m4r",
			".m4v",
			".mp4",
			".oga",
			".ogg",
			".ogm",
			".ogv",
			".ogx",
			".opus",
			".spx",
			".webm",
		},
	}

	reYTVideoID = regexp.MustCompile(
		`^.*(youtu.be\/|v\/|u\/\w\/|embed\/|watch\?v=|\&v=)([^#\&\?]*).*`)
)

// MediaObj represents the relevant data that will eventually be passed to
// the connected minions. It is used to generate the information that connected
// minions would use to properly embed the desired content.
//
// It also provides several functions that can be used to more easily work with
// the data, so that command modules aren't filled with a lot of excessive
// code.
type MediaObj struct {
	// 'Src' is formatted over time and is what will eventually be passed to
	// the connected minions.
	Src  string `json:"src"`
	url  string
	host string
	// 'Format' tells the connected minions how to embed the desired content
	// using 'Src'.
	Format      string `json:"format"`
	mediaType   string
	// End represents where in the desired content's timeline to stop playing.
	End string `json:"end"`
	// Muted represents whether or not the desired content should be muted.
	Muted             bool `json:"muted"`
	Loop              bool `json:"loop"`
	track             string
	acceptableFormats []string
}

// SetAcceptableFormats takes in a string array of acceptable media types,
// which will be checked against during SetSrc. If the determined media type is
// not acceptable, the url will be rejected.
func (mObj *MediaObj) SetAcceptableFormats(formats []string) {
	mObj.acceptableFormats = formats
}

// checkFormatIsAcceptable checks to make sure that the determined media
// type is acceptable. If the MediaObj's acceptableFormats attribute is not
// set, it is assumed that the media type is acceptable.
func (mObj *MediaObj) checkFormatIsAcceptable() error {
	if len(mObj.acceptableFormats) == 0 {
		// if acceptableFormats is not set, all media types are acceptable
		return nil
	}

	for _, acceptableFormat := range mObj.acceptableFormats {
		if mObj.Format == acceptableFormat {
			// The determined media type is acceptable.
			return nil
		}
	}

	// If it made it here, the determined media type must not be acceptable.
	errMsg := "error: content-type (" + mObj.mediaType + ") not supported " +
		"by this command"
	return errors.New(errMsg)
}

// SetSrc takes in a string that represents a URL. This function determines if
// the URL is a valid URL, formats imgur links to use .webm instead of .gif(v),
// and determines the Format that the URL represents.
//
// The MediaObj's 'Src' attribute will either be set to the passed URL, or the
// formatted imgur URL (if it was an imgur link).
//
// The MediaObj's 'Src' attribute can be retrieved using the MediaObj's
// 'GetSrc()' function.
//
// The URL that was originally passed, is saved as the MediaObj's 'url'
// attribute, and can be retrieved with the MediaObj's 'GetURL()' function.
func (mObj *MediaObj) SetSrc(link string) error {
	uri, linkErr := url.ParseRequestURI(link)
	if linkErr != nil {
		errorMsg := "error: not a valid URL"
		return errors.New(errorMsg)
	}
	// Strip any query or fragment attached to the URL
	mObj.Src = uri.String()
	mObj.url = link
	mObj.host = uri.Host

	// Check that the URL returns a status code of 200.
	res, err := http.Head(mObj.Src)
	if err != nil {
		errMsg := "error: " + err.Error()
		return errors.New(errMsg)
	}
	statusCode := strconv.Itoa(res.StatusCode)
	if statusCode != "200" {
		errMsg := "error: response status code is " + statusCode
		return errors.New(errMsg)
	}

	headErr := mObj.setFormat(res.Header)
	if headErr != nil {
		return headErr
	}

	// If it's an imgur link, and the content-type contains "image/gif", modify
	// the MediaObj so minions embed the far more efficient webm version.
	if mObj.isImgur() {
		isGIF := strings.Contains(strings.ToLower(mObj.mediaType), "image/gif")
		hasGIFVExt := mObj.GetExt() == ".gifv"
		if isGIF || hasGIFVExt {
			mObj.replaceSrcExt(".webm")
			mObj.Format = "video"
			mObj.mediaType = "video/webm"
		}
	}

	merr := mObj.checkFormatIsAcceptable()
	if merr != nil {
		return merr
	}

	return nil
}

// GetSrc returns the MediaObj's 'Src' attribute (this is what should get
// passed to the connected minions).
func (mObj *MediaObj) GetSrc() string {
	return mObj.Src
}

// GetURL returns the URL that was originally passed to the 'SetSrc()'
// function.
func (mObj *MediaObj) GetURL() string {
	return mObj.url
}

// setFormat sets the 'Format' attribute of the MediaObj. This tells the
// connected minions what kind of content they should be trying to embed.
func (mObj *MediaObj) setFormat(header map[string][]string) error {
	// If it's a YouTube link, check if there's a video ID we can grab.
	if mObj.isYouTube() {
		match := reYTVideoID.FindAllStringSubmatch(mObj.Src, -1)
		if len(match) > 0 {
			mObj.Src = match[0][2]
			mObj.Format = "youtube"
			mObj.mediaType = "youtube"
			return nil
		}
	}

	// Is the media type in the contentType an image|audio|video type that
	// Chromium supports?
	if contentType, ok := header["Content-Type"]; ok {
		// Check for standard, supported media types.
		for format, formatMediaTypes := range supportedFormatsAndTypes {
			for _, mediaType := range formatMediaTypes {
				for _, cType := range contentType {
					if strings.Contains(cType, mediaType) {
						mObj.Format = format
						mObj.mediaType = mediaType
						return nil
					}
				}
			}
		}

		// Fallback to known supported file extensions if content-type isn't
		// recognized as supported.
		ext := mObj.GetExt()
		for format, formatExtensions := range supportedFormatsAndExtensions {
			for _, extension := range formatExtensions {
				if extension == ext {
					mObj.Format = format
					mObj.mediaType = ext
					return nil
				}
			}
		}

		// If the media type isn't supported, return an error.
		errMsg := "error: unsupported content-type " +
			"(" + strings.Join(contentType, ", ") + ")"
		return errors.New(errMsg)
	}

	// It will only get here if it didn't have a content-type in the header.
	errMsg := "error: no content-type found"
	return errors.New(errMsg)
}

// GetFormat returnes the MediaObj's 'Format' attribute. The 'Format'
// tells the connected minions what kind of content they should be trying to
// embed when using the MediaObj's 'Src' attribute.
func (mObj *MediaObj) GetFormat() string {
	return mObj.Format
}

// IsOfFormat determines if the MediaObj's Format is contained in the
// passed string array.
func (mObj *MediaObj) IsOfFormat(formats []string) bool {
	format := mObj.GetFormat()
	for _, mt := range formats {
		if format == mt {
			return true
		}
	}
	return false
}

// GetExt is a convenience function to get the extension of theMediaObj's
// current Src.
func (mObj *MediaObj) GetExt() string {
	return strings.ToLower(path.Ext(mObj.Src))
}

// isImgur attempts to determine if the desired content is hosted on imgur.
func (mObj *MediaObj) isImgur() bool {
	for _, d := range imgurHostNames {
		if mObj.host == d {
			return true
		}
	}
	return false
}

// isYouTube attempts to determine if the desired content is a video hosted on
// YouTube
func (mObj *MediaObj) isYouTube() bool {
	for _, d := range youtubeHostNames {
		if mObj.host == d {
			return true
		}
	}
	return false
}

// replaceSrcExt is a convenience function to replace the extension of the
// MediaObj's current Src.
func (mObj *MediaObj) replaceSrcExt(newExt string) {
	mObj.Src = mObj.Src[0:len(mObj.Src)-len(mObj.GetExt())] + newExt
}

// Serialize generates and returns the JSON string out of the MediaObj. This
// JSON string is what should be sent to the connected minions.
func (mObj *MediaObj) Serialize() string {
	serializedJSON, _ := json.Marshal(struct {
		MediaObj *MediaObj `json:"mediaObj"`
		Status   string    `json:"status"`
		Track    string    `json:"track"`
	}{
		Status:   "media",
		Track:    mObj.track,
		MediaObj: mObj,
	})
	return string(serializedJSON)
}

// NewMediaObj is a convenience function meant to clean up the code of modules.
// It builds the MediaObj.
func NewMediaObj(mediaItem map[string]string, track string, muted bool, loop bool, acceptableFormats []string) (*MediaObj, error) {
	// Parse the mediaItem map into a MediaObj.
	mObj := new(MediaObj)
	mObj.End = mediaItem["end"]
	mObj.Muted = muted
	mObj.Loop = loop
	mObj.track = track
	mObj.SetAcceptableFormats(acceptableFormats)

	setSrcErr := mObj.SetSrc(mediaItem["url"])
	if setSrcErr != nil {
		return nil, setSrcErr
	}

	return mObj, nil
}
<|endoftext|>


<commit_before>// Copyright © 2013-14 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bufferpool

import (
	"bytes"
	"sync"
)

// AverageBufferSize should be adjusted to the average size of a bytes.buffer
// in your application
var AverageBufferSize int = 16

var bufferPool = &sync.Pool{
	New: func() interface{} {
		b := bytes.NewBuffer(make([]byte, AverageBufferSize))
		b.Reset()
		return b
	},
}

// Get returns a buffer from the pool.
func Get() (buf *bytes.Buffer) {
	return bufferPool.Get().(*bytes.Buffer)
}

// Put returns a buffer to the pool.
// The buffer is reset before it is put back into circulation.
func Put(buf *bytes.Buffer) {
	// println(buf.Len()) todo for some statistics
	buf.Reset()
	bufferPool.Put(buf)
}
<commit_msg>util/bufferpool: Todo add https://github.com/thejerf/gomempool<commit_after>// Copyright © 2013-14 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bufferpool

import (
	"bytes"
	"sync"
)

// AverageBufferSize should be adjusted to the average size of a bytes.buffer
// in your application.
// TODO: https://github.com/thejerf/gomempool
var AverageBufferSize int = 16

var bufferPool = &sync.Pool{
	New: func() interface{} {
		b := bytes.NewBuffer(make([]byte, AverageBufferSize))
		b.Reset()
		return b
	},
}

// Get returns a buffer from the pool.
func Get() (buf *bytes.Buffer) {
	return bufferPool.Get().(*bytes.Buffer)
}

// Put returns a buffer to the pool.
// The buffer is reset before it is put back into circulation.
func Put(buf *bytes.Buffer) {
	// println(buf.Len()) todo for some statistics
	buf.Reset()
	bufferPool.Put(buf)
}
<|endoftext|>