<commit_before>package etcdhttp

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"

	crand "crypto/rand"
	"math/rand"

	"github.com/coreos/etcd/elog"
	etcdErr "github.com/coreos/etcd/error"
	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/store"
	"github.com/coreos/etcd/third_party/code.google.com/p/go.net/context"
)

const (
	keysPrefix     = "/v2/keys"
	machinesPrefix = "/v2/machines"
)

type Peers map[int64][]string

func (ps Peers) Pick(id int64) string {
	addrs := ps[id]
	if len(addrs) == 0 {
		return ""
	}
	return addScheme(addrs[rand.Intn(len(addrs))])
}

// TODO: improve this when implementing TLS
func addScheme(addr string) string {
	return fmt.Sprintf("http://%s", addr)
}

// Set parses command line sets of names to ips formatted like:
// a=1.1.1.1&a=1.1.1.2&b=2.2.2.2
func (ps *Peers) Set(s string) error {
	m := make(map[int64][]string)
	v, err := url.ParseQuery(s)
	if err != nil {
		return err
	}
	for k, v := range v {
		id, err := strconv.ParseInt(k, 0, 64)
		if err != nil {
			return err
		}
		m[id] = v
	}
	*ps = m
	return nil
}

func (ps *Peers) String() string {
	v := url.Values{}
	for k, vv := range *ps {
		for i := range vv {
			v.Add(strconv.FormatInt(k, 16), vv[i])
		}
	}
	return v.Encode()
}

func (ps Peers) Ids() []int64 {
	var ids []int64
	for id := range ps {
		ids = append(ids, id)
	}
	return ids
}

var errClosed = errors.New("etcdhttp: client closed connection")

const DefaultTimeout = 500 * time.Millisecond

func Sender(p Peers) func(msgs []raftpb.Message) {
	return func(msgs []raftpb.Message) {
		for _, m := range msgs {
			// TODO: reuse go routines
			// limit the number of outgoing connections for the same receiver
			go send(p, m)
		}
	}
}

func send(p Peers, m raftpb.Message) {
	// TODO (xiangli): reasonable retry logic
	for i := 0; i < 3; i++ {
		url := p.Pick(m.To)
		if url == "" {
			// TODO: unknown peer id.. what do we do? I
			// don't think his should ever happen, need to
			// look into this further.
			log.Println("etcdhttp: no addr for %d", m.To)
			return
		}

		url += "/raft"

		// TODO: don't block. we should be able to have 1000s
		// of messages out at a time.
		data, err := m.Marshal()
		if err != nil {
			log.Println("etcdhttp: dropping message:", err)
			return // drop bad message
		}
		if httpPost(url, data) {
			return // success
		}
		// TODO: backoff
	}
}

func httpPost(url string, data []byte) bool {
	// TODO: set timeouts
	resp, err := http.Post(url, "application/protobuf", bytes.NewBuffer(data))
	if err != nil {
		elog.TODO()
		return false
	}
	resp.Body.Close()
	if resp.StatusCode != 200 {
		elog.TODO()
		return false
	}
	return true
}

// Handler implements the http.Handler interface and serves etcd client and
// raft communication.
type Handler struct {
	Timeout time.Duration
	Server  *etcdserver.Server
	// TODO: dynamic configuration may make this outdated. take care of it.
	// TODO: dynamic configuration may introduce race also.
	Peers Peers
}

func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// TODO: set read/write timeout?

	timeout := h.Timeout
	if timeout == 0 {
		timeout = DefaultTimeout
	}

	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	switch {
	case strings.HasPrefix(r.URL.Path, "/raft"):
		h.serveRaft(ctx, w, r)
	case strings.HasPrefix(r.URL.Path, keysPrefix):
		h.serveKeys(ctx, w, r)
	case strings.HasPrefix(r.URL.Path, machinesPrefix):
		h.serveMachines(w, r)
	default:
		http.NotFound(w, r)
	}
}

func (h Handler) serveKeys(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	rr, err := parseRequest(r, genId())
	if err != nil {
		writeError(w, err)
		return
	}

	resp, err := h.Server.Do(ctx, rr)
	if err != nil {
		writeError(w, err)
		return
	}

	var ev *store.Event
	switch {
	case resp.Event != nil:
		ev = resp.Event
	case resp.Watcher != nil:
		if ev, err = waitForEvent(ctx, w, resp.Watcher); err != nil {
			http.Error(w, err.Error(), http.StatusGatewayTimeout)
			return
		}
	default:
		writeError(w, errors.New("received response with no Event/Watcher!"))
		return
	}

	if err = writeEvent(w, ev); err != nil {
		// Should never be reached
		log.Println("error writing event: %v", err)
	}
}

// serveMachines responds address list in the format '0.0.0.0, 1.1.1.1'.
// TODO: rethink the format of machine list because it is not json format.
func (h Handler) serveMachines(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" && r.Method != "HEAD" {
		allow(w, "GET", "HEAD")
		return
	}
	urls := make([]string, 0)
	for _, addrs := range h.Peers {
		for _, addr := range addrs {
			urls = append(urls, addScheme(addr))
		}
	}
	sort.Strings(urls)
	w.Write([]byte(strings.Join(urls, ", ")))
}

func (h Handler) serveRaft(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Println("etcdhttp: error reading raft message:", err)
	}
	var m raftpb.Message
	if err := m.Unmarshal(b); err != nil {
		log.Println("etcdhttp: error unmarshaling raft message:", err)
	}
	log.Printf("etcdhttp: raft recv message from %#x: %+v", m.From, m)
	if err := h.Server.Node.Step(ctx, m); err != nil {
		log.Println("etcdhttp: error stepping raft messages:", err)
	}
}

// genId generates a random id that is: n < 0 < n.
func genId() int64 {
	for {
		b := make([]byte, 8)
		if _, err := io.ReadFull(crand.Reader, b); err != nil {
			panic(err) // really bad stuff happened
		}
		n := int64(binary.BigEndian.Uint64(b))
		if n != 0 {
			return n
		}
	}
}

// parseRequest converts a received http.Request to a server Request,
// performing validation of supplied fields as appropriate.
// If any validation fails, an empty Request and non-nil error is returned.
func parseRequest(r *http.Request, id int64) (etcdserverpb.Request, error) {
	emptyReq := etcdserverpb.Request{}

	err := r.ParseForm()
	if err != nil {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeInvalidForm,
			err.Error(),
		)
	}

	if !strings.HasPrefix(r.URL.Path, keysPrefix) {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeInvalidForm,
			"incorrect key prefix",
		)
	}
	p := r.URL.Path[len(keysPrefix):]

	var pIdx, wIdx, ttl uint64
	if pIdx, err = getUint64(r.Form, "prevIndex"); err != nil {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeIndexNaN,
			fmt.Sprintf("invalid value for prevIndex"),
		)
	}
	if wIdx, err = getUint64(r.Form, "waitIndex"); err != nil {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeIndexNaN,
			fmt.Sprintf("invalid value for waitIndex"),
		)
	}
	if ttl, err = getUint64(r.Form, "ttl"); err != nil {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeTTLNaN,
			`invalid value for "ttl"`,
		)
	}

	var rec, sort, wait bool
	if rec, err = getBool(r.Form, "recursive"); err != nil {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeInvalidField,
			`invalid value for "recursive"`,
		)
	}
	if sort, err = getBool(r.Form, "sorted"); err != nil {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeInvalidField,
			`invalid value for "sorted"`,
		)
	}
	if wait, err = getBool(r.Form, "wait"); err != nil {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeInvalidField,
			`invalid value for "wait"`,
		)
	}

	// prevExists is nullable, so leave it null if not specified
	var pe *bool
	if _, ok := r.Form["prevExists"]; ok {
		bv, err := getBool(r.Form, "prevExists")
		if err != nil {
			return emptyReq, etcdErr.NewRequestError(
				etcdErr.EcodeInvalidField,
				"invalid value for prevExists",
			)
		}
		pe = &bv
	}

	rr := etcdserverpb.Request{
		Id:         id,
		Method:     r.Method,
		Path:       p,
		Val:        r.FormValue("value"),
		PrevValue:  r.FormValue("prevValue"),
		PrevIndex:  pIdx,
		PrevExists: pe,
		Recursive:  rec,
		Since:      wIdx,
		Sorted:     sort,
		Wait:       wait,
	}

	if pe != nil {
		rr.PrevExists = pe
	}

	if ttl > 0 {
		expr := time.Duration(ttl) * time.Second
		// TODO(jonboulle): use fake clock instead of time module
		// https://github.com/coreos/etcd/issues/1021
		rr.Expiration = time.Now().Add(expr).UnixNano()
	}

	return rr, nil
}

// getUint64 extracts a uint64 by the given key from a Form. If the key does
// not exist in the form, 0 is returned. If the key exists but the value is
// badly formed, an error is returned. If multiple values are present only the
// first is considered.
func getUint64(form url.Values, key string) (i uint64, err error) {
	if vals, ok := form[key]; ok {
		i, err = strconv.ParseUint(vals[0], 10, 64)
	}
	return
}

// getBool extracts a bool by the given key from a Form. If the key does not
// exist in the form, false is returned. If the key exists but the value is
// badly formed, an error is returned. If multiple values are present only the
// first is considered.
func getBool(form url.Values, key string) (b bool, err error) {
	if vals, ok := form[key]; ok {
		b, err = strconv.ParseBool(vals[0])
	}
	return
}

// writeError logs and writes the given Error to the ResponseWriter
// If Error is an etcdErr, it is rendered to the ResponseWriter
// Otherwise, it is assumed to be an InternalServerError
func writeError(w http.ResponseWriter, err error) {
	if err == nil {
		return
	}
	log.Println(err)
	if e, ok := err.(*etcdErr.Error); ok {
		e.Write(w)
	} else {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
	}
}

// writeEvent serializes the given Event and writes the resulting JSON to the
// given ResponseWriter
func writeEvent(w http.ResponseWriter, ev *store.Event) error {
	if ev == nil {
		return errors.New("cannot write empty Event!")
	}
	w.Header().Set("Content-Type", "application/json")
	w.Header().Add("X-Etcd-Index", fmt.Sprint(ev.Index()))

	if ev.IsCreated() {
		w.WriteHeader(http.StatusCreated)
	}

	return json.NewEncoder(w).Encode(ev)
}

// waitForEvent waits for a given Watcher to return its associated
// event. It returns a non-nil error if the given Context times out
// or the given ResponseWriter triggers a CloseNotify.
func waitForEvent(ctx context.Context, w http.ResponseWriter, wa store.Watcher) (*store.Event, error) {
	// TODO(bmizerany): support streaming?
	defer wa.Remove()
	var nch <-chan bool
	if x, ok := w.(http.CloseNotifier); ok {
		nch = x.CloseNotify()
	}
	select {
	case ev := <-wa.EventChan():
		return ev, nil
	case <-nch:
		elog.TODO()
		return nil, errClosed
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

// allow writes response for the case that Method Not Allowed
func allow(w http.ResponseWriter, m ...string) {
	w.Header().Set("Allow", strings.Join(m, ","))
	http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
}
<commit_msg>simplify<commit_after>package etcdhttp

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"

	crand "crypto/rand"
	"math/rand"

	"github.com/coreos/etcd/elog"
	etcdErr "github.com/coreos/etcd/error"
	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/store"
	"github.com/coreos/etcd/third_party/code.google.com/p/go.net/context"
)

const (
	keysPrefix     = "/v2/keys"
	machinesPrefix = "/v2/machines"
)

type Peers map[int64][]string

func (ps Peers) Pick(id int64) string {
	addrs := ps[id]
	if len(addrs) == 0 {
		return ""
	}
	return addScheme(addrs[rand.Intn(len(addrs))])
}

// TODO: improve this when implementing TLS
func addScheme(addr string) string {
	return fmt.Sprintf("http://%s", addr)
}

// Set parses command line sets of names to ips formatted like:
// a=1.1.1.1&a=1.1.1.2&b=2.2.2.2
func (ps *Peers) Set(s string) error {
	m := make(map[int64][]string)
	v, err := url.ParseQuery(s)
	if err != nil {
		return err
	}
	for k, v := range v {
		id, err := strconv.ParseInt(k, 0, 64)
		if err != nil {
			return err
		}
		m[id] = v
	}
	*ps = m
	return nil
}

func (ps *Peers) String() string {
	v := url.Values{}
	for k, vv := range *ps {
		for i := range vv {
			v.Add(strconv.FormatInt(k, 16), vv[i])
		}
	}
	return v.Encode()
}

func (ps Peers) Ids() []int64 {
	var ids []int64
	for id := range ps {
		ids = append(ids, id)
	}
	return ids
}

var errClosed = errors.New("etcdhttp: client closed connection")

const DefaultTimeout = 500 * time.Millisecond

func Sender(p Peers) func(msgs []raftpb.Message) {
	return func(msgs []raftpb.Message) {
		for _, m := range msgs {
			// TODO: reuse go routines
			// limit the number of outgoing connections for the same receiver
			go send(p, m)
		}
	}
}

func send(p Peers, m raftpb.Message) {
	// TODO (xiangli): reasonable retry logic
	for i := 0; i < 3; i++ {
		url := p.Pick(m.To)
		if url == "" {
			// TODO: unknown peer id.. what do we do? I
			// don't think his should ever happen, need to
			// look into this further.
			log.Println("etcdhttp: no addr for %d", m.To)
			return
		}

		url += "/raft"

		// TODO: don't block. we should be able to have 1000s
		// of messages out at a time.
		data, err := m.Marshal()
		if err != nil {
			log.Println("etcdhttp: dropping message:", err)
			return // drop bad message
		}
		if httpPost(url, data) {
			return // success
		}
		// TODO: backoff
	}
}

func httpPost(url string, data []byte) bool {
	// TODO: set timeouts
	resp, err := http.Post(url, "application/protobuf", bytes.NewBuffer(data))
	if err != nil {
		elog.TODO()
		return false
	}
	resp.Body.Close()
	if resp.StatusCode != 200 {
		elog.TODO()
		return false
	}
	return true
}

// Handler implements the http.Handler interface and serves etcd client and
// raft communication.
type Handler struct {
	Timeout time.Duration
	Server  *etcdserver.Server
	// TODO: dynamic configuration may make this outdated. take care of it.
	// TODO: dynamic configuration may introduce race also.
	Peers Peers
}

func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// TODO: set read/write timeout?

	timeout := h.Timeout
	if timeout == 0 {
		timeout = DefaultTimeout
	}

	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	switch {
	case strings.HasPrefix(r.URL.Path, "/raft"):
		h.serveRaft(ctx, w, r)
	case strings.HasPrefix(r.URL.Path, keysPrefix):
		h.serveKeys(ctx, w, r)
	case strings.HasPrefix(r.URL.Path, machinesPrefix):
		h.serveMachines(w, r)
	default:
		http.NotFound(w, r)
	}
}

func (h Handler) serveKeys(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	rr, err := parseRequest(r, genId())
	if err != nil {
		writeError(w, err)
		return
	}

	resp, err := h.Server.Do(ctx, rr)
	if err != nil {
		writeError(w, err)
		return
	}

	var ev *store.Event
	switch {
	case resp.Event != nil:
		ev = resp.Event
	case resp.Watcher != nil:
		if ev, err = waitForEvent(ctx, w, resp.Watcher); err != nil {
			http.Error(w, err.Error(), http.StatusGatewayTimeout)
			return
		}
	default:
		writeError(w, errors.New("received response with no Event/Watcher!"))
		return
	}

	if err = writeEvent(w, ev); err != nil {
		// Should never be reached
		log.Println("error writing event: %v", err)
	}
}

// serveMachines responds address list in the format '0.0.0.0, 1.1.1.1'.
// TODO: rethink the format of machine list because it is not json format.
func (h Handler) serveMachines(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" && r.Method != "HEAD" {
		allow(w, "GET", "HEAD")
		return
	}
	urls := make([]string, 0)
	for _, addrs := range h.Peers {
		for _, addr := range addrs {
			urls = append(urls, addScheme(addr))
		}
	}
	sort.Strings(urls)
	w.Write([]byte(strings.Join(urls, ", ")))
}

func (h Handler) serveRaft(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Println("etcdhttp: error reading raft message:", err)
	}
	var m raftpb.Message
	if err := m.Unmarshal(b); err != nil {
		log.Println("etcdhttp: error unmarshaling raft message:", err)
	}
	log.Printf("etcdhttp: raft recv message from %#x: %+v", m.From, m)
	if err := h.Server.Node.Step(ctx, m); err != nil {
		log.Println("etcdhttp: error stepping raft messages:", err)
	}
}

// genId generates a random id that is: n < 0 < n.
func genId() int64 {
	for {
		b := make([]byte, 8)
		if _, err := io.ReadFull(crand.Reader, b); err != nil {
			panic(err) // really bad stuff happened
		}
		n := int64(binary.BigEndian.Uint64(b))
		if n != 0 {
			return n
		}
	}
}

// parseRequest converts a received http.Request to a server Request,
// performing validation of supplied fields as appropriate.
// If any validation fails, an empty Request and non-nil error is returned.
func parseRequest(r *http.Request, id int64) (etcdserverpb.Request, error) {
	emptyReq := etcdserverpb.Request{}

	err := r.ParseForm()
	if err != nil {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeInvalidForm,
			err.Error(),
		)
	}

	if !strings.HasPrefix(r.URL.Path, keysPrefix) {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeInvalidForm,
			"incorrect key prefix",
		)
	}
	p := r.URL.Path[len(keysPrefix):]

	var pIdx, wIdx, ttl uint64
	if pIdx, err = getUint64(r.Form, "prevIndex"); err != nil {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeIndexNaN,
			`invalid value for "prevIndex"`,
		)
	}
	if wIdx, err = getUint64(r.Form, "waitIndex"); err != nil {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeIndexNaN,
			`invalid value for "waitIndex"`,
		)
	}
	if ttl, err = getUint64(r.Form, "ttl"); err != nil {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeTTLNaN,
			`invalid value for "ttl"`,
		)
	}

	var rec, sort, wait bool
	if rec, err = getBool(r.Form, "recursive"); err != nil {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeInvalidField,
			`invalid value for "recursive"`,
		)
	}
	if sort, err = getBool(r.Form, "sorted"); err != nil {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeInvalidField,
			`invalid value for "sorted"`,
		)
	}
	if wait, err = getBool(r.Form, "wait"); err != nil {
		return emptyReq, etcdErr.NewRequestError(
			etcdErr.EcodeInvalidField,
			`invalid value for "wait"`,
		)
	}

	// prevExists is nullable, so leave it null if not specified
	var pe *bool
	if _, ok := r.Form["prevExists"]; ok {
		bv, err := getBool(r.Form, "prevExists")
		if err != nil {
			return emptyReq, etcdErr.NewRequestError(
				etcdErr.EcodeInvalidField,
				"invalid value for prevExists",
			)
		}
		pe = &bv
	}

	rr := etcdserverpb.Request{
		Id:         id,
		Method:     r.Method,
		Path:       p,
		Val:        r.FormValue("value"),
		PrevValue:  r.FormValue("prevValue"),
		PrevIndex:  pIdx,
		PrevExists: pe,
		Recursive:  rec,
		Since:      wIdx,
		Sorted:     sort,
		Wait:       wait,
	}

	if pe != nil {
		rr.PrevExists = pe
	}

	if ttl > 0 {
		expr := time.Duration(ttl) * time.Second
		// TODO(jonboulle): use fake clock instead of time module
		// https://github.com/coreos/etcd/issues/1021
		rr.Expiration = time.Now().Add(expr).UnixNano()
	}

	return rr, nil
}

// getUint64 extracts a uint64 by the given key from a Form. If the key does
// not exist in the form, 0 is returned. If the key exists but the value is
// badly formed, an error is returned. If multiple values are present only the
// first is considered.
func getUint64(form url.Values, key string) (i uint64, err error) {
	if vals, ok := form[key]; ok {
		i, err = strconv.ParseUint(vals[0], 10, 64)
	}
	return
}

// getBool extracts a bool by the given key from a Form. If the key does not
// exist in the form, false is returned. If the key exists but the value is
// badly formed, an error is returned. If multiple values are present only the
// first is considered.
func getBool(form url.Values, key string) (b bool, err error) {
	if vals, ok := form[key]; ok {
		b, err = strconv.ParseBool(vals[0])
	}
	return
}

// writeError logs and writes the given Error to the ResponseWriter
// If Error is an etcdErr, it is rendered to the ResponseWriter
// Otherwise, it is assumed to be an InternalServerError
func writeError(w http.ResponseWriter, err error) {
	if err == nil {
		return
	}
	log.Println(err)
	if e, ok := err.(*etcdErr.Error); ok {
		e.Write(w)
	} else {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
	}
}

// writeEvent serializes the given Event and writes the resulting JSON to the
// given ResponseWriter
func writeEvent(w http.ResponseWriter, ev *store.Event) error {
	if ev == nil {
		return errors.New("cannot write empty Event!")
	}
	w.Header().Set("Content-Type", "application/json")
	w.Header().Add("X-Etcd-Index", fmt.Sprint(ev.Index()))

	if ev.IsCreated() {
		w.WriteHeader(http.StatusCreated)
	}

	return json.NewEncoder(w).Encode(ev)
}

// waitForEvent waits for a given Watcher to return its associated
// event. It returns a non-nil error if the given Context times out
// or the given ResponseWriter triggers a CloseNotify.
func waitForEvent(ctx context.Context, w http.ResponseWriter, wa store.Watcher) (*store.Event, error) {
	// TODO(bmizerany): support streaming?
	defer wa.Remove()
	var nch <-chan bool
	if x, ok := w.(http.CloseNotifier); ok {
		nch = x.CloseNotify()
	}
	select {
	case ev := <-wa.EventChan():
		return ev, nil
	case <-nch:
		elog.TODO()
		return nil, errClosed
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

// allow writes response for the case that Method Not Allowed
func allow(w http.ResponseWriter, m ...string) {
	w.Header().Set("Allow", strings.Join(m, ","))
	http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
}
<|endoftext|>


<commit_before>package dalga

// TODO list
// write basic integration tests
// handle mysql disconnect
// handle rabbitmq disconnect

import (
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"github.com/streadway/amqp"
	"log"
	"net"
	"strconv"
	"strings"
	"time"
)

var debugging = flag.Bool("d", false, "turn on debug messages")

func debug(args ...interface{}) {
	if *debugging {
		log.Println(args...)
	}
}

type Dalga struct {
	C            *Config
	db           *sql.DB
	rabbit       *amqp.Connection
	channel      *amqp.Channel
	listener     net.Listener
	newJobs      chan *Job
	canceledJobs chan *Job
	quit         chan bool
}

func NewDalga(config *Config) *Dalga {
	return &Dalga{
		C:            config,
		newJobs:      make(chan *Job),
		canceledJobs: make(chan *Job),
		quit:         make(chan bool, 1),
	}
}

func (d *Dalga) Run() error {
	err := d.connectDB()
	if err != nil {
		return err
	}

	err = d.connectMQ()
	if err != nil {
		return err
	}

	server, err := d.makeServer()
	if err != nil {
		return err
	}

	go d.publisher()
	go server()

	debug("Waiting a message from quit channel")
	<-d.quit
	debug("Got quit message")
	return nil
}

func (d *Dalga) Shutdown() error {
	return d.listener.Close()
}

func (d *Dalga) connectDB() error {
	var err error
	my := d.C.MySQL
	dsn := my.User + ":" + my.Password + "@" + "tcp(" + my.Host + ":" + my.Port + ")/" + my.Db + "?parseTime=true"
	d.db, err = sql.Open("mysql", dsn)
	if err != nil {
		return err
	}
	fmt.Println("Connected to MySQL")
	return d.db.Ping()
}

func (d *Dalga) connectMQ() error {
	var err error
	rabbit := d.C.RabbitMQ
	uri := "amqp://" + rabbit.User + ":" + rabbit.Password + "@" + rabbit.Host + ":" + rabbit.Port + rabbit.VHost
	d.rabbit, err = amqp.Dial(uri)
	if err != nil {
		return err
	}
	d.channel, err = d.rabbit.Channel()
	fmt.Println("Connected to RabbitMQ")
	return err
}

// front returns the first job to be run in the queue.
func (d *Dalga) front() (*Job, error) {
	var interval uint
	j := Job{}
	row := d.db.QueryRow("SELECT routing_key, body, `interval`, next_run " +
		"FROM " + d.C.MySQL.Table + " " +
		"ORDER BY next_run ASC LIMIT 1")
	err := row.Scan(&j.RoutingKey, &j.Body, &interval, &j.NextRun)
	if err != nil {
		return nil, err
	}
	j.Interval = time.Duration(interval) * time.Second
	return &j, nil
}

// publish sends a message to exchange defined in the config and
// updates the Job's next run time on the database.
func (d *Dalga) publish(j *Job) error {
	debug("publish", *j)

	// Update next run time
	_, err := d.db.Exec("UPDATE "+d.C.MySQL.Table+" "+
		"SET next_run=? "+
		"WHERE routing_key=? AND body=?",
		time.Now().UTC().Add(j.Interval), j.RoutingKey, j.Body)
	if err != nil {
		return err
	}

	// Send a message to RabbitMQ
	err = d.channel.Publish(d.C.RabbitMQ.Exchange, j.RoutingKey, false, false, amqp.Publishing{
		Headers: amqp.Table{
			"interval":     j.Interval.Seconds(),
			"published_at": time.Now().UTC().String(),
		},
		ContentType:     "text/plain",
		ContentEncoding: "UTF-8",
		Body:            []byte(j.Body),
		DeliveryMode:    amqp.Persistent,
		Priority:        0,
		Expiration:      strconv.FormatUint(uint64(j.Interval.Seconds()), 10) + "000",
	})
	if err != nil {
		return err
	}

	return nil
}

// enter puts the job to the waiting queue.
func (d *Dalga) enter(j *Job) error {
	interval := j.Interval.Seconds()
	_, err := d.db.Exec("INSERT INTO "+d.C.MySQL.Table+" "+
		"(routing_key, body, `interval`, next_run) "+
		"VALUES(?, ?, ?, ?) "+
		"ON DUPLICATE KEY UPDATE "+
		"next_run=DATE_ADD(next_run, INTERVAL (? - `interval`) SECOND), "+
		"`interval`=?",
		j.RoutingKey, j.Body, interval, j.NextRun, interval, interval)
	return err
}

// cancel removes the job from the waiting queue.
func (d *Dalga) cancel(routingKey, body string) error {
	_, err := d.db.Exec("DELETE FROM "+d.C.MySQL.Table+" "+
		"WHERE routing_key=? AND body=?", routingKey, body)
	return err
}

// publisher runs a loop that reads the next Job from the queue and publishes it.
func (d *Dalga) publisher() {
	publish := func(j *Job) {
		err := d.publish(j)
		if err != nil {
			fmt.Println(err)
		}
	}

	for {
		debug("")

		job, err := d.front()
		if err != nil {
			if strings.Contains(err.Error(), "no rows in result set") {
				debug("No waiting jobs in the queue")
				debug("Waiting wakeup signal")
				job = <-d.newJobs
				debug("Got wakeup signal")
			} else {
				fmt.Println(err)
				continue
			}
		}

	CheckNextRun:
		remaining := job.Remaining()
		debug("Next job:", job, "Remaining:", remaining)

		now := time.Now().UTC()
		if job.NextRun.After(now) {
			// Wait until the next Job time or
			// the webserver's /schedule handler wakes us up
			debug("Sleeping for job:", remaining)
			select {
			case <-time.After(remaining):
				debug("Job sleep time finished")
				publish(job)
			case newJob := <-d.newJobs:
				debug("A new job has been scheduled")
				if newJob.NextRun.Before(job.NextRun) {
					debug("The new job comes before out current job")
					job = newJob // Process the new job next
				}
				// Continue processing the current job without fetching from database
				goto CheckNextRun
			case canceledJob := <-d.canceledJobs:
				debug("A job has been cancelled")
				if (job.RoutingKey == canceledJob.RoutingKey) && (job.Body == canceledJob.Body) {
					// The job we are waiting for has been canceled.
					// We need to fetch the next job in the queue.
					debug("The cancelled job is our current job")
					continue
				}
				// Continue to process our current job
				goto CheckNextRun
			}
		} else {
			publish(job)
		}

	}
}
<commit_msg>new todos<commit_after>package dalga

// TODO list
// use bytes
// option for creating table
// write basic integration tests
// handle mysql disconnect
// handle rabbitmq disconnect

import (
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"github.com/streadway/amqp"
	"log"
	"net"
	"strconv"
	"strings"
	"time"
)

var debugging = flag.Bool("d", false, "turn on debug messages")

func debug(args ...interface{}) {
	if *debugging {
		log.Println(args...)
	}
}

type Dalga struct {
	C            *Config
	db           *sql.DB
	rabbit       *amqp.Connection
	channel      *amqp.Channel
	listener     net.Listener
	newJobs      chan *Job
	canceledJobs chan *Job
	quit         chan bool
}

func NewDalga(config *Config) *Dalga {
	return &Dalga{
		C:            config,
		newJobs:      make(chan *Job),
		canceledJobs: make(chan *Job),
		quit:         make(chan bool, 1),
	}
}

func (d *Dalga) Run() error {
	err := d.connectDB()
	if err != nil {
		return err
	}

	err = d.connectMQ()
	if err != nil {
		return err
	}

	server, err := d.makeServer()
	if err != nil {
		return err
	}

	go d.publisher()
	go server()

	debug("Waiting a message from quit channel")
	<-d.quit
	debug("Got quit message")
	return nil
}

func (d *Dalga) Shutdown() error {
	return d.listener.Close()
}

func (d *Dalga) connectDB() error {
	var err error
	my := d.C.MySQL
	dsn := my.User + ":" + my.Password + "@" + "tcp(" + my.Host + ":" + my.Port + ")/" + my.Db + "?parseTime=true"
	d.db, err = sql.Open("mysql", dsn)
	if err != nil {
		return err
	}
	fmt.Println("Connected to MySQL")
	return d.db.Ping()
}

func (d *Dalga) connectMQ() error {
	var err error
	rabbit := d.C.RabbitMQ
	uri := "amqp://" + rabbit.User + ":" + rabbit.Password + "@" + rabbit.Host + ":" + rabbit.Port + rabbit.VHost
	d.rabbit, err = amqp.Dial(uri)
	if err != nil {
		return err
	}
	d.channel, err = d.rabbit.Channel()
	fmt.Println("Connected to RabbitMQ")
	return err
}

// front returns the first job to be run in the queue.
func (d *Dalga) front() (*Job, error) {
	var interval uint
	j := Job{}
	row := d.db.QueryRow("SELECT routing_key, body, `interval`, next_run " +
		"FROM " + d.C.MySQL.Table + " " +
		"ORDER BY next_run ASC LIMIT 1")
	err := row.Scan(&j.RoutingKey, &j.Body, &interval, &j.NextRun)
	if err != nil {
		return nil, err
	}
	j.Interval = time.Duration(interval) * time.Second
	return &j, nil
}

// publish sends a message to exchange defined in the config and
// updates the Job's next run time on the database.
func (d *Dalga) publish(j *Job) error {
	debug("publish", *j)

	// Update next run time
	_, err := d.db.Exec("UPDATE "+d.C.MySQL.Table+" "+
		"SET next_run=? "+
		"WHERE routing_key=? AND body=?",
		time.Now().UTC().Add(j.Interval), j.RoutingKey, j.Body)
	if err != nil {
		return err
	}

	// Send a message to RabbitMQ
	err = d.channel.Publish(d.C.RabbitMQ.Exchange, j.RoutingKey, false, false, amqp.Publishing{
		Headers: amqp.Table{
			"interval":     j.Interval.Seconds(),
			"published_at": time.Now().UTC().String(),
		},
		ContentType:     "text/plain",
		ContentEncoding: "UTF-8",
		Body:            []byte(j.Body),
		DeliveryMode:    amqp.Persistent,
		Priority:        0,
		Expiration:      strconv.FormatUint(uint64(j.Interval.Seconds()), 10) + "000",
	})
	if err != nil {
		return err
	}

	return nil
}

// enter puts the job to the waiting queue.
func (d *Dalga) enter(j *Job) error {
	interval := j.Interval.Seconds()
	_, err := d.db.Exec("INSERT INTO "+d.C.MySQL.Table+" "+
		"(routing_key, body, `interval`, next_run) "+
		"VALUES(?, ?, ?, ?) "+
		"ON DUPLICATE KEY UPDATE "+
		"next_run=DATE_ADD(next_run, INTERVAL (? - `interval`) SECOND), "+
		"`interval`=?",
		j.RoutingKey, j.Body, interval, j.NextRun, interval, interval)
	return err
}

// cancel removes the job from the waiting queue.
func (d *Dalga) cancel(routingKey, body string) error {
	_, err := d.db.Exec("DELETE FROM "+d.C.MySQL.Table+" "+
		"WHERE routing_key=? AND body=?", routingKey, body)
	return err
}

// publisher runs a loop that reads the next Job from the queue and publishes it.
func (d *Dalga) publisher() {
	publish := func(j *Job) {
		err := d.publish(j)
		if err != nil {
			fmt.Println(err)
		}
	}

	for {
		debug("")

		job, err := d.front()
		if err != nil {
			if strings.Contains(err.Error(), "no rows in result set") {
				debug("No waiting jobs in the queue")
				debug("Waiting wakeup signal")
				job = <-d.newJobs
				debug("Got wakeup signal")
			} else {
				fmt.Println(err)
				continue
			}
		}

	CheckNextRun:
		remaining := job.Remaining()
		debug("Next job:", job, "Remaining:", remaining)

		now := time.Now().UTC()
		if job.NextRun.After(now) {
			// Wait until the next Job time or
			// the webserver's /schedule handler wakes us up
			debug("Sleeping for job:", remaining)
			select {
			case <-time.After(remaining):
				debug("Job sleep time finished")
				publish(job)
			case newJob := <-d.newJobs:
				debug("A new job has been scheduled")
				if newJob.NextRun.Before(job.NextRun) {
					debug("The new job comes before out current job")
					job = newJob // Process the new job next
				}
				// Continue processing the current job without fetching from database
				goto CheckNextRun
			case canceledJob := <-d.canceledJobs:
				debug("A job has been cancelled")
				if (job.RoutingKey == canceledJob.RoutingKey) && (job.Body == canceledJob.Body) {
					// The job we are waiting for has been canceled.
					// We need to fetch the next job in the queue.
					debug("The cancelled job is our current job")
					continue
				}
				// Continue to process our current job
				goto CheckNextRun
			}
		} else {
			publish(job)
		}

	}
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/gcfg"
	"database/sql"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"github.com/streadway/amqp"
	"log"
	"net/http"
	"strconv"
	"time"
)

var (
	cfg struct {
		DB struct {
			Driver string
			Dsn    string
			Table  string
		}
		RabbitMQ struct {
			Uri      string
			Exchange string
		}
		HTTP struct {
			Host string
			Port string
		}
	}

	db     *sql.DB
	broker *amqp.Connection
)

func handleSchedule(w http.ResponseWriter, r *http.Request) {
	routingKey, body, interval_s := r.FormValue("routing_key"), r.FormValue("body"), r.FormValue("interval")
	log.Println("/schedule", routingKey, body)
	interval, err := strconv.ParseInt(interval_s, 10, 64)
	if err != nil {
		panic(err)
	}

	next_run := time.Now().Add(time.Duration(interval) * time.Second)
	_, err = db.Exec("INSERT IGNORE INTO "+cfg.DB.Table+" "+
		"(routing_key, body, `interval`, next_run, state) "+
		"VALUES(?, ?, ?, ?, 'WAITING')",
		routingKey, body, interval, next_run)
	if err != nil {
		panic(err)
	}
}

func handleCancel(w http.ResponseWriter, r *http.Request) {
	fmt.Println("/cancel")
}

func main() {
	// Read config
	err := gcfg.ReadFileInto(&cfg, "dalga.ini")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("Read config: ", cfg)

	// Connect to database
	db, err = sql.Open(cfg.DB.Driver, cfg.DB.Dsn)
	if err != nil {
		log.Fatal(err)
	}
	err = db.Ping()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("Connected to DB")

	// Connect to RabbitMQ
	_, err = amqp.Dial(cfg.RabbitMQ.Uri)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("Connected to RabbitMQ")

	// Start HTTP server
	addr := cfg.HTTP.Host + ":" + cfg.HTTP.Port
	http.HandleFunc("/schedule", handleSchedule)
	http.HandleFunc("/cancel", handleSchedule)
	http.ListenAndServe(addr, nil)
}
<commit_msg>cancel<commit_after>package main

import (
	"code.google.com/p/gcfg"
	"database/sql"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"github.com/streadway/amqp"
	"log"
	"net/http"
	"strconv"
	"time"
)

var (
	cfg struct {
		DB struct {
			Driver string
			Dsn    string
			Table  string
		}
		RabbitMQ struct {
			Uri      string
			Exchange string
		}
		HTTP struct {
			Host string
			Port string
		}
	}

	db     *sql.DB
	broker *amqp.Connection
)

func handleSchedule(w http.ResponseWriter, r *http.Request) {
	routingKey, body, interval_s := r.FormValue("routing_key"), r.FormValue("body"), r.FormValue("interval")
	log.Println("/schedule", routingKey, body)

	interval, err := strconv.ParseInt(interval_s, 10, 64)
	if err != nil {
		panic(err)
	}

	next_run := time.Now().Add(time.Duration(interval) * time.Second)
	_, err = db.Exec("INSERT IGNORE INTO "+cfg.DB.Table+" "+
		"(routing_key, body, `interval`, next_run, state) "+
		"VALUES(?, ?, ?, ?, 'WAITING')",
		routingKey, body, interval, next_run)
	if err != nil {
		panic(err)
	}
}

func handleCancel(w http.ResponseWriter, r *http.Request) {
	routingKey, body := r.FormValue("routing_key"), r.FormValue("body")
	fmt.Println("/cancel", routingKey, body)

	_, err := db.Exec("DELETE FROM "+cfg.DB.Table+" "+
		"WHERE routing_key=? AND body=?", routingKey, body)
	if err != nil {
		panic(err)
	}
}

func main() {
	// Read config
	err := gcfg.ReadFileInto(&cfg, "dalga.ini")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("Read config: ", cfg)

	// Connect to database
	db, err = sql.Open(cfg.DB.Driver, cfg.DB.Dsn)
	if err != nil {
		log.Fatal(err)
	}
	err = db.Ping()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("Connected to DB")

	// Connect to RabbitMQ
	_, err = amqp.Dial(cfg.RabbitMQ.Uri)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("Connected to RabbitMQ")

	// Start HTTP server
	addr := cfg.HTTP.Host + ":" + cfg.HTTP.Port
	http.HandleFunc("/schedule", handleSchedule)
	http.HandleFunc("/cancel", handleCancel)
	http.ListenAndServe(addr, nil)
}
<|endoftext|>


<commit_before>package finalize

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/cloudfoundry/libbuildpack"
	"github.com/kr/text"
)

type Stager interface {
	BuildDir() string
	DepsIdx() string
	DepDir() string
}

type Finalizer struct {
	Stager           Stager
	Versions         Versions
	Log              *libbuildpack.Logger
	Gem12Factor      bool
	GemStaticAssets  bool
	GemStdoutLogging bool
	RailsVersion     int
}

func Run(f *Finalizer) error {
	f.Log.BeginStep("Finalizing Ruby")

	if err := f.Setup(); err != nil {
		f.Log.Error("Error determining versions: %v", err)
		return err
	}

	if err := f.RestoreGemfileLock(); err != nil {
		f.Log.Error("Error copying Gemfile.lock to app: %v", err)
		return err
	}

	if err := f.InstallPlugins(); err != nil {
		f.Log.Error("Error installing plugins: %v", err)
		return err
	}

	if err := f.WriteDatabaseYml(); err != nil {
		f.Log.Error("Error writing database.yml: %v", err)
		return err
	}

	if err := f.PrecompileAssets(); err != nil {
		f.Log.Error("Error precompiling assets: %v", err)
		return err
	}

	f.BestPracticeWarnings()

	if err := f.DeleteVendorBundle(); err != nil {
		f.Log.Error("Error deleting vendor/bundle: %v", err)
		return err
	}

	if err := f.CopyToAppBin(); err != nil {
		f.Log.Error("Error creating files in bin: %v", err)
		return err
	}

	data, err := f.GenerateReleaseYaml()
	if err != nil {
		f.Log.Error("Error generating release YAML: %v", err)
		return err
	}
	releasePath := filepath.Join(f.Stager.BuildDir(), "tmp", "ruby-buildpack-release-step.yml")
	libbuildpack.NewYAML().Write(releasePath, data)

	return nil
}

func (f *Finalizer) Setup() error {
	var err error

	f.Gem12Factor, err = f.Versions.HasGem("rails_12factor")
	if err != nil {
		return err
	}

	f.GemStdoutLogging, err = f.Versions.HasGem("rails_stdout_logging")
	if err != nil {
		return err
	}

	f.GemStaticAssets, err = f.Versions.HasGem("rails_serve_static_assets")
	if err != nil {
		return err
	}

	f.RailsVersion, err = f.Versions.GemMajorVersion("rails")
	if err != nil {
		return err
	}

	return nil
}

func (f *Finalizer) RestoreGemfileLock() error {
	source := filepath.Join(f.Stager.DepDir(), "Gemfile.lock")
	f.Log.Debug("RestoreGemfileLock; %s", source)
	if exists, err := libbuildpack.FileExists(source); err != nil {
		return err
	} else if exists {
		gemfile := "Gemfile"
		if os.Getenv("BUNDLE_GEMFILE") != "" {
			gemfile = os.Getenv("BUNDLE_GEMFILE")
		}
		target := filepath.Join(f.Stager.BuildDir(), gemfile) + ".lock"
		f.Log.Debug("RestoreGemfileLock; exists, copy to %s", target)
		return os.Rename(source, target)
	}
	return nil
}

func (f *Finalizer) WriteDatabaseYml() error {
	if exists, err := libbuildpack.FileExists(filepath.Join(f.Stager.BuildDir(), "config")); err != nil {
		return err
	} else if !exists {
		return nil
	}
	if rails41Plus, err := f.Versions.HasGemVersion("activerecord", ">=4.1.0.beta"); err != nil {
		return err
	} else if rails41Plus {
		return nil
	}

	f.Log.BeginStep("Writing config/database.yml to read from DATABASE_URL")
	if err := ioutil.WriteFile(filepath.Join(f.Stager.BuildDir(), "config", "database.yml"), []byte(config_database_yml), 0644); err != nil {
		return err
	}

	return nil
}

func (f *Finalizer) PrecompileAssets() error {
	cmd := exec.Command("bundle", "exec", "rake", "-n", "assets:precompile")
	cmd.Dir = f.Stager.BuildDir()
	if err := cmd.Run(); err != nil {
		return nil
	}

	f.Log.BeginStep("Precompiling assets")
	startTime := time.Now()
	cmd = exec.Command("bundle", "exec", "rake", "assets:precompile")
	cmd.Dir = f.Stager.BuildDir()
	cmd.Stdout = text.NewIndentWriter(os.Stdout, []byte("       "))
	cmd.Stderr = text.NewIndentWriter(os.Stderr, []byte("       "))
	err := cmd.Run()

	f.Log.Info("Asset precompilation completed (%v)", time.Since(startTime))

	if f.RailsVersion >= 4 && err == nil {
		f.Log.Info("Cleaning assets")
		cmd = exec.Command("bundle", "exec", "rake", "assets:clean")
		cmd.Dir = f.Stager.BuildDir()
		cmd.Stdout = text.NewIndentWriter(os.Stdout, []byte("       "))
		cmd.Stderr = text.NewIndentWriter(os.Stderr, []byte("       "))
		err = cmd.Run()
	}

	return err
}

func (f *Finalizer) InstallPlugins() error {
	if f.Gem12Factor {
		return nil
	}

	if f.RailsVersion == 4 {
		if !(f.GemStdoutLogging && f.GemStaticAssets) {
			f.Log.Protip("Include 'rails_12factor' gem to enable all platform features", "https://devcenter.heroku.com/articles/rails-integration-gems")
		}
		return nil
	}

	if f.RailsVersion == 2 || f.RailsVersion == 3 {
		if err := f.installPluginStdoutLogger(); err != nil {
			return err
		}
		if err := f.installPluginServeStaticAssets(); err != nil {
			return err
		}
	}
	return nil
}

func (f *Finalizer) installPluginStdoutLogger() error {
	if f.GemStdoutLogging {
		return nil
	}

	f.Log.BeginStep("Injecting plugin 'rails_log_stdout'")

	code := `
begin
  STDOUT.sync = true
  def Rails.cloudfoundry_stdout_logger
    logger = Logger.new(STDOUT)
    logger = ActiveSupport::TaggedLogging.new(logger) if defined?(ActiveSupport::TaggedLogging)
    level = ENV['LOG_LEVEL'].to_s.upcase
    level = 'INFO' unless %w[DEBUG INFO WARN ERROR FATAL UNKNOWN].include?(level)
    logger.level = Logger.const_get(level)
    logger
  end
  Rails.logger = Rails.application.config.logger = Rails.cloudfoundry_stdout_logger
rescue Exception => ex
  puts %Q{WARNING: Exception during rails_log_stdout init: #{ex.message}}
end
`

	if err := os.MkdirAll(filepath.Join(f.Stager.BuildDir(), "vendor", "plugins", "rails_log_stdout"), 0755); err != nil {
		return fmt.Errorf("Error creating rails_log_stdout plugin directory: %v", err)
	}
	if err := ioutil.WriteFile(filepath.Join(f.Stager.BuildDir(), "vendor", "plugins", "rails_log_stdout", "init.rb"), []byte(code), 0644); err != nil {
		return fmt.Errorf("Error writing rails_log_stdout plugin file: %v", err)
	}
	return nil
}

func (f *Finalizer) installPluginServeStaticAssets() error {
	if f.GemStaticAssets {
		return nil
	}

	f.Log.BeginStep("Injecting plugin 'rails3_serve_static_assets'")

	code := "Rails.application.class.config.serve_static_assets = true\n"

	if err := os.MkdirAll(filepath.Join(f.Stager.BuildDir(), "vendor", "plugins", "rails3_serve_static_assets"), 0755); err != nil {
		return fmt.Errorf("Error creating rails3_serve_static_assets plugin directory: %v", err)
	}
	if err := ioutil.WriteFile(filepath.Join(f.Stager.BuildDir(), "vendor", "plugins", "rails3_serve_static_assets", "init.rb"), []byte(code), 0644); err != nil {
		return fmt.Errorf("Error writing rails3_serve_static_assets plugin file: %v", err)
	}
	return nil
}

func (f *Finalizer) BestPracticeWarnings() {
	if os.Getenv("RAILS_ENV") != "production" {
		f.Log.Warning("You are deploying to a non-production environment: %s", os.Getenv("RAILS_ENV"))
	}
}

func (f *Finalizer) DeleteVendorBundle() error {
	if exists, err := libbuildpack.FileExists(filepath.Join(f.Stager.BuildDir(), "vendor", "bundle")); err != nil {
		return err
	} else if exists {
		f.Log.Warning("Removing `vendor/bundle`.\nChecking in `vendor/bundle` is not supported. Please remove this directory and add it to your .gitignore. To vendor your gems with Bundler, use `bundle pack` instead.")
		return os.RemoveAll(filepath.Join(f.Stager.BuildDir(), "vendor", "bundle"))
	}

	return nil
}

func (f *Finalizer) CopyToAppBin() error {
	f.Log.BeginStep("Copy binaries to app/bin directory")

	binDir := filepath.Join(f.Stager.BuildDir(), "bin")
	if err := os.MkdirAll(binDir, 0755); err != nil {
		return fmt.Errorf("Could not create /app/bin directory: %v", err)
	}

	files, err := ioutil.ReadDir(filepath.Join(f.Stager.DepDir(), "binstubs"))
	if err != nil {
		if !os.IsNotExist(err) {
			return fmt.Errorf("Could not read dep/binstubs directory: %v", err)
		}
		files = []os.FileInfo{}
	}
	for _, file := range files {
		source := filepath.Join(f.Stager.DepDir(), "binstubs", file.Name())
		target := filepath.Join(binDir, file.Name())
		if exists, err := libbuildpack.FileExists(target); err != nil {
			return fmt.Errorf("Checking existence: %v", err)
		} else if !exists {
			if err := libbuildpack.CopyFile(source, target); err != nil {
				return fmt.Errorf("CopyFile: %v", err)
			}
		}
	}

	files, err = ioutil.ReadDir(filepath.Join(f.Stager.DepDir(), "bin"))
	if err != nil {
		if !os.IsNotExist(err) {
			return fmt.Errorf("Could not read dep/bin directory: %v", err)
		}
		files = []os.FileInfo{}
	}
	for _, file := range files {
		target := filepath.Join(binDir, file.Name())
		if exists, err := libbuildpack.FileExists(target); err != nil {
			return fmt.Errorf("Checking existence: %v", err)
		} else if !exists {
			contents := fmt.Sprintf("#!/bin/bash\nexec $DEPS_DIR/%s/bin/%s \"$@\"\n", f.Stager.DepsIdx(), file.Name())
			if err := ioutil.WriteFile(target, []byte(contents), 0755); err != nil {
				return fmt.Errorf("WriteFile: %v", err)
			}
		}
	}
	return nil
}
<commit_msg>Limit environment for assets:precompile [#149435225]<commit_after>package finalize

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/cloudfoundry/libbuildpack"
	"github.com/kr/text"
)

type Stager interface {
	BuildDir() string
	DepsIdx() string
	DepDir() string
}

type Finalizer struct {
	Stager           Stager
	Versions         Versions
	Log              *libbuildpack.Logger
	Gem12Factor      bool
	GemStaticAssets  bool
	GemStdoutLogging bool
	RailsVersion     int
}

func Run(f *Finalizer) error {
	f.Log.BeginStep("Finalizing Ruby")

	if err := f.Setup(); err != nil {
		f.Log.Error("Error determining versions: %v", err)
		return err
	}

	if err := f.RestoreGemfileLock(); err != nil {
		f.Log.Error("Error copying Gemfile.lock to app: %v", err)
		return err
	}

	if err := f.InstallPlugins(); err != nil {
		f.Log.Error("Error installing plugins: %v", err)
		return err
	}

	if err := f.WriteDatabaseYml(); err != nil {
		f.Log.Error("Error writing database.yml: %v", err)
		return err
	}

	if err := f.PrecompileAssets(); err != nil {
		f.Log.Error("Error precompiling assets: %v", err)
		return err
	}

	f.BestPracticeWarnings()

	if err := f.DeleteVendorBundle(); err != nil {
		f.Log.Error("Error deleting vendor/bundle: %v", err)
		return err
	}

	if err := f.CopyToAppBin(); err != nil {
		f.Log.Error("Error creating files in bin: %v", err)
		return err
	}

	data, err := f.GenerateReleaseYaml()
	if err != nil {
		f.Log.Error("Error generating release YAML: %v", err)
		return err
	}
	releasePath := filepath.Join(f.Stager.BuildDir(), "tmp", "ruby-buildpack-release-step.yml")
	libbuildpack.NewYAML().Write(releasePath, data)

	return nil
}

func (f *Finalizer) Setup() error {
	var err error

	f.Gem12Factor, err = f.Versions.HasGem("rails_12factor")
	if err != nil {
		return err
	}

	f.GemStdoutLogging, err = f.Versions.HasGem("rails_stdout_logging")
	if err != nil {
		return err
	}

	f.GemStaticAssets, err = f.Versions.HasGem("rails_serve_static_assets")
	if err != nil {
		return err
	}

	f.RailsVersion, err = f.Versions.GemMajorVersion("rails")
	if err != nil {
		return err
	}

	return nil
}

func (f *Finalizer) RestoreGemfileLock() error {
	source := filepath.Join(f.Stager.DepDir(), "Gemfile.lock")
	f.Log.Debug("RestoreGemfileLock; %s", source)
	if exists, err := libbuildpack.FileExists(source); err != nil {
		return err
	} else if exists {
		gemfile := "Gemfile"
		if os.Getenv("BUNDLE_GEMFILE") != "" {
			gemfile = os.Getenv("BUNDLE_GEMFILE")
		}
		target := filepath.Join(f.Stager.BuildDir(), gemfile) + ".lock"
		f.Log.Debug("RestoreGemfileLock; exists, copy to %s", target)
		return os.Rename(source, target)
	}
	return nil
}

func (f *Finalizer) WriteDatabaseYml() error {
	if exists, err := libbuildpack.FileExists(filepath.Join(f.Stager.BuildDir(), "config")); err != nil {
		return err
	} else if !exists {
		return nil
	}
	if rails41Plus, err := f.Versions.HasGemVersion("activerecord", ">=4.1.0.beta"); err != nil {
		return err
	} else if rails41Plus {
		return nil
	}

	f.Log.BeginStep("Writing config/database.yml to read from DATABASE_URL")
	if err := ioutil.WriteFile(filepath.Join(f.Stager.BuildDir(), "config", "database.yml"), []byte(config_database_yml), 0644); err != nil {
		return err
	}

	return nil
}

func (f *Finalizer) PrecompileAssets() error {
	cmd := exec.Command("bundle", "exec", "rake", "-n", "assets:precompile")
	cmd.Dir = f.Stager.BuildDir()
	if err := cmd.Run(); err != nil {
		return nil
	}

	env := []string{
		fmt.Sprintf("BUNDLE_BIN=%s", os.Getenv("BUNDLE_BIN")),
		fmt.Sprintf("BUNDLE_CONFIG=%s", os.Getenv("BUNDLE_CONFIG")),
		fmt.Sprintf("BUNDLE_PATH=%s", os.Getenv("BUNDLE_PATH")),
		fmt.Sprintf("BUNDLE_WITHOUT=%s", os.Getenv("BUNDLE_WITHOUT")),
		fmt.Sprintf("GEM_HOME=%s", os.Getenv("GEM_HOME")),
		fmt.Sprintf("GEM_PATH=%s", os.Getenv("GEM_PATH")),
		fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
	}

	f.Log.BeginStep("Precompiling assets")
	startTime := time.Now()
	cmd = exec.Command("bundle", "exec", "rake", "assets:precompile")
	cmd.Dir = f.Stager.BuildDir()
	cmd.Stdout = text.NewIndentWriter(os.Stdout, []byte("       "))
	cmd.Stderr = text.NewIndentWriter(os.Stderr, []byte("       "))
	cmd.Env = env
	err := cmd.Run()

	f.Log.Info("Asset precompilation completed (%v)", time.Since(startTime))

	if f.RailsVersion >= 4 && err == nil {
		f.Log.Info("Cleaning assets")
		cmd = exec.Command("bundle", "exec", "rake", "assets:clean")
		cmd.Dir = f.Stager.BuildDir()
		cmd.Stdout = text.NewIndentWriter(os.Stdout, []byte("       "))
		cmd.Stderr = text.NewIndentWriter(os.Stderr, []byte("       "))
		cmd.Env = env
		err = cmd.Run()
	}

	return err
}

func (f *Finalizer) InstallPlugins() error {
	if f.Gem12Factor {
		return nil
	}

	if f.RailsVersion == 4 {
		if !(f.GemStdoutLogging && f.GemStaticAssets) {
			f.Log.Protip("Include 'rails_12factor' gem to enable all platform features", "https://devcenter.heroku.com/articles/rails-integration-gems")
		}
		return nil
	}

	if f.RailsVersion == 2 || f.RailsVersion == 3 {
		if err := f.installPluginStdoutLogger(); err != nil {
			return err
		}
		if err := f.installPluginServeStaticAssets(); err != nil {
			return err
		}
	}
	return nil
}

func (f *Finalizer) installPluginStdoutLogger() error {
	if f.GemStdoutLogging {
		return nil
	}

	f.Log.BeginStep("Injecting plugin 'rails_log_stdout'")

	code := `
begin
  STDOUT.sync = true
  def Rails.cloudfoundry_stdout_logger
    logger = Logger.new(STDOUT)
    logger = ActiveSupport::TaggedLogging.new(logger) if defined?(ActiveSupport::TaggedLogging)
    level = ENV['LOG_LEVEL'].to_s.upcase
    level = 'INFO' unless %w[DEBUG INFO WARN ERROR FATAL UNKNOWN].include?(level)
    logger.level = Logger.const_get(level)
    logger
  end
  Rails.logger = Rails.application.config.logger = Rails.cloudfoundry_stdout_logger
rescue Exception => ex
  puts %Q{WARNING: Exception during rails_log_stdout init: #{ex.message}}
end
`

	if err := os.MkdirAll(filepath.Join(f.Stager.BuildDir(), "vendor", "plugins", "rails_log_stdout"), 0755); err != nil {
		return fmt.Errorf("Error creating rails_log_stdout plugin directory: %v", err)
	}
	if err := ioutil.WriteFile(filepath.Join(f.Stager.BuildDir(), "vendor", "plugins", "rails_log_stdout", "init.rb"), []byte(code), 0644); err != nil {
		return fmt.Errorf("Error writing rails_log_stdout plugin file: %v", err)
	}
	return nil
}

func (f *Finalizer) installPluginServeStaticAssets() error {
	if f.GemStaticAssets {
		return nil
	}

	f.Log.BeginStep("Injecting plugin 'rails3_serve_static_assets'")

	code := "Rails.application.class.config.serve_static_assets = true\n"

	if err := os.MkdirAll(filepath.Join(f.Stager.BuildDir(), "vendor", "plugins", "rails3_serve_static_assets"), 0755); err != nil {
		return fmt.Errorf("Error creating rails3_serve_static_assets plugin directory: %v", err)
	}
	if err := ioutil.WriteFile(filepath.Join(f.Stager.BuildDir(), "vendor", "plugins", "rails3_serve_static_assets", "init.rb"), []byte(code), 0644); err != nil {
		return fmt.Errorf("Error writing rails3_serve_static_assets plugin file: %v", err)
	}
	return nil
}

func (f *Finalizer) BestPracticeWarnings() {
	if os.Getenv("RAILS_ENV") != "production" {
		f.Log.Warning("You are deploying to a non-production environment: %s", os.Getenv("RAILS_ENV"))
	}
}

func (f *Finalizer) DeleteVendorBundle() error {
	if exists, err := libbuildpack.FileExists(filepath.Join(f.Stager.BuildDir(), "vendor", "bundle")); err != nil {
		return err
	} else if exists {
		f.Log.Warning("Removing `vendor/bundle`.\nChecking in `vendor/bundle` is not supported. Please remove this directory and add it to your .gitignore. To vendor your gems with Bundler, use `bundle pack` instead.")
		return os.RemoveAll(filepath.Join(f.Stager.BuildDir(), "vendor", "bundle"))
	}

	return nil
}

func (f *Finalizer) CopyToAppBin() error {
	f.Log.BeginStep("Copy binaries to app/bin directory")

	binDir := filepath.Join(f.Stager.BuildDir(), "bin")
	if err := os.MkdirAll(binDir, 0755); err != nil {
		return fmt.Errorf("Could not create /app/bin directory: %v", err)
	}

	files, err := ioutil.ReadDir(filepath.Join(f.Stager.DepDir(), "binstubs"))
	if err != nil {
		if !os.IsNotExist(err) {
			return fmt.Errorf("Could not read dep/binstubs directory: %v", err)
		}
		files = []os.FileInfo{}
	}
	for _, file := range files {
		source := filepath.Join(f.Stager.DepDir(), "binstubs", file.Name())
		target := filepath.Join(binDir, file.Name())
		if exists, err := libbuildpack.FileExists(target); err != nil {
			return fmt.Errorf("Checking existence: %v", err)
		} else if !exists {
			if err := libbuildpack.CopyFile(source, target); err != nil {
				return fmt.Errorf("CopyFile: %v", err)
			}
		}
	}

	files, err = ioutil.ReadDir(filepath.Join(f.Stager.DepDir(), "bin"))
	if err != nil {
		if !os.IsNotExist(err) {
			return fmt.Errorf("Could not read dep/bin directory: %v", err)
		}
		files = []os.FileInfo{}
	}
	for _, file := range files {
		target := filepath.Join(binDir, file.Name())
		if exists, err := libbuildpack.FileExists(target); err != nil {
			return fmt.Errorf("Checking existence: %v", err)
		} else if !exists {
			contents := fmt.Sprintf("#!/bin/bash\nexec $DEPS_DIR/%s/bin/%s \"$@\"\n", f.Stager.DepsIdx(), file.Name())
			if err := ioutil.WriteFile(target, []byte(contents), 0755); err != nil {
				return fmt.Errorf("WriteFile: %v", err)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package components

import (
	"sync"

	"github.com/TheThingsNetwork/ttn/core"
	"github.com/brocaar/lorawan"
)

type handlerStorage interface {
	store(lorawan.DevAddr, handlerEntry) error
	partition([]core.Packet) ([]handlerPartition, error)
}

type handlerPartition struct {
	handlerEntry
	Packets []core.Packet
}

type handlerEntry struct {
	AppEUI  lorawan.EUI64
	NwkSKey lorawan.AES128Key
	AppSKey lorawan.AES128Key
	DevAddr lorawan.DevAddr
}

type handlerDB struct {
	sync.RWMutex // Guards entries
	entries      map[lorawan.DevAddr][]handlerEntry
}

// newHandlerDB construct a new local handlerStorage
func newHandlerDB() handlerStorage {
	return &handlerDB{entries: make(map[lorawan.DevAddr][]handlerEntry)}
}

// store implements the handlerStorage interface
func (db *handlerDB) store(devAddr lorawan.DevAddr, entry handlerEntry) error {
	db.Lock()
	db.entries[devAddr] = append(db.entries[devAddr], entry)
	db.Unlock()
	return nil
}

// partition implements the handlerStorage interface
func (db *handlerDB) partition(packets []core.Packet) ([]handlerPartition, error) {
	// Create a map in order to do the partition
	partitions := make(map[lorawan.EUI64]handlerPartition)

	db.RLock() // We require lock on the whole block because we don't want the entries to change while building the partition.
	for _, packet := range packets {
		// First, determine devAddr and get the macPayload. Those are mandatory.
		devAddr, err := packet.DevAddr()
		if err != nil {
			return nil, ErrInvalidPacket
		}
		macPayload, ok := packet.Payload.MACPayload.(*lorawan.MACPayload)
		if !ok {
			return nil, ErrInvalidPacket
		}

		// Now, get all tuples associated to that device address, and choose the right one
		for _, entry := range db.entries[devAddr] {
			// Try to decrypt the frame payload with those keys
			key := entry.AppSKey
			if macPayload.FPort == 0 {
				key = entry.NwkSKey
			}
			err := macPayload.DecryptFRMPayload(key)
			if err != nil { // Weren't the good keys
				continue
			}
			// #Easy
			packet.Payload.MACPayload = macPayload
			partitions[entry.AppEUI] = handlerPartition{
				handlerEntry: entry,
				Packets:      append(partitions[entry.AppEUI].Packets, packet),
			}
			break // We don't need to look for other entries, we've found the right one
		}
	}
	db.RUnlock()

	// Transform the map in a slice
	res := make([]handlerPartition, 0, len(partitions))
	for _, p := range partitions {
		res = append(res, p)
	}

	return res, nil
}
<commit_msg>[handler] Make tests pass.<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package components

import (
	"sync"

	"github.com/TheThingsNetwork/ttn/core"
	"github.com/brocaar/lorawan"
)

type handlerStorage interface {
	store(lorawan.DevAddr, handlerEntry) error
	partition([]core.Packet) ([]handlerPartition, error)
}

type handlerPartition struct {
	handlerEntry
	Packets []core.Packet
}

type handlerEntry struct {
	AppEUI  lorawan.EUI64
	NwkSKey lorawan.AES128Key
	AppSKey lorawan.AES128Key
	DevAddr lorawan.DevAddr
}

type handlerDB struct {
	sync.RWMutex // Guards entries
	entries      map[lorawan.DevAddr][]handlerEntry
}

// newHandlerDB construct a new local handlerStorage
func newHandlerDB() handlerStorage {
	return &handlerDB{entries: make(map[lorawan.DevAddr][]handlerEntry)}
}

// store implements the handlerStorage interface
func (db *handlerDB) store(devAddr lorawan.DevAddr, entry handlerEntry) error {
	db.Lock()
	db.entries[devAddr] = append(db.entries[devAddr], entry)
	db.Unlock()
	return nil
}

// partition implements the handlerStorage interface
func (db *handlerDB) partition(packets []core.Packet) ([]handlerPartition, error) {
	// Create a map in order to do the partition
	partitions := make(map[[20]byte]handlerPartition)

	db.RLock() // We require lock on the whole block because we don't want the entries to change while building the partition.
	for _, packet := range packets {
		// First, determine devAddr and get the macPayload. Those are mandatory.
		devAddr, err := packet.DevAddr()
		if err != nil {
			return nil, ErrInvalidPacket
		}

		// Now, get all tuples associated to that device address, and choose the right one
		for _, entry := range db.entries[devAddr] {
			// Compute MIC check to find the right keys
			ok, err := packet.Payload.ValidateMIC(entry.NwkSKey)
			if err != nil || !ok {
				continue // These aren't the droid you're looking for
			}

			// #Easy
			var id [20]byte
			copy(id[:16], entry.AppEUI[:])
			copy(id[16:], entry.DevAddr[:])
			partitions[id] = handlerPartition{
				handlerEntry: entry,
				Packets:      append(partitions[id].Packets, packet),
			}
			break // We shouldn't look for other entries, we've found the right one
		}
	}
	db.RUnlock()

	// Transform the map in a slice
	res := make([]handlerPartition, 0, len(partitions))
	for _, p := range partitions {
		res = append(res, p)
	}

	if len(res) == 0 {
		return nil, ErrNotFound
	}
	return res, nil
}
<|endoftext|>


<commit_before>package cc_messages

import "encoding/json"

type DesireAppRequestFromCC struct {
	ProcessGuid     string      `json:"process_guid"`
	DropletUri      string      `json:"droplet_uri"`
	DockerImageUrl  string      `json:"docker_image"`
	Stack           string      `json:"stack"`
	StartCommand    string      `json:"start_command"`
	Environment     Environment `json:"environment"`
	MemoryMB        int         `json:"memory_mb"`
	DiskMB          int         `json:"disk_mb"`
	FileDescriptors uint64      `json:"file_descriptors"`
	NumInstances    int         `json:"num_instances"`
	Routes          []string    `json:"routes"`
	LogGuid         string      `json:"log_guid"`
}

func (d DesireAppRequestFromCC) ToJSON() []byte {
	encoded, _ := json.Marshal(d)
	return encoded
}

type CCDesiredStateServerResponse struct {
	Apps        []DesireAppRequestFromCC `json:"apps"`
	CCBulkToken *json.RawMessage         `json:"token"`
}

type CCBulkToken struct {
	Id int `json:"id"`
}
<commit_msg>Add staging_metadata to DesireAppMessage<commit_after>package cc_messages

import "encoding/json"

type DesireAppRequestFromCC struct {
	ProcessGuid     string      `json:"process_guid"`
	DropletUri      string      `json:"droplet_uri"`
	DockerImageUrl  string      `json:"docker_image"`
	Stack           string      `json:"stack"`
	StartCommand    string      `json:"start_command"`
	StagingMetadata string      `json:"staging_metadata"`
	Environment     Environment `json:"environment"`
	MemoryMB        int         `json:"memory_mb"`
	DiskMB          int         `json:"disk_mb"`
	FileDescriptors uint64      `json:"file_descriptors"`
	NumInstances    int         `json:"num_instances"`
	Routes          []string    `json:"routes"`
	LogGuid         string      `json:"log_guid"`
}

func (d DesireAppRequestFromCC) ToJSON() []byte {
	encoded, _ := json.Marshal(d)
	return encoded
}

type CCDesiredStateServerResponse struct {
	Apps        []DesireAppRequestFromCC `json:"apps"`
	CCBulkToken *json.RawMessage         `json:"token"`
}

type CCBulkToken struct {
	Id int `json:"id"`
}
<|endoftext|>


<commit_before>package dsl_test
<commit_msg>a few more tests<commit_after>package dsl_test

import (
	"github.com/bketelsen/gorma"
	gdsl "github.com/bketelsen/gorma/dsl"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/raphael/goa/design"
	. "github.com/raphael/goa/design/dsl"
)

var _ = Describe("RelationalStore", func() {
	var sgname, name string
	var dsl func()

	BeforeEach(func() {
		Design = nil
		Errors = nil
		sgname = "production"
		dsl = nil
		name = ""
		gorma.GormaConstructs = nil

	})

	JustBeforeEach(func() {

		gdsl.StorageGroup(sgname, func() {
			gdsl.RelationalStore(name, gorma.MySQL, dsl)
		})

		RunDSL()

	})

	Context("with no DSL", func() {
		BeforeEach(func() {
			name = "mysql"
		})

		It("produces a valid Relational Store definition", func() {
			Ω(Design.Validate()).ShouldNot(HaveOccurred())
			sg := gorma.GormaConstructs[gorma.StorageGroup].(*gorma.StorageGroupDefinition)
			Ω(sg.RelationalStores[name].Name).Should(Equal(name))
		})
	})

	Context("with an already defined Relational Store with the same name", func() {
		BeforeEach(func() {
			name = "mysql"
		})

		It("produces an error", func() {
			gdsl.StorageGroup(sgname, func() {
				gdsl.RelationalStore(name, gorma.MySQL, dsl)
			})
			Ω(Errors).Should(HaveOccurred())
		})
	})

	Context("with an already defined Relational Store with a different name", func() {
		BeforeEach(func() {
			sgname = "mysql"
		})

		It("returns an error", func() {
			gdsl.StorageGroup("news", dsl)
			Ω(Errors).Should(HaveOccurred())
		})
	})

	Context("with valid DSL", func() {
		JustBeforeEach(func() {
			Ω(Errors).ShouldNot(HaveOccurred())
			Ω(Design.Validate()).ShouldNot(HaveOccurred())
		})

		Context("with a description", func() {
			const description = "description"

			BeforeEach(func() {
				name = "mysql"
				dsl = func() {
					gdsl.Description(description)
				}
			})

			It("sets the relational store description", func() {
				sg := gorma.GormaConstructs[gorma.StorageGroup].(*gorma.StorageGroupDefinition)
				Ω(sg.RelationalStores[name].Description).Should(Equal(description))
			})
		})

	})
})
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"

	goselenium "github.com/bunsenapp/go-selenium"
)

func main() {
	// Create capabilities, driver etc.
	capabilities := goselenium.Capabilities{}
	capabilities.SetBrowser(goselenium.FirefoxBrowser())

	driver, err := goselenium.NewSeleniumWebDriver("http://localhost:4444/wd/hub", capabilities)
	if err != nil {
		fmt.Println(err)
		return
	}

	_, err = driver.CreateSession()
	if err != nil {
		fmt.Println(err)
		return
	}

	// Delete the session once this function is completed.
	defer driver.DeleteSession()

	// Navigate to the HackerNews website.
	_, err = driver.Go("https://news.ycombinator.com")
	if err != nil {
		fmt.Println(err)
		return
	}

	// Click the 'new' link at the top
	el, err := driver.FindElement(goselenium.ByCSSSelector("a[href='newest']"))
	if err != nil {
		fmt.Println(err)
		return
	}

	// Click the link.
	_, err = el.Click()
	if err != nil {
		fmt.Println(err)
		return
	}

	// Wait until the URL has changed with a timeout of 1 second and a check
	// interval of 10ms..
	newLink := "https://news.ycombinator.com/newest"
	ok := driver.Wait(goselenium.UntilURLIs(newLink), 10*time.Second, 10*time.Millisecond)
	if !ok {
		fmt.Println("Wait timed out :<")
		return
	}

	// Woohoo! We have successfully navigated to a page.
	fmt.Println("Successfully navigated to URL " + newLink)
}
<commit_msg>Change time back to 1 second<commit_after>package main

import (
	"fmt"
	"time"

	goselenium "github.com/bunsenapp/go-selenium"
)

func main() {
	// Create capabilities, driver etc.
	capabilities := goselenium.Capabilities{}
	capabilities.SetBrowser(goselenium.FirefoxBrowser())

	driver, err := goselenium.NewSeleniumWebDriver("http://localhost:4444/wd/hub", capabilities)
	if err != nil {
		fmt.Println(err)
		return
	}

	_, err = driver.CreateSession()
	if err != nil {
		fmt.Println(err)
		return
	}

	// Delete the session once this function is completed.
	defer driver.DeleteSession()

	// Navigate to the HackerNews website.
	_, err = driver.Go("https://news.ycombinator.com")
	if err != nil {
		fmt.Println(err)
		return
	}

	// Click the 'new' link at the top
	el, err := driver.FindElement(goselenium.ByCSSSelector("a[href='newest']"))
	if err != nil {
		fmt.Println(err)
		return
	}

	// Click the link.
	_, err = el.Click()
	if err != nil {
		fmt.Println(err)
		return
	}

	// Wait until the URL has changed with a timeout of 1 second and a check
	// interval of 10ms..
	newLink := "https://news.ycombinator.com/newest"
	ok := driver.Wait(goselenium.UntilURLIs(newLink), 1*time.Second, 10*time.Millisecond)
	if !ok {
		fmt.Println("Wait timed out :<")
		return
	}

	// Woohoo! We have successfully navigated to a page.
	fmt.Println("Successfully navigated to URL " + newLink)
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"github.com/jacobsa/aws/exp/sdb"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"math/rand"
	"sync"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type integrationTest struct {
	db sdb.SimpleDB
}

func (t *integrationTest) SetUp(i *TestInfo) {
	var err error

	// Open a connection.
	t.db, err = sdb.NewSimpleDB(g_region, g_accessKey)
	AssertEq(nil, err)
}

// Generate an item name likely to be unique.
func (t *integrationTest) makeItemName() sdb.ItemName {
	return sdb.ItemName(fmt.Sprintf("item.%16x", uint64(rand.Int63())))
}

////////////////////////////////////////////////////////////////////////
// Domains
////////////////////////////////////////////////////////////////////////

var g_domainsTestDb sdb.SimpleDB
var g_domainsTestDomain0 sdb.Domain
var g_domainsTestDomain1 sdb.Domain

type DomainsTest struct {
	integrationTest

	mutex           sync.Mutex
	domainsToDelete []sdb.Domain  // Protected by mutex
}

func init() { RegisterTestSuite(&DomainsTest{}) }

func (t *DomainsTest) SetUpTestSuite() {
	var err error

	// Open a connection.
	g_domainsTestDb, err = sdb.NewSimpleDB(g_region, g_accessKey)
	if err != nil {
		panic(err)
	}

	// Create domain 0.
	g_domainsTestDomain0, err = g_domainsTestDb.OpenDomain("DomainsTest.domain0")
	if err != nil {
		panic(err)
	}

	// Create domain 1.
	g_domainsTestDomain1, err = g_domainsTestDb.OpenDomain("DomainsTest.domain1")
	if err != nil {
		panic(err)
	}
}

func (t *DomainsTest) TearDownTestSuite() {
	// Delete both domains.
	if err := g_domainsTestDb.DeleteDomain(g_domainsTestDomain0); err != nil {
		panic(err)
	}

	if err := g_domainsTestDb.DeleteDomain(g_domainsTestDomain1); err != nil {
		panic(err)
	}

	// Clear variables.
	g_domainsTestDb = nil
	g_domainsTestDomain0 = nil
	g_domainsTestDomain1 = nil
}

func (t *DomainsTest) TearDown() {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	// Delete each of the domains created during the test.
	for _, d := range t.domainsToDelete {
		ExpectEq(nil, t.db.DeleteDomain(d), "Domain: %s", d.Name())
	}
}

func (t *DomainsTest) InvalidAccessKey() {
	// Open a connection with an unknown key ID.
	wrongKey := g_accessKey
	wrongKey.Id += "taco"

	db, err := sdb.NewSimpleDB(g_region, wrongKey)
	AssertEq(nil, err)

	// Attempt to create a domain.
	_, err = db.OpenDomain("some_domain")

	ExpectThat(err, Error(HasSubstr("403")))
	ExpectThat(err, Error(HasSubstr("Key Id")))
	ExpectThat(err, Error(HasSubstr("exist")))
}

func (t *DomainsTest) SeparatelyNamedDomainsHaveIndependentItems() {
	var err error

	// Set up an item in the first domain.
	itemName := t.makeItemName()
	err = g_domainsTestDomain0.PutAttributes(
		itemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "enchilada", Value: "queso"},
		},
		[]sdb.Precondition{},
	)

	AssertEq(nil, err)

	// Get attributes for the same name in the other domain. There should be
	// none.
	attrs, err := g_domainsTestDomain1.GetAttributes(itemName, true, []string{})
	AssertEq(nil, err)

	ExpectThat(attrs, ElementsAre())
}

func (t *DomainsTest) IdenticallyNamedDomainsHaveIdenticalItems() {
	var err error

	// Set up an item in the first domain.
	itemName := t.makeItemName()
	err = g_domainsTestDomain0.PutAttributes(
		itemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "enchilada", Value: "queso"},
		},
		[]sdb.Precondition{},
	)

	AssertEq(nil, err)

	// Get attributes for the same name in another domain object opened with the
	// same name.
	domain1, err := t.db.OpenDomain(g_domainsTestDomain0.Name())
	AssertEq(nil, err)

	attrs, err := domain1.GetAttributes(itemName, true, []string{})
	AssertEq(nil, err)

	ExpectThat(
		attrs,
		ElementsAre(
			DeepEquals(sdb.Attribute{Name: "enchilada", Value: "queso"}),
		),
	)
}

func (t *DomainsTest) Delete() {
	var err error
	domainName := "DomainsTest.Delete"

	// Create a domain, then delete it.
	domain, err := t.db.OpenDomain(domainName)
	AssertEq(nil, err)

	err = t.db.DeleteDomain(domain)
	AssertEq(nil, err)

	// Delete again; nothing should go wrong.
	err = t.db.DeleteDomain(domain)
	AssertEq(nil, err)

	// Attempt to write to the domain.
	err = domain.PutAttributes(
		"some_item",
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "foo", Value: "bar"},
		},
		[]sdb.Precondition{},
	)

	ExpectThat(err, Error(HasSubstr("NoSuchDomain")))
	ExpectThat(err, Error(HasSubstr("domain")))
	ExpectThat(err, Error(HasSubstr("exist")))
}

////////////////////////////////////////////////////////////////////////
// Items
////////////////////////////////////////////////////////////////////////

var g_itemsTestDb     sdb.SimpleDB
var g_itemsTestDomain sdb.Domain

type ItemsTest struct {
	integrationTest
}

func init() { RegisterTestSuite(&ItemsTest{}) }

func (t *ItemsTest) SetUpTestSuite() {
	var err error

	// Open a connection.
	g_itemsTestDb, err = sdb.NewSimpleDB(g_region, g_accessKey)
	if err != nil {
		panic(err)
	}

	// Create a domain.
	g_itemsTestDomain, err = g_itemsTestDb.OpenDomain("ItemsTest.domain")
	if err != nil {
		panic(err)
	}
}

func (t *ItemsTest) TearDownTestSuite() {
	// Delete the domain.
	if err := g_itemsTestDb.DeleteDomain(g_itemsTestDomain); err != nil {
		panic(err)
	}

	// Clear variables.
	g_itemsTestDb = nil
	g_itemsTestDomain = nil
}

func (t *ItemsTest) WrongAccessKeySecret() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidUtf8ItemName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidUtf8AttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidUtf8AttributeValue() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) LongItemName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) LongAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) LongAttributeValue() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) PutThenGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchPutThenGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchPutThenBatchGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetForNonExistentItem() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetForNonExistentItems() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetNonExistentAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetNonExistentAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedValuePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedExistencePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedNonExistencePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SuccessfulPreconditions() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) DeleteParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) DeleteAllAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchDelete() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidSelectQuery() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectAll() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectItemName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectCount() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithPredicates() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithSortOrder() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithLimit() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectEmptyResultSet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectLargeResultSet() {
	ExpectEq("TODO", "")
}
<commit_msg>Deleted a redundant test.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"github.com/jacobsa/aws/exp/sdb"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"math/rand"
	"sync"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type integrationTest struct {
	db sdb.SimpleDB
}

func (t *integrationTest) SetUp(i *TestInfo) {
	var err error

	// Open a connection.
	t.db, err = sdb.NewSimpleDB(g_region, g_accessKey)
	AssertEq(nil, err)
}

// Generate an item name likely to be unique.
func (t *integrationTest) makeItemName() sdb.ItemName {
	return sdb.ItemName(fmt.Sprintf("item.%16x", uint64(rand.Int63())))
}

////////////////////////////////////////////////////////////////////////
// Domains
////////////////////////////////////////////////////////////////////////

var g_domainsTestDb sdb.SimpleDB
var g_domainsTestDomain0 sdb.Domain
var g_domainsTestDomain1 sdb.Domain

type DomainsTest struct {
	integrationTest

	mutex           sync.Mutex
	domainsToDelete []sdb.Domain  // Protected by mutex
}

func init() { RegisterTestSuite(&DomainsTest{}) }

func (t *DomainsTest) SetUpTestSuite() {
	var err error

	// Open a connection.
	g_domainsTestDb, err = sdb.NewSimpleDB(g_region, g_accessKey)
	if err != nil {
		panic(err)
	}

	// Create domain 0.
	g_domainsTestDomain0, err = g_domainsTestDb.OpenDomain("DomainsTest.domain0")
	if err != nil {
		panic(err)
	}

	// Create domain 1.
	g_domainsTestDomain1, err = g_domainsTestDb.OpenDomain("DomainsTest.domain1")
	if err != nil {
		panic(err)
	}
}

func (t *DomainsTest) TearDownTestSuite() {
	// Delete both domains.
	if err := g_domainsTestDb.DeleteDomain(g_domainsTestDomain0); err != nil {
		panic(err)
	}

	if err := g_domainsTestDb.DeleteDomain(g_domainsTestDomain1); err != nil {
		panic(err)
	}

	// Clear variables.
	g_domainsTestDb = nil
	g_domainsTestDomain0 = nil
	g_domainsTestDomain1 = nil
}

func (t *DomainsTest) TearDown() {
	t.mutex.Lock()
	defer t.mutex.Unlock()

	// Delete each of the domains created during the test.
	for _, d := range t.domainsToDelete {
		ExpectEq(nil, t.db.DeleteDomain(d), "Domain: %s", d.Name())
	}
}

func (t *DomainsTest) InvalidAccessKey() {
	// Open a connection with an unknown key ID.
	wrongKey := g_accessKey
	wrongKey.Id += "taco"

	db, err := sdb.NewSimpleDB(g_region, wrongKey)
	AssertEq(nil, err)

	// Attempt to create a domain.
	_, err = db.OpenDomain("some_domain")

	ExpectThat(err, Error(HasSubstr("403")))
	ExpectThat(err, Error(HasSubstr("Key Id")))
	ExpectThat(err, Error(HasSubstr("exist")))
}

func (t *DomainsTest) SeparatelyNamedDomainsHaveIndependentItems() {
	var err error

	// Set up an item in the first domain.
	itemName := t.makeItemName()
	err = g_domainsTestDomain0.PutAttributes(
		itemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "enchilada", Value: "queso"},
		},
		[]sdb.Precondition{},
	)

	AssertEq(nil, err)

	// Get attributes for the same name in the other domain. There should be
	// none.
	attrs, err := g_domainsTestDomain1.GetAttributes(itemName, true, []string{})
	AssertEq(nil, err)

	ExpectThat(attrs, ElementsAre())
}

func (t *DomainsTest) IdenticallyNamedDomainsHaveIdenticalItems() {
	var err error

	// Set up an item in the first domain.
	itemName := t.makeItemName()
	err = g_domainsTestDomain0.PutAttributes(
		itemName,
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "enchilada", Value: "queso"},
		},
		[]sdb.Precondition{},
	)

	AssertEq(nil, err)

	// Get attributes for the same name in another domain object opened with the
	// same name.
	domain1, err := t.db.OpenDomain(g_domainsTestDomain0.Name())
	AssertEq(nil, err)

	attrs, err := domain1.GetAttributes(itemName, true, []string{})
	AssertEq(nil, err)

	ExpectThat(
		attrs,
		ElementsAre(
			DeepEquals(sdb.Attribute{Name: "enchilada", Value: "queso"}),
		),
	)
}

func (t *DomainsTest) Delete() {
	var err error
	domainName := "DomainsTest.Delete"

	// Create a domain, then delete it.
	domain, err := t.db.OpenDomain(domainName)
	AssertEq(nil, err)

	err = t.db.DeleteDomain(domain)
	AssertEq(nil, err)

	// Delete again; nothing should go wrong.
	err = t.db.DeleteDomain(domain)
	AssertEq(nil, err)

	// Attempt to write to the domain.
	err = domain.PutAttributes(
		"some_item",
		[]sdb.PutUpdate{
			sdb.PutUpdate{Name: "foo", Value: "bar"},
		},
		[]sdb.Precondition{},
	)

	ExpectThat(err, Error(HasSubstr("NoSuchDomain")))
	ExpectThat(err, Error(HasSubstr("domain")))
	ExpectThat(err, Error(HasSubstr("exist")))
}

////////////////////////////////////////////////////////////////////////
// Items
////////////////////////////////////////////////////////////////////////

var g_itemsTestDb     sdb.SimpleDB
var g_itemsTestDomain sdb.Domain

type ItemsTest struct {
	integrationTest
}

func init() { RegisterTestSuite(&ItemsTest{}) }

func (t *ItemsTest) SetUpTestSuite() {
	var err error

	// Open a connection.
	g_itemsTestDb, err = sdb.NewSimpleDB(g_region, g_accessKey)
	if err != nil {
		panic(err)
	}

	// Create a domain.
	g_itemsTestDomain, err = g_itemsTestDb.OpenDomain("ItemsTest.domain")
	if err != nil {
		panic(err)
	}
}

func (t *ItemsTest) TearDownTestSuite() {
	// Delete the domain.
	if err := g_itemsTestDb.DeleteDomain(g_itemsTestDomain); err != nil {
		panic(err)
	}

	// Clear variables.
	g_itemsTestDb = nil
	g_itemsTestDomain = nil
}

func (t *ItemsTest) InvalidUtf8ItemName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidUtf8AttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidUtf8AttributeValue() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) LongItemName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) LongAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) LongAttributeValue() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) PutThenGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchPutThenGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchPutThenBatchGet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetForNonExistentItem() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetForNonExistentItems() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) GetNonExistentAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchGetNonExistentAttributeName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedValuePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedExistencePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) FailedNonExistencePrecondition() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SuccessfulPreconditions() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) DeleteParticularAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) DeleteAllAttributes() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) BatchDelete() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) InvalidSelectQuery() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectAll() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectItemName() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectCount() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithPredicates() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithSortOrder() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectWithLimit() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectEmptyResultSet() {
	ExpectEq("TODO", "")
}

func (t *ItemsTest) SelectLargeResultSet() {
	ExpectEq("TODO", "")
}
<|endoftext|>


<commit_before>package maimok

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"

	libvirt "github.com/libvirt/libvirt-go"
	"github.com/satori/go.uuid"
)

// VM data model
type VM struct {
	ID     uint    `json:"id"`
	Name   string  `json:"name"`
	Memory uint64  `json:"memory"`
	State  VMState `json:"state"`
}

// VMState is the state of the vm
type VMState int

const (
	// Running vm
	Running VMState = iota + 1
	// Stopped vm
	Stopped
)

// CreateVMStruct struct for the CreateVM call
type CreateVMStruct struct {
	// Generated
	ID         string
	MACAddress string
	// Provided by API
	Name        string
	Hostname    string
	RAMMB       uint
	DiskSpaceGB uint
	Image       string
	IPAddress   string
	// Set in config file
	SSHKey  string
	Gateway string
	Netmask string
}

func generateMACAddress() (string, error) {
	buf := make(net.HardwareAddr, 6)
	_, err := rand.Read(buf)
	if err != nil {
		return "", err
	}
	buf[0] = (buf[0] | 2) & 0xfe // Set local bit, ensure unicast address
	return buf.String(), nil
}

// CreateVM createds a virtual machine
func CreateVM(state *globalState, createVM CreateVMStruct) error {
	createVM.ID = uuid.NewV4().String()
	createVM.SSHKey = state.config.SSHKey
	createVM.Gateway = state.config.Gateway
	createVM.Netmask = state.config.Netmask

	macAddress, err := generateMACAddress()
	if err != nil {
		return fmt.Errorf("Cannot generate MAC Address, %s", err)
	}
	createVM.MACAddress = macAddress

	// create config iso
	dir, err := ioutil.TempDir("", "maimok")
	if err != nil {
		return fmt.Errorf("Cannot create temp directory, %s", err)
	}
	defer os.RemoveAll(dir)

	metaDataFile, err := os.OpenFile(filepath.Join(dir, "meta-data"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer metaDataFile.Close()
	if err := state.tpl.ExecuteTemplate(metaDataFile, "meta-data.yml", createVM); err != nil {
		return err
	}

	userDataFile, err := os.OpenFile(filepath.Join(dir, "user-data"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer userDataFile.Close()
	if err := state.tpl.ExecuteTemplate(userDataFile, "user-data.yml", createVM); err != nil {
		return err
	}

	networkConfigFile, err := os.OpenFile(filepath.Join(dir, "network-config"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer networkConfigFile.Close()
	if err := state.tpl.ExecuteTemplate(networkConfigFile, "network-config.yml", createVM); err != nil {
		return err
	}

	cmd := exec.Command("genisoimage", "-volid", "cidata", "-joliet", "-rock",
		metaDataFile.Name(), userDataFile.Name(), networkConfigFile.Name())
	var isoFile bytes.Buffer
	cmd.Stdout = &isoFile
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("Generation of iso image failed: %s", err)
	}

	storagePool, err := state.conn.LookupStoragePoolByName("default")
	if err != nil {
		return err
	}

	// create iso volume
	buf := new(bytes.Buffer)
	if err := state.tpl.ExecuteTemplate(buf, "iso-volume.xml", createVM); err != nil {
		return err
	}
	volume, err := storagePool.StorageVolCreateXML(buf.String(), 0)
	if err != nil {
		return err
	}

	stream, err := state.conn.NewStream(0)
	if err != nil {
		return err
	}
	volume.Upload(stream, 0, 0, 0)
	len, err := stream.Send(isoFile.Bytes())
	if err != nil {
		return fmt.Errorf("Could not send config iso image to storage pool: %s", err)
	}
	if len < buf.Len() {
		return fmt.Errorf("Could not send all iso image data to storage pool")
	}
	if err := stream.Finish(); err != nil {
		return fmt.Errorf("Could not send config iso image to storage pool: %s", err)
	}

	// create harddisk volume
	buf = new(bytes.Buffer)
	if err := state.tpl.ExecuteTemplate(buf, "volume.xml", createVM); err != nil {
		return err
	}
	if _, err = storagePool.StorageVolCreateXML(buf.String(), 0); err != nil {
		return err
	}

	// define domain
	buf = new(bytes.Buffer)
	if err = state.tpl.ExecuteTemplate(buf, "domain.xml", createVM); err != nil {
		return err
	}
	domain, err := state.conn.DomainDefineXML(buf.String())
	if err != nil {
		return fmt.Errorf("Failed to define domain: %s", err)
	}

	// start domain
	if err = domain.Create(); err != nil {
		return fmt.Errorf("Failed to start domain: %s", err)
	}

	return nil
}

// ListVMs returns a list of all virtual machines
func ListVMs(state *globalState) []*VM {
	domains, err := state.conn.ListAllDomains(0)
	if err != nil {
		return nil
	}

	vms := []*VM{}
	for _, domain := range domains {
		name, _ := domain.GetName()
		memory, _ := domain.GetMaxMemory()
		id, _ := domain.GetID()
		state, _, _ := domain.GetState()

		var vmState VMState
		if state == libvirt.DOMAIN_RUNNING {
			vmState = Running
		} else {
			vmState = Stopped
		}

		vms = append(vms, &VM{ID: id, Name: name, Memory: memory, State: vmState})
	}
	return vms
}
<commit_msg>backend: show running state as boolean<commit_after>package maimok

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"

	libvirt "github.com/libvirt/libvirt-go"
	uuid "github.com/satori/go.uuid"
)

// VM data model
type VM struct {
	ID      uint   `json:"id"`
	Name    string `json:"name"`
	Memory  uint64 `json:"memory"`
	Running bool   `json:"running"`
}

// CreateVMStruct struct for the CreateVM call
type CreateVMStruct struct {
	// Generated
	ID         string
	MACAddress string
	// Provided by API
	Name        string
	Hostname    string
	RAMMB       uint
	DiskSpaceGB uint
	Image       string
	IPAddress   string
	// Set in config file
	SSHKey  string
	Gateway string
	Netmask string
}

func generateMACAddress() (string, error) {
	buf := make(net.HardwareAddr, 6)
	_, err := rand.Read(buf)
	if err != nil {
		return "", err
	}
	buf[0] = (buf[0] | 2) & 0xfe // Set local bit, ensure unicast address
	return buf.String(), nil
}

// CreateVM createds a virtual machine
func CreateVM(state *globalState, createVM CreateVMStruct) error {
	createVM.ID = uuid.NewV4().String()
	createVM.SSHKey = state.config.SSHKey
	createVM.Gateway = state.config.Gateway
	createVM.Netmask = state.config.Netmask

	macAddress, err := generateMACAddress()
	if err != nil {
		return fmt.Errorf("Cannot generate MAC Address, %s", err)
	}
	createVM.MACAddress = macAddress

	// create config iso
	dir, err := ioutil.TempDir("", "maimok")
	if err != nil {
		return fmt.Errorf("Cannot create temp directory, %s", err)
	}
	defer os.RemoveAll(dir)

	metaDataFile, err := os.OpenFile(filepath.Join(dir, "meta-data"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer metaDataFile.Close()
	if err := state.tpl.ExecuteTemplate(metaDataFile, "meta-data.yml", createVM); err != nil {
		return err
	}

	userDataFile, err := os.OpenFile(filepath.Join(dir, "user-data"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer userDataFile.Close()
	if err := state.tpl.ExecuteTemplate(userDataFile, "user-data.yml", createVM); err != nil {
		return err
	}

	networkConfigFile, err := os.OpenFile(filepath.Join(dir, "network-config"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer networkConfigFile.Close()
	if err := state.tpl.ExecuteTemplate(networkConfigFile, "network-config.yml", createVM); err != nil {
		return err
	}

	cmd := exec.Command("genisoimage", "-volid", "cidata", "-joliet", "-rock",
		metaDataFile.Name(), userDataFile.Name(), networkConfigFile.Name())
	var isoFile bytes.Buffer
	cmd.Stdout = &isoFile
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("Generation of iso image failed: %s", err)
	}

	storagePool, err := state.conn.LookupStoragePoolByName("default")
	if err != nil {
		return err
	}

	// create iso volume
	buf := new(bytes.Buffer)
	if err := state.tpl.ExecuteTemplate(buf, "iso-volume.xml", createVM); err != nil {
		return err
	}
	volume, err := storagePool.StorageVolCreateXML(buf.String(), 0)
	if err != nil {
		return err
	}

	stream, err := state.conn.NewStream(0)
	if err != nil {
		return err
	}
	volume.Upload(stream, 0, 0, 0)
	len, err := stream.Send(isoFile.Bytes())
	if err != nil {
		return fmt.Errorf("Could not send config iso image to storage pool: %s", err)
	}
	if len < buf.Len() {
		return fmt.Errorf("Could not send all iso image data to storage pool")
	}
	if err := stream.Finish(); err != nil {
		return fmt.Errorf("Could not send config iso image to storage pool: %s", err)
	}

	// create harddisk volume
	buf = new(bytes.Buffer)
	if err := state.tpl.ExecuteTemplate(buf, "volume.xml", createVM); err != nil {
		return err
	}
	if _, err = storagePool.StorageVolCreateXML(buf.String(), 0); err != nil {
		return err
	}

	// define domain
	buf = new(bytes.Buffer)
	if err = state.tpl.ExecuteTemplate(buf, "domain.xml", createVM); err != nil {
		return err
	}
	domain, err := state.conn.DomainDefineXML(buf.String())
	if err != nil {
		return fmt.Errorf("Failed to define domain: %s", err)
	}

	// start domain
	if err = domain.Create(); err != nil {
		return fmt.Errorf("Failed to start domain: %s", err)
	}

	return nil
}

// ListVMs returns a list of all virtual machines
func ListVMs(state *globalState) []*VM {
	domains, err := state.conn.ListAllDomains(0)
	if err != nil {
		return nil
	}

	vms := []*VM{}
	for _, domain := range domains {
		name, _ := domain.GetName()
		memory, _ := domain.GetMaxMemory()
		id, _ := domain.GetID()
		state, _, _ := domain.GetState()

		var running bool
		if state == libvirt.DOMAIN_RUNNING {
			running = true
		} else {
			running = false
		}

		vms = append(vms, &VM{ID: id, Name: name, Memory: memory, Running: running})
	}
	return vms
}
<|endoftext|>