<commit_before>package backend

import (
	"bytes"
	"context"
	"fmt"
	"os"
	"path"

	gw "github.com/cvmfs/gateway/internal/gateway"
	"github.com/pkg/errors"
	bolt "go.etcd.io/bbolt"
)

// NotificationMessage is an alias for a UTF-8 string
type NotificationMessage string

// SubscriberHandle is the writable end of a channel of notification messages
type SubscriberHandle chan NotificationMessage

// SubscriberSet is a set of subscriber handles (implemented as a map handler -> void)
type SubscriberSet map[SubscriberHandle]struct{}

// SubscriberMap holds a set of subscriber handles for each repository
type SubscriberMap map[string]SubscriberSet

// NotificationSystem encapsulates the functionality of the repository
// activity notification system
type NotificationSystem struct {
	Subscribers SubscriberMap
	WorkDir     string
	Store       *bolt.DB
}

// NewNotificationSystem is a constructor function for the NotificationSystem type
func NewNotificationSystem(workDir string) (*NotificationSystem, error) {
	workDir = path.Join(workDir, "notify")
	if err := os.MkdirAll(workDir, 0777); err != nil {
		return nil, errors.Wrap(err, "could not create directory for backing store")
	}
	store, err := bolt.Open(workDir+"/messages.db", 0666, nil)
	if err != nil {
		return nil, errors.Wrap(err, "could not open backing store")
	}

	// Create a bucket in the backing store to hold the last message published for
	// each repository
	store.Update(func(txn *bolt.Tx) error {
		_, err := txn.CreateBucketIfNotExists([]byte("last_message"))
		if err != nil {
			return errors.Wrap(err, "could not create bucket: 'last_message'")
		}

		return nil
	})

	gw.Log("notify", gw.LogInfo).
		Msgf("database opened (work dir: %v)", workDir)

	ns := &NotificationSystem{
		Subscribers: make(SubscriberMap),
		WorkDir:     workDir,
		Store:       store,
	}

	return ns, nil
}

// Publish a new repository manifest to the notification system
func (ns *NotificationSystem) Publish(
	ctx context.Context, repository string, message []byte) error {

	shouldNotify := false
	ns.Store.Update(func(txn *bolt.Tx) error {
		existing := getLastMessage(txn, repository)
		if existing == nil || !bytes.Equal(message, existing) {
			if err := setMessage(txn, repository, message); err != nil {
				return err
			}
			shouldNotify = true
		}

		return nil
	})

	if shouldNotify {
		ns.notify(repository, message)
	}

	gw.LogC(ctx, "notify", gw.LogDebug).
		Str("repository", repository).
		Msg("manifest published")

	return nil
}

// Subscribe the handle to messages for the given repository
func (ns *NotificationSystem) Subscribe(
	ctx context.Context, repository string, handle SubscriberHandle) {

	added := false
	subsForRepo, present := ns.Subscribers[repository]
	if present {
		_, pres := subsForRepo[handle]
		if !pres {
			subsForRepo[handle] = struct{}{}
			added = true
		}
	} else {
		subsForRepo = make(SubscriberSet)
		subsForRepo[handle] = struct{}{}
		added = true
		ns.Subscribers[repository] = subsForRepo
	}

	if added {
		message := make([]byte, 0)
		ns.Store.View(func(txn *bolt.Tx) error {
			message = getLastMessage(txn, repository)
			return nil
		})

		if message != nil {
			ns.notify(repository, message)
		}

		gw.LogC(ctx, "notify", gw.LogDebug).
			Str("repository", repository).
			Msg("subscription added")
	}
}

// Unsubscribe from messages for the given repository
func (ns *NotificationSystem) Unsubscribe(
	ctx context.Context, repository string, handle SubscriberHandle) error {

	subsForRepo, hasSubs := ns.Subscribers[repository]
	if !hasSubs {
		return fmt.Errorf("no_subscriptions_for_repository")
	}

	_, found := subsForRepo[handle]
	if !found {
		return fmt.Errorf("invalid_handle")
	}

	delete(subsForRepo, handle)
	close(handle)

	gw.LogC(ctx, "notify", gw.LogDebug).
		Str("repository", repository).
		Msg("subscription removed")

	return nil
}

func (ns *NotificationSystem) notify(repository string, message []byte) {
	subsForRepo, present := ns.Subscribers[repository]
	if present {
		for s := range subsForRepo {
			s <- NotificationMessage(message)
		}
	}
}

func getLastMessage(txn *bolt.Tx, repository string) []byte {
	b := txn.Bucket([]byte("last_message"))
	return b.Get([]byte(repository))
}

func setMessage(txn *bolt.Tx, repository string, message []byte) error {
	b := txn.Bucket([]byte("last_message"))
	return b.Put([]byte(repository), message)
}
<commit_msg>Ensure subscriber list operations are thread-safe<commit_after>package backend

import (
	"bytes"
	"context"
	"fmt"
	"os"
	"path"
	"sync"

	gw "github.com/cvmfs/gateway/internal/gateway"
	"github.com/pkg/errors"
	bolt "go.etcd.io/bbolt"
)

// NotificationMessage is an alias for a UTF-8 string
type NotificationMessage string

// SubscriberHandle is the writable end of a channel of notification messages
type SubscriberHandle chan NotificationMessage

// SubscriberSet is a set of subscriber handles (implemented as a map handler -> void)
type SubscriberSet map[SubscriberHandle]struct{}

// SubscriberMap holds a set of subscriber handles for each repository
type SubscriberMap map[string]SubscriberSet

// NotificationSystem encapsulates the functionality of the repository
// activity notification system
type NotificationSystem struct {
	Subscribers    SubscriberMap
	SubscriberLock sync.RWMutex
	WorkDir        string
	Store          *bolt.DB
}

// NewNotificationSystem is a constructor function for the NotificationSystem type
func NewNotificationSystem(workDir string) (*NotificationSystem, error) {
	workDir = path.Join(workDir, "notify")
	if err := os.MkdirAll(workDir, 0777); err != nil {
		return nil, errors.Wrap(err, "could not create directory for backing store")
	}
	store, err := bolt.Open(workDir+"/messages.db", 0666, nil)
	if err != nil {
		return nil, errors.Wrap(err, "could not open backing store")
	}

	// Create a bucket in the backing store to hold the last message published for
	// each repository
	store.Update(func(txn *bolt.Tx) error {
		_, err := txn.CreateBucketIfNotExists([]byte("last_message"))
		if err != nil {
			return errors.Wrap(err, "could not create bucket: 'last_message'")
		}

		return nil
	})

	gw.Log("notify", gw.LogInfo).
		Msgf("database opened (work dir: %v)", workDir)

	ns := &NotificationSystem{
		Subscribers:    make(SubscriberMap),
		SubscriberLock: sync.RWMutex{},
		WorkDir:        workDir,
		Store:          store,
	}

	return ns, nil
}

// Publish a new repository manifest to the notification system
func (ns *NotificationSystem) Publish(
	ctx context.Context, repository string, message []byte) error {

	shouldNotify := false
	ns.Store.Update(func(txn *bolt.Tx) error {
		existing := getLastMessage(txn, repository)
		if existing == nil || !bytes.Equal(message, existing) {
			if err := setMessage(txn, repository, message); err != nil {
				return err
			}
			shouldNotify = true
		}

		return nil
	})

	if shouldNotify {
		ns.notify(repository, message)
	}

	gw.LogC(ctx, "notify", gw.LogDebug).
		Str("repository", repository).
		Msg("manifest published")

	return nil
}

// Subscribe the handle to messages for the given repository
func (ns *NotificationSystem) Subscribe(
	ctx context.Context, repository string, handle SubscriberHandle) {

	added := false
	go func() {
		ns.SubscriberLock.Lock()
		defer ns.SubscriberLock.Unlock()
		subsForRepo, present := ns.Subscribers[repository]
		if present {
			_, pres := subsForRepo[handle]
			if !pres {
				subsForRepo[handle] = struct{}{}
				added = true
			}
		} else {
			subsForRepo = make(SubscriberSet)
			subsForRepo[handle] = struct{}{}
			added = true
			ns.Subscribers[repository] = subsForRepo
		}
	}()

	if added {
		message := make([]byte, 0)
		ns.Store.View(func(txn *bolt.Tx) error {
			message = getLastMessage(txn, repository)
			return nil
		})

		if message != nil {
			ns.notify(repository, message)
		}

		gw.LogC(ctx, "notify", gw.LogDebug).
			Str("repository", repository).
			Msg("subscription added")
	}
}

// Unsubscribe from messages for the given repository. Closes the subscriber handle (chan)
func (ns *NotificationSystem) Unsubscribe(
	ctx context.Context, repository string, handle SubscriberHandle) error {

	ns.SubscriberLock.Lock()
	defer ns.SubscriberLock.Unlock()
	subsForRepo, hasSubs := ns.Subscribers[repository]
	if !hasSubs {
		return fmt.Errorf("no_subscriptions_for_repository")
	}

	_, found := subsForRepo[handle]
	if !found {
		return fmt.Errorf("invalid_handle")
	}

	delete(subsForRepo, handle)
	close(handle)

	gw.LogC(ctx, "notify", gw.LogDebug).
		Str("repository", repository).
		Msg("subscription removed")

	return nil
}

func (ns *NotificationSystem) notify(repository string, message []byte) {
	ns.SubscriberLock.RLock()
	defer ns.SubscriberLock.RUnlock()
	subsForRepo, present := ns.Subscribers[repository]
	if present {
		for s := range subsForRepo {
			s <- NotificationMessage(message)
		}
	}
}

func getLastMessage(txn *bolt.Tx, repository string) []byte {
	b := txn.Bucket([]byte("last_message"))
	return b.Get([]byte(repository))
}

func setMessage(txn *bolt.Tx, repository string, message []byte) error {
	b := txn.Bucket([]byte("last_message"))
	return b.Put([]byte(repository), message)
}
<|endoftext|>


<commit_before>package frontend

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/julienschmidt/httprouter"

	gw "github.com/cvmfs/gateway/internal/gateway"
	be "github.com/cvmfs/gateway/internal/gateway/backend"
)

type message map[string]interface{}

// WithAdminAuthz returns an HMAC authorization middleware used for administrative
// operations (disable/enable repositories and keys, cancel leases, trigger GC, etc.)
func WithAdminAuthz(ac be.ActionController, next httprouter.Handle) httprouter.Handle {
	return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		ctx := req.Context()
		tokens := strings.Split(req.Header.Get("Authorization"), " ")
		if len(tokens) != 2 {
			gw.LogC(ctx, "http", gw.LogError).
				Msg("missing tokens in authorization header")
			replyJSON(ctx, w, message{"status": "error", "reason": "invalid_authorization_header"})
			return
		}

		keyID := tokens[0]
		keyCfg := ac.GetKey(keyID)
		if keyCfg == nil {
			gw.LogC(ctx, "http", gw.LogError).
				Msg("invalid key ID specified")
			replyJSON(ctx, w, message{"status": "error", "reason": "invalid_key"})
			return
		}

		if !keyCfg.Admin {
			gw.LogC(ctx, "http", gw.LogError).
				Msg("key does not have admin rights")
			replyJSON(ctx, w, message{"status": "error", "reason": "no_admin_key"})
			return
		}

		HMAC, err := base64.StdEncoding.DecodeString(tokens[1])
		if err != nil {
			gw.LogC(ctx, "http", gw.LogError).
				Err(err).Msg("could not base64 decode HMAC")
			replyJSON(ctx, w, message{"status": "error", "reason": "invalid_hmac"})
			return
		}

		var HMACInput []byte
		switch req.Method {
		case "DELETE":
			// For DELETE requests, use the path component of the URL to compute the HMAC
			HMACInput = []byte(req.URL.Path)
		case "POST":
			// For POST requests, the request body is used to compute HMAC
			var err error
			HMACInput, err = ioutil.ReadAll(req.Body)
			if err != nil {
				httpWrapError(ctx, err, "could not read request body", w, http.StatusInternalServerError)
				return
			}
			// Body needs to be read again in the next handler, reset it
			// using a copy of the original body
			bodyCopy := ioutil.NopCloser(bytes.NewReader(HMACInput))
			req.Body.Close()
			req.Body = bodyCopy
		default:
			msg := fmt.Sprintf("authorization middleware not implemented for HTTP method: %v", req.Method)
			gw.LogC(ctx, "http", gw.LogError).
				Msgf(msg)
			http.Error(w, msg, http.StatusMethodNotAllowed)
		}

		if !CheckHMAC(HMACInput, HMAC, keyCfg.Secret) {
			gw.LogC(ctx, "http", gw.LogError).
				Msg("invalid HMAC")
			replyJSON(ctx, w, message{"status": "error", "reason": "invalid_hmac"})
			return
		}

		next(w, req, ps)
	}
}

// WithAuthz returns an HMAC authorization middleware
func WithAuthz(ac be.ActionController, next httprouter.Handle) httprouter.Handle {
	return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		ctx := req.Context()
		tokens := strings.Split(req.Header.Get("Authorization"), " ")
		if len(tokens) != 2 {
			gw.LogC(ctx, "http", gw.LogError).
				Msg("missing tokens in authorization header")
			replyJSON(ctx, w, message{"status": "error", "reason": "invalid_hmac"})
			return
		}

		keyID := tokens[0]
		HMAC, err := base64.StdEncoding.DecodeString(tokens[1])
		if err != nil {
			gw.LogC(ctx, "http", gw.LogError).
				Err(err).Msg("could not base64 decode HMAC")
			replyJSON(ctx, w, message{"status": "error", "reason": "invalid_hmac"})
			return
		}

		keyCfg := ac.GetKey(keyID)
		if keyCfg == nil {
			gw.LogC(ctx, "http", gw.LogError).
				Msg("invalid key ID specified")
			replyJSON(ctx, w, message{"status": "error", "reason": "invalid_hmac"})
			return
		}

		// Different parts of the request are used to compute then HMAC, depending
		// in HTTP method and route

		var HMACInput []byte
		if strings.HasPrefix(req.URL.Path, APIRoot+"/leases") {
			token := ps.ByName("token")
			if token != "" {
				// For commit/drop lease requests use the token to compute HMAC
				HMACInput = []byte(token)
			} else {
				// For new lease request used the request body to compute HMAC
				HMACInput, err = ioutil.ReadAll(req.Body)
				if err != nil {
					httpWrapError(ctx, err, "could not read request body", w, http.StatusInternalServerError)
					return
				}
				// Body needs to be read again in the next handler, reset it
				// using a copy of the original body
				bodyCopy := ioutil.NopCloser(bytes.NewReader(HMACInput))
				req.Body.Close()
				req.Body = bodyCopy
			}
		} else if strings.HasPrefix(req.URL.Path, APIRoot+"/payloads") {
			token := ps.ByName("token")
			if token != "" {
				// For the new style of payload submission requests, use the token to compute HMAC
				HMACInput = []byte(token)
			} else {
				// For legacy payload submission requests, the JSON msg at the beginning of the body
				// is used to compute the HMAC
				sz := req.Header.Get("message-size")
				msgSize, err := strconv.Atoi(sz)
				if err != nil {
					httpWrapError(ctx, err, "missing message-size header", w, http.StatusBadRequest)
					return
				}
				msg := make([]byte, msgSize)
				if _, err := io.ReadFull(req.Body, msg); err != nil {
					httpWrapError(ctx, err, "invalid request body", w, http.StatusBadRequest)
					return
				}

				HMACInput = msg

				// replace the request body with a new ReadCLoser which includes the already-read
				// head part
				req.Body = newRecombineReadCloser(msg, req.Body)
			}
		}

		if !CheckHMAC(HMACInput, HMAC, keyCfg.Secret) {
			gw.LogC(ctx, "http", gw.LogError).
				Msg("invalid HMAC")
			replyJSON(ctx, w, message{"status": "error", "reason": "invalid_hmac"})
			return
		}
		next(w, req, ps)
	}
}

// The recombineReadCloser is used during payload submission requests to recombine the request message,
// already read inside the authorization middleware with the remaining request body and ensure that the
// body (io.ReadCloser) is eventually closed and does not leak
type recombineReadCloser struct {
	combined io.Reader
	original io.ReadCloser
}

func newRecombineReadCloser(head []byte, tail io.ReadCloser) *recombineReadCloser {
	return &recombineReadCloser{io.MultiReader(bytes.NewReader(head), tail), tail}
}

func (r recombineReadCloser) Read(p []byte) (int, error) {
	return r.combined.Read(p)
}

func (r recombineReadCloser) Close() error {
	return r.original.Close()
}
<commit_msg>Factor out common parts of the two authorization middlewares<commit_after>package frontend

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/julienschmidt/httprouter"
	"github.com/pkg/errors"

	gw "github.com/cvmfs/gateway/internal/gateway"
	be "github.com/cvmfs/gateway/internal/gateway/backend"
)

type message map[string]interface{}

// WithAdminAuthz returns an HMAC authorization middleware used for administrative
// operations (disable/enable repositories and keys, cancel leases, trigger GC, etc.)
func WithAdminAuthz(ac be.ActionController, next httprouter.Handle) httprouter.Handle {
	return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		ctx := req.Context()
		keyID, HMAC, err := parseHeader(&req.Header)
		if err != nil {
			gw.LogC(ctx, "http", gw.LogError).
				Err(err).
				Msg("authorization failure")
			replyJSON(ctx, w, message{"status": "error", "reason": "invalid_authorization_header"})
			return
		}

		keyCfg := ac.GetKey(keyID)
		if keyCfg == nil {
			gw.LogC(ctx, "http", gw.LogError).
				Msg("invalid key ID specified")
			replyJSON(ctx, w, message{"status": "error", "reason": "invalid_key"})
			return
		}

		if !keyCfg.Admin {
			gw.LogC(ctx, "http", gw.LogError).
				Msg("key does not have admin rights")
			replyJSON(ctx, w, message{"status": "error", "reason": "no_admin_key"})
			return
		}

		var HMACInput []byte
		switch req.Method {
		case "DELETE":
			// For DELETE requests, use the path component of the URL to compute the HMAC
			HMACInput = []byte(req.URL.Path)
		case "POST":
			// For POST requests, the request body is used to compute HMAC
			HMACInput, err = readBody(req, req.ContentLength)
			if err != nil {
				httpWrapError(ctx, err, "could not read request body", w, http.StatusInternalServerError)
				return
			}
		default:
			msg := fmt.Sprintf("authorization middleware not implemented for HTTP method: %v", req.Method)
			gw.LogC(ctx, "http", gw.LogError).
				Msgf(msg)
			http.Error(w, msg, http.StatusMethodNotAllowed)
		}

		if !CheckHMAC(HMACInput, HMAC, keyCfg.Secret) {
			gw.LogC(ctx, "http", gw.LogError).
				Msg("invalid HMAC")
			replyJSON(ctx, w, message{"status": "error", "reason": "invalid_hmac"})
			return
		}

		next(w, req, ps)
	}
}

// WithAuthz returns an HMAC authorization middleware
func WithAuthz(ac be.ActionController, next httprouter.Handle) httprouter.Handle {
	return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		ctx := req.Context()
		keyID, HMAC, err := parseHeader(&req.Header)
		if err != nil {
			gw.LogC(ctx, "http", gw.LogError).
				Err(err).
				Msg("authorization failure")
			replyJSON(ctx, w, message{"status": "error", "reason": "invalid_hmac"})
			return
		}

		keyCfg := ac.GetKey(keyID)
		if keyCfg == nil {
			gw.LogC(ctx, "http", gw.LogError).
				Msg("invalid key ID specified")
			replyJSON(ctx, w, message{"status": "error", "reason": "invalid_hmac"})
			return
		}

		// Different parts of the request are used to compute then HMAC, depending
		// in HTTP method and route

		var HMACInput []byte
		if strings.HasPrefix(req.URL.Path, APIRoot+"/leases") {
			token := ps.ByName("token")
			if token != "" {
				// For commit/drop lease requests use the token to compute HMAC
				HMACInput = []byte(token)
			} else {
				// For new lease request used the request body to compute HMAC
				HMACInput, err = readBody(req, req.ContentLength)
				if err != nil {
					httpWrapError(ctx, err, "could not read request body", w, http.StatusInternalServerError)
					return
				}
			}
		} else if strings.HasPrefix(req.URL.Path, APIRoot+"/payloads") {
			token := ps.ByName("token")
			if token != "" {
				// For the new style of payload submission requests, use the token to compute HMAC
				HMACInput = []byte(token)
			} else {
				// For legacy payload submission requests, the JSON msg at the beginning of the body
				// is used to compute the HMAC
				sz := req.Header.Get("message-size")
				msgSize, err := strconv.Atoi(sz)
				if err != nil {
					httpWrapError(ctx, err, "missing message-size header", w, http.StatusBadRequest)
					return
				}
				HMACInput, err = readBody(req, int64(msgSize))
				if err != nil {
					httpWrapError(ctx, err, "could not read request body", w, http.StatusInternalServerError)
					return
				}
			}
		}

		if !CheckHMAC(HMACInput, HMAC, keyCfg.Secret) {
			gw.LogC(ctx, "http", gw.LogError).
				Msg("invalid HMAC")
			replyJSON(ctx, w, message{"status": "error", "reason": "invalid_hmac"})
			return
		}
		next(w, req, ps)
	}
}

// The recombineReadCloser is used during payload submission requests to recombine the request message,
// already read inside the authorization middleware with the remaining request body and ensure that the
// body (io.ReadCloser) is eventually closed and does not leak
type recombineReadCloser struct {
	combined io.Reader
	original io.ReadCloser
}

func newRecombineReadCloser(head []byte, tail io.ReadCloser) *recombineReadCloser {
	return &recombineReadCloser{io.MultiReader(bytes.NewReader(head), tail), tail}
}

func (r recombineReadCloser) Read(p []byte) (int, error) {
	return r.combined.Read(p)
}

func (r recombineReadCloser) Close() error {
	return r.original.Close()
}

func parseHeader(h *http.Header) (string, []byte, error) {
	tokens := strings.Split(h.Get("Authorization"), " ")
	if len(tokens) != 2 {
		return "", nil, fmt.Errorf("missing tokens in authoriation header")
	}

	keyID := tokens[0]

	HMAC, err := base64.StdEncoding.DecodeString(tokens[1])
	if err != nil {
		return "", nil, errors.Wrap(err, "could not base64 decode HMAC")
	}

	return keyID, HMAC, nil
}

// Read the request body and place and resets the consumed Reader, allowing the
// body to be re-read in the next HTTP handler. Only the first "readSize" bytes
// will be read from the body and returned.
func readBody(req *http.Request, readSize int64) ([]byte, error) {
	partial := readSize < req.ContentLength
	body := make([]byte, readSize)
	if _, err := io.ReadFull(req.Body, body); err != nil {
		return nil, err
	}

	if partial {
		// replace the request body with a new ReadCLoser which includes the already-read
		// head part
		req.Body = newRecombineReadCloser(body, req.Body)
	} else {
		bodyCopy := ioutil.NopCloser(bytes.NewReader(body))
		req.Body.Close()
		req.Body = bodyCopy
	}

	return body, nil
}
<|endoftext|>


<commit_before>package builtin_test

import (
	"bytes"
	"testing"

	"github.com/NeowayLabs/nash"
)

func TestFormat(t *testing.T) {
	type formatDesc struct {
		script string
		output string
	}

	tests := map[string]formatDesc{
		"textonly": {
			script: `
				r <= format("helloworld")
				echo $r
			`,
			output: "helloworld\n",
		},
		"fmtstring": {
			script: `
				r <= format("%s:%s", "hello", "world")
				echo $r
			`,
			output: "hello:world\n",
		},
		"fmtlist": {
			script: `
				list = ("1" "2" "3")
				r <= format("%s:%s", "list", $list)
				echo $r
			`,
			output: "list:1 2 3\n",
		},
		"funconly": {
			script: `
				fn func() {}
				r <= format($func)
				echo $r
			`,
			output: "<fn func>\n",
		},
		"funcfmt": {
			script: `
				fn func() {}
				r <= format("calling:%s", $func)
				echo $r
			`,
			output: "calling:<fn func>\n",
		},
		"listonly": {
			script: `
				list = ("1" "2" "3")
				r <= format($list)
				echo $r
			`,
			output: "1 2 3\n",
		},
		"listoflists": {
			script: `
				list = (("1" "2" "3") ("4" "5" "6"))
				r <= format("%s:%s", "listoflists", $list)
				echo $r
			`,
			output: "listoflists:1 2 3 4 5 6\n",
		},
		"listasfmt": {
			script: `
				list = ("%s" "%s")
				r <= format($list, "1", "2")
				echo $r
			`,
			output: "1 2\n",
		},
		"invalidFmt": {
			script: `
				r <= format("%d%s", "invalid")
				echo $r
			`,
			output: "%!d(string=invalid)%!s(MISSING)\n",
		},
	}

	for name, desc := range tests {
		t.Run(name, func(t *testing.T) {
			var output bytes.Buffer
			shell, err := nash.New()

			if err != nil {
				t.Fatalf("unexpected err: %s", err)
			}

			shell.SetStdout(&output)
			err = shell.Exec("", desc.script)

			if err != nil {
				t.Fatalf("unexpected err: %s", err)
			}

			if output.String() != desc.output {
				t.Fatalf("got %q expected %q", output.String(), desc.output)
			}
		})
	}
}

func TestFormatfErrors(t *testing.T) {
	type formatDesc struct {
		script string
	}

	tests := map[string]formatDesc{
		"noParams": {script: `format()`},
	}

	for name, desc := range tests {
		t.Run(name, func(t *testing.T) {
			var output bytes.Buffer
			shell, err := nash.New()

			if err != nil {
				t.Fatalf("unexpected err: %s", err)
			}

			shell.SetStdout(&output)
			err = shell.Exec("", desc.script)

			if err == nil {
				t.Fatalf("expected err, got success, output: %s", output)
			}

			if output.Len() > 0 {
				t.Fatalf("expected empty output, got: %s", output)
			}
		})
	}
}
<commit_msg>Add tests to reproduce problem<commit_after>package builtin_test

import (
	"bytes"
	"testing"

	"github.com/NeowayLabs/nash"
)

func TestFormat(t *testing.T) {
	type formatDesc struct {
		script string
		output string
	}

	tests := map[string]formatDesc{
		"textonly": {
			script: `
				r <= format("helloworld")
				echo $r
			`,
			output: "helloworld\n",
		},
		"ncallsRegressionTest": {
			script: `
				fn formatstuff() {
					r <= format("helloworld")
					s <= format("hacktheworld")
					echo $r
					echo $s
				}
				formatstuff()
				formatstuff()
			`,
			output: "helloworld\nhacktheworld\nhelloworld\nhacktheworld\n",
		},
		"ncallsWithVarsRegressionTest": {
			script: `
				fn formatstuff() {
					b = "world"
					r <= format("hello%s", $b)
					s <= format("hackthe%s", $b)
					echo $r
					echo $s
				}
				formatstuff()
				formatstuff()
			`,
			output: "helloworld\nhacktheworld\nhelloworld\nhacktheworld\n",
		},
		"fmtstring": {
			script: `
				r <= format("%s:%s", "hello", "world")
				echo $r
			`,
			output: "hello:world\n",
		},
		"fmtlist": {
			script: `
				list = ("1" "2" "3")
				r <= format("%s:%s", "list", $list)
				echo $r
			`,
			output: "list:1 2 3\n",
		},
		"funconly": {
			script: `
				fn func() {}
				r <= format($func)
				echo $r
			`,
			output: "<fn func>\n",
		},
		"funcfmt": {
			script: `
				fn func() {}
				r <= format("calling:%s", $func)
				echo $r
			`,
			output: "calling:<fn func>\n",
		},
		"listonly": {
			script: `
				list = ("1" "2" "3")
				r <= format($list)
				echo $r
			`,
			output: "1 2 3\n",
		},
		"listoflists": {
			script: `
				list = (("1" "2" "3") ("4" "5" "6"))
				r <= format("%s:%s", "listoflists", $list)
				echo $r
			`,
			output: "listoflists:1 2 3 4 5 6\n",
		},
		"listasfmt": {
			script: `
				list = ("%s" "%s")
				r <= format($list, "1", "2")
				echo $r
			`,
			output: "1 2\n",
		},
		"invalidFmt": {
			script: `
				r <= format("%d%s", "invalid")
				echo $r
			`,
			output: "%!d(string=invalid)%!s(MISSING)\n",
		},
	}

	for name, desc := range tests {
		t.Run(name, func(t *testing.T) {
			var output bytes.Buffer
			shell, err := nash.New()

			if err != nil {
				t.Fatalf("unexpected err: %s", err)
			}

			shell.SetStdout(&output)
			err = shell.Exec("", desc.script)

			if err != nil {
				t.Fatalf("unexpected err: %s", err)
			}

			if output.String() != desc.output {
				t.Fatalf("got %q expected %q", output.String(), desc.output)
			}
		})
	}
}

func TestFormatfErrors(t *testing.T) {
	type formatDesc struct {
		script string
	}

	tests := map[string]formatDesc{
		"noParams": {script: `format()`},
	}

	for name, desc := range tests {
		t.Run(name, func(t *testing.T) {
			var output bytes.Buffer
			shell, err := nash.New()

			if err != nil {
				t.Fatalf("unexpected err: %s", err)
			}

			shell.SetStdout(&output)
			err = shell.Exec("", desc.script)

			if err == nil {
				t.Fatalf("expected err, got success, output: %s", output)
			}

			if output.Len() > 0 {
				t.Fatalf("expected empty output, got: %s", output)
			}
		})
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provider

import (
	"fmt"
	"io/ioutil"
	"strings"

	log "github.com/sirupsen/logrus"

	yaml "gopkg.in/yaml.v2"

	"github.com/Azure/azure-sdk-for-go/arm/dns"
	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/adal"
	"github.com/Azure/go-autorest/autorest/azure"
	"github.com/Azure/go-autorest/autorest/to"

	"github.com/kubernetes-incubator/external-dns/endpoint"
	"github.com/kubernetes-incubator/external-dns/plan"
)

const (
	azureRecordTTL = 300
)

type config struct {
	Cloud          string `json:"cloud" yaml:"cloud"`
	TenantID       string `json:"tenantId" yaml:"tenantId"`
	SubscriptionID string `json:"subscriptionId" yaml:"subscriptionId"`
	ResourceGroup  string `json:"resourceGroup" yaml:"resourceGroup"`
	Location       string `json:"location" yaml:"location"`
	ClientID       string `json:"aadClientId" yaml:"aadClientId"`
	ClientSecret   string `json:"aadClientSecret" yaml:"aadClientSecret"`
}

// ZonesClient is an interface of dns.ZoneClient that can be stubbed for testing.
type ZonesClient interface {
	ListByResourceGroup(resourceGroupName string, top *int32) (result dns.ZoneListResult, err error)
	ListByResourceGroupNextResults(lastResults dns.ZoneListResult) (result dns.ZoneListResult, err error)
}

// RecordsClient is an interface of dns.RecordClient that can be stubbed for testing.
type RecordsClient interface {
	ListByDNSZone(resourceGroupName string, zoneName string, top *int32) (result dns.RecordSetListResult, err error)
	ListByDNSZoneNextResults(list dns.RecordSetListResult) (result dns.RecordSetListResult, err error)
	Delete(resourceGroupName string, zoneName string, relativeRecordSetName string, recordType dns.RecordType, ifMatch string) (result autorest.Response, err error)
	CreateOrUpdate(resourceGroupName string, zoneName string, relativeRecordSetName string, recordType dns.RecordType, parameters dns.RecordSet, ifMatch string, ifNoneMatch string) (result dns.RecordSet, err error)
}

// AzureProvider implements the DNS provider for Microsoft's Azure cloud platform.
type AzureProvider struct {
	domainFilter  DomainFilter
	zoneIDFilter  ZoneIDFilter
	dryRun        bool
	resourceGroup string
	zonesClient   ZonesClient
	recordsClient RecordsClient
}

// NewAzureProvider creates a new Azure provider.
//
// Returns the provider or an error if a provider could not be created.
func NewAzureProvider(configFile string, domainFilter DomainFilter, zoneIDFilter ZoneIDFilter, resourceGroup string, dryRun bool) (*AzureProvider, error) {
	contents, err := ioutil.ReadFile(configFile)
	if err != nil {
		return nil, fmt.Errorf("failed to read Azure config file '%s': %v", configFile, err)
	}
	cfg := config{}
	err = yaml.Unmarshal(contents, &cfg)
	if err != nil {
		return nil, fmt.Errorf("failed to read Azure config file '%s': %v", configFile, err)
	}

	// If a resource group was given, override what was present in the config file
	if resourceGroup != "" {
		cfg.ResourceGroup = resourceGroup
	}

	var environment azure.Environment
	if cfg.Cloud == "" {
		environment = azure.PublicCloud
	} else {
		environment, err = azure.EnvironmentFromName(cfg.Cloud)
		if err != nil {
			return nil, fmt.Errorf("invalid cloud value '%s': %v", cfg.Cloud, err)
		}
	}

	oauthConfig, err := adal.NewOAuthConfig(environment.ActiveDirectoryEndpoint, cfg.TenantID)
	if err != nil {
		return nil, fmt.Errorf("failed to retrieve OAuth config: %v", err)
	}

	token, err := adal.NewServicePrincipalToken(*oauthConfig, cfg.ClientID, cfg.ClientSecret, environment.ResourceManagerEndpoint)
	if err != nil {
		return nil, fmt.Errorf("failed to create service principal token: %v", err)
	}

	zonesClient := dns.NewZonesClient(cfg.SubscriptionID)
	zonesClient.Authorizer = autorest.NewBearerAuthorizer(token)
	recordsClient := dns.NewRecordSetsClient(cfg.SubscriptionID)
	recordsClient.Authorizer = autorest.NewBearerAuthorizer(token)

	provider := &AzureProvider{
		domainFilter:  domainFilter,
		zoneIDFilter:  zoneIDFilter,
		dryRun:        dryRun,
		resourceGroup: cfg.ResourceGroup,
		zonesClient:   zonesClient,
		recordsClient: recordsClient,
	}
	return provider, nil
}

// Records gets the current records.
//
// Returns the current records or an error if the operation failed.
func (p *AzureProvider) Records() (endpoints []*endpoint.Endpoint, _ error) {
	zones, err := p.zones()
	if err != nil {
		return nil, err
	}

	for _, zone := range zones {
		err := p.iterateRecords(*zone.Name, func(recordSet dns.RecordSet) bool {
			if recordSet.Name == nil || recordSet.Type == nil {
				log.Error("Skipping invalid record set with nil name or type.")
				return true
			}
			recordType := strings.TrimLeft(*recordSet.Type, "Microsoft.Network/dnszones/")
			if !supportedRecordType(recordType) {
				return true
			}
			name := formatAzureDNSName(*recordSet.Name, *zone.Name)
			target := extractAzureTarget(&recordSet)
			if target == "" {
				log.Errorf("Failed to extract target for '%s' with type '%s'.", name, recordType)
				return true
			}
			var ttl endpoint.TTL
			if recordSet.TTL != nil {
				ttl = endpoint.TTL(*recordSet.TTL)
			}

			ep := endpoint.NewEndpointWithTTL(name, target, recordType, endpoint.TTL(ttl))
			log.Debugf(
				"Found %s record for '%s' with target '%s'.",
				ep.RecordType,
				ep.DNSName,
				ep.Targets,
			)
			endpoints = append(endpoints, ep)
			return true
		})
		if err != nil {
			return nil, err
		}
	}
	return endpoints, nil
}

// ApplyChanges applies the given changes.
//
// Returns nil if the operation was successful or an error if the operation failed.
func (p *AzureProvider) ApplyChanges(changes *plan.Changes) error {
	zones, err := p.zones()
	if err != nil {
		return err
	}

	deleted, updated := p.mapChanges(zones, changes)
	p.deleteRecords(deleted)
	p.updateRecords(updated)
	return nil
}

func (p *AzureProvider) zones() ([]dns.Zone, error) {
	log.Debug("Retrieving Azure DNS zones.")

	var zones []dns.Zone
	list, err := p.zonesClient.ListByResourceGroup(p.resourceGroup, nil)
	if err != nil {
		return nil, err
	}

	for list.Value != nil && len(*list.Value) > 0 {
		for _, zone := range *list.Value {
			if zone.Name == nil {
				continue
			}

			if !p.domainFilter.Match(*zone.Name) {
				continue
			}

			if !p.zoneIDFilter.Match(*zone.ID) {
				continue
			}

			zones = append(zones, zone)
		}

		list, err = p.zonesClient.ListByResourceGroupNextResults(list)
		if err != nil {
			return nil, err
		}
	}
	log.Debugf("Found %d Azure DNS zone(s).", len(zones))
	return zones, nil
}

func (p *AzureProvider) iterateRecords(zoneName string, callback func(dns.RecordSet) bool) error {
	log.Debugf("Retrieving Azure DNS records for zone '%s'.", zoneName)

	list, err := p.recordsClient.ListByDNSZone(p.resourceGroup, zoneName, nil)
	if err != nil {
		return err
	}

	for list.Value != nil && len(*list.Value) > 0 {
		for _, recordSet := range *list.Value {
			if !callback(recordSet) {
				return nil
			}
		}

		list, err = p.recordsClient.ListByDNSZoneNextResults(list)
		if err != nil {
			return err
		}
	}
	return nil
}

type azureChangeMap map[string][]*endpoint.Endpoint

func (p *AzureProvider) mapChanges(zones []dns.Zone, changes *plan.Changes) (azureChangeMap, azureChangeMap) {
	ignored := map[string]bool{}
	deleted := azureChangeMap{}
	updated := azureChangeMap{}
	zoneNameIDMapper := zoneIDName{}
	for _, z := range zones {
		if z.Name != nil {
			zoneNameIDMapper.Add(*z.Name, *z.Name)
		}
	}
	mapChange := func(changeMap azureChangeMap, change *endpoint.Endpoint) {
		zone, _ := zoneNameIDMapper.FindZone(change.DNSName)
		if zone == "" {
			if _, ok := ignored[change.DNSName]; !ok {
				ignored[change.DNSName] = true
				log.Infof("Ignoring changes to '%s' because a suitable Azure DNS zone was not found.", change.DNSName)
			}
			return
		}
		// Ensure the record type is suitable
		changeMap[zone] = append(changeMap[zone], change)
	}

	for _, change := range changes.Delete {
		mapChange(deleted, change)
	}

	for _, change := range changes.UpdateOld {
		mapChange(deleted, change)
	}

	for _, change := range changes.Create {
		mapChange(updated, change)
	}

	for _, change := range changes.UpdateNew {
		mapChange(updated, change)
	}
	return deleted, updated
}

func (p *AzureProvider) deleteRecords(deleted azureChangeMap) {
	// Delete records first
	for zone, endpoints := range deleted {
		for _, endpoint := range endpoints {
			name := p.recordSetNameForZone(zone, endpoint)
			if p.dryRun {
				log.Infof("Would delete %s record named '%s' for Azure DNS zone '%s'.", endpoint.RecordType, name, zone)
			} else {
				log.Infof("Deleting %s record named '%s' for Azure DNS zone '%s'.", endpoint.RecordType, name, zone)
				if _, err := p.recordsClient.Delete(p.resourceGroup, zone, name, dns.RecordType(endpoint.RecordType), ""); err != nil {
					log.Errorf(
						"Failed to delete %s record named '%s' for Azure DNS zone '%s': %v",
						endpoint.RecordType,
						name,
						zone,
						err,
					)
				}
			}
		}
	}
}

func (p *AzureProvider) updateRecords(updated azureChangeMap) {
	for zone, endpoints := range updated {
		for _, endpoint := range endpoints {
			name := p.recordSetNameForZone(zone, endpoint)
			if p.dryRun {
				log.Infof(
					"Would update %s record named '%s' to '%s' for Azure DNS zone '%s'.",
					endpoint.RecordType,
					name,
					endpoint.Targets,
					zone,
				)
				continue
			}

			log.Infof(
				"Updating %s record named '%s' to '%s' for Azure DNS zone '%s'.",
				endpoint.RecordType,
				name,
				endpoint.Targets,
				zone,
			)

			recordSet, err := p.newRecordSet(endpoint)
			if err == nil {
				_, err = p.recordsClient.CreateOrUpdate(
					p.resourceGroup,
					zone,
					name,
					dns.RecordType(endpoint.RecordType),
					recordSet,
					"",
					"",
				)
			}
			if err != nil {
				log.Errorf(
					"Failed to update %s record named '%s' to '%s' for DNS zone '%s': %v",
					endpoint.RecordType,
					name,
					endpoint.Targets,
					zone,
					err,
				)
			}
		}
	}
}

func (p *AzureProvider) recordSetNameForZone(zone string, endpoint *endpoint.Endpoint) string {
	// Remove the zone from the record set
	name := endpoint.DNSName
	name = name[:len(name)-len(zone)]
	name = strings.TrimSuffix(name, ".")

	// For root, use @
	if name == "" {
		return "@"
	}
	return name
}

func (p *AzureProvider) newRecordSet(endpoint *endpoint.Endpoint) (dns.RecordSet, error) {
	var ttl int64 = azureRecordTTL
	if endpoint.RecordTTL.IsConfigured() {
		ttl = int64(endpoint.RecordTTL)
	}
	switch dns.RecordType(endpoint.RecordType) {
	case dns.A:
		return dns.RecordSet{
			RecordSetProperties: &dns.RecordSetProperties{
				TTL: to.Int64Ptr(ttl),
				ARecords: &[]dns.ARecord{
					{
						Ipv4Address: to.StringPtr(endpoint.Targets[0]),
					},
				},
			},
		}, nil
	case dns.CNAME:
		return dns.RecordSet{
			RecordSetProperties: &dns.RecordSetProperties{
				TTL: to.Int64Ptr(ttl),
				CnameRecord: &dns.CnameRecord{
					Cname: to.StringPtr(endpoint.Targets[0]),
				},
			},
		}, nil
	case dns.TXT:
		return dns.RecordSet{
			RecordSetProperties: &dns.RecordSetProperties{
				TTL: to.Int64Ptr(ttl),
				TxtRecords: &[]dns.TxtRecord{
					{
						Value: &[]string{
							endpoint.Targets[0],
						},
					},
				},
			},
		}, nil
	}
	return dns.RecordSet{}, fmt.Errorf("unsupported record type '%s'", endpoint.RecordType)
}

// Helper function (shared with test code)
func formatAzureDNSName(recordName, zoneName string) string {
	if recordName == "@" {
		return zoneName
	}
	return fmt.Sprintf("%s.%s", recordName, zoneName)
}

// Helper function (shared with text code)
func extractAzureTarget(recordSet *dns.RecordSet) string {
	properties := recordSet.RecordSetProperties
	if properties == nil {
		return ""
	}

	// Check for A records
	aRecords := properties.ARecords
	if aRecords != nil && len(*aRecords) > 0 && (*aRecords)[0].Ipv4Address != nil {
		return *(*aRecords)[0].Ipv4Address
	}

	// Check for CNAME records
	cnameRecord := properties.CnameRecord
	if cnameRecord != nil && cnameRecord.Cname != nil {
		return *cnameRecord.Cname
	}

	// Check for TXT records
	txtRecords := properties.TxtRecords
	if txtRecords != nil && len(*txtRecords) > 0 && (*txtRecords)[0].Value != nil {
		values := (*txtRecords)[0].Value
		if values != nil && len(*values) > 0 {
			return (*values)[0]
		}
	}
	return ""
}
<commit_msg>Add Foreign Cloud support to Azure Provider (#485)<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provider

import (
	"fmt"
	"io/ioutil"
	"strings"

	log "github.com/sirupsen/logrus"

	yaml "gopkg.in/yaml.v2"

	"github.com/Azure/azure-sdk-for-go/arm/dns"
	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/adal"
	"github.com/Azure/go-autorest/autorest/azure"
	"github.com/Azure/go-autorest/autorest/to"

	"github.com/kubernetes-incubator/external-dns/endpoint"
	"github.com/kubernetes-incubator/external-dns/plan"
)

const (
	azureRecordTTL = 300
)

type config struct {
	Cloud          string `json:"cloud" yaml:"cloud"`
	TenantID       string `json:"tenantId" yaml:"tenantId"`
	SubscriptionID string `json:"subscriptionId" yaml:"subscriptionId"`
	ResourceGroup  string `json:"resourceGroup" yaml:"resourceGroup"`
	Location       string `json:"location" yaml:"location"`
	ClientID       string `json:"aadClientId" yaml:"aadClientId"`
	ClientSecret   string `json:"aadClientSecret" yaml:"aadClientSecret"`
}

// ZonesClient is an interface of dns.ZoneClient that can be stubbed for testing.
type ZonesClient interface {
	ListByResourceGroup(resourceGroupName string, top *int32) (result dns.ZoneListResult, err error)
	ListByResourceGroupNextResults(lastResults dns.ZoneListResult) (result dns.ZoneListResult, err error)
}

// RecordsClient is an interface of dns.RecordClient that can be stubbed for testing.
type RecordsClient interface {
	ListByDNSZone(resourceGroupName string, zoneName string, top *int32) (result dns.RecordSetListResult, err error)
	ListByDNSZoneNextResults(list dns.RecordSetListResult) (result dns.RecordSetListResult, err error)
	Delete(resourceGroupName string, zoneName string, relativeRecordSetName string, recordType dns.RecordType, ifMatch string) (result autorest.Response, err error)
	CreateOrUpdate(resourceGroupName string, zoneName string, relativeRecordSetName string, recordType dns.RecordType, parameters dns.RecordSet, ifMatch string, ifNoneMatch string) (result dns.RecordSet, err error)
}

// AzureProvider implements the DNS provider for Microsoft's Azure cloud platform.
type AzureProvider struct {
	domainFilter  DomainFilter
	zoneIDFilter  ZoneIDFilter
	dryRun        bool
	resourceGroup string
	zonesClient   ZonesClient
	recordsClient RecordsClient
}

// NewAzureProvider creates a new Azure provider.
//
// Returns the provider or an error if a provider could not be created.
func NewAzureProvider(configFile string, domainFilter DomainFilter, zoneIDFilter ZoneIDFilter, resourceGroup string, dryRun bool) (*AzureProvider, error) {
	contents, err := ioutil.ReadFile(configFile)
	if err != nil {
		return nil, fmt.Errorf("failed to read Azure config file '%s': %v", configFile, err)
	}
	cfg := config{}
	err = yaml.Unmarshal(contents, &cfg)
	if err != nil {
		return nil, fmt.Errorf("failed to read Azure config file '%s': %v", configFile, err)
	}

	// If a resource group was given, override what was present in the config file
	if resourceGroup != "" {
		cfg.ResourceGroup = resourceGroup
	}

	var environment azure.Environment
	if cfg.Cloud == "" {
		environment = azure.PublicCloud
	} else {
		environment, err = azure.EnvironmentFromName(cfg.Cloud)
		if err != nil {
			return nil, fmt.Errorf("invalid cloud value '%s': %v", cfg.Cloud, err)
		}
	}

	oauthConfig, err := adal.NewOAuthConfig(environment.ActiveDirectoryEndpoint, cfg.TenantID)
	if err != nil {
		return nil, fmt.Errorf("failed to retrieve OAuth config: %v", err)
	}

	token, err := adal.NewServicePrincipalToken(*oauthConfig, cfg.ClientID, cfg.ClientSecret, environment.ResourceManagerEndpoint)
	if err != nil {
		return nil, fmt.Errorf("failed to create service principal token: %v", err)
	}

	zonesClient := dns.NewZonesClientWithBaseURI(environment.ResourceManagerEndpoint, cfg.SubscriptionID)
	zonesClient.Authorizer = autorest.NewBearerAuthorizer(token)
	recordsClient := dns.NewRecordSetsClientWithBaseURI(environment.ResourceManagerEndpoint, cfg.SubscriptionID)
	recordsClient.Authorizer = autorest.NewBearerAuthorizer(token)

	provider := &AzureProvider{
		domainFilter:  domainFilter,
		zoneIDFilter:  zoneIDFilter,
		dryRun:        dryRun,
		resourceGroup: cfg.ResourceGroup,
		zonesClient:   zonesClient,
		recordsClient: recordsClient,
	}
	return provider, nil
}

// Records gets the current records.
//
// Returns the current records or an error if the operation failed.
func (p *AzureProvider) Records() (endpoints []*endpoint.Endpoint, _ error) {
	zones, err := p.zones()
	if err != nil {
		return nil, err
	}

	for _, zone := range zones {
		err := p.iterateRecords(*zone.Name, func(recordSet dns.RecordSet) bool {
			if recordSet.Name == nil || recordSet.Type == nil {
				log.Error("Skipping invalid record set with nil name or type.")
				return true
			}
			recordType := strings.TrimLeft(*recordSet.Type, "Microsoft.Network/dnszones/")
			if !supportedRecordType(recordType) {
				return true
			}
			name := formatAzureDNSName(*recordSet.Name, *zone.Name)
			target := extractAzureTarget(&recordSet)
			if target == "" {
				log.Errorf("Failed to extract target for '%s' with type '%s'.", name, recordType)
				return true
			}
			var ttl endpoint.TTL
			if recordSet.TTL != nil {
				ttl = endpoint.TTL(*recordSet.TTL)
			}

			ep := endpoint.NewEndpointWithTTL(name, target, recordType, endpoint.TTL(ttl))
			log.Debugf(
				"Found %s record for '%s' with target '%s'.",
				ep.RecordType,
				ep.DNSName,
				ep.Targets,
			)
			endpoints = append(endpoints, ep)
			return true
		})
		if err != nil {
			return nil, err
		}
	}
	return endpoints, nil
}

// ApplyChanges applies the given changes.
//
// Returns nil if the operation was successful or an error if the operation failed.
func (p *AzureProvider) ApplyChanges(changes *plan.Changes) error {
	zones, err := p.zones()
	if err != nil {
		return err
	}

	deleted, updated := p.mapChanges(zones, changes)
	p.deleteRecords(deleted)
	p.updateRecords(updated)
	return nil
}

func (p *AzureProvider) zones() ([]dns.Zone, error) {
	log.Debug("Retrieving Azure DNS zones.")

	var zones []dns.Zone
	list, err := p.zonesClient.ListByResourceGroup(p.resourceGroup, nil)
	if err != nil {
		return nil, err
	}

	for list.Value != nil && len(*list.Value) > 0 {
		for _, zone := range *list.Value {
			if zone.Name == nil {
				continue
			}

			if !p.domainFilter.Match(*zone.Name) {
				continue
			}

			if !p.zoneIDFilter.Match(*zone.ID) {
				continue
			}

			zones = append(zones, zone)
		}

		list, err = p.zonesClient.ListByResourceGroupNextResults(list)
		if err != nil {
			return nil, err
		}
	}
	log.Debugf("Found %d Azure DNS zone(s).", len(zones))
	return zones, nil
}

func (p *AzureProvider) iterateRecords(zoneName string, callback func(dns.RecordSet) bool) error {
	log.Debugf("Retrieving Azure DNS records for zone '%s'.", zoneName)

	list, err := p.recordsClient.ListByDNSZone(p.resourceGroup, zoneName, nil)
	if err != nil {
		return err
	}

	for list.Value != nil && len(*list.Value) > 0 {
		for _, recordSet := range *list.Value {
			if !callback(recordSet) {
				return nil
			}
		}

		list, err = p.recordsClient.ListByDNSZoneNextResults(list)
		if err != nil {
			return err
		}
	}
	return nil
}

type azureChangeMap map[string][]*endpoint.Endpoint

func (p *AzureProvider) mapChanges(zones []dns.Zone, changes *plan.Changes) (azureChangeMap, azureChangeMap) {
	ignored := map[string]bool{}
	deleted := azureChangeMap{}
	updated := azureChangeMap{}
	zoneNameIDMapper := zoneIDName{}
	for _, z := range zones {
		if z.Name != nil {
			zoneNameIDMapper.Add(*z.Name, *z.Name)
		}
	}
	mapChange := func(changeMap azureChangeMap, change *endpoint.Endpoint) {
		zone, _ := zoneNameIDMapper.FindZone(change.DNSName)
		if zone == "" {
			if _, ok := ignored[change.DNSName]; !ok {
				ignored[change.DNSName] = true
				log.Infof("Ignoring changes to '%s' because a suitable Azure DNS zone was not found.", change.DNSName)
			}
			return
		}
		// Ensure the record type is suitable
		changeMap[zone] = append(changeMap[zone], change)
	}

	for _, change := range changes.Delete {
		mapChange(deleted, change)
	}

	for _, change := range changes.UpdateOld {
		mapChange(deleted, change)
	}

	for _, change := range changes.Create {
		mapChange(updated, change)
	}

	for _, change := range changes.UpdateNew {
		mapChange(updated, change)
	}
	return deleted, updated
}

func (p *AzureProvider) deleteRecords(deleted azureChangeMap) {
	// Delete records first
	for zone, endpoints := range deleted {
		for _, endpoint := range endpoints {
			name := p.recordSetNameForZone(zone, endpoint)
			if p.dryRun {
				log.Infof("Would delete %s record named '%s' for Azure DNS zone '%s'.", endpoint.RecordType, name, zone)
			} else {
				log.Infof("Deleting %s record named '%s' for Azure DNS zone '%s'.", endpoint.RecordType, name, zone)
				if _, err := p.recordsClient.Delete(p.resourceGroup, zone, name, dns.RecordType(endpoint.RecordType), ""); err != nil {
					log.Errorf(
						"Failed to delete %s record named '%s' for Azure DNS zone '%s': %v",
						endpoint.RecordType,
						name,
						zone,
						err,
					)
				}
			}
		}
	}
}

func (p *AzureProvider) updateRecords(updated azureChangeMap) {
	for zone, endpoints := range updated {
		for _, endpoint := range endpoints {
			name := p.recordSetNameForZone(zone, endpoint)
			if p.dryRun {
				log.Infof(
					"Would update %s record named '%s' to '%s' for Azure DNS zone '%s'.",
					endpoint.RecordType,
					name,
					endpoint.Targets,
					zone,
				)
				continue
			}

			log.Infof(
				"Updating %s record named '%s' to '%s' for Azure DNS zone '%s'.",
				endpoint.RecordType,
				name,
				endpoint.Targets,
				zone,
			)

			recordSet, err := p.newRecordSet(endpoint)
			if err == nil {
				_, err = p.recordsClient.CreateOrUpdate(
					p.resourceGroup,
					zone,
					name,
					dns.RecordType(endpoint.RecordType),
					recordSet,
					"",
					"",
				)
			}
			if err != nil {
				log.Errorf(
					"Failed to update %s record named '%s' to '%s' for DNS zone '%s': %v",
					endpoint.RecordType,
					name,
					endpoint.Targets,
					zone,
					err,
				)
			}
		}
	}
}

func (p *AzureProvider) recordSetNameForZone(zone string, endpoint *endpoint.Endpoint) string {
	// Remove the zone from the record set
	name := endpoint.DNSName
	name = name[:len(name)-len(zone)]
	name = strings.TrimSuffix(name, ".")

	// For root, use @
	if name == "" {
		return "@"
	}
	return name
}

func (p *AzureProvider) newRecordSet(endpoint *endpoint.Endpoint) (dns.RecordSet, error) {
	var ttl int64 = azureRecordTTL
	if endpoint.RecordTTL.IsConfigured() {
		ttl = int64(endpoint.RecordTTL)
	}
	switch dns.RecordType(endpoint.RecordType) {
	case dns.A:
		return dns.RecordSet{
			RecordSetProperties: &dns.RecordSetProperties{
				TTL: to.Int64Ptr(ttl),
				ARecords: &[]dns.ARecord{
					{
						Ipv4Address: to.StringPtr(endpoint.Targets[0]),
					},
				},
			},
		}, nil
	case dns.CNAME:
		return dns.RecordSet{
			RecordSetProperties: &dns.RecordSetProperties{
				TTL: to.Int64Ptr(ttl),
				CnameRecord: &dns.CnameRecord{
					Cname: to.StringPtr(endpoint.Targets[0]),
				},
			},
		}, nil
	case dns.TXT:
		return dns.RecordSet{
			RecordSetProperties: &dns.RecordSetProperties{
				TTL: to.Int64Ptr(ttl),
				TxtRecords: &[]dns.TxtRecord{
					{
						Value: &[]string{
							endpoint.Targets[0],
						},
					},
				},
			},
		}, nil
	}
	return dns.RecordSet{}, fmt.Errorf("unsupported record type '%s'", endpoint.RecordType)
}

// Helper function (shared with test code)
func formatAzureDNSName(recordName, zoneName string) string {
	if recordName == "@" {
		return zoneName
	}
	return fmt.Sprintf("%s.%s", recordName, zoneName)
}

// Helper function (shared with text code)
func extractAzureTarget(recordSet *dns.RecordSet) string {
	properties := recordSet.RecordSetProperties
	if properties == nil {
		return ""
	}

	// Check for A records
	aRecords := properties.ARecords
	if aRecords != nil && len(*aRecords) > 0 && (*aRecords)[0].Ipv4Address != nil {
		return *(*aRecords)[0].Ipv4Address
	}

	// Check for CNAME records
	cnameRecord := properties.CnameRecord
	if cnameRecord != nil && cnameRecord.Cname != nil {
		return *cnameRecord.Cname
	}

	// Check for TXT records
	txtRecords := properties.TxtRecords
	if txtRecords != nil && len(*txtRecords) > 0 && (*txtRecords)[0].Value != nil {
		values := (*txtRecords)[0].Value
		if values != nil && len(*values) > 0 {
			return (*values)[0]
		}
	}
	return ""
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "net/http"
import "net/http/httputil"
import "net/url"
import "time"
import "log"
import "encoding/base64"
import "flag"
import "os"
import "github.com/elazarl/goproxy"

func Log(handler http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
        handler.ServeHTTP(w, r)
    })
}

var record = flag.Bool("record", false, "run a http proxy that will record requests")
var playback = flag.String("playback", "", "playback a recoreded session")
var target = flag.String("target", "", "the target host to record")

func getTimeMilis() int64 {
    return time.Now().UnixNano()/1e6
}

func runProxy() {
    proxy := goproxy.NewProxyHttpServer()
    proxy.Verbose = true
    client := http.Client{}
    fo, err := os.Create(".requests")
    if err != nil { panic(err) }
    // close fo on exit and check for its returned error
    defer func() {
        if err := fo.Close(); err != nil {
            panic(err)
        }
    }()

    startTimeMilis := getTimeMilis()

    proxy.OnRequest().DoFunc(func(r *http.Request,ctx *goproxy.ProxyCtx)(*http.Request,*http.Response) {
        fmt.Println(r.Host)
        request_url, err := url.Parse(r.RequestURI)
        if err != nil {
            panic(err)
        }

        log.Print("Rewriting request on ", r.Host, " to ", *target)

        r.RequestURI = ""
        r.URL        = request_url
        r.URL.Scheme = "https"
        r.Host       = *target
        r.URL.Host   = *target
        r.URL.User   = nil

        currentTimeMilis := getTimeMilis()
        fo.WriteString(fmt.Sprint(currentTimeMilis-startTimeMilis))
        fo.WriteString("\nLOLPONIES\n")

        var requestBytes []byte
        requestBytes, err = httputil.DumpRequest(r, true)
        fo.WriteString(base64.StdEncoding.EncodeToString(requestBytes))

        fo.WriteString("\nLOLPONIES\n")
        fo.Sync()

        resp, err := client.Do(r)
        if err != nil {
            panic(err)
        }

        return nil, resp
    })
    http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static/"))))
    fmt.Println("Serving on http://localhost:8080")
    log.Fatal(http.ListenAndServe(":8080", proxy))
}

func main() {
    flag.Parse()
    if (*playback == "" && *record == false) {
        fmt.Println("One of --playback <filename> or --record is required")
        flag.Usage()
        os.Exit(1)
    } else if (*record && *target == "") {
        fmt.Println("--target <url> is required with --record")
        flag.Usage()
        os.Exit(1)
    } else if (*record && *target != "") {
        runProxy()
    }
}
<commit_msg>concurrency<commit_after>package main

import "fmt"
import "net/http"
import "net/http/httputil"
import "net/url"
import "time"
import "log"
import "encoding/base64"
import "flag"
import "os"
import "github.com/elazarl/goproxy"

func Log(handler http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
        handler.ServeHTTP(w, r)
    })
}

var record = flag.Bool("record", false, "run a http proxy that will record requests")
var playback = flag.String("playback", "", "playback a recoreded session")
var target = flag.String("target", "", "the target host to record")

func getTimeMilis() int64 {
    return time.Now().UnixNano()/1e6
}

func runProxy() {
    proxy := goproxy.NewProxyHttpServer()
    proxy.Verbose = true
    client := http.Client{}
    fo, err := os.Create(".requests")
    if err != nil { panic(err) }
    // close fo on exit and check for its returned error
    defer func() {
        if err := fo.Close(); err != nil {
            panic(err)
        }
    }()

    startTimeMilis := getTimeMilis()

    var c chan string = make(chan string)
    go func() {
        for {
            line := <- c
            fo.WriteString(line)
            fo.Sync()
        }
    }()

    proxy.OnRequest().DoFunc(func(r *http.Request,ctx *goproxy.ProxyCtx)(*http.Request,*http.Response) {
        fmt.Println(r.Host)
        request_url, err := url.Parse(r.RequestURI)
        if err != nil {
            panic(err)
        }

        log.Print("Rewriting request on ", r.Host, " to ", *target)

        r.RequestURI = ""
        r.URL        = request_url
        r.URL.Scheme = "https"
        r.Host       = *target
        r.URL.Host   = *target
        r.URL.User   = nil

        currentTimeMilis := getTimeMilis()
        var requestBytes []byte
        requestBytes, err = httputil.DumpRequest(r, true)

        c <- fmt.Sprint(currentTimeMilis-startTimeMilis)
        c <- "\nLOLPONIES\n"
        c <- base64.StdEncoding.EncodeToString(requestBytes)
        c <- "\nLOLPONIES\n"

        resp, err := client.Do(r)
        if err != nil {
            panic(err)
        }

        return nil, resp
    })
    http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static/"))))
    fmt.Println("Serving on http://localhost:8080")
    log.Fatal(http.ListenAndServe(":8080", proxy))
}

func runPlayback() {
}

func main() {
    flag.Parse()
    if (*playback == "" && *record == false) {
        fmt.Println("One of --playback <filename> or --record is required")
        flag.Usage()
        os.Exit(1)
    } else if (*record && *target == "") {
        fmt.Println("--target <url> is required with --record")
        flag.Usage()
        os.Exit(1)
    } else if (*record && *target != "") {
        runProxy()
    }
}
<|endoftext|>


<commit_before><commit_msg>updating routing to remove /entry at start of each request<commit_after><|endoftext|>


<commit_before>package pg_test

import (
	"net"
	"sync"
	"time"

	. "launchpad.net/gocheck"

	"github.com/go-pg/pg"
)

var _ = Suite(&PoolTest{})

type PoolTest struct {
	db *pg.DB
}

func (t *PoolTest) SetUpTest(c *C) {
	t.db = pg.Connect(&pg.Options{
		User:     "postgres",
		Database: "test",
		PoolSize: 10,

		DialTimeout:  3 * time.Second,
		ReadTimeout:  3 * time.Second,
		WriteTimeout: 3 * time.Second,
	})
}

func (t *PoolTest) TearDownTest(c *C) {
	t.db.Close()
}

func (t *PoolTest) TestPoolReusesConnection(c *C) {
	for i := 0; i < 100; i++ {
		_, err := t.db.Exec("SELECT 1")
		c.Assert(err, IsNil)
	}

	c.Assert(t.db.Pool().Size(), Equals, 1)
	c.Assert(t.db.Pool().Len(), Equals, 1)
}

func (t *PoolTest) TestPoolMaxSize(c *C) {
	N := 1000

	wg := &sync.WaitGroup{}
	wg.Add(N)
	for i := 0; i < 1000; i++ {
		go func() {
			_, err := t.db.Exec("SELECT 1")
			c.Assert(err, IsNil)
			wg.Done()
		}()
	}
	wg.Wait()

	c.Assert(t.db.Pool().Size(), Equals, 10)
	c.Assert(t.db.Pool().Len(), Equals, 10)
}

func (t *PoolTest) TestTimeoutAndCancelRequest(c *C) {
	_, err := t.db.Exec("SELECT pg_sleep(60)")
	c.Assert(err.(net.Error).Timeout(), Equals, true)

	c.Assert(t.db.Pool().Size(), Equals, 0)
	c.Assert(t.db.Pool().Len(), Equals, 0)

	// Give PostgreSQL some time to cancel request.
	time.Sleep(time.Second)

	// Unreliable check that previous query was cancelled.
	var count int
	_, err = t.db.QueryOne(pg.LoadInto(&count), "SELECT COUNT(*) FROM pg_stat_activity")
	c.Assert(err, IsNil)
	c.Assert(count, Equals, 1)
}

func (t *PoolTest) TestCloseClosesAllConnections(c *C) {
	ln, err := t.db.Listen("test_channel")
	c.Assert(err, IsNil)

	started := make(chan struct{})
	done := make(chan struct{})
	go func() {
		close(started)
		_, _, err := ln.Receive()
		c.Assert(err, Not(IsNil))
		c.Assert(err.Error(), Equals, "read tcp 127.0.0.1:5432: use of closed network connection")
		close(done)
	}()

	<-started
	c.Assert(t.db.Close(), IsNil)
	<-done
}

func (t *PoolTest) TestClosedDB(c *C) {
	c.Assert(t.db.Close(), IsNil)

	err := t.db.Close()
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: database is closed")

	_, err = t.db.Exec("SELECT 1")
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: database is closed")
}

func (t *PoolTest) TestClosedListener(c *C) {
	ln, err := t.db.Listen("test_channel")
	c.Assert(err, IsNil)

	c.Assert(ln.Close(), IsNil)

	err = ln.Close()
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: listener is closed")

	_, _, err = ln.Receive()
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: listener is closed")
}

func (t *PoolTest) TestClosedTx(c *C) {
	tx, err := t.db.Begin()
	c.Assert(err, IsNil)

	c.Assert(tx.Rollback(), IsNil)

	err = tx.Rollback()
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: transaction has already been committed or rolled back")

	_, err = tx.Exec("SELECT 1")
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: transaction has already been committed or rolled back")
}

func (t *PoolTest) TestClosedStmt(c *C) {
	stmt, err := t.db.Prepare("SELECT $1::int")
	c.Assert(err, IsNil)

	c.Assert(stmt.Close(), IsNil)

	err = stmt.Close()
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: statement is closed")

	_, err = stmt.Exec(1)
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: statement is closed")
}
<commit_msg>fixing import in unittests<commit_after>package pg_test

import (
	"net"
	"sync"
	"time"

	. "launchpad.net/gocheck"

	"gopkg.in/pg.v1"
)

var _ = Suite(&PoolTest{})

type PoolTest struct {
	db *pg.DB
}

func (t *PoolTest) SetUpTest(c *C) {
	t.db = pg.Connect(&pg.Options{
		User:     "postgres",
		Database: "test",
		PoolSize: 10,

		DialTimeout:  3 * time.Second,
		ReadTimeout:  3 * time.Second,
		WriteTimeout: 3 * time.Second,
	})
}

func (t *PoolTest) TearDownTest(c *C) {
	t.db.Close()
}

func (t *PoolTest) TestPoolReusesConnection(c *C) {
	for i := 0; i < 100; i++ {
		_, err := t.db.Exec("SELECT 1")
		c.Assert(err, IsNil)
	}

	c.Assert(t.db.Pool().Size(), Equals, 1)
	c.Assert(t.db.Pool().Len(), Equals, 1)
}

func (t *PoolTest) TestPoolMaxSize(c *C) {
	N := 1000

	wg := &sync.WaitGroup{}
	wg.Add(N)
	for i := 0; i < 1000; i++ {
		go func() {
			_, err := t.db.Exec("SELECT 1")
			c.Assert(err, IsNil)
			wg.Done()
		}()
	}
	wg.Wait()

	c.Assert(t.db.Pool().Size(), Equals, 10)
	c.Assert(t.db.Pool().Len(), Equals, 10)
}

func (t *PoolTest) TestTimeoutAndCancelRequest(c *C) {
	_, err := t.db.Exec("SELECT pg_sleep(60)")
	c.Assert(err.(net.Error).Timeout(), Equals, true)

	c.Assert(t.db.Pool().Size(), Equals, 0)
	c.Assert(t.db.Pool().Len(), Equals, 0)

	// Give PostgreSQL some time to cancel request.
	time.Sleep(time.Second)

	// Unreliable check that previous query was cancelled.
	var count int
	_, err = t.db.QueryOne(pg.LoadInto(&count), "SELECT COUNT(*) FROM pg_stat_activity")
	c.Assert(err, IsNil)
	c.Assert(count, Equals, 1)
}

func (t *PoolTest) TestCloseClosesAllConnections(c *C) {
	ln, err := t.db.Listen("test_channel")
	c.Assert(err, IsNil)

	started := make(chan struct{})
	done := make(chan struct{})
	go func() {
		close(started)
		_, _, err := ln.Receive()
		c.Assert(err, Not(IsNil))
		c.Assert(err.Error(), Equals, "read tcp 127.0.0.1:5432: use of closed network connection")
		close(done)
	}()

	<-started
	c.Assert(t.db.Close(), IsNil)
	<-done
}

func (t *PoolTest) TestClosedDB(c *C) {
	c.Assert(t.db.Close(), IsNil)

	err := t.db.Close()
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: database is closed")

	_, err = t.db.Exec("SELECT 1")
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: database is closed")
}

func (t *PoolTest) TestClosedListener(c *C) {
	ln, err := t.db.Listen("test_channel")
	c.Assert(err, IsNil)

	c.Assert(ln.Close(), IsNil)

	err = ln.Close()
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: listener is closed")

	_, _, err = ln.Receive()
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: listener is closed")
}

func (t *PoolTest) TestClosedTx(c *C) {
	tx, err := t.db.Begin()
	c.Assert(err, IsNil)

	c.Assert(tx.Rollback(), IsNil)

	err = tx.Rollback()
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: transaction has already been committed or rolled back")

	_, err = tx.Exec("SELECT 1")
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: transaction has already been committed or rolled back")
}

func (t *PoolTest) TestClosedStmt(c *C) {
	stmt, err := t.db.Prepare("SELECT $1::int")
	c.Assert(err, IsNil)

	c.Assert(stmt.Close(), IsNil)

	err = stmt.Close()
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: statement is closed")

	_, err = stmt.Exec(1)
	c.Assert(err, Not(IsNil))
	c.Assert(err.Error(), Equals, "pg: statement is closed")
}
<|endoftext|>


<commit_before>package dynamodb

import (
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/gruntwork-io/terragrunt/locks"
	"time"
	"github.com/gruntwork-io/terragrunt/util"
	"github.com/aws/aws-sdk-go/aws"
	"fmt"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
)

// Create a DynamoDB key for the given item id
func createKeyFromItemId(itemId string) map[string]*dynamodb.AttributeValue {
	return map[string]*dynamodb.AttributeValue {
		ATTR_STATE_FILE_ID: &dynamodb.AttributeValue{S: aws.String(itemId)},
	}
}

// Fetch the metadata for the given item from DynamoDB and display it to stdout. This metadata will contain info about
// who currently has the lock.
func displayLockMetadata(itemId string, tableName string, client *dynamodb.DynamoDB) {
	lockMetadata, err := getLockMetadata(itemId, tableName, client)
	if err != nil {
		util.Logger.Printf("Someone already has a lock on state file %s in table %s in DynamoDB! However, failed to fetch metadata for the lock (perhaps the lock has since been released?): %s", itemId, tableName, err.Error())
	} else {
		util.Logger.Printf("Someone already has a lock on state file %s! %s@%s acquired the lock on %s.", itemId, lockMetadata.Username, lockMetadata.IpAddress, lockMetadata.DateCreated.String())
	}
}

// Fetch the lock metadata for the given item from DynamoDB. This metadata will contain info about who currently has
// the lock.
func getLockMetadata(itemId string, tableName string, client *dynamodb.DynamoDB) (*locks.LockMetadata, error) {
	output, err := client.GetItem(&dynamodb.GetItemInput{
		Key: createKeyFromItemId(itemId),
		ConsistentRead: aws.Bool(true),
		TableName: aws.String(tableName),
	})

	if err != nil {
		return nil, errors.WithStackTrace(err)
	}

	return toLockMetadata(itemId, output.Item)
}

// Convert the AttributeValue map returned by DynamoDB into a LockMetadata struct
func toLockMetadata(itemId string, item map[string]*dynamodb.AttributeValue) (*locks.LockMetadata, error) {
	username, err := getAttribute(item, ATTR_USERNAME)
	if err != nil {
		return nil, err
	}

	ipAddress, err := getAttribute(item, ATTR_IP)
	if err != nil {
		return nil, err
	}

	dateCreatedStr, err := getAttribute(item, ATTR_CREATION_DATE)
	if err != nil {
		return nil, err
	}

	dateCreated, err := time.Parse(locks.DEFAULT_TIME_FORMAT, dateCreatedStr)
	if err != nil {
		return nil, errors.WithStackTrace(InvalidDateFormat{Date: dateCreatedStr, UnderlyingErr: err})
	}

	return &locks.LockMetadata{
		StateFileId: itemId,
		Username: username,
		IpAddress: ipAddress,
		DateCreated: dateCreated,
	}, nil
}

// Return the value for the given attribute from the given attribute map, or return an error if that attribute is
// missing from the map
func getAttribute(item map[string]*dynamodb.AttributeValue, attribute string) (string, error) {
	value, exists := item[attribute]
	if !exists {
		return "", errors.WithStackTrace(AttributeMissing{AttributeName: attribute})
	}

	return *value.S, nil
}

// Create a DynamoDB item for the given item id. This item represents a lock and will include metadata about the
// current user, who is trying to acquire the lock.
func createItemAttributes(itemId string, client *dynamodb.DynamoDB) (map[string]*dynamodb.AttributeValue, error) {
	callerIdentity, err := getCallerIdentity(client)
	if err != nil {
		return nil, err
	}

	lockMetadata, err := locks.CreateLockMetadata(itemId, callerIdentity)
	if err != nil {
		return nil, err
	}

	return map[string]*dynamodb.AttributeValue{
		ATTR_STATE_FILE_ID: &dynamodb.AttributeValue{S: aws.String(itemId)},
		ATTR_USERNAME: &dynamodb.AttributeValue{S: aws.String(lockMetadata.Username)},
		ATTR_IP: &dynamodb.AttributeValue{S: aws.String(lockMetadata.IpAddress)},
		ATTR_CREATION_DATE: &dynamodb.AttributeValue{S: aws.String(lockMetadata.DateCreated.String())},
	}, nil
}

// Return the UserID
func getCallerIdentity(client *dynamodb.DynamoDB) (string, error) {
	stsconn := sts.New(session.New(), &client.Config)
	output, err := stsconn.GetCallerIdentity(&sts.GetCallerIdentityInput{})
	if err != nil {
		return "", errors.WithStackTrace(err)
	}

	return *output.UserId, nil
}

type AttributeMissing struct {
	AttributeName string
}

func (err AttributeMissing) Error() string {
	return fmt.Sprintf("Could not find attribute %s", err.AttributeName)
}

type InvalidDateFormat struct {
	Date 		string
	UnderlyingErr 	error
}

func (err InvalidDateFormat) Error() string {
	return fmt.Sprintf("Unable to parse date %s: %s", err.Date, err.UnderlyingErr.Error())
}<commit_msg>saving username instead of userid into db items<commit_after>package dynamodb

import (
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/gruntwork-io/terragrunt/locks"
	"time"
	"github.com/gruntwork-io/terragrunt/util"
	"github.com/aws/aws-sdk-go/aws"
	"fmt"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
)

// Create a DynamoDB key for the given item id
func createKeyFromItemId(itemId string) map[string]*dynamodb.AttributeValue {
	return map[string]*dynamodb.AttributeValue {
		ATTR_STATE_FILE_ID: &dynamodb.AttributeValue{S: aws.String(itemId)},
	}
}

// Fetch the metadata for the given item from DynamoDB and display it to stdout. This metadata will contain info about
// who currently has the lock.
func displayLockMetadata(itemId string, tableName string, client *dynamodb.DynamoDB) {
	lockMetadata, err := getLockMetadata(itemId, tableName, client)
	if err != nil {
		util.Logger.Printf("Someone already has a lock on state file %s in table %s in DynamoDB! However, failed to fetch metadata for the lock (perhaps the lock has since been released?): %s", itemId, tableName, err.Error())
	} else {
		util.Logger.Printf("Someone already has a lock on state file %s! %s@%s acquired the lock on %s.", itemId, lockMetadata.Username, lockMetadata.IpAddress, lockMetadata.DateCreated.String())
	}
}

// Fetch the lock metadata for the given item from DynamoDB. This metadata will contain info about who currently has
// the lock.
func getLockMetadata(itemId string, tableName string, client *dynamodb.DynamoDB) (*locks.LockMetadata, error) {
	output, err := client.GetItem(&dynamodb.GetItemInput{
		Key: createKeyFromItemId(itemId),
		ConsistentRead: aws.Bool(true),
		TableName: aws.String(tableName),
	})

	if err != nil {
		return nil, errors.WithStackTrace(err)
	}

	return toLockMetadata(itemId, output.Item)
}

// Convert the AttributeValue map returned by DynamoDB into a LockMetadata struct
func toLockMetadata(itemId string, item map[string]*dynamodb.AttributeValue) (*locks.LockMetadata, error) {
	username, err := getAttribute(item, ATTR_USERNAME)
	if err != nil {
		return nil, err
	}

	ipAddress, err := getAttribute(item, ATTR_IP)
	if err != nil {
		return nil, err
	}

	dateCreatedStr, err := getAttribute(item, ATTR_CREATION_DATE)
	if err != nil {
		return nil, err
	}

	dateCreated, err := time.Parse(locks.DEFAULT_TIME_FORMAT, dateCreatedStr)
	if err != nil {
		return nil, errors.WithStackTrace(InvalidDateFormat{Date: dateCreatedStr, UnderlyingErr: err})
	}

	return &locks.LockMetadata{
		StateFileId: itemId,
		Username: username,
		IpAddress: ipAddress,
		DateCreated: dateCreated,
	}, nil
}

// Return the value for the given attribute from the given attribute map, or return an error if that attribute is
// missing from the map
func getAttribute(item map[string]*dynamodb.AttributeValue, attribute string) (string, error) {
	value, exists := item[attribute]
	if !exists {
		return "", errors.WithStackTrace(AttributeMissing{AttributeName: attribute})
	}

	return *value.S, nil
}

// Create a DynamoDB item for the given item id. This item represents a lock and will include metadata about the
// current user, who is trying to acquire the lock.
func createItemAttributes(itemId string, client *dynamodb.DynamoDB) (map[string]*dynamodb.AttributeValue, error) {
	callerIdentity, err := getCallerIdentity(client)
	if err != nil {
		return nil, err
	}

	lockMetadata, err := locks.CreateLockMetadata(itemId, callerIdentity)
	if err != nil {
		return nil, err
	}

	return map[string]*dynamodb.AttributeValue{
		ATTR_STATE_FILE_ID: &dynamodb.AttributeValue{S: aws.String(itemId)},
		ATTR_USERNAME: &dynamodb.AttributeValue{S: aws.String(lockMetadata.Username)},
		ATTR_IP: &dynamodb.AttributeValue{S: aws.String(lockMetadata.IpAddress)},
		ATTR_CREATION_DATE: &dynamodb.AttributeValue{S: aws.String(lockMetadata.DateCreated.String())},
	}, nil
}

// Return the UserArn
func getCallerIdentity(client *dynamodb.DynamoDB) (string, error) {
	stsconn := sts.New(session.New(), &client.Config)
	output, err := stsconn.GetCallerIdentity(&sts.GetCallerIdentityInput{})
	if err != nil {
		return "", errors.WithStackTrace(err)
	}

	return *output.Arn, nil
}

type AttributeMissing struct {
	AttributeName string
}

func (err AttributeMissing) Error() string {
	return fmt.Sprintf("Could not find attribute %s", err.AttributeName)
}

type InvalidDateFormat struct {
	Date 		string
	UnderlyingErr 	error
}

func (err InvalidDateFormat) Error() string {
	return fmt.Sprintf("Unable to parse date %s: %s", err.Date, err.UnderlyingErr.Error())
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/ip"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/logger"
)

// A variation of the standard tls.Listener that supports atomically swapping
// the underlying TLS configuration. Requests served before the swap will
// continue using the old configuration.
type networkListener struct {
	net.Listener
	mu     sync.RWMutex
	config *tls.Config
}

func networkTLSListener(inner net.Listener, config *tls.Config) *networkListener {
	listener := &networkListener{
		Listener: inner,
		config:   config,
	}

	return listener
}

// Accept waits for and returns the next incoming TLS connection then use the
// current TLS configuration to handle it.
func (l *networkListener) Accept() (net.Conn, error) {
	var c net.Conn
	var err error

	// Accept() is non-blocking in go < 1.12 hence the loop and error check.
	for {
		c, err = l.Listener.Accept()
		if err == nil {
			break
		}

		if err.(net.Error).Timeout() {
			time.Sleep(100 * time.Millisecond)
			continue
		}

		return nil, err
	}

	l.mu.RLock()
	defer l.mu.RUnlock()

	return tls.Server(c, l.config), nil
}

func serverTLSConfig() (*tls.Config, error) {
	certInfo, err := shared.KeyPairAndCA(".", "agent", shared.CertServer, false)
	if err != nil {
		return nil, err
	}

	tlsConfig := util.ServerTLSConfig(certInfo)
	return tlsConfig, nil
}

// reconfigureNetworkInterfaces checks for the existence of files under NICConfigDir in the config share.
// Each file is named <device>.json and contains the Device Name, NIC Name, MTU and MAC address.
func reconfigureNetworkInterfaces() {
	nicDirEntries, err := ioutil.ReadDir(deviceConfig.NICConfigDir)
	if err != nil {
		// Abort if configuration folder does not exist (nothing to do), otherwise log and return.
		if os.IsNotExist(err) {
			return
		}

		logger.Error("Could not read network interface configuration directory", logger.Ctx{"err": err})
		return
	}

	// nicData is a map of MAC address to NICConfig.
	nicData := make(map[string]deviceConfig.NICConfig, len(nicDirEntries))

	for _, f := range nicDirEntries {
		nicBytes, err := ioutil.ReadFile(filepath.Join(deviceConfig.NICConfigDir, f.Name()))
		if err != nil {
			logger.Error("Could not read network interface configuration file", logger.Ctx{"err": err})
		}

		var conf deviceConfig.NICConfig
		err = json.Unmarshal(nicBytes, &conf)
		if err != nil {
			logger.Error("Could not parse network interface configuration file", logger.Ctx{"err": err})
			return
		}

		if conf.MACAddress != "" {
			nicData[conf.MACAddress] = conf
		}
	}

	// configureNIC applies any config specified for the interface based on its current MAC address.
	configureNIC := func(currentNIC net.Interface) error {
		revert := revert.New()
		defer revert.Fail()

		// Look for a NIC config entry for this interface based on its MAC address.
		nic, ok := nicData[currentNIC.HardwareAddr.String()]
		if !ok {
			return nil
		}

		var changeName, changeMTU bool
		if nic.NICName != "" && currentNIC.Name != nic.NICName {
			changeName = true
		}

		if nic.MTU > 0 && currentNIC.MTU != int(nic.MTU) {
			changeMTU = true
		}

		if !changeName && !changeMTU {
			return nil // Nothing to do.
		}

		link := ip.Link{
			Name: currentNIC.Name,
			MTU:  fmt.Sprintf("%d", currentNIC.MTU),
		}

		err := link.SetDown()
		if err != nil {
			return err
		}

		revert.Add(func() {
			_ = link.SetUp()
		})

		// Apply the name from the NIC config if needed.
		if changeName {
			err = link.SetName(nic.NICName)
			if err != nil {
				return err
			}

			revert.Add(func() {
				err := link.SetName(currentNIC.Name)
				if err != nil {
					return
				}

				link.Name = currentNIC.Name
			})

			link.Name = nic.NICName
		}

		// Apply the MTU from the NIC config if needed.
		if changeMTU {
			newMTU := fmt.Sprintf("%d", nic.MTU)
			err = link.SetMTU(newMTU)
			if err != nil {
				return err
			}

			revert.Add(func() {
				currentMTU := fmt.Sprintf("%d", currentNIC.MTU)
				err := link.SetMTU(currentMTU)
				if err != nil {
					return
				}

				link.MTU = currentMTU
			})

			link.MTU = newMTU
		}

		err = link.SetUp()
		if err != nil {
			return err
		}

		revert.Success()
		return nil
	}

	ifaces, err := net.Interfaces()
	if err != nil {
		logger.Error("Unable to read network interfaces", logger.Ctx{"err": err})
	}

	for _, iface := range ifaces {
		err = configureNIC(iface)
		if err != nil {
			logger.Error("Unable to reconfigure network interface", logger.Ctx{"interface": iface.Name, "err": err})
		}
	}
}
<commit_msg>lxd-agent: Remove legacy code<commit_after>package main

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"sync"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/ip"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/logger"
)

// A variation of the standard tls.Listener that supports atomically swapping
// the underlying TLS configuration. Requests served before the swap will
// continue using the old configuration.
type networkListener struct {
	net.Listener
	mu     sync.RWMutex
	config *tls.Config
}

func networkTLSListener(inner net.Listener, config *tls.Config) *networkListener {
	listener := &networkListener{
		Listener: inner,
		config:   config,
	}

	return listener
}

// Accept waits for and returns the next incoming TLS connection then use the
// current TLS configuration to handle it.
func (l *networkListener) Accept() (net.Conn, error) {
	c, err := l.Listener.Accept()
	if err != nil {
		return nil, err
	}

	l.mu.RLock()
	defer l.mu.RUnlock()

	return tls.Server(c, l.config), nil
}

func serverTLSConfig() (*tls.Config, error) {
	certInfo, err := shared.KeyPairAndCA(".", "agent", shared.CertServer, false)
	if err != nil {
		return nil, err
	}

	tlsConfig := util.ServerTLSConfig(certInfo)
	return tlsConfig, nil
}

// reconfigureNetworkInterfaces checks for the existence of files under NICConfigDir in the config share.
// Each file is named <device>.json and contains the Device Name, NIC Name, MTU and MAC address.
func reconfigureNetworkInterfaces() {
	nicDirEntries, err := ioutil.ReadDir(deviceConfig.NICConfigDir)
	if err != nil {
		// Abort if configuration folder does not exist (nothing to do), otherwise log and return.
		if os.IsNotExist(err) {
			return
		}

		logger.Error("Could not read network interface configuration directory", logger.Ctx{"err": err})
		return
	}

	// nicData is a map of MAC address to NICConfig.
	nicData := make(map[string]deviceConfig.NICConfig, len(nicDirEntries))

	for _, f := range nicDirEntries {
		nicBytes, err := ioutil.ReadFile(filepath.Join(deviceConfig.NICConfigDir, f.Name()))
		if err != nil {
			logger.Error("Could not read network interface configuration file", logger.Ctx{"err": err})
		}

		var conf deviceConfig.NICConfig
		err = json.Unmarshal(nicBytes, &conf)
		if err != nil {
			logger.Error("Could not parse network interface configuration file", logger.Ctx{"err": err})
			return
		}

		if conf.MACAddress != "" {
			nicData[conf.MACAddress] = conf
		}
	}

	// configureNIC applies any config specified for the interface based on its current MAC address.
	configureNIC := func(currentNIC net.Interface) error {
		revert := revert.New()
		defer revert.Fail()

		// Look for a NIC config entry for this interface based on its MAC address.
		nic, ok := nicData[currentNIC.HardwareAddr.String()]
		if !ok {
			return nil
		}

		var changeName, changeMTU bool
		if nic.NICName != "" && currentNIC.Name != nic.NICName {
			changeName = true
		}

		if nic.MTU > 0 && currentNIC.MTU != int(nic.MTU) {
			changeMTU = true
		}

		if !changeName && !changeMTU {
			return nil // Nothing to do.
		}

		link := ip.Link{
			Name: currentNIC.Name,
			MTU:  fmt.Sprintf("%d", currentNIC.MTU),
		}

		err := link.SetDown()
		if err != nil {
			return err
		}

		revert.Add(func() {
			_ = link.SetUp()
		})

		// Apply the name from the NIC config if needed.
		if changeName {
			err = link.SetName(nic.NICName)
			if err != nil {
				return err
			}

			revert.Add(func() {
				err := link.SetName(currentNIC.Name)
				if err != nil {
					return
				}

				link.Name = currentNIC.Name
			})

			link.Name = nic.NICName
		}

		// Apply the MTU from the NIC config if needed.
		if changeMTU {
			newMTU := fmt.Sprintf("%d", nic.MTU)
			err = link.SetMTU(newMTU)
			if err != nil {
				return err
			}

			revert.Add(func() {
				currentMTU := fmt.Sprintf("%d", currentNIC.MTU)
				err := link.SetMTU(currentMTU)
				if err != nil {
					return
				}

				link.MTU = currentMTU
			})

			link.MTU = newMTU
		}

		err = link.SetUp()
		if err != nil {
			return err
		}

		revert.Success()
		return nil
	}

	ifaces, err := net.Interfaces()
	if err != nil {
		logger.Error("Unable to read network interfaces", logger.Ctx{"err": err})
	}

	for _, iface := range ifaces {
		err = configureNIC(iface)
		if err != nil {
			logger.Error("Unable to reconfigure network interface", logger.Ctx{"interface": iface.Name, "err": err})
		}
	}
}
<|endoftext|>