<commit_before>package subscription

import (
	"bytes"
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"net/http"
	"net/mail"
	"time"

	mailUtils "github.com/coduno/app/mail"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
)

type Subscription struct {
	Address          string
	EntryTime        time.Time
	Token            []byte
	VerificationTime time.Time
}

func Subscriptions(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		subscribe(w, r)
	} else if r.Method == "GET" {
		action := r.FormValue("action")
		if action == "confirm" {
			confirm(w, r)
		} else if action == "delete" {
			delete(w, r)
		} else {
			c := appengine.NewContext(r)
			log.Infof(c, "What happend?")
			http.Error(w, "Unknown action.", http.StatusBadRequest)
		}
	} else {
		w.Header().Set("Allow", "GET, POST")
		http.Error(w, "", http.StatusMethodNotAllowed)
	}
}

func delete(w http.ResponseWriter, r *http.Request) {
	address, err := mail.ParseAddress(r.FormValue("email"))

	if err != nil {
		http.Error(w, "Invalid email address: "+err.Error(), 422)
		return
	}

	token, err := hex.DecodeString(r.FormValue("token"))

	if err != nil {
		http.Error(w, "Invalid token: "+err.Error(), 422)
		return
	}

	c := appengine.NewContext(r)
	q := datastore.NewQuery("subscription").Filter("Address =", address.Address).Limit(1)

	var subs []Subscription
	keys, err := q.GetAll(c, &subs)

	if len(subs) != 1 {
		http.Error(w, "Unable to identify your subscription. Got "+fmt.Sprintf("%d", len(subs))+" matches on "+address.Address, http.StatusInternalServerError)
		return
	}

	sub := subs[0]

	if !bytes.Equal(sub.Token, token) {
		http.Error(w, "Forbidden", http.StatusUnauthorized)
		return
	}

	err = datastore.Delete(c, keys[0])

	if err != nil {
		http.Error(w, "Failed to delete your subscription. Please contact root@cod.uno.", http.StatusInternalServerError)
		return
	}

	w.Write([]byte("Your subscription was removed, we're sorry to see you go :("))
}

func confirm(w http.ResponseWriter, r *http.Request) {
	address, err := mail.ParseAddress(r.FormValue("email"))

	if err != nil {
		http.Error(w, "Invalid email address: "+err.Error(), 422)
		return
	}

	token, err := hex.DecodeString(r.FormValue("token"))

	if err != nil {
		http.Error(w, "Invalid token: "+err.Error(), 422)
		return
	}

	c := appengine.NewContext(r)
	q := datastore.NewQuery("subscription").Filter("Address =", address.Address).Limit(1)

	var subs []Subscription
	keys, err := q.GetAll(c, &subs)

	if len(subs) != 1 {
		http.Error(w, "Unable to identify your subscription. Got "+fmt.Sprintf("%d", len(subs))+" matches on "+address.Address, http.StatusInternalServerError)
		return
	}

	sub := subs[0]

	if !bytes.Equal(sub.Token, token) {
		http.Error(w, "Forbidden", http.StatusUnauthorized)
		return
	}

	sub.VerificationTime = time.Now()
	token, _, err = newToken()

	if err != nil {
		http.Error(w, "Unable to generate your new token.", http.StatusInternalServerError)
		return
	}

	sub.Token = token

	_, err = datastore.Put(c, keys[0], &sub)

	if err != nil {
		http.Error(w, "Failed to store subscription: "+err.Error(), http.StatusInternalServerError)
		return
	}

	w.Write([]byte("Your subscription was verified, yay!"))
}

func newToken() ([]byte, int, error) {
	token := make([]byte, 16)
	n, err := rand.Read(token)
	return token, n, err
}

func subscribe(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	address, err := mail.ParseAddress(r.FormValue("email"))

	if err != nil {
		http.Error(w, "Invalid email address: "+err.Error(), 422)
		return
	}

	q := datastore.NewQuery("subscription").Filter("Address =", address.Address).Limit(1).KeysOnly()
	collisions, err := q.GetAll(c, nil)

	if err != nil {
		http.Error(w, "Failed to check for duplicates.", http.StatusInternalServerError)
		return
	}

	if len(collisions) > 0 {
		http.Error(w, "Duplicate email address.", 422)
		return
	}

	revocationBytes, _, err := newToken()

	if err != nil {
		http.Error(w, "Failed to generate revocation secret: "+err.Error(), http.StatusInternalServerError)
		return
	}

	sub := Subscription{
		Address:   address.Address,
		EntryTime: time.Now(),
		Token:     revocationBytes,
	}

	_, err = datastore.Put(c, datastore.NewIncompleteKey(c, "subscription", nil), &sub)

	if err != nil {
		http.Error(w, "Failed to store subscription: "+err.Error(), http.StatusInternalServerError)
		return
	}

	err = sub.RequestConfirmation(c)

	if err != nil {
		http.Error(w, "Failed to send confirmation request: "+err.Error(), http.StatusInternalServerError)
		return
	}

	w.Write([]byte("A message to confirm your subscription was sent."))
}

func (sub Subscription) RequestConfirmation(c context.Context) error {
	body, err := mailUtils.PrepareMailTemplate(mailUtils.SubscriptionTemplate, sub)
	if err != nil {
		return err
	}
	return mailUtils.SendMail(c, []string{sub.Address}, "Hello from Coduno", body)
}
<commit_msg>Adopt changes to app/mail for Subscriptions<commit_after>package subscription

import (
	"bytes"
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"net/http"
	"net/mail"
	"time"

	mailUtils "github.com/coduno/app/mail"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
)

type Subscription struct {
	Address          mail.Address
	EntryTime        time.Time
	Token            []byte
	VerificationTime time.Time
}

func Subscriptions(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		subscribe(w, r)
	} else if r.Method == "GET" {
		action := r.FormValue("action")
		if action == "confirm" {
			confirm(w, r)
		} else if action == "delete" {
			delete(w, r)
		} else {
			c := appengine.NewContext(r)
			log.Infof(c, "What happend?")
			http.Error(w, "Unknown action.", http.StatusBadRequest)
		}
	} else {
		w.Header().Set("Allow", "GET, POST")
		http.Error(w, "", http.StatusMethodNotAllowed)
	}
}

func delete(w http.ResponseWriter, r *http.Request) {
	address, err := mail.ParseAddress(r.FormValue("email"))

	if err != nil {
		http.Error(w, "Invalid email address: "+err.Error(), 422)
		return
	}

	token, err := hex.DecodeString(r.FormValue("token"))

	if err != nil {
		http.Error(w, "Invalid token: "+err.Error(), 422)
		return
	}

	c := appengine.NewContext(r)
	q := datastore.NewQuery("subscription").Filter("Address =", address.Address).Limit(1)

	var subs []Subscription
	keys, err := q.GetAll(c, &subs)

	if len(subs) != 1 {
		http.Error(w, "Unable to identify your subscription. Got "+fmt.Sprintf("%d", len(subs))+" matches on "+address.Address, http.StatusInternalServerError)
		return
	}

	sub := subs[0]

	if !bytes.Equal(sub.Token, token) {
		http.Error(w, "Forbidden", http.StatusUnauthorized)
		return
	}

	err = datastore.Delete(c, keys[0])

	if err != nil {
		http.Error(w, "Failed to delete your subscription. Please contact root@cod.uno.", http.StatusInternalServerError)
		return
	}

	w.Write([]byte("Your subscription was removed, we're sorry to see you go :("))
}

func confirm(w http.ResponseWriter, r *http.Request) {
	address, err := mail.ParseAddress(r.FormValue("email"))

	if err != nil {
		http.Error(w, "Invalid email address: "+err.Error(), 422)
		return
	}

	token, err := hex.DecodeString(r.FormValue("token"))

	if err != nil {
		http.Error(w, "Invalid token: "+err.Error(), 422)
		return
	}

	c := appengine.NewContext(r)
	q := datastore.NewQuery("subscription").Filter("Address =", address.Address).Limit(1)

	var subs []Subscription
	keys, err := q.GetAll(c, &subs)

	if len(subs) != 1 {
		http.Error(w, "Unable to identify your subscription. Got "+fmt.Sprintf("%d", len(subs))+" matches on "+address.Address, http.StatusInternalServerError)
		return
	}

	sub := subs[0]

	if !bytes.Equal(sub.Token, token) {
		http.Error(w, "Forbidden", http.StatusUnauthorized)
		return
	}

	sub.VerificationTime = time.Now()
	token, _, err = newToken()

	if err != nil {
		http.Error(w, "Unable to generate your new token.", http.StatusInternalServerError)
		return
	}

	sub.Token = token

	_, err = datastore.Put(c, keys[0], &sub)

	if err != nil {
		http.Error(w, "Failed to store subscription: "+err.Error(), http.StatusInternalServerError)
		return
	}

	w.Write([]byte("Your subscription was verified, yay!"))
}

func newToken() ([]byte, int, error) {
	token := make([]byte, 16)
	n, err := rand.Read(token)
	return token, n, err
}

func subscribe(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	address, err := mail.ParseAddress(r.FormValue("email"))

	if err != nil {
		http.Error(w, "Invalid email address: "+err.Error(), 422)
		return
	}

	q := datastore.NewQuery("subscription").Filter("Address =", address.Address).Limit(1).KeysOnly()
	collisions, err := q.GetAll(c, nil)

	if err != nil {
		http.Error(w, "Failed to check for duplicates.", http.StatusInternalServerError)
		return
	}

	if len(collisions) > 0 {
		http.Error(w, "Duplicate email address.", 422)
		return
	}

	revocationBytes, _, err := newToken()

	if err != nil {
		http.Error(w, "Failed to generate revocation secret: "+err.Error(), http.StatusInternalServerError)
		return
	}

	sub := Subscription{
		Address:   *address,
		EntryTime: time.Now(),
		Token:     revocationBytes,
	}

	_, err = datastore.Put(c, datastore.NewIncompleteKey(c, "subscription", nil), &sub)

	if err != nil {
		http.Error(w, "Failed to store subscription: "+err.Error(), http.StatusInternalServerError)
		return
	}

	err = sub.RequestConfirmation(c)

	if err != nil {
		http.Error(w, "Failed to send confirmation request: "+err.Error(), http.StatusInternalServerError)
		return
	}

	w.Write([]byte("A message to confirm your subscription was sent."))
}

func (sub Subscription) RequestConfirmation(ctx context.Context) error {
	buf := new(bytes.Buffer)
	if err := mailUtils.Subscription.Execute(buf, sub); err != nil {
		return err
	}
	return mailUtils.Send(ctx, sub.Address, "Hello from Coduno", buf.String())
}
<|endoftext|>


<commit_before>/**
 * (c) 2014, Tonnerre Lombard <tonnerre@ancient-solutions.com>,
 *	     Ancient Solutions. All rights reserved.
 *
 * Redistribution and use in source  and binary forms, with or without
 * modification, are permitted  provided that the following conditions
 * are met:
 *
 * * Redistributions of  source code  must retain the  above copyright
 *   notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 *   notice, this  list of conditions and the  following disclaimer in
 *   the  documentation  and/or  other  materials  provided  with  the
 *   distribution.
 * * Neither  the  name  of  Ancient Solutions  nor  the  name  of its
 *   contributors may  be used to endorse or  promote products derived
 *   from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS"  AND ANY EXPRESS  OR IMPLIED WARRANTIES  OF MERCHANTABILITY
 * AND FITNESS  FOR A PARTICULAR  PURPOSE ARE DISCLAIMED. IN  NO EVENT
 * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED  TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE,  DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT  LIABILITY,  OR  TORT  (INCLUDING NEGLIGENCE  OR  OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package blubberstore

import (
	"code.google.com/p/goprotobuf/proto"
	"errors"
	"github.com/tonnerre/go-serialdata"
	"log"
	"os"
	"path"
	"strings"
	"sync"
	"time"
)

// Block directory service for blubberstore.
type BlubberBlockDirectory struct {
	// Pointer to the currently active journal.
	currentJournal *serialdata.SerialDataWriter
	journalFile    *os.File
	journalPrefix  string

	// Mapping of block IDs to hosts which contain them to block states.
	blockMap       map[string]map[string]*ServerBlockStatus
	blockMapMtx    *sync.RWMutex
	blockMapPrefix string
}

func NewBlubberBlockDirectory(
	journalPrefix, blockMapPrefix string) (*BlubberBlockDirectory, error) {
	var now time.Time = time.Now()
	var newpath string = journalPrefix + now.Format("2006-01-02.150405")
	var blockMap map[string]map[string]*ServerBlockStatus = make(map[string]map[string]*ServerBlockStatus)
	var parentDir *os.File
	var stateDumpFile, newJournal *os.File
	var reader *serialdata.SerialDataReader
	var ret *BlubberBlockDirectory
	var names []string
	var name string
	var bs BlockStatus
	var err error

	// Try to find the latest complete block map.
	stateDumpFile, err = os.Open(blockMapPrefix + ".blockmap")
	if err != nil && os.IsNotExist(err) {
		stateDumpFile, err = os.Open(blockMapPrefix + ".old")
	}
	if err != nil && os.IsNotExist(err) {
		log.Print("Warning: created new empty block map")
		stateDumpFile, err = os.Create(blockMapPrefix + ".blockmap")
	} else if err != nil {
		return nil, errors.New("Unable to open block map: " + err.Error())
	}

	// Read all data from the state dump file.
	reader = serialdata.NewSerialDataReader(stateDumpFile)

	for err = reader.ReadMessage(&bs); err == nil; err = reader.ReadMessage(&bs) {
		var blockId string = string(bs.GetBlockId())
		var srv *ServerBlockStatus

		blockMap[blockId] = make(map[string]*ServerBlockStatus)

		for _, srv = range bs.Servers {
			blockMap[blockId][srv.GetHostPort()] = srv
		}

		bs.Reset()
	}

	newJournal, err = os.Create(newpath)
	if err != nil {
		return nil, errors.New("Error creating " + newpath + ": " +
			err.Error())
	}

	ret = &BlubberBlockDirectory{
		currentJournal: serialdata.NewSerialDataWriter(newJournal),
		journalFile:    newJournal,
		journalPrefix:  journalPrefix,
		blockMap:       blockMap,
		blockMapMtx:    new(sync.RWMutex),
		blockMapPrefix: blockMapPrefix,
	}

	// Now replay all the journal files we can find.
	parentDir, err = os.Open(path.Dir(journalPrefix))
	if err != nil {
		return nil, errors.New("Unable to open " + path.Dir(journalPrefix) +
			": " + err.Error())
	}
	names, err = parentDir.Readdirnames(-1)
	if err != nil {
		return nil, errors.New("Unable to list " + path.Dir(journalPrefix) +
			": " + err.Error())
	}
	parentDir.Close()

	for _, name = range names {
		if strings.HasPrefix(name, path.Base(journalPrefix)) {
			var journal *os.File
			var br BlockReport

			journal, err = os.Open(parentDir.Name() + "/" + name)
			if err != nil {
				return nil, err
			}

			reader = serialdata.NewSerialDataReader(journal)
			for err = reader.ReadMessage(&br); err == nil; err = reader.ReadMessage(&br) {
				ret.ApplyBlockReport(&br)
			}
			journal.Close()
		}
	}

	// Start syncing the state dump to disk.
	go ret.periodicallyWriteStateDump()

	return ret, nil
}

// Write a complete state dump to disk regularly and remove the journals.
func (b *BlubberBlockDirectory) periodicallyWriteStateDump() {
	var c <-chan time.Time

	c = time.Tick(2 * time.Minute)
	for _ = range c {
		b.dumpState()
	}
}

// Apply a single BlockReport to the internal state without writing it to
// the journal. This is useful e.g. for replaying the journal.
func (b *BlubberBlockDirectory) ApplyBlockReport(status *BlockReport) {
	var host string

	// Register the updated block for all reported servers.
	b.blockMapMtx.Lock()
	defer b.blockMapMtx.Unlock()

	for _, host = range status.Server {
		// Create a new ServerBlockStatus record with the information we have.
		var sbs *ServerBlockStatus = &ServerBlockStatus{
			HostPort:  proto.String(host),
			Checksum:  make([]byte, len(status.Status.Checksum)),
			Timestamp: proto.Uint64(status.Status.GetTimestamp()),
		}
		var ok bool
		copy(sbs.Checksum, status.Status.Checksum)

		_, ok = b.blockMap[string(status.Status.BlockId)]
		if !ok {
			b.blockMap[string(status.Status.BlockId)] =
				make(map[string]*ServerBlockStatus)
		}
		b.blockMap[string(status.Status.BlockId)][host] = sbs
	}
}

// Report to the directory that the block with the given properties is now
// stored on the specified server.
func (b *BlubberBlockDirectory) ReportBlob(status BlockReport, res *BlockId) error {
	var err error

	res.BlockId = make([]byte, len(status.Status.BlockId))
	copy(res.BlockId, status.Status.BlockId)

	// Write all data about the reported blob to the journal.
	err = b.currentJournal.WriteMessage(&status)
	if err != nil {
		return err
	}

	// Now apply the report to the internal state.
	b.ApplyBlockReport(&status)

	return nil
}

// Look up the block with the given ID.
func (b *BlubberBlockDirectory) LookupBlob(
	req BlockId, res *BlockHolderList) error {
	var host string
	var ok bool

	b.blockMapMtx.RLock()
	defer b.blockMapMtx.RUnlock()
	_, ok = b.blockMap[string(req.BlockId)]
	if !ok {
		// We don't know anything about the block.
		return nil
	}

	// FIXME: we should check the majority holders here instead of just
	// returning a list of all hosts.
	for host, _ = range b.blockMap[string(req.BlockId)] {
		res.HostPort = append(res.HostPort, host)
	}

	return nil
}

// Remove the given host from the holders of the blob.
func (b *BlubberBlockDirectory) RemoveBlobHolder(report BlockRemovalReport,
	res *BlockId) error {
	var server string
	b.blockMapMtx.Lock()
	defer b.blockMapMtx.Unlock()

	for _, server = range report.GetServer() {
		delete(b.blockMap[string(report.GetBlockId())],
			server)
	}
	res.BlockId = make([]byte, len(report.GetBlockId()))
	copy(res.BlockId, report.GetBlockId())
	return nil
}

// Write a new dump of the current state of block/server mappings.
func (b *BlubberBlockDirectory) dumpState() {
	var now time.Time = time.Now()
	var newpath string = b.journalPrefix + now.Format("2006-01-02.150405")
	var oldJournalFile *os.File = b.journalFile
	var newJournalFile *os.File
	var newStateDumpFile *os.File
	var stateDumpWriter *serialdata.SerialDataWriter
	var blockId string
	var serverMap map[string]*ServerBlockStatus
	var err error

	os.Remove(b.blockMapPrefix + ".new")
	newStateDumpFile, err = os.Create(b.blockMapPrefix + ".new")
	if err != nil {
		log.Print("Unable to open new journal file ", newpath, ": ", err)
		log.Print("Skipping state dump")
		return
	}

	newJournalFile, err = os.Create(newpath)
	if err != nil {
		log.Print("Unable to open new journal file ", newpath, ": ", err)
		log.Print("Skipping state dump")
		return
	}

	// Start a new journal to track future differences. We'll assume they're
	// idempotent so we can just replay them on top of an already-modified
	// tree.
	b.currentJournal = serialdata.NewSerialDataWriter(newJournalFile)
	b.journalFile = newJournalFile

	stateDumpWriter = serialdata.NewSerialDataWriter(newStateDumpFile)

	b.blockMapMtx.RLock()
	defer b.blockMapMtx.RUnlock()
	for blockId, serverMap = range b.blockMap {
		var bs BlockStatus
		var host string
		var srv *ServerBlockStatus

		bs.BlockId = make([]byte, len([]byte(blockId)))
		copy(bs.BlockId, []byte(blockId))

		// TODO(tonnerre): make this something user specified.
		bs.ReplicationFactor = proto.Uint32(3)

		for host, srv = range serverMap {
			if srv.GetHostPort() != host {
				log.Print("host:port mismatch for block ", []byte(blockId),
					": ", srv.GetHostPort(), " vs. ", host)
				srv.HostPort = proto.String(host)
			}

			bs.Servers = append(bs.Servers, srv)
		}

		err = stateDumpWriter.WriteMessage(&bs)
		if err != nil {
			log.Print("Error writing record to state dump file: ", err)
			newStateDumpFile.Close()
			os.Remove(b.blockMapPrefix + ".new")
		}
	}

	err = newStateDumpFile.Close()
	if err != nil {
		log.Print("Error writing state dump: ", err)
		os.Remove(b.blockMapPrefix + ".new")
		return
	}

	// Try moving the old state dump out of the way.
	err = os.Rename(b.blockMapPrefix+".blockmap", b.blockMapPrefix+".old")
	if err != nil && !os.IsNotExist(err) {
		log.Print("Error moving away old state dump: ", err)
	}

	// Move the new state dump into the place where the old one used to be.
	err = os.Rename(b.blockMapPrefix+".new", b.blockMapPrefix+".blockmap")
	if err != nil {
		log.Print("Error moving new state dump into place: ", err)
		os.Remove(b.blockMapPrefix + ".new")
		return
	}

	err = os.Remove(b.blockMapPrefix + ".old")
	if err != nil && !os.IsNotExist(err) {
		log.Print("Error deleting old state dump: ", err)
	}

	err = os.Remove(oldJournalFile.Name())
	if err != nil {
		log.Print("Error deleting old journal file ", oldJournalFile.Name(),
			": ", err)
	}
}
<commit_msg>Add a list of hosts holding blocks by hosts.<commit_after>/**
 * (c) 2014, Tonnerre Lombard <tonnerre@ancient-solutions.com>,
 *	     Ancient Solutions. All rights reserved.
 *
 * Redistribution and use in source  and binary forms, with or without
 * modification, are permitted  provided that the following conditions
 * are met:
 *
 * * Redistributions of  source code  must retain the  above copyright
 *   notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 *   notice, this  list of conditions and the  following disclaimer in
 *   the  documentation  and/or  other  materials  provided  with  the
 *   distribution.
 * * Neither  the  name  of  Ancient Solutions  nor  the  name  of its
 *   contributors may  be used to endorse or  promote products derived
 *   from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS"  AND ANY EXPRESS  OR IMPLIED WARRANTIES  OF MERCHANTABILITY
 * AND FITNESS  FOR A PARTICULAR  PURPOSE ARE DISCLAIMED. IN  NO EVENT
 * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED  TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE,  DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT  LIABILITY,  OR  TORT  (INCLUDING NEGLIGENCE  OR  OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package blubberstore

import (
	"code.google.com/p/goprotobuf/proto"
	"errors"
	"github.com/tonnerre/go-serialdata"
	"log"
	"os"
	"path"
	"strings"
	"sync"
	"time"
)

// Block directory service for blubberstore.
type BlubberBlockDirectory struct {
	// Pointer to the currently active journal.
	currentJournal *serialdata.SerialDataWriter
	journalFile    *os.File
	journalPrefix  string

	// Mapping of block IDs to hosts which contain them to block states.
	blockHostMap   map[string][]string
	blockMap       map[string]map[string]*ServerBlockStatus
	blockMapMtx    *sync.RWMutex
	blockMapPrefix string
}

func NewBlubberBlockDirectory(
	journalPrefix, blockMapPrefix string) (*BlubberBlockDirectory, error) {
	var now time.Time = time.Now()
	var newpath string = journalPrefix + now.Format("2006-01-02.150405")
	var blockMap map[string]map[string]*ServerBlockStatus = make(map[string]map[string]*ServerBlockStatus)
	var parentDir *os.File
	var stateDumpFile, newJournal *os.File
	var reader *serialdata.SerialDataReader
	var ret *BlubberBlockDirectory
	var names []string
	var name string
	var bs BlockStatus
	var err error

	// Try to find the latest complete block map.
	stateDumpFile, err = os.Open(blockMapPrefix + ".blockmap")
	if err != nil && os.IsNotExist(err) {
		stateDumpFile, err = os.Open(blockMapPrefix + ".old")
	}
	if err != nil && os.IsNotExist(err) {
		log.Print("Warning: created new empty block map")
		stateDumpFile, err = os.Create(blockMapPrefix + ".blockmap")
	} else if err != nil {
		return nil, errors.New("Unable to open block map: " + err.Error())
	}

	// Read all data from the state dump file.
	reader = serialdata.NewSerialDataReader(stateDumpFile)

	for err = reader.ReadMessage(&bs); err == nil; err = reader.ReadMessage(&bs) {
		var blockId string = string(bs.GetBlockId())
		var srv *ServerBlockStatus

		blockMap[blockId] = make(map[string]*ServerBlockStatus)

		for _, srv = range bs.Servers {
			blockMap[blockId][srv.GetHostPort()] = srv
		}

		bs.Reset()
	}

	newJournal, err = os.Create(newpath)
	if err != nil {
		return nil, errors.New("Error creating " + newpath + ": " +
			err.Error())
	}

	ret = &BlubberBlockDirectory{
		currentJournal: serialdata.NewSerialDataWriter(newJournal),
		journalFile:    newJournal,
		journalPrefix:  journalPrefix,
		blockMap:       blockMap,
		blockMapMtx:    new(sync.RWMutex),
		blockMapPrefix: blockMapPrefix,
	}

	// Now replay all the journal files we can find.
	parentDir, err = os.Open(path.Dir(journalPrefix))
	if err != nil {
		return nil, errors.New("Unable to open " + path.Dir(journalPrefix) +
			": " + err.Error())
	}
	names, err = parentDir.Readdirnames(-1)
	if err != nil {
		return nil, errors.New("Unable to list " + path.Dir(journalPrefix) +
			": " + err.Error())
	}
	parentDir.Close()

	for _, name = range names {
		if strings.HasPrefix(name, path.Base(journalPrefix)) {
			var journal *os.File
			var br BlockReport

			journal, err = os.Open(parentDir.Name() + "/" + name)
			if err != nil {
				return nil, err
			}

			reader = serialdata.NewSerialDataReader(journal)
			for err = reader.ReadMessage(&br); err == nil; err = reader.ReadMessage(&br) {
				ret.ApplyBlockReport(&br)
			}
			journal.Close()
		}
	}

	// Start syncing the state dump to disk.
	go ret.periodicallyWriteStateDump()

	return ret, nil
}

// Write a complete state dump to disk regularly and remove the journals.
func (b *BlubberBlockDirectory) periodicallyWriteStateDump() {
	var c <-chan time.Time

	c = time.Tick(2 * time.Minute)
	for _ = range c {
		b.dumpState()
	}
}

// Apply a single BlockReport to the internal state without writing it to
// the journal. This is useful e.g. for replaying the journal.
func (b *BlubberBlockDirectory) ApplyBlockReport(status *BlockReport) {
	var host string

	// Register the updated block for all reported servers.
	b.blockMapMtx.Lock()
	defer b.blockMapMtx.Unlock()

	for _, host = range status.Server {
		// Create a new ServerBlockStatus record with the information we have.
		var sbs *ServerBlockStatus = &ServerBlockStatus{
			HostPort:  proto.String(host),
			Checksum:  make([]byte, len(status.Status.Checksum)),
			Timestamp: proto.Uint64(status.Status.GetTimestamp()),
		}
		var ok bool
		copy(sbs.Checksum, status.Status.Checksum)

		_, ok = b.blockMap[string(status.Status.BlockId)]
		if !ok {
			b.blockMap[string(status.Status.BlockId)] =
				make(map[string]*ServerBlockStatus)
		}
		b.blockMap[string(status.Status.BlockId)][host] = sbs
		b.blockHostMap[host] = append(b.blockHostMap[host],
			string(status.Status.BlockId))
	}
}

// Report to the directory that the block with the given properties is now
// stored on the specified server.
func (b *BlubberBlockDirectory) ReportBlob(status BlockReport, res *BlockId) error {
	var err error

	res.BlockId = make([]byte, len(status.Status.BlockId))
	copy(res.BlockId, status.Status.BlockId)

	// Write all data about the reported blob to the journal.
	err = b.currentJournal.WriteMessage(&status)
	if err != nil {
		return err
	}

	// Now apply the report to the internal state.
	b.ApplyBlockReport(&status)

	return nil
}

// Look up the block with the given ID.
func (b *BlubberBlockDirectory) LookupBlob(
	req BlockId, res *BlockHolderList) error {
	var host string
	var ok bool

	b.blockMapMtx.RLock()
	defer b.blockMapMtx.RUnlock()
	_, ok = b.blockMap[string(req.BlockId)]
	if !ok {
		// We don't know anything about the block.
		return nil
	}

	// FIXME: we should check the majority holders here instead of just
	// returning a list of all hosts.
	for host, _ = range b.blockMap[string(req.BlockId)] {
		res.HostPort = append(res.HostPort, host)
	}

	return nil
}

// Remove the given host from the holders of the blob.
func (b *BlubberBlockDirectory) RemoveBlobHolder(report BlockRemovalReport,
	res *BlockId) error {
	var server string
	b.blockMapMtx.Lock()
	defer b.blockMapMtx.Unlock()

	for _, server = range report.GetServer() {
		var blocks []string
		var block string

		delete(b.blockMap[string(report.GetBlockId())],
			server)

		blocks = b.blockHostMap[server]
		b.blockHostMap[server] = make([]string, 0)
		for _, block = range blocks {
			if block != string(report.GetBlockId()) {
				b.blockHostMap[server] = append(b.blockHostMap[server],
					block)
			}
		}
	}

	res.BlockId = make([]byte, len(report.GetBlockId()))
	copy(res.BlockId, report.GetBlockId())

	return nil
}

// Write a new dump of the current state of block/server mappings.
func (b *BlubberBlockDirectory) dumpState() {
	var now time.Time = time.Now()
	var newpath string = b.journalPrefix + now.Format("2006-01-02.150405")
	var oldJournalFile *os.File = b.journalFile
	var newJournalFile *os.File
	var newStateDumpFile *os.File
	var stateDumpWriter *serialdata.SerialDataWriter
	var blockId string
	var serverMap map[string]*ServerBlockStatus
	var err error

	os.Remove(b.blockMapPrefix + ".new")
	newStateDumpFile, err = os.Create(b.blockMapPrefix + ".new")
	if err != nil {
		log.Print("Unable to open new journal file ", newpath, ": ", err)
		log.Print("Skipping state dump")
		return
	}

	newJournalFile, err = os.Create(newpath)
	if err != nil {
		log.Print("Unable to open new journal file ", newpath, ": ", err)
		log.Print("Skipping state dump")
		return
	}

	// Start a new journal to track future differences. We'll assume they're
	// idempotent so we can just replay them on top of an already-modified
	// tree.
	b.currentJournal = serialdata.NewSerialDataWriter(newJournalFile)
	b.journalFile = newJournalFile

	stateDumpWriter = serialdata.NewSerialDataWriter(newStateDumpFile)

	b.blockMapMtx.RLock()
	defer b.blockMapMtx.RUnlock()
	for blockId, serverMap = range b.blockMap {
		var bs BlockStatus
		var host string
		var srv *ServerBlockStatus

		bs.BlockId = make([]byte, len([]byte(blockId)))
		copy(bs.BlockId, []byte(blockId))

		// TODO(tonnerre): make this something user specified.
		bs.ReplicationFactor = proto.Uint32(3)

		for host, srv = range serverMap {
			if srv.GetHostPort() != host {
				log.Print("host:port mismatch for block ", []byte(blockId),
					": ", srv.GetHostPort(), " vs. ", host)
				srv.HostPort = proto.String(host)
			}

			bs.Servers = append(bs.Servers, srv)
		}

		err = stateDumpWriter.WriteMessage(&bs)
		if err != nil {
			log.Print("Error writing record to state dump file: ", err)
			newStateDumpFile.Close()
			os.Remove(b.blockMapPrefix + ".new")
		}
	}

	err = newStateDumpFile.Close()
	if err != nil {
		log.Print("Error writing state dump: ", err)
		os.Remove(b.blockMapPrefix + ".new")
		return
	}

	// Try moving the old state dump out of the way.
	err = os.Rename(b.blockMapPrefix+".blockmap", b.blockMapPrefix+".old")
	if err != nil && !os.IsNotExist(err) {
		log.Print("Error moving away old state dump: ", err)
	}

	// Move the new state dump into the place where the old one used to be.
	err = os.Rename(b.blockMapPrefix+".new", b.blockMapPrefix+".blockmap")
	if err != nil {
		log.Print("Error moving new state dump into place: ", err)
		os.Remove(b.blockMapPrefix + ".new")
		return
	}

	err = os.Remove(b.blockMapPrefix + ".old")
	if err != nil && !os.IsNotExist(err) {
		log.Print("Error deleting old state dump: ", err)
	}

	err = os.Remove(oldJournalFile.Name())
	if err != nil {
		log.Print("Error deleting old journal file ", oldJournalFile.Name(),
			": ", err)
	}
}
<|endoftext|>


<commit_before>package login

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/client/rest"
	"github.com/Aptomi/aptomi/pkg/client/rest/http"
	"github.com/Aptomi/aptomi/pkg/config"
	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cobra"
	"gopkg.in/yaml.v2"
	"io/ioutil"
)

// NewCommand returns instance of cobra command that allows to login into aptomi
func NewCommand(cfg *config.Client, cfgFile *string) *cobra.Command {
	var username, password string

	cmd := &cobra.Command{
		Use:   "login",
		Short: "login into the Aptomi",
		Run: func(cmd *cobra.Command, args []string) {
			if len(username) == 0 || len(password) == 0 {
				panic(fmt.Sprintf("Both username and password should be non-empty"))
			}

			authSuccess, err := rest.New(cfg, http.NewClient(cfg)).User().Login(username, password)
			if err != nil {
				panic(fmt.Sprintf("Error while user login: %s", err))
			}

			cfg.Auth.Token = authSuccess.Token

			data, err := yaml.Marshal(cfg)
			if err != nil {
				panic(fmt.Sprintf("Error marshaling client config: %s", err))
			}
			err = ioutil.WriteFile(*cfgFile, data, 0644)
			if err != nil {
				panic(fmt.Sprintf("Error while saving config with token to %s: %s", *cfgFile, err))
			}

			log.Infof("Config successfully updated with token")
		},
	}

	cmd.Flags().StringVarP(&username, "username", "u", "", "Username")
	if err := cmd.MarkFlagRequired("username"); err != nil {
		panic(err)
	}
	cmd.Flags().StringVarP(&password, "password", "p", "", "Password")
	if err := cmd.MarkFlagRequired("password"); err != nil {
		panic(err)
	}

	return cmd
}
<commit_msg>Backup existing config if it differs from updated one on login cmd<commit_after>package login

import (
	"bytes"
	"fmt"
	"github.com/Aptomi/aptomi/cmd/common"
	"github.com/Aptomi/aptomi/pkg/client/rest"
	"github.com/Aptomi/aptomi/pkg/client/rest/http"
	"github.com/Aptomi/aptomi/pkg/config"
	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cobra"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
)

// NewCommand returns instance of cobra command that allows to login into aptomi
func NewCommand(cfg *config.Client, cfgFile *string) *cobra.Command {
	var username, password string

	cmd := &cobra.Command{
		Use:   "login",
		Short: "login into the Aptomi",
		Run: func(cmd *cobra.Command, args []string) {
			if len(username) == 0 || len(password) == 0 {
				panic(fmt.Sprintf("Both username and password should be non-empty"))
			}

			authSuccess, err := rest.New(cfg, http.NewClient(cfg)).User().Login(username, password)
			if err != nil {
				panic(fmt.Sprintf("Error while user login: %s", err))
			}

			cfg.Auth.Token = authSuccess.Token

			writeConfig(cfg, cfgFile)

			log.Infof("Config successfully updated with token")
		},
	}

	cmd.Flags().StringVarP(&username, "username", "u", "", "Username")
	if err := cmd.MarkFlagRequired("username"); err != nil {
		panic(err)
	}
	cmd.Flags().StringVarP(&password, "password", "p", "", "Password")
	if err := cmd.MarkFlagRequired("password"); err != nil {
		panic(err)
	}

	return cmd
}

func writeConfig(cfg *config.Client, cfgFile *string) {
	data, err := yaml.Marshal(cfg)
	if err != nil {
		panic(fmt.Sprintf("Error marshaling client config: %s", err))
	}

	backupConfigIfDiffers(*cfgFile, data)

	err = ioutil.WriteFile(*cfgFile, data, 0644)
	if err != nil {
		panic(fmt.Sprintf("Error while saving config with token to %s: %s", *cfgFile, err))
	}
}

func backupConfigIfDiffers(cfgFile string, newData []byte) {
	cfg, err := os.Open(cfgFile)
	if err != nil {
		panic(fmt.Sprintf("Error while opening current config file %s: %s", cfgFile, err))
	}
	defer cfg.Close()

	data, err := ioutil.ReadAll(cfg)
	if err != nil {
		panic(fmt.Sprintf("Error while reading current config file %s: %s", cfgFile, err))
	}

	if !bytes.Equal(data, newData) {
		backupCfgFile := cfgFile + ".bak"
		err = ioutil.WriteFile(backupCfgFile, data, 0644)
		if err != nil {
			panic(fmt.Sprintf("Error while writing backup of the current config %s: %s", cfgFile, err))
		}
		log.Infof("Current config saved to: %s", backupCfgFile)
	}
}
<|endoftext|>


<commit_before>package search // import "bosun.org/cmd/bosun/search"

import (
	"fmt"
	"math/rand"
	"reflect"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"bosun.org/cmd/bosun/database"
	"bosun.org/collect"
	"bosun.org/metadata"
	"bosun.org/opentsdb"
	"bosun.org/slog"
)

// Search is a struct to hold indexed data about OpenTSDB metric and tag data.
// It is suited to answering questions about: available metrics for a tag set,
// available tag keys for a metric, and available tag values for a metric and
// tag key.
type Search struct {
	DataAccess database.DataAccess

	Last map[string]*lastInfo

	indexQueue chan *opentsdb.DataPoint
	sync.RWMutex
}

type lastInfo struct {
	lastVal      float64
	diffFromPrev float64
	timestamp    int64
}

func init() {
	metadata.AddMetricMeta("bosun.search.index_queue", metadata.Gauge, metadata.Count, "Number of datapoints queued for indexing to redis")
	metadata.AddMetricMeta("bosun.search.dropped", metadata.Counter, metadata.Count, "Number of datapoints discarded without being saved to redis")
}
func NewSearch(data database.DataAccess) *Search {
	s := Search{
		DataAccess: data,
		Last:       make(map[string]*lastInfo),
		indexQueue: make(chan *opentsdb.DataPoint, 300000),
	}
	collect.Set("search.index_queue", opentsdb.TagSet{}, func() interface{} { return len(s.indexQueue) })
	go s.redisIndex(s.indexQueue)
	return &s
}

func (s *Search) Index(mdp opentsdb.MultiDataPoint) {
	for _, dp := range mdp {
		s.Lock()
		metric := dp.Metric
		key := metric + dp.Tags.String()
		p := s.Last[key]
		if p == nil {
			p = &lastInfo{}
			s.Last[key] = p
		}
		if p.timestamp < dp.Timestamp {
			if fv, err := getFloat(dp.Value); err == nil {
				p.diffFromPrev = (fv - p.lastVal) / float64(dp.Timestamp-p.timestamp)
				p.lastVal = fv
			} else {
				slog.Error(err)
			}
			p.timestamp = dp.Timestamp
		}
		s.Unlock()
		select {
		case s.indexQueue <- dp:
		default:
			collect.Add("search.dropped", opentsdb.TagSet{}, 1)
		}
	}
}

func (s *Search) redisIndex(c <-chan *opentsdb.DataPoint) {
	now := time.Now().Unix()
	nextUpdateTimes := make(map[string]int64)
	updateIfTime := func(key string, f func()) {
		nextUpdate, ok := nextUpdateTimes[key]
		if !ok || now > nextUpdate {
			f()
			nextUpdateTimes[key] = now + int64(30*60+rand.Intn(15*60)) //pick a random time between 30 and 45 minutes from now
		}
	}
	for dp := range c {
		now = time.Now().Unix()
		metric := dp.Metric
		for k, v := range dp.Tags {
			updateIfTime(fmt.Sprintf("kvm:%s:%s:%s", k, v, metric), func() {
				if err := s.DataAccess.Search_AddMetricForTag(k, v, metric, now); err != nil {
					slog.Error(err)
				}
				if err := s.DataAccess.Search_AddTagValue(metric, k, v, now); err != nil {
					slog.Error(err)
				}
			})
			updateIfTime(fmt.Sprintf("mk:%s:%s", metric, k), func() {
				if err := s.DataAccess.Search_AddTagKeyForMetric(metric, k, now); err != nil {
					slog.Error(err)
				}
			})
			updateIfTime(fmt.Sprintf("kv:%s:%s", k, v), func() {
				if err := s.DataAccess.Search_AddTagValue(database.Search_All, k, v, now); err != nil {
					slog.Error(err)
				}
			})
			updateIfTime(fmt.Sprintf("m:%s", metric), func() {
				if err := s.DataAccess.Search_AddMetric(metric, now); err != nil {
					slog.Error(err)
				}
			})
		}
	}
}

var floatType = reflect.TypeOf(float64(0))

func getFloat(unk interface{}) (float64, error) {
	v := reflect.ValueOf(unk)
	v = reflect.Indirect(v)
	if !v.Type().ConvertibleTo(floatType) {
		return 0, fmt.Errorf("cannot convert %v to float64", v.Type())
	}
	fv := v.Convert(floatType)
	return fv.Float(), nil
}

// Match returns all matching values against search. search is a regex, except
// that `.` is literal, `*` can be used for `.*`, and the entire string is
// searched (`^` and `&` added to ends of search).
func Match(search string, values []string) ([]string, error) {
	v := strings.Replace(search, ".", `\.`, -1)
	v = strings.Replace(v, "*", ".*", -1)
	v = "^" + v + "$"
	re, err := regexp.Compile(v)
	if err != nil {
		return nil, err
	}
	var nvs []string
	for _, nv := range values {
		if re.MatchString(nv) {
			nvs = append(nvs, nv)
		}
	}
	return nvs, nil
}

var errNotFloat = fmt.Errorf("last: expected float64")

// GetLast returns the value of the most recent data point for the given metric
// and tag. tags should be of the form "{key=val,key2=val2}". If diff is true,
// the value is treated as a counter. err is non nil if there is no match.
func (s *Search) GetLast(metric, tags string, diff bool) (v float64, err error) {
	s.RLock()
	p := s.Last[metric+tags]
	if p != nil {
		if diff {
			return p.diffFromPrev, nil
		}
		return p.lastVal, nil
	}
	s.RUnlock()
	return 0, nil
}

func (s *Search) Expand(q *opentsdb.Query) error {
	for k, ov := range q.Tags {
		var nvs []string
		for _, v := range strings.Split(ov, "|") {
			v = strings.TrimSpace(v)
			if v == "*" || !strings.Contains(v, "*") {
				nvs = append(nvs, v)
			} else {
				vs, err := s.TagValuesByMetricTagKey(q.Metric, k, 0)
				if err != nil {
					return err
				}
				ns, err := Match(v, vs)
				if err != nil {
					return err
				}
				nvs = append(nvs, ns...)
			}
		}
		if len(nvs) == 0 {
			return fmt.Errorf("expr: no tags matching %s=%s", k, ov)
		}
		q.Tags[k] = strings.Join(nvs, "|")
	}
	return nil
}

func (s *Search) UniqueMetrics() ([]string, error) {
	m, err := s.DataAccess.Search_GetAllMetrics()
	if err != nil {
		return nil, err
	}
	metrics := make([]string, len(m))
	i := 0
	for k, _ := range m {
		metrics[i] = k
		i++
	}
	sort.Strings(metrics)
	return metrics, nil
}

func (s *Search) TagValuesByTagKey(Tagk string, since time.Duration) ([]string, error) {
	return s.TagValuesByMetricTagKey(database.Search_All, Tagk, since)
}

func (s *Search) MetricsByTagPair(tagk, tagv string) ([]string, error) {
	metrics, err := s.DataAccess.Search_GetMetricsForTag(tagk, tagv)
	if err != nil {
		return nil, err
	}
	r := []string{}
	for k, _ := range metrics {
		r = append(r, k)
	}
	sort.Strings(r)
	return r, nil
}

func (s *Search) TagKeysByMetric(metric string) ([]string, error) {
	keys, err := s.DataAccess.Search_GetTagKeysForMetric(metric)
	if err != nil {
		return nil, err
	}
	r := []string{}
	for k := range keys {
		r = append(r, k)
	}
	sort.Strings(r)
	return r, nil
}

func (s *Search) TagValuesByMetricTagKey(metric, tagK string, since time.Duration) ([]string, error) {
	var t int64
	if since > 0 {
		t = time.Now().Add(-since).Unix()
	}
	vals, err := s.DataAccess.Search_GetTagValues(metric, tagK)
	if err != nil {
		return nil, err
	}
	r := []string{}
	for k, ts := range vals {
		if t <= ts {
			r = append(r, k)
		}
	}
	sort.Strings(r)
	return r, nil
}
<commit_msg>gofmt<commit_after>package search // import "bosun.org/cmd/bosun/search"

import (
	"fmt"
	"math/rand"
	"reflect"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"bosun.org/cmd/bosun/database"
	"bosun.org/collect"
	"bosun.org/metadata"
	"bosun.org/opentsdb"
	"bosun.org/slog"
)

// Search is a struct to hold indexed data about OpenTSDB metric and tag data.
// It is suited to answering questions about: available metrics for a tag set,
// available tag keys for a metric, and available tag values for a metric and
// tag key.
type Search struct {
	DataAccess database.DataAccess

	Last map[string]*lastInfo

	indexQueue chan *opentsdb.DataPoint
	sync.RWMutex
}

type lastInfo struct {
	lastVal      float64
	diffFromPrev float64
	timestamp    int64
}

func init() {
	metadata.AddMetricMeta("bosun.search.index_queue", metadata.Gauge, metadata.Count, "Number of datapoints queued for indexing to redis")
	metadata.AddMetricMeta("bosun.search.dropped", metadata.Counter, metadata.Count, "Number of datapoints discarded without being saved to redis")
}
func NewSearch(data database.DataAccess) *Search {
	s := Search{
		DataAccess: data,
		Last:       make(map[string]*lastInfo),
		indexQueue: make(chan *opentsdb.DataPoint, 300000),
	}
	collect.Set("search.index_queue", opentsdb.TagSet{}, func() interface{} { return len(s.indexQueue) })
	go s.redisIndex(s.indexQueue)
	return &s
}

func (s *Search) Index(mdp opentsdb.MultiDataPoint) {
	for _, dp := range mdp {
		s.Lock()
		metric := dp.Metric
		key := metric + dp.Tags.String()
		p := s.Last[key]
		if p == nil {
			p = &lastInfo{}
			s.Last[key] = p
		}
		if p.timestamp < dp.Timestamp {
			if fv, err := getFloat(dp.Value); err == nil {
				p.diffFromPrev = (fv - p.lastVal) / float64(dp.Timestamp-p.timestamp)
				p.lastVal = fv
			} else {
				slog.Error(err)
			}
			p.timestamp = dp.Timestamp
		}
		s.Unlock()
		select {
		case s.indexQueue <- dp:
		default:
			collect.Add("search.dropped", opentsdb.TagSet{}, 1)
		}
	}
}

func (s *Search) redisIndex(c <-chan *opentsdb.DataPoint) {
	now := time.Now().Unix()
	nextUpdateTimes := make(map[string]int64)
	updateIfTime := func(key string, f func()) {
		nextUpdate, ok := nextUpdateTimes[key]
		if !ok || now > nextUpdate {
			f()
			nextUpdateTimes[key] = now + int64(30*60+rand.Intn(15*60)) //pick a random time between 30 and 45 minutes from now
		}
	}
	for dp := range c {
		now = time.Now().Unix()
		metric := dp.Metric
		for k, v := range dp.Tags {
			updateIfTime(fmt.Sprintf("kvm:%s:%s:%s", k, v, metric), func() {
				if err := s.DataAccess.Search_AddMetricForTag(k, v, metric, now); err != nil {
					slog.Error(err)
				}
				if err := s.DataAccess.Search_AddTagValue(metric, k, v, now); err != nil {
					slog.Error(err)
				}
			})
			updateIfTime(fmt.Sprintf("mk:%s:%s", metric, k), func() {
				if err := s.DataAccess.Search_AddTagKeyForMetric(metric, k, now); err != nil {
					slog.Error(err)
				}
			})
			updateIfTime(fmt.Sprintf("kv:%s:%s", k, v), func() {
				if err := s.DataAccess.Search_AddTagValue(database.Search_All, k, v, now); err != nil {
					slog.Error(err)
				}
			})
			updateIfTime(fmt.Sprintf("m:%s", metric), func() {
				if err := s.DataAccess.Search_AddMetric(metric, now); err != nil {
					slog.Error(err)
				}
			})
		}
	}
}

var floatType = reflect.TypeOf(float64(0))

func getFloat(unk interface{}) (float64, error) {
	v := reflect.ValueOf(unk)
	v = reflect.Indirect(v)
	if !v.Type().ConvertibleTo(floatType) {
		return 0, fmt.Errorf("cannot convert %v to float64", v.Type())
	}
	fv := v.Convert(floatType)
	return fv.Float(), nil
}

// Match returns all matching values against search. search is a regex, except
// that `.` is literal, `*` can be used for `.*`, and the entire string is
// searched (`^` and `&` added to ends of search).
func Match(search string, values []string) ([]string, error) {
	v := strings.Replace(search, ".", `\.`, -1)
	v = strings.Replace(v, "*", ".*", -1)
	v = "^" + v + "$"
	re, err := regexp.Compile(v)
	if err != nil {
		return nil, err
	}
	var nvs []string
	for _, nv := range values {
		if re.MatchString(nv) {
			nvs = append(nvs, nv)
		}
	}
	return nvs, nil
}

var errNotFloat = fmt.Errorf("last: expected float64")

// GetLast returns the value of the most recent data point for the given metric
// and tag. tags should be of the form "{key=val,key2=val2}". If diff is true,
// the value is treated as a counter. err is non nil if there is no match.
func (s *Search) GetLast(metric, tags string, diff bool) (v float64, err error) {
	s.RLock()
	p := s.Last[metric+tags]
	if p != nil {
		if diff {
			return p.diffFromPrev, nil
		}
		return p.lastVal, nil
	}
	s.RUnlock()
	return 0, nil
}

func (s *Search) Expand(q *opentsdb.Query) error {
	for k, ov := range q.Tags {
		var nvs []string
		for _, v := range strings.Split(ov, "|") {
			v = strings.TrimSpace(v)
			if v == "*" || !strings.Contains(v, "*") {
				nvs = append(nvs, v)
			} else {
				vs, err := s.TagValuesByMetricTagKey(q.Metric, k, 0)
				if err != nil {
					return err
				}
				ns, err := Match(v, vs)
				if err != nil {
					return err
				}
				nvs = append(nvs, ns...)
			}
		}
		if len(nvs) == 0 {
			return fmt.Errorf("expr: no tags matching %s=%s", k, ov)
		}
		q.Tags[k] = strings.Join(nvs, "|")
	}
	return nil
}

func (s *Search) UniqueMetrics() ([]string, error) {
	m, err := s.DataAccess.Search_GetAllMetrics()
	if err != nil {
		return nil, err
	}
	metrics := make([]string, len(m))
	i := 0
	for k := range m {
		metrics[i] = k
		i++
	}
	sort.Strings(metrics)
	return metrics, nil
}

func (s *Search) TagValuesByTagKey(Tagk string, since time.Duration) ([]string, error) {
	return s.TagValuesByMetricTagKey(database.Search_All, Tagk, since)
}

func (s *Search) MetricsByTagPair(tagk, tagv string) ([]string, error) {
	metrics, err := s.DataAccess.Search_GetMetricsForTag(tagk, tagv)
	if err != nil {
		return nil, err
	}
	r := []string{}
	for k := range metrics {
		r = append(r, k)
	}
	sort.Strings(r)
	return r, nil
}

func (s *Search) TagKeysByMetric(metric string) ([]string, error) {
	keys, err := s.DataAccess.Search_GetTagKeysForMetric(metric)
	if err != nil {
		return nil, err
	}
	r := []string{}
	for k := range keys {
		r = append(r, k)
	}
	sort.Strings(r)
	return r, nil
}

func (s *Search) TagValuesByMetricTagKey(metric, tagK string, since time.Duration) ([]string, error) {
	var t int64
	if since > 0 {
		t = time.Now().Add(-since).Unix()
	}
	vals, err := s.DataAccess.Search_GetTagValues(metric, tagK)
	if err != nil {
		return nil, err
	}
	r := []string{}
	for k, ts := range vals {
		if t <= ts {
			r = append(r, k)
		}
	}
	sort.Strings(r)
	return r, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"

	"github.com/flant/dapp/pkg/lock"
	"github.com/flant/dapp/pkg/ruby2go"

	"github.com/flant/kubedog/pkg/kube"
	"github.com/flant/kubedog/pkg/kubedog"
	"github.com/flant/kubedog/pkg/monitor"
)

func main() {
	var err error

	err = lock.Init()
	if err != nil {
		panic(err)
	}

	err = kube.Init()
	if err != nil {
		panic(err)
	}

	ruby2go.RunCli("deploy-watcher", func(args map[string]interface{}) (interface{}, error) {
		namespace := args["namespace"].(string)
		if namespace == "" {
			return nil, fmt.Errorf("namespace argument required!")
		}

		resourceName := args["resourceName"].(string)
		if resourceName == "" {
			return nil, fmt.Errorf("resourceName argument required!")
		}

		timeout := args["timeout"].(float64)

		switch action := args["action"]; action {
		case "watch job":
			err := kubedog.WatchJobTillDone(resourceName, namespace, kube.Kubernetes, monitor.WatchOptions{Timeout: time.Second * time.Duration(timeout)})
			if err != nil {
				return nil, fmt.Errorf("error watching job `%s` in namespace `%s`: %s", resourceName, namespace, err)
			}
		case "watch deployment":
			err := kubedog.WatchDeploymentTillReady(resourceName, namespace, kube.Kubernetes)
			if err != nil {
				return nil, fmt.Errorf("error watching deployment `%s` in namespace `%s`: %s", resourceName, namespace, err)
			}
		default:
			return nil, fmt.Errorf("unknown action \"%s\"", action)
		}

		return nil, nil
	})
}
<commit_msg>[go-deeper] Fix kubedog api change<commit_after>package main

import (
	"fmt"
	"time"

	"github.com/flant/dapp/pkg/lock"
	"github.com/flant/dapp/pkg/ruby2go"

	"github.com/flant/kubedog/pkg/kube"
	"github.com/flant/kubedog/pkg/tracker"
	"github.com/flant/kubedog/pkg/trackers/rollout"
)

func main() {
	var err error

	err = lock.Init()
	if err != nil {
		panic(err)
	}

	err = kube.Init()
	if err != nil {
		panic(err)
	}

	ruby2go.RunCli("deploy-watcher", func(args map[string]interface{}) (interface{}, error) {
		namespace := args["namespace"].(string)
		if namespace == "" {
			return nil, fmt.Errorf("namespace argument required!")
		}

		resourceName := args["resourceName"].(string)
		if resourceName == "" {
			return nil, fmt.Errorf("resourceName argument required!")
		}

		timeout := args["timeout"].(float64)

		switch action := args["action"]; action {
		case "watch job":
			err := rollout.TrackJobTillDone(resourceName, namespace, kube.Kubernetes, tracker.Options{Timeout: time.Second * time.Duration(timeout)})
			if err != nil {
				return nil, fmt.Errorf("error tracking job `%s` in namespace `%s`: %s", resourceName, namespace, err)
			}
		case "watch deployment":
			err := rollout.TrackDeploymentTillReady(resourceName, namespace, kube.Kubernetes, tracker.Options{Timeout: time.Second * time.Duration(timeout)})
			if err != nil {
				return nil, fmt.Errorf("error tracking deployment `%s` in namespace `%s`: %s", resourceName, namespace, err)
			}
		default:
			return nil, fmt.Errorf("unknown action \"%s\"", action)
		}

		return nil, nil
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"os"

	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/mcnerror"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
	cmdUtil "k8s.io/minikube/cmd/util"
	"k8s.io/minikube/pkg/minikube/cluster"
	pkg_config "k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/console"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/machine"
	pkgutil "k8s.io/minikube/pkg/util"
)

var deleteAll bool

// deleteCmd represents the delete command
var deleteCmd = &cobra.Command{
	Use:   "delete",
	Short: "Deletes a local kubernetes cluster",
	Long: `Deletes a local kubernetes cluster. This command deletes the VM, and removes all
associated files.`,
	Run: runDelete,
}

// runDelete handles the executes the flow of "minikube delete"
func runDelete(cmd *cobra.Command, args []string) {
	profileFlag, _ := cmd.Flags().GetString("profile")
	deleteAllFlag, _ := cmd.Flags().GetBool("all")

	if profileFlag != constants.DefaultMachineName && deleteAllFlag {
		exit.Usage("usage: minikube delete --all")
	}

	if deleteAllFlag {
		profiles := cmdcfg.GetAllProfiles()
		deleteAllProfiles(profiles)
	} else {
		if len(args) > 0 {
			exit.Usage("usage: minikube delete")
		}

		profileName := viper.GetString(pkg_config.MachineProfile)
		deleteProfile(profileName)
	}
}

func deleteProfile(profileName string) {
	api, err := machine.NewAPIClient()
	if err != nil {
		exit.WithError("Error getting client", err)
	}
	defer api.Close()

	cc, err := pkg_config.Load()
	if err != nil && !os.IsNotExist(err) {
		console.ErrLn("Error loading profile config: %v", err)
	}

	// In the case of "none", we want to uninstall Kubernetes as there is no VM to delete
	if err == nil && cc.MachineConfig.VMDriver == constants.DriverNone {
		uninstallKubernetes(api, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper))
	}

	if err = cluster.DeleteHost(api); err != nil {
		switch err := errors.Cause(err).(type) {
		case mcnerror.ErrHostDoesNotExist:
			console.OutStyle(console.Meh, "%q cluster does not exist", profileName)
		default:
			exit.WithError("Failed to delete cluster", err)
		}
	}

	if err := cmdUtil.KillMountProcess(); err != nil {
		console.Fatal("Failed to kill mount process: %v", err)
	}

	if err := os.RemoveAll(constants.GetProfilePath(viper.GetString(pkg_config.MachineProfile))); err != nil {
		if os.IsNotExist(err) {
			console.OutStyle(console.Meh, "%q profile does not exist", profileName)
			os.Exit(0)
		}
		exit.WithError("Failed to remove profile", err)
	}
	console.OutStyle(console.Crushed, "The %q cluster has been deleted.", profileName)

	machineName := pkg_config.GetMachineName()
	if err := pkgutil.DeleteKubeConfigContext(constants.KubeconfigPath, machineName); err != nil {
		exit.WithError("update config", err)
	}
}

func deleteAllProfiles(profiles []string) {
	for _, profile := range profiles {
		fmt.Println(profile)
	}
	os.Exit(0)
}

func uninstallKubernetes(api libmachine.API, kc pkg_config.KubernetesConfig, bsName string) {
	console.OutStyle(console.Resetting, "Uninstalling Kubernetes %s using %s ...", kc.KubernetesVersion, bsName)
	clusterBootstrapper, err := GetClusterBootstrapper(api, bsName)
	if err != nil {
		console.ErrLn("Unable to get bootstrapper: %v", err)
	} else if err = clusterBootstrapper.DeleteCluster(kc); err != nil {
		console.ErrLn("Failed to delete cluster: %v", err)
	}
}

func init() {
	deleteCmd.Flags().BoolVar(&deleteAll, "all", false, "Set flag to delete all profiles")
	RootCmd.AddCommand(deleteCmd)
}
<commit_msg>Implemented deleteAllProfiles<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"os"

	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/mcnerror"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
	cmdUtil "k8s.io/minikube/cmd/util"
	"k8s.io/minikube/pkg/minikube/cluster"
	pkg_config "k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/console"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/machine"
	pkgutil "k8s.io/minikube/pkg/util"
)

var deleteAll bool

// deleteCmd represents the delete command
var deleteCmd = &cobra.Command{
	Use:   "delete",
	Short: "Deletes a local kubernetes cluster",
	Long: `Deletes a local kubernetes cluster. This command deletes the VM, and removes all
associated files.`,
	Run: runDelete,
}

// runDelete handles the executes the flow of "minikube delete"
func runDelete(cmd *cobra.Command, args []string) {
	profileFlag, _ := cmd.Flags().GetString("profile")
	deleteAllFlag, _ := cmd.Flags().GetBool("all")

	if profileFlag != constants.DefaultMachineName && deleteAllFlag {
		exit.Usage("usage: minikube delete --all")
	}

	if deleteAllFlag {
		profiles := cmdcfg.GetAllProfiles()
		deleteAllProfiles(profiles)
	} else {
		if len(args) > 0 {
			exit.Usage("usage: minikube delete")
		}

		profileName := viper.GetString(pkg_config.MachineProfile)
		deleteProfile(profileName)
	}
}

func deleteProfile(profileName string) {
	api, err := machine.NewAPIClient()
	if err != nil {
		exit.WithError("Error getting client", err)
	}
	defer api.Close()

	cc, err := pkg_config.Load()
	if err != nil && !os.IsNotExist(err) {
		console.ErrLn("Error loading profile config: %v", err)
	}

	// In the case of "none", we want to uninstall Kubernetes as there is no VM to delete
	if err == nil && cc.MachineConfig.VMDriver == constants.DriverNone {
		uninstallKubernetes(api, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper))
	}

	if err = cluster.DeleteHost(api); err != nil {
		switch err := errors.Cause(err).(type) {
		case mcnerror.ErrHostDoesNotExist:
			console.OutStyle(console.Meh, "%q cluster does not exist", profileName)
		default:
			exit.WithError("Failed to delete cluster", err)
		}
	}

	if err := cmdUtil.KillMountProcess(); err != nil {
		console.Fatal("Failed to kill mount process: %v", err)
	}

	if err := os.RemoveAll(constants.GetProfilePath(viper.GetString(pkg_config.MachineProfile))); err != nil {
		if os.IsNotExist(err) {
			console.OutStyle(console.Meh, "%q profile does not exist", profileName)
			os.Exit(0)
		}
		exit.WithError("Failed to remove profile", err)
	}
	console.OutStyle(console.Crushed, "The %q cluster has been deleted.", profileName)

	machineName := pkg_config.GetMachineName()
	if err := pkgutil.DeleteKubeConfigContext(constants.KubeconfigPath, machineName); err != nil {
		exit.WithError("update config", err)
	}
}

func deleteAllProfiles(profiles []string) {
	for _, profile := range profiles {
		// TODO: Refactor: viper.Set seems to be in the wrong place
		viper.Set(pkg_config.MachineProfile, profile)
		deleteProfile(profile)
	}
	os.Exit(0)
}

func uninstallKubernetes(api libmachine.API, kc pkg_config.KubernetesConfig, bsName string) {
	console.OutStyle(console.Resetting, "Uninstalling Kubernetes %s using %s ...", kc.KubernetesVersion, bsName)
	clusterBootstrapper, err := GetClusterBootstrapper(api, bsName)
	if err != nil {
		console.ErrLn("Unable to get bootstrapper: %v", err)
	} else if err = clusterBootstrapper.DeleteCluster(kc); err != nil {
		console.ErrLn("Failed to delete cluster: %v", err)
	}
}

func init() {
	deleteCmd.Flags().BoolVar(&deleteAll, "all", false, "Set flag to delete all profiles")
	RootCmd.AddCommand(deleteCmd)
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package provision

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/apigee/apigee-remote-service-cli/shared"
	"github.com/apigee/apigee-remote-service-envoy/server"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"go.uber.org/multierr"
)

const (
	kvmName       = "remote-service"
	cacheName     = "remote-service"
	encryptKVM    = true
	authProxyName = "remote-service"

	remoteServiceProxyZip = "remote-service-gcp.zip"

	apiProductsPath = "apiproducts"

	certsURLFormat        = "%s/certs"        // RemoteServiceProxyURL
	productsURLFormat     = "%s/products"     // RemoteServiceProxyURL
	verifyAPIKeyURLFormat = "%s/verifyApiKey" // RemoteServiceProxyURL
	quotasURLFormat       = "%s/quotas"       // RemoteServiceProxyURL
)

// default durations for the proxy verification retry
var (
	duration time.Duration = 180  // second
	interval time.Duration = 5000 // millisecond
)

type provision struct {
	*shared.RootArgs
	forceProxyInstall bool
	virtualHosts      string
	rotate            int
}

// Cmd returns base command
func Cmd(rootArgs *shared.RootArgs, printf shared.FormatFn) *cobra.Command {
	p := &provision{RootArgs: rootArgs}

	c := &cobra.Command{
		Use:   "provision",
		Short: "Provision your Apigee environment for remote services",
		Long: `The provision command will set up your Apigee environment for remote services. This includes creating
and installing a remote-service kvm with certificates, creating credentials, and deploying a remote-service proxy
to your organization and environment.`,
		Args: cobra.NoArgs,
		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
			if err := rootArgs.Resolve(false, true); err != nil {
				return err
			}
			if !p.IsGCPManaged && p.rotate > 0 {
				return fmt.Errorf(`--rotate only valid for hybrid, use 'token rotate-cert' for others`)
			}
			return nil
		},

		RunE: func(cmd *cobra.Command, _ []string) error {
			return p.run(printf)
		},
	}

	c.Flags().StringVarP(&rootArgs.ManagementBase, "management", "m",
		shared.DefaultManagementBase, "Apigee management base URL")
	c.Flags().BoolVarP(&rootArgs.IsLegacySaaS, "legacy", "", false,
		"Apigee SaaS (sets management and runtime URL)")
	c.Flags().BoolVarP(&rootArgs.IsOPDK, "opdk", "", false,
		"Apigee opdk")

	c.Flags().StringVarP(&rootArgs.Token, "token", "t", "",
		"Apigee OAuth or SAML token (hybrid only)")
	c.Flags().StringVarP(&rootArgs.Username, "username", "u", "",
		"Apigee username (legacy or OPDK only)")
	c.Flags().StringVarP(&rootArgs.Password, "password", "p", "",
		"Apigee password (legacy or OPDK only)")

	c.Flags().BoolVarP(&p.forceProxyInstall, "force-proxy-install", "f", false,
		"force new proxy install (upgrades proxy)")
	c.Flags().StringVarP(&p.virtualHosts, "virtual-hosts", "", "default,secure",
		"override proxy virtualHosts")
	c.Flags().StringVarP(&p.Namespace, "namespace", "n", "apigee",
		"emit configuration in the specified namespace")

	c.Flags().IntVarP(&p.rotate, "rotate", "", 0, "if n > 0, generate new private key and keep n public keys (hybrid only)")

	return c
}

func (p *provision) run(printf shared.FormatFn) error {

	var cred *keySecret

	var verbosef = shared.NoPrintf
	if p.Verbose {
		verbosef = shared.Errorf
	}

	tempDir, err := ioutil.TempDir("", "apigee")
	if err != nil {
		return errors.Wrap(err, "creating temp dir")
	}
	defer os.RemoveAll(tempDir)

	replaceVH := func(proxyDir string) error {
		proxiesFile := filepath.Join(proxyDir, "proxies", "default.xml")
		bytes, err := ioutil.ReadFile(proxiesFile)
		if err != nil {
			return errors.Wrapf(err, "reading file %s", proxiesFile)
		}
		newVH := ""
		for _, vh := range strings.Split(p.virtualHosts, ",") {
			if strings.TrimSpace(vh) != "" {
				newVH = newVH + fmt.Sprintf(virtualHostReplacementFmt, vh)
			}
		}
		// remove all "secure" virtualhost
		bytes = []byte(strings.ReplaceAll(string(bytes), virtualHostDeleteText, ""))
		// replace the "default" virtualhost
		bytes = []byte(strings.Replace(string(bytes), virtualHostReplaceText, newVH, 1))
		if err := ioutil.WriteFile(proxiesFile, bytes, 0); err != nil {
			return errors.Wrapf(err, "writing file %s", proxiesFile)
		}
		return nil
	}

	replaceInFile := func(file, old, new string) error {
		bytes, err := ioutil.ReadFile(file)
		if err != nil {
			return errors.Wrapf(err, "reading file %s", file)
		}
		bytes = []byte(strings.Replace(string(bytes), old, new, 1))
		if err := ioutil.WriteFile(file, bytes, 0); err != nil {
			return errors.Wrapf(err, "writing file %s", file)
		}
		return nil
	}

	replaceVHAndAuthTarget := func(proxyDir string) error {
		if err := replaceVH(proxyDir); err != nil {
			return err
		}

		if p.IsOPDK {
			// OPDK must target local internal proxy
			authFile := filepath.Join(proxyDir, "policies", "Authenticate-Call.xml")
			oldTarget := "https://edgemicroservices.apigee.net"
			newTarget := p.RuntimeBase
			if err := replaceInFile(authFile, oldTarget, newTarget); err != nil {
				return err
			}

			// OPDK must have org.noncps = true for products callout
			calloutFile := filepath.Join(proxyDir, "policies", "JavaCallout.xml")
			oldValue := "</Properties>"
			newValue := `<Property name="org.noncps">true</Property>
			</Properties>`
			if err := replaceInFile(calloutFile, oldValue, newValue); err != nil {
				return err
			}
		}
		return nil
	}

	if p.IsOPDK {
		if err := p.deployInternalProxy(replaceVH, tempDir, verbosef); err != nil {
			return errors.Wrap(err, "deploying internal proxy")
		}
	}

	// input remote-service proxy
	var customizedProxy string
	if p.IsGCPManaged {
		customizedProxy, err = getCustomizedProxy(tempDir, remoteServiceProxyZip, nil)
	} else {
		customizedProxy, err = getCustomizedProxy(tempDir, legacyAuthProxyZip, replaceVHAndAuthTarget)
	}
	if err != nil {
		return err
	}

	if err := p.checkAndDeployProxy(authProxyName, customizedProxy, verbosef); err != nil {
		return errors.Wrapf(err, "deploying proxy %s", authProxyName)
	}

	// create API product
	if err := p.createAPIProduct(verbosef); err != nil {
		return errors.Wrapf(err, "creating remote-service API product")
	}

	if !p.IsGCPManaged {
		cred, err = p.createLegacyCredential(verbosef) // TODO: on missing or force new cred
		if err != nil {
			return errors.Wrapf(err, "generating credential")
		}

		if err := p.getOrCreateKVM(cred, verbosef); err != nil {
			return errors.Wrapf(err, "retrieving or creating kvm")
		}
	}

	config := p.ServerConfig
	if config == nil {
		config = p.createConfig(cred)
	}

	if p.IsGCPManaged && config.Tenant.PrivateKey == nil {
		keyID, privateKey, jwks, err := p.CreateNewKey()
		if err != nil {
			return err
		}
		config.Tenant.PrivateKey = privateKey
		config.Tenant.PrivateKeyID = keyID

		if jwks, err = p.RotateJKWS(jwks, p.rotate); err != nil {
			return err
		}

		config.Tenant.JWKS = jwks
	}

	var verifyErrors error
	if p.IsGCPManaged {
		verifyErrors = p.verifyWithRetry(config, verbosef)
	} else {
		verifyErrors = p.verifyWithoutRetry(config, verbosef)
	}

	if err := p.printConfig(config, printf, verifyErrors); err != nil {
		return errors.Wrapf(err, "generating config")
	}

	if verifyErrors == nil {
		verbosef("provisioning verified OK")
	}

	return verifyErrors
}

func (p *provision) createAuthorizedClient(config *server.Config) (*http.Client, error) {

	// add authorization to transport
	tr := http.DefaultTransport
	if config.Tenant.AllowUnverifiedSSLCert {
		tr = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			DialContext: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
				DualStack: true,
			}).DialContext,
			MaxIdleConns:          100,
			IdleConnTimeout:       90 * time.Second,
			TLSHandshakeTimeout:   10 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
			TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		}
	}

	tr, err := server.AuthorizationRoundTripper(config, tr)
	if err != nil {
		return nil, err
	}

	return &http.Client{
		Timeout:   config.Tenant.ClientTimeout,
		Transport: tr,
	}, nil
}

func (p *provision) verifyWithRetry(config *server.Config, verbosef shared.FormatFn) error {
	var verifyErrors error
	timeout := time.After(duration * time.Second)
	tick := time.Tick(interval * time.Millisecond)
	for {
		select {
		case <-timeout:
			if verifyErrors != nil {
				shared.Errorf("\nWARNING: Apigee may not be provisioned properly.")
				shared.Errorf("Unable to verify proxy endpoint(s). Errors:\n")
				for _, err := range multierr.Errors(verifyErrors) {
					if strings.Contains(err.Error(), "Unable to get the runtime version") {
						p.encodeUDCAEndpoint(config, verbosef)
					}
					shared.Errorf("  %s", err)
				}
				shared.Errorf("\n")
			}
			return verifyErrors
		case <-tick:
			verifyErrors = p.verify(config, verbosef)
			if verifyErrors == nil {
				return nil
			}
			verbosef("verifying proxies failed, trying again...")
		}
	}
}

func (p *provision) verifyWithoutRetry(config *server.Config, verbosef shared.FormatFn) error {
	verifyErrors := p.verify(config, verbosef)
	if verifyErrors != nil {
		shared.Errorf("\nWARNING: Apigee may not be provisioned properly.")
		shared.Errorf("Unable to verify proxy endpoint(s). Errors:\n")
		for _, err := range multierr.Errors(verifyErrors) {
			shared.Errorf("  %s", err)
		}
		shared.Errorf("\n")
	}
	return verifyErrors
}

func (p *provision) verify(config *server.Config, verbosef shared.FormatFn) error {

	client, err := p.createAuthorizedClient(config)
	if err != nil {
		return err
	}

	var verifyErrors error
	if p.IsLegacySaaS || p.IsOPDK {
		verbosef("verifying internal proxy...")
		verifyErrors = p.verifyInternalProxy(client, verbosef)
	}

	verbosef("verifying remote-service proxy...")
	verifyErrors = multierr.Combine(verifyErrors, p.verifyRemoteServiceProxy(client, verbosef))

	if p.IsGCPManaged {
		if version, err := p.checkRuntimeVersion(config, client, verbosef); err != nil {
			err = errors.Wrapf(err, "Unable to get the runtime version")
			verifyErrors = multierr.Combine(verifyErrors, err)
		} else if version >= "1.3.0" || version == "unknown" {
			p.encodeUDCAEndpoint(config, verbosef)
		}
	}

	return verifyErrors
}

// verify GET RemoteServiceProxyURL/certs
// verify GET RemoteServiceProxyURL/products
// verify POST RemoteServiceProxyURL/verifyApiKey
// verify POST RemoteServiceProxyURL/quotas
func (p *provision) verifyRemoteServiceProxy(client *http.Client, printf shared.FormatFn) error {

	verifyGET := func(targetURL string) error {
		req, err := http.NewRequest(http.MethodGet, targetURL, nil)
		if err != nil {
			return errors.Wrapf(err, "creating request")
		}
		res, err := client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		return err
	}

	var res *http.Response
	var verifyErrors error
	certsURL := fmt.Sprintf(certsURLFormat, p.RemoteServiceProxyURL)
	err := verifyGET(certsURL)
	verifyErrors = multierr.Append(verifyErrors, err)

	productsURL := fmt.Sprintf(productsURLFormat, p.RemoteServiceProxyURL)
	err = verifyGET(productsURL)
	verifyErrors = multierr.Append(verifyErrors, err)

	verifyAPIKeyURL := fmt.Sprintf(verifyAPIKeyURLFormat, p.RemoteServiceProxyURL)
	req, err := http.NewRequest(http.MethodPost, verifyAPIKeyURL, strings.NewReader(`{ "apiKey": "x" }`))
	if err == nil {
		req.Header.Add("Content-Type", "application/json")
		res, err = client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
	}
	if err != nil && (res == nil || res.StatusCode != 401) { // 401 is ok, we didn't use a valid api key
		verifyErrors = multierr.Append(verifyErrors, err)
	}

	quotasURL := fmt.Sprintf(quotasURLFormat, p.RemoteServiceProxyURL)
	req, err = http.NewRequest(http.MethodPost, quotasURL, strings.NewReader("{}"))
	if err == nil {
		req.Header.Add("Content-Type", "application/json")
		res, err = client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
	}
	if err != nil {
		verifyErrors = multierr.Append(verifyErrors, err)
	}

	return verifyErrors
}

type keySecret struct {
	Key    string `json:"key"`
	Secret string `json:"secret"`
}
<commit_msg>perform rotate when specified (#28)<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package provision

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/apigee/apigee-remote-service-cli/shared"
	"github.com/apigee/apigee-remote-service-envoy/server"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"go.uber.org/multierr"
)

const (
	kvmName       = "remote-service"
	cacheName     = "remote-service"
	encryptKVM    = true
	authProxyName = "remote-service"

	remoteServiceProxyZip = "remote-service-gcp.zip"

	apiProductsPath = "apiproducts"

	certsURLFormat        = "%s/certs"        // RemoteServiceProxyURL
	productsURLFormat     = "%s/products"     // RemoteServiceProxyURL
	verifyAPIKeyURLFormat = "%s/verifyApiKey" // RemoteServiceProxyURL
	quotasURLFormat       = "%s/quotas"       // RemoteServiceProxyURL
)

// default durations for the proxy verification retry
var (
	duration time.Duration = 180  // second
	interval time.Duration = 5000 // millisecond
)

type provision struct {
	*shared.RootArgs
	forceProxyInstall bool
	virtualHosts      string
	rotate            int
}

// Cmd returns base command
func Cmd(rootArgs *shared.RootArgs, printf shared.FormatFn) *cobra.Command {
	p := &provision{RootArgs: rootArgs}

	c := &cobra.Command{
		Use:   "provision",
		Short: "Provision your Apigee environment for remote services",
		Long: `The provision command will set up your Apigee environment for remote services. This includes creating
and installing a remote-service kvm with certificates, creating credentials, and deploying a remote-service proxy
to your organization and environment.`,
		Args: cobra.NoArgs,
		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
			if err := rootArgs.Resolve(false, true); err != nil {
				return err
			}
			if !p.IsGCPManaged && p.rotate > 0 {
				return fmt.Errorf(`--rotate only valid for hybrid, use 'token rotate-cert' for others`)
			}
			return nil
		},

		RunE: func(cmd *cobra.Command, _ []string) error {
			return p.run(printf)
		},
	}

	c.Flags().StringVarP(&rootArgs.ManagementBase, "management", "m",
		shared.DefaultManagementBase, "Apigee management base URL")
	c.Flags().BoolVarP(&rootArgs.IsLegacySaaS, "legacy", "", false,
		"Apigee SaaS (sets management and runtime URL)")
	c.Flags().BoolVarP(&rootArgs.IsOPDK, "opdk", "", false,
		"Apigee opdk")

	c.Flags().StringVarP(&rootArgs.Token, "token", "t", "",
		"Apigee OAuth or SAML token (hybrid only)")
	c.Flags().StringVarP(&rootArgs.Username, "username", "u", "",
		"Apigee username (legacy or OPDK only)")
	c.Flags().StringVarP(&rootArgs.Password, "password", "p", "",
		"Apigee password (legacy or OPDK only)")

	c.Flags().BoolVarP(&p.forceProxyInstall, "force-proxy-install", "f", false,
		"force new proxy install (upgrades proxy)")
	c.Flags().StringVarP(&p.virtualHosts, "virtual-hosts", "", "default,secure",
		"override proxy virtualHosts")
	c.Flags().StringVarP(&p.Namespace, "namespace", "n", "apigee",
		"emit configuration in the specified namespace")

	c.Flags().IntVarP(&p.rotate, "rotate", "", 0, "if n > 0, generate new private key and keep n public keys (hybrid only)")

	return c
}

func (p *provision) run(printf shared.FormatFn) error {

	var cred *keySecret

	var verbosef = shared.NoPrintf
	if p.Verbose {
		verbosef = shared.Errorf
	}

	tempDir, err := ioutil.TempDir("", "apigee")
	if err != nil {
		return errors.Wrap(err, "creating temp dir")
	}
	defer os.RemoveAll(tempDir)

	replaceVH := func(proxyDir string) error {
		proxiesFile := filepath.Join(proxyDir, "proxies", "default.xml")
		bytes, err := ioutil.ReadFile(proxiesFile)
		if err != nil {
			return errors.Wrapf(err, "reading file %s", proxiesFile)
		}
		newVH := ""
		for _, vh := range strings.Split(p.virtualHosts, ",") {
			if strings.TrimSpace(vh) != "" {
				newVH = newVH + fmt.Sprintf(virtualHostReplacementFmt, vh)
			}
		}
		// remove all "secure" virtualhost
		bytes = []byte(strings.ReplaceAll(string(bytes), virtualHostDeleteText, ""))
		// replace the "default" virtualhost
		bytes = []byte(strings.Replace(string(bytes), virtualHostReplaceText, newVH, 1))
		if err := ioutil.WriteFile(proxiesFile, bytes, 0); err != nil {
			return errors.Wrapf(err, "writing file %s", proxiesFile)
		}
		return nil
	}

	replaceInFile := func(file, old, new string) error {
		bytes, err := ioutil.ReadFile(file)
		if err != nil {
			return errors.Wrapf(err, "reading file %s", file)
		}
		bytes = []byte(strings.Replace(string(bytes), old, new, 1))
		if err := ioutil.WriteFile(file, bytes, 0); err != nil {
			return errors.Wrapf(err, "writing file %s", file)
		}
		return nil
	}

	replaceVHAndAuthTarget := func(proxyDir string) error {
		if err := replaceVH(proxyDir); err != nil {
			return err
		}

		if p.IsOPDK {
			// OPDK must target local internal proxy
			authFile := filepath.Join(proxyDir, "policies", "Authenticate-Call.xml")
			oldTarget := "https://edgemicroservices.apigee.net"
			newTarget := p.RuntimeBase
			if err := replaceInFile(authFile, oldTarget, newTarget); err != nil {
				return err
			}

			// OPDK must have org.noncps = true for products callout
			calloutFile := filepath.Join(proxyDir, "policies", "JavaCallout.xml")
			oldValue := "</Properties>"
			newValue := `<Property name="org.noncps">true</Property>
			</Properties>`
			if err := replaceInFile(calloutFile, oldValue, newValue); err != nil {
				return err
			}
		}
		return nil
	}

	if p.IsOPDK {
		if err := p.deployInternalProxy(replaceVH, tempDir, verbosef); err != nil {
			return errors.Wrap(err, "deploying internal proxy")
		}
	}

	// input remote-service proxy
	var customizedProxy string
	if p.IsGCPManaged {
		customizedProxy, err = getCustomizedProxy(tempDir, remoteServiceProxyZip, nil)
	} else {
		customizedProxy, err = getCustomizedProxy(tempDir, legacyAuthProxyZip, replaceVHAndAuthTarget)
	}
	if err != nil {
		return err
	}

	if err := p.checkAndDeployProxy(authProxyName, customizedProxy, verbosef); err != nil {
		return errors.Wrapf(err, "deploying proxy %s", authProxyName)
	}

	// create API product
	if err := p.createAPIProduct(verbosef); err != nil {
		return errors.Wrapf(err, "creating remote-service API product")
	}

	if !p.IsGCPManaged {
		cred, err = p.createLegacyCredential(verbosef) // TODO: on missing or force new cred
		if err != nil {
			return errors.Wrapf(err, "generating credential")
		}

		if err := p.getOrCreateKVM(cred, verbosef); err != nil {
			return errors.Wrapf(err, "retrieving or creating kvm")
		}
	}

	config := p.ServerConfig
	if config == nil {
		config = p.createConfig(cred)
	}

	if p.IsGCPManaged && (config.Tenant.PrivateKey == nil || p.rotate > 0) {
		keyID, privateKey, jwks, err := p.CreateNewKey()
		if err != nil {
			return err
		}
		config.Tenant.PrivateKey = privateKey
		config.Tenant.PrivateKeyID = keyID

		if jwks, err = p.RotateJKWS(jwks, p.rotate); err != nil {
			return err
		}

		config.Tenant.JWKS = jwks
	}

	var verifyErrors error
	if p.IsGCPManaged {
		verifyErrors = p.verifyWithRetry(config, verbosef)
	} else {
		verifyErrors = p.verifyWithoutRetry(config, verbosef)
	}

	if err := p.printConfig(config, printf, verifyErrors); err != nil {
		return errors.Wrapf(err, "generating config")
	}

	if verifyErrors == nil {
		verbosef("provisioning verified OK")
	}

	return verifyErrors
}

func (p *provision) createAuthorizedClient(config *server.Config) (*http.Client, error) {

	// add authorization to transport
	tr := http.DefaultTransport
	if config.Tenant.AllowUnverifiedSSLCert {
		tr = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			DialContext: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
				DualStack: true,
			}).DialContext,
			MaxIdleConns:          100,
			IdleConnTimeout:       90 * time.Second,
			TLSHandshakeTimeout:   10 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
			TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		}
	}

	tr, err := server.AuthorizationRoundTripper(config, tr)
	if err != nil {
		return nil, err
	}

	return &http.Client{
		Timeout:   config.Tenant.ClientTimeout,
		Transport: tr,
	}, nil
}

func (p *provision) verifyWithRetry(config *server.Config, verbosef shared.FormatFn) error {
	var verifyErrors error
	timeout := time.After(duration * time.Second)
	tick := time.Tick(interval * time.Millisecond)
	for {
		select {
		case <-timeout:
			if verifyErrors != nil {
				shared.Errorf("\nWARNING: Apigee may not be provisioned properly.")
				shared.Errorf("Unable to verify proxy endpoint(s). Errors:\n")
				for _, err := range multierr.Errors(verifyErrors) {
					if strings.Contains(err.Error(), "Unable to get the runtime version") {
						p.encodeUDCAEndpoint(config, verbosef)
					}
					shared.Errorf("  %s", err)
				}
				shared.Errorf("\n")
			}
			return verifyErrors
		case <-tick:
			verifyErrors = p.verify(config, verbosef)
			if verifyErrors == nil {
				return nil
			}
			verbosef("verifying proxies failed, trying again...")
		}
	}
}

func (p *provision) verifyWithoutRetry(config *server.Config, verbosef shared.FormatFn) error {
	verifyErrors := p.verify(config, verbosef)
	if verifyErrors != nil {
		shared.Errorf("\nWARNING: Apigee may not be provisioned properly.")
		shared.Errorf("Unable to verify proxy endpoint(s). Errors:\n")
		for _, err := range multierr.Errors(verifyErrors) {
			shared.Errorf("  %s", err)
		}
		shared.Errorf("\n")
	}
	return verifyErrors
}

func (p *provision) verify(config *server.Config, verbosef shared.FormatFn) error {

	client, err := p.createAuthorizedClient(config)
	if err != nil {
		return err
	}

	var verifyErrors error
	if p.IsLegacySaaS || p.IsOPDK {
		verbosef("verifying internal proxy...")
		verifyErrors = p.verifyInternalProxy(client, verbosef)
	}

	verbosef("verifying remote-service proxy...")
	verifyErrors = multierr.Combine(verifyErrors, p.verifyRemoteServiceProxy(client, verbosef))

	if p.IsGCPManaged {
		if version, err := p.checkRuntimeVersion(config, client, verbosef); err != nil {
			err = errors.Wrapf(err, "Unable to get the runtime version")
			verifyErrors = multierr.Combine(verifyErrors, err)
		} else if version >= "1.3.0" || version == "unknown" {
			p.encodeUDCAEndpoint(config, verbosef)
		}
	}

	return verifyErrors
}

// verify GET RemoteServiceProxyURL/certs
// verify GET RemoteServiceProxyURL/products
// verify POST RemoteServiceProxyURL/verifyApiKey
// verify POST RemoteServiceProxyURL/quotas
func (p *provision) verifyRemoteServiceProxy(client *http.Client, printf shared.FormatFn) error {

	verifyGET := func(targetURL string) error {
		req, err := http.NewRequest(http.MethodGet, targetURL, nil)
		if err != nil {
			return errors.Wrapf(err, "creating request")
		}
		res, err := client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		return err
	}

	var res *http.Response
	var verifyErrors error
	certsURL := fmt.Sprintf(certsURLFormat, p.RemoteServiceProxyURL)
	err := verifyGET(certsURL)
	verifyErrors = multierr.Append(verifyErrors, err)

	productsURL := fmt.Sprintf(productsURLFormat, p.RemoteServiceProxyURL)
	err = verifyGET(productsURL)
	verifyErrors = multierr.Append(verifyErrors, err)

	verifyAPIKeyURL := fmt.Sprintf(verifyAPIKeyURLFormat, p.RemoteServiceProxyURL)
	req, err := http.NewRequest(http.MethodPost, verifyAPIKeyURL, strings.NewReader(`{ "apiKey": "x" }`))
	if err == nil {
		req.Header.Add("Content-Type", "application/json")
		res, err = client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
	}
	if err != nil && (res == nil || res.StatusCode != 401) { // 401 is ok, we didn't use a valid api key
		verifyErrors = multierr.Append(verifyErrors, err)
	}

	quotasURL := fmt.Sprintf(quotasURLFormat, p.RemoteServiceProxyURL)
	req, err = http.NewRequest(http.MethodPost, quotasURL, strings.NewReader("{}"))
	if err == nil {
		req.Header.Add("Content-Type", "application/json")
		res, err = client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
	}
	if err != nil {
		verifyErrors = multierr.Append(verifyErrors, err)
	}

	return verifyErrors
}

type keySecret struct {
	Key    string `json:"key"`
	Secret string `json:"secret"`
}
<|endoftext|>