<commit_before>package move

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"os"
	"path"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/contacts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/vfs"
	vcardParser "github.com/emersion/go-vcard"
)

func createContact(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, db couchdb.Database) error {
	decoder := vcardParser.NewDecoder(tr)
	vcard, err := decoder.Decode()
	if err != nil {
		return err
	}

	fullname := "John Doe"
	contactname := contacts.Name{
		GivenName:  "John",
		FamilyName: "Doe",
	}

	name := vcard.Name()
	if name != nil {
		contactname = contacts.Name{
			FamilyName:     name.FamilyName,
			GivenName:      name.GivenName,
			AdditionalName: name.AdditionalName,
			NamePrefix:     name.HonorificPrefix,
			NameSuffix:     name.HonorificSuffix,
		}
		fullname = name.Value
	}
	if names := vcard.FormattedNames(); len(names) > 0 {
		fullname = names[0].Value
	}

	var bday string
	if field := vcard.Get("BDAY"); field != nil {
		bday = field.Value
	}

	var note string
	if field := vcard.Get("NOTE"); field != nil {
		note = field.Value
	}

	var contactemail []contacts.Email
	for _, mail := range vcard.Values("EMAIL") {
		ce := contacts.Email{
			Address: mail,
		}
		contactemail = append(contactemail, ce)
	}

	var contactphone []contacts.Phone
	for _, phone := range vcard.Values("TEL") {
		cp := contacts.Phone{
			Number: phone,
		}
		contactphone = append(contactphone, cp)
	}

	var contactaddress []contacts.Address
	for _, address := range vcard.Addresses() {
		ca := contacts.Address{
			Street:           address.StreetAddress,
			Pobox:            address.PostOfficeBox,
			City:             address.Locality,
			Region:           address.Region,
			Postcode:         address.PostalCode,
			Country:          address.Country,
			FormattedAddress: address.Value,
		}
		contactaddress = append(contactaddress, ca)
	}

	contact := &contacts.Contact{
		FullName: fullname,
		Name:     contactname,
		Birthday: bday,
		Note:     note,
		Address:  contactaddress,
		Email:    contactemail,
		Phone:    contactphone,
	}

	return couchdb.CreateDoc(db, contact)
}

func createAlbums(i *instance.Instance, tr *tar.Reader, albums *AlbumReferences) error {
	bs := bufio.NewScanner(tr)

	for bs.Scan() {
		jsondoc := &couchdb.JSONDoc{}
		if err := jsondoc.UnmarshalJSON(bs.Bytes()); err != nil {
			return err
		}
		delete(jsondoc.M, "type")
		id := jsondoc.ID()
		jsondoc.SetID("")
		jsondoc.SetRev("")
		jsondoc.Type = consts.PhotosAlbums

		if err := couchdb.CreateDoc(i, jsondoc); err != nil {
			return err
		}
		(*albums)[id] = couchdb.DocReference{
			ID:   jsondoc.ID(),
			Type: consts.PhotosAlbums,
		}
	}

	return nil
}

// AlbumReferences is used to associate photos to their albums, though we don't
// force the ID of the albums to the values in the tarball.
type AlbumReferences map[string]couchdb.DocReference

func fillAlbums(i *instance.Instance, tr *tar.Reader, dstDoc *vfs.DirDoc, albums *AlbumReferences) error {
	fs := i.VFS()
	bs := bufio.NewScanner(tr)

	for bs.Scan() {
		ref := Reference{}
		if err := json.Unmarshal(bs.Bytes(), &ref); err != nil {
			return err
		}

		file, err := fs.FileByPath(dstDoc.Fullpath + ref.Filepath)
		if err != nil {
			// XXX Ignore missing photos (we have this for migrating some cozy v2)
			continue
		}

		if docRef, ok := (*albums)[ref.Albumid]; ok {
			file.AddReferencedBy(docRef)
			if err = couchdb.UpdateDoc(i, file); err != nil {
				return err
			}
		}
	}

	return nil
}

func createFile(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, dstDoc *vfs.DirDoc) error {
	name := strings.TrimPrefix(hdr.Name, "files/")
	filename := path.Base(name)
	mime, class := vfs.ExtractMimeAndClassFromFilename(filename)
	now := time.Now()
	executable := hdr.FileInfo().Mode()&0100 != 0

	dirname := path.Join(dstDoc.Fullpath, path.Dir(name))
	dirDoc, err := fs.DirByPath(dirname)
	if err != nil {
		// XXX Tarball from cozy v2 exports can have files in a non-existant directory
		if dirDoc, err = vfs.MkdirAll(fs, dirname, nil); err != nil {
			return err
		}
	}
	fileDoc, err := vfs.NewFileDoc(filename, dirDoc.ID(), hdr.Size, nil, mime, class, now, executable, false, nil)
	if err != nil {
		return err
	}

	file, err := fs.CreateFile(fileDoc, nil)
	if err != nil {
		ext := path.Ext(fileDoc.DocName)
		fileName := fileDoc.DocName[0 : len(fileDoc.DocName)-len(ext)]
		fileDoc.DocName = fmt.Sprintf("%s-conflict-%d%s", fileName, rand.Int(), ext)
		file, err = fs.CreateFile(fileDoc, nil)
		if err != nil {
			return err
		}
	}

	_, err = io.Copy(file, tr)
	cerr := file.Close()
	if err != nil {
		return err
	}
	return cerr
}

// untar untar doc directory
func untar(r io.Reader, dst *vfs.DirDoc, instance *instance.Instance) error {
	fs := instance.VFS()

	// tar+gzip reader
	gr, err := gzip.NewReader(r)
	if err != nil {
		logger.WithDomain(instance.Domain).Errorf("Can't open gzip reader for import: %s", err)
		return err
	}
	defer gr.Close()
	tgz := tar.NewReader(gr)

	albumsRef := make(AlbumReferences)

	for {
		hdr, errb := tgz.Next()
		if errb == io.EOF {
			break
		}
		if errb != nil {
			logger.WithDomain(instance.Domain).Errorf("Error on import: %s", errb)
			return errb
		}

		parts := strings.SplitN(path.Clean(hdr.Name), "/", 2)
		var name, doctype string
		if len(parts) > 1 {
			doctype = parts[0]
			name = parts[1]
		}

		switch hdr.Typeflag {
		case tar.TypeDir:
			if doctype == "files" {
				dirname := path.Join(dst.Fullpath, name)
				if _, err = vfs.MkdirAll(fs, dirname, nil); err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import directory %s: %s", hdr.Name, err)
				}
			}

		case tar.TypeReg:
			if doctype == "albums" && name == albumsFile {
				err = createAlbums(instance, tgz, &albumsRef)
				if err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import album %s: %s", hdr.Name, err)
				}
			} else if doctype == "albums" && name == referencesFile {
				err = fillAlbums(instance, tgz, dst, &albumsRef)
				if err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import album %s: %s", hdr.Name, err)
				}
			} else if doctype == "contacts" {
				if err = createContact(fs, hdr, tgz, instance); err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import contact %s: %s", hdr.Name, err)
				}
			} else if doctype == "files" {
				if err = createFile(fs, hdr, tgz, dst); err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import file %s: %s", hdr.Name, err)
				}
			}

		default:
			logger.WithDomain(instance.Domain).Errorf("Unknown typeflag for import: %v", hdr.Typeflag)
			return errors.New("Unknown typeflag")
		}
	}

	return err
}

// Import is used to import a tarball with files, photos, contacts to an instance
func Import(instance *instance.Instance, filename, destination string) error {
	r, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer r.Close()

	fs := instance.VFS()
	exist, err := vfs.DirExists(fs, destination)
	if err != nil {
		logger.WithDomain(instance.Domain).Errorf("Error for destination %s: %s", destination, err)
		return err
	}
	var dst *vfs.DirDoc
	if !exist {
		dst, err = vfs.Mkdir(fs, destination, nil)
		if err != nil {
			logger.WithDomain(instance.Domain).Errorf("Can't create destination directory %s: %s", destination, err)
			return err
		}
	} else {
		dst, err = fs.DirByPath(destination)
		if err != nil {
			logger.WithDomain(instance.Domain).Errorf("Can't find destination directory %s: %s", destination, err)
			return err
		}
	}

	return untar(r, dst, instance)
}
<commit_msg>Avoid duplicate directories when important a tarball<commit_after>package move

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"os"
	"path"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/contacts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/vfs"
	vcardParser "github.com/emersion/go-vcard"
)

func createContact(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, db couchdb.Database) error {
	decoder := vcardParser.NewDecoder(tr)
	vcard, err := decoder.Decode()
	if err != nil {
		return err
	}

	fullname := "John Doe"
	contactname := contacts.Name{
		GivenName:  "John",
		FamilyName: "Doe",
	}

	name := vcard.Name()
	if name != nil {
		contactname = contacts.Name{
			FamilyName:     name.FamilyName,
			GivenName:      name.GivenName,
			AdditionalName: name.AdditionalName,
			NamePrefix:     name.HonorificPrefix,
			NameSuffix:     name.HonorificSuffix,
		}
		fullname = name.Value
	}
	if names := vcard.FormattedNames(); len(names) > 0 {
		fullname = names[0].Value
	}

	var bday string
	if field := vcard.Get("BDAY"); field != nil {
		bday = field.Value
	}

	var note string
	if field := vcard.Get("NOTE"); field != nil {
		note = field.Value
	}

	var contactemail []contacts.Email
	for _, mail := range vcard.Values("EMAIL") {
		ce := contacts.Email{
			Address: mail,
		}
		contactemail = append(contactemail, ce)
	}

	var contactphone []contacts.Phone
	for _, phone := range vcard.Values("TEL") {
		cp := contacts.Phone{
			Number: phone,
		}
		contactphone = append(contactphone, cp)
	}

	var contactaddress []contacts.Address
	for _, address := range vcard.Addresses() {
		ca := contacts.Address{
			Street:           address.StreetAddress,
			Pobox:            address.PostOfficeBox,
			City:             address.Locality,
			Region:           address.Region,
			Postcode:         address.PostalCode,
			Country:          address.Country,
			FormattedAddress: address.Value,
		}
		contactaddress = append(contactaddress, ca)
	}

	contact := &contacts.Contact{
		FullName: fullname,
		Name:     contactname,
		Birthday: bday,
		Note:     note,
		Address:  contactaddress,
		Email:    contactemail,
		Phone:    contactphone,
	}

	return couchdb.CreateDoc(db, contact)
}

func createAlbums(i *instance.Instance, tr *tar.Reader, albums *AlbumReferences) error {
	bs := bufio.NewScanner(tr)

	for bs.Scan() {
		jsondoc := &couchdb.JSONDoc{}
		if err := jsondoc.UnmarshalJSON(bs.Bytes()); err != nil {
			return err
		}
		delete(jsondoc.M, "type")
		id := jsondoc.ID()
		jsondoc.SetID("")
		jsondoc.SetRev("")
		jsondoc.Type = consts.PhotosAlbums

		if err := couchdb.CreateDoc(i, jsondoc); err != nil {
			return err
		}
		(*albums)[id] = couchdb.DocReference{
			ID:   jsondoc.ID(),
			Type: consts.PhotosAlbums,
		}
	}

	return nil
}

// AlbumReferences is used to associate photos to their albums, though we don't
// force the ID of the albums to the values in the tarball.
type AlbumReferences map[string]couchdb.DocReference

func fillAlbums(i *instance.Instance, tr *tar.Reader, dstDoc *vfs.DirDoc, albums *AlbumReferences) error {
	fs := i.VFS()
	bs := bufio.NewScanner(tr)

	for bs.Scan() {
		ref := Reference{}
		if err := json.Unmarshal(bs.Bytes(), &ref); err != nil {
			return err
		}

		file, err := fs.FileByPath(dstDoc.Fullpath + ref.Filepath)
		if err != nil {
			// XXX Ignore missing photos (we have this for migrating some cozy v2)
			continue
		}

		if docRef, ok := (*albums)[ref.Albumid]; ok {
			file.AddReferencedBy(docRef)
			if err = couchdb.UpdateDoc(i, file); err != nil {
				return err
			}
		}
	}

	return nil
}

func createFile(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, dstDoc *vfs.DirDoc, dirs map[string]*vfs.DirDoc) error {
	var err error
	name := strings.TrimPrefix(hdr.Name, "files/")
	filename := path.Base(name)
	mime, class := vfs.ExtractMimeAndClassFromFilename(filename)
	now := time.Now()
	executable := hdr.FileInfo().Mode()&0100 != 0

	dirname := path.Join(dstDoc.Fullpath, path.Dir(name))
	dirDoc, ok := dirs[dirname]
	if !ok {
		// XXX Tarball from cozy v2 exports can have files in a non-existant directory
		if dirDoc, err = vfs.MkdirAll(fs, dirname, nil); err != nil {
			return err
		}
		dirs[dirname] = dirDoc
	}
	fileDoc, err := vfs.NewFileDoc(filename, dirDoc.ID(), hdr.Size, nil, mime, class, now, executable, false, nil)
	if err != nil {
		return err
	}

	file, err := fs.CreateFile(fileDoc, nil)
	if err != nil {
		ext := path.Ext(fileDoc.DocName)
		fileName := fileDoc.DocName[0 : len(fileDoc.DocName)-len(ext)]
		fileDoc.DocName = fmt.Sprintf("%s-conflict-%d%s", fileName, rand.Int(), ext)
		file, err = fs.CreateFile(fileDoc, nil)
		if err != nil {
			return err
		}
	}

	_, err = io.Copy(file, tr)
	cerr := file.Close()
	if err != nil {
		return err
	}
	return cerr
}

// untar untar doc directory
func untar(r io.Reader, dst *vfs.DirDoc, instance *instance.Instance) error {
	fs := instance.VFS()

	// tar+gzip reader
	gr, err := gzip.NewReader(r)
	if err != nil {
		logger.WithDomain(instance.Domain).Errorf("Can't open gzip reader for import: %s", err)
		return err
	}
	defer gr.Close()
	tgz := tar.NewReader(gr)

	albumsRef := make(AlbumReferences)
	dirs := make(map[string]*vfs.DirDoc)

	for {
		hdr, errb := tgz.Next()
		if errb == io.EOF {
			break
		}
		if errb != nil {
			logger.WithDomain(instance.Domain).Errorf("Error on import: %s", errb)
			return errb
		}

		parts := strings.SplitN(path.Clean(hdr.Name), "/", 2)
		var name, doctype string
		if len(parts) > 1 {
			doctype = parts[0]
			name = parts[1]
		}

		switch hdr.Typeflag {
		case tar.TypeDir:
			if doctype == "files" {
				dirname := path.Join(dst.Fullpath, name)
				if dir, err := vfs.MkdirAll(fs, dirname, nil); err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import directory %s: %s", hdr.Name, err)
				} else {
					dirs[dirname] = dir
				}
			}

		case tar.TypeReg:
			if doctype == "albums" && name == albumsFile {
				err = createAlbums(instance, tgz, &albumsRef)
				if err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import album %s: %s", hdr.Name, err)
				}
			} else if doctype == "albums" && name == referencesFile {
				err = fillAlbums(instance, tgz, dst, &albumsRef)
				if err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import album %s: %s", hdr.Name, err)
				}
			} else if doctype == "contacts" {
				if err = createContact(fs, hdr, tgz, instance); err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import contact %s: %s", hdr.Name, err)
				}
			} else if doctype == "files" {
				if err = createFile(fs, hdr, tgz, dst, dirs); err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import file %s: %s", hdr.Name, err)
				}
			}

		default:
			logger.WithDomain(instance.Domain).Errorf("Unknown typeflag for import: %v", hdr.Typeflag)
			return errors.New("Unknown typeflag")
		}
	}

	return err
}

// Import is used to import a tarball with files, photos, contacts to an instance
func Import(instance *instance.Instance, filename, destination string) error {
	r, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer r.Close()

	fs := instance.VFS()
	exist, err := vfs.DirExists(fs, destination)
	if err != nil {
		logger.WithDomain(instance.Domain).Errorf("Error for destination %s: %s", destination, err)
		return err
	}
	var dst *vfs.DirDoc
	if !exist {
		dst, err = vfs.Mkdir(fs, destination, nil)
		if err != nil {
			logger.WithDomain(instance.Domain).Errorf("Can't create destination directory %s: %s", destination, err)
			return err
		}
	} else {
		dst, err = fs.DirByPath(destination)
		if err != nil {
			logger.WithDomain(instance.Domain).Errorf("Can't find destination directory %s: %s", destination, err)
			return err
		}
	}

	return untar(r, dst, instance)
}
<|endoftext|>


<commit_before>/*

Package oplog provides an HTTP Server-Sent Events (SSE) endpoint for real-time replication of the BlobStore.

*/
package oplog // import "a4.io/blobstash/pkg/oplog"

import (
	"context"
	"fmt"
	"net/http"

	"a4.io/blobstash/pkg/blob"
	"a4.io/blobstash/pkg/config"
	"a4.io/blobstash/pkg/hub"

	"github.com/gorilla/mux"
	log "github.com/inconshreveable/log15"
)

type Oplog struct {
	broker *Broker
	hub    *hub.Hub
	log    log.Logger
}

func New(logger log.Logger, conf *config.Config, h *hub.Hub) (*Oplog, error) {
	oplog := &Oplog{
		log: logger,
		broker: &Broker{
			log:            logger.New("submodule", "broker"),
			clients:        make(map[chan string]bool),
			newClients:     make(chan (chan string)),
			defunctClients: make(chan (chan string)),
			messages:       make(chan string),
		},
		hub: h,
	}
	oplog.init()
	return oplog, nil
}

func (o *Oplog) newBlobCallback(ctx context.Context, blob *blob.Blob, _ interface{}) error {
	// Send the blob hash to the broker
	o.broker.messages <- blob.Hash
	return nil
}

func (o *Oplog) Register(r *mux.Router, basicAuth func(http.Handler) http.Handler) {
	// Register the SSE HTTP endpoint
	r.Handle("/", basicAuth(o.broker))
}

func (o *Oplog) init() {
	// Start the SSE broker worker
	o.broker.start()
	// Register to the new blob event
	o.hub.Subscribe(hub.NewBlob, "oplog", o.newBlobCallback)
}

type Broker struct {
	log            log.Logger
	clients        map[chan string]bool
	newClients     chan chan string
	defunctClients chan chan string
	messages       chan string
}

func (b *Broker) start() {
	go func() {
		for {
			select {

			case s := <-b.newClients:
				// There is a new client attached and we
				// want to start sending them messages.
				b.clients[s] = true
				b.log.Debug("added new client", "client", s)

			case s := <-b.defunctClients:
				// A client has dettached and we want to
				// stop sending them messages.
				delete(b.clients, s)
				close(s)
				b.log.Debug("removed client", "client", s)

			case msg := <-b.messages:
				// There is a new message to send.  For each
				// attached client, push the new message
				// into the client's message channel.
				for s, _ := range b.clients {
					s <- msg
				}
				b.log.Info("message sent", "msg", msg, "clients_count", len(b.clients))
			}
		}
	}()
}

func (b *Broker) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	f, ok := w.(http.Flusher)
	if !ok {
		http.Error(w, "Streaming unsupported!", http.StatusInternalServerError)
		return
	}

	// Create a new channel, over which the broker can
	// send this client messages.
	messageChan := make(chan string)

	// Add this client to the map of those that should
	// receive updates
	b.newClients <- messageChan

	// Listen to the closing of the http connection via the CloseNotifier
	notify := w.(http.CloseNotifier).CloseNotify()
	go func() {
		<-notify
		// Remove this client from the map of attached clients
		// when `EventHandler` exits.
		b.defunctClients <- messageChan
	}()

	// Set the headers related to event streaming.
	w.Header().Set("Content-Type", "text/event-stream")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Connection", "keep-alive")

	for {

		// Read from our messageChan.
		msg, open := <-messageChan

		if !open {
			// If our messageChan was closed, this means that the client has
			// disconnected.
			break
		}

		// Write to the ResponseWriter, `w`.
		fmt.Fprintf(w, "data: Message: %s\n\n", msg)

		// Flush the response.  This is only possible if
		// the repsonse supports streaming.
		f.Flush()
	}
}
<commit_msg>oplog: send heatbeat to keep connection open<commit_after>/*

Package oplog provides an HTTP Server-Sent Events (SSE) endpoint for real-time replication of the BlobStore.

*/
package oplog // import "a4.io/blobstash/pkg/oplog"

import (
	"context"
	"fmt"
	"net/http"
	"time"

	"a4.io/blobstash/pkg/blob"
	"a4.io/blobstash/pkg/config"
	"a4.io/blobstash/pkg/hub"

	"github.com/gorilla/mux"
	log "github.com/inconshreveable/log15"
)

type Oplog struct {
	broker    *Broker
	hub       *hub.Hub
	log       log.Logger
	heartbeat *time.Ticker
}

type Op struct {
	Event, Data string
}

func New(logger log.Logger, conf *config.Config, h *hub.Hub) (*Oplog, error) {
	oplog := &Oplog{
		log:       logger,
		heartbeat: time.NewTicker(30 * time.Second),
		broker: &Broker{
			log:            logger.New("submodule", "broker"),
			clients:        make(map[chan *Op]bool),
			newClients:     make(chan (chan *Op)),
			defunctClients: make(chan (chan *Op)),
			ops:            make(chan *Op),
		},
		hub: h,
	}
	oplog.init()
	return oplog, nil
}

func (o *Oplog) newBlobCallback(ctx context.Context, blob *blob.Blob, _ interface{}) error {
	// Send the blob hash to the broker
	o.broker.ops <- &Op{Event: "blob", Data: blob.Hash}
	return nil
}

func (o *Oplog) Register(r *mux.Router, basicAuth func(http.Handler) http.Handler) {
	// Register the SSE HTTP endpoint
	r.Handle("/", basicAuth(o.broker))
}

func (o *Oplog) init() {
	// Start the SSE broker worker
	o.broker.start()
	// Register to the new blob event
	o.hub.Subscribe(hub.NewBlob, "oplog", o.newBlobCallback)

	go func() {
		for {
			<-o.heartbeat.C
			o.broker.ops <- &Op{Event: "heatbeat", Data: ""}
		}
	}()
}

type Broker struct {
	log            log.Logger
	clients        map[chan *Op]bool
	newClients     chan chan *Op
	defunctClients chan chan *Op
	ops            chan *Op
}

func (b *Broker) start() {
	go func() {
		for {
			select {
			case s := <-b.newClients:
				// There is a new client attached and we
				// want to start sending them messages.
				b.clients[s] = true
				b.log.Debug("added new client")

			case s := <-b.defunctClients:
				// A client has dettached and we want to
				// stop sending them messages.
				delete(b.clients, s)
				close(s)
				b.log.Debug("removed client")

			case op := <-b.ops:
				// There is a new message to send.  For each
				// attached client, push the new message
				// into the client's message channel.
				for s, _ := range b.clients {
					s <- op
				}
				b.log.Info("message sent", "op", op, "clients_count", len(b.clients))
			}
		}
	}()
}

func (b *Broker) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	f, ok := w.(http.Flusher)
	if !ok {
		http.Error(w, "Streaming unsupported!", http.StatusInternalServerError)
		return
	}

	// Create a new channel, over which the broker can
	// send this client messages.
	opChan := make(chan *Op)

	// Add this client to the map of those that should
	// receive updates
	b.newClients <- opChan

	// Listen to the closing of the http connection via the CloseNotifier
	notify := w.(http.CloseNotifier).CloseNotify()
	go func() {
		<-notify
		// Remove this client from the map of attached clients
		// when `EventHandler` exits.
		b.defunctClients <- opChan
	}()

	// Set the headers related to event streaming.
	w.Header().Set("Content-Type", "text/event-stream")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Connection", "keep-alive")

	// Send an initial heatbeat
	fmt.Fprintf(w, "event: heatbeat\ndata: \n\n")
	f.Flush()

	for {

		// Read from our messageChan.
		op, open := <-opChan

		if !open {
			// If our messageChan was closed, this means that the client has
			// disconnected.
			break
		}

		// Write to the ResponseWriter, `w`.
		fmt.Fprintf(w, "event: %s\n", op.Event)
		fmt.Fprintf(w, "data: %s\n\n", op.Data)

		// Flush the response.  This is only possible if
		// the repsonse supports streaming.
		f.Flush()
	}
}
<|endoftext|>


<commit_before>/*******************************************************************************
*
* Copyright 2018 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package plugins

import (
	"errors"
	"time"

	"github.com/gophercloud/gophercloud"
	"github.com/sapcc/go-bits/logg"
	"github.com/sapcc/limes"
	"github.com/sapcc/limes/pkg/core"
)

type cfmPlugin struct {
	cfg       core.ServiceConfiguration
	projectID string

	shareserversCache        []cfmShareserver
	shareserversCacheExpires time.Time
}

func init() {
	core.RegisterQuotaPlugin(func(c core.ServiceConfiguration, scrapeSubresources map[string]bool) core.QuotaPlugin {
		return &cfmPlugin{cfg: c}
	})
}

//Init implements the core.QuotaPlugin interface.
func (p *cfmPlugin) Init(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts) (err error) {
	p.projectID, err = getProjectIDForToken(provider, eo)
	return err
}

//ServiceInfo implements the core.QuotaPlugin interface.
func (p *cfmPlugin) ServiceInfo() limes.ServiceInfo {
	return limes.ServiceInfo{
		Type:        "database",
		ProductName: "cfm",
		Area:        "storage",
	}
}

//Resources implements the core.QuotaPlugin interface.
func (p *cfmPlugin) Resources() []limes.ResourceInfo {
	return []limes.ResourceInfo{{
		Name: "cfm_share_capacity",
		Unit: limes.UnitBytes,
		//need explicit permission to set quota for this service
		ExternallyManaged: !p.cfg.CFM.Authoritative,
	}}
}

//Scrape implements the core.QuotaPlugin interface.
func (p *cfmPlugin) Scrape(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts, clusterID, domainUUID, projectUUID string) (map[string]core.ResourceData, error) {
	client, err := newCFMClient(provider, eo, p.projectID)
	if err != nil {
		return nil, err
	}

	//prefer the new quota API if it is available
	var data struct {
		StorageQuota struct {
			SizeLimitBytes int64 `json:"size_limit"`
			Usage          struct {
				PotentialGrowthSize uint64 `json:"potential_growth_size"`
				BytesUsed           uint64 `json:"size_used"`
			} `json:"usage"`
		} `json:"storage_quota"`
	}
	err = client.GetQuotaSet(projectUUID).ExtractInto(&data)
	if err == nil {
		logg.Info("using CFM quota set for project %s", projectUUID)
		if !p.cfg.CFM.ReportPhysicalUsage {
			return map[string]core.ResourceData{
				"cfm_share_capacity": {
					Quota: data.StorageQuota.SizeLimitBytes,
					Usage: data.StorageQuota.Usage.BytesUsed,
				},
			}, nil
		}
		physicalUsage := data.StorageQuota.Usage.BytesUsed
		return map[string]core.ResourceData{
			"cfm_share_capacity": {
				Quota:         data.StorageQuota.SizeLimitBytes,
				Usage:         data.StorageQuota.Usage.PotentialGrowthSize,
				PhysicalUsage: &physicalUsage,
			},
		}, nil
	}

	//never use the old API when we're instructed to only read quotas
	if p.cfg.CFM.Authoritative {
		if _, ok := err.(cfmNotFoundError); ok {
			return map[string]core.ResourceData{"cfm_share_capacity": {Quota: 0, Usage: 0}}, nil
		}
		return nil, err
	}

	return p.scrapeOld(client, projectUUID)
}

func (p *cfmPlugin) scrapeOld(client *cfmClient, projectUUID string) (map[string]core.ResourceData, error) {
	//cache the result of cfmListShareservers(), it's mildly expensive
	now := time.Now()
	if p.shareserversCache == nil || p.shareserversCacheExpires.Before(now) {
		shareservers, err := client.ListShareservers()
		if err != nil {
			return nil, err
		}
		p.shareserversCache = shareservers
		p.shareserversCacheExpires = now.Add(5 * time.Minute)
	}
	shareservers := p.shareserversCache

	result := core.ResourceData{Quota: 0, Usage: 0}
	for _, shareserver := range shareservers {
		if shareserver.ProjectUUID != projectUUID {
			continue
		}

		shareserverDetailed, err := client.GetShareserver(shareserver.DetailsURL)
		if err != nil {
			return nil, err
		}

		result.Quota += int64(shareserverDetailed.BytesUsed)
		result.Usage += shareserverDetailed.BytesUsed
	}

	return map[string]core.ResourceData{"cfm_share_capacity": result}, nil
}

//SetQuota implements the core.QuotaPlugin interface.
func (p *cfmPlugin) SetQuota(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts, clusterID, domainUUID, projectUUID string, quotas map[string]uint64) error {
	if !p.cfg.CFM.Authoritative {
		return errors.New("the database/cfm_share_capacity resource is externally managed")
	}

	client, err := newCFMClient(provider, eo, p.projectID)
	if err != nil {
		return err
	}
	quotaBytes := quotas["cfm_share_capacity"]
	err = client.UpdateQuotaSet(projectUUID, quotaBytes)
	if _, ok := err.(cfmNotFoundError); ok {
		if quotaBytes == 0 {
			return nil //nothing to do: quota does not exist, but is also not wanted
		}
		err = client.CreateQuotaSet(projectUUID, quotaBytes)
	}
	return err
}
<commit_msg>[cfm] Project usage based on storage_quota.usage.size<commit_after>/*******************************************************************************
*
* Copyright 2018 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package plugins

import (
	"errors"
	"time"

	"github.com/gophercloud/gophercloud"
	"github.com/sapcc/go-bits/logg"
	"github.com/sapcc/limes"
	"github.com/sapcc/limes/pkg/core"
)

type cfmPlugin struct {
	cfg       core.ServiceConfiguration
	projectID string

	shareserversCache        []cfmShareserver
	shareserversCacheExpires time.Time
}

func init() {
	core.RegisterQuotaPlugin(func(c core.ServiceConfiguration, scrapeSubresources map[string]bool) core.QuotaPlugin {
		return &cfmPlugin{cfg: c}
	})
}

//Init implements the core.QuotaPlugin interface.
func (p *cfmPlugin) Init(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts) (err error) {
	p.projectID, err = getProjectIDForToken(provider, eo)
	return err
}

//ServiceInfo implements the core.QuotaPlugin interface.
func (p *cfmPlugin) ServiceInfo() limes.ServiceInfo {
	return limes.ServiceInfo{
		Type:        "database",
		ProductName: "cfm",
		Area:        "storage",
	}
}

//Resources implements the core.QuotaPlugin interface.
func (p *cfmPlugin) Resources() []limes.ResourceInfo {
	return []limes.ResourceInfo{{
		Name: "cfm_share_capacity",
		Unit: limes.UnitBytes,
		//need explicit permission to set quota for this service
		ExternallyManaged: !p.cfg.CFM.Authoritative,
	}}
}

//Scrape implements the core.QuotaPlugin interface.
func (p *cfmPlugin) Scrape(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts, clusterID, domainUUID, projectUUID string) (map[string]core.ResourceData, error) {
	client, err := newCFMClient(provider, eo, p.projectID)
	if err != nil {
		return nil, err
	}

	//prefer the new quota API if it is available
	var data struct {
		StorageQuota struct {
			SizeLimitBytes int64 `json:"size_limit"`
			Usage          struct {
				Size      uint64 `json:"size"`
				BytesUsed uint64 `json:"size_used"`
			} `json:"usage"`
		} `json:"storage_quota"`
	}
	err = client.GetQuotaSet(projectUUID).ExtractInto(&data)
	if err == nil {
		logg.Info("using CFM quota set for project %s", projectUUID)
		if !p.cfg.CFM.ReportPhysicalUsage {
			return map[string]core.ResourceData{
				"cfm_share_capacity": {
					Quota: data.StorageQuota.SizeLimitBytes,
					Usage: data.StorageQuota.Usage.BytesUsed,
				},
			}, nil
		}
		physicalUsage := data.StorageQuota.Usage.BytesUsed
		return map[string]core.ResourceData{
			"cfm_share_capacity": {
				Quota:         data.StorageQuota.SizeLimitBytes,
				Usage:         data.StorageQuota.Usage.Size,
				PhysicalUsage: &physicalUsage,
			},
		}, nil
	}

	//never use the old API when we're instructed to only read quotas
	if p.cfg.CFM.Authoritative {
		if _, ok := err.(cfmNotFoundError); ok {
			return map[string]core.ResourceData{"cfm_share_capacity": {Quota: 0, Usage: 0}}, nil
		}
		return nil, err
	}

	return p.scrapeOld(client, projectUUID)
}

func (p *cfmPlugin) scrapeOld(client *cfmClient, projectUUID string) (map[string]core.ResourceData, error) {
	//cache the result of cfmListShareservers(), it's mildly expensive
	now := time.Now()
	if p.shareserversCache == nil || p.shareserversCacheExpires.Before(now) {
		shareservers, err := client.ListShareservers()
		if err != nil {
			return nil, err
		}
		p.shareserversCache = shareservers
		p.shareserversCacheExpires = now.Add(5 * time.Minute)
	}
	shareservers := p.shareserversCache

	result := core.ResourceData{Quota: 0, Usage: 0}
	for _, shareserver := range shareservers {
		if shareserver.ProjectUUID != projectUUID {
			continue
		}

		shareserverDetailed, err := client.GetShareserver(shareserver.DetailsURL)
		if err != nil {
			return nil, err
		}

		result.Quota += int64(shareserverDetailed.BytesUsed)
		result.Usage += shareserverDetailed.BytesUsed
	}

	return map[string]core.ResourceData{"cfm_share_capacity": result}, nil
}

//SetQuota implements the core.QuotaPlugin interface.
func (p *cfmPlugin) SetQuota(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts, clusterID, domainUUID, projectUUID string, quotas map[string]uint64) error {
	if !p.cfg.CFM.Authoritative {
		return errors.New("the database/cfm_share_capacity resource is externally managed")
	}

	client, err := newCFMClient(provider, eo, p.projectID)
	if err != nil {
		return err
	}
	quotaBytes := quotas["cfm_share_capacity"]
	err = client.UpdateQuotaSet(projectUUID, quotaBytes)
	if _, ok := err.(cfmNotFoundError); ok {
		if quotaBytes == 0 {
			return nil //nothing to do: quota does not exist, but is also not wanted
		}
		err = client.CreateQuotaSet(projectUUID, quotaBytes)
	}
	return err
}
<|endoftext|>


<commit_before>/*
Copyright 2013 The Camlistore Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package search

import (
	"errors"
	"fmt"
	"log"
	"regexp"
	"strconv"
	"strings"

	"camlistore.org/pkg/context"
	"camlistore.org/pkg/geocode"
)

var (
	tagExpr = regexp.MustCompile(`^tag:(\w+)$`)

	// used for width/height ranges. 10 is max length of 32-bit
	// int (strconv.Atoi on 32-bit platforms), even though a max
	// JPEG dimension is only 16-bit.
	whRangeExpr = regexp.MustCompile(`^(\d{0,10})-(\d{0,10})$`)
)

// parseExpression parses a search expression (e.g. "tag:funny
// near:portland") and returns a SearchQuery for that search text. The
// Constraint field will always be set. The Limit and Sort may also be
// set.
func parseExpression(ctx *context.Context, exp string) (*SearchQuery, error) {
	base := &Constraint{
		Permanode: &PermanodeConstraint{
			SkipHidden: true,
		},
	}
	sq := &SearchQuery{
		Constraint: base,
	}

	exp = strings.TrimSpace(exp)
	if exp == "" {
		return sq, nil
	}

	andNot := false // whether the next and(x) is really a and(!x)
	and := func(c *Constraint) {
		old := sq.Constraint
		if andNot {
			c = &Constraint{
				Logical: &LogicalConstraint{
					Op: "not",
					A:  c,
				},
			}
		}
		sq.Constraint = &Constraint{
			Logical: &LogicalConstraint{
				Op: "and",
				A:  old,
				B:  c,
			},
		}
	}
	permOfFile := func(fc *FileConstraint) *Constraint {
		return &Constraint{
			Permanode: &PermanodeConstraint{
				Attr:       "camliContent",
				ValueInSet: &Constraint{File: fc},
			},
		}
	}
	orConst := func(a, b *Constraint) *Constraint {
		return &Constraint{
			Logical: &LogicalConstraint{
				Op: "or",
				A:  a,
				B:  b,
			},
		}
	}
	andFile := func(fc *FileConstraint) {
		and(permOfFile(fc))
	}
	andWHRatio := func(fc *FloatConstraint) {
		andFile(&FileConstraint{
			IsImage: true,
			WHRatio: fc,
		})
	}

	words := strings.Fields(exp)
	for _, word := range words {
		andNot = false
		if strings.HasPrefix(word, "-") {
			andNot = true
			word = word[1:]
		}
		if m := tagExpr.FindStringSubmatch(word); m != nil {
			and(&Constraint{
				Permanode: &PermanodeConstraint{
					Attr:       "tag",
					SkipHidden: true,
					Value:      m[1],
				},
			})
			continue
		}
		if word == "is:image" {
			and(&Constraint{
				Permanode: &PermanodeConstraint{
					Attr: "camliContent",
					ValueInSet: &Constraint{
						File: &FileConstraint{
							IsImage: true,
						},
					},
				},
			})
			continue
		}
		if word == "is:landscape" {
			andWHRatio(&FloatConstraint{Min: 1.0})
			continue
		}
		if word == "is:portrait" {
			andWHRatio(&FloatConstraint{Max: 1.0})
			continue
		}
		if word == "is:pano" {
			andWHRatio(&FloatConstraint{Min: 1.5})
			continue
		}
		if word == "has:location" {
			andFile(&FileConstraint{
				IsImage: true,
				Location: &LocationConstraint{
					Any: true,
				},
			})
			continue
		}
		if strings.HasPrefix(word, "format:") {
			andFile(&FileConstraint{
				MIMEType: &StringConstraint{
					Equals: mimeFromFormat(strings.TrimPrefix(word, "format:")),
				},
			})
			continue
		}
		if strings.HasPrefix(word, "width:") {
			m := whRangeExpr.FindStringSubmatch(strings.TrimPrefix(word, "width:"))
			if m == nil {
				return nil, errors.New("bogus width range")
			}
			andFile(&FileConstraint{
				IsImage: true,
				Width:   whIntConstraint(m[1], m[2]),
			})
			continue
		}
		if strings.HasPrefix(word, "height:") {
			m := whRangeExpr.FindStringSubmatch(strings.TrimPrefix(word, "height:"))
			if m == nil {
				return nil, errors.New("bogus height range")
			}
			andFile(&FileConstraint{
				IsImage: true,
				Height:  whIntConstraint(m[1], m[2]),
			})
			continue
		}
		if strings.HasPrefix(word, "loc:") {
			where := strings.TrimPrefix(word, "loc:")
			rects, err := geocode.Lookup(ctx, where)
			if err != nil {
				return nil, err
			}
			if len(rects) == 0 {
				return nil, fmt.Errorf("No location found for %q", where)
			}
			var locConstraint *Constraint
			for i, rect := range rects {
				rectConstraint := permOfFile(&FileConstraint{
					IsImage: true,
					Location: &LocationConstraint{
						West:  rect.SouthWest.Long,
						East:  rect.NorthEast.Long,
						North: rect.NorthEast.Lat,
						South: rect.SouthWest.Lat,
					},
				})
				if i == 0 {
					locConstraint = rectConstraint
				} else {
					locConstraint = orConst(locConstraint, rectConstraint)
				}
			}
			and(locConstraint)
			continue
		}
		log.Printf("Unknown search expression word %q", word)
		// TODO: finish. better tokenization. non-operator tokens
		// are text searches, etc.
	}

	return sq, nil
}

func whIntConstraint(mins, maxs string) *IntConstraint {
	ic := &IntConstraint{}
	if mins != "" {
		if mins == "0" {
			ic.ZeroMin = true
		} else {
			n, _ := strconv.Atoi(mins)
			ic.Min = int64(n)
		}
	}
	if maxs != "" {
		if maxs == "0" {
			ic.ZeroMax = true
		} else {
			n, _ := strconv.Atoi(maxs)
			ic.Max = int64(n)
		}
	}
	return ic
}

func mimeFromFormat(v string) string {
	if strings.Contains(v, "/") {
		return v
	}
	switch v {
	case "jpg", "jpeg":
		return "image/jpeg"
	case "gif":
		return "image/gif"
	case "png":
		return "image/png"
	case "pdf":
		return "application/pdf" // RFC 3778
	}
	return "???"
}
<commit_msg>search: make is:pano be a bit wider.<commit_after>/*
Copyright 2013 The Camlistore Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package search

import (
	"errors"
	"fmt"
	"log"
	"regexp"
	"strconv"
	"strings"

	"camlistore.org/pkg/context"
	"camlistore.org/pkg/geocode"
)

var (
	tagExpr = regexp.MustCompile(`^tag:(\w+)$`)

	// used for width/height ranges. 10 is max length of 32-bit
	// int (strconv.Atoi on 32-bit platforms), even though a max
	// JPEG dimension is only 16-bit.
	whRangeExpr = regexp.MustCompile(`^(\d{0,10})-(\d{0,10})$`)
)

// parseExpression parses a search expression (e.g. "tag:funny
// near:portland") and returns a SearchQuery for that search text. The
// Constraint field will always be set. The Limit and Sort may also be
// set.
func parseExpression(ctx *context.Context, exp string) (*SearchQuery, error) {
	base := &Constraint{
		Permanode: &PermanodeConstraint{
			SkipHidden: true,
		},
	}
	sq := &SearchQuery{
		Constraint: base,
	}

	exp = strings.TrimSpace(exp)
	if exp == "" {
		return sq, nil
	}

	andNot := false // whether the next and(x) is really a and(!x)
	and := func(c *Constraint) {
		old := sq.Constraint
		if andNot {
			c = &Constraint{
				Logical: &LogicalConstraint{
					Op: "not",
					A:  c,
				},
			}
		}
		sq.Constraint = &Constraint{
			Logical: &LogicalConstraint{
				Op: "and",
				A:  old,
				B:  c,
			},
		}
	}
	permOfFile := func(fc *FileConstraint) *Constraint {
		return &Constraint{
			Permanode: &PermanodeConstraint{
				Attr:       "camliContent",
				ValueInSet: &Constraint{File: fc},
			},
		}
	}
	orConst := func(a, b *Constraint) *Constraint {
		return &Constraint{
			Logical: &LogicalConstraint{
				Op: "or",
				A:  a,
				B:  b,
			},
		}
	}
	andFile := func(fc *FileConstraint) {
		and(permOfFile(fc))
	}
	andWHRatio := func(fc *FloatConstraint) {
		andFile(&FileConstraint{
			IsImage: true,
			WHRatio: fc,
		})
	}

	words := strings.Fields(exp)
	for _, word := range words {
		andNot = false
		if strings.HasPrefix(word, "-") {
			andNot = true
			word = word[1:]
		}
		if m := tagExpr.FindStringSubmatch(word); m != nil {
			and(&Constraint{
				Permanode: &PermanodeConstraint{
					Attr:       "tag",
					SkipHidden: true,
					Value:      m[1],
				},
			})
			continue
		}
		if word == "is:image" {
			and(&Constraint{
				Permanode: &PermanodeConstraint{
					Attr: "camliContent",
					ValueInSet: &Constraint{
						File: &FileConstraint{
							IsImage: true,
						},
					},
				},
			})
			continue
		}
		if word == "is:landscape" {
			andWHRatio(&FloatConstraint{Min: 1.0})
			continue
		}
		if word == "is:portrait" {
			andWHRatio(&FloatConstraint{Max: 1.0})
			continue
		}
		if word == "is:pano" {
			andWHRatio(&FloatConstraint{Min: 1.6})
			continue
		}
		if word == "has:location" {
			andFile(&FileConstraint{
				IsImage: true,
				Location: &LocationConstraint{
					Any: true,
				},
			})
			continue
		}
		if strings.HasPrefix(word, "format:") {
			andFile(&FileConstraint{
				MIMEType: &StringConstraint{
					Equals: mimeFromFormat(strings.TrimPrefix(word, "format:")),
				},
			})
			continue
		}
		if strings.HasPrefix(word, "width:") {
			m := whRangeExpr.FindStringSubmatch(strings.TrimPrefix(word, "width:"))
			if m == nil {
				return nil, errors.New("bogus width range")
			}
			andFile(&FileConstraint{
				IsImage: true,
				Width:   whIntConstraint(m[1], m[2]),
			})
			continue
		}
		if strings.HasPrefix(word, "height:") {
			m := whRangeExpr.FindStringSubmatch(strings.TrimPrefix(word, "height:"))
			if m == nil {
				return nil, errors.New("bogus height range")
			}
			andFile(&FileConstraint{
				IsImage: true,
				Height:  whIntConstraint(m[1], m[2]),
			})
			continue
		}
		if strings.HasPrefix(word, "loc:") {
			where := strings.TrimPrefix(word, "loc:")
			rects, err := geocode.Lookup(ctx, where)
			if err != nil {
				return nil, err
			}
			if len(rects) == 0 {
				return nil, fmt.Errorf("No location found for %q", where)
			}
			var locConstraint *Constraint
			for i, rect := range rects {
				rectConstraint := permOfFile(&FileConstraint{
					IsImage: true,
					Location: &LocationConstraint{
						West:  rect.SouthWest.Long,
						East:  rect.NorthEast.Long,
						North: rect.NorthEast.Lat,
						South: rect.SouthWest.Lat,
					},
				})
				if i == 0 {
					locConstraint = rectConstraint
				} else {
					locConstraint = orConst(locConstraint, rectConstraint)
				}
			}
			and(locConstraint)
			continue
		}
		log.Printf("Unknown search expression word %q", word)
		// TODO: finish. better tokenization. non-operator tokens
		// are text searches, etc.
	}

	return sq, nil
}

func whIntConstraint(mins, maxs string) *IntConstraint {
	ic := &IntConstraint{}
	if mins != "" {
		if mins == "0" {
			ic.ZeroMin = true
		} else {
			n, _ := strconv.Atoi(mins)
			ic.Min = int64(n)
		}
	}
	if maxs != "" {
		if maxs == "0" {
			ic.ZeroMax = true
		} else {
			n, _ := strconv.Atoi(maxs)
			ic.Max = int64(n)
		}
	}
	return ic
}

func mimeFromFormat(v string) string {
	if strings.Contains(v, "/") {
		return v
	}
	switch v {
	case "jpg", "jpeg":
		return "image/jpeg"
	case "gif":
		return "image/gif"
	case "png":
		return "image/png"
	case "pdf":
		return "application/pdf" // RFC 3778
	}
	return "???"
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package spec

import "errors"

type BackupStorageType string

const (
	BackupStorageTypeDefault          = ""
	BackupStorageTypePersistentVolume = "PersistentVolume"
	BackupStorageTypeS3               = "S3"
)

var errPVZeroSize = errors.New("PV backup should not have 0 size volume")

type BackupPolicy struct {
	// Pod defines the policy to create the backup pod.
	Pod *PodPolicy `json:"pod,omitempty"`

	// StorageType specifies the type of storage device to store backup files.
	// If it's not set by user, the default is "PersistentVolume".
	StorageType BackupStorageType `json:"storageType"`

	StorageSource `json:",inline"`

	// BackupIntervalInSecond specifies the interval between two backups.
	// The default interval is 1800 seconds.
	BackupIntervalInSecond int `json:"backupIntervalInSecond"`

	// MaxBackups is the maximum number of backup files to retain. 0 is disable backup.
	// If backup is disabled, the etcd cluster cannot recover from a
	// disaster failure (lose more than half of its members at the same
	// time).
	MaxBackups int `json:"maxBackups"`

	// CleanupBackupsOnClusterDelete tells whether to cleanup backup data if cluster is deleted.
	// By default, operator will keep the backup data.
	CleanupBackupsOnClusterDelete bool `json:"cleanupBackupsOnClusterDelete"`
}

func (bp *BackupPolicy) Validate() error {
	if bp.StorageType == BackupStorageTypePersistentVolume {
		if pv := bp.StorageSource.PV; pv == nil || pv.VolumeSizeInMB <= 0 {
			return errPVZeroSize
		}
	}
	return nil
}

type StorageSource struct {
	PV *PVSource `json:"pv,omitempty"`
	S3 *S3Source `json:"s3,omitempty"`
}

type PVSource struct {
	// VolumeSizeInMB specifies the required volume size to perform backups.
	// Operator will claim the required size before creating the etcd cluster for backup
	// purpose.
	// If the snapshot size is larger than the size specified, backup fails.
	VolumeSizeInMB int `json:"volumeSizeInMB"`
}

type S3Source struct {
}

type BackupServiceStatus struct {
	// RecentBackup is status of the most recent backup created by
	// the backup service
	RecentBackup *BackupStatus `json:"recentBackup,omitempty"`

	// Backups is the totoal number of existing backups
	Backups int `json:"backups"`

	// BackupSize is the total size of existing backups in MB.
	BackupSize float64 `json:"backupSize"`
}

type BackupStatus struct {
	// Creation time of the backup.
	CreationTime string `json:"creationTime"`

	// Size is the size of the backup in MB.
	Size float64 `json:"size"`

	// Version is the version of the backup cluster.
	Version string `json:"version"`

	// TimeTookInSecond is the total time took to create the backup.
	TimeTookInSecond int `json:"timeTookInSecond"`
}
<commit_msg>spec: add s3 configuration into s3 source<commit_after>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package spec

import "errors"

type BackupStorageType string

const (
	BackupStorageTypeDefault          = ""
	BackupStorageTypePersistentVolume = "PersistentVolume"
	BackupStorageTypeS3               = "S3"
)

var errPVZeroSize = errors.New("PV backup should not have 0 size volume")

type BackupPolicy struct {
	// Pod defines the policy to create the backup pod.
	Pod *PodPolicy `json:"pod,omitempty"`

	// StorageType specifies the type of storage device to store backup files.
	// If it's not set by user, the default is "PersistentVolume".
	StorageType BackupStorageType `json:"storageType"`

	StorageSource `json:",inline"`

	// BackupIntervalInSecond specifies the interval between two backups.
	// The default interval is 1800 seconds.
	BackupIntervalInSecond int `json:"backupIntervalInSecond"`

	// MaxBackups is the maximum number of backup files to retain. 0 is disable backup.
	// If backup is disabled, the etcd cluster cannot recover from a
	// disaster failure (lose more than half of its members at the same
	// time).
	MaxBackups int `json:"maxBackups"`

	// CleanupBackupsOnClusterDelete tells whether to cleanup backup data if cluster is deleted.
	// By default, operator will keep the backup data.
	CleanupBackupsOnClusterDelete bool `json:"cleanupBackupsOnClusterDelete"`
}

func (bp *BackupPolicy) Validate() error {
	if bp.StorageType == BackupStorageTypePersistentVolume {
		if pv := bp.StorageSource.PV; pv == nil || pv.VolumeSizeInMB <= 0 {
			return errPVZeroSize
		}
	}
	return nil
}

type StorageSource struct {
	PV *PVSource `json:"pv,omitempty"`
	S3 *S3Source `json:"s3,omitempty"`
}

type PVSource struct {
	// VolumeSizeInMB specifies the required volume size to perform backups.
	// Operator will claim the required size before creating the etcd cluster for backup
	// purpose.
	// If the snapshot size is larger than the size specified, backup fails.
	VolumeSizeInMB int `json:"volumeSizeInMB"`
}

// TODO: support per cluster S3 Source configuration.
type S3Source struct {
	// The name of the AWS S3 bucket to store backups in.
	//
	// S3Bucket overwrites the default etcd operator wide bucket.
	S3Bucket string `json:"s3Bucket,omitempty"`

	// The name of the secret object that stores the AWS credential and config files.
	// The file name of the credential MUST be 'credentials'.
	// The file name of the config MUST be 'config'
	//
	// AWSCredentialSecret overwrites the default etcd operator wide AWS credential and
	// config.
	AWSSecret string `json:"awsSecret,omitempty"`
}

type BackupServiceStatus struct {
	// RecentBackup is status of the most recent backup created by
	// the backup service
	RecentBackup *BackupStatus `json:"recentBackup,omitempty"`

	// Backups is the totoal number of existing backups
	Backups int `json:"backups"`

	// BackupSize is the total size of existing backups in MB.
	BackupSize float64 `json:"backupSize"`
}

type BackupStatus struct {
	// Creation time of the backup.
	CreationTime string `json:"creationTime"`

	// Size is the size of the backup in MB.
	Size float64 `json:"size"`

	// Version is the version of the backup cluster.
	Version string `json:"version"`

	// TimeTookInSecond is the total time took to create the backup.
	TimeTookInSecond int `json:"timeTookInSecond"`
}
<|endoftext|>


<commit_before>// Package stack provides a basic implementation
// of a stack using a linked list.
package stack

// Stack represents the stack data structure as a linked list,
// containing a pointer to the first Frame as a head and the
// size of the stack.
type Stack struct {
	head *frame
	size int
}

// frame represents an element of the stack. It contains
// data and the link to the next Frame as a pointer.
type frame struct {
	data interface{}
	next *frame
}

// NewStack returns a blank stack, where head is nil and size
// is 0.
func NewStack() *Stack {
	return &Stack{}
}

// Push adds a new element on top of the stack, creating
// a new head holding this data and updating its head to
// the previous stack's head.
func (s *Stack) Push(element interface{}) {
	head := &frame{
		data: element,
		next: s.head,
	}
	s.head = head
	s.size++
}

// Pop removes and returns the element on top of the stack,
// updating its head to the next Frame. If the stack was empty,
// it returns false.
func (s *Stack) Pop() (interface{}, bool) {
	if s.head == nil {
		return nil, false
	}

	element := s.head.data
	s.head = s.head.next
	s.size--
	return element, true
}

// Size returns the number of elements that a stack contains.
func (s *Stack) Size() int {
	return s.size
}

// Peek returns the element on top of the stack.
func (s *Stack) Peek() interface{} {
	if s.head == nil {
		return nil
	}
	return s.head.data
}
<commit_msg>pkg/stack: use mutexes to avoid race conditions<commit_after>// Package stack provides a basic implementation
// of a stack using a linked list.
package stack

import "sync"

// Stack represents the stack data structure as a linked list,
// containing a pointer to the first Frame as a head and the
// size of the stack.
type Stack struct {
	head *frame
	size int
	mux  sync.Mutex
}

// frame represents an element of the stack. It contains
// data and the link to the next Frame as a pointer.
type frame struct {
	data interface{}
	next *frame
}

// NewStack returns a blank stack, where head is nil and size
// is 0.
func NewStack() *Stack {
	return &Stack{}
}

// Push adds a new element on top of the stack, creating
// a new head holding this data and updating its head to
// the previous stack's head.
func (s *Stack) Push(element interface{}) {
	s.mux.Lock()
	defer s.mux.Unlock()

	head := &frame{
		data: element,
		next: s.head,
	}
	s.head = head
	s.size++
}

// Pop removes and returns the element on top of the stack,
// updating its head to the next Frame. If the stack was empty,
// it returns false.
func (s *Stack) Pop() (interface{}, bool) {
	s.mux.Lock()
	defer s.mux.Unlock()

	if s.head == nil {
		return nil, false
	}

	element := s.head.data
	s.head = s.head.next
	s.size--
	return element, true
}

// Size returns the number of elements that a stack contains.
func (s *Stack) Size() int {
	return s.size
}

// Peek returns the element on top of the stack.
func (s *Stack) Peek() interface{} {
	if s.head == nil {
		return nil
	}
	return s.head.data
}
<|endoftext|>


<commit_before>package topom

import (
	"net"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/wandoulabs/codis/pkg/models"
	"github.com/wandoulabs/codis/pkg/proxy"
	"github.com/wandoulabs/codis/pkg/utils"
	"github.com/wandoulabs/codis/pkg/utils/atomic2"
	"github.com/wandoulabs/codis/pkg/utils/errors"
	"github.com/wandoulabs/codis/pkg/utils/log"
	"github.com/wandoulabs/codis/pkg/utils/rpc"
)

type Topom struct {
	mu sync.RWMutex

	xauth string
	model *models.Topom
	store models.Store

	intvl atomic2.Int64

	exit struct {
		C chan struct{}
	}

	online bool
	closed bool

	ladmin net.Listener
	redisp *RedisPool

	config *Config

	mappings [models.MaxSlotNum]*models.SlotMapping

	groups  map[int]*models.Group
	proxies map[string]*models.Proxy
	clients map[string]*proxy.ApiClient

	stats struct {
		servers map[string]*ServerStats
		proxies map[string]*ProxyStats
	}
	start sync.Once
}

var (
	ErrClosedTopom = errors.New("use of closed topom")
	ErrUpdateStore = errors.New("update store failed")
)

func NewWithConfig(store models.Store, config *Config) (*Topom, error) {
	s := &Topom{config: config, store: store}
	s.xauth = rpc.NewXAuth(config.ProductName, config.ProductAuth)
	s.model = &models.Topom{
		StartTime: time.Now().String(),
	}
	s.model.ProductName = config.ProductName
	s.model.Pid = os.Getpid()
	s.model.Pwd, _ = os.Getwd()

	s.intvl.Set(1000)

	s.redisp = NewRedisPool(config.ProductAuth, time.Minute)

	s.exit.C = make(chan struct{})

	s.groups = make(map[int]*models.Group)
	s.proxies = make(map[string]*models.Proxy)
	s.clients = make(map[string]*proxy.ApiClient)

	s.stats.servers = make(map[string]*ServerStats)
	s.stats.proxies = make(map[string]*ProxyStats)

	if err := s.setup(); err != nil {
		s.Close()
		return nil, err
	}

	log.Infof("[%p] create new topom:\n%s", s, s.model.Encode())

	go s.serveAdmin()

	return s, nil
}

func (s *Topom) setup() error {
	if !utils.IsValidName(s.config.ProductName) {
		return errors.New("invalid product name, empty or using invalid character")
	}

	if l, err := net.Listen("tcp", s.config.AdminAddr); err != nil {
		return errors.Trace(err)
	} else {
		s.ladmin = l
	}

	if addr, err := utils.ResolveAddr("tcp", s.ladmin.Addr().String()); err != nil {
		return err
	} else {
		s.model.AdminAddr = addr
	}

	if err := s.store.Acquire(s.config.ProductName, s.model); err != nil {
		return err
	} else {
		s.online = true
	}

	for i := 0; i < len(s.mappings); i++ {
		if m, err := s.store.LoadSlotMapping(i); err != nil {
			return err
		} else {
			s.mappings[i] = m
		}
	}

	if glist, err := s.store.ListGroup(); err != nil {
		return err
	} else {
		for _, g := range glist {
			s.groups[g.Id] = g
		}
	}

	if plist, err := s.store.ListProxy(); err != nil {
		return err
	} else {
		for _, p := range plist {
			c := proxy.NewApiClient(p.AdminAddr)
			c.SetXAuth(s.config.ProductName, s.config.ProductAuth, p.Token)
			s.proxies[p.Token] = p
			s.clients[p.Token] = c
			s.stats.proxies[p.Token] = nil
		}
	}
	return nil
}

func (s *Topom) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed {
		return nil
	}
	s.closed = true
	close(s.exit.C)

	s.ladmin.Close()
	s.redisp.Close()

	defer s.store.Close()

	if !s.online {
		return nil
	} else {
		return s.store.Release()
	}
}

func (s *Topom) GetXAuth() string {
	return s.xauth
}

func (s *Topom) GetModel() *models.Topom {
	return s.model
}

func (s *Topom) GetConfig() *Config {
	return s.config
}

func (s *Topom) IsOnline() bool {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.online && !s.closed
}

func (s *Topom) IsClosed() bool {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.closed
}

func (s *Topom) GetInterval() int {
	return int(s.intvl.Get())
}

func (s *Topom) SetInterval(ms int) {
	ms = utils.MaxInt(ms, 0)
	ms = utils.MinInt(ms, 1000)
	s.intvl.Set(int64(ms))
}

func (s *Topom) serveAdmin() {
	if s.IsClosed() {
		return
	}
	defer s.Close()

	log.Infof("[%p] admin start service on %s", s, s.ladmin.Addr())

	eh := make(chan error, 1)
	go func(l net.Listener) {
		h := http.NewServeMux()
		h.Handle("/", newApiServer(s))
		hs := &http.Server{Handler: h}
		eh <- hs.Serve(l)
	}(s.ladmin)

	select {
	case <-s.exit.C:
		log.Infof("[%p] admin shutdown", s)
	case err := <-eh:
		log.ErrorErrorf(err, "[%p] admin exit on error", s)
	}
}

func (s *Topom) StartDaemonRoutines() {
	s.start.Do(func() {
		go func() {
			for {
				wg := s.RefreshServerStats(time.Second)
				if wg != nil {
					wg.Wait()
				} else {
					return
				}
				time.Sleep(time.Second)
			}
		}()

		go func() {
			for {
				wg := s.RefreshProxyStats(time.Second)
				if wg != nil {
					wg.Wait()
				} else {
					return
				}
				time.Sleep(time.Second)
			}
		}()

		go func() {
			for !s.IsClosed() {
				if a := s.NextAction(); a != nil {
					if err := a.Do(); err != nil {
						log.WarnErrorf(err, "[%p] action on slot-[%d] failed", s, a.SlotId)
						time.Sleep(time.Second * 3)
					}
				} else {
					time.Sleep(time.Millisecond * 200)
				}
			}
		}()
	})
}
<commit_msg>Fix, init stats.servers with groups<commit_after>package topom

import (
	"net"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/wandoulabs/codis/pkg/models"
	"github.com/wandoulabs/codis/pkg/proxy"
	"github.com/wandoulabs/codis/pkg/utils"
	"github.com/wandoulabs/codis/pkg/utils/atomic2"
	"github.com/wandoulabs/codis/pkg/utils/errors"
	"github.com/wandoulabs/codis/pkg/utils/log"
	"github.com/wandoulabs/codis/pkg/utils/rpc"
)

type Topom struct {
	mu sync.RWMutex

	xauth string
	model *models.Topom
	store models.Store

	intvl atomic2.Int64

	exit struct {
		C chan struct{}
	}

	online bool
	closed bool

	ladmin net.Listener
	redisp *RedisPool

	config *Config

	mappings [models.MaxSlotNum]*models.SlotMapping

	groups  map[int]*models.Group
	proxies map[string]*models.Proxy
	clients map[string]*proxy.ApiClient

	stats struct {
		servers map[string]*ServerStats
		proxies map[string]*ProxyStats
	}
	start sync.Once
}

var (
	ErrClosedTopom = errors.New("use of closed topom")
	ErrUpdateStore = errors.New("update store failed")
)

func NewWithConfig(store models.Store, config *Config) (*Topom, error) {
	s := &Topom{config: config, store: store}
	s.xauth = rpc.NewXAuth(config.ProductName, config.ProductAuth)
	s.model = &models.Topom{
		StartTime: time.Now().String(),
	}
	s.model.ProductName = config.ProductName
	s.model.Pid = os.Getpid()
	s.model.Pwd, _ = os.Getwd()

	s.intvl.Set(1000)

	s.redisp = NewRedisPool(config.ProductAuth, time.Minute)

	s.exit.C = make(chan struct{})

	s.groups = make(map[int]*models.Group)
	s.proxies = make(map[string]*models.Proxy)
	s.clients = make(map[string]*proxy.ApiClient)

	s.stats.servers = make(map[string]*ServerStats)
	s.stats.proxies = make(map[string]*ProxyStats)

	if err := s.setup(); err != nil {
		s.Close()
		return nil, err
	}

	log.Infof("[%p] create new topom:\n%s", s, s.model.Encode())

	go s.serveAdmin()

	return s, nil
}

func (s *Topom) setup() error {
	if !utils.IsValidName(s.config.ProductName) {
		return errors.New("invalid product name, empty or using invalid character")
	}

	if l, err := net.Listen("tcp", s.config.AdminAddr); err != nil {
		return errors.Trace(err)
	} else {
		s.ladmin = l
	}

	if addr, err := utils.ResolveAddr("tcp", s.ladmin.Addr().String()); err != nil {
		return err
	} else {
		s.model.AdminAddr = addr
	}

	if err := s.store.Acquire(s.config.ProductName, s.model); err != nil {
		return err
	} else {
		s.online = true
	}

	for i := 0; i < len(s.mappings); i++ {
		if m, err := s.store.LoadSlotMapping(i); err != nil {
			return err
		} else {
			s.mappings[i] = m
		}
	}

	if glist, err := s.store.ListGroup(); err != nil {
		return err
	} else {
		for _, g := range glist {
			s.groups[g.Id] = g
		}
		for _, g := range glist {
			for _, addr := range g.Servers {
				s.stats.servers[addr] = nil
			}
		}
	}

	if plist, err := s.store.ListProxy(); err != nil {
		return err
	} else {
		for _, p := range plist {
			c := proxy.NewApiClient(p.AdminAddr)
			c.SetXAuth(s.config.ProductName, s.config.ProductAuth, p.Token)
			s.proxies[p.Token] = p
			s.clients[p.Token] = c
		}
		for _, p := range plist {
			s.stats.servers[p.Token] = nil
		}
	}
	return nil
}

func (s *Topom) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed {
		return nil
	}
	s.closed = true
	close(s.exit.C)

	s.ladmin.Close()
	s.redisp.Close()

	defer s.store.Close()

	if !s.online {
		return nil
	} else {
		return s.store.Release()
	}
}

func (s *Topom) GetXAuth() string {
	return s.xauth
}

func (s *Topom) GetModel() *models.Topom {
	return s.model
}

func (s *Topom) GetConfig() *Config {
	return s.config
}

func (s *Topom) IsOnline() bool {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.online && !s.closed
}

func (s *Topom) IsClosed() bool {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.closed
}

func (s *Topom) GetInterval() int {
	return int(s.intvl.Get())
}

func (s *Topom) SetInterval(ms int) {
	ms = utils.MaxInt(ms, 0)
	ms = utils.MinInt(ms, 1000)
	s.intvl.Set(int64(ms))
}

func (s *Topom) serveAdmin() {
	if s.IsClosed() {
		return
	}
	defer s.Close()

	log.Infof("[%p] admin start service on %s", s, s.ladmin.Addr())

	eh := make(chan error, 1)
	go func(l net.Listener) {
		h := http.NewServeMux()
		h.Handle("/", newApiServer(s))
		hs := &http.Server{Handler: h}
		eh <- hs.Serve(l)
	}(s.ladmin)

	select {
	case <-s.exit.C:
		log.Infof("[%p] admin shutdown", s)
	case err := <-eh:
		log.ErrorErrorf(err, "[%p] admin exit on error", s)
	}
}

func (s *Topom) StartDaemonRoutines() {
	s.start.Do(func() {
		go func() {
			for {
				wg := s.RefreshServerStats(time.Second)
				if wg != nil {
					wg.Wait()
				} else {
					return
				}
				time.Sleep(time.Second)
			}
		}()

		go func() {
			for {
				wg := s.RefreshProxyStats(time.Second)
				if wg != nil {
					wg.Wait()
				} else {
					return
				}
				time.Sleep(time.Second)
			}
		}()

		go func() {
			for !s.IsClosed() {
				if a := s.NextAction(); a != nil {
					if err := a.Do(); err != nil {
						log.WarnErrorf(err, "[%p] action on slot-[%d] failed", s, a.SlotId)
						time.Sleep(time.Second * 3)
					}
				} else {
					time.Sleep(time.Millisecond * 200)
				}
			}
		}()
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Tuenti Technologies S.L. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vault

import (
	"fmt"
	"net/http"

	"github.com/hashicorp/vault/api"
)

const (
	TokenHeader   = "X-Vault-Token"
	WrapTTLHeader = "X-Vault-Wrap-Ttl"

	SysHealthURL = "/v1/sys/health"

	AuthAppRoleURL  = "/v1/sys/auth/approle"
	AppRoleLoginURL = "/v1/auth/approle/login"
	AppRoleURL      = "/v1/auth/approle/role"
)

type RequestOptions struct {
	WrapTTL string

	Data map[string]interface{}
}

type Vault interface {
	Login() error
	Request(method, urlPath string, options *RequestOptions) (*api.Secret, error)
	UnwrapSecretID(token string) error
	GetToken() string
}

type Config struct {
	Address  string `json:"address,omitempty"`
	RoleID   string `json:"role_id,omitempty"`
	SecretID string `json:"secret_id,omitempty"`
	Token    string `json:"token,omitempty"`
}

type vaultApi struct {
	Address  string
	RoleID   string
	SecretID string
	Token    string
}

func New(c Config) Vault {
	return &vaultApi{
		Address:  c.Address,
		RoleID:   c.RoleID,
		SecretID: c.SecretID,
		Token:    c.Token,
	}
}

func (v *vaultApi) getClient() (*api.Client, error) {
	config := api.DefaultConfig()
	if v.Address != "" {
		config.Address = v.Address
	}
	return api.NewClient(config)
}

func (v *vaultApi) Login() error {
	if v.Token != "" {
		return nil
	}
	if v.RoleID == "" {
		return fmt.Errorf("role ID needed")
	}
	data := make(map[string]interface{})
	data["role_id"] = v.RoleID
	if v.SecretID != "" {
		data["secret_id"] = v.SecretID
	}
	options := RequestOptions{Data: data}
	resp, err := v.Request(http.MethodPost, AppRoleLoginURL, &options)
	if err != nil {
		return err
	}

	v.Token = resp.Auth.ClientToken

	return nil
}

func (v *vaultApi) UnwrapSecretID(token string) error {
	c, err := v.getClient()
	if err != nil {
		return err
	}
	c.SetToken(token)
	resp, err := c.Logical().Unwrap(token)
	if err != nil {
		return err
	}
	if resp == nil {
		return fmt.Errorf("no response?")
	}
	secretID, ok := resp.Data["secret_id"]
	if !ok {
		return fmt.Errorf("no secret ID found in response")
	}
	v.SecretID, ok = secretID.(string)
	if !ok {
		return fmt.Errorf("secret_id in response is not a string")
	}
	return nil
}

func (v *vaultApi) Request(method, urlPath string, options *RequestOptions) (*api.Secret, error) {
	c, err := v.getClient()
	if err != nil {
		return nil, err
	}
	if v.Token != "" {
		c.SetToken(v.Token)
	}

	r := c.NewRequest(method, urlPath)
	if options != nil {
		if len(options.Data) > 0 {
			err = r.SetJSONBody(options.Data)
			if err != nil {
				return nil, err
			}
		}

		if options.WrapTTL != "" {
			r.WrapTTL = options.WrapTTL
		}
	}

	resp, err := c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusNoContent {
		return nil, nil
	}
	return api.ParseSecret(resp.Body)
}

func (v *vaultApi) GetToken() string {
	return v.Token
}
<commit_msg>Token autorenew<commit_after>/*
Copyright 2017 Tuenti Technologies S.L. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vault

import (
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/hashicorp/vault/api"
)

const (
	// Token renewal period is its TTL multiplied by this ratio
	AutoRenewPeriodRatio = 0.5

	TokenHeader   = "X-Vault-Token"
	WrapTTLHeader = "X-Vault-Wrap-Ttl"

	SelfTokenURL      = "/v1//auth/token/lookup-self"
	SelfTokenRenewURL = "/v1/auth/token/renew-self"

	SysHealthURL = "/v1/sys/health"

	AuthAppRoleURL  = "/v1/sys/auth/approle"
	AppRoleLoginURL = "/v1/auth/approle/login"
	AppRoleURL      = "/v1/auth/approle/role"
)

type RequestOptions struct {
	WrapTTL string

	Data map[string]interface{}
}

type Vault interface {
	Login() error
	Request(method, urlPath string, options *RequestOptions) (*api.Secret, error)
	UnwrapSecretID(token string) error
	GetToken() string
}

type Config struct {
	Address  string `json:"address,omitempty"`
	RoleID   string `json:"role_id,omitempty"`
	SecretID string `json:"secret_id,omitempty"`
	Token    string `json:"token,omitempty"`
}

type vaultApi struct {
	Address  string
	RoleID   string
	SecretID string
	Token    string
}

func New(c Config) Vault {
	return &vaultApi{
		Address:  c.Address,
		RoleID:   c.RoleID,
		SecretID: c.SecretID,
		Token:    c.Token,
	}
}

func (v *vaultApi) getClient() (*api.Client, error) {
	config := api.DefaultConfig()
	if v.Address != "" {
		config.Address = v.Address
	}
	return api.NewClient(config)
}

func (v *vaultApi) tokenTTL() (int, error) {
	resp, err := v.Request(http.MethodGet, SelfTokenURL, nil)
	if err != nil || resp == nil {
		return 0, fmt.Errorf("couldn't obtain self token information: %s", err)
	}
	ttl, ok := resp.Data["ttl"].(int)
	if !ok {
		return 0, fmt.Errorf("couldn't obtain token TTL")
	}
	return ttl, nil
}

func (v *vaultApi) autoRenewToken() {
	var ttl int
	var err error

	retry := func(attempts int, t time.Duration, f func() error) {
		for attempt := 0; attempt < attempts; attempt-- {
			err = f()
			if err == nil {
				<-time.After(t)
				break
			}
		}
	}

	for {
		retry(5, 1*time.Second, func() error {
			ttl, err = v.tokenTTL()
			return err
		})
		if err != nil {
			log.Printf("Couldn't obtain token TTL: %s\n")
			break
		}
		if ttl == 0 {
			log.Println("Using token without expiration")
			return
		}
		period := time.Duration(float64(ttl)*AutoRenewPeriodRatio) * time.Second
		<-time.After(period)
		retry(5, 1*time.Second, func() error {
			_, err = v.Request(http.MethodPost, SelfTokenRenewURL, nil)
			return err
		})
		if err != nil {
			log.Printf("Failed to renew token: %s", err)
			break
		}
		log.Println("Token succesfuly renewed")
	}
	log.Println("Won't autorenew token")
}

func (v *vaultApi) Login() error {
	if v.Token != "" {
		return nil
	}
	if v.RoleID == "" {
		return fmt.Errorf("role ID needed")
	}
	data := make(map[string]interface{})
	data["role_id"] = v.RoleID
	if v.SecretID != "" {
		data["secret_id"] = v.SecretID
	}
	options := RequestOptions{Data: data}
	resp, err := v.Request(http.MethodPost, AppRoleLoginURL, &options)
	if err != nil {
		return err
	}

	v.Token = resp.Auth.ClientToken
	go v.autoRenewToken()

	return nil
}

func (v *vaultApi) UnwrapSecretID(token string) error {
	c, err := v.getClient()
	if err != nil {
		return err
	}
	c.SetToken(token)
	resp, err := c.Logical().Unwrap(token)
	if err != nil {
		return err
	}
	if resp == nil {
		return fmt.Errorf("no response?")
	}
	secretID, ok := resp.Data["secret_id"]
	if !ok {
		return fmt.Errorf("no secret ID found in response")
	}
	v.SecretID, ok = secretID.(string)
	if !ok {
		return fmt.Errorf("secret_id in response is not a string")
	}
	return nil
}

func (v *vaultApi) Request(method, urlPath string, options *RequestOptions) (*api.Secret, error) {
	c, err := v.getClient()
	if err != nil {
		return nil, err
	}
	if v.Token != "" {
		c.SetToken(v.Token)
	}

	r := c.NewRequest(method, urlPath)
	if options != nil {
		if len(options.Data) > 0 {
			err = r.SetJSONBody(options.Data)
			if err != nil {
				return nil, err
			}
		}

		if options.WrapTTL != "" {
			r.WrapTTL = options.WrapTTL
		}
	}

	resp, err := c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusNoContent {
		return nil, nil
	}
	return api.ParseSecret(resp.Body)
}

func (v *vaultApi) GetToken() string {
	return v.Token
}
<|endoftext|>


<commit_before>package tinygo

import (
	"io"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"os"
	"strconv"
	"time"

	"github.com/kdada/tinygo/router"
	"github.com/kdada/tinygo/session"
)

// 表单文件
type FormFile struct {
	file   multipart.File        //表单文件
	header *multipart.FileHeader //文件头信息
}

// FileName 返回文件名
func (this *FormFile) FileName() string {
	return this.header.Filename
}

// Header 返回文件头信息
func (this *FormFile) Header() textproto.MIMEHeader {
	return this.header.Header
}

// File 返回表单文件
func (this *FormFile) File() multipart.File {
	return this.file
}

// Close 关闭表单文件
func (this *FormFile) Close() {
	this.file.Close()
}

// SaveTo 将文件保存到指定路径,保存完毕后自动关闭表单文件
func (this *FormFile) SaveTo(path string) error {
	var lf, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0666)
	if err == nil {
		defer lf.Close()
		_, err = io.Copy(lf, this.file)
		if err == nil {
			this.Close()
		}
	}
	return err
}

// 路由环境
type HttpContext struct {
	urlParts       []string               //Url分段信息,每段一个字符串
	request        *http.Request          //http请求
	responseWriter http.ResponseWriter    //http响应
	session        session.Session        //http会话
	csrf           session.Session        //csrf会话
	params         map[string]string      //http参数,包含url,query,form的所有参数
	parsed         bool                   //存储参数是否已经解析过
	routers        []router.Router        //分派成功的路由链
	executor       router.ContextExecutor //存储最终执行Context的执行器
	static         bool                   //是否是静态路由
}

// Method 返回Http方法
func (this *HttpContext) Method() string {
	return this.request.Method
}

// ResponseWriter 返回ResponseWriter
func (this *HttpContext) ResponseWriter() http.ResponseWriter {
	return this.responseWriter
}

// Request 返回Request
func (this *HttpContext) Request() *http.Request {
	return this.request
}

// CsrfToken生成一个新的csrf token
func (this *HttpContext) CsrfToken() string {
	if this.csrf != nil {
		var newToken = session.Guid()
		this.csrf.SetInt(newToken, time.Now().Unix())
		return newToken
	}
	return ""
}

// ValidateCsrfToken 验证表单请求中是否存在csrf的token并且该token有效,验证后token即失效
func (this *HttpContext) ValidateCsrfToken() bool {
	if this.csrf != nil {
		var token = this.ParamString(DefaultCSRFTokenName)
		if token != "" {
			var t, ok = this.csrf.Int(token)
			if ok {
				this.csrf.Delete(token)
				return time.Now().Unix()-t <= tinyConfig.csrfexpire
			}
		}
	}
	return false
}

// Session 返回Session
func (this *HttpContext) Session() session.Session {
	return this.session
}

// Cookie 返回指定cookie的值
func (this *HttpContext) Cookie(name string) (string, error) {
	var cookie, err = this.request.Cookie(name)
	if err == nil {
		return cookie.Value, nil
	}
	return "", err
}

// AddSimpleCookie 添加一个简单的cookie,该cookie使用/作为path
//  name:最好只使用英文和数字作为名称,不得包含换行符回车符分号冒号等http特殊字符
//  value:最好只使用英文和数字作为值,不得包含换行符回车符分号冒号等http特殊字符
func (this *HttpContext) AddSimpleCookie(name string, value string, age int) {
	var cookie = new(http.Cookie)
	cookie.Name = name
	cookie.Value = value
	if age > 0 {
		cookie.MaxAge = age
		cookie.Expires = time.Now().Add(time.Duration(cookie.MaxAge) * time.Second)
	}
	cookie.HttpOnly = false
	cookie.Path = "/"
	this.responseWriter.Header().Add("Set-Cookie", cookie.String())
}

// AddCookie 添加一个cookie,cookie信息必须完整,若设置了cookie.MaxAge则必须设置cookie.Expires
func (this *HttpContext) AddCookie(cookie *http.Cookie) {
	this.responseWriter.Header().Add("Set-Cookie", cookie.String())
}

// ParseParams 解析参数,将路由参数,query string,表单都解析到this.Request.Form中
func (this *HttpContext) ParseParams() error {
	if !this.parsed {
		this.parsed = true
		var err = this.request.ParseMultipartForm(DefaultMaxMemory)
		if err != nil {
			return err
		}
		for k, v := range this.params {
			this.request.Form.Set(k, v)
		}
	}
	return nil
}

// ParamString 获取http参数字符串
func (this *HttpContext) ParamString(key string) string {
	return this.request.FormValue(key)
}

// ParamString 获取http参数字符串数组
func (this *HttpContext) ParamStringArray(key string) ([]string, error) {
	var result, ok = this.request.Form[key]
	if ok {
		return result, nil
	}
	return nil, TinyGoErrorParamNotFoundError.Format(key).Error()
}

// ParamString 获取http参数字符串
func (this *HttpContext) ParamBool(key string) (bool, error) {
	var result = this.request.FormValue(key)
	return strconv.ParseBool(result)
}

// ParamString 获取http参数字符串
func (this *HttpContext) ParamInt(key string) (int64, error) {
	var result = this.request.FormValue(key)
	return strconv.ParseInt(result, 10, 64)

}

// ParamString 获取http参数字符串
func (this *HttpContext) ParamFloat(key string) (float64, error) {
	var result = this.request.FormValue(key)
	return strconv.ParseFloat(result, 64)

}

// ParamString 获取http参数文件
func (this *HttpContext) ParamFile(key string) (*FormFile, error) {
	var file, header, err = this.request.FormFile(key)
	if err == nil {
		return &FormFile{file, header}, nil
	}
	return nil, err
}

// WriteString 将字符串写入http response流
func (this *HttpContext) WriteString(value string) error {
	var _, err = this.responseWriter.Write([]byte(value))
	return err
}

/////////以下为路由使用方法,一般不需要使用/////////

// RouterParts 返回路由段
func (this *HttpContext) RouterParts() []string {
	return this.urlParts
}

// SetRouterParts 设置路由段
func (this *HttpContext) SetRouterParts(parts []string) {
	this.urlParts = parts
}

// Static 返回是否是静态路由
func (this *HttpContext) Static() bool {
	return this.static
}

// SetStatic 设置当前上下文为静态路由上下文
func (this *HttpContext) SetStatic(static bool) {
	this.static = static
}

// AddParams 添加路由参数
func (this *HttpContext) AddRouterParams(key, value string) {
	this.params[key] = value
}

// RemoveRouterParams 移除路由参数
func (this *HttpContext) RemoveRouterParams(key string) {
	delete(this.params, key)
}

// AddRouter 添加执行路由,最后一级路由最先添加
func (this *HttpContext) AddRouter(router router.Router) {
	this.routers = append(this.routers, router)
}

// AddContextExector 添加执行器
func (this *HttpContext) AddContextExecutor(exector router.ContextExecutor) {
	this.executor = exector
}

// 处理该HttpContext
func (this *HttpContext) execute() {
	var ok = this.executeBeforeFilters()
	if ok {
		this.executor.Exec(this)
		this.executeAfterFilters()
	}

}

func (this *HttpContext) executeBeforeFilters() bool {
	for i := len(this.routers) - 1; i >= 0; i-- {
		var router = this.routers[i]
		var ok = router.ExecBeforeFilter(this)
		if !ok {
			return false
		}
	}
	return true
}

func (this *HttpContext) executeAfterFilters() bool {
	for i := 0; i < len(this.routers); i++ {
		var router = this.routers[i]
		var ok = router.ExecAfterFilter(this)
		if !ok {
			return false
		}
	}
	return true
}
<commit_msg>0.1.6.8<commit_after>package tinygo

import (
	"io"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/kdada/tinygo/router"
	"github.com/kdada/tinygo/session"
)

// 表单文件
type FormFile struct {
	file   multipart.File        //表单文件
	header *multipart.FileHeader //文件头信息
}

// FileName 返回文件名
func (this *FormFile) FileName() string {
	return this.header.Filename
}

// Header 返回文件头信息
func (this *FormFile) Header() textproto.MIMEHeader {
	return this.header.Header
}

// File 返回表单文件
func (this *FormFile) File() multipart.File {
	return this.file
}

// Close 关闭表单文件
func (this *FormFile) Close() {
	this.file.Close()
}

// SaveTo 将文件保存到指定路径,保存完毕后自动关闭表单文件
func (this *FormFile) SaveTo(path string) error {
	var lf, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0666)
	if err == nil {
		defer lf.Close()
		_, err = io.Copy(lf, this.file)
		if err == nil {
			this.Close()
		}
	}
	return err
}

// 路由环境
type HttpContext struct {
	urlParts       []string               //Url分段信息,每段一个字符串
	request        *http.Request          //http请求
	responseWriter http.ResponseWriter    //http响应
	session        session.Session        //http会话
	csrf           session.Session        //csrf会话
	params         map[string]string      //http参数,包含url,query,form的所有参数
	parsed         bool                   //存储参数是否已经解析过
	routers        []router.Router        //分派成功的路由链
	executor       router.ContextExecutor //存储最终执行Context的执行器
	static         bool                   //是否是静态路由
}

// Method 返回Http方法
func (this *HttpContext) Method() string {
	return this.request.Method
}

// ResponseWriter 返回ResponseWriter
func (this *HttpContext) ResponseWriter() http.ResponseWriter {
	return this.responseWriter
}

// Request 返回Request
func (this *HttpContext) Request() *http.Request {
	return this.request
}

// CsrfToken生成一个新的csrf token
func (this *HttpContext) CsrfToken() string {
	if this.csrf != nil {
		var newToken = session.Guid()
		this.csrf.SetInt(newToken, time.Now().Unix())
		return newToken
	}
	return ""
}

// ValidateCsrfToken 验证表单请求中是否存在csrf的token并且该token有效,验证后token即失效
func (this *HttpContext) ValidateCsrfToken() bool {
	if this.csrf != nil {
		var token = this.ParamString(DefaultCSRFTokenName)
		if token != "" {
			var t, ok = this.csrf.Int(token)
			if ok {
				this.csrf.Delete(token)
				return time.Now().Unix()-t <= tinyConfig.csrfexpire
			}
		}
	}
	return false
}

// Session 返回Session
func (this *HttpContext) Session() session.Session {
	return this.session
}

// Cookie 返回指定cookie的值
func (this *HttpContext) Cookie(name string) (string, error) {
	var cookie, err = this.request.Cookie(name)
	if err == nil {
		return cookie.Value, nil
	}
	return "", err
}

// AddSimpleCookie 添加一个简单的cookie,该cookie使用/作为path
//  name:最好只使用英文和数字作为名称,不得包含换行符回车符分号冒号等http特殊字符
//  value:最好只使用英文和数字作为值,不得包含换行符回车符分号冒号等http特殊字符
func (this *HttpContext) AddSimpleCookie(name string, value string, age int) {
	var cookie = new(http.Cookie)
	cookie.Name = name
	cookie.Value = value
	if age > 0 {
		cookie.MaxAge = age
		cookie.Expires = time.Now().Add(time.Duration(cookie.MaxAge) * time.Second)
	}
	cookie.HttpOnly = false
	cookie.Path = "/"
	this.responseWriter.Header().Add("Set-Cookie", cookie.String())
}

// AddCookie 添加一个cookie,cookie信息必须完整,若设置了cookie.MaxAge则必须设置cookie.Expires
func (this *HttpContext) AddCookie(cookie *http.Cookie) {
	this.responseWriter.Header().Add("Set-Cookie", cookie.String())
}

// ParseParams 解析参数,将路由参数,query string,表单都解析到this.Request.Form中
func (this *HttpContext) ParseParams() error {
	if !this.parsed {
		this.parsed = true
		var ct = this.request.Header.Get("Content-Type")
		var err error
		if strings.Contains(ct, "multipart/form-data") {
			err = this.request.ParseMultipartForm(DefaultMaxMemory)
		} else {
			err = this.request.ParseForm()
		}
		if err != nil {
			return err
		}
		for k, v := range this.params {
			this.request.Form.Set(k, v)
		}
	}
	return nil
}

// ParamString 获取http参数字符串
func (this *HttpContext) ParamString(key string) string {
	return this.request.FormValue(key)
}

// ParamString 获取http参数字符串数组
func (this *HttpContext) ParamStringArray(key string) ([]string, error) {
	var result, ok = this.request.Form[key]
	if ok {
		return result, nil
	}
	return nil, TinyGoErrorParamNotFoundError.Format(key).Error()
}

// ParamString 获取http参数字符串
func (this *HttpContext) ParamBool(key string) (bool, error) {
	var result = this.request.FormValue(key)
	return strconv.ParseBool(result)
}

// ParamString 获取http参数字符串
func (this *HttpContext) ParamInt(key string) (int64, error) {
	var result = this.request.FormValue(key)
	return strconv.ParseInt(result, 10, 64)

}

// ParamString 获取http参数字符串
func (this *HttpContext) ParamFloat(key string) (float64, error) {
	var result = this.request.FormValue(key)
	return strconv.ParseFloat(result, 64)

}

// ParamString 获取http参数文件
func (this *HttpContext) ParamFile(key string) (*FormFile, error) {
	var file, header, err = this.request.FormFile(key)
	if err == nil {
		return &FormFile{file, header}, nil
	}
	return nil, err
}

// WriteString 将字符串写入http response流
func (this *HttpContext) WriteString(value string) error {
	var _, err = this.responseWriter.Write([]byte(value))
	return err
}

/////////以下为路由使用方法,一般不需要使用/////////

// RouterParts 返回路由段
func (this *HttpContext) RouterParts() []string {
	return this.urlParts
}

// SetRouterParts 设置路由段
func (this *HttpContext) SetRouterParts(parts []string) {
	this.urlParts = parts
}

// Static 返回是否是静态路由
func (this *HttpContext) Static() bool {
	return this.static
}

// SetStatic 设置当前上下文为静态路由上下文
func (this *HttpContext) SetStatic(static bool) {
	this.static = static
}

// AddParams 添加路由参数
func (this *HttpContext) AddRouterParams(key, value string) {
	this.params[key] = value
}

// RemoveRouterParams 移除路由参数
func (this *HttpContext) RemoveRouterParams(key string) {
	delete(this.params, key)
}

// AddRouter 添加执行路由,最后一级路由最先添加
func (this *HttpContext) AddRouter(router router.Router) {
	this.routers = append(this.routers, router)
}

// AddContextExector 添加执行器
func (this *HttpContext) AddContextExecutor(exector router.ContextExecutor) {
	this.executor = exector
}

// 处理该HttpContext
func (this *HttpContext) execute() {
	var ok = this.executeBeforeFilters()
	if ok {
		this.executor.Exec(this)
		this.executeAfterFilters()
	}

}

func (this *HttpContext) executeBeforeFilters() bool {
	for i := len(this.routers) - 1; i >= 0; i-- {
		var router = this.routers[i]
		var ok = router.ExecBeforeFilter(this)
		if !ok {
			return false
		}
	}
	return true
}

func (this *HttpContext) executeAfterFilters() bool {
	for i := 0; i < len(this.routers); i++ {
		var router = this.routers[i]
		var ok = router.ExecAfterFilter(this)
		if !ok {
			return false
		}
	}
	return true
}
<|endoftext|>