<commit_before>package api

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"sync"
	"time"

	"github.com/docker/swarm/cluster"
)

// EventsHandler broadcasts events to multiple client listeners.
type eventsHandler struct {
	sync.RWMutex
	ws map[string]io.Writer
	cs map[string]chan struct{}
}

// NewEventsHandler creates a new EventsHandler for a cluster.
// The new eventsHandler is initialized with no writers or channels.
func newEventsHandler() *eventsHandler {
	return &eventsHandler{
		ws: make(map[string]io.Writer),
		cs: make(map[string]chan struct{}),
	}
}

// Add adds the writer and a new channel for the remote address.
func (eh *eventsHandler) Add(remoteAddr string, w io.Writer) {
	eh.Lock()
	eh.ws[remoteAddr] = w
	eh.cs[remoteAddr] = make(chan struct{})
	eh.Unlock()
}

// Wait waits on a signal from the remote address.
func (eh *eventsHandler) Wait(remoteAddr string, until int64) {

	timer := time.NewTimer(0)

	// Based on issue https://github.com/golang/go/issues/14383.
	// If timer has already expired, `time.Stop` will return false.
	// And we have to drain the channel manually.
	if !timer.Stop() {
		<-timer.C
	}

	if until > 0 {
		dur := time.Unix(until, 0).Sub(time.Now())
		timer = time.NewTimer(dur)
	}

	// subscribe to http client close event
	eh.RLock()
	w := eh.ws[remoteAddr]
	ch := eh.cs[remoteAddr]
	eh.RUnlock()
	var closeNotify <-chan bool
	if closeNotifier, ok := w.(http.CloseNotifier); ok {
		closeNotify = closeNotifier.CloseNotify()
	}

	select {
	case <-ch:
	case <-closeNotify:
	case <-timer.C: // `--until` timeout
		close(ch)
	}
	eh.cleanupHandler(remoteAddr)
}

func (eh *eventsHandler) cleanupHandler(remoteAddr string) {
	eh.Lock()
	// the maps are expected to have the same keys
	delete(eh.cs, remoteAddr)
	delete(eh.ws, remoteAddr)
	eh.Unlock()

}

// Handle writes information about a cluster event to each remote address in the cluster that has been added to the events handler.
// After an unsuccessful write to a remote address, the associated channel is closed and the address is removed from the events handler.
func (eh *eventsHandler) Handle(e *cluster.Event) error {
	// remove this hack once 1.10 is broadly adopted
	from := e.From
	e.From = e.From + " node:" + e.Engine.Name

	// Attributes will be nil if the event was sent by engine < 1.10
	if e.Actor.Attributes == nil {
		e.Actor.Attributes = make(map[string]string)
	}
	e.Actor.Attributes["node.name"] = e.Engine.Name
	e.Actor.Attributes["node.id"] = e.Engine.ID
	e.Actor.Attributes["node.addr"] = e.Engine.Addr
	e.Actor.Attributes["node.ip"] = e.Engine.IP

	data, err := json.Marshal(e)
	e.From = from
	if err != nil {
		return err
	}

	// remove the node field once 1.10 is broadly adopted & interlock stop relying on it
	node := fmt.Sprintf(",%q:{%q:%q,%q:%q,%q:%q,%q:%q}}",
		"node",
		"Name", e.Engine.Name,
		"Id", e.Engine.ID,
		"Addr", e.Engine.Addr,
		"Ip", e.Engine.IP,
	)

	// insert Node field
	data = data[:len(data)-1]
	data = append(data, []byte(node)...)

	var failed []string

	eh.RLock()

	for key, w := range eh.ws {
		if _, err := fmt.Fprint(w, string(data)); err != nil {
			// collect them to handle later under Lock
			failed = append(failed, key)
			continue
		}

		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	eh.RUnlock()
	eh.Lock()
	if len(failed) > 0 {
		for _, key := range failed {
			if ch, ok := eh.cs[key]; ok {
				close(ch)
			}
			delete(eh.cs, key)
		}
	}

	eh.Unlock()

	return nil
}

// Size returns the number of remote addresses that the events handler currently contains.
func (eh *eventsHandler) Size() int {
	eh.RLock()
	defer eh.RUnlock()
	return len(eh.ws)
}
<commit_msg>concurrent write/flush to http.ResponseWriter or http.Flusher would cause data race<commit_after>package api

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"sync"
	"time"

	"github.com/docker/swarm/cluster"
)

// EventsHandler broadcasts events to multiple client listeners.
type eventsHandler struct {
	sync.RWMutex
	ws map[string]io.Writer
	cs map[string]chan struct{}
}

// NewEventsHandler creates a new EventsHandler for a cluster.
// The new eventsHandler is initialized with no writers or channels.
func newEventsHandler() *eventsHandler {
	return &eventsHandler{
		ws: make(map[string]io.Writer),
		cs: make(map[string]chan struct{}),
	}
}

// Add adds the writer and a new channel for the remote address.
func (eh *eventsHandler) Add(remoteAddr string, w io.Writer) {
	eh.Lock()
	eh.ws[remoteAddr] = w
	eh.cs[remoteAddr] = make(chan struct{})
	eh.Unlock()
}

// Wait waits on a signal from the remote address.
func (eh *eventsHandler) Wait(remoteAddr string, until int64) {

	timer := time.NewTimer(0)

	// Based on issue https://github.com/golang/go/issues/14383.
	// If timer has already expired, `time.Stop` will return false.
	// And we have to drain the channel manually.
	if !timer.Stop() {
		<-timer.C
	}

	if until > 0 {
		dur := time.Unix(until, 0).Sub(time.Now())
		timer = time.NewTimer(dur)
	}

	// subscribe to http client close event
	eh.RLock()
	w := eh.ws[remoteAddr]
	ch := eh.cs[remoteAddr]
	eh.RUnlock()
	var closeNotify <-chan bool
	if closeNotifier, ok := w.(http.CloseNotifier); ok {
		closeNotify = closeNotifier.CloseNotify()
	}

	select {
	case <-ch:
	case <-closeNotify:
	case <-timer.C: // `--until` timeout
		close(ch)
	}
	eh.cleanupHandler(remoteAddr)
}

func (eh *eventsHandler) cleanupHandler(remoteAddr string) {
	eh.Lock()
	// the maps are expected to have the same keys
	delete(eh.cs, remoteAddr)
	delete(eh.ws, remoteAddr)
	eh.Unlock()

}

// Handle writes information about a cluster event to each remote address in the cluster that has been added to the events handler.
// After an unsuccessful write to a remote address, the associated channel is closed and the address is removed from the events handler.
func (eh *eventsHandler) Handle(e *cluster.Event) error {
	// remove this hack once 1.10 is broadly adopted
	from := e.From
	e.From = e.From + " node:" + e.Engine.Name

	// Attributes will be nil if the event was sent by engine < 1.10
	if e.Actor.Attributes == nil {
		e.Actor.Attributes = make(map[string]string)
	}
	e.Actor.Attributes["node.name"] = e.Engine.Name
	e.Actor.Attributes["node.id"] = e.Engine.ID
	e.Actor.Attributes["node.addr"] = e.Engine.Addr
	e.Actor.Attributes["node.ip"] = e.Engine.IP

	data, err := json.Marshal(e)
	e.From = from
	if err != nil {
		return err
	}

	// remove the node field once 1.10 is broadly adopted & interlock stop relying on it
	node := fmt.Sprintf(",%q:{%q:%q,%q:%q,%q:%q,%q:%q}}",
		"node",
		"Name", e.Engine.Name,
		"Id", e.Engine.ID,
		"Addr", e.Engine.Addr,
		"Ip", e.Engine.IP,
	)

	// insert Node field
	data = data[:len(data)-1]
	data = append(data, []byte(node)...)

	var failed []string

	eh.Lock()

	for key, w := range eh.ws {
		if _, err := fmt.Fprint(w, string(data)); err != nil {
			// collect them to handle later under Lock
			failed = append(failed, key)
			continue
		}

		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	if len(failed) > 0 {
		for _, key := range failed {
			if ch, ok := eh.cs[key]; ok {
				close(ch)
			}
			delete(eh.cs, key)
		}
	}

	eh.Unlock()

	return nil
}

// Size returns the number of remote addresses that the events handler currently contains.
func (eh *eventsHandler) Size() int {
	eh.RLock()
	defer eh.RUnlock()
	return len(eh.ws)
}
<|endoftext|>


<commit_before>// Package email is designed to provide an "email interface for humans."
// Designed to be robust and flexible, the email package aims to make sending email easy without getting in the way.
package email

import (
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"net/mail"
	"net/smtp"
	"net/textproto"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const (
	// MaxLineLength is the maximum line length per RFC 2045
	MaxLineLength = 76
)

// Email is the type used for email messages
type Email struct {
	From        string
	To          []string
	Bcc         []string
	Cc          []string
	Subject     string
	Text        []byte // Plaintext message (optional)
	HTML        []byte // Html message (optional)
	Headers     textproto.MIMEHeader
	Attachments []*Attachment
	ReadReceipt []string
}

// NewEmail creates an Email, and returns the pointer to it.
func NewEmail() *Email {
	return &Email{Headers: textproto.MIMEHeader{}}
}

// Attach is used to attach content from an io.Reader to the email.
// Required parameters include an io.Reader, the desired filename for the attachment, and the Content-Type
// The function will return the created Attachment for reference, as well as nil for the error, if successful.
func (e *Email) Attach(r io.Reader, filename string, c string) (a *Attachment, err error) {
	var buffer bytes.Buffer
	if _, err = io.Copy(&buffer, r); err != nil {
		return
	}
	at := &Attachment{
		Filename: filename,
		Header:   textproto.MIMEHeader{},
		Content:  buffer.Bytes(),
	}
	// Get the Content-Type to be used in the MIMEHeader
	if c != "" {
		at.Header.Set("Content-Type", c)
	} else {
		// If the Content-Type is blank, set the Content-Type to "application/octet-stream"
		at.Header.Set("Content-Type", "application/octet-stream")
	}
	at.Header.Set("Content-Disposition", fmt.Sprintf("attachment;\r\n filename=\"%s\"", filename))
	at.Header.Set("Content-Transfer-Encoding", "base64")
	e.Attachments = append(e.Attachments, at)
	return at, nil
}

// AttachFile is used to attach content to the email.
// It attempts to open the file referenced by filename and, if successful, creates an Attachment.
// This Attachment is then appended to the slice of Email.Attachments.
// The function will then return the Attachment for reference, as well as nil for the error, if successful.
func (e *Email) AttachFile(filename string) (a *Attachment, err error) {
	f, err := os.Open(filename)
	if err != nil {
		return
	}
	ct := mime.TypeByExtension(filepath.Ext(filename))
	basename := filepath.Base(filename)
	return e.Attach(f, basename, ct)
}

// msgHeaders merges the Email's various fields and custom headers together in a
// standards compliant way to create a MIMEHeader to be used in the resulting
// message. It does not alter e.Headers.
//
// "e"'s fields To, Cc, From, Subject will be used unless they are present in
// e.Headers. Unless set in e.Headers, "Date" will filled with the current time.
func (e *Email) msgHeaders() textproto.MIMEHeader {
	res := make(textproto.MIMEHeader, len(e.Headers)+4)
	if e.Headers != nil {
		for _, h := range []string{"To", "Cc", "From", "Subject", "Date"} {
			if v, ok := e.Headers[h]; ok {
				res[h] = v
			}
		}
	}
	// Set headers if there are values.
	if _, ok := res["To"]; !ok && len(e.To) > 0 {
		res.Set("To", strings.Join(e.To, ", "))
	}
	if _, ok := res["Cc"]; !ok && len(e.Cc) > 0 {
		res.Set("Cc", strings.Join(e.Cc, ", "))
	}
	if _, ok := res["Subject"]; !ok && e.Subject != "" {
		res.Set("Subject", e.Subject)
	}
	// Date and From are required headers.
	if _, ok := res["From"]; !ok {
		res.Set("From", e.From)
	}
	if _, ok := res["Date"]; !ok {
		res.Set("Date", time.Now().Format(time.RFC1123Z))
	}
	for field, vals := range e.Headers {
		if _, ok := res[field]; !ok {
			res[field] = vals
		}
	}
	return res
}

// Bytes converts the Email object to a []byte representation, including all needed MIMEHeaders, boundaries, etc.
func (e *Email) Bytes() ([]byte, error) {
	// TODO: better guess buffer size
	buff := bytes.NewBuffer(make([]byte, 0, 4096))

	headers := e.msgHeaders()
	w := multipart.NewWriter(buff)
	// TODO: determine the content type based on message/attachment mix.
	headers.Set("Content-Type", "multipart/mixed;\r\n boundary="+w.Boundary())
	headerToBytes(buff, headers)
	io.WriteString(buff, "\r\n")

	// Start the multipart/mixed part
	fmt.Fprintf(buff, "--%s\r\n", w.Boundary())
	header := textproto.MIMEHeader{}
	// Check to see if there is a Text or HTML field
	if len(e.Text) > 0 || len(e.HTML) > 0 {
		subWriter := multipart.NewWriter(buff)
		// Create the multipart alternative part
		header.Set("Content-Type", fmt.Sprintf("multipart/alternative;\r\n boundary=%s\r\n", subWriter.Boundary()))
		// Write the header
		headerToBytes(buff, header)
		// Create the body sections
		if len(e.Text) > 0 {
			header.Set("Content-Type", fmt.Sprintf("text/plain; charset=UTF-8"))
			header.Set("Content-Transfer-Encoding", "quoted-printable")
			if _, err := subWriter.CreatePart(header); err != nil {
				return nil, err
			}
			// Write the text
			if err := quotePrintEncode(buff, e.Text); err != nil {
				return nil, err
			}
		}
		if len(e.HTML) > 0 {
			header.Set("Content-Type", fmt.Sprintf("text/html; charset=UTF-8"))
			header.Set("Content-Transfer-Encoding", "quoted-printable")
			if _, err := subWriter.CreatePart(header); err != nil {
				return nil, err
			}
			// Write the text
			if err := quotePrintEncode(buff, e.HTML); err != nil {
				return nil, err
			}
		}
		if err := subWriter.Close(); err != nil {
			return nil, err
		}
	}
	// Create attachment part, if necessary
	for _, a := range e.Attachments {
		ap, err := w.CreatePart(a.Header)
		if err != nil {
			return nil, err
		}
		// Write the base64Wrapped content to the part
		base64Wrap(ap, a.Content)
	}
	if err := w.Close(); err != nil {
		return nil, err
	}
	return buff.Bytes(), nil
}

// Send an email using the given host and SMTP auth (optional), returns any error thrown by smtp.SendMail
// This function merges the To, Cc, and Bcc fields and calls the smtp.SendMail function using the Email.Bytes() output as the message
func (e *Email) Send(addr string, a smtp.Auth) error {
	// Merge the To, Cc, and Bcc fields
	to := make([]string, 0, len(e.To)+len(e.Cc)+len(e.Bcc))
	to = append(append(append(to, e.To...), e.Cc...), e.Bcc...)
	for i := 0; i < len(to); i++ {
		addr, _ := mail.ParseAddress(to[i])
		to[i] = addr.Address
	}
	// Check to make sure there is at least one recipient and one "From" address
	if e.From == "" || len(to) == 0 {
		return errors.New("Must specify at least one From address and one To address")
	}
	from, err := mail.ParseAddress(e.From)
	if err != nil {
		return err
	}
	raw, err := e.Bytes()
	if err != nil {
		return err
	}
	return smtp.SendMail(addr, a, from.Address, to, raw)
}

// Attachment is a struct representing an email attachment.
// Based on the mime/multipart.FileHeader struct, Attachment contains the name, MIMEHeader, and content of the attachment in question
type Attachment struct {
	Filename string
	Header   textproto.MIMEHeader
	Content  []byte
}

// quotePrintEncode writes the quoted-printable text to the IO Writer (according to RFC 2045)
func quotePrintEncode(w io.Writer, body []byte) error {
	var buf [3]byte
	mc := 0
	for _, c := range body {
		// We're assuming Unix style text formats as input (LF line break), and
		// quoted-printable uses CRLF line breaks. (Literal CRs will become
		// "=0D", but probably shouldn't be there to begin with!)
		if c == '\n' {
			io.WriteString(w, "\r\n")
			mc = 0
			continue
		}

		var nextOut []byte
		if isPrintable[c] {
			buf[0] = c
			nextOut = buf[:1]
		} else {
			nextOut = buf[:]
			qpEscape(nextOut, c)
		}

		// Add a soft line break if the next (encoded) byte would push this line
		// to or past the limit.
		if mc+len(nextOut) >= MaxLineLength {
			if _, err := io.WriteString(w, "=\r\n"); err != nil {
				return err
			}
			mc = 0
		}

		if _, err := w.Write(nextOut); err != nil {
			return err
		}
		mc += len(nextOut)
	}
	// No trailing end-of-line?? Soft line break, then. TODO: is this sane?
	if mc > 0 {
		io.WriteString(w, "=\r\n")
	}
	return nil
}

// isPrintable holds true if the byte given is "printable" according to RFC 2045, false otherwise
var isPrintable [256]bool

func init() {
	for c := '!'; c <= '<'; c++ {
		isPrintable[c] = true
	}
	for c := '>'; c <= '~'; c++ {
		isPrintable[c] = true
	}
	isPrintable[' '] = true
	isPrintable['\n'] = true
	isPrintable['\t'] = true
}

// qpEscape is a helper function for quotePrintEncode which escapes a
// non-printable byte. Expects len(dest) == 3.
func qpEscape(dest []byte, c byte) {
	const nums = "0123456789ABCDEF"
	dest[0] = '='
	dest[1] = nums[(c&0xf0)>>4]
	dest[2] = nums[(c & 0xf)]
}

// base64Wrap encodes the attachment content, and wraps it according to RFC 2045 standards (every 76 chars)
// The output is then written to the specified io.Writer
func base64Wrap(w io.Writer, b []byte) {
	// 57 raw bytes per 76-byte base64 line.
	const maxRaw = 57
	// Buffer for each line, including trailing CRLF.
	buffer := make([]byte, MaxLineLength+len("\r\n"))
	copy(buffer[MaxLineLength:], "\r\n")
	// Process raw chunks until there's no longer enough to fill a line.
	for len(b) >= maxRaw {
		base64.StdEncoding.Encode(buffer, b[:maxRaw])
		w.Write(buffer)
		b = b[maxRaw:]
	}
	// Handle the last chunk of bytes.
	if len(b) > 0 {
		out := buffer[:base64.StdEncoding.EncodedLen(len(b))]
		base64.StdEncoding.Encode(out, b)
		out = append(out, "\r\n"...)
		w.Write(out)
	}
}

// headerToBytes renders "header" to "buff". If there are multiple values for a
// field, multiple "Field: value\r\n" lines will be emitted.
func headerToBytes(buff *bytes.Buffer, header textproto.MIMEHeader) {
	for field, vals := range header {
		for _, subval := range vals {
			// bytes.Buffer.Write() never returns an error.
			io.WriteString(buff, field)
			io.WriteString(buff, ": ")
			io.WriteString(buff, subval)
			io.WriteString(buff, "\r\n")
		}
	}
}
<commit_msg>Added Mime-Version header - fixes #8<commit_after>// Package email is designed to provide an "email interface for humans."
// Designed to be robust and flexible, the email package aims to make sending email easy without getting in the way.
package email

import (
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"net/mail"
	"net/smtp"
	"net/textproto"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const (
	// MaxLineLength is the maximum line length per RFC 2045
	MaxLineLength = 76
)

// Email is the type used for email messages
type Email struct {
	From        string
	To          []string
	Bcc         []string
	Cc          []string
	Subject     string
	Text        []byte // Plaintext message (optional)
	HTML        []byte // Html message (optional)
	Headers     textproto.MIMEHeader
	Attachments []*Attachment
	ReadReceipt []string
}

// NewEmail creates an Email, and returns the pointer to it.
func NewEmail() *Email {
	return &Email{Headers: textproto.MIMEHeader{}}
}

// Attach is used to attach content from an io.Reader to the email.
// Required parameters include an io.Reader, the desired filename for the attachment, and the Content-Type
// The function will return the created Attachment for reference, as well as nil for the error, if successful.
func (e *Email) Attach(r io.Reader, filename string, c string) (a *Attachment, err error) {
	var buffer bytes.Buffer
	if _, err = io.Copy(&buffer, r); err != nil {
		return
	}
	at := &Attachment{
		Filename: filename,
		Header:   textproto.MIMEHeader{},
		Content:  buffer.Bytes(),
	}
	// Get the Content-Type to be used in the MIMEHeader
	if c != "" {
		at.Header.Set("Content-Type", c)
	} else {
		// If the Content-Type is blank, set the Content-Type to "application/octet-stream"
		at.Header.Set("Content-Type", "application/octet-stream")
	}
	at.Header.Set("Content-Disposition", fmt.Sprintf("attachment;\r\n filename=\"%s\"", filename))
	at.Header.Set("Content-Transfer-Encoding", "base64")
	e.Attachments = append(e.Attachments, at)
	return at, nil
}

// AttachFile is used to attach content to the email.
// It attempts to open the file referenced by filename and, if successful, creates an Attachment.
// This Attachment is then appended to the slice of Email.Attachments.
// The function will then return the Attachment for reference, as well as nil for the error, if successful.
func (e *Email) AttachFile(filename string) (a *Attachment, err error) {
	f, err := os.Open(filename)
	if err != nil {
		return
	}
	ct := mime.TypeByExtension(filepath.Ext(filename))
	basename := filepath.Base(filename)
	return e.Attach(f, basename, ct)
}

// msgHeaders merges the Email's various fields and custom headers together in a
// standards compliant way to create a MIMEHeader to be used in the resulting
// message. It does not alter e.Headers.
//
// "e"'s fields To, Cc, From, Subject will be used unless they are present in
// e.Headers. Unless set in e.Headers, "Date" will filled with the current time.
func (e *Email) msgHeaders() textproto.MIMEHeader {
	res := make(textproto.MIMEHeader, len(e.Headers)+4)
	if e.Headers != nil {
		for _, h := range []string{"To", "Cc", "From", "Subject", "Date"} {
			if v, ok := e.Headers[h]; ok {
				res[h] = v
			}
		}
	}
	// Set headers if there are values.
	if _, ok := res["To"]; !ok && len(e.To) > 0 {
		res.Set("To", strings.Join(e.To, ", "))
	}
	if _, ok := res["Cc"]; !ok && len(e.Cc) > 0 {
		res.Set("Cc", strings.Join(e.Cc, ", "))
	}
	if _, ok := res["Subject"]; !ok && e.Subject != "" {
		res.Set("Subject", e.Subject)
	}
	// Date and From are required headers.
	if _, ok := res["From"]; !ok {
		res.Set("From", e.From)
	}
	if _, ok := res["Date"]; !ok {
		res.Set("Date", time.Now().Format(time.RFC1123Z))
	}
	if _, ok := res["Mime-Version"]; !ok {
		res.Set("Mime-Version", "1.0")
	}
	for field, vals := range e.Headers {
		if _, ok := res[field]; !ok {
			res[field] = vals
		}
	}
	return res
}

// Bytes converts the Email object to a []byte representation, including all needed MIMEHeaders, boundaries, etc.
func (e *Email) Bytes() ([]byte, error) {
	// TODO: better guess buffer size
	buff := bytes.NewBuffer(make([]byte, 0, 4096))

	headers := e.msgHeaders()
	w := multipart.NewWriter(buff)
	// TODO: determine the content type based on message/attachment mix.
	headers.Set("Content-Type", "multipart/mixed;\r\n boundary="+w.Boundary())
	headerToBytes(buff, headers)
	io.WriteString(buff, "\r\n")

	// Start the multipart/mixed part
	fmt.Fprintf(buff, "--%s\r\n", w.Boundary())
	header := textproto.MIMEHeader{}
	// Check to see if there is a Text or HTML field
	if len(e.Text) > 0 || len(e.HTML) > 0 {
		subWriter := multipart.NewWriter(buff)
		// Create the multipart alternative part
		header.Set("Content-Type", fmt.Sprintf("multipart/alternative;\r\n boundary=%s\r\n", subWriter.Boundary()))
		// Write the header
		headerToBytes(buff, header)
		// Create the body sections
		if len(e.Text) > 0 {
			header.Set("Content-Type", fmt.Sprintf("text/plain; charset=UTF-8"))
			header.Set("Content-Transfer-Encoding", "quoted-printable")
			if _, err := subWriter.CreatePart(header); err != nil {
				return nil, err
			}
			// Write the text
			if err := quotePrintEncode(buff, e.Text); err != nil {
				return nil, err
			}
		}
		if len(e.HTML) > 0 {
			header.Set("Content-Type", fmt.Sprintf("text/html; charset=UTF-8"))
			header.Set("Content-Transfer-Encoding", "quoted-printable")
			if _, err := subWriter.CreatePart(header); err != nil {
				return nil, err
			}
			// Write the text
			if err := quotePrintEncode(buff, e.HTML); err != nil {
				return nil, err
			}
		}
		if err := subWriter.Close(); err != nil {
			return nil, err
		}
	}
	// Create attachment part, if necessary
	for _, a := range e.Attachments {
		ap, err := w.CreatePart(a.Header)
		if err != nil {
			return nil, err
		}
		// Write the base64Wrapped content to the part
		base64Wrap(ap, a.Content)
	}
	if err := w.Close(); err != nil {
		return nil, err
	}
	return buff.Bytes(), nil
}

// Send an email using the given host and SMTP auth (optional), returns any error thrown by smtp.SendMail
// This function merges the To, Cc, and Bcc fields and calls the smtp.SendMail function using the Email.Bytes() output as the message
func (e *Email) Send(addr string, a smtp.Auth) error {
	// Merge the To, Cc, and Bcc fields
	to := make([]string, 0, len(e.To)+len(e.Cc)+len(e.Bcc))
	to = append(append(append(to, e.To...), e.Cc...), e.Bcc...)
	for i := 0; i < len(to); i++ {
		addr, _ := mail.ParseAddress(to[i])
		to[i] = addr.Address
	}
	// Check to make sure there is at least one recipient and one "From" address
	if e.From == "" || len(to) == 0 {
		return errors.New("Must specify at least one From address and one To address")
	}
	from, err := mail.ParseAddress(e.From)
	if err != nil {
		return err
	}
	raw, err := e.Bytes()
	if err != nil {
		return err
	}
	return smtp.SendMail(addr, a, from.Address, to, raw)
}

// Attachment is a struct representing an email attachment.
// Based on the mime/multipart.FileHeader struct, Attachment contains the name, MIMEHeader, and content of the attachment in question
type Attachment struct {
	Filename string
	Header   textproto.MIMEHeader
	Content  []byte
}

// quotePrintEncode writes the quoted-printable text to the IO Writer (according to RFC 2045)
func quotePrintEncode(w io.Writer, body []byte) error {
	var buf [3]byte
	mc := 0
	for _, c := range body {
		// We're assuming Unix style text formats as input (LF line break), and
		// quoted-printable uses CRLF line breaks. (Literal CRs will become
		// "=0D", but probably shouldn't be there to begin with!)
		if c == '\n' {
			io.WriteString(w, "\r\n")
			mc = 0
			continue
		}

		var nextOut []byte
		if isPrintable[c] {
			buf[0] = c
			nextOut = buf[:1]
		} else {
			nextOut = buf[:]
			qpEscape(nextOut, c)
		}

		// Add a soft line break if the next (encoded) byte would push this line
		// to or past the limit.
		if mc+len(nextOut) >= MaxLineLength {
			if _, err := io.WriteString(w, "=\r\n"); err != nil {
				return err
			}
			mc = 0
		}

		if _, err := w.Write(nextOut); err != nil {
			return err
		}
		mc += len(nextOut)
	}
	// No trailing end-of-line?? Soft line break, then. TODO: is this sane?
	if mc > 0 {
		io.WriteString(w, "=\r\n")
	}
	return nil
}

// isPrintable holds true if the byte given is "printable" according to RFC 2045, false otherwise
var isPrintable [256]bool

func init() {
	for c := '!'; c <= '<'; c++ {
		isPrintable[c] = true
	}
	for c := '>'; c <= '~'; c++ {
		isPrintable[c] = true
	}
	isPrintable[' '] = true
	isPrintable['\n'] = true
	isPrintable['\t'] = true
}

// qpEscape is a helper function for quotePrintEncode which escapes a
// non-printable byte. Expects len(dest) == 3.
func qpEscape(dest []byte, c byte) {
	const nums = "0123456789ABCDEF"
	dest[0] = '='
	dest[1] = nums[(c&0xf0)>>4]
	dest[2] = nums[(c & 0xf)]
}

// base64Wrap encodes the attachment content, and wraps it according to RFC 2045 standards (every 76 chars)
// The output is then written to the specified io.Writer
func base64Wrap(w io.Writer, b []byte) {
	// 57 raw bytes per 76-byte base64 line.
	const maxRaw = 57
	// Buffer for each line, including trailing CRLF.
	buffer := make([]byte, MaxLineLength+len("\r\n"))
	copy(buffer[MaxLineLength:], "\r\n")
	// Process raw chunks until there's no longer enough to fill a line.
	for len(b) >= maxRaw {
		base64.StdEncoding.Encode(buffer, b[:maxRaw])
		w.Write(buffer)
		b = b[maxRaw:]
	}
	// Handle the last chunk of bytes.
	if len(b) > 0 {
		out := buffer[:base64.StdEncoding.EncodedLen(len(b))]
		base64.StdEncoding.Encode(out, b)
		out = append(out, "\r\n"...)
		w.Write(out)
	}
}

// headerToBytes renders "header" to "buff". If there are multiple values for a
// field, multiple "Field: value\r\n" lines will be emitted.
func headerToBytes(buff *bytes.Buffer, header textproto.MIMEHeader) {
	for field, vals := range header {
		for _, subval := range vals {
			// bytes.Buffer.Write() never returns an error.
			io.WriteString(buff, field)
			io.WriteString(buff, ": ")
			io.WriteString(buff, subval)
			io.WriteString(buff, "\r\n")
		}
	}
}
<|endoftext|>


<commit_before>//Package email is designed to provide an "email interface for humans."
//Designed to be robust and flexible, the email package aims to make sending email easy without getting in the way.
package email

import (
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/mail"
	"net/smtp"
	"net/textproto"
	"os"
	"path/filepath"
	"strings"
)

//Email is the type used for email messages
type Email struct {
	From        string
	To          []string
	Bcc         []string
	Cc          []string
	Subject     string
	Text        string //Plaintext message (optional)
	Html        string //Html message (optional)
	Headers     textproto.MIMEHeader
	Attachments map[string]*Attachment
	ReadReceipt []string
}

//NewEmail creates an Email, and returns the pointer to it.
func NewEmail() *Email {
	return &Email{Attachments: make(map[string]*Attachment), Headers: textproto.MIMEHeader{}}
}

//Attach is used to attach a file to the email.
//It attempts to open the file referenced by filename and, if successful, creates an Attachment.
//This Attachment is then appended to the slice of Email.Attachments.
//The function will then return the Attachment for reference, as well as nil for the error, if successful.
func (e *Email) Attach(filename string) (a *Attachment, err error) {
	//Check if the file exists, return any error
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		return nil, err
	}
	//Read the file, and set the appropriate headers
	buffer, _ := ioutil.ReadFile(filename)
	e.Attachments[filename] = &Attachment{
		Filename: filename,
		Header:   textproto.MIMEHeader{},
		Content:  buffer}
	at := e.Attachments[filename]
	//Get the Content-Type to be used in the MIMEHeader
	ct := mime.TypeByExtension(filepath.Ext(filename))
	if ct != "" {
		at.Header.Set("Content-Type", ct)
	} else {
		//If the Content-Type is blank, set the Content-Type to "application/octet-stream"
		at.Header.Set("Content-Type", "application/octet-stream")
	}
	at.Header.Set("Content-Disposition", fmt.Sprintf("attachment;\r\n filename=\"%s\"", filename))
	at.Header.Set("Content-Transfer-Encoding", "base64")
	return e.Attachments[filename], nil
}

//Bytes converts the Email object to a []byte representation, including all needed MIMEHeaders, boundaries, etc.
func (e *Email) Bytes() ([]byte, error) {
	buff := &bytes.Buffer{}
	w := multipart.NewWriter(buff)
	//Set the appropriate headers (overwriting any conflicts)
	//Leave out Bcc (only included in envelope headers)
	//TODO: Support wrapping on 76 characters (ref: MIME RFC)
	e.Headers.Set("To", strings.Join(e.To, ","))
	if e.Cc != nil {
		e.Headers.Set("Cc", strings.Join(e.Cc, ","))
	}
	e.Headers.Set("From", e.From)
	e.Headers.Set("Subject", e.Subject)
	if len(e.ReadReceipt) != 0 {
		e.Headers.Set("Disposition-Notification-To", strings.Join(e.ReadReceipt, ","))
	}
	e.Headers.Set("MIME-Version", "1.0")
	e.Headers.Set("Content-Type", fmt.Sprintf("multipart/mixed;\r\n boundary=%s\r\n", w.Boundary()))

	//Write the envelope headers (including any custom headers)
	if err := headerToBytes(buff, e.Headers); err != nil {
	}
	//Start the multipart/mixed part
	fmt.Fprintf(buff, "--%s\r\n", w.Boundary())
	header := textproto.MIMEHeader{}
	//Check to see if there is a Text or HTML field
	if e.Text != "" || e.Html != "" {
		subWriter := multipart.NewWriter(buff)
		//Create the multipart alternative part
		header.Set("Content-Type", fmt.Sprintf("multipart/alternative;\r\n boundary=%s\r\n", subWriter.Boundary()))
		//Write the header
		if err := headerToBytes(buff, header); err != nil {

		}
		//Create the body sections
		if e.Text != "" {
			header.Set("Content-Type", fmt.Sprintf("text/plain; charset=UTF-8"))
			part, err := subWriter.CreatePart(header)
			if err != nil {

			}
			// Write the text
			if err := writeMIME(part, e.Text); err != nil {

			}
		}
		if e.Html != "" {
			header.Set("Content-Type", fmt.Sprintf("text/html; charset=UTF-8"))
			subWriter.CreatePart(header)
			// Write the text
			if err := writeMIME(buff, e.Html); err != nil {

			}
		}
		subWriter.Close()
	}
	//Create attachment part, if necessary
	if e.Attachments != nil {
		for _, a := range e.Attachments {
			ap, err := w.CreatePart(a.Header)
			if err != nil {
				return nil, err
			}
			//Write the base64Wrapped content to the part
			base64Wrap(ap, a.Content)
		}
	}
	w.Close()
	return buff.Bytes(), nil
}

//Send an email using the given host and SMTP auth (optional), returns any error thrown by smtp.SendMail
//This function merges the To, Cc, and Bcc fields and calls the smtp.SendMail function using the Email.Bytes() output as the message
func (e *Email) Send(addr string, a smtp.Auth) error {
	//Check to make sure there is at least one recipient and one "From" address
	if e.From == "" || (len(e.To) == 0 && len(e.Cc) == 0 && len(e.Bcc) == 0) {
		return errors.New("Must specify at least one From address and one To address")
	}
	// Merge the To, Cc, and Bcc fields
	to := append(append(e.To, e.Cc...), e.Bcc...)
	from, err := mail.ParseAddress(e.From)
	if err != nil {
		return err
	}
	raw, err := e.Bytes()
	if err != nil {
		return err
	}
	return smtp.SendMail(addr, a, from.Address, to, raw)
}

//writeMIME writes the quoted-printable text to the IO Writer
func writeMIME(w io.Writer, s string) error {
	_, err := fmt.Fprintf(w, "%s\r\n", s)
	if err != nil {
		return err
	}
	return nil
}

//base64Wrap encodeds the attachment content, and wraps it according to RFC 2045 standards (every 76 chars)
//The output is then written to the specified io.Writer
func base64Wrap(w io.Writer, b []byte) {
	encoded := base64.StdEncoding.EncodeToString(b)
	for i := 0; i < len(encoded); i += 76 {
		//Do we need to print 76 characters, or the rest of the string?
		if len(encoded)-i < 76 {
			fmt.Fprintf(w, "%s\r\n", encoded[i:])
		} else {
			fmt.Fprintf(w, "%s\r\n", encoded[i:i+76])
		}
	}
}

//headerToBytes enumerates the key and values in the header, and writes the results to the IO Writer
func headerToBytes(w io.Writer, t textproto.MIMEHeader) error {
	for k, v := range t {
		//Write the header key
		_, err := fmt.Fprintf(w, "%s: ", k)
		if err != nil {
			return err
		}
		//Write each value in the header
		for _, c := range v {
			_, err := fmt.Fprintf(w, "%s\r\n", c)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

//Attachment is a struct representing an email attachment.
//Based on the mime/multipart.FileHeader struct, Attachment contains the name, MIMEHeader, and content of the attachment in question
type Attachment struct {
	Filename string
	Header   textproto.MIMEHeader
	Content  []byte
}
<commit_msg>Working on writeMime function (may make this a full encoder later)<commit_after>//Package email is designed to provide an "email interface for humans."
//Designed to be robust and flexible, the email package aims to make sending email easy without getting in the way.
package email

import (
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/mail"
	"net/smtp"
	"net/textproto"
	"os"
	"path/filepath"
	"strings"
)

//Email is the type used for email messages
type Email struct {
	From        string
	To          []string
	Bcc         []string
	Cc          []string
	Subject     string
	Text        string //Plaintext message (optional)
	Html        string //Html message (optional)
	Headers     textproto.MIMEHeader
	Attachments map[string]*Attachment
	ReadReceipt []string
}

//NewEmail creates an Email, and returns the pointer to it.
func NewEmail() *Email {
	return &Email{Attachments: make(map[string]*Attachment), Headers: textproto.MIMEHeader{}}
}

//Attach is used to attach a file to the email.
//It attempts to open the file referenced by filename and, if successful, creates an Attachment.
//This Attachment is then appended to the slice of Email.Attachments.
//The function will then return the Attachment for reference, as well as nil for the error, if successful.
func (e *Email) Attach(filename string) (a *Attachment, err error) {
	//Check if the file exists, return any error
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		return nil, err
	}
	//Read the file, and set the appropriate headers
	buffer, _ := ioutil.ReadFile(filename)
	e.Attachments[filename] = &Attachment{
		Filename: filename,
		Header:   textproto.MIMEHeader{},
		Content:  buffer}
	at := e.Attachments[filename]
	//Get the Content-Type to be used in the MIMEHeader
	ct := mime.TypeByExtension(filepath.Ext(filename))
	if ct != "" {
		at.Header.Set("Content-Type", ct)
	} else {
		//If the Content-Type is blank, set the Content-Type to "application/octet-stream"
		at.Header.Set("Content-Type", "application/octet-stream")
	}
	at.Header.Set("Content-Disposition", fmt.Sprintf("attachment;\r\n filename=\"%s\"", filename))
	at.Header.Set("Content-Transfer-Encoding", "base64")
	return e.Attachments[filename], nil
}

//Bytes converts the Email object to a []byte representation, including all needed MIMEHeaders, boundaries, etc.
func (e *Email) Bytes() ([]byte, error) {
	buff := &bytes.Buffer{}
	w := multipart.NewWriter(buff)
	//Set the appropriate headers (overwriting any conflicts)
	//Leave out Bcc (only included in envelope headers)
	//TODO: Support wrapping on 76 characters (ref: MIME RFC)
	e.Headers.Set("To", strings.Join(e.To, ","))
	if e.Cc != nil {
		e.Headers.Set("Cc", strings.Join(e.Cc, ","))
	}
	e.Headers.Set("From", e.From)
	e.Headers.Set("Subject", e.Subject)
	if len(e.ReadReceipt) != 0 {
		e.Headers.Set("Disposition-Notification-To", strings.Join(e.ReadReceipt, ","))
	}
	e.Headers.Set("MIME-Version", "1.0")
	e.Headers.Set("Content-Type", fmt.Sprintf("multipart/mixed;\r\n boundary=%s\r\n", w.Boundary()))

	//Write the envelope headers (including any custom headers)
	if err := headerToBytes(buff, e.Headers); err != nil {
	}
	//Start the multipart/mixed part
	fmt.Fprintf(buff, "--%s\r\n", w.Boundary())
	header := textproto.MIMEHeader{}
	//Check to see if there is a Text or HTML field
	if e.Text != "" || e.Html != "" {
		subWriter := multipart.NewWriter(buff)
		//Create the multipart alternative part
		header.Set("Content-Type", fmt.Sprintf("multipart/alternative;\r\n boundary=%s\r\n", subWriter.Boundary()))
		//Write the header
		if err := headerToBytes(buff, header); err != nil {

		}
		//Create the body sections
		if e.Text != "" {
			header.Set("Content-Type", fmt.Sprintf("text/plain; charset=UTF-8"))
			part, err := subWriter.CreatePart(header)
			if err != nil {

			}
			// Write the text
			if err := writeMIME(part, e.Text); err != nil {

			}
		}
		if e.Html != "" {
			header.Set("Content-Type", fmt.Sprintf("text/html; charset=UTF-8"))
			subWriter.CreatePart(header)
			// Write the text
			if err := writeMIME(buff, e.Html); err != nil {

			}
		}
		subWriter.Close()
	}
	//Create attachment part, if necessary
	if e.Attachments != nil {
		for _, a := range e.Attachments {
			ap, err := w.CreatePart(a.Header)
			if err != nil {
				return nil, err
			}
			//Write the base64Wrapped content to the part
			base64Wrap(ap, a.Content)
		}
	}
	w.Close()
	return buff.Bytes(), nil
}

//Send an email using the given host and SMTP auth (optional), returns any error thrown by smtp.SendMail
//This function merges the To, Cc, and Bcc fields and calls the smtp.SendMail function using the Email.Bytes() output as the message
func (e *Email) Send(addr string, a smtp.Auth) error {
	//Check to make sure there is at least one recipient and one "From" address
	if e.From == "" || (len(e.To) == 0 && len(e.Cc) == 0 && len(e.Bcc) == 0) {
		return errors.New("Must specify at least one From address and one To address")
	}
	// Merge the To, Cc, and Bcc fields
	to := append(append(e.To, e.Cc...), e.Bcc...)
	from, err := mail.ParseAddress(e.From)
	if err != nil {
		return err
	}
	raw, err := e.Bytes()
	if err != nil {
		return err
	}
	return smtp.SendMail(addr, a, from.Address, to, raw)
}

//writeMIME writes the quoted-printable text to the IO Writer
func writeMIME(w io.Writer, s string) error {
	// Basic rules (comments to be removed once this function is fully implemented)
	// * If character is printable, it can be represented AS IS
	// * Lines must have a max of 76 characters
	// * Lines must not end with whitespace
	//		- Rather, append a soft break (=) to the end of the line after the space for preservation
	// *
	_, err := fmt.Fprintf(w, "%s\r\n", s)
	for _, c := range s {
		// Check if we can just print the character
		if (c >= '!' && c < '=') || (c >= '>' && c < '~') {
			return nil
		}
	}
	if err != nil {
		return err
	}
	return nil
}

//base64Wrap encodeds the attachment content, and wraps it according to RFC 2045 standards (every 76 chars)
//The output is then written to the specified io.Writer
func base64Wrap(w io.Writer, b []byte) {
	encoded := base64.StdEncoding.EncodeToString(b)
	for i := 0; i < len(encoded); i += 76 {
		//Do we need to print 76 characters, or the rest of the string?
		if len(encoded)-i < 76 {
			fmt.Fprintf(w, "%s\r\n", encoded[i:])
		} else {
			fmt.Fprintf(w, "%s\r\n", encoded[i:i+76])
		}
	}
}

//headerToBytes enumerates the key and values in the header, and writes the results to the IO Writer
func headerToBytes(w io.Writer, t textproto.MIMEHeader) error {
	for k, v := range t {
		//Write the header key
		_, err := fmt.Fprintf(w, "%s: ", k)
		if err != nil {
			return err
		}
		//Write each value in the header
		for _, c := range v {
			_, err := fmt.Fprintf(w, "%s\r\n", c)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

//Attachment is a struct representing an email attachment.
//Based on the mime/multipart.FileHeader struct, Attachment contains the name, MIMEHeader, and content of the attachment in question
type Attachment struct {
	Filename string
	Header   textproto.MIMEHeader
	Content  []byte
}
<|endoftext|>


<commit_before>package emogo

import (
	"errors"
)

// #include <emokit/emokit.h>
// #include <stdint.h>
// #cgo LDFLAGS: -lemokit
import "C"

// These are defined in emokit.h and reproduced here as cgo isn't
// linking them in for some reason.
const (
	EMOKIT_VID int = 0x21a1
	EMOKIT_PID int = 0x0001
)

// EmokitContext represents a connection to an EPOC device.
type EmokitContext struct {
	eeg *C.struct_emokit_device
}

func NewEmokitContext() (*EmokitContext, error) {
	e := new(EmokitContext)
	e.eeg = C.emokit_create()
	ret := C.emokit_open(e.eeg, C.int(EMOKIT_VID), C.int(EMOKIT_PID), 0)
	if ret != 0 {
		return nil, errors.New("Cannot access device.")
	}
	return e, nil
}
<commit_msg>Read EPOC frames<commit_after>package emogo

import (
	"errors"
	"unsafe"
	"syscall"
)

// #include <emokit/emokit.h>
// #include <stdint.h>
// #cgo LDFLAGS: -lemokit
import "C"

// These are defined in emokit.h and reproduced here as cgo isn't
// linking them in for some reason.
const (
	EMOKIT_VID int = 0x21a1
	EMOKIT_PID int = 0x0001
	EmokitPacketSize = 32
)

// EmokitContext represents a connection to an EPOC device.
type EmokitContext struct {
	eeg *C.struct_emokit_device
}

func NewEmokitContext() (*EmokitContext, error) {
	e := new(EmokitContext)
	e.eeg = C.emokit_create()
	ret := C.emokit_open(e.eeg, C.int(EMOKIT_VID), C.int(EMOKIT_PID), 0)
	if ret != 0 {
		return nil, errors.New("Cannot access device.")
	}
	return e, nil
}

type EmokitFrame struct {
	raw []byte
	rendered C.struct_emokit_frame
}

func NewEmokitFrame() *EmokitFrame {
	f := new(EmokitFrame)
	f.raw = make([]byte, EmokitPacketSize)
	return f
}

// readData reads data from the EPOC dongle and returns the number of
// bytes read.
func (e *EmokitContext) readData() int {
	n := C.emokit_read_data(e.eeg)
	return int(n)
}

func (e *EmokitContext) getNextFrame() (*EmokitFrame, error) {
	f := NewEmokitFrame()
	f.rendered = C.emokit_get_next_frame(e.eeg)
	if f.rendered.counter == 0 {
		return nil, errors.New("Could not read raw packet.")
	}
	C.emokit_get_raw_frame(e.eeg, (*C.uchar)(unsafe.Pointer(&f.raw[0])))
	return f, nil
}

// GetFrame returns the next available EPOC frame. If there is no frame
// to be read, the error value will be EAGAIN.
func (e *EmokitContext) GetFrame() (*EmokitFrame, error) {
	if e.readData() > 0 {
		f, err := e.getNextFrame()
		if err != nil {
			return nil, err
		}
		return f, nil
	}
	return nil, syscall.EAGAIN
}
<|endoftext|>


<commit_before>package api

import (
	"github.com/go-macaron/binding"
	"github.com/raintank/gziper"
	"github.com/raintank/metrictank/api/middleware"
	"github.com/raintank/metrictank/api/models"
	"gopkg.in/macaron.v1"
)

func (s *Server) RegisterRoutes() {
	r := s.Macaron
	if useGzip {
		r.Use(gziper.Gziper())
	}
	r.Use(middleware.RequestStats())
	r.Use(macaron.Renderer())
	r.Use(middleware.OrgMiddleware(multiTenant))
	r.Use(middleware.CorsHandler())

	bind := binding.Bind
	withOrg := middleware.RequireOrg()
	cBody := middleware.CaptureBody

	r.Get("/", s.appStatus)
	r.Get("/node", s.getNodeStatus)
	r.Post("/node", bind(models.NodeStatus{}), s.setNodeStatus)

	r.Get("/cluster", s.getClusterStatus)

	r.Combo("/getdata", bind(models.GetData{})).Get(s.getData).Post(s.getData)
	// equivalent to /render but used by graphite-metrictank so we can keep the stats separate
	// note, you should not request anything from this endpoint that may trigger a loop
	// (where MT proxies the req to graphite, graphite hits /get again, and so on).
	// just don't request any function processing, or maybe just some of the stable functions
	r.Combo("/get", withOrg, bind(models.GraphiteRender{})).Get(s.renderMetrics).Post(s.renderMetrics)

	r.Combo("/index/find", bind(models.IndexFind{})).Get(s.indexFind).Post(s.indexFind)
	r.Combo("/index/list", bind(models.IndexList{})).Get(s.indexList).Post(s.indexList)
	r.Combo("/index/delete", bind(models.IndexDelete{})).Get(s.indexDelete).Post(s.indexDelete)
	r.Combo("/index/get", bind(models.IndexGet{})).Get(s.indexGet).Post(s.indexGet)

	r.Options("/*", func(ctx *macaron.Context) {
		ctx.Write(nil)
	})

	// Graphite endpoints
	r.Combo("/render", cBody, withOrg, bind(models.GraphiteRender{})).Get(s.renderMetrics).Post(s.renderMetrics)
	r.Combo("/metrics/find", withOrg, bind(models.GraphiteFind{})).Get(s.metricsFind).Post(s.metricsFind)
	r.Get("/metrics/index.json", withOrg, s.metricsIndex)
	r.Post("/metrics/delete", withOrg, bind(models.MetricsDelete{}), s.metricsDelete)

}
<commit_msg>remove route which was only used by deprecated graphite-api finder<commit_after>package api

import (
	"github.com/go-macaron/binding"
	"github.com/raintank/gziper"
	"github.com/raintank/metrictank/api/middleware"
	"github.com/raintank/metrictank/api/models"
	"gopkg.in/macaron.v1"
)

func (s *Server) RegisterRoutes() {
	r := s.Macaron
	if useGzip {
		r.Use(gziper.Gziper())
	}
	r.Use(middleware.RequestStats())
	r.Use(macaron.Renderer())
	r.Use(middleware.OrgMiddleware(multiTenant))
	r.Use(middleware.CorsHandler())

	bind := binding.Bind
	withOrg := middleware.RequireOrg()
	cBody := middleware.CaptureBody

	r.Get("/", s.appStatus)
	r.Get("/node", s.getNodeStatus)
	r.Post("/node", bind(models.NodeStatus{}), s.setNodeStatus)

	r.Get("/cluster", s.getClusterStatus)

	r.Combo("/getdata", bind(models.GetData{})).Get(s.getData).Post(s.getData)

	r.Combo("/index/find", bind(models.IndexFind{})).Get(s.indexFind).Post(s.indexFind)
	r.Combo("/index/list", bind(models.IndexList{})).Get(s.indexList).Post(s.indexList)
	r.Combo("/index/delete", bind(models.IndexDelete{})).Get(s.indexDelete).Post(s.indexDelete)
	r.Combo("/index/get", bind(models.IndexGet{})).Get(s.indexGet).Post(s.indexGet)

	r.Options("/*", func(ctx *macaron.Context) {
		ctx.Write(nil)
	})

	// Graphite endpoints
	r.Combo("/render", cBody, withOrg, bind(models.GraphiteRender{})).Get(s.renderMetrics).Post(s.renderMetrics)
	r.Combo("/metrics/find", withOrg, bind(models.GraphiteFind{})).Get(s.metricsFind).Post(s.metricsFind)
	r.Get("/metrics/index.json", withOrg, s.metricsIndex)
	r.Post("/metrics/delete", withOrg, bind(models.MetricsDelete{}), s.metricsDelete)

}
<|endoftext|>


<commit_before>//
// Copyright 2021, Sander van Harmelen
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"net/http"
	"time"
)

// EpicsService handles communication with the epic related methods
// of the GitLab API.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html
type EpicsService struct {
	client *Client
}

// EpicAuthor represents a author of the epic.
type EpicAuthor struct {
	ID        int    `json:"id"`
	State     string `json:"state"`
	WebURL    string `json:"web_url"`
	Name      string `json:"name"`
	AvatarURL string `json:"avatar_url"`
	Username  string `json:"username"`
}

// Epic represents a GitLab epic.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html
type Epic struct {
	ID                      int         `json:"id"`
	IID                     int         `json:"iid"`
	GroupID                 int         `json:"group_id"`
	ParentID                int         `json:"parent_id"`
	Title                   string      `json:"title"`
	Description             string      `json:"description"`
	State                   string      `json:"state"`
	WebURL                  string      `json:"web_url"`
	Author                  *EpicAuthor `json:"author"`
	StartDate               *ISOTime    `json:"start_date"`
	StartDateIsFixed        bool        `json:"start_date_is_fixed"`
	StartDateFixed          *ISOTime    `json:"start_date_fixed"`
	StartDateFromMilestones *ISOTime    `json:"start_date_from_milestones"`
	DueDate                 *ISOTime    `json:"due_date"`
	DueDateIsFixed          bool        `json:"due_date_is_fixed"`
	DueDateFixed            *ISOTime    `json:"due_date_fixed"`
	DueDateFromMilestones   *ISOTime    `json:"due_date_from_milestones"`
	CreatedAt               *time.Time  `json:"created_at"`
	UpdatedAt               *time.Time  `json:"updated_at"`
	Labels                  []string    `json:"labels"`
	Upvotes                 int         `json:"upvotes"`
	Downvotes               int         `json:"downvotes"`
	UserNotesCount          int         `json:"user_notes_count"`
	URL                     string      `json:"url"`
}

func (e Epic) String() string {
	return Stringify(e)
}

// ListGroupEpicsOptions represents the available ListGroupEpics() options.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#list-epics-for-a-group
type ListGroupEpicsOptions struct {
	ListOptions
	AuthorID                *int       `url:"author_id,omitempty" json:"author_id,omitempty"`
	Labels                  *Labels    `url:"labels,comma,omitempty" json:"labels,omitempty"`
	WithLabelDetails        *bool      `url:"with_labels_details,omitempty" json:"with_labels_details,omitempty"`
	OrderBy                 *string    `url:"order_by,omitempty" json:"order_by,omitempty"`
	Sort                    *string    `url:"sort,omitempty" json:"sort,omitempty"`
	Search                  *string    `url:"search,omitempty" json:"search,omitempty"`
	State                   *string    `url:"state,omitempty" json:"state,omitempty"`
	CreatedAfter            *time.Time `url:"created_after,omitempty" json:"created_after,omitempty"`
	CreatedBefore           *time.Time `url:"created_before,omitempty" json:"created_before,omitempty"`
	UpdatedAfter            *time.Time `url:"updated_after,omitempty" json:"updated_after,omitempty"`
	UpdatedBefore           *time.Time `url:"updated_before,omitempty" json:"updated_before,omitempty"`
	IncludeAncestorGroups   *bool      `url:"include_ancestor_groups,omitempty" json:"include_ancestor_groups,omitempty"`
	IncludeDescendantGroups *bool      `url:"include_descendant_groups,omitempty" json:"include_descendant_groups,omitempty"`
	MyReactionEmoji         *string    `url:"my_reaction_emoji,omitempty" json:"my_reaction_emoji,omitempty"`
}

// ListGroupEpics gets a list of group epics. This function accepts pagination
// parameters page and per_page to return the list of group epics.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#list-epics-for-a-group
func (s *EpicsService) ListGroupEpics(gid interface{}, opt *ListGroupEpicsOptions, options ...RequestOptionFunc) ([]*Epic, *Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/epics", PathEscape(group))

	req, err := s.client.NewRequest(http.MethodGet, u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	var es []*Epic
	resp, err := s.client.Do(req, &es)
	if err != nil {
		return nil, resp, err
	}

	return es, resp, err
}

// GetEpic gets a single group epic.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#single-epic
func (s *EpicsService) GetEpic(gid interface{}, epic int, options ...RequestOptionFunc) (*Epic, *Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/epics/%d", PathEscape(group), epic)

	req, err := s.client.NewRequest(http.MethodGet, u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	e := new(Epic)
	resp, err := s.client.Do(req, e)
	if err != nil {
		return nil, resp, err
	}

	return e, resp, err
}

// GetEpicLinks gets all child epics of an epic.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epic_links.html
func (s *EpicsService) GetEpicLinks(gid interface{}, epic int, options ...RequestOptionFunc) ([]*Epic, *Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/epics/%d/epics", PathEscape(group), epic)

	req, err := s.client.NewRequest(http.MethodGet, u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	var e []*Epic
	resp, err := s.client.Do(req, &e)
	if err != nil {
		return nil, resp, err
	}

	return e, resp, err
}

// CreateEpicOptions represents the available CreateEpic() options.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#new-epic
type CreateEpicOptions struct {
	Title            *string  `url:"title,omitempty" json:"title,omitempty"`
	Description      *string  `url:"description,omitempty" json:"description,omitempty"`
	Labels           *Labels  `url:"labels,comma,omitempty" json:"labels,omitempty"`
	StartDateIsFixed *bool    `url:"start_date_is_fixed,omitempty" json:"start_date_is_fixed,omitempty"`
	StartDateFixed   *ISOTime `url:"start_date_fixed,omitempty" json:"start_date_fixed,omitempty"`
	DueDateIsFixed   *bool    `url:"due_date_is_fixed,omitempty" json:"due_date_is_fixed,omitempty"`
	DueDateFixed     *ISOTime `url:"due_date_fixed,omitempty" json:"due_date_fixed,omitempty"`
}

// CreateEpic creates a new group epic.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#new-epic
func (s *EpicsService) CreateEpic(gid interface{}, opt *CreateEpicOptions, options ...RequestOptionFunc) (*Epic, *Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/epics", PathEscape(group))

	req, err := s.client.NewRequest(http.MethodPost, u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	e := new(Epic)
	resp, err := s.client.Do(req, e)
	if err != nil {
		return nil, resp, err
	}

	return e, resp, err
}

// UpdateEpicOptions represents the available UpdateEpic() options.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#update-epic
type UpdateEpicOptions struct {
	Title            *string  `url:"title,omitempty" json:"title,omitempty"`
	Description      *string  `url:"description,omitempty" json:"description,omitempty"`
	Labels           *Labels  `url:"labels,comma,omitempty" json:"labels,omitempty"`
	StartDateIsFixed *bool    `url:"start_date_is_fixed,omitempty" json:"start_date_is_fixed,omitempty"`
	StartDateFixed   *ISOTime `url:"start_date_fixed,omitempty" json:"start_date_fixed,omitempty"`
	DueDateIsFixed   *bool    `url:"due_date_is_fixed,omitempty" json:"due_date_is_fixed,omitempty"`
	DueDateFixed     *ISOTime `url:"due_date_fixed,omitempty" json:"due_date_fixed,omitempty"`
	StateEvent       *string  `url:"state_event,omitempty" json:"state_event,omitempty"`
}

// UpdateEpic updates an existing group epic. This function is also used
// to mark an epic as closed.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#update-epic
func (s *EpicsService) UpdateEpic(gid interface{}, epic int, opt *UpdateEpicOptions, options ...RequestOptionFunc) (*Epic, *Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/epics/%d", PathEscape(group), epic)

	req, err := s.client.NewRequest(http.MethodPut, u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	e := new(Epic)
	resp, err := s.client.Do(req, e)
	if err != nil {
		return nil, resp, err
	}

	return e, resp, err
}

// DeleteEpic deletes a single group epic.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#delete-epic
func (s *EpicsService) DeleteEpic(gid interface{}, epic int, options ...RequestOptionFunc) (*Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("groups/%s/epics/%d", PathEscape(group), epic)

	req, err := s.client.NewRequest(http.MethodDelete, u, nil, options)
	if err != nil {
		return nil, err
	}

	return s.client.Do(req, nil)
}
<commit_msg>Fetch ClosedAt field for epics<commit_after>//
// Copyright 2021, Sander van Harmelen
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"net/http"
	"time"
)

// EpicsService handles communication with the epic related methods
// of the GitLab API.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html
type EpicsService struct {
	client *Client
}

// EpicAuthor represents a author of the epic.
type EpicAuthor struct {
	ID        int    `json:"id"`
	State     string `json:"state"`
	WebURL    string `json:"web_url"`
	Name      string `json:"name"`
	AvatarURL string `json:"avatar_url"`
	Username  string `json:"username"`
}

// Epic represents a GitLab epic.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html
type Epic struct {
	ID                      int         `json:"id"`
	IID                     int         `json:"iid"`
	GroupID                 int         `json:"group_id"`
	ParentID                int         `json:"parent_id"`
	Title                   string      `json:"title"`
	Description             string      `json:"description"`
	State                   string      `json:"state"`
	WebURL                  string      `json:"web_url"`
	Author                  *EpicAuthor `json:"author"`
	StartDate               *ISOTime    `json:"start_date"`
	StartDateIsFixed        bool        `json:"start_date_is_fixed"`
	StartDateFixed          *ISOTime    `json:"start_date_fixed"`
	StartDateFromMilestones *ISOTime    `json:"start_date_from_milestones"`
	DueDate                 *ISOTime    `json:"due_date"`
	DueDateIsFixed          bool        `json:"due_date_is_fixed"`
	DueDateFixed            *ISOTime    `json:"due_date_fixed"`
	DueDateFromMilestones   *ISOTime    `json:"due_date_from_milestones"`
	CreatedAt               *time.Time  `json:"created_at"`
	UpdatedAt               *time.Time  `json:"updated_at"`
	ClosedAt                *time.Time  `json:"closed_at"`
	Labels                  []string    `json:"labels"`
	Upvotes                 int         `json:"upvotes"`
	Downvotes               int         `json:"downvotes"`
	UserNotesCount          int         `json:"user_notes_count"`
	URL                     string      `json:"url"`
}

func (e Epic) String() string {
	return Stringify(e)
}

// ListGroupEpicsOptions represents the available ListGroupEpics() options.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#list-epics-for-a-group
type ListGroupEpicsOptions struct {
	ListOptions
	AuthorID                *int       `url:"author_id,omitempty" json:"author_id,omitempty"`
	Labels                  *Labels    `url:"labels,comma,omitempty" json:"labels,omitempty"`
	WithLabelDetails        *bool      `url:"with_labels_details,omitempty" json:"with_labels_details,omitempty"`
	OrderBy                 *string    `url:"order_by,omitempty" json:"order_by,omitempty"`
	Sort                    *string    `url:"sort,omitempty" json:"sort,omitempty"`
	Search                  *string    `url:"search,omitempty" json:"search,omitempty"`
	State                   *string    `url:"state,omitempty" json:"state,omitempty"`
	CreatedAfter            *time.Time `url:"created_after,omitempty" json:"created_after,omitempty"`
	CreatedBefore           *time.Time `url:"created_before,omitempty" json:"created_before,omitempty"`
	UpdatedAfter            *time.Time `url:"updated_after,omitempty" json:"updated_after,omitempty"`
	UpdatedBefore           *time.Time `url:"updated_before,omitempty" json:"updated_before,omitempty"`
	IncludeAncestorGroups   *bool      `url:"include_ancestor_groups,omitempty" json:"include_ancestor_groups,omitempty"`
	IncludeDescendantGroups *bool      `url:"include_descendant_groups,omitempty" json:"include_descendant_groups,omitempty"`
	MyReactionEmoji         *string    `url:"my_reaction_emoji,omitempty" json:"my_reaction_emoji,omitempty"`
}

// ListGroupEpics gets a list of group epics. This function accepts pagination
// parameters page and per_page to return the list of group epics.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#list-epics-for-a-group
func (s *EpicsService) ListGroupEpics(gid interface{}, opt *ListGroupEpicsOptions, options ...RequestOptionFunc) ([]*Epic, *Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/epics", PathEscape(group))

	req, err := s.client.NewRequest(http.MethodGet, u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	var es []*Epic
	resp, err := s.client.Do(req, &es)
	if err != nil {
		return nil, resp, err
	}

	return es, resp, err
}

// GetEpic gets a single group epic.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#single-epic
func (s *EpicsService) GetEpic(gid interface{}, epic int, options ...RequestOptionFunc) (*Epic, *Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/epics/%d", PathEscape(group), epic)

	req, err := s.client.NewRequest(http.MethodGet, u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	e := new(Epic)
	resp, err := s.client.Do(req, e)
	if err != nil {
		return nil, resp, err
	}

	return e, resp, err
}

// GetEpicLinks gets all child epics of an epic.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epic_links.html
func (s *EpicsService) GetEpicLinks(gid interface{}, epic int, options ...RequestOptionFunc) ([]*Epic, *Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/epics/%d/epics", PathEscape(group), epic)

	req, err := s.client.NewRequest(http.MethodGet, u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	var e []*Epic
	resp, err := s.client.Do(req, &e)
	if err != nil {
		return nil, resp, err
	}

	return e, resp, err
}

// CreateEpicOptions represents the available CreateEpic() options.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#new-epic
type CreateEpicOptions struct {
	Title            *string  `url:"title,omitempty" json:"title,omitempty"`
	Description      *string  `url:"description,omitempty" json:"description,omitempty"`
	Labels           *Labels  `url:"labels,comma,omitempty" json:"labels,omitempty"`
	StartDateIsFixed *bool    `url:"start_date_is_fixed,omitempty" json:"start_date_is_fixed,omitempty"`
	StartDateFixed   *ISOTime `url:"start_date_fixed,omitempty" json:"start_date_fixed,omitempty"`
	DueDateIsFixed   *bool    `url:"due_date_is_fixed,omitempty" json:"due_date_is_fixed,omitempty"`
	DueDateFixed     *ISOTime `url:"due_date_fixed,omitempty" json:"due_date_fixed,omitempty"`
}

// CreateEpic creates a new group epic.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#new-epic
func (s *EpicsService) CreateEpic(gid interface{}, opt *CreateEpicOptions, options ...RequestOptionFunc) (*Epic, *Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/epics", PathEscape(group))

	req, err := s.client.NewRequest(http.MethodPost, u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	e := new(Epic)
	resp, err := s.client.Do(req, e)
	if err != nil {
		return nil, resp, err
	}

	return e, resp, err
}

// UpdateEpicOptions represents the available UpdateEpic() options.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#update-epic
type UpdateEpicOptions struct {
	Title            *string  `url:"title,omitempty" json:"title,omitempty"`
	Description      *string  `url:"description,omitempty" json:"description,omitempty"`
	Labels           *Labels  `url:"labels,comma,omitempty" json:"labels,omitempty"`
	StartDateIsFixed *bool    `url:"start_date_is_fixed,omitempty" json:"start_date_is_fixed,omitempty"`
	StartDateFixed   *ISOTime `url:"start_date_fixed,omitempty" json:"start_date_fixed,omitempty"`
	DueDateIsFixed   *bool    `url:"due_date_is_fixed,omitempty" json:"due_date_is_fixed,omitempty"`
	DueDateFixed     *ISOTime `url:"due_date_fixed,omitempty" json:"due_date_fixed,omitempty"`
	StateEvent       *string  `url:"state_event,omitempty" json:"state_event,omitempty"`
}

// UpdateEpic updates an existing group epic. This function is also used
// to mark an epic as closed.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#update-epic
func (s *EpicsService) UpdateEpic(gid interface{}, epic int, opt *UpdateEpicOptions, options ...RequestOptionFunc) (*Epic, *Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("groups/%s/epics/%d", PathEscape(group), epic)

	req, err := s.client.NewRequest(http.MethodPut, u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	e := new(Epic)
	resp, err := s.client.Do(req, e)
	if err != nil {
		return nil, resp, err
	}

	return e, resp, err
}

// DeleteEpic deletes a single group epic.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/epics.html#delete-epic
func (s *EpicsService) DeleteEpic(gid interface{}, epic int, options ...RequestOptionFunc) (*Response, error) {
	group, err := parseID(gid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("groups/%s/epics/%d", PathEscape(group), epic)

	req, err := s.client.NewRequest(http.MethodDelete, u, nil, options)
	if err != nil {
		return nil, err
	}

	return s.client.Do(req, nil)
}
<|endoftext|>


<commit_before>package api

import (
	"io"
	"time"

	"github.com/hashicorp/vault/helper/jsonutil"
)

// Secret is the structure returned for every secret within Vault.
type Secret struct {
	LeaseID       string `json:"lease_id"`
	LeaseDuration int    `json:"lease_duration"`
	Renewable     bool   `json:"renewable"`

	// Data is the actual contents of the secret. The format of the data
	// is arbitrary and up to the secret backend.
	Data map[string]interface{} `json:"data"`

	// Warnings contains any warnings related to the operation. These
	// are not issues that caused the command to fail, but that the
	// client should be aware of.
	Warnings []string `json:"warnings"`

	// Auth, if non-nil, means that there was authentication information
	// attached to this response.
	Auth *SecretAuth `json:"auth,omitempty"`

	// WrapInfo, if non-nil, means that the initial response was wrapped in the
	// cubbyhole of the given token (which has a TTL of the given number of
	// seconds)
	WrapInfo *SecretWrapInfo `json:"wrap_info,omitempty"`
}

// SecretWrapInfo contains wrapping information if we have it. If what is
// contained is an authentication token, the accessor for the token will be
// available in WrappedAccessor.
type SecretWrapInfo struct {
	Token           string    `json:"token"`
	TTL             int       `json:"ttl"`
	CreationTime    time.Time `json:"creation_time"`
	WrappedAccessor string    `json:"wrapped_accessor"`
}

// SecretAuth is the structure containing auth information if we have it.
type SecretAuth struct {
	ClientToken string            `json:"client_token"`
	Accessor    string            `json:"accessor"`
	Policies    []string          `json:"policies"`
	Metadata    map[string]string `json:"metadata"`

	LeaseDuration int  `json:"lease_duration"`
	Renewable     bool `json:"renewable"`
}

// ParseSecret is used to parse a secret value from JSON from an io.Reader.
func ParseSecret(r io.Reader) (*Secret, error) {
	// First decode the JSON into a map[string]interface{}
	var secret Secret
	if err := jsonutil.DecodeJSONFromReader(r, &secret); err != nil {
		return nil, err
	}

	return &secret, nil
}
<commit_msg>Plumb request UUID through the API<commit_after>package api

import (
	"io"
	"time"

	"github.com/hashicorp/vault/helper/jsonutil"
)

// Secret is the structure returned for every secret within Vault.
type Secret struct {
	// The request ID that generated this response
	RequestID string `json:"request_id"`

	LeaseID       string `json:"lease_id"`
	LeaseDuration int    `json:"lease_duration"`
	Renewable     bool   `json:"renewable"`

	// Data is the actual contents of the secret. The format of the data
	// is arbitrary and up to the secret backend.
	Data map[string]interface{} `json:"data"`

	// Warnings contains any warnings related to the operation. These
	// are not issues that caused the command to fail, but that the
	// client should be aware of.
	Warnings []string `json:"warnings"`

	// Auth, if non-nil, means that there was authentication information
	// attached to this response.
	Auth *SecretAuth `json:"auth,omitempty"`

	// WrapInfo, if non-nil, means that the initial response was wrapped in the
	// cubbyhole of the given token (which has a TTL of the given number of
	// seconds)
	WrapInfo *SecretWrapInfo `json:"wrap_info,omitempty"`
}

// SecretWrapInfo contains wrapping information if we have it. If what is
// contained is an authentication token, the accessor for the token will be
// available in WrappedAccessor.
type SecretWrapInfo struct {
	Token           string    `json:"token"`
	TTL             int       `json:"ttl"`
	CreationTime    time.Time `json:"creation_time"`
	WrappedAccessor string    `json:"wrapped_accessor"`
}

// SecretAuth is the structure containing auth information if we have it.
type SecretAuth struct {
	ClientToken string            `json:"client_token"`
	Accessor    string            `json:"accessor"`
	Policies    []string          `json:"policies"`
	Metadata    map[string]string `json:"metadata"`

	LeaseDuration int  `json:"lease_duration"`
	Renewable     bool `json:"renewable"`
}

// ParseSecret is used to parse a secret value from JSON from an io.Reader.
func ParseSecret(r io.Reader) (*Secret, error) {
	// First decode the JSON into a map[string]interface{}
	var secret Secret
	if err := jsonutil.DecodeJSONFromReader(r, &secret); err != nil {
		return nil, err
	}

	return &secret, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 The gocql Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gocql

import (
	"io"
	"net"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

type TestServer struct {
	Address  string
	t        *testing.T
	nreq     uint64
	listen   net.Listener
	nKillReq uint64
}

func TestSimple(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	db, err := NewCluster(srv.Address).CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}

	if err := db.Query("void").Exec(); err != nil {
		t.Error(err)
	}
}

func TestClosed(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	session, err := NewCluster(srv.Address).CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}
	session.Close()

	if err := session.Query("void").Exec(); err != ErrSessionClosed {
		t.Errorf("expected %#v, got %#v", ErrSessionClosed, err)
	}
}

func TestTimeout(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	db, err := NewCluster(srv.Address).CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}

	go func() {
		<-time.After(2 * time.Second)
		t.Fatal("no timeout")
	}()

	if err := db.Query("kill").Exec(); err == nil {
		t.Fatal("expected error")
	}
}

// TestQueryRetry will test to make sure that gocql will execute
// the exact amount of retry queries designated by the user.
func TestQueryRetry(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	db, err := NewCluster(srv.Address).CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}

	go func() {
		<-time.After(5 * time.Second)
		t.Fatal("no timeout")
	}()
	rt := RetryPolicy{NumRetries: 1}

	if err := db.Query("kill").RetryPolicy(rt).Exec(); err == nil {
		t.Fatal("expected error")
	}
	//Minus 1 from the nKillReq variable since there is the initial query attempt
	if srv.nKillReq-1 != uint64(rt.NumRetries) {
		t.Fatalf("failed to retry the query %v time(s). Query executed %v times", rt.NumRetries, srv.nKillReq-1)
	}
}

func TestSlowQuery(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	db, err := NewCluster(srv.Address).CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}

	if err := db.Query("slow").Exec(); err != nil {
		t.Fatal(err)
	}
}

func TestRoundRobin(t *testing.T) {
	servers := make([]*TestServer, 5)
	addrs := make([]string, len(servers))
	for i := 0; i < len(servers); i++ {
		servers[i] = NewTestServer(t)
		addrs[i] = servers[i].Address
		defer servers[i].Stop()
	}
	cluster := NewCluster(addrs...)
	db, err := cluster.CreateSession()
	time.Sleep(1 * time.Second) //Sleep to allow the Cluster.fillPool to complete

	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}

	var wg sync.WaitGroup
	wg.Add(5)
	for i := 0; i < 5; i++ {
		go func() {
			for j := 0; j < 5; j++ {
				if err := db.Query("void").Exec(); err != nil {
					t.Fatal(err)
				}
			}
			wg.Done()
		}()
	}
	wg.Wait()

	diff := 0
	for i := 1; i < len(servers); i++ {
		d := 0
		if servers[i].nreq > servers[i-1].nreq {
			d = int(servers[i].nreq - servers[i-1].nreq)
		} else {
			d = int(servers[i-1].nreq - servers[i].nreq)
		}
		if d > diff {
			diff = d
		}
	}

	if diff > 0 {
		t.Fatal("diff:", diff)
	}
}

func TestConnClosing(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	db, err := NewCluster(srv.Address).CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}
	defer db.Close()

	numConns := db.cfg.NumConns
	count := db.cfg.NumStreams * numConns

	wg := &sync.WaitGroup{}
	wg.Add(count)
	for i := 0; i < count; i++ {
		go func(wg *sync.WaitGroup) {
			wg.Done()
			db.Query("kill").Exec()
		}(wg)
	}

	wg.Wait()

	time.Sleep(1 * time.Second) //Sleep so the fillPool can complete.
	pool := db.Pool.(ConnectionPool)
	conns := pool.Size()

	if conns != numConns {
		t.Fatalf("Expected to have %d connections but have %d", numConns, conns)
	}
}

func TestConnectHander(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	called := false

	cluster_conf := NewCluster(srv.Address)
	cluster_conf.ConnectHandler = func(p ConnectionPool) {
		called = true
		if p.Size() <= 0 {
			t.Errorf("No new connection detected in pool: %v", p)
		}
	}

	db, err := cluster_conf.CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}

	// Run some query to make sure we connect
	// before checking that handler fired off
	if err := db.Query("void").Exec(); err != nil {
		t.Error(err)
	}

	if !called {
		t.Errorf("Connect handler not called")
	}
}

func TestDisconnectHander(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	called := false

	cluster_conf := NewCluster(srv.Address)
	cluster_conf.DisconnectHandler = func(p ConnectionPool) {
		called = true
	}

	db, err := cluster_conf.CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}

	// Run some query to make sure we connect
	if err := db.Query("void").Exec(); err != nil {
		t.Error(err)
	}

	// Run close to make sure we connect
	db.Close()

	if err := db.Query("void").Exec(); err != ErrSessionClosed {
		t.Errorf("expected %#v, got %#v", ErrSessionClosed, err)
	}

	if !called {
		t.Errorf("Disconnect handler not called")
	}
}

func NewTestServer(t *testing.T) *TestServer {
	laddr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	listen, err := net.ListenTCP("tcp", laddr)
	if err != nil {
		t.Fatal(err)
	}
	srv := &TestServer{Address: listen.Addr().String(), listen: listen, t: t}
	go srv.serve()
	return srv
}

func (srv *TestServer) serve() {
	defer srv.listen.Close()
	for {
		conn, err := srv.listen.Accept()
		if err != nil {
			break
		}
		go func(conn net.Conn) {
			defer conn.Close()
			for {
				frame := srv.readFrame(conn)
				atomic.AddUint64(&srv.nreq, 1)
				srv.process(frame, conn)
			}
		}(conn)
	}
}

func (srv *TestServer) Stop() {
	srv.listen.Close()
}

func (srv *TestServer) process(frame frame, conn net.Conn) {
	switch frame[3] {
	case opStartup:
		frame = frame[:headerSize]
		frame.setHeader(protoResponse, 0, frame[2], opReady)
	case opQuery:
		input := frame
		input.skipHeader()
		query := strings.TrimSpace(input.readLongString())
		frame = frame[:headerSize]
		frame.setHeader(protoResponse, 0, frame[2], opResult)
		first := query
		if n := strings.Index(query, " "); n > 0 {
			first = first[:n]
		}
		switch strings.ToLower(first) {
		case "kill":
			atomic.AddUint64(&srv.nKillReq, 1)
			select {}
		case "slow":
			go func() {
				<-time.After(1 * time.Second)
				frame.writeInt(resultKindVoid)
				frame.setLength(len(frame) - headerSize)
				if _, err := conn.Write(frame); err != nil {
					return
				}
			}()
			return
		case "use":
			frame.writeInt(3)
			frame.writeString(strings.TrimSpace(query[3:]))
		case "void":
			frame.writeInt(resultKindVoid)
		default:
			frame.writeInt(resultKindVoid)
		}
	default:
		frame = frame[:headerSize]
		frame.setHeader(protoResponse, 0, frame[2], opError)
		frame.writeInt(0)
		frame.writeString("not supported")
	}
	frame.setLength(len(frame) - headerSize)
	if _, err := conn.Write(frame); err != nil {
		return
	}
}

func (srv *TestServer) readFrame(conn net.Conn) frame {
	frame := make(frame, headerSize, headerSize+512)
	if _, err := io.ReadFull(conn, frame); err != nil {
		srv.t.Fatal(err)
	}
	if n := frame.Length(); n > 0 {
		frame.grow(n)
		if _, err := io.ReadFull(conn, frame[headerSize:]); err != nil {
			srv.t.Fatal(err)
		}
	}
	return frame
}
<commit_msg>wait on mutex before checking if disconnect handler is called<commit_after>// Copyright (c) 2012 The gocql Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gocql

import (
	"io"
	"net"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

type TestServer struct {
	Address  string
	t        *testing.T
	nreq     uint64
	listen   net.Listener
	nKillReq uint64
}

func TestSimple(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	db, err := NewCluster(srv.Address).CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}

	if err := db.Query("void").Exec(); err != nil {
		t.Error(err)
	}
}

func TestClosed(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	session, err := NewCluster(srv.Address).CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}
	session.Close()

	if err := session.Query("void").Exec(); err != ErrSessionClosed {
		t.Errorf("expected %#v, got %#v", ErrSessionClosed, err)
	}
}

func TestTimeout(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	db, err := NewCluster(srv.Address).CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}

	go func() {
		<-time.After(2 * time.Second)
		t.Fatal("no timeout")
	}()

	if err := db.Query("kill").Exec(); err == nil {
		t.Fatal("expected error")
	}
}

// TestQueryRetry will test to make sure that gocql will execute
// the exact amount of retry queries designated by the user.
func TestQueryRetry(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	db, err := NewCluster(srv.Address).CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}

	go func() {
		<-time.After(5 * time.Second)
		t.Fatal("no timeout")
	}()
	rt := RetryPolicy{NumRetries: 1}

	if err := db.Query("kill").RetryPolicy(rt).Exec(); err == nil {
		t.Fatal("expected error")
	}
	//Minus 1 from the nKillReq variable since there is the initial query attempt
	if srv.nKillReq-1 != uint64(rt.NumRetries) {
		t.Fatalf("failed to retry the query %v time(s). Query executed %v times", rt.NumRetries, srv.nKillReq-1)
	}
}

func TestSlowQuery(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	db, err := NewCluster(srv.Address).CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}

	if err := db.Query("slow").Exec(); err != nil {
		t.Fatal(err)
	}
}

func TestRoundRobin(t *testing.T) {
	servers := make([]*TestServer, 5)
	addrs := make([]string, len(servers))
	for i := 0; i < len(servers); i++ {
		servers[i] = NewTestServer(t)
		addrs[i] = servers[i].Address
		defer servers[i].Stop()
	}
	cluster := NewCluster(addrs...)
	db, err := cluster.CreateSession()
	time.Sleep(1 * time.Second) //Sleep to allow the Cluster.fillPool to complete

	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}

	var wg sync.WaitGroup
	wg.Add(5)
	for i := 0; i < 5; i++ {
		go func() {
			for j := 0; j < 5; j++ {
				if err := db.Query("void").Exec(); err != nil {
					t.Fatal(err)
				}
			}
			wg.Done()
		}()
	}
	wg.Wait()

	diff := 0
	for i := 1; i < len(servers); i++ {
		d := 0
		if servers[i].nreq > servers[i-1].nreq {
			d = int(servers[i].nreq - servers[i-1].nreq)
		} else {
			d = int(servers[i-1].nreq - servers[i].nreq)
		}
		if d > diff {
			diff = d
		}
	}

	if diff > 0 {
		t.Fatal("diff:", diff)
	}
}

func TestConnClosing(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	db, err := NewCluster(srv.Address).CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}
	defer db.Close()

	numConns := db.cfg.NumConns
	count := db.cfg.NumStreams * numConns

	wg := &sync.WaitGroup{}
	wg.Add(count)
	for i := 0; i < count; i++ {
		go func(wg *sync.WaitGroup) {
			wg.Done()
			db.Query("kill").Exec()
		}(wg)
	}

	wg.Wait()

	time.Sleep(1 * time.Second) //Sleep so the fillPool can complete.
	pool := db.Pool.(ConnectionPool)
	conns := pool.Size()

	if conns != numConns {
		t.Fatalf("Expected to have %d connections but have %d", numConns, conns)
	}
}

func TestConnectHander(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	called := false

	cluster_conf := NewCluster(srv.Address)
	cluster_conf.ConnectHandler = func(p ConnectionPool) {
		called = true
		if p.Size() <= 0 {
			t.Errorf("No new connection detected in pool: %v", p)
		}
	}

	db, err := cluster_conf.CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}

	// Run some query to make sure we connect
	// before checking that handler fired off
	if err := db.Query("void").Exec(); err != nil {
		t.Error(err)
	}

	if !called {
		t.Errorf("Connect handler not called")
	}
}

func TestDisconnectHander(t *testing.T) {
	srv := NewTestServer(t)
	defer srv.Stop()

	called := false

	cluster_conf := NewCluster(srv.Address)
	cluster_conf.DisconnectHandler = func(p ConnectionPool) {
		called = true
	}

	db, err := cluster_conf.CreateSession()
	if err != nil {
		t.Errorf("NewCluster: %v", err)
	}

	// Run some query to make sure we connect
	if err := db.Query("void").Exec(); err != nil {
		t.Error(err)
	}

	// Run close to make sure we connect
	db.Close()

	// Calling Closed() should wait on the mutex to ensure we're done
	// closing before checking called
	if db.Closed() && !called {
		t.Errorf("Disconnect handler not called")
	}
}

func NewTestServer(t *testing.T) *TestServer {
	laddr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	listen, err := net.ListenTCP("tcp", laddr)
	if err != nil {
		t.Fatal(err)
	}
	srv := &TestServer{Address: listen.Addr().String(), listen: listen, t: t}
	go srv.serve()
	return srv
}

func (srv *TestServer) serve() {
	defer srv.listen.Close()
	for {
		conn, err := srv.listen.Accept()
		if err != nil {
			break
		}
		go func(conn net.Conn) {
			defer conn.Close()
			for {
				frame := srv.readFrame(conn)
				atomic.AddUint64(&srv.nreq, 1)
				srv.process(frame, conn)
			}
		}(conn)
	}
}

func (srv *TestServer) Stop() {
	srv.listen.Close()
}

func (srv *TestServer) process(frame frame, conn net.Conn) {
	switch frame[3] {
	case opStartup:
		frame = frame[:headerSize]
		frame.setHeader(protoResponse, 0, frame[2], opReady)
	case opQuery:
		input := frame
		input.skipHeader()
		query := strings.TrimSpace(input.readLongString())
		frame = frame[:headerSize]
		frame.setHeader(protoResponse, 0, frame[2], opResult)
		first := query
		if n := strings.Index(query, " "); n > 0 {
			first = first[:n]
		}
		switch strings.ToLower(first) {
		case "kill":
			atomic.AddUint64(&srv.nKillReq, 1)
			select {}
		case "slow":
			go func() {
				<-time.After(1 * time.Second)
				frame.writeInt(resultKindVoid)
				frame.setLength(len(frame) - headerSize)
				if _, err := conn.Write(frame); err != nil {
					return
				}
			}()
			return
		case "use":
			frame.writeInt(3)
			frame.writeString(strings.TrimSpace(query[3:]))
		case "void":
			frame.writeInt(resultKindVoid)
		default:
			frame.writeInt(resultKindVoid)
		}
	default:
		frame = frame[:headerSize]
		frame.setHeader(protoResponse, 0, frame[2], opError)
		frame.writeInt(0)
		frame.writeString("not supported")
	}
	frame.setLength(len(frame) - headerSize)
	if _, err := conn.Write(frame); err != nil {
		return
	}
}

func (srv *TestServer) readFrame(conn net.Conn) frame {
	frame := make(frame, headerSize, headerSize+512)
	if _, err := io.ReadFull(conn, frame); err != nil {
		srv.t.Fatal(err)
	}
	if n := frame.Length(); n > 0 {
		frame.grow(n)
		if _, err := io.ReadFull(conn, frame[headerSize:]); err != nil {
			srv.t.Fatal(err)
		}
	}
	return frame
}
<|endoftext|>


<commit_before>package uploads

import (
	"crypto/md5"
	"os"

	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/db/schema"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/materials-commons/mcstore/server/mcstored/service/uploads/processor"
)

// A finisher finishes the rest of the book keeping
// when a file has been successfully uploaded and
// reconstructed.
type finisher struct {
	files dai.Files
	dirs  dai.Dirs
}

// newFinisher creates a new finisher.
func newFinisher(files dai.Files, dirs dai.Dirs) *finisher {
	return &finisher{
		files: files,
		dirs:  dirs,
	}
}

// finish takes care of updating the file and directory pointers, determining
// if a matching file (by checksum) has already been uploaded, and making the
// file ready for the user to access.
// TODO: refactor this method into a few separate methods that contain the
// logical blocks.
func (f *finisher) finish(req *UploadRequest, fileID string, upload *schema.Upload) error {
	filePath := app.MCDir.FilePath(fileID)
	checksum, err := file.HashStr(md5.New(), filePath)
	if err != nil {
		app.Log.Errorf("Failed creating checksum for '%s': %s", filePath, err)
		return err
	}

	parentID, err := f.parentID(upload.File.Name, upload.DirectoryID)
	if err != nil {
		app.Log.Errorf("Looking up parentID for %s/%s returned unexpected error: %s.", upload.File.Name, upload.DirectoryID, err)
		return err
	}

	size := f.size(fileID)

	if size != req.FlowTotalSize {
		app.Log.Errorf("Uploaded file (%s/%s) doesn't match the expected size. Expected:%d, Got: %d", upload.File.Name, req.FlowIdentifier, req.FlowTotalSize, size)
		return app.ErrInvalid
	}

	mediatype := files.MediaType(upload.File.Name, filePath)
	fields := map[string]interface{}{
		schema.FileFields.Current():   true,
		schema.FileFields.Parent():    parentID,
		schema.FileFields.Uploaded():  req.FlowTotalSize,
		schema.FileFields.Size():      req.FlowTotalSize,
		schema.FileFields.Checksum():  checksum,
		schema.FileFields.MediaType(): mediatype,
	}

	matchingFile, err := f.files.ByChecksum(checksum)
	switch {
	case err != nil && err == app.ErrNotFound:
		// This is a brand new upload for a file we haven't seen before. There are processing
		// steps that may need to be done on the file. For example we convert tif and bmp
		// image files so they can be displayed in the browser.
		f.processFile(fileID, mediatype)
	case err != nil:
		// Some type of error accessing the database
		app.Log.Errorf("Looking up file by checksum for %s/%s returned unexpected error: %s.", checksum, req.FlowFileName, err)
		return err
	default:
		// Found a matching checksum. There are two cases
		// 1. The existing file is the file we uploaded.
		// 2. The existing file is not the file we uploaded

		// Case 1: Is matching file. Delete it completely.
		if f.fileInDir(checksum, upload.File.Name, upload.DirectoryID) {
			app.Log.Infof("Found exact matching file (%s/%s) in same directory (%s), deleting", upload.File.Name, fileID, upload.DirectoryID)
			f.deleteUploadedFile(fileID, upload)
			return nil
		}

		// Case 2: Existing file is not the file we uploaded:
		// File found is not the file we uploaded, so we need to do a couple of things:
		// First set file entry to point this file (set UsesID) and secondly
		// remove the file (on the file system) that was just uploaded since we don't
		// keep duplicates.

		fields[schema.FileFields.UsesID()] = matchingFile.ID
		os.Remove(filePath)
	}

	return f.files.UpdateFields(fileID, fields)
}

// Size gets the size of the reconstructed file.
func (f *finisher) size(fileID string) int64 {
	finfo, err := os.Stat(app.MCDir.FilePath(fileID))
	if err != nil {
		return 0
	}
	return finfo.Size()
}

// parentID returns the parent for this file (if any).
func (f *finisher) parentID(fileName, dirID string) (parentID string, err error) {
	parent, err := f.files.ByPath(fileName, dirID)

	if parent != nil {
		parentID = parent.ID
	}

	if err != nil && err == app.ErrNotFound {
		// log error
		return parentID, nil
	}

	return parentID, err
}

// processFile will process the file on disk.
func (f *finisher) processFile(fileID string, mediatype schema.MediaType) {
	fp := processor.New(fileID, mediatype)
	fp.Process()
}

// fileInDir determines if this exact file has already been uploaded
// to this directory.
func (f *finisher) fileInDir(checksum, fileName, dirID string) bool {
	files, err := f.dirs.Files(dirID)
	if err != nil {
		return false
	}

	for _, fileEntry := range files {
		if fileEntry.Name == fileName && fileEntry.Checksum == checksum {
			return true
		}
	}
	return false
}

// deleteUploadedFile will completely delete the file from the system. This
// happens when the file has already been uploaded to the given directory.
func (f *finisher) deleteUploadedFile(fileID string, upload *schema.Upload) {
	f.files.Delete(fileID, upload.DirectoryID, upload.ProjectID)
	os.Remove(app.MCDir.FilePath(fileID))
}
<commit_msg>Clear up a comment by defining what completely means.<commit_after>package uploads

import (
	"crypto/md5"
	"os"

	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/db/schema"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/materials-commons/mcstore/server/mcstored/service/uploads/processor"
)

// A finisher finishes the rest of the book keeping
// when a file has been successfully uploaded and
// reconstructed.
type finisher struct {
	files dai.Files
	dirs  dai.Dirs
}

// newFinisher creates a new finisher.
func newFinisher(files dai.Files, dirs dai.Dirs) *finisher {
	return &finisher{
		files: files,
		dirs:  dirs,
	}
}

// finish takes care of updating the file and directory pointers, determining
// if a matching file (by checksum) has already been uploaded, and making the
// file ready for the user to access.
// TODO: refactor this method into a few separate methods that contain the
// logical blocks.
func (f *finisher) finish(req *UploadRequest, fileID string, upload *schema.Upload) error {
	filePath := app.MCDir.FilePath(fileID)
	checksum, err := file.HashStr(md5.New(), filePath)
	if err != nil {
		app.Log.Errorf("Failed creating checksum for '%s': %s", filePath, err)
		return err
	}

	parentID, err := f.parentID(upload.File.Name, upload.DirectoryID)
	if err != nil {
		app.Log.Errorf("Looking up parentID for %s/%s returned unexpected error: %s.", upload.File.Name, upload.DirectoryID, err)
		return err
	}

	size := f.size(fileID)

	if size != req.FlowTotalSize {
		app.Log.Errorf("Uploaded file (%s/%s) doesn't match the expected size. Expected:%d, Got: %d", upload.File.Name, req.FlowIdentifier, req.FlowTotalSize, size)
		return app.ErrInvalid
	}

	mediatype := files.MediaType(upload.File.Name, filePath)
	fields := map[string]interface{}{
		schema.FileFields.Current():   true,
		schema.FileFields.Parent():    parentID,
		schema.FileFields.Uploaded():  req.FlowTotalSize,
		schema.FileFields.Size():      req.FlowTotalSize,
		schema.FileFields.Checksum():  checksum,
		schema.FileFields.MediaType(): mediatype,
	}

	matchingFile, err := f.files.ByChecksum(checksum)
	switch {
	case err != nil && err == app.ErrNotFound:
		// This is a brand new upload for a file we haven't seen before. There are processing
		// steps that may need to be done on the file. For example we convert tif and bmp
		// image files so they can be displayed in the browser.
		f.processFile(fileID, mediatype)
	case err != nil:
		// Some type of error accessing the database
		app.Log.Errorf("Looking up file by checksum for %s/%s returned unexpected error: %s.", checksum, req.FlowFileName, err)
		return err
	default:
		// Found a matching checksum. There are two cases
		// 1. The existing file is the file we uploaded.
		// 2. The existing file is not the file we uploaded

		// Case 1: Is matching file. Delete it completely.
		if f.fileInDir(checksum, upload.File.Name, upload.DirectoryID) {
			app.Log.Infof("Found exact matching file (%s/%s) in same directory (%s), deleting", upload.File.Name, fileID, upload.DirectoryID)
			f.deleteUploadedFile(fileID, upload)
			return nil
		}

		// Case 2: Existing file is not the file we uploaded:
		// File found is not the file we uploaded, so we need to do a couple of things:
		// First set file entry to point this file (set UsesID) and secondly
		// remove the file (on the file system) that was just uploaded since we don't
		// keep duplicates.

		fields[schema.FileFields.UsesID()] = matchingFile.ID
		os.Remove(filePath)
	}

	return f.files.UpdateFields(fileID, fields)
}

// Size gets the size of the reconstructed file.
func (f *finisher) size(fileID string) int64 {
	finfo, err := os.Stat(app.MCDir.FilePath(fileID))
	if err != nil {
		return 0
	}
	return finfo.Size()
}

// parentID returns the parent for this file (if any).
func (f *finisher) parentID(fileName, dirID string) (parentID string, err error) {
	parent, err := f.files.ByPath(fileName, dirID)

	if parent != nil {
		parentID = parent.ID
	}

	if err != nil && err == app.ErrNotFound {
		// log error
		return parentID, nil
	}

	return parentID, err
}

// processFile will process the file on disk.
func (f *finisher) processFile(fileID string, mediatype schema.MediaType) {
	fp := processor.New(fileID, mediatype)
	fp.Process()
}

// fileInDir determines if this exact file has already been uploaded
// to this directory.
func (f *finisher) fileInDir(checksum, fileName, dirID string) bool {
	files, err := f.dirs.Files(dirID)
	if err != nil {
		return false
	}

	for _, fileEntry := range files {
		if fileEntry.Name == fileName && fileEntry.Checksum == checksum {
			return true
		}
	}
	return false
}

// deleteUploadedFile will completely delete the file from the system. Completely
// means it also deletes the database entries for the file.
func (f *finisher) deleteUploadedFile(fileID string, upload *schema.Upload) {
	f.files.Delete(fileID, upload.DirectoryID, upload.ProjectID)
	os.Remove(app.MCDir.FilePath(fileID))
}
<|endoftext|>