<commit_before>//Package models provides models used in application.
package presentation

import (
	"appengine"
	"appengine/blobstore"
	"appengine/datastore"
	"fmt"
	"math"
	"models/action"
	"time"
)

//Presentation stores data about uploaded presentations in Datastore.
type Presentation struct {
	BlobKey     appengine.BlobKey //Key linking the metadata with file stored in Blobstore
	Created     time.Time         //Time of upload
	Name        string            //Name used to identify presentation in administration
	Description []byte            //Textual description used in UI
	FileType    string            //Name that clients should store the presentation under
	Active      bool              //If true, presentation is distributed to clients. Only one presentation can be active.

	//Used for storage of Datastore key when passing the struct around. Doesn't get saved to DS.
	//The key is stored as an encoded string
	Key string `datastore:"-"`
}

func (p Presentation) GetKey(ctx appengine.Context) (k *datastore.Key, err error) {
	k, err = datastore.DecodeKey(p.Key)
	return
}

//GetActive gets the active presentation from the Datastore.
//If more than one presentation has the Active field set to true,
//its results are unpredictable.
func GetActive(c appengine.Context) (*Presentation, error) {
	var p Presentation
	i := datastore.NewQuery("Presentation").Filter("Active =", true).Limit(1).Run(c)
	key, err := i.Next(&p)
	if err != nil {
		return &p, err
	}
	p.Key = key.Encode()
	return &p, err
}

//New returns pointer to a presentation with fields set to given values.
func New(k appengine.BlobKey, fileType, name string, desc []byte, active bool) *Presentation {
	p := new(Presentation)
	p.BlobKey = k
	p.Created = time.Now()
	p.FileType = fileType
	p.Name = name
	p.Description = desc
	p.Active = active
	return p
}

//Make creates a new presentation with New, saves it to Datastore and returns a pointer to it.
func Make(k appengine.BlobKey, fileType, name string, desc []byte, active bool, c appengine.Context) (p *Presentation, err error) {
	p = New(k, fileType, name, desc, active)
	err = p.Save(c)
	return
}

//Sve saves presentation to Datastore. If its Active field is true,
//the currently active presentation is deactivated.
//TODO: Use transactions to deactivate and activate presentations.
func (p *Presentation) Save(c appengine.Context) (err error) {
	var k *datastore.Key
	if p.Key == "" {

		if p.Active {
			err = deactivateCurrent(c)
			if err != nil {
				return
			}
		}
		k, err = datastore.Put(c, datastore.NewIncompleteKey(c, "Presentation", nil), p)
		if err != nil {
			return
		}

		p.Key = k.Encode()
	} else {
		k, err = datastore.DecodeKey(p.Key)
		if err != nil {
			return
		}

		if p.Active {
			err = deactivateCurrent(c)
			if err != nil {
				return
			}
			action.DeleteFor(p, c)
		}
		_, err = datastore.Put(c, k, p)
		if err != nil {
			return
		}

	}
	return
}

//Delete deletes the presentation record from Datastore and
//its data file from Blobstore.
func (p *Presentation) Delete(c appengine.Context) (err error) {
	if p.Active {
		err = fmt.Errorf("Active presentation can't be deleted.")
		return
	}
	k, err := datastore.DecodeKey(p.Key)
	if err != nil {
		return
	}

	err = datastore.Delete(c, k)
	if err != nil {
		return
	}
	p.Key = ""

	err = blobstore.Delete(c, p.BlobKey)

	return
}

//GetAll returns a slice of all the presentations in Datastore.
func GetAll(c appengine.Context) (ps []*Presentation, err error) {
	ps = make([]*Presentation, 0)
	keys, err := datastore.NewQuery("Presentation").Order("-Created").GetAll(c, &ps)
	for i := 0; i < len(ps); i++ {
		ps[i].Key = keys[i].Encode()
	}
	return
}

func GetListing(page int, perPage int, c appengine.Context) (ps []*Presentation, err error) {
	var q *datastore.Query
	if page == 1 {
		q = datastore.NewQuery("Presentation").Limit(perPage).Order("-Created")
	} else {
		q = datastore.NewQuery("Presentation").Limit(perPage).Offset(perPage * (page - 1)).Order("-Created")
	}

	keys, err := q.GetAll(c, &ps)
	if err != nil {
		return
	}
	for i, p := range ps {
		p.Key = keys[i].Encode()
	}
	return
}

func PageCount(perPage int, c appengine.Context) (pgs int, err error) {
	ps, err := datastore.NewQuery("Presentation").Count(c)
	if err != nil {
		return
	}
	pgs = int(math.Ceil(float64(ps) / float64(perPage)))
	return
}

//GetByKey gets the presentation with given key from Datastore and returns a pointer to it.
func GetByKey(key string, c appengine.Context) (p *Presentation, err error) {
	k, err := datastore.DecodeKey(key)
	if err != nil {
		return
	}
	p = new(Presentation)
	err = datastore.Get(c, k, p)
	if err != nil {
		return
	}
	p.Key = key
	return
}

//deactivateCurrent is a helper function that deactivates current active presentation.
//It should only be used just before saving another active presentation.
func deactivateCurrent(c appengine.Context) (err error) {
	curr, err := GetActive(c)
	if err != nil {
		err = fmt.Errorf("Couldn't get active presentation: %v", err)
		return
	}
	curr.Active = false
	err = curr.Save(c)
	if err != nil {
		err = fmt.Errorf("Couldn't deactivate active presentation: %v", err)
		return
	}
	return
}
<commit_msg>Allow activation of more than one presentation<commit_after>//Package models provides models used in application.
package presentation

import (
	"appengine"
	"appengine/blobstore"
	"appengine/datastore"
	"fmt"
	"math"
	"models/action"
	"time"
)

//Presentation stores data about uploaded presentations in Datastore.
type Presentation struct {
	BlobKey     appengine.BlobKey //Key linking the metadata with file stored in Blobstore
	Created     time.Time         //Time of upload
	Name        string            //Name used to identify presentation in administration
	Description []byte            //Textual description used in UI
	FileType    string            //Name that clients should store the presentation under
	Active      bool              //If true, presentation is distributed to clients. Only one presentation can be active.

	//Used for storage of Datastore key when passing the struct around. Doesn't get saved to DS.
	//The key is stored as an encoded string
	Key string `datastore:"-"`
}

func (p Presentation) GetKey(ctx appengine.Context) (k *datastore.Key, err error) {
	k, err = datastore.DecodeKey(p.Key)
	return
}

//GetActive gets the active presentation from the Datastore.
//If more than one presentation has the Active field set to true,
//its results are unpredictable.
func GetActive(c appengine.Context) (*Presentation, error) {
	var p Presentation
	i := datastore.NewQuery("Presentation").Filter("Active =", true).Limit(1).Run(c)
	key, err := i.Next(&p)
	if err != nil {
		return &p, err
	}
	p.Key = key.Encode()
	return &p, err
}

//New returns pointer to a presentation with fields set to given values.
func New(k appengine.BlobKey, fileType, name string, desc []byte, active bool) *Presentation {
	p := new(Presentation)
	p.BlobKey = k
	p.Created = time.Now()
	p.FileType = fileType
	p.Name = name
	p.Description = desc
	p.Active = active
	return p
}

//Make creates a new presentation with New, saves it to Datastore and returns a pointer to it.
func Make(k appengine.BlobKey, fileType, name string, desc []byte, active bool, c appengine.Context) (p *Presentation, err error) {
	p = New(k, fileType, name, desc, active)
	err = p.Save(c)
	return
}

//Sve saves presentation to Datastore. If its Active field is true,
//the currently active presentation is deactivated.
//TODO: Use transactions to deactivate and activate presentations.
func (p *Presentation) Save(c appengine.Context) (err error) {
	var k *datastore.Key
	if p.Key == "" {

		k, err = datastore.Put(c, datastore.NewIncompleteKey(c, "Presentation", nil), p)
		if err != nil {
			return
		}

		p.Key = k.Encode()
	} else {
		k, err = datastore.DecodeKey(p.Key)
		if err != nil {
			return
		}

		if p.Active {
			action.DeleteFor(p, c)
		}
		_, err = datastore.Put(c, k, p)
		if err != nil {
			return
		}

	}
	return
}

//Delete deletes the presentation record from Datastore and
//its data file from Blobstore.
func (p *Presentation) Delete(c appengine.Context) (err error) {
	if p.Active {
		err = fmt.Errorf("Active presentation can't be deleted.")
		return
	}
	k, err := datastore.DecodeKey(p.Key)
	if err != nil {
		return
	}

	err = datastore.Delete(c, k)
	if err != nil {
		return
	}
	p.Key = ""

	err = blobstore.Delete(c, p.BlobKey)

	return
}

//GetAll returns a slice of all the presentations in Datastore.
func GetAll(c appengine.Context) (ps []*Presentation, err error) {
	ps = make([]*Presentation, 0)
	keys, err := datastore.NewQuery("Presentation").Order("-Created").GetAll(c, &ps)
	for i := 0; i < len(ps); i++ {
		ps[i].Key = keys[i].Encode()
	}
	return
}

func GetListing(page int, perPage int, c appengine.Context) (ps []*Presentation, err error) {
	var q *datastore.Query
	if page == 1 {
		q = datastore.NewQuery("Presentation").Limit(perPage).Order("-Created")
	} else {
		q = datastore.NewQuery("Presentation").Limit(perPage).Offset(perPage * (page - 1)).Order("-Created")
	}

	keys, err := q.GetAll(c, &ps)
	if err != nil {
		return
	}
	for i, p := range ps {
		p.Key = keys[i].Encode()
	}
	return
}

func PageCount(perPage int, c appengine.Context) (pgs int, err error) {
	ps, err := datastore.NewQuery("Presentation").Count(c)
	if err != nil {
		return
	}
	pgs = int(math.Ceil(float64(ps) / float64(perPage)))
	return
}

//GetByKey gets the presentation with given key from Datastore and returns a pointer to it.
func GetByKey(key string, c appengine.Context) (p *Presentation, err error) {
	k, err := datastore.DecodeKey(key)
	if err != nil {
		return
	}
	p = new(Presentation)
	err = datastore.Get(c, k, p)
	if err != nil {
		return
	}
	p.Key = key
	return
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package provides basic interfaces to I/O primitives.
// Its primary job is to wrap existing implementations of such primitives,
// such as those in package os, into shared public interfaces that
// abstract the functionality.
// It also provides buffering primitives and some other basic operations.
package io

import (
	"bytes";
	"os";
	"strings";
)

// Error represents an unexpected I/O behavior.
type Error struct {
	os.ErrorString
}

// ErrShortWrite means that a write accepted fewer bytes than requested
// but failed to return an explicit error.
var ErrShortWrite os.Error = &Error{"short write"}

// ErrUnexpectedEOF means that os.EOF was encountered in the
// middle of reading a fixed-size block or data structure.
var ErrUnexpectedEOF os.Error = &Error{"unexpected EOF"}

// Reader is the interface that wraps the basic Read method.
//
// Read reads up to len(p) bytes into p.  It returns the number of bytes
// read (0 <= n <= len(p)) and any error encountered.
// Even if Read returns n < len(p),
// it may use all of p as scratch space during the call.
// If some data is available but not len(p) bytes, Read conventionally
// returns what is available rather than block waiting for more.
//
// At the end of the input stream, Read returns 0, os.EOF.
// Read may return a non-zero number of bytes with a non-nil err.
// In particular, a Read that exhausts the input may return n > 0, os.EOF.
type Reader interface {
	Read(p []byte) (n int, err os.Error);
}

// Writer is the interface that wraps the basic Write method.
//
// Write writes len(p) bytes from p to the underlying data stream.
// It returns the number of bytes written from p (0 <= n <= len(p))
// and any error encountered that caused the write to stop early.
// Write must return a non-nil error if it returns n < len(p).
type Writer interface {
	Write(p []byte) (n int, err os.Error);
}

// Closer is the interface that wraps the basic Close method.
type Closer interface {
	Close() os.Error;
}

// Seeker is the interface that wraps the basic Seek method.
//
// Seek sets the offset for the next Read or Write to offset,
// interpreted according to whence: 0 means relative to the origin of
// the file, 1 means relative to the current offset, and 2 means
// relative to the end.  Seek returns the new offset and an Error, if
// any.
type Seeker interface {
	Seek(offset int64, whence int) (ret int64, err os.Error);
}

// ReadWrite is the interface that groups the basic Read and Write methods.
type ReadWriter interface {
	Reader;
	Writer;
}

// ReadCloser is the interface that groups the basic Read and Close methods.
type ReadCloser interface {
	Reader;
	Closer;
}

// WriteCloser is the interface that groups the basic Write and Close methods.
type WriteCloser interface {
	Writer;
	Closer;
}

// ReadWriteCloser is the interface that groups the basic Read, Write and Close methods.
type ReadWriteCloser interface {
	Reader;
	Writer;
	Closer;
}

// ReadSeeker is the interface that groups the basic Read and Seek methods.
type ReadSeeker interface {
	Reader;
	Seeker;
}

// WriteSeeker is the interface that groups the basic Write and Seek methods.
type WriteSeeker interface {
	Writer;
	Seeker;
}

// ReadWriteSeeker is the interface that groups the basic Read, Write and Seek methods.
type ReadWriteSeeker interface {
	Reader;
	Writer;
	Seeker;
}

// WriteString writes the contents of the string s to w, which accepts an array of bytes.
func WriteString(w Writer, s string) (n int, err os.Error) {
	return w.Write(strings.Bytes(s))
}

// ReadAtLeast reads from r into buf until it has read at least min bytes.
// It returns the number of bytes copied and an error if fewer bytes were read.
// The error is os.EOF only if no bytes were read.
// If an EOF happens after reading fewer than min bytes,
// ReadAtLeast returns ErrUnexpectedEOF.
func ReadAtLeast(r Reader, buf []byte, min int) (n int, err os.Error) {
	n = 0;
	for n < min {
		nn, e := r.Read(buf[n:len(buf)]);
		if nn > 0 {
			n += nn
		}
		if e != nil {
			if e == os.EOF && n > 0 {
				e = ErrUnexpectedEOF;
			}
			return n, e
		}
	}
	return n, nil
}

// ReadFull reads exactly len(buf) bytes from r into buf.
// It returns the number of bytes copied and an error if fewer bytes were read.
// The error is os.EOF only if no bytes were read.
// If an EOF happens after reading some but not all the bytes,
// ReadFull returns ErrUnexpectedEOF.
func ReadFull(r Reader, buf []byte) (n int, err os.Error) {
	return ReadAtLeast(r, buf, len(buf));
}

// Copyn copies n bytes (or until an error) from src to dst.
// It returns the number of bytes copied and the error, if any.
func Copyn(src Reader, dst Writer, n int64) (written int64, err os.Error) {
	buf := make([]byte, 32*1024);
	for written < n {
		l := len(buf);
		if d := n - written; d < int64(l) {
			l = int(d);
		}
		nr, er := src.Read(buf[0 : l]);
		if nr > 0 {
			nw, ew := dst.Write(buf[0 : nr]);
			if nw > 0 {
				written += int64(nw);
			}
			if ew != nil {
				err = ew;
				break;
			}
			if nr != nw {
				err = ErrShortWrite;
				break;
			}
		}
		if er != nil {
			err = er;
			break;
		}
	}
	return written, err
}

// Copy copies from src to dst until either EOF is reached
// on src or an error occurs.  It returns the number of bytes
// copied and the error, if any.
func Copy(src Reader, dst Writer) (written int64, err os.Error) {
	buf := make([]byte, 32*1024);
	for {
		nr, er := src.Read(buf);
		if nr > 0 {
			nw, ew := dst.Write(buf[0:nr]);
			if nw > 0 {
				written += int64(nw);
			}
			if ew != nil {
				err = ew;
				break;
			}
			if nr != nw {
				err = ErrShortWrite;
				break;
			}
		}
		if er == os.EOF {
			break;
		}
		if er != nil {
			err = er;
			break;
		}
	}
	return written, err
}

// LimitReader returns a Reader that reads from r
// but stops with os.EOF after n bytes.
func LimitReader(r Reader, n int64) Reader {
	return &limitedReader{r, n};
}

type limitedReader struct {
	r Reader;
	n int64;
}

func (l *limitedReader) Read(p []byte) (n int, err os.Error) {
	if l.n <= 0 {
		return 0, os.EOF;
	}
	if int64(len(p)) > l.n {
		p = p[0:l.n];
	}
	n, err = l.r.Read(p);
	l.n -= int64(n);
	return;
}
<commit_msg>add SectionReader, ReaderAt.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package provides basic interfaces to I/O primitives.
// Its primary job is to wrap existing implementations of such primitives,
// such as those in package os, into shared public interfaces that
// abstract the functionality.
// It also provides buffering primitives and some other basic operations.
package io

import (
	"bytes";
	"os";
	"strings";
)

// Error represents an unexpected I/O behavior.
type Error struct {
	os.ErrorString
}

// ErrShortWrite means that a write accepted fewer bytes than requested
// but failed to return an explicit error.
var ErrShortWrite os.Error = &Error{"short write"}

// ErrUnexpectedEOF means that os.EOF was encountered in the
// middle of reading a fixed-size block or data structure.
var ErrUnexpectedEOF os.Error = &Error{"unexpected EOF"}

// Reader is the interface that wraps the basic Read method.
//
// Read reads up to len(p) bytes into p.  It returns the number of bytes
// read (0 <= n <= len(p)) and any error encountered.
// Even if Read returns n < len(p),
// it may use all of p as scratch space during the call.
// If some data is available but not len(p) bytes, Read conventionally
// returns what is available rather than block waiting for more.
//
// At the end of the input stream, Read returns 0, os.EOF.
// Read may return a non-zero number of bytes with a non-nil err.
// In particular, a Read that exhausts the input may return n > 0, os.EOF.
type Reader interface {
	Read(p []byte) (n int, err os.Error);
}

// Writer is the interface that wraps the basic Write method.
//
// Write writes len(p) bytes from p to the underlying data stream.
// It returns the number of bytes written from p (0 <= n <= len(p))
// and any error encountered that caused the write to stop early.
// Write must return a non-nil error if it returns n < len(p).
type Writer interface {
	Write(p []byte) (n int, err os.Error);
}

// Closer is the interface that wraps the basic Close method.
type Closer interface {
	Close() os.Error;
}

// Seeker is the interface that wraps the basic Seek method.
//
// Seek sets the offset for the next Read or Write to offset,
// interpreted according to whence: 0 means relative to the origin of
// the file, 1 means relative to the current offset, and 2 means
// relative to the end.  Seek returns the new offset and an Error, if
// any.
type Seeker interface {
	Seek(offset int64, whence int) (ret int64, err os.Error);
}

// ReadWrite is the interface that groups the basic Read and Write methods.
type ReadWriter interface {
	Reader;
	Writer;
}

// ReadCloser is the interface that groups the basic Read and Close methods.
type ReadCloser interface {
	Reader;
	Closer;
}

// WriteCloser is the interface that groups the basic Write and Close methods.
type WriteCloser interface {
	Writer;
	Closer;
}

// ReadWriteCloser is the interface that groups the basic Read, Write and Close methods.
type ReadWriteCloser interface {
	Reader;
	Writer;
	Closer;
}

// ReadSeeker is the interface that groups the basic Read and Seek methods.
type ReadSeeker interface {
	Reader;
	Seeker;
}

// WriteSeeker is the interface that groups the basic Write and Seek methods.
type WriteSeeker interface {
	Writer;
	Seeker;
}

// ReadWriteSeeker is the interface that groups the basic Read, Write and Seek methods.
type ReadWriteSeeker interface {
	Reader;
	Writer;
	Seeker;
}

// ReaderAt is the interface that wraps the basic ReadAt method.
//
// ReadAt reads len(p) bytes into p starting at offset off in the
// underlying data stream.  It returns the number of bytes
// read (0 <= n <= len(p)) and any error encountered.
//
// Even if ReadAt returns n < len(p),
// it may use all of p as scratch space during the call.
// If some data is available but not len(p) bytes, ReadAt blocks
// until either all the data is available or an error occurs.
//
// At the end of the input stream, ReadAt returns 0, os.EOF.
// ReadAt may return a non-zero number of bytes with a non-nil err.
// In particular, a ReadAt that exhausts the input may return n > 0, os.EOF.
type ReaderAt interface {
	ReadAt(p []byte, off int64) (n int, err os.Error);
}

// WriterAt is the interface that wraps the basic WriteAt method.
//
// WriteAt writes len(p) bytes from p to the underlying data stream
// at offset off.  It returns the number of bytes written from p (0 <= n <= len(p))
// and any error encountered that caused the write to stop early.
// WriteAt must return a non-nil error if it returns n < len(p).
type WriterAt interface {
	WriteAt(p []byte, off int64) (n int, err os.Error);
}

// WriteString writes the contents of the string s to w, which accepts an array of bytes.
func WriteString(w Writer, s string) (n int, err os.Error) {
	return w.Write(strings.Bytes(s))
}

// ReadAtLeast reads from r into buf until it has read at least min bytes.
// It returns the number of bytes copied and an error if fewer bytes were read.
// The error is os.EOF only if no bytes were read.
// If an EOF happens after reading fewer than min bytes,
// ReadAtLeast returns ErrUnexpectedEOF.
func ReadAtLeast(r Reader, buf []byte, min int) (n int, err os.Error) {
	n = 0;
	for n < min {
		nn, e := r.Read(buf[n:len(buf)]);
		if nn > 0 {
			n += nn
		}
		if e != nil {
			if e == os.EOF && n > 0 {
				e = ErrUnexpectedEOF;
			}
			return n, e
		}
	}
	return n, nil
}

// ReadFull reads exactly len(buf) bytes from r into buf.
// It returns the number of bytes copied and an error if fewer bytes were read.
// The error is os.EOF only if no bytes were read.
// If an EOF happens after reading some but not all the bytes,
// ReadFull returns ErrUnexpectedEOF.
func ReadFull(r Reader, buf []byte) (n int, err os.Error) {
	return ReadAtLeast(r, buf, len(buf));
}

// Copyn copies n bytes (or until an error) from src to dst.
// It returns the number of bytes copied and the error, if any.
func Copyn(src Reader, dst Writer, n int64) (written int64, err os.Error) {
	buf := make([]byte, 32*1024);
	for written < n {
		l := len(buf);
		if d := n - written; d < int64(l) {
			l = int(d);
		}
		nr, er := src.Read(buf[0 : l]);
		if nr > 0 {
			nw, ew := dst.Write(buf[0 : nr]);
			if nw > 0 {
				written += int64(nw);
			}
			if ew != nil {
				err = ew;
				break;
			}
			if nr != nw {
				err = ErrShortWrite;
				break;
			}
		}
		if er != nil {
			err = er;
			break;
		}
	}
	return written, err
}

// Copy copies from src to dst until either EOF is reached
// on src or an error occurs.  It returns the number of bytes
// copied and the error, if any.
func Copy(src Reader, dst Writer) (written int64, err os.Error) {
	buf := make([]byte, 32*1024);
	for {
		nr, er := src.Read(buf);
		if nr > 0 {
			nw, ew := dst.Write(buf[0:nr]);
			if nw > 0 {
				written += int64(nw);
			}
			if ew != nil {
				err = ew;
				break;
			}
			if nr != nw {
				err = ErrShortWrite;
				break;
			}
		}
		if er == os.EOF {
			break;
		}
		if er != nil {
			err = er;
			break;
		}
	}
	return written, err
}

// LimitReader returns a Reader that reads from r
// but stops with os.EOF after n bytes.
func LimitReader(r Reader, n int64) Reader {
	return &limitedReader{r, n};
}

type limitedReader struct {
	r Reader;
	n int64;
}

func (l *limitedReader) Read(p []byte) (n int, err os.Error) {
	if l.n <= 0 {
		return 0, os.EOF;
	}
	if int64(len(p)) > l.n {
		p = p[0:l.n];
	}
	n, err = l.r.Read(p);
	l.n -= int64(n);
	return;
}

// NewSectionReader returns a SectionReader that reads from r
// starting at offset off and stops with os.EOF after n bytes.
func NewSectionReader(r ReaderAt, off int64, n int64) *SectionReader {
	return &SectionReader{r, off, off, off+n};
}

// SectionReader implements Read, Seek, and ReadAt on a section
// of an underlying ReaderAt.
type SectionReader struct {
	r ReaderAt;
	base int64;
	off int64;
	limit int64;
}

func (s *SectionReader) Read(p []byte) (n int, err os.Error) {
	if s.off >= s.limit {
		return 0, os.EOF;
	}
	if max := s.limit - s.off; int64(len(p)) > max {
		p = p[0:max];
	}
	n, err = s.r.ReadAt(p, s.off);
	s.off += int64(n);
	return;
}

func (s *SectionReader) Seek(offset int64, whence int) (ret int64, err os.Error) {
	switch whence {
	default:
		return 0, os.EINVAL
	case 0:
		offset += s.base
	case 1:
		offset += s.off
	case 2:
		offset += s.limit
	}
	if offset < s.off || offset > s.limit {
		return 0, os.EINVAL
	}
	s.off = offset;
	return offset - s.base, nil
}

func (s *SectionReader) ReadAt(p []byte, off int64) (n int, err os.Error) {
	if off < 0 || off >= s.limit - s.base {
		return 0, os.EOF;
	}
	off += s.base;
	if max := s.limit - off; int64(len(p)) > max {
		p = p[0:max];
	}
	return s.r.ReadAt(p, off);
}

<|endoftext|>


<commit_before>package search_test

import (
	"github.com/gonum/graph/concrete"
	"github.com/gonum/graph/search"
	"math"
	"testing"
)

func TestFWOneEdge(t *testing.T) {
	dg := concrete.NewDenseGraph(2, true)
	aPaths, sPath := search.FloydWarshall(dg, nil)

	path, cost, err := sPath(concrete.GonumNode(0), concrete.GonumNode(1))
	if err != nil {
		t.Fatal(err)
	}

	if math.Abs(cost-1.0) > .000001 {
		t.Errorf("FW got wrong cost %f", cost)
	}

	if len(path) != 2 || path[0].ID() != 0 && path[1].ID() != 1 {
		t.Errorf("Wrong path in FW %v", path)
	}

	paths, cost, err := aPaths(concrete.GonumNode(0), concrete.GonumNode(1))
	if err != nil {
		t.Fatal(err)
	}

	if math.Abs(cost-1.0) > .000001 {
		t.Errorf("FW got wrong cost %f", cost)
	}

	if len(paths) != 1 {
		t.Errorf("Didn't get right paths in FW %v", paths)
	}

	path = paths[0]
	if len(path) != 2 || path[0].ID() != 0 && path[1].ID() != 1 {
		t.Errorf("Wrong path in FW allpaths %v", path)
	}
}

func TestFWTwoPaths(t *testing.T) {
	dg := concrete.NewDenseGraph(5, false)
	// Adds two paths from 0->2 of equal length
	dg.SetEdgeCost(concrete.GonumNode(0), concrete.GonumNode(2), 2.0, true)
	dg.SetEdgeCost(concrete.GonumNode(0), concrete.GonumNode(1), 1.0, true)
	dg.SetEdgeCost(concrete.GonumNode(1), concrete.GonumNode(2), 1.0, true)

	aPaths, sPath := search.FloydWarshall(dg, nil)
	path, cost, err := sPath(concrete.GonumNode(0), concrete.GonumNode(2))
	if err != nil {
		t.Fatal(err)
	}

	if math.Abs(cost-2.0) > .00001 {
		t.Errorf("Path has incorrect cost, %f", cost)
	}

	if len(path) == 2 && path[0].ID() == 0 && path[1].ID() == 2 {
		t.Logf("Got correct path: %v", path)
	} else if len(path) == 3 && path[0].ID() == 0 && path[1].ID() == 1 && path[2].ID() == 2 {
		t.Logf("Got correct path %v", path)
	} else {
		t.Errorf("Got wrong path %v", path)
	}

	paths, cost, err := aPaths(concrete.GonumNode(0), concrete.GonumNode(2))

	if err != nil {
		t.Fatal(err)
	}

	if math.Abs(cost-2.0) > .00001 {
		t.Errorf("All paths function gets incorrect cost, %f", cost)
	}

	if len(paths) != 2 {
		t.Fatalf("Didn't get all shortest paths %v", paths)
	}

	for _, path := range paths {
		if len(path) == 2 && path[0].ID() == 0 && path[1].ID() == 2 {
			t.Logf("Got correct path for all paths: %v", path)
		} else if len(path) == 3 && path[0].ID() == 0 && path[1].ID() == 1 && path[2].ID() == 2 {
			t.Logf("Got correct path for all paths %v", path)
		} else {
			t.Errorf("Got wrong path for all paths %v", path)
		}
	}
}

// Tests with multiple right paths, but also one dead-end path
// and one path that reaches the goal, but not optimally
func TestFWConfoundingPath(t *testing.T) {
	dg := concrete.NewDenseGraph(6, false)

	// Add a path from 0->5 of cost 4
	dg.SetEdgeCost(concrete.GonumNode(0), concrete.GonumNode(1), 1.0, true)
	dg.SetEdgeCost(concrete.GonumNode(1), concrete.GonumNode(2), 1.0, true)
	dg.SetEdgeCost(concrete.GonumNode(2), concrete.GonumNode(3), 1.0, true)
	dg.SetEdgeCost(concrete.GonumNode(3), concrete.GonumNode(5), 1.0, true)

	// Add direct edge to goal of cost 4
	dg.SetEdgeCost(concrete.GonumNode(0), concrete.GonumNode(5), 4.0, true)

	// Add edge to 3 that's overpriced
	dg.SetEdgeCost(concrete.GonumNode(0), concrete.GonumNode(3), 4.0, true)

	// Add very cheap edge to 4 which is a dead end
	dg.SetEdgeCost(concrete.GonumNode(0), concrete.GonumNode(4), 0.25, true)

	aPaths, sPath := search.FloydWarshall(dg, nil)

	path, cost, err := sPath(concrete.GonumNode(0), concrete.GonumNode(5))
	if err != nil {
		t.Fatal(err)
	}

	if math.Abs(cost-4.0) > .000001 {
		t.Error("Incorrect cost %f", cost)
	}

	if len(path) == 5 && path[0].ID() == 0 && path[1].ID() == 1 && path[2].ID() == 2 && path[3].ID() == 3 && path[4].ID() == 5 {
		t.Log("Correct path found for single path %v", path)
	} else if len(path) == 2 && path[0].ID() == 0 && path[1].ID() == 5 {
		t.Log("Correct path found for single path %v", path)
	} else {
		t.Error("Wrong path found for single path %v", path)
	}

	paths, cost, err := aPaths(concrete.GonumNode(0), concrete.GonumNode(5))
	if err != nil {
		t.Fatal(err)
	}

	if math.Abs(cost-4.0) > .000001 {
		t.Error("Incorrect cost %f", cost)
	}

	if len(paths) != 2 {
		t.Error("Wrong paths gooten for all paths %v", paths)
	}

	for _, path := range paths {
		if len(path) == 5 && path[0].ID() == 0 && path[1].ID() == 1 && path[2].ID() == 2 && path[3].ID() == 3 && path[4].ID() == 5 {
			t.Log("Correct path found for all paths %v", path)
		} else if len(path) == 2 && path[0].ID() == 0 && path[1].ID() == 5 {
			t.Log("Correct path found for all paths %v", path)
		} else {
			t.Error("Wrong path found for all paths %v", path)
		}
	}

	path, _, err = sPath(concrete.GonumNode(4), concrete.GonumNode(5))
	if err != nil {
		t.Log("Success!", err)
	} else {
		t.Error("Path was found by FW single path where one shouldn't be %v", path)
	}

	paths, _, err = aPaths(concrete.GonumNode(4), concrete.GonumNode(5))
	if err != nil {
		t.Log("Success!", err)
	} else {
		t.Error("Path was found by FW multi-path where one shouldn't be %v", paths)
	}
}
<commit_msg>Added a little more to the complex test<commit_after>package search_test

import (
	"github.com/gonum/graph/concrete"
	"github.com/gonum/graph/search"
	"math"
	"testing"
)

func TestFWOneEdge(t *testing.T) {
	dg := concrete.NewDenseGraph(2, true)
	aPaths, sPath := search.FloydWarshall(dg, nil)

	path, cost, err := sPath(concrete.GonumNode(0), concrete.GonumNode(1))
	if err != nil {
		t.Fatal(err)
	}

	if math.Abs(cost-1.0) > .000001 {
		t.Errorf("FW got wrong cost %f", cost)
	}

	if len(path) != 2 || path[0].ID() != 0 && path[1].ID() != 1 {
		t.Errorf("Wrong path in FW %v", path)
	}

	paths, cost, err := aPaths(concrete.GonumNode(0), concrete.GonumNode(1))
	if err != nil {
		t.Fatal(err)
	}

	if math.Abs(cost-1.0) > .000001 {
		t.Errorf("FW got wrong cost %f", cost)
	}

	if len(paths) != 1 {
		t.Errorf("Didn't get right paths in FW %v", paths)
	}

	path = paths[0]
	if len(path) != 2 || path[0].ID() != 0 && path[1].ID() != 1 {
		t.Errorf("Wrong path in FW allpaths %v", path)
	}
}

func TestFWTwoPaths(t *testing.T) {
	dg := concrete.NewDenseGraph(5, false)
	// Adds two paths from 0->2 of equal length
	dg.SetEdgeCost(concrete.GonumNode(0), concrete.GonumNode(2), 2.0, true)
	dg.SetEdgeCost(concrete.GonumNode(0), concrete.GonumNode(1), 1.0, true)
	dg.SetEdgeCost(concrete.GonumNode(1), concrete.GonumNode(2), 1.0, true)

	aPaths, sPath := search.FloydWarshall(dg, nil)
	path, cost, err := sPath(concrete.GonumNode(0), concrete.GonumNode(2))
	if err != nil {
		t.Fatal(err)
	}

	if math.Abs(cost-2.0) > .00001 {
		t.Errorf("Path has incorrect cost, %f", cost)
	}

	if len(path) == 2 && path[0].ID() == 0 && path[1].ID() == 2 {
		t.Logf("Got correct path: %v", path)
	} else if len(path) == 3 && path[0].ID() == 0 && path[1].ID() == 1 && path[2].ID() == 2 {
		t.Logf("Got correct path %v", path)
	} else {
		t.Errorf("Got wrong path %v", path)
	}

	paths, cost, err := aPaths(concrete.GonumNode(0), concrete.GonumNode(2))

	if err != nil {
		t.Fatal(err)
	}

	if math.Abs(cost-2.0) > .00001 {
		t.Errorf("All paths function gets incorrect cost, %f", cost)
	}

	if len(paths) != 2 {
		t.Fatalf("Didn't get all shortest paths %v", paths)
	}

	for _, path := range paths {
		if len(path) == 2 && path[0].ID() == 0 && path[1].ID() == 2 {
			t.Logf("Got correct path for all paths: %v", path)
		} else if len(path) == 3 && path[0].ID() == 0 && path[1].ID() == 1 && path[2].ID() == 2 {
			t.Logf("Got correct path for all paths %v", path)
		} else {
			t.Errorf("Got wrong path for all paths %v", path)
		}
	}
}

// Tests with multiple right paths, but also one dead-end path
// and one path that reaches the goal, but not optimally
func TestFWConfoundingPath(t *testing.T) {
	dg := concrete.NewDenseGraph(6, false)

	// Add a path from 0->5 of cost 4
	dg.SetEdgeCost(concrete.GonumNode(0), concrete.GonumNode(1), 1.0, true)
	dg.SetEdgeCost(concrete.GonumNode(1), concrete.GonumNode(2), 1.0, true)
	dg.SetEdgeCost(concrete.GonumNode(2), concrete.GonumNode(3), 1.0, true)
	dg.SetEdgeCost(concrete.GonumNode(3), concrete.GonumNode(5), 1.0, true)

	// Add direct edge to goal of cost 4
	dg.SetEdgeCost(concrete.GonumNode(0), concrete.GonumNode(5), 4.0, true)

	// Add edge to a node that's still optimal
	dg.SetEdgeCost(concrete.GonumNode(0), concrete.GonumNode(2), 2.0, true)

	// Add edge to 3 that's overpriced
	dg.SetEdgeCost(concrete.GonumNode(0), concrete.GonumNode(3), 4.0, true)

	// Add very cheap edge to 4 which is a dead end
	dg.SetEdgeCost(concrete.GonumNode(0), concrete.GonumNode(4), 0.25, true)

	aPaths, sPath := search.FloydWarshall(dg, nil)

	path, cost, err := sPath(concrete.GonumNode(0), concrete.GonumNode(5))
	if err != nil {
		t.Fatal(err)
	}

	if math.Abs(cost-4.0) > .000001 {
		t.Errorf("Incorrect cost %f", cost)
	}

	if len(path) == 5 && path[0].ID() == 0 && path[1].ID() == 1 && path[2].ID() == 2 && path[3].ID() == 3 && path[4].ID() == 5 {
		t.Logf("Correct path found for single path %v", path)
	} else if len(path) == 2 && path[0].ID() == 0 && path[1].ID() == 5 {
		t.Logf("Correct path found for single path %v", path)
	} else if len(path) == 4 && path[0].ID() == 0 && path[1].ID() == 2 && path[2].ID() == 3 && path[3].ID() == 5 {
		t.Logf("Correct path found for single path %v", path)
	} else {
		t.Errorf("Wrong path found for single path %v", path)
	}

	paths, cost, err := aPaths(concrete.GonumNode(0), concrete.GonumNode(5))
	if err != nil {
		t.Fatal(err)
	}

	if math.Abs(cost-4.0) > .000001 {
		t.Errorf("Incorrect cost %f", cost)
	}

	if len(paths) != 3 {
		t.Errorf("Wrong paths gotten for all paths %v", paths)
	}

	for _, path := range paths {
		if len(path) == 5 && path[0].ID() == 0 && path[1].ID() == 1 && path[2].ID() == 2 && path[3].ID() == 3 && path[4].ID() == 5 {
			t.Logf("Correct path found for multi path %v", path)
		} else if len(path) == 2 && path[0].ID() == 0 && path[1].ID() == 5 {
			t.Logf("Correct path found for multi path %v", path)
		} else if len(path) == 4 && path[0].ID() == 0 && path[1].ID() == 2 && path[2].ID() == 3 && path[3].ID() == 5 {
			t.Logf("Correct path found for multi path %v", path)
		} else {
			t.Errorf("Wrong path found for multi path %v", path)
		}
	}

	path, _, err = sPath(concrete.GonumNode(4), concrete.GonumNode(5))
	if err != nil {
		t.Log("Success!", err)
	} else {
		t.Errorf("Path was found by FW single path where one shouldn't be %v", path)
	}

	paths, _, err = aPaths(concrete.GonumNode(4), concrete.GonumNode(5))
	if err != nil {
		t.Log("Success!", err)
	} else {
		t.Errorf("Path was found by FW multi-path where one shouldn't be %v", paths)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package rpcreflect

import (
	"errors"
	"reflect"
	"sort"
	"sync"

	"launchpad.net/juju-core/log"
)

var (
	errorType  = reflect.TypeOf((*error)(nil)).Elem()
	stringType = reflect.TypeOf("")
)

var (
	typeMutex     sync.RWMutex
	typesByGoType = make(map[reflect.Type]*Type)

	objTypeMutex     sync.RWMutex
	objTypesByGoType = make(map[reflect.Type]*ObjType)
)

var ErrMethodNotFound = errors.New("no such method")

// Type holds information about a type that implements RPC server methods,
// a root-level RPC type.
type Type struct {
	// root holds the root type.
	root reflect.Type

	// method maps from root-object method name to
	// information about that method. The term "obtain"
	// is because these methods obtain an object to
	// call an action on.
	method map[string]*RootMethod

	// discarded holds names of all discarded methods.
	discarded []string
}

// MethodNames returns the names of all the root object
// methods on the receiving object.
func (r *Type) MethodNames() []string {
	var names []string
	for name := range r.method {
		names = append(names, name)
	}
	sort.Strings(names)
	return names
}

// Method returns information on the method with the given name,
// or the zero Method and ErrMethodNotFound if there is no such method.
func (r *Type) Method(name string) (RootMethod, error) {
	m, ok := r.method[name]
	if !ok {
		return RootMethod{}, ErrMethodNotFound
	}
	return *m, nil
}

func (r *Type) DiscardedMethods() []string {
	return append([]string(nil), r.discarded...)
}

// RootMethod holds information on a root-level method.
type RootMethod struct {
	// Call invokes the method. The rcvr parameter must be
	// the same type as the root object. The given id is passed
	// as an argument to the method.
	Call func(rcvr reflect.Value, id string) (reflect.Value, error)

	// Methods holds RPC object-method information about
	// objects returned from the above call.
	ObjType *ObjType
}

// TypeOf returns information on all root-level RPC methods
// implemented by an object of the given Go type.
func TypeOf(goType reflect.Type) *Type {
	typeMutex.RLock()
	t := typesByGoType[goType]
	typeMutex.RUnlock()
	if t != nil {
		return t
	}
	typeMutex.Lock()
	defer typeMutex.Unlock()
	t = typesByGoType[goType]
	if t != nil {
		return t
	}
	t = rootInfo(goType)
	typesByGoType[goType] = t
	return t
}

// rootInfo is like RootInfo but without the cache - it
// always allocates. Called with rootTypeMutex locked.
func rootInfo(goType reflect.Type) *Type {
	rm := &Type{
		method: make(map[string]*RootMethod),
	}
	for i := 0; i < goType.NumMethod(); i++ {
		m := goType.Method(i)
		if m.PkgPath != "" || isKillMethod(m) {
			// The Kill method gets a special exception because
			// it fulfils the Killer interface which we're expecting,
			// so it's not really discarded as such.
			continue
		}
		if o := newRootMethod(m); o != nil {
			rm.method[m.Name] = o
		} else {
			rm.discarded = append(rm.discarded, m.Name)
		}
	}
	return rm
}

func isKillMethod(m reflect.Method) bool {
	return m.Name == "Kill" && m.Type.NumIn() == 1 && m.Type.NumOut() == 0
}

func newRootMethod(m reflect.Method) *RootMethod {
	if m.PkgPath != "" {
		return nil
	}
	t := m.Type
	if t.NumIn() != 2 ||
		t.NumOut() != 2 ||
		t.In(1) != stringType ||
		t.Out(1) != errorType {
		return nil
	}
	f := func(rcvr reflect.Value, id string) (r reflect.Value, err error) {
		out := rcvr.Method(m.Index).Call([]reflect.Value{reflect.ValueOf(id)})
		if !out[1].IsNil() {
			err = out[1].Interface().(error)
		}
		r = out[0]
		return
	}
	return &RootMethod{
		Call:    f,
		ObjType: ObjTypeOf(t.Out(0)),
	}
}

// ObjType holds information on RPC methods implemented on
// an RPC object.
type ObjType struct {
	goType    reflect.Type
	method    map[string]*ObjMethod
	discarded []string
}

func (t *ObjType) GoType() reflect.Type {
	return t.goType
}

// Method returns information on the method with the given name,
// or the zero Method and an error if there is no such method.
func (t *ObjType) Method(name string) (ObjMethod, error) {
	m, ok := t.method[name]
	if !ok {
		return ObjMethod{}, ErrMethodNotFound
	}
	return *m, nil
}

// DiscardedMethods returns the names of all methods that cannot
// implement RPC calls because their type signature is inappropriate.
func (t *ObjType) DiscardedMethods() []string {
	return append([]string(nil), t.discarded...)
}

// MethodNames returns the names of all the RPC methods
// defined on the type.
func (t *ObjType) MethodNames() []string {
	var names []string
	for name := range t.method {
		names = append(names, name)
	}
	sort.Strings(names)
	return names
}

// ObjMethod holds information about an RPC method on an
// object returned by a root-level method.
type ObjMethod struct {
	// Params holds the argument type of the method, or nil
	// if there is no argument.
	Params reflect.Type

	// Result holds the return type of the method, or nil
	// if the method returns no value.
	Result reflect.Type

	// Call calls the method with the given argument
	// on the given receiver value. If the method does
	// not return a value, the returned value will not be valid.
	Call func(rcvr, arg reflect.Value) (reflect.Value, error)
}

// ObjTypeOf returns information on all RPC methods
// implemented by an object of the given Go type,
// as returned from a root-level method.
func ObjTypeOf(objType reflect.Type) *ObjType {
	objTypeMutex.RLock()
	methods := objTypesByGoType[objType]
	objTypeMutex.RUnlock()
	if methods != nil {
		return methods
	}
	objTypeMutex.Lock()
	defer objTypeMutex.Unlock()
	methods = objTypesByGoType[objType]
	if methods != nil {
		return methods
	}
	methods = objTypeOf(objType)
	objTypesByGoType[objType] = methods
	return methods
}

// objTypeOf is like ObjTypeOf but without the cache.
// Called with objTypeMutex locked.
func objTypeOf(goType reflect.Type) *ObjType {
	objType := &ObjType{
		method: make(map[string]*ObjMethod),
		goType: goType,
	}
	for i := 0; i < goType.NumMethod(); i++ {
		m := goType.Method(i)
		if m.PkgPath != "" {
			continue
		}
		log.Infof("considering method %#v\n", m)
		if objm := newMethod(m, goType.Kind()); objm != nil {
			objType.method[m.Name] = objm
		} else {
			objType.discarded = append(objType.discarded, m.Name)
		}
	}
	return objType
}

func newMethod(m reflect.Method, receiverKind reflect.Kind) *ObjMethod {
	if m.PkgPath != "" {
		return nil
	}
	var p ObjMethod
	var assemble func(arg reflect.Value) []reflect.Value
	// N.B. The method type has the receiver as its first argument
	// unless the receiver is an interface.
	receiverArgCount := 1
	if receiverKind == reflect.Interface {
		receiverArgCount = 0
	}
	t := m.Type
	switch {
	case t.NumIn() == 0+receiverArgCount:
		// Method() ...
		assemble = func(arg reflect.Value) []reflect.Value {
			return nil
		}
	case t.NumIn() == 1+receiverArgCount:
		// Method(T) ...
		p.Params = t.In(receiverArgCount)
		assemble = func(arg reflect.Value) []reflect.Value {
			return []reflect.Value{arg}
		}
	default:
		return nil
	}

	switch {
	case t.NumOut() == 0:
		// Method(...)
		p.Call = func(rcvr, arg reflect.Value) (r reflect.Value, err error) {
			rcvr.Method(m.Index).Call(assemble(arg))
			return
		}
	case t.NumOut() == 1 && t.Out(0) == errorType:
		// Method(...) error
		p.Call = func(rcvr, arg reflect.Value) (r reflect.Value, err error) {
			out := rcvr.Method(m.Index).Call(assemble(arg))
			if !out[0].IsNil() {
				err = out[0].Interface().(error)
			}
			return
		}
	case t.NumOut() == 1:
		// Method(...) R
		p.Result = t.Out(0)
		p.Call = func(rcvr, arg reflect.Value) (reflect.Value, error) {
			out := rcvr.Method(m.Index).Call(assemble(arg))
			return out[0], nil
		}
	case t.NumOut() == 2 && t.Out(1) == errorType:
		// Method(...) (R, error)
		p.Result = t.Out(0)
		p.Call = func(rcvr, arg reflect.Value) (r reflect.Value, err error) {
			out := rcvr.Method(m.Index).Call(assemble(arg))
			r = out[0]
			if !out[1].IsNil() {
				err = out[1].Interface().(error)
			}
			return
		}
	default:
		return nil
	}
	// The parameters and return value must be of struct type.
	if p.Params != nil && p.Params.Kind() != reflect.Struct {
		return nil
	}
	if p.Result != nil && p.Result.Kind() != reflect.Struct {
		return nil
	}
	return &p
}
<commit_msg>rpc/rpcreflect: minor changes for review<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package rpcreflect

import (
	"errors"
	"reflect"
	"sort"
	"sync"
)

var (
	errorType  = reflect.TypeOf((*error)(nil)).Elem()
	stringType = reflect.TypeOf("")
)

var (
	typeMutex     sync.RWMutex
	typesByGoType = make(map[reflect.Type]*Type)

	objTypeMutex     sync.RWMutex
	objTypesByGoType = make(map[reflect.Type]*ObjType)
)

var ErrMethodNotFound = errors.New("no such method")

// Type holds information about a type that implements RPC server methods,
// a root-level RPC type.
type Type struct {
	// root holds the root type.
	root reflect.Type

	// method maps from root-object method name to
	// information about that method. The term "obtain"
	// is because these methods obtain an object to
	// call an action on.
	method map[string]*RootMethod

	// discarded holds names of all discarded methods.
	discarded []string
}

// MethodNames returns the names of all the root object
// methods on the receiving object.
func (r *Type) MethodNames() []string {
	names := make([]string, 0, len(r.method))
	for name := range r.method {
		names = append(names, name)
	}
	sort.Strings(names)
	return names
}

// Method returns information on the method with the given name,
// or the zero Method and ErrMethodNotFound if there is no such method.
func (r *Type) Method(name string) (RootMethod, error) {
	m, ok := r.method[name]
	if !ok {
		return RootMethod{}, ErrMethodNotFound
	}
	return *m, nil
}

func (r *Type) DiscardedMethods() []string {
	return append([]string(nil), r.discarded...)
}

// RootMethod holds information on a root-level method.
type RootMethod struct {
	// Call invokes the method. The rcvr parameter must be
	// the same type as the root object. The given id is passed
	// as an argument to the method.
	Call func(rcvr reflect.Value, id string) (reflect.Value, error)

	// Methods holds RPC object-method information about
	// objects returned from the above call.
	ObjType *ObjType
}

// TypeOf returns information on all root-level RPC methods
// implemented by an object of the given Go type.
func TypeOf(goType reflect.Type) *Type {
	typeMutex.RLock()
	t := typesByGoType[goType]
	typeMutex.RUnlock()
	if t != nil {
		return t
	}
	typeMutex.Lock()
	defer typeMutex.Unlock()
	t = typesByGoType[goType]
	if t != nil {
		return t
	}
	t = rootInfo(goType)
	typesByGoType[goType] = t
	return t
}

// rootInfo is like RootInfo but without the cache - it
// always allocates. Called with rootTypeMutex locked.
func rootInfo(goType reflect.Type) *Type {
	rm := &Type{
		method: make(map[string]*RootMethod),
	}
	for i := 0; i < goType.NumMethod(); i++ {
		m := goType.Method(i)
		if m.PkgPath != "" || isKillMethod(m) {
			// The Kill method gets a special exception because
			// it fulfils the Killer interface which we're expecting,
			// so it's not really discarded as such.
			continue
		}
		if o := newRootMethod(m); o != nil {
			rm.method[m.Name] = o
		} else {
			rm.discarded = append(rm.discarded, m.Name)
		}
	}
	return rm
}

func isKillMethod(m reflect.Method) bool {
	return m.Name == "Kill" && m.Type.NumIn() == 1 && m.Type.NumOut() == 0
}

func newRootMethod(m reflect.Method) *RootMethod {
	if m.PkgPath != "" {
		return nil
	}
	t := m.Type
	if t.NumIn() != 2 ||
		t.NumOut() != 2 ||
		t.In(1) != stringType ||
		t.Out(1) != errorType {
		return nil
	}
	f := func(rcvr reflect.Value, id string) (r reflect.Value, err error) {
		out := rcvr.Method(m.Index).Call([]reflect.Value{reflect.ValueOf(id)})
		if !out[1].IsNil() {
			err = out[1].Interface().(error)
		}
		r = out[0]
		return
	}
	return &RootMethod{
		Call:    f,
		ObjType: ObjTypeOf(t.Out(0)),
	}
}

// ObjType holds information on RPC methods implemented on
// an RPC object.
type ObjType struct {
	goType    reflect.Type
	method    map[string]*ObjMethod
	discarded []string
}

func (t *ObjType) GoType() reflect.Type {
	return t.goType
}

// Method returns information on the method with the given name,
// or the zero Method and an error if there is no such method.
func (t *ObjType) Method(name string) (ObjMethod, error) {
	m, ok := t.method[name]
	if !ok {
		return ObjMethod{}, ErrMethodNotFound
	}
	return *m, nil
}

// DiscardedMethods returns the names of all methods that cannot
// implement RPC calls because their type signature is inappropriate.
func (t *ObjType) DiscardedMethods() []string {
	return append([]string(nil), t.discarded...)
}

// MethodNames returns the names of all the RPC methods
// defined on the type.
func (t *ObjType) MethodNames() []string {
	names := make([]string, 0, len(t.method))
	for name := range t.method {
		names = append(names, name)
	}
	sort.Strings(names)
	return names
}

// ObjMethod holds information about an RPC method on an
// object returned by a root-level method.
type ObjMethod struct {
	// Params holds the argument type of the method, or nil
	// if there is no argument.
	Params reflect.Type

	// Result holds the return type of the method, or nil
	// if the method returns no value.
	Result reflect.Type

	// Call calls the method with the given argument
	// on the given receiver value. If the method does
	// not return a value, the returned value will not be valid.
	Call func(rcvr, arg reflect.Value) (reflect.Value, error)
}

// ObjTypeOf returns information on all RPC methods
// implemented by an object of the given Go type,
// as returned from a root-level method.
func ObjTypeOf(objType reflect.Type) *ObjType {
	objTypeMutex.RLock()
	methods := objTypesByGoType[objType]
	objTypeMutex.RUnlock()
	if methods != nil {
		return methods
	}
	objTypeMutex.Lock()
	defer objTypeMutex.Unlock()
	methods = objTypesByGoType[objType]
	if methods != nil {
		return methods
	}
	methods = objTypeOf(objType)
	objTypesByGoType[objType] = methods
	return methods
}

// objTypeOf is like ObjTypeOf but without the cache.
// Called with objTypeMutex locked.
func objTypeOf(goType reflect.Type) *ObjType {
	objType := &ObjType{
		method: make(map[string]*ObjMethod),
		goType: goType,
	}
	for i := 0; i < goType.NumMethod(); i++ {
		m := goType.Method(i)
		if m.PkgPath != "" {
			continue
		}
		if objm := newMethod(m, goType.Kind()); objm != nil {
			objType.method[m.Name] = objm
		} else {
			objType.discarded = append(objType.discarded, m.Name)
		}
	}
	return objType
}

func newMethod(m reflect.Method, receiverKind reflect.Kind) *ObjMethod {
	if m.PkgPath != "" {
		return nil
	}
	var p ObjMethod
	var assemble func(arg reflect.Value) []reflect.Value
	// N.B. The method type has the receiver as its first argument
	// unless the receiver is an interface.
	receiverArgCount := 1
	if receiverKind == reflect.Interface {
		receiverArgCount = 0
	}
	t := m.Type
	switch {
	case t.NumIn() == 0+receiverArgCount:
		// Method() ...
		assemble = func(arg reflect.Value) []reflect.Value {
			return nil
		}
	case t.NumIn() == 1+receiverArgCount:
		// Method(T) ...
		p.Params = t.In(receiverArgCount)
		assemble = func(arg reflect.Value) []reflect.Value {
			return []reflect.Value{arg}
		}
	default:
		return nil
	}

	switch {
	case t.NumOut() == 0:
		// Method(...)
		p.Call = func(rcvr, arg reflect.Value) (r reflect.Value, err error) {
			rcvr.Method(m.Index).Call(assemble(arg))
			return
		}
	case t.NumOut() == 1 && t.Out(0) == errorType:
		// Method(...) error
		p.Call = func(rcvr, arg reflect.Value) (r reflect.Value, err error) {
			out := rcvr.Method(m.Index).Call(assemble(arg))
			if !out[0].IsNil() {
				err = out[0].Interface().(error)
			}
			return
		}
	case t.NumOut() == 1:
		// Method(...) R
		p.Result = t.Out(0)
		p.Call = func(rcvr, arg reflect.Value) (reflect.Value, error) {
			out := rcvr.Method(m.Index).Call(assemble(arg))
			return out[0], nil
		}
	case t.NumOut() == 2 && t.Out(1) == errorType:
		// Method(...) (R, error)
		p.Result = t.Out(0)
		p.Call = func(rcvr, arg reflect.Value) (r reflect.Value, err error) {
			out := rcvr.Method(m.Index).Call(assemble(arg))
			r = out[0]
			if !out[1].IsNil() {
				err = out[1].Interface().(error)
			}
			return
		}
	default:
		return nil
	}
	// The parameters and return value must be of struct type.
	if p.Params != nil && p.Params.Kind() != reflect.Struct {
		return nil
	}
	if p.Result != nil && p.Result.Kind() != reflect.Struct {
		return nil
	}
	return &p
}
<|endoftext|>


<commit_before>// BucketReader gets a list of files awaiting processing in the S3
// intake buckets and adds each item to the queue for processing.
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/APTrust/bagman"
	"github.com/APTrust/bagman/fluctus/client"
	"github.com/crowdmob/goamz/aws"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

// Send S3 files to queue in batches of 500.
// Wait X milliseconds between batches. The wait time is really
// only necessary when the queue is running on an AWS small EC2
// instance, where the number of open network connections is
// severely restricted.
const (
	batchSize        = 500
	waitMilliseconds = 5000
)

var (
	config        bagman.Config
	jsonLog       *log.Logger
	messageLog    *log.Logger
	fluctusClient *client.Client
	statusCache   map[string]*bagman.ProcessStatus
)

func main() {
	err := initialize()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Initialization failed for bucket_reader: %v", err)
		os.Exit(1)
	}
	run()
}

func initialize() (err error) {
	// Load the config or die.
	requestedConfig := flag.String("config", "", "configuration to run")
	flag.Parse()
	config = bagman.LoadRequestedConfig(requestedConfig)
	jsonLog, messageLog = bagman.InitLoggers(config.LogDirectory, "bucket_reader")
	fluctusClient, err = client.New(
		config.FluctusURL,
		os.Getenv("FLUCTUS_API_USER"),
		os.Getenv("FLUCTUS_API_KEY"),
		messageLog)
	return err
}

func run() {
	s3Client, err := bagman.NewS3Client(aws.USEast)
	if err != nil {
		messageLog.Println("[ERROR]", err)
		return
	}
	bucketSummaries, err := s3Client.CheckAllBuckets(config.Buckets)
	if err != nil {
		messageLog.Println("[ERROR]", err)
		return
	}
	loadStatusCache()
	url := fmt.Sprintf("%s/mput?topic=%s", config.NsqdHttpAddress,
		config.BagProcessorTopic)
	messageLog.Printf("[INFO] Sending S3 file info to %s \n", url)
	s3Files := filterLargeFiles(bucketSummaries)
	messageLog.Printf("[INFO] %d S3 Files are within our size limit\n",
		len(s3Files))
	filesToProcess := s3Files
	// SkipAlreadyProcessed will almost always be true.
	// The exception is when we want to reprocess items to test new code.
	if config.SkipAlreadyProcessed == true {
		messageLog.Println("[INFO] Skipping already processed files, because config says so")
		filesToProcess = filterProcessedFiles(s3Files)
	} else {
		messageLog.Println("[INFO] Reprocessing already processed files, because config says so")
	}
	start := 0
	end := min(len(filesToProcess), batchSize)
	messageLog.Printf("[INFO] %d Unprocessed files\n", len(filesToProcess))
	for start <= end {
		batch := filesToProcess[start:end]
		messageLog.Printf("[INFO] Queuing batch of %d items\n", len(batch))
		enqueue(url, batch)
		start = end + 1
		if start < len(filesToProcess) {
			end = min(len(filesToProcess), start+batchSize)
		}
		// Sleep so we don't max out connections on EC2 small.
		// The utility server is running a lot of other network I/O
		// in addition to our queue.
		time.Sleep(time.Millisecond * waitMilliseconds)
	}
}

// min returns the minimum of x or y. The Math package has this function
// but you have to cast to floats.
func min(x, y int) int {
	if x < y {
		return x
	} else {
		return y
	}
}

// filterLargeFiles returns only those S3 files that are not larger
// than config.MaxFileSize. This is useful when running tests and
// demos on your local machine, so that you can limit your test
// runs to files under 100k (or whatever you set in config.json).
// You don't want to pull down lots of multi-gig files when you're
// just running local tests. In production, set maxFileSize to
// zero, or to some huge value to get all files.
func filterLargeFiles(bucketSummaries []*bagman.BucketSummary) (s3Files []*bagman.S3File) {
	for _, bucketSummary := range bucketSummaries {
		for _, key := range bucketSummary.Keys {
			if config.MaxFileSize == 0 || key.Size < config.MaxFileSize {
				s3Files = append(s3Files, &bagman.S3File{bucketSummary.BucketName, key})
			}
		}
	}
	return s3Files
}

// Remove S3 files that have been processed successfully.
// No need to reprocess those!
func filterProcessedFiles(s3Files []*bagman.S3File) (filesToProcess []*bagman.S3File) {
	for _, s3File := range s3Files {
		bagDate, err := time.Parse(bagman.S3DateFormat, s3File.Key.LastModified)
		if err != nil {
			messageLog.Printf("[ERROR] Cannot parse S3File mod date '%s'. "+
				"File %s will be re-processed.",
				s3File.Key.LastModified, s3File.Key.Key)
			filesToProcess = append(filesToProcess, s3File)
			continue
		}
		etag := strings.Replace(s3File.Key.ETag, "\"", "", 2)
		status := findInStatusCache(etag, s3File.Key.Key, bagDate)
		if status == nil {
			status, err = fluctusClient.GetBagStatus(etag, s3File.Key.Key, bagDate)
		}
		if err != nil {
			messageLog.Printf("[ERROR] Cannot get Fluctus bag status for %s. "+
				"Will re-process bag. Error was %v", s3File.Key.Key, err)
			filesToProcess = append(filesToProcess, s3File)
		} else if  status == nil ||
			(status.Stage == "Receive" && time.Since(status.Date) > time.Hour*24) ||
			(status.Status == "Processing" && status.Retry == true) {
			reason := "Bag has not yet been successfully processed."
			if status == nil {
				err = createFluctusRecord(s3File)
				if err != nil {
					// TODO: Notify someone?
					messageLog.Printf("[ERROR] Could not create Fluctus ProcessedItem "+
						"for %s: %v", s3File.Key.Key, err)
				}
			}
			messageLog.Printf("[INFO] Will process bag %s: %s", s3File.Key.Key, reason)
			filesToProcess = append(filesToProcess, s3File)
		} else if status.Status != "Failed" {
			messageLog.Printf("[INFO] Skipping %s: already processed successfully.", s3File.Key.Key)
		} else if status.Retry == false {
			messageLog.Printf("[INFO] Skipping %s: retry flag is set to false.", s3File.Key.Key)
		}
	}
	return filesToProcess
}

// Loads status of all bags received in the past two hours from fluctus
// in a single call.
func loadStatusCache() {
	twoHoursAgo := time.Now().Add(time.Hour * -2)
	statusRecords, err := fluctusClient.BulkStatusGet(twoHoursAgo)
	if err != nil {
		messageLog.Println("[WARNING] Could not get bulk status records")
	} else {
		messageLog.Printf("[INFO] Got %d status records from the fluctopus\n", len(statusRecords))
		statusCache = make(map[string]*bagman.ProcessStatus, len(statusRecords))
		for i := range(statusRecords) {
			record := statusRecords[i]
			key := fmt.Sprintf("%s%s%s", record.ETag, record.Name, record.BagDate)
			statusCache[key] = record
		}
	}
}

// Finds the status of the specified tar bag in the cache that
// we retrieved from Fluctus. The cache can save us hundreds or
// thousands of HTTP calls each time the bucket reader runs.
func findInStatusCache(etag, name string, bagDate time.Time) (*bagman.ProcessStatus) {
	key := fmt.Sprintf("%s%s%s", etag, name, bagDate)
	item, exists := statusCache[key]
	if exists {
		messageLog.Printf("[INFO] Found item in cache: %s\n", name)
		return item
	}
	messageLog.Printf("[INFO] Item not in cache. Will have to ask the fluctopus for %s\n", name)
	return nil
}

func createFluctusRecord(s3File *bagman.S3File) (err error) {
	status := &bagman.ProcessStatus{}
	status.Date = time.Now().UTC()
	status.Action = "Ingest"
	status.Name = s3File.Key.Key
	bagDate, _ := time.Parse(bagman.S3DateFormat, s3File.Key.LastModified)
	status.BagDate = bagDate
	status.Bucket = s3File.BucketName
	// Strip the quotes off the ETag
	status.ETag = strings.Replace(s3File.Key.ETag, "\"", "", 2)
	status.Stage = "Receive"
	status.Status = "Succeeded"
	status.Note = "Item is in receiving bucket. Processing has not started."
	status.Institution = bagman.OwnerOf(s3File.BucketName)
	status.Outcome = status.Status

	// Retry should be true until we know for sure that there
	// is something wrong with the bag.
	status.Retry = true

	err = fluctusClient.UpdateBagStatus(status)
	if err != nil {
		return err
	}
	messageLog.Printf("[INFO] Created Fluctus ProcessedItem for %s\n",
		s3File.Key.Key)
	return nil
}

// enqueue adds a batch of items to the nsqd work queue
func enqueue(url string, s3Files []*bagman.S3File) {
	jsonData := make([]string, len(s3Files))
	for i, s3File := range s3Files {
		json, err := json.Marshal(s3File)
		if err != nil {
			messageLog.Printf("[ERROR] Error marshalling s3 file to JSON: %v", err)
		} else {
			jsonData[i] = string(json)
			messageLog.Println("[INFO]", "Put", s3File.Key.Key, "into fetch queue")
		}
	}
	batch := strings.Join(jsonData, "\n")
	resp, err := http.Post(url, "application/json", bytes.NewBuffer([]byte(batch)))
	if err != nil {
		messageLog.Printf("[ERROR] nsqd returned an error: %v", err)
	}
	if resp == nil {
		msg := "[ERROR] No response from nsqd. Is it running? bucket_reader is quitting."
		messageLog.Printf(msg)
		fmt.Println(msg)
		os.Exit(1)
	} else if resp.StatusCode != 200 {
		messageLog.Printf("[ERROR] nsqd returned status code %d on last mput", resp.StatusCode)
	}
}
<commit_msg>Removed time limit on receive<commit_after>// BucketReader gets a list of files awaiting processing in the S3
// intake buckets and adds each item to the queue for processing.
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/APTrust/bagman"
	"github.com/APTrust/bagman/fluctus/client"
	"github.com/crowdmob/goamz/aws"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

// Send S3 files to queue in batches of 500.
// Wait X milliseconds between batches. The wait time is really
// only necessary when the queue is running on an AWS small EC2
// instance, where the number of open network connections is
// severely restricted.
const (
	batchSize        = 500
	waitMilliseconds = 5000
)

var (
	config        bagman.Config
	jsonLog       *log.Logger
	messageLog    *log.Logger
	fluctusClient *client.Client
	statusCache   map[string]*bagman.ProcessStatus
)

func main() {
	err := initialize()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Initialization failed for bucket_reader: %v", err)
		os.Exit(1)
	}
	run()
}

func initialize() (err error) {
	// Load the config or die.
	requestedConfig := flag.String("config", "", "configuration to run")
	flag.Parse()
	config = bagman.LoadRequestedConfig(requestedConfig)
	jsonLog, messageLog = bagman.InitLoggers(config.LogDirectory, "bucket_reader")
	fluctusClient, err = client.New(
		config.FluctusURL,
		os.Getenv("FLUCTUS_API_USER"),
		os.Getenv("FLUCTUS_API_KEY"),
		messageLog)
	return err
}

func run() {
	s3Client, err := bagman.NewS3Client(aws.USEast)
	if err != nil {
		messageLog.Println("[ERROR]", err)
		return
	}
	bucketSummaries, err := s3Client.CheckAllBuckets(config.Buckets)
	if err != nil {
		messageLog.Println("[ERROR]", err)
		return
	}
	loadStatusCache()
	url := fmt.Sprintf("%s/mput?topic=%s", config.NsqdHttpAddress,
		config.BagProcessorTopic)
	messageLog.Printf("[INFO] Sending S3 file info to %s \n", url)
	s3Files := filterLargeFiles(bucketSummaries)
	messageLog.Printf("[INFO] %d S3 Files are within our size limit\n",
		len(s3Files))
	filesToProcess := s3Files
	// SkipAlreadyProcessed will almost always be true.
	// The exception is when we want to reprocess items to test new code.
	if config.SkipAlreadyProcessed == true {
		messageLog.Println("[INFO] Skipping already processed files, because config says so")
		filesToProcess = filterProcessedFiles(s3Files)
	} else {
		messageLog.Println("[INFO] Reprocessing already processed files, because config says so")
	}
	start := 0
	end := min(len(filesToProcess), batchSize)
	messageLog.Printf("[INFO] %d Unprocessed files\n", len(filesToProcess))
	for start <= end {
		batch := filesToProcess[start:end]
		messageLog.Printf("[INFO] Queuing batch of %d items\n", len(batch))
		enqueue(url, batch)
		start = end + 1
		if start < len(filesToProcess) {
			end = min(len(filesToProcess), start+batchSize)
		}
		// Sleep so we don't max out connections on EC2 small.
		// The utility server is running a lot of other network I/O
		// in addition to our queue.
		time.Sleep(time.Millisecond * waitMilliseconds)
	}
}

// min returns the minimum of x or y. The Math package has this function
// but you have to cast to floats.
func min(x, y int) int {
	if x < y {
		return x
	} else {
		return y
	}
}

// filterLargeFiles returns only those S3 files that are not larger
// than config.MaxFileSize. This is useful when running tests and
// demos on your local machine, so that you can limit your test
// runs to files under 100k (or whatever you set in config.json).
// You don't want to pull down lots of multi-gig files when you're
// just running local tests. In production, set maxFileSize to
// zero, or to some huge value to get all files.
func filterLargeFiles(bucketSummaries []*bagman.BucketSummary) (s3Files []*bagman.S3File) {
	for _, bucketSummary := range bucketSummaries {
		for _, key := range bucketSummary.Keys {
			if config.MaxFileSize == 0 || key.Size < config.MaxFileSize {
				s3Files = append(s3Files, &bagman.S3File{bucketSummary.BucketName, key})
			}
		}
	}
	return s3Files
}

// Remove S3 files that have been processed successfully.
// No need to reprocess those!
func filterProcessedFiles(s3Files []*bagman.S3File) (filesToProcess []*bagman.S3File) {
	for _, s3File := range s3Files {
		bagDate, err := time.Parse(bagman.S3DateFormat, s3File.Key.LastModified)
		if err != nil {
			messageLog.Printf("[ERROR] Cannot parse S3File mod date '%s'. "+
				"File %s will be re-processed.",
				s3File.Key.LastModified, s3File.Key.Key)
			filesToProcess = append(filesToProcess, s3File)
			continue
		}
		etag := strings.Replace(s3File.Key.ETag, "\"", "", 2)
		status := findInStatusCache(etag, s3File.Key.Key, bagDate)
		if status == nil {
			status, err = fluctusClient.GetBagStatus(etag, s3File.Key.Key, bagDate)
		}
		if err != nil {
			messageLog.Printf("[ERROR] Cannot get Fluctus bag status for %s. "+
				"Will re-process bag. Error was %v", s3File.Key.Key, err)
			filesToProcess = append(filesToProcess, s3File)
		} else if  status == nil || status.Stage == "Receive" ||
			(status.Status == "Processing" && status.Retry == true) {
			reason := "Bag has not yet been successfully processed."
			if status == nil {
				err = createFluctusRecord(s3File)
				if err != nil {
					// TODO: Notify someone?
					messageLog.Printf("[ERROR] Could not create Fluctus ProcessedItem "+
						"for %s: %v", s3File.Key.Key, err)
				}
			}
			messageLog.Printf("[INFO] Will process bag %s: %s", s3File.Key.Key, reason)
			filesToProcess = append(filesToProcess, s3File)
		} else if status.Status != "Failed" {
			messageLog.Printf("[INFO] Skipping %s: already processed successfully.", s3File.Key.Key)
		} else if status.Retry == false {
			messageLog.Printf("[INFO] Skipping %s: retry flag is set to false.", s3File.Key.Key)
		}
	}
	return filesToProcess
}

// Loads status of all bags received in the past two hours from fluctus
// in a single call.
func loadStatusCache() {
	twoHoursAgo := time.Now().Add(time.Hour * -2)
	statusRecords, err := fluctusClient.BulkStatusGet(twoHoursAgo)
	if err != nil {
		messageLog.Println("[WARNING] Could not get bulk status records")
	} else {
		messageLog.Printf("[INFO] Got %d status records from the fluctopus\n", len(statusRecords))
		statusCache = make(map[string]*bagman.ProcessStatus, len(statusRecords))
		for i := range(statusRecords) {
			record := statusRecords[i]
			key := fmt.Sprintf("%s%s%s", record.ETag, record.Name, record.BagDate)
			statusCache[key] = record
		}
	}
}

// Finds the status of the specified tar bag in the cache that
// we retrieved from Fluctus. The cache can save us hundreds or
// thousands of HTTP calls each time the bucket reader runs.
func findInStatusCache(etag, name string, bagDate time.Time) (*bagman.ProcessStatus) {
	key := fmt.Sprintf("%s%s%s", etag, name, bagDate)
	item, exists := statusCache[key]
	if exists {
		messageLog.Printf("[INFO] Found item in cache: %s\n", name)
		return item
	}
	messageLog.Printf("[INFO] Item not in cache. Will have to ask the fluctopus for %s\n", name)
	return nil
}

func createFluctusRecord(s3File *bagman.S3File) (err error) {
	status := &bagman.ProcessStatus{}
	status.Date = time.Now().UTC()
	status.Action = "Ingest"
	status.Name = s3File.Key.Key
	bagDate, _ := time.Parse(bagman.S3DateFormat, s3File.Key.LastModified)
	status.BagDate = bagDate
	status.Bucket = s3File.BucketName
	// Strip the quotes off the ETag
	status.ETag = strings.Replace(s3File.Key.ETag, "\"", "", 2)
	status.Stage = "Receive"
	status.Status = "Succeeded"
	status.Note = "Item is in receiving bucket. Processing has not started."
	status.Institution = bagman.OwnerOf(s3File.BucketName)
	status.Outcome = status.Status

	// Retry should be true until we know for sure that there
	// is something wrong with the bag.
	status.Retry = true

	err = fluctusClient.UpdateBagStatus(status)
	if err != nil {
		return err
	}
	messageLog.Printf("[INFO] Created Fluctus ProcessedItem for %s\n",
		s3File.Key.Key)
	return nil
}

// enqueue adds a batch of items to the nsqd work queue
func enqueue(url string, s3Files []*bagman.S3File) {
	jsonData := make([]string, len(s3Files))
	for i, s3File := range s3Files {
		json, err := json.Marshal(s3File)
		if err != nil {
			messageLog.Printf("[ERROR] Error marshalling s3 file to JSON: %v", err)
		} else {
			jsonData[i] = string(json)
			messageLog.Println("[INFO]", "Put", s3File.Key.Key, "into fetch queue")
		}
	}
	batch := strings.Join(jsonData, "\n")
	resp, err := http.Post(url, "application/json", bytes.NewBuffer([]byte(batch)))
	if err != nil {
		messageLog.Printf("[ERROR] nsqd returned an error: %v", err)
	}
	if resp == nil {
		msg := "[ERROR] No response from nsqd. Is it running? bucket_reader is quitting."
		messageLog.Printf(msg)
		fmt.Println(msg)
		os.Exit(1)
	} else if resp.StatusCode != 200 {
		messageLog.Printf("[ERROR] nsqd returned status code %d on last mput", resp.StatusCode)
	}
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"strings"
	"time"

	"github.com/gempir/bitraft/pkg/log"
	"github.com/gempir/bitraft/pkg/slice"
	"github.com/labstack/echo/v4"
	nickHelix "github.com/nicklaw5/helix"
)

type Redemptions struct {
	Bttv Redemption
}

type Redemption struct {
	Title  string
	Active bool
}

const (
	bttvPrompt = "Add a BetterTTV emote! In the text field, send a link to the BetterTTV emote. powered by bitraft.gempir.com"
)

type channelPointRedemption struct {
	Subscription struct {
		ID        string `json:"id"`
		Status    string `json:"status"`
		Type      string `json:"type"`
		Version   string `json:"version"`
		Condition struct {
			BroadcasterUserID string `json:"broadcaster_user_id"`
			RewardID          string `json:"reward_id"`
		} `json:"condition"`
		Transport struct {
			Method   string `json:"method"`
			Callback string `json:"callback"`
		} `json:"transport"`
		CreatedAt time.Time `json:"created_at"`
		Cost      int       `json:"cost"`
	} `json:"subscription"`
	Event struct {
		BroadcasterUserID    string    `json:"broadcaster_user_id"`
		BroadcasterUserLogin string    `json:"broadcaster_user_login"`
		BroadcasterUserName  string    `json:"broadcaster_user_name"`
		ID                   string    `json:"id"`
		UserID               string    `json:"user_id"`
		UserLogin            string    `json:"user_login"`
		UserName             string    `json:"user_name"`
		UserInput            string    `json:"user_input"`
		Status               string    `json:"status"`
		RedeemedAt           time.Time `json:"redeemed_at"`
		Reward               struct {
			ID     string `json:"id"`
			Title  string `json:"title"`
			Prompt string `json:"prompt"`
			Cost   int    `json:"cost"`
		} `json:"reward"`
	} `json:"event"`
}

var bttvRegex = regexp.MustCompile(`https?:\/\/betterttv.com\/emotes\/(\w*)`)

func (s *Server) subscribeChannelPoints(userID string) error {
	response, err := s.helixClient.CreateChannelPointsRewardAdd(userID, s.cfg.WebhookApiBaseUrl+"/api/redemption")
	if err != nil {
		log.Errorf("Error subscribing: %s", err)
		return nil
	}

	if response.StatusCode == http.StatusForbidden {
		return errors.New("forbidden")
	}

	log.Infof("[%d] created subscription %s", response.StatusCode, response.Error)
	for _, sub := range response.Data.EventSubSubscriptions {
		log.Infof("new subscription for %s id: %s", userID, sub.ID)
		s.db.AddEventSubSubscription(userID, sub.ID, sub.Version)
	}

	return nil
}

func (s *Server) removeEventSubSubscription(userID string, subscriptionID string, reason string) error {
	response, err := s.helixClient.Client.RemoveEventSubSubscription(subscriptionID)
	if err != nil {
		return err
	}

	log.Infof("[%d] removed EventSubSubscription for %s reason: %s", response.StatusCode, userID, reason)
	s.db.RemoveEventSubSubscription(userID, subscriptionID)

	return nil
}

func (s *Server) syncSubscriptions() {
	resp, err := s.helixClient.Client.GetEventSubSubscriptions(&nickHelix.EventSubSubscriptionsParams{})
	if err != nil {
		log.Errorf("Failed to get subscriptions: %s", err)
		return
	}

	log.Infof("Found %d total subscriptions, syncing to DB", resp.Data.Total)
	subscribed := []string{}

	for _, sub := range resp.Data.EventSubSubscriptions {
		if !strings.Contains(sub.Transport.Callback, s.cfg.WebhookApiBaseUrl) || sub.Status == nickHelix.EventSubStatusFailed {
			err := s.removeEventSubSubscription(sub.Condition.BroadcasterUserID, sub.ID, "bad EventSub subscription, unsubscribing")
			if err != nil {
				log.Errorf("Failed to unsubscribe %s error: %s", sub.Condition.BroadcasterUserID, err.Error())
			}
			continue
		}

		_, err = s.db.GetEventSubSubscription(sub.Condition.BroadcasterUserID, sub.ID)
		if err != nil {
			log.Infof("Found unknown subscription, adding %s", sub.Condition.BroadcasterUserID)
			s.db.AddEventSubSubscription(sub.Condition.BroadcasterUserID, sub.ID, sub.Version)
		}

		subscribed = append(subscribed, sub.Condition.BroadcasterUserID)
	}

	rewards := s.db.GetDistinctRewardsPerUser()
	log.Infof("Found %d total distinct rewards, checking missing subscriptions", len(rewards))

	for _, dbReward := range rewards {
		if !slice.Contains(subscribed, dbReward.OwnerTwitchID) {
			log.Infof("Found no subscription for existing reward, creating subscription %s", dbReward.OwnerTwitchID)
			err := s.subscribeChannelPoints(dbReward.OwnerTwitchID)
			if err != nil {
				log.Infof("Removing reward for user %s because we didn't get permission to subscribe eventsub", dbReward.OwnerTwitchID)
				s.db.DeleteChannelPointReward(dbReward.OwnerTwitchID, dbReward.Type)
			}
		}
	}
}

func (s *Server) handleChannelPointsRedemption(c echo.Context) error {
	body, err := ioutil.ReadAll(c.Request().Body)
	if err != nil {
		log.Error(err)
		return echo.NewHTTPError(http.StatusBadRequest, "Failed reading body")
	}

	verified := nickHelix.VerifyEventSubNotification(s.cfg.Secret, c.Request().Header, string(body))
	if !verified {
		log.Errorf("Failed verification %s", c.Request().Header.Get("Twitch-Eventsub-Message-Id"))
		return echo.NewHTTPError(http.StatusPreconditionFailed, "failed verfication")
	}

	if c.Request().Header.Get("Twitch-Eventsub-Message-Type") == "webhook_callback_verification" {
		return s.handleChallenge(c, body)
	}

	var redemption channelPointRedemption
	err = json.Unmarshal(body, &redemption)
	if err != nil {
		return echo.NewHTTPError(http.StatusBadRequest, "Failed decoding body: "+err.Error())
	}

	err = s.handleRedemption(redemption)
	if err != nil {
		return c.NoContent(http.StatusInternalServerError)
	}

	return c.String(http.StatusOK, "success")
}

func (s *Server) handleRedemption(redemption channelPointRedemption) error {
	exists, err := s.store.Client.Exists("redemption:" + redemption.Event.ID).Result()
	if err != nil {
		log.Error(err)
		return err
	}
	if exists == 1 {
		log.Infof("Redemption already handled before, ignoring retry %s", redemption.Event.ID)
		return nil
	}

	s.store.Client.SetNX("redemption:"+redemption.Event.ID, "1", time.Minute*10)

	if redemption.Subscription.Version != "1" {
		log.Errorf("Unknown subscription version found %s %s", redemption.Subscription.Version, redemption.Subscription.ID)
		return nil
	}

	reward, err := s.db.GetEnabledChannelPointRewardByID(redemption.Event.Reward.ID)
	if err != nil {
		// no redemption found
		return nil
	}

	// Err is only returned when it's worth responding with a bad response code
	if reward.Type == TYPE_BTTV {
		err = s.handleBttvRedemption(redemption)
	}

	if err != nil {
		return err
	}

	return nil
}

func (s *Server) handleBttvRedemption(redemption channelPointRedemption) error {
	success := false

	matches := bttvRegex.FindAllStringSubmatch(redemption.Event.UserInput, -1)
	if len(matches) == 1 && len(matches[0]) == 2 {
		emoteAdded, emoteRemoved, err := s.emotechief.SetEmote(redemption.Event.BroadcasterUserID, matches[0][1], redemption.Event.BroadcasterUserLogin)
		if err != nil {
			log.Warnf("Bttv error %s %s", redemption.Event.BroadcasterUserLogin, err)
			s.store.PublishSpeakerMessage(redemption.Event.BroadcasterUserLogin, fmt.Sprintf("⚠ Failed to add emote from: @%s error: %s", redemption.Event.UserName, err.Error()))
		} else if emoteAdded != nil && emoteRemoved != nil {
			success = true
			s.store.PublishSpeakerMessage(redemption.Event.BroadcasterUserLogin, fmt.Sprintf("✅ Added new emote: %s redeemed by @%s removed: %s", emoteAdded.Code, redemption.Event.UserName, emoteRemoved.Code))
		} else if emoteAdded != nil {
			success = true
			s.store.PublishSpeakerMessage(redemption.Event.BroadcasterUserLogin, fmt.Sprintf("✅ Added new emote: %s redeemed by @%s", emoteAdded.Code, redemption.Event.UserName))
		} else {
			success = true
			s.store.PublishSpeakerMessage(redemption.Event.BroadcasterUserLogin, fmt.Sprintf("✅ Added new emote: [unknown] redeemed by @%s", redemption.Event.UserName))
		}
	} else {
		s.store.PublishSpeakerMessage(redemption.Event.BroadcasterUserLogin, fmt.Sprintf("⚠ Failed to add emote from @%s error: no bttv link found in message", redemption.Event.UserName))
	}

	token, err := s.db.GetUserAccessToken(redemption.Event.BroadcasterUserID)
	if err != nil {
		log.Errorf("Failed to get userAccess token to update redemption status for %s", redemption.Event.BroadcasterUserID)
		return nil
	} else {
		err := s.helixClient.UpdateRedemptionStatus(redemption.Event.BroadcasterUserID, token.AccessToken, redemption.Event.Reward.ID, redemption.Event.ID, success)
		if err != nil {
			log.Errorf("Failed to update redemption status %s", err.Error())
			return nil
		}
	}

	return nil
}

func (s *Server) handleChallenge(c echo.Context, body []byte) error {
	var event struct {
		Challenge string `json:"challenge"`
	}
	err := json.Unmarshal(body, &event)
	if err != nil {
		return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Failed to handle challenge: "+err.Error(), http.StatusBadRequest))
	}

	log.Infof("Challenge success: %s", event.Challenge)
	return c.String(http.StatusOK, event.Challenge)
}
<commit_msg>handle unknown versions<commit_after>package server

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"strings"
	"time"

	"github.com/gempir/bitraft/pkg/log"
	"github.com/gempir/bitraft/pkg/slice"
	"github.com/labstack/echo/v4"
	nickHelix "github.com/nicklaw5/helix"
)

type Redemptions struct {
	Bttv Redemption
}

type Redemption struct {
	Title  string
	Active bool
}

const (
	bttvPrompt = "Add a BetterTTV emote! In the text field, send a link to the BetterTTV emote. powered by bitraft.gempir.com"
)

type channelPointRedemption struct {
	Subscription struct {
		ID        string `json:"id"`
		Status    string `json:"status"`
		Type      string `json:"type"`
		Version   string `json:"version"`
		Condition struct {
			BroadcasterUserID string `json:"broadcaster_user_id"`
			RewardID          string `json:"reward_id"`
		} `json:"condition"`
		Transport struct {
			Method   string `json:"method"`
			Callback string `json:"callback"`
		} `json:"transport"`
		CreatedAt time.Time `json:"created_at"`
		Cost      int       `json:"cost"`
	} `json:"subscription"`
	Event struct {
		BroadcasterUserID    string    `json:"broadcaster_user_id"`
		BroadcasterUserLogin string    `json:"broadcaster_user_login"`
		BroadcasterUserName  string    `json:"broadcaster_user_name"`
		ID                   string    `json:"id"`
		UserID               string    `json:"user_id"`
		UserLogin            string    `json:"user_login"`
		UserName             string    `json:"user_name"`
		UserInput            string    `json:"user_input"`
		Status               string    `json:"status"`
		RedeemedAt           time.Time `json:"redeemed_at"`
		Reward               struct {
			ID     string `json:"id"`
			Title  string `json:"title"`
			Prompt string `json:"prompt"`
			Cost   int    `json:"cost"`
		} `json:"reward"`
	} `json:"event"`
}

var bttvRegex = regexp.MustCompile(`https?:\/\/betterttv.com\/emotes\/(\w*)`)

func (s *Server) subscribeChannelPoints(userID string) error {
	response, err := s.helixClient.CreateChannelPointsRewardAdd(userID, s.cfg.WebhookApiBaseUrl+"/api/redemption")
	if err != nil {
		log.Errorf("Error subscribing: %s", err)
		return nil
	}

	if response.StatusCode == http.StatusForbidden {
		return errors.New("forbidden")
	}

	log.Infof("[%d] created subscription %s", response.StatusCode, response.Error)
	for _, sub := range response.Data.EventSubSubscriptions {
		log.Infof("new subscription for %s id: %s", userID, sub.ID)
		s.db.AddEventSubSubscription(userID, sub.ID, sub.Version)
	}

	return nil
}

func (s *Server) removeEventSubSubscription(userID string, subscriptionID string, reason string) error {
	response, err := s.helixClient.Client.RemoveEventSubSubscription(subscriptionID)
	if err != nil {
		return err
	}

	log.Infof("[%d] removed EventSubSubscription for %s reason: %s", response.StatusCode, userID, reason)
	s.db.RemoveEventSubSubscription(userID, subscriptionID)

	return nil
}

func (s *Server) syncSubscriptions() {
	resp, err := s.helixClient.Client.GetEventSubSubscriptions(&nickHelix.EventSubSubscriptionsParams{})
	if err != nil {
		log.Errorf("Failed to get subscriptions: %s", err)
		return
	}

	log.Infof("Found %d total subscriptions, syncing to DB", resp.Data.Total)
	subscribed := []string{}

	for _, sub := range resp.Data.EventSubSubscriptions {
		if !strings.Contains(sub.Transport.Callback, s.cfg.WebhookApiBaseUrl) || sub.Status == nickHelix.EventSubStatusFailed {
			err := s.removeEventSubSubscription(sub.Condition.BroadcasterUserID, sub.ID, "bad EventSub subscription, unsubscribing")
			if err != nil {
				log.Errorf("Failed to unsubscribe %s error: %s", sub.Condition.BroadcasterUserID, err.Error())
			}
			continue
		}

		_, err = s.db.GetEventSubSubscription(sub.Condition.BroadcasterUserID, sub.ID)
		if err != nil {
			log.Infof("Found unknown subscription, adding %s", sub.Condition.BroadcasterUserID)
			s.db.AddEventSubSubscription(sub.Condition.BroadcasterUserID, sub.ID, sub.Version)
		}

		subscribed = append(subscribed, sub.Condition.BroadcasterUserID)
	}

	rewards := s.db.GetDistinctRewardsPerUser()
	log.Infof("Found %d total distinct rewards, checking missing subscriptions", len(rewards))

	for _, dbReward := range rewards {
		if !slice.Contains(subscribed, dbReward.OwnerTwitchID) {
			log.Infof("Found no subscription for existing reward, creating subscription %s", dbReward.OwnerTwitchID)
			err := s.subscribeChannelPoints(dbReward.OwnerTwitchID)
			if err != nil {
				log.Infof("Removing reward for user %s because we didn't get permission to subscribe eventsub", dbReward.OwnerTwitchID)
				s.db.DeleteChannelPointReward(dbReward.OwnerTwitchID, dbReward.Type)
			}
		}
	}
}

func (s *Server) handleChannelPointsRedemption(c echo.Context) error {
	body, err := ioutil.ReadAll(c.Request().Body)
	if err != nil {
		log.Error(err)
		return echo.NewHTTPError(http.StatusBadRequest, "Failed reading body")
	}

	verified := nickHelix.VerifyEventSubNotification(s.cfg.Secret, c.Request().Header, string(body))
	if !verified {
		log.Errorf("Failed verification %s", c.Request().Header.Get("Twitch-Eventsub-Message-Id"))
		return echo.NewHTTPError(http.StatusPreconditionFailed, "failed verfication")
	}

	if c.Request().Header.Get("Twitch-Eventsub-Message-Type") == "webhook_callback_verification" {
		return s.handleChallenge(c, body)
	}

	var redemption channelPointRedemption
	err = json.Unmarshal(body, &redemption)
	if err != nil {
		return echo.NewHTTPError(http.StatusBadRequest, "Failed decoding body: "+err.Error())
	}

	err = s.handleRedemption(redemption)
	if err != nil {
		return c.NoContent(http.StatusInternalServerError)
	}

	return c.String(http.StatusOK, "success")
}

func (s *Server) handleRedemption(redemption channelPointRedemption) error {
	exists, err := s.store.Client.Exists("redemption:" + redemption.Event.ID).Result()
	if err != nil {
		log.Error(err)
		return err
	}
	if exists == 1 {
		log.Infof("Redemption already handled before, ignoring retry %s", redemption.Event.ID)
		return nil
	}

	s.store.Client.SetNX("redemption:"+redemption.Event.ID, "1", time.Minute*10)

	if redemption.Subscription.Version != "1" && redemption.Subscription.Version != "" {
		log.Errorf("Unknown subscription version found %s %s", redemption.Subscription.Version, redemption.Subscription.ID)
		return nil
	}

	reward, err := s.db.GetEnabledChannelPointRewardByID(redemption.Event.Reward.ID)
	if err != nil {
		// no redemption found
		return nil
	}

	// Err is only returned when it's worth responding with a bad response code
	if reward.Type == TYPE_BTTV {
		err = s.handleBttvRedemption(redemption)
	}

	if err != nil {
		return err
	}

	return nil
}

func (s *Server) handleBttvRedemption(redemption channelPointRedemption) error {
	success := false

	matches := bttvRegex.FindAllStringSubmatch(redemption.Event.UserInput, -1)
	if len(matches) == 1 && len(matches[0]) == 2 {
		emoteAdded, emoteRemoved, err := s.emotechief.SetEmote(redemption.Event.BroadcasterUserID, matches[0][1], redemption.Event.BroadcasterUserLogin)
		if err != nil {
			log.Warnf("Bttv error %s %s", redemption.Event.BroadcasterUserLogin, err)
			s.store.PublishSpeakerMessage(redemption.Event.BroadcasterUserLogin, fmt.Sprintf("⚠ Failed to add emote from: @%s error: %s", redemption.Event.UserName, err.Error()))
		} else if emoteAdded != nil && emoteRemoved != nil {
			success = true
			s.store.PublishSpeakerMessage(redemption.Event.BroadcasterUserLogin, fmt.Sprintf("✅ Added new emote: %s redeemed by @%s removed: %s", emoteAdded.Code, redemption.Event.UserName, emoteRemoved.Code))
		} else if emoteAdded != nil {
			success = true
			s.store.PublishSpeakerMessage(redemption.Event.BroadcasterUserLogin, fmt.Sprintf("✅ Added new emote: %s redeemed by @%s", emoteAdded.Code, redemption.Event.UserName))
		} else {
			success = true
			s.store.PublishSpeakerMessage(redemption.Event.BroadcasterUserLogin, fmt.Sprintf("✅ Added new emote: [unknown] redeemed by @%s", redemption.Event.UserName))
		}
	} else {
		s.store.PublishSpeakerMessage(redemption.Event.BroadcasterUserLogin, fmt.Sprintf("⚠ Failed to add emote from @%s error: no bttv link found in message", redemption.Event.UserName))
	}

	token, err := s.db.GetUserAccessToken(redemption.Event.BroadcasterUserID)
	if err != nil {
		log.Errorf("Failed to get userAccess token to update redemption status for %s", redemption.Event.BroadcasterUserID)
		return nil
	} else {
		err := s.helixClient.UpdateRedemptionStatus(redemption.Event.BroadcasterUserID, token.AccessToken, redemption.Event.Reward.ID, redemption.Event.ID, success)
		if err != nil {
			log.Errorf("Failed to update redemption status %s", err.Error())
			return nil
		}
	}

	return nil
}

func (s *Server) handleChallenge(c echo.Context, body []byte) error {
	var event struct {
		Challenge string `json:"challenge"`
	}
	err := json.Unmarshal(body, &event)
	if err != nil {
		return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Failed to handle challenge: "+err.Error(), http.StatusBadRequest))
	}

	log.Infof("Challenge success: %s", event.Challenge)
	return c.String(http.StatusOK, event.Challenge)
}
<|endoftext|>