<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package provides basic interfaces to I/O primitives.
// Its primary job is to wrap existing implementations of such primitives,
// such as those in package os, into shared public interfaces that
// abstract the functionality, plus some other related primitives.
package io

import "os"

// Error represents an unexpected I/O behavior.
type Error struct {
	os.ErrorString
}

// ErrShortWrite means that a write accepted fewer bytes than requested
// but failed to return an explicit error.
var ErrShortWrite os.Error = &Error{"short write"}

// ErrUnexpectedEOF means that os.EOF was encountered in the
// middle of reading a fixed-size block or data structure.
var ErrUnexpectedEOF os.Error = &Error{"unexpected EOF"}

// Reader is the interface that wraps the basic Read method.
//
// Read reads up to len(p) bytes into p.  It returns the number of bytes
// read (0 <= n <= len(p)) and any error encountered.
// Even if Read returns n < len(p),
// it may use all of p as scratch space during the call.
// If some data is available but not len(p) bytes, Read conventionally
// returns what is available rather than block waiting for more.
//
// At the end of the input stream, Read returns 0, os.EOF.
// Read may return a non-zero number of bytes with a non-nil err.
// In particular, a Read that exhausts the input may return n > 0, os.EOF.
type Reader interface {
	Read(p []byte) (n int, err os.Error)
}

// Writer is the interface that wraps the basic Write method.
//
// Write writes len(p) bytes from p to the underlying data stream.
// It returns the number of bytes written from p (0 <= n <= len(p))
// and any error encountered that caused the write to stop early.
// Write must return a non-nil error if it returns n < len(p).
type Writer interface {
	Write(p []byte) (n int, err os.Error)
}

// Closer is the interface that wraps the basic Close method.
type Closer interface {
	Close() os.Error
}

// Seeker is the interface that wraps the basic Seek method.
//
// Seek sets the offset for the next Read or Write to offset,
// interpreted according to whence: 0 means relative to the origin of
// the file, 1 means relative to the current offset, and 2 means
// relative to the end.  Seek returns the new offset and an Error, if
// any.
type Seeker interface {
	Seek(offset int64, whence int) (ret int64, err os.Error)
}

// ReadWriter is the interface that groups the basic Read and Write methods.
type ReadWriter interface {
	Reader
	Writer
}

// ReadCloser is the interface that groups the basic Read and Close methods.
type ReadCloser interface {
	Reader
	Closer
}

// WriteCloser is the interface that groups the basic Write and Close methods.
type WriteCloser interface {
	Writer
	Closer
}

// ReadWriteCloser is the interface that groups the basic Read, Write and Close methods.
type ReadWriteCloser interface {
	Reader
	Writer
	Closer
}

// ReadSeeker is the interface that groups the basic Read and Seek methods.
type ReadSeeker interface {
	Reader
	Seeker
}

// WriteSeeker is the interface that groups the basic Write and Seek methods.
type WriteSeeker interface {
	Writer
	Seeker
}

// ReadWriteSeeker is the interface that groups the basic Read, Write and Seek methods.
type ReadWriteSeeker interface {
	Reader
	Writer
	Seeker
}

// ReaderFrom is the interface that wraps the ReadFrom method.
type ReaderFrom interface {
	ReadFrom(r Reader) (n int64, err os.Error)
}

// WriterTo is the interface that wraps the WriteTo method.
type WriterTo interface {
	WriteTo(w Writer) (n int64, err os.Error)
}

// ReaderAt is the interface that wraps the basic ReadAt method.
//
// ReadAt reads len(p) bytes into p starting at offset off in the
// underlying data stream.  It returns the number of bytes
// read (0 <= n <= len(p)) and any error encountered.
//
// Even if ReadAt returns n < len(p),
// it may use all of p as scratch space during the call.
// If some data is available but not len(p) bytes, ReadAt blocks
// until either all the data is available or an error occurs.
//
// At the end of the input stream, ReadAt returns 0, os.EOF.
// ReadAt may return a non-zero number of bytes with a non-nil err.
// In particular, a ReadAt that exhausts the input may return n > 0, os.EOF.
type ReaderAt interface {
	ReadAt(p []byte, off int64) (n int, err os.Error)
}

// WriterAt is the interface that wraps the basic WriteAt method.
//
// WriteAt writes len(p) bytes from p to the underlying data stream
// at offset off.  It returns the number of bytes written from p (0 <= n <= len(p))
// and any error encountered that caused the write to stop early.
// WriteAt must return a non-nil error if it returns n < len(p).
type WriterAt interface {
	WriteAt(p []byte, off int64) (n int, err os.Error)
}

// ReadByter is the interface that wraps the ReadByte method.
//
// ReadByte reads and returns the next byte from the input.
// If no byte is available, err will be set.
type ReadByter interface {
	ReadByte() (c byte, err os.Error)
}

// WriteString writes the contents of the string s to w, which accepts an array of bytes.
func WriteString(w Writer, s string) (n int, err os.Error) {
	return w.Write([]byte(s))
}

// ReadAtLeast reads from r into buf until it has read at least min bytes.
// It returns the number of bytes copied and an error if fewer bytes were read.
// The error is os.EOF only if no bytes were read.
// If an EOF happens after reading fewer than min bytes,
// ReadAtLeast returns ErrUnexpectedEOF.
func ReadAtLeast(r Reader, buf []byte, min int) (n int, err os.Error) {
	n = 0
	for n < min {
		nn, e := r.Read(buf[n:])
		if nn > 0 {
			n += nn
		}
		if e != nil {
			if e == os.EOF && n > 0 {
				e = ErrUnexpectedEOF
			}
			return n, e
		}
	}
	return n, nil
}

// ReadFull reads exactly len(buf) bytes from r into buf.
// It returns the number of bytes copied and an error if fewer bytes were read.
// The error is os.EOF only if no bytes were read.
// If an EOF happens after reading some but not all the bytes,
// ReadFull returns ErrUnexpectedEOF.
func ReadFull(r Reader, buf []byte) (n int, err os.Error) {
	return ReadAtLeast(r, buf, len(buf))
}

// Copyn copies n bytes (or until an error) from src to dst.
// It returns the number of bytes copied and the error, if any.
//
// If dst implements the ReaderFrom interface,
// the copy is implemented by calling dst.ReadFrom(src).
func Copyn(dst Writer, src Reader, n int64) (written int64, err os.Error) {
	// If the writer has a ReadFrom method, use it to to do the copy.
	// Avoids a buffer allocation and a copy.
	if rt, ok := dst.(ReaderFrom); ok {
		return rt.ReadFrom(LimitReader(src, n))
	}
	buf := make([]byte, 32*1024)
	for written < n {
		l := len(buf)
		if d := n - written; d < int64(l) {
			l = int(d)
		}
		nr, er := src.Read(buf[0:l])
		if nr > 0 {
			nw, ew := dst.Write(buf[0:nr])
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
				break
			}
			if nr != nw {
				err = ErrShortWrite
				break
			}
		}
		if er != nil {
			err = er
			break
		}
	}
	return written, err
}

// Copy copies from src to dst until either EOF is reached
// on src or an error occurs.  It returns the number of bytes
// copied and the error, if any.
//
// If dst implements the ReaderFrom interface,
// the copy is implemented by calling dst.ReadFrom(src).
// Otherwise, if src implements the WriterTo interface,
// the copy is implemented by calling src.WriteTo(dst).
func Copy(dst Writer, src Reader) (written int64, err os.Error) {
	// If the writer has a ReadFrom method, use it to to do the copy.
	// Avoids an allocation and a copy.
	if rt, ok := dst.(ReaderFrom); ok {
		return rt.ReadFrom(src)
	}
	// Similarly, if the reader has a WriteTo method, use it to to do the copy.
	if wt, ok := src.(WriterTo); ok {
		return wt.WriteTo(dst)
	}
	buf := make([]byte, 32*1024)
	for {
		nr, er := src.Read(buf)
		if nr > 0 {
			nw, ew := dst.Write(buf[0:nr])
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
				break
			}
			if nr != nw {
				err = ErrShortWrite
				break
			}
		}
		if er == os.EOF {
			break
		}
		if er != nil {
			err = er
			break
		}
	}
	return written, err
}

// LimitReader returns a Reader that reads from r
// but stops with os.EOF after n bytes.
func LimitReader(r Reader, n int64) Reader { return &limitedReader{r, n} }

type limitedReader struct {
	r Reader
	n int64
}

func (l *limitedReader) Read(p []byte) (n int, err os.Error) {
	if l.n <= 0 {
		return 0, os.EOF
	}
	if int64(len(p)) > l.n {
		p = p[0:l.n]
	}
	n, err = l.r.Read(p)
	l.n -= int64(n)
	return
}

// NewSectionReader returns a SectionReader that reads from r
// starting at offset off and stops with os.EOF after n bytes.
func NewSectionReader(r ReaderAt, off int64, n int64) *SectionReader {
	return &SectionReader{r, off, off, off + n}
}

// SectionReader implements Read, Seek, and ReadAt on a section
// of an underlying ReaderAt.
type SectionReader struct {
	r     ReaderAt
	base  int64
	off   int64
	limit int64
}

func (s *SectionReader) Read(p []byte) (n int, err os.Error) {
	if s.off >= s.limit {
		return 0, os.EOF
	}
	if max := s.limit - s.off; int64(len(p)) > max {
		p = p[0:max]
	}
	n, err = s.r.ReadAt(p, s.off)
	s.off += int64(n)
	return
}

func (s *SectionReader) Seek(offset int64, whence int) (ret int64, err os.Error) {
	switch whence {
	default:
		return 0, os.EINVAL
	case 0:
		offset += s.base
	case 1:
		offset += s.off
	case 2:
		offset += s.limit
	}
	if offset < s.off || offset > s.limit {
		return 0, os.EINVAL
	}
	s.off = offset
	return offset - s.base, nil
}

func (s *SectionReader) ReadAt(p []byte, off int64) (n int, err os.Error) {
	if off < 0 || off >= s.limit-s.base {
		return 0, os.EOF
	}
	off += s.base
	if max := s.limit - off; int64(len(p)) > max {
		p = p[0:max]
	}
	return s.r.ReadAt(p, off)
}

// Size returns the size of the section in bytes.
func (s *SectionReader) Size() int64 { return s.limit - s.base }
<commit_msg>io: fix SectionReader Seek to seek backwards<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package provides basic interfaces to I/O primitives.
// Its primary job is to wrap existing implementations of such primitives,
// such as those in package os, into shared public interfaces that
// abstract the functionality, plus some other related primitives.
package io

import "os"

// Error represents an unexpected I/O behavior.
type Error struct {
	os.ErrorString
}

// ErrShortWrite means that a write accepted fewer bytes than requested
// but failed to return an explicit error.
var ErrShortWrite os.Error = &Error{"short write"}

// ErrUnexpectedEOF means that os.EOF was encountered in the
// middle of reading a fixed-size block or data structure.
var ErrUnexpectedEOF os.Error = &Error{"unexpected EOF"}

// Reader is the interface that wraps the basic Read method.
//
// Read reads up to len(p) bytes into p.  It returns the number of bytes
// read (0 <= n <= len(p)) and any error encountered.
// Even if Read returns n < len(p),
// it may use all of p as scratch space during the call.
// If some data is available but not len(p) bytes, Read conventionally
// returns what is available rather than block waiting for more.
//
// At the end of the input stream, Read returns 0, os.EOF.
// Read may return a non-zero number of bytes with a non-nil err.
// In particular, a Read that exhausts the input may return n > 0, os.EOF.
type Reader interface {
	Read(p []byte) (n int, err os.Error)
}

// Writer is the interface that wraps the basic Write method.
//
// Write writes len(p) bytes from p to the underlying data stream.
// It returns the number of bytes written from p (0 <= n <= len(p))
// and any error encountered that caused the write to stop early.
// Write must return a non-nil error if it returns n < len(p).
type Writer interface {
	Write(p []byte) (n int, err os.Error)
}

// Closer is the interface that wraps the basic Close method.
type Closer interface {
	Close() os.Error
}

// Seeker is the interface that wraps the basic Seek method.
//
// Seek sets the offset for the next Read or Write to offset,
// interpreted according to whence: 0 means relative to the origin of
// the file, 1 means relative to the current offset, and 2 means
// relative to the end.  Seek returns the new offset and an Error, if
// any.
type Seeker interface {
	Seek(offset int64, whence int) (ret int64, err os.Error)
}

// ReadWriter is the interface that groups the basic Read and Write methods.
type ReadWriter interface {
	Reader
	Writer
}

// ReadCloser is the interface that groups the basic Read and Close methods.
type ReadCloser interface {
	Reader
	Closer
}

// WriteCloser is the interface that groups the basic Write and Close methods.
type WriteCloser interface {
	Writer
	Closer
}

// ReadWriteCloser is the interface that groups the basic Read, Write and Close methods.
type ReadWriteCloser interface {
	Reader
	Writer
	Closer
}

// ReadSeeker is the interface that groups the basic Read and Seek methods.
type ReadSeeker interface {
	Reader
	Seeker
}

// WriteSeeker is the interface that groups the basic Write and Seek methods.
type WriteSeeker interface {
	Writer
	Seeker
}

// ReadWriteSeeker is the interface that groups the basic Read, Write and Seek methods.
type ReadWriteSeeker interface {
	Reader
	Writer
	Seeker
}

// ReaderFrom is the interface that wraps the ReadFrom method.
type ReaderFrom interface {
	ReadFrom(r Reader) (n int64, err os.Error)
}

// WriterTo is the interface that wraps the WriteTo method.
type WriterTo interface {
	WriteTo(w Writer) (n int64, err os.Error)
}

// ReaderAt is the interface that wraps the basic ReadAt method.
//
// ReadAt reads len(p) bytes into p starting at offset off in the
// underlying data stream.  It returns the number of bytes
// read (0 <= n <= len(p)) and any error encountered.
//
// Even if ReadAt returns n < len(p),
// it may use all of p as scratch space during the call.
// If some data is available but not len(p) bytes, ReadAt blocks
// until either all the data is available or an error occurs.
//
// At the end of the input stream, ReadAt returns 0, os.EOF.
// ReadAt may return a non-zero number of bytes with a non-nil err.
// In particular, a ReadAt that exhausts the input may return n > 0, os.EOF.
type ReaderAt interface {
	ReadAt(p []byte, off int64) (n int, err os.Error)
}

// WriterAt is the interface that wraps the basic WriteAt method.
//
// WriteAt writes len(p) bytes from p to the underlying data stream
// at offset off.  It returns the number of bytes written from p (0 <= n <= len(p))
// and any error encountered that caused the write to stop early.
// WriteAt must return a non-nil error if it returns n < len(p).
type WriterAt interface {
	WriteAt(p []byte, off int64) (n int, err os.Error)
}

// ReadByter is the interface that wraps the ReadByte method.
//
// ReadByte reads and returns the next byte from the input.
// If no byte is available, err will be set.
type ReadByter interface {
	ReadByte() (c byte, err os.Error)
}

// WriteString writes the contents of the string s to w, which accepts an array of bytes.
func WriteString(w Writer, s string) (n int, err os.Error) {
	return w.Write([]byte(s))
}

// ReadAtLeast reads from r into buf until it has read at least min bytes.
// It returns the number of bytes copied and an error if fewer bytes were read.
// The error is os.EOF only if no bytes were read.
// If an EOF happens after reading fewer than min bytes,
// ReadAtLeast returns ErrUnexpectedEOF.
func ReadAtLeast(r Reader, buf []byte, min int) (n int, err os.Error) {
	n = 0
	for n < min {
		nn, e := r.Read(buf[n:])
		if nn > 0 {
			n += nn
		}
		if e != nil {
			if e == os.EOF && n > 0 {
				e = ErrUnexpectedEOF
			}
			return n, e
		}
	}
	return n, nil
}

// ReadFull reads exactly len(buf) bytes from r into buf.
// It returns the number of bytes copied and an error if fewer bytes were read.
// The error is os.EOF only if no bytes were read.
// If an EOF happens after reading some but not all the bytes,
// ReadFull returns ErrUnexpectedEOF.
func ReadFull(r Reader, buf []byte) (n int, err os.Error) {
	return ReadAtLeast(r, buf, len(buf))
}

// Copyn copies n bytes (or until an error) from src to dst.
// It returns the number of bytes copied and the error, if any.
//
// If dst implements the ReaderFrom interface,
// the copy is implemented by calling dst.ReadFrom(src).
func Copyn(dst Writer, src Reader, n int64) (written int64, err os.Error) {
	// If the writer has a ReadFrom method, use it to to do the copy.
	// Avoids a buffer allocation and a copy.
	if rt, ok := dst.(ReaderFrom); ok {
		return rt.ReadFrom(LimitReader(src, n))
	}
	buf := make([]byte, 32*1024)
	for written < n {
		l := len(buf)
		if d := n - written; d < int64(l) {
			l = int(d)
		}
		nr, er := src.Read(buf[0:l])
		if nr > 0 {
			nw, ew := dst.Write(buf[0:nr])
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
				break
			}
			if nr != nw {
				err = ErrShortWrite
				break
			}
		}
		if er != nil {
			err = er
			break
		}
	}
	return written, err
}

// Copy copies from src to dst until either EOF is reached
// on src or an error occurs.  It returns the number of bytes
// copied and the error, if any.
//
// If dst implements the ReaderFrom interface,
// the copy is implemented by calling dst.ReadFrom(src).
// Otherwise, if src implements the WriterTo interface,
// the copy is implemented by calling src.WriteTo(dst).
func Copy(dst Writer, src Reader) (written int64, err os.Error) {
	// If the writer has a ReadFrom method, use it to to do the copy.
	// Avoids an allocation and a copy.
	if rt, ok := dst.(ReaderFrom); ok {
		return rt.ReadFrom(src)
	}
	// Similarly, if the reader has a WriteTo method, use it to to do the copy.
	if wt, ok := src.(WriterTo); ok {
		return wt.WriteTo(dst)
	}
	buf := make([]byte, 32*1024)
	for {
		nr, er := src.Read(buf)
		if nr > 0 {
			nw, ew := dst.Write(buf[0:nr])
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
				break
			}
			if nr != nw {
				err = ErrShortWrite
				break
			}
		}
		if er == os.EOF {
			break
		}
		if er != nil {
			err = er
			break
		}
	}
	return written, err
}

// LimitReader returns a Reader that reads from r
// but stops with os.EOF after n bytes.
func LimitReader(r Reader, n int64) Reader { return &limitedReader{r, n} }

type limitedReader struct {
	r Reader
	n int64
}

func (l *limitedReader) Read(p []byte) (n int, err os.Error) {
	if l.n <= 0 {
		return 0, os.EOF
	}
	if int64(len(p)) > l.n {
		p = p[0:l.n]
	}
	n, err = l.r.Read(p)
	l.n -= int64(n)
	return
}

// NewSectionReader returns a SectionReader that reads from r
// starting at offset off and stops with os.EOF after n bytes.
func NewSectionReader(r ReaderAt, off int64, n int64) *SectionReader {
	return &SectionReader{r, off, off, off + n}
}

// SectionReader implements Read, Seek, and ReadAt on a section
// of an underlying ReaderAt.
type SectionReader struct {
	r     ReaderAt
	base  int64
	off   int64
	limit int64
}

func (s *SectionReader) Read(p []byte) (n int, err os.Error) {
	if s.off >= s.limit {
		return 0, os.EOF
	}
	if max := s.limit - s.off; int64(len(p)) > max {
		p = p[0:max]
	}
	n, err = s.r.ReadAt(p, s.off)
	s.off += int64(n)
	return
}

func (s *SectionReader) Seek(offset int64, whence int) (ret int64, err os.Error) {
	switch whence {
	default:
		return 0, os.EINVAL
	case 0:
		offset += s.base
	case 1:
		offset += s.off
	case 2:
		offset += s.limit
	}
	if offset < s.base || offset > s.limit {
		return 0, os.EINVAL
	}
	s.off = offset
	return offset - s.base, nil
}

func (s *SectionReader) ReadAt(p []byte, off int64) (n int, err os.Error) {
	if off < 0 || off >= s.limit-s.base {
		return 0, os.EOF
	}
	off += s.base
	if max := s.limit - off; int64(len(p)) > max {
		p = p[0:max]
	}
	return s.r.ReadAt(p, off)
}

// Size returns the size of the section in bytes.
func (s *SectionReader) Size() int64 { return s.limit - s.base }
<|endoftext|>


<commit_before>package model

import (
	"context"
	"database/sql"
	"fmt"
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"go.opentelemetry.io/otel"

	"github.com/oinume/lekcije/backend/domain/config"
	"github.com/oinume/lekcije/backend/errors"
)

type User struct {
	ID                 uint32 `gorm:"primary_key;AUTO_INCREMENT"`
	Name               string
	Email              string
	EmailVerified      bool
	PlanID             uint8
	FollowedTeacherAt  sql.NullTime
	OpenNotificationAt sql.NullTime
	CreatedAt          time.Time
	UpdatedAt          time.Time
}

func (*User) TableName() string {
	return "user"
}

func (u *User) IsFollowedTeacher() bool {
	return u.FollowedTeacherAt.Valid && !u.FollowedTeacherAt.Time.IsZero()
}

type UserService struct {
	db *gorm.DB
}

func NewUserService(db *gorm.DB) *UserService {
	return &UserService{db: db}
}

func (s *UserService) TableName() string {
	return (&User{}).TableName()
}

func (s *UserService) FindByPK(id uint32) (*User, error) {
	user := &User{}
	if result := s.db.First(user, &User{ID: id}); result.Error != nil {
		if err := wrapNotFound(result, user.TableName(), "id", fmt.Sprint(id)); err != nil {
			return nil, err
		}
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithResource(errors.NewResource(user.TableName(), "id", id)),
		)
	}
	if err := s.db.First(user, &User{ID: id}).Error; err != nil {
		return nil, err
	}
	return user, nil
}

// FindAllEmailVerifiedIsTrue returns an empty slice if no users found
func (s *UserService) FindAllEmailVerifiedIsTrue(ctx context.Context, notificationInterval int) ([]*User, error) {
	_, span := otel.Tracer(config.DefaultTracerName).Start(ctx, "UserService.FindAllEmailVerifiedIsTrue")
	defer span.End()

	var users []*User
	sql := `
	SELECT u.* FROM (SELECT DISTINCT(user_id) FROM following_teacher) AS ft
	INNER JOIN user AS u ON ft.user_id = u.id
	INNER JOIN m_plan AS mp ON u.plan_id = mp.id
	WHERE
	  u.email_verified = 1
	  AND mp.notification_interval = ?
	ORDER BY u.open_notification_at DESC
	`
	result := s.db.Raw(strings.TrimSpace(sql), notificationInterval).Scan(&users)
	if result.Error != nil && !result.RecordNotFound() {
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithMessage("Failed to find Users"),
		)
	}
	return users, nil
}

// FindAllFollowedTeacherAtIsNull returns an empty slice if no users found
func (s *UserService) FindAllFollowedTeacherAtIsNull(createdAt time.Time) ([]*User, error) {
	var users []*User
	sql := `SELECT * FROM user WHERE followed_teacher_at IS NULL AND CAST(created_at AS DATE) = ? ORDER BY id`
	result := s.db.Raw(sql, createdAt.Format(dbDateFormat)).Scan(&users)
	if result.Error != nil && !result.RecordNotFound() {
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithMessage("Failed to find users"),
		)
	}
	return users, nil
}

func (s *UserService) Create(name, email string) (*User, error) {
	user := &User{
		Name:          name,
		Email:         email,
		EmailVerified: true,
		PlanID:        DefaultMPlanID,
	}
	if result := s.db.Create(user); result.Error != nil {
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithMessage("Failed to Create user"),
			errors.WithResource(errors.NewResource("user", "email", email)),
		)
	}
	return user, nil
}

func (s *UserService) UpdateFollowedTeacherAt(user *User) error { // TODO: delete
	sql := "UPDATE user SET followed_teacher_at = NOW() WHERE id = ?"
	if err := s.db.Exec(sql, user.ID).Error; err != nil {
		return errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessage("Failed to update followed_teacher_at"),
			errors.WithResource(errors.NewResource(user.TableName(), "id", user.ID)),
		)
	}
	return nil
}

func (s *UserService) UpdateOpenNotificationAt(userID uint32, t time.Time) error {
	sql := "UPDATE user SET open_notification_at = ? WHERE id = ?"
	if err := s.db.Exec(sql, t.Format(dbDatetimeFormat), userID).Error; err != nil {
		return errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessage("Failed to update open_notification_at"),
			errors.WithResource(errors.NewResource((&User{}).TableName(), "id", userID)),
		)
	}
	return nil
}

func (s *UserService) FindLoggedInUser(token string) (*User, error) {
	user := &User{}
	sql := `
		SELECT * FROM user AS u
		INNER JOIN user_api_token AS uat ON u.id = uat.user_id
		WHERE uat.token = ?
		`
	result := s.db.Model(&User{}).Raw(strings.TrimSpace(sql), token).Scan(user)
	if result.Error != nil {
		if err := wrapNotFound(result, user.TableName(), "token", token); err != nil {
			return nil, err
		}
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithResource(errors.NewResource(user.TableName(), "token", token)),
		)
	}
	return user, nil
}
<commit_msg>Remove unused methods from model.UserService<commit_after>package model

import (
	"context"
	"database/sql"
	"fmt"
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"go.opentelemetry.io/otel"

	"github.com/oinume/lekcije/backend/domain/config"
	"github.com/oinume/lekcije/backend/errors"
)

type User struct {
	ID                 uint32 `gorm:"primary_key;AUTO_INCREMENT"`
	Name               string
	Email              string
	EmailVerified      bool
	PlanID             uint8
	FollowedTeacherAt  sql.NullTime
	OpenNotificationAt sql.NullTime
	CreatedAt          time.Time
	UpdatedAt          time.Time
}

func (*User) TableName() string {
	return "user"
}

func (u *User) IsFollowedTeacher() bool {
	return u.FollowedTeacherAt.Valid && !u.FollowedTeacherAt.Time.IsZero()
}

type UserService struct {
	db *gorm.DB
}

func NewUserService(db *gorm.DB) *UserService {
	return &UserService{db: db}
}

func (s *UserService) TableName() string {
	return (&User{}).TableName()
}

func (s *UserService) FindByPK(id uint32) (*User, error) {
	user := &User{}
	if result := s.db.First(user, &User{ID: id}); result.Error != nil {
		if err := wrapNotFound(result, user.TableName(), "id", fmt.Sprint(id)); err != nil {
			return nil, err
		}
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithResource(errors.NewResource(user.TableName(), "id", id)),
		)
	}
	if err := s.db.First(user, &User{ID: id}).Error; err != nil {
		return nil, err
	}
	return user, nil
}

// FindAllEmailVerifiedIsTrue returns an empty slice if no users found
func (s *UserService) FindAllEmailVerifiedIsTrue(ctx context.Context, notificationInterval int) ([]*User, error) {
	_, span := otel.Tracer(config.DefaultTracerName).Start(ctx, "UserService.FindAllEmailVerifiedIsTrue")
	defer span.End()

	var users []*User
	sql := `
	SELECT u.* FROM (SELECT DISTINCT(user_id) FROM following_teacher) AS ft
	INNER JOIN user AS u ON ft.user_id = u.id
	INNER JOIN m_plan AS mp ON u.plan_id = mp.id
	WHERE
	  u.email_verified = 1
	  AND mp.notification_interval = ?
	ORDER BY u.open_notification_at DESC
	`
	result := s.db.Raw(strings.TrimSpace(sql), notificationInterval).Scan(&users)
	if result.Error != nil && !result.RecordNotFound() {
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithMessage("Failed to find Users"),
		)
	}
	return users, nil
}

// FindAllFollowedTeacherAtIsNull returns an empty slice if no users found
func (s *UserService) FindAllFollowedTeacherAtIsNull(createdAt time.Time) ([]*User, error) {
	var users []*User
	sql := `SELECT * FROM user WHERE followed_teacher_at IS NULL AND CAST(created_at AS DATE) = ? ORDER BY id`
	result := s.db.Raw(sql, createdAt.Format(dbDateFormat)).Scan(&users)
	if result.Error != nil && !result.RecordNotFound() {
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithMessage("Failed to find users"),
		)
	}
	return users, nil
}

func (s *UserService) Create(name, email string) (*User, error) {
	user := &User{
		Name:          name,
		Email:         email,
		EmailVerified: true,
		PlanID:        DefaultMPlanID,
	}
	if result := s.db.Create(user); result.Error != nil {
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithMessage("Failed to Create user"),
			errors.WithResource(errors.NewResource("user", "email", email)),
		)
	}
	return user, nil
}

func (s *UserService) UpdateOpenNotificationAt(userID uint32, t time.Time) error {
	sql := "UPDATE user SET open_notification_at = ? WHERE id = ?"
	if err := s.db.Exec(sql, t.Format(dbDatetimeFormat), userID).Error; err != nil {
		return errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessage("Failed to update open_notification_at"),
			errors.WithResource(errors.NewResource((&User{}).TableName(), "id", userID)),
		)
	}
	return nil
}

func (s *UserService) FindLoggedInUser(token string) (*User, error) {
	user := &User{}
	sql := `
		SELECT * FROM user AS u
		INNER JOIN user_api_token AS uat ON u.id = uat.user_id
		WHERE uat.token = ?
		`
	result := s.db.Model(&User{}).Raw(strings.TrimSpace(sql), token).Scan(user)
	if result.Error != nil {
		if err := wrapNotFound(result, user.TableName(), "token", token); err != nil {
			return nil, err
		}
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithResource(errors.NewResource(user.TableName(), "token", token)),
		)
	}
	return user, nil
}
<|endoftext|>


<commit_before>/*

	filesystem is a storage layer that stores information about games as JSON
	files within a given folder, one per game. It's extremely inefficient and
	doesn't even persist extended game information to disk. It's most useful
	for cases where having an easy-to-read, diffable representation for games
	makes sense, for example to create golden tester games for use in testing.

	filesystem stores files according to their gametype in the given base
	folder, for example 'checkers/a22ffcdef.json'. If the sub-folders don't
	exist, they will be created. Folders may be soft-linked from within the
	base folder; often when using the filesystem storage layer to help
	generate test cases you set up soft-links from a central location to a
	folder for test files in each game's sub-directory, so the test files can
	be in the same place.

*/
package filesystem

import (
	"errors"
	"github.com/jkomoros/boardgame"
	"github.com/jkomoros/boardgame/server/api/extendedgame"
	"github.com/jkomoros/boardgame/server/api/listing"
	"github.com/jkomoros/boardgame/storage/filesystem/record"
	"github.com/jkomoros/boardgame/storage/internal/helpers"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

type StorageManager struct {
	//Fall back on those methods
	*helpers.ExtendedMemoryStorageManager
	basePath string
	managers []*boardgame.GameManager
}

//Store seen ids and remember where the path was
var idToPath map[string]string

func init() {
	idToPath = make(map[string]string)
}

//NewStorageManager returns a new filesystem storage manager. basePath is the
//folder, relative to this executable, to have as the root of the storage
//pool.
func NewStorageManager(basePath string) *StorageManager {

	result := &StorageManager{
		basePath: basePath,
	}

	result.ExtendedMemoryStorageManager = helpers.NewExtendedMemoryStorageManager(result)

	return result
}

func (s *StorageManager) Name() string {
	return "filesystem"
}

func (s *StorageManager) Connect(config string) error {

	if _, err := os.Stat(s.basePath); os.IsNotExist(err) {
		if err := os.Mkdir(s.basePath, 0700); err != nil {
			return errors.New("Base path didn't exist and couldn't create it: " + err.Error())
		}
	}

	return nil
}

func (s *StorageManager) WithManagers(managers []*boardgame.GameManager) {
	s.managers = managers
}

func (s *StorageManager) CleanUp() {
	os.RemoveAll(s.basePath)
}

//pathForId will look through each sub-folder and look for a file named
//gameId.json, returning its relative path if it is found, "" otherwise.
func pathForId(basePath, gameId string) string {

	if path, ok := idToPath[gameId]; ok {
		return path
	}

	items, err := ioutil.ReadDir(basePath)
	if err != nil {
		return ""
	}
	for _, item := range items {
		if item.IsDir() {
			if recursiveResult := pathForId(filepath.Join(basePath, item.Name()), gameId); recursiveResult != "" {
				return recursiveResult
			}
			continue
		}

		if item.Name() == gameId+".json" {
			result := filepath.Join(basePath, item.Name())
			idToPath[gameId] = result
			return result
		}
	}
	return ""
}

func (s *StorageManager) recordForId(gameId string) (*record.Record, error) {
	if s.basePath == "" {
		return nil, errors.New("No base path provided")
	}

	gameId = strings.ToLower(gameId)

	path := pathForId(s.basePath, gameId)

	if path == "" {
		return nil, errors.New("Couldn't find file matching: " + gameId)
	}

	return record.New(path)
}

func (s *StorageManager) saveRecordForId(gameId string, rec *record.Record) error {
	if s.basePath == "" {
		return errors.New("Invalid base path")
	}

	if rec.Game() == nil {
		return errors.New("Game record in rec was nil")
	}

	gameId = strings.ToLower(gameId)

	path := filepath.Join(s.basePath, rec.Game().Name, gameId+".json")

	dir, _ := filepath.Split(path)

	if err := os.MkdirAll(dir, 0700); err != nil {
		return errors.New("Couldn't create all necessary sub-paths: " + err.Error())
	}

	if err := rec.Save(path); err != nil {
		return err
	}

	idToPath[gameId] = path

	return nil
}

func (s *StorageManager) State(gameId string, version int) (boardgame.StateStorageRecord, error) {
	rec, err := s.recordForId(gameId)

	if err != nil {
		return nil, err
	}

	result, err := rec.State(version)

	if err != nil {
		return nil, err
	}

	return boardgame.StateStorageRecord(result), nil

}

func (s *StorageManager) Move(gameId string, version int) (*boardgame.MoveStorageRecord, error) {
	rec, err := s.recordForId(gameId)

	if err != nil {
		return nil, err
	}

	return rec.Move(version)
}

func (s *StorageManager) Moves(gameId string, fromVersion, toVersion int) ([]*boardgame.MoveStorageRecord, error) {
	return helpers.MovesHelper(s, gameId, fromVersion, toVersion)
}

func (s *StorageManager) Game(id string) (*boardgame.GameStorageRecord, error) {

	rec, err := s.recordForId(id)

	if err != nil {
		return nil, err
	}

	return rec.Game(), nil
}

func (s *StorageManager) SaveGameAndCurrentState(game *boardgame.GameStorageRecord, state boardgame.StateStorageRecord, move *boardgame.MoveStorageRecord) error {
	rec, err := s.recordForId(game.Id)

	if err != nil {
		//Must be the first save.
		rec = &record.Record{}
	}

	if err := rec.AddGameAndCurrentState(game, state, move); err != nil {
		return errors.New("Couldn't add state: " + err.Error())
	}

	return s.saveRecordForId(game.Id, rec)

}

func (s *StorageManager) CombinedGame(id string) (*extendedgame.CombinedStorageRecord, error) {
	rec, err := s.recordForId(id)

	if err != nil {
		return nil, err
	}

	eGame, err := s.ExtendedGame(id)

	if err != nil {
		return nil, err
	}

	return &extendedgame.CombinedStorageRecord{
		*rec.Game(),
		*eGame,
	}, nil
}

func idFromPath(path string) string {
	_, filename := filepath.Split(path)
	return strings.TrimSuffix(filename, ".json")
}

func (s *StorageManager) recursiveAllGames(basePath string) []*boardgame.GameStorageRecord {

	files, err := ioutil.ReadDir(basePath)

	if err != nil {
		return nil
	}

	var result []*boardgame.GameStorageRecord

	for _, file := range files {

		if file.IsDir() {
			result = append(result, s.recursiveAllGames(filepath.Join(basePath, file.Name()))...)
			continue
		}
		ext := filepath.Ext(file.Name())
		if ext != ".json" {
			continue
		}
		rec, err := s.recordForId(idFromPath(file.Name()))
		if err != nil {
			return nil
		}
		result = append(result, rec.Game())
	}
	return result
}

func (s *StorageManager) AllGames() []*boardgame.GameStorageRecord {
	return s.recursiveAllGames(s.basePath)
}

func (s *StorageManager) ListGames(max int, list listing.Type, userId string, gameType string) []*extendedgame.CombinedStorageRecord {
	return helpers.ListGamesHelper(s, max, list, userId, gameType)
}
<commit_msg>Fix only `go vet` error in the packages. Fixes #659.<commit_after>/*

	filesystem is a storage layer that stores information about games as JSON
	files within a given folder, one per game. It's extremely inefficient and
	doesn't even persist extended game information to disk. It's most useful
	for cases where having an easy-to-read, diffable representation for games
	makes sense, for example to create golden tester games for use in testing.

	filesystem stores files according to their gametype in the given base
	folder, for example 'checkers/a22ffcdef.json'. If the sub-folders don't
	exist, they will be created. Folders may be soft-linked from within the
	base folder; often when using the filesystem storage layer to help
	generate test cases you set up soft-links from a central location to a
	folder for test files in each game's sub-directory, so the test files can
	be in the same place.

*/
package filesystem

import (
	"errors"
	"github.com/jkomoros/boardgame"
	"github.com/jkomoros/boardgame/server/api/extendedgame"
	"github.com/jkomoros/boardgame/server/api/listing"
	"github.com/jkomoros/boardgame/storage/filesystem/record"
	"github.com/jkomoros/boardgame/storage/internal/helpers"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

type StorageManager struct {
	//Fall back on those methods
	*helpers.ExtendedMemoryStorageManager
	basePath string
	managers []*boardgame.GameManager
}

//Store seen ids and remember where the path was
var idToPath map[string]string

func init() {
	idToPath = make(map[string]string)
}

//NewStorageManager returns a new filesystem storage manager. basePath is the
//folder, relative to this executable, to have as the root of the storage
//pool.
func NewStorageManager(basePath string) *StorageManager {

	result := &StorageManager{
		basePath: basePath,
	}

	result.ExtendedMemoryStorageManager = helpers.NewExtendedMemoryStorageManager(result)

	return result
}

func (s *StorageManager) Name() string {
	return "filesystem"
}

func (s *StorageManager) Connect(config string) error {

	if _, err := os.Stat(s.basePath); os.IsNotExist(err) {
		if err := os.Mkdir(s.basePath, 0700); err != nil {
			return errors.New("Base path didn't exist and couldn't create it: " + err.Error())
		}
	}

	return nil
}

func (s *StorageManager) WithManagers(managers []*boardgame.GameManager) {
	s.managers = managers
}

func (s *StorageManager) CleanUp() {
	os.RemoveAll(s.basePath)
}

//pathForId will look through each sub-folder and look for a file named
//gameId.json, returning its relative path if it is found, "" otherwise.
func pathForId(basePath, gameId string) string {

	if path, ok := idToPath[gameId]; ok {
		return path
	}

	items, err := ioutil.ReadDir(basePath)
	if err != nil {
		return ""
	}
	for _, item := range items {
		if item.IsDir() {
			if recursiveResult := pathForId(filepath.Join(basePath, item.Name()), gameId); recursiveResult != "" {
				return recursiveResult
			}
			continue
		}

		if item.Name() == gameId+".json" {
			result := filepath.Join(basePath, item.Name())
			idToPath[gameId] = result
			return result
		}
	}
	return ""
}

func (s *StorageManager) recordForId(gameId string) (*record.Record, error) {
	if s.basePath == "" {
		return nil, errors.New("No base path provided")
	}

	gameId = strings.ToLower(gameId)

	path := pathForId(s.basePath, gameId)

	if path == "" {
		return nil, errors.New("Couldn't find file matching: " + gameId)
	}

	return record.New(path)
}

func (s *StorageManager) saveRecordForId(gameId string, rec *record.Record) error {
	if s.basePath == "" {
		return errors.New("Invalid base path")
	}

	if rec.Game() == nil {
		return errors.New("Game record in rec was nil")
	}

	gameId = strings.ToLower(gameId)

	path := filepath.Join(s.basePath, rec.Game().Name, gameId+".json")

	dir, _ := filepath.Split(path)

	if err := os.MkdirAll(dir, 0700); err != nil {
		return errors.New("Couldn't create all necessary sub-paths: " + err.Error())
	}

	if err := rec.Save(path); err != nil {
		return err
	}

	idToPath[gameId] = path

	return nil
}

func (s *StorageManager) State(gameId string, version int) (boardgame.StateStorageRecord, error) {
	rec, err := s.recordForId(gameId)

	if err != nil {
		return nil, err
	}

	result, err := rec.State(version)

	if err != nil {
		return nil, err
	}

	return boardgame.StateStorageRecord(result), nil

}

func (s *StorageManager) Move(gameId string, version int) (*boardgame.MoveStorageRecord, error) {
	rec, err := s.recordForId(gameId)

	if err != nil {
		return nil, err
	}

	return rec.Move(version)
}

func (s *StorageManager) Moves(gameId string, fromVersion, toVersion int) ([]*boardgame.MoveStorageRecord, error) {
	return helpers.MovesHelper(s, gameId, fromVersion, toVersion)
}

func (s *StorageManager) Game(id string) (*boardgame.GameStorageRecord, error) {

	rec, err := s.recordForId(id)

	if err != nil {
		return nil, err
	}

	return rec.Game(), nil
}

func (s *StorageManager) SaveGameAndCurrentState(game *boardgame.GameStorageRecord, state boardgame.StateStorageRecord, move *boardgame.MoveStorageRecord) error {
	rec, err := s.recordForId(game.Id)

	if err != nil {
		//Must be the first save.
		rec = &record.Record{}
	}

	if err := rec.AddGameAndCurrentState(game, state, move); err != nil {
		return errors.New("Couldn't add state: " + err.Error())
	}

	return s.saveRecordForId(game.Id, rec)

}

func (s *StorageManager) CombinedGame(id string) (*extendedgame.CombinedStorageRecord, error) {
	rec, err := s.recordForId(id)

	if err != nil {
		return nil, err
	}

	eGame, err := s.ExtendedGame(id)

	if err != nil {
		return nil, err
	}

	return &extendedgame.CombinedStorageRecord{
		GameStorageRecord: *rec.Game(),
		StorageRecord:     *eGame,
	}, nil
}

func idFromPath(path string) string {
	_, filename := filepath.Split(path)
	return strings.TrimSuffix(filename, ".json")
}

func (s *StorageManager) recursiveAllGames(basePath string) []*boardgame.GameStorageRecord {

	files, err := ioutil.ReadDir(basePath)

	if err != nil {
		return nil
	}

	var result []*boardgame.GameStorageRecord

	for _, file := range files {

		if file.IsDir() {
			result = append(result, s.recursiveAllGames(filepath.Join(basePath, file.Name()))...)
			continue
		}
		ext := filepath.Ext(file.Name())
		if ext != ".json" {
			continue
		}
		rec, err := s.recordForId(idFromPath(file.Name()))
		if err != nil {
			return nil
		}
		result = append(result, rec.Game())
	}
	return result
}

func (s *StorageManager) AllGames() []*boardgame.GameStorageRecord {
	return s.recursiveAllGames(s.basePath)
}

func (s *StorageManager) ListGames(max int, list listing.Type, userId string, gameType string) []*extendedgame.CombinedStorageRecord {
	return helpers.ListGamesHelper(s, max, list, userId, gameType)
}
<|endoftext|>


<commit_before>package cap

import (
	"encoding/xml"
	"io/ioutil"
	"testing"
)

func getCAPAlertExample() (*Alert11, error) {
	xmlData, err := ioutil.ReadFile("examples/nws_alert.xml")

	if err != nil {
		return nil, err
	}

	var alert Alert11

	err = xml.Unmarshal(xmlData, &alert)

	if err != nil {
		return nil, err
	}

	return &alert, nil
}

func TestUnmarshalAlertHasProperValues(t *testing.T) {
	alert, err := getCAPAlertExample()

	if err != nil {
		t.Fatal(err)
	}

	assertEqual(t,
		alert.MessageID,
		"NOAA-NWS-ALERTS-AR1253BA3B00A4.FloodWarning.1253BA3D4A94AR.LZKFLSLZK.342064b5a5aafb8265dfc3707d6a3b09",
		"MessageID does not match!")

	assertEqual(t,
		alert.SenderID,
		"w-nws.webmaster@noaa.gov",
		"SenderID does not match!")

	assertEqual(t,
		alert.SentDate,
		"2015-08-15T20:45:00-05:00",
		"SenderDate does not match!")

	assertEqual(t,
		alert.MessageStatus,
		"Actual",
		"MessageStatus does not match!")

	assertEqual(t,
		alert.MessageType,
		"Alert",
		"MessageType does not match!")

	assertEqual(t,
		alert.Scope,
		"Public",
		"Scope does not match!")

	assertEqual(t,
		alert.Note,
		"Alert for Jackson; Woodruff (Arkansas) Issued by the National Weather Service",
		"Note does not match!")

	assertEqual(t,
		len(alert.Infos),
		1,
		"One <info> should be present")
}

func TestUnmarshalAlertInfoHasProperValues(t *testing.T) {
	alert, err := getCAPAlertExample()

	if err != nil {
		t.Fatal(err)
	}

	var info = alert.Infos[0]
	assertEqual(t,
		info.EventCategory,
		"Met",
		"EventCategory does not match!")

	assertEqual(t,
		info.EventType,
		"Flood Warning",
		"EventType does not match!")

	assertEqual(t,
		info.Urgency,
		"Expected",
		"Urgency does not match!")

	assertEqual(t,
		info.Certainty,
		"Likely",
		"Certainty does not match!")

	assertEqual(t,
		info.EventCode[0].ValueName,
		"SAME",
		"EventCode-ValueName does not match!")

	assertEqual(t,
		info.EventCode[0].Value,
		"",
		"EventCode-Value does not match!")

	assertEqual(t,
		info.EffectiveDate,
		"2015-08-15T20:45:00-05:00",
		"EffectiveDate does not match!")

	assertEqual(t,
		info.ExpiresDate,
		"2015-08-16T11:45:00-05:00",
		"ExpiresDate does not match")

	assertEqual(t,
		info.SenderName,
		"NWS Little Rock (Arkansas)",
		"SenderName does not match")

	assertEqual(t,
		info.Headline,
		"Flood Warning issued August 15 at 8:45PM CDT until further notice by NWS Little Rock",
		"Headline does not match!")

	assertStartsWith(t,
		info.EventDescription,
		"...From the National Weather Service in Little Rock",
		"EventDescription does not match!")

	assertStartsWith(t,
		info.Instruction,
		"Safety message...",
		"Instruction does not match!")

	assertEqual(t,
		len(info.Parameters),
		4,
		"Number of Parameters does not match!")

	assertEqual(t,
		len(info.Areas),
		1,
		"Number of Areas does not match!")
}

func TestUnmarshalAlertInfoParameterHasProperValues(t *testing.T) {
	alert, err := getCAPAlertExample()

	if err != nil {
		t.Fatal(err)
	}

	var info = alert.Infos[0]

	assertEqual(t,
		info.Parameter("WMOHEADER"),
		"",
		"WMOHEADER does not match!")

	assertEqual(t,
		info.Parameter("UGC"),
		"ARC067-147",
		"UGC does not match!")

	assertStartsWith(t,
		info.Parameter("VTEC"),
		"/O.CON.KLZK.FL.W.0108.000000T0000Z-000000T0000Z/\n",
		"VTEC does not match!")

	assertEqual(t,
		info.Parameter("TIME...MOT...LOC"),
		"",
		"TIME...MOT...LOC does not match!")

	assertEqual(t,
		info.Parameter("TIME...MOT...LOC"),
		"",
		"TIME...MOT...LOC does not match!")

	assertEqual(t,
		info.Parameter("TIME...MOT...LOC"),
		"",
		"TIME...MOT...LOC does not match!")
}

func TestUnmarshalAlertInfoAreaHasProperValues(t *testing.T) {
	alert, err := getCAPAlertExample()

	if err != nil {
		t.Fatal(err)
	}

	var info = alert.Infos[0]
	var area = info.Areas[0]

	assertEqual(t,
		area.Description,
		"Jackson; Woodruff",
		"Description does not match!")

	assertEqual(t,
		area.Polygon,
		"35.1,-91.33 35.22,-91.28 35.39,-91.23 35.38,-91.13 35.21,-91.17 35.08,-91.22 35.1,-91.33",
		"Polygon does not match!")

	assertEqual(t,
		len(area.Geocodes),
		4,
		"Area does not have the proper number of Geocode elements")

	assertIn(t,
		"005067",
		area.GeocodeAll("FIPS6"),
		"Value not found in Geocode[FIPS6]!")

	assertIn(t,
		"005147",
		area.GeocodeAll("FIPS6"),
		"Value not found in Geocode[FIPS6]!")

	assertIn(t,
		"ARC067",
		area.GeocodeAll("UGC"),
		"Value not found in Geocode[UGC]!")

	assertIn(t,
		"ARC147",
		area.GeocodeAll("UGC"),
		"Value not found in Geocode[UGC]!")
}
<commit_msg>Added some additional tests to cap_test.go to improve coverage<commit_after>package cap

import (
	"encoding/xml"
	"io/ioutil"
	"testing"
)

func getCAPAlertExample() (*Alert11, error) {
	xmlData, err := ioutil.ReadFile("examples/nws_alert.xml")

	if err != nil {
		return nil, err
	}

	var alert Alert11

	err = xml.Unmarshal(xmlData, &alert)

	if err != nil {
		return nil, err
	}

	return &alert, nil
}

func TestUnmarshalAlertHasProperValues(t *testing.T) {
	alert, err := getCAPAlertExample()

	if err != nil {
		t.Fatal(err)
	}

	assertEqual(t,
		alert.MessageID,
		"NOAA-NWS-ALERTS-AR1253BA3B00A4.FloodWarning.1253BA3D4A94AR.LZKFLSLZK.342064b5a5aafb8265dfc3707d6a3b09",
		"MessageID does not match!")

	assertEqual(t,
		alert.SenderID,
		"w-nws.webmaster@noaa.gov",
		"SenderID does not match!")

	assertEqual(t,
		alert.SentDate,
		"2015-08-15T20:45:00-05:00",
		"SenderDate does not match!")

	assertEqual(t,
		alert.MessageStatus,
		"Actual",
		"MessageStatus does not match!")

	assertEqual(t,
		alert.MessageType,
		"Alert",
		"MessageType does not match!")

	assertEqual(t,
		alert.Scope,
		"Public",
		"Scope does not match!")

	assertEqual(t,
		alert.Note,
		"Alert for Jackson; Woodruff (Arkansas) Issued by the National Weather Service",
		"Note does not match!")

	assertEqual(t,
		len(alert.Infos),
		1,
		"One <info> should be present")
}

func TestUnmarshalAlertInfoHasProperValues(t *testing.T) {
	alert, err := getCAPAlertExample()

	if err != nil {
		t.Fatal(err)
	}

	var info = alert.Infos[0]
	assertEqual(t,
		info.EventCategory,
		"Met",
		"EventCategory does not match!")

	assertEqual(t,
		info.EventType,
		"Flood Warning",
		"EventType does not match!")

	assertEqual(t,
		info.Urgency,
		"Expected",
		"Urgency does not match!")

	assertEqual(t,
		info.Certainty,
		"Likely",
		"Certainty does not match!")

	assertEqual(t,
		info.EventCode[0].ValueName,
		"SAME",
		"EventCode-ValueName does not match!")

	assertEqual(t,
		info.EventCode[0].Value,
		"",
		"EventCode-Value does not match!")

	assertEqual(t,
		info.EffectiveDate,
		"2015-08-15T20:45:00-05:00",
		"EffectiveDate does not match!")

	assertEqual(t,
		info.ExpiresDate,
		"2015-08-16T11:45:00-05:00",
		"ExpiresDate does not match")

	assertEqual(t,
		info.SenderName,
		"NWS Little Rock (Arkansas)",
		"SenderName does not match")

	assertEqual(t,
		info.Headline,
		"Flood Warning issued August 15 at 8:45PM CDT until further notice by NWS Little Rock",
		"Headline does not match!")

	assertStartsWith(t,
		info.EventDescription,
		"...From the National Weather Service in Little Rock",
		"EventDescription does not match!")

	assertStartsWith(t,
		info.Instruction,
		"Safety message...",
		"Instruction does not match!")

	assertEqual(t,
		len(info.Parameters),
		4,
		"Number of Parameters does not match!")

	assertEqual(t,
		len(info.Areas),
		1,
		"Number of Areas does not match!")
}

func TestUnmarshalAlertInfoParameterHasProperValues(t *testing.T) {
	alert, err := getCAPAlertExample()

	if err != nil {
		t.Fatal(err)
	}

	var info = alert.Infos[0]

	assertEqual(t,
		info.Parameter("WMOHEADER"),
		"",
		"WMOHEADER does not match!")

	assertEqual(t,
		info.Parameter("UGC"),
		"ARC067-147",
		"UGC does not match!")

	assertStartsWith(t,
		info.Parameter("VTEC"),
		"/O.CON.KLZK.FL.W.0108.000000T0000Z-000000T0000Z/\n",
		"VTEC does not match!")

	assertEqual(t,
		info.Parameter("TIME...MOT...LOC"),
		"",
		"TIME...MOT...LOC does not match!")

	assertEqual(t,
		info.Parameter("TIME...MOT...LOC"),
		"",
		"TIME...MOT...LOC does not match!")

	assertEqual(t,
		info.Parameter("TIME...MOT...LOC"),
		"",
		"TIME...MOT...LOC does not match!")
}

func TestUnmarshalAlertInfoAreaHasProperValues(t *testing.T) {
	alert, err := getCAPAlertExample()

	if err != nil {
		t.Fatal(err)
	}

	var info = alert.Infos[0]
	var area = info.Areas[0]

	assertEqual(t,
		area.Description,
		"Jackson; Woodruff",
		"Description does not match!")

	assertEqual(t,
		area.Polygon,
		"35.1,-91.33 35.22,-91.28 35.39,-91.23 35.38,-91.13 35.21,-91.17 35.08,-91.22 35.1,-91.33",
		"Polygon does not match!")

	assertEqual(t,
		len(area.Geocodes),
		4,
		"Area does not have the proper number of Geocode elements")

	assertIn(t,
		"005067",
		area.GeocodeAll("FIPS6"),
		"Value not found in Geocode[FIPS6]!")

	assertIn(t,
		"005147",
		area.GeocodeAll("FIPS6"),
		"Value not found in Geocode[FIPS6]!")

	assertIn(t,
		"ARC067",
		area.GeocodeAll("UGC"),
		"Value not found in Geocode[UGC]!")

	assertIn(t,
		"ARC147",
		area.GeocodeAll("UGC"),
		"Value not found in Geocode[UGC]!")
}

func TestAddParameterToInfoSetsProperValue(t *testing.T) {
	parameterName := "testcode"
	parameterValue := "1234"
	var info Info

	assertEqual(t, len(info.Parameters), 0, "info.Parameters should be empty")

	info.AddParameter(parameterName, parameterValue)

	assertEqual(t, len(info.Parameters), 1, "info.Parameters should have len = 1")

	parameter := info.Parameters[0]
	assertEqual(t, parameter.ValueName, parameterName, "info.Parameters[0] does not have the correct name")
	assertEqual(t, parameter.Value, parameterValue, "info.Parameters[0] does not have the correct value")
}

func TestAddGeocodeToAreaSetsProperValue(t *testing.T) {
	geocodeName := "testcode"
	geocodeValue := "1234"
	var area Area

	assertEqual(t, len(area.Geocodes), 0, "area.Geocodes should be empty")

	area.AddGeocode(geocodeName, geocodeValue)

	assertEqual(t, len(area.Geocodes), 1, "area.Geocodes should have len = 1")

	geocode := area.Geocodes[0]
	assertEqual(t, geocode.ValueName, geocodeName, "area.Geocodes[0] does not have the correct name")
	assertEqual(t, geocode.Value, geocodeValue, "area.Geocodes[0] does not have the correct value")
}

func TestAreaGecodeReturnsFirstValue(t *testing.T) {
	geocode1 := NamedValue{"test-name", "1234"}
	geocode2 := NamedValue{"test-name", "5678"}

	var area Area

	area.AddGeocode(geocode1.ValueName, geocode1.Value)
	area.AddGeocode(geocode2.ValueName, geocode2.Value)

	assertEqual(t, len(area.Geocodes), 2, "area.Geocodes should have len = 2")

	geocodeValue := area.Geocode("test-name")
	assertEqual(t, geocodeValue, geocode1.Value, "Geocode does not have the correct name")
}

func TestAreaGecodeReturnsEmptyStringIfNotFound(t *testing.T) {
	geocode := NamedValue{"test-name", "1234"}

	var area Area

	area.AddGeocode(geocode.ValueName, geocode.Value)

	geocodeValue := area.Geocode("not-a-real-key")
	assertEqual(t, geocodeValue, "", "Geocode did not return an empty string")
}
<|endoftext|>


<commit_before>/***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****/

package s3splitfile

import (
	"bufio"
	"fmt"
	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/s3"
	"github.com/mozilla-services/heka/message"
	"github.com/mozilla-services/heka/pipeline"
	"io/ioutil"
	"math"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type MessageLocation struct {
	Key    string
	Offset uint32
	Length uint32
}

type S3OffsetInput struct {
	processMessageCount    int64
	processMessageFailures int64
	processMessageBytes    int64

	*S3OffsetInputConfig
	clientids  map[string]struct{}
	bucket     *s3.Bucket
	metaBucket *s3.Bucket
	stop       chan bool
	offsetChan chan MessageLocation
}

type S3OffsetInputConfig struct {
	// So we can default to using ProtobufDecoder.
	Decoder            string
	Splitter           string
	ClientIdListFile   string `toml:"client_id_list"`
	StartDate          string `toml:"start_date"`
	EndDate            string `toml:"end_date"`
	AWSKey             string `toml:"aws_key"`
	AWSSecretKey       string `toml:"aws_secret_key"`
	AWSRegion          string `toml:"aws_region"`
	S3MetaBucket       string `toml:"s3_meta_bucket"`
	S3MetaBucketPrefix string `toml:"s3_meta_bucket_prefix"`
	S3Bucket           string `toml:"s3_bucket"`
	S3Retries          uint32 `toml:"s3_retries"`
	S3WorkerCount      uint32 `toml:"s3_worker_count"`
}

func (input *S3OffsetInput) ConfigStruct() interface{} {
	return &S3OffsetInputConfig{
		Decoder:            "ProtobufDecoder",
		Splitter:           "NullSplitter",
		StartDate:          "20150101",
		EndDate:            time.Now().UTC().Format("20060102"),
		AWSKey:             "",
		AWSSecretKey:       "",
		AWSRegion:          "us-west-2",
		S3MetaBucket:       "",
		S3MetaBucketPrefix: "",
		S3Bucket:           "",
		S3Retries:          5,
		S3WorkerCount:      16,
	}
}

func (input *S3OffsetInput) Init(config interface{}) (err error) {
	conf := config.(*S3OffsetInputConfig)
	input.S3OffsetInputConfig = conf

	// Load clientids from file.
	input.clientids, err = readLines(conf.ClientIdListFile)
	if err != nil {
		return fmt.Errorf("Error reading file %s for 'client_id_list': %s", conf.ClientIdListFile, err)
	}

	auth, err := aws.GetAuth(conf.AWSKey, conf.AWSSecretKey, "", time.Now())
	if err != nil {
		return fmt.Errorf("Authentication error: %s\n", err)
	}
	region, ok := aws.Regions[conf.AWSRegion]
	if !ok {
		return fmt.Errorf("Parameter 'aws_region' must be a valid AWS Region")
	}
	s := s3.New(auth, region)
	// TODO: ensure we can read from (and list, for meta) the buckets.
	input.bucket = s.Bucket(conf.S3Bucket)
	input.metaBucket = s.Bucket(conf.S3MetaBucket)

	// Remove any excess path separators from the bucket prefix.
	conf.S3MetaBucketPrefix = CleanBucketPrefix(conf.S3MetaBucketPrefix)

	input.stop = make(chan bool)
	input.offsetChan = make(chan MessageLocation, 1000)

	return nil
}

func (input *S3OffsetInput) Stop() {
	close(input.stop)
}

func (input *S3OffsetInput) Run(runner pipeline.InputRunner, helper pipeline.PluginHelper) error {
	// List offset metadata index files
	// For each index D >= start and <= end
	//   Read index D
	//   Write offsets for any desired clients to offsetChan
	// Meanwhile, for each item in offsetChan
	//   Go fetch that record, inject resulting message into pipeline.

	var (
		wg          sync.WaitGroup
		i           uint32
		emptySchema Schema
	)

	wg.Add(1)
	go func() {
		runner.LogMessage("Starting S3 list")
		for r := range S3Iterator(input.metaBucket, input.S3MetaBucketPrefix, emptySchema) {
			if r.Err != nil {
				runner.LogError(fmt.Errorf("Error getting S3 list: %s", r.Err))
			} else {
				base := path.Base(r.Key.Key)[0:8]
				// Check if r is in the desired date range.
				if base >= input.StartDate && base <= input.EndDate {
					err := input.grep(r)
					if err != nil {
						runner.LogMessage(fmt.Sprintf("Error reading index: %s", err))
					}
				}
			}
		}
		// All done listing, close the channel
		runner.LogMessage("All done listing. Closing channel")
		close(input.offsetChan)
		wg.Done()
	}()

	// Run a pool of concurrent readers.
	for i = 0; i < input.S3WorkerCount; i++ {
		wg.Add(1)
		go input.fetcher(runner, &wg, i)
	}
	wg.Wait()

	return nil
}

func (input *S3OffsetInput) grep(result S3ListResult) (err error) {
	// Read the file from S3, grep for desired clients.
	// It appears that goamz helpfully gunzips the content for you if the
	// correct headers are set.
	reader, err := input.metaBucket.GetReader(result.Key.Key)
	if err != nil {
		return err
	}
	defer reader.Close()

	lineNum := 0
	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		pieces := strings.Split(scanner.Text(), "\t")
		lineNum++
		if len(pieces) != 4 {
			return fmt.Errorf("Error on %s line %d: invalid line. Expected 4 values, found %d.", result.Key.Key, lineNum, len(pieces))
		}

		// Check if this client is in our list.
		_, ok := input.clientids[pieces[1]]
		if !ok {
			continue
		}
		o, err := makeInt(pieces[2])
		if err != nil {
			return err
		}
		l, err := makeInt(pieces[3])
		if err != nil {
			return err
		}
		input.offsetChan <- MessageLocation{pieces[0], o, l}
	}
	return scanner.Err()
}

func (input *S3OffsetInput) fetcher(runner pipeline.InputRunner, wg *sync.WaitGroup, workerId uint32) {
	var (
		loc       MessageLocation
		startTime time.Time
		duration  float64
		headers   map[string][]string
		record    []byte
		err       error
	)

	headers = map[string][]string{
		"Range": []string{""},
	}

	fetcherName := fmt.Sprintf("S3Reader%d", workerId)
	deliverer := runner.NewDeliverer(fetcherName)
	defer deliverer.Done()
	splitterRunner := runner.NewSplitterRunner(fetcherName)

	ok := true
	for ok {
		select {
		case loc, ok = <-input.offsetChan:
			if !ok {
				// Channel is closed => we're shutting down, exit cleanly.
				runner.LogMessage("Fetcher all done! shutting down.")
				break
			}

			startTime = time.Now().UTC()
			// Read one message from the given location
			headers["Range"][0] = fmt.Sprintf("bytes=%d-%d", loc.Offset, loc.Offset+loc.Length-1)
			atomic.AddInt64(&input.processMessageCount, 1)
			atomic.AddInt64(&input.processMessageBytes, int64(loc.Length))
			for attempt := uint32(1); attempt <= input.S3Retries; attempt++ {
				record, err = getClientRecord(input.bucket, &loc, headers)
				if err != nil {
					runner.LogMessage(fmt.Sprintf("Error #%d fetching %s @ %d+%d: %s\n", attempt, loc.Key, loc.Offset, loc.Length, err))
				} else {
					break
				}
			}
			if err != nil {
				atomic.AddInt64(&input.processMessageFailures, 1)
				continue
			}
			splitterRunner.DeliverRecord(record, deliverer)
			duration = time.Now().UTC().Sub(startTime).Seconds()
			runner.LogMessage(fmt.Sprintf("Successfully fetched %s in %.2fs ", loc.Key, duration))
		}
	}

	wg.Done()
}

func (input *S3OffsetInput) ReportMsg(msg *message.Message) error {
	message.NewInt64Field(msg, "ProcessMessageCount", atomic.LoadInt64(&input.processMessageCount), "count")
	message.NewInt64Field(msg, "ProcessMessageFailures", atomic.LoadInt64(&input.processMessageFailures), "count")
	message.NewInt64Field(msg, "ProcessMessageBytes", atomic.LoadInt64(&input.processMessageBytes), "B")

	return nil
}

// Read all lines from specified file into an array.
func readLines(path string) (map[string]struct{}, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	lines := map[string]struct{}{}
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		lines[scanner.Text()] = struct{}{}
	}
	return lines, scanner.Err()
}

// Parse a string as a uint32 value.
func makeInt(numstr string) (uint32, error) {
	i, err := strconv.ParseInt(string(numstr), 10, 64)
	if err != nil {
		return 0, err
	}
	if i < 0 || i > math.MaxUint32 {
		return 0, fmt.Errorf("Error parsing %d as uint32")
	}
	return uint32(i), nil
}

func init() {
	pipeline.RegisterPlugin("S3OffsetInput", func() interface{} {
		return new(S3OffsetInput)
	})
}

// Read a single client record using a partial read from S3 using the given
// headers, which should contain a "Range: bytes=M-N" header.
func getClientRecord(bucket *s3.Bucket, o *MessageLocation, headers map[string][]string) ([]byte, error) {
	resp, err := bucket.GetResponseWithHeaders(o.Key, headers)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err == nil && len(body) != int(o.Length) {
		err = fmt.Errorf("Unexpected body length: %d != %d\n", len(body), o.Length)
	}
	return body, err
}
<commit_msg>Add support for reading metadata from a file.<commit_after>/***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****/

package s3splitfile

import (
	"bufio"
	"fmt"
	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/s3"
	"github.com/mozilla-services/heka/message"
	"github.com/mozilla-services/heka/pipeline"
	"io"
	"io/ioutil"
	"math"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type MessageLocation struct {
	Key    string
	Offset uint32
	Length uint32
}

type S3OffsetInput struct {
	processMessageCount    int64
	processMessageFailures int64
	processMessageBytes    int64

	*S3OffsetInputConfig
	clientids    map[string]struct{}
	metaFileName string
	bucket       *s3.Bucket
	metaBucket   *s3.Bucket
	stop         chan bool
	offsetChan   chan MessageLocation
}

type S3OffsetInputConfig struct {
	// So we can default to using ProtobufDecoder.
	Decoder            string
	Splitter           string
	ClientIdListFile   string `toml:"client_id_list"`
	MetaFile           string `toml:"metadata_file"`
	StartDate          string `toml:"start_date"`
	EndDate            string `toml:"end_date"`
	AWSKey             string `toml:"aws_key"`
	AWSSecretKey       string `toml:"aws_secret_key"`
	AWSRegion          string `toml:"aws_region"`
	S3MetaBucket       string `toml:"s3_meta_bucket"`
	S3MetaBucketPrefix string `toml:"s3_meta_bucket_prefix"`
	S3Bucket           string `toml:"s3_bucket"`
	S3Retries          uint32 `toml:"s3_retries"`
	S3ConnectTimeout   uint32 `toml:"s3_connect_timeout"`
	S3ReadTimeout      uint32 `toml:"s3_read_timeout"`
	S3WorkerCount      uint32 `toml:"s3_worker_count"`
}

func (input *S3OffsetInput) ConfigStruct() interface{} {
	return &S3OffsetInputConfig{
		Decoder:            "ProtobufDecoder",
		Splitter:           "NullSplitter",
		StartDate:          "20150101",
		EndDate:            time.Now().UTC().Format("20060102"),
		AWSKey:             "",
		AWSSecretKey:       "",
		AWSRegion:          "us-west-2",
		S3MetaBucket:       "",
		S3MetaBucketPrefix: "",
		S3Bucket:           "",
		S3Retries:          5,
		S3ConnectTimeout:   60,
		S3ReadTimeout:      60,
		S3WorkerCount:      16,
	}
}

func (input *S3OffsetInput) Init(config interface{}) (err error) {
	conf := config.(*S3OffsetInputConfig)
	input.S3OffsetInputConfig = conf

	if conf.MetaFile != "" {
		// We already have the required metadata. Don't need to fetch it.
		input.metaFileName = conf.MetaFile
	} else if conf.ClientIdListFile != "" {
		// Load clientids from file.
		input.clientids, err = readLines(conf.ClientIdListFile)
		if err != nil {
			return fmt.Errorf("Error reading file %s for 'client_id_list': %s", conf.ClientIdListFile, err)
		}
	} else {
		return fmt.Errorf("Missing parameter: You must specify either 'client_id_list' or 'metadata_file'")
	}

	auth, err := aws.GetAuth(conf.AWSKey, conf.AWSSecretKey, "", time.Now())
	if err != nil {
		return fmt.Errorf("Authentication error: %s\n", err)
	}
	region, ok := aws.Regions[conf.AWSRegion]
	if !ok {
		return fmt.Errorf("Parameter 'aws_region' must be a valid AWS Region")
	}
	s := s3.New(auth, region)
	s.ConnectTimeout = time.Duration(conf.S3ConnectTimeout) * time.Second
	s.ReadTimeout = time.Duration(conf.S3ReadTimeout) * time.Second

	// TODO: ensure we can read from (and list, for meta) the buckets.
	input.bucket = s.Bucket(conf.S3Bucket)

	if conf.S3MetaBucket != "" {
		input.metaBucket = s.Bucket(conf.S3MetaBucket)
	} else if conf.MetaFile == "" {
		return fmt.Errorf("Parameter 's3_meta_bucket' is required unless using 'metadata_file'")
	}

	// Remove any excess path separators from the bucket prefix.
	conf.S3MetaBucketPrefix = CleanBucketPrefix(conf.S3MetaBucketPrefix)

	input.stop = make(chan bool)
	input.offsetChan = make(chan MessageLocation, 1000)

	return nil
}

func (input *S3OffsetInput) Stop() {
	close(input.stop)
}

func (input *S3OffsetInput) Run(runner pipeline.InputRunner, helper pipeline.PluginHelper) error {
	// List offset metadata index files
	// For each index D >= start and <= end
	//   Read index D
	//   Write offsets for any desired clients to offsetChan
	// Meanwhile, for each item in offsetChan
	//   Go fetch that record, inject resulting message into pipeline.

	var (
		wg          sync.WaitGroup
		i           uint32
		emptySchema Schema
	)

	if input.metaFileName != "" {
		wg.Add(1)
		go func() {
			reader, err := os.Open(input.metaFileName)
			if err != nil {
				runner.LogMessage(fmt.Sprintf("Error opening metadata file '%s': %s", input.metaFileName, err))
			}
			defer reader.Close()
			err = input.parseMessageLocations(reader, input.metaFileName)
			if err != nil {
				runner.LogMessage(fmt.Sprintf("Error reading metadata: %s", err))
			}
			// All done with metadata, close the channel
			runner.LogMessage("All done with metadata. Closing channel")
			close(input.offsetChan)
			wg.Done()
		}()
	} else if input.metaBucket != nil {
		wg.Add(1)
		go func() {
			runner.LogMessage("Starting S3 list")
		iteratorLoop:
			for r := range S3Iterator(input.metaBucket, input.S3MetaBucketPrefix, emptySchema) {
				select {
				case <-input.stop:
					runner.LogMessage("Stopping S3 list")
					break iteratorLoop
				default:
				}
				if r.Err != nil {
					runner.LogError(fmt.Errorf("Error getting S3 list: %s", r.Err))
				} else {
					base := path.Base(r.Key.Key)[0:8]
					// Check if r is in the desired date range.
					if base >= input.StartDate && base <= input.EndDate {
						err := input.grep(r)
						if err != nil {
							runner.LogMessage(fmt.Sprintf("Error reading index: %s", err))
						}
					}
				}
			}
			// All done listing, close the channel
			runner.LogMessage("All done listing. Closing channel")
			close(input.offsetChan)
			wg.Done()
		}()
	} else {
		runner.LogMessage("Nothing to do, no metadata available. Closing channel")
		close(input.offsetChan)
		wg.Done()
	}

	// Run a pool of concurrent readers.
	for i = 0; i < input.S3WorkerCount; i++ {
		wg.Add(1)
		go input.fetcher(runner, &wg, i)
	}
	wg.Wait()

	return nil
}

func (input *S3OffsetInput) grep(result S3ListResult) (err error) {
	// Read the file from S3, grep for desired clients.
	// It appears that goamz helpfully gunzips the content for you if the
	// correct headers are set.
	reader, err := input.metaBucket.GetReader(result.Key.Key)
	if err != nil {
		return err
	}
	defer reader.Close()
	return input.parseMessageLocations(reader, result.Key.Key)
}

// Not spec-compliant, but should work well enough for our purposes.
func (input *S3OffsetInput) detectFieldSeparator(line string, expectedCount int) (sep string) {
	possible := [...]string{"\t", ",", "|", " "}
	for _, s := range possible {
		pieces := strings.Split(line, s)
		if len(pieces) == expectedCount {
			return s
		}
	}
	// Don't know... default to tab.
	return possible[0]
}

func (input *S3OffsetInput) parseMessageLocations(reader io.Reader, name string) (err error) {
	lineNum := 0
	// TODO: use "encoding/csv" and set .Comma to the detected separator.
	scanner := bufio.NewScanner(reader)
	delim := ""
	expectedTokens := 4
	for scanner.Scan() {
		if lineNum == 0 {
			delim = input.detectFieldSeparator(scanner.Text(), expectedTokens)
		}
		pieces := strings.Split(scanner.Text(), delim)
		if len(pieces) != expectedTokens {
			return fmt.Errorf("Error on %s line %d: invalid line. Expected %d values, found %d.", name, lineNum, expectedTokens, len(pieces))
		}
		lineNum++

		// Skip optional header.
		if pieces[0] == "file_name" {
			continue
		}

		if input.metaFileName == "" {
			// Check if this client is in our list.
			_, ok := input.clientids[pieces[1]]
			if !ok {
				continue
			}
		}
		o, err := makeInt(pieces[2])
		if err != nil {
			return err
		}
		l, err := makeInt(pieces[3])
		if err != nil {
			return err
		}
		input.offsetChan <- MessageLocation{pieces[0], o, l}
	}
	return scanner.Err()
}

func (input *S3OffsetInput) fetcher(runner pipeline.InputRunner, wg *sync.WaitGroup, workerId uint32) {
	var (
		loc       MessageLocation
		startTime time.Time
		duration  float64
		headers   map[string][]string
		record    []byte
		err       error
	)

	headers = map[string][]string{
		"Range": []string{""},
	}

	fetcherName := fmt.Sprintf("S3Reader%d", workerId)
	deliverer := runner.NewDeliverer(fetcherName)
	defer deliverer.Done()
	splitterRunner := runner.NewSplitterRunner(fetcherName)

	ok := true
	for ok {
		select {
		case loc, ok = <-input.offsetChan:
			if !ok {
				// Channel is closed => we're shutting down, exit cleanly.
				runner.LogMessage("Fetcher all done! shutting down.")
				break
			}

			startTime = time.Now().UTC()
			// Read one message from the given location
			headers["Range"][0] = fmt.Sprintf("bytes=%d-%d", loc.Offset, loc.Offset+loc.Length-1)
			atomic.AddInt64(&input.processMessageCount, 1)
			atomic.AddInt64(&input.processMessageBytes, int64(loc.Length))
			for attempt := uint32(1); attempt <= input.S3Retries; attempt++ {
				record, err = getClientRecord(input.bucket, &loc, headers)
				if err != nil {
					runner.LogMessage(fmt.Sprintf("Error #%d fetching %s @ %d+%d: %s\n", attempt, loc.Key, loc.Offset, loc.Length, err))
				} else {
					break
				}
			}
			if err != nil {
				atomic.AddInt64(&input.processMessageFailures, 1)
				continue
			}
			splitterRunner.DeliverRecord(record, deliverer)
			duration = time.Now().UTC().Sub(startTime).Seconds()
			runner.LogMessage(fmt.Sprintf("Successfully fetched %s in %.2fs ", loc.Key, duration))

		case <-input.stop:
			runner.LogMessage("Stopping fetcher...")
			for _ = range input.offsetChan {
				// Drain the channel without processing anything.
			}
			ok = false
		}
	}

	wg.Done()
}

func (input *S3OffsetInput) ReportMsg(msg *message.Message) error {
	message.NewInt64Field(msg, "ProcessMessageCount", atomic.LoadInt64(&input.processMessageCount), "count")
	message.NewInt64Field(msg, "ProcessMessageFailures", atomic.LoadInt64(&input.processMessageFailures), "count")
	message.NewInt64Field(msg, "ProcessMessageBytes", atomic.LoadInt64(&input.processMessageBytes), "B")

	return nil
}

// Read all lines from specified file into an array.
func readLines(path string) (map[string]struct{}, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	lines := map[string]struct{}{}
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		lines[scanner.Text()] = struct{}{}
	}
	return lines, scanner.Err()
}

// Parse a string as a uint32 value.
func makeInt(numstr string) (uint32, error) {
	i, err := strconv.ParseInt(string(numstr), 10, 64)
	if err != nil {
		return 0, err
	}
	if i < 0 || i > math.MaxUint32 {
		return 0, fmt.Errorf("Error parsing %d as uint32")
	}
	return uint32(i), nil
}

func init() {
	pipeline.RegisterPlugin("S3OffsetInput", func() interface{} {
		return new(S3OffsetInput)
	})
}

// Read a single client record using a partial read from S3 using the given
// headers, which should contain a "Range: bytes=M-N" header.
func getClientRecord(bucket *s3.Bucket, o *MessageLocation, headers map[string][]string) ([]byte, error) {
	resp, err := bucket.GetResponseWithHeaders(o.Key, headers)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err == nil && len(body) != int(o.Length) {
		err = fmt.Errorf("Unexpected body length: %d != %d\n", len(body), o.Length)
	}
	return body, err
}
<|endoftext|>


<commit_before>package mandrill

import (
	"fmt"

	"github.com/freeusd/solebtc/Godeps/_workspace/src/github.com/keighl/mandrill"
	"github.com/freeusd/solebtc/errors"
	"github.com/freeusd/solebtc/services/mail"
)

// Mailer implements Mail interface for sending email
type Mailer struct {
	fromEmail string
	fromName  string
	client    *mandrill.Client
}

var _ mail.Mailer = Mailer{}

// New returns a Mailer with mandrill client
func New(key, fromEmail, fromName string) Mailer {
	return Mailer{
		fromEmail: fromEmail,
		fromName:  fromName,
		client:    mandrill.ClientWithKey(key),
	}
}

// SendEmail sends email using mandrill api
func (m Mailer) SendEmail(recipients []string, subject, html string) *errors.Error {
	message := &mandrill.Message{}
	for _, recipient := range recipients {
		message.AddRecipient(recipient, "", "to")
	}
	message.FromEmail = m.fromEmail
	message.FromName = m.fromName
	message.Subject = subject
	message.HTML = html

	_, err := m.client.MessagesSend(message)
	if err != nil {
		return &errors.Error{
			ErrCode:             errors.ErrCodeMandrill,
			ErrStringForLogging: fmt.Sprintf("Send email via mandrill error: %v", err),
		}
	}

	return nil
}
<commit_msg>Config mandrill message to be async, inlineCss, important<commit_after>package mandrill

import (
	"fmt"

	"github.com/freeusd/solebtc/Godeps/_workspace/src/github.com/keighl/mandrill"
	"github.com/freeusd/solebtc/errors"
	"github.com/freeusd/solebtc/services/mail"
)

// Mailer implements Mail interface for sending email
type Mailer struct {
	fromEmail string
	fromName  string
	client    *mandrill.Client
}

var _ mail.Mailer = Mailer{}

// New returns a Mailer with mandrill client
func New(key, fromEmail, fromName string) Mailer {
	return Mailer{
		fromEmail: fromEmail,
		fromName:  fromName,
		client:    mandrill.ClientWithKey(key),
	}
}

// SendEmail sends email using mandrill api
func (m Mailer) SendEmail(recipients []string, subject, html string) *errors.Error {
	message := &mandrill.Message{}
	message.Async = true
	message.InlineCSS = true
	message.Important = true
	for _, recipient := range recipients {
		message.AddRecipient(recipient, "", "to")
	}
	message.FromEmail = m.fromEmail
	message.FromName = m.fromName
	message.Subject = subject
	message.HTML = html

	_, err := m.client.MessagesSend(message)
	if err != nil {
		return &errors.Error{
			ErrCode:             errors.ErrCodeMandrill,
			ErrStringForLogging: fmt.Sprintf("Send email via mandrill error: %v", err),
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012-2015, Rolf Veen and contributors.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ogdl

import (
	"errors"
	"fmt"
	// "log"
	"reflect"
	"runtime"
)

// Function enables calling Go functions from templates.
//
// INPUT FORMAT
//
// g is the Function's context. g.This contains the presumed class name.
// The _type subnode of g, if present, contains the function type (a Go
// interface name or 'rfunction'
//
// p is the input path, where i points to the current position to be processed.
// The arguments of the function are 1 level higher than the function name.
// p[ix] points to the class name.
//
// Example 1
//
// !p
//   T
//   !g
//     'some text'
//
// Example 2
// !p
//   math
//   Sin
//   !g
//    !e
//      1.0
//
// Functions calls are limited to whole paths.
//
// TODO: Catch panic() att Call(). Return named variables so that defer/recover
// return something usefull
func (g *Graph) function(path *Graph, typ interface{}) (interface{}, error) {

	defer func() {
		if err := recover(); err != nil {
			fmt.Println(err)
			return
		}
	}()

	// log.Printf("\n%s\n", path.Show())

	v := reflect.ValueOf(typ)

	// Build arguments in the form []reflect.Value
	var vargs []reflect.Value

	switch v.Kind() {

	case reflect.Func:

		//log.Println("function.Func", path.Out[1].ThisString(), path.Out[1].Len())
		//log.Println(runtime.FuncForPC(v.Pointer()).Name())
		//log.Println(reflect.TypeOf(typ).String())

		// Pre-evaluate
		var args []interface{}
		for _, arg := range path.Out[1].Out {
			args = append(args, g.evalExpression(arg))
			// log.Printf("%v\n", args[len(args)-1])
		}

		for i, arg := range args {
			if arg == nil {
				// No untyped nil support :-(
				vargs = append(vargs, reflect.Zero(v.Type().In(i)))
			} else {
				vargs = append(vargs, reflect.ValueOf(arg))
			}
		}

		/* DEBUG CODE
		for i := 0; i < v.Type().NumIn(); i++ {
			log.Println("> ", v.Type().In(i).String())
		}
		for i := 0; i < len(vargs); i++ {
			log.Println("< ", vargs[i].Type().String())
		} /**/

		if v.Type().NumIn() != len(args) {
			// TODO Check that we print the name of the function
			return nil, fmt.Errorf("Invalid number of arguments in function %s (is %d, soll %d)\n%s", runtime.FuncForPC(v.Pointer()).Name(), len(args), v.Type().NumIn(), path.Show())
		}

		// TODO: return 0..2 values
		vv := v.Call(vargs)
		if len(vv) > 0 {
			return vv[0].Interface(), nil
		}
		return nil, nil

	case reflect.Ptr:

		// log.Println("function.Ptr")

		fn := path.GetAt(1)
		if fn == nil {
			return nil, errors.New("No method")
		}
		fname := fn.ThisString()

		// Check if it is a method
		me := v.MethodByName(fname)

		if !me.IsValid() {
			// Try field
			if v.Kind() == reflect.Struct {
				v = v.FieldByName(fname)
				if v.IsValid() {
					return v.Interface(), nil
				}
			}

			return nil, errors.New("No method: " + fname)
		}

		// Pre-evaluate
		var args []interface{}
		if len(path.Out) > 2 {
			for _, arg := range path.Out[2].Out {
				args = append(args, g.evalExpression(arg))
			}
		}

		for i, arg := range args {
			if arg == nil {
				// No untyped nil support :-(
				vargs = append(vargs, reflect.Zero(me.Type().In(i)))
			} else {
				vargs = append(vargs, reflect.ValueOf(arg))
			}
		}

		// TODO: variadic
		/*
			if me.Type().NumIn() != len(args) {
				return nil, errors.New("Invalid number of arguments in method " + fname)
			}

			for i, arg := range args {
				v := reflect.TypeOf(arg)
				if v == nil || me.Type().In(i).String() != v.String() {
					return nil, errors.New("Invalid argument for method " + fname)
				}
			}*/

		// TODO: return 0..2 values
		vv := me.Call(vargs)
		if len(vv) > 0 {
			return vv[0].Interface(), nil
		}
		return nil, nil

	default:
		return nil, nil
	}

}
<commit_msg>detecting remote functions. Much to be improved<commit_after>// Copyright 2012-2015, Rolf Veen and contributors.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ogdl

import (
	"errors"
	"fmt"
	"log"
	"reflect"
	"runtime"
)

// Function enables calling Go functions from templates.
//
// INPUT FORMAT
//
// g is the Function's context. g.This contains the presumed class name.
// The _type subnode of g, if present, contains the function type (a Go
// interface name or 'rfunction'
//
// p is the input path, where i points to the current position to be processed.
// The arguments of the function are 1 level higher than the function name.
// p[ix] points to the class name.
//
// Example 1
//
// !p
//   T
//   !g
//     'some text'
//
// Example 2
// !p
//   math
//   Sin
//   !g
//    !e
//      1.0
//
// Functions calls are limited to whole paths.
//
// TODO: Catch panic() att Call(). Return named variables so that defer/recover
// return something usefull

func (g *Graph) function(path *Graph, typ interface{}) (interface{}, error) {

	defer func() {
		if err := recover(); err != nil {
			fmt.Println(err)
			return
		}
	}()

	// log.Printf("\n%s\n", path.Show())

	v := reflect.ValueOf(typ)

	// Remote functions have this signature
	var f func(*Graph) (*Graph, error)
	rfType := reflect.ValueOf(f).Type()

	// Build arguments in the form []reflect.Value
	var vargs []reflect.Value

	switch v.Kind() {

	case reflect.Func:

		//log.Println("function.Func", path.Out[1].ThisString(), path.Out[1].Len())
		// log.Println("Func type", v.Type())
		//log.Println(runtime.FuncForPC(v.Pointer()).Name())
		//log.Println(reflect.TypeOf(typ).String())

		// Pre-evaluate
		var args []interface{}

		if v.Type() == rfType {
			// Remote function
			n := New()
			nn := n.Add(path.Out[1].This)
			if len(path.Out) > 2 {
				for _, arg := range path.Out[2].Out {
					// log.Printf("arg:\n%s\n", arg.Show())
					nn.Add(g.evalExpression(arg))
				}
			}

			log.Println(n.Show())
			args = append(args, n)
		} else {
			// Local function
			for _, arg := range path.Out[1].Out {
				args = append(args, g.evalExpression(arg))
				// log.Printf("%v\n", args[len(args)-1])
			}
		}

		for i, arg := range args {
			if arg == nil {
				// No untyped nil support :-(
				vargs = append(vargs, reflect.Zero(v.Type().In(i)))
			} else {
				vargs = append(vargs, reflect.ValueOf(arg))
			}
		}

		/* DEBUG CODE
		for i := 0; i < v.Type().NumIn(); i++ {
			log.Println("> ", v.Type().In(i).String())
		}
		for i := 0; i < len(vargs); i++ {
			log.Println("< ", vargs[i].Type().String())
		} /**/

		if v.Type().NumIn() != len(args) {
			// TODO Check that we print the name of the function
			return nil, fmt.Errorf("Invalid number of arguments in function %s (is %d, soll %d)\n%s", runtime.FuncForPC(v.Pointer()).Name(), len(args), v.Type().NumIn(), path.Show())
		}

		// TODO: return 0..2 values
		vv := v.Call(vargs)
		if len(vv) > 0 {
			return vv[0].Interface(), nil
		}
		return nil, nil

	case reflect.Ptr:

		// log.Println("function.Ptr")

		fn := path.GetAt(1)
		if fn == nil {
			return nil, errors.New("No method")
		}
		fname := fn.ThisString()

		// Check if it is a method
		me := v.MethodByName(fname)

		if !me.IsValid() {
			// Try field
			if v.Kind() == reflect.Struct {
				v = v.FieldByName(fname)
				if v.IsValid() {
					return v.Interface(), nil
				}
			}

			return nil, errors.New("No method: " + fname)
		}

		// Pre-evaluate
		var args []interface{}
		if len(path.Out) > 2 {
			for _, arg := range path.Out[2].Out {
				args = append(args, g.evalExpression(arg))
			}
		}

		for i, arg := range args {
			if arg == nil {
				// No untyped nil support :-(
				vargs = append(vargs, reflect.Zero(me.Type().In(i)))
			} else {
				vargs = append(vargs, reflect.ValueOf(arg))
			}
		}

		// TODO: variadic
		/*
			if me.Type().NumIn() != len(args) {
				return nil, errors.New("Invalid number of arguments in method " + fname)
			}

			for i, arg := range args {
				v := reflect.TypeOf(arg)
				if v == nil || me.Type().In(i).String() != v.String() {
					return nil, errors.New("Invalid argument for method " + fname)
				}
			}*/

		// TODO: return 0..2 values
		vv := me.Call(vargs)
		if len(vv) > 0 {
			return vv[0].Interface(), nil
		}
		return nil, nil

	default:
		return nil, nil
	}

}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"regexp"
	"strings"
	"testing"
)

func testcmd(cmd string) (string, error) {
	fmt.Println("(T) " + cmd)
	switch {
	case cmd == "sudo ls -a1F /mnt/sda1/var/lib/docker/vfs/dir":
		return currenttest.vs.ls(), nil
	case cmd == "docker ps -aq --no-trunc":
		return "", nil
	case strings.HasPrefix(cmd, "docker inspect -f '{{ .Name }},{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' "):
		return "", nil
	case strings.HasPrefix(cmd, "sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):
		deleted := cmd[len("sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):]
		deletions = append(deletions, deleted)
		return "", nil
	case strings.HasPrefix(cmd, "sudo readlink /mnt/sda1/var/lib/docker/vfs/dir/"):
		if strings.Contains(cmd, ",nonexistent") {
			return "", errors.New("non-existent linked folder")
		}
		r := regexp.MustCompile(`.*\$([^#]+)###.*`)
		ss := r.FindStringSubmatch(cmd)
		if len(ss) == 2 {
			folder := ss[1]
			folder = folder + strings.Repeat("1", 64-len(folder))
			return folder, nil
		}
		return "", nil
	case strings.HasPrefix(cmd, "sudo ls /mnt/sda1/var/lib/docker/vfs/dir/"):
		if cmd == "sudo ls /mnt/sda1/var/lib/docker/vfs/dir/" {
			return "", errors.New("non-ls linked folder")
		}
		return "", nil
	default:
		currentT.Fatalf("test '%s': unknown command!\n", cmd)
		return fmt.Sprintf("test '%s'", cmd), errors.New("unknown command")
	}
}

type volspecs []string
type Test struct {
	title string
	vs    volspecs
	res   []int
}

func (vs volspecs) ls() string {
	if len(vs) == 0 {
		return ""
	}
	res := ""
	for i, spec := range vs {
		if strings.HasSuffix(spec, "/") {
			spec = spec[:len(spec)-1]
			res = res + spec + strings.Repeat(fmt.Sprintf("%d", i), 64-len(spec)) + "/\n"
		}
		if strings.HasSuffix(spec, "@") {
			mp := "." + strings.Replace(spec, ";", "###", -1)
			mp = strings.Replace(mp, "/", ",#,", -1)
			res = res + mp + "\n"
		}
	}
	return res
}

var deletions = []string{}
var tests = []Test{
	Test{"empty vfs", []string{}, []int{0, 0, 0, 0, 0}},
	Test{"two volumes", []string{"fa/", "fb/"}, []int{0, 0, 2, 2, 0}},
	Test{"Invalid (ill-formed) markers must be deleted", []string{"cainv/path/a@"}, []int{0, 0, 0, 0, -1}},
	Test{"Invalid (no readlink) markers must be deleted", []string{"ca;/path/nonexistenta@", "cb;/path/nonexistentb@"}, []int{0, 0, 0, 0, -2}},
	Test{"Invalid (no ls) markers must be deleted", []string{"ca;/path/nolsa@", "cb;/path/nolsb@"}, []int{0, 0, 0, 0, -2}},
	Test{"Invalid (no vdir) markers must be deleted", []string{"ca$novdira;/path/nolsa@", "cb$novdirb;/path/nolsb@"}, []int{0, 0, 0, 0, -2}},
}
var currenttest Test
var currentT *testing.T

// TestContainers test different vfs scenarios
func TestContainers(t *testing.T) {
	cmd = testcmd
	currentT = t
	for i, test := range tests {
		currenttest = test
		deletions = []string{}
		fmt.Println("------ vvv " + test.title + " vvv ------")
		main()
		tc := Containers()
		toc := OrphanedContainers()
		tv := Volumes()
		tov := OrphanedVolumes()
		tm := Markers()
		if len(tc) != test.res[0] {
			t.Errorf("Test %d: '%s' expected '%d' containers, got '%d'", i+1, test.title, test.res[0], len(tc))
		}
		if len(toc) != test.res[1] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned containers, got '%d'", i+1, test.title, test.res[1], len(toc))
		}
		if len(tv) != test.res[2] {
			t.Errorf("Test %d: '%s' expected '%d' volumes, got '%d'", i+1, test.title, test.res[2], len(tv))
		}
		if len(tov) != test.res[3] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned volumes, got '%d'", i+1, test.title, test.res[3], len(tov))
		}
		if nbmarkers(tm) != test.res[4] {
			t.Errorf("Test %d: '%s' expected '%d' markers, got '%d'", i+1, test.title, test.res[4], nbmarkers(tm))
		}
		fmt.Println("------ ^^^ " + test.title + " ^^^ ------")
		fmt.Println("----------")
	}
}

func nbmarkers(tm markers) int {
	res := len(tm)
	for _, d := range deletions {
		if strings.HasPrefix(d, ".") {
			res = res - 1
		}
	}
	return res
}
<commit_msg>gcl_test.go:add valid markers test<commit_after>package main

import (
	"errors"
	"fmt"
	"regexp"
	"strings"
	"testing"
)

func testcmd(cmd string) (string, error) {
	fmt.Println("(T) " + cmd)
	switch {
	case cmd == "sudo ls -a1F /mnt/sda1/var/lib/docker/vfs/dir":
		return currenttest.vs.ls(), nil
	case cmd == "docker ps -aq --no-trunc":
		return "", nil
	case strings.HasPrefix(cmd, "docker inspect -f '{{ .Name }},{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' "):
		return "", nil
	case strings.HasPrefix(cmd, "sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):
		deleted := cmd[len("sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):]
		deletions = append(deletions, deleted)
		return "", nil
	case strings.HasPrefix(cmd, "sudo readlink /mnt/sda1/var/lib/docker/vfs/dir/"):
		if strings.Contains(cmd, ",nonexistent") {
			return "", errors.New("non-existent linked folder")
		}
		r := regexp.MustCompile(`.*\$([^#]+)###.*`)
		ss := r.FindStringSubmatch(cmd)
		if len(ss) == 2 {
			folder := ss[1]
			folder = folder + strings.Repeat("1", 64-len(folder))
			return folder, nil
		}
		return "", nil
	case strings.HasPrefix(cmd, "sudo ls /mnt/sda1/var/lib/docker/vfs/dir/"):
		if cmd == "sudo ls /mnt/sda1/var/lib/docker/vfs/dir/" {
			return "", errors.New("non-ls linked folder")
		}
		return "", nil
	default:
		currentT.Fatalf("test '%s': unknown command!\n", cmd)
		return fmt.Sprintf("test '%s'", cmd), errors.New("unknown command")
	}
}

type volspecs []string
type Test struct {
	title string
	vs    volspecs
	res   []int
}

func (vs volspecs) ls() string {
	if len(vs) == 0 {
		return ""
	}
	res := ""
	for i, spec := range vs {
		if strings.HasSuffix(spec, "/") {
			spec = spec[:len(spec)-1]
			res = res + spec + strings.Repeat(fmt.Sprintf("%d", i), 64-len(spec)) + "/\n"
		}
		if strings.HasSuffix(spec, "@") {
			mp := "." + strings.Replace(spec, ";", "###", -1)
			mp = strings.Replace(mp, "/", ",#,", -1)
			res = res + mp + "\n"
		}
	}
	return res
}

var deletions = []string{}
var tests = []Test{
	Test{"empty vfs", []string{}, []int{0, 0, 0, 0, 0}},
	Test{"two volumes", []string{"fa/", "fb/"}, []int{0, 0, 2, 2, 0}},
	Test{"Invalid (ill-formed) markers must be deleted", []string{"cainv/path/a@"}, []int{0, 0, 0, 0, -1}},
	Test{"Invalid (no readlink) markers must be deleted", []string{"ca;/path/nonexistenta@", "cb;/path/nonexistentb@"}, []int{0, 0, 0, 0, -2}},
	Test{"Invalid (no ls) markers must be deleted", []string{"ca;/path/nolsa@", "cb;/path/nolsb@"}, []int{0, 0, 0, 0, -2}},
	Test{"Invalid (no vdir) markers must be deleted", []string{"ca$novdira;/path/nolsa@", "cb$novdirb;/path/nolsb@"}, []int{0, 0, 0, 0, -2}},
	Test{"two valid markers", []string{"ca$fa;/path/vola@", "cb$fb;/path/volb@"}, []int{0, 0, 0, 0, 2}},
}
var currenttest Test
var currentT *testing.T

// TestContainers test different vfs scenarios
func TestContainers(t *testing.T) {
	cmd = testcmd
	currentT = t
	for i, test := range tests {
		currenttest = test
		deletions = []string{}
		fmt.Println("------ vvv " + test.title + " vvv ------")
		main()
		tc := Containers()
		toc := OrphanedContainers()
		tv := Volumes()
		tov := OrphanedVolumes()
		tm := Markers()
		if len(tc) != test.res[0] {
			t.Errorf("Test %d: '%s' expected '%d' containers, got '%d'", i+1, test.title, test.res[0], len(tc))
		}
		if len(toc) != test.res[1] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned containers, got '%d'", i+1, test.title, test.res[1], len(toc))
		}
		if len(tv) != test.res[2] {
			t.Errorf("Test %d: '%s' expected '%d' volumes, got '%d'", i+1, test.title, test.res[2], len(tv))
		}
		if len(tov) != test.res[3] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned volumes, got '%d'", i+1, test.title, test.res[3], len(tov))
		}
		if nbmarkers(tm) != test.res[4] {
			t.Errorf("Test %d: '%s' expected '%d' markers, got '%d'", i+1, test.title, test.res[4], nbmarkers(tm))
		}
		fmt.Println("------ ^^^ " + test.title + " ^^^ ------")
		fmt.Println("----------")
	}
}

func nbmarkers(tm markers) int {
	res := len(tm)
	for _, d := range deletions {
		if strings.HasPrefix(d, ".") {
			res = res - 1
		}
	}
	return res
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"go/build"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
)

var cmdTool = &Command{
	Run:       runTool,
	UsageLine: "tool command [args...]",
	Short:     "run specified go tool",
	Long: `
Tool runs the go tool command identified by the arguments.
With no arguments it prints the list of known tools.

For more about each tool command, see 'go tool command -h'.
`,
}

var (
	toolGOOS      = runtime.GOOS
	toolGOARCH    = runtime.GOARCH
	toolIsWindows = toolGOOS == "windows"
	toolDir       = filepath.Join(build.Path[0].Path, "bin", "go-tool")
)

const toolWindowsExtension = ".exe"

func tool(name string) string {
	p := filepath.Join(toolDir, name)
	if toolIsWindows {
		p += toolWindowsExtension
	}
	return p
}

func runTool(cmd *Command, args []string) {
	if len(args) == 0 {
		listTools()
		return
	}
	toolName := args[0]
	// The tool name must be lower-case letters and numbers.
	for _, c := range toolName {
		switch {
		case 'a' <= c && c <= 'z', '0' <= c && c <= '9':
		default:
			fmt.Fprintf(os.Stderr, "go tool: bad tool name %q\n", tool)
			setExitStatus(2)
			return
		}
	}
	toolPath := tool(toolName)
	// Give a nice message if there is no tool with that name.
	if _, err := os.Stat(toolPath); err != nil {
		fmt.Fprintf(os.Stderr, "go tool: no such tool %q\n", tool)
		setExitStatus(3)
		return
	}
	toolCmd := &exec.Cmd{
		Path:   toolPath,
		Args:   args,
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}
	err := toolCmd.Run()
	if err != nil {
		fmt.Fprintf(os.Stderr, "go tool %s: %s\n", toolName, err)
		setExitStatus(1)
		return
	}
}

// listTools prints a list of the available tools in the go-tools directory.
func listTools() {
	f, err := os.Open(toolDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "go tool: no tool directory: %s\n", err)
		setExitStatus(2)
		return
	}
	defer f.Close()
	names, err := f.Readdirnames(-1)
	if err != nil {
		fmt.Fprintf(os.Stderr, "go tool: can't read directory: %s\n", err)
		setExitStatus(2)
		return
	}
	sort.Strings(names)
	for _, name := range names {
		// Unify presentation by going to lower case.
		name = strings.ToLower(name)
		// If it's windows, don't show the .exe suffix.
		if toolIsWindows && strings.HasSuffix(name, toolWindowsExtension) {
			name = name[:len(name)-len(toolWindowsExtension)]
		}
		fmt.Println(name)
	}
}
<commit_msg>cmd/go: fix error message on non-existing tools.<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"go/build"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
)

var cmdTool = &Command{
	Run:       runTool,
	UsageLine: "tool command [args...]",
	Short:     "run specified go tool",
	Long: `
Tool runs the go tool command identified by the arguments.
With no arguments it prints the list of known tools.

For more about each tool command, see 'go tool command -h'.
`,
}

var (
	toolGOOS      = runtime.GOOS
	toolGOARCH    = runtime.GOARCH
	toolIsWindows = toolGOOS == "windows"
	toolDir       = filepath.Join(build.Path[0].Path, "bin", "go-tool")
)

const toolWindowsExtension = ".exe"

func tool(name string) string {
	p := filepath.Join(toolDir, name)
	if toolIsWindows {
		p += toolWindowsExtension
	}
	return p
}

func runTool(cmd *Command, args []string) {
	if len(args) == 0 {
		listTools()
		return
	}
	toolName := args[0]
	// The tool name must be lower-case letters and numbers.
	for _, c := range toolName {
		switch {
		case 'a' <= c && c <= 'z', '0' <= c && c <= '9':
		default:
			fmt.Fprintf(os.Stderr, "go tool: bad tool name %q\n", toolName)
			setExitStatus(2)
			return
		}
	}
	toolPath := tool(toolName)
	// Give a nice message if there is no tool with that name.
	if _, err := os.Stat(toolPath); err != nil {
		fmt.Fprintf(os.Stderr, "go tool: no such tool %q\n", toolName)
		setExitStatus(3)
		return
	}
	toolCmd := &exec.Cmd{
		Path:   toolPath,
		Args:   args,
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}
	err := toolCmd.Run()
	if err != nil {
		fmt.Fprintf(os.Stderr, "go tool %s: %s\n", toolName, err)
		setExitStatus(1)
		return
	}
}

// listTools prints a list of the available tools in the go-tools directory.
func listTools() {
	f, err := os.Open(toolDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "go tool: no tool directory: %s\n", err)
		setExitStatus(2)
		return
	}
	defer f.Close()
	names, err := f.Readdirnames(-1)
	if err != nil {
		fmt.Fprintf(os.Stderr, "go tool: can't read directory: %s\n", err)
		setExitStatus(2)
		return
	}
	sort.Strings(names)
	for _, name := range names {
		// Unify presentation by going to lower case.
		name = strings.ToLower(name)
		// If it's windows, don't show the .exe suffix.
		if toolIsWindows && strings.HasSuffix(name, toolWindowsExtension) {
			name = name[:len(name)-len(toolWindowsExtension)]
		}
		fmt.Println(name)
	}
}
<|endoftext|>