<commit_before>package s3

import (
	"context"
	"fmt"
	"io"
	"os"
	"path"
	"restic"
	"strings"
	"time"

	"restic/backend"
	"restic/errors"

	"github.com/minio/minio-go"

	"restic/debug"
)

// Backend stores data on an S3 endpoint.
type Backend struct {
	client *minio.Client
	sem    *backend.Semaphore
	cfg    Config
	backend.Layout
}

// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}

const defaultLayout = "default"

func open(cfg Config) (*Backend, error) {
	debug.Log("open, config %#v", cfg)

	if cfg.MaxRetries > 0 {
		minio.MaxRetry = int(cfg.MaxRetries)
	}

	client, err := minio.New(cfg.Endpoint, cfg.KeyID, cfg.Secret, !cfg.UseHTTP)
	if err != nil {
		return nil, errors.Wrap(err, "minio.New")
	}

	sem, err := backend.NewSemaphore(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &Backend{
		client: client,
		sem:    sem,
		cfg:    cfg,
	}

	client.SetCustomTransport(backend.Transport())

	l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix)
	if err != nil {
		return nil, err
	}

	be.Layout = l

	return be, nil
}

// Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet.
func Open(cfg Config) (restic.Backend, error) {
	return open(cfg)
}

// Create opens the S3 backend at bucket and region and creates the bucket if
// it does not exist yet.
func Create(cfg Config) (restic.Backend, error) {
	be, err := open(cfg)
	found, err := be.client.BucketExists(cfg.Bucket)
	if err != nil {
		debug.Log("BucketExists(%v) returned err %v", cfg.Bucket, err)
		return nil, errors.Wrap(err, "client.BucketExists")
	}

	if !found {
		// create new bucket with default ACL in default region
		err = be.client.MakeBucket(cfg.Bucket, "")
		if err != nil {
			return nil, errors.Wrap(err, "client.MakeBucket")
		}
	}

	return be, nil
}

// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
	debug.Log("IsNotExist(%T, %#v)", err, err)
	if os.IsNotExist(errors.Cause(err)) {
		return true
	}

	if e, ok := errors.Cause(err).(minio.ErrorResponse); ok && e.Code == "NoSuchKey" {
		return true
	}

	return false
}

// Join combines path components with slashes.
func (be *Backend) Join(p ...string) string {
	return path.Join(p...)
}

type fileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
	isDir   bool
}

func (fi fileInfo) Name() string       { return fi.name }    // base name of the file
func (fi fileInfo) Size() int64        { return fi.size }    // length in bytes for regular files; system-dependent for others
func (fi fileInfo) Mode() os.FileMode  { return fi.mode }    // file mode bits
func (fi fileInfo) ModTime() time.Time { return fi.modTime } // modification time
func (fi fileInfo) IsDir() bool        { return fi.isDir }   // abbreviation for Mode().IsDir()
func (fi fileInfo) Sys() interface{}   { return nil }        // underlying data source (can return nil)

// ReadDir returns the entries for a directory.
func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) {
	debug.Log("ReadDir(%v)", dir)

	// make sure dir ends with a slash
	if dir[len(dir)-1] != '/' {
		dir += "/"
	}

	done := make(chan struct{})
	defer close(done)

	for obj := range be.client.ListObjects(be.cfg.Bucket, dir, false, done) {
		if obj.Key == "" {
			continue
		}

		name := strings.TrimPrefix(obj.Key, dir)
		if name == "" {
			return nil, errors.Errorf("invalid key name %v, removing prefix %v yielded empty string", obj.Key, dir)
		}
		entry := fileInfo{
			name:    name,
			size:    obj.Size,
			modTime: obj.LastModified,
		}

		if name[len(name)-1] == '/' {
			entry.isDir = true
			entry.mode = os.ModeDir | 0755
			entry.name = name[:len(name)-1]
		} else {
			entry.mode = 0644
		}

		list = append(list, entry)
	}

	return list, nil
}

// Location returns this backend's location (the bucket name).
func (be *Backend) Location() string {
	return be.Join(be.cfg.Bucket, be.cfg.Prefix)
}

// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
	return be.cfg.Prefix
}

// nopCloserFile wraps *os.File and overwrites the Close() method with method
// that does nothing. In addition, the method Len() is implemented, which
// returns the size of the file (filesize - current offset).
type nopCloserFile struct {
	*os.File
}

func (f nopCloserFile) Close() error {
	debug.Log("prevented Close()")
	return nil
}

// Len returns the remaining length of the file (filesize - current offset).
func (f nopCloserFile) Len() int {
	debug.Log("Len() called")
	fi, err := f.Stat()
	if err != nil {
		panic(err)
	}

	pos, err := f.Seek(0, io.SeekCurrent)
	if err != nil {
		panic(err)
	}

	size := fi.Size() - pos
	debug.Log("returning file size %v", size)
	return int(size)
}

type lenner interface {
	Len() int
	io.Reader
}

// nopCloserLenner wraps a lenner and overwrites the Close() method with method
// that does nothing. In addition, the method Size() is implemented, which
// returns the size of the file (filesize - current offset).
type nopCloserLenner struct {
	lenner
}

func (f *nopCloserLenner) Close() error {
	debug.Log("prevented Close()")
	return nil
}

// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
	debug.Log("Save %v", h)

	if err := h.Valid(); err != nil {
		return err
	}

	objName := be.Filename(h)

	// Check key does not already exist
	_, err = be.client.StatObject(be.cfg.Bucket, objName)
	if err == nil {
		debug.Log("%v already exists", h)
		return errors.New("key already exists")
	}

	// prevent the HTTP client from closing a file
	if f, ok := rd.(*os.File); ok {
		debug.Log("reader is %#T, using nopCloserFile{}", rd)
		rd = nopCloserFile{f}
	} else if l, ok := rd.(lenner); ok {
		debug.Log("reader is %#T, using nopCloserLenner{}", rd)
		rd = nopCloserLenner{l}
	} else {
		debug.Log("reader is %#T, no specific workaround enabled", rd)
	}

	be.sem.GetToken()
	debug.Log("PutObject(%v, %v)", be.cfg.Bucket, objName)
	n, err := be.client.PutObject(be.cfg.Bucket, objName, rd, "application/octet-stream")
	be.sem.ReleaseToken()

	debug.Log("%v -> %v bytes, err %#v: %v", objName, n, err, err)

	return errors.Wrap(err, "client.PutObject")
}

// wrapReader wraps an io.ReadCloser to run an additional function on Close.
type wrapReader struct {
	io.ReadCloser
	f func()
}

func (wr wrapReader) Close() error {
	err := wr.ReadCloser.Close()
	wr.f()
	return err
}

// Load returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
	if err := h.Valid(); err != nil {
		return nil, err
	}

	if offset < 0 {
		return nil, errors.New("offset is negative")
	}

	if length < 0 {
		return nil, errors.Errorf("invalid length %d", length)
	}

	objName := be.Filename(h)

	byteRange := fmt.Sprintf("bytes=%d-", offset)
	if length > 0 {
		byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
	}
	headers := minio.NewGetReqHeaders()
	headers.Add("Range", byteRange)

	be.sem.GetToken()
	debug.Log("Load(%v) send range %v", h, byteRange)

	coreClient := minio.Core{Client: be.client}
	rd, _, err := coreClient.GetObject(be.cfg.Bucket, objName, headers)
	if err != nil {
		be.sem.ReleaseToken()
		return nil, err
	}

	closeRd := wrapReader{
		ReadCloser: rd,
		f: func() {
			debug.Log("Close()")
			be.sem.ReleaseToken()
		},
	}

	return closeRd, err
}

// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
	debug.Log("%v", h)

	objName := be.Filename(h)
	var obj *minio.Object

	obj, err = be.client.GetObject(be.cfg.Bucket, objName)
	if err != nil {
		debug.Log("GetObject() err %v", err)
		return restic.FileInfo{}, errors.Wrap(err, "client.GetObject")
	}

	// make sure that the object is closed properly.
	defer func() {
		e := obj.Close()
		if err == nil {
			err = errors.Wrap(e, "Close")
		}
	}()

	fi, err := obj.Stat()
	if err != nil {
		debug.Log("Stat() err %v", err)
		return restic.FileInfo{}, errors.Wrap(err, "Stat")
	}

	return restic.FileInfo{Size: fi.Size}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
	found := false
	objName := be.Filename(h)
	_, err := be.client.StatObject(be.cfg.Bucket, objName)
	if err == nil {
		found = true
	}

	// If error, then not found
	return found, nil
}

// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
	objName := be.Filename(h)
	err := be.client.RemoveObject(be.cfg.Bucket, objName)
	debug.Log("Remove(%v) at %v -> err %v", h, objName, err)

	if be.IsNotExist(err) {
		err = nil
	}

	return errors.Wrap(err, "client.RemoveObject")
}

// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
	debug.Log("listing %v", t)
	ch := make(chan string)

	prefix := be.Dirname(restic.Handle{Type: t})

	// make sure prefix ends with a slash
	if prefix[len(prefix)-1] != '/' {
		prefix += "/"
	}

	listresp := be.client.ListObjects(be.cfg.Bucket, prefix, true, ctx.Done())

	go func() {
		defer close(ch)
		for obj := range listresp {
			m := strings.TrimPrefix(obj.Key, prefix)
			if m == "" {
				continue
			}

			select {
			case ch <- path.Base(m):
			case <-ctx.Done():
				return
			}
		}
	}()

	return ch
}

// Remove keys for a specified backend type.
func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error {
	for key := range be.List(ctx, restic.DataFile) {
		err := be.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key})
		if err != nil {
			return err
		}
	}

	return nil
}

// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
	alltypes := []restic.FileType{
		restic.DataFile,
		restic.KeyFile,
		restic.LockFile,
		restic.SnapshotFile,
		restic.IndexFile}

	for _, t := range alltypes {
		err := be.removeKeys(ctx, t)
		if err != nil {
			return nil
		}
	}

	return be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
}

// Close does nothing
func (be *Backend) Close() error { return nil }

// Rename moves a file based on the new layout l.
func (be *Backend) Rename(h restic.Handle, l backend.Layout) error {
	debug.Log("Rename %v to %v", h, l)
	oldname := be.Filename(h)
	newname := l.Filename(h)

	if oldname == newname {
		debug.Log("  %v is already renamed", newname)
		return nil
	}

	debug.Log("  %v -> %v", oldname, newname)

	coreClient := minio.Core{Client: be.client}
	err := coreClient.CopyObject(be.cfg.Bucket, newname, path.Join(be.cfg.Bucket, oldname), minio.CopyConditions{})
	if err != nil {
		debug.Log("copy failed: %v", err)
		return err
	}

	return be.client.RemoveObject(be.cfg.Bucket, oldname)
}
<commit_msg>s3 migrate layout: Ignore already renamed files<commit_after>package s3

import (
	"context"
	"fmt"
	"io"
	"os"
	"path"
	"restic"
	"strings"
	"time"

	"restic/backend"
	"restic/errors"

	"github.com/minio/minio-go"

	"restic/debug"
)

// Backend stores data on an S3 endpoint.
type Backend struct {
	client *minio.Client
	sem    *backend.Semaphore
	cfg    Config
	backend.Layout
}

// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}

const defaultLayout = "default"

func open(cfg Config) (*Backend, error) {
	debug.Log("open, config %#v", cfg)

	if cfg.MaxRetries > 0 {
		minio.MaxRetry = int(cfg.MaxRetries)
	}

	client, err := minio.New(cfg.Endpoint, cfg.KeyID, cfg.Secret, !cfg.UseHTTP)
	if err != nil {
		return nil, errors.Wrap(err, "minio.New")
	}

	sem, err := backend.NewSemaphore(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &Backend{
		client: client,
		sem:    sem,
		cfg:    cfg,
	}

	client.SetCustomTransport(backend.Transport())

	l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix)
	if err != nil {
		return nil, err
	}

	be.Layout = l

	return be, nil
}

// Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet.
func Open(cfg Config) (restic.Backend, error) {
	return open(cfg)
}

// Create opens the S3 backend at bucket and region and creates the bucket if
// it does not exist yet.
func Create(cfg Config) (restic.Backend, error) {
	be, err := open(cfg)
	found, err := be.client.BucketExists(cfg.Bucket)
	if err != nil {
		debug.Log("BucketExists(%v) returned err %v", cfg.Bucket, err)
		return nil, errors.Wrap(err, "client.BucketExists")
	}

	if !found {
		// create new bucket with default ACL in default region
		err = be.client.MakeBucket(cfg.Bucket, "")
		if err != nil {
			return nil, errors.Wrap(err, "client.MakeBucket")
		}
	}

	return be, nil
}

// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
	debug.Log("IsNotExist(%T, %#v)", err, err)
	if os.IsNotExist(errors.Cause(err)) {
		return true
	}

	if e, ok := errors.Cause(err).(minio.ErrorResponse); ok && e.Code == "NoSuchKey" {
		return true
	}

	return false
}

// Join combines path components with slashes.
func (be *Backend) Join(p ...string) string {
	return path.Join(p...)
}

type fileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
	isDir   bool
}

func (fi fileInfo) Name() string       { return fi.name }    // base name of the file
func (fi fileInfo) Size() int64        { return fi.size }    // length in bytes for regular files; system-dependent for others
func (fi fileInfo) Mode() os.FileMode  { return fi.mode }    // file mode bits
func (fi fileInfo) ModTime() time.Time { return fi.modTime } // modification time
func (fi fileInfo) IsDir() bool        { return fi.isDir }   // abbreviation for Mode().IsDir()
func (fi fileInfo) Sys() interface{}   { return nil }        // underlying data source (can return nil)

// ReadDir returns the entries for a directory.
func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) {
	debug.Log("ReadDir(%v)", dir)

	// make sure dir ends with a slash
	if dir[len(dir)-1] != '/' {
		dir += "/"
	}

	done := make(chan struct{})
	defer close(done)

	for obj := range be.client.ListObjects(be.cfg.Bucket, dir, false, done) {
		if obj.Key == "" {
			continue
		}

		name := strings.TrimPrefix(obj.Key, dir)
		if name == "" {
			return nil, errors.Errorf("invalid key name %v, removing prefix %v yielded empty string", obj.Key, dir)
		}
		entry := fileInfo{
			name:    name,
			size:    obj.Size,
			modTime: obj.LastModified,
		}

		if name[len(name)-1] == '/' {
			entry.isDir = true
			entry.mode = os.ModeDir | 0755
			entry.name = name[:len(name)-1]
		} else {
			entry.mode = 0644
		}

		list = append(list, entry)
	}

	return list, nil
}

// Location returns this backend's location (the bucket name).
func (be *Backend) Location() string {
	return be.Join(be.cfg.Bucket, be.cfg.Prefix)
}

// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
	return be.cfg.Prefix
}

// nopCloserFile wraps *os.File and overwrites the Close() method with method
// that does nothing. In addition, the method Len() is implemented, which
// returns the size of the file (filesize - current offset).
type nopCloserFile struct {
	*os.File
}

func (f nopCloserFile) Close() error {
	debug.Log("prevented Close()")
	return nil
}

// Len returns the remaining length of the file (filesize - current offset).
func (f nopCloserFile) Len() int {
	debug.Log("Len() called")
	fi, err := f.Stat()
	if err != nil {
		panic(err)
	}

	pos, err := f.Seek(0, io.SeekCurrent)
	if err != nil {
		panic(err)
	}

	size := fi.Size() - pos
	debug.Log("returning file size %v", size)
	return int(size)
}

type lenner interface {
	Len() int
	io.Reader
}

// nopCloserLenner wraps a lenner and overwrites the Close() method with method
// that does nothing. In addition, the method Size() is implemented, which
// returns the size of the file (filesize - current offset).
type nopCloserLenner struct {
	lenner
}

func (f *nopCloserLenner) Close() error {
	debug.Log("prevented Close()")
	return nil
}

// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
	debug.Log("Save %v", h)

	if err := h.Valid(); err != nil {
		return err
	}

	objName := be.Filename(h)

	// Check key does not already exist
	_, err = be.client.StatObject(be.cfg.Bucket, objName)
	if err == nil {
		debug.Log("%v already exists", h)
		return errors.New("key already exists")
	}

	// prevent the HTTP client from closing a file
	if f, ok := rd.(*os.File); ok {
		debug.Log("reader is %#T, using nopCloserFile{}", rd)
		rd = nopCloserFile{f}
	} else if l, ok := rd.(lenner); ok {
		debug.Log("reader is %#T, using nopCloserLenner{}", rd)
		rd = nopCloserLenner{l}
	} else {
		debug.Log("reader is %#T, no specific workaround enabled", rd)
	}

	be.sem.GetToken()
	debug.Log("PutObject(%v, %v)", be.cfg.Bucket, objName)
	n, err := be.client.PutObject(be.cfg.Bucket, objName, rd, "application/octet-stream")
	be.sem.ReleaseToken()

	debug.Log("%v -> %v bytes, err %#v: %v", objName, n, err, err)

	return errors.Wrap(err, "client.PutObject")
}

// wrapReader wraps an io.ReadCloser to run an additional function on Close.
type wrapReader struct {
	io.ReadCloser
	f func()
}

func (wr wrapReader) Close() error {
	err := wr.ReadCloser.Close()
	wr.f()
	return err
}

// Load returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
	if err := h.Valid(); err != nil {
		return nil, err
	}

	if offset < 0 {
		return nil, errors.New("offset is negative")
	}

	if length < 0 {
		return nil, errors.Errorf("invalid length %d", length)
	}

	objName := be.Filename(h)

	byteRange := fmt.Sprintf("bytes=%d-", offset)
	if length > 0 {
		byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
	}
	headers := minio.NewGetReqHeaders()
	headers.Add("Range", byteRange)

	be.sem.GetToken()
	debug.Log("Load(%v) send range %v", h, byteRange)

	coreClient := minio.Core{Client: be.client}
	rd, _, err := coreClient.GetObject(be.cfg.Bucket, objName, headers)
	if err != nil {
		be.sem.ReleaseToken()
		return nil, err
	}

	closeRd := wrapReader{
		ReadCloser: rd,
		f: func() {
			debug.Log("Close()")
			be.sem.ReleaseToken()
		},
	}

	return closeRd, err
}

// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
	debug.Log("%v", h)

	objName := be.Filename(h)
	var obj *minio.Object

	obj, err = be.client.GetObject(be.cfg.Bucket, objName)
	if err != nil {
		debug.Log("GetObject() err %v", err)
		return restic.FileInfo{}, errors.Wrap(err, "client.GetObject")
	}

	// make sure that the object is closed properly.
	defer func() {
		e := obj.Close()
		if err == nil {
			err = errors.Wrap(e, "Close")
		}
	}()

	fi, err := obj.Stat()
	if err != nil {
		debug.Log("Stat() err %v", err)
		return restic.FileInfo{}, errors.Wrap(err, "Stat")
	}

	return restic.FileInfo{Size: fi.Size}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
	found := false
	objName := be.Filename(h)
	_, err := be.client.StatObject(be.cfg.Bucket, objName)
	if err == nil {
		found = true
	}

	// If error, then not found
	return found, nil
}

// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
	objName := be.Filename(h)
	err := be.client.RemoveObject(be.cfg.Bucket, objName)
	debug.Log("Remove(%v) at %v -> err %v", h, objName, err)

	if be.IsNotExist(err) {
		err = nil
	}

	return errors.Wrap(err, "client.RemoveObject")
}

// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
	debug.Log("listing %v", t)
	ch := make(chan string)

	prefix := be.Dirname(restic.Handle{Type: t})

	// make sure prefix ends with a slash
	if prefix[len(prefix)-1] != '/' {
		prefix += "/"
	}

	listresp := be.client.ListObjects(be.cfg.Bucket, prefix, true, ctx.Done())

	go func() {
		defer close(ch)
		for obj := range listresp {
			m := strings.TrimPrefix(obj.Key, prefix)
			if m == "" {
				continue
			}

			select {
			case ch <- path.Base(m):
			case <-ctx.Done():
				return
			}
		}
	}()

	return ch
}

// Remove keys for a specified backend type.
func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error {
	for key := range be.List(ctx, restic.DataFile) {
		err := be.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key})
		if err != nil {
			return err
		}
	}

	return nil
}

// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
	alltypes := []restic.FileType{
		restic.DataFile,
		restic.KeyFile,
		restic.LockFile,
		restic.SnapshotFile,
		restic.IndexFile}

	for _, t := range alltypes {
		err := be.removeKeys(ctx, t)
		if err != nil {
			return nil
		}
	}

	return be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
}

// Close does nothing
func (be *Backend) Close() error { return nil }

// Rename moves a file based on the new layout l.
func (be *Backend) Rename(h restic.Handle, l backend.Layout) error {
	debug.Log("Rename %v to %v", h, l)
	oldname := be.Filename(h)
	newname := l.Filename(h)

	if oldname == newname {
		debug.Log("  %v is already renamed", newname)
		return nil
	}

	debug.Log("  %v -> %v", oldname, newname)

	coreClient := minio.Core{Client: be.client}
	err := coreClient.CopyObject(be.cfg.Bucket, newname, path.Join(be.cfg.Bucket, oldname), minio.CopyConditions{})
	if err != nil && be.IsNotExist(err) {
		debug.Log("copy failed: %v, seems to already have been renamed", err)
		return nil
	}

	if err != nil {
		debug.Log("copy failed: %v", err)
		return err
	}

	return be.client.RemoveObject(be.cfg.Bucket, oldname)
}
<|endoftext|>


<commit_before>package dynamics

import (
	"encoding/csv"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/soniakeys/meeus/julian"
)

// CgCatalog definiton.
type CgCatalog struct {
	Version string     `json:"version"`
	Name    string     `json:"name"`
	Items   []*CgItems `json:"items"`
	Require []string   `json:"require,omitempty"`
}

func (c *CgCatalog) String() string {
	return c.Name + "(" + c.Version + ")"
}

// CgItems definiton.
type CgItems struct {
	Class          string            `json:"class"`
	Name           string            `json:"name"`
	StartTime      string            `json:"startTime"`
	EndTime        string            `json:"endTime"`
	Center         string            `json:"center"`
	Trajectory     *CgTrajectory     `json:"trajectory,omitempty"`
	Bodyframe      *CgBodyFrame      `json:"bodyFrame,omitempty"`
	Geometry       *CgGeometry       `json:"geometry,omitempty"`
	Label          *CgLabel          `json:"label,omitempty"`
	TrajectoryPlot *CgTrajectoryPlot `json:"trajectoryPlot,omitempty"`
}

// CgTrajectory definition.
type CgTrajectory struct {
	Type   string `json:"type,omitempty"`
	Source string `json:"source,omitempty"`
}

// Validate validates a CgTrajectory.
func (t *CgTrajectory) Validate() error {
	if t.Type != "InterpolatedStates" || !strings.HasSuffix(t.Source, "xyzv") {
		return errors.New("Only InterpolatedStates are currently supported in Cosmographia trajectory types.")
	}
	return nil
}

func (t *CgTrajectory) String() string {
	return t.Source + " as " + t.Type
}

// CgBodyFrame definiton.
type CgBodyFrame struct {
	Type string `json:"type,omitempty"`
	Name string `json:"name,omitempty"`
}

func (c *CgBodyFrame) String() string {
	return c.Name + " (type: " + c.Type + ")"
}

// CgGeometry definiton.
type CgGeometry struct {
	Type   string    `json:"type,omitempty"`
	Mesh   []float64 `json:"meshRotation,omitempty"`
	Size   float64   `json:"size,omitempty"`
	Source string    `json:"source,omitempty"`
}

// CgLabel definiton.
type CgLabel struct {
	Color    []float64 `json:"color,omitempty"`
	FadeSize int       `json:"fadeSize,omitempty"`
	ShowText bool      `json:"showText,omitempty"`
}

func (l *CgLabel) String() string {
	return fmt.Sprintf("color %v, fade %d, show %v", l.Color, l.FadeSize, l.ShowText)
}

// CgTrajectoryPlot definition.
type CgTrajectoryPlot struct {
	Color       []float64 `json:"color,omitempty"`
	LineWidth   int       `json:"lineWidth,omitempty"`
	Duration    string    `json:"duration,omitempty"`
	Lead        string    `json:"lead,omitempty"`
	Fade        int       `json:"fade,omitempty"`
	SampleCount int       `json:"sampleCount,omitempty"`
}

// CgInterpolatedState definiton.
type CgInterpolatedState struct {
	JD       float64
	Position []float64
	Velocity []float64
}

// FromText initializes from text.
// The `record` parameter must be an array of seven items.
func (i *CgInterpolatedState) FromText(record []string) {
	if val, err := strconv.ParseFloat(record[0], 64); err != nil {
		panic(err)
	} else {
		i.JD = val
	}

	if posX, err := strconv.ParseFloat(record[1], 64); err != nil {
		panic(err)
	} else if posY, err := strconv.ParseFloat(record[2], 64); err != nil {
		panic(err)
	} else if posZ, err := strconv.ParseFloat(record[3], 64); err != nil {
		panic(err)
	} else {
		i.Position = []float64{posX, posY, posZ}
	}

	if velX, err := strconv.ParseFloat(record[4], 64); err != nil {
		panic(err)
	} else if velY, err := strconv.ParseFloat(record[5], 64); err != nil {
		panic(err)
	} else if velZ, err := strconv.ParseFloat(record[6], 64); err != nil {
		panic(err)
	} else {
		i.Velocity = []float64{velX, velY, velZ}
	}
}

// ToText converts to text for written output.
func (i *CgInterpolatedState) ToText() string {
	return fmt.Sprintf("%f %f %f %f %f %f %f", i.JD, i.Position[0], i.Position[1], i.Position[2], i.Velocity[0], i.Velocity[1], i.Velocity[2])
}

// ParseInterpolatedStates takes a string and converts that into a CgInterpolatedState.
func ParseInterpolatedStates(s string) []*CgInterpolatedState {
	var states = []*CgInterpolatedState{}
	r := csv.NewReader(strings.NewReader(s))
	r.Comma = ' '
	r.Comment = '#'
	for {
		record, err := r.Read()
		if err != nil {
			if err == io.EOF {
				break
			}
			panic(err)
		}
		state := CgInterpolatedState{}
		state.FromText(record)
		states = append(states, &state)
	}

	return states
}

// createInterpolatedFile returns a file which requires a defer close statement!
func createInterpolatedFile(filename string, stamped bool, stateDT time.Time) *os.File {
	if stamped {
		t := time.Now()
		filename = fmt.Sprintf("%s/prop-%s-%d-%02d-%02dT%02d.%02d.%02d.xyzv", os.Getenv("DATAOUT"), filename, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
	} else {
		filename = fmt.Sprintf("%s/prop-%s.xyzv", os.Getenv("DATAOUT"), filename)
	}
	f, err := os.Create(filename)
	if err != nil {
		panic(err)
	}
	// Header
	f.WriteString(fmt.Sprintf(`# Creation date (UTC): %s
# Records are <jd> <x> <y> <z> <vel x> <vel y> <vel z>
#   Time is a TDB Julian date
#   Position in km
#   Velocity in km/sec
#   Simulation time start (UTC): %s`, time.Now(), stateDT.UTC()))
	return f
}

// StreamStates streams the output of the channel to the provided file.
func StreamStates(filename string, stateChan <-chan (AstroState), stamped bool) {
	// Read from channel
	var prevStatePtr, firstStatePtr *AstroState
	var fileNo uint8
	var f *os.File
	fileNo = 0
	cgItems := []*CgItems{}
	var curCgItem CgItems
	for {
		state, more := <-stateChan
		if more {
			// Determine whether a new Cosmographia interpolated state file is needed.
			if prevStatePtr == nil {
				firstStatePtr = &state
				f = createInterpolatedFile(fmt.Sprintf("%s-%d", filename, fileNo), stamped, state.dt)
				fileNo++
				traj := CgTrajectory{Type: "InterpolatedStates", Source: fmt.Sprintf("prop-%s-%d.xyzv", filename, fileNo)}
				// TODO: Switch color based on SC state (e.g. no fuel, not thrusting, etc.)
				label := CgLabel{Color: []float64{0.6, 1, 1}, FadeSize: 1000000, ShowText: true}
				plot := CgTrajectoryPlot{Color: []float64{0.6, 1, 1}, LineWidth: 1, Duration: "", Lead: "0 d", Fade: 0, SampleCount: 10}
				curCgItem = CgItems{Class: "spacecraft", Name: fmt.Sprintf("%s-%d", state.sc.Name, fileNo), StartTime: fmt.Sprintf("%s", state.dt.UTC()), EndTime: "", Center: state.orbit.Origin.Name, Trajectory: &traj, Bodyframe: nil, Geometry: nil, Label: &label, TrajectoryPlot: &plot}
			} else {
				if !prevStatePtr.orbit.Origin.Equals(state.orbit.Origin) {
					// Before switching files, let's write the end of simulation time.
					f.WriteString(fmt.Sprintf("\n# Simulation time end (UTC): %s\n", prevStatePtr.dt.UTC()))
					// Update plot time propagation.
					curCgItem.EndTime = fmt.Sprintf("%s", prevStatePtr.dt.UTC())
					fmt.Printf("duration %s\n", prevStatePtr.dt.Sub(firstStatePtr.dt))
					curCgItem.TrajectoryPlot.Duration = fmt.Sprintf("%d d", int(prevStatePtr.dt.Sub(firstStatePtr.dt).Hours()/24+1))
					// Add this CG item to the list of items.
					cgItems = append(cgItems, &curCgItem)
					// Switch files.
					f.Close()
					f = createInterpolatedFile(fmt.Sprintf("%s-%d", filename, fileNo), stamped, state.dt)
					fileNo++
					// Force writing this data point now instead of creating N new files.
					prevStatePtr = &state
					asTxt := CgInterpolatedState{JD: julian.TimeToJD(state.dt), Position: state.orbit.R, Velocity: state.orbit.V}
					_, err := f.WriteString("\n" + asTxt.ToText())
					if err != nil {
						panic(err)
					}
					continue
				}
			}
			// Only write one datapoint per minute.
			if prevStatePtr != nil && state.dt.Sub(prevStatePtr.dt) <= time.Duration(1)*time.Minute {
				continue
			}
			prevStatePtr = &state
			asTxt := CgInterpolatedState{JD: julian.TimeToJD(state.dt), Position: state.orbit.R, Velocity: state.orbit.V}
			_, err := f.WriteString("\n" + asTxt.ToText())
			if err != nil {
				panic(err)
			}
		} else {
			// The channel is closed, hence the simulation is over.
			f.WriteString(fmt.Sprintf("\n# Simulation time end (UTC): %s\n", prevStatePtr.dt.UTC()))
			cgItems = append(cgItems, &curCgItem)
			break
		}
	}
	// Let's write the catalog.
	c := CgCatalog{Version: "1.0", Name: prevStatePtr.sc.Name, Items: cgItems, Require: nil}
	// Create JSON file.
	f, err := os.Create(os.Getenv("DATAOUT") + "/catalog-" + filename + ".json")
	if err != nil {
		panic(err)
	}
	defer f.Close()
	fmt.Printf("Saving file to %s.\n", f.Name())
	if marsh, err := json.Marshal(c); err != nil {
		panic(err)
	} else {
		f.Write(marsh)
	}
}
<commit_msg>Woop seems to work<commit_after>package dynamics

import (
	"encoding/csv"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/soniakeys/meeus/julian"
)

// CgCatalog definiton.
type CgCatalog struct {
	Version string     `json:"version"`
	Name    string     `json:"name"`
	Items   []*CgItems `json:"items"`
	Require []string   `json:"require,omitempty"`
}

func (c *CgCatalog) String() string {
	return c.Name + "(" + c.Version + ")"
}

// CgItems definiton.
type CgItems struct {
	Class          string            `json:"class"`
	Name           string            `json:"name"`
	StartTime      string            `json:"startTime"`
	EndTime        string            `json:"endTime"`
	Center         string            `json:"center"`
	Trajectory     *CgTrajectory     `json:"trajectory,omitempty"`
	Bodyframe      *CgBodyFrame      `json:"bodyFrame,omitempty"`
	Geometry       *CgGeometry       `json:"geometry,omitempty"`
	Label          *CgLabel          `json:"label,omitempty"`
	TrajectoryPlot *CgTrajectoryPlot `json:"trajectoryPlot,omitempty"`
}

// CgTrajectory definition.
type CgTrajectory struct {
	Type   string `json:"type,omitempty"`
	Source string `json:"source,omitempty"`
}

// Validate validates a CgTrajectory.
func (t *CgTrajectory) Validate() error {
	if t.Type != "InterpolatedStates" || !strings.HasSuffix(t.Source, "xyzv") {
		return errors.New("Only InterpolatedStates are currently supported in Cosmographia trajectory types.")
	}
	return nil
}

func (t *CgTrajectory) String() string {
	return t.Source + " as " + t.Type
}

// CgBodyFrame definiton.
type CgBodyFrame struct {
	Type string `json:"type,omitempty"`
	Name string `json:"name,omitempty"`
}

func (c *CgBodyFrame) String() string {
	return c.Name + " (type: " + c.Type + ")"
}

// CgGeometry definiton.
type CgGeometry struct {
	Type   string    `json:"type,omitempty"`
	Mesh   []float64 `json:"meshRotation,omitempty"`
	Size   float64   `json:"size,omitempty"`
	Source string    `json:"source,omitempty"`
}

// CgLabel definiton.
type CgLabel struct {
	Color    []float64 `json:"color,omitempty"`
	FadeSize int       `json:"fadeSize,omitempty"`
	ShowText bool      `json:"showText,omitempty"`
}

func (l *CgLabel) String() string {
	return fmt.Sprintf("color %v, fade %d, show %v", l.Color, l.FadeSize, l.ShowText)
}

// CgTrajectoryPlot definition.
type CgTrajectoryPlot struct {
	Color       []float64 `json:"color,omitempty"`
	LineWidth   int       `json:"lineWidth,omitempty"`
	Duration    string    `json:"duration,omitempty"`
	Lead        string    `json:"lead,omitempty"`
	Fade        int       `json:"fade,omitempty"`
	SampleCount int       `json:"sampleCount,omitempty"`
}

// CgInterpolatedState definiton.
type CgInterpolatedState struct {
	JD       float64
	Position []float64
	Velocity []float64
}

// FromText initializes from text.
// The `record` parameter must be an array of seven items.
func (i *CgInterpolatedState) FromText(record []string) {
	if val, err := strconv.ParseFloat(record[0], 64); err != nil {
		panic(err)
	} else {
		i.JD = val
	}

	if posX, err := strconv.ParseFloat(record[1], 64); err != nil {
		panic(err)
	} else if posY, err := strconv.ParseFloat(record[2], 64); err != nil {
		panic(err)
	} else if posZ, err := strconv.ParseFloat(record[3], 64); err != nil {
		panic(err)
	} else {
		i.Position = []float64{posX, posY, posZ}
	}

	if velX, err := strconv.ParseFloat(record[4], 64); err != nil {
		panic(err)
	} else if velY, err := strconv.ParseFloat(record[5], 64); err != nil {
		panic(err)
	} else if velZ, err := strconv.ParseFloat(record[6], 64); err != nil {
		panic(err)
	} else {
		i.Velocity = []float64{velX, velY, velZ}
	}
}

// ToText converts to text for written output.
func (i *CgInterpolatedState) ToText() string {
	return fmt.Sprintf("%f %f %f %f %f %f %f", i.JD, i.Position[0], i.Position[1], i.Position[2], i.Velocity[0], i.Velocity[1], i.Velocity[2])
}

// ParseInterpolatedStates takes a string and converts that into a CgInterpolatedState.
func ParseInterpolatedStates(s string) []*CgInterpolatedState {
	var states = []*CgInterpolatedState{}
	r := csv.NewReader(strings.NewReader(s))
	r.Comma = ' '
	r.Comment = '#'
	for {
		record, err := r.Read()
		if err != nil {
			if err == io.EOF {
				break
			}
			panic(err)
		}
		state := CgInterpolatedState{}
		state.FromText(record)
		states = append(states, &state)
	}

	return states
}

// createInterpolatedFile returns a file which requires a defer close statement!
func createInterpolatedFile(filename string, stamped bool, stateDT time.Time) *os.File {
	if stamped {
		t := time.Now()
		filename = fmt.Sprintf("%s/prop-%s-%d-%02d-%02dT%02d.%02d.%02d.xyzv", os.Getenv("DATAOUT"), filename, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
	} else {
		filename = fmt.Sprintf("%s/prop-%s.xyzv", os.Getenv("DATAOUT"), filename)
	}
	f, err := os.Create(filename)
	if err != nil {
		panic(err)
	}
	// Header
	f.WriteString(fmt.Sprintf(`# Creation date (UTC): %s
# Records are <jd> <x> <y> <z> <vel x> <vel y> <vel z>
#   Time is a TDB Julian date
#   Position in km
#   Velocity in km/sec
#   Simulation time start (UTC): %s`, time.Now(), stateDT.UTC()))
	return f
}

// StreamStates streams the output of the channel to the provided file.
func StreamStates(filename string, stateChan <-chan (AstroState), stamped bool) {
	// Read from channel
	var prevStatePtr, firstStatePtr *AstroState
	var fileNo uint8
	var f *os.File
	fileNo = 0
	cgItems := []*CgItems{}
	var curCgItem *CgItems
	for {
		state, more := <-stateChan
		if more {
			// Determine whether a new Cosmographia interpolated state file is needed.
			if prevStatePtr == nil {
				firstStatePtr = &state
				f = createInterpolatedFile(fmt.Sprintf("%s-%d", filename, fileNo), stamped, state.dt)
				traj := CgTrajectory{Type: "InterpolatedStates", Source: fmt.Sprintf("prop-%s-%d.xyzv", filename, fileNo)}
				// TODO: Switch color based on SC state (e.g. no fuel, not thrusting, etc.)
				label := CgLabel{Color: []float64{0.6, 1, 1}, FadeSize: 1000000, ShowText: true}
				plot := CgTrajectoryPlot{Color: []float64{0.6, 1, 1}, LineWidth: 1, Duration: "", Lead: "0 d", Fade: 0, SampleCount: 10}
				curCgItem = &CgItems{Class: "spacecraft", Name: fmt.Sprintf("%s-%d", state.sc.Name, fileNo), StartTime: fmt.Sprintf("%s", state.dt.UTC()), EndTime: "", Center: state.orbit.Origin.Name, Trajectory: &traj, Bodyframe: nil, Geometry: nil, Label: &label, TrajectoryPlot: &plot}
				fileNo++
			} else {
				if !prevStatePtr.orbit.Origin.Equals(state.orbit.Origin) {
					// Before switching files, let's write the end of simulation time.
					f.WriteString(fmt.Sprintf("\n# Simulation time end (UTC): %s\n", state.dt.UTC()))
					// Update plot time propagation.
					curCgItem.EndTime = fmt.Sprintf("%s", state.dt.UTC())
					curCgItem.TrajectoryPlot.Duration = fmt.Sprintf("%d d", int(state.dt.Sub(firstStatePtr.dt).Hours()/24+1))
					// Add this CG item to the list of items.
					cgItems = append(cgItems, curCgItem) // TODO: SWITCH TO POINTER AND CHECK
					// Switch files.
					f.Close()
					// XXX: Copy/paste from above :'(
					f = createInterpolatedFile(fmt.Sprintf("%s-%d", filename, fileNo), stamped, state.dt)
					traj := CgTrajectory{Type: "InterpolatedStates", Source: fmt.Sprintf("prop-%s-%d.xyzv", filename, fileNo)}
					label := CgLabel{Color: []float64{0.6, 1, 1}, FadeSize: 1000000, ShowText: true}
					plot := CgTrajectoryPlot{Color: []float64{0.6, 1, 1}, LineWidth: 1, Duration: "", Lead: "0 d", Fade: 0, SampleCount: 10}
					curCgItem = &CgItems{Class: "spacecraft", Name: fmt.Sprintf("%s-%d", state.sc.Name, fileNo), StartTime: fmt.Sprintf("%s", state.dt.UTC()), EndTime: "", Center: state.orbit.Origin.Name, Trajectory: &traj, Bodyframe: nil, Geometry: nil, Label: &label, TrajectoryPlot: &plot}
					fileNo++
					// Force writing this data point now instead of creating N new files.
					prevStatePtr = &state
					asTxt := CgInterpolatedState{JD: julian.TimeToJD(state.dt), Position: state.orbit.R, Velocity: state.orbit.V}
					if _, err := f.WriteString("\n" + asTxt.ToText()); err != nil {
						panic(err)
					}
					continue
				}
			}
			// Only write one datapoint per simulation minute.
			if prevStatePtr != nil && state.dt.Sub(prevStatePtr.dt) <= time.Duration(1)*time.Minute {
				continue
			}
			prevStatePtr = &state
			asTxt := CgInterpolatedState{JD: julian.TimeToJD(state.dt), Position: state.orbit.R, Velocity: state.orbit.V}
			if _, err := f.WriteString("\n" + asTxt.ToText()); err != nil {
				panic(err)
			}
		} else {
			// The channel is closed, hence the simulation is over.
			f.WriteString(fmt.Sprintf("\n# Simulation time end (UTC): %s\n", prevStatePtr.dt.UTC()))
			f.Close()
			curCgItem.EndTime = fmt.Sprintf("%s", prevStatePtr.dt.UTC())
			curCgItem.TrajectoryPlot.Duration = fmt.Sprintf("%d d", int(prevStatePtr.dt.Sub(firstStatePtr.dt).Hours()/24+1))
			cgItems = append(cgItems, curCgItem)
			break
		}
	}
	// Let's write the catalog.
	c := CgCatalog{Version: "1.0", Name: prevStatePtr.sc.Name, Items: cgItems, Require: nil}
	// Create JSON file.
	f, err := os.Create(os.Getenv("DATAOUT") + "/catalog-" + filename + ".json")
	if err != nil {
		panic(err)
	}
	defer f.Close()
	fmt.Printf("Saving file to %s.\n", f.Name())
	if marsh, err := json.Marshal(c); err != nil {
		panic(err)
	} else {
		f.Write(marsh)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Meador Inge.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package scanner

import (
	"testing"
)

type test struct {
	tok TokenKind
	str string
}

var test_single_token = [...]*Token{
	NewToken(EOF, ""),
	NewToken(COMMENT, "// this is a BCPL a comment!\n"),

	// Names.
	NewToken(NAME, "Global"),
	NewToken(NAME, "Let"),
	NewToken(NAME, "a"),
	NewToken(NAME, "Z"),

	// Numbers.
	NewToken(NUMBER, "1"),
	NewToken(NUMBER, "98765"),

	// String constants.
	NewToken(STRINGCONST, "\"foo bar baz\n\""),

	// Operators.
	NewToken(VALOF, "valof"),
	NewToken(LV, "lv"),
	NewToken(RV, "rv"),
	NewToken(DIV, "/"),
	NewToken(REM, "rem"),
	NewToken(PLUS, "+"),
	NewToken(MINUS, "-"),
	NewToken(EQ, "="),
	NewToken(NE, "!="),
	NewToken(LS, "<"),
	NewToken(GR, ">"),
	NewToken(LE, "<="),
	NewToken(GE, ">="),
	NewToken(NOT, "!"),
	NewToken(LSHIFT, "<<"),
	NewToken(RSHIFT, ">>"),
	NewToken(LOGAND, "&"),
	NewToken(LOGOR, "|"),
	NewToken(EQV, "eqv"),
	NewToken(NEQV, "neqv"),
	NewToken(COND, "->"),
	NewToken(COMMA, ","),
	NewToken(ASS, ":="),
	NewToken(SECTBRA, "$("),
	NewToken(SECTKET, "$)"),
	NewToken(RBRA, "("),
	NewToken(RKET, ")"),
	NewToken(SBRA, "["),
	NewToken(SKET, "]"),
	NewToken(COLON, ":"),
	NewToken(SEMICOLON, ";"),
	NewToken(STAR, "*"),

	// Keywords.
	NewToken(TRUE, "true"),
	NewToken(FALSE, "false"),
	NewToken(AND, "and"),
	NewToken(GOTO, "goto"),
	NewToken(RESULTIS, "resultis"),
	NewToken(TEST, "test"),
	NewToken(FOR, "for"),
	NewToken(IF, "if"),
	NewToken(UNLESS, "unless"),
	NewToken(WHILE, "while"),
	NewToken(UNTIL, "until"),
	NewToken(REPEAT, "repeat"),
	NewToken(REPEATWHILE, "repeatwhile"),
	NewToken(REPEATUNTIL, "repeatuntil"),
	NewToken(BREAK, "break"),
	NewToken(RETURN, "return"),
	NewToken(FINISH, "finish"),
	NewToken(SWITCHON, "switchon"),
	NewToken(CASE, "case"),
	NewToken(DEFAULT, "default"),
	NewToken(LET, "let"),
	NewToken(MANIFEST, "manifest"),
	NewToken(GLOBAL, "global"),
	NewToken(BE, "be"),
	NewToken(INTO, "into"),
	NewToken(TO, "to"),
	NewToken(DO, "do"),
	NewToken(OR, "or"),
	NewToken(VEC, "vec"),
}

func TestSingleToken(t *testing.T) {
	var s Scanner

	for _, etok := range test_single_token {
		s.Init([]byte(etok.literal))

		tok := s.Next()
		if tok.kind != etok.kind {
			t.Errorf("bad token: got '%s', expected '%s'", tok, etok)
		}
		if tok.literal != etok.literal {
			t.Errorf("bad token: got '%s', expected '%s'", tok, etok)
		}
	}
}

var test_fact_str = `get "libhdr"
let START() = valof $(
        for I = 1 to 5 do
                writef("%N! = %I4*N", I, FACT(I))
        resultis 0
$)

and FACT(N) = N = 0 -> 1, N * FACT(N - 1)`

var test_fact_tokens = [...]*Token{
	NewToken(GET, "get"),
	NewToken(STRINGCONST, "\"libhdr\""),
	NewToken(LET, "let"),
	NewToken(NAME, "START"),
	NewToken(RBRA, "("),
	NewToken(RKET, ")"),
	NewToken(EQ, "="),
	NewToken(VALOF, "valof"),
	NewToken(SECTBRA, "$("),
	NewToken(FOR, "for"),
	NewToken(NAME, "I"),
	NewToken(EQ, "="),
	NewToken(NUMBER, "1"),
	NewToken(TO, "to"),
	NewToken(NUMBER, "5"),
	NewToken(DO, "do"),
	NewToken(NAME, "writef"),
	NewToken(RBRA, "("),
	NewToken(STRINGCONST, "\"%N! = %I4*N\""),
	NewToken(COMMA, ","),
	NewToken(NAME, "I"),
	NewToken(COMMA, ","),
	NewToken(NAME, "FACT"),
	NewToken(RBRA, "("),
	NewToken(NAME, "I"),
	NewToken(RKET, ")"),
	NewToken(RKET, ")"),
	NewToken(SEMICOLON, ";"),
	NewToken(RESULTIS, "resultis"),
	NewToken(NUMBER, "0"),
	NewToken(SECTKET, "$)"),
	NewToken(AND, "and"),
	NewToken(NAME, "FACT"),
	NewToken(RBRA, "("),
	NewToken(NAME, "N"),
	NewToken(RKET, ")"),
	NewToken(EQ, "="),
	NewToken(NAME, "N"),
	NewToken(EQ, "="),
	NewToken(NUMBER, "0"),
	NewToken(COND, "->"),
	NewToken(NUMBER, "1"),
	NewToken(COMMA, ","),
	NewToken(NAME, "N"),
	NewToken(STAR, "*"),
	NewToken(NAME, "FACT"),
	NewToken(RBRA, "("),
	NewToken(NAME, "N"),
	NewToken(MINUS, "-"),
	NewToken(NUMBER, "1"),
	NewToken(RKET, ")"),
	NewToken(EOF, ""),
}

func TestMultiple(t *testing.T) {
	var s Scanner
	s.Init([]byte(test_fact_str))

	for _, etok := range test_fact_tokens {
		tok := s.Next()
		if tok.kind != etok.kind {
			t.Errorf("bad token: got '%s', expected '%s'", tok, etok)
		}
		if tok.literal != etok.literal {
			t.Errorf("bad token: got '%s', expected '%s'", tok, etok)
		}
	}
}

var test_hello_str = `get "libhdr"
let start() = valof
$( writes("Hello, World!*n")
   resultis 0
$)`

var test_hello_tokens = [...]*Token{
	NewToken(GET, "get"),
	NewToken(STRINGCONST, "\"libhdr\""),
	NewToken(LET, "let"),
	NewToken(NAME, "start"),
	NewToken(RBRA, "("),
	NewToken(RKET, ")"),
	NewToken(EQ, "="),
	NewToken(VALOF, "valof"),
	NewToken(SECTBRA, "$("),
	NewToken(NAME, "writes"),
	NewToken(RBRA, "("),
	NewToken(STRINGCONST, "\"Hello, World!*n\""),
	NewToken(RKET, ")"),
	NewToken(SEMICOLON, ";"),
	NewToken(RESULTIS, "resultis"),
	NewToken(NUMBER, "0"),
	NewToken(SECTKET, "$)"),
	NewToken(EOF, ""),
}

func TestHello(t *testing.T) {
	var s Scanner
	s.Init([]byte(test_hello_str))

	for _, etok := range test_hello_tokens {
		tok := s.Next()
		if tok.kind != etok.kind {
			t.Errorf("bad token: got '%s', expected '%s'", tok, etok)
		}
		if tok.literal != etok.literal {
			t.Errorf("bad token: got '%s', expected '%s'", tok, etok)
		}
	}
}

var test_semi_str = `global $(
        COUNT: 200
        ALL: 201
$)`

var test_semi_tokens = [...]*Token{
	NewToken(GLOBAL, "global"),
	NewToken(SECTBRA, "$("),
	NewToken(NAME, "COUNT"),
	NewToken(COLON, ":"),
	NewToken(NUMBER, "200"),
	NewToken(SEMICOLON, ";"),
	NewToken(NAME, "ALL"),
	NewToken(COLON, ":"),
	NewToken(NUMBER, "201"),
	NewToken(SECTKET, "$)"),
	NewToken(EOF, ""),
}

func TestSemiInsertion(t *testing.T) {
	var s Scanner
	s.Init([]byte(test_semi_str))

	for _, etok := range test_semi_tokens {
		tok := s.Next()
		if tok.kind != etok.kind {
			t.Errorf("bad token: got '%s', expected '%s'", tok, etok)
		}
		if tok.literal != etok.literal {
			t.Errorf("bad token: got '%s', expected '%s'", tok, etok)
		}
	}
}

var test_do_str = `
let START() = valof $(
        for I = 1 to 5 for J = 1 to 5 do
                writef("%N * %N = %N", I, J, I * J)
        resultis 0
$)`

var test_do_tokens = [...]*Token{
	NewToken(LET, "let"),
	NewToken(NAME, "START"),
	NewToken(RBRA, "("),
	NewToken(RKET, ")"),
	NewToken(EQ, "="),
	NewToken(VALOF, "valof"),
	NewToken(SECTBRA, "$("),
	NewToken(FOR, "for"),
	NewToken(NAME, "I"),
	NewToken(EQ, "="),
	NewToken(NUMBER, "1"),
	NewToken(TO, "to"),
	NewToken(NUMBER, "5"),
	NewToken(DO, "do"), // This DO is inserted.
	NewToken(FOR, "for"),
	NewToken(NAME, "J"),
	NewToken(EQ, "="),
	NewToken(NUMBER, "1"),
	NewToken(TO, "to"),
	NewToken(NUMBER, "5"),
	NewToken(DO, "do"),
	NewToken(NAME, "writef"),
	NewToken(RBRA, "("),
	NewToken(STRINGCONST, "\"%N * %N = %N\""),
	NewToken(COMMA, ","),
	NewToken(NAME, "I"),
	NewToken(COMMA, ","),
	NewToken(NAME, "J"),
	NewToken(COMMA, ","),
	NewToken(NAME, "I"),
	NewToken(STAR, "*"),
	NewToken(NAME, "J"),
	NewToken(RKET, ")"),
	NewToken(SEMICOLON, ";"),
	NewToken(RESULTIS, "resultis"),
	NewToken(NUMBER, "0"),
	NewToken(SECTKET, "$)"),
	NewToken(EOF, ""),
}

func TestDoInsertion(t *testing.T) {
	var s Scanner
	s.Init([]byte(test_do_str))

	for _, etok := range test_do_tokens {
		tok := s.Next()
		if tok.kind != etok.kind {
			t.Errorf("bad token: got '%s', expected '%s'", tok, etok)
		}
		if tok.literal != etok.literal {
			t.Errorf("bad token: got '%s', expected '%s'", tok, etok)
		}
	}
}
<commit_msg>scanner: Refactor test code.<commit_after>// Copyright 2015 Meador Inge.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package scanner

import (
	"testing"
)

// Helper test functions.

func assertTokensEqual(t *testing.T, tok, etok *Token) {
	if tok.kind != etok.kind {
		t.Errorf("bad token: got '%s', expected '%s'", tok, etok)
	}
	if tok.literal != etok.literal {
		t.Errorf("bad token: got '%s', expected '%s'", tok, etok)
	}
}

func assertTokensEqualSource(t *testing.T, toks []*Token, str string) {
	var s Scanner
	s.Init([]byte(str))
	for _, etok := range toks {
		tok := s.Next()
		assertTokensEqual(t, tok, etok)
	}

}

type test struct {
	tok TokenKind
	str string
}

var test_single_token = [...]*Token{
	NewToken(EOF, ""),
	NewToken(COMMENT, "// this is a BCPL a comment!\n"),

	// Names.
	NewToken(NAME, "Global"),
	NewToken(NAME, "Let"),
	NewToken(NAME, "a"),
	NewToken(NAME, "Z"),

	// Numbers.
	NewToken(NUMBER, "1"),
	NewToken(NUMBER, "98765"),

	// String constants.
	NewToken(STRINGCONST, "\"foo bar baz\n\""),

	// Operators.
	NewToken(VALOF, "valof"),
	NewToken(LV, "lv"),
	NewToken(RV, "rv"),
	NewToken(DIV, "/"),
	NewToken(REM, "rem"),
	NewToken(PLUS, "+"),
	NewToken(MINUS, "-"),
	NewToken(EQ, "="),
	NewToken(NE, "!="),
	NewToken(LS, "<"),
	NewToken(GR, ">"),
	NewToken(LE, "<="),
	NewToken(GE, ">="),
	NewToken(NOT, "!"),
	NewToken(LSHIFT, "<<"),
	NewToken(RSHIFT, ">>"),
	NewToken(LOGAND, "&"),
	NewToken(LOGOR, "|"),
	NewToken(EQV, "eqv"),
	NewToken(NEQV, "neqv"),
	NewToken(COND, "->"),
	NewToken(COMMA, ","),
	NewToken(ASS, ":="),
	NewToken(SECTBRA, "$("),
	NewToken(SECTKET, "$)"),
	NewToken(RBRA, "("),
	NewToken(RKET, ")"),
	NewToken(SBRA, "["),
	NewToken(SKET, "]"),
	NewToken(COLON, ":"),
	NewToken(SEMICOLON, ";"),
	NewToken(STAR, "*"),

	// Keywords.
	NewToken(TRUE, "true"),
	NewToken(FALSE, "false"),
	NewToken(AND, "and"),
	NewToken(GOTO, "goto"),
	NewToken(RESULTIS, "resultis"),
	NewToken(TEST, "test"),
	NewToken(FOR, "for"),
	NewToken(IF, "if"),
	NewToken(UNLESS, "unless"),
	NewToken(WHILE, "while"),
	NewToken(UNTIL, "until"),
	NewToken(REPEAT, "repeat"),
	NewToken(REPEATWHILE, "repeatwhile"),
	NewToken(REPEATUNTIL, "repeatuntil"),
	NewToken(BREAK, "break"),
	NewToken(RETURN, "return"),
	NewToken(FINISH, "finish"),
	NewToken(SWITCHON, "switchon"),
	NewToken(CASE, "case"),
	NewToken(DEFAULT, "default"),
	NewToken(LET, "let"),
	NewToken(MANIFEST, "manifest"),
	NewToken(GLOBAL, "global"),
	NewToken(BE, "be"),
	NewToken(INTO, "into"),
	NewToken(TO, "to"),
	NewToken(DO, "do"),
	NewToken(OR, "or"),
	NewToken(VEC, "vec"),
}

func TestSingleToken(t *testing.T) {
	var s Scanner

	for _, etok := range test_single_token {
		s.Init([]byte(etok.literal))
		tok := s.Next()
		assertTokensEqual(t, tok, etok)
	}
}

var test_fact_str = `get "libhdr"
let START() = valof $(
        for I = 1 to 5 do
                writef("%N! = %I4*N", I, FACT(I))
        resultis 0
$)

and FACT(N) = N = 0 -> 1, N * FACT(N - 1)`

var test_fact_tokens = []*Token{
	NewToken(GET, "get"),
	NewToken(STRINGCONST, "\"libhdr\""),
	NewToken(LET, "let"),
	NewToken(NAME, "START"),
	NewToken(RBRA, "("),
	NewToken(RKET, ")"),
	NewToken(EQ, "="),
	NewToken(VALOF, "valof"),
	NewToken(SECTBRA, "$("),
	NewToken(FOR, "for"),
	NewToken(NAME, "I"),
	NewToken(EQ, "="),
	NewToken(NUMBER, "1"),
	NewToken(TO, "to"),
	NewToken(NUMBER, "5"),
	NewToken(DO, "do"),
	NewToken(NAME, "writef"),
	NewToken(RBRA, "("),
	NewToken(STRINGCONST, "\"%N! = %I4*N\""),
	NewToken(COMMA, ","),
	NewToken(NAME, "I"),
	NewToken(COMMA, ","),
	NewToken(NAME, "FACT"),
	NewToken(RBRA, "("),
	NewToken(NAME, "I"),
	NewToken(RKET, ")"),
	NewToken(RKET, ")"),
	NewToken(SEMICOLON, ";"),
	NewToken(RESULTIS, "resultis"),
	NewToken(NUMBER, "0"),
	NewToken(SECTKET, "$)"),
	NewToken(AND, "and"),
	NewToken(NAME, "FACT"),
	NewToken(RBRA, "("),
	NewToken(NAME, "N"),
	NewToken(RKET, ")"),
	NewToken(EQ, "="),
	NewToken(NAME, "N"),
	NewToken(EQ, "="),
	NewToken(NUMBER, "0"),
	NewToken(COND, "->"),
	NewToken(NUMBER, "1"),
	NewToken(COMMA, ","),
	NewToken(NAME, "N"),
	NewToken(STAR, "*"),
	NewToken(NAME, "FACT"),
	NewToken(RBRA, "("),
	NewToken(NAME, "N"),
	NewToken(MINUS, "-"),
	NewToken(NUMBER, "1"),
	NewToken(RKET, ")"),
	NewToken(EOF, ""),
}

func TestMultiple(t *testing.T) {
	assertTokensEqualSource(t, test_fact_tokens, test_fact_str)
}

var test_hello_str = `get "libhdr"
let start() = valof
$( writes("Hello, World!*n")
   resultis 0
$)`

var test_hello_tokens = []*Token{
	NewToken(GET, "get"),
	NewToken(STRINGCONST, "\"libhdr\""),
	NewToken(LET, "let"),
	NewToken(NAME, "start"),
	NewToken(RBRA, "("),
	NewToken(RKET, ")"),
	NewToken(EQ, "="),
	NewToken(VALOF, "valof"),
	NewToken(SECTBRA, "$("),
	NewToken(NAME, "writes"),
	NewToken(RBRA, "("),
	NewToken(STRINGCONST, "\"Hello, World!*n\""),
	NewToken(RKET, ")"),
	NewToken(SEMICOLON, ";"),
	NewToken(RESULTIS, "resultis"),
	NewToken(NUMBER, "0"),
	NewToken(SECTKET, "$)"),
	NewToken(EOF, ""),
}

func TestHello(t *testing.T) {
	assertTokensEqualSource(t, test_hello_tokens, test_hello_str)
}

var test_semi_str = `global $(
        COUNT: 200
        ALL: 201
$)`

var test_semi_tokens = []*Token{
	NewToken(GLOBAL, "global"),
	NewToken(SECTBRA, "$("),
	NewToken(NAME, "COUNT"),
	NewToken(COLON, ":"),
	NewToken(NUMBER, "200"),
	NewToken(SEMICOLON, ";"),
	NewToken(NAME, "ALL"),
	NewToken(COLON, ":"),
	NewToken(NUMBER, "201"),
	NewToken(SECTKET, "$)"),
	NewToken(EOF, ""),
}

func TestSemiInsertion(t *testing.T) {
	assertTokensEqualSource(t, test_semi_tokens, test_semi_str)
}

var test_do_str = `
let START() = valof $(
        for I = 1 to 5 for J = 1 to 5 do
                writef("%N * %N = %N", I, J, I * J)
        resultis 0
$)`

var test_do_tokens = []*Token{
	NewToken(LET, "let"),
	NewToken(NAME, "START"),
	NewToken(RBRA, "("),
	NewToken(RKET, ")"),
	NewToken(EQ, "="),
	NewToken(VALOF, "valof"),
	NewToken(SECTBRA, "$("),
	NewToken(FOR, "for"),
	NewToken(NAME, "I"),
	NewToken(EQ, "="),
	NewToken(NUMBER, "1"),
	NewToken(TO, "to"),
	NewToken(NUMBER, "5"),
	NewToken(DO, "do"), // This DO is inserted.
	NewToken(FOR, "for"),
	NewToken(NAME, "J"),
	NewToken(EQ, "="),
	NewToken(NUMBER, "1"),
	NewToken(TO, "to"),
	NewToken(NUMBER, "5"),
	NewToken(DO, "do"),
	NewToken(NAME, "writef"),
	NewToken(RBRA, "("),
	NewToken(STRINGCONST, "\"%N * %N = %N\""),
	NewToken(COMMA, ","),
	NewToken(NAME, "I"),
	NewToken(COMMA, ","),
	NewToken(NAME, "J"),
	NewToken(COMMA, ","),
	NewToken(NAME, "I"),
	NewToken(STAR, "*"),
	NewToken(NAME, "J"),
	NewToken(RKET, ")"),
	NewToken(SEMICOLON, ";"),
	NewToken(RESULTIS, "resultis"),
	NewToken(NUMBER, "0"),
	NewToken(SECTKET, "$)"),
	NewToken(EOF, ""),
}

func TestDoInsertion(t *testing.T) {
	assertTokensEqualSource(t, test_do_tokens, test_do_str)
}
<|endoftext|>


<commit_before>package cluster

import (
	"strings"

	"github.com/rancher/norman/types"
	"github.com/rancher/norman/types/convert"
	"github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/sirupsen/logrus"
)

type Formatter struct {
	KontainerDriverLister v3.KontainerDriverLister
}

func (f *Formatter) Formatter(request *types.APIContext, resource *types.RawResource) {
	if convert.ToBool(resource.Values["internal"]) {
		delete(resource.Links, "remove")
	}
	shellLink := request.URLBuilder.Link("shell", resource)
	shellLink = strings.Replace(shellLink, "http", "ws", 1)
	shellLink = strings.Replace(shellLink, "/shell", "?shell=true", 1)
	resource.Links["shell"] = shellLink
	resource.AddAction(request, "generateKubeconfig")
	resource.AddAction(request, "importYaml")
	resource.AddAction(request, "exportYaml")
	if _, ok := resource.Values["rancherKubernetesEngineConfig"]; ok {
		resource.AddAction(request, "rotateCertificates")
		resource.AddAction(request, "backupEtcd")
		resource.AddAction(request, "restoreFromEtcdBackup")
	}

	if err := request.AccessControl.CanDo(v3.ClusterGroupVersionKind.Group, v3.ClusterResource.Name, "update", request, resource.Values, request.Schema); err == nil {
		if convert.ToBool(resource.Values["enableClusterMonitoring"]) {
			resource.AddAction(request, "disableMonitoring")
			resource.AddAction(request, "editMonitoring")
		} else {
			resource.AddAction(request, "enableMonitoring")
		}
	}

	if convert.ToBool(resource.Values["enableClusterMonitoring"]) {
		resource.AddAction(request, "viewMonitoring")
	}

	if gkeConfig, ok := resource.Values["googleKubernetesEngineConfig"]; ok {
		configMap, ok := gkeConfig.(map[string]interface{})
		if !ok {
			logrus.Errorf("could not convert gke config to map")
			return
		}

		setTrueIfNil(configMap, "enableStackdriverLogging")
		setTrueIfNil(configMap, "enableStackdriverMonitoring")
		setTrueIfNil(configMap, "enableHorizontalPodAutoscaling")
		setTrueIfNil(configMap, "enableHttpLoadBalancing")
		setTrueIfNil(configMap, "enableNetworkPolicyConfig")
	}

	if eksConfig, ok := resource.Values["amazonElasticContainerServiceConfig"]; ok {
		configMap, ok := eksConfig.(map[string]interface{})
		if !ok {
			logrus.Errorf("could not convert eks config to map")
			return
		}

		setTrueIfNil(configMap, "associateWorkerNodePublicIp")
		setIntIfNil(configMap, "nodeVolumeSize", 20)
	}
}

func setTrueIfNil(configMap map[string]interface{}, fieldName string) {
	if configMap[fieldName] == nil {
		configMap[fieldName] = true
	}
}

func setIntIfNil(configMap map[string]interface{}, fieldName string, replaceVal int) {
	if configMap[fieldName] == nil {
		configMap[fieldName] = replaceVal
	}
}
<commit_msg>Handle unset backup config on the API level<commit_after>package cluster

import (
	"strings"

	"github.com/rancher/norman/types"
	"github.com/rancher/norman/types/convert"
	"github.com/rancher/norman/types/values"
	"github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/sirupsen/logrus"
)

type Formatter struct {
	KontainerDriverLister v3.KontainerDriverLister
}

func (f *Formatter) Formatter(request *types.APIContext, resource *types.RawResource) {
	if convert.ToBool(resource.Values["internal"]) {
		delete(resource.Links, "remove")
	}
	shellLink := request.URLBuilder.Link("shell", resource)
	shellLink = strings.Replace(shellLink, "http", "ws", 1)
	shellLink = strings.Replace(shellLink, "/shell", "?shell=true", 1)
	resource.Links["shell"] = shellLink
	resource.AddAction(request, "generateKubeconfig")
	resource.AddAction(request, "importYaml")
	resource.AddAction(request, "exportYaml")
	if _, ok := resource.Values["rancherKubernetesEngineConfig"]; ok {
		resource.AddAction(request, "rotateCertificates")
		if _, ok := values.GetValue(resource.Values, "rancherKubernetesEngineConfig", "services", "etcd", "backupConfig"); ok {
			resource.AddAction(request, "backupEtcd")
			resource.AddAction(request, "restoreFromEtcdBackup")
		}
	}

	if err := request.AccessControl.CanDo(v3.ClusterGroupVersionKind.Group, v3.ClusterResource.Name, "update", request, resource.Values, request.Schema); err == nil {
		if convert.ToBool(resource.Values["enableClusterMonitoring"]) {
			resource.AddAction(request, "disableMonitoring")
			resource.AddAction(request, "editMonitoring")
		} else {
			resource.AddAction(request, "enableMonitoring")
		}
	}

	if convert.ToBool(resource.Values["enableClusterMonitoring"]) {
		resource.AddAction(request, "viewMonitoring")
	}

	if gkeConfig, ok := resource.Values["googleKubernetesEngineConfig"]; ok {
		configMap, ok := gkeConfig.(map[string]interface{})
		if !ok {
			logrus.Errorf("could not convert gke config to map")
			return
		}

		setTrueIfNil(configMap, "enableStackdriverLogging")
		setTrueIfNil(configMap, "enableStackdriverMonitoring")
		setTrueIfNil(configMap, "enableHorizontalPodAutoscaling")
		setTrueIfNil(configMap, "enableHttpLoadBalancing")
		setTrueIfNil(configMap, "enableNetworkPolicyConfig")
	}

	if eksConfig, ok := resource.Values["amazonElasticContainerServiceConfig"]; ok {
		configMap, ok := eksConfig.(map[string]interface{})
		if !ok {
			logrus.Errorf("could not convert eks config to map")
			return
		}

		setTrueIfNil(configMap, "associateWorkerNodePublicIp")
		setIntIfNil(configMap, "nodeVolumeSize", 20)
	}
}

func setTrueIfNil(configMap map[string]interface{}, fieldName string) {
	if configMap[fieldName] == nil {
		configMap[fieldName] = true
	}
}

func setIntIfNil(configMap map[string]interface{}, fieldName string, replaceVal int) {
	if configMap[fieldName] == nil {
		configMap[fieldName] = replaceVal
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Tobias Schottdorf

package server

import (
	"fmt"
	"reflect"
	"sort"
	"sync"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/storage"
	"github.com/cockroachdb/cockroach/util/leaktest"
)

func TestIntentResolution(t *testing.T) {
	defer leaktest.AfterTest(t)
	t.Skip("https://github.com/cockroachdb/cockroach/issues/4579")

	testCases := []struct {
		keys   []string
		ranges [][2]string
		exp    []string
	}{
		// Note that the first key (or, range, if no keys present) determines
		// the base key of the Txn. In these examples, it's always the first
		// range, so "a"-"s" is local.
		{
			// All local points, except for "s" and "x"
			keys:   []string{"a", "x", "b", "c", "s"},
			ranges: [][2]string{{"d", "e"}},
			exp:    []string{"s", "x"},
		},
		{
			// h is local, y is covered by the Range below but still gets an
			// explicit request since we do it all in the same batch (didn't
			// seem worth optimizing), and z is an explicit end point.
			keys:   []string{"h", "y", "z"},
			ranges: [][2]string{{"g", "z"}},
			exp:    []string{`"s"-"z"`, "z" /* optimizable: */, "y"},
		},
		{
			// This test demonstrates another redundancy. Two overlapping key
			// ranges aren't reduced to a wider range intent for the non-local
			// part, though the contained ranges "a"-"u" and "t"-"u" are.
			// Might make sense to optimize this away at some point, along with
			// "s" which is also already covered by the range intents.
			keys:   []string{"q", "s"},
			ranges: [][2]string{{"a", "w"}, {"b", "x"}, {"t", "u"}},
			exp:    []string{`"s"-"w"`, `"s"-"x"` /* optimizable: */, `"t"-"u"`, "s"},
		},
	}

	splitKey := []byte("s")
	defer func() { storage.TestingCommandFilter = nil }()
	for i, tc := range testCases {
		var result []string
		var mu sync.Mutex
		closer := make(chan struct{}, 2)
		storage.TestingCommandFilter = func(_ roachpb.StoreID, args roachpb.Request, _ roachpb.Header) error {
			mu.Lock()
			defer mu.Unlock()
			header := args.Header()
			// Ignore anything outside of the intent key range of "a" - "z"
			// TODO: Implement "ContainsKey()" for Span, currently only
			// implemented for RSpan
			if header.Key.Compare(roachpb.Key("a")) < 0 || header.Key.Compare(roachpb.Key("z")) > 0 {
				return nil
			}
			switch args.(type) {
			case *roachpb.ResolveIntentRequest:
				result = append(result, string(header.Key))
			case *roachpb.ResolveIntentRangeRequest:
				result = append(result, fmt.Sprintf("%s-%s", header.Key, header.EndKey))
			}
			if len(result) == len(tc.exp) {
				closer <- struct{}{}
			}
			return nil
		}
		func() {
			s := StartTestServer(t)
			defer s.Stop()

			go func() {
				// Sets a timeout, cut short by the stopper having drained.
				select {
				case <-time.After(time.Second):
				case <-s.Server.stopper.ShouldStop():
					return
				}
				closer <- struct{}{}
			}()

			// Split the Range. This should not have any asynchronous intents.
			if err := s.db.AdminSplit(splitKey); err != nil {
				t.Fatal(err)
			}

			if pErr := s.db.Txn(func(txn *client.Txn) *roachpb.Error {
				b := txn.NewBatch()
				for _, key := range tc.keys {
					b.Put(key, "test")
				}
				for _, kr := range tc.ranges {
					b.DelRange(kr[0], kr[1], false)
				}
				return txn.CommitInBatch(b)
			}); pErr != nil {
				t.Fatalf("%d: %s", i, pErr)
			}
			<-closer // wait for async intents
		}()
		// Verification. Note that this runs after the system has stopped, so that
		// everything asynchronous has already happened (while tearing down, intent
		// resolution still takes place, synchronously).
		expResult := tc.exp
		sort.Strings(expResult)
		sort.Strings(result)
		if !reflect.DeepEqual(result, expResult) {
			t.Fatalf("%d: unexpected non-local intents, expected %s: %s", i, expResult, result)
		}

	}
}
<commit_msg>Ignore duplicates in TestIntentResolution<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Tobias Schottdorf

package server

import (
	"fmt"
	"reflect"
	"sort"
	"sync"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/storage"
	"github.com/cockroachdb/cockroach/util/leaktest"
)

func TestIntentResolution(t *testing.T) {
	defer leaktest.AfterTest(t)

	testCases := []struct {
		keys   []string
		ranges [][2]string
		exp    []string
	}{
		// Note that the first key (or, range, if no keys present) determines
		// the base key of the Txn. In these examples, it's always the first
		// range, so "a"-"s" is local.
		{
			// All local points, except for "s" and "x"
			keys:   []string{"a", "x", "b", "c", "s"},
			ranges: [][2]string{{"d", "e"}},
			exp:    []string{"s", "x"},
		},
		{
			// h is local, y is covered by the Range below but still gets an
			// explicit request since we do it all in the same batch (didn't
			// seem worth optimizing), and z is an explicit end point.
			keys:   []string{"h", "y", "z"},
			ranges: [][2]string{{"g", "z"}},
			exp:    []string{`"s"-"z"`, "z" /* optimizable: */, "y"},
		},
		{
			// This test demonstrates another redundancy. Two overlapping key
			// ranges aren't reduced to a wider range intent for the non-local
			// part, though the contained ranges "a"-"u" and "t"-"u" are.
			// Might make sense to optimize this away at some point, along with
			// "s" which is also already covered by the range intents.
			keys:   []string{"q", "s"},
			ranges: [][2]string{{"a", "w"}, {"b", "x"}, {"t", "u"}},
			exp:    []string{`"s"-"w"`, `"s"-"x"` /* optimizable: */, `"t"-"u"`, "s"},
		},
	}

	splitKey := []byte("s")
	results := map[string]struct{}{}
	defer func() { storage.TestingCommandFilter = nil }()
	for i, tc := range testCases {
		var mu sync.Mutex
		closer := make(chan struct{}, 2)
		storage.TestingCommandFilter = func(_ roachpb.StoreID, args roachpb.Request, _ roachpb.Header) error {
			mu.Lock()
			defer mu.Unlock()
			header := args.Header()
			// Ignore anything outside of the intent key range of "a" - "z"
			if header.Key.Compare(roachpb.Key("a")) < 0 || header.Key.Compare(roachpb.Key("z")) > 0 {
				return nil
			}
			switch args.(type) {
			case *roachpb.ResolveIntentRequest:
				results[string(header.Key)] = struct{}{}
			case *roachpb.ResolveIntentRangeRequest:
				results[fmt.Sprintf("%s-%s", header.Key, header.EndKey)] = struct{}{}
			}
			if len(results) == len(tc.exp) {
				closer <- struct{}{}
			}
			return nil
		}
		func() {
			s := StartTestServer(t)
			defer s.Stop()

			go func() {
				// Sets a timeout, cut short by the stopper having drained.
				select {
				case <-time.After(time.Second):
				case <-s.Server.stopper.ShouldStop():
					return
				}
				closer <- struct{}{}
			}()

			// Split the Range. This should not have any asynchronous intents.
			if err := s.db.AdminSplit(splitKey); err != nil {
				t.Fatal(err)
			}

			if pErr := s.db.Txn(func(txn *client.Txn) *roachpb.Error {
				b := txn.NewBatch()
				for _, key := range tc.keys {
					b.Put(key, "test")
				}
				for _, kr := range tc.ranges {
					b.DelRange(kr[0], kr[1], false)
				}
				return txn.CommitInBatch(b)
			}); pErr != nil {
				t.Fatalf("%d: %s", i, pErr)
			}
			<-closer // wait for async intents
		}()
		// Verification. Note that this runs after the system has stopped, so that
		// everything asynchronous has already happened.
		expResult := tc.exp
		sort.Strings(expResult)
		var actResult []string
		for k := range results {
			actResult = append(actResult, k)
		}
		sort.Strings(actResult)
		if !reflect.DeepEqual(expResult, expResult) {
			t.Fatalf("%d: unexpected non-local intents, expected %s: %s", i, expResult, expResult)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package commands

import (
	. "github.com/limetext/lime/backend"
	"testing"
)

func TestNewWindow(t *testing.T) {
	ed := GetEditor()
	l := len(ed.Windows())
	ed.CommandHandler().RunWindowCommand(ed.ActiveWindow(), "new_window", nil)

	if len(ed.Windows()) != l+1 {
		t.Errorf("Expected %d window, but got %d", l+1, len(ed.Windows()))
	}
}

func TestCloseAll(t *testing.T) {
	ed := GetEditor()

	w := ed.NewWindow()
	defer w.Close()

	l := len(w.Views())

	ed.CommandHandler().RunWindowCommand(w, "new_file", nil)
	ed.CommandHandler().RunWindowCommand(w, "new_file", nil)
	ed.CommandHandler().RunWindowCommand(w, "new_file", nil)

	ed.CommandHandler().RunWindowCommand(w, "close_all", nil)

	if len(w.Views()) != l {
		t.Errorf("Expected %d views, but got %d", l, len(w.Views()))
	}
}

func TestCloseWindow(t *testing.T) {
	ed := GetEditor()
	w := ed.NewWindow()
	l := len(ed.Windows())
	ed.CommandHandler().RunWindowCommand(w, "close_window", nil)

	if len(ed.Windows()) != l-1 {
		t.Errorf("Expected %d window, but got %d", l-1, len(ed.Windows()))
	}
}

func TestNewAppWindow(t *testing.T) {
	ed := GetEditor()
	l := len(ed.Windows())
	ed.CommandHandler().RunApplicationCommand("new_window", nil)

	if len(ed.Windows()) != l+1 {
		t.Errorf("Expected %d window, but got %d", l+1, len(ed.Windows()))
	}
}

func TestCloseAppWindow(t *testing.T) {
	ed := GetEditor()
	_ = ed.NewWindow()
	l := len(ed.Windows())
	ed.CommandHandler().RunApplicationCommand("close_window", nil)

	if len(ed.Windows()) != l-1 {
		t.Errorf("Expected %d window, but got %d", l-1, len(ed.Windows()))
	}
}
<commit_msg>Make sure the test checks that all the views are closed.<commit_after>// Copyright 2013 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package commands

import (
	. "github.com/limetext/lime/backend"
	"testing"
)

func TestNewWindow(t *testing.T) {
	ed := GetEditor()
	l := len(ed.Windows())
	ed.CommandHandler().RunWindowCommand(ed.ActiveWindow(), "new_window", nil)

	if len(ed.Windows()) != l+1 {
		t.Errorf("Expected %d window, but got %d", l+1, len(ed.Windows()))
	}
}

func TestCloseAll(t *testing.T) {
	ed := GetEditor()

	w := ed.NewWindow()
	defer w.Close()

	ed.CommandHandler().RunWindowCommand(w, "new_file", nil)
	ed.CommandHandler().RunWindowCommand(w, "new_file", nil)
	ed.CommandHandler().RunWindowCommand(w, "new_file", nil)

	ed.CommandHandler().RunWindowCommand(w, "close_all", nil)

	if len(w.Views()) != 0 {
		t.Errorf("Expected no views, but got %d", len(w.Views()))
	}
}

func TestCloseWindow(t *testing.T) {
	ed := GetEditor()
	w := ed.NewWindow()
	l := len(ed.Windows())
	ed.CommandHandler().RunWindowCommand(w, "close_window", nil)

	if len(ed.Windows()) != l-1 {
		t.Errorf("Expected %d window, but got %d", l-1, len(ed.Windows()))
	}
}

func TestNewAppWindow(t *testing.T) {
	ed := GetEditor()
	l := len(ed.Windows())
	ed.CommandHandler().RunApplicationCommand("new_window", nil)

	if len(ed.Windows()) != l+1 {
		t.Errorf("Expected %d window, but got %d", l+1, len(ed.Windows()))
	}
}

func TestCloseAppWindow(t *testing.T) {
	ed := GetEditor()
	_ = ed.NewWindow()
	l := len(ed.Windows())
	ed.CommandHandler().RunApplicationCommand("close_window", nil)

	if len(ed.Windows()) != l-1 {
		t.Errorf("Expected %d window, but got %d", l-1, len(ed.Windows()))
	}
}
<|endoftext|>


<commit_before>package communication

import (
	"errors"
	"fmt"
	"log"
	"strings"
	"superstellar/backend/constants"
	"superstellar/backend/events"
	"superstellar/backend/pb"
	"time"

	"github.com/golang/protobuf/proto"

	"superstellar/backend/monitor"

	"superstellar/backend/utils"

	"github.com/gorilla/websocket"
)

const channelBufSize = 100

// Client struct holds client-specific variables.
type Client struct {
	id               uint32
	ws               *websocket.Conn
	ch               chan *[]byte
	doneCh           chan bool
	monitor          *monitor.Monitor
	eventDispatcher  *events.EventDispatcher
	userNameRegistry *utils.UserNamesRegistry
}

// NewClient initializes a new Client struct with given websocket and Server.
func NewClient(ws *websocket.Conn, monitor *monitor.Monitor, eventsDispatcher *events.EventDispatcher,
	userNameRegistry *utils.UserNamesRegistry, clientID uint32) *Client {
	if ws == nil {
		panic("ws cannot be nil")
	}

	ch := make(chan *[]byte, channelBufSize)
	doneCh := make(chan bool)

	return &Client{clientID, ws, ch, doneCh, monitor, eventsDispatcher, userNameRegistry}
}

// Conn returns client's websocket.Conn struct.
func (c *Client) Conn() *websocket.Conn {
	return c.ws
}

// SendMessage sends game state to the client.
func (c *Client) SendMessage(bytes *[]byte) {
	select {
	case c.ch <- bytes:
	default:
		c.monitor.AddDroppedMessage()
	}
}

// Done sends done message to the Client which closes the conection.
func (c *Client) Done() {
	c.doneCh <- true
}

// Listen Write and Read request via chanel
func (c *Client) Listen() {
	go c.listenWrite()
	c.listenRead()
}

// Listen write request via chanel
func (c *Client) listenWrite() {
	log.Println("Listening write to client")
	for {
		select {

		case bytes := <-c.ch:
			before := time.Now()
			err := c.ws.WriteMessage(websocket.BinaryMessage, *bytes)
			after := time.Now()

			if err != nil {
				log.Println(err)
			} else {
				elapsed := after.Sub(before)
				c.monitor.AddSendTime(elapsed)
			}

		case <-c.doneCh:
			c.doneCh <- true
			return
		}
	}
}

func (c *Client) listenRead() {
	log.Println("Listening read from client")
	for {
		select {

		case <-c.doneCh:
			c.doneCh <- true
			return

		default:
			c.readFromWebSocket()
		}
	}
}

func (c *Client) readFromWebSocket() {
	messageType, data, err := c.ws.ReadMessage()
	if err != nil {
		log.Println(err)

		c.doneCh <- true
		c.eventDispatcher.FireUserLeft(&events.UserLeft{ClientID: c.id})
	} else if messageType != websocket.BinaryMessage {
		log.Println("Non binary message recived, ignoring")
	} else {
		c.unmarshalUserInput(data)
	}
}

func (c *Client) unmarshalUserInput(data []byte) {
	protoUserMessage := &pb.UserMessage{}
	if err := proto.Unmarshal(data, protoUserMessage); err != nil {
		log.Fatalln("Failed to unmarshal UserInput:", err)
		return
	}

	switch x := protoUserMessage.Content.(type) {
	case *pb.UserMessage_UserAction:
		userInputEvent := events.UserInputFromProto(protoUserMessage.GetUserAction(), c.id)
		c.eventDispatcher.FireUserInput(userInputEvent)
	case *pb.UserMessage_TargetAngle:
		targetAngleEvent := events.TargetAngleFromProto(protoUserMessage.GetTargetAngle(), c.id)
		c.eventDispatcher.FireTargetAngle(targetAngleEvent)
	case *pb.UserMessage_JoinGame:
		c.tryToJoinGame(protoUserMessage.GetJoinGame())
	case *pb.UserMessage_Ping:
		c.sendPong(protoUserMessage.GetPing().Id)
	default:
		log.Fatalln("Unknown message type %T", x)
	}
}

func (c *Client) tryToJoinGame(joinGameMsg *pb.JoinGame) {
	username := strings.TrimSpace(joinGameMsg.Username)
	ok, err := validateUsername(username)

	if ok {
		c.userNameRegistry.AddUserName(c.id, username)
		c.sendJoinGameAckMessage(&pb.JoinGameAck{Success: true})
		c.eventDispatcher.FireUserJoined(&events.UserJoined{ClientID: c.id, UserName: username})
	} else {
		c.sendJoinGameAckMessage(
			&pb.JoinGameAck{Success: false, Error: err.Error()},
		)
	}
}

func validateUsername(username string) (bool, error) {
	length := len(username)

	if length < constants.MinimumUsernameLength {
		return false, errors.New(fmt.Sprintf("I doubt your name is shorter than %d characters, Captain.", constants.MinimumUsernameLength))
	}

	if length > constants.MaximumUsernameLength {
		return false, errors.New(fmt.Sprintf("Space fleet doesn't allow names longer than %d characters!", constants.MaximumUsernameLength))
	}

	return true, nil
}

func (c *Client) sendJoinGameAckMessage(joinGameAck *pb.JoinGameAck) {
	message := &pb.Message{
		Content: &pb.Message_JoinGameAck{
			JoinGameAck: joinGameAck,
		},
	}

	c.SendMessage(marshalMessage(message))
}

func (c *Client) sendPong(id uint32) {
	message := &pb.Message{
		Content: &pb.Message_Pong{
			Pong: &pb.Pong{Id: id},
		},
	}

	c.SendMessage(marshalMessage(message))
}
<commit_msg>Add defer to websocket connections<commit_after>package communication

import (
	"errors"
	"fmt"
	"log"
	"strings"
	"superstellar/backend/constants"
	"superstellar/backend/events"
	"superstellar/backend/pb"
	"time"

	"github.com/golang/protobuf/proto"

	"superstellar/backend/monitor"

	"superstellar/backend/utils"

	"github.com/gorilla/websocket"
)

const channelBufSize = 100

// Client struct holds client-specific variables.
type Client struct {
	id               uint32
	ws               *websocket.Conn
	ch               chan *[]byte
	doneCh           chan bool
	monitor          *monitor.Monitor
	eventDispatcher  *events.EventDispatcher
	userNameRegistry *utils.UserNamesRegistry
}

// NewClient initializes a new Client struct with given websocket and Server.
func NewClient(ws *websocket.Conn, monitor *monitor.Monitor, eventsDispatcher *events.EventDispatcher,
	userNameRegistry *utils.UserNamesRegistry, clientID uint32) *Client {
	if ws == nil {
		panic("ws cannot be nil")
	}

	ch := make(chan *[]byte, channelBufSize)
	doneCh := make(chan bool)

	return &Client{clientID, ws, ch, doneCh, monitor, eventsDispatcher, userNameRegistry}
}

// Conn returns client's websocket.Conn struct.
func (c *Client) Conn() *websocket.Conn {
	return c.ws
}

// SendMessage sends game state to the client.
func (c *Client) SendMessage(bytes *[]byte) {
	select {
	case c.ch <- bytes:
	default:
		c.monitor.AddDroppedMessage()
	}
}

// Done sends done message to the Client which closes the conection.
func (c *Client) Done() {
	c.doneCh <- true
}

// Listen Write and Read request via chanel
func (c *Client) Listen() {
	go c.listenWrite()
	c.listenRead()
}

// Listen write request via chanel
func (c *Client) listenWrite() {
	defer func() {
		err := c.ws.Close()
		if err != nil {
			log.Println("Error:", err.Error())
		}
	}()

	log.Println("Listening write to client")
	for {
		select {

		case bytes := <-c.ch:
			before := time.Now()
			err := c.ws.WriteMessage(websocket.BinaryMessage, *bytes)
			after := time.Now()

			if err != nil {
				log.Println(err)
			} else {
				elapsed := after.Sub(before)
				c.monitor.AddSendTime(elapsed)
			}

		case <-c.doneCh:
			c.doneCh <- true
			return
		}
	}
}

func (c *Client) listenRead() {
	defer func() {
		err := c.ws.Close()
		if err != nil {
			log.Println("Error:", err.Error())
		}
	}()

	log.Println("Listening read from client")
	for {
		select {

		case <-c.doneCh:
			c.doneCh <- true
			return

		default:
			c.readFromWebSocket()
		}
	}
}

func (c *Client) readFromWebSocket() {
	messageType, data, err := c.ws.ReadMessage()
	if err != nil {
		log.Println(err)

		c.doneCh <- true
		c.eventDispatcher.FireUserLeft(&events.UserLeft{ClientID: c.id})
	} else if messageType != websocket.BinaryMessage {
		log.Println("Non binary message recived, ignoring")
	} else {
		c.unmarshalUserInput(data)
	}
}

func (c *Client) unmarshalUserInput(data []byte) {
	protoUserMessage := &pb.UserMessage{}
	if err := proto.Unmarshal(data, protoUserMessage); err != nil {
		log.Fatalln("Failed to unmarshal UserInput:", err)
		return
	}

	switch x := protoUserMessage.Content.(type) {
	case *pb.UserMessage_UserAction:
		userInputEvent := events.UserInputFromProto(protoUserMessage.GetUserAction(), c.id)
		c.eventDispatcher.FireUserInput(userInputEvent)
	case *pb.UserMessage_TargetAngle:
		targetAngleEvent := events.TargetAngleFromProto(protoUserMessage.GetTargetAngle(), c.id)
		c.eventDispatcher.FireTargetAngle(targetAngleEvent)
	case *pb.UserMessage_JoinGame:
		c.tryToJoinGame(protoUserMessage.GetJoinGame())
	case *pb.UserMessage_Ping:
		c.sendPong(protoUserMessage.GetPing().Id)
	default:
		log.Fatalln("Unknown message type %T", x)
	}
}

func (c *Client) tryToJoinGame(joinGameMsg *pb.JoinGame) {
	username := strings.TrimSpace(joinGameMsg.Username)
	ok, err := validateUsername(username)

	if ok {
		c.userNameRegistry.AddUserName(c.id, username)
		c.sendJoinGameAckMessage(&pb.JoinGameAck{Success: true})
		c.eventDispatcher.FireUserJoined(&events.UserJoined{ClientID: c.id, UserName: username})
	} else {
		c.sendJoinGameAckMessage(
			&pb.JoinGameAck{Success: false, Error: err.Error()},
		)
	}
}

func validateUsername(username string) (bool, error) {
	length := len(username)

	if length < constants.MinimumUsernameLength {
		return false, errors.New(fmt.Sprintf("I doubt your name is shorter than %d characters, Captain.", constants.MinimumUsernameLength))
	}

	if length > constants.MaximumUsernameLength {
		return false, errors.New(fmt.Sprintf("Space fleet doesn't allow names longer than %d characters!", constants.MaximumUsernameLength))
	}

	return true, nil
}

func (c *Client) sendJoinGameAckMessage(joinGameAck *pb.JoinGameAck) {
	message := &pb.Message{
		Content: &pb.Message_JoinGameAck{
			JoinGameAck: joinGameAck,
		},
	}

	c.SendMessage(marshalMessage(message))
}

func (c *Client) sendPong(id uint32) {
	message := &pb.Message{
		Content: &pb.Message_Pong{
			Pong: &pb.Pong{Id: id},
		},
	}

	c.SendMessage(marshalMessage(message))
}
<|endoftext|>


<commit_before>package kyuko

import (
	"time"

	"github.com/PuerkitoBio/goquery"
	goTwitter "github.com/dghubble/go-twitter/twitter"
	"github.com/g-hyoga/kyuko/go/model"
	"github.com/g-hyoga/kyuko/go/scrape"
	"github.com/g-hyoga/kyuko/go/twitter"
)

// scrapingを実行してデータベースに保存する
// その後twitterに投稿
func Exec(place int, client *goTwitter.Client) ([]model.KyukoData, error) {
	var kyukoResult []model.KyukoData
	var err error

	//今日の日付
	weekday := int(time.Now().Weekday())
	//今の時間
	nowTime := time.Now().Hour()
	// 18:00超えてたら次の日の情報にする
	if nowTime >= 18 {
		weekday += 1
	}
	// 日曜なら月曜の情報にする
	if weekday == 7 {
		weekday = 1
	}

	//第一引数:校地
	//第二引数:曜日
	url, err := scrape.SetUrl(place, weekday)
	if err != nil {
		return nil, err
	}

	//http
	reader, err := scrape.Get(url)
	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromReader(reader)
	if err != nil {
		return nil, err
	}

	kyukoData, err := scrape.Scrape(doc)
	if err != nil {
		return nil, err
	}

	var db model.DB
	err = db.Connect()
	if err != nil {
		return nil, err
	}
	defer db.Close()

	for _, data := range kyukoData {
		_, err = db.Insert(data)
		kyukoResult = append(kyukoResult, data)
		if err != nil {
			return nil, err
		}

		canceledClass, err := model.KyukoToCanceled(data)
		if err != nil {
			return nil, err
		}

		//挿入するデータが存在するのか確認
		id, err := db.ShowCanceledClassID(canceledClass)
		if err != nil {
			return nil, err
		}

		//DBに存在するデータかつ今日のデータでないなら
		if isExist, _ := db.IsExistToday(data); id != -1 && !isExist {
			canceledClass.ID = id
			_, err = db.AddCanceled(canceledClass.ID)
			if err != nil {
				return nil, err
			}
		}

		_, err = db.InsertCanceledClass(canceledClass)
		if err != nil {
			return nil, err
		}
	}

	tws, err := twitter.CreateContent(kyukoData)
	//_, err = twitter.CreateContent(kyukoData)
	if err != nil {
		return nil, err
	}

	for _, tw := range tws {
		err := twitter.Update(client, tw)
		if err != nil {
			return nil, err
			return nil, err
		}
	}

	return kyukoData, nil
}
<commit_msg>処理を分けた<commit_after>package kyuko

import (
	"time"

	"github.com/PuerkitoBio/goquery"
	goTwitter "github.com/dghubble/go-twitter/twitter"
	"github.com/g-hyoga/kyuko/go/model"
	"github.com/g-hyoga/kyuko/go/scrape"
	"github.com/g-hyoga/kyuko/go/twitter"
)

func weekdayToday() int {
	//今日の曜日
	weekday := int(time.Now().Weekday())
	//今の時間
	nowTime := time.Now().Hour()
	// 18:00超えてたら次の日の情報にする
	if nowTime >= 18 {
		weekday += 1
	}
	// 日曜なら月曜の情報にする
	if weekday == 7 {
		weekday = 1
	}
	return weekday
}

func scraper(place, weekday int) ([]model.KyukoData, error) {
	//第一引数:校地
	//第二引数:曜日
	url, err := scrape.SetUrl(place, weekday)
	if err != nil {
		return nil, err
	}

	//http
	reader, err := scrape.Get(url)
	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromReader(reader)
	if err != nil {
		return nil, err
	}

	kyukoData, err := scrape.Scrape(doc)
	if err != nil {
		return nil, err
	}
	return kyukoData, nil
}

func ManageDB(kyukoData []model.KyukoData) error {
	var db model.DB
	err = db.Connect()
	if err != nil {
		return err
	}
	defer db.Close()

	for _, data := range kyukoData {
		_, err = db.Insert(data)
		kyukoResult = append(kyukoResult, data)
		if err != nil {
			return err
		}

		canceledClass, err := model.KyukoToCanceled(data)
		if err != nil {
			return err
		}

		//挿入するデータが存在するのか確認
		id, err := db.ShowCanceledClassID(canceledClass)
		if err != nil {
			return err
		}

		//DBに存在するデータかつ今日のデータでないなら
		if isExist, _ := db.IsExistToday(data); id != -1 && !isExist {
			canceledClass.ID = id
			_, err = db.AddCanceled(canceledClass.ID)
			if err != nil {
				return err
			}
		}

		_, err = db.InsertCanceledClass(canceledClass)
		if err != nil {
			return err
		}
	}
	return nil
}

func manageTwitter(kyukoData model.KyukoData) error {
	tws, err := twitter.CreateContent(kyukoData)
	//_, err = twitter.CreateContent(kyukoData)
	if err != nil {
		return err
	}

	for _, tw := range tws {
		err := twitter.Update(client, tw)
		if err != nil {
			return err
		}
	}

	return nil
}

// scrapingを実行してデータベースに保存する
// その後twitterに投稿
func Exec(place int, client *goTwitter.Client) ([]model.KyukoData, error) {
	var kyukoResult []model.KyukoData

	weekday := weekdayToday()

	kyukoData, err := scraper(place, weekday)
	if err != nil {
		return kyukoResult, err
	}

	err := manageDB(kyuokData)
	if err != nil {
		return kyukoResult, err
	}

	err := manageTwitter(kyukoData)
	if err != nil {
		return kyukoResult, err
	}

	return kyukoData, nil
}
<|endoftext|>


<commit_before>package cas

import (
	"io"
	"os"
	"errors"
	"io/ioutil"
	"github.com/s3git/s3git-go/internal/kv"
	"github.com/s3git/s3git-go/internal/backend"
	"encoding/hex"
)

func MakeReader(hash string) *Reader {
	cr := Reader{hash: hash}
	err := cr.open(hash)
	if err != nil {
		return nil
	}
	return &cr
}

type Reader struct {
	hash        string
	offset      int64
	leafNr      int
	leaves      []Key
	chunkBuf    []byte
	chunkOffset int
	chunkLast   bool
}

func openRoot(hash string) ([]Key, error) {

	key, _ := hex.DecodeString(hash)
	leafHashes, _, err := kv.GetLevel1(key)
	if err != nil {
		return nil, err
	}

	// Has blob already been pulled down to disk?
	if len(leafHashes) == 0 {

		var err error
		leafHashes, err = pullDownOnDemand(hash)
		if err != nil {
			return nil, err
		}
	}
	leaves := make([]Key, 0, len(leafHashes)/KeySize)
	for i := 0; i < len(leafHashes); i += KeySize {
		leaves = append(leaves, NewKey(leafHashes[i:i+KeySize]))
	}
	return leaves, nil
}

// Pull a blob on demand from the back end store
func pullDownOnDemand(hash string) ([]byte, error) {

	// TODO: implement streaming mode for large blobs, spawn off multiple GET range-headers

	var client backend.Backend
	if client == nil  {
		client = backend.GetDefaultClient()
	}

	b, _ := hex.DecodeString(hash)
	_, objType, err := kv.GetLevel1(b)
	if err != nil {
		return nil, err
	}
	leafHashes, err := pullBlobDownToLocalDisk(hash, objType, client)
	if err != nil {
		return nil, err
	}

	return leafHashes, nil
}

// Open the root hash of a blob
func (cr *Reader) open(hash string) error {

	var err error
	cr.leaves, err = openRoot(hash)
	return err
}

// Read the contents of a blob
func (cr *Reader) Read(p []byte) (n int, err error) {

	// TODO: Consider returning EOF in case of errors/failed functions?

	bytesToRead := len(p)

	if cr.chunkLast && cr.chunkOffset == 0 {
		return 0, io.EOF
	}

	for {
		if cr.chunkBuf == nil {
			key := cr.leaves[cr.leafNr].String()
			// TODO: Consider using a memory pool and reusing chunks?

			// Check whether chunk is available on local disk, if not, pull down to local disk
			chunkFile := getBlobPath(key)
			if _, err := os.Stat(chunkFile); os.IsNotExist(err) {

				// Chunk is missing, load file from back end
				// TODO: Ideally optimize here to just get the missing chunk (not whole file)
				_, err = pullDownOnDemand(cr.hash)
				if err != nil {
					return 0, err
				}

				// Double check that missing chunk is now available
				if _, err := os.Stat(chunkFile); os.IsNotExist(err) {
					return 0, errors.New("Failed to fetch missing chunk from back end")
				}
			}

			cr.chunkBuf, err = ioutil.ReadFile(chunkFile)
			if err != nil {
				return 0, err
			}
			cr.chunkOffset = 0
			cr.leafNr++
			cr.chunkLast = cr.leafNr == len(cr.leaves) // Mark when we have the last chunk
		}

		remainingInBuffer := len(cr.chunkBuf) - cr.chunkOffset
		bytesRead := min(bytesToRead, remainingInBuffer)
		copy(p[len(p)-bytesToRead:], cr.chunkBuf[cr.chunkOffset:cr.chunkOffset+bytesRead])
		bytesToRead -= bytesRead
		remainingInBuffer -= bytesRead
		cr.chunkOffset += bytesRead

		if bytesToRead <= 0 {
			return bytesRead, nil
		}

		if remainingInBuffer <= 0 {
			if cr.chunkLast {
				return bytesRead, io.EOF
			}
			cr.chunkBuf = nil
			cr.chunkOffset = 0
		}
	}
}

// Fetch blob down to temp file in order to load
func FetchBlobToTempFile(hash string, client backend.Backend) (tempFile string, err error) {

	file, err := ioutil.TempFile("", "pull-")
	if err != nil {
		return "", err
	}

	err = client.DownloadWithWriter(hash, file)
	if err != nil {
		return "", err
	}
	name := file.Name()
	file.Close()

	return name, nil
}

// Store the blob in the caching area of the cas
func StoreBlobInCache(name, objType string) ([]byte, error) {

	in, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	defer in.Close()

	var cw *Writer
	if objType == kv.PREFIX {
		cw = MakeWriterInCheatMode(objType)
	} else {
		cw = MakeWriter(objType)
	}

	// For pulls write to the cache area (not stage)
	cw.setAreaDir(cacheDir)

	io.Copy(cw, in)

	_, leafHashes, _, err := cw.Flush()
	if err != nil {
		return nil, err
	}

	return leafHashes, nil
}

func pullBlobDownToLocalDisk(hash, objType string, client backend.Backend) ([]byte, error) {

	// TODO: Remove work around by using separate file
	name, err := FetchBlobToTempFile(hash, client)
	if err != nil {
		return nil, err
	}
	defer os.Remove(name)

	return StoreBlobInCache(name, objType)
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<commit_msg>Keep function internal<commit_after>package cas

import (
	"io"
	"os"
	"errors"
	"io/ioutil"
	"github.com/s3git/s3git-go/internal/kv"
	"github.com/s3git/s3git-go/internal/backend"
	"encoding/hex"
)

func MakeReader(hash string) *Reader {
	cr := Reader{hash: hash}
	err := cr.open(hash)
	if err != nil {
		return nil
	}
	return &cr
}

type Reader struct {
	hash        string
	offset      int64
	leafNr      int
	leaves      []Key
	chunkBuf    []byte
	chunkOffset int
	chunkLast   bool
}

func openRoot(hash string) ([]Key, error) {

	key, _ := hex.DecodeString(hash)
	leafHashes, _, err := kv.GetLevel1(key)
	if err != nil {
		return nil, err
	}

	// Has blob already been pulled down to disk?
	if len(leafHashes) == 0 {

		var err error
		leafHashes, err = pullDownOnDemand(hash)
		if err != nil {
			return nil, err
		}
	}
	leaves := make([]Key, 0, len(leafHashes)/KeySize)
	for i := 0; i < len(leafHashes); i += KeySize {
		leaves = append(leaves, NewKey(leafHashes[i:i+KeySize]))
	}
	return leaves, nil
}

// Pull a blob on demand from the back end store
func pullDownOnDemand(hash string) ([]byte, error) {

	// TODO: implement streaming mode for large blobs, spawn off multiple GET range-headers

	var client backend.Backend
	if client == nil  {
		client = backend.GetDefaultClient()
	}

	b, _ := hex.DecodeString(hash)
	_, objType, err := kv.GetLevel1(b)
	if err != nil {
		return nil, err
	}
	leafHashes, err := PullBlobDownToLocalDisk(hash, objType, client)
	if err != nil {
		return nil, err
	}

	return leafHashes, nil
}

// Open the root hash of a blob
func (cr *Reader) open(hash string) error {

	var err error
	cr.leaves, err = openRoot(hash)
	return err
}

// Read the contents of a blob
func (cr *Reader) Read(p []byte) (n int, err error) {

	// TODO: Consider returning EOF in case of errors/failed functions?

	bytesToRead := len(p)

	if cr.chunkLast && cr.chunkOffset == 0 {
		return 0, io.EOF
	}

	for {
		if cr.chunkBuf == nil {
			key := cr.leaves[cr.leafNr].String()
			// TODO: Consider using a memory pool and reusing chunks?

			// Check whether chunk is available on local disk, if not, pull down to local disk
			chunkFile := getBlobPath(key)
			if _, err := os.Stat(chunkFile); os.IsNotExist(err) {

				// Chunk is missing, load file from back end
				// TODO: Ideally optimize here to just get the missing chunk (not whole file)
				_, err = pullDownOnDemand(cr.hash)
				if err != nil {
					return 0, err
				}

				// Double check that missing chunk is now available
				if _, err := os.Stat(chunkFile); os.IsNotExist(err) {
					return 0, errors.New("Failed to fetch missing chunk from back end")
				}
			}

			cr.chunkBuf, err = ioutil.ReadFile(chunkFile)
			if err != nil {
				return 0, err
			}
			cr.chunkOffset = 0
			cr.leafNr++
			cr.chunkLast = cr.leafNr == len(cr.leaves) // Mark when we have the last chunk
		}

		remainingInBuffer := len(cr.chunkBuf) - cr.chunkOffset
		bytesRead := min(bytesToRead, remainingInBuffer)
		copy(p[len(p)-bytesToRead:], cr.chunkBuf[cr.chunkOffset:cr.chunkOffset+bytesRead])
		bytesToRead -= bytesRead
		remainingInBuffer -= bytesRead
		cr.chunkOffset += bytesRead

		if bytesToRead <= 0 {
			return bytesRead, nil
		}

		if remainingInBuffer <= 0 {
			if cr.chunkLast {
				return bytesRead, io.EOF
			}
			cr.chunkBuf = nil
			cr.chunkOffset = 0
		}
	}
}

// Fetch blob down to temp file in order to load
func FetchBlobToTempFile(hash string, client backend.Backend) (tempFile string, err error) {

	file, err := ioutil.TempFile("", "pull-")
	if err != nil {
		return "", err
	}

	err = client.DownloadWithWriter(hash, file)
	if err != nil {
		return "", err
	}
	name := file.Name()
	file.Close()

	return name, nil
}

// Store the blob in the caching area of the cas
func StoreBlobInCache(name, objType string) ([]byte, error) {

	in, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	defer in.Close()

	var cw *Writer
	if objType == kv.PREFIX {
		cw = MakeWriterInCheatMode(objType)
	} else {
		cw = MakeWriter(objType)
	}

	// For pulls write to the cache area (not stage)
	cw.setAreaDir(cacheDir)

	io.Copy(cw, in)

	_, leafHashes, _, err := cw.Flush()
	if err != nil {
		return nil, err
	}

	return leafHashes, nil
}

func PullBlobDownToLocalDisk(hash, objType string, client backend.Backend) ([]byte, error) {

	// TODO: Remove work around by using separate file
	name, err := FetchBlobToTempFile(hash, client)
	if err != nil {
		return nil, err
	}
	defer os.Remove(name)

	return StoreBlobInCache(name, objType)
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>