<commit_before>package filesys

import (
	"context"
	"os"
	"sort"
	"time"

	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

const blockSize = 512

var _ = fs.Node(&File{})
var _ = fs.NodeIdentifier(&File{})
var _ = fs.NodeOpener(&File{})
var _ = fs.NodeFsyncer(&File{})
var _ = fs.NodeSetattrer(&File{})
var _ = fs.NodeGetxattrer(&File{})
var _ = fs.NodeSetxattrer(&File{})
var _ = fs.NodeRemovexattrer(&File{})
var _ = fs.NodeListxattrer(&File{})
var _ = fs.NodeForgetter(&File{})

type File struct {
	Name          string
	dir           *Dir
	wfs           *WFS
	entry         *filer_pb.Entry
	isOpen        int
	dirtyMetadata bool
	id            uint64
}

func (file *File) fullpath() util.FullPath {
	return util.NewFullPath(file.dir.FullPath(), file.Name)
}

func (file *File) Id() uint64 {
	return file.id
}

func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) {

	glog.V(4).Infof("file Attr %s, open:%v existing:%v", file.fullpath(), file.isOpen, attr)

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if entry == nil {
		return fuse.ENOENT
	}

	attr.Inode = file.Id()
	attr.Valid = time.Second
	attr.Mode = os.FileMode(entry.Attributes.FileMode)
	attr.Size = filer.FileSize(entry)
	if file.isOpen > 0 {
		attr.Size = entry.Attributes.FileSize
		glog.V(4).Infof("file Attr %s, open:%v, size: %d", file.fullpath(), file.isOpen, attr.Size)
	}
	attr.Crtime = time.Unix(entry.Attributes.Crtime, 0)
	attr.Ctime = time.Unix(entry.Attributes.Mtime, 0)
	attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
	attr.Gid = entry.Attributes.Gid
	attr.Uid = entry.Attributes.Uid
	attr.Blocks = attr.Size/blockSize + 1
	attr.BlockSize = uint32(file.wfs.option.ChunkSizeLimit)
	if entry.HardLinkCounter > 0 {
		attr.Nlink = uint32(entry.HardLinkCounter)
	}

	return nil

}

func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {

	// glog.V(4).Infof("file Getxattr %s", file.fullpath())

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	return getxattr(entry, req, resp)
}

func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {

	glog.V(4).Infof("file %v open %+v", file.fullpath(), req)
	// resp.Flags |= fuse.OpenDirectIO

	handle := file.wfs.AcquireHandle(file, req.Uid, req.Gid)

	resp.Handle = fuse.HandleID(handle.handle)

	glog.V(4).Infof("%v file open handle id = %d", file.fullpath(), handle.handle)

	return handle, nil

}

func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {

	glog.V(4).Infof("%v file setattr %+v mode=%d", file.fullpath(), req, req.Mode)

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if req.Valid.Size() {

		glog.V(4).Infof("%v file setattr set size=%v chunks=%d", file.fullpath(), req.Size, len(entry.Chunks))
		if req.Size < filer.FileSize(entry) {
			// fmt.Printf("truncate %v \n", fullPath)
			var chunks []*filer_pb.FileChunk
			var truncatedChunks []*filer_pb.FileChunk
			for _, chunk := range entry.Chunks {
				int64Size := int64(chunk.Size)
				if chunk.Offset+int64Size > int64(req.Size) {
					// this chunk is truncated
					int64Size = int64(req.Size) - chunk.Offset
					if int64Size > 0 {
						chunks = append(chunks, chunk)
						glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
						chunk.Size = uint64(int64Size)
					} else {
						glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
						truncatedChunks = append(truncatedChunks, chunk)
					}
				}
			}
			// set the new chunks and reset entry cache
			entry.Chunks = chunks
			file.wfs.handlesLock.Lock()
			existingHandle, found := file.wfs.handles[file.Id()]
			file.wfs.handlesLock.Unlock()
			if found {
				existingHandle.entryViewCache = nil
			}

		}
		entry.Attributes.Mtime = time.Now().Unix()
		entry.Attributes.FileSize = req.Size
		file.dirtyMetadata = true
	}

	if req.Valid.Mode() && entry.Attributes.FileMode != uint32(req.Mode) {
		entry.Attributes.FileMode = uint32(req.Mode)
		entry.Attributes.Mtime = time.Now().Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Uid() && entry.Attributes.Uid != req.Uid {
		entry.Attributes.Uid = req.Uid
		entry.Attributes.Mtime = time.Now().Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Gid() && entry.Attributes.Gid != req.Gid {
		entry.Attributes.Gid = req.Gid
		entry.Attributes.Mtime = time.Now().Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Crtime() {
		entry.Attributes.Crtime = req.Crtime.Unix()
		entry.Attributes.Mtime = time.Now().Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Mtime() && entry.Attributes.Mtime != req.Mtime.Unix() {
		entry.Attributes.Mtime = req.Mtime.Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Handle() {
		// fmt.Printf("file handle => %d\n", req.Handle)
	}

	if file.isOpen > 0 {
		return nil
	}

	if !file.dirtyMetadata {
		return nil
	}

	return file.saveEntry(entry)

}

func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {

	glog.V(4).Infof("file Setxattr %s: %s", file.fullpath(), req.Name)

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if err := setxattr(entry, req); err != nil {
		return err
	}
	file.dirtyMetadata = true

	if file.isOpen > 0 {
		return nil
	}

	return file.saveEntry(entry)

}

func (file *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {

	glog.V(4).Infof("file Removexattr %s: %s", file.fullpath(), req.Name)

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if err := removexattr(entry, req); err != nil {
		return err
	}
	file.dirtyMetadata = true

	if file.isOpen > 0 {
		return nil
	}

	return file.saveEntry(entry)

}

func (file *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {

	glog.V(4).Infof("file Listxattr %s", file.fullpath())

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if err := listxattr(entry, req, resp); err != nil {
		return err
	}

	return nil

}

func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {

	// write the file chunks to the filerGrpcAddress
	glog.V(4).Infof("%s/%s fsync file %+v", file.dir.FullPath(), file.Name, req)

	return file.wfs.Fsync(file, req.Header)

}

func (file *File) Forget() {
	t := util.NewFullPath(file.dir.FullPath(), file.Name)
	glog.V(4).Infof("Forget file %s", t)
	file.wfs.ReleaseHandle(t, fuse.HandleID(t.AsInode(file.entry.FileMode())))

}

func (file *File) maybeLoadEntry(ctx context.Context) (entry *filer_pb.Entry, err error) {

	file.wfs.handlesLock.Lock()
	handle, found := file.wfs.handles[file.Id()]
	file.wfs.handlesLock.Unlock()
	entry = file.entry
	if found {
		// glog.V(4).Infof("maybeLoadEntry found opened file %s/%s", file.dir.FullPath(), file.Name)
		entry = handle.f.entry
	}

	if entry != nil {
		if len(entry.HardLinkId) == 0 {
			// only always reload hard link
			return entry, nil
		}
	}
	entry, err = file.wfs.maybeLoadEntry(file.dir.FullPath(), file.Name)
	if err != nil {
		glog.V(3).Infof("maybeLoadEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err)
		return entry, err
	}
	if entry != nil {
		// file.entry = entry
	} else {
		glog.Warningf("maybeLoadEntry not found entry %s/%s: %v", file.dir.FullPath(), file.Name, err)
	}
	return entry, nil
}

func lessThan(a, b *filer_pb.FileChunk) bool {
	if a.Mtime == b.Mtime {
		return a.Fid.FileKey < b.Fid.FileKey
	}
	return a.Mtime < b.Mtime
}

func (file *File) addChunks(chunks []*filer_pb.FileChunk) {

	// find the earliest incoming chunk
	newChunks := chunks
	earliestChunk := newChunks[0]
	for i := 1; i < len(newChunks); i++ {
		if lessThan(earliestChunk, newChunks[i]) {
			earliestChunk = newChunks[i]
		}
	}

	entry := file.getEntry()
	if entry == nil {
		return
	}

	// pick out-of-order chunks from existing chunks
	for _, chunk := range entry.Chunks {
		if lessThan(earliestChunk, chunk) {
			chunks = append(chunks, chunk)
		}
	}

	// sort incoming chunks
	sort.Slice(chunks, func(i, j int) bool {
		return lessThan(chunks[i], chunks[j])
	})

	glog.V(4).Infof("%s existing %d chunks adds %d more", file.fullpath(), len(entry.Chunks), len(chunks))

	entry.Chunks = append(entry.Chunks, newChunks...)
}

func (file *File) saveEntry(entry *filer_pb.Entry) error {
	return file.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

		file.wfs.mapPbIdFromLocalToFiler(entry)
		defer file.wfs.mapPbIdFromFilerToLocal(entry)

		request := &filer_pb.CreateEntryRequest{
			Directory:  file.dir.FullPath(),
			Entry:      entry,
			Signatures: []int32{file.wfs.signature},
		}

		glog.V(4).Infof("save file entry: %v", request)
		_, err := client.CreateEntry(context.Background(), request)
		if err != nil {
			glog.Errorf("UpdateEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err)
			return fuse.EIO
		}

		file.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry))

		file.dirtyMetadata = false

		return nil
	})
}

func (file *File) getEntry() *filer_pb.Entry {
	return file.entry
}

func (file *File) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entry, error) {
	err := file.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.CacheRemoteObjectToLocalClusterRequest{
			Directory: file.dir.FullPath(),
			Name:      entry.Name,
		}

		glog.V(4).Infof("download entry: %v", request)
		resp, err := client.CacheRemoteObjectToLocalCluster(context.Background(), request)
		if err != nil {
			glog.Errorf("CacheRemoteObjectToLocalCluster file %s/%s: %v", file.dir.FullPath(), file.Name, err)
			return fuse.EIO
		}

		entry = resp.Entry

		file.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, resp.Entry))

		file.dirtyMetadata = false

		return nil
	})

	return entry, err
}
<commit_msg>blocks count<commit_after>package filesys

import (
	"context"
	"os"
	"sort"
	"time"

	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

const blockSize = 512

var _ = fs.Node(&File{})
var _ = fs.NodeIdentifier(&File{})
var _ = fs.NodeOpener(&File{})
var _ = fs.NodeFsyncer(&File{})
var _ = fs.NodeSetattrer(&File{})
var _ = fs.NodeGetxattrer(&File{})
var _ = fs.NodeSetxattrer(&File{})
var _ = fs.NodeRemovexattrer(&File{})
var _ = fs.NodeListxattrer(&File{})
var _ = fs.NodeForgetter(&File{})

type File struct {
	Name          string
	dir           *Dir
	wfs           *WFS
	entry         *filer_pb.Entry
	isOpen        int
	dirtyMetadata bool
	id            uint64
}

func (file *File) fullpath() util.FullPath {
	return util.NewFullPath(file.dir.FullPath(), file.Name)
}

func (file *File) Id() uint64 {
	return file.id
}

func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) {

	glog.V(4).Infof("file Attr %s, open:%v existing:%v", file.fullpath(), file.isOpen, attr)

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if entry == nil {
		return fuse.ENOENT
	}

	attr.Inode = file.Id()
	attr.Valid = time.Second
	attr.Mode = os.FileMode(entry.Attributes.FileMode)
	attr.Size = filer.FileSize(entry)
	if file.isOpen > 0 {
		attr.Size = entry.Attributes.FileSize
		glog.V(4).Infof("file Attr %s, open:%v, size: %d", file.fullpath(), file.isOpen, attr.Size)
	}
	attr.Crtime = time.Unix(entry.Attributes.Crtime, 0)
	attr.Ctime = time.Unix(entry.Attributes.Mtime, 0)
	attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
	attr.Gid = entry.Attributes.Gid
	attr.Uid = entry.Attributes.Uid
	attr.BlockSize = blockSize
	attr.Blocks = (attr.Size + blockSize - 1) / blockSize
	if entry.HardLinkCounter > 0 {
		attr.Nlink = uint32(entry.HardLinkCounter)
	}

	return nil

}

func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {

	// glog.V(4).Infof("file Getxattr %s", file.fullpath())

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	return getxattr(entry, req, resp)
}

func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {

	glog.V(4).Infof("file %v open %+v", file.fullpath(), req)
	// resp.Flags |= fuse.OpenDirectIO

	handle := file.wfs.AcquireHandle(file, req.Uid, req.Gid)

	resp.Handle = fuse.HandleID(handle.handle)

	glog.V(4).Infof("%v file open handle id = %d", file.fullpath(), handle.handle)

	return handle, nil

}

func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {

	glog.V(4).Infof("%v file setattr %+v mode=%d", file.fullpath(), req, req.Mode)

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if req.Valid.Size() {

		glog.V(4).Infof("%v file setattr set size=%v chunks=%d", file.fullpath(), req.Size, len(entry.Chunks))
		if req.Size < filer.FileSize(entry) {
			// fmt.Printf("truncate %v \n", fullPath)
			var chunks []*filer_pb.FileChunk
			var truncatedChunks []*filer_pb.FileChunk
			for _, chunk := range entry.Chunks {
				int64Size := int64(chunk.Size)
				if chunk.Offset+int64Size > int64(req.Size) {
					// this chunk is truncated
					int64Size = int64(req.Size) - chunk.Offset
					if int64Size > 0 {
						chunks = append(chunks, chunk)
						glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
						chunk.Size = uint64(int64Size)
					} else {
						glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
						truncatedChunks = append(truncatedChunks, chunk)
					}
				}
			}
			// set the new chunks and reset entry cache
			entry.Chunks = chunks
			file.wfs.handlesLock.Lock()
			existingHandle, found := file.wfs.handles[file.Id()]
			file.wfs.handlesLock.Unlock()
			if found {
				existingHandle.entryViewCache = nil
			}

		}
		entry.Attributes.Mtime = time.Now().Unix()
		entry.Attributes.FileSize = req.Size
		file.dirtyMetadata = true
	}

	if req.Valid.Mode() && entry.Attributes.FileMode != uint32(req.Mode) {
		entry.Attributes.FileMode = uint32(req.Mode)
		entry.Attributes.Mtime = time.Now().Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Uid() && entry.Attributes.Uid != req.Uid {
		entry.Attributes.Uid = req.Uid
		entry.Attributes.Mtime = time.Now().Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Gid() && entry.Attributes.Gid != req.Gid {
		entry.Attributes.Gid = req.Gid
		entry.Attributes.Mtime = time.Now().Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Crtime() {
		entry.Attributes.Crtime = req.Crtime.Unix()
		entry.Attributes.Mtime = time.Now().Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Mtime() && entry.Attributes.Mtime != req.Mtime.Unix() {
		entry.Attributes.Mtime = req.Mtime.Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Handle() {
		// fmt.Printf("file handle => %d\n", req.Handle)
	}

	if file.isOpen > 0 {
		return nil
	}

	if !file.dirtyMetadata {
		return nil
	}

	return file.saveEntry(entry)

}

func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {

	glog.V(4).Infof("file Setxattr %s: %s", file.fullpath(), req.Name)

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if err := setxattr(entry, req); err != nil {
		return err
	}
	file.dirtyMetadata = true

	if file.isOpen > 0 {
		return nil
	}

	return file.saveEntry(entry)

}

func (file *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {

	glog.V(4).Infof("file Removexattr %s: %s", file.fullpath(), req.Name)

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if err := removexattr(entry, req); err != nil {
		return err
	}
	file.dirtyMetadata = true

	if file.isOpen > 0 {
		return nil
	}

	return file.saveEntry(entry)

}

func (file *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {

	glog.V(4).Infof("file Listxattr %s", file.fullpath())

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if err := listxattr(entry, req, resp); err != nil {
		return err
	}

	return nil

}

func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {

	// write the file chunks to the filerGrpcAddress
	glog.V(4).Infof("%s/%s fsync file %+v", file.dir.FullPath(), file.Name, req)

	return file.wfs.Fsync(file, req.Header)

}

func (file *File) Forget() {
	t := util.NewFullPath(file.dir.FullPath(), file.Name)
	glog.V(4).Infof("Forget file %s", t)
	file.wfs.ReleaseHandle(t, fuse.HandleID(t.AsInode(file.entry.FileMode())))

}

func (file *File) maybeLoadEntry(ctx context.Context) (entry *filer_pb.Entry, err error) {

	file.wfs.handlesLock.Lock()
	handle, found := file.wfs.handles[file.Id()]
	file.wfs.handlesLock.Unlock()
	entry = file.entry
	if found {
		// glog.V(4).Infof("maybeLoadEntry found opened file %s/%s", file.dir.FullPath(), file.Name)
		entry = handle.f.entry
	}

	if entry != nil {
		if len(entry.HardLinkId) == 0 {
			// only always reload hard link
			return entry, nil
		}
	}
	entry, err = file.wfs.maybeLoadEntry(file.dir.FullPath(), file.Name)
	if err != nil {
		glog.V(3).Infof("maybeLoadEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err)
		return entry, err
	}
	if entry != nil {
		// file.entry = entry
	} else {
		glog.Warningf("maybeLoadEntry not found entry %s/%s: %v", file.dir.FullPath(), file.Name, err)
	}
	return entry, nil
}

func lessThan(a, b *filer_pb.FileChunk) bool {
	if a.Mtime == b.Mtime {
		return a.Fid.FileKey < b.Fid.FileKey
	}
	return a.Mtime < b.Mtime
}

func (file *File) addChunks(chunks []*filer_pb.FileChunk) {

	// find the earliest incoming chunk
	newChunks := chunks
	earliestChunk := newChunks[0]
	for i := 1; i < len(newChunks); i++ {
		if lessThan(earliestChunk, newChunks[i]) {
			earliestChunk = newChunks[i]
		}
	}

	entry := file.getEntry()
	if entry == nil {
		return
	}

	// pick out-of-order chunks from existing chunks
	for _, chunk := range entry.Chunks {
		if lessThan(earliestChunk, chunk) {
			chunks = append(chunks, chunk)
		}
	}

	// sort incoming chunks
	sort.Slice(chunks, func(i, j int) bool {
		return lessThan(chunks[i], chunks[j])
	})

	glog.V(4).Infof("%s existing %d chunks adds %d more", file.fullpath(), len(entry.Chunks), len(chunks))

	entry.Chunks = append(entry.Chunks, newChunks...)
}

func (file *File) saveEntry(entry *filer_pb.Entry) error {
	return file.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

		file.wfs.mapPbIdFromLocalToFiler(entry)
		defer file.wfs.mapPbIdFromFilerToLocal(entry)

		request := &filer_pb.CreateEntryRequest{
			Directory:  file.dir.FullPath(),
			Entry:      entry,
			Signatures: []int32{file.wfs.signature},
		}

		glog.V(4).Infof("save file entry: %v", request)
		_, err := client.CreateEntry(context.Background(), request)
		if err != nil {
			glog.Errorf("UpdateEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err)
			return fuse.EIO
		}

		file.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry))

		file.dirtyMetadata = false

		return nil
	})
}

func (file *File) getEntry() *filer_pb.Entry {
	return file.entry
}

func (file *File) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entry, error) {
	err := file.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.CacheRemoteObjectToLocalClusterRequest{
			Directory: file.dir.FullPath(),
			Name:      entry.Name,
		}

		glog.V(4).Infof("download entry: %v", request)
		resp, err := client.CacheRemoteObjectToLocalCluster(context.Background(), request)
		if err != nil {
			glog.Errorf("CacheRemoteObjectToLocalCluster file %s/%s: %v", file.dir.FullPath(), file.Name, err)
			return fuse.EIO
		}

		entry = resp.Entry

		file.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, resp.Entry))

		file.dirtyMetadata = false

		return nil
	})

	return entry, err
}
<|endoftext|>


<commit_before>package smoke_tests

import (
	"fmt"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/pborman/uuid"
	"github.com/pivotal-cf/cf-rabbitmq-smoke-tests/tests/helper"
)

var _ = Describe("Smoke tests", func() {

	const appName = "rmq-smoke-tests-ruby"
	const appPath = "../assets/rabbit-example-app"

	AfterEach(func() {
		helper.DeleteApp(appName)
	})

	smokeTestForPlan := func(planName string) func() {
		return func() {
			serviceName := fmt.Sprintf("rmq-smoke-test-instance-%s", uuid.New()[:18])
			helper.CreateService(testConfig.ServiceOffering, planName, serviceName)

			if useTLS {
				By("enabling TLS")
				helper.EnableTLSForODB(serviceName)
			}

			By("pushing and binding an app")
			appURL := helper.PushAndBindApp(appName, serviceName, appPath)

			By("RabbitMQ protocol")
			appEnv := helper.GetAppEnv(appName)
			if useTLS {
				Expect(appEnv).To(ContainSubstring("amqps://"), "bind should expose amqps protocol")
				Expect(appEnv).ToNot(ContainSubstring("amqp://"), "bind should not expose amqp protocol")
			} else {
				Expect(appEnv).ToNot(ContainSubstring("amqps://"), "bind should not expose amqps protocol")
				Expect(appEnv).To(ContainSubstring("amqp://"), "bind should expose amqp protocol")
			}

			By("sending and receiving rabbit messages")
			queue := fmt.Sprintf("%s-queue", appName)

			helper.SendMessage(appURL, queue, "foo")
			helper.SendMessage(appURL, queue, "bar")
			Expect(helper.ReceiveMessage(appURL, queue)).To(Equal("foo"))
			Expect(helper.ReceiveMessage(appURL, queue)).To(Equal("bar"))

			helper.UnbindService(appName, serviceName)

			helper.DeleteService(serviceName)
		}
	}

	for _, plan := range testConfig.TestPlans {
		It(fmt.Sprintf("pushes an app, sends, and reads a message from RabbitMQ: plan '%s'", plan.Name),
			smokeTestForPlan(plan.Name), 300.0) // seconds
	}
})
<commit_msg>Enable TLS for ODB by creating service key and updating service<commit_after>package smoke_tests

import (
	"fmt"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/pborman/uuid"
	"github.com/pivotal-cf/cf-rabbitmq-smoke-tests/tests/helper"
)

var _ = Describe("Smoke tests", func() {

	const appName = "rmq-smoke-tests-ruby"
	const appPath = "../assets/rabbit-example-app"

	AfterEach(func() {
		helper.DeleteApp(appName)
	})

	smokeTestForPlan := func(planName string) func() {
		return func() {
			serviceName := fmt.Sprintf("rmq-smoke-test-instance-%s", uuid.New()[:18])
			helper.CreateService(testConfig.ServiceOffering, planName, serviceName)

			if useTLS && testConfig.ServiceOffering == "p.rabbitmq" {
				By("enabling TLS")
				helper.EnableTLSForODB(serviceName)
			}

			By("pushing and binding an app")
			appURL := helper.PushAndBindApp(appName, serviceName, appPath)

			By("RabbitMQ protocol")
			appEnv := helper.GetAppEnv(appName)
			if useTLS {
				Expect(appEnv).To(ContainSubstring("amqps://"), "bind should expose amqps protocol")
				Expect(appEnv).ToNot(ContainSubstring("amqp://"), "bind should not expose amqp protocol")
			} else {
				Expect(appEnv).ToNot(ContainSubstring("amqps://"), "bind should not expose amqps protocol")
				Expect(appEnv).To(ContainSubstring("amqp://"), "bind should expose amqp protocol")
			}

			By("sending and receiving rabbit messages")
			queue := fmt.Sprintf("%s-queue", appName)

			helper.SendMessage(appURL, queue, "foo")
			helper.SendMessage(appURL, queue, "bar")
			Expect(helper.ReceiveMessage(appURL, queue)).To(Equal("foo"))
			Expect(helper.ReceiveMessage(appURL, queue)).To(Equal("bar"))

			helper.UnbindService(appName, serviceName)

			helper.DeleteService(serviceName)
		}
	}

	for _, plan := range testConfig.TestPlans {
		It(fmt.Sprintf("pushes an app, sends, and reads a message from RabbitMQ: plan '%s'", plan.Name),
			smokeTestForPlan(plan.Name), 300.0) // seconds
	}
})
<|endoftext|>


<commit_before>package wikidump

import (
	"fmt"
	"github.com/cheggaaa/pb"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
)

func nullLogger(string, ...interface{}) {
}

// Writer with progressbar.
type pbWriter struct {
	w   io.WriteCloser
	bar *pb.ProgressBar
}

func newPbWriter(w io.WriteCloser, total int64) *pbWriter {
	pbw := &pbWriter{w, pb.New64(total)}
	pbw.bar.Start()
	return pbw
}

func (w *pbWriter) Close() error {
	w.bar.Finish()
	return w.w.Close()
}

func (w *pbWriter) Write(p []byte) (n int, err error) {
	n, err = w.w.Write(p)
	w.bar.Add(n)
	return
}

// Download database dump for wikiname (e.g., "en", "sco", "nds_nl") from
// WikiMedia.
//
// Returns the local file path of the dump, derived from the URL.
//
// Logs its progress on the standard log if logProgress is true.
func Download(wikiname string, logProgress bool) (filepath string, err error) {
	return download(wikiname, ".", logProgress, http.DefaultClient)
}

func download(wikiname, directory string, logProgress bool,
	client *http.Client) (filepath string, err error) {

	logprint := nullLogger
	if logProgress {
		logprint = log.Printf
	}

	urlstr := fmt.Sprintf(
		"https://dumps.wikimedia.org/%s/latest/%s-latest-pages-articles.xml.bz2",
		wikiname, wikiname)
	resp, err := client.Get(urlstr)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("HTTP error %d for %s", resp.StatusCode, urlstr)
	}

	u, err := url.Parse(urlstr)
	if err != nil {
		return
	}
	filepath = path.Base(u.Path)

	var out io.WriteCloser
	out, err = os.OpenFile(filepath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
	if err != nil {
		return
	}
	defer out.Close()

	logprint("downloading from %s to %s", urlstr, filepath)
	if logProgress && resp.ContentLength >= 0 {
		out = newPbWriter(out, resp.ContentLength)
	}
	_, err = io.Copy(out, resp.Body)
	logprint("download of %s done", urlstr)
	return
}
<commit_msg>fix downloader: actually report HTTP errors<commit_after>package wikidump

import (
	"fmt"
	"github.com/cheggaaa/pb"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
)

func nullLogger(string, ...interface{}) {
}

// Writer with progressbar.
type pbWriter struct {
	w   io.WriteCloser
	bar *pb.ProgressBar
}

func newPbWriter(w io.WriteCloser, total int64) *pbWriter {
	pbw := &pbWriter{w, pb.New64(total)}
	pbw.bar.Start()
	return pbw
}

func (w *pbWriter) Close() error {
	w.bar.Finish()
	return w.w.Close()
}

func (w *pbWriter) Write(p []byte) (n int, err error) {
	n, err = w.w.Write(p)
	w.bar.Add(n)
	return
}

// Download database dump for wikiname (e.g., "en", "sco", "nds_nl") from
// WikiMedia.
//
// Returns the local file path of the dump, derived from the URL.
//
// Logs its progress on the standard log if logProgress is true.
func Download(wikiname string, logProgress bool) (filepath string, err error) {
	return download(wikiname, ".", logProgress, http.DefaultClient)
}

func download(wikiname, directory string, logProgress bool,
	client *http.Client) (filepath string, err error) {

	logprint := nullLogger
	if logProgress {
		logprint = log.Printf
	}

	urlstr := fmt.Sprintf(
		"https://dumps.wikimedia.org/%s/latest/%s-latest-pages-articles.xml.bz2",
		wikiname, wikiname)
	resp, err := client.Get(urlstr)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("HTTP error %d for %s", resp.StatusCode, urlstr)
		return
	}

	u, err := url.Parse(urlstr)
	if err != nil {
		return
	}
	filepath = path.Base(u.Path)

	var out io.WriteCloser
	out, err = os.OpenFile(filepath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
	if err != nil {
		return
	}
	defer out.Close()

	logprint("downloading from %s to %s", urlstr, filepath)
	if logProgress && resp.ContentLength >= 0 {
		out = newPbWriter(out, resp.ContentLength)
	}
	_, err = io.Copy(out, resp.Body)
	logprint("download of %s done", urlstr)
	return
}
<|endoftext|>


<commit_before>package registry

import (
	"fmt"
	"log"

	"github.com/golang/protobuf/protoc-gen-go/descriptor"
)

type Enum struct {
	Type *descriptor.EnumDescriptorProto

	File     *File
	Registry *Registry

	Filename string
	Package  string
	Comment  string
	Index    int
	Name     string

	Values []*EnumValue
}

func (e *Enum) String() string {
	return fmt.Sprintf(".%s.%s", e.Package, e.Name)
}

func NewEnum(d *descriptor.EnumDescriptorProto, f *File, index int) *Enum {
	e := &Enum{
		Type:     d,
		File:     f,
		Registry: f.Registry,
		Package:  f.Package,
		Index:    index,
		Name:     d.GetName(),
	}

	valuesMap := map[int]*descriptor.EnumValueDescriptorProto{}

	for _, v := range d.Value {
		valuesMap[int(*v.Number)] = v
	}

	for i := 0; i < len(valuesMap); i++ {
		if v, ok := valuesMap[i]; ok {
			e.Values = append(e.Values, NewEnumValue(v, e, f.Registry))
		} else {
			log.Fatalf("error on enum %s: Values from 0..%d should be present. Value %d not found ", e.Name, len(valuesMap), i)
		}
	}

	return e
}

type EnumValue struct {
	Type *descriptor.EnumValueDescriptorProto
	Enum *Enum

	Name   string
	Number int32

	Registry *Registry
}

type Enums []*Enum

func (es *Enums) Add(ne *Enum) {
	for _, e := range *es {
		if e.String() == ne.String() {
			return
		}
	}
	*es = append(*es, ne)
}

func (es *Enums) Get(name string) (*Enum, bool) {
	for _, e := range *es {
		if e.Name == name {
			return e, true
		}
	}

	return nil, false
}

func NewEnumValue(d *descriptor.EnumValueDescriptorProto, e *Enum, r *Registry) *EnumValue {
	return &EnumValue{
		Enum:     e,
		Registry: r,
		Type:     d,
		Name:     *d.Name,
		Number:   *d.Number,
	}
}
<commit_msg>Match enums from external packages<commit_after>package registry

import (
	"fmt"
	"log"

	"github.com/golang/protobuf/protoc-gen-go/descriptor"
)

type Enum struct {
	Type *descriptor.EnumDescriptorProto

	File     *File
	Registry *Registry

	Filename string
	Package  string
	Comment  string
	Index    int
	Name     string

	Values []*EnumValue
}

func (e *Enum) String() string {
	return fmt.Sprintf(".%s.%s", e.Package, e.Name)
}

func NewEnum(d *descriptor.EnumDescriptorProto, f *File, index int) *Enum {
	e := &Enum{
		Type:     d,
		File:     f,
		Registry: f.Registry,
		Package:  f.Package,
		Index:    index,
		Name:     d.GetName(),
	}

	valuesMap := map[int]*descriptor.EnumValueDescriptorProto{}

	for _, v := range d.Value {
		valuesMap[int(*v.Number)] = v
	}

	for i := 0; i < len(valuesMap); i++ {
		if v, ok := valuesMap[i]; ok {
			e.Values = append(e.Values, NewEnumValue(v, e, f.Registry))
		} else {
			log.Fatalf("error on enum %s: Values from 0..%d should be present. Value %d not found ", e.Name, len(valuesMap), i)
		}
	}

	return e
}

type EnumValue struct {
	Type *descriptor.EnumValueDescriptorProto
	Enum *Enum

	Name   string
	Number int32

	Registry *Registry
}

type Enums []*Enum

func (es *Enums) Add(ne *Enum) {
	for _, e := range *es {
		if e.String() == ne.String() {
			return
		}
	}
	*es = append(*es, ne)
}

func (es *Enums) Get(name string) (*Enum, bool) {
	for _, e := range *es {
		if e.Name == name || e.String() == name {
			return e, true
		}
	}

	return nil, false
}

func NewEnumValue(d *descriptor.EnumValueDescriptorProto, e *Enum, r *Registry) *EnumValue {
	return &EnumValue{
		Enum:     e,
		Registry: r,
		Type:     d,
		Name:     *d.Name,
		Number:   *d.Number,
	}
}
<|endoftext|>


<commit_before>package registry

import (

	"github.com/coreos/fleet/third_party/github.com/coreos/go-semver/semver"

	"github.com/coreos/fleet/job"
	"github.com/coreos/fleet/machine"
	"github.com/coreos/fleet/unit"
)

func NewFakeRegistry() *FakeRegistry {
	return &FakeRegistry{
		machines:  []machine.MachineState{},
		jobStates: map[string]*unit.UnitState{},
		jobs:      map[string]job.Job{},
		units:     []unit.Unit{},
		version:   nil,
	}
}

type FakeRegistry struct {
	// Not all methods of required by the Registry interface are implemented
	// by the TestRegistry. Any calls to these unimplemented methods will
	// result in a panic.
	Registry

	machines  []machine.MachineState
	jobStates map[string]*unit.UnitState
	jobs      map[string]job.Job
	units     []unit.Unit
	version   *semver.Version
}

func (f *FakeRegistry) SetMachines(machines []machine.MachineState) {
	f.machines = machines
}

func (f *FakeRegistry) SetJobs(jobs []job.Job) {
	f.jobs = make(map[string]job.Job, len(jobs))
	for _, j := range jobs {
		f.jobs[j.Name] = j
	}
}

func (f *FakeRegistry) SetUnitStates(jobStates map[string]*unit.UnitState) {
	f.jobStates = jobStates
}

func (f *FakeRegistry) SetUnits(units []unit.Unit) {
	f.units = units
}

func (f *FakeRegistry) SetLatestVersion(v semver.Version) {
	f.version = &v
}

func (f *FakeRegistry) GetActiveMachines() ([]machine.MachineState, error) {
	return f.machines, nil
}

func (f *FakeRegistry) GetAllJobs() ([]job.Job, error) {
	jobs := make([]job.Job, 0, len(f.jobs))
	for _, j := range f.jobs {
		jobs = append(jobs, j)
	}
	return jobs, nil
}

func (f *FakeRegistry) GetJob(name string) (*job.Job, error) {
	j, ok := f.jobs[name]
	if !ok {
		return nil, nil
	}

	j.UnitState = f.jobStates[name]
	return &j, nil
}

func (f *FakeRegistry) GetJobTarget(name string) (string, error) {
	js := f.jobStates[name]
	if js != nil {
		return js.MachineState.ID, nil
	}
	return "", nil
}

func (f *FakeRegistry) GetMachineState(machID string) (*machine.MachineState, error) {
	for _, ms := range f.machines {
		if ms.ID == machID {
			return &ms, nil
		}
	}
	return nil, nil
}

func (f *FakeRegistry) GetLatestVersion() (*semver.Version, error) {
	return f.version, nil
}
<commit_msg>fix(FakeRegistry): Protect access with mutex<commit_after>package registry

import (
	"sync"

	"github.com/coreos/fleet/third_party/github.com/coreos/go-semver/semver"

	"github.com/coreos/fleet/job"
	"github.com/coreos/fleet/machine"
	"github.com/coreos/fleet/unit"
)

func NewFakeRegistry() *FakeRegistry {
	return &FakeRegistry{
		machines:  []machine.MachineState{},
		jobStates: map[string]*unit.UnitState{},
		jobs:      map[string]job.Job{},
		units:     []unit.Unit{},
		version:   nil,
	}
}

type FakeRegistry struct {
	// Not all methods of required by the Registry interface are implemented
	// by the TestRegistry. Any calls to these unimplemented methods will
	// result in a panic.
	Registry
	sync.RWMutex

	machines  []machine.MachineState
	jobStates map[string]*unit.UnitState
	jobs      map[string]job.Job
	units     []unit.Unit
	version   *semver.Version
}

func (f *FakeRegistry) SetMachines(machines []machine.MachineState) {
	f.Lock()
	defer f.Unlock()

	f.machines = machines
}

func (f *FakeRegistry) SetJobs(jobs []job.Job) {
	f.Lock()
	defer f.Unlock()

	f.jobs = make(map[string]job.Job, len(jobs))
	for _, j := range jobs {
		f.jobs[j.Name] = j
	}
}

func (f *FakeRegistry) SetUnitStates(jobStates map[string]*unit.UnitState) {
	f.Lock()
	defer f.Unlock()

	f.jobStates = jobStates
}

func (f *FakeRegistry) SetUnits(units []unit.Unit) {
	f.Lock()
	defer f.Unlock()

	f.units = units
}

func (f *FakeRegistry) SetLatestVersion(v semver.Version) {
	f.Lock()
	defer f.Unlock()

	f.version = &v
}

func (f *FakeRegistry) GetActiveMachines() ([]machine.MachineState, error) {
	f.RLock()
	defer f.RUnlock()

	return f.machines, nil
}

func (f *FakeRegistry) GetAllJobs() ([]job.Job, error) {
	f.RLock()
	defer f.RUnlock()

	jobs := make([]job.Job, 0, len(f.jobs))
	for _, j := range f.jobs {
		jobs = append(jobs, j)
	}
	return jobs, nil
}

func (f *FakeRegistry) GetJob(name string) (*job.Job, error) {
	f.RLock()
	defer f.RUnlock()

	j, ok := f.jobs[name]
	if !ok {
		return nil, nil
	}

	j.UnitState = f.jobStates[name]
	return &j, nil
}

func (f *FakeRegistry) GetJobTarget(name string) (string, error) {
	f.RLock()
	defer f.RUnlock()

	js := f.jobStates[name]
	if js != nil {
		return js.MachineState.ID, nil
	}
	return "", nil
}

func (f *FakeRegistry) GetMachineState(machID string) (*machine.MachineState, error) {
	f.RLock()
	defer f.RUnlock()

	for _, ms := range f.machines {
		if ms.ID == machID {
			return &ms, nil
		}
	}
	return nil, nil
}

func (f *FakeRegistry) GetLatestVersion() (*semver.Version, error) {
	f.RLock()
	defer f.RUnlock()

	return f.version, nil
}
<|endoftext|>


<commit_before>/*
Command db-massdelete handles mass deletion of derpibooru images.

Please use this sparingly.

Usage:

	Usage of ./db-massdelete:
	  -keyFile="/home/xena/.local/share/within/db.cadance.key": file with the derpibooru key to use
	  -reason="": reason to use when deleting images

Then give it the image ID's you want to delete.

	./db-massdelete -reason "OP is a duck" 123 325 1561 136324
*/
package main

import (
	"flag"
	"io/ioutil"
	"log"

	"github.com/Xe/derpigo"
)

var (
	keyFile = flag.String("keyFile", "/home/xena/.local/share/within/db.cadance.key", "file with the derpibooru key to use")
	reason  = flag.String("reason", "", "reason to use when deleting images")
)

func main() {
	flag.Parse()

	if *reason == "" {
		log.Fatal("Need a reason")
	}

	key, err := ioutil.ReadFile(*keyFile)
	if err != nil {
		log.Fatal(err)
	}

	c := derpigo.New(string(key))

	for _, i := range flag.Args() {
		err := c.DeleteImage(i, *reason)
		if err != nil {
			panic(err)
		}

		log.Printf("Deleted %s because %s", i, *reason)
	}
}
<commit_msg>db-massdelete: also back up the file being deleted<commit_after>/*
Command db-massdelete handles mass deletion of derpibooru images.

Please use this sparingly.

Usage:

	Usage of ./db-massdelete:
	  -keyFile="/home/xena/.local/share/within/db.cadance.key": file with the derpibooru key to use
	  -reason="": reason to use when deleting images

Then give it the image ID's you want to delete.

	./db-massdelete -reason "OP is a duck" 123 325 1561 136324
*/
package main

import (
	"flag"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/Xe/derpigo"
)

var (
	keyFile = flag.String("keyFile", "/home/xena/.local/share/within/db.cadance.key", "file with the derpibooru key to use")
	reason  = flag.String("reason", "", "reason to use when deleting images")
	needTag = flag.String("needtag", "", "optional tag an image must have to be deleted")
)

func main() {
	flag.Parse()

	if *reason == "" {
		log.Fatal("Need a reason")
	}

	key, err := ioutil.ReadFile(*keyFile)
	if err != nil {
		log.Fatal(err)
	}

	c := derpigo.New(string(key))

	for _, i := range flag.Args() {
		id, err := strconv.Atoi(i)
		if err != nil {
			log.Printf("Bad number %s", i)
			continue
		}

		img, err := c.GetImage(id)
		if err != nil {
			log.Printf("couldn't fetch info on image %d: %v", id, err)
		}

		if *needTag != "" {
			ok := false

			for _, tag := range strings.Split(img.Tags, ", ") {
				if tag == *needTag {
					ok = true
				}
			}

			if !ok && *needTag != "" {
				log.Printf("Can't delete %d, doesn't have the tag %s", id, *needTag)
			}
		}

		fout, err := os.Create("/home/xena/pictures/derpi/" + img.FileName)
		if err != nil {
			panic(err)
		}
		defer fout.Close()

		resp, err := http.Get("https:" + img.Image)
		if err != nil {
			log.Printf("could not download image: %v", err)
			continue
		}
		defer resp.Body.Close()

		io.Copy(fout, resp.Body)

		err = c.DeleteImage(i, *reason)
		if err != nil {
			panic(err)
		}

		log.Printf("Deleted %s because %s", i, *reason)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"regexp"
	"strings"

	"helm.sh/helm/v3/pkg/strvals"
)

//
// cluster information
//

const (
	clusterUnknown = iota
	clusterDockerDesktop
	clusterMinikube
	clusterKIND
	clusterK3D
	clusterGKE
	clusterAKS
	clusterEKS
	clusterEC2
)

// clusterInfoMessages represent custom messages for some environments
type clusterInfoMessages struct {
	// getServiceIP is the message printed for how to get the service IP
	getServiceIP string
}

// clusterInfo describes some properties about the cluster where the installation is performed
type clusterInfo struct {
	// a name for this kind of cluster (ie, gke)
	name string

	// True if this is a local environment (ie, minikube)
	isLocal bool

	// extra Chart values to set in this environment, as a list of assignments
	chartValues []string

	// customMessages are some custom messages for this environment
	customMessages clusterInfoMessages
}

func (c clusterInfo) CopyChartValuesTo(chartValues map[string]interface{}) {
	for _, assignment := range c.chartValues {
		err := strvals.ParseInto(assignment, chartValues)
		if err != nil {
			panic(err) // this should never happen: only if we have created a wrong `chartValues`
		}
	}
}

// clusterInfoDatabase is the database of information about
var clusterInfoDatabase = map[int]clusterInfo{
	clusterUnknown: {
		name: "unknown",
	},
	clusterDockerDesktop: {
		name:    "docker-desktop",
		isLocal: true,
	},
	clusterMinikube: {
		name:    "minikube",
		isLocal: true,
		customMessages: clusterInfoMessages{
			getServiceIP: "minikube service -n ambassador ambassador",
		},
	},
	clusterKIND: {
		name:    "KIND",
		isLocal: true,
		chartValues: []string{
			"service.type=NodePort",
		},
	},
	clusterK3D: {
		name:    "K3D",
		isLocal: true,
		chartValues: []string{
			"service.type=NodePort",
		},
	},
	clusterGKE: {
		name: "GKE",
	},
	clusterAKS: {
		name: "AKS",
	},
	clusterEKS: {
		name: "EKS",
	},
	clusterEC2: {
		name: "AEC2",
	},
}

func newClusterInfoFromNodeLabels(clusterNodeLabels string) clusterInfo {
	if strings.Contains(clusterNodeLabels, "docker-desktop") {
		return clusterInfoDatabase[clusterDockerDesktop]
	} else if strings.Contains(clusterNodeLabels, "minikube") {
		return clusterInfoDatabase[clusterMinikube]
	} else if strings.Contains(clusterNodeLabels, "kind") {
		return clusterInfoDatabase[clusterKIND]
	} else if strings.Contains(clusterNodeLabels, "k3d") {
		return clusterInfoDatabase[clusterK3D]
	} else if strings.Contains(clusterNodeLabels, "gke") {
		return clusterInfoDatabase[clusterGKE]
	} else if strings.Contains(clusterNodeLabels, "aks") {
		return clusterInfoDatabase[clusterAKS]
	} else if strings.Contains(clusterNodeLabels, "compute") {
		return clusterInfoDatabase[clusterEKS]
	} else if strings.Contains(clusterNodeLabels, "ec2") {
		return clusterInfoDatabase[clusterEC2]
	}
	return clusterInfoDatabase[clusterUnknown]
}

//
// installation methods
//

const (
	instNone = iota
	instOSS
	instAES
	instEdgectl
	instOperator
	instHelm
)

type installationMethodInfo struct {
	Method   int
	Label    string
	Name     string
	LongName string
	Image    *regexp.Regexp
}

// defInstallationMethodsInfo contains information
// about different installation methods. It can be used for detecting previous
// installation methods.
// NOTE: this is an ordered-list: higher-precision labels are first
var defInstallationMethodsInfo = []installationMethodInfo{
	{
		Method:   instEdgectl,
		Label:    "app.kubernetes.io/managed-by=edgectl",
		Name:     "edgectl",
		LongName: "edgectl",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instOperator,
		Label:    "app.kubernetes.io/managed-by=amb-oper",
		Name:     "operator",
		LongName: "the Ambassador Operator",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instHelm,
		Label:    "app.kubernetes.io/name=ambassador",
		Name:     "helm",
		LongName: "Helm",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instAES,
		Label:    "product=aes",
		Name:     "aes",
		LongName: "AES manifests",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instOSS,
		Label:    "service=ambassador",
		Name:     "oss",
		LongName: "OSS manifests",
		Image:    regexp.MustCompile("quay[.]io/datawire/ambassador:([[:^space:]]+)"),
	},
}

type deployGetter func(string) (string, error)

// getExistingInstallation tries to find an existing deployment by looking at a list of predefined labels,
// If such a deployment is found, it returns the image and the installation "family" (aes, oss, helm, etc).
// It returns an empty string if no installation could be found.
//
// TODO: Try to search all namespaces (which may fail due to RBAC) and capture a
//       correct namespace for an Ambassador installation (what if there is more than
//       one?), then proceed operating on that Ambassador in that namespace. Right now
//       we hard-code the "ambassador" namespace in a number of spots.
//
func getExistingInstallation(deploys deployGetter) (string, installationMethodInfo, error) {
	findFor := func(label string, imageRe *regexp.Regexp) (string, error) {
		deploys, err := deploys(label)
		if err != nil {
			return "", err
		}
		scanner := bufio.NewScanner(strings.NewReader(deploys))
		for scanner.Scan() {
			image := strings.TrimSpace(scanner.Text())
			if matches := imageRe.FindStringSubmatch(image); len(matches) == 2 {
				return matches[1], nil
			}
		}
		return "", scanner.Err()
	}

	for _, info := range defInstallationMethodsInfo {
		if info.Label == "" {
			continue
		}
		version, err := findFor(info.Label, info.Image)
		if err != nil {
			continue // ignore errors
		}
		if version != "" {
			return version, info, nil
		}
	}
	return "", installationMethodInfo{Method: instNone}, nil
}
<commit_msg>Recognize other Operator installations<commit_after>package main

import (
	"bufio"
	"regexp"
	"strings"

	"helm.sh/helm/v3/pkg/strvals"
)

//
// cluster information
//

const (
	clusterUnknown = iota
	clusterDockerDesktop
	clusterMinikube
	clusterKIND
	clusterK3D
	clusterGKE
	clusterAKS
	clusterEKS
	clusterEC2
)

// clusterInfoMessages represent custom messages for some environments
type clusterInfoMessages struct {
	// getServiceIP is the message printed for how to get the service IP
	getServiceIP string
}

// clusterInfo describes some properties about the cluster where the installation is performed
type clusterInfo struct {
	// a name for this kind of cluster (ie, gke)
	name string

	// True if this is a local environment (ie, minikube)
	isLocal bool

	// extra Chart values to set in this environment, as a list of assignments
	chartValues []string

	// customMessages are some custom messages for this environment
	customMessages clusterInfoMessages
}

func (c clusterInfo) CopyChartValuesTo(chartValues map[string]interface{}) {
	for _, assignment := range c.chartValues {
		err := strvals.ParseInto(assignment, chartValues)
		if err != nil {
			panic(err) // this should never happen: only if we have created a wrong `chartValues`
		}
	}
}

// clusterInfoDatabase is the database of information about
var clusterInfoDatabase = map[int]clusterInfo{
	clusterUnknown: {
		name: "unknown",
	},
	clusterDockerDesktop: {
		name:    "docker-desktop",
		isLocal: true,
	},
	clusterMinikube: {
		name:    "minikube",
		isLocal: true,
		customMessages: clusterInfoMessages{
			getServiceIP: "minikube service -n ambassador ambassador",
		},
	},
	clusterKIND: {
		name:    "KIND",
		isLocal: true,
		chartValues: []string{
			"service.type=NodePort",
		},
	},
	clusterK3D: {
		name:    "K3D",
		isLocal: true,
		chartValues: []string{
			"service.type=NodePort",
		},
	},
	clusterGKE: {
		name: "GKE",
	},
	clusterAKS: {
		name: "AKS",
	},
	clusterEKS: {
		name: "EKS",
	},
	clusterEC2: {
		name: "AEC2",
	},
}

func newClusterInfoFromNodeLabels(clusterNodeLabels string) clusterInfo {
	if strings.Contains(clusterNodeLabels, "docker-desktop") {
		return clusterInfoDatabase[clusterDockerDesktop]
	} else if strings.Contains(clusterNodeLabels, "minikube") {
		return clusterInfoDatabase[clusterMinikube]
	} else if strings.Contains(clusterNodeLabels, "kind") {
		return clusterInfoDatabase[clusterKIND]
	} else if strings.Contains(clusterNodeLabels, "k3d") {
		return clusterInfoDatabase[clusterK3D]
	} else if strings.Contains(clusterNodeLabels, "gke") {
		return clusterInfoDatabase[clusterGKE]
	} else if strings.Contains(clusterNodeLabels, "aks") {
		return clusterInfoDatabase[clusterAKS]
	} else if strings.Contains(clusterNodeLabels, "compute") {
		return clusterInfoDatabase[clusterEKS]
	} else if strings.Contains(clusterNodeLabels, "ec2") {
		return clusterInfoDatabase[clusterEC2]
	}
	return clusterInfoDatabase[clusterUnknown]
}

//
// installation methods
//

const (
	instNone = iota
	instOSS
	instAES
	instEdgectl
	instOperator
	instHelm
)

type installationMethodInfo struct {
	Method   int
	Label    string
	Name     string
	LongName string
	Image    *regexp.Regexp
}

// defInstallationMethodsInfo contains information
// about different installation methods. It can be used for detecting previous
// installation methods.
// NOTE: this is an ordered-list: higher-precision labels are first
var defInstallationMethodsInfo = []installationMethodInfo{
	{
		Method:   instEdgectl,
		Label:    "app.kubernetes.io/managed-by=edgectl",
		Name:     "edgectl",
		LongName: "edgectl",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instOperator,
		Label:    "getambassador.io/installer=operator",
		Name:     "operator",
		LongName: "the Ambassador Operator",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instHelm,
		Label:    "app.kubernetes.io/name=ambassador",
		Name:     "helm",
		LongName: "Helm",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instAES,
		Label:    "product=aes",
		Name:     "aes",
		LongName: "AES manifests",
		Image:    regexp.MustCompile("quay[.]io/datawire/aes:([[:^space:]]+)"),
	},
	{
		Method:   instOSS,
		Label:    "service=ambassador",
		Name:     "oss",
		LongName: "OSS manifests",
		Image:    regexp.MustCompile("quay[.]io/datawire/ambassador:([[:^space:]]+)"),
	},
}

type deployGetter func(string) (string, error)

// getExistingInstallation tries to find an existing deployment by looking at a list of predefined labels,
// If such a deployment is found, it returns the image and the installation "family" (aes, oss, helm, etc).
// It returns an empty string if no installation could be found.
//
// TODO: Try to search all namespaces (which may fail due to RBAC) and capture a
//       correct namespace for an Ambassador installation (what if there is more than
//       one?), then proceed operating on that Ambassador in that namespace. Right now
//       we hard-code the "ambassador" namespace in a number of spots.
//
func getExistingInstallation(deploys deployGetter) (string, installationMethodInfo, error) {
	findFor := func(label string, imageRe *regexp.Regexp) (string, error) {
		deploys, err := deploys(label)
		if err != nil {
			return "", err
		}
		scanner := bufio.NewScanner(strings.NewReader(deploys))
		for scanner.Scan() {
			image := strings.TrimSpace(scanner.Text())
			if matches := imageRe.FindStringSubmatch(image); len(matches) == 2 {
				return matches[1], nil
			}
		}
		return "", scanner.Err()
	}

	for _, info := range defInstallationMethodsInfo {
		if info.Label == "" {
			continue
		}
		version, err := findFor(info.Label, info.Image)
		if err != nil {
			continue // ignore errors
		}
		if version != "" {
			return version, info, nil
		}
	}
	return "", installationMethodInfo{Method: instNone}, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Intel Corporation. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package agent

import (
	"encoding/json"
	"fmt"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"

	"github.com/intel-hpdd/lemur/cmd/lhsmd/agent/fileid"
	pb "github.com/intel-hpdd/lemur/pdm"
	"github.com/intel-hpdd/logging/alert"
	"github.com/intel-hpdd/logging/audit"
	"github.com/intel-hpdd/logging/debug"

	"github.com/intel-hpdd/go-lustre/fs"
	"github.com/intel-hpdd/go-lustre/hsm"
	"github.com/intel-hpdd/go-lustre/llapi"
)

type (
	// ActionID is a unique (per agent instance) ID for HSM actions
	ActionID uint64

	// Action represents an HSM action
	Action struct {
		id    ActionID
		aih   hsm.ActionHandle
		agent *HsmAgent
		start time.Time
		UUID  string
		Hash  []byte
		URL   string
		Data  []byte
	}

	// ActionData is extra data passed to the Agent by policy engine
	ActionData struct {
		FileID    []byte `json:"file_id"`
		MoverData []byte `json:"mover_data"`
	}
)

var actionIDCounter ActionID

// NextActionID returns monotonically-increasing ActionIDs
func NextActionID() ActionID {
	return ActionID(atomic.AddUint64((*uint64)(&actionIDCounter), 1))
}

func (action *Action) String() string {
	return fmt.Sprintf("id:%d %s %v ", action.id, action.aih.Action(), action.aih.Fid())
}

func hsm2Command(a llapi.HsmAction) (c pb.Command) {
	switch a {
	case llapi.HsmActionArchive:
		c = pb.Command_ARCHIVE
	case llapi.HsmActionRestore:
		c = pb.Command_RESTORE
	case llapi.HsmActionRemove:
		c = pb.Command_REMOVE
	case llapi.HsmActionCancel:
		c = pb.Command_CANCEL
	default:
		alert.Abort(errors.Errorf("unknown command: %v", a))
	}

	return
}

// Handle returns the raw hsm.ActionHandle (temporary function until queue
// transport is updated)
func (action *Action) Handle() hsm.ActionHandle {
	return action.aih
}

// ID Returns the action id.
func (action *Action) ID() ActionID {
	return action.id
}

// MarshalActionData returns an initallized and marshalled ActionData struct. The moverData
// value is also marshalled before adding it to the ActionData.
func MarshalActionData(fileID []byte, moverData interface{}) ([]byte, error) {
	mdata, err := json.Marshal(moverData)
	if err != nil {
		return nil, err
	}
	return json.Marshal(
		&ActionData{
			FileID:    fileID,
			MoverData: mdata,
		})
}

// Prepare ensure action is ready to be sent.
// Complete any actions that may require accessing the filesystem.
func (action *Action) Prepare() error {
	var data ActionData
	if len(action.aih.Data()) > 0 {
		err := json.Unmarshal(action.aih.Data(), &data)
		if err != nil {
			alert.Warnf("unrecognized data passed to agent: %v: %v", action.aih.Data(), err)
			action.Data = action.aih.Data()
		}
	}

	if len(data.MoverData) > 0 {
		action.Data = data.MoverData
	}

	if len(data.FileID) > 0 {
		debug.Printf("found fileID from user: %v %d", data.FileID, len(data.FileID))
		action.UUID = string(data.FileID)
	} else {
		switch action.aih.Action() {
		case llapi.HsmActionRestore, llapi.HsmActionRemove:
			uuid, err := fileid.UUID.Get(action.agent.Root(), action.aih.Fid())
			if err != nil {
				alert.Warnf("Error reading UUID: %v (%v)", err, action)
			} else {
				action.UUID = string(uuid)
			}

			action.Hash, err = fileid.Hash.Get(action.agent.Root(), action.aih.Fid())
			if err != nil {
				alert.Warnf("Error reading Hash: %v (%v)", err, action)
			}

			url, err := fileid.URL.Get(action.agent.Root(), action.aih.Fid())
			if err != nil {
				alert.Warnf("Error reading URL: %v (%v)", err, action)
			} else {
				action.URL = string(url)
			}

		}
	}
	return nil
}

// AsMessage returns the protobuf version of an Action.
func (action *Action) AsMessage() *pb.ActionItem {
	msg := &pb.ActionItem{
		Id:          uint64(action.id),
		Op:          hsm2Command(action.aih.Action()),
		PrimaryPath: fs.FidRelativePath(action.aih.Fid()),
		Offset:      action.aih.Offset(),
		Length:      action.aih.Length(),
		Uuid:        action.UUID,
		Hash:        action.Hash,
		Url:         action.URL,
		Data:        action.Data,
	}

	dfid, err := action.aih.DataFid()
	if err == nil {
		msg.WritePath = fs.FidRelativePath(dfid)
	}

	return msg
}

// Update handles the Status messages from the data mover. The Status
// updates the current progress of the Action. if the Completed flag is true,
// then the Action is completed and true is returned so the transport can remove
// any related state. After an action is completed any further status updates
// should be ignored.
//
// If this function returns an error then the transport layer should notify
// the mover that this action has been terminated. In this case the Action will
// be completed immediately and no further updates are required.
//
func (action *Action) Update(status *pb.ActionStatus) (bool, error) {
	debug.Printf("id:%d update offset: %d length: %d complete: %v status: %d", status.Id,
		status.Offset,
		status.Length,
		status.Completed, status.Error)
	if status.Completed {
		duration := time.Since(action.start)
		debug.Printf("id:%d completed status: %v in %v", status.Id, status.Error, duration)

		if status.Uuid != "" {
			fileid.UUID.Update(action.agent.Root(), action.aih.Fid(), []byte(status.Uuid))
		}
		if status.Hash != nil {
			fileid.Hash.Update(action.agent.Root(), action.aih.Fid(), status.Hash)
		}
		if status.Url != "" {
			fileid.URL.Update(action.agent.Root(), action.aih.Fid(), []byte(status.Url))
		}
		action.agent.stats.CompleteAction(action, int(status.Error))
		err := action.aih.End(status.Offset, status.Length, 0, int(status.Error))
		if err != nil {
			audit.Logf("id:%d completion failed: %v", status.Id, err)
			return true, err // Completed, but Failed. Internal HSM state is not updated
		}
		<-action.agent.rpcsInFlight
		if action.aih.Action() == llapi.HsmActionArchive && action.agent.config.Snapshots.Enabled && status.Uuid != "" {
			createSnapshot(action.agent.Root(), action.aih.ArchiveID(), action.aih.Fid(), []byte(status.Uuid))
		}
		return true, nil // Completed
	}
	err := action.aih.Progress(status.Offset, status.Length, action.aih.Length(), 0)
	if err != nil {
		debug.Printf("id:%d progress update failed: %v", status.Id, err)
		action.agent.stats.CompleteAction(action, -1)
		if err2 := action.aih.End(0, 0, 0, -1); err2 != nil {
			<-action.agent.rpcsInFlight
			debug.Printf("id:%d completion after error failed: %v", status.Id, err2)
			return false, fmt.Errorf("err: %s/err2: %s", err, err2)
		}
		<-action.agent.rpcsInFlight
		return false, err // Incomplete Failed Action
	}

	return false, nil
}

// Fail signals that the action has failed
func (action *Action) Fail(rc int) error {
	audit.Logf("id:%d fail %x %v: %v", action.id, action.aih.Cookie(), action.aih.Fid(), rc)
	action.agent.stats.CompleteAction(action, rc)
	err := action.aih.End(0, 0, 0, rc)
	if err != nil {
		audit.Logf("id:%d fail after fail %x: %v", action.id, action.aih.Cookie(), err)
	}
	<-action.agent.rpcsInFlight
	return errors.Wrap(err, "end action failed")

}
<commit_msg>Change secondary EA errors to debug messages for now.<commit_after>// Copyright (c) 2016 Intel Corporation. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package agent

import (
	"encoding/json"
	"fmt"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"

	"github.com/intel-hpdd/lemur/cmd/lhsmd/agent/fileid"
	pb "github.com/intel-hpdd/lemur/pdm"
	"github.com/intel-hpdd/logging/alert"
	"github.com/intel-hpdd/logging/audit"
	"github.com/intel-hpdd/logging/debug"

	"github.com/intel-hpdd/go-lustre/fs"
	"github.com/intel-hpdd/go-lustre/hsm"
	"github.com/intel-hpdd/go-lustre/llapi"
)

type (
	// ActionID is a unique (per agent instance) ID for HSM actions
	ActionID uint64

	// Action represents an HSM action
	Action struct {
		id    ActionID
		aih   hsm.ActionHandle
		agent *HsmAgent
		start time.Time
		UUID  string
		Hash  []byte
		URL   string
		Data  []byte
	}

	// ActionData is extra data passed to the Agent by policy engine
	ActionData struct {
		FileID    []byte `json:"file_id"`
		MoverData []byte `json:"mover_data"`
	}
)

var actionIDCounter ActionID

// NextActionID returns monotonically-increasing ActionIDs
func NextActionID() ActionID {
	return ActionID(atomic.AddUint64((*uint64)(&actionIDCounter), 1))
}

func (action *Action) String() string {
	return fmt.Sprintf("id:%d %s %v ", action.id, action.aih.Action(), action.aih.Fid())
}

func hsm2Command(a llapi.HsmAction) (c pb.Command) {
	switch a {
	case llapi.HsmActionArchive:
		c = pb.Command_ARCHIVE
	case llapi.HsmActionRestore:
		c = pb.Command_RESTORE
	case llapi.HsmActionRemove:
		c = pb.Command_REMOVE
	case llapi.HsmActionCancel:
		c = pb.Command_CANCEL
	default:
		alert.Abort(errors.Errorf("unknown command: %v", a))
	}

	return
}

// Handle returns the raw hsm.ActionHandle (temporary function until queue
// transport is updated)
func (action *Action) Handle() hsm.ActionHandle {
	return action.aih
}

// ID Returns the action id.
func (action *Action) ID() ActionID {
	return action.id
}

// MarshalActionData returns an initallized and marshalled ActionData struct. The moverData
// value is also marshalled before adding it to the ActionData.
func MarshalActionData(fileID []byte, moverData interface{}) ([]byte, error) {
	mdata, err := json.Marshal(moverData)
	if err != nil {
		return nil, err
	}
	return json.Marshal(
		&ActionData{
			FileID:    fileID,
			MoverData: mdata,
		})
}

// Prepare ensure action is ready to be sent.
// Complete any actions that may require accessing the filesystem.
func (action *Action) Prepare() error {
	var data ActionData
	if len(action.aih.Data()) > 0 {
		err := json.Unmarshal(action.aih.Data(), &data)
		if err != nil {
			alert.Warnf("unrecognized data passed to agent: %v: %v", action.aih.Data(), err)
			action.Data = action.aih.Data()
		}
	}

	if len(data.MoverData) > 0 {
		action.Data = data.MoverData
	}

	if len(data.FileID) > 0 {
		debug.Printf("found fileID from user: %v %d", data.FileID, len(data.FileID))
		action.UUID = string(data.FileID)
	} else {
		switch action.aih.Action() {
		case llapi.HsmActionRestore, llapi.HsmActionRemove:
			uuid, err := fileid.UUID.Get(action.agent.Root(), action.aih.Fid())
			if err != nil {
				alert.Warnf("Error reading UUID: %v (%v)", err, action)
			} else {
				action.UUID = string(uuid)
			}

			action.Hash, err = fileid.Hash.Get(action.agent.Root(), action.aih.Fid())
			if err != nil {
				debug.Printf("Error reading Hash: %v (%v)", err, action)
			}

			url, err := fileid.URL.Get(action.agent.Root(), action.aih.Fid())
			if err != nil {
				debug.Printf("Error reading URL: %v (%v)", err, action)
			} else {
				action.URL = string(url)
			}

		}
	}
	return nil
}

// AsMessage returns the protobuf version of an Action.
func (action *Action) AsMessage() *pb.ActionItem {
	msg := &pb.ActionItem{
		Id:          uint64(action.id),
		Op:          hsm2Command(action.aih.Action()),
		PrimaryPath: fs.FidRelativePath(action.aih.Fid()),
		Offset:      action.aih.Offset(),
		Length:      action.aih.Length(),
		Uuid:        action.UUID,
		Hash:        action.Hash,
		Url:         action.URL,
		Data:        action.Data,
	}

	dfid, err := action.aih.DataFid()
	if err == nil {
		msg.WritePath = fs.FidRelativePath(dfid)
	}

	return msg
}

// Update handles the Status messages from the data mover. The Status
// updates the current progress of the Action. if the Completed flag is true,
// then the Action is completed and true is returned so the transport can remove
// any related state. After an action is completed any further status updates
// should be ignored.
//
// If this function returns an error then the transport layer should notify
// the mover that this action has been terminated. In this case the Action will
// be completed immediately and no further updates are required.
//
func (action *Action) Update(status *pb.ActionStatus) (bool, error) {
	debug.Printf("id:%d update offset: %d length: %d complete: %v status: %d", status.Id,
		status.Offset,
		status.Length,
		status.Completed, status.Error)
	if status.Completed {
		duration := time.Since(action.start)
		debug.Printf("id:%d completed status: %v in %v", status.Id, status.Error, duration)

		if status.Uuid != "" {
			fileid.UUID.Update(action.agent.Root(), action.aih.Fid(), []byte(status.Uuid))
		}
		if status.Hash != nil {
			fileid.Hash.Update(action.agent.Root(), action.aih.Fid(), status.Hash)
		}
		if status.Url != "" {
			fileid.URL.Update(action.agent.Root(), action.aih.Fid(), []byte(status.Url))
		}
		action.agent.stats.CompleteAction(action, int(status.Error))
		err := action.aih.End(status.Offset, status.Length, 0, int(status.Error))
		if err != nil {
			audit.Logf("id:%d completion failed: %v", status.Id, err)
			return true, err // Completed, but Failed. Internal HSM state is not updated
		}
		<-action.agent.rpcsInFlight
		if action.aih.Action() == llapi.HsmActionArchive && action.agent.config.Snapshots.Enabled && status.Uuid != "" {
			createSnapshot(action.agent.Root(), action.aih.ArchiveID(), action.aih.Fid(), []byte(status.Uuid))
		}
		return true, nil // Completed
	}
	err := action.aih.Progress(status.Offset, status.Length, action.aih.Length(), 0)
	if err != nil {
		debug.Printf("id:%d progress update failed: %v", status.Id, err)
		action.agent.stats.CompleteAction(action, -1)
		if err2 := action.aih.End(0, 0, 0, -1); err2 != nil {
			<-action.agent.rpcsInFlight
			debug.Printf("id:%d completion after error failed: %v", status.Id, err2)
			return false, fmt.Errorf("err: %s/err2: %s", err, err2)
		}
		<-action.agent.rpcsInFlight
		return false, err // Incomplete Failed Action
	}

	return false, nil
}

// Fail signals that the action has failed
func (action *Action) Fail(rc int) error {
	audit.Logf("id:%d fail %x %v: %v", action.id, action.aih.Cookie(), action.aih.Fid(), rc)
	action.agent.stats.CompleteAction(action, rc)
	err := action.aih.End(0, 0, 0, rc)
	if err != nil {
		audit.Logf("id:%d fail after fail %x: %v", action.id, action.aih.Cookie(), err)
	}
	<-action.agent.rpcsInFlight
	return errors.Wrap(err, "end action failed")

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"
	"unicode"
)

type dockerPullStat struct {
	ExitCode int
	Duration time.Duration
}

type dockerRunStat struct {
	ExitCode int
	Duration time.Duration
}

type estafettePipelineStat struct {
	Pipeline       estafettePipeline
	DockerPullStat dockerPullStat
	DockerRunStat  dockerRunStat
}

func (c *estafettePipelineStat) ExitCode() int {
	if c.DockerPullStat.ExitCode > 0 {
		return c.DockerPullStat.ExitCode
	}
	if c.DockerRunStat.ExitCode > 0 {
		return c.DockerRunStat.ExitCode
	}
	return 0
}

func runDockerPull(p estafettePipeline) (stat dockerPullStat, err error) {

	start := time.Now()

	cmd := "docker"

	// add docker command and options
	argsSlice := make([]string, 0)
	argsSlice = append(argsSlice, "pull")
	argsSlice = append(argsSlice, p.ContainerImage)

	fmt.Printf("[estafette] Running command '%v %v'\n", cmd, strings.Join(argsSlice, " "))
	dockerPullCmd := exec.Command(cmd, argsSlice...)

	// run and wait until completion
	if err := dockerPullCmd.Run(); err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok && status.ExitStatus() > 0 {
				stat.ExitCode = status.ExitStatus()
				return stat, err
			}
		} else {
			return stat, err
		}
	}

	stat.Duration = time.Since(start)

	return
}

func runDockerRun(dir string, envvars map[string]string, p estafettePipeline) (stat dockerRunStat, err error) {

	// run docker with image and commands from yaml
	start := time.Now()

	cmd := "docker"

	// add docker command and options
	argsSlice := make([]string, 0)
	argsSlice = append(argsSlice, "run")
	argsSlice = append(argsSlice, "--privileged")
	argsSlice = append(argsSlice, "--rm")
	argsSlice = append(argsSlice, "--entrypoint=\"\"")
	argsSlice = append(argsSlice, fmt.Sprintf("--volume=\"%v:%v\"", dir, p.WorkingDirectory))
	argsSlice = append(argsSlice, "--volume=\"/var/run/docker.sock:/var/run/docker.sock\"")
	argsSlice = append(argsSlice, fmt.Sprintf("--workdir==\"%v\"", p.WorkingDirectory))
	if envvars != nil && len(envvars) > 0 {
		for k, v := range envvars {
			argsSlice = append(argsSlice, fmt.Sprintf("--env=\"%v=%v\"", k, v))
		}
	}

	// the actual container to run
	argsSlice = append(argsSlice, p.ContainerImage)

	// the commands to execute in the container
	argsSlice = append(argsSlice, p.Shell)
	argsSlice = append(argsSlice, "-c")
	argsSlice = append(argsSlice, strings.Join(p.Commands, ";"))

	fmt.Printf("[estafette] Running command '%v %v'\n", cmd, strings.Join(argsSlice, " "))
	dockerRunCmd := exec.Command(cmd, argsSlice...)

	// pipe logs
	stdout, err := dockerRunCmd.StdoutPipe()
	if err != nil {
		return stat, err
	}
	stderr, err := dockerRunCmd.StderrPipe()
	if err != nil {
		return stat, err
	}

	// start
	if err := dockerRunCmd.Start(); err != nil {
		return stat, err
	}

	// tail logs
	multi := io.MultiReader(stdout, stderr)

	in := bufio.NewScanner(multi)

	for in.Scan() {
		log.Printf("[%v] %s", p.Name, in.Text()) // write each line to your log, or anything you need
	}
	if err := in.Err(); err != nil {
		log.Printf("[%v] Error: %s", p.Name, err)
	}

	// wait for completion
	if err := dockerRunCmd.Wait(); err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok && status.ExitStatus() > 0 {
				stat.ExitCode = status.ExitStatus()
				return stat, err
			}
		} else {
			return stat, err
		}
	}

	stat.Duration = time.Since(start)

	return
}

// https://gist.github.com/elwinar/14e1e897fdbe4d3432e1
func toUpperSnake(in string) string {
	runes := []rune(in)
	length := len(runes)

	var out []rune
	for i := 0; i < length; i++ {
		if i > 0 && unicode.IsUpper(runes[i]) && ((i+1 < length && unicode.IsLower(runes[i+1])) || unicode.IsLower(runes[i-1])) {
			out = append(out, '_')
		}
		out = append(out, unicode.ToUpper(runes[i]))
	}

	return string(out)
}

func collectEstafetteEnvvars(m estafetteManifest) (envvars map[string]string) {

	envvars = map[string]string{}

	for _, e := range os.Environ() {
		kvPair := strings.Split(e, "=")
		if len(kvPair) == 2 {
			envvarName := kvPair[0]
			envvarValue := kvPair[1]

			if strings.HasPrefix(envvarName, "ESTAFETTE_") {
				envvars[envvarName] = envvarValue
			}
		}
	}

	// add the labels as envvars
	if m.Labels != nil && len(m.Labels) > 0 {
		for key, value := range m.Labels {

			envvarName := "ESTAFETTE_LABEL_" + toUpperSnake(key)
			envvars[envvarName] = value
		}
	}

	return
}

func runPipeline(dir string, envvars map[string]string, p estafettePipeline) (stat estafettePipelineStat, err error) {

	stat.Pipeline = p

	fmt.Printf("[estafette] Starting pipeline '%v'\n", p.Name)

	// pull docker image
	stat.DockerPullStat, err = runDockerPull(p)
	if err != nil {
		return
	}

	stat.DockerRunStat, err = runDockerRun(dir, envvars, p)
	if err != nil {
		return
	}

	fmt.Printf("[estafette] Finished pipeline '%v' successfully\n", p.Name)

	return
}
<commit_msg>fix workdir flag<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"
	"unicode"
)

type dockerPullStat struct {
	ExitCode int
	Duration time.Duration
}

type dockerRunStat struct {
	ExitCode int
	Duration time.Duration
}

type estafettePipelineStat struct {
	Pipeline       estafettePipeline
	DockerPullStat dockerPullStat
	DockerRunStat  dockerRunStat
}

func (c *estafettePipelineStat) ExitCode() int {
	if c.DockerPullStat.ExitCode > 0 {
		return c.DockerPullStat.ExitCode
	}
	if c.DockerRunStat.ExitCode > 0 {
		return c.DockerRunStat.ExitCode
	}
	return 0
}

func runDockerPull(p estafettePipeline) (stat dockerPullStat, err error) {

	start := time.Now()

	cmd := "docker"

	// add docker command and options
	argsSlice := make([]string, 0)
	argsSlice = append(argsSlice, "pull")
	argsSlice = append(argsSlice, p.ContainerImage)

	fmt.Printf("[estafette] Running command '%v %v'\n", cmd, strings.Join(argsSlice, " "))
	dockerPullCmd := exec.Command(cmd, argsSlice...)

	// run and wait until completion
	if err := dockerPullCmd.Run(); err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok && status.ExitStatus() > 0 {
				stat.ExitCode = status.ExitStatus()
				return stat, err
			}
		} else {
			return stat, err
		}
	}

	stat.Duration = time.Since(start)

	return
}

func runDockerRun(dir string, envvars map[string]string, p estafettePipeline) (stat dockerRunStat, err error) {

	// run docker with image and commands from yaml
	start := time.Now()

	cmd := "docker"

	// add docker command and options
	argsSlice := make([]string, 0)
	argsSlice = append(argsSlice, "run")
	argsSlice = append(argsSlice, "--privileged")
	argsSlice = append(argsSlice, "--rm")
	argsSlice = append(argsSlice, "--entrypoint=\"\"")
	argsSlice = append(argsSlice, fmt.Sprintf("--volume=\"%v:%v\"", dir, p.WorkingDirectory))
	argsSlice = append(argsSlice, "--volume=\"/var/run/docker.sock:/var/run/docker.sock\"")
	argsSlice = append(argsSlice, fmt.Sprintf("--workdir=\"%v\"", p.WorkingDirectory))
	if envvars != nil && len(envvars) > 0 {
		for k, v := range envvars {
			argsSlice = append(argsSlice, fmt.Sprintf("--env=\"%v=%v\"", k, v))
		}
	}

	// the actual container to run
	argsSlice = append(argsSlice, p.ContainerImage)

	// the commands to execute in the container
	argsSlice = append(argsSlice, p.Shell)
	argsSlice = append(argsSlice, "-c")
	argsSlice = append(argsSlice, strings.Join(p.Commands, ";"))

	fmt.Printf("[estafette] Running command '%v %v'\n", cmd, strings.Join(argsSlice, " "))
	dockerRunCmd := exec.Command(cmd, argsSlice...)

	// pipe logs
	stdout, err := dockerRunCmd.StdoutPipe()
	if err != nil {
		return stat, err
	}
	stderr, err := dockerRunCmd.StderrPipe()
	if err != nil {
		return stat, err
	}

	// start
	if err := dockerRunCmd.Start(); err != nil {
		return stat, err
	}

	// tail logs
	multi := io.MultiReader(stdout, stderr)

	in := bufio.NewScanner(multi)

	for in.Scan() {
		log.Printf("[%v] %s", p.Name, in.Text()) // write each line to your log, or anything you need
	}
	if err := in.Err(); err != nil {
		log.Printf("[%v] Error: %s", p.Name, err)
	}

	// wait for completion
	if err := dockerRunCmd.Wait(); err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok && status.ExitStatus() > 0 {
				stat.ExitCode = status.ExitStatus()
				return stat, err
			}
		} else {
			return stat, err
		}
	}

	stat.Duration = time.Since(start)

	return
}

// https://gist.github.com/elwinar/14e1e897fdbe4d3432e1
func toUpperSnake(in string) string {
	runes := []rune(in)
	length := len(runes)

	var out []rune
	for i := 0; i < length; i++ {
		if i > 0 && unicode.IsUpper(runes[i]) && ((i+1 < length && unicode.IsLower(runes[i+1])) || unicode.IsLower(runes[i-1])) {
			out = append(out, '_')
		}
		out = append(out, unicode.ToUpper(runes[i]))
	}

	return string(out)
}

func collectEstafetteEnvvars(m estafetteManifest) (envvars map[string]string) {

	envvars = map[string]string{}

	for _, e := range os.Environ() {
		kvPair := strings.Split(e, "=")
		if len(kvPair) == 2 {
			envvarName := kvPair[0]
			envvarValue := kvPair[1]

			if strings.HasPrefix(envvarName, "ESTAFETTE_") {
				envvars[envvarName] = envvarValue
			}
		}
	}

	// add the labels as envvars
	if m.Labels != nil && len(m.Labels) > 0 {
		for key, value := range m.Labels {

			envvarName := "ESTAFETTE_LABEL_" + toUpperSnake(key)
			envvars[envvarName] = value
		}
	}

	return
}

func runPipeline(dir string, envvars map[string]string, p estafettePipeline) (stat estafettePipelineStat, err error) {

	stat.Pipeline = p

	fmt.Printf("[estafette] Starting pipeline '%v'\n", p.Name)

	// pull docker image
	stat.DockerPullStat, err = runDockerPull(p)
	if err != nil {
		return
	}

	stat.DockerRunStat, err = runDockerRun(dir, envvars, p)
	if err != nil {
		return
	}

	fmt.Printf("[estafette] Finished pipeline '%v' successfully\n", p.Name)

	return
}
<|endoftext|>


<commit_before>/*

api is a package that can create and cleanup api server binaries.

Its cousin is the build/static package, which contains considerably more
logic.

The point of this package is primarily to create an `api/main.go` output that
statically links the games and storage type configured via config.json, and
then build that binary using `go build`.

There's nothing magic about this package; it's legal to create your own server
binary by hand. This package just automates that for you so when you add a
game to your server you only have to worry about adding it in your config.json
and everything else happens automatically.

Typically it is not used directly, but via the `boardgame-util build api`,
`boardgame-util cleanup api`, and `boardgame-util serve` commands.

*/
package api

import (
	"path/filepath"
	"strings"
	"text/template"
)

type StorageType int

const (
	StorageInvalid StorageType = iota
	StorageDefault
	StorageMemory
	StorageBolt
	StorageMysql
	StorageFilesystem
)

var apiTemplate *template.Template

func init() {
	apiTemplate = template.Must(template.New("api").Parse(apiTemplateText))
}

//ValidStorageTypeStrings returns an array of strings that are the normal
//(i.e. not invalid) strings that would return useful values if passed to
//StorageTypeFromString.
func ValidStorageTypeStrings() []string {
	return []string{
		StorageDefault.String(),
		StorageMemory.String(),
		StorageBolt.String(),
		StorageMysql.String(),
		StorageFilesystem.String(),
	}
}

func StorageTypeFromString(in string) StorageType {
	in = strings.ToLower(in)
	in = strings.TrimSpace(in)

	switch in {
	case "default":
		return StorageDefault
	case "":
		return StorageDefault
	case "memory":
		return StorageMemory
	case "bolt":
		return StorageBolt
	case "mysql":
		return StorageMysql
	case "filesystem":
		return StorageFilesystem
	}

	return StorageInvalid
}

func (s StorageType) String() string {
	switch s {
	case StorageDefault:
		return "default"
	case StorageMemory:
		return "memory"
	case StorageBolt:
		return "bolt"
	case StorageMysql:
		return "mysql"
	case StorageFilesystem:
		return "filesystem"
	}
	return "invalid"
}

//Import is the string denting the import path for this storage type.
func (s StorageType) Import() string {

	if s == StorageDefault {
		//api package already imported
		return ""
	}

	base := "github.com/jkomoros/boardgame/storage"
	return filepath.Join(base, s.String())
}

//Constructor is a string representing a default constructor for this storage
//type, e.g. `bolt.NewStorageManager(".database")`
func (s StorageType) Constructor() string {

	if s == StorageDefault {
		return "api.NewDefaultStorageManager()"
	}

	args := ""

	switch s {
	case StorageFilesystem:
		args = "\"games/\""
	case StorageBolt:
		args = "\".database\""
	case StorageMysql:
		args = "false"
	}

	return s.String() + ".NewStorageManager(" + args + ")"

}
<commit_msg>Update the pacakge doc for build/api to show sample output and mention directory parameters. Part of #671.<commit_after>/*

api is a package that can create and cleanup api server binaries. Its cousin
is the build/static package, which contains considerably more logic.

The point of this package is primarily to create an `api/main.go` output in
the given directory that statically links the games and storage type
configured via config.json, and then build that binary using `go build`.

The directory parameter gives the build directory; the build command will
create an `api` sub-folder within that, and static.Build() will create a
static directory. A directory of "" is legal and is effectively ".".

There's nothing magic about this package; it's legal to create your own server
binary by hand. This package just automates that for you so when you add a
game to your server you only have to worry about adding it in your config.json
and everything else happens automatically.

For a config json that has a defaultstoragetype of bolt and lists the games
`github.com/jkomoros/boardgame/examples/checkers`,
`github.com/jkomoros/boardgame/examples/memory`, and
`github.com/jkomoros/boardgame/examples/pig` it would output (with the package
doc comment omitted):

	package main

	import (
		"github.com/jkomoros/boardgame/examples/checkers"
		"github.com/jkomoros/boardgame/examples/memory"
		"github.com/jkomoros/boardgame/examples/pig"
		"github.com/jkomoros/boardgame/server/api"
		"github.com/jkomoros/boardgame/storage/bolt"
	)

	func main() {

		storage := api.NewServerStorageManager(bolt.NewStorageManager(".database"))
		defer storage.Close()
		api.NewServer(storage,
			checkers.NewDelegate(),
			memory.NewDelegate(),
			pig.NewDelegate(),
		).Start()
	}

Typically it is not used directly, but via the `boardgame-util build api
`,`boardgame-util cleanup api`, and `boardgame-util serve` commands.

*/
package api

import (
	"path/filepath"
	"strings"
	"text/template"
)

type StorageType int

const (
	StorageInvalid StorageType = iota
	StorageDefault
	StorageMemory
	StorageBolt
	StorageMysql
	StorageFilesystem
)

var apiTemplate *template.Template

func init() {
	apiTemplate = template.Must(template.New("api").Parse(apiTemplateText))
}

//ValidStorageTypeStrings returns an array of strings that are the normal
//(i.e. not invalid) strings that would return useful values if passed to
//StorageTypeFromString.
func ValidStorageTypeStrings() []string {
	return []string{
		StorageDefault.String(),
		StorageMemory.String(),
		StorageBolt.String(),
		StorageMysql.String(),
		StorageFilesystem.String(),
	}
}

func StorageTypeFromString(in string) StorageType {
	in = strings.ToLower(in)
	in = strings.TrimSpace(in)

	switch in {
	case "default":
		return StorageDefault
	case "":
		return StorageDefault
	case "memory":
		return StorageMemory
	case "bolt":
		return StorageBolt
	case "mysql":
		return StorageMysql
	case "filesystem":
		return StorageFilesystem
	}

	return StorageInvalid
}

func (s StorageType) String() string {
	switch s {
	case StorageDefault:
		return "default"
	case StorageMemory:
		return "memory"
	case StorageBolt:
		return "bolt"
	case StorageMysql:
		return "mysql"
	case StorageFilesystem:
		return "filesystem"
	}
	return "invalid"
}

//Import is the string denting the import path for this storage type.
func (s StorageType) Import() string {

	if s == StorageDefault {
		//api package already imported
		return ""
	}

	base := "github.com/jkomoros/boardgame/storage"
	return filepath.Join(base, s.String())
}

//Constructor is a string representing a default constructor for this storage
//type, e.g. `bolt.NewStorageManager(".database")`
func (s StorageType) Constructor() string {

	if s == StorageDefault {
		return "api.NewDefaultStorageManager()"
	}

	args := ""

	switch s {
	case StorageFilesystem:
		args = "\"games/\""
	case StorageBolt:
		args = "\".database\""
	case StorageMysql:
		args = "false"
	}

	return s.String() + ".NewStorageManager(" + args + ")"

}
<|endoftext|>