<commit_before>package herd

import (
	"fmt"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filter"
	subproto "github.com/Symantec/Dominator/proto/sub"
	"os"
	"path"
)

func (sub *Sub) buildUpdateRequest(request *subproto.UpdateRequest) {
	fmt.Println("buildUpdateRequest()") // TODO(rgooch): Delete debugging.
	subFS := sub.fileSystem
	requiredImage := sub.herd.getImage(sub.requiredImage)
	requiredFS := requiredImage.FileSystem
	filter := requiredImage.Filter
	compareDirectories(request, &subFS.Directory, &requiredFS.Directory, "",
		filter)
	// TODO(rgooch): Implement this.
}

func compareDirectories(request *subproto.UpdateRequest,
	subDirectory, requiredDirectory *filesystem.Directory,
	parentName string, filter *filter.Filter) {
	// First look for entries that should be deleted.
	subPathName := path.Join(parentName, subDirectory.Name)
	for name, subEntry := range subDirectory.EntriesByName {
		pathname := path.Join(subPathName, entryName(subEntry))
		if filter.Match(pathname) {
			continue
		}
		if requiredEntry, ok := requiredDirectory.EntriesByName[name]; ok {
			compareEntries(request, subEntry, requiredEntry, subPathName,
				filter)
		} else {
			request.PathsToDelete = append(request.PathsToDelete, pathname)
		}
	}
}

func entryName(entry interface{}) string {
	switch e := entry.(type) {
	case *filesystem.RegularFile:
		return e.Name
	case *filesystem.Symlink:
		return e.Name
	case *filesystem.File:
		return e.Name
	case *filesystem.Directory:
		return e.Name
	}
	panic("Unsupported entry type")
}

func compareEntries(request *subproto.UpdateRequest,
	subEntry, requiredEntry interface{},
	parentName string, filter *filter.Filter) {
	switch se := subEntry.(type) {
	case *filesystem.RegularFile:
		compareSubRegularFileWithEntry(request, se, requiredEntry)
		return
	case *filesystem.Symlink:
		return
	case *filesystem.File:
		return
	case *filesystem.Directory:
		compareSubDirectoryWithEntry(request, se, requiredEntry, parentName,
			filter)
		return
	}
	panic("Unsupported entry type")
}

func compareSubRegularFileWithEntry(request *subproto.UpdateRequest,
	subRegularFile *filesystem.RegularFile, requiredEntry interface{}) {
	if requiredRegularFile, ok := requiredEntry.(*filesystem.RegularFile); ok {
		if filesystem.CompareRegularFiles(subRegularFile, requiredRegularFile,
			os.Stdout) {
			return
		}
		fmt.Printf("Different: %s...\n", subRegularFile.Name) // HACK
	}
	// TODO(rgooch): Delete regular file and replace.
}

func compareSubDirectoryWithEntry(request *subproto.UpdateRequest,
	subDirectory *filesystem.Directory, requiredEntry interface{},
	parentName string, filter *filter.Filter) {
	if requiredDirectory, ok := requiredEntry.(*filesystem.Directory); ok {
		compareDirectories(request, subDirectory, requiredDirectory,
			parentName,
			filter)
	} else {
		// TODO(rgooch): Delete directory and replace.
	}
}
<commit_msg>Incremental work on buildUpdateRequest().<commit_after>package herd

import (
	"fmt"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filter"
	subproto "github.com/Symantec/Dominator/proto/sub"
	"os"
	"path"
)

func (sub *Sub) buildUpdateRequest(request *subproto.UpdateRequest) {
	fmt.Println("buildUpdateRequest()") // TODO(rgooch): Delete debugging.
	subFS := sub.fileSystem
	requiredImage := sub.herd.getImage(sub.requiredImage)
	requiredFS := requiredImage.FileSystem
	filter := requiredImage.Filter
	compareDirectories(request, &subFS.Directory, &requiredFS.Directory, "",
		filter)
	// TODO(rgooch): Implement this.
}

func compareDirectories(request *subproto.UpdateRequest,
	subDirectory, requiredDirectory *filesystem.Directory,
	parentName string, filter *filter.Filter) {
	// First look for entries that should be deleted.
	if subDirectory != nil {
		subPathName := path.Join(parentName, subDirectory.Name)
		for name, subEntry := range subDirectory.EntriesByName {
			pathname := path.Join(subPathName, entryName(subEntry))
			if filter.Match(pathname) {
				continue
			}
			if _, ok := requiredDirectory.EntriesByName[name]; !ok {
				request.PathsToDelete = append(request.PathsToDelete, pathname)
				fmt.Printf("Delete: %s\n", pathname) // HACK
			}
		}
	}
	requiredPathName := path.Join(parentName, requiredDirectory.Name)
	for name, requiredEntry := range requiredDirectory.EntriesByName {
		pathname := path.Join(requiredPathName, entryName(requiredEntry))
		if filter.Match(pathname) {
			continue
		}
		if subDirectory == nil {
			compareEntries(request, nil, requiredEntry, requiredPathName,
				filter)
		} else {
			if subEntry, ok := subDirectory.EntriesByName[name]; ok {
				compareEntries(request, subEntry, requiredEntry,
					requiredPathName, filter)
			} else {
				compareEntries(request, nil, requiredEntry, requiredPathName,
					filter)
			}
		}
	}
}

func entryName(entry interface{}) string {
	switch e := entry.(type) {
	case *filesystem.RegularFile:
		return e.Name
	case *filesystem.Symlink:
		return e.Name
	case *filesystem.File:
		return e.Name
	case *filesystem.Directory:
		return e.Name
	}
	panic("Unsupported entry type")
}

func compareEntries(request *subproto.UpdateRequest,
	subEntry, requiredEntry interface{},
	parentName string, filter *filter.Filter) {
	switch re := requiredEntry.(type) {
	case *filesystem.RegularFile:
		compareRegularFile(request, subEntry, re, parentName)
		return
	case *filesystem.Symlink:
		//compareSymlink(request, subEntry, re, parentName)
		return
	case *filesystem.File:
		//compareFile(request, subEntry, re, parentName)
		return
	case *filesystem.Directory:
		compareDirectory(request, subEntry, re, parentName, filter)
		return
	}
	panic("Unsupported entry type")
}

func compareRegularFile(request *subproto.UpdateRequest,
	subEntry interface{}, requiredRegularFile *filesystem.RegularFile,
	parentName string) {
	if subRegularFile, ok := subEntry.(*filesystem.RegularFile); ok {
		sameMetadata := filesystem.CompareRegularInodesMetadata(
			subRegularFile.Inode(), requiredRegularFile.Inode(),
			os.Stdout)
		sameData := filesystem.CompareRegularInodesData(subRegularFile.Inode(),
			requiredRegularFile.Inode(), os.Stdout)
		if sameMetadata && sameData {
			return
		}
		fmt.Printf("Different: %s...\n", subRegularFile.Name) // HACK
	} else {
		fmt.Printf("Add: %s...\n", subRegularFile.Name) // HACK
	}
	// TODO(rgooch): Delete regular file and replace.
}

func compareDirectory(request *subproto.UpdateRequest,
	subEntry interface{}, requiredDirectory *filesystem.Directory,
	parentName string, filter *filter.Filter) {
	if subDirectory, ok := subEntry.(*filesystem.Directory); ok {
		compareDirectories(request, subDirectory, requiredDirectory,
			parentName, filter)
	} else {
		// TODO(rgooch): Delete directory and replace.
	}
}
<|endoftext|>


<commit_before>package main

import (
)

func SetAnnouncementHandler(w http.ResponseWriter, r *http.Request) {
	//Set Announcement in Memcache.
	if r.Method != "GET" {
		w.WriteHeader(http.StatusNotAcceptable)
		return
	}
	header := r.Header.Get("X-AppEngine-Cron")
	if header == "" {
		w.WriteHeader(http.StatusForbidden)
		w.Write([]byte("attempt to access cron handler directly, missing custom App Engine header"))
		return
	}
	cacheAnnouncement(r)
	w.WriteHeader(http.StatusNoContent)
}

func init() {
	http.HandleFunc("/crons/set_announcement", SetAnnouncementHandler)
}
<commit_msg>3rd update in 3<commit_after>package main

import (
	"net/http"
	"log"
	"appengine"
)

func SetAnnouncementHandler(w http.ResponseWriter, r *http.Request) {
	//Set Announcement in Memcache.
	if r.Method != "GET" {
		w.WriteHeader(http.StatusNotAcceptable)
		return
	}
	header := r.Header.Get("X-AppEngine-Cron")
	if header == "" {
		w.WriteHeader(http.StatusForbidden)
		w.Write([]byte("attempt to access cron handler directly, missing custom App Engine header"))
		return
	}
	cacheAnnouncement(r)
	w.WriteHeader(http.StatusNoContent)
}

func init() {
	http.HandleFunc("/crons/set_announcement", SetAnnouncementHandler)
}
<|endoftext|>


<commit_before>package command

import (
	"context"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
	"github.com/chrislusf/seaweedfs/weed/remote_storage"
	"github.com/chrislusf/seaweedfs/weed/replication/source"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"
)

func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *source.FilerSource, mountedDir string) error {

	// read filer remote storage mount mappings
	_, _, remoteStorageMountLocation, remoteStorage, detectErr := filer.DetectMountInfo(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir)
	if detectErr != nil {
		return fmt.Errorf("read mount info: %v", detectErr)
	}

	eachEntryFunc, err := makeEventProcessor(remoteStorage, mountedDir, remoteStorageMountLocation, filerSource)
	if err != nil {
		return err
	}

	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
		lastTime := time.Unix(0, lastTsNs)
		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, lastTsNs)
	})

	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo)

	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync",
		mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
}

func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
	client, err := remote_storage.GetRemoteStorage(remoteStorage)
	if err != nil {
		return nil, err
	}

	handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if message.NewEntry == nil {
			return nil
		}
		if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
			mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
			if readErr != nil {
				return fmt.Errorf("unmarshal mappings: %v", readErr)
			}
			if remoteLoc, found := mappings.Mappings[mountedDir]; found {
				if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
					glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
				}
			} else {
				glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
				os.Exit(0)
			}
		}
		if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX {
			conf := &remote_pb.RemoteConf{}
			if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
				return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
			}
			remoteStorage = conf
			if newClient, err := remote_storage.GetRemoteStorage(remoteStorage); err == nil {
				client = newClient
			} else {
				return err
			}
		}

		return nil
	}

	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
			return handleEtcRemoteChanges(resp)
		}

		if message.OldEntry == nil && message.NewEntry == nil {
			return nil
		}
		if message.OldEntry == nil && message.NewEntry != nil {
			if !filer.HasData(message.NewEntry) {
				return nil
			}
			glog.V(2).Infof("create: %+v", resp)
			if !shouldSendToRemote(message.NewEntry) {
				glog.V(2).Infof("skipping creating: %+v", resp)
				return nil
			}
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
			if message.NewEntry.IsDirectory {
				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
				return client.WriteDirectory(dest, message.NewEntry)
			}
			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
			if writeErr != nil {
				return writeErr
			}
			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
		}
		if message.OldEntry != nil && message.NewEntry == nil {
			glog.V(2).Infof("delete: %+v", resp)
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
			if message.OldEntry.IsDirectory {
				glog.V(0).Infof("rmdir  %s", remote_storage.FormatLocation(dest))
				return client.RemoveDirectory(dest)
			}
			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
			return client.DeleteFile(dest)
		}
		if message.OldEntry != nil && message.NewEntry != nil {
			oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
			if !shouldSendToRemote(message.NewEntry) {
				glog.V(2).Infof("skipping updating: %+v", resp)
				return nil
			}
			if message.NewEntry.IsDirectory {
				return client.WriteDirectory(dest, message.NewEntry)
			}
			if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
				if filer.IsSameData(message.OldEntry, message.NewEntry) {
					glog.V(2).Infof("update meta: %+v", resp)
					return client.UpdateFileMetadata(dest, message.OldEntry, message.NewEntry)
				}
			}
			glog.V(2).Infof("update: %+v", resp)
			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
			if err := client.DeleteFile(oldDest); err != nil {
				return err
			}
			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
			if writeErr != nil {
				return writeErr
			}
			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
		}

		return nil
	}
	return eachEntryFunc, nil
}

func retriedWriteFile(client remote_storage.RemoteStorageClient, filerSource *source.FilerSource, newEntry *filer_pb.Entry, dest *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
	var writeErr error
	err = util.Retry("writeFile", func() error {
		reader := filer.NewFileReader(filerSource, newEntry)
		glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
		remoteEntry, writeErr = client.WriteFile(dest, newEntry, reader)
		if writeErr != nil {
			return writeErr
		}
		return nil
	})
	glog.Errorf("write to %s: %v", dest, err)
	return
}

func collectLastSyncOffset(filerClient filer_pb.FilerClient, grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, mountedDir string, timeAgo time.Duration) time.Time {
	// 1. specified by timeAgo
	// 2. last offset timestamp for this directory
	// 3. directory creation time
	var lastOffsetTs time.Time
	if timeAgo == 0 {
		mountedDirEntry, err := filer_pb.GetEntry(filerClient, util.FullPath(mountedDir))
		if err != nil {
			glog.V(0).Infof("get mounted directory %s: %v", mountedDir, err)
			return time.Now()
		}

		lastOffsetTsNs, err := remote_storage.GetSyncOffset(grpcDialOption, filerAddress, mountedDir)
		if mountedDirEntry != nil {
			if err == nil && mountedDirEntry.Attributes.Crtime < lastOffsetTsNs/1000000 {
				lastOffsetTs = time.Unix(0, lastOffsetTsNs)
				glog.V(0).Infof("resume from %v", lastOffsetTs)
			} else {
				lastOffsetTs = time.Unix(mountedDirEntry.Attributes.Crtime, 0)
			}
		} else {
			lastOffsetTs = time.Now()
		}
	} else {
		lastOffsetTs = time.Now().Add(-timeAgo)
	}
	return lastOffsetTs
}

func toRemoteStorageLocation(mountDir, sourcePath util.FullPath, remoteMountLocation *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation {
	source := string(sourcePath[len(mountDir):])
	dest := util.FullPath(remoteMountLocation.Path).Child(source)
	return &remote_pb.RemoteStorageLocation{
		Name:   remoteMountLocation.Name,
		Bucket: remoteMountLocation.Bucket,
		Path:   string(dest),
	}
}

func shouldSendToRemote(entry *filer_pb.Entry) bool {
	if entry.RemoteEntry == nil {
		return true
	}
	if entry.RemoteEntry.RemoteMtime < entry.Attributes.Mtime {
		return true
	}
	return false
}

func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
	remoteEntry.LastLocalSyncTsNs = time.Now().UnixNano()
	entry.RemoteEntry = remoteEntry
	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
		_, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
			Directory: dir,
			Entry:     entry,
		})
		return err
	})
}
<commit_msg>adjust error log<commit_after>package command

import (
	"context"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
	"github.com/chrislusf/seaweedfs/weed/remote_storage"
	"github.com/chrislusf/seaweedfs/weed/replication/source"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"
)

func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *source.FilerSource, mountedDir string) error {

	// read filer remote storage mount mappings
	_, _, remoteStorageMountLocation, remoteStorage, detectErr := filer.DetectMountInfo(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir)
	if detectErr != nil {
		return fmt.Errorf("read mount info: %v", detectErr)
	}

	eachEntryFunc, err := makeEventProcessor(remoteStorage, mountedDir, remoteStorageMountLocation, filerSource)
	if err != nil {
		return err
	}

	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
		lastTime := time.Unix(0, lastTsNs)
		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, lastTsNs)
	})

	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo)

	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync",
		mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
}

func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
	client, err := remote_storage.GetRemoteStorage(remoteStorage)
	if err != nil {
		return nil, err
	}

	handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if message.NewEntry == nil {
			return nil
		}
		if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
			mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
			if readErr != nil {
				return fmt.Errorf("unmarshal mappings: %v", readErr)
			}
			if remoteLoc, found := mappings.Mappings[mountedDir]; found {
				if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
					glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
				}
			} else {
				glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
				os.Exit(0)
			}
		}
		if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX {
			conf := &remote_pb.RemoteConf{}
			if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
				return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
			}
			remoteStorage = conf
			if newClient, err := remote_storage.GetRemoteStorage(remoteStorage); err == nil {
				client = newClient
			} else {
				return err
			}
		}

		return nil
	}

	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
			return handleEtcRemoteChanges(resp)
		}

		if message.OldEntry == nil && message.NewEntry == nil {
			return nil
		}
		if message.OldEntry == nil && message.NewEntry != nil {
			if !filer.HasData(message.NewEntry) {
				return nil
			}
			glog.V(2).Infof("create: %+v", resp)
			if !shouldSendToRemote(message.NewEntry) {
				glog.V(2).Infof("skipping creating: %+v", resp)
				return nil
			}
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
			if message.NewEntry.IsDirectory {
				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
				return client.WriteDirectory(dest, message.NewEntry)
			}
			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
			if writeErr != nil {
				return writeErr
			}
			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
		}
		if message.OldEntry != nil && message.NewEntry == nil {
			glog.V(2).Infof("delete: %+v", resp)
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
			if message.OldEntry.IsDirectory {
				glog.V(0).Infof("rmdir  %s", remote_storage.FormatLocation(dest))
				return client.RemoveDirectory(dest)
			}
			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
			return client.DeleteFile(dest)
		}
		if message.OldEntry != nil && message.NewEntry != nil {
			oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
			if !shouldSendToRemote(message.NewEntry) {
				glog.V(2).Infof("skipping updating: %+v", resp)
				return nil
			}
			if message.NewEntry.IsDirectory {
				return client.WriteDirectory(dest, message.NewEntry)
			}
			if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
				if filer.IsSameData(message.OldEntry, message.NewEntry) {
					glog.V(2).Infof("update meta: %+v", resp)
					return client.UpdateFileMetadata(dest, message.OldEntry, message.NewEntry)
				}
			}
			glog.V(2).Infof("update: %+v", resp)
			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
			if err := client.DeleteFile(oldDest); err != nil {
				return err
			}
			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
			if writeErr != nil {
				return writeErr
			}
			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
		}

		return nil
	}
	return eachEntryFunc, nil
}

func retriedWriteFile(client remote_storage.RemoteStorageClient, filerSource *source.FilerSource, newEntry *filer_pb.Entry, dest *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
	var writeErr error
	err = util.Retry("writeFile", func() error {
		reader := filer.NewFileReader(filerSource, newEntry)
		glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
		remoteEntry, writeErr = client.WriteFile(dest, newEntry, reader)
		if writeErr != nil {
			return writeErr
		}
		return nil
	})
	if err != nil {
		glog.Errorf("write to %s: %v", dest, err)
	}
	return
}

func collectLastSyncOffset(filerClient filer_pb.FilerClient, grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, mountedDir string, timeAgo time.Duration) time.Time {
	// 1. specified by timeAgo
	// 2. last offset timestamp for this directory
	// 3. directory creation time
	var lastOffsetTs time.Time
	if timeAgo == 0 {
		mountedDirEntry, err := filer_pb.GetEntry(filerClient, util.FullPath(mountedDir))
		if err != nil {
			glog.V(0).Infof("get mounted directory %s: %v", mountedDir, err)
			return time.Now()
		}

		lastOffsetTsNs, err := remote_storage.GetSyncOffset(grpcDialOption, filerAddress, mountedDir)
		if mountedDirEntry != nil {
			if err == nil && mountedDirEntry.Attributes.Crtime < lastOffsetTsNs/1000000 {
				lastOffsetTs = time.Unix(0, lastOffsetTsNs)
				glog.V(0).Infof("resume from %v", lastOffsetTs)
			} else {
				lastOffsetTs = time.Unix(mountedDirEntry.Attributes.Crtime, 0)
			}
		} else {
			lastOffsetTs = time.Now()
		}
	} else {
		lastOffsetTs = time.Now().Add(-timeAgo)
	}
	return lastOffsetTs
}

func toRemoteStorageLocation(mountDir, sourcePath util.FullPath, remoteMountLocation *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation {
	source := string(sourcePath[len(mountDir):])
	dest := util.FullPath(remoteMountLocation.Path).Child(source)
	return &remote_pb.RemoteStorageLocation{
		Name:   remoteMountLocation.Name,
		Bucket: remoteMountLocation.Bucket,
		Path:   string(dest),
	}
}

func shouldSendToRemote(entry *filer_pb.Entry) bool {
	if entry.RemoteEntry == nil {
		return true
	}
	if entry.RemoteEntry.RemoteMtime < entry.Attributes.Mtime {
		return true
	}
	return false
}

func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
	remoteEntry.LastLocalSyncTsNs = time.Now().UnixNano()
	entry.RemoteEntry = remoteEntry
	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
		_, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
			Directory: dir,
			Entry:     entry,
		})
		return err
	})
}
<|endoftext|>


<commit_before>// Copyright 2014 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package local

import (
	"encoding/binary"
	"fmt"
	"io"
	"math"

	"github.com/prometheus/common/model"
)

// The 21-byte header of a delta-encoded chunk looks like:
//
// - time delta bytes:  1 bytes
// - value delta bytes: 1 bytes
// - is integer:        1 byte
// - base time:         8 bytes
// - base value:        8 bytes
// - used buf bytes:    2 bytes
const (
	deltaHeaderBytes = 21

	deltaHeaderTimeBytesOffset  = 0
	deltaHeaderValueBytesOffset = 1
	deltaHeaderIsIntOffset      = 2
	deltaHeaderBaseTimeOffset   = 3
	deltaHeaderBaseValueOffset  = 11
	deltaHeaderBufLenOffset     = 19
)

// A deltaEncodedChunk adaptively stores sample timestamps and values with a
// delta encoding of various types (int, float) and bit widths. However, once 8
// bytes would be needed to encode a delta value, a fall-back to the absolute
// numbers happens (so that timestamps are saved directly as int64 and values as
// float64). It implements the chunk interface.
type deltaEncodedChunk []byte

// newDeltaEncodedChunk returns a newly allocated deltaEncodedChunk.
func newDeltaEncodedChunk(tb, vb deltaBytes, isInt bool, length int) *deltaEncodedChunk {
	if tb < 1 {
		panic("need at least 1 time delta byte")
	}
	if length < deltaHeaderBytes+16 {
		panic(fmt.Errorf(
			"chunk length %d bytes is insufficient, need at least %d",
			length, deltaHeaderBytes+16,
		))
	}
	c := make(deltaEncodedChunk, deltaHeaderIsIntOffset+1, length)

	c[deltaHeaderTimeBytesOffset] = byte(tb)
	c[deltaHeaderValueBytesOffset] = byte(vb)
	if vb < d8 && isInt { // Only use int for fewer than 8 value delta bytes.
		c[deltaHeaderIsIntOffset] = 1
	} else {
		c[deltaHeaderIsIntOffset] = 0
	}

	return &c
}

// add implements chunk.
func (c deltaEncodedChunk) add(s model.SamplePair) ([]chunk, error) {
	if c.len() == 0 {
		c = c[:deltaHeaderBytes]
		binary.LittleEndian.PutUint64(c[deltaHeaderBaseTimeOffset:], uint64(s.Timestamp))
		binary.LittleEndian.PutUint64(c[deltaHeaderBaseValueOffset:], math.Float64bits(float64(s.Value)))
	}

	remainingBytes := cap(c) - len(c)
	sampleSize := c.sampleSize()

	// Do we generally have space for another sample in this chunk? If not,
	// overflow into a new one.
	if remainingBytes < sampleSize {
		overflowChunks, err := newChunk().add(s)
		if err != nil {
			return nil, err
		}
		return []chunk{&c, overflowChunks[0]}, nil
	}

	baseValue := c.baseValue()
	dt := s.Timestamp - c.baseTime()
	if dt < 0 {
		return nil, fmt.Errorf("time delta is less than zero: %v", dt)
	}

	dv := s.Value - baseValue
	tb := c.timeBytes()
	vb := c.valueBytes()
	isInt := c.isInt()

	// If the new sample is incompatible with the current encoding, reencode the
	// existing chunk data into new chunk(s).

	ntb, nvb, nInt := tb, vb, isInt
	if isInt && !isInt64(dv) {
		// int->float.
		nvb = d4
		nInt = false
	} else if !isInt && vb == d4 && baseValue+model.SampleValue(float32(dv)) != s.Value {
		// float32->float64.
		nvb = d8
	} else {
		if tb < d8 {
			// Maybe more bytes for timestamp.
			ntb = max(tb, bytesNeededForUnsignedTimestampDelta(dt))
		}
		if c.isInt() && vb < d8 {
			// Maybe more bytes for sample value.
			nvb = max(vb, bytesNeededForIntegerSampleValueDelta(dv))
		}
	}
	if tb != ntb || vb != nvb || isInt != nInt {
		if len(c)*2 < cap(c) {
			return transcodeAndAdd(newDeltaEncodedChunk(ntb, nvb, nInt, cap(c)), &c, s)
		}
		// Chunk is already half full. Better create a new one and save the transcoding efforts.
		overflowChunks, err := newChunk().add(s)
		if err != nil {
			return nil, err
		}
		return []chunk{&c, overflowChunks[0]}, nil
	}

	offset := len(c)
	c = c[:offset+sampleSize]

	switch tb {
	case d1:
		c[offset] = byte(dt)
	case d2:
		binary.LittleEndian.PutUint16(c[offset:], uint16(dt))
	case d4:
		binary.LittleEndian.PutUint32(c[offset:], uint32(dt))
	case d8:
		// Store the absolute value (no delta) in case of d8.
		binary.LittleEndian.PutUint64(c[offset:], uint64(s.Timestamp))
	default:
		return nil, fmt.Errorf("invalid number of bytes for time delta: %d", tb)
	}

	offset += int(tb)

	if c.isInt() {
		switch vb {
		case d0:
			// No-op. Constant value is stored as base value.
		case d1:
			c[offset] = byte(int8(dv))
		case d2:
			binary.LittleEndian.PutUint16(c[offset:], uint16(int16(dv)))
		case d4:
			binary.LittleEndian.PutUint32(c[offset:], uint32(int32(dv)))
		// d8 must not happen. Those samples are encoded as float64.
		default:
			return nil, fmt.Errorf("invalid number of bytes for integer delta: %d", vb)
		}
	} else {
		switch vb {
		case d4:
			binary.LittleEndian.PutUint32(c[offset:], math.Float32bits(float32(dv)))
		case d8:
			// Store the absolute value (no delta) in case of d8.
			binary.LittleEndian.PutUint64(c[offset:], math.Float64bits(float64(s.Value)))
		default:
			return nil, fmt.Errorf("invalid number of bytes for floating point delta: %d", vb)
		}
	}
	return []chunk{&c}, nil
}

// clone implements chunk.
func (c deltaEncodedChunk) clone() chunk {
	clone := make(deltaEncodedChunk, len(c), cap(c))
	copy(clone, c)
	return &clone
}

// firstTime implements chunk.
func (c deltaEncodedChunk) firstTime() model.Time {
	return c.baseTime()
}

// newIterator implements chunk.
func (c *deltaEncodedChunk) newIterator() chunkIterator {
	return newIndexAccessingChunkIterator(c.len(), &deltaEncodedIndexAccessor{
		c:      *c,
		baseT:  c.baseTime(),
		baseV:  c.baseValue(),
		tBytes: c.timeBytes(),
		vBytes: c.valueBytes(),
		isInt:  c.isInt(),
	})
}

// marshal implements chunk.
func (c deltaEncodedChunk) marshal(w io.Writer) error {
	if len(c) > math.MaxUint16 {
		panic("chunk buffer length would overflow a 16 bit uint.")
	}
	binary.LittleEndian.PutUint16(c[deltaHeaderBufLenOffset:], uint16(len(c)))

	n, err := w.Write(c[:cap(c)])
	if err != nil {
		return err
	}
	if n != cap(c) {
		return fmt.Errorf("wanted to write %d bytes, wrote %d", cap(c), n)
	}
	return nil
}

// marshalToBuf implements chunk.
func (c deltaEncodedChunk) marshalToBuf(buf []byte) error {
	if len(c) > math.MaxUint16 {
		panic("chunk buffer length would overflow a 16 bit uint")
	}
	binary.LittleEndian.PutUint16(c[deltaHeaderBufLenOffset:], uint16(len(c)))

	n := copy(buf, c)
	if n != len(c) {
		return fmt.Errorf("wanted to copy %d bytes to buffer, copied %d", len(c), n)
	}
	return nil
}

// unmarshal implements chunk.
func (c *deltaEncodedChunk) unmarshal(r io.Reader) error {
	*c = (*c)[:cap(*c)]
	if _, err := io.ReadFull(r, *c); err != nil {
		return err
	}
	l := binary.LittleEndian.Uint16((*c)[deltaHeaderBufLenOffset:])
	if int(l) > cap(*c) {
		return fmt.Errorf("chunk length exceeded during unmarshaling: %d", l)
	}
	*c = (*c)[:l]
	return nil
}

// unmarshalFromBuf implements chunk.
func (c *deltaEncodedChunk) unmarshalFromBuf(buf []byte) error {
	*c = (*c)[:cap(*c)]
	copy(*c, buf)
	l := binary.LittleEndian.Uint16((*c)[deltaHeaderBufLenOffset:])
	if int(l) > cap(*c) {
		return fmt.Errorf("chunk length exceeded during unmarshaling: %d", l)
	}
	*c = (*c)[:l]
	return nil
}

// encoding implements chunk.
func (c deltaEncodedChunk) encoding() chunkEncoding { return delta }

func (c deltaEncodedChunk) timeBytes() deltaBytes {
	return deltaBytes(c[deltaHeaderTimeBytesOffset])
}

func (c deltaEncodedChunk) valueBytes() deltaBytes {
	return deltaBytes(c[deltaHeaderValueBytesOffset])
}

func (c deltaEncodedChunk) isInt() bool {
	return c[deltaHeaderIsIntOffset] == 1
}

func (c deltaEncodedChunk) baseTime() model.Time {
	return model.Time(binary.LittleEndian.Uint64(c[deltaHeaderBaseTimeOffset:]))
}

func (c deltaEncodedChunk) baseValue() model.SampleValue {
	return model.SampleValue(math.Float64frombits(binary.LittleEndian.Uint64(c[deltaHeaderBaseValueOffset:])))
}

func (c deltaEncodedChunk) sampleSize() int {
	return int(c.timeBytes() + c.valueBytes())
}

func (c deltaEncodedChunk) len() int {
	if len(c) < deltaHeaderBytes {
		return 0
	}
	return (len(c) - deltaHeaderBytes) / c.sampleSize()
}

// deltaEncodedIndexAccessor implements indexAccessor.
type deltaEncodedIndexAccessor struct {
	c              deltaEncodedChunk
	baseT          model.Time
	baseV          model.SampleValue
	tBytes, vBytes deltaBytes
	isInt          bool
	lastErr        error
}

func (acc *deltaEncodedIndexAccessor) err() error {
	return acc.lastErr
}

func (acc *deltaEncodedIndexAccessor) timestampAtIndex(idx int) model.Time {
	offset := deltaHeaderBytes + idx*int(acc.tBytes+acc.vBytes)

	switch acc.tBytes {
	case d1:
		return acc.baseT + model.Time(uint8(acc.c[offset]))
	case d2:
		return acc.baseT + model.Time(binary.LittleEndian.Uint16(acc.c[offset:]))
	case d4:
		return acc.baseT + model.Time(binary.LittleEndian.Uint32(acc.c[offset:]))
	case d8:
		// Take absolute value for d8.
		return model.Time(binary.LittleEndian.Uint64(acc.c[offset:]))
	default:
		acc.lastErr = fmt.Errorf("invalid number of bytes for time delta: %d", acc.tBytes)
		return model.Earliest
	}
}

func (acc *deltaEncodedIndexAccessor) sampleValueAtIndex(idx int) model.SampleValue {
	offset := deltaHeaderBytes + idx*int(acc.tBytes+acc.vBytes) + int(acc.tBytes)

	if acc.isInt {
		switch acc.vBytes {
		case d0:
			return acc.baseV
		case d1:
			return acc.baseV + model.SampleValue(int8(acc.c[offset]))
		case d2:
			return acc.baseV + model.SampleValue(int16(binary.LittleEndian.Uint16(acc.c[offset:])))
		case d4:
			return acc.baseV + model.SampleValue(int32(binary.LittleEndian.Uint32(acc.c[offset:])))
		// No d8 for ints.
		default:
			acc.lastErr = fmt.Errorf("invalid number of bytes for integer delta: %d", acc.vBytes)
		}
	} else {
		switch acc.vBytes {
		case d4:
			return acc.baseV + model.SampleValue(math.Float32frombits(binary.LittleEndian.Uint32(acc.c[offset:])))
		case d8:
			// Take absolute value for d8.
			return model.SampleValue(math.Float64frombits(binary.LittleEndian.Uint64(acc.c[offset:])))
		default:
			acc.lastErr = fmt.Errorf("invalid number of bytes for floating point delta: %d", acc.vBytes)
			return 0
		}
	}
}
<commit_msg>Fix return statement.<commit_after>// Copyright 2014 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package local

import (
	"encoding/binary"
	"fmt"
	"io"
	"math"

	"github.com/prometheus/common/model"
)

// The 21-byte header of a delta-encoded chunk looks like:
//
// - time delta bytes:  1 bytes
// - value delta bytes: 1 bytes
// - is integer:        1 byte
// - base time:         8 bytes
// - base value:        8 bytes
// - used buf bytes:    2 bytes
const (
	deltaHeaderBytes = 21

	deltaHeaderTimeBytesOffset  = 0
	deltaHeaderValueBytesOffset = 1
	deltaHeaderIsIntOffset      = 2
	deltaHeaderBaseTimeOffset   = 3
	deltaHeaderBaseValueOffset  = 11
	deltaHeaderBufLenOffset     = 19
)

// A deltaEncodedChunk adaptively stores sample timestamps and values with a
// delta encoding of various types (int, float) and bit widths. However, once 8
// bytes would be needed to encode a delta value, a fall-back to the absolute
// numbers happens (so that timestamps are saved directly as int64 and values as
// float64). It implements the chunk interface.
type deltaEncodedChunk []byte

// newDeltaEncodedChunk returns a newly allocated deltaEncodedChunk.
func newDeltaEncodedChunk(tb, vb deltaBytes, isInt bool, length int) *deltaEncodedChunk {
	if tb < 1 {
		panic("need at least 1 time delta byte")
	}
	if length < deltaHeaderBytes+16 {
		panic(fmt.Errorf(
			"chunk length %d bytes is insufficient, need at least %d",
			length, deltaHeaderBytes+16,
		))
	}
	c := make(deltaEncodedChunk, deltaHeaderIsIntOffset+1, length)

	c[deltaHeaderTimeBytesOffset] = byte(tb)
	c[deltaHeaderValueBytesOffset] = byte(vb)
	if vb < d8 && isInt { // Only use int for fewer than 8 value delta bytes.
		c[deltaHeaderIsIntOffset] = 1
	} else {
		c[deltaHeaderIsIntOffset] = 0
	}

	return &c
}

// add implements chunk.
func (c deltaEncodedChunk) add(s model.SamplePair) ([]chunk, error) {
	if c.len() == 0 {
		c = c[:deltaHeaderBytes]
		binary.LittleEndian.PutUint64(c[deltaHeaderBaseTimeOffset:], uint64(s.Timestamp))
		binary.LittleEndian.PutUint64(c[deltaHeaderBaseValueOffset:], math.Float64bits(float64(s.Value)))
	}

	remainingBytes := cap(c) - len(c)
	sampleSize := c.sampleSize()

	// Do we generally have space for another sample in this chunk? If not,
	// overflow into a new one.
	if remainingBytes < sampleSize {
		overflowChunks, err := newChunk().add(s)
		if err != nil {
			return nil, err
		}
		return []chunk{&c, overflowChunks[0]}, nil
	}

	baseValue := c.baseValue()
	dt := s.Timestamp - c.baseTime()
	if dt < 0 {
		return nil, fmt.Errorf("time delta is less than zero: %v", dt)
	}

	dv := s.Value - baseValue
	tb := c.timeBytes()
	vb := c.valueBytes()
	isInt := c.isInt()

	// If the new sample is incompatible with the current encoding, reencode the
	// existing chunk data into new chunk(s).

	ntb, nvb, nInt := tb, vb, isInt
	if isInt && !isInt64(dv) {
		// int->float.
		nvb = d4
		nInt = false
	} else if !isInt && vb == d4 && baseValue+model.SampleValue(float32(dv)) != s.Value {
		// float32->float64.
		nvb = d8
	} else {
		if tb < d8 {
			// Maybe more bytes for timestamp.
			ntb = max(tb, bytesNeededForUnsignedTimestampDelta(dt))
		}
		if c.isInt() && vb < d8 {
			// Maybe more bytes for sample value.
			nvb = max(vb, bytesNeededForIntegerSampleValueDelta(dv))
		}
	}
	if tb != ntb || vb != nvb || isInt != nInt {
		if len(c)*2 < cap(c) {
			return transcodeAndAdd(newDeltaEncodedChunk(ntb, nvb, nInt, cap(c)), &c, s)
		}
		// Chunk is already half full. Better create a new one and save the transcoding efforts.
		overflowChunks, err := newChunk().add(s)
		if err != nil {
			return nil, err
		}
		return []chunk{&c, overflowChunks[0]}, nil
	}

	offset := len(c)
	c = c[:offset+sampleSize]

	switch tb {
	case d1:
		c[offset] = byte(dt)
	case d2:
		binary.LittleEndian.PutUint16(c[offset:], uint16(dt))
	case d4:
		binary.LittleEndian.PutUint32(c[offset:], uint32(dt))
	case d8:
		// Store the absolute value (no delta) in case of d8.
		binary.LittleEndian.PutUint64(c[offset:], uint64(s.Timestamp))
	default:
		return nil, fmt.Errorf("invalid number of bytes for time delta: %d", tb)
	}

	offset += int(tb)

	if c.isInt() {
		switch vb {
		case d0:
			// No-op. Constant value is stored as base value.
		case d1:
			c[offset] = byte(int8(dv))
		case d2:
			binary.LittleEndian.PutUint16(c[offset:], uint16(int16(dv)))
		case d4:
			binary.LittleEndian.PutUint32(c[offset:], uint32(int32(dv)))
		// d8 must not happen. Those samples are encoded as float64.
		default:
			return nil, fmt.Errorf("invalid number of bytes for integer delta: %d", vb)
		}
	} else {
		switch vb {
		case d4:
			binary.LittleEndian.PutUint32(c[offset:], math.Float32bits(float32(dv)))
		case d8:
			// Store the absolute value (no delta) in case of d8.
			binary.LittleEndian.PutUint64(c[offset:], math.Float64bits(float64(s.Value)))
		default:
			return nil, fmt.Errorf("invalid number of bytes for floating point delta: %d", vb)
		}
	}
	return []chunk{&c}, nil
}

// clone implements chunk.
func (c deltaEncodedChunk) clone() chunk {
	clone := make(deltaEncodedChunk, len(c), cap(c))
	copy(clone, c)
	return &clone
}

// firstTime implements chunk.
func (c deltaEncodedChunk) firstTime() model.Time {
	return c.baseTime()
}

// newIterator implements chunk.
func (c *deltaEncodedChunk) newIterator() chunkIterator {
	return newIndexAccessingChunkIterator(c.len(), &deltaEncodedIndexAccessor{
		c:      *c,
		baseT:  c.baseTime(),
		baseV:  c.baseValue(),
		tBytes: c.timeBytes(),
		vBytes: c.valueBytes(),
		isInt:  c.isInt(),
	})
}

// marshal implements chunk.
func (c deltaEncodedChunk) marshal(w io.Writer) error {
	if len(c) > math.MaxUint16 {
		panic("chunk buffer length would overflow a 16 bit uint.")
	}
	binary.LittleEndian.PutUint16(c[deltaHeaderBufLenOffset:], uint16(len(c)))

	n, err := w.Write(c[:cap(c)])
	if err != nil {
		return err
	}
	if n != cap(c) {
		return fmt.Errorf("wanted to write %d bytes, wrote %d", cap(c), n)
	}
	return nil
}

// marshalToBuf implements chunk.
func (c deltaEncodedChunk) marshalToBuf(buf []byte) error {
	if len(c) > math.MaxUint16 {
		panic("chunk buffer length would overflow a 16 bit uint")
	}
	binary.LittleEndian.PutUint16(c[deltaHeaderBufLenOffset:], uint16(len(c)))

	n := copy(buf, c)
	if n != len(c) {
		return fmt.Errorf("wanted to copy %d bytes to buffer, copied %d", len(c), n)
	}
	return nil
}

// unmarshal implements chunk.
func (c *deltaEncodedChunk) unmarshal(r io.Reader) error {
	*c = (*c)[:cap(*c)]
	if _, err := io.ReadFull(r, *c); err != nil {
		return err
	}
	l := binary.LittleEndian.Uint16((*c)[deltaHeaderBufLenOffset:])
	if int(l) > cap(*c) {
		return fmt.Errorf("chunk length exceeded during unmarshaling: %d", l)
	}
	*c = (*c)[:l]
	return nil
}

// unmarshalFromBuf implements chunk.
func (c *deltaEncodedChunk) unmarshalFromBuf(buf []byte) error {
	*c = (*c)[:cap(*c)]
	copy(*c, buf)
	l := binary.LittleEndian.Uint16((*c)[deltaHeaderBufLenOffset:])
	if int(l) > cap(*c) {
		return fmt.Errorf("chunk length exceeded during unmarshaling: %d", l)
	}
	*c = (*c)[:l]
	return nil
}

// encoding implements chunk.
func (c deltaEncodedChunk) encoding() chunkEncoding { return delta }

func (c deltaEncodedChunk) timeBytes() deltaBytes {
	return deltaBytes(c[deltaHeaderTimeBytesOffset])
}

func (c deltaEncodedChunk) valueBytes() deltaBytes {
	return deltaBytes(c[deltaHeaderValueBytesOffset])
}

func (c deltaEncodedChunk) isInt() bool {
	return c[deltaHeaderIsIntOffset] == 1
}

func (c deltaEncodedChunk) baseTime() model.Time {
	return model.Time(binary.LittleEndian.Uint64(c[deltaHeaderBaseTimeOffset:]))
}

func (c deltaEncodedChunk) baseValue() model.SampleValue {
	return model.SampleValue(math.Float64frombits(binary.LittleEndian.Uint64(c[deltaHeaderBaseValueOffset:])))
}

func (c deltaEncodedChunk) sampleSize() int {
	return int(c.timeBytes() + c.valueBytes())
}

func (c deltaEncodedChunk) len() int {
	if len(c) < deltaHeaderBytes {
		return 0
	}
	return (len(c) - deltaHeaderBytes) / c.sampleSize()
}

// deltaEncodedIndexAccessor implements indexAccessor.
type deltaEncodedIndexAccessor struct {
	c              deltaEncodedChunk
	baseT          model.Time
	baseV          model.SampleValue
	tBytes, vBytes deltaBytes
	isInt          bool
	lastErr        error
}

func (acc *deltaEncodedIndexAccessor) err() error {
	return acc.lastErr
}

func (acc *deltaEncodedIndexAccessor) timestampAtIndex(idx int) model.Time {
	offset := deltaHeaderBytes + idx*int(acc.tBytes+acc.vBytes)

	switch acc.tBytes {
	case d1:
		return acc.baseT + model.Time(uint8(acc.c[offset]))
	case d2:
		return acc.baseT + model.Time(binary.LittleEndian.Uint16(acc.c[offset:]))
	case d4:
		return acc.baseT + model.Time(binary.LittleEndian.Uint32(acc.c[offset:]))
	case d8:
		// Take absolute value for d8.
		return model.Time(binary.LittleEndian.Uint64(acc.c[offset:]))
	default:
		acc.lastErr = fmt.Errorf("invalid number of bytes for time delta: %d", acc.tBytes)
		return model.Earliest
	}
}

func (acc *deltaEncodedIndexAccessor) sampleValueAtIndex(idx int) model.SampleValue {
	offset := deltaHeaderBytes + idx*int(acc.tBytes+acc.vBytes) + int(acc.tBytes)

	if acc.isInt {
		switch acc.vBytes {
		case d0:
			return acc.baseV
		case d1:
			return acc.baseV + model.SampleValue(int8(acc.c[offset]))
		case d2:
			return acc.baseV + model.SampleValue(int16(binary.LittleEndian.Uint16(acc.c[offset:])))
		case d4:
			return acc.baseV + model.SampleValue(int32(binary.LittleEndian.Uint32(acc.c[offset:])))
		// No d8 for ints.
		default:
			acc.lastErr = fmt.Errorf("invalid number of bytes for integer delta: %d", acc.vBytes)
			return 0
		}
	} else {
		switch acc.vBytes {
		case d4:
			return acc.baseV + model.SampleValue(math.Float32frombits(binary.LittleEndian.Uint32(acc.c[offset:])))
		case d8:
			// Take absolute value for d8.
			return model.SampleValue(math.Float64frombits(binary.LittleEndian.Uint64(acc.c[offset:])))
		default:
			acc.lastErr = fmt.Errorf("invalid number of bytes for floating point delta: %d", acc.vBytes)
			return 0
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package orchestrator

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/paths"
	"github.com/andreaskoch/allmark2/common/route"
	"github.com/andreaskoch/allmark2/model"
	"github.com/andreaskoch/allmark2/web/view/viewmodel"
	"strings"
)

func getBaseModel(root, item *model.Item, pathProvider paths.Pather) viewmodel.Base {

	baseModel := viewmodel.Base{
		RepositoryName:        root.Title,
		RepositoryDescription: root.Description,

		Type:    item.Type.String(),
		Route:   pathProvider.Path(item.Route().Value()),
		Level:   item.Route().Level(),
		BaseUrl: GetBaseUrl(item.Route()),
		Alias:   item.MetaData.Alias,

		PrintUrl: GetTypedItemUrl(item.Route(), "print"),
		JsonUrl:  GetTypedItemUrl(item.Route(), "json"),
		RtfUrl:   GetTypedItemUrl(item.Route(), "rtf"),

		Title:       item.Title,
		Description: item.Description,
	}

	return baseModel

}

func GetBaseUrl(route route.Route) string {
	url := route.Value()
	if url != "" {
		return "/" + url + "/"
	}

	return "/"
}

func GetTypedItemUrl(route route.Route, urlType string) string {
	itemPath := GetBaseUrl(route)
	itemPath = strings.TrimSuffix(itemPath, "/")

	if len(itemPath) > 0 {
		return fmt.Sprintf("%s.%s", itemPath, urlType)
	}

	return urlType
}
<commit_msg>Include creation date, modified date and language in the base model<commit_after>// Copyright 2014 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package orchestrator

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/paths"
	"github.com/andreaskoch/allmark2/common/route"
	"github.com/andreaskoch/allmark2/model"
	"github.com/andreaskoch/allmark2/web/view/viewmodel"
	"strings"
)

func getBaseModel(root, item *model.Item, pathProvider paths.Pather) viewmodel.Base {

	baseModel := viewmodel.Base{
		RepositoryName:        root.Title,
		RepositoryDescription: root.Description,

		Type:    item.Type.String(),
		Route:   pathProvider.Path(item.Route().Value()),
		Level:   item.Route().Level(),
		BaseUrl: GetBaseUrl(item.Route()),
		Alias:   item.MetaData.Alias,

		PrintUrl: GetTypedItemUrl(item.Route(), "print"),
		JsonUrl:  GetTypedItemUrl(item.Route(), "json"),
		RtfUrl:   GetTypedItemUrl(item.Route(), "rtf"),

		Title:       item.Title,
		Description: item.Description,

		LanguageTag:      item.MetaData.Language,
		CreationDate:     item.MetaData.CreationDate.Format("2006-01-02"),
		LastModifiedDate: item.MetaData.LastModifiedDate.Format("2006-01-02"),
	}

	return baseModel

}

func GetBaseUrl(route route.Route) string {
	url := route.Value()
	if url != "" {
		return "/" + url + "/"
	}

	return "/"
}

func GetTypedItemUrl(route route.Route, urlType string) string {
	itemPath := GetBaseUrl(route)
	itemPath = strings.TrimSuffix(itemPath, "/")

	if len(itemPath) > 0 {
		return fmt.Sprintf("%s.%s", itemPath, urlType)
	}

	return urlType
}
<|endoftext|>


<commit_before>// Copyright 2019 CUE Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package json converts JSON to and from CUE.
package json

import (
	gojson "encoding/json"
	"io"
	"strconv"
	"strings"

	"cuelang.org/go/cue"
	"cuelang.org/go/cue/ast"
	"cuelang.org/go/cue/ast/astutil"
	"cuelang.org/go/cue/errors"
	"cuelang.org/go/cue/literal"
	"cuelang.org/go/cue/parser"
	"cuelang.org/go/cue/token"
	"cuelang.org/go/pkg/encoding/json"
)

// Valid reports whether data is a valid JSON encoding.
func Valid(b []byte) bool {
	return gojson.Valid(b)
}

// Validate validates JSON and confirms it matches the constraints
// specified by v.
func Validate(b []byte, v cue.Value) error {
	_, err := json.Validate(b, v)
	return err
}

// Extract parses the JSON to a CUE expression.
//
// If src != nil, Extract parses the source from src and the path is for
// position information. The type of the argument for the src parameter must be
// string, []byte, or io.Reader. If src == nil, ParseFile parses the file
// specified by filename.
func Extract(path string, b []byte) (ast.Expr, error) {
	expr, err := extract(path, b)
	if err != nil {
		return nil, err
	}
	patchExpr(expr)
	return expr, nil
}

// Decode converts a JSON file to a CUE value.
//
// If src != nil, Extract parses the source from src and the path is for
// position information. The type of the argument for the src parameter must be
// string, []byte, or io.Reader. If src == nil, ParseFile parses the file
// specified by filename.
func Decode(r *cue.Runtime, path string, b []byte) (*cue.Instance, error) {
	expr, err := extract(path, b)
	if err != nil {
		return nil, err
	}
	return r.CompileExpr(expr)
}

func extract(path string, b []byte) (ast.Expr, error) {
	expr, err := parser.ParseExpr(path, b)
	if err != nil || !gojson.Valid(b) {
		p := token.NoPos
		if pos := errors.Positions(err); len(pos) > 0 {
			p = pos[0]
		}
		var x interface{}
		err := gojson.Unmarshal(b, &x)
		return nil, errors.Wrapf(err, p, "invalid JSON for file %q", path)
	}
	return expr, nil
}

// NewDecoder configures a JSON decoder. The path is used to associate position
// information with each node. The runtime may be nil if the decoder
// is only used to extract to CUE ast objects.
func NewDecoder(r *cue.Runtime, path string, src io.Reader) *Decoder {
	return &Decoder{
		r:      r,
		path:   path,
		dec:    gojson.NewDecoder(src),
		offset: 1,
	}
}

// A Decoder converts JSON values to CUE.
type Decoder struct {
	r      *cue.Runtime
	path   string
	dec    *gojson.Decoder
	offset int
}

// Extract converts the current JSON value to a CUE ast. It returns io.EOF
// if the input has been exhausted.
func (d *Decoder) Extract() (ast.Expr, error) {
	expr, err := d.extract()
	if err != nil {
		return expr, err
	}
	patchExpr(expr)
	return expr, nil
}

func (d *Decoder) extract() (ast.Expr, error) {
	var raw gojson.RawMessage
	err := d.dec.Decode(&raw)
	if err == io.EOF {
		return nil, err
	}
	offset := d.offset
	d.offset += len(raw)
	if err != nil {
		pos := token.NewFile(d.path, offset, len(raw)).Pos(0, 0)
		return nil, errors.Wrapf(err, pos, "invalid JSON for file %q", d.path)
	}
	expr, err := parser.ParseExpr(d.path, []byte(raw), parser.FileOffset(offset))
	if err != nil {
		return nil, err
	}
	return expr, nil
}

// Decode converts the current JSON value to a CUE instance. It returns io.EOF
// if the input has been exhausted.
func (d *Decoder) Decode() (*cue.Instance, error) {
	expr, err := d.Extract()
	if err != nil {
		return nil, err
	}
	return d.r.CompileExpr(expr)
}

// patchExpr simplifies the AST parsed from JSON.
// TODO: some of the modifications are already done in format, but are
// a package deal of a more aggressive simplify. Other pieces of modification
// should probably be moved to format.
func patchExpr(n ast.Node) {
	type info struct {
		reflow bool
	}
	stack := []info{{true}}

	afterFn := func(n ast.Node) {
		switch n.(type) {
		case *ast.ListLit, *ast.StructLit:
			stack = stack[:len(stack)-1]
		}
	}

	var beforeFn func(n ast.Node) bool

	beforeFn = func(n ast.Node) bool {
		isLarge := n.End().Offset()-n.Pos().Offset() > 50
		descent := true

		switch x := n.(type) {
		case *ast.ListLit:
			reflow := true
			if !isLarge {
				for _, e := range x.Elts {
					if hasSpaces(e) {
						reflow = false
						break
					}
				}
			}
			stack = append(stack, info{reflow})
			if reflow {
				x.Lbrack = x.Lbrack.WithRel(token.NoRelPos)
				x.Rbrack = x.Rbrack.WithRel(token.NoRelPos)
			}
			return true

		case *ast.StructLit:
			reflow := true
			if !isLarge {
				for _, e := range x.Elts {
					if f, ok := e.(*ast.Field); !ok || hasSpaces(f) || hasSpaces(f.Value) {
						reflow = false
						break
					}
				}
			}
			stack = append(stack, info{reflow})
			if reflow {
				x.Lbrace = x.Lbrace.WithRel(token.NoRelPos)
				x.Rbrace = x.Rbrace.WithRel(token.NoRelPos)
			}
			return true

		case *ast.Field:
			// label is always a string for JSON.
			switch {
			case true:
				s, ok := x.Label.(*ast.BasicLit)
				if !ok || s.Kind != token.STRING {
					break // should not happen: implies invalid JSON
				}

				u, err := literal.Unquote(s.Value)
				if err != nil {
					break // should not happen: implies invalid JSON
				}

				// TODO(legacy): remove checking for '_' prefix once hidden
				// fields are removed.
				if !ast.IsValidIdent(u) || strings.HasPrefix(u, "_") {
					break // keep string
				}

				x.Label = ast.NewIdent(u)
				astutil.CopyMeta(x.Label, s)
			}
			ast.Walk(x.Value, beforeFn, afterFn)
			descent = false

		case *ast.BasicLit:
			if x.Kind == token.STRING && len(x.Value) > 10 {
				s, err := literal.Unquote(x.Value)
				if err != nil {
					break // should not happen: implies invalid JSON
				}

				lines := strings.Split(s, "\n")
				if len(lines) == 1 {
					break
				}
				x.Value = quoteMulti(lines, len(stack))
			}
		}

		if stack[len(stack)-1].reflow {
			ast.SetRelPos(n, token.NoRelPos)
		}
		return descent
	}

	ast.Walk(n, beforeFn, afterFn)
}

func hasSpaces(n ast.Node) bool {
	return n.Pos().RelPos() > token.NoSpace
}

func quoteMulti(a []string, indent int) string {
	b := strings.Builder{}
	prefix := "\n" + strings.Repeat("\t", indent)
	b.WriteString(`"""`)
	for _, s := range a {
		b.WriteString(prefix)
		q := strconv.Quote(s)
		b.WriteString(q[1 : len(q)-1])
	}
	b.WriteString(prefix)
	b.WriteString(`"""`)
	return b.String()
}
<commit_msg>encoding/json: fix Doc comments for Extract and Decode<commit_after>// Copyright 2019 CUE Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package json converts JSON to and from CUE.
package json

import (
	gojson "encoding/json"
	"io"
	"strconv"
	"strings"

	"cuelang.org/go/cue"
	"cuelang.org/go/cue/ast"
	"cuelang.org/go/cue/ast/astutil"
	"cuelang.org/go/cue/errors"
	"cuelang.org/go/cue/literal"
	"cuelang.org/go/cue/parser"
	"cuelang.org/go/cue/token"
	"cuelang.org/go/pkg/encoding/json"
)

// Valid reports whether data is a valid JSON encoding.
func Valid(b []byte) bool {
	return gojson.Valid(b)
}

// Validate validates JSON and confirms it matches the constraints
// specified by v.
func Validate(b []byte, v cue.Value) error {
	_, err := json.Validate(b, v)
	return err
}

// Extract parses JSON-encoded data to a CUE expression, using path for
// position information.
func Extract(path string, data []byte) (ast.Expr, error) {
	expr, err := extract(path, data)
	if err != nil {
		return nil, err
	}
	patchExpr(expr)
	return expr, nil
}

// Decode parses JSON-encoded data to a CUE value, using path for position
// information.
func Decode(r *cue.Runtime, path string, data []byte) (*cue.Instance, error) {
	expr, err := extract(path, data)
	if err != nil {
		return nil, err
	}
	return r.CompileExpr(expr)
}

func extract(path string, b []byte) (ast.Expr, error) {
	expr, err := parser.ParseExpr(path, b)
	if err != nil || !gojson.Valid(b) {
		p := token.NoPos
		if pos := errors.Positions(err); len(pos) > 0 {
			p = pos[0]
		}
		var x interface{}
		err := gojson.Unmarshal(b, &x)
		return nil, errors.Wrapf(err, p, "invalid JSON for file %q", path)
	}
	return expr, nil
}

// NewDecoder configures a JSON decoder. The path is used to associate position
// information with each node. The runtime may be nil if the decoder
// is only used to extract to CUE ast objects.
func NewDecoder(r *cue.Runtime, path string, src io.Reader) *Decoder {
	return &Decoder{
		r:      r,
		path:   path,
		dec:    gojson.NewDecoder(src),
		offset: 1,
	}
}

// A Decoder converts JSON values to CUE.
type Decoder struct {
	r      *cue.Runtime
	path   string
	dec    *gojson.Decoder
	offset int
}

// Extract converts the current JSON value to a CUE ast. It returns io.EOF
// if the input has been exhausted.
func (d *Decoder) Extract() (ast.Expr, error) {
	expr, err := d.extract()
	if err != nil {
		return expr, err
	}
	patchExpr(expr)
	return expr, nil
}

func (d *Decoder) extract() (ast.Expr, error) {
	var raw gojson.RawMessage
	err := d.dec.Decode(&raw)
	if err == io.EOF {
		return nil, err
	}
	offset := d.offset
	d.offset += len(raw)
	if err != nil {
		pos := token.NewFile(d.path, offset, len(raw)).Pos(0, 0)
		return nil, errors.Wrapf(err, pos, "invalid JSON for file %q", d.path)
	}
	expr, err := parser.ParseExpr(d.path, []byte(raw), parser.FileOffset(offset))
	if err != nil {
		return nil, err
	}
	return expr, nil
}

// Decode converts the current JSON value to a CUE instance. It returns io.EOF
// if the input has been exhausted.
func (d *Decoder) Decode() (*cue.Instance, error) {
	expr, err := d.Extract()
	if err != nil {
		return nil, err
	}
	return d.r.CompileExpr(expr)
}

// patchExpr simplifies the AST parsed from JSON.
// TODO: some of the modifications are already done in format, but are
// a package deal of a more aggressive simplify. Other pieces of modification
// should probably be moved to format.
func patchExpr(n ast.Node) {
	type info struct {
		reflow bool
	}
	stack := []info{{true}}

	afterFn := func(n ast.Node) {
		switch n.(type) {
		case *ast.ListLit, *ast.StructLit:
			stack = stack[:len(stack)-1]
		}
	}

	var beforeFn func(n ast.Node) bool

	beforeFn = func(n ast.Node) bool {
		isLarge := n.End().Offset()-n.Pos().Offset() > 50
		descent := true

		switch x := n.(type) {
		case *ast.ListLit:
			reflow := true
			if !isLarge {
				for _, e := range x.Elts {
					if hasSpaces(e) {
						reflow = false
						break
					}
				}
			}
			stack = append(stack, info{reflow})
			if reflow {
				x.Lbrack = x.Lbrack.WithRel(token.NoRelPos)
				x.Rbrack = x.Rbrack.WithRel(token.NoRelPos)
			}
			return true

		case *ast.StructLit:
			reflow := true
			if !isLarge {
				for _, e := range x.Elts {
					if f, ok := e.(*ast.Field); !ok || hasSpaces(f) || hasSpaces(f.Value) {
						reflow = false
						break
					}
				}
			}
			stack = append(stack, info{reflow})
			if reflow {
				x.Lbrace = x.Lbrace.WithRel(token.NoRelPos)
				x.Rbrace = x.Rbrace.WithRel(token.NoRelPos)
			}
			return true

		case *ast.Field:
			// label is always a string for JSON.
			switch {
			case true:
				s, ok := x.Label.(*ast.BasicLit)
				if !ok || s.Kind != token.STRING {
					break // should not happen: implies invalid JSON
				}

				u, err := literal.Unquote(s.Value)
				if err != nil {
					break // should not happen: implies invalid JSON
				}

				// TODO(legacy): remove checking for '_' prefix once hidden
				// fields are removed.
				if !ast.IsValidIdent(u) || strings.HasPrefix(u, "_") {
					break // keep string
				}

				x.Label = ast.NewIdent(u)
				astutil.CopyMeta(x.Label, s)
			}
			ast.Walk(x.Value, beforeFn, afterFn)
			descent = false

		case *ast.BasicLit:
			if x.Kind == token.STRING && len(x.Value) > 10 {
				s, err := literal.Unquote(x.Value)
				if err != nil {
					break // should not happen: implies invalid JSON
				}

				lines := strings.Split(s, "\n")
				if len(lines) == 1 {
					break
				}
				x.Value = quoteMulti(lines, len(stack))
			}
		}

		if stack[len(stack)-1].reflow {
			ast.SetRelPos(n, token.NoRelPos)
		}
		return descent
	}

	ast.Walk(n, beforeFn, afterFn)
}

func hasSpaces(n ast.Node) bool {
	return n.Pos().RelPos() > token.NoSpace
}

func quoteMulti(a []string, indent int) string {
	b := strings.Builder{}
	prefix := "\n" + strings.Repeat("\t", indent)
	b.WriteString(`"""`)
	for _, s := range a {
		b.WriteString(prefix)
		q := strconv.Quote(s)
		b.WriteString(q[1 : len(q)-1])
	}
	b.WriteString(prefix)
	b.WriteString(`"""`)
	return b.String()
}
<|endoftext|>


<commit_before>// Package path provides Marshaling and Unmarshaling for [ ]byte-encoded paths.
//
// For example, if you have a path like "/pies/apple/0/", of the standard form
// "/pies/<Type:string>/<Index:uint32>/", this helps you Unmarshal that into a
// Pies struct with the fields "Type" and "Index", and vis-versa.
//
// This encodes into a binary, non-human-readable format when using anything but
// strings. It also only supports Marshaling and Unmarshaling into structs using
// primitive types (with fixed byte sizes), which means that your paths must be
// strictly defined before using this.
//
// Warning: do not use this for anything that will change over time and that
// needs to remain backwards compatible. For that, you should use something like
// capnproto.
package path

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"reflect"
	"strings"
)

// Separator allows you to change the path separator used.
type Separator struct {
	b  byte
	s  string
	bs []byte
}

// Static is used to place a static, unchanging element into the path.
type Static struct{}

var staticType = reflect.TypeOf(Static{})

// NewSeparator is used to change the path separator
func NewSeparator(delim byte) Separator {
	return Separator{
		b:  delim,
		s:  string(delim),
		bs: []byte{delim},
	}
}

// Marshal turns the given struct into a structured path
func (s Separator) Marshal(p interface{}) (b []byte, err error) {
	buff := bytes.Buffer{}
	err = s.MarshalInto(p, &buff)
	if err == nil {
		b = buff.Bytes()
	}

	return
}

// MarshalInto works exactly like Marshal, except it writes the path to the
// given Buffer instead of returning a [ ]byte.
func (s Separator) MarshalInto(p interface{}, buff *bytes.Buffer) error {
	err := s.marshalInto(p, buff, true)
	if err == nil {
		buff.WriteByte(s.b)
	}

	return err
}

func (s Separator) marshalInto(p interface{}, buff *bytes.Buffer, needDelim bool) (err error) {
	pt := reflect.TypeOf(p)

	// A nil obviously can't be a path
	if pt == nil {
		return fmt.Errorf("path: a nil cannot be turned into a path")
	}

	pv := reflect.ValueOf(p)

	if pt.Kind() == reflect.Ptr {
		if pv.IsNil() {
			return fmt.Errorf("path: cannot Marshal a nil pointer, expected a struct")
		}

		pv = reflect.Indirect(pv)
		pt = pv.Type()
	}

	if pt.Kind() != reflect.Struct {
		return fmt.Errorf("path: paths may only be read into structs")
	}

	n := pt.NumField()
	for i := 0; i < n; i++ {
		v := pv.Field(i)

		if !v.CanInterface() { // unexported
			continue
		}

		if needDelim {
			buff.WriteByte(s.b)
		}

		needDelim = true

		switch v.Kind() {
		case reflect.Bool:
			b := byte('\x00')
			if v.Bool() {
				b = '\x01'
			}

			buff.WriteByte(b)

		case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			i := v.Int()
			sign := byte(0)
			if i < 0 {
				i = -i
				sign = 0x80
			}

			ib := make([]byte, 8)
			binary.BigEndian.PutUint64(ib, uint64(i))
			ibs := ib[8-v.Type().Size():]
			ibs[0] |= sign

			_, err = buff.Write(ibs)

		case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			ib := make([]byte, 8)
			binary.BigEndian.PutUint64(ib, v.Uint())
			_, err = buff.Write(ib[8-v.Type().Size():])

		case reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
			err = binary.Write(buff, binary.BigEndian, v.Interface())

		case reflect.String:
			err = s.writeString(buff, v.String())

		case reflect.Struct:
			if v.Type() == staticType {
				err = s.writeString(buff, pt.Field(i).Tag.Get("path"))
			} else {
				err = s.marshalInto(v.Interface(), buff, false)
			}

		default:
			err = fmt.Errorf("path: unsupported type: %s", v.Kind())
		}

		if err != nil {
			return
		}
	}

	return
}

func (s Separator) writeString(buff *bytes.Buffer, ss string) error {
	if strings.Contains(ss, s.s) {
		return fmt.Errorf("path: invalid string: may not contain a `%s`", s.s)
	}

	_, err := buff.WriteString(ss)
	return err
}

// Unmarshal is the reverse of Marshal, reading a serialized path into a struct.
func (s Separator) Unmarshal(b []byte, p interface{}) error {
	buff := bytes.NewBuffer(b)
	return s.UnmarshalFrom(buff, p)
}

// UnmarshalFrom works exactly like Unmarshal, except it reads from the given
// Buffer instead of a [ ]byte.
func (s Separator) UnmarshalFrom(buff *bytes.Buffer, p interface{}) (err error) {
	return s.unmarshalFrom(buff, p, true)
}

func (s Separator) unmarshalFrom(buff *bytes.Buffer, p interface{}, expectDelim bool) (err error) {
	pv := reflect.ValueOf(p)

	if pv.Kind() != reflect.Ptr || pv.IsNil() {
		return fmt.Errorf("path: expected pointer to unmarshal into")
	}

	pv = reflect.Indirect(pv)
	pt := pv.Type()

	readString := func() (val string, err error) {
		expectDelim = false

		val, err = buff.ReadString(s.b)
		if err == nil {
			val = val[:len(val)-1]
		}

		return
	}

	n := pv.NumField()
	for i := 0; i < n; i++ {
		v := pv.Field(i)

		if !v.CanSet() {
			continue
		}

		if expectDelim {
			c, err := buff.ReadByte()
			if c != s.b || err != nil {
				return fmt.Errorf("path: invalid path: missing delimiter")
			}
		}

		expectDelim = true

		switch v.Kind() {
		case reflect.Bool:
			var c byte
			c, err = buff.ReadByte()
			if err == nil {
				v.SetBool(c != '\x00')
			}

		case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			var ibs []byte
			ib := make([]byte, 8)
			ibs, err = readInt(ib, buff, v)
			if err == nil {
				sign := ibs[0]&0x80 != 0
				if sign {
					ibs[0] ^= 0x80

					min := true
					for _, b := range ibs {
						if b != 0 {
							min = false
							break
						}
					}

					if min {
						ibs[0] |= 0x80
					}
				}

				val := int64(binary.BigEndian.Uint64(ib))

				if sign {
					val = -val
				}

				v.SetInt(val)
			}

		case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			ib := make([]byte, 8)
			_, err = readInt(ib, buff, v)
			if err == nil {
				v.SetUint(binary.BigEndian.Uint64(ib))
			}

		case reflect.Float32:
			val := float32(0)
			err = binary.Read(buff, binary.BigEndian, &val)
			if err == nil {
				v.SetFloat(float64(val))
			}

		case reflect.Float64:
			val := float64(0)
			err = binary.Read(buff, binary.BigEndian, &val)
			if err == nil {
				v.SetFloat(val)
			}

		case reflect.Complex64:
			val := complex64(0)
			err = binary.Read(buff, binary.BigEndian, &val)
			if err == nil {
				v.SetComplex(complex128(val))
			}

		case reflect.Complex128:
			val := complex128(0)
			err = binary.Read(buff, binary.BigEndian, &val)
			if err == nil {
				v.SetComplex(val)
			}

		case reflect.String:
			var val string
			val, err = readString()
			if err == nil {
				v.SetString(val)
			}

		case reflect.Struct:
			if v.Type() == staticType {
				var val string
				val, err = readString()
				tag := pt.Field(i).Tag.Get("path")
				if err == nil && val != tag {
					err = fmt.Errorf("path: tag mismatch: %s != %s", val, tag)
				}
			} else {
				err = s.unmarshalFrom(buff, v.Addr().Interface(), false)
			}

		default:
			err = fmt.Errorf("path: unsupported type: %s", v.Kind())
		}

		if err != nil {
			return
		}
	}

	return
}

func readInt(ib []byte, buff *bytes.Buffer, v reflect.Value) ([]byte, error) {
	size := v.Type().Size()
	ibs := ib[8-size:]

	n, err := buff.Read(ibs)
	if err != nil || n != len(ibs) {
		return nil, fmt.Errorf("path: failed to read %s", v.Kind())
	}

	return ibs, nil
}
<commit_msg>Ugh, no good English will come of this.<commit_after>// Package path provides Marshaling and Unmarshaling for [ ]byte-encoded paths.
//
// For example, if you have a path like "/pies/apple/0/", of the standard form
// "/pies/<Type:string>/<Index:uint32>/", this helps you Unmarshal that into a
// Pies struct with the fields "Type" and "Index", and vis-versa.
//
// This encodes into a binary, non-human-readable format when using anything but
// strings. It also only supports Marshaling and Unmarshaling into structs using
// primitive types (with fixed byte sizes), which means that your paths must be
// strictly defined before using this.
//
// Warning: do not use this for anything that will change over time and that
// needs to remain backwards compatible. For that, you should use something like
// capnproto.
package path

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"reflect"
	"strings"
)

// Separator allows you to change the path separator used.
type Separator struct {
	b  byte
	s  string
	bs []byte
}

// Static is used to place a static, unchanging element into the path.
type Static struct{}

var staticType = reflect.TypeOf(Static{})

// NewSeparator uses the given delimiter instead of "/"
func NewSeparator(delim byte) Separator {
	return Separator{
		b:  delim,
		s:  string(delim),
		bs: []byte{delim},
	}
}

// Marshal turns the given struct into a structured path
func (s Separator) Marshal(p interface{}) (b []byte, err error) {
	buff := bytes.Buffer{}
	err = s.MarshalInto(p, &buff)
	if err == nil {
		b = buff.Bytes()
	}

	return
}

// MarshalInto works exactly like Marshal, except it writes the path to the
// given Buffer instead of returning a [ ]byte.
func (s Separator) MarshalInto(p interface{}, buff *bytes.Buffer) error {
	err := s.marshalInto(p, buff, true)
	if err == nil {
		buff.WriteByte(s.b)
	}

	return err
}

func (s Separator) marshalInto(p interface{}, buff *bytes.Buffer, needDelim bool) (err error) {
	pt := reflect.TypeOf(p)

	// A nil obviously can't be a path
	if pt == nil {
		return fmt.Errorf("path: a nil cannot be turned into a path")
	}

	pv := reflect.ValueOf(p)

	if pt.Kind() == reflect.Ptr {
		if pv.IsNil() {
			return fmt.Errorf("path: cannot Marshal a nil pointer, expected a struct")
		}

		pv = reflect.Indirect(pv)
		pt = pv.Type()
	}

	if pt.Kind() != reflect.Struct {
		return fmt.Errorf("path: paths may only be read into structs")
	}

	n := pt.NumField()
	for i := 0; i < n; i++ {
		v := pv.Field(i)

		if !v.CanInterface() { // unexported
			continue
		}

		if needDelim {
			buff.WriteByte(s.b)
		}

		needDelim = true

		switch v.Kind() {
		case reflect.Bool:
			b := byte('\x00')
			if v.Bool() {
				b = '\x01'
			}

			buff.WriteByte(b)

		case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			i := v.Int()
			sign := byte(0)
			if i < 0 {
				i = -i
				sign = 0x80
			}

			ib := make([]byte, 8)
			binary.BigEndian.PutUint64(ib, uint64(i))
			ibs := ib[8-v.Type().Size():]
			ibs[0] |= sign

			_, err = buff.Write(ibs)

		case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			ib := make([]byte, 8)
			binary.BigEndian.PutUint64(ib, v.Uint())
			_, err = buff.Write(ib[8-v.Type().Size():])

		case reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
			err = binary.Write(buff, binary.BigEndian, v.Interface())

		case reflect.String:
			err = s.writeString(buff, v.String())

		case reflect.Struct:
			if v.Type() == staticType {
				err = s.writeString(buff, pt.Field(i).Tag.Get("path"))
			} else {
				err = s.marshalInto(v.Interface(), buff, false)
			}

		default:
			err = fmt.Errorf("path: unsupported type: %s", v.Kind())
		}

		if err != nil {
			return
		}
	}

	return
}

func (s Separator) writeString(buff *bytes.Buffer, ss string) error {
	if strings.Contains(ss, s.s) {
		return fmt.Errorf("path: invalid string: may not contain a `%s`", s.s)
	}

	_, err := buff.WriteString(ss)
	return err
}

// Unmarshal is the reverse of Marshal, reading a serialized path into a struct.
func (s Separator) Unmarshal(b []byte, p interface{}) error {
	buff := bytes.NewBuffer(b)
	return s.UnmarshalFrom(buff, p)
}

// UnmarshalFrom works exactly like Unmarshal, except it reads from the given
// Buffer instead of a [ ]byte.
func (s Separator) UnmarshalFrom(buff *bytes.Buffer, p interface{}) (err error) {
	return s.unmarshalFrom(buff, p, true)
}

func (s Separator) unmarshalFrom(buff *bytes.Buffer, p interface{}, expectDelim bool) (err error) {
	pv := reflect.ValueOf(p)

	if pv.Kind() != reflect.Ptr || pv.IsNil() {
		return fmt.Errorf("path: expected pointer to unmarshal into")
	}

	pv = reflect.Indirect(pv)
	pt := pv.Type()

	readString := func() (val string, err error) {
		expectDelim = false

		val, err = buff.ReadString(s.b)
		if err == nil {
			val = val[:len(val)-1]
		}

		return
	}

	n := pv.NumField()
	for i := 0; i < n; i++ {
		v := pv.Field(i)

		if !v.CanSet() {
			continue
		}

		if expectDelim {
			c, err := buff.ReadByte()
			if c != s.b || err != nil {
				return fmt.Errorf("path: invalid path: missing delimiter")
			}
		}

		expectDelim = true

		switch v.Kind() {
		case reflect.Bool:
			var c byte
			c, err = buff.ReadByte()
			if err == nil {
				v.SetBool(c != '\x00')
			}

		case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			var ibs []byte
			ib := make([]byte, 8)
			ibs, err = readInt(ib, buff, v)
			if err == nil {
				sign := ibs[0]&0x80 != 0
				if sign {
					ibs[0] ^= 0x80

					min := true
					for _, b := range ibs {
						if b != 0 {
							min = false
							break
						}
					}

					if min {
						ibs[0] |= 0x80
					}
				}

				val := int64(binary.BigEndian.Uint64(ib))

				if sign {
					val = -val
				}

				v.SetInt(val)
			}

		case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			ib := make([]byte, 8)
			_, err = readInt(ib, buff, v)
			if err == nil {
				v.SetUint(binary.BigEndian.Uint64(ib))
			}

		case reflect.Float32:
			val := float32(0)
			err = binary.Read(buff, binary.BigEndian, &val)
			if err == nil {
				v.SetFloat(float64(val))
			}

		case reflect.Float64:
			val := float64(0)
			err = binary.Read(buff, binary.BigEndian, &val)
			if err == nil {
				v.SetFloat(val)
			}

		case reflect.Complex64:
			val := complex64(0)
			err = binary.Read(buff, binary.BigEndian, &val)
			if err == nil {
				v.SetComplex(complex128(val))
			}

		case reflect.Complex128:
			val := complex128(0)
			err = binary.Read(buff, binary.BigEndian, &val)
			if err == nil {
				v.SetComplex(val)
			}

		case reflect.String:
			var val string
			val, err = readString()
			if err == nil {
				v.SetString(val)
			}

		case reflect.Struct:
			if v.Type() == staticType {
				var val string
				val, err = readString()
				tag := pt.Field(i).Tag.Get("path")
				if err == nil && val != tag {
					err = fmt.Errorf("path: tag mismatch: %s != %s", val, tag)
				}
			} else {
				err = s.unmarshalFrom(buff, v.Addr().Interface(), false)
			}

		default:
			err = fmt.Errorf("path: unsupported type: %s", v.Kind())
		}

		if err != nil {
			return
		}
	}

	return
}

func readInt(ib []byte, buff *bytes.Buffer, v reflect.Value) ([]byte, error) {
	size := v.Type().Size()
	ibs := ib[8-size:]

	n, err := buff.Read(ibs)
	if err != nil || n != len(ibs) {
		return nil, fmt.Errorf("path: failed to read %s", v.Kind())
	}

	return ibs, nil
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2017 Cavium
//
// SPDX-License-Identifier: Apache-2.0
//

package distro

import (
	"io"
	"net/http"

	"github.com/go-zoo/bone"
)

func replyPing(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/text; charset=utf-8")
	w.WriteHeader(http.StatusOK)
	str := `pong`
	io.WriteString(w, str)
}

func replyUpdateRegistrations(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/text; charset=utf-8")
	w.WriteHeader(http.StatusOK)
	io.WriteString(w, "")
	RefreshRegistrations()
}

// HTTPServer function
func HTTPServer() http.Handler {
	mux := bone.New()

	mux.Get("/api/v1/ping", http.HandlerFunc(replyPing))
	mux.Get("/api/v1/updateregistrations", http.HandlerFunc(replyUpdateRegistrations))

	return mux
}
<commit_msg>Changing ending point to notify/registrations<commit_after>//
// Copyright (c) 2017 Cavium
//
// SPDX-License-Identifier: Apache-2.0
//

package distro

import (
	"io"
	"net/http"

	"github.com/go-zoo/bone"
)

func replyPing(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/text; charset=utf-8")
	w.WriteHeader(http.StatusOK)
	str := `pong`
	io.WriteString(w, str)
}

func replyNotifyRegistrations(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/text; charset=utf-8")
	w.WriteHeader(http.StatusOK)
	io.WriteString(w, "")
	RefreshRegistrations()
}

// HTTPServer function
func HTTPServer() http.Handler {
	mux := bone.New()

	mux.Get("/api/v1/ping", http.HandlerFunc(replyPing))
	mux.Get("/api/v1/notify/registrations", http.HandlerFunc(replyNotifyRegistrations))

	return mux
}
<|endoftext|>