<commit_before>package shell

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"math"
	"sort"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"google.golang.org/grpc"
)

func moveMountedShardToEcNode(commandEnv *CommandEnv, existingLocation *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, destinationEcNode *EcNode, applyBalancing bool) (err error) {

	copiedShardIds := []uint32{uint32(shardId)}

	if applyBalancing {

		existingServerAddress := pb.NewServerAddressFromDataNode(existingLocation.info)

		// ask destination node to copy shard and the ecx file from source node, and mount it
		copiedShardIds, err = oneServerCopyAndMountEcShardsFromSource(commandEnv.option.GrpcDialOption, destinationEcNode, []uint32{uint32(shardId)}, vid, collection, existingServerAddress)
		if err != nil {
			return err
		}

		// unmount the to be deleted shards
		err = unmountEcShards(commandEnv.option.GrpcDialOption, vid, existingServerAddress, copiedShardIds)
		if err != nil {
			return err
		}

		// ask source node to delete the shard, and maybe the ecx file
		err = sourceServerDeleteEcShards(commandEnv.option.GrpcDialOption, collection, vid, existingServerAddress, copiedShardIds)
		if err != nil {
			return err
		}

		fmt.Printf("moved ec shard %d.%d %s => %s\n", vid, shardId, existingLocation.info.Id, destinationEcNode.info.Id)

	}

	destinationEcNode.addEcVolumeShards(vid, collection, copiedShardIds)
	existingLocation.deleteEcVolumeShards(vid, copiedShardIds)

	return nil

}

func oneServerCopyAndMountEcShardsFromSource(grpcDialOption grpc.DialOption,
	targetServer *EcNode, shardIdsToCopy []uint32,
	volumeId needle.VolumeId, collection string, existingLocation pb.ServerAddress) (copiedShardIds []uint32, err error) {

	fmt.Printf("allocate %d.%v %s => %s\n", volumeId, shardIdsToCopy, existingLocation, targetServer.info.Id)

	targetAddress := pb.NewServerAddressFromDataNode(targetServer.info)
	err = operation.WithVolumeServerClient(false, targetAddress, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {

		if targetAddress != existingLocation {

			fmt.Printf("copy %d.%v %s => %s\n", volumeId, shardIdsToCopy, existingLocation, targetServer.info.Id)
			_, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{
				VolumeId:       uint32(volumeId),
				Collection:     collection,
				ShardIds:       shardIdsToCopy,
				CopyEcxFile:    true,
				CopyEcjFile:    true,
				CopyVifFile:    true,
				SourceDataNode: string(existingLocation),
			})
			if copyErr != nil {
				return fmt.Errorf("copy %d.%v %s => %s : %v\n", volumeId, shardIdsToCopy, existingLocation, targetServer.info.Id, copyErr)
			}
		}

		fmt.Printf("mount %d.%v on %s\n", volumeId, shardIdsToCopy, targetServer.info.Id)
		_, mountErr := volumeServerClient.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
			VolumeId:   uint32(volumeId),
			Collection: collection,
			ShardIds:   shardIdsToCopy,
		})
		if mountErr != nil {
			return fmt.Errorf("mount %d.%v on %s : %v\n", volumeId, shardIdsToCopy, targetServer.info.Id, mountErr)
		}

		if targetAddress != existingLocation {
			copiedShardIds = shardIdsToCopy
			glog.V(0).Infof("%s ec volume %d deletes shards %+v", existingLocation, volumeId, copiedShardIds)
		}

		return nil
	})

	if err != nil {
		return
	}

	return
}

func eachDataNode(topo *master_pb.TopologyInfo, fn func(dc string, rack RackId, dn *master_pb.DataNodeInfo)) {
	for _, dc := range topo.DataCenterInfos {
		for _, rack := range dc.RackInfos {
			for _, dn := range rack.DataNodeInfos {
				fn(dc.Id, RackId(rack.Id), dn)
			}
		}
	}
}

func sortEcNodesByFreeslotsDecending(ecNodes []*EcNode) {
	sort.Slice(ecNodes, func(i, j int) bool {
		return ecNodes[i].freeEcSlot > ecNodes[j].freeEcSlot
	})
}

func sortEcNodesByFreeslotsAscending(ecNodes []*EcNode) {
	sort.Slice(ecNodes, func(i, j int) bool {
		return ecNodes[i].freeEcSlot < ecNodes[j].freeEcSlot
	})
}

type CandidateEcNode struct {
	ecNode     *EcNode
	shardCount int
}

// if the index node changed the freeEcSlot, need to keep every EcNode still sorted
func ensureSortedEcNodes(data []*CandidateEcNode, index int, lessThan func(i, j int) bool) {
	for i := index - 1; i >= 0; i-- {
		if lessThan(i+1, i) {
			swap(data, i, i+1)
		} else {
			break
		}
	}
	for i := index + 1; i < len(data); i++ {
		if lessThan(i, i-1) {
			swap(data, i, i-1)
		} else {
			break
		}
	}
}

func swap(data []*CandidateEcNode, i, j int) {
	t := data[i]
	data[i] = data[j]
	data[j] = t
}

func countShards(ecShardInfos []*master_pb.VolumeEcShardInformationMessage) (count int) {
	for _, ecShardInfo := range ecShardInfos {
		shardBits := erasure_coding.ShardBits(ecShardInfo.EcIndexBits)
		count += shardBits.ShardIdCount()
	}
	return
}

func countFreeShardSlots(dn *master_pb.DataNodeInfo, diskType types.DiskType) (count int) {
	if dn.DiskInfos == nil {
		return 0
	}
	diskInfo := dn.DiskInfos[string(diskType)]
	if diskInfo == nil {
		return 0
	}
	return int(diskInfo.MaxVolumeCount-diskInfo.ActiveVolumeCount)*erasure_coding.DataShardsCount - countShards(diskInfo.EcShardInfos)
}

type RackId string
type EcNodeId string

type EcNode struct {
	info       *master_pb.DataNodeInfo
	dc         string
	rack       RackId
	freeEcSlot int
}

func (ecNode *EcNode) localShardIdCount(vid uint32) int {
	for _, diskInfo := range ecNode.info.DiskInfos {
		for _, ecShardInfo := range diskInfo.EcShardInfos {
			if vid == ecShardInfo.Id {
				shardBits := erasure_coding.ShardBits(ecShardInfo.EcIndexBits)
				return shardBits.ShardIdCount()
			}
		}
	}
	return 0
}

type EcRack struct {
	ecNodes    map[EcNodeId]*EcNode
	freeEcSlot int
}

func collectEcNodes(commandEnv *CommandEnv, selectedDataCenter string) (ecNodes []*EcNode, totalFreeEcSlots int, err error) {

	// list all possible locations
	// collect topology information
	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
	if err != nil {
		return
	}

	// find out all volume servers with one slot left.
	ecNodes, totalFreeEcSlots = collectEcVolumeServersByDc(topologyInfo, selectedDataCenter)

	sortEcNodesByFreeslotsDecending(ecNodes)

	return
}

func collectEcVolumeServersByDc(topo *master_pb.TopologyInfo, selectedDataCenter string) (ecNodes []*EcNode, totalFreeEcSlots int) {
	eachDataNode(topo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
		if selectedDataCenter != "" && selectedDataCenter != dc {
			return
		}

		freeEcSlots := countFreeShardSlots(dn, types.HardDriveType)
		ecNodes = append(ecNodes, &EcNode{
			info:       dn,
			dc:         dc,
			rack:       rack,
			freeEcSlot: int(freeEcSlots),
		})
		totalFreeEcSlots += freeEcSlots
	})
	return
}

func sourceServerDeleteEcShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeDeletedShardIds []uint32) error {

	fmt.Printf("delete %d.%v from %s\n", volumeId, toBeDeletedShardIds, sourceLocation)

	return operation.WithVolumeServerClient(false, sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
		_, deleteErr := volumeServerClient.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{
			VolumeId:   uint32(volumeId),
			Collection: collection,
			ShardIds:   toBeDeletedShardIds,
		})
		return deleteErr
	})

}

func unmountEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeUnmountedhardIds []uint32) error {

	fmt.Printf("unmount %d.%v from %s\n", volumeId, toBeUnmountedhardIds, sourceLocation)

	return operation.WithVolumeServerClient(false, sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
		_, deleteErr := volumeServerClient.VolumeEcShardsUnmount(context.Background(), &volume_server_pb.VolumeEcShardsUnmountRequest{
			VolumeId: uint32(volumeId),
			ShardIds: toBeUnmountedhardIds,
		})
		return deleteErr
	})
}

func mountEcShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeMountedhardIds []uint32) error {

	fmt.Printf("mount %d.%v on %s\n", volumeId, toBeMountedhardIds, sourceLocation)

	return operation.WithVolumeServerClient(false, sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
		_, mountErr := volumeServerClient.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
			VolumeId:   uint32(volumeId),
			Collection: collection,
			ShardIds:   toBeMountedhardIds,
		})
		return mountErr
	})
}

func divide(total, n int) float64 {
	return float64(total) / float64(n)
}

func ceilDivide(total, n int) int {
	return int(math.Ceil(float64(total) / float64(n)))
}

func findEcVolumeShards(ecNode *EcNode, vid needle.VolumeId) erasure_coding.ShardBits {

	if diskInfo, found := ecNode.info.DiskInfos[string(types.HardDriveType)]; found {
		for _, shardInfo := range diskInfo.EcShardInfos {
			if needle.VolumeId(shardInfo.Id) == vid {
				return erasure_coding.ShardBits(shardInfo.EcIndexBits)
			}
		}
	}

	return 0
}

func (ecNode *EcNode) addEcVolumeShards(vid needle.VolumeId, collection string, shardIds []uint32) *EcNode {

	foundVolume := false
	diskInfo, found := ecNode.info.DiskInfos[string(types.HardDriveType)]
	if found {
		for _, shardInfo := range diskInfo.EcShardInfos {
			if needle.VolumeId(shardInfo.Id) == vid {
				oldShardBits := erasure_coding.ShardBits(shardInfo.EcIndexBits)
				newShardBits := oldShardBits
				for _, shardId := range shardIds {
					newShardBits = newShardBits.AddShardId(erasure_coding.ShardId(shardId))
				}
				shardInfo.EcIndexBits = uint32(newShardBits)
				ecNode.freeEcSlot -= newShardBits.ShardIdCount() - oldShardBits.ShardIdCount()
				foundVolume = true
				break
			}
		}
	} else {
		diskInfo = &master_pb.DiskInfo{
			Type: string(types.HardDriveType),
		}
		ecNode.info.DiskInfos[string(types.HardDriveType)] = diskInfo
	}

	if !foundVolume {
		var newShardBits erasure_coding.ShardBits
		for _, shardId := range shardIds {
			newShardBits = newShardBits.AddShardId(erasure_coding.ShardId(shardId))
		}
		diskInfo.EcShardInfos = append(diskInfo.EcShardInfos, &master_pb.VolumeEcShardInformationMessage{
			Id:          uint32(vid),
			Collection:  collection,
			EcIndexBits: uint32(newShardBits),
			DiskType:    string(types.HardDriveType),
		})
		ecNode.freeEcSlot -= len(shardIds)
	}

	return ecNode
}

func (ecNode *EcNode) deleteEcVolumeShards(vid needle.VolumeId, shardIds []uint32) *EcNode {

	if diskInfo, found := ecNode.info.DiskInfos[string(types.HardDriveType)]; found {
		for _, shardInfo := range diskInfo.EcShardInfos {
			if needle.VolumeId(shardInfo.Id) == vid {
				oldShardBits := erasure_coding.ShardBits(shardInfo.EcIndexBits)
				newShardBits := oldShardBits
				for _, shardId := range shardIds {
					newShardBits = newShardBits.RemoveShardId(erasure_coding.ShardId(shardId))
				}
				shardInfo.EcIndexBits = uint32(newShardBits)
				ecNode.freeEcSlot -= newShardBits.ShardIdCount() - oldShardBits.ShardIdCount()
			}
		}
	}

	return ecNode
}

func groupByCount(data []*EcNode, identifierFn func(*EcNode) (id string, count int)) map[string]int {
	countMap := make(map[string]int)
	for _, d := range data {
		id, count := identifierFn(d)
		countMap[id] += count
	}
	return countMap
}

func groupBy(data []*EcNode, identifierFn func(*EcNode) (id string)) map[string][]*EcNode {
	groupMap := make(map[string][]*EcNode)
	for _, d := range data {
		id := identifierFn(d)
		groupMap[id] = append(groupMap[id], d)
	}
	return groupMap
}
<commit_msg>ec.encode: calculate free ec slots based on (maxVolumeCount-volumeCount)<commit_after>package shell

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"math"
	"sort"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"google.golang.org/grpc"
)

func moveMountedShardToEcNode(commandEnv *CommandEnv, existingLocation *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, destinationEcNode *EcNode, applyBalancing bool) (err error) {

	copiedShardIds := []uint32{uint32(shardId)}

	if applyBalancing {

		existingServerAddress := pb.NewServerAddressFromDataNode(existingLocation.info)

		// ask destination node to copy shard and the ecx file from source node, and mount it
		copiedShardIds, err = oneServerCopyAndMountEcShardsFromSource(commandEnv.option.GrpcDialOption, destinationEcNode, []uint32{uint32(shardId)}, vid, collection, existingServerAddress)
		if err != nil {
			return err
		}

		// unmount the to be deleted shards
		err = unmountEcShards(commandEnv.option.GrpcDialOption, vid, existingServerAddress, copiedShardIds)
		if err != nil {
			return err
		}

		// ask source node to delete the shard, and maybe the ecx file
		err = sourceServerDeleteEcShards(commandEnv.option.GrpcDialOption, collection, vid, existingServerAddress, copiedShardIds)
		if err != nil {
			return err
		}

		fmt.Printf("moved ec shard %d.%d %s => %s\n", vid, shardId, existingLocation.info.Id, destinationEcNode.info.Id)

	}

	destinationEcNode.addEcVolumeShards(vid, collection, copiedShardIds)
	existingLocation.deleteEcVolumeShards(vid, copiedShardIds)

	return nil

}

func oneServerCopyAndMountEcShardsFromSource(grpcDialOption grpc.DialOption,
	targetServer *EcNode, shardIdsToCopy []uint32,
	volumeId needle.VolumeId, collection string, existingLocation pb.ServerAddress) (copiedShardIds []uint32, err error) {

	fmt.Printf("allocate %d.%v %s => %s\n", volumeId, shardIdsToCopy, existingLocation, targetServer.info.Id)

	targetAddress := pb.NewServerAddressFromDataNode(targetServer.info)
	err = operation.WithVolumeServerClient(false, targetAddress, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {

		if targetAddress != existingLocation {

			fmt.Printf("copy %d.%v %s => %s\n", volumeId, shardIdsToCopy, existingLocation, targetServer.info.Id)
			_, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{
				VolumeId:       uint32(volumeId),
				Collection:     collection,
				ShardIds:       shardIdsToCopy,
				CopyEcxFile:    true,
				CopyEcjFile:    true,
				CopyVifFile:    true,
				SourceDataNode: string(existingLocation),
			})
			if copyErr != nil {
				return fmt.Errorf("copy %d.%v %s => %s : %v\n", volumeId, shardIdsToCopy, existingLocation, targetServer.info.Id, copyErr)
			}
		}

		fmt.Printf("mount %d.%v on %s\n", volumeId, shardIdsToCopy, targetServer.info.Id)
		_, mountErr := volumeServerClient.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
			VolumeId:   uint32(volumeId),
			Collection: collection,
			ShardIds:   shardIdsToCopy,
		})
		if mountErr != nil {
			return fmt.Errorf("mount %d.%v on %s : %v\n", volumeId, shardIdsToCopy, targetServer.info.Id, mountErr)
		}

		if targetAddress != existingLocation {
			copiedShardIds = shardIdsToCopy
			glog.V(0).Infof("%s ec volume %d deletes shards %+v", existingLocation, volumeId, copiedShardIds)
		}

		return nil
	})

	if err != nil {
		return
	}

	return
}

func eachDataNode(topo *master_pb.TopologyInfo, fn func(dc string, rack RackId, dn *master_pb.DataNodeInfo)) {
	for _, dc := range topo.DataCenterInfos {
		for _, rack := range dc.RackInfos {
			for _, dn := range rack.DataNodeInfos {
				fn(dc.Id, RackId(rack.Id), dn)
			}
		}
	}
}

func sortEcNodesByFreeslotsDecending(ecNodes []*EcNode) {
	sort.Slice(ecNodes, func(i, j int) bool {
		return ecNodes[i].freeEcSlot > ecNodes[j].freeEcSlot
	})
}

func sortEcNodesByFreeslotsAscending(ecNodes []*EcNode) {
	sort.Slice(ecNodes, func(i, j int) bool {
		return ecNodes[i].freeEcSlot < ecNodes[j].freeEcSlot
	})
}

type CandidateEcNode struct {
	ecNode     *EcNode
	shardCount int
}

// if the index node changed the freeEcSlot, need to keep every EcNode still sorted
func ensureSortedEcNodes(data []*CandidateEcNode, index int, lessThan func(i, j int) bool) {
	for i := index - 1; i >= 0; i-- {
		if lessThan(i+1, i) {
			swap(data, i, i+1)
		} else {
			break
		}
	}
	for i := index + 1; i < len(data); i++ {
		if lessThan(i, i-1) {
			swap(data, i, i-1)
		} else {
			break
		}
	}
}

func swap(data []*CandidateEcNode, i, j int) {
	t := data[i]
	data[i] = data[j]
	data[j] = t
}

func countShards(ecShardInfos []*master_pb.VolumeEcShardInformationMessage) (count int) {
	for _, ecShardInfo := range ecShardInfos {
		shardBits := erasure_coding.ShardBits(ecShardInfo.EcIndexBits)
		count += shardBits.ShardIdCount()
	}
	return
}

func countFreeShardSlots(dn *master_pb.DataNodeInfo, diskType types.DiskType) (count int) {
	if dn.DiskInfos == nil {
		return 0
	}
	diskInfo := dn.DiskInfos[string(diskType)]
	if diskInfo == nil {
		return 0
	}
	return int(diskInfo.MaxVolumeCount-diskInfo.VolumeCount)*erasure_coding.DataShardsCount - countShards(diskInfo.EcShardInfos)
}

type RackId string
type EcNodeId string

type EcNode struct {
	info       *master_pb.DataNodeInfo
	dc         string
	rack       RackId
	freeEcSlot int
}

func (ecNode *EcNode) localShardIdCount(vid uint32) int {
	for _, diskInfo := range ecNode.info.DiskInfos {
		for _, ecShardInfo := range diskInfo.EcShardInfos {
			if vid == ecShardInfo.Id {
				shardBits := erasure_coding.ShardBits(ecShardInfo.EcIndexBits)
				return shardBits.ShardIdCount()
			}
		}
	}
	return 0
}

type EcRack struct {
	ecNodes    map[EcNodeId]*EcNode
	freeEcSlot int
}

func collectEcNodes(commandEnv *CommandEnv, selectedDataCenter string) (ecNodes []*EcNode, totalFreeEcSlots int, err error) {

	// list all possible locations
	// collect topology information
	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
	if err != nil {
		return
	}

	// find out all volume servers with one slot left.
	ecNodes, totalFreeEcSlots = collectEcVolumeServersByDc(topologyInfo, selectedDataCenter)

	sortEcNodesByFreeslotsDecending(ecNodes)

	return
}

func collectEcVolumeServersByDc(topo *master_pb.TopologyInfo, selectedDataCenter string) (ecNodes []*EcNode, totalFreeEcSlots int) {
	eachDataNode(topo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
		if selectedDataCenter != "" && selectedDataCenter != dc {
			return
		}

		freeEcSlots := countFreeShardSlots(dn, types.HardDriveType)
		ecNodes = append(ecNodes, &EcNode{
			info:       dn,
			dc:         dc,
			rack:       rack,
			freeEcSlot: int(freeEcSlots),
		})
		totalFreeEcSlots += freeEcSlots
	})
	return
}

func sourceServerDeleteEcShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeDeletedShardIds []uint32) error {

	fmt.Printf("delete %d.%v from %s\n", volumeId, toBeDeletedShardIds, sourceLocation)

	return operation.WithVolumeServerClient(false, sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
		_, deleteErr := volumeServerClient.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{
			VolumeId:   uint32(volumeId),
			Collection: collection,
			ShardIds:   toBeDeletedShardIds,
		})
		return deleteErr
	})

}

func unmountEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeUnmountedhardIds []uint32) error {

	fmt.Printf("unmount %d.%v from %s\n", volumeId, toBeUnmountedhardIds, sourceLocation)

	return operation.WithVolumeServerClient(false, sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
		_, deleteErr := volumeServerClient.VolumeEcShardsUnmount(context.Background(), &volume_server_pb.VolumeEcShardsUnmountRequest{
			VolumeId: uint32(volumeId),
			ShardIds: toBeUnmountedhardIds,
		})
		return deleteErr
	})
}

func mountEcShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeMountedhardIds []uint32) error {

	fmt.Printf("mount %d.%v on %s\n", volumeId, toBeMountedhardIds, sourceLocation)

	return operation.WithVolumeServerClient(false, sourceLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
		_, mountErr := volumeServerClient.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
			VolumeId:   uint32(volumeId),
			Collection: collection,
			ShardIds:   toBeMountedhardIds,
		})
		return mountErr
	})
}

func divide(total, n int) float64 {
	return float64(total) / float64(n)
}

func ceilDivide(total, n int) int {
	return int(math.Ceil(float64(total) / float64(n)))
}

func findEcVolumeShards(ecNode *EcNode, vid needle.VolumeId) erasure_coding.ShardBits {

	if diskInfo, found := ecNode.info.DiskInfos[string(types.HardDriveType)]; found {
		for _, shardInfo := range diskInfo.EcShardInfos {
			if needle.VolumeId(shardInfo.Id) == vid {
				return erasure_coding.ShardBits(shardInfo.EcIndexBits)
			}
		}
	}

	return 0
}

func (ecNode *EcNode) addEcVolumeShards(vid needle.VolumeId, collection string, shardIds []uint32) *EcNode {

	foundVolume := false
	diskInfo, found := ecNode.info.DiskInfos[string(types.HardDriveType)]
	if found {
		for _, shardInfo := range diskInfo.EcShardInfos {
			if needle.VolumeId(shardInfo.Id) == vid {
				oldShardBits := erasure_coding.ShardBits(shardInfo.EcIndexBits)
				newShardBits := oldShardBits
				for _, shardId := range shardIds {
					newShardBits = newShardBits.AddShardId(erasure_coding.ShardId(shardId))
				}
				shardInfo.EcIndexBits = uint32(newShardBits)
				ecNode.freeEcSlot -= newShardBits.ShardIdCount() - oldShardBits.ShardIdCount()
				foundVolume = true
				break
			}
		}
	} else {
		diskInfo = &master_pb.DiskInfo{
			Type: string(types.HardDriveType),
		}
		ecNode.info.DiskInfos[string(types.HardDriveType)] = diskInfo
	}

	if !foundVolume {
		var newShardBits erasure_coding.ShardBits
		for _, shardId := range shardIds {
			newShardBits = newShardBits.AddShardId(erasure_coding.ShardId(shardId))
		}
		diskInfo.EcShardInfos = append(diskInfo.EcShardInfos, &master_pb.VolumeEcShardInformationMessage{
			Id:          uint32(vid),
			Collection:  collection,
			EcIndexBits: uint32(newShardBits),
			DiskType:    string(types.HardDriveType),
		})
		ecNode.freeEcSlot -= len(shardIds)
	}

	return ecNode
}

func (ecNode *EcNode) deleteEcVolumeShards(vid needle.VolumeId, shardIds []uint32) *EcNode {

	if diskInfo, found := ecNode.info.DiskInfos[string(types.HardDriveType)]; found {
		for _, shardInfo := range diskInfo.EcShardInfos {
			if needle.VolumeId(shardInfo.Id) == vid {
				oldShardBits := erasure_coding.ShardBits(shardInfo.EcIndexBits)
				newShardBits := oldShardBits
				for _, shardId := range shardIds {
					newShardBits = newShardBits.RemoveShardId(erasure_coding.ShardId(shardId))
				}
				shardInfo.EcIndexBits = uint32(newShardBits)
				ecNode.freeEcSlot -= newShardBits.ShardIdCount() - oldShardBits.ShardIdCount()
			}
		}
	}

	return ecNode
}

func groupByCount(data []*EcNode, identifierFn func(*EcNode) (id string, count int)) map[string]int {
	countMap := make(map[string]int)
	for _, d := range data {
		id, count := identifierFn(d)
		countMap[id] += count
	}
	return countMap
}

func groupBy(data []*EcNode, identifierFn func(*EcNode) (id string)) map[string][]*EcNode {
	groupMap := make(map[string][]*EcNode)
	for _, d := range data {
		id := identifierFn(d)
		groupMap[id] = append(groupMap[id], d)
	}
	return groupMap
}
<|endoftext|>


<commit_before>package ot

type OpType int

const (
	OpRetain OpType = iota
	OpInsert
	OpDelete
)

type Op interface {
	Type() OpType
	Span() int
}

type RetainOp int

func (p RetainOp) Type() OpType {
	return OpRetain
}

func (p RetainOp) Span() int {
	return int(p)
}

func (p RetainOp) ComposeRetain(q RetainOp) (Op, Op, Op) {
	switch {
	case p > q:
		return q, p - q, Noop

	default:
		return p, Noop, q - p
	}
}

func (p RetainOp) ComposeDelete(q DeleteOp) (Op, Op, Op) {
	switch {
	case int(p) < int(q):
		return DeleteOp(p), Noop, q - DeleteOp(p)

	default:
		return q, p - RetainOp(q), Noop
	}
}

func (p RetainOp) TransformRetain(q RetainOp) (Op, Op, Op, Op) {
	switch {
	case p > q:
		return q, q, p - q, Noop

	default:
		return p, p, Noop, q - p
	}
}

func (p RetainOp) TransformDelete(q DeleteOp) (Op, Op, Op, Op) {
	switch {
	case int(p) > int(q):
		return Noop, q, p - RetainOp(q), Noop

	case int(p) < int(q):
		return Noop, DeleteOp(p), Noop, q - DeleteOp(p)

	default:
		return Noop, DeleteOp(p), Noop, Noop
	}
}

var Noop = RetainOp(0)

type InsertOp string

func (p InsertOp) Type() OpType {
	return OpInsert
}

func (p InsertOp) Span() int {
	return len(p)
}

func (p InsertOp) ComposeRetain(q RetainOp) (Op, Op, Op) {
	switch {
	case len(p) > int(q):
		return p[:int(q)], p[int(q):], Noop

	default:
		return p, Noop, q - RetainOp(len(p))
	}
}

func (p InsertOp) ComposeDelete(q DeleteOp) (Op, Op, Op) {
	switch {
	case len(p) > int(q):
		return Noop, p[int(q):], Noop

	case len(p) < int(q):
		return Noop, Noop, q - DeleteOp(len(p))

	default:
		return Noop, Noop, Noop
	}
}

type DeleteOp int

func (p DeleteOp) Type() OpType {
	return OpDelete
}

func (p DeleteOp) Span() int {
	return int(p)
}

func (p DeleteOp) TransformRetain(q RetainOp) (Op, Op, Op, Op) {
	switch {
	case int(p) > int(q):
		return DeleteOp(q), Noop, p - DeleteOp(q), Noop

	default:
		return p, Noop, Noop, q - RetainOp(p)
	}
}

func (p DeleteOp) TransformDelete(q DeleteOp) (Op, Op, Op, Op) {
	switch {
	case p > q:
		return Noop, Noop, p - q, Noop

	default:
		return Noop, Noop, Noop, q - p
	}
}

type RetainComposer interface {
	ComposeRetain(q RetainOp) (Op, Op, Op)
}

type DeleteComposer interface {
	ComposeDelete(q DeleteOp) (Op, Op, Op)
}

type RetainTransformer interface {
	TransformRetain(q RetainOp) (Op, Op, Op, Op)
}

type DeleteTransformer interface {
	TransformDelete(q DeleteOp) (Op, Op, Op, Op)
}
<commit_msg>Fix delete-delete transform bug<commit_after>package ot

type OpType int

const (
	OpRetain OpType = iota
	OpInsert
	OpDelete
)

type Op interface {
	Type() OpType
	Span() int
}

type RetainOp int

func (p RetainOp) Type() OpType {
	return OpRetain
}

func (p RetainOp) Span() int {
	return int(p)
}

func (p RetainOp) ComposeRetain(q RetainOp) (Op, Op, Op) {
	switch {
	case p > q:
		return q, p - q, Noop

	default:
		return p, Noop, q - p
	}
}

func (p RetainOp) ComposeDelete(q DeleteOp) (Op, Op, Op) {
	switch {
	case int(p) < int(q):
		return DeleteOp(p), Noop, q - DeleteOp(p)

	default:
		return q, p - RetainOp(q), Noop
	}
}

func (p RetainOp) TransformRetain(q RetainOp) (Op, Op, Op, Op) {
	switch {
	case p > q:
		return q, q, p - q, Noop

	default:
		return p, p, Noop, q - p
	}
}

func (p RetainOp) TransformDelete(q DeleteOp) (Op, Op, Op, Op) {
	switch {
	case int(p) > int(q):
		return Noop, q, p - RetainOp(q), Noop

	case int(p) < int(q):
		return Noop, DeleteOp(p), Noop, q - DeleteOp(p)

	default:
		return Noop, DeleteOp(p), Noop, Noop
	}
}

var Noop = RetainOp(0)

type InsertOp string

func (p InsertOp) Type() OpType {
	return OpInsert
}

func (p InsertOp) Span() int {
	return len(p)
}

func (p InsertOp) ComposeRetain(q RetainOp) (Op, Op, Op) {
	switch {
	case len(p) > int(q):
		return p[:int(q)], p[int(q):], Noop

	default:
		return p, Noop, q - RetainOp(len(p))
	}
}

func (p InsertOp) ComposeDelete(q DeleteOp) (Op, Op, Op) {
	switch {
	case len(p) > int(q):
		return Noop, p[int(q):], Noop

	case len(p) < int(q):
		return Noop, Noop, q - DeleteOp(len(p))

	default:
		return Noop, Noop, Noop
	}
}

type DeleteOp int

func (p DeleteOp) Type() OpType {
	return OpDelete
}

func (p DeleteOp) Span() int {
	return int(p)
}

func (p DeleteOp) TransformRetain(q RetainOp) (Op, Op, Op, Op) {
	switch {
	case int(p) > int(q):
		return DeleteOp(q), Noop, p - DeleteOp(q), Noop

	default:
		return p, Noop, Noop, q - RetainOp(p)
	}
}

func (p DeleteOp) TransformDelete(q DeleteOp) (Op, Op, Op, Op) {
	switch {
	case p > q:
		return Noop, Noop, p - q, Noop

	case p < q:
		return Noop, Noop, Noop, q - p

	default:
		return Noop, Noop, Noop, Noop
	}
}

type RetainComposer interface {
	ComposeRetain(q RetainOp) (Op, Op, Op)
}

type DeleteComposer interface {
	ComposeDelete(q DeleteOp) (Op, Op, Op)
}

type RetainTransformer interface {
	TransformRetain(q RetainOp) (Op, Op, Op, Op)
}

type DeleteTransformer interface {
	TransformDelete(q DeleteOp) (Op, Op, Op, Op)
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 The Syncthing Authors.
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
// more details.
//
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.

package versioner

import (
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/syncthing/syncthing/internal/osutil"
)

func init() {
	// Register the constructor for this type of versioner with the name "staggered"
	Factories["staggered"] = NewStaggered
}

type Interval struct {
	step int64
	end  int64
}

// The type holds our configuration
type Staggered struct {
	versionsPath  string
	cleanInterval int64
	folderPath    string
	interval      [4]Interval
	mutex         *sync.Mutex
}

// Rename versions with old version format
func (v Staggered) renameOld() {
	err := filepath.Walk(v.versionsPath, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if f.Mode().IsRegular() {
			versionUnix, err := strconv.ParseInt(strings.Replace(filepath.Ext(path), ".v", "", 1), 10, 0)
			if err == nil {
				l.Infoln("Renaming file", path, "from old to new version format")
				versiondate := time.Unix(versionUnix, 0)
				name := path[:len(path)-len(filepath.Ext(path))]
				err = osutil.Rename(path, taggedFilename(name, versiondate.Format(TimeFormat)))
				if err != nil {
					l.Infoln("Error renaming to new format", err)
				}
			}
		}
		return nil
	})
	if err != nil {
		l.Infoln("Versioner: error scanning versions dir", err)
		return
	}
}

// The constructor function takes a map of parameters and creates the type.
func NewStaggered(folderID, folderPath string, params map[string]string) Versioner {
	maxAge, err := strconv.ParseInt(params["maxAge"], 10, 0)
	if err != nil {
		maxAge = 31536000 // Default: ~1 year
	}
	cleanInterval, err := strconv.ParseInt(params["cleanInterval"], 10, 0)
	if err != nil {
		cleanInterval = 3600 // Default: clean once per hour
	}

	// Use custom path if set, otherwise .stversions in folderPath
	var versionsDir string
	if params["versionsPath"] == "" {
		if debug {
			l.Debugln("using default dir .stversions")
		}
		versionsDir = filepath.Join(folderPath, ".stversions")
	} else {
		if debug {
			l.Debugln("using dir", params["versionsPath"])
		}
		versionsDir = params["versionsPath"]
	}

	var mutex sync.Mutex
	s := Staggered{
		versionsPath:  versionsDir,
		cleanInterval: cleanInterval,
		folderPath:    folderPath,
		interval: [4]Interval{
			{30, 3600},       // first hour -> 30 sec between versions
			{3600, 86400},    // next day -> 1 h between versions
			{86400, 592000},  // next 30 days -> 1 day between versions
			{604800, maxAge}, // next year -> 1 week between versions
		},
		mutex: &mutex,
	}

	if debug {
		l.Debugf("instantiated %#v", s)
	}

	// Rename version with old version format
	s.renameOld()

	go func() {
		s.clean()
		for _ = range time.Tick(time.Duration(cleanInterval) * time.Second) {
			s.clean()
		}
	}()

	return s
}

func (v Staggered) clean() {
	if debug {
		l.Debugln("Versioner clean: Waiting for lock on", v.versionsPath)
	}
	v.mutex.Lock()
	defer v.mutex.Unlock()
	if debug {
		l.Debugln("Versioner clean: Cleaning", v.versionsPath)
	}

	_, err := os.Stat(v.versionsPath)
	if err != nil {
		if os.IsNotExist(err) {
			if debug {
				l.Debugln("creating versions dir", v.versionsPath)
			}
			os.MkdirAll(v.versionsPath, 0755)
			osutil.HideFile(v.versionsPath)
		} else {
			l.Warnln("Versioner: can't create versions dir", err)
		}
	}

	versionsPerFile := make(map[string][]string)
	filesPerDir := make(map[string]int)

	err = filepath.Walk(v.versionsPath, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if f.Mode().IsDir() && f.Mode()&os.ModeSymlink == 0 {
			filesPerDir[path] = 0
			if path != v.versionsPath {
				dir := filepath.Dir(path)
				filesPerDir[dir]++
			}
		} else {
			// Regular file, or possibly a symlink.

			extension := filenameTag(path)
			dir := filepath.Dir(path)
			name := path[:len(path)-len(extension)-1]

			filesPerDir[dir]++
			versionsPerFile[name] = append(versionsPerFile[name], path)
		}

		return nil
	})
	if err != nil {
		l.Warnln("Versioner: error scanning versions dir", err)
		return
	}

	for _, versionList := range versionsPerFile {
		// List from filepath.Walk is sorted
		v.expire(versionList)
	}

	for path, numFiles := range filesPerDir {
		if numFiles > 0 {
			continue
		}

		if path == v.versionsPath {
			if debug {
				l.Debugln("Cleaner: versions dir is empty, don't delete", path)
			}
			continue
		}

		if debug {
			l.Debugln("Cleaner: deleting empty directory", path)
		}
		err = os.Remove(path)
		if err != nil {
			l.Warnln("Versioner: can't remove directory", path, err)
		}
	}
	if debug {
		l.Debugln("Cleaner: Finished cleaning", v.versionsPath)
	}
}

func (v Staggered) expire(versions []string) {
	if debug {
		l.Debugln("Versioner: Expiring versions", versions)
	}
	var prevAge int64
	firstFile := true
	for _, file := range versions {
		fi, err := os.Stat(file)
		if err != nil {
			l.Warnln("versioner:", err)
			continue
		}

		if fi.IsDir() {
			l.Infof("non-file %q is named like a file version", file)
			continue
		}

		versionTime, err := time.Parse(TimeFormat, filenameTag(file))
		if err != nil {
			l.Infof("Versioner: file name %q is invalid: %v", file, err)
			continue
		}
		age := int64(time.Since(versionTime).Seconds())

		// If the file is older than the max age of the last interval, remove it
		if lastIntv := v.interval[len(v.interval)-1]; lastIntv.end > 0 && age > lastIntv.end {
			if debug {
				l.Debugln("Versioner: File over maximum age -> delete ", file)
			}
			err = os.Remove(file)
			if err != nil {
				l.Warnf("Versioner: can't remove %q: %v", file, err)
			}
			continue
		}

		// If it's the first (oldest) file in the list we can skip the interval checks
		if firstFile {
			prevAge = age
			firstFile = false
			continue
		}

		// Find the interval the file fits in
		var usedInterval Interval
		for _, usedInterval = range v.interval {
			if age < usedInterval.end {
				break
			}
		}

		if prevAge-age < usedInterval.step {
			if debug {
				l.Debugln("too many files in step -> delete", file)
			}
			err = os.Remove(file)
			if err != nil {
				l.Warnf("Versioner: can't remove %q: %v", file, err)
			}
			continue
		}

		prevAge = age
	}
}

// Move away the named file to a version archive. If this function returns
// nil, the named file does not exist any more (has been archived).
func (v Staggered) Archive(filePath string) error {
	if debug {
		l.Debugln("Waiting for lock on ", v.versionsPath)
	}
	v.mutex.Lock()
	defer v.mutex.Unlock()

	if _, err := os.Lstat(filePath); err != nil {
		if os.IsNotExist(err) {
			if debug {
				l.Debugln("not archiving nonexistent file", filePath)
			}
			return nil
		}
		return err
	}

	if _, err := os.Stat(v.versionsPath); err != nil {
		if os.IsNotExist(err) {
			if debug {
				l.Debugln("creating versions dir", v.versionsPath)
			}
			os.MkdirAll(v.versionsPath, 0755)
			osutil.HideFile(v.versionsPath)
		} else {
			return err
		}
	}

	if debug {
		l.Debugln("archiving", filePath)
	}

	file := filepath.Base(filePath)
	inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath))
	if err != nil {
		return err
	}

	dir := filepath.Join(v.versionsPath, inFolderPath)
	err = os.MkdirAll(dir, 0755)
	if err != nil && !os.IsExist(err) {
		return err
	}

	ver := taggedFilename(file, time.Now().Format(TimeFormat))
	dst := filepath.Join(dir, ver)
	if debug {
		l.Debugln("moving to", dst)
	}
	err = osutil.Rename(filePath, dst)
	if err != nil {
		return err
	}

	// Glob according to the new file~timestamp.ext pattern.
	newVersions, err := filepath.Glob(filepath.Join(dir, taggedFilename(file, TimeGlob)))
	if err != nil {
		l.Warnln("globbing:", err)
		return nil
	}

	// Also according to the old file.ext~timestamp pattern.
	oldVersions, err := filepath.Glob(filepath.Join(dir, file+"~"+TimeGlob))
	if err != nil {
		l.Warnln("globbing:", err)
		return nil
	}

	// Use all the found filenames.
	versions := append(oldVersions, newVersions...)
	v.expire(uniqueSortedStrings(versions))

	return nil
}
<commit_msg>Silence versioner warnings for unmatched files (fixes #1117)<commit_after>// Copyright (C) 2014 The Syncthing Authors.
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
// more details.
//
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.

package versioner

import (
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/syncthing/syncthing/internal/osutil"
)

func init() {
	// Register the constructor for this type of versioner with the name "staggered"
	Factories["staggered"] = NewStaggered
}

type Interval struct {
	step int64
	end  int64
}

// The type holds our configuration
type Staggered struct {
	versionsPath  string
	cleanInterval int64
	folderPath    string
	interval      [4]Interval
	mutex         *sync.Mutex
}

// Rename versions with old version format
func (v Staggered) renameOld() {
	err := filepath.Walk(v.versionsPath, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if f.Mode().IsRegular() {
			versionUnix, err := strconv.ParseInt(strings.Replace(filepath.Ext(path), ".v", "", 1), 10, 0)
			if err == nil {
				l.Infoln("Renaming file", path, "from old to new version format")
				versiondate := time.Unix(versionUnix, 0)
				name := path[:len(path)-len(filepath.Ext(path))]
				err = osutil.Rename(path, taggedFilename(name, versiondate.Format(TimeFormat)))
				if err != nil {
					l.Infoln("Error renaming to new format", err)
				}
			}
		}
		return nil
	})
	if err != nil {
		l.Infoln("Versioner: error scanning versions dir", err)
		return
	}
}

// The constructor function takes a map of parameters and creates the type.
func NewStaggered(folderID, folderPath string, params map[string]string) Versioner {
	maxAge, err := strconv.ParseInt(params["maxAge"], 10, 0)
	if err != nil {
		maxAge = 31536000 // Default: ~1 year
	}
	cleanInterval, err := strconv.ParseInt(params["cleanInterval"], 10, 0)
	if err != nil {
		cleanInterval = 3600 // Default: clean once per hour
	}

	// Use custom path if set, otherwise .stversions in folderPath
	var versionsDir string
	if params["versionsPath"] == "" {
		if debug {
			l.Debugln("using default dir .stversions")
		}
		versionsDir = filepath.Join(folderPath, ".stversions")
	} else {
		if debug {
			l.Debugln("using dir", params["versionsPath"])
		}
		versionsDir = params["versionsPath"]
	}

	var mutex sync.Mutex
	s := Staggered{
		versionsPath:  versionsDir,
		cleanInterval: cleanInterval,
		folderPath:    folderPath,
		interval: [4]Interval{
			{30, 3600},       // first hour -> 30 sec between versions
			{3600, 86400},    // next day -> 1 h between versions
			{86400, 592000},  // next 30 days -> 1 day between versions
			{604800, maxAge}, // next year -> 1 week between versions
		},
		mutex: &mutex,
	}

	if debug {
		l.Debugf("instantiated %#v", s)
	}

	// Rename version with old version format
	s.renameOld()

	go func() {
		s.clean()
		for _ = range time.Tick(time.Duration(cleanInterval) * time.Second) {
			s.clean()
		}
	}()

	return s
}

func (v Staggered) clean() {
	if debug {
		l.Debugln("Versioner clean: Waiting for lock on", v.versionsPath)
	}
	v.mutex.Lock()
	defer v.mutex.Unlock()
	if debug {
		l.Debugln("Versioner clean: Cleaning", v.versionsPath)
	}

	_, err := os.Stat(v.versionsPath)
	if err != nil {
		if os.IsNotExist(err) {
			if debug {
				l.Debugln("creating versions dir", v.versionsPath)
			}
			os.MkdirAll(v.versionsPath, 0755)
			osutil.HideFile(v.versionsPath)
		} else {
			l.Warnln("Versioner: can't create versions dir", err)
		}
	}

	versionsPerFile := make(map[string][]string)
	filesPerDir := make(map[string]int)

	err = filepath.Walk(v.versionsPath, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if f.Mode().IsDir() && f.Mode()&os.ModeSymlink == 0 {
			filesPerDir[path] = 0
			if path != v.versionsPath {
				dir := filepath.Dir(path)
				filesPerDir[dir]++
			}
		} else {
			// Regular file, or possibly a symlink.

			extension := filenameTag(path)
			dir := filepath.Dir(path)
			name := path[:len(path)-len(extension)-1]

			filesPerDir[dir]++
			versionsPerFile[name] = append(versionsPerFile[name], path)
		}

		return nil
	})
	if err != nil {
		l.Warnln("Versioner: error scanning versions dir", err)
		return
	}

	for _, versionList := range versionsPerFile {
		// List from filepath.Walk is sorted
		v.expire(versionList)
	}

	for path, numFiles := range filesPerDir {
		if numFiles > 0 {
			continue
		}

		if path == v.versionsPath {
			if debug {
				l.Debugln("Cleaner: versions dir is empty, don't delete", path)
			}
			continue
		}

		if debug {
			l.Debugln("Cleaner: deleting empty directory", path)
		}
		err = os.Remove(path)
		if err != nil {
			l.Warnln("Versioner: can't remove directory", path, err)
		}
	}
	if debug {
		l.Debugln("Cleaner: Finished cleaning", v.versionsPath)
	}
}

func (v Staggered) expire(versions []string) {
	if debug {
		l.Debugln("Versioner: Expiring versions", versions)
	}
	var prevAge int64
	firstFile := true
	for _, file := range versions {
		fi, err := os.Stat(file)
		if err != nil {
			l.Warnln("versioner:", err)
			continue
		}

		if fi.IsDir() {
			l.Infof("non-file %q is named like a file version", file)
			continue
		}

		versionTime, err := time.Parse(TimeFormat, filenameTag(file))
		if err != nil {
			if debug {
				l.Debugf("Versioner: file name %q is invalid: %v", file, err)
			}
			continue
		}
		age := int64(time.Since(versionTime).Seconds())

		// If the file is older than the max age of the last interval, remove it
		if lastIntv := v.interval[len(v.interval)-1]; lastIntv.end > 0 && age > lastIntv.end {
			if debug {
				l.Debugln("Versioner: File over maximum age -> delete ", file)
			}
			err = os.Remove(file)
			if err != nil {
				l.Warnf("Versioner: can't remove %q: %v", file, err)
			}
			continue
		}

		// If it's the first (oldest) file in the list we can skip the interval checks
		if firstFile {
			prevAge = age
			firstFile = false
			continue
		}

		// Find the interval the file fits in
		var usedInterval Interval
		for _, usedInterval = range v.interval {
			if age < usedInterval.end {
				break
			}
		}

		if prevAge-age < usedInterval.step {
			if debug {
				l.Debugln("too many files in step -> delete", file)
			}
			err = os.Remove(file)
			if err != nil {
				l.Warnf("Versioner: can't remove %q: %v", file, err)
			}
			continue
		}

		prevAge = age
	}
}

// Move away the named file to a version archive. If this function returns
// nil, the named file does not exist any more (has been archived).
func (v Staggered) Archive(filePath string) error {
	if debug {
		l.Debugln("Waiting for lock on ", v.versionsPath)
	}
	v.mutex.Lock()
	defer v.mutex.Unlock()

	if _, err := os.Lstat(filePath); err != nil {
		if os.IsNotExist(err) {
			if debug {
				l.Debugln("not archiving nonexistent file", filePath)
			}
			return nil
		}
		return err
	}

	if _, err := os.Stat(v.versionsPath); err != nil {
		if os.IsNotExist(err) {
			if debug {
				l.Debugln("creating versions dir", v.versionsPath)
			}
			os.MkdirAll(v.versionsPath, 0755)
			osutil.HideFile(v.versionsPath)
		} else {
			return err
		}
	}

	if debug {
		l.Debugln("archiving", filePath)
	}

	file := filepath.Base(filePath)
	inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath))
	if err != nil {
		return err
	}

	dir := filepath.Join(v.versionsPath, inFolderPath)
	err = os.MkdirAll(dir, 0755)
	if err != nil && !os.IsExist(err) {
		return err
	}

	ver := taggedFilename(file, time.Now().Format(TimeFormat))
	dst := filepath.Join(dir, ver)
	if debug {
		l.Debugln("moving to", dst)
	}
	err = osutil.Rename(filePath, dst)
	if err != nil {
		return err
	}

	// Glob according to the new file~timestamp.ext pattern.
	newVersions, err := filepath.Glob(filepath.Join(dir, taggedFilename(file, TimeGlob)))
	if err != nil {
		l.Warnln("globbing:", err)
		return nil
	}

	// Also according to the old file.ext~timestamp pattern.
	oldVersions, err := filepath.Glob(filepath.Join(dir, file+"~"+TimeGlob))
	if err != nil {
		l.Warnln("globbing:", err)
		return nil
	}

	// Use all the found filenames.
	versions := append(oldVersions, newVersions...)
	v.expire(uniqueSortedStrings(versions))

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package watcher

import (
	"context"
	"expvar"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/golang/glog"
	"github.com/pkg/errors"
)

var (
	errorCount = expvar.NewInt("log_watcher_errors_total")
)

type watch struct {
	ps []Processor
	fi os.FileInfo
}

// hasChanged indicates that a FileInfo has changed.
// http://apenwarr.ca/log/20181113 suggests that comparing mtime is
// insufficient for sub-second resolution on many platforms, and we can do
// better by comparing a few fields in the FileInfo.  This set of tests is less
// than the ones suggested in the blog post, but seem sufficient for making
// tests (notably, sub-millisecond accuracy) pass quickly.  mtime-only diff has
// caused race conditions in test and likely caused strange behaviour in
// production environments.
func hasChanged(a, b os.FileInfo) bool {
	if a == nil || b == nil {
		glog.V(2).Info("One or both FileInfos are nil")
		return true
	}
	if a.ModTime() != b.ModTime() {
		glog.V(2).Info("modtimes differ")
		return true
	}
	if a.Size() != b.Size() {
		glog.V(2).Info("sizes differ")
		return true
	}
	if a.Mode() != b.Mode() {
		glog.V(2).Info("modes differ")
		return true
	}
	return false
}

// LogWatcher implements a Watcher for watching real filesystems.
type LogWatcher struct {
	watcher    *fsnotify.Watcher
	pollTicker *time.Ticker

	watchedMu sync.RWMutex // protects `watched'
	watched   map[string]*watch

	stopTicks chan struct{} // Channel to notify ticker to stop.

	ticksDone  chan struct{} // Channel to notify when the ticks handler is done.
	eventsDone chan struct{} // Channel to notify when the events handler is done.

	pollMu sync.Mutex // protects `Poll()`

	closeOnce sync.Once
}

// NewLogWatcher returns a new LogWatcher, or returns an error.
func NewLogWatcher(pollInterval time.Duration, enableFsnotify bool) (*LogWatcher, error) {
	var f *fsnotify.Watcher
	if enableFsnotify {
		var err error
		f, err = fsnotify.NewWatcher()
		if err != nil {
			glog.Warning(err)
		}
	}
	if f == nil && pollInterval == 0 {
		glog.Infof("fsnotify disabled and no poll interval specified; defaulting to 250ms poll")
		pollInterval = time.Millisecond * 250
	}
	w := &LogWatcher{
		watcher: f,
		watched: make(map[string]*watch),
	}
	if pollInterval > 0 {
		w.pollTicker = time.NewTicker(pollInterval)
		w.stopTicks = make(chan struct{})
		w.ticksDone = make(chan struct{})
		go w.runTicks()
		glog.V(2).Infof("started ticker with %s interval", pollInterval)
	}
	if f != nil {
		w.eventsDone = make(chan struct{})
		go w.runEvents()
	}
	return w, nil
}

func (w *LogWatcher) sendEvent(e Event) {
	w.watchedMu.RLock()
	watch, ok := w.watched[e.Pathname]
	w.watchedMu.RUnlock()
	if !ok {
		d := filepath.Dir(e.Pathname)
		w.watchedMu.RLock()
		watch, ok = w.watched[d]
		w.watchedMu.RUnlock()
		if !ok {
			glog.V(2).Infof("No watch for path %q", e.Pathname)
			return
		}
	}
	w.sendWatchedEvent(watch, e)
}

// Send an event to a watch; all locks assumed to be held.
func (w *LogWatcher) sendWatchedEvent(watch *watch, e Event) {
	for _, p := range watch.ps {
		p.ProcessFileEvent(context.TODO(), e)
	}
}

func (w *LogWatcher) runTicks() {
	defer close(w.ticksDone)

	if w.pollTicker == nil {
		return
	}

	for {
		select {
		case <-w.pollTicker.C:
			w.Poll()
		case <-w.stopTicks:
			w.pollTicker.Stop()
			return
		}
	}
}

// Poll all watched objects for updates, dispatching events if required.
func (w *LogWatcher) Poll() {
	w.pollMu.Lock()
	defer w.pollMu.Unlock()
	glog.V(2).Info("Polling watched files.")
	w.watchedMu.RLock()
	for n, watch := range w.watched {
		w.watchedMu.RUnlock()
		w.pollWatchedPath(n, watch)
		w.watchedMu.RLock()
	}
	w.watchedMu.RUnlock()
}

// pollWatchedPathLocked polls an already-watched path for updates.
func (w *LogWatcher) pollWatchedPath(pathname string, watched *watch) {
	glog.V(2).Infof("Stat %q", pathname)
	fi, err := os.Stat(pathname)
	if err != nil {
		glog.V(1).Info(err)
		return
	}

	// fsnotify does not send update events for the directory itself.
	if fi.IsDir() {
		w.pollDirectory(watched, pathname)
	} else if hasChanged(fi, watched.fi) {
		glog.V(2).Infof("sending update for %s", pathname)
		w.sendWatchedEvent(watched, Event{Update, pathname})
	}

	w.watchedMu.Lock()
	if _, ok := w.watched[pathname]; ok {
		w.watched[pathname].fi = fi
	}
	w.watchedMu.Unlock()
}

func (w *LogWatcher) pollDirectory(parentWatch *watch, pathname string) {
	matches, err := filepath.Glob(path.Join(pathname, "*"))
	if err != nil {
		glog.V(1).Info(err)
		return
	}
	// TODO(jaq): how do we avoid duplicate notifies for things that are already in the watch list?
	for _, match := range matches {
		fi, err := os.Stat(match)
		if err != nil {
			glog.V(1).Info(err)
			continue
		}

		w.watchedMu.RLock()
		watched, ok := w.watched[match]
		w.watchedMu.RUnlock()
		switch {
		case !ok:
			// The object has no watch object so it must be new, but we can't
			// decide that -- wait for the Tailer to match pattern and instruct
			// us to Observe it directly.
			glog.V(2).Infof("sending create for %s", match)
			w.sendWatchedEvent(parentWatch, Event{Create, match})
		case hasChanged(fi, watched.fi):
			glog.V(2).Infof("sending update for %s", match)
			w.sendWatchedEvent(watched, Event{Update, match})
			w.watchedMu.Lock()
			w.watched[match].fi = fi
			w.watchedMu.Unlock()
		default:
			glog.V(2).Infof("No change for %s, no send", match)
		}
		if fi.IsDir() {
			w.pollDirectory(parentWatch, match)
		}
	}
}

// runEvents assumes that w.watcher is not nil
func (w *LogWatcher) runEvents() {
	defer close(w.eventsDone)

	// Suck out errors and dump them to the error log.
	go func() {
		for err := range w.watcher.Errors {
			errorCount.Add(1)
			glog.Errorf("fsnotify error: %s\n", err)
		}
	}()

	for e := range w.watcher.Events {
		glog.V(2).Infof("watcher event %v", e)
		switch {
		case e.Op&fsnotify.Create == fsnotify.Create:
			w.sendEvent(Event{Create, e.Name})
		case e.Op&fsnotify.Write == fsnotify.Write,
			e.Op&fsnotify.Chmod == fsnotify.Chmod:
			w.sendEvent(Event{Update, e.Name})
		case e.Op&fsnotify.Remove == fsnotify.Remove:
			w.sendEvent(Event{Delete, e.Name})
		case e.Op&fsnotify.Rename == fsnotify.Rename:
			// Rename is only issued on the original file path; the new name receives a Create event
			w.sendEvent(Event{Delete, e.Name})
		default:
			panic(fmt.Sprintf("unknown op type %v", e.Op))
		}
	}
	glog.Infof("Shutting down log watcher.")
}

// Close shuts down the LogWatcher.  It is safe to call this from multiple clients.
func (w *LogWatcher) Close() (err error) {
	w.closeOnce.Do(func() {
		if w.watcher != nil {
			err = w.watcher.Close()
			<-w.eventsDone
		}
		if w.pollTicker != nil {
			close(w.stopTicks)
			<-w.ticksDone
		}
	})
	return nil
}

// Observe adds a path to the list of watched items.
// If this path has a new event, then the processor being registered will be sent the event.
func (w *LogWatcher) Observe(path string, processor Processor) error {
	absPath, err := w.addWatch(path)
	if err != nil {
		return err
	}
	w.watchedMu.Lock()
	defer w.watchedMu.Unlock()
	watched, ok := w.watched[absPath]
	if !ok {
		fi, err := os.Stat(absPath)
		if err != nil {
			glog.V(1).Info(err)
		}
		w.watched[absPath] = &watch{ps: []Processor{processor}, fi: fi}
		glog.Infof("No abspath in watched list, added new one for %s", absPath)
		return nil
	}
	for _, p := range watched.ps {
		if p == processor {
			glog.Infof("Found this processor in watched list")
			return nil
		}
	}
	watched.ps = append(watched.ps, processor)
	glog.Infof("appended this processor")
	return nil
}

func (w *LogWatcher) addWatch(path string) (string, error) {
	absPath, err := filepath.Abs(path)
	if err != nil {
		return "", errors.Wrapf(err, "Failed to lookup absolutepath of %q", path)
	}
	glog.V(2).Infof("Adding a watch on resolved path %q", absPath)
	if w.watcher != nil {
		err = w.watcher.Add(absPath)
		if err != nil {
			if os.IsPermission(err) {
				glog.V(2).Infof("Skipping permission denied error on adding a watch.")
			} else {
				return "", errors.Wrapf(err, "Failed to create a new watch on %q", absPath)
			}
		}
	}
	return absPath, nil
}

// IsWatching indicates if the path is being watched. It includes both
// filenames and directories.
func (w *LogWatcher) IsWatching(path string) bool {
	absPath, err := filepath.Abs(path)
	if err != nil {
		glog.V(2).Infof("Couldn't resolve path %q: %s", absPath, err)
		return false
	}
	glog.V(2).Infof("Resolved path for lookup %q", absPath)
	w.watchedMu.RLock()
	_, ok := w.watched[absPath]
	w.watchedMu.RUnlock()
	return ok
}

func (w *LogWatcher) Unobserve(path string, processor Processor) error {
	w.watchedMu.Lock()
	defer w.watchedMu.Unlock()
	_, ok := w.watched[path]
	if !ok {
		return nil
	}

	for i, p := range w.watched[path].ps {
		if p == processor {
			w.watched[path].ps = append(w.watched[path].ps[0:i], w.watched[path].ps[i+1:]...)
			break
		}
	}
	if len(w.watched[path].ps) == 0 {
		delete(w.watched, path)
	}
	if w.watcher != nil {
		return w.watcher.Remove(path)
	}
	return nil
}
<commit_msg>Move os.Stat until after we know we watch the file.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package watcher

import (
	"context"
	"expvar"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/golang/glog"
	"github.com/pkg/errors"
)

var (
	errorCount = expvar.NewInt("log_watcher_errors_total")
)

type watch struct {
	ps []Processor
	fi os.FileInfo
}

// hasChanged indicates that a FileInfo has changed.
// http://apenwarr.ca/log/20181113 suggests that comparing mtime is
// insufficient for sub-second resolution on many platforms, and we can do
// better by comparing a few fields in the FileInfo.  This set of tests is less
// than the ones suggested in the blog post, but seem sufficient for making
// tests (notably, sub-millisecond accuracy) pass quickly.  mtime-only diff has
// caused race conditions in test and likely caused strange behaviour in
// production environments.
func hasChanged(a, b os.FileInfo) bool {
	if a == nil || b == nil {
		glog.V(2).Info("One or both FileInfos are nil")
		return true
	}
	if a.ModTime() != b.ModTime() {
		glog.V(2).Info("modtimes differ")
		return true
	}
	if a.Size() != b.Size() {
		glog.V(2).Info("sizes differ")
		return true
	}
	if a.Mode() != b.Mode() {
		glog.V(2).Info("modes differ")
		return true
	}
	return false
}

// LogWatcher implements a Watcher for watching real filesystems.
type LogWatcher struct {
	watcher    *fsnotify.Watcher
	pollTicker *time.Ticker

	watchedMu sync.RWMutex // protects `watched'
	watched   map[string]*watch

	stopTicks chan struct{} // Channel to notify ticker to stop.

	ticksDone  chan struct{} // Channel to notify when the ticks handler is done.
	eventsDone chan struct{} // Channel to notify when the events handler is done.

	pollMu sync.Mutex // protects `Poll()`

	closeOnce sync.Once
}

// NewLogWatcher returns a new LogWatcher, or returns an error.
func NewLogWatcher(pollInterval time.Duration, enableFsnotify bool) (*LogWatcher, error) {
	var f *fsnotify.Watcher
	if enableFsnotify {
		var err error
		f, err = fsnotify.NewWatcher()
		if err != nil {
			glog.Warning(err)
		}
	}
	if f == nil && pollInterval == 0 {
		glog.Infof("fsnotify disabled and no poll interval specified; defaulting to 250ms poll")
		pollInterval = time.Millisecond * 250
	}
	w := &LogWatcher{
		watcher: f,
		watched: make(map[string]*watch),
	}
	if pollInterval > 0 {
		w.pollTicker = time.NewTicker(pollInterval)
		w.stopTicks = make(chan struct{})
		w.ticksDone = make(chan struct{})
		go w.runTicks()
		glog.V(2).Infof("started ticker with %s interval", pollInterval)
	}
	if f != nil {
		w.eventsDone = make(chan struct{})
		go w.runEvents()
	}
	return w, nil
}

func (w *LogWatcher) sendEvent(e Event) {
	w.watchedMu.RLock()
	watch, ok := w.watched[e.Pathname]
	w.watchedMu.RUnlock()
	if !ok {
		d := filepath.Dir(e.Pathname)
		w.watchedMu.RLock()
		watch, ok = w.watched[d]
		w.watchedMu.RUnlock()
		if !ok {
			glog.V(2).Infof("No watch for path %q", e.Pathname)
			return
		}
	}
	w.sendWatchedEvent(watch, e)
}

// Send an event to a watch; all locks assumed to be held.
func (w *LogWatcher) sendWatchedEvent(watch *watch, e Event) {
	for _, p := range watch.ps {
		p.ProcessFileEvent(context.TODO(), e)
	}
}

func (w *LogWatcher) runTicks() {
	defer close(w.ticksDone)

	if w.pollTicker == nil {
		return
	}

	for {
		select {
		case <-w.pollTicker.C:
			w.Poll()
		case <-w.stopTicks:
			w.pollTicker.Stop()
			return
		}
	}
}

// Poll all watched objects for updates, dispatching events if required.
func (w *LogWatcher) Poll() {
	w.pollMu.Lock()
	defer w.pollMu.Unlock()
	glog.V(2).Info("Polling watched files.")
	w.watchedMu.RLock()
	for n, watch := range w.watched {
		w.watchedMu.RUnlock()
		w.pollWatchedPath(n, watch)
		w.watchedMu.RLock()
	}
	w.watchedMu.RUnlock()
}

// pollWatchedPathLocked polls an already-watched path for updates.
func (w *LogWatcher) pollWatchedPath(pathname string, watched *watch) {
	glog.V(2).Infof("Stat %q", pathname)
	fi, err := os.Stat(pathname)
	if err != nil {
		glog.V(1).Info(err)
		return
	}

	// fsnotify does not send update events for the directory itself.
	if fi.IsDir() {
		w.pollDirectory(watched, pathname)
	} else if hasChanged(fi, watched.fi) {
		glog.V(2).Infof("sending update for %s", pathname)
		w.sendWatchedEvent(watched, Event{Update, pathname})
	}

	w.watchedMu.Lock()
	if _, ok := w.watched[pathname]; ok {
		w.watched[pathname].fi = fi
	}
	w.watchedMu.Unlock()
}

func (w *LogWatcher) pollDirectory(parentWatch *watch, pathname string) {
	matches, err := filepath.Glob(path.Join(pathname, "*"))
	if err != nil {
		glog.V(1).Info(err)
		return
	}
	// TODO(jaq): how do we avoid duplicate notifies for things that are already in the watch list?
	for _, match := range matches {
		w.watchedMu.RLock()
		watched, ok := w.watched[match]
		w.watchedMu.RUnlock()
		if !ok {
			// The object has no watch object so it must be new, but we can't
			// decide that -- wait for the Tailer to match pattern and instruct
			// us to Observe it directly.
			glog.V(2).Infof("sending create for %s", match)
			w.sendWatchedEvent(parentWatch, Event{Create, match})
		}
		fi, err := os.Stat(match)
		if err != nil {
			glog.V(1).Info(err)
			continue
		}
		if ok && hasChanged(fi, watched.fi) {
			glog.V(2).Infof("sending update for %s", match)
			w.sendWatchedEvent(watched, Event{Update, match})
			w.watchedMu.Lock()
			w.watched[match].fi = fi
			w.watchedMu.Unlock()
		} else {
			glog.V(2).Infof("No change for %s, no send", match)
		}
		if fi.IsDir() {
			w.pollDirectory(parentWatch, match)
		}
	}
}

// runEvents assumes that w.watcher is not nil
func (w *LogWatcher) runEvents() {
	defer close(w.eventsDone)

	// Suck out errors and dump them to the error log.
	go func() {
		for err := range w.watcher.Errors {
			errorCount.Add(1)
			glog.Errorf("fsnotify error: %s\n", err)
		}
	}()

	for e := range w.watcher.Events {
		glog.V(2).Infof("watcher event %v", e)
		switch {
		case e.Op&fsnotify.Create == fsnotify.Create:
			w.sendEvent(Event{Create, e.Name})
		case e.Op&fsnotify.Write == fsnotify.Write,
			e.Op&fsnotify.Chmod == fsnotify.Chmod:
			w.sendEvent(Event{Update, e.Name})
		case e.Op&fsnotify.Remove == fsnotify.Remove:
			w.sendEvent(Event{Delete, e.Name})
		case e.Op&fsnotify.Rename == fsnotify.Rename:
			// Rename is only issued on the original file path; the new name receives a Create event
			w.sendEvent(Event{Delete, e.Name})
		default:
			panic(fmt.Sprintf("unknown op type %v", e.Op))
		}
	}
	glog.Infof("Shutting down log watcher.")
}

// Close shuts down the LogWatcher.  It is safe to call this from multiple clients.
func (w *LogWatcher) Close() (err error) {
	w.closeOnce.Do(func() {
		if w.watcher != nil {
			err = w.watcher.Close()
			<-w.eventsDone
		}
		if w.pollTicker != nil {
			close(w.stopTicks)
			<-w.ticksDone
		}
	})
	return nil
}

// Observe adds a path to the list of watched items.
// If this path has a new event, then the processor being registered will be sent the event.
func (w *LogWatcher) Observe(path string, processor Processor) error {
	absPath, err := w.addWatch(path)
	if err != nil {
		return err
	}
	w.watchedMu.Lock()
	defer w.watchedMu.Unlock()
	watched, ok := w.watched[absPath]
	if !ok {
		fi, err := os.Stat(absPath)
		if err != nil {
			glog.V(1).Info(err)
		}
		w.watched[absPath] = &watch{ps: []Processor{processor}, fi: fi}
		glog.Infof("No abspath in watched list, added new one for %s", absPath)
		return nil
	}
	for _, p := range watched.ps {
		if p == processor {
			glog.Infof("Found this processor in watched list")
			return nil
		}
	}
	watched.ps = append(watched.ps, processor)
	glog.Infof("appended this processor")
	return nil
}

func (w *LogWatcher) addWatch(path string) (string, error) {
	absPath, err := filepath.Abs(path)
	if err != nil {
		return "", errors.Wrapf(err, "Failed to lookup absolutepath of %q", path)
	}
	glog.V(2).Infof("Adding a watch on resolved path %q", absPath)
	if w.watcher != nil {
		err = w.watcher.Add(absPath)
		if err != nil {
			if os.IsPermission(err) {
				glog.V(2).Infof("Skipping permission denied error on adding a watch.")
			} else {
				return "", errors.Wrapf(err, "Failed to create a new watch on %q", absPath)
			}
		}
	}
	return absPath, nil
}

// IsWatching indicates if the path is being watched. It includes both
// filenames and directories.
func (w *LogWatcher) IsWatching(path string) bool {
	absPath, err := filepath.Abs(path)
	if err != nil {
		glog.V(2).Infof("Couldn't resolve path %q: %s", absPath, err)
		return false
	}
	glog.V(2).Infof("Resolved path for lookup %q", absPath)
	w.watchedMu.RLock()
	_, ok := w.watched[absPath]
	w.watchedMu.RUnlock()
	return ok
}

func (w *LogWatcher) Unobserve(path string, processor Processor) error {
	w.watchedMu.Lock()
	defer w.watchedMu.Unlock()
	_, ok := w.watched[path]
	if !ok {
		return nil
	}

	for i, p := range w.watched[path].ps {
		if p == processor {
			w.watched[path].ps = append(w.watched[path].ps[0:i], w.watched[path].ps[i+1:]...)
			break
		}
	}
	if len(w.watched[path].ps) == 0 {
		delete(w.watched, path)
	}
	if w.watcher != nil {
		return w.watcher.Remove(path)
	}
	return nil
}
<|endoftext|>


<commit_before>package node

import (
  "github.com/nictuku/dht"
  "net"
  "crypto/rsa"
  "crypto/x509"
  "sync"
  "fmt"
  "io"
  "encoding/binary"
  "errors"
  "golang.org/x/crypto/nacl/box"
  "crypto/rand"
  "crypto/sha1"
  "bytes"
  "encoding/hex"
  "time"

  "github.com/danoctavian/bluntly/stream"
)

/* NODE */
type Node struct {
  dht *DHT
  config *Config
}

/* CONFIG */
type FileConfig struct {
  dhtPort int
  ownKeyFile string
  id string
  configRoot string
  holePunch HolePunchConf
}

type Config struct {
  dhtPort int
  ownKey rsa.PrivateKey
  id string
  configRoot string
  holePunch HolePunchConf
  contactList *ContactList
}

type HolePunchConf struct {
	recvPort int
}

type ContactList struct {
  contacts *map[rsa.PublicKey]string
  mut *sync.Mutex
}

func NewNode(conf *Config) (node *Node, err error) {
  // setup the DHT
  dhtConf := dht.DefaultConfig
  dhtConf.Port = conf.dhtPort
  dhtClient, err := dht.New(dhtConf)
  if err != nil { return }
  dht := newDHT(dhtClient)

  go dht.Run()

  node.dht = dht

	return node, nil
}

/* CLIENT */
func (n *Node) Dial(peerPubKey *rsa.PublicKey) (conn Conn, err error) {

}

/* LISTENER */
type Listener struct {
  connChan chan *Conn
}

func (n *Node) Listen(port int) (listener *Listener, err error) {
  connChan := make(chan *Conn)
  listener = &Listener{connChan: connChan}
  // setup TCP listener
  tcpListener, err := net.Listen("tcp", fmt.Sprintf(":"))
  if err != nil {return}

  // loop accepting TCP connections
  go func() {
    for {
      tcpConn, tcpErr := tcpListener.Accept()
      if tcpErr != nil {
        Log(LOG_ERROR, "%s", tcpErr)
      }

      go func() {
        conn, handshakeError := handleClientConn(tcpConn, &n.config.ownKey, n.config.contactList)
        if err != nil {
          Log(LOG_INFO,
              "handling client connection from address %s %s",
              tcpConn.RemoteAddr().String(), handshakeError)
        } else {
          connChan <- conn
        }
      }()
    }
  }()

  // announce yourself to the DHT
  go n.announceSelf()

  return
}

func (n *Node) announceSelf() error {
  infoHash, err := pubKeyToInfoHash(&n.config.ownKey.PublicKey)
  if (err != nil) {return err}

  for {
    // the peers request is done only for the purpose of announcing
    n.dht.PeersRequest(infoHash, true)
    time.Sleep(10 * time.Second)
  }
}

func pubKeyToInfoHash(pub *rsa.PublicKey) (string, error) {
  pubKeyBytes, err := x509.MarshalPKIXPublicKey(pub)
  if err != nil { return "", err}
  sha1Bytes := sha1.Sum(pubKeyBytes)
  return hex.EncodeToString(sha1Bytes[:]), nil
}

func (l *Listener) Accept() (c net.Conn, err error) {
  conn := <- l.connChan
  return conn, nil
}

func (l *Listener) Close() error {
  return nil
}

func Addr() net.Addr {
  return nil
}

func handleClientConn(rawConn net.Conn,
                      ownKey *rsa.PrivateKey,
                      contacts *ContactList) (conn *Conn, err error) {

  var handshakeLen int64
  err = binary.Read(rawConn, binary.BigEndian, &handshakeLen)
  if (err != nil) { return }

  ciphertext := make([]byte, handshakeLen)
  _, err = io.ReadFull(rawConn, ciphertext)
  if (err != nil) { return }

  plain, err := rsa.DecryptPKCS1v15(nil, ownKey, ciphertext)
  if (err != nil) { return }

  connReq := ConnRequest{}
  err = connReq.UnmarshalBinary(plain)
  if (err != nil) { return }

  _, privKey, err := box.GenerateKey(rand.Reader)
  if (err != nil) { return }

  var sharedKey [sessionKeyLen]byte
  box.Precompute(&sharedKey, connReq.sessionKey, privKey)

  return &Conn{Conn: rawConn,
              sharedKey: &sharedKey,
              // TODO: circular buffer capacity may cause
              // streaming to fail. to avoid,
              // put a cap on the size of the encrypted chunks
              readBuf: stream.NewCircularBuf(2048),
              readerBufMutex: &sync.Mutex{}}, nil
}

const sessionKeyLen = 32

/* connection request */
type ConnRequest struct {
  peerPub *rsa.PublicKey
  sessionKey *[sessionKeyLen]byte
}

type ConnResponse struct {

}

func (r *ConnRequest) MarshalBinary() (data []byte, err error) {
  pubKeyBytes, err := x509.MarshalPKIXPublicKey(r.peerPub)
  if (err != nil) { return }

  return append((*r.sessionKey)[:], pubKeyBytes...), nil
}

func (r *ConnRequest) UnmarshalBinary(data []byte) (err error) {
  copiedBytes := copy(r.sessionKey[:], data[:sessionKeyLen])
  if (copiedBytes < sessionKeyLen) {
    return errors.New("session key too short.")
  }

  someKey, err := x509.ParsePKIXPublicKey(data)
  pubKey := someKey.(*rsa.PublicKey)
  if (err != nil) { return }
  r.peerPub = pubKey

  return
}

/* CONNECTION */
type Conn struct {
  net.Conn // underlying network connection
  sharedKey *[sessionKeyLen]byte

  // a buffer that is used to store in excess data, not yet read
  // but already decrypted
  readBuf *stream.CircularBuf
  readerBufMutex *sync.Mutex
}

func (c *Conn) Read(b []byte) (readBytes int, err error) {
  c.readerBufMutex.Lock()
  defer c.readerBufMutex.Unlock()

  readBytes, _ = c.readBuf.Read(b)
  if readBytes > 0 { return readBytes, nil } // just serve the data from the buffer

  // if the buffer is empty
  msg, err := c.readFromConn()
  readBytes = copy(b, msg)
  if readBytes < len(msg) {
    // there's data unread that needs to be buffered
    _, err = c.readBuf.Write(msg[readBytes:])
    if (err != nil) {return 0, err}
  }

  return
}

// reads data from the network connection
func (c *Conn) readFromConn() (msg []byte, err error) {
  var msgLen uint64
  err = binary.Read(c.Conn, binary.BigEndian, &msgLen)
  if (err != nil) { return nil, err}

  cipherBuf := make([]byte, msgLen)
  _, err = io.ReadFull(c.Conn, cipherBuf)
  if (err != nil) {return nil, err}
  msg, err = Decrypt(cipherBuf, c.sharedKey)
  return
}

func (c *Conn) Write(msg []byte) (n int, err error) {
  ciphertext, err := Encrypt(msg, c.sharedKey)
  if err != nil {return 0, err}
  final := lenPrefix(ciphertext)

  return c.Conn.Write(final)
}

func (c Conn) Close() error {
  return nil
}

const nonceLen = 24

func CiphertextLength(msgLen int) int {
  return box.Overhead + nonceLen + msgLen
}

func MsgLength(cipherLen int) int {
  return cipherLen - box.Overhead - nonceLen
}

func Encrypt(msg []byte, sharedKey *[sessionKeyLen]byte) (ciphertext []byte, err error) {
  err = nil
  ciphertext = make([]byte, nonceLen)
  var nonce [nonceLen]byte
   _, err = rand.Read(ciphertext[:nonceLen])
  if err != nil { return }

  copy(nonce[:], ciphertext[:nonceLen])

  ciphertext = box.SealAfterPrecomputation(ciphertext, msg, &nonce, sharedKey)
  return
}

func Decrypt(ciphertext []byte, sharedKey *[sessionKeyLen]byte) (msg []byte, err error) {
  nonceSlice := ciphertext[:nonceLen]
  var nonce [nonceLen]byte
  copy(nonce[:], nonceSlice)

  msgLen := MsgLength(len(ciphertext))
  msg = make([]byte, msgLen)

  _, success := box.OpenAfterPrecomputation(msg, ciphertext[nonceLen:], &nonce, sharedKey)

  if !success {
    return nil, DecryptError{}
  } else {
    return msg, nil
  }
}

// prefix a buffer with its length
func lenPrefix(b []byte) ([]byte) {
  buf := new(bytes.Buffer)
  binary.Write(buf, binary.BigEndian, uint64(len(b)))
  return append(buf.Bytes(), b...)
}

type DecryptError struct {
}
func (e DecryptError) Error() string { return "failed to decrypt message."}

type PeerChan (chan<- string)
/* DHT */
// an enhanced version of the DHT that allows per infohash subscriptions
// with notifications sent down channels
type DHT struct {
  *dht.DHT

  subscribers *map[dht.InfoHash](*ChanSet)
  subMutex *sync.Mutex
}

// holy mother of god... no set type and no generics
type ChanSet struct {
    set map[PeerChan]bool
}

func (set *ChanSet) Add(ch PeerChan) bool {
    _, found := set.set[ch]
    set.set[ch] = true
    return !found
}
func (set *ChanSet) Remove(ch PeerChan) {
  delete(set.set, ch)
}

func newDHT(dhtClient *dht.DHT) *DHT {
  mp := make(map[dht.InfoHash](*ChanSet))
  return &DHT{dhtClient, &mp, &sync.Mutex{}}
}

func (d *DHT) Run() {
  // launch the goroutine that deals with dispatching notifications about peers
  go d.drainResults()

  // run the underlying dht client normally
  d.DHT.Run()
}

// subscribe for peer notifications on that infohash
func (d *DHT) subscribeToInfoHash(infoHash dht.InfoHash, notificationChan PeerChan) {
  d.subMutex.Lock()
  defer d.subMutex.Unlock()
  chanSet := (*d.subscribers)[infoHash]
  if (chanSet == nil) {
    chanSet = &ChanSet{}
    (*d.subscribers)[infoHash] = chanSet
  }
  chanSet.Add(notificationChan)
}

func (dht *DHT) unsubscribeToInfoHash(infoHash dht.InfoHash, notificationChan PeerChan) {
  dht.subMutex.Lock()
  defer dht.subMutex.Unlock()

  chanSet := (*dht.subscribers)[infoHash]
  if (chanSet != nil) {
    chanSet.Remove(notificationChan)
  }
}

func (dht *DHT) drainResults() {
  for batch := range dht.PeersRequestResults {
    dht.notifyOfPeers(batch)
  }
}

func (dht *DHT) notifyOfPeers(batch map[dht.InfoHash][]string) {
  dht.subMutex.Lock()
  defer dht.subMutex.Unlock()

  for infoHash, peers := range batch {
    for _, peer := range peers {
      peerChanSet := (*dht.subscribers)[infoHash]
      for peerChan, _ := range peerChanSet.set {
        // do something with e.Value
        peerChan <- peer
      }
    }
  }
}
<commit_msg>rethinking the use of locks<commit_after>package node

import (
  "github.com/nictuku/dht"
  "net"
  "crypto/rsa"
  "crypto/x509"
  "sync"
  "fmt"
  "io"
  "encoding/binary"
  "errors"
  "golang.org/x/crypto/nacl/box"
  "crypto/rand"
  "crypto/sha1"
  "bytes"
  "encoding/hex"
  "time"

  "github.com/danoctavian/bluntly/stream"
)

/* CONSTANTS */

const sessionKeyLen = 32
const nonceLen = 24
const readBufferCapacity = 2048

// size of the buffer for peer notifications
const peerChanBufferCapacity = 100

/* NODE */
type Node struct {
  dht *DHT
  config *Config
}

/* CONFIG */
type FileConfig struct {
  dhtPort int
  ownKeyFile string
  id string
  configRoot string
  holePunch HolePunchConf
}

type Config struct {
  dhtPort int
  ownKey rsa.PrivateKey
  id string
  configRoot string
  holePunch HolePunchConf
  contactList *ContactList
}

type HolePunchConf struct {
	recvPort int
}

type ContactList struct {
  contacts *map[rsa.PublicKey]string
  mut *sync.Mutex
}

func NewNode(conf *Config) (node *Node, err error) {
  // setup the DHT
  dhtConf := dht.DefaultConfig
  dhtConf.Port = conf.dhtPort
  dhtClient, err := dht.New(dhtConf)
  if err != nil { return }
  dht := newDHT(dhtClient)

  go dht.Run()

  node.dht = dht

	return node, nil
}

/* CLIENT */
func (n *Node) Dial(peerPubKey *rsa.PublicKey) (conn Conn, err error) {
  infoHash, err := pubKeyToInfoHash(peerPubKey)
  if (err != nil) {return }

  peerNotifications := make(chan string, peerChanBufferCapacity)
  n.dht.subscribeToInfoHash(dht.InfoHash(infoHash), peerNotifications)

  // ask for the infohash
  n.dht.PeersRequest(infoHash, false)

  for peerNotification := range peerNotifications {
    go func(peerNotification string) {

    }(peerNotification)
  }

  return
}

/* LISTENER */
type Listener struct {
  connChan chan *Conn
}

func (n *Node) Listen(port int) (listener *Listener, err error) {
  connChan := make(chan *Conn)
  listener = &Listener{connChan: connChan}
  // setup TCP listener
  tcpListener, err := net.Listen("tcp", fmt.Sprintf(":"))
  if err != nil {return}

  // loop accepting TCP connections
  go func() {
    for {
      tcpConn, tcpErr := tcpListener.Accept()
      if tcpErr != nil {
        Log(LOG_ERROR, "%s", tcpErr)
      }

      go func() {
        conn, handshakeError := handleClientConn(tcpConn, &n.config.ownKey, n.config.contactList)
        if err != nil {
          Log(LOG_INFO,
              "handling client connection from address %s %s",
              tcpConn.RemoteAddr().String(), handshakeError)
        } else {
          connChan <- conn
        }
      }()
    }
  }()

  // announce yourself to the DHT
  go n.announceSelf()

  return
}

func (n *Node) announceSelf() error {
  infoHash, err := pubKeyToInfoHash(&n.config.ownKey.PublicKey)
  if (err != nil) {return err}

  for {
    // the peers request is done only for the purpose of announcing
    n.dht.PeersRequest(infoHash, true)
    time.Sleep(10 * time.Second)
  }
}

func pubKeyToInfoHash(pub *rsa.PublicKey) (string, error) {
  pubKeyBytes, err := x509.MarshalPKIXPublicKey(pub)
  if err != nil { return "", err}
  sha1Bytes := sha1.Sum(pubKeyBytes)
  return hex.EncodeToString(sha1Bytes[:]), nil
}

func (l *Listener) Accept() (c net.Conn, err error) {
  conn := <- l.connChan
  return conn, nil
}

func (l *Listener) Close() error {
  return nil
}

func Addr() net.Addr {
  return nil
}

func handleClientConn(rawConn net.Conn,
                      ownKey *rsa.PrivateKey,
                      contacts *ContactList) (conn *Conn, err error) {

  var handshakeLen int64
  err = binary.Read(rawConn, binary.BigEndian, &handshakeLen)
  if (err != nil) { return }

  ciphertext := make([]byte, handshakeLen)
  _, err = io.ReadFull(rawConn, ciphertext)
  if (err != nil) { return }

  plain, err := rsa.DecryptPKCS1v15(nil, ownKey, ciphertext)
  if (err != nil) { return }

  connReq := ConnRequest{}
  err = connReq.UnmarshalBinary(plain)
  if (err != nil) { return }

  _, privKey, err := box.GenerateKey(rand.Reader)
  if (err != nil) { return }

  var sharedKey [sessionKeyLen]byte
  box.Precompute(&sharedKey, connReq.sessionKey, privKey)

  return &Conn{Conn: rawConn,
              sharedKey: &sharedKey,
              // TODO: circular buffer capacity may cause
              // streaming to fail. to avoid,
              // put a cap on the size of the encrypted chunks
              readBuf: stream.NewCircularBuf(readBufferCapacity),
              readerBufMutex: &sync.Mutex{}}, nil
}

/* connection request */
type ConnRequest struct {
  peerPub *rsa.PublicKey
  sessionKey *[sessionKeyLen]byte
}

type ConnResponse struct {

}

func (r *ConnRequest) MarshalBinary() (data []byte, err error) {
  pubKeyBytes, err := x509.MarshalPKIXPublicKey(r.peerPub)
  if (err != nil) { return }

  return append((*r.sessionKey)[:], pubKeyBytes...), nil
}

func (r *ConnRequest) UnmarshalBinary(data []byte) (err error) {
  copiedBytes := copy(r.sessionKey[:], data[:sessionKeyLen])
  if (copiedBytes < sessionKeyLen) {
    return errors.New("session key too short.")
  }

  someKey, err := x509.ParsePKIXPublicKey(data)
  pubKey := someKey.(*rsa.PublicKey)
  if (err != nil) { return }
  r.peerPub = pubKey

  return
}

/* CONNECTION */
type Conn struct {
  net.Conn // underlying network connection
  sharedKey *[sessionKeyLen]byte

  // a buffer that is used to store in excess data, not yet read
  // but already decrypted
  readBuf *stream.CircularBuf
  readerBufMutex *sync.Mutex
}

func (c *Conn) Read(b []byte) (readBytes int, err error) {
  c.readerBufMutex.Lock()
  defer c.readerBufMutex.Unlock()

  readBytes, _ = c.readBuf.Read(b)
  if readBytes > 0 { return readBytes, nil } // just serve the data from the buffer

  // if the buffer is empty
  msg, err := c.readFromConn()
  readBytes = copy(b, msg)
  if readBytes < len(msg) {
    // there's data unread that needs to be buffered
    _, err = c.readBuf.Write(msg[readBytes:])
    if (err != nil) {return 0, err}
  }

  return
}

// reads data from the network connection
func (c *Conn) readFromConn() (msg []byte, err error) {
  var msgLen uint64
  err = binary.Read(c.Conn, binary.BigEndian, &msgLen)
  if (err != nil) { return nil, err}

  cipherBuf := make([]byte, msgLen)
  _, err = io.ReadFull(c.Conn, cipherBuf)
  if (err != nil) {return nil, err}
  msg, err = Decrypt(cipherBuf, c.sharedKey)
  return
}

func (c *Conn) Write(msg []byte) (n int, err error) {
  ciphertext, err := Encrypt(msg, c.sharedKey)
  if err != nil {return 0, err}
  final := lenPrefix(ciphertext)

  return c.Conn.Write(final)
}

func (c Conn) Close() error {
  return nil
}

func CiphertextLength(msgLen int) int {
  return box.Overhead + nonceLen + msgLen
}

func MsgLength(cipherLen int) int {
  return cipherLen - box.Overhead - nonceLen
}

func Encrypt(msg []byte, sharedKey *[sessionKeyLen]byte) (ciphertext []byte, err error) {
  err = nil
  ciphertext = make([]byte, nonceLen)
  var nonce [nonceLen]byte
   _, err = rand.Read(ciphertext[:nonceLen])
  if err != nil { return }

  copy(nonce[:], ciphertext[:nonceLen])

  ciphertext = box.SealAfterPrecomputation(ciphertext, msg, &nonce, sharedKey)
  return
}

func Decrypt(ciphertext []byte, sharedKey *[sessionKeyLen]byte) (msg []byte, err error) {
  nonceSlice := ciphertext[:nonceLen]
  var nonce [nonceLen]byte
  copy(nonce[:], nonceSlice)

  msgLen := MsgLength(len(ciphertext))
  msg = make([]byte, msgLen)

  _, success := box.OpenAfterPrecomputation(msg, ciphertext[nonceLen:], &nonce, sharedKey)

  if !success {
    return nil, DecryptError{}
  } else {
    return msg, nil
  }
}

// prefix a buffer with its length
func lenPrefix(b []byte) ([]byte) {
  buf := new(bytes.Buffer)
  binary.Write(buf, binary.BigEndian, uint64(len(b)))
  return append(buf.Bytes(), b...)
}

type DecryptError struct {
}
func (e DecryptError) Error() string { return "failed to decrypt message."}

type PeerChan (chan<- string)
/* DHT */
// an enhanced version of the DHT that allows per infohash subscriptions
// with notifications sent down channels
type DHT struct {
  *dht.DHT

  subscribers *map[dht.InfoHash](*ChanSet)
  subMutex *sync.Mutex
}

// holy mother of god... no set type and no generics
type ChanSet struct {
    set map[PeerChan]bool
}

func (set *ChanSet) Add(ch PeerChan) bool {
    _, found := set.set[ch]
    set.set[ch] = true
    return !found
}
func (set *ChanSet) Remove(ch PeerChan) {
  delete(set.set, ch)
}

func newDHT(dhtClient *dht.DHT) *DHT {
  mp := make(map[dht.InfoHash](*ChanSet))
  return &DHT{dhtClient, &mp, &sync.Mutex{}}
}

func (d *DHT) Run() {
  // launch the goroutine that deals with dispatching notifications about peers
  go d.drainResults()

  // run the underlying dht client normally
  d.DHT.Run()
}

// subscribe for peer notifications on that infohash
func (d *DHT) subscribeToInfoHash(infoHash dht.InfoHash, notificationChan PeerChan) {
  d.subMutex.Lock()
  defer d.subMutex.Unlock()
  chanSet := (*d.subscribers)[infoHash]
  if (chanSet == nil) {
    chanSet = &ChanSet{}
    (*d.subscribers)[infoHash] = chanSet
  }
  chanSet.Add(notificationChan)
}

func (dht *DHT) unsubscribeToInfoHash(infoHash dht.InfoHash, notificationChan PeerChan) {
  dht.subMutex.Lock()
  defer dht.subMutex.Unlock()

  chanSet := (*dht.subscribers)[infoHash]
  if (chanSet != nil) {
    chanSet.Remove(notificationChan)
  }
}

func (dht *DHT) drainResults() {
  for batch := range dht.PeersRequestResults {
    dht.notifyOfPeers(batch)
  }
}

func (dht *DHT) notifyOfPeers(batch map[dht.InfoHash][]string) {
  dht.subMutex.Lock()
  defer dht.subMutex.Unlock()

  for infoHash, peers := range batch {
    for _, peer := range peers {
      peerChanSet := (*dht.subscribers)[infoHash]
      for peerChan, _ := range peerChanSet.set {
        // do something with e.Value
        peerChan <- peer
      }
    }
  }
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package workflow

import (
	"encoding/json"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type TestElasticsearchDB struct {
	*ResourceDB
	mapping string
}

type TestElasticsearchBody struct {
	ID    piazza.Ident `json:"id"`
	Value int          `json:"value"`
}

const TestElasticsearchSettings = `{
	"mappings" : {
		"Obj5":{
			"properties":{
				"id5": {
					"type":"integer",
					"index": "not_analyzed"
				},
				"data5": {
					"type":"string",
					"index": "not_analyzed"
				},
				"foo5": {
					"type":"boolean",
					"index": "not_analyzed"
				}
			}
		}
	}
}`

const TestElasticsearchMapping = "TestElasticsearch"

func NewTestElasticsearchDB(service *Service, esi elasticsearch.IIndex) (*TestElasticsearchDB, error) {

	rdb, err := NewResourceDB(service, esi, "")
	if err != nil {
		return nil, err
	}

	err = esi.SetMapping(TestElasticsearchMapping, TestElasticsearchSettings)
	if err != nil {
		return nil, err
	}

	etrdb := TestElasticsearchDB{ResourceDB: rdb, mapping: TestElasticsearchMapping}

	/*time.Sleep(5 * time.Second)

	ok, err := esi.IndexExists()
	if err != nil {
		return nil, fmt.Errorf("Index %s failes to exist after creation", esi.IndexName())
	}
	if !ok {
		return nil, fmt.Errorf("Index %s does not exist after creation", esi.IndexName())
	}

	ok, err = esi.TypeExists(TestElasticsearchDBMapping)
	if err != nil {
		return nil, fmt.Errorf("Type %s fails to exist in index %s after creation", TestElasticsearchDBMapping, esi.IndexName())
	}
	if !ok {
		return nil, fmt.Errorf("Index %s does not exist in index %s after creation", TestElasticsearchDBMapping, esi.IndexName())
	}*/

	return &etrdb, nil
}

func (db *TestElasticsearchDB) PostData(obj interface{}, id piazza.Ident) (piazza.Ident, error) {
	var p *TestElasticsearchBody
	ok1 := false
	temp1, ok1 := obj.(TestElasticsearchBody)
	if !ok1 {
		temp2, ok2 := obj.(*TestElasticsearchBody)
		if !ok2 {
			return piazza.NoIdent, LoggedError("TestElasticsearchDB.PostData failed: was not given an TestElasticsearchBody to post")
		}
		p = temp2
	} else {
		p = &temp1
	}

	indexResult, err := db.Esi.PostData(db.mapping, id.String(), p)
	if err != nil {
		return piazza.NoIdent, LoggedError("TestElasticsearchDB.PostData failed: %s\n%#v\n%#v", err, db.mapping, p)
	}
	if !indexResult.Created {
		return piazza.NoIdent, LoggedError("TestElasticsearchDB.PostData failed: not created")
	}

	return id, nil
}

func (db *TestElasticsearchDB) GetAll(format *piazza.JsonPagination) ([]TestElasticsearchBody, int64, error) {
	bodies := []TestElasticsearchBody{}

	exists, err := db.Esi.TypeExists(db.mapping)
	if err != nil {
		return bodies, 0, err
	}
	if !exists {
		return bodies, 0, nil
	}

	searchResult, err := db.Esi.FilterByMatchAll(db.mapping, format)
	if err != nil {
		return nil, 0, LoggedError("TestElasticsearchDB.GetAll failed: %s", err)
	}
	if searchResult == nil {
		return nil, 0, LoggedError("TestElasticsearchDB.GetAll failed: no searchResult")
	}

	if searchResult != nil && searchResult.GetHits() != nil {
		for _, hit := range *searchResult.GetHits() {
			var body TestElasticsearchBody
			err := json.Unmarshal(*hit.Source, &body)
			if err != nil {
				return nil, 0, err
			}
			bodies = append(bodies, body)
		}
	}

	return bodies, searchResult.TotalHits(), nil
}

func (db *TestElasticsearchDB) GetVersion() (string, error) {
	v := db.Esi.GetVersion()
	return v, nil
}
<commit_msg>fix payload<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package workflow

import (
	"encoding/json"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type TestElasticsearchDB struct {
	*ResourceDB
	mapping string
}

type TestElasticsearchBody struct {
	ID    piazza.Ident `json:"id"`
	Value int          `json:"value"`
}

const TestElasticsearchSettings = `{
		"Obj5":{
			"properties":{
				"id5": {
					"type":"integer",
					"index": "not_analyzed"
				},
				"data5": {
					"type":"string",
					"index": "not_analyzed"
				},
				"foo5": {
					"type":"boolean",
					"index": "not_analyzed"
				}
			}
		}
	}`

const TestElasticsearchMapping = "TestElasticsearch"

func NewTestElasticsearchDB(service *Service, esi elasticsearch.IIndex) (*TestElasticsearchDB, error) {

	rdb, err := NewResourceDB(service, esi, "")
	if err != nil {
		return nil, err
	}

	err = esi.SetMapping(TestElasticsearchMapping, TestElasticsearchSettings)
	if err != nil {
		return nil, err
	}

	etrdb := TestElasticsearchDB{ResourceDB: rdb, mapping: TestElasticsearchMapping}

	/*time.Sleep(5 * time.Second)

	ok, err := esi.IndexExists()
	if err != nil {
		return nil, fmt.Errorf("Index %s failes to exist after creation", esi.IndexName())
	}
	if !ok {
		return nil, fmt.Errorf("Index %s does not exist after creation", esi.IndexName())
	}

	ok, err = esi.TypeExists(TestElasticsearchDBMapping)
	if err != nil {
		return nil, fmt.Errorf("Type %s fails to exist in index %s after creation", TestElasticsearchDBMapping, esi.IndexName())
	}
	if !ok {
		return nil, fmt.Errorf("Index %s does not exist in index %s after creation", TestElasticsearchDBMapping, esi.IndexName())
	}*/

	return &etrdb, nil
}

func (db *TestElasticsearchDB) PostData(obj interface{}, id piazza.Ident) (piazza.Ident, error) {
	var p *TestElasticsearchBody
	ok1 := false
	temp1, ok1 := obj.(TestElasticsearchBody)
	if !ok1 {
		temp2, ok2 := obj.(*TestElasticsearchBody)
		if !ok2 {
			return piazza.NoIdent, LoggedError("TestElasticsearchDB.PostData failed: was not given an TestElasticsearchBody to post")
		}
		p = temp2
	} else {
		p = &temp1
	}

	indexResult, err := db.Esi.PostData(db.mapping, id.String(), p)
	if err != nil {
		return piazza.NoIdent, LoggedError("TestElasticsearchDB.PostData failed: %s\n%#v\n%#v", err, db.mapping, p)
	}
	if !indexResult.Created {
		return piazza.NoIdent, LoggedError("TestElasticsearchDB.PostData failed: not created")
	}

	return id, nil
}

func (db *TestElasticsearchDB) GetAll(format *piazza.JsonPagination) ([]TestElasticsearchBody, int64, error) {
	bodies := []TestElasticsearchBody{}

	exists, err := db.Esi.TypeExists(db.mapping)
	if err != nil {
		return bodies, 0, err
	}
	if !exists {
		return bodies, 0, nil
	}

	searchResult, err := db.Esi.FilterByMatchAll(db.mapping, format)
	if err != nil {
		return nil, 0, LoggedError("TestElasticsearchDB.GetAll failed: %s", err)
	}
	if searchResult == nil {
		return nil, 0, LoggedError("TestElasticsearchDB.GetAll failed: no searchResult")
	}

	if searchResult != nil && searchResult.GetHits() != nil {
		for _, hit := range *searchResult.GetHits() {
			var body TestElasticsearchBody
			err := json.Unmarshal(*hit.Source, &body)
			if err != nil {
				return nil, 0, err
			}
			bodies = append(bodies, body)
		}
	}

	return bodies, searchResult.TotalHits(), nil
}

func (db *TestElasticsearchDB) GetVersion() (string, error) {
	v := db.Esi.GetVersion()
	return v, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package nycbcra provides non-alpha-premultiplied Y'CbCr-with-alpha image and
// color types.
//
// Deprecated: as of Go 1.6. Use the standard image and image/color packages
// instead.
package nycbcra // import "golang.org/x/image/webp/nycbcra"

import (
	"image"
	"image/color"
)

func init() {
	println("The golang.org/x/image/webp/nycbcra package is deprecated, as of Go 1.6. " +
		"Use the standard image and image/color packages instead.")
}

// TODO: move this to the standard image and image/color packages, so that the
// image/draw package can have fast-path code. Moving would rename:
//	nycbcra.Color      to color.NYCbCrA
//	nycbcra.ColorModel to color.NYCbCrAModel
//	nycbcra.Image      to image.NYCbCrA

// Color represents a non-alpha-premultiplied Y'CbCr-with-alpha color, having
// 8 bits each for one luma, two chroma and one alpha component.
type Color struct {
	color.YCbCr
	A uint8
}

func (c Color) RGBA() (r, g, b, a uint32) {
	r8, g8, b8 := color.YCbCrToRGB(c.Y, c.Cb, c.Cr)
	a = uint32(c.A) * 0x101
	r = uint32(r8) * 0x101 * a / 0xffff
	g = uint32(g8) * 0x101 * a / 0xffff
	b = uint32(b8) * 0x101 * a / 0xffff
	return
}

// ColorModel is the Model for non-alpha-premultiplied Y'CbCr-with-alpha colors.
var ColorModel color.Model = color.ModelFunc(nYCbCrAModel)

func nYCbCrAModel(c color.Color) color.Color {
	switch c := c.(type) {
	case Color:
		return c
	case color.YCbCr:
		return Color{c, 0xff}
	}
	r, g, b, a := c.RGBA()

	// Convert from alpha-premultiplied to non-alpha-premultiplied.
	if a != 0 {
		r = (r * 0xffff) / a
		g = (g * 0xffff) / a
		b = (b * 0xffff) / a
	}

	y, u, v := color.RGBToYCbCr(uint8(r>>8), uint8(g>>8), uint8(b>>8))
	return Color{color.YCbCr{Y: y, Cb: u, Cr: v}, uint8(a >> 8)}
}

// Image is an in-memory image of non-alpha-premultiplied Y'CbCr-with-alpha
// colors. A and AStride are analogous to the Y and YStride fields of the
// embedded YCbCr.
type Image struct {
	image.YCbCr
	A       []uint8
	AStride int
}

func (p *Image) ColorModel() color.Model {
	return ColorModel
}

func (p *Image) At(x, y int) color.Color {
	return p.NYCbCrAAt(x, y)
}

func (p *Image) NYCbCrAAt(x, y int) Color {
	if !(image.Point{X: x, Y: y}.In(p.Rect)) {
		return Color{}
	}
	yi := p.YOffset(x, y)
	ci := p.COffset(x, y)
	ai := p.AOffset(x, y)
	return Color{
		color.YCbCr{
			Y:  p.Y[yi],
			Cb: p.Cb[ci],
			Cr: p.Cr[ci],
		},
		p.A[ai],
	}
}

// AOffset returns the index of the first element of A that corresponds to
// the pixel at (x, y).
func (p *Image) AOffset(x, y int) int {
	return (y-p.Rect.Min.Y)*p.AStride + (x - p.Rect.Min.X)
}

// SubImage returns an image representing the portion of the image p visible
// through r. The returned value shares pixels with the original image.
func (p *Image) SubImage(r image.Rectangle) image.Image {
	// TODO: share code with image.NewYCbCr when this type moves into the
	// standard image package.
	r = r.Intersect(p.Rect)
	// If r1 and r2 are Rectangles, r1.Intersect(r2) is not guaranteed to be inside
	// either r1 or r2 if the intersection is empty. Without explicitly checking for
	// this, the Pix[i:] expression below can panic.
	if r.Empty() {
		return &Image{
			YCbCr: image.YCbCr{
				SubsampleRatio: p.SubsampleRatio,
			},
		}
	}
	yi := p.YOffset(r.Min.X, r.Min.Y)
	ci := p.COffset(r.Min.X, r.Min.Y)
	ai := p.AOffset(r.Min.X, r.Min.Y)
	return &Image{
		YCbCr: image.YCbCr{
			Y:              p.Y[yi:],
			Cb:             p.Cb[ci:],
			Cr:             p.Cr[ci:],
			SubsampleRatio: p.SubsampleRatio,
			YStride:        p.YStride,
			CStride:        p.CStride,
			Rect:           r,
		},
		A:       p.A[ai:],
		AStride: p.AStride,
	}
}

// Opaque scans the entire image and reports whether it is fully opaque.
func (p *Image) Opaque() bool {
	if p.Rect.Empty() {
		return true
	}
	i0, i1 := 0, p.Rect.Dx()
	for y := p.Rect.Min.Y; y < p.Rect.Max.Y; y++ {
		for _, a := range p.A[i0:i1] {
			if a != 0xff {
				return false
			}
		}
		i0 += p.AStride
		i1 += p.AStride
	}
	return true
}

// New returns a new Image with the given bounds and subsample ratio.
func New(r image.Rectangle, subsampleRatio image.YCbCrSubsampleRatio) *Image {
	// TODO: share code with image.NewYCbCr when this type moves into the
	// standard image package.
	w, h, cw, ch := r.Dx(), r.Dy(), 0, 0
	switch subsampleRatio {
	case image.YCbCrSubsampleRatio422:
		cw = (r.Max.X+1)/2 - r.Min.X/2
		ch = h
	case image.YCbCrSubsampleRatio420:
		cw = (r.Max.X+1)/2 - r.Min.X/2
		ch = (r.Max.Y+1)/2 - r.Min.Y/2
	case image.YCbCrSubsampleRatio440:
		cw = w
		ch = (r.Max.Y+1)/2 - r.Min.Y/2
	default:
		// Default to 4:4:4 subsampling.
		cw = w
		ch = h
	}
	b := make([]byte, 2*w*h+2*cw*ch)
	// TODO: use s[i:j:k] notation to set the cap.
	return &Image{
		YCbCr: image.YCbCr{
			Y:              b[:w*h],
			Cb:             b[w*h+0*cw*ch : w*h+1*cw*ch],
			Cr:             b[w*h+1*cw*ch : w*h+2*cw*ch],
			SubsampleRatio: subsampleRatio,
			YStride:        w,
			CStride:        cw,
			Rect:           r,
		},
		A:       b[w*h+2*cw*ch:],
		AStride: w,
	}
}
<commit_msg>webp/nycbcra: delete deprecated package<commit_after><|endoftext|>