<commit_before>package shell

import (
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"golang.org/x/exp/slices"
	"io"
	"os"
	"time"

	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
)

func init() {
	Commands = append(Commands, &commandVolumeBalance{})
}

type commandVolumeBalance struct {
}

func (c *commandVolumeBalance) Name() string {
	return "volume.balance"
}

func (c *commandVolumeBalance) Help() string {
	return `balance all volumes among volume servers

	volume.balance [-collection ALL|EACH_COLLECTION|<collection_name>] [-force] [-dataCenter=<data_center_name>]

	Algorithm:

	For each type of volume server (different max volume count limit){
		for each collection {
			balanceWritableVolumes()
			balanceReadOnlyVolumes()
		}
	}

	func balanceWritableVolumes(){
		idealWritableVolumeRatio = totalWritableVolumes / totalNumberOfMaxVolumes
		for hasMovedOneVolume {
			sort all volume servers ordered by the localWritableVolumeRatio = localWritableVolumes to localVolumeMax
			pick the volume server B with the highest localWritableVolumeRatio y
			for any the volume server A with the number of writable volumes x + 1 <= idealWritableVolumeRatio * localVolumeMax {
				if y > localWritableVolumeRatio {
					if B has a writable volume id v that A does not have, and satisfy v replication requirements {
						move writable volume v from A to B
					}
				}
			}
		}
	}
	func balanceReadOnlyVolumes(){
		//similar to balanceWritableVolumes
	}

`
}

func (c *commandVolumeBalance) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	balanceCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	collection := balanceCommand.String("collection", "EACH_COLLECTION", "collection name, or use \"ALL_COLLECTIONS\" across collections, \"EACH_COLLECTION\" for each collection")
	dc := balanceCommand.String("dataCenter", "", "only apply the balancing for this dataCenter")
	applyBalancing := balanceCommand.Bool("force", false, "apply the balancing plan.")
	if err = balanceCommand.Parse(args); err != nil {
		return nil
	}
	infoAboutSimulationMode(writer, *applyBalancing, "-force")

	if err = commandEnv.confirmIsLocked(args); err != nil {
		return
	}

	// collect topology information
	topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 15*time.Second)
	if err != nil {
		return err
	}

	volumeServers := collectVolumeServersByDc(topologyInfo, *dc)
	volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)
	diskTypes := collectVolumeDiskTypes(topologyInfo)

	if *collection == "EACH_COLLECTION" {
		collections, err := ListCollectionNames(commandEnv, true, false)
		if err != nil {
			return err
		}
		for _, c := range collections {
			if err = balanceVolumeServers(commandEnv, diskTypes, volumeReplicas, volumeServers, volumeSizeLimitMb*1024*1024, c, *applyBalancing); err != nil {
				return err
			}
		}
	} else if *collection == "ALL_COLLECTIONS" {
		if err = balanceVolumeServers(commandEnv, diskTypes, volumeReplicas, volumeServers, volumeSizeLimitMb*1024*1024, "ALL_COLLECTIONS", *applyBalancing); err != nil {
			return err
		}
	} else {
		if err = balanceVolumeServers(commandEnv, diskTypes, volumeReplicas, volumeServers, volumeSizeLimitMb*1024*1024, *collection, *applyBalancing); err != nil {
			return err
		}
	}

	return nil
}

func balanceVolumeServers(commandEnv *CommandEnv, diskTypes []types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, volumeSizeLimit uint64, collection string, applyBalancing bool) error {

	for _, diskType := range diskTypes {
		if err := balanceVolumeServersByDiskType(commandEnv, diskType, volumeReplicas, nodes, volumeSizeLimit, collection, applyBalancing); err != nil {
			return err
		}
	}
	return nil

}

func balanceVolumeServersByDiskType(commandEnv *CommandEnv, diskType types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, volumeSizeLimit uint64, collection string, applyBalancing bool) error {

	for _, n := range nodes {
		n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool {
			if collection != "ALL_COLLECTIONS" {
				if v.Collection != collection {
					return false
				}
			}
			return v.DiskType == string(diskType)
		})
	}
	if err := balanceSelectedVolume(commandEnv, diskType, volumeReplicas, nodes, capacityByMaxVolumeCount(diskType), sortWritableVolumes, applyBalancing); err != nil {
		return err
	}

	return nil
}

func collectVolumeServersByDc(t *master_pb.TopologyInfo, selectedDataCenter string) (nodes []*Node) {
	for _, dc := range t.DataCenterInfos {
		if selectedDataCenter != "" && dc.Id != selectedDataCenter {
			continue
		}
		for _, r := range dc.RackInfos {
			for _, dn := range r.DataNodeInfos {
				nodes = append(nodes, &Node{
					info: dn,
					dc:   dc.Id,
					rack: r.Id,
				})
			}
		}
	}
	return
}

func collectVolumeDiskTypes(t *master_pb.TopologyInfo) (diskTypes []types.DiskType) {
	knownTypes := make(map[string]bool)
	for _, dc := range t.DataCenterInfos {
		for _, r := range dc.RackInfos {
			for _, dn := range r.DataNodeInfos {
				for diskType, _ := range dn.DiskInfos {
					if _, found := knownTypes[diskType]; !found {
						knownTypes[diskType] = true
					}
				}
			}
		}
	}
	for diskType, _ := range knownTypes {
		diskTypes = append(diskTypes, types.ToDiskType(diskType))
	}
	return
}

type Node struct {
	info            *master_pb.DataNodeInfo
	selectedVolumes map[uint32]*master_pb.VolumeInformationMessage
	dc              string
	rack            string
}

type CapacityFunc func(*master_pb.DataNodeInfo) int

func capacityByMaxVolumeCount(diskType types.DiskType) CapacityFunc {
	return func(info *master_pb.DataNodeInfo) int {
		diskInfo, found := info.DiskInfos[string(diskType)]
		if !found {
			return 0
		}
		return int(diskInfo.MaxVolumeCount)
	}
}

func capacityByFreeVolumeCount(diskType types.DiskType) CapacityFunc {
	return func(info *master_pb.DataNodeInfo) int {
		diskInfo, found := info.DiskInfos[string(diskType)]
		if !found {
			return 0
		}
		return int(diskInfo.MaxVolumeCount - diskInfo.VolumeCount)
	}
}

func (n *Node) localVolumeRatio(capacityFunc CapacityFunc) float64 {
	return divide(len(n.selectedVolumes), capacityFunc(n.info))
}

func (n *Node) localVolumeNextRatio(capacityFunc CapacityFunc) float64 {
	return divide(len(n.selectedVolumes)+1, capacityFunc(n.info))
}

func (n *Node) selectVolumes(fn func(v *master_pb.VolumeInformationMessage) bool) {
	n.selectedVolumes = make(map[uint32]*master_pb.VolumeInformationMessage)
	for _, diskInfo := range n.info.DiskInfos {
		for _, v := range diskInfo.VolumeInfos {
			if fn(v) {
				n.selectedVolumes[v.Id] = v
			}
		}
	}
}

func sortWritableVolumes(volumes []*master_pb.VolumeInformationMessage) {
	slices.SortFunc(volumes, func(a, b *master_pb.VolumeInformationMessage) bool {
		return a.Size < b.Size
	})
}

func sortReadOnlyVolumes(volumes []*master_pb.VolumeInformationMessage) {
	slices.SortFunc(volumes, func(a, b *master_pb.VolumeInformationMessage) bool {
		return a.Id < b.Id
	})
}

func balanceSelectedVolume(commandEnv *CommandEnv, diskType types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, capacityFunc CapacityFunc, sortCandidatesFn func(volumes []*master_pb.VolumeInformationMessage), applyBalancing bool) (err error) {
	selectedVolumeCount, volumeMaxCount := 0, 0
	var nodesWithCapacity []*Node
	for _, dn := range nodes {
		selectedVolumeCount += len(dn.selectedVolumes)
		capacity := capacityFunc(dn.info)
		if capacity > 0 {
			nodesWithCapacity = append(nodesWithCapacity, dn)
		}
		volumeMaxCount += capacity
	}

	idealVolumeRatio := divide(selectedVolumeCount, volumeMaxCount)

	hasMoved := true

	// fmt.Fprintf(os.Stdout, " total %d volumes, max %d volumes, idealVolumeRatio %f\n", selectedVolumeCount, volumeMaxCount, idealVolumeRatio)

	for hasMoved {
		hasMoved = false
		slices.SortFunc(nodesWithCapacity, func(a, b *Node) bool {
			return a.localVolumeRatio(capacityFunc) < b.localVolumeRatio(capacityFunc)
		})
		fullNode := nodesWithCapacity[len(nodesWithCapacity)-1]
		var candidateVolumes []*master_pb.VolumeInformationMessage
		for _, v := range fullNode.selectedVolumes {
			candidateVolumes = append(candidateVolumes, v)
		}
		sortCandidatesFn(candidateVolumes)

		for i := 0; i < len(nodesWithCapacity)-1; i++ {
			emptyNode := nodesWithCapacity[i]
			if !(fullNode.localVolumeRatio(capacityFunc) > idealVolumeRatio && emptyNode.localVolumeNextRatio(capacityFunc) <= idealVolumeRatio) {
				// no more volume servers with empty slots
				break
			}
			fmt.Fprintf(os.Stdout, "%s %.2f %.2f:%.2f\t", diskType.ReadableString(), idealVolumeRatio, fullNode.localVolumeRatio(capacityFunc), emptyNode.localVolumeNextRatio(capacityFunc))
			hasMoved, err = attemptToMoveOneVolume(commandEnv, volumeReplicas, fullNode, candidateVolumes, emptyNode, applyBalancing)
			if err != nil {
				return
			}
			if hasMoved {
				// moved one volume
				break
			}
		}
	}
	return nil
}

func attemptToMoveOneVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*VolumeReplica, fullNode *Node, candidateVolumes []*master_pb.VolumeInformationMessage, emptyNode *Node, applyBalancing bool) (hasMoved bool, err error) {

	for _, v := range candidateVolumes {
		hasMoved, err = maybeMoveOneVolume(commandEnv, volumeReplicas, fullNode, v, emptyNode, applyBalancing)
		if err != nil {
			return
		}
		if hasMoved {
			break
		}
	}
	return
}

func maybeMoveOneVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*VolumeReplica, fullNode *Node, candidateVolume *master_pb.VolumeInformationMessage, emptyNode *Node, applyChange bool) (hasMoved bool, err error) {

	if candidateVolume.ReplicaPlacement > 0 {
		replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(candidateVolume.ReplicaPlacement))
		if !isGoodMove(replicaPlacement, volumeReplicas[candidateVolume.Id], fullNode, emptyNode) {
			return false, nil
		}
	}
	if _, found := emptyNode.selectedVolumes[candidateVolume.Id]; !found {
		if err = moveVolume(commandEnv, candidateVolume, fullNode, emptyNode, applyChange); err == nil {
			adjustAfterMove(candidateVolume, volumeReplicas, fullNode, emptyNode)
			return true, nil
		} else {
			return
		}
	}
	return
}

func moveVolume(commandEnv *CommandEnv, v *master_pb.VolumeInformationMessage, fullNode *Node, emptyNode *Node, applyChange bool) error {
	collectionPrefix := v.Collection + "_"
	if v.Collection == "" {
		collectionPrefix = ""
	}
	fmt.Fprintf(os.Stdout, "  moving %s volume %s%d %s => %s\n", v.DiskType, collectionPrefix, v.Id, fullNode.info.Id, emptyNode.info.Id)
	if applyChange {
		return LiveMoveVolume(commandEnv.option.GrpcDialOption, os.Stderr, needle.VolumeId(v.Id), pb.NewServerAddressFromDataNode(fullNode.info), pb.NewServerAddressFromDataNode(emptyNode.info), 5*time.Second, v.DiskType, false)
	}
	return nil
}

func isGoodMove(placement *super_block.ReplicaPlacement, existingReplicas []*VolumeReplica, sourceNode, targetNode *Node) bool {
	for _, replica := range existingReplicas {
		if replica.location.dataNode.Id == targetNode.info.Id &&
			replica.location.rack == targetNode.rack &&
			replica.location.dc == targetNode.dc {
			// never move to existing nodes
			return false
		}
	}
	dcs, racks := make(map[string]bool), make(map[string]int)
	for _, replica := range existingReplicas {
		if replica.location.dataNode.Id != sourceNode.info.Id {
			dcs[replica.location.DataCenter()] = true
			racks[replica.location.Rack()]++
		}
	}

	dcs[targetNode.dc] = true
	racks[fmt.Sprintf("%s %s", targetNode.dc, targetNode.rack)]++

	if len(dcs) != placement.DiffDataCenterCount+1 {
		return false
	}

	if len(racks) != placement.DiffRackCount+placement.DiffDataCenterCount+1 {
		return false
	}

	for _, sameRackCount := range racks {
		if sameRackCount != placement.SameRackCount+1 {
			return false
		}
	}

	return true

}

func adjustAfterMove(v *master_pb.VolumeInformationMessage, volumeReplicas map[uint32][]*VolumeReplica, fullNode *Node, emptyNode *Node) {
	delete(fullNode.selectedVolumes, v.Id)
	if emptyNode.selectedVolumes != nil {
		emptyNode.selectedVolumes[v.Id] = v
	}
	existingReplicas := volumeReplicas[v.Id]
	for _, replica := range existingReplicas {
		if replica.location.dataNode.Id == fullNode.info.Id &&
			replica.location.rack == fullNode.rack &&
			replica.location.dc == fullNode.dc {
			loc := newLocation(emptyNode.dc, emptyNode.rack, emptyNode.info)
			replica.location = &loc
			return
		}
	}
}
<commit_msg>edge case checking when volume server does not have capacity to balance<commit_after>package shell

import (
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"golang.org/x/exp/slices"
	"io"
	"os"
	"time"

	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
)

func init() {
	Commands = append(Commands, &commandVolumeBalance{})
}

type commandVolumeBalance struct {
}

func (c *commandVolumeBalance) Name() string {
	return "volume.balance"
}

func (c *commandVolumeBalance) Help() string {
	return `balance all volumes among volume servers

	volume.balance [-collection ALL|EACH_COLLECTION|<collection_name>] [-force] [-dataCenter=<data_center_name>]

	Algorithm:

	For each type of volume server (different max volume count limit){
		for each collection {
			balanceWritableVolumes()
			balanceReadOnlyVolumes()
		}
	}

	func balanceWritableVolumes(){
		idealWritableVolumeRatio = totalWritableVolumes / totalNumberOfMaxVolumes
		for hasMovedOneVolume {
			sort all volume servers ordered by the localWritableVolumeRatio = localWritableVolumes to localVolumeMax
			pick the volume server B with the highest localWritableVolumeRatio y
			for any the volume server A with the number of writable volumes x + 1 <= idealWritableVolumeRatio * localVolumeMax {
				if y > localWritableVolumeRatio {
					if B has a writable volume id v that A does not have, and satisfy v replication requirements {
						move writable volume v from A to B
					}
				}
			}
		}
	}
	func balanceReadOnlyVolumes(){
		//similar to balanceWritableVolumes
	}

`
}

func (c *commandVolumeBalance) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	balanceCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	collection := balanceCommand.String("collection", "EACH_COLLECTION", "collection name, or use \"ALL_COLLECTIONS\" across collections, \"EACH_COLLECTION\" for each collection")
	dc := balanceCommand.String("dataCenter", "", "only apply the balancing for this dataCenter")
	applyBalancing := balanceCommand.Bool("force", false, "apply the balancing plan.")
	if err = balanceCommand.Parse(args); err != nil {
		return nil
	}
	infoAboutSimulationMode(writer, *applyBalancing, "-force")

	if err = commandEnv.confirmIsLocked(args); err != nil {
		return
	}

	// collect topology information
	topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 15*time.Second)
	if err != nil {
		return err
	}

	volumeServers := collectVolumeServersByDc(topologyInfo, *dc)
	volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)
	diskTypes := collectVolumeDiskTypes(topologyInfo)

	if *collection == "EACH_COLLECTION" {
		collections, err := ListCollectionNames(commandEnv, true, false)
		if err != nil {
			return err
		}
		for _, c := range collections {
			if err = balanceVolumeServers(commandEnv, diskTypes, volumeReplicas, volumeServers, volumeSizeLimitMb*1024*1024, c, *applyBalancing); err != nil {
				return err
			}
		}
	} else if *collection == "ALL_COLLECTIONS" {
		if err = balanceVolumeServers(commandEnv, diskTypes, volumeReplicas, volumeServers, volumeSizeLimitMb*1024*1024, "ALL_COLLECTIONS", *applyBalancing); err != nil {
			return err
		}
	} else {
		if err = balanceVolumeServers(commandEnv, diskTypes, volumeReplicas, volumeServers, volumeSizeLimitMb*1024*1024, *collection, *applyBalancing); err != nil {
			return err
		}
	}

	return nil
}

func balanceVolumeServers(commandEnv *CommandEnv, diskTypes []types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, volumeSizeLimit uint64, collection string, applyBalancing bool) error {

	for _, diskType := range diskTypes {
		if err := balanceVolumeServersByDiskType(commandEnv, diskType, volumeReplicas, nodes, volumeSizeLimit, collection, applyBalancing); err != nil {
			return err
		}
	}
	return nil

}

func balanceVolumeServersByDiskType(commandEnv *CommandEnv, diskType types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, volumeSizeLimit uint64, collection string, applyBalancing bool) error {

	for _, n := range nodes {
		n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool {
			if collection != "ALL_COLLECTIONS" {
				if v.Collection != collection {
					return false
				}
			}
			return v.DiskType == string(diskType)
		})
	}
	if err := balanceSelectedVolume(commandEnv, diskType, volumeReplicas, nodes, capacityByMaxVolumeCount(diskType), sortWritableVolumes, applyBalancing); err != nil {
		return err
	}

	return nil
}

func collectVolumeServersByDc(t *master_pb.TopologyInfo, selectedDataCenter string) (nodes []*Node) {
	for _, dc := range t.DataCenterInfos {
		if selectedDataCenter != "" && dc.Id != selectedDataCenter {
			continue
		}
		for _, r := range dc.RackInfos {
			for _, dn := range r.DataNodeInfos {
				nodes = append(nodes, &Node{
					info: dn,
					dc:   dc.Id,
					rack: r.Id,
				})
			}
		}
	}
	return
}

func collectVolumeDiskTypes(t *master_pb.TopologyInfo) (diskTypes []types.DiskType) {
	knownTypes := make(map[string]bool)
	for _, dc := range t.DataCenterInfos {
		for _, r := range dc.RackInfos {
			for _, dn := range r.DataNodeInfos {
				for diskType, _ := range dn.DiskInfos {
					if _, found := knownTypes[diskType]; !found {
						knownTypes[diskType] = true
					}
				}
			}
		}
	}
	for diskType, _ := range knownTypes {
		diskTypes = append(diskTypes, types.ToDiskType(diskType))
	}
	return
}

type Node struct {
	info            *master_pb.DataNodeInfo
	selectedVolumes map[uint32]*master_pb.VolumeInformationMessage
	dc              string
	rack            string
}

type CapacityFunc func(*master_pb.DataNodeInfo) int

func capacityByMaxVolumeCount(diskType types.DiskType) CapacityFunc {
	return func(info *master_pb.DataNodeInfo) int {
		diskInfo, found := info.DiskInfos[string(diskType)]
		if !found {
			return 0
		}
		return int(diskInfo.MaxVolumeCount)
	}
}

func capacityByFreeVolumeCount(diskType types.DiskType) CapacityFunc {
	return func(info *master_pb.DataNodeInfo) int {
		diskInfo, found := info.DiskInfos[string(diskType)]
		if !found {
			return 0
		}
		return int(diskInfo.MaxVolumeCount - diskInfo.VolumeCount)
	}
}

func (n *Node) localVolumeRatio(capacityFunc CapacityFunc) float64 {
	return divide(len(n.selectedVolumes), capacityFunc(n.info))
}

func (n *Node) localVolumeNextRatio(capacityFunc CapacityFunc) float64 {
	return divide(len(n.selectedVolumes)+1, capacityFunc(n.info))
}

func (n *Node) selectVolumes(fn func(v *master_pb.VolumeInformationMessage) bool) {
	n.selectedVolumes = make(map[uint32]*master_pb.VolumeInformationMessage)
	for _, diskInfo := range n.info.DiskInfos {
		for _, v := range diskInfo.VolumeInfos {
			if fn(v) {
				n.selectedVolumes[v.Id] = v
			}
		}
	}
}

func sortWritableVolumes(volumes []*master_pb.VolumeInformationMessage) {
	slices.SortFunc(volumes, func(a, b *master_pb.VolumeInformationMessage) bool {
		return a.Size < b.Size
	})
}

func sortReadOnlyVolumes(volumes []*master_pb.VolumeInformationMessage) {
	slices.SortFunc(volumes, func(a, b *master_pb.VolumeInformationMessage) bool {
		return a.Id < b.Id
	})
}

func balanceSelectedVolume(commandEnv *CommandEnv, diskType types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, capacityFunc CapacityFunc, sortCandidatesFn func(volumes []*master_pb.VolumeInformationMessage), applyBalancing bool) (err error) {
	selectedVolumeCount, volumeMaxCount := 0, 0
	var nodesWithCapacity []*Node
	for _, dn := range nodes {
		selectedVolumeCount += len(dn.selectedVolumes)
		capacity := capacityFunc(dn.info)
		if capacity > 0 {
			nodesWithCapacity = append(nodesWithCapacity, dn)
		}
		volumeMaxCount += capacity
	}

	idealVolumeRatio := divide(selectedVolumeCount, volumeMaxCount)

	hasMoved := true

	// fmt.Fprintf(os.Stdout, " total %d volumes, max %d volumes, idealVolumeRatio %f\n", selectedVolumeCount, volumeMaxCount, idealVolumeRatio)

	for hasMoved {
		hasMoved = false
		slices.SortFunc(nodesWithCapacity, func(a, b *Node) bool {
			return a.localVolumeRatio(capacityFunc) < b.localVolumeRatio(capacityFunc)
		})
		if len(nodesWithCapacity) == 0 {
			fmt.Printf("no volume server found with capacity for %s", diskType.ReadableString())
			return nil
		}
		fullNode := nodesWithCapacity[len(nodesWithCapacity)-1]
		var candidateVolumes []*master_pb.VolumeInformationMessage
		for _, v := range fullNode.selectedVolumes {
			candidateVolumes = append(candidateVolumes, v)
		}
		sortCandidatesFn(candidateVolumes)

		for i := 0; i < len(nodesWithCapacity)-1; i++ {
			emptyNode := nodesWithCapacity[i]
			if !(fullNode.localVolumeRatio(capacityFunc) > idealVolumeRatio && emptyNode.localVolumeNextRatio(capacityFunc) <= idealVolumeRatio) {
				// no more volume servers with empty slots
				break
			}
			fmt.Fprintf(os.Stdout, "%s %.2f %.2f:%.2f\t", diskType.ReadableString(), idealVolumeRatio, fullNode.localVolumeRatio(capacityFunc), emptyNode.localVolumeNextRatio(capacityFunc))
			hasMoved, err = attemptToMoveOneVolume(commandEnv, volumeReplicas, fullNode, candidateVolumes, emptyNode, applyBalancing)
			if err != nil {
				return
			}
			if hasMoved {
				// moved one volume
				break
			}
		}
	}
	return nil
}

func attemptToMoveOneVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*VolumeReplica, fullNode *Node, candidateVolumes []*master_pb.VolumeInformationMessage, emptyNode *Node, applyBalancing bool) (hasMoved bool, err error) {

	for _, v := range candidateVolumes {
		hasMoved, err = maybeMoveOneVolume(commandEnv, volumeReplicas, fullNode, v, emptyNode, applyBalancing)
		if err != nil {
			return
		}
		if hasMoved {
			break
		}
	}
	return
}

func maybeMoveOneVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*VolumeReplica, fullNode *Node, candidateVolume *master_pb.VolumeInformationMessage, emptyNode *Node, applyChange bool) (hasMoved bool, err error) {

	if candidateVolume.ReplicaPlacement > 0 {
		replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(candidateVolume.ReplicaPlacement))
		if !isGoodMove(replicaPlacement, volumeReplicas[candidateVolume.Id], fullNode, emptyNode) {
			return false, nil
		}
	}
	if _, found := emptyNode.selectedVolumes[candidateVolume.Id]; !found {
		if err = moveVolume(commandEnv, candidateVolume, fullNode, emptyNode, applyChange); err == nil {
			adjustAfterMove(candidateVolume, volumeReplicas, fullNode, emptyNode)
			return true, nil
		} else {
			return
		}
	}
	return
}

func moveVolume(commandEnv *CommandEnv, v *master_pb.VolumeInformationMessage, fullNode *Node, emptyNode *Node, applyChange bool) error {
	collectionPrefix := v.Collection + "_"
	if v.Collection == "" {
		collectionPrefix = ""
	}
	fmt.Fprintf(os.Stdout, "  moving %s volume %s%d %s => %s\n", v.DiskType, collectionPrefix, v.Id, fullNode.info.Id, emptyNode.info.Id)
	if applyChange {
		return LiveMoveVolume(commandEnv.option.GrpcDialOption, os.Stderr, needle.VolumeId(v.Id), pb.NewServerAddressFromDataNode(fullNode.info), pb.NewServerAddressFromDataNode(emptyNode.info), 5*time.Second, v.DiskType, false)
	}
	return nil
}

func isGoodMove(placement *super_block.ReplicaPlacement, existingReplicas []*VolumeReplica, sourceNode, targetNode *Node) bool {
	for _, replica := range existingReplicas {
		if replica.location.dataNode.Id == targetNode.info.Id &&
			replica.location.rack == targetNode.rack &&
			replica.location.dc == targetNode.dc {
			// never move to existing nodes
			return false
		}
	}
	dcs, racks := make(map[string]bool), make(map[string]int)
	for _, replica := range existingReplicas {
		if replica.location.dataNode.Id != sourceNode.info.Id {
			dcs[replica.location.DataCenter()] = true
			racks[replica.location.Rack()]++
		}
	}

	dcs[targetNode.dc] = true
	racks[fmt.Sprintf("%s %s", targetNode.dc, targetNode.rack)]++

	if len(dcs) != placement.DiffDataCenterCount+1 {
		return false
	}

	if len(racks) != placement.DiffRackCount+placement.DiffDataCenterCount+1 {
		return false
	}

	for _, sameRackCount := range racks {
		if sameRackCount != placement.SameRackCount+1 {
			return false
		}
	}

	return true

}

func adjustAfterMove(v *master_pb.VolumeInformationMessage, volumeReplicas map[uint32][]*VolumeReplica, fullNode *Node, emptyNode *Node) {
	delete(fullNode.selectedVolumes, v.Id)
	if emptyNode.selectedVolumes != nil {
		emptyNode.selectedVolumes[v.Id] = v
	}
	existingReplicas := volumeReplicas[v.Id]
	for _, replica := range existingReplicas {
		if replica.location.dataNode.Id == fullNode.info.Id &&
			replica.location.rack == fullNode.rack &&
			replica.location.dc == fullNode.dc {
			loc := newLocation(emptyNode.dc, emptyNode.rack, emptyNode.info)
			replica.location = &loc
			return
		}
	}
}
<|endoftext|>


<commit_before>package fit

import (
	"math"
	"strconv"
)

const (
	sint32Invalid = 0x7FFFFFFF
	precision     = 5 // 1.1 m
)

var (
	semiToDegFactor = 180 / math.Pow(2, 31)
	degToSemiFactor = math.Pow(2, 31) / 180
)

// Latitude represents the geographical coordinate latitude.
type Latitude struct {
	semicircles int32
}

// NewLatitude returns a new latitude from a semicircle. If semicircles is
// outside the range of a latitude, (math.MinInt32/2, math.MaxInt32/2) then an
// invalid latitude is returned.
func NewLatitude(semicircles int32) Latitude {
	if semicircles == sint32Invalid {
		return NewLatitudeInvalid()
	}
	if semicircles < math.MinInt32/2 || semicircles > math.MaxInt32/2 {
		return NewLatitudeInvalid()
	}
	return Latitude{semicircles: semicircles}
}

// NewLatitudeDegrees returns a new latitude from a degree. If degrees is
// outside the range of a latitude (+/- 90°) then an invalid latitude is
// returned.
func NewLatitudeDegrees(degrees float64) Latitude {
	if degrees >= 90 || degrees <= -90 {
		return NewLatitudeInvalid()
	}
	return Latitude{semicircles: int32(degrees * degToSemiFactor)}
}

// NewLatitudeInvalid returns an invalid latitude. The underlying storage is
// set to the invalid value of the FIT base type (sint32) used to represent a
// latitude.
func NewLatitudeInvalid() Latitude {
	return Latitude{semicircles: sint32Invalid}
}

// Semicircles returns l in semicircles.
func (l Latitude) Semicircles() int32 {
	return l.semicircles
}

// Degrees returns l in degrees. If l is invalid then NaN is returned.
func (l Latitude) Degrees() float64 {
	if l.semicircles == sint32Invalid {
		return math.NaN()
	}
	return float64(l.semicircles) * semiToDegFactor
}

// Invalid reports whether l represents an invalid latitude.
func (l Latitude) Invalid() bool {
	return l.semicircles == sint32Invalid
}

// String returns a string representation of l in degrees with 5 decimal
// places. If l is invalid then the string "Invalid" is returned.
func (l Latitude) String() string {
	if l.semicircles == sint32Invalid {
		return "Invalid"
	}
	return strconv.FormatFloat(l.Degrees(), 'f', precision, 32)
}

// Longitude represents the geographical coordinate longitude.
type Longitude struct {
	semicircles int32
}

// NewLongitude returns a new longitude from a semicircle.
func NewLongitude(semicircles int32) Longitude {
	return Longitude{semicircles: semicircles}
}

// NewLongitudeDegrees returns a new longitude from a degree. If degrees is
// outside the range of a longitude (+/- 180°) then an invalid longitude is
// returned.
func NewLongitudeDegrees(degrees float64) Longitude {
	if degrees >= 180 || degrees <= -180 {
		return Longitude{semicircles: sint32Invalid}
	}
	return Longitude{semicircles: int32(degrees * degToSemiFactor)}
}

// NewLongitudeInvalid returns an invalid longitude. The underlying storage is
// set to the invalid value of the FIT base type (sint32) used to represent a
// longitude.
func NewLongitudeInvalid() Longitude {
	return Longitude{semicircles: sint32Invalid}
}

// Semicircles returns l in semicircles.
func (l Longitude) Semicircles() int32 {
	return l.semicircles
}

// Degrees returns l in degrees. If l is invalid then NaN is returned.
func (l Longitude) Degrees() float64 {
	if l.semicircles == sint32Invalid {
		return math.NaN()
	}
	return float64(l.semicircles) * semiToDegFactor
}

// Invalid reports whether l represents an invalid longitude.
func (l Longitude) Invalid() bool {
	return l.semicircles == sint32Invalid
}

// String returns a string representation of l in degrees with 5 decimal
// places. If l is invalid then the string "Invalid" is returned.
func (l Longitude) String() string {
	if l.semicircles == sint32Invalid {
		return "Invalid"
	}
	return strconv.FormatFloat(l.Degrees(), 'f', precision, 32)
}
<commit_msg>latlng: move repeated string into constant<commit_after>package fit

import (
	"math"
	"strconv"
)

const (
	sint32Invalid = 0x7FFFFFFF
	stringInvalid = "Invalid"
	precision     = 5 // 1.1 m
)

var (
	semiToDegFactor = 180 / math.Pow(2, 31)
	degToSemiFactor = math.Pow(2, 31) / 180
)

// Latitude represents the geographical coordinate latitude.
type Latitude struct {
	semicircles int32
}

// NewLatitude returns a new latitude from a semicircle. If semicircles is
// outside the range of a latitude, (math.MinInt32/2, math.MaxInt32/2) then an
// invalid latitude is returned.
func NewLatitude(semicircles int32) Latitude {
	if semicircles == sint32Invalid {
		return NewLatitudeInvalid()
	}
	if semicircles < math.MinInt32/2 || semicircles > math.MaxInt32/2 {
		return NewLatitudeInvalid()
	}
	return Latitude{semicircles: semicircles}
}

// NewLatitudeDegrees returns a new latitude from a degree. If degrees is
// outside the range of a latitude (+/- 90°) then an invalid latitude is
// returned.
func NewLatitudeDegrees(degrees float64) Latitude {
	if degrees >= 90 || degrees <= -90 {
		return NewLatitudeInvalid()
	}
	return Latitude{semicircles: int32(degrees * degToSemiFactor)}
}

// NewLatitudeInvalid returns an invalid latitude. The underlying storage is
// set to the invalid value of the FIT base type (sint32) used to represent a
// latitude.
func NewLatitudeInvalid() Latitude {
	return Latitude{semicircles: sint32Invalid}
}

// Semicircles returns l in semicircles.
func (l Latitude) Semicircles() int32 {
	return l.semicircles
}

// Degrees returns l in degrees. If l is invalid then NaN is returned.
func (l Latitude) Degrees() float64 {
	if l.semicircles == sint32Invalid {
		return math.NaN()
	}
	return float64(l.semicircles) * semiToDegFactor
}

// Invalid reports whether l represents an invalid latitude.
func (l Latitude) Invalid() bool {
	return l.semicircles == sint32Invalid
}

// String returns a string representation of l in degrees with 5 decimal
// places. If l is invalid then the string "Invalid" is returned.
func (l Latitude) String() string {
	if l.semicircles == sint32Invalid {
		return stringInvalid
	}
	return strconv.FormatFloat(l.Degrees(), 'f', precision, 32)
}

// Longitude represents the geographical coordinate longitude.
type Longitude struct {
	semicircles int32
}

// NewLongitude returns a new longitude from a semicircle.
func NewLongitude(semicircles int32) Longitude {
	return Longitude{semicircles: semicircles}
}

// NewLongitudeDegrees returns a new longitude from a degree. If degrees is
// outside the range of a longitude (+/- 180°) then an invalid longitude is
// returned.
func NewLongitudeDegrees(degrees float64) Longitude {
	if degrees >= 180 || degrees <= -180 {
		return Longitude{semicircles: sint32Invalid}
	}
	return Longitude{semicircles: int32(degrees * degToSemiFactor)}
}

// NewLongitudeInvalid returns an invalid longitude. The underlying storage is
// set to the invalid value of the FIT base type (sint32) used to represent a
// longitude.
func NewLongitudeInvalid() Longitude {
	return Longitude{semicircles: sint32Invalid}
}

// Semicircles returns l in semicircles.
func (l Longitude) Semicircles() int32 {
	return l.semicircles
}

// Degrees returns l in degrees. If l is invalid then NaN is returned.
func (l Longitude) Degrees() float64 {
	if l.semicircles == sint32Invalid {
		return math.NaN()
	}
	return float64(l.semicircles) * semiToDegFactor
}

// Invalid reports whether l represents an invalid longitude.
func (l Longitude) Invalid() bool {
	return l.semicircles == sint32Invalid
}

// String returns a string representation of l in degrees with 5 decimal
// places. If l is invalid then the string "Invalid" is returned.
func (l Longitude) String() string {
	if l.semicircles == sint32Invalid {
		return stringInvalid
	}
	return strconv.FormatFloat(l.Degrees(), 'f', precision, 32)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"leeroy/build"
	"leeroy/callbacks"
	"leeroy/config"
	"leeroy/logging"
	"leeroy/web"
	"log"
	"net/http"
)

var cfgFlag = flag.String("config", "leeroy.json", "JSON formatted config")

func main() {
	flag.Parse()

	c := config.FromFile(*cfgFlag)

	jobs := make(chan logging.Job, 100)
	b := logging.New(c.BuildLogPath)

	go build.Build(jobs, &c, b)

	log.Println("leeroy up an running!")

	http.HandleFunc("/callback/", web.Auth(func(w http.ResponseWriter, r *http.Request) {
		callbacks.Callback(w, r, jobs, &c, b)
	}, c.Secret))
	http.HandleFunc("/status/", func(w http.ResponseWriter, r *http.Request) {
		web.Status(w, r, &c, b)
	})
	http.HandleFunc("/status/repo/", func(w http.ResponseWriter, r *http.Request) {
		web.Repo(w, r, &c, b)
	})
	http.HandleFunc("/status/branch/", func(w http.ResponseWriter, r *http.Request) {
		web.Branch(w, r, &c, b)
	})
	http.HandleFunc("/status/commit/", func(w http.ResponseWriter, r *http.Request) {
		web.Commit(w, r, &c, b)
	})
	log.Fatal(http.ListenAndServe(":8082", nil))
}
<commit_msg>move status to /<commit_after>package main

import (
	"flag"
	"leeroy/build"
	"leeroy/callbacks"
	"leeroy/config"
	"leeroy/logging"
	"leeroy/web"
	"log"
	"net/http"
)

var cfgFlag = flag.String("config", "leeroy.json", "JSON formatted config")

func main() {
	flag.Parse()

	c := config.FromFile(*cfgFlag)

	jobs := make(chan logging.Job, 100)
	b := logging.New(c.BuildLogPath)

	go build.Build(jobs, &c, b)

	log.Println("leeroy up an running!")

	http.HandleFunc("/callback/", web.Auth(func(w http.ResponseWriter, r *http.Request) {
		callbacks.Callback(w, r, jobs, &c, b)
	}, c.Secret))
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		web.Status(w, r, &c, b)
	})
	http.HandleFunc("/status/repo/", func(w http.ResponseWriter, r *http.Request) {
		web.Repo(w, r, &c, b)
	})
	http.HandleFunc("/status/branch/", func(w http.ResponseWriter, r *http.Request) {
		web.Branch(w, r, &c, b)
	})
	http.HandleFunc("/status/commit/", func(w http.ResponseWriter, r *http.Request) {
		web.Commit(w, r, &c, b)
	})
	log.Fatal(http.ListenAndServe(":8082", nil))
}
<|endoftext|>


<commit_before>package php

import (
	"strings"
	"unicode"
)

const shortPHPBegin = "<?"
const longPHPBegin = "<?php"
const phpEnd = "?>"

const eof = -1

// lexHTML consumes and emits an html item until it
// finds a php begin
func lexHTML(l *lexer) stateFn {
	for {
		if strings.HasPrefix(l.input[l.pos:], shortPHPBegin) {
			if l.pos > l.start {
				l.emit(itemHTML)
			}
			return lexPHPBegin
		}
		if l.next() == eof {
			break
		}
	}
	if l.pos > l.start {
		l.emit(itemHTML)
	}
	l.emit(itemEOF)
	return nil
}

func lexPHPBegin(l *lexer) stateFn {
	if strings.HasPrefix(l.input[l.pos:], longPHPBegin) {
		l.pos += len(longPHPBegin)
	}
	if strings.HasPrefix(l.input[l.pos:], shortPHPBegin) {
		l.pos += len(shortPHPBegin)
	}
	l.emit(itemPHPBegin)
	return lexPHP
}

func lexPHP(l *lexer) stateFn {
	l.skipSpace()

	if r := l.peek(); unicode.IsDigit(r) {
		return lexNumberLiteral
	} else if r == '.' {
		l.next()
		if unicode.IsDigit(l.peek()) {
			l.backup()
			return lexNumberLiteral
		}
		l.backup()
	}

	if strings.HasPrefix(l.input[l.pos:], "?>") {
		return lexPHPEnd
	}

	if strings.HasPrefix(l.input[l.pos:], "//") {
		return lexLineComment
	}

	if strings.HasPrefix(l.input[l.pos:], "/*") {
		return lexBlockComment
	}

	for _, token := range tokenList {
		item := tokenMap[token]
		if strings.HasPrefix(l.input[l.pos:], token) {
			l.pos += len(token)
			if isKeyword(item) && l.accept(alphabet+underscore+digits) {
				l.backup()
				l.pos -= len(token)
				break
			}
			l.emit(item)
			return lexPHP
		}
	}

	if strings.HasPrefix(l.input[l.pos:], "$") {
		return lexIdentifier
	}

	if l.next() == eof {
		l.emit(itemEOF)
		return nil
	}
	l.backup()

	if l.peek() == '\'' {
		return lexSingleQuotedStringLiteral
	}

	if l.peek() == '"' {
		return lexDoubleQuotedStringLiteral
	}

	l.acceptRun(alphabet + underscore + digits + "\\")
	l.emit(itemNonVariableIdentifier)
	return lexPHP
}

func lexNumberLiteral(l *lexer) stateFn {
	// is negative?
	l.accept("-")
	l.acceptRun(digits)

	// is decimal?
	if l.accept(".") {
		l.acceptRun(digits)
	}

	l.emit(itemNumberLiteral)
	return lexPHP
}

func lexSingleQuotedStringLiteral(l *lexer) stateFn {
	l.next()
	for {
		switch l.next() {
		case '\\':
			l.next()
			continue
		case '\'':
			l.emit(itemStringLiteral)
			return lexPHP
		}
	}
}

func lexDoubleQuotedStringLiteral(l *lexer) stateFn {
	l.next()
	for {
		switch l.next() {
		case '\\':
			l.next()
			continue
		case '"':
			l.emit(itemStringLiteral)
			return lexPHP
		}
	}
}

func lexIf(l *lexer) stateFn {
	return l.errorf("if is not supported")
}

func lexCondition(l *lexer) stateFn {
	// this could be useful in the condition of a while, do-while, for terminator, if, and if-else block
	// what state should it return?
	// in all cases except do-while, after this is done, a block-begin is the correct state

	// how can this take advantage of the lexPHP function?
	return lexPHP
}

const alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const digits = "0123456789"
const underscore = "_"

func lexIdentifier(l *lexer) stateFn {
	l.accept("$")
	l.accept(underscore + alphabet)
	l.acceptRun(underscore + alphabet + digits)
	l.emit(itemIdentifier)
	return lexPHP
}

func lexFunctionArgs(l *lexer) stateFn {
	l.skipSpace()
	switch r := l.next(); {
	case r == '(':
		l.emit(itemOpenParen)
		if l.peek() == ')' {
			return lexFunctionArgs
		}
		return lexFunctionArg
	case r == ')':
		l.emit(itemCloseParen)
		return lexPHP
	case r == ',':
		l.emit(itemArgumentSeparator)
		return lexFunctionArg
	default:
		return l.errorf("invalid function argument separator: '%s'", string(r))
	}
}

func lexFunctionArg(l *lexer) stateFn {
	l.skipSpace()
	if l.peek() != '$' {
		l.accept(underscore + alphabet)
		l.acceptRun(underscore + alphabet + digits)
		l.emit(itemTypeHint)
	}
	l.skipSpace()
	l.next()
	l.accept(underscore + alphabet)
	l.acceptRun(underscore + alphabet + digits)
	l.emit(itemArgumentName)
	return lexFunctionArgs
}

// lexBlockBegin lexes the beginning of a code block delimited by '{'.
// This state occurs after the declaration of control flow structures.
func lexBlockBegin(l *lexer) stateFn {
	for isSpace(l.peek()) {
		l.next()
	}
	if l.next() == '{' {
		l.emit(itemBlockBegin)
	} else {
		l.errorf("expecting { to begin a new block")
	}
	return lexPHP
}

// lexBlockEnd lexes the end of a code block delimited by '}'.
func lexBlockEnd(l *lexer) stateFn {
	l.pos += 1
	l.emit(itemBlockEnd)
	return lexPHP
}

// lexPHPEnd lexes the end of a PHP section returning the context to HTML
func lexPHPEnd(l *lexer) stateFn {
	l.pos += len(phpEnd)
	l.emit(itemPHPEnd)
	return lexHTML
}

func lexLineComment(l *lexer) stateFn {
	l.pos += strings.Index(l.input[l.pos:], "\n") + 1
	l.ignore()
	return lexPHP
}

func lexBlockComment(l *lexer) stateFn {
	l.pos += strings.Index(l.input[l.pos:], "*/") + 2
	l.ignore()
	return lexPHP
}
<commit_msg>Removed dead code<commit_after>package php

import (
	"strings"
	"unicode"
)

const shortPHPBegin = "<?"
const longPHPBegin = "<?php"
const phpEnd = "?>"

const eof = -1

// lexHTML consumes and emits an html item until it
// finds a php begin
func lexHTML(l *lexer) stateFn {
	for {
		if strings.HasPrefix(l.input[l.pos:], shortPHPBegin) {
			if l.pos > l.start {
				l.emit(itemHTML)
			}
			return lexPHPBegin
		}
		if l.next() == eof {
			break
		}
	}
	if l.pos > l.start {
		l.emit(itemHTML)
	}
	l.emit(itemEOF)
	return nil
}

func lexPHPBegin(l *lexer) stateFn {
	if strings.HasPrefix(l.input[l.pos:], longPHPBegin) {
		l.pos += len(longPHPBegin)
	}
	if strings.HasPrefix(l.input[l.pos:], shortPHPBegin) {
		l.pos += len(shortPHPBegin)
	}
	l.emit(itemPHPBegin)
	return lexPHP
}

func lexPHP(l *lexer) stateFn {
	l.skipSpace()

	if r := l.peek(); unicode.IsDigit(r) {
		return lexNumberLiteral
	} else if r == '.' {
		l.next()
		if unicode.IsDigit(l.peek()) {
			l.backup()
			return lexNumberLiteral
		}
		l.backup()
	}

	if strings.HasPrefix(l.input[l.pos:], "?>") {
		return lexPHPEnd
	}

	if strings.HasPrefix(l.input[l.pos:], "//") {
		return lexLineComment
	}

	if strings.HasPrefix(l.input[l.pos:], "/*") {
		return lexBlockComment
	}

	for _, token := range tokenList {
		item := tokenMap[token]
		if strings.HasPrefix(l.input[l.pos:], token) {
			l.pos += len(token)
			if isKeyword(item) && l.accept(alphabet+underscore+digits) {
				l.backup()
				l.pos -= len(token)
				break
			}
			l.emit(item)
			return lexPHP
		}
	}

	if strings.HasPrefix(l.input[l.pos:], "$") {
		return lexIdentifier
	}

	if l.next() == eof {
		l.emit(itemEOF)
		return nil
	}
	l.backup()

	if l.peek() == '\'' {
		return lexSingleQuotedStringLiteral
	}

	if l.peek() == '"' {
		return lexDoubleQuotedStringLiteral
	}

	l.acceptRun(alphabet + underscore + digits + "\\")
	l.emit(itemNonVariableIdentifier)
	return lexPHP
}

func lexNumberLiteral(l *lexer) stateFn {
	// is decimal?
	l.acceptRun(digits)
	if l.accept(".") {
		l.acceptRun(digits)
	}

	l.emit(itemNumberLiteral)
	return lexPHP
}

func lexSingleQuotedStringLiteral(l *lexer) stateFn {
	l.next()
	for {
		switch l.next() {
		case '\\':
			l.next()
			continue
		case '\'':
			l.emit(itemStringLiteral)
			return lexPHP
		}
	}
}

func lexDoubleQuotedStringLiteral(l *lexer) stateFn {
	l.next()
	for {
		switch l.next() {
		case '\\':
			l.next()
			continue
		case '"':
			l.emit(itemStringLiteral)
			return lexPHP
		}
	}
}

func lexIf(l *lexer) stateFn {
	return l.errorf("if is not supported")
}

func lexCondition(l *lexer) stateFn {
	// this could be useful in the condition of a while, do-while, for terminator, if, and if-else block
	// what state should it return?
	// in all cases except do-while, after this is done, a block-begin is the correct state

	// how can this take advantage of the lexPHP function?
	return lexPHP
}

const alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const digits = "0123456789"
const underscore = "_"

func lexIdentifier(l *lexer) stateFn {
	l.accept("$")
	l.accept(underscore + alphabet)
	l.acceptRun(underscore + alphabet + digits)
	l.emit(itemIdentifier)
	return lexPHP
}

// lexBlockBegin lexes the beginning of a code block delimited by '{'.
// This state occurs after the declaration of control flow structures.
func lexBlockBegin(l *lexer) stateFn {
	for isSpace(l.peek()) {
		l.next()
	}
	if l.next() == '{' {
		l.emit(itemBlockBegin)
	} else {
		l.errorf("expecting { to begin a new block")
	}
	return lexPHP
}

// lexBlockEnd lexes the end of a code block delimited by '}'.
func lexBlockEnd(l *lexer) stateFn {
	l.pos += 1
	l.emit(itemBlockEnd)
	return lexPHP
}

// lexPHPEnd lexes the end of a PHP section returning the context to HTML
func lexPHPEnd(l *lexer) stateFn {
	l.pos += len(phpEnd)
	l.emit(itemPHPEnd)
	return lexHTML
}

func lexLineComment(l *lexer) stateFn {
	l.pos += strings.Index(l.input[l.pos:], "\n") + 1
	l.ignore()
	return lexPHP
}

func lexBlockComment(l *lexer) stateFn {
	l.pos += strings.Index(l.input[l.pos:], "*/") + 2
	l.ignore()
	return lexPHP
}
<|endoftext|>


<commit_before>package xbinary

import (
	"encoding/binary"
	"math"
)

type littleEndian struct{}

var LittleEndian littleEndian

func (littleEndian) Uint8(b []byte, index int) (uint8, error) {
	if len(b) < index || index < 0 {
		return 0, ErrOutOfRange
	}
	return uint8(b[index]), nil
}

func (littleEndian) Int8(b []byte, index int) (int8, error) {
	if len(b) < index || index < 0 {
		return 0, ErrOutOfRange
	}
	return int8(b[index]), nil
}

func (littleEndian) Uint16(b []byte, index int) (uint16, error) {
	if len(b) < index+1 || index < 0 {
		return 0, ErrOutOfRange
	}
	return binary.LittleEndian.Uint16(b[index:]), nil
}

func (littleEndian) Int16(b []byte, index int) (int16, error) {
	if len(b) < index+1 || index < 0 {
		return 0, ErrOutOfRange
	}
	return int16(binary.LittleEndian.Uint16(b[index:])), nil
}

func (littleEndian) Uint32(b []byte, index int) (uint32, error) {
	if len(b) < index+3 || index < 0 {
		return 0, ErrOutOfRange
	}
	return binary.LittleEndian.Uint32(b[index:]), nil
}

func (littleEndian) Int32(b []byte, index int) (int32, error) {
	if len(b) < index+3 || index < 0 {
		return 0, ErrOutOfRange
	}
	return int32(binary.LittleEndian.Uint32(b[index:])), nil
}

func (littleEndian) Uint64(b []byte, index int) (uint64, error) {
	if len(b) < index+7 || index < 0 {
		return 0, ErrOutOfRange
	}
	return binary.LittleEndian.Uint64(b[index:]), nil
}

func (littleEndian) Int64(b []byte, index int) (int64, error) {
	if len(b) < index+7 || index < 0 {
		return 0, ErrOutOfRange
	}
	return int64(binary.LittleEndian.Uint64(b[index:])), nil
}

func (littleEndian) Float32(b []byte, index int) (float32, error) {
	if len(b) < index+3 || index < 0 {
		return 0, ErrOutOfRange
	}
	return math.Float32frombits(binary.LittleEndian.Uint32(b[index:])), nil
}

func (l littleEndian) Float64(b []byte, index int) (float64, error) {
	val, err := l.Uint64(b, index)
	if err != nil {
		return 0.0, err
	}
	return math.Float64frombits(val), nil
}

func (littleEndian) PutUint8(b []byte, index int, v uint8) (int, error) {
	if len(b) < index || index < 0 {
		return 0, ErrOutOfRange
	}
	b[index] = byte(v)
	return 1, nil
}

func (littleEndian) PutUint16(b []byte, index int, v uint16) (int, error) {
	if len(b) < index+1 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint16(b[index:], v)
	return 2, nil
}

func (littleEndian) PutUint32(b []byte, index int, v uint32) (int, error) {
	if len(b) < index+3 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint32(b[index:], v)
	return 4, nil
}

func (littleEndian) PutUint64(b []byte, index int, v uint64) (int, error) {
	if len(b) < index+7 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint64(b[index:], v)
	return 8, nil
}

func (littleEndian) PutInt8(b []byte, index int, v int8) (int, error) {
	if len(b) < index || index < 0 {
		return 0, ErrOutOfRange
	}
	b[index] = byte(v)
	return 1, nil
}

func (littleEndian) PutInt16(b []byte, index int, v int16) (int, error) {
	if len(b) < index+1 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint16(b[index:], uint16(v))
	return 2, nil
}

func (littleEndian) PutInt32(b []byte, index int, v int32) (int, error) {
	if len(b) < index+3 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint32(b[index:], uint32(v))
	return 4, nil
}

func (littleEndian) PutInt64(b []byte, index int, v int64) (int, error) {
	if len(b) < index+7 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint64(b[index:], uint64(v))
	return 8, nil
}

func (littleEndian) PutFloat64(b []byte, index int, v float64) (int, error) {
	if len(b) < index+7 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint64(b[index:], math.Float64bits(v))
	return 8, nil
}

func (littleEndian) PutFloat32(b []byte, index int, v float32) (int, error) {
	if len(b) < index+3 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint32(b[index:], math.Float32bits(v))
	return 4, nil
}

func (littleEndian) Uint16Array(b []byte, index int, dest *[]uint16) error {
	if (len(*dest)*2+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = binary.LittleEndian.Uint16(b[index+i*2:])
	}
	return nil
}

func (littleEndian) Uint32Array(b []byte, index int, dest *[]uint32) error {
	if (len(*dest)*4+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = binary.LittleEndian.Uint32(b[index+i*4:])
	}
	return nil
}

func (littleEndian) Uint64Array(b []byte, index int, dest *[]uint64) error {
	if (len(*dest)*8+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = binary.LittleEndian.Uint64(b[index+i*8:])
	}
	return nil
}

func (littleEndian) Int16Array(b []byte, index int, dest *[]int16) error {
	if (len(*dest)*2+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = int16(binary.LittleEndian.Uint16(b[index+i*2:]))
	}
	return nil
}

func (littleEndian) Int32Array(b []byte, index int, dest *[]int32) error {
	if (len(*dest)*4+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = int32(binary.LittleEndian.Uint32(b[index+i*4:]))
	}
	return nil
}

func (littleEndian) Int64Array(b []byte, index int, dest *[]int64) error {
	if (len(*dest)*8+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = int64(binary.LittleEndian.Uint64(b[index+i*8:]))
	}
	return nil
}
func (littleEndian) Float32Array(b []byte, index int, dest *[]float32) error {
	if (len(*dest)*4+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = math.Float32frombits(binary.LittleEndian.Uint32(b[index+i*4:]))
	}
	return nil
}

func (littleEndian) Float64Array(b []byte, index int, dest *[]float64) error {
	if (len(*dest)*8+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = math.Float64frombits(binary.LittleEndian.Uint64(b[index+i*8:]))
	}
	return nil
}

func (littleEndian) PutUint16Array(b []byte, index int, value []uint16) (int, error) {
	var reqsize = (len(value)*2 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint16(b[index+i*2:], value[i])
	}
	return (len(value) * 2), nil
}

func (littleEndian) PutUint32Array(b []byte, index int, value []uint32) (int, error) {
	var reqsize = (len(value)*4 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint32(b[index+i*4:], value[i])
	}
	return (reqsize - index), nil
}

func (littleEndian) PutUint64Array(b []byte, index int, value []uint64) (int, error) {
	var reqsize = (len(value)*8 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint64(b[index+i*8:], value[i])
	}
	return (reqsize - index), nil
}

func (littleEndian) PutInt16Array(b []byte, index int, value []int16) (int, error) {
	var reqsize = (len(value)*2 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint16(b[index+i*2:], uint16(value[i]))
	}
	return (len(value) * 2), nil
}

func (littleEndian) PutInt32Array(b []byte, index int, value []int32) (int, error) {
	var reqsize = (len(value)*4 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint32(b[index+i*4:], uint32(value[i]))
	}
	return (reqsize - index), nil
}

func (littleEndian) PutInt64Array(b []byte, index int, value []int64) (int, error) {
	var reqsize = (len(value)*8 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint64(b[index+i*8:], uint64(value[i]))
	}
	return (reqsize - index), nil
}

func (littleEndian) PutFloat32Array(b []byte, index int, value []float32) (int, error) {
	var reqsize = (len(value)*4 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint32(b[index+i*4:], math.Float32bits(value[i]))
	}
	return (reqsize - index), nil
}

func (littleEndian) PutFloat64Array(b []byte, index int, value []float64) (int, error) {
	var reqsize = (len(value)*8 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint64(b[index+i*8:], math.Float64bits(value[i]))
	}
	return (reqsize - index), nil
}

func (littleEndian) String(b []byte, index, size int) (string, error) {
	if index+size > len(b) || index < 0 {
		return "", ErrOutOfRange
	}
	return string(b[index:]), nil
}

func (littleEndian) PutString(b []byte, index int, value string) (int, error) {
	if index+len(value) > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	copy(b[index:index+len(value)], []byte(value))
	return (len(value)), nil
}

func (littleEndian) PutUint8Array(b []byte, index int, value []uint8) (int, error) {
	var reqsize = (len(value) + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		b[index+i] = byte(value[i])
	}
	return (len(value)), nil
}

func (littleEndian) PutInt8Array(b []byte, index int, value []int8) (int, error) {
	var reqsize = (len(value) + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		b[index+i] = byte(value[i])
	}
	return (len(value)), nil
}
<commit_msg>Resolve test failures<commit_after>package xbinary

import (
	"encoding/binary"
	"math"
)

type littleEndian struct{}

var LittleEndian littleEndian

func (littleEndian) Uint8(b []byte, index int) (uint8, error) {
	if len(b) < index || index < 0 {
		return 0, ErrOutOfRange
	}
	return uint8(b[index]), nil
}

func (littleEndian) Int8(b []byte, index int) (int8, error) {
	if len(b) < index || index < 0 {
		return 0, ErrOutOfRange
	}
	return int8(b[index]), nil
}

func (littleEndian) Uint16(b []byte, index int) (uint16, error) {
	if len(b) < index+1 || index < 0 {
		return 0, ErrOutOfRange
	}
	return binary.LittleEndian.Uint16(b[index:]), nil
}

func (littleEndian) Int16(b []byte, index int) (int16, error) {
	if len(b) < index+1 || index < 0 {
		return 0, ErrOutOfRange
	}
	return int16(binary.LittleEndian.Uint16(b[index:])), nil
}

func (littleEndian) Uint32(b []byte, index int) (uint32, error) {
	if len(b) < index+3 || index < 0 {
		return 0, ErrOutOfRange
	}
	return binary.LittleEndian.Uint32(b[index:]), nil
}

func (littleEndian) Int32(b []byte, index int) (int32, error) {
	if len(b) < index+3 || index < 0 {
		return 0, ErrOutOfRange
	}
	return int32(binary.LittleEndian.Uint32(b[index:])), nil
}

func (littleEndian) Uint64(b []byte, index int) (uint64, error) {
	if len(b) < index+7 || index < 0 {
		return 0, ErrOutOfRange
	}
	return binary.LittleEndian.Uint64(b[index:]), nil
}

func (littleEndian) Int64(b []byte, index int) (int64, error) {
	if len(b) < index+7 || index < 0 {
		return 0, ErrOutOfRange
	}
	return int64(binary.LittleEndian.Uint64(b[index:])), nil
}

func (littleEndian) Float32(b []byte, index int) (float32, error) {
	if len(b) < index+3 || index < 0 {
		return 0, ErrOutOfRange
	}
	return math.Float32frombits(binary.LittleEndian.Uint32(b[index:])), nil
}

func (l littleEndian) Float64(b []byte, index int) (float64, error) {
	val, err := l.Uint64(b, index)
	if err != nil {
		return 0.0, err
	}
	return math.Float64frombits(val), nil
}

func (littleEndian) PutUint8(b []byte, index int, v uint8) (uint64, error) {
	if len(b) < index || index < 0 {
		return 0, ErrOutOfRange
	}
	b[index] = byte(v)
	return 1, nil
}

func (littleEndian) PutUint16(b []byte, index int, v uint16) (uint64, error) {
	if len(b) < index+1 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint16(b[index:], v)
	return 2, nil
}

func (littleEndian) PutUint32(b []byte, index int, v uint32) (uint64, error) {
	if len(b) < index+3 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint32(b[index:], v)
	return 4, nil
}

func (littleEndian) PutUint64(b []byte, index int, v uint64) (uint64, error) {
	if len(b) < index+7 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint64(b[index:], v)
	return 8, nil
}

func (littleEndian) PutInt8(b []byte, index int, v int8) (uint64, error) {
	if len(b) < index || index < 0 {
		return 0, ErrOutOfRange
	}
	b[index] = byte(v)
	return 1, nil
}

func (littleEndian) PutInt16(b []byte, index int, v int16) (uint64, error) {
	if len(b) < index+1 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint16(b[index:], uint16(v))
	return 2, nil
}

func (littleEndian) PutInt32(b []byte, index int, v int32) (uint64, error) {
	if len(b) < index+3 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint32(b[index:], uint32(v))
	return 4, nil
}

func (littleEndian) PutInt64(b []byte, index int, v int64) (uint64, error) {
	if len(b) < index+7 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint64(b[index:], uint64(v))
	return 8, nil
}

func (littleEndian) PutFloat64(b []byte, index int, v float64) (uint64, error) {
	if len(b) < index+7 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint64(b[index:], math.Float64bits(v))
	return 8, nil
}

func (littleEndian) PutFloat32(b []byte, index int, v float32) (uint64, error) {
	if len(b) < index+3 || index < 0 {
		return 0, ErrOutOfRange
	}
	binary.LittleEndian.PutUint32(b[index:], math.Float32bits(v))
	return 4, nil
}

func (littleEndian) Uint16Array(b []byte, index int, dest *[]uint16) error {
	if (len(*dest)*2+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = binary.LittleEndian.Uint16(b[index+i*2:])
	}
	return nil
}

func (littleEndian) Uint32Array(b []byte, index int, dest *[]uint32) error {
	if (len(*dest)*4+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = binary.LittleEndian.Uint32(b[index+i*4:])
	}
	return nil
}

func (littleEndian) Uint64Array(b []byte, index int, dest *[]uint64) error {
	if (len(*dest)*8+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = binary.LittleEndian.Uint64(b[index+i*8:])
	}
	return nil
}

func (littleEndian) Int16Array(b []byte, index int, dest *[]int16) error {
	if (len(*dest)*2+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = int16(binary.LittleEndian.Uint16(b[index+i*2:]))
	}
	return nil
}

func (littleEndian) Int32Array(b []byte, index int, dest *[]int32) error {
	if (len(*dest)*4+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = int32(binary.LittleEndian.Uint32(b[index+i*4:]))
	}
	return nil
}

func (littleEndian) Int64Array(b []byte, index int, dest *[]int64) error {
	if (len(*dest)*8+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = int64(binary.LittleEndian.Uint64(b[index+i*8:]))
	}
	return nil
}
func (littleEndian) Float32Array(b []byte, index int, dest *[]float32) error {
	if (len(*dest)*4+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = math.Float32frombits(binary.LittleEndian.Uint32(b[index+i*4:]))
	}
	return nil
}

func (littleEndian) Float64Array(b []byte, index int, dest *[]float64) error {
	if (len(*dest)*8+index) > len(b) || index < 0 {
		return ErrOutOfRange
	}
	for i := 0; i < len(*dest); i++ {
		(*dest)[i] = math.Float64frombits(binary.LittleEndian.Uint64(b[index+i*8:]))
	}
	return nil
}

func (littleEndian) PutUint16Array(b []byte, index int, value []uint16) (uint64, error) {
	var reqsize = (len(value)*2 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint16(b[index+i*2:], value[i])
	}
	return uint64(len(value) * 2), nil
}

func (littleEndian) PutUint32Array(b []byte, index int, value []uint32) (uint64, error) {
	var reqsize = (len(value)*4 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint32(b[index+i*4:], value[i])
	}
	return uint64(reqsize - index), nil
}

func (littleEndian) PutUint64Array(b []byte, index int, value []uint64) (uint64, error) {
	var reqsize = (len(value)*8 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint64(b[index+i*8:], value[i])
	}
	return uint64(reqsize - index), nil
}

func (littleEndian) PutInt16Array(b []byte, index int, value []int16) (uint64, error) {
	var reqsize = (len(value)*2 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint16(b[index+i*2:], uint16(value[i]))
	}
	return uint64(len(value) * 2), nil
}

func (littleEndian) PutInt32Array(b []byte, index int, value []int32) (uint64, error) {
	var reqsize = (len(value)*4 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint32(b[index+i*4:], uint32(value[i]))
	}
	return uint64(reqsize - index), nil
}

func (littleEndian) PutInt64Array(b []byte, index int, value []int64) (uint64, error) {
	var reqsize = (len(value)*8 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint64(b[index+i*8:], uint64(value[i]))
	}
	return uint64(reqsize - index), nil
}

func (littleEndian) PutFloat32Array(b []byte, index int, value []float32) (uint64, error) {
	var reqsize = (len(value)*4 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint32(b[index+i*4:], math.Float32bits(value[i]))
	}
	return uint64(reqsize - index), nil
}

func (littleEndian) PutFloat64Array(b []byte, index int, value []float64) (uint64, error) {
	var reqsize = (len(value)*8 + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		binary.LittleEndian.PutUint64(b[index+i*8:], math.Float64bits(value[i]))
	}
	return uint64(reqsize - index), nil
}

func (littleEndian) String(b []byte, index, size int) (string, error) {
	if index+size > len(b) || index < 0 {
		return "", ErrOutOfRange
	}
	return string(b[index:]), nil
}

func (littleEndian) PutString(b []byte, index int, value string) (uint64, error) {
	if index+len(value) > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	copy(b[index:index+len(value)], []byte(value))
	return uint64(len(value)), nil
}

func (littleEndian) PutUint8Array(b []byte, index int, value []uint8) (uint64, error) {
	var reqsize = (len(value) + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		b[index+i] = byte(value[i])
	}
	return uint64(len(value)), nil
}

func (littleEndian) PutInt8Array(b []byte, index int, value []int8) (uint64, error) {
	var reqsize = (len(value) + index)
	if reqsize > len(b) || index < 0 {
		return 0, ErrOutOfRange
	}
	for i := 0; i < len(value); i++ {
		b[index+i] = byte(value[i])
	}
	return uint64(len(value)), nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"fmt"
	"runtime"

	"github.com/jacobsa/bazilfuse"
	"golang.org/x/net/context"
)

// A type that knows how to serve ops read from a connection.
type Server interface {
	// Read and serve ops from the supplied connection until EOF.
	ServeOps(*Connection)
}

// A struct representing the status of a mount operation, with a method that
// waits for unmounting.
type MountedFileSystem struct {
	dir string

	// The result to return from Join. Not valid until the channel is closed.
	joinStatus          error
	joinStatusAvailable chan struct{}
}

// Return the directory on which the file system is mounted (or where we
// attempted to mount it.)
func (mfs *MountedFileSystem) Dir() string {
	return mfs.dir
}

// Block until a mounted file system has been unmounted. Do not return
// successfully until all ops read from the connection have been responded to
// (i.e. the file system server has finished processing all in-flight ops).
//
// The return value will be non-nil if anything unexpected happened while
// serving. May be called multiple times.
func (mfs *MountedFileSystem) Join(ctx context.Context) error {
	select {
	case <-mfs.joinStatusAvailable:
		return mfs.joinStatus
	case <-ctx.Done():
		return ctx.Err()
	}
}

// Optional configuration accepted by Mount.
type MountConfig struct {
	// The context from which every op read from the connetion by the sever
	// should inherit. If nil, context.Background() will be used.
	OpContext context.Context

	// OS X only.
	//
	// Normally on OS X we mount with the novncache option
	// (cf. http://goo.gl/1pTjuk), which disables entry caching in the kernel.
	// This is because osxfuse does not honor the entry expiration values we
	// return to it, instead caching potentially forever (cf.
	// http://goo.gl/8yR0Ie), and it is probably better to fail to cache than to
	// cache for too long, since the latter is more likely to hide consistency
	// bugs that are difficult to detect and diagnose.
	//
	// This field disables the use of novncache, restoring entry caching. Beware:
	// the value of ChildInodeEntry.EntryExpiration is ignored by the kernel, and
	// entries will be cached for an arbitrarily long time.
	EnableVnodeCaching bool
}

// Convert to mount options to be passed to package bazilfuse.
func (c *MountConfig) bazilfuseOptions() (opts []bazilfuse.MountOption) {
	isDarwin := runtime.GOOS == "darwin"

	// Enable permissions checking in the kernel. See the comments on
	// InodeAttributes.Mode.
	opts = append(opts, bazilfuse.SetOption("default_permissions", ""))

	// OS X: set novncache when appropriate.
	if isDarwin && !c.EnableVnodeCaching {
		opts = append(opts, bazilfuse.SetOption("novncache", ""))
	}

	// OS X: disable the use of "Apple Double" (._foo and .DS_Store) files, which
	// just add noise to debug output and can have significant cost on
	// network-based file systems.
	//
	// Cf. https://github.com/osxfuse/osxfuse/wiki/Mount-options
	if isDarwin {
		opts = append(opts, bazilfuse.SetOption("noappledouble", ""))
	}

	return
}

// Attempt to mount a file system on the given directory, using the supplied
// Server to serve connection requests. This function blocks until the file
// system is successfully mounted. On some systems, this requires the supplied
// Server to make forward progress (in particular, to respond to
// fuseops.InitOp).
func Mount(
	dir string,
	server Server,
	config *MountConfig) (mfs *MountedFileSystem, err error) {
	logger := getLogger()

	// Initialize the struct.
	mfs = &MountedFileSystem{
		dir:                 dir,
		joinStatusAvailable: make(chan struct{}),
	}

	// Open a bazilfuse connection.
	logger.Println("Opening a bazilfuse connection.")
	bfConn, err := bazilfuse.Mount(mfs.dir, config.bazilfuseOptions()...)
	if err != nil {
		err = fmt.Errorf("bazilfuse.Mount: %v", err)
		return
	}

	// Choose a parent context for ops.
	opContext := config.OpContext
	if opContext == nil {
		opContext = context.Background()
	}

	// Create our own Connection object wrapping it.
	connection, err := newConnection(opContext, logger, bfConn)
	if err != nil {
		bfConn.Close()
		err = fmt.Errorf("newConnection: %v", err)
		return
	}

	// Serve the connection in the background. When done, set the join status.
	go func() {
		server.ServeOps(connection)
		mfs.joinStatus = connection.close()
		close(mfs.joinStatusAvailable)
	}()

	// Wait for the connection to say it is ready.
	if err = connection.waitForReady(); err != nil {
		err = fmt.Errorf("WaitForReady: %v", err)
		return
	}

	return
}
<commit_msg>Added MountConfig.ReadOnly.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"fmt"
	"runtime"

	"github.com/jacobsa/bazilfuse"
	"golang.org/x/net/context"
)

// A type that knows how to serve ops read from a connection.
type Server interface {
	// Read and serve ops from the supplied connection until EOF.
	ServeOps(*Connection)
}

// A struct representing the status of a mount operation, with a method that
// waits for unmounting.
type MountedFileSystem struct {
	dir string

	// The result to return from Join. Not valid until the channel is closed.
	joinStatus          error
	joinStatusAvailable chan struct{}
}

// Return the directory on which the file system is mounted (or where we
// attempted to mount it.)
func (mfs *MountedFileSystem) Dir() string {
	return mfs.dir
}

// Block until a mounted file system has been unmounted. Do not return
// successfully until all ops read from the connection have been responded to
// (i.e. the file system server has finished processing all in-flight ops).
//
// The return value will be non-nil if anything unexpected happened while
// serving. May be called multiple times.
func (mfs *MountedFileSystem) Join(ctx context.Context) error {
	select {
	case <-mfs.joinStatusAvailable:
		return mfs.joinStatus
	case <-ctx.Done():
		return ctx.Err()
	}
}

// Optional configuration accepted by Mount.
type MountConfig struct {
	// The context from which every op read from the connetion by the sever
	// should inherit. If nil, context.Background() will be used.
	OpContext context.Context

	// TODO(jacobsa): Add comments.
	ReadOnly bool

	// OS X only.
	//
	// Normally on OS X we mount with the novncache option
	// (cf. http://goo.gl/1pTjuk), which disables entry caching in the kernel.
	// This is because osxfuse does not honor the entry expiration values we
	// return to it, instead caching potentially forever (cf.
	// http://goo.gl/8yR0Ie), and it is probably better to fail to cache than to
	// cache for too long, since the latter is more likely to hide consistency
	// bugs that are difficult to detect and diagnose.
	//
	// This field disables the use of novncache, restoring entry caching. Beware:
	// the value of ChildInodeEntry.EntryExpiration is ignored by the kernel, and
	// entries will be cached for an arbitrarily long time.
	EnableVnodeCaching bool
}

// Convert to mount options to be passed to package bazilfuse.
func (c *MountConfig) bazilfuseOptions() (opts []bazilfuse.MountOption) {
	isDarwin := runtime.GOOS == "darwin"

	// Enable permissions checking in the kernel. See the comments on
	// InodeAttributes.Mode.
	opts = append(opts, bazilfuse.SetOption("default_permissions", ""))

	// OS X: set novncache when appropriate.
	if isDarwin && !c.EnableVnodeCaching {
		opts = append(opts, bazilfuse.SetOption("novncache", ""))
	}

	// OS X: disable the use of "Apple Double" (._foo and .DS_Store) files, which
	// just add noise to debug output and can have significant cost on
	// network-based file systems.
	//
	// Cf. https://github.com/osxfuse/osxfuse/wiki/Mount-options
	if isDarwin {
		opts = append(opts, bazilfuse.SetOption("noappledouble", ""))
	}

	return
}

// Attempt to mount a file system on the given directory, using the supplied
// Server to serve connection requests. This function blocks until the file
// system is successfully mounted. On some systems, this requires the supplied
// Server to make forward progress (in particular, to respond to
// fuseops.InitOp).
func Mount(
	dir string,
	server Server,
	config *MountConfig) (mfs *MountedFileSystem, err error) {
	logger := getLogger()

	// Initialize the struct.
	mfs = &MountedFileSystem{
		dir:                 dir,
		joinStatusAvailable: make(chan struct{}),
	}

	// Open a bazilfuse connection.
	logger.Println("Opening a bazilfuse connection.")
	bfConn, err := bazilfuse.Mount(mfs.dir, config.bazilfuseOptions()...)
	if err != nil {
		err = fmt.Errorf("bazilfuse.Mount: %v", err)
		return
	}

	// Choose a parent context for ops.
	opContext := config.OpContext
	if opContext == nil {
		opContext = context.Background()
	}

	// Create our own Connection object wrapping it.
	connection, err := newConnection(opContext, logger, bfConn)
	if err != nil {
		bfConn.Close()
		err = fmt.Errorf("newConnection: %v", err)
		return
	}

	// Serve the connection in the background. When done, set the join status.
	go func() {
		server.ServeOps(connection)
		mfs.joinStatus = connection.close()
		close(mfs.joinStatusAvailable)
	}()

	// Wait for the connection to say it is ready.
	if err = connection.waitForReady(); err != nil {
		err = fmt.Errorf("WaitForReady: %v", err)
		return
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"log"
	"net"
)

// Listener accepts connections from devices.
type Listener struct {
	debug    bool
	listener net.Listener
}

// Listen creates a listener with the given PEM encoded X.509 certificate and the private key on the local network address laddr.
// Debug mode logs all server activity.
func Listen(cert, priv []byte, laddr string, debug bool) (*Listener, error) {
	c, err := x509.ParseCertificate(cert)
	p, err := x509.ParsePKCS1PrivateKey(priv)
	if err != nil {
		return nil, fmt.Errorf("failed to parse the certificate or the private key with error: %v", err)
	}

	pool := x509.NewCertPool()
	pool.AddCert(c)

	tlsCert := tls.Certificate{
		Certificate: [][]byte{cert},
		PrivateKey:  p,
	}

	config := tls.Config{
		Certificates: []tls.Certificate{tlsCert},
		ClientCAs:    pool,
	}

	if laddr == "" {
		laddr = "0.0.0.0:443"
	}

	listener, err := tls.Listen("tcp", laddr, &config)
	if err != nil {
		return nil, err
	}

	if debug {
		log.Printf("New CCS connection established with XMPP parameters: %+v\n", c)
	}

	return &Listener{
		debug:    debug,
		listener: listener,
	}, nil
}

// Accept waits for incoming connections and forwards incoming messages to handleMsg in a new goroutine.
// This function never returns, unless there is an error while accepting new connections.
func (l *Listener) Accept(handleMsg func(string)) error {
	for {
		conn, err := l.listener.Accept()
		if err != nil {
			log.Printf("error while accepting a new connection from a client: %v", err)
			return err // todo: it might not be appropriate to break the loop on all errors (like client disconnect during handshake)
		}

		defer conn.Close()
		log.Printf("server: accepted from %s", conn.RemoteAddr())
		go handleClient(conn)
	}
}

func handleClient(conn net.Conn) {
	defer conn.Close()
	buf := make([]byte, 512)
	for {
		log.Print("server: conn: waiting")
		n, err := conn.Read(buf)
		if err != nil {
			if err != nil {
				log.Printf("server: conn: read: %s", err)
			}
			break
		}

		tlscon, ok := conn.(*tls.Conn)
		if ok {
			state := tlscon.ConnectionState()
			sub := state.PeerCertificates[0].Subject
			log.Println(sub)
		}

		log.Printf("server: conn: echo %q\n", string(buf[:n]))
		n, err = conn.Write(buf[:n])

		n, err = conn.Write(buf[:n])
		log.Printf("server: conn: wrote %d bytes", n)

		if err != nil {
			log.Printf("server: write: %s", err)
			break
		}
	}
	log.Println("server: conn: closed")
}
<commit_msg>fix listerner close<commit_after>package main

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"log"
	"net"
)

// Listener accepts connections from devices.
type Listener struct {
	debug    bool
	listener net.Listener
}

// Listen creates a listener with the given PEM encoded X.509 certificate and the private key on the local network address laddr.
// Debug mode logs all server activity.
func Listen(cert, priv []byte, laddr string, debug bool) (*Listener, error) {
	c, err := x509.ParseCertificate(cert)
	p, err := x509.ParsePKCS1PrivateKey(priv)
	if err != nil {
		return nil, fmt.Errorf("failed to parse the certificate or the private key with error: %v", err)
	}

	pool := x509.NewCertPool()
	pool.AddCert(c)

	tlsCert := tls.Certificate{
		Certificate: [][]byte{cert},
		PrivateKey:  p,
	}

	config := tls.Config{
		Certificates: []tls.Certificate{tlsCert},
		ClientCAs:    pool,
	}

	if laddr == "" {
		laddr = "0.0.0.0:443"
	}

	listener, err := tls.Listen("tcp", laddr, &config)
	if err != nil {
		return nil, err
	}

	if debug {
		log.Printf("New CCS connection established with XMPP parameters: %+v\n", c)
	}

	return &Listener{
		debug:    debug,
		listener: listener,
	}, nil
}

// Accept waits for incoming connections and forwards incoming messages to handleMsg in a new goroutine.
// This function never returns, unless there is an error while accepting new connections.
// Listener is closed upon error return.
func (l *Listener) Accept(handleMsg func(string)) error {
	defer l.listener.Close()
	for {
		conn, err := l.listener.Accept()
		if err != nil {
			log.Printf("error while accepting a new connection from a client: %v", err)
			return err
			// todo: it might not be appropriate to break the loop on recoverable errors (like client disconnect during handshake)
			// the underlying fd.accept() does some basic recovery though we might need more: http://golang.org/src/net/fd_unix.go
		}
		log.Printf("server: accepted connection from client IP: %s", conn.RemoteAddr())

		go handleConn(conn)
	}
}

func handleConn(conn net.Conn) {
	defer conn.Close()
	buf := make([]byte, 512)
	for {
		log.Print("server: conn: waiting")
		n, err := conn.Read(buf)
		if err != nil {
			if err != nil {
				log.Printf("server: conn: read: %s", err)
			}
			break
		}

		tlscon, ok := conn.(*tls.Conn)
		if ok {
			state := tlscon.ConnectionState()
			sub := state.PeerCertificates[0].Subject
			log.Println(sub)
		}

		log.Printf("server: conn: echo %q\n", string(buf[:n]))
		n, err = conn.Write(buf[:n])

		n, err = conn.Write(buf[:n])
		log.Printf("server: conn: wrote %d bytes", n)

		if err != nil {
			log.Printf("server: write: %s", err)
			break
		}
	}
	log.Println("server: conn: closed")
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net"
	"strconv"

	baps3 "github.com/UniversityRadioYork/baps3-go"
)

type clientAndMessage struct {
	c   *Client
	msg baps3.Message
}

// Maintains communications with the downstream service and connected clients.
// Also does any processing needed with the commands.
type hub struct {
	// All current clients.
	clients map[*Client]bool

	// Dump state from the downstream service (playd)
	downstreamVersion  string
	downstreamFeatures baps3.FeatureSet

	// Playlist instance
	pl *Playlist

	// For communication with the downstream service.
	cReqCh chan<- baps3.Message
	cResCh <-chan baps3.Message

	// Where new requests from clients come through.
	reqCh chan clientAndMessage

	// Handlers for adding/removing connections.
	addCh chan *Client
	rmCh  chan *Client
	Quit  chan bool
}

// Handles a new client connection.
// conn is the new connection object.
func (h *hub) handleNewConnection(conn net.Conn) {
	defer conn.Close()
	client := &Client{
		conn:  conn,
		resCh: make(chan baps3.Message),
		tok:   baps3.NewTokeniser(),
	}

	// Register user
	h.addCh <- client

	go client.Read(h.reqCh, h.rmCh)
	client.Write(client.resCh, h.rmCh)
}

// Appends the downstream service's version (from the OHAI) to the listd version.
func (h *hub) makeWelcomeMsg() *baps3.Message {
	return baps3.NewMessage(baps3.RsOhai).AddArg("listd " + LD_VERSION + "/" + h.downstreamVersion)
}

// Crafts the features message by adding listd's features to the downstream service's and removing
// features listd intercepts.
func (h *hub) makeFeaturesMsg() (msg *baps3.Message) {
	features := h.downstreamFeatures
	features.DelFeature(baps3.FtFileLoad) // 'Mask' the features listd intercepts
	features.AddFeature(baps3.FtPlaylist)
	features.AddFeature(baps3.FtPlaylistTextItems)
	msg = features.ToMessage()
	return
}

func sendInvalidCmd(c *Client, errType baps3.MessageWord, errStr string, oldCmd baps3.Message) {
	msg := baps3.NewMessage(errType).AddArg(errStr)
	for _, w := range oldCmd.AsSlice() {
		msg.AddArg(w)
	}
	c.resCh <- *msg
}

func (h *hub) processReqDequeue(req baps3.Message) (baps3.MessageWord, string) {
	iStr, err := req.Arg(0)
	if err != nil {
		return baps3.RsWhat, "Bad command"
	}

	hash, err := req.Arg(1)
	if err != nil {
		return baps3.RsWhat, "Bad command"
	}

	i, err := strconv.Atoi(iStr)
	if err != nil {
		return baps3.RsWhat, "Bad index"
	}

	rmIdx, rmHash, err := h.pl.Dequeue(i, hash)
	if err != nil {
		return baps3.RsFail, err.Error()
	}
	h.broadcast(*baps3.NewMessage(baps3.RsDequeue).AddArg(strconv.Itoa(rmIdx)).AddArg(rmHash))
	return baps3.BadWord, "" // No error
}

// Handles a request from a client.
// Falls through to the connector cReqCh if command is "not understood".
func (h *hub) processRequest(c *Client, req baps3.Message) {
	// TODO: Do something else
	log.Println("New request:", req.String())
	switch req.Word() {
	case baps3.RqDequeue:
		if errType, errStr := h.processReqDequeue(req); errType != baps3.BadWord {
			sendInvalidCmd(c, errType, errStr, req)
		}

	default:
		h.cReqCh <- req
	}
}

// Processes a response from the downstream service.
func (h *hub) processResponse(res baps3.Message) {
	// TODO: Do something else
	log.Println("New response:", res.String())
	switch res.Word() {
	case baps3.RsOhai:
		h.downstreamVersion, _ = res.Arg(0)
	case baps3.RsFeatures:
		fs, err := baps3.FeatureSetFromMsg(&res)
		if err != nil {
			log.Fatal("Error reading features: " + err.Error())
		}
		h.downstreamFeatures = fs
	default:
		h.broadcast(res)
	}
}

// Send a response message to all clients.
func (h *hub) broadcast(res baps3.Message) {
	for c, _ := range h.clients {
		c.resCh <- res
	}
}

// Listens for new connections on addr:port and spins up the relevant goroutines.
func (h *hub) runListener(addr string, port string) {
	netListener, err := net.Listen("tcp", addr+":"+port)
	if err != nil {
		log.Println("Listening error:", err.Error())
		return
	}

	// Get new connections
	go func() {
		for {
			conn, err := netListener.Accept()
			if err != nil {
				log.Println("Error accepting connection:", err.Error())
				continue
			}

			go h.handleNewConnection(conn)
		}
	}()

	for {
		select {
		case msg := <-h.cResCh:
			h.processResponse(msg)
		case data := <-h.reqCh:
			h.processRequest(data.c, data.msg)
		case client := <-h.addCh:
			h.clients[client] = true
			client.resCh <- *h.makeWelcomeMsg()
			client.resCh <- *h.makeFeaturesMsg()
			log.Println("New connection from", client.conn.RemoteAddr())
		case client := <-h.rmCh:
			close(client.resCh)
			delete(h.clients, client)
			log.Println("Closed connection from", client.conn.RemoteAddr())
		case <-h.Quit:
			log.Println("Closing all connections")
			for c, _ := range h.clients {
				close(c.resCh)
				delete(h.clients, c)
			}
			//			h.Quit <- true
		}
	}
}

// Sets up the connector channels for the hub object.
func (h *hub) setConnector(cReqCh chan<- baps3.Message, cResCh <-chan baps3.Message) {
	h.cReqCh = cReqCh
	h.cResCh = cResCh
}
<commit_msg>Add enqueue<commit_after>package main

import (
	"log"
	"net"
	"strconv"

	baps3 "github.com/UniversityRadioYork/baps3-go"
)

type clientAndMessage struct {
	c   *Client
	msg baps3.Message
}

// Maintains communications with the downstream service and connected clients.
// Also does any processing needed with the commands.
type hub struct {
	// All current clients.
	clients map[*Client]bool

	// Dump state from the downstream service (playd)
	downstreamVersion  string
	downstreamFeatures baps3.FeatureSet

	// Playlist instance
	pl *Playlist

	// For communication with the downstream service.
	cReqCh chan<- baps3.Message
	cResCh <-chan baps3.Message

	// Where new requests from clients come through.
	reqCh chan clientAndMessage

	// Handlers for adding/removing connections.
	addCh chan *Client
	rmCh  chan *Client
	Quit  chan bool
}

// Handles a new client connection.
// conn is the new connection object.
func (h *hub) handleNewConnection(conn net.Conn) {
	defer conn.Close()
	client := &Client{
		conn:  conn,
		resCh: make(chan baps3.Message),
		tok:   baps3.NewTokeniser(),
	}

	// Register user
	h.addCh <- client

	go client.Read(h.reqCh, h.rmCh)
	client.Write(client.resCh, h.rmCh)
}

// Appends the downstream service's version (from the OHAI) to the listd version.
func (h *hub) makeWelcomeMsg() *baps3.Message {
	return baps3.NewMessage(baps3.RsOhai).AddArg("listd " + LD_VERSION + "/" + h.downstreamVersion)
}

// Crafts the features message by adding listd's features to the downstream service's and removing
// features listd intercepts.
func (h *hub) makeFeaturesMsg() (msg *baps3.Message) {
	features := h.downstreamFeatures
	features.DelFeature(baps3.FtFileLoad) // 'Mask' the features listd intercepts
	features.AddFeature(baps3.FtPlaylist)
	features.AddFeature(baps3.FtPlaylistTextItems)
	msg = features.ToMessage()
	return
}

func sendInvalidCmd(c *Client, errType baps3.MessageWord, errStr string, oldCmd baps3.Message) {
	msg := baps3.NewMessage(errType).AddArg(errStr)
	for _, w := range oldCmd.AsSlice() {
		msg.AddArg(w)
	}
	c.resCh <- *msg
}

func (h *hub) processReqDequeue(req baps3.Message) (baps3.MessageWord, string) {
	iStr, err := req.Arg(0)
	if err != nil {
		return baps3.RsWhat, "Bad command"
	}

	hash, err := req.Arg(1)
	if err != nil {
		return baps3.RsWhat, "Bad command"
	}

	i, err := strconv.Atoi(iStr)
	if err != nil {
		return baps3.RsWhat, "Bad index"
	}

	rmIdx, rmHash, err := h.pl.Dequeue(i, hash)
	if err != nil {
		return baps3.RsFail, err.Error()
	}
	h.broadcast(*baps3.NewMessage(baps3.RsDequeue).AddArg(strconv.Itoa(rmIdx)).AddArg(rmHash))
	return baps3.BadWord, "" // No error
}

func (h *hub) processReqEnqueue(req baps3.Message) (baps3.MessageWord, string) {
	iStr, err := req.Arg(0)
	if err != nil {
		return baps3.RsWhat, "Bad command"
	}

	hash, err := req.Arg(1)
	if err != nil {
		return baps3.RsWhat, "Bad command"
	}

	itemType, err := req.Arg(2)
	if err != nil {
		return baps3.RsWhat, "Bad command"
	}

	arg, err := req.Arg(3)
	if err != nil {
		return baps3.RsWhat, "Bad command"
	}

	i, err := strconv.Atoi(iStr)
	if err != nil {
		return baps3.RsWhat, "Bad index"
	}

	if itemType != "file" && itemType != "text" {
		return baps3.RsWhat, "Bad item type"
	}

	item := &PlaylistItem{Data: arg, Hash: hash, IsFile: itemType == "file"}
	newIdx, err := h.pl.Enqueue(i, item)
	if err != nil {
		return baps3.RsFail, err.Error()
	}
	h.broadcast(*baps3.NewMessage(baps3.RsEnqueue).AddArg(strconv.Itoa(newIdx)).AddArg(item.Hash).AddArg(itemType).AddArg(arg))
	return baps3.BadWord, "" // No error
}

// Handles a request from a client.
// Falls through to the connector cReqCh if command is "not understood".
func (h *hub) processRequest(c *Client, req baps3.Message) {
	// TODO: Do something else
	log.Println("New request:", req.String())
	switch req.Word() {
	case baps3.RqDequeue:
		if errType, errStr := h.processReqDequeue(req); errType != baps3.BadWord {
			sendInvalidCmd(c, errType, errStr, req)
		}
	case baps3.RqEnqueue:
		if errType, errStr := h.processReqEnqueue(req); errType != baps3.BadWord {
			sendInvalidCmd(c, errType, errStr, req)
		}

	default:
		h.cReqCh <- req
	}
}

// Processes a response from the downstream service.
func (h *hub) processResponse(res baps3.Message) {
	// TODO: Do something else
	log.Println("New response:", res.String())
	switch res.Word() {
	case baps3.RsOhai:
		h.downstreamVersion, _ = res.Arg(0)
	case baps3.RsFeatures:
		fs, err := baps3.FeatureSetFromMsg(&res)
		if err != nil {
			log.Fatal("Error reading features: " + err.Error())
		}
		h.downstreamFeatures = fs
	default:
		h.broadcast(res)
	}
}

// Send a response message to all clients.
func (h *hub) broadcast(res baps3.Message) {
	for c, _ := range h.clients {
		c.resCh <- res
	}
}

// Listens for new connections on addr:port and spins up the relevant goroutines.
func (h *hub) runListener(addr string, port string) {
	netListener, err := net.Listen("tcp", addr+":"+port)
	if err != nil {
		log.Println("Listening error:", err.Error())
		return
	}

	// Get new connections
	go func() {
		for {
			conn, err := netListener.Accept()
			if err != nil {
				log.Println("Error accepting connection:", err.Error())
				continue
			}

			go h.handleNewConnection(conn)
		}
	}()

	for {
		select {
		case msg := <-h.cResCh:
			h.processResponse(msg)
		case data := <-h.reqCh:
			h.processRequest(data.c, data.msg)
		case client := <-h.addCh:
			h.clients[client] = true
			client.resCh <- *h.makeWelcomeMsg()
			client.resCh <- *h.makeFeaturesMsg()
			log.Println("New connection from", client.conn.RemoteAddr())
		case client := <-h.rmCh:
			close(client.resCh)
			delete(h.clients, client)
			log.Println("Closed connection from", client.conn.RemoteAddr())
		case <-h.Quit:
			log.Println("Closing all connections")
			for c, _ := range h.clients {
				close(c.resCh)
				delete(h.clients, c)
			}
			//			h.Quit <- true
		}
	}
}

// Sets up the connector channels for the hub object.
func (h *hub) setConnector(cReqCh chan<- baps3.Message, cResCh <-chan baps3.Message) {
	h.cReqCh = cReqCh
	h.cResCh = cResCh
}
<|endoftext|>


<commit_before>package grouper

import (
	"fmt"
	"github.com/SumoLogic/sumoshell/util"
	"os"
	"strings"
	"sync"
)

type Grouper struct {
	constructor func(map[string]interface{}) util.SumoAggOperator
	operators   map[string]util.SumoAggOperator
	merger      Merger
	by          []string
	key         string
	mu          *sync.Mutex
}

type builder func(Merger, string, map[string]interface{}) util.SumoAggOperator

func NewAggregate(
	constructor builder,
	by []string,
	key string,
	sortCol string) Grouper {
	merger := NewMerger(sortCol)
	ctor := func(base map[string]interface{}) util.SumoAggOperator {
		return constructor(merger, key, base)
	}

	mu := &sync.Mutex{}
	return Grouper{ctor, make(map[string]util.SumoAggOperator), merger, by, key, mu}
}

func (g Grouper) Flush() {
	g.mu.Lock()
	defer g.mu.Unlock()
	for _, v := range g.operators {
		v.Flush()
	}
	g.merger.Flush()
}

func (g Grouper) Process(inp map[string]interface{}) {
	g.mu.Lock()
	defer g.mu.Unlock()
	var keys []string
	for _, key := range g.by {
		val, ok := inp[key]
		if ok {
			keys = append(keys, fmt.Sprint(val))
		} else {
			keys = append(keys, "")
		}
	}

	groupKey := strings.Join(keys, "-")

	_, ok := g.operators[groupKey]
	if !ok {
		nextId := len(g.operators)
		base := make(map[string]interface{})
		for i, key := range g.by {
			base[key] = keys[i]
		}
		base[Id] = nextId
		g.operators[groupKey] = g.constructor(base)
	}
	op, _ := g.operators[groupKey]
	op.Process(inp)
}

type Merger struct {
	// one map for each grouper
	aggregate map[int]map[string]interface{}
	output    *util.JsonWriter
	mu        *sync.Mutex
	sortCol   string
}

func NewMerger(sortCol string) Merger {
	mu := &sync.Mutex{}
	m := Merger{make(map[int]map[string]interface{}), util.NewJsonWriter(), mu, sortCol}
	return m
}

const Id = "_Id"

func ExtractId(inp map[string]interface{}) int {
	raw, ok := inp[Id].(int)
	if ok {
		return raw
	} else {
		return -1
	}
}

func WithId(id int) map[string]interface{} {
	return map[string]interface{}{Id: id}
}

func (m Merger) Process(inp map[string]interface{}) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if !util.IsStartRelation(inp) && !util.IsEndRelation(inp) {
		m.aggregate[ExtractId(inp)] = inp
	}
}

func (m Merger) Write(inp map[string]interface{}) {
	m.Process(inp)
}

func (m Merger) Flush() {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.output.Write(util.CreateStartRelationMeta("merger"))
	// Output keys sorted by index so the ui is consistent
	if m.sortCol == "" {
		for i := 0; i < len(m.aggregate); i++ {
			m.output.Write(util.CreateRelation(m.aggregate[i]))
		}
	} else {
		aggs := make([]map[string]interface{}, 0, len(m.aggregate))
		for i := 0; i < len(m.aggregate); i++ {
			aggs = append(aggs, m.aggregate[i])
		}
		util.SortByField(m.sortCol, aggs)
		for i := 0; i < len(aggs); i++ {
			m.output.Write(util.CreateRelation(aggs[i]))

		}
	}
	m.output.Write(util.CreateEndRelation())
	queryString := strings.Join(os.Args[0:], " ")
	m.output.Write(util.CreateMeta(map[string]interface{}{"_queryString": queryString}))
}
<commit_msg>Delete unused code<commit_after>package grouper

import (
	"fmt"
	"github.com/SumoLogic/sumoshell/util"
	"os"
	"strings"
	"sync"
)

type Grouper struct {
	constructor func(map[string]interface{}) util.SumoAggOperator
	operators   map[string]util.SumoAggOperator
	merger      Merger
	by          []string
	key         string
	mu          *sync.Mutex
}

type builder func(Merger, string, map[string]interface{}) util.SumoAggOperator

func NewAggregate(
	constructor builder,
	by []string,
	key string,
	sortCol string) Grouper {
	merger := NewMerger(sortCol)
	ctor := func(base map[string]interface{}) util.SumoAggOperator {
		return constructor(merger, key, base)
	}

	mu := &sync.Mutex{}
	return Grouper{ctor, make(map[string]util.SumoAggOperator), merger, by, key, mu}
}

func (g Grouper) Flush() {
	g.mu.Lock()
	defer g.mu.Unlock()
	for _, v := range g.operators {
		v.Flush()
	}
	g.merger.Flush()
}

func (g Grouper) Process(inp map[string]interface{}) {
	g.mu.Lock()
	defer g.mu.Unlock()
	var keys []string
	for _, key := range g.by {
		val, ok := inp[key]
		if ok {
			keys = append(keys, fmt.Sprint(val))
		} else {
			keys = append(keys, "")
		}
	}

	groupKey := strings.Join(keys, "-")

	_, ok := g.operators[groupKey]
	if !ok {
		nextId := len(g.operators)
		base := make(map[string]interface{})
		for i, key := range g.by {
			base[key] = keys[i]
		}
		base[Id] = nextId
		g.operators[groupKey] = g.constructor(base)
	}
	op, _ := g.operators[groupKey]
	op.Process(inp)
}

type Merger struct {
	// one map for each grouper
	aggregate map[int]map[string]interface{}
	output    *util.JsonWriter
	mu        *sync.Mutex
	sortCol   string
}

func NewMerger(sortCol string) Merger {
	mu := &sync.Mutex{}
	m := Merger{make(map[int]map[string]interface{}), util.NewJsonWriter(), mu, sortCol}
	return m
}

const Id = "_Id"

func ExtractId(inp map[string]interface{}) int {
	raw, ok := inp[Id].(int)
	if ok {
		return raw
	} else {
		return -1
	}
}

func (m Merger) Process(inp map[string]interface{}) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if !util.IsStartRelation(inp) && !util.IsEndRelation(inp) {
		m.aggregate[ExtractId(inp)] = inp
	}
}

func (m Merger) Write(inp map[string]interface{}) {
	m.Process(inp)
}

func (m Merger) Flush() {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.output.Write(util.CreateStartRelationMeta("merger"))
	// Output keys sorted by index so the ui is consistent
	if m.sortCol == "" {
		for i := 0; i < len(m.aggregate); i++ {
			m.output.Write(util.CreateRelation(m.aggregate[i]))
		}
	} else {
		aggs := make([]map[string]interface{}, 0, len(m.aggregate))
		for i := 0; i < len(m.aggregate); i++ {
			aggs = append(aggs, m.aggregate[i])
		}
		util.SortByField(m.sortCol, aggs)
		for i := 0; i < len(aggs); i++ {
			m.output.Write(util.CreateRelation(aggs[i]))

		}
	}
	m.output.Write(util.CreateEndRelation())
	queryString := strings.Join(os.Args[0:], " ")
	m.output.Write(util.CreateMeta(map[string]interface{}{"_queryString": queryString}))
}
<|endoftext|>