<commit_before>package shell

import (
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"io"
	"os"
	"sort"
)

func init() {
	Commands = append(Commands, &commandVolumeServerEvacuate{})
}

type commandVolumeServerEvacuate struct {
}

func (c *commandVolumeServerEvacuate) Name() string {
	return "volumeServer.evacuate"
}

func (c *commandVolumeServerEvacuate) Help() string {
	return `move out all data on a volume server

	volumeServer.evacuate -node <host:port>

	This command moves all data away from the volume server.
	The volumes on the volume servers will be redistributed.

	Usually this is used to prepare to shutdown or upgrade the volume server.

	Sometimes a volume can not be moved because there are no
	good destination to meet the replication requirement.
	E.g. a volume replication 001 in a cluster with 2 volume servers can not be moved.
	You can use "-skipNonMoveable" to move the rest volumes.

`
}

func (c *commandVolumeServerEvacuate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	if err = commandEnv.confirmIsLocked(); err != nil {
		return
	}

	vsEvacuateCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	volumeServer := vsEvacuateCommand.String("node", "", "<host>:<port> of the volume server")
	skipNonMoveable := vsEvacuateCommand.Bool("skipNonMoveable", false, "skip volumes that can not be moved")
	applyChange := vsEvacuateCommand.Bool("force", false, "actually apply the changes")
	if err = vsEvacuateCommand.Parse(args); err != nil {
		return nil
	}

	if *volumeServer == "" {
		return fmt.Errorf("need to specify volume server by -node=<host>:<port>")
	}

	return volumeServerEvacuate(commandEnv, *volumeServer, *skipNonMoveable, *applyChange, writer)

}

func volumeServerEvacuate(commandEnv *CommandEnv, volumeServer string, skipNonMoveable, applyChange bool, writer io.Writer) (err error) {
	// 1. confirm the volume server is part of the cluster
	// 2. collect all other volume servers, sort by empty slots
	// 3. move to any other volume server as long as it satisfy the replication requirements

	// list all the volumes
	// collect topology information
	topologyInfo, _, err := collectTopologyInfo(commandEnv)
	if err != nil {
		return err
	}

	if err := evacuateNormalVolumes(commandEnv, topologyInfo, volumeServer, skipNonMoveable, applyChange, writer); err != nil {
		return err
	}

	if err := evacuateEcVolumes(commandEnv, topologyInfo, volumeServer, skipNonMoveable, applyChange, writer); err != nil {
		return err
	}

	return nil
}

func evacuateNormalVolumes(commandEnv *CommandEnv, topologyInfo *master_pb.TopologyInfo, volumeServer string, skipNonMoveable, applyChange bool, writer io.Writer) error {
	// find this volume server
	volumeServers := collectVolumeServersByDc(topologyInfo, "")
	thisNode, otherNodes := nodesOtherThan(volumeServers, volumeServer)
	if thisNode == nil {
		return fmt.Errorf("%s is not found in this cluster", volumeServer)
	}

	// move away normal volumes
	volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)
	for _, diskInfo := range thisNode.info.DiskInfos {
		for _, vol := range diskInfo.VolumeInfos {
			hasMoved, err := moveAwayOneNormalVolume(commandEnv, volumeReplicas, vol, thisNode, otherNodes, applyChange)
			if err != nil {
				return fmt.Errorf("move away volume %d from %s: %v", vol.Id, volumeServer, err)
			}
			if !hasMoved {
				if skipNonMoveable {
					replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(vol.ReplicaPlacement))
					fmt.Fprintf(writer, "skipping non moveable volume %d replication:%s\n", vol.Id, replicaPlacement.String())
				} else {
					return fmt.Errorf("failed to move volume %d from %s", vol.Id, volumeServer)
				}
			}
		}
	}
	return nil
}

func evacuateEcVolumes(commandEnv *CommandEnv, topologyInfo *master_pb.TopologyInfo, volumeServer string, skipNonMoveable, applyChange bool, writer io.Writer) error {
	// find this ec volume server
	ecNodes, _ := collectEcVolumeServersByDc(topologyInfo, "")
	thisNode, otherNodes := ecNodesOtherThan(ecNodes, volumeServer)
	if thisNode == nil {
		return fmt.Errorf("%s is not found in this cluster\n", volumeServer)
	}

	// move away ec volumes
	for _, diskInfo := range thisNode.info.DiskInfos {
		for _, ecShardInfo := range diskInfo.EcShardInfos {
			hasMoved, err := moveAwayOneEcVolume(commandEnv, ecShardInfo, thisNode, otherNodes, applyChange)
			if err != nil {
				return fmt.Errorf("move away volume %d from %s: %v", ecShardInfo.Id, volumeServer, err)
			}
			if !hasMoved {
				if skipNonMoveable {
					fmt.Fprintf(writer, "failed to move away ec volume %d from %s\n", ecShardInfo.Id, volumeServer)
				} else {
					return fmt.Errorf("failed to move away ec volume %d from %s", ecShardInfo.Id, volumeServer)
				}
			}
		}
	}
	return nil
}

func moveAwayOneEcVolume(commandEnv *CommandEnv, ecShardInfo *master_pb.VolumeEcShardInformationMessage, thisNode *EcNode, otherNodes []*EcNode, applyChange bool) (hasMoved bool, err error) {

	for _, shardId := range erasure_coding.ShardBits(ecShardInfo.EcIndexBits).ShardIds() {

		sort.Slice(otherNodes, func(i, j int) bool {
			return otherNodes[i].localShardIdCount(ecShardInfo.Id) < otherNodes[j].localShardIdCount(ecShardInfo.Id)
		})

		for i := 0; i < len(otherNodes); i++ {
			emptyNode := otherNodes[i]
			collectionPrefix := ""
			if ecShardInfo.Collection != "" {
				collectionPrefix = ecShardInfo.Collection + "_"
			}
			fmt.Fprintf(os.Stdout, "moving ec volume %s%d.%d %s => %s\n", collectionPrefix, ecShardInfo.Id, shardId, thisNode.info.Id, emptyNode.info.Id)
			err = moveMountedShardToEcNode(commandEnv, thisNode, ecShardInfo.Collection, needle.VolumeId(ecShardInfo.Id), shardId, emptyNode, applyChange)
			if err != nil {
				return
			} else {
				hasMoved = true
				break
			}
		}
		if !hasMoved {
			return
		}
	}

	return
}

func moveAwayOneNormalVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*VolumeReplica, vol *master_pb.VolumeInformationMessage, thisNode *Node, otherNodes []*Node, applyChange bool) (hasMoved bool, err error) {
	fn := capacityByFreeVolumeCount(types.ToDiskType(vol.DiskType))
	sort.Slice(otherNodes, func(i, j int) bool {
		return otherNodes[i].localVolumeRatio(fn) > otherNodes[j].localVolumeRatio(fn)
	})

	for i := 0; i < len(otherNodes); i++ {
		emptyNode := otherNodes[i]
		hasMoved, err = maybeMoveOneVolume(commandEnv, volumeReplicas, thisNode, vol, emptyNode, applyChange)
		if err != nil {
			return
		}
		if hasMoved {
			break
		}
	}
	return
}

func nodesOtherThan(volumeServers []*Node, thisServer string) (thisNode *Node, otherNodes []*Node) {
	for _, node := range volumeServers {
		if node.info.Id == thisServer {
			thisNode = node
			continue
		}
		otherNodes = append(otherNodes, node)
	}
	return
}

func ecNodesOtherThan(volumeServers []*EcNode, thisServer string) (thisNode *EcNode, otherNodes []*EcNode) {
	for _, node := range volumeServers {
		if node.info.Id == thisServer {
			thisNode = node
			continue
		}
		otherNodes = append(otherNodes, node)
	}
	return
}
<commit_msg>add node.selectedVolumes<commit_after>package shell

import (
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"io"
	"os"
	"sort"
)

func init() {
	Commands = append(Commands, &commandVolumeServerEvacuate{})
}

type commandVolumeServerEvacuate struct {
}

func (c *commandVolumeServerEvacuate) Name() string {
	return "volumeServer.evacuate"
}

func (c *commandVolumeServerEvacuate) Help() string {
	return `move out all data on a volume server

	volumeServer.evacuate -node <host:port>

	This command moves all data away from the volume server.
	The volumes on the volume servers will be redistributed.

	Usually this is used to prepare to shutdown or upgrade the volume server.

	Sometimes a volume can not be moved because there are no
	good destination to meet the replication requirement.
	E.g. a volume replication 001 in a cluster with 2 volume servers can not be moved.
	You can use "-skipNonMoveable" to move the rest volumes.

`
}

func (c *commandVolumeServerEvacuate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	if err = commandEnv.confirmIsLocked(); err != nil {
		return
	}

	vsEvacuateCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	volumeServer := vsEvacuateCommand.String("node", "", "<host>:<port> of the volume server")
	skipNonMoveable := vsEvacuateCommand.Bool("skipNonMoveable", false, "skip volumes that can not be moved")
	applyChange := vsEvacuateCommand.Bool("force", false, "actually apply the changes")
	if err = vsEvacuateCommand.Parse(args); err != nil {
		return nil
	}

	if *volumeServer == "" {
		return fmt.Errorf("need to specify volume server by -node=<host>:<port>")
	}

	return volumeServerEvacuate(commandEnv, *volumeServer, *skipNonMoveable, *applyChange, writer)

}

func volumeServerEvacuate(commandEnv *CommandEnv, volumeServer string, skipNonMoveable, applyChange bool, writer io.Writer) (err error) {
	// 1. confirm the volume server is part of the cluster
	// 2. collect all other volume servers, sort by empty slots
	// 3. move to any other volume server as long as it satisfy the replication requirements

	// list all the volumes
	// collect topology information
	topologyInfo, _, err := collectTopologyInfo(commandEnv)
	if err != nil {
		return err
	}

	if err := evacuateNormalVolumes(commandEnv, topologyInfo, volumeServer, skipNonMoveable, applyChange, writer); err != nil {
		return err
	}

	if err := evacuateEcVolumes(commandEnv, topologyInfo, volumeServer, skipNonMoveable, applyChange, writer); err != nil {
		return err
	}

	return nil
}

func evacuateNormalVolumes(commandEnv *CommandEnv, topologyInfo *master_pb.TopologyInfo, volumeServer string, skipNonMoveable, applyChange bool, writer io.Writer) error {
	// find this volume server
	volumeServers := collectVolumeServersByDc(topologyInfo, "")
	thisNode, otherNodes := nodesOtherThan(volumeServers, volumeServer)
	if thisNode == nil {
		return fmt.Errorf("%s is not found in this cluster", volumeServer)
	}

	// move away normal volumes
	volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)
	for _, diskInfo := range thisNode.info.DiskInfos {
		for _, vol := range diskInfo.VolumeInfos {
			hasMoved, err := moveAwayOneNormalVolume(commandEnv, volumeReplicas, vol, thisNode, otherNodes, applyChange)
			if err != nil {
				return fmt.Errorf("move away volume %d from %s: %v", vol.Id, volumeServer, err)
			}
			if !hasMoved {
				if skipNonMoveable {
					replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(vol.ReplicaPlacement))
					fmt.Fprintf(writer, "skipping non moveable volume %d replication:%s\n", vol.Id, replicaPlacement.String())
				} else {
					return fmt.Errorf("failed to move volume %d from %s", vol.Id, volumeServer)
				}
			}
		}
	}
	return nil
}

func evacuateEcVolumes(commandEnv *CommandEnv, topologyInfo *master_pb.TopologyInfo, volumeServer string, skipNonMoveable, applyChange bool, writer io.Writer) error {
	// find this ec volume server
	ecNodes, _ := collectEcVolumeServersByDc(topologyInfo, "")
	thisNode, otherNodes := ecNodesOtherThan(ecNodes, volumeServer)
	if thisNode == nil {
		return fmt.Errorf("%s is not found in this cluster\n", volumeServer)
	}

	// move away ec volumes
	for _, diskInfo := range thisNode.info.DiskInfos {
		for _, ecShardInfo := range diskInfo.EcShardInfos {
			hasMoved, err := moveAwayOneEcVolume(commandEnv, ecShardInfo, thisNode, otherNodes, applyChange)
			if err != nil {
				return fmt.Errorf("move away volume %d from %s: %v", ecShardInfo.Id, volumeServer, err)
			}
			if !hasMoved {
				if skipNonMoveable {
					fmt.Fprintf(writer, "failed to move away ec volume %d from %s\n", ecShardInfo.Id, volumeServer)
				} else {
					return fmt.Errorf("failed to move away ec volume %d from %s", ecShardInfo.Id, volumeServer)
				}
			}
		}
	}
	return nil
}

func moveAwayOneEcVolume(commandEnv *CommandEnv, ecShardInfo *master_pb.VolumeEcShardInformationMessage, thisNode *EcNode, otherNodes []*EcNode, applyChange bool) (hasMoved bool, err error) {

	for _, shardId := range erasure_coding.ShardBits(ecShardInfo.EcIndexBits).ShardIds() {

		sort.Slice(otherNodes, func(i, j int) bool {
			return otherNodes[i].localShardIdCount(ecShardInfo.Id) < otherNodes[j].localShardIdCount(ecShardInfo.Id)
		})

		for i := 0; i < len(otherNodes); i++ {
			emptyNode := otherNodes[i]
			collectionPrefix := ""
			if ecShardInfo.Collection != "" {
				collectionPrefix = ecShardInfo.Collection + "_"
			}
			fmt.Fprintf(os.Stdout, "moving ec volume %s%d.%d %s => %s\n", collectionPrefix, ecShardInfo.Id, shardId, thisNode.info.Id, emptyNode.info.Id)
			err = moveMountedShardToEcNode(commandEnv, thisNode, ecShardInfo.Collection, needle.VolumeId(ecShardInfo.Id), shardId, emptyNode, applyChange)
			if err != nil {
				return
			} else {
				hasMoved = true
				break
			}
		}
		if !hasMoved {
			return
		}
	}

	return
}

func moveAwayOneNormalVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*VolumeReplica, vol *master_pb.VolumeInformationMessage, thisNode *Node, otherNodes []*Node, applyChange bool) (hasMoved bool, err error) {
	fn := capacityByFreeVolumeCount(types.ToDiskType(vol.DiskType))
	for _, n := range otherNodes {
		n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool {
			return v.DiskType == vol.DiskType
		})
	}
	sort.Slice(otherNodes, func(i, j int) bool {
		return otherNodes[i].localVolumeRatio(fn) > otherNodes[j].localVolumeRatio(fn)
	})

	for i := 0; i < len(otherNodes); i++ {
		emptyNode := otherNodes[i]
		hasMoved, err = maybeMoveOneVolume(commandEnv, volumeReplicas, thisNode, vol, emptyNode, applyChange)
		if err != nil {
			return
		}
		if hasMoved {
			break
		}
	}
	return
}

func nodesOtherThan(volumeServers []*Node, thisServer string) (thisNode *Node, otherNodes []*Node) {
	for _, node := range volumeServers {
		if node.info.Id == thisServer {
			thisNode = node
			continue
		}
		otherNodes = append(otherNodes, node)
	}
	return
}

func ecNodesOtherThan(volumeServers []*EcNode, thisServer string) (thisNode *EcNode, otherNodes []*EcNode) {
	for _, node := range volumeServers {
		if node.info.Id == thisServer {
			thisNode = node
			continue
		}
		otherNodes = append(otherNodes, node)
	}
	return
}
<|endoftext|>


<commit_before>package node

import (
	"fmt"
	"io/ioutil"
	"math"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"path"
	"reflect"
	"testing"
	"time"

	"github.com/ethereum/go-ethereum/les"
	gethnode "github.com/ethereum/go-ethereum/node"
	"github.com/ethereum/go-ethereum/p2p"
	"github.com/ethereum/go-ethereum/p2p/enode"

	"github.com/status-im/status-go/whisper/v6"

	"github.com/stretchr/testify/require"

	"github.com/status-im/status-go/discovery"
	"github.com/status-im/status-go/params"
	"github.com/status-im/status-go/t/helpers"
	"github.com/status-im/status-go/t/utils"
)

func TestStatusNodeStart(t *testing.T) {
	config, err := utils.MakeTestNodeConfigWithDataDir("", "", params.StatusChainNetworkID)
	require.NoError(t, err)
	n := New()

	// checks before node is started
	require.Nil(t, n.GethNode())
	require.Nil(t, n.Config())
	require.Nil(t, n.RPCClient())
	require.Equal(t, 0, n.PeerCount())
	// start node
	require.NoError(t, n.Start(config, nil))

	// checks after node is started
	require.True(t, n.IsRunning())
	require.NotNil(t, n.GethNode())
	require.NotNil(t, n.Config())
	require.NotNil(t, n.RPCClient())
	require.Equal(t, 0, n.PeerCount())
	accountManager, err := n.AccountManager()
	require.Nil(t, err)
	require.NotNil(t, accountManager)
	// try to start already started node
	require.EqualError(t, n.Start(config, nil), ErrNodeRunning.Error())

	// stop node
	require.NoError(t, n.Stop())
	// try to stop already stopped node
	require.EqualError(t, n.Stop(), ErrNoRunningNode.Error())

	// checks after node is stopped
	require.Nil(t, n.GethNode())
	require.Nil(t, n.RPCClient())
	require.Equal(t, 0, n.PeerCount())
}

func TestStatusNodeWithDataDir(t *testing.T) {
	var err error

	dir, err := ioutil.TempDir("", "status-node-test")
	require.NoError(t, err)
	defer func() {
		require.NoError(t, os.RemoveAll(dir))
	}()

	// keystore directory
	keyStoreDir := path.Join(dir, "keystore")
	err = os.MkdirAll(keyStoreDir, os.ModePerm)
	require.NoError(t, err)

	config := params.NodeConfig{
		DataDir:     dir,
		KeyStoreDir: keyStoreDir,
	}
	n := New()

	require.NoError(t, n.Start(&config, nil))
	require.NoError(t, n.Stop())
}

func TestStatusNodeServiceGetters(t *testing.T) {
	config := params.NodeConfig{
		EnableNTPSync: true,
		WhisperConfig: params.WhisperConfig{
			Enabled: true,
		},
		LightEthConfig: params.LightEthConfig{
			Enabled: true,
		},
	}
	n := New()

	var (
		instance interface{}
		err      error
	)

	services := []struct {
		getter func() (interface{}, error)
		typ    reflect.Type
	}{
		{
			getter: func() (interface{}, error) {
				return n.WhisperService()
			},
			typ: reflect.TypeOf(&whisper.Whisper{}),
		},
		{
			getter: func() (interface{}, error) {
				return n.LightEthereumService()
			},
			typ: reflect.TypeOf(&les.LightEthereum{}),
		},
	}

	for _, service := range services {
		t.Run(service.typ.String(), func(t *testing.T) {
			// checks before node is started
			instance, err = service.getter()
			require.EqualError(t, err, ErrNoRunningNode.Error())
			require.Nil(t, instance)

			// start node
			require.NoError(t, n.Start(&config, nil))

			// checks after node is started
			instance, err = service.getter()
			require.NoError(t, err)
			require.NotNil(t, instance)
			require.Equal(t, service.typ, reflect.TypeOf(instance))

			// stop node
			require.NoError(t, n.Stop())

			// checks after node is stopped
			instance, err = service.getter()
			require.EqualError(t, err, ErrNoRunningNode.Error())
			require.Nil(t, instance)
		})
	}
}

func TestStatusNodeAddPeer(t *testing.T) {
	var err error

	peer, err := gethnode.New(&gethnode.Config{
		P2P: p2p.Config{
			MaxPeers:    math.MaxInt32,
			NoDiscovery: true,
			ListenAddr:  ":0",
		},
		NoUSB: true,
	})
	require.NoError(t, err)
	require.NoError(t, peer.Start())
	defer func() { require.NoError(t, peer.Stop()) }()
	peerURL := peer.Server().Self().URLv4()

	n := New()

	// checks before node is started
	require.EqualError(t, n.AddPeer(peerURL), ErrNoRunningNode.Error())

	// start status node
	config := params.NodeConfig{
		MaxPeers: math.MaxInt32,
	}
	require.NoError(t, n.Start(&config, nil))
	defer func() { require.NoError(t, n.Stop()) }()

	errCh := helpers.WaitForPeerAsync(n.Server(), peerURL, p2p.PeerEventTypeAdd, time.Second*5)

	// checks after node is started
	require.NoError(t, n.AddPeer(peerURL))
	require.NoError(t, <-errCh)
	require.Equal(t, 1, n.PeerCount())
}

func TestStatusNodeReconnectStaticPeers(t *testing.T) {
	var err error

	peer, err := gethnode.New(&gethnode.Config{
		P2P: p2p.Config{
			MaxPeers:    math.MaxInt32,
			NoDiscovery: true,
			ListenAddr:  ":0",
		},
		NoUSB: true,
	})
	require.NoError(t, err)
	require.NoError(t, peer.Start())
	defer func() { require.NoError(t, peer.Stop()) }()

	var errCh <-chan error

	peerURL := peer.Server().Self().URLv4()
	n := New()

	// checks before node is started
	require.EqualError(t, n.ReconnectStaticPeers(), ErrNoRunningNode.Error())

	// start status node
	config := params.NodeConfig{
		MaxPeers: math.MaxInt32,
		ClusterConfig: params.ClusterConfig{
			Enabled:     true,
			StaticNodes: []string{peerURL},
		},
	}
	require.NoError(t, n.Start(&config, nil))
	defer func() { require.NoError(t, n.Stop()) }()

	// checks after node is started
	// it may happen that the peer is already connected
	// because it was already added to `StaticNodes`
	connected, err := isPeerConnected(n, peerURL)
	require.NoError(t, err)
	if !connected {
		errCh = helpers.WaitForPeerAsync(n.Server(), peerURL, p2p.PeerEventTypeAdd, time.Second*30)
		require.NoError(t, <-errCh)
	}
	require.Equal(t, 1, n.PeerCount())
	require.Equal(t, peer.Server().Self().ID().String(), n.GethNode().Server().PeersInfo()[0].ID)

	// reconnect static peers
	errDropCh := helpers.WaitForPeerAsync(n.Server(), peerURL, p2p.PeerEventTypeDrop, time.Second*30)

	// it takes at least 30 seconds to bring back previously connected peer
	errAddCh := helpers.WaitForPeerAsync(n.Server(), peerURL, p2p.PeerEventTypeAdd, time.Second*60)
	require.NoError(t, n.ReconnectStaticPeers())
	// first check if a peer gets disconnected
	require.NoError(t, <-errDropCh)
	require.NoError(t, <-errAddCh)
}

func isPeerConnected(node *StatusNode, peerURL string) (bool, error) {
	if !node.IsRunning() {
		return false, ErrNoRunningNode
	}

	parsedPeer, err := enode.ParseV4(peerURL)
	if err != nil {
		return false, err
	}

	server := node.GethNode().Server()

	for _, peer := range server.PeersInfo() {
		if peer.ID == parsedPeer.ID().String() {
			return true, nil
		}
	}

	return false, nil
}

func TestStatusNodeRendezvousDiscovery(t *testing.T) {
	config := params.NodeConfig{
		Rendezvous:  true,
		NoDiscovery: true,
		ClusterConfig: params.ClusterConfig{
			Enabled: true,
			// not necessarily with id, just valid multiaddr
			RendezvousNodes: []string{"/ip4/127.0.0.1/tcp/34012", "/ip4/127.0.0.1/tcp/34011"},
		},
		// use custom address to test the all possibilities
		AdvertiseAddr: "127.0.0.1",
	}
	n := New()
	require.NoError(t, n.Start(&config, nil))
	require.NotNil(t, n.discovery)
	require.True(t, n.discovery.Running())
	require.IsType(t, &discovery.Rendezvous{}, n.discovery)
}

func TestStatusNodeStartDiscoveryManual(t *testing.T) {
	config := params.NodeConfig{
		Rendezvous:  true,
		NoDiscovery: true,
		ClusterConfig: params.ClusterConfig{
			Enabled: true,
			// not necessarily with id, just valid multiaddr
			RendezvousNodes: []string{"/ip4/127.0.0.1/tcp/34012", "/ip4/127.0.0.1/tcp/34011"},
		},
		// use custom address to test the all possibilities
		AdvertiseAddr: "127.0.0.1",
	}
	n := New()
	require.NoError(t, n.StartWithOptions(&config, StartOptions{}))
	require.Nil(t, n.discovery)
	// start discovery manually
	require.NoError(t, n.StartDiscovery())
	require.NotNil(t, n.discovery)
	require.True(t, n.discovery.Running())
	require.IsType(t, &discovery.Rendezvous{}, n.discovery)
}

func TestStatusNodeDiscoverNode(t *testing.T) {
	config := params.NodeConfig{
		NoDiscovery: true,
		ListenAddr:  "127.0.0.1:0",
	}
	n := New()
	require.NoError(t, n.Start(&config, nil))
	node, err := n.discoverNode()
	require.NoError(t, err)
	require.Equal(t, net.ParseIP("127.0.0.1").To4(), node.IP())

	config = params.NodeConfig{
		NoDiscovery:   true,
		AdvertiseAddr: "127.0.0.2",
		ListenAddr:    "127.0.0.1:0",
	}
	n = New()
	require.NoError(t, n.Start(&config, nil))
	node, err = n.discoverNode()
	require.NoError(t, err)
	require.Equal(t, net.ParseIP("127.0.0.2").To4(), node.IP())
}

func TestChaosModeCheckRPCClientsUpstreamURL(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, `{
			"id": 1,
			"jsonrpc": "2.0",
			"result": 1
		}`)
	}))
	defer ts.Close()

	config := params.NodeConfig{
		NoDiscovery: true,
		ListenAddr:  "127.0.0.1:0",
		UpstreamConfig: params.UpstreamRPCConfig{
			Enabled: true,
			// put "infura.io" substring to simulate blocking an actual infura.io URLs
			URL: ts.URL + "?actualURL=infura.io",
		},
	}
	n := New()
	require.NoError(t, n.Start(&config, nil))
	defer func() { require.NoError(t, n.Stop()) }()
	require.NotNil(t, n.RPCClient())

	client := n.RPCClient()
	require.NotNil(t, client)

	err := client.Call(nil, "net_version")
	require.NoError(t, err)

	// act
	err = n.ChaosModeCheckRPCClientsUpstreamURL(true)
	require.NoError(t, err)

	// assert
	err = client.Call(nil, "net_version")
	require.Error(t, err)

	// act
	err = n.ChaosModeCheckRPCClientsUpstreamURL(false)
	require.NoError(t, err)

	// assert
	err = client.Call(nil, "net_version")
	require.NoError(t, err)
}
<commit_msg>Skip flaky test<commit_after>package node

import (
	"fmt"
	"io/ioutil"
	"math"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"path"
	"reflect"
	"testing"
	"time"

	"github.com/ethereum/go-ethereum/les"
	gethnode "github.com/ethereum/go-ethereum/node"
	"github.com/ethereum/go-ethereum/p2p"
	"github.com/ethereum/go-ethereum/p2p/enode"

	"github.com/status-im/status-go/whisper/v6"

	"github.com/stretchr/testify/require"

	"github.com/status-im/status-go/discovery"
	"github.com/status-im/status-go/params"
	"github.com/status-im/status-go/t/helpers"
	"github.com/status-im/status-go/t/utils"
)

func TestStatusNodeStart(t *testing.T) {
	config, err := utils.MakeTestNodeConfigWithDataDir("", "", params.StatusChainNetworkID)
	require.NoError(t, err)
	n := New()

	// checks before node is started
	require.Nil(t, n.GethNode())
	require.Nil(t, n.Config())
	require.Nil(t, n.RPCClient())
	require.Equal(t, 0, n.PeerCount())
	// start node
	require.NoError(t, n.Start(config, nil))

	// checks after node is started
	require.True(t, n.IsRunning())
	require.NotNil(t, n.GethNode())
	require.NotNil(t, n.Config())
	require.NotNil(t, n.RPCClient())
	require.Equal(t, 0, n.PeerCount())
	accountManager, err := n.AccountManager()
	require.Nil(t, err)
	require.NotNil(t, accountManager)
	// try to start already started node
	require.EqualError(t, n.Start(config, nil), ErrNodeRunning.Error())

	// stop node
	require.NoError(t, n.Stop())
	// try to stop already stopped node
	require.EqualError(t, n.Stop(), ErrNoRunningNode.Error())

	// checks after node is stopped
	require.Nil(t, n.GethNode())
	require.Nil(t, n.RPCClient())
	require.Equal(t, 0, n.PeerCount())
}

func TestStatusNodeWithDataDir(t *testing.T) {
	var err error

	dir, err := ioutil.TempDir("", "status-node-test")
	require.NoError(t, err)
	defer func() {
		require.NoError(t, os.RemoveAll(dir))
	}()

	// keystore directory
	keyStoreDir := path.Join(dir, "keystore")
	err = os.MkdirAll(keyStoreDir, os.ModePerm)
	require.NoError(t, err)

	config := params.NodeConfig{
		DataDir:     dir,
		KeyStoreDir: keyStoreDir,
	}
	n := New()

	require.NoError(t, n.Start(&config, nil))
	require.NoError(t, n.Stop())
}

func TestStatusNodeServiceGetters(t *testing.T) {
	config := params.NodeConfig{
		EnableNTPSync: true,
		WhisperConfig: params.WhisperConfig{
			Enabled: true,
		},
		LightEthConfig: params.LightEthConfig{
			Enabled: true,
		},
	}
	n := New()

	var (
		instance interface{}
		err      error
	)

	services := []struct {
		getter func() (interface{}, error)
		typ    reflect.Type
	}{
		{
			getter: func() (interface{}, error) {
				return n.WhisperService()
			},
			typ: reflect.TypeOf(&whisper.Whisper{}),
		},
		{
			getter: func() (interface{}, error) {
				return n.LightEthereumService()
			},
			typ: reflect.TypeOf(&les.LightEthereum{}),
		},
	}

	for _, service := range services {
		t.Run(service.typ.String(), func(t *testing.T) {
			// checks before node is started
			instance, err = service.getter()
			require.EqualError(t, err, ErrNoRunningNode.Error())
			require.Nil(t, instance)

			// start node
			require.NoError(t, n.Start(&config, nil))

			// checks after node is started
			instance, err = service.getter()
			require.NoError(t, err)
			require.NotNil(t, instance)
			require.Equal(t, service.typ, reflect.TypeOf(instance))

			// stop node
			require.NoError(t, n.Stop())

			// checks after node is stopped
			instance, err = service.getter()
			require.EqualError(t, err, ErrNoRunningNode.Error())
			require.Nil(t, instance)
		})
	}
}

func TestStatusNodeAddPeer(t *testing.T) {
	var err error

	peer, err := gethnode.New(&gethnode.Config{
		P2P: p2p.Config{
			MaxPeers:    math.MaxInt32,
			NoDiscovery: true,
			ListenAddr:  ":0",
		},
		NoUSB: true,
	})
	require.NoError(t, err)
	require.NoError(t, peer.Start())
	defer func() { require.NoError(t, peer.Stop()) }()
	peerURL := peer.Server().Self().URLv4()

	n := New()

	// checks before node is started
	require.EqualError(t, n.AddPeer(peerURL), ErrNoRunningNode.Error())

	// start status node
	config := params.NodeConfig{
		MaxPeers: math.MaxInt32,
	}
	require.NoError(t, n.Start(&config, nil))
	defer func() { require.NoError(t, n.Stop()) }()

	errCh := helpers.WaitForPeerAsync(n.Server(), peerURL, p2p.PeerEventTypeAdd, time.Second*5)

	// checks after node is started
	require.NoError(t, n.AddPeer(peerURL))
	require.NoError(t, <-errCh)
	require.Equal(t, 1, n.PeerCount())
}

func TestStatusNodeReconnectStaticPeers(t *testing.T) {
	// Skipping as flaky
	t.Skip()
	var err error

	peer, err := gethnode.New(&gethnode.Config{
		P2P: p2p.Config{
			MaxPeers:    math.MaxInt32,
			NoDiscovery: true,
			ListenAddr:  ":0",
		},
		NoUSB: true,
	})
	require.NoError(t, err)
	require.NoError(t, peer.Start())
	defer func() { require.NoError(t, peer.Stop()) }()

	var errCh <-chan error

	peerURL := peer.Server().Self().URLv4()
	n := New()

	// checks before node is started
	require.EqualError(t, n.ReconnectStaticPeers(), ErrNoRunningNode.Error())

	// start status node
	config := params.NodeConfig{
		MaxPeers: math.MaxInt32,
		ClusterConfig: params.ClusterConfig{
			Enabled:     true,
			StaticNodes: []string{peerURL},
		},
	}
	require.NoError(t, n.Start(&config, nil))
	defer func() { require.NoError(t, n.Stop()) }()

	// checks after node is started
	// it may happen that the peer is already connected
	// because it was already added to `StaticNodes`
	connected, err := isPeerConnected(n, peerURL)
	require.NoError(t, err)
	if !connected {
		errCh = helpers.WaitForPeerAsync(n.Server(), peerURL, p2p.PeerEventTypeAdd, time.Second*30)
		require.NoError(t, <-errCh)
	}
	require.Equal(t, 1, n.PeerCount())
	require.Equal(t, peer.Server().Self().ID().String(), n.GethNode().Server().PeersInfo()[0].ID)

	// reconnect static peers
	errDropCh := helpers.WaitForPeerAsync(n.Server(), peerURL, p2p.PeerEventTypeDrop, time.Second*30)

	// it takes at least 30 seconds to bring back previously connected peer
	errAddCh := helpers.WaitForPeerAsync(n.Server(), peerURL, p2p.PeerEventTypeAdd, time.Second*60)
	require.NoError(t, n.ReconnectStaticPeers())
	// first check if a peer gets disconnected
	require.NoError(t, <-errDropCh)
	require.NoError(t, <-errAddCh)
}

func isPeerConnected(node *StatusNode, peerURL string) (bool, error) {
	if !node.IsRunning() {
		return false, ErrNoRunningNode
	}

	parsedPeer, err := enode.ParseV4(peerURL)
	if err != nil {
		return false, err
	}

	server := node.GethNode().Server()

	for _, peer := range server.PeersInfo() {
		if peer.ID == parsedPeer.ID().String() {
			return true, nil
		}
	}

	return false, nil
}

func TestStatusNodeRendezvousDiscovery(t *testing.T) {
	config := params.NodeConfig{
		Rendezvous:  true,
		NoDiscovery: true,
		ClusterConfig: params.ClusterConfig{
			Enabled: true,
			// not necessarily with id, just valid multiaddr
			RendezvousNodes: []string{"/ip4/127.0.0.1/tcp/34012", "/ip4/127.0.0.1/tcp/34011"},
		},
		// use custom address to test the all possibilities
		AdvertiseAddr: "127.0.0.1",
	}
	n := New()
	require.NoError(t, n.Start(&config, nil))
	require.NotNil(t, n.discovery)
	require.True(t, n.discovery.Running())
	require.IsType(t, &discovery.Rendezvous{}, n.discovery)
}

func TestStatusNodeStartDiscoveryManual(t *testing.T) {
	config := params.NodeConfig{
		Rendezvous:  true,
		NoDiscovery: true,
		ClusterConfig: params.ClusterConfig{
			Enabled: true,
			// not necessarily with id, just valid multiaddr
			RendezvousNodes: []string{"/ip4/127.0.0.1/tcp/34012", "/ip4/127.0.0.1/tcp/34011"},
		},
		// use custom address to test the all possibilities
		AdvertiseAddr: "127.0.0.1",
	}
	n := New()
	require.NoError(t, n.StartWithOptions(&config, StartOptions{}))
	require.Nil(t, n.discovery)
	// start discovery manually
	require.NoError(t, n.StartDiscovery())
	require.NotNil(t, n.discovery)
	require.True(t, n.discovery.Running())
	require.IsType(t, &discovery.Rendezvous{}, n.discovery)
}

func TestStatusNodeDiscoverNode(t *testing.T) {
	config := params.NodeConfig{
		NoDiscovery: true,
		ListenAddr:  "127.0.0.1:0",
	}
	n := New()
	require.NoError(t, n.Start(&config, nil))
	node, err := n.discoverNode()
	require.NoError(t, err)
	require.Equal(t, net.ParseIP("127.0.0.1").To4(), node.IP())

	config = params.NodeConfig{
		NoDiscovery:   true,
		AdvertiseAddr: "127.0.0.2",
		ListenAddr:    "127.0.0.1:0",
	}
	n = New()
	require.NoError(t, n.Start(&config, nil))
	node, err = n.discoverNode()
	require.NoError(t, err)
	require.Equal(t, net.ParseIP("127.0.0.2").To4(), node.IP())
}

func TestChaosModeCheckRPCClientsUpstreamURL(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, `{
			"id": 1,
			"jsonrpc": "2.0",
			"result": 1
		}`)
	}))
	defer ts.Close()

	config := params.NodeConfig{
		NoDiscovery: true,
		ListenAddr:  "127.0.0.1:0",
		UpstreamConfig: params.UpstreamRPCConfig{
			Enabled: true,
			// put "infura.io" substring to simulate blocking an actual infura.io URLs
			URL: ts.URL + "?actualURL=infura.io",
		},
	}
	n := New()
	require.NoError(t, n.Start(&config, nil))
	defer func() { require.NoError(t, n.Stop()) }()
	require.NotNil(t, n.RPCClient())

	client := n.RPCClient()
	require.NotNil(t, client)

	err := client.Call(nil, "net_version")
	require.NoError(t, err)

	// act
	err = n.ChaosModeCheckRPCClientsUpstreamURL(true)
	require.NoError(t, err)

	// assert
	err = client.Call(nil, "net_version")
	require.Error(t, err)

	// act
	err = n.ChaosModeCheckRPCClientsUpstreamURL(false)
	require.NoError(t, err)

	// assert
	err = client.Call(nil, "net_version")
	require.NoError(t, err)
}
<|endoftext|>


<commit_before>// Copyright 2016 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package data

import (
	"fmt"
	"io/ioutil"
	"path"

	"gopkg.in/yaml.v2"
)

// 描述标签信息
type Tag struct {
	Slug      string  `yaml:"slug"`
	Title     string  `yaml:"title"`
	Color     string  `yaml:"color,omitempty"` // 未指定，则继承父容器
	Content   string  `yaml:"content"`
	Posts     []*Post `yaml:"-"` // 关联的文章
	Premalink string  `yaml:"-"`
}

func (d *Data) loadTags(p string) error {
	data, err := ioutil.ReadFile(p)
	if err != nil {
		return err
	}

	tags := make([]*Tag, 0, 100)
	if err = yaml.Unmarshal(data, &tags); err != nil {
		return err
	}
	for index, tag := range tags {
		if len(tag.Slug) == 0 {
			return fmt.Errorf("第[%v]个标签未指slug", index)
		}

		if len(tag.Title) == 0 {
			return fmt.Errorf("第[%v]个标签未指title", index)
		}

		if len(tag.Content) == 0 {
			return fmt.Errorf("第[%v]个标签未指content", index)
		}

		tag.Posts = make([]*Post, 0, 10)
		tag.Premalink = path.Join(d.URLS.Root, d.URLS.Tag, tag.Slug+d.URLS.Suffix)
	}
	d.Tags = tags
	return nil
}
<commit_msg>修正拼写错误<commit_after>// Copyright 2016 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package data

import (
	"fmt"
	"io/ioutil"
	"path"

	"gopkg.in/yaml.v2"
)

// 描述标签信息
type Tag struct {
	Slug      string  `yaml:"slug"`
	Title     string  `yaml:"title"`
	Color     string  `yaml:"color,omitempty"` // 未指定，则继承父容器
	Content   string  `yaml:"content"`
	Posts     []*Post `yaml:"-"` // 关联的文章
	Permalink string  `yaml:"-"`
}

func (d *Data) loadTags(p string) error {
	data, err := ioutil.ReadFile(p)
	if err != nil {
		return err
	}

	tags := make([]*Tag, 0, 100)
	if err = yaml.Unmarshal(data, &tags); err != nil {
		return err
	}
	for index, tag := range tags {
		if len(tag.Slug) == 0 {
			return fmt.Errorf("第[%v]个标签未指slug", index)
		}

		if len(tag.Title) == 0 {
			return fmt.Errorf("第[%v]个标签未指title", index)
		}

		if len(tag.Content) == 0 {
			return fmt.Errorf("第[%v]个标签未指content", index)
		}

		tag.Posts = make([]*Post, 0, 10)
		tag.Permalink = path.Join(d.URLS.Root, d.URLS.Tag, tag.Slug+d.URLS.Suffix)
	}
	d.Tags = tags
	return nil
}
<|endoftext|>


<commit_before>package SchemaOutput

import (
	"fmt"

	"github.com/xitongsys/parquet-go/parquet"
)

func ParquetTypeToParquetTypeStr(pT *parquet.Type, cT *parquet.ConvertedType) (string, string) {
	var pTStr, cTStr string
	if pT != nil {
		switch *pT {
		case parquet.Type_BOOLEAN:
			pTStr = "BOOLEAN"
		case parquet.Type_INT32:
			pTStr = "INT32"
		case parquet.Type_INT64:
			pTStr = "INT64"
		case parquet.Type_INT96:
			pTStr = "INT96"
		case parquet.Type_FLOAT:
			pTStr = "FLOAT"
		case parquet.Type_DOUBLE:
			pTStr = "DOUBLE"
		case parquet.Type_BYTE_ARRAY:
			pTStr = "BYTE_ARRAY"
		case parquet.Type_FIXED_LEN_BYTE_ARRAY:
			pTStr = "FIXED_LEN_BYTE_ARRAY"
		}
	}
	if cT != nil {
		switch *cT {
		case parquet.ConvertedType_UTF8:
			cTStr = "UTF8"
		case parquet.ConvertedType_INT_8:
			cTStr = "INT_8"
		case parquet.ConvertedType_INT_16:
			cTStr = "INT_16"
		case parquet.ConvertedType_INT_32:
			cTStr = "INT_32"
		case parquet.ConvertedType_INT_64:
			cTStr = "INT_64"
		case parquet.ConvertedType_UINT_8:
			cTStr = "UINT_8"
		case parquet.ConvertedType_UINT_16:
			cTStr = "UINT_16"
		case parquet.ConvertedType_UINT_32:
			cTStr = "UINT_32"
		case parquet.ConvertedType_UINT_64:
			cTStr = "UINT_64"
		case parquet.ConvertedType_DATE:
			cTStr = "DATE"
		case parquet.ConvertedType_TIME_MILLIS:
			cTStr = "TIME_MILLIS"
		case parquet.ConvertedType_TIME_MICROS:
			cTStr = "TIME_MICROS"
		case parquet.ConvertedType_TIMESTAMP_MILLIS:
			cTStr = "TIMESTAMP_MILLIS"
		case parquet.ConvertedType_TIMESTAMP_MICROS:
			cTStr = "TIMESTAMP_MICROS"
		case parquet.ConvertedType_INTERVAL:
			cTStr = "INTERVAL"
		case parquet.ConvertedType_DECIMAL:
			cTStr = "DECIMAL"
		case parquet.ConvertedType_MAP:
			cTStr = "MAP"
		case parquet.ConvertedType_LIST:
			cTStr = "LIST"
		}
	}
	return pTStr, cTStr
}

func ParquetTypeToGoTypeStr(pT *parquet.Type, cT *parquet.ConvertedType) string {
	res := ""
	if pT != nil {
		switch *pT {
		case parquet.Type_BOOLEAN:
			res = "bool"
		case parquet.Type_INT32:
			res = "int32"
		case parquet.Type_INT64:
			res = "int64"
		case parquet.Type_INT96:
			res = "string"
		case parquet.Type_FLOAT:
			res = "float32"
		case parquet.Type_DOUBLE:
			res = "float64"
		case parquet.Type_BYTE_ARRAY:
			res = "string"
		case parquet.Type_FIXED_LEN_BYTE_ARRAY:
			res = "string"
		}
	}
	if cT != nil {
		switch *cT {
		case parquet.ConvertedType_UINT_8:
			res = "uint32"
		case parquet.ConvertedType_UINT_16:
			res = "uint32"
		case parquet.ConvertedType_UINT_32:
			res = "uint32"
		case parquet.ConvertedType_UINT_64:
			res = "uint64"
		}
	}
	return res
}

type Node struct {
	SE       *parquet.SchemaElement
	Children []*Node
}

func NewNode(schema *parquet.SchemaElement) *Node {
	node := &(Node{
		SE:       schema,
		Children: []*Node{},
	})
	return node
}

func (self *Node) OutputJsonSchema() string {
	res := "{\"Tag\":"
	pT, cT := self.SE.Type, self.SE.ConvertedType
	rTStr := "REQUIRED"
	if self.SE.GetRepetitionType() == parquet.FieldRepetitionType_OPTIONAL {
		rTStr = "OPTIONAL"
	} else if self.SE.GetRepetitionType() == parquet.FieldRepetitionType_REPEATED {
		rTStr = "REPEATED"
	}

	pTStr, cTStr := ParquetTypeToParquetTypeStr(pT, cT)
	tagStr := "\"name=%s, type=%s, repetitiontype=%s\""
	name := self.SE.GetName()

	if len(self.Children) == 0 {
		if *pT == parquet.Type_FIXED_LEN_BYTE_ARRAY && cT == nil {
			length := self.SE.GetTypeLength()
			tagStr = "\"name=%s, type=%s, length=%d, repetitiontype=%s\""
			res += fmt.Sprintf(tagStr, name, pTStr, length, rTStr) + "}"

		} else if cT != nil && *cT == parquet.ConvertedType_DECIMAL {
			scale, precision := self.SE.GetScale(), self.SE.GetPrecision()
			if *pT == parquet.Type_FIXED_LEN_BYTE_ARRAY {
				length := self.SE.GetTypeLength()
				tagStr = "\"name=%s, type=%s, basetype=%s, scale=%d, precision=%s, length=%d, repetitiontype=%s\""
				res += fmt.Sprintf(tagStr, name, cTStr, pTStr, scale, precision, length, rTStr) + "}"
			} else {
				tagStr = "\"name=%s, type=%s, basetype=%s, scale=%d, precision=%s, repetitiontype\""
				res += fmt.Sprintf(tagStr, name, cTStr, pTStr, scale, precision, rTStr) + "}"
			}

		} else {
			typeStr := pTStr
			if cT != nil {
				typeStr = pTStr
			}
			res += fmt.Sprintf(tagStr, name, typeStr, rTStr) + "}"

		}
	} else {

		if cT != nil {
			tagStr = "\"name=%s, type=%s, repetitiontype=%s\""
			res += fmt.Sprintf(tagStr, name, cTStr, rTStr)
		} else {
			tagStr = "\"name=%s, repetitiontype=%s\""
			res += fmt.Sprintf(tagStr, name, rTStr)
		}
		res += ",\n"
		res += "\"Fields\":[\n"

		nodes := self.Children
		if cT != nil {
			nodes = self.Children[0].Children
		}

		for _, cNode := range nodes {
			res += "\t" + cNode.OutputJsonSchema() + "\n"
		}

		res += "]\n"
		res += "}"

	}
	return res
}

func (self *Node) OutputStruct() string {
	name := self.SE.GetName()
	res := name
	pT, cT := self.SE.Type, self.SE.ConvertedType
	rTStr := " "
	if self.SE.GetRepetitionType() == parquet.FieldRepetitionType_OPTIONAL {
		rTStr = " *"
	} else if self.SE.GetRepetitionType() == parquet.FieldRepetitionType_REPEATED {
		rTStr = " []"
	}

	if pT == nil && cT == nil {
		res += rTStr + "struct {\n"
		for _, cNode := range self.Children {
			res += "\t" + cNode.OutputStruct() + "\n"
		}
		res += "}"

	} else if cT != nil && *cT == parquet.ConvertedType_MAP {
		keyNode := self.Children[0].Children[0]
		keyPT, keyCT := keyNode.SE.Type, keyNode.SE.ConvertedType
		keyGoTypeStr := ParquetTypeToGoTypeStr(keyPT, keyCT)
		valNode := self.Children[0].Children[1]
		res += rTStr + "map[" + keyGoTypeStr + "]" + valNode.OutputStruct()[6:] //filter "value "

	} else if cT != nil && *cT == parquet.ConvertedType_LIST {
		cNode := self.Children[0].Children[0]
		res += rTStr + "[]" + cNode.OutputStruct()[8:] //filter "element "

	} else {
		goTypeStr := ParquetTypeToGoTypeStr(pT, cT)
		res += rTStr + goTypeStr
	}
	return res

}

func CreateTree(schemas []*parquet.SchemaElement) *Node {
	pos := 0
	stack := make([]*Node, 0)
	root := NewNode(schemas[0])
	stack = append(stack, root)
	pos++

	for len(stack) > 0 {
		node := stack[len(stack)-1]
		numChildren := int(node.SE.GetNumChildren())
		lc := len(node.Children)
		if lc < numChildren {
			newNode := NewNode(schemas[pos])
			node.Children = append(node.Children, newNode)
			stack = append(stack, newNode)
			pos++
		} else {
			stack = stack[:len(stack)-1]
		}
	}
	return root

}
<commit_msg>del tab in SchemaOutput<commit_after>package SchemaOutput

import (
	"fmt"

	"github.com/xitongsys/parquet-go/parquet"
)

func ParquetTypeToParquetTypeStr(pT *parquet.Type, cT *parquet.ConvertedType) (string, string) {
	var pTStr, cTStr string
	if pT != nil {
		switch *pT {
		case parquet.Type_BOOLEAN:
			pTStr = "BOOLEAN"
		case parquet.Type_INT32:
			pTStr = "INT32"
		case parquet.Type_INT64:
			pTStr = "INT64"
		case parquet.Type_INT96:
			pTStr = "INT96"
		case parquet.Type_FLOAT:
			pTStr = "FLOAT"
		case parquet.Type_DOUBLE:
			pTStr = "DOUBLE"
		case parquet.Type_BYTE_ARRAY:
			pTStr = "BYTE_ARRAY"
		case parquet.Type_FIXED_LEN_BYTE_ARRAY:
			pTStr = "FIXED_LEN_BYTE_ARRAY"
		}
	}
	if cT != nil {
		switch *cT {
		case parquet.ConvertedType_UTF8:
			cTStr = "UTF8"
		case parquet.ConvertedType_INT_8:
			cTStr = "INT_8"
		case parquet.ConvertedType_INT_16:
			cTStr = "INT_16"
		case parquet.ConvertedType_INT_32:
			cTStr = "INT_32"
		case parquet.ConvertedType_INT_64:
			cTStr = "INT_64"
		case parquet.ConvertedType_UINT_8:
			cTStr = "UINT_8"
		case parquet.ConvertedType_UINT_16:
			cTStr = "UINT_16"
		case parquet.ConvertedType_UINT_32:
			cTStr = "UINT_32"
		case parquet.ConvertedType_UINT_64:
			cTStr = "UINT_64"
		case parquet.ConvertedType_DATE:
			cTStr = "DATE"
		case parquet.ConvertedType_TIME_MILLIS:
			cTStr = "TIME_MILLIS"
		case parquet.ConvertedType_TIME_MICROS:
			cTStr = "TIME_MICROS"
		case parquet.ConvertedType_TIMESTAMP_MILLIS:
			cTStr = "TIMESTAMP_MILLIS"
		case parquet.ConvertedType_TIMESTAMP_MICROS:
			cTStr = "TIMESTAMP_MICROS"
		case parquet.ConvertedType_INTERVAL:
			cTStr = "INTERVAL"
		case parquet.ConvertedType_DECIMAL:
			cTStr = "DECIMAL"
		case parquet.ConvertedType_MAP:
			cTStr = "MAP"
		case parquet.ConvertedType_LIST:
			cTStr = "LIST"
		}
	}
	return pTStr, cTStr
}

func ParquetTypeToGoTypeStr(pT *parquet.Type, cT *parquet.ConvertedType) string {
	res := ""
	if pT != nil {
		switch *pT {
		case parquet.Type_BOOLEAN:
			res = "bool"
		case parquet.Type_INT32:
			res = "int32"
		case parquet.Type_INT64:
			res = "int64"
		case parquet.Type_INT96:
			res = "string"
		case parquet.Type_FLOAT:
			res = "float32"
		case parquet.Type_DOUBLE:
			res = "float64"
		case parquet.Type_BYTE_ARRAY:
			res = "string"
		case parquet.Type_FIXED_LEN_BYTE_ARRAY:
			res = "string"
		}
	}
	if cT != nil {
		switch *cT {
		case parquet.ConvertedType_UINT_8:
			res = "uint32"
		case parquet.ConvertedType_UINT_16:
			res = "uint32"
		case parquet.ConvertedType_UINT_32:
			res = "uint32"
		case parquet.ConvertedType_UINT_64:
			res = "uint64"
		}
	}
	return res
}

type Node struct {
	Indent   string
	SE       *parquet.SchemaElement
	Children []*Node
}

func NewNode(schema *parquet.SchemaElement) *Node {
	node := &(Node{
		Indent:   "",
		SE:       schema,
		Children: []*Node{},
	})
	return node
}

func (self *Node) OutputJsonSchema() string {
	res := "{\"Tag\":"
	pT, cT := self.SE.Type, self.SE.ConvertedType
	rTStr := "REQUIRED"
	if self.SE.GetRepetitionType() == parquet.FieldRepetitionType_OPTIONAL {
		rTStr = "OPTIONAL"
	} else if self.SE.GetRepetitionType() == parquet.FieldRepetitionType_REPEATED {
		rTStr = "REPEATED"
	}

	pTStr, cTStr := ParquetTypeToParquetTypeStr(pT, cT)
	tagStr := "\"name=%s, type=%s, repetitiontype=%s\""
	name := self.SE.GetName()

	if len(self.Children) == 0 {
		if *pT == parquet.Type_FIXED_LEN_BYTE_ARRAY && cT == nil {
			length := self.SE.GetTypeLength()
			tagStr = "\"name=%s, type=%s, length=%d, repetitiontype=%s\""
			res += fmt.Sprintf(tagStr, name, pTStr, length, rTStr) + "}"

		} else if cT != nil && *cT == parquet.ConvertedType_DECIMAL {
			scale, precision := self.SE.GetScale(), self.SE.GetPrecision()
			if *pT == parquet.Type_FIXED_LEN_BYTE_ARRAY {
				length := self.SE.GetTypeLength()
				tagStr = "\"name=%s, type=%s, basetype=%s, scale=%d, precision=%s, length=%d, repetitiontype=%s\""
				res += fmt.Sprintf(tagStr, name, cTStr, pTStr, scale, precision, length, rTStr) + "}"
			} else {
				tagStr = "\"name=%s, type=%s, basetype=%s, scale=%d, precision=%s, repetitiontype\""
				res += fmt.Sprintf(tagStr, name, cTStr, pTStr, scale, precision, rTStr) + "}"
			}

		} else {
			typeStr := pTStr
			if cT != nil {
				typeStr = pTStr
			}
			res += fmt.Sprintf(tagStr, name, typeStr, rTStr) + "}"

		}
	} else {

		if cT != nil {
			tagStr = "\"name=%s, type=%s, repetitiontype=%s\""
			res += fmt.Sprintf(tagStr, name, cTStr, rTStr)
		} else {
			tagStr = "\"name=%s, repetitiontype=%s\""
			res += fmt.Sprintf(tagStr, name, rTStr)
		}
		res += ",\n"
		res += "\"Fields\":[\n"

		nodes := self.Children
		if cT != nil {
			nodes = self.Children[0].Children
		}

		for _, cNode := range nodes {
			res += cNode.OutputJsonSchema() + "\n"
		}

		res += "]\n"
		res += "}"

	}
	return res
}

func (self *Node) OutputStruct() string {
	name := self.SE.GetName()
	res := name
	pT, cT := self.SE.Type, self.SE.ConvertedType
	rTStr := " "
	if self.SE.GetRepetitionType() == parquet.FieldRepetitionType_OPTIONAL {
		rTStr = " *"
	} else if self.SE.GetRepetitionType() == parquet.FieldRepetitionType_REPEATED {
		rTStr = " []"
	}

	if pT == nil && cT == nil {
		res += rTStr + "struct {\n"
		for _, cNode := range self.Children {
			res += cNode.OutputStruct() + "\n"
		}
		res += "}"

	} else if cT != nil && *cT == parquet.ConvertedType_MAP {
		keyNode := self.Children[0].Children[0]
		keyPT, keyCT := keyNode.SE.Type, keyNode.SE.ConvertedType
		keyGoTypeStr := ParquetTypeToGoTypeStr(keyPT, keyCT)
		valNode := self.Children[0].Children[1]
		res += rTStr + "map[" + keyGoTypeStr + "]" + valNode.OutputStruct()[6:] //filter "value "

	} else if cT != nil && *cT == parquet.ConvertedType_LIST {
		cNode := self.Children[0].Children[0]
		res += rTStr + "[]" + cNode.OutputStruct()[8:] //filter "element "

	} else {
		goTypeStr := ParquetTypeToGoTypeStr(pT, cT)
		res += rTStr + goTypeStr
	}
	return res

}

func CreateTree(schemas []*parquet.SchemaElement) *Node {
	pos := 0
	stack := make([]*Node, 0)
	root := NewNode(schemas[0])
	stack = append(stack, root)
	pos++

	for len(stack) > 0 {
		node := stack[len(stack)-1]
		numChildren := int(node.SE.GetNumChildren())
		lc := len(node.Children)
		if lc < numChildren {
			newNode := NewNode(schemas[pos])
			newNode.Indent += "\t"
			node.Children = append(node.Children, newNode)
			stack = append(stack, newNode)
			pos++
		} else {
			stack = stack[:len(stack)-1]
		}
	}
	return root

}
<|endoftext|>


<commit_before>package main

import "github.com/go-xorm/xorm"

var orm *xorm.Engine

// InitDatabase sets up the database and xorm
func InitDatabase() error {
	var err error

	// connect to our database
	orm, err = xorm.NewEngine("sqlite3", "/tmp/requests.db")

	if err != nil {
		return err
	}

	err = orm.Sync(new(ClientRequest))
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>This didn't make it in the last commit<commit_after>package main

import (
	"github.com/go-xorm/xorm"
	_ "github.com/mattn/go-sqlite3"
)

var orm *xorm.Engine

// InitDatabase sets up the database and xorm
func InitDatabase() error {
	var err error

	// connect to our database
	orm, err = xorm.NewEngine("sqlite3", "/tmp/requests.db")

	if err != nil {
		return err
	}

	err = orm.Sync(new(ClientRequest))
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"io"
	"strings"

	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
)

// DDB is *sql.DB wrapper.
type DDB struct {
	driver    string
	dsn       string
	escape    string
	rewritten []string
	maxbulk   int
	*sql.DB
	tx *sql.Tx
}

// iTable is import Table data
type iTable struct {
	tablename string
	header    []string
	columns   []string
	place     string
	firstrow  bool
	count     int
	rows      []interface{}
	rownum    int
	sqlpre    string
	sqlstr    string
	stmt      *sql.Stmt
}

// ImportData is import to the table.
func (db *DDB) ImportData(tablename string, header []string, input Input, firstrow bool) error {
	var err error
	columns := make([]string, len(header))
	for i := range header {
		columns[i] = db.escape + header[i] + db.escape
	}
	rows := make([]interface{}, 0, db.maxbulk*len(header))
	itable := &iTable{
		tablename: tablename,
		header:    header,
		columns:   columns,
		firstrow:  firstrow,
		rows:      rows,
		rownum:    0,
	}
	if db.driver == "postgres" {
		err = db.copyImport(itable, input)
	} else {
		err = db.insertImport(itable, input)
	}
	return err
}

func (db *DDB) copyImport(itable *iTable, input Input) error {
	sqlstr := fmt.Sprintf("COPY %s (%s) FROM STDIN", itable.tablename, strings.Join(itable.columns, ","))
	debug.Printf(sqlstr)
	stmt, err := db.tx.Prepare(sqlstr)
	if err != nil {
		return fmt.Errorf("COPY Prepare: %s", err)
	}
	row := make([]interface{}, len(itable.header))
	if itable.firstrow {
		row = input.firstRow(row)
		_, err = stmt.Exec(row...)
		if err != nil {
			return err
		}
	}

	for {
		row, err = input.rowRead(row)
		if err == io.EOF {
			break
		} else if err != nil {
			return fmt.Errorf("Read: %s", err)
		}
		_, err = stmt.Exec(row...)
		if err != nil {
			return err
		}
	}
	stmt.Exec()
	stmt.Close()
	return nil
}

func (db *DDB) insertImport(itable *iTable, input Input) error {
	var err error
	var stmt *sql.Stmt

	row := make([]interface{}, len(itable.header))

	itable.sqlpre = fmt.Sprintf("INSERT INTO %s (%s) VALUES ",
		itable.tablename, strings.Join(itable.columns, ","))

	if itable.firstrow {
		itable.place = "(" + strings.Repeat("?,", len(itable.header)-1) + "?)"
		sqlstr := itable.sqlpre + itable.place
		debug.Printf(sqlstr)

		stmt, err = db.tx.Prepare(sqlstr)
		if err != nil {
			return fmt.Errorf("INSERT Prepare: %s:%s", sqlstr, err)
		}
		row = input.firstRow(row)
		_, err = stmt.Exec(row...)
		if err != nil {
			return err
		}
		stmt.Exec()
		stmt.Close()
	}

	for {
		row, err = input.rowRead(row)
		if err == io.EOF {
			err = nil
			break
		} else if err != nil {
			return fmt.Errorf("Read: %s", err)
		}
		err = db.bulkpush(itable, row)
		if err != nil {
			return err
		}
	}
	if itable.count > 0 {
		err = db.bulkimport(itable)
	}
	itable.stmt.Close()
	return err
}

func (db *DDB) bulkpush(itable *iTable, row []interface{}) error {
	var err error
	itable.count++
	for _, r := range row {
		itable.rows = append(itable.rows, r)
	}
	if (itable.count*len(row))+len(row) > db.maxbulk {
		err = db.bulkimport(itable)
		itable.rows = itable.rows[:0]
		itable.count = 0
	}
	return err
}

func (db *DDB) bulkimport(itable *iTable) error {
	var err error
	if itable.rownum != itable.count {
		if itable.stmt != nil {
			itable.stmt.Close()
		}
		itable.sqlstr = itable.sqlpre +
			strings.Repeat(itable.place+",", itable.count-1) + itable.place
		itable.rownum = itable.count
		debug.Printf(itable.sqlstr)
		itable.stmt, err = db.tx.Prepare(itable.sqlstr)
		if err != nil {
			return fmt.Errorf("INSERT Prepare: %s:%s", itable.sqlstr, err)
		}
	}
	_, err = itable.stmt.Exec(itable.rows...)
	return err
}

// Connect is connects to the database
func Connect(driver, dsn string) (*DDB, error) {
	var db DDB
	var err error
	db.driver = driver
	db.dsn = dsn
	switch driver {
	case "sqlite3":
		db.maxbulk = 1000
		db.escape = "`"
		if dsn == "" {
			db.dsn = ":memory:"
		}
	case "mysql":
		db.escape = "`"
		db.maxbulk = 1000
	case "postgres":
		db.escape = "\""
	}
	db.DB, err = sql.Open(db.driver, db.dsn)
	return &db, err
}

// Disconnect is disconnect the database
func (db *DDB) Disconnect() error {
	err := db.Close()
	return err
}

// Create is create a temporary table
func (db *DDB) Create(table string, header []string) error {
	var sqlstr string
	columns := make([]string, len(header))
	for i := 0; i < len(header); i++ {
		columns[i] = db.escape + header[i] + db.escape + " text"
	}
	sqlstr = "CREATE TEMPORARY TABLE "
	sqlstr = sqlstr + table + " ( " + strings.Join(columns, ",") + " );"
	debug.Printf(sqlstr)
	_, err := db.tx.Exec(sqlstr)
	return err
}

// Select is executes SQL select statements
func (db *DDB) Select(sqlstr string) (*sql.Rows, error) {
	sqlstr = strings.TrimSpace(sqlstr)
	if sqlstr == "" {
		return nil, errors.New("no SQL statement")
	}
	debug.Printf(sqlstr)
	rows, err := db.tx.Query(sqlstr)
	if err != nil {
		return rows, fmt.Errorf("SQL:%s\n[%s]", err, sqlstr)
	}
	return rows, nil
}

func (db *DDB) escapetable(oldname string) string {
	var newname string
	if oldname[0] != db.escape[0] {
		newname = db.escape + oldname + db.escape
	} else {
		newname = oldname
	}
	return newname
}

func (db *DDB) rewrite(sqlstr string, oldname string, newname string) (rewrite string) {
	for _, rewritten := range db.rewritten {
		if rewritten == newname {
			// Rewritten
			return sqlstr
		}
	}
	rewrite = strings.Replace(sqlstr, oldname, newname, -1)
	db.rewritten = append(db.rewritten, newname)
	return rewrite
}

func sqlparse(sqlstr string) []string {
	var tablenames []string
	word := strings.Fields(sqlstr)
	for i, w := range word {
		if element := strings.ToUpper(w); element == "FROM" || element == "JOIN" {
			if (i + 1) < len(word) {
				tablenames = append(tablenames, word[i+1])
			}
		}
	}
	return tablenames
}
<commit_msg>fix place was empty at -ih<commit_after>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"io"
	"strings"

	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
)

// DDB is *sql.DB wrapper.
type DDB struct {
	driver    string
	dsn       string
	escape    string
	rewritten []string
	maxbulk   int
	*sql.DB
	tx *sql.Tx
}

// iTable is import Table data
type iTable struct {
	tablename string
	header    []string
	columns   []string
	place     string
	firstrow  bool
	count     int
	rows      []interface{}
	rownum    int
	sqlpre    string
	sqlstr    string
	stmt      *sql.Stmt
}

// ImportData is import to the table.
func (db *DDB) ImportData(tablename string, header []string, input Input, firstrow bool) error {
	var err error
	columns := make([]string, len(header))
	for i := range header {
		columns[i] = db.escape + header[i] + db.escape
	}
	rows := make([]interface{}, 0, db.maxbulk*len(header))
	itable := &iTable{
		tablename: tablename,
		header:    header,
		columns:   columns,
		firstrow:  firstrow,
		rows:      rows,
		rownum:    0,
	}
	if db.driver == "postgres" {
		err = db.copyImport(itable, input)
	} else {
		err = db.insertImport(itable, input)
	}
	return err
}

func (db *DDB) copyImport(itable *iTable, input Input) error {
	sqlstr := fmt.Sprintf("COPY %s (%s) FROM STDIN", itable.tablename, strings.Join(itable.columns, ","))
	debug.Printf(sqlstr)
	stmt, err := db.tx.Prepare(sqlstr)
	if err != nil {
		return fmt.Errorf("COPY Prepare: %s", err)
	}
	row := make([]interface{}, len(itable.header))
	if itable.firstrow {
		row = input.firstRow(row)
		_, err = stmt.Exec(row...)
		if err != nil {
			return err
		}
	}

	for {
		row, err = input.rowRead(row)
		if err == io.EOF {
			break
		} else if err != nil {
			return fmt.Errorf("Read: %s", err)
		}
		_, err = stmt.Exec(row...)
		if err != nil {
			return err
		}
	}
	stmt.Exec()
	stmt.Close()
	return nil
}

func (db *DDB) insertImport(itable *iTable, input Input) error {
	var err error
	var stmt *sql.Stmt

	row := make([]interface{}, len(itable.header))

	itable.sqlpre = fmt.Sprintf("INSERT INTO %s (%s) VALUES ",
		itable.tablename, strings.Join(itable.columns, ","))

	itable.place = "(" + strings.Repeat("?,", len(itable.header)-1) + "?)"
	if itable.firstrow {
		sqlstr := itable.sqlpre + itable.place
		debug.Printf(sqlstr)

		stmt, err = db.tx.Prepare(sqlstr)
		if err != nil {
			return fmt.Errorf("INSERT Prepare: %s:%s", sqlstr, err)
		}
		row = input.firstRow(row)
		_, err = stmt.Exec(row...)
		if err != nil {
			return err
		}
		stmt.Exec()
		stmt.Close()
	}
	for {
		row, err = input.rowRead(row)
		if err == io.EOF {
			err = nil
			break
		} else if err != nil {
			return fmt.Errorf("Read: %s", err)
		}
		err = db.bulkpush(itable, row)
		if err != nil {
			return err
		}
	}
	if itable.count > 0 {
		err = db.bulkimport(itable)
	}
	itable.stmt.Close()
	return err
}

func (db *DDB) bulkpush(itable *iTable, row []interface{}) error {
	var err error
	itable.count++
	for _, r := range row {
		itable.rows = append(itable.rows, r)
	}
	if (itable.count*len(row))+len(row) > db.maxbulk {
		err = db.bulkimport(itable)
		itable.rows = itable.rows[:0]
		itable.count = 0
	}
	return err
}

func (db *DDB) bulkimport(itable *iTable) error {
	var err error
	if itable.rownum != itable.count {
		if itable.stmt != nil {
			itable.stmt.Close()
		}
		itable.sqlstr = itable.sqlpre +
			strings.Repeat(itable.place+",", itable.count-1) + itable.place
		itable.rownum = itable.count
		debug.Printf(itable.sqlstr)
		itable.stmt, err = db.tx.Prepare(itable.sqlstr)
		if err != nil {
			return fmt.Errorf("INSERT Prepare: %s:%s", itable.sqlstr, err)
		}
	}
	_, err = itable.stmt.Exec(itable.rows...)
	return err
}

// Connect is connects to the database
func Connect(driver, dsn string) (*DDB, error) {
	var db DDB
	var err error
	db.driver = driver
	db.dsn = dsn
	switch driver {
	case "sqlite3":
		db.maxbulk = 1000
		db.escape = "`"
		if dsn == "" {
			db.dsn = ":memory:"
		}
	case "mysql":
		db.escape = "`"
		db.maxbulk = 1000
	case "postgres":
		db.escape = "\""
	}
	db.DB, err = sql.Open(db.driver, db.dsn)
	return &db, err
}

// Disconnect is disconnect the database
func (db *DDB) Disconnect() error {
	err := db.Close()
	return err
}

// Create is create a temporary table
func (db *DDB) Create(table string, header []string) error {
	var sqlstr string
	columns := make([]string, len(header))
	for i := 0; i < len(header); i++ {
		columns[i] = db.escape + header[i] + db.escape + " text"
	}
	sqlstr = "CREATE TEMPORARY TABLE "
	sqlstr = sqlstr + table + " ( " + strings.Join(columns, ",") + " );"
	debug.Printf(sqlstr)
	_, err := db.tx.Exec(sqlstr)
	return err
}

// Select is executes SQL select statements
func (db *DDB) Select(sqlstr string) (*sql.Rows, error) {
	sqlstr = strings.TrimSpace(sqlstr)
	if sqlstr == "" {
		return nil, errors.New("no SQL statement")
	}
	debug.Printf(sqlstr)
	rows, err := db.tx.Query(sqlstr)
	if err != nil {
		return rows, fmt.Errorf("SQL:%s\n[%s]", err, sqlstr)
	}
	return rows, nil
}

func (db *DDB) escapetable(oldname string) string {
	var newname string
	if oldname[0] != db.escape[0] {
		newname = db.escape + oldname + db.escape
	} else {
		newname = oldname
	}
	return newname
}

func (db *DDB) rewrite(sqlstr string, oldname string, newname string) (rewrite string) {
	for _, rewritten := range db.rewritten {
		if rewritten == newname {
			// Rewritten
			return sqlstr
		}
	}
	rewrite = strings.Replace(sqlstr, oldname, newname, -1)
	db.rewritten = append(db.rewritten, newname)
	return rewrite
}

func sqlparse(sqlstr string) []string {
	var tablenames []string
	word := strings.Fields(sqlstr)
	for i, w := range word {
		if element := strings.ToUpper(w); element == "FROM" || element == "JOIN" {
			if (i + 1) < len(word) {
				tablenames = append(tablenames, word[i+1])
			}
		}
	}
	return tablenames
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/packer/packer"
)

// AMIConfig is for common configuration related to creating AMIs.
type AMIConfig struct {
	AMIName         string            `mapstructure:"ami_name"`
	AMIDescription  string            `mapstructure:"ami_description"`
	AMIUsers        []string          `mapstructure:"ami_users"`
	AMIGroups       []string          `mapstructure:"ami_groups"`
	AMIProductCodes []string          `mapstructure:"ami_product_codes"`
	AMIRegions      []string          `mapstructure:"ami_regions"`
	AMITags         map[string]string `mapstructure:"tags"`
	AMIVisibility	string			  `mapstructure:"ami_visibility"`
}

func (c *AMIConfig) Prepare(t *packer.ConfigTemplate) []error {
	if t == nil {
		var err error
		t, err = packer.NewConfigTemplate()
		if err != nil {
			return []error{err}
		}
	}

	templates := map[string]*string{
		"ami_name":        &c.AMIName,
		"ami_description": &c.AMIDescription,
		"ami_visibility":  &c.AMIVisibility,
	}

	errs := make([]error, 0)
	for n, ptr := range templates {
		var err error
		*ptr, err = t.Process(*ptr, nil)
		if err != nil {
			errs = append(
				errs, fmt.Errorf("Error processing %s: %s", n, err))
		}
	}

	sliceTemplates := map[string][]string{
		"ami_users":         c.AMIUsers,
		"ami_groups":        c.AMIGroups,
		"ami_product_codes": c.AMIProductCodes,
		"ami_regions":       c.AMIRegions,
	}

	for n, slice := range sliceTemplates {
		for i, elem := range slice {
			var err error
			slice[i], err = t.Process(elem, nil)
			if err != nil {
				errs = append(
					errs, fmt.Errorf("Error processing %s[%d]: %s", n, i, err))
			}
		}
	}

	if c.AMIName == "" {
		errs = append(errs, fmt.Errorf("ami_name must be specified"))
	}

	if len(c.AMIRegions) > 0 {
		regionSet := make(map[string]struct{})
		regions := make([]string, 0, len(c.AMIRegions))

		for _, region := range c.AMIRegions {
			// If we already saw the region, then don't look again
			if _, ok := regionSet[region]; ok {
				continue
			}

			// Mark that we saw the region
			regionSet[region] = struct{}{}

			// Verify the region is real
			if _, ok := aws.Regions[region]; !ok {
				errs = append(errs, fmt.Errorf("Unknown region: %s", region))
				continue
			}

			regions = append(regions, region)
		}

		c.AMIRegions = regions
	}

	newTags := make(map[string]string)
	for k, v := range c.AMITags {
		k, err := t.Process(k, nil)
		if err != nil {
			errs = append(errs,
				fmt.Errorf("Error processing tag key %s: %s", k, err))
			continue
		}

		v, err := t.Process(v, nil)
		if err != nil {
			errs = append(errs,
				fmt.Errorf("Error processing tag value '%s': %s", v, err))
			continue
		}

		newTags[k] = v
	}

	c.AMITags = newTags

	if len(errs) > 0 {
		return errs
	}

	return nil
}
<commit_msg>fixed format issue<commit_after>package common

import (
	"fmt"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/packer/packer"
)

// AMIConfig is for common configuration related to creating AMIs.
type AMIConfig struct {
	AMIName         string            `mapstructure:"ami_name"`
	AMIDescription  string            `mapstructure:"ami_description"`
	AMIUsers        []string          `mapstructure:"ami_users"`
	AMIGroups       []string          `mapstructure:"ami_groups"`
	AMIProductCodes []string          `mapstructure:"ami_product_codes"`
	AMIRegions      []string          `mapstructure:"ami_regions"`
	AMITags         map[string]string `mapstructure:"tags"`
	AMIVisibility   string            `mapstructure:"ami_visibility"`
}

func (c *AMIConfig) Prepare(t *packer.ConfigTemplate) []error {
	if t == nil {
		var err error
		t, err = packer.NewConfigTemplate()
		if err != nil {
			return []error{err}
		}
	}

	templates := map[string]*string{
		"ami_name":        &c.AMIName,
		"ami_description": &c.AMIDescription,
		"ami_visibility":  &c.AMIVisibility,
	}

	errs := make([]error, 0)
	for n, ptr := range templates {
		var err error
		*ptr, err = t.Process(*ptr, nil)
		if err != nil {
			errs = append(
				errs, fmt.Errorf("Error processing %s: %s", n, err))
		}
	}

	sliceTemplates := map[string][]string{
		"ami_users":         c.AMIUsers,
		"ami_groups":        c.AMIGroups,
		"ami_product_codes": c.AMIProductCodes,
		"ami_regions":       c.AMIRegions,
	}

	for n, slice := range sliceTemplates {
		for i, elem := range slice {
			var err error
			slice[i], err = t.Process(elem, nil)
			if err != nil {
				errs = append(
					errs, fmt.Errorf("Error processing %s[%d]: %s", n, i, err))
			}
		}
	}

	if c.AMIName == "" {
		errs = append(errs, fmt.Errorf("ami_name must be specified"))
	}

	if len(c.AMIRegions) > 0 {
		regionSet := make(map[string]struct{})
		regions := make([]string, 0, len(c.AMIRegions))

		for _, region := range c.AMIRegions {
			// If we already saw the region, then don't look again
			if _, ok := regionSet[region]; ok {
				continue
			}

			// Mark that we saw the region
			regionSet[region] = struct{}{}

			// Verify the region is real
			if _, ok := aws.Regions[region]; !ok {
				errs = append(errs, fmt.Errorf("Unknown region: %s", region))
				continue
			}

			regions = append(regions, region)
		}

		c.AMIRegions = regions
	}

	newTags := make(map[string]string)
	for k, v := range c.AMITags {
		k, err := t.Process(k, nil)
		if err != nil {
			errs = append(errs,
				fmt.Errorf("Error processing tag key %s: %s", k, err))
			continue
		}

		v, err := t.Process(v, nil)
		if err != nil {
			errs = append(errs,
				fmt.Errorf("Error processing tag value '%s': %s", v, err))
			continue
		}

		newTags[k] = v
	}

	c.AMITags = newTags

	if len(errs) > 0 {
		return errs
	}

	return nil
}
<|endoftext|>


<commit_before>package prefixedio

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"sync"
)

// Reader reads from another io.Reader and de-multiplexes line-oriented
// data into different io.Reader streams.
//
// Lines are delimited with the '\n' character.
//
// When `Read` is called, any data that doesn't currently have a prefix
// registered will be discarded. Data won't start being discarded until
// the first Read is called on a prefix. Once the first Read is called,
// data is read until EOF. Therefore, be sure to request all prefix
// readers before issuing any Read calls on any prefixes.
//
// Reads will block if all the readers aren't routinely draining their
// buffers. Therefore, be sure to be actively reading from all registered
// prefixes, otherwise you can encounter deadlock scenarios.
type Reader struct {
	done     bool
	prefixes map[string]*io.PipeWriter
	r        io.Reader

	l    sync.Mutex
	once sync.Once
}

// NewReader creates a new Reader with the given io.Reader.
func NewReader(r io.Reader) (*Reader, error) {
	if r == nil {
		return nil, errors.New("Reader must not be nil")
	}

	return &Reader{r: r}, nil
}

// Prefix returns a new io.Reader that will read data that
// is prefixed with the given prefix.
//
// The read data is line-oriented so calling Read will result
// in a full line of output (including the line separator),
// but is exposed as an io.Reader for useful utility interoperating
// with other Go libraries.
//
// The data read has the prefix stripped, but contains the line
// delimiter.
//
// An empty prefix "" will read the data before any other prefix match is
// found, allowing you to have a default reader before a prefix is matched.
func (r *Reader) Prefix(p string) (io.Reader, error) {
	r.l.Lock()
	defer r.l.Unlock()

	if r.prefixes == nil {
		r.prefixes = make(map[string]*io.PipeWriter)
	}

	if _, ok := r.prefixes[p]; ok {
		return nil, fmt.Errorf("Prefix already registered: %s", p)
	}

	pr, pw := io.Pipe()
	r.prefixes[p] = pw

	if r.done {
		pw.Close()
	}

	return &prefixReader{
		r:  r,
		pr: pr,
	}, nil
}

// init starts the goroutine that reads from the underlying reader
// and sends data to the proper place.
//
// This is safe to call multiple times.
func (r *Reader) init() {
	r.once.Do(func() {
		go r.read()
	})
}

// read runs in a goroutine and performs a read on the reader,
// dispatching lines to prefixes where necessary.
func (r *Reader) read() {
	var err error
	var lastPrefix string
	buf := bufio.NewReader(r.r)

	for {
		var line []byte
		line, err = buf.ReadBytes('\n')

		// If an error occurred and its not an EOF, then report that
		// error to all pipes and exit.
		if err != nil && err != io.EOF {
			break
		}

		// Go through each prefix and write if the line matches.
		// If no lines match, the data is lost.
		var prefix string
		r.l.Lock()
		for p, _ := range r.prefixes {
			if p == "" {
				continue
			}

			if bytes.HasPrefix(line, []byte(p)) {
				prefix = p
				line = line[len(p):]
				break
			}
		}

		if prefix == "" {
			prefix = lastPrefix
		}

		pw, ok := r.prefixes[prefix]
		if ok {
			lastPrefix = prefix

			// Make sure we write all the data before we exit.
			n := 0
			for n < len(line) {
				ni, err := pw.Write(line[n:])
				if err != nil {
					break
				}

				n += ni
			}
		}
		r.l.Unlock()

		if err == io.EOF {
			break
		}
	}

	r.l.Lock()
	defer r.l.Unlock()

	// Mark us done so that we don't create anymore readers
	r.done = true

	// All previous writers should be closed so that the readers
	// properly return an EOF (or another error if we had one)
	for _, pw := range r.prefixes {
		if err != nil && err != io.EOF {
			pw.CloseWithError(err)
		} else {
			pw.Close()
		}
	}
}

type prefixReader struct {
	r  *Reader
	pr io.Reader
}

func (r *prefixReader) Read(p []byte) (int, error) {
	r.r.init()
	return r.pr.Read(p)
}
<commit_msg>reader should flush data after not seeing it for awhile<commit_after>package prefixedio

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"sync"
	"time"
)

// Reader reads from another io.Reader and de-multiplexes line-oriented
// data into different io.Reader streams.
//
// Lines are delimited with the '\n' character.
//
// When `Read` is called, any data that doesn't currently have a prefix
// registered will be discarded. Data won't start being discarded until
// the first Read is called on a prefix. Once the first Read is called,
// data is read until EOF. Therefore, be sure to request all prefix
// readers before issuing any Read calls on any prefixes.
//
// Reads will block if all the readers aren't routinely draining their
// buffers. Therefore, be sure to be actively reading from all registered
// prefixes, otherwise you can encounter deadlock scenarios.
type Reader struct {
	FlushTimeout time.Duration

	done     bool
	prefixes map[string]*io.PipeWriter
	r        io.Reader

	l    sync.Mutex
	once sync.Once
}

// NewReader creates a new Reader with the given io.Reader.
func NewReader(r io.Reader) (*Reader, error) {
	if r == nil {
		return nil, errors.New("Reader must not be nil")
	}

	return &Reader{r: r}, nil
}

// Prefix returns a new io.Reader that will read data that
// is prefixed with the given prefix.
//
// The read data is line-oriented so calling Read will result
// in a full line of output (including the line separator),
// but is exposed as an io.Reader for useful utility interoperating
// with other Go libraries.
//
// The data read has the prefix stripped, but contains the line
// delimiter.
//
// An empty prefix "" will read the data before any other prefix match is
// found, allowing you to have a default reader before a prefix is matched.
func (r *Reader) Prefix(p string) (io.Reader, error) {
	r.l.Lock()
	defer r.l.Unlock()

	if r.prefixes == nil {
		r.prefixes = make(map[string]*io.PipeWriter)
	}

	if _, ok := r.prefixes[p]; ok {
		return nil, fmt.Errorf("Prefix already registered: %s", p)
	}

	pr, pw := io.Pipe()
	r.prefixes[p] = pw

	if r.done {
		pw.Close()
	}

	return &prefixReader{
		r:  r,
		pr: pr,
	}, nil
}

// init starts the goroutine that reads from the underlying reader
// and sends data to the proper place.
//
// This is safe to call multiple times.
func (r *Reader) init() {
	r.once.Do(func() {
		go r.read()
	})
}

// read runs in a goroutine and performs a read on the reader,
// dispatching lines to prefixes where necessary.
func (r *Reader) read() {
	var err error
	var lastPrefix string
	buf := bufio.NewReader(r.r)

	// Listen for bytes in a goroutine. We do this so that if we're blocking
	// we can flush the bytes we have after some configured time. There is
	// probably a way to make this a lot faster but this works for now.
	byteCh := make(chan byte)
	doneCh := make(chan error)
	go func() {
		defer close(doneCh)
		for {
			b, err := buf.ReadByte()
			if err != nil {
				doneCh <- err
				return
			}

			byteCh <- b
		}
	}()

	// Figure out the timeout we wait until we flush if we see no data
	ft := r.FlushTimeout
	if ft == 0 {
		ft = 100 * time.Millisecond
	}

	lineBuf := make([]byte, 0, 80)
	for {
		line := lineBuf[0:0]
		for {
			brk := false

			select {
			case b := <-byteCh:
				line = append(line, b)
				brk = b == '\n'
			case err = <-doneCh:
				brk = true
			case <-time.After(ft):
				brk = true
			}

			if brk {
				break
			}
		}

		// If an error occurred and its not an EOF, then report that
		// error to all pipes and exit.
		if err != nil && err != io.EOF {
			break
		}

		// Go through each prefix and write if the line matches.
		// If no lines match, the data is lost.
		var prefix string
		r.l.Lock()
		for p, _ := range r.prefixes {
			if p == "" {
				continue
			}

			if bytes.HasPrefix(line, []byte(p)) {
				prefix = p
				line = line[len(p):]
				break
			}
		}

		if prefix == "" {
			prefix = lastPrefix
		}

		pw, ok := r.prefixes[prefix]
		if ok {
			lastPrefix = prefix

			// Make sure we write all the data before we exit.
			n := 0
			for n < len(line) {
				ni, err := pw.Write(line[n:])
				if err != nil {
					break
				}

				n += ni
			}
		}
		r.l.Unlock()

		if err == io.EOF {
			break
		}
	}

	r.l.Lock()
	defer r.l.Unlock()

	// Mark us done so that we don't create anymore readers
	r.done = true

	// All previous writers should be closed so that the readers
	// properly return an EOF (or another error if we had one)
	for _, pw := range r.prefixes {
		if err != nil && err != io.EOF {
			pw.CloseWithError(err)
		} else {
			pw.Close()
		}
	}
}

type prefixReader struct {
	r  *Reader
	pr io.Reader
}

func (r *prefixReader) Read(p []byte) (int, error) {
	r.r.init()
	return r.pr.Read(p)
}
<|endoftext|>


<commit_before>package backend

import (
	"regexp"
	"sync"

	"github.com/hashicorp/vault/vault"
)

// Backend is an implementation of vault.LogicalBackend that allows
// the implementer to code a backend using a much more programmer-friendly
// framework that handles a lot of the routing and validation for you.
//
// This is recommended over implementing vault.LogicalBackend directly.
type Backend struct {
	// Paths are the various routes that the backend responds to.
	// This cannot be modified after construction (i.e. dynamically changing
	// paths, including adding or removing, is not allowed once the
	// backend is in use).
	Paths []*Path

	once    sync.Once
	pathsRe []*regexp.Regexp
}

// Path is a single path that the backend responds to.
type Path struct {
	// Pattern is the pattern of the URL that matches this path.
	//
	// This should be a valid regular expression. Named captures will be
	// exposed as fields that should map to a schema in Fields. If a named
	// capture is not a field in the Fields map, then it will be ignored.
	Pattern string

	// Fields is the mapping of data fields to a schema describing that
	// field. Named captures in the Pattern also map to fields. If a named
	// capture name matches a PUT body name, the named capture takes
	// priority.
	//
	// Note that only named capture fields are available in every operation,
	// whereas all fields are avaiable in the Write operation.
	Fields map[string]*FieldSchema

	// Root if not blank, denotes that this path requires root
	// privileges and the path pattern that is the root path. This can't
	// be a regular expression and must be an exact path. It may have a
	// trailing '*' to denote that it is a prefix, and not an exact match.
	Root string

	// Callback is what is called when this path is requested with
	// a valid set of data.
	Callback func(*vault.Request, *FieldData) (*vault.Response, error)
}

func (b *Backend) Route(path string) *Path {
	b.once.Do(b.init)

	for i, re := range b.pathsRe {
		if re.MatchString(path) {
			return b.Paths[i]
		}
	}

	return nil
}

func (b *Backend) init() {
	b.pathsRe = make([]*regexp.Regexp, len(b.Paths))
	for i, p := range b.Paths {
		b.pathsRe[i] = regexp.MustCompile(p.Pattern)
	}
}

// FieldSchema is a basic schema to describe the format of a path field.
type FieldSchema struct {
	Type    FieldType
	Default interface{}
}

// DefaultOrZero returns the default value if it is set, or otherwise
// the zero value of the type.
func (s *FieldSchema) DefaultOrZero() interface{} {
	if s.Default != nil {
		return s.Default
	}

	return s.Type.Zero()
}

func (t FieldType) Zero() interface{} {
	switch t {
	case TypeString:
		return ""
	case TypeInt:
		return 0
	case TypeBool:
		return false
	default:
		panic("unknown type: " + t.String())
	}
}
<commit_msg>helper/backend: store captures for a path<commit_after>package backend

import (
	"regexp"
	"sync"

	"github.com/hashicorp/vault/vault"
)

// Backend is an implementation of vault.LogicalBackend that allows
// the implementer to code a backend using a much more programmer-friendly
// framework that handles a lot of the routing and validation for you.
//
// This is recommended over implementing vault.LogicalBackend directly.
type Backend struct {
	// Paths are the various routes that the backend responds to.
	// This cannot be modified after construction (i.e. dynamically changing
	// paths, including adding or removing, is not allowed once the
	// backend is in use).
	Paths []*Path

	once    sync.Once
	pathsRe []*regexp.Regexp
}

// Path is a single path that the backend responds to.
type Path struct {
	// Pattern is the pattern of the URL that matches this path.
	//
	// This should be a valid regular expression. Named captures will be
	// exposed as fields that should map to a schema in Fields. If a named
	// capture is not a field in the Fields map, then it will be ignored.
	Pattern string

	// Fields is the mapping of data fields to a schema describing that
	// field. Named captures in the Pattern also map to fields. If a named
	// capture name matches a PUT body name, the named capture takes
	// priority.
	//
	// Note that only named capture fields are available in every operation,
	// whereas all fields are avaiable in the Write operation.
	Fields map[string]*FieldSchema

	// Root if not blank, denotes that this path requires root
	// privileges and the path pattern that is the root path. This can't
	// be a regular expression and must be an exact path. It may have a
	// trailing '*' to denote that it is a prefix, and not an exact match.
	Root string

	// Callback is what is called when this path is requested with
	// a valid set of data.
	Callback func(*vault.Request, *FieldData) (*vault.Response, error)
}

// Route looks up the path that would be used for a given path string.
func (b *Backend) Route(path string) *Path {
	result, _ := b.route(path)
	return result
}

func (b *Backend) init() {
	b.pathsRe = make([]*regexp.Regexp, len(b.Paths))
	for i, p := range b.Paths {
		b.pathsRe[i] = regexp.MustCompile(p.Pattern)
	}
}

func (b *Backend) route(path string) (*Path, map[string]string) {
	b.once.Do(b.init)

	for i, re := range b.pathsRe {
		matches := re.FindStringSubmatch(path)
		if matches == nil {
			continue
		}

		// We have a match, determine the mapping of the captures and
		// store that for returning.
		var captures map[string]string
		path := b.Paths[i]
		if captureNames := re.SubexpNames(); len(captureNames) > 1 {
			captures = make(map[string]string, len(captureNames))
			for i, name := range captureNames {
				if name != "" {
					captures[name] = matches[i]
				}
			}
		}

		return path, captures
	}

	return nil, nil
}

// FieldSchema is a basic schema to describe the format of a path field.
type FieldSchema struct {
	Type    FieldType
	Default interface{}
}

// DefaultOrZero returns the default value if it is set, or otherwise
// the zero value of the type.
func (s *FieldSchema) DefaultOrZero() interface{} {
	if s.Default != nil {
		return s.Default
	}

	return s.Type.Zero()
}

func (t FieldType) Zero() interface{} {
	switch t {
	case TypeString:
		return ""
	case TypeInt:
		return 0
	case TypeBool:
		return false
	default:
		panic("unknown type: " + t.String())
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package discovery_test

import (
	"testing"

	v23 "v.io/v23"
	"v.io/v23/discovery"
	"v.io/v23/naming"
	idiscovery "v.io/x/ref/lib/discovery"
	fdiscovery "v.io/x/ref/lib/discovery/factory"
	"v.io/x/ref/lib/discovery/plugins/mock"
	"v.io/x/ref/lib/discovery/testutil"
	_ "v.io/x/ref/runtime/factories/generic"
	"v.io/x/ref/test"
)

func withNewAddresses(ad *discovery.Advertisement, eps []naming.Endpoint, suffix string) discovery.Advertisement {
	newAd := *ad
	newAd.Addresses = make([]string, len(eps))
	for i, ep := range eps {
		newAd.Addresses[i] = naming.JoinAddressName(ep.Name(), suffix)
	}
	return newAd
}

func TestAdvertiseServer(t *testing.T) {
	ctx, shutdown := test.V23Init()
	defer shutdown()

	df, _ := idiscovery.NewFactory(ctx, mock.New())
	fdiscovery.InjectFactory(df)

	const suffix = "test"

	eps := testutil.ToEndpoints("addr1:123")
	mockServer := testutil.NewMockServer(eps)

	ad := discovery.Advertisement{
		InterfaceName: "v.io/v23/a",
		Attributes:    map[string]string{"a1": "v1"},
	}

	_, err := idiscovery.AdvertiseServer(ctx, nil, mockServer, suffix, &ad, nil)
	if err != nil {
		t.Fatal(err)
	}

	d, _ := v23.NewDiscovery(ctx)

	nad := ad
	newAd := withNewAddresses(&nad, eps, suffix)
	if err := testutil.ScanAndMatch(ctx, d, "", newAd); err != nil {
		t.Error(err)
	}

	tests := [][]naming.Endpoint{
		testutil.ToEndpoints("addr2:123", "addr3:456"),
		testutil.ToEndpoints("addr4:123"),
		testutil.ToEndpoints("addr5:123", "addr6:456"),
	}
	for _, eps := range tests {
		mockServer.UpdateNetwork(eps)

		newAd = withNewAddresses(&ad, eps, suffix)
		if err := testutil.ScanAndMatch(ctx, d, "", newAd); err != nil {
			t.Error(err)
		}
	}
}
<commit_msg>fix race in test (#226)<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package discovery_test

import (
	"testing"

	v23 "v.io/v23"
	"v.io/v23/discovery"
	"v.io/v23/naming"
	idiscovery "v.io/x/ref/lib/discovery"
	fdiscovery "v.io/x/ref/lib/discovery/factory"
	"v.io/x/ref/lib/discovery/plugins/mock"
	"v.io/x/ref/lib/discovery/testutil"
	_ "v.io/x/ref/runtime/factories/generic"
	"v.io/x/ref/test"
)

func withNewAddresses(ad *discovery.Advertisement, eps []naming.Endpoint, suffix string) discovery.Advertisement {
	newAd := *ad
	newAd.Addresses = make([]string, len(eps))
	for i, ep := range eps {
		newAd.Addresses[i] = naming.JoinAddressName(ep.Name(), suffix)
	}
	return newAd
}

func TestAdvertiseServer(t *testing.T) {
	ctx, shutdown := test.V23Init()
	defer shutdown()

	df, _ := idiscovery.NewFactory(ctx, mock.New())
	fdiscovery.InjectFactory(df)

	const suffix = "test"

	eps := testutil.ToEndpoints("addr1:123")
	mockServer := testutil.NewMockServer(eps)

	ad := discovery.Advertisement{
		InterfaceName: "v.io/v23/a",
		Attributes:    map[string]string{"a1": "v1"},
	}

	_, err := idiscovery.AdvertiseServer(ctx, nil, mockServer, suffix, &ad, nil)
	if err != nil {
		t.Fatal(err)
	}

	d, _ := v23.NewDiscovery(ctx)

	nad := ad
	newAd := withNewAddresses(&nad, eps, suffix)
	if err := testutil.ScanAndMatch(ctx, d, "", newAd); err != nil {
		t.Error(err)
	}

	tests := [][]naming.Endpoint{
		testutil.ToEndpoints("addr2:123", "addr3:456"),
		testutil.ToEndpoints("addr4:123"),
		testutil.ToEndpoints("addr5:123", "addr6:456"),
	}
	for _, eps := range tests {
		mockServer.UpdateNetwork(eps)

		newAd = withNewAddresses(&nad, eps, suffix)
		if err := testutil.ScanAndMatch(ctx, d, "", newAd); err != nil {
			t.Error(err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The nvim-go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package autocmd

import (
	"context"
	"sync"

	"github.com/neovim/go-client/nvim"
	"github.com/neovim/go-client/nvim/plugin"
	"github.com/zchee/nvim-go/src/buildctx"
	"github.com/zchee/nvim-go/src/command"
	"github.com/zchee/nvim-go/src/logger"
	"go.uber.org/zap"
	"golang.org/x/sync/syncmap"
)

// Autocmd represents a autocmd context.
type Autocmd struct {
	ctx context.Context
	log *zap.Logger

	Nvim         *nvim.Nvim
	buildContext *buildctx.Context
	cmd          *command.Command

	bufWritePostChan chan error
	bufWritePreChan  chan interface{}
	mu               sync.Mutex
	wg               sync.WaitGroup

	errs *syncmap.Map
}

// Register register autocmd to nvim.
func Register(ctx context.Context, p *plugin.Plugin, buildContext *buildctx.Context, cmd *command.Command) {
	autocmd := &Autocmd{
		ctx:              ctx,
		log:              logger.FromContext(ctx).Named("autocmd"),
		Nvim:             p.Nvim,
		buildContext:     buildContext,
		cmd:              cmd,
		bufWritePreChan:  make(chan interface{}),
		bufWritePostChan: make(chan error),
		errs:             new(syncmap.Map),
	}

	p.HandleAutocmd(&plugin.AutocmdOptions{Event: "BufEnter", Pattern: "*.go", Group: "nvim-go", Eval: "*"}, autocmd.BufEnter)
	p.HandleAutocmd(&plugin.AutocmdOptions{Event: "BufWritePost", Pattern: "*.go", Group: "nvim-go", Eval: "*"}, autocmd.bufWritePost)
	p.HandleAutocmd(&plugin.AutocmdOptions{Event: "BufWritePre", Pattern: "*.go", Group: "nvim-go", Eval: "*"}, autocmd.bufWritePre)
	p.HandleAutocmd(&plugin.AutocmdOptions{Event: "VimEnter", Pattern: "*.go", Group: "nvim-go", Eval: "*"}, autocmd.VimEnter)
}
<commit_msg>autocmd: add cancel field and wrap parent ctx to WithCancel<commit_after>// Copyright 2016 The nvim-go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package autocmd

import (
	"context"
	"sync"

	"github.com/neovim/go-client/nvim"
	"github.com/neovim/go-client/nvim/plugin"
	"github.com/zchee/nvim-go/src/buildctx"
	"github.com/zchee/nvim-go/src/command"
	"github.com/zchee/nvim-go/src/logger"
	"go.uber.org/zap"
	"golang.org/x/sync/syncmap"
)

// Autocmd represents a autocmd context.
type Autocmd struct {
	ctx    context.Context
	cancel context.CancelFunc
	log    *zap.Logger

	Nvim         *nvim.Nvim
	buildContext *buildctx.Context
	cmd          *command.Command

	bufWritePostChan chan error
	bufWritePreChan  chan interface{}
	mu               sync.Mutex
	wg               sync.WaitGroup

	errs *syncmap.Map
}

// Register register autocmd to nvim.
func Register(pctx context.Context, p *plugin.Plugin, buildContext *buildctx.Context, cmd *command.Command) {
	ctx, cancel := context.WithCancel(pctx)

	autocmd := &Autocmd{
		ctx:              ctx,
		cancel:           cancel,
		log:              logger.FromContext(ctx).Named("autocmd"),
		Nvim:             p.Nvim,
		buildContext:     buildContext,
		cmd:              cmd,
		bufWritePreChan:  make(chan interface{}),
		bufWritePostChan: make(chan error),
		errs:             new(syncmap.Map),
	}

	p.HandleAutocmd(&plugin.AutocmdOptions{Event: "BufEnter", Pattern: "*.go", Group: "nvim-go", Eval: "*"}, autocmd.BufEnter)
	p.HandleAutocmd(&plugin.AutocmdOptions{Event: "BufWritePost", Pattern: "*.go", Group: "nvim-go", Eval: "*"}, autocmd.bufWritePost)
	p.HandleAutocmd(&plugin.AutocmdOptions{Event: "BufWritePre", Pattern: "*.go", Group: "nvim-go", Eval: "*"}, autocmd.bufWritePre)
	p.HandleAutocmd(&plugin.AutocmdOptions{Event: "VimEnter", Pattern: "*.go", Group: "nvim-go", Eval: "*"}, autocmd.VimEnter)
}
<|endoftext|>


<commit_before>package check

import (
	"testing"
)

func TestAny(t *testing.T) {
	var small, fifteen, sixteen string

	// Check for a small number
	small = "4111"
	if CreditCard(small).CardIs("Any") {
		t.Errorf("Any(%v) = true, want false", small)
	}

	// Check for a AMEX style card number
	fifteen = "822233334444555"
	if !CreditCard(fifteen).CardIs("Any") {
		t.Errorf("Any(%v) = false, want true", fifteen)
	}

	// Check for all other style card numbers
	sixteen = "9111222233334444"
	if !CreditCard(sixteen).CardIs("Any") {
		t.Errorf("Any(%v) = false, want true", sixteen)
	}

}

func TestMasterCard(t *testing.T) {
	var small, fifteen, sixteen string

	// Check for a small number
	small = "4111"
	if CreditCard(small).CardIs("MasterCard") {
		t.Errorf("Any(%v) = true, want false", small)
	}

	// Check for a AMEX style card number (numbers)
	fifteen = "822233334444555"
	if CreditCard(fifteen).CardIs("MasterCard") {
		t.Errorf("Any(%v) = true, want false", fifteen)
	}

	// Check for MasterCard card number
	sixteen = "5011122223333444"
	if !CreditCard(sixteen).CardIs("MasterCard") {
		t.Log(len(sixteen))
		t.Errorf("Any(%v) = false, want true", sixteen)
	}

	// Check for all other style card numbers
	sixteen = "9111222233334444"
	if CreditCard(sixteen).CardIs("MasterCard") {
		t.Errorf("Any(%v) = true, want false", sixteen)
	}
}

func TestVisa(t *testing.T) {
	var small, fifteen, sixteen string

	// Check for a small number
	small = "4111"
	if CreditCard(small).CardIs("Visa") {
		t.Errorf("Any(%v) = true, want false", small)
	}

	// Check for a AMEX style card number (numbers)
	fifteen = "822233334444555"
	if CreditCard(fifteen).CardIs("Visa") {
		t.Errorf("Any(%v) = true, want false", fifteen)
	}

	// Check for Visa card number
	sixteen = "4011122223333444"
	if !CreditCard(sixteen).CardIs("Visa") {
		t.Log(len(sixteen))
		t.Errorf("Any(%v) = false, want true", sixteen)
	}

	// Check for all other style card numbers
	sixteen = "9111222233334444"
	if CreditCard(sixteen).CardIs("Visa") {
		t.Errorf("Any(%v) = true, want false", sixteen)
	}
}

func TestDiscover(t *testing.T) {
	var small, fifteen, sixteen string

	// Check for a small number
	small = "4111"
	if CreditCard(small).CardIs("Discover") {
		t.Errorf("Any(%v) = true, want false", small)
	}

	// Check for a AMEX style card number (numbers)
	fifteen = "822233334444555"
	if CreditCard(fifteen).CardIs("Discover") {
		t.Errorf("Any(%v) = true, want false", fifteen)
	}

	// Check for Discover card number
	sixteen = "6011122223333444"
	if !CreditCard(sixteen).CardIs("Discover") {
		t.Log(len(sixteen))
		t.Errorf("Any(%v) = false, want true", sixteen)
	}

	// Check for all other style card numbers
	sixteen = "9111222233334444"
	if CreditCard(sixteen).CardIs("Discover") {
		t.Errorf("Any(%v) = true, want false", sixteen)
	}
}

func TestAmex(t *testing.T) {
	var small, fifteen, sixteen string

	// Check for a small number
	small = "4111"
	if CreditCard(small).CardIs("AmericanExpress") {
		t.Errorf("Any(%v) = true, want false", small)
	}

	// Check for a AMEX style card number (numbers)
	fifteen = "822233334444555"
	if CreditCard(fifteen).CardIs("AmericanExpress") {
		t.Errorf("Any(%v) = true, want false", fifteen)
	}

	// Check for AMEX style card number (sequence) Format 1
	fifteen = "3417122223333444"
	if !CreditCard(fifteen).CardIs("AmericanExpress") {
		t.Log(len(fifteen))
		t.Log("Failure format 1")
		t.Errorf("Any(%v) = false, want true", fifteen)
	}

	// Check for AMEX style card number (sequence) Format 2
	fifteen = "3717122223333444"
	if !CreditCard(fifteen).CardIs("AmericanExpress") {
		t.Log(len(fifteen))
		t.Log("Failure format 2")
		t.Errorf("Any(%v) = false, want true", fifteen)
	}

	// Check for all other style card numbers
	sixteen = "9111222233334444"
	if CreditCard(sixteen).CardIs("AmericanExpress") {
		t.Errorf("Any(%v) = true, want false", sixteen)
	}
}
<commit_msg>Add letters to test<commit_after>package check

import (
	"testing"
)

func TestAny(t *testing.T) {
	var small, fifteen, sixteen string

	// Check for a small number
	small = "4111"
	if CreditCard(small).CardIs("Any") {
		t.Errorf("Any(%v) = true, want false", small)
	}

	// Check for a AMEX style card number
	fifteen = "822233334444555"
	if !CreditCard(fifteen).CardIs("Any") {
		t.Errorf("Any(%v) = false, want true", fifteen)
	}

	// Check for all other style card numbers
	sixteen = "9111222233334444"
	if !CreditCard(sixteen).CardIs("Any") {
		t.Errorf("Any(%v) = false, want true", sixteen)
	}

	// Check for letters
	sixteen = "a111222233334444"
	if CreditCard(sixteen).CardIs("Any") {
		t.Errorf("Any(%v) = true, want false", sixteen)
	}

}

func TestMasterCard(t *testing.T) {
	var small, fifteen, sixteen string

	// Check for a small number
	small = "4111"
	if CreditCard(small).CardIs("MasterCard") {
		t.Errorf("Any(%v) = true, want false", small)
	}

	// Check for a AMEX style card number (numbers)
	fifteen = "822233334444555"
	if CreditCard(fifteen).CardIs("MasterCard") {
		t.Errorf("Any(%v) = true, want false", fifteen)
	}

	// Check for MasterCard card number
	sixteen = "5011122223333444"
	if !CreditCard(sixteen).CardIs("MasterCard") {
		t.Log(len(sixteen))
		t.Errorf("Any(%v) = false, want true", sixteen)
	}

	// Check for all other style card numbers
	sixteen = "9111222233334444"
	if CreditCard(sixteen).CardIs("MasterCard") {
		t.Errorf("Any(%v) = true, want false", sixteen)
	}

	// Check for letters
	sixteen = "a111222233334444"
	if CreditCard(sixteen).CardIs("MasterCard") {
		t.Errorf("Any(%v) = true, want false", sixteen)
	}
}

func TestVisa(t *testing.T) {
	var small, fifteen, sixteen string

	// Check for a small number
	small = "4111"
	if CreditCard(small).CardIs("Visa") {
		t.Errorf("Any(%v) = true, want false", small)
	}

	// Check for a AMEX style card number (numbers)
	fifteen = "822233334444555"
	if CreditCard(fifteen).CardIs("Visa") {
		t.Errorf("Any(%v) = true, want false", fifteen)
	}

	// Check for Visa card number
	sixteen = "4011122223333444"
	if !CreditCard(sixteen).CardIs("Visa") {
		t.Log(len(sixteen))
		t.Errorf("Any(%v) = false, want true", sixteen)
	}

	// Check for all other style card numbers
	sixteen = "9111222233334444"
	if CreditCard(sixteen).CardIs("Visa") {
		t.Errorf("Any(%v) = true, want false", sixteen)
	}

	// Check for letters
	sixteen = "a111222233334444"
	if CreditCard(sixteen).CardIs("Visa") {
		t.Errorf("Any(%v) = true, want false", sixteen)
	}
}

func TestDiscover(t *testing.T) {
	var small, fifteen, sixteen string

	// Check for a small number
	small = "4111"
	if CreditCard(small).CardIs("Discover") {
		t.Errorf("Any(%v) = true, want false", small)
	}

	// Check for a AMEX style card number (numbers)
	fifteen = "822233334444555"
	if CreditCard(fifteen).CardIs("Discover") {
		t.Errorf("Any(%v) = true, want false", fifteen)
	}

	// Check for Discover card number
	sixteen = "6011122223333444"
	if !CreditCard(sixteen).CardIs("Discover") {
		t.Log(len(sixteen))
		t.Errorf("Any(%v) = false, want true", sixteen)
	}

	// Check for all other style card numbers
	sixteen = "9111222233334444"
	if CreditCard(sixteen).CardIs("Discover") {
		t.Errorf("Any(%v) = true, want false", sixteen)
	}

	// Check for letters
	sixteen = "a111222233334444"
	if CreditCard(sixteen).CardIs("Discover") {
		t.Errorf("Any(%v) = true, want false", sixteen)
	}
}

func TestAmex(t *testing.T) {
	var small, fifteen, sixteen string

	// Check for a small number
	small = "4111"
	if CreditCard(small).CardIs("AmericanExpress") {
		t.Errorf("Any(%v) = true, want false", small)
	}

	// Check for a AMEX style card number (numbers)
	fifteen = "822233334444555"
	if CreditCard(fifteen).CardIs("AmericanExpress") {
		t.Errorf("Any(%v) = true, want false", fifteen)
	}

	// Check for AMEX style card number (sequence) Format 1
	fifteen = "3417122223333444"
	if !CreditCard(fifteen).CardIs("AmericanExpress") {
		t.Log(len(fifteen))
		t.Log("Failure format 1")
		t.Errorf("Any(%v) = false, want true", fifteen)
	}

	// Check for AMEX style card number (sequence) Format 2
	fifteen = "3717122223333444"
	if !CreditCard(fifteen).CardIs("AmericanExpress") {
		t.Log(len(fifteen))
		t.Log("Failure format 2")
		t.Errorf("Any(%v) = false, want true", fifteen)
	}

	// Check for all other style card numbers
	sixteen = "9111222233334444"
	if CreditCard(sixteen).CardIs("AmericanExpress") {
		t.Errorf("Any(%v) = true, want false", sixteen)
	}

	// Check for letters
	fifteen = "a11222233334444"
	if CreditCard(sixteen).CardIs("AmericanExpress") {
		t.Errorf("Any(%v) = true, want false", fifteen)
	}
}
<|endoftext|>