<commit_before>package procspy

// /proc based implementation

import (
	"bufio"
	"encoding/hex"
	"errors"
	"io"
	"net"
	"os"
	"strconv"
	"strings"
	"syscall"
)

const procRoot = "/proc"

// SpyProc uses /proc directly to make the connection list.
func SpyProc() ([]ConnProc, error) {
	// A map of inode -> pid
	inodes, err := walkProcPid()
	if err != nil {
		return nil, err
	}

	res := []ConnProc{}
	for _, procFile := range []string{
		procRoot + "/net/tcp",
		procRoot + "/net/tcp6",
	} {
		fh, err := os.Open(procFile)
		if err != nil {
			// File might not be there if IPv{4,6} is not supported.
			continue
		}
		defer fh.Close()
		for _, tp := range parseTransport(fh) {
			if pid, ok := inodes[tp.inode]; ok {
				name, err := procName(pid)
				if err != nil {
					// Process might be gone by now
					continue
				}
				if tp.remoteAddress.IsUnspecified() {
					// Remote address is zero. This is a listen entry.
					continue
				}
				res = append(res, ConnProc{
					Transport:  "tcp",
					LocalAddr:  tp.localAddress.String(),
					LocalPort:  tp.localPort,
					RemoteAddr: tp.remoteAddress.String(),
					RemotePort: tp.remotePort,
					PID:        pid,
					Name:       name,
				})
			}
		}
	}
	return res, nil
}

func walkProcPid() (map[uint64]uint, error) {
	// Walk over all /proc entries (numerical ones, those are PIDs), and see if
	// their ./fd/* files are symlink to sockets.
	// Returns a map from socket id ('inode`) to PID.
	// Will return an error if /proc/ isn't there.
	fh, err := os.Open(procRoot)
	if err != nil {
		return nil, err
	}
	defer fh.Close()
	dirNames, err := fh.Readdirnames(-1)
	if err != nil {
		return nil, err
	}
	procmap := map[uint64]uint{}
	for _, dirName := range dirNames {
		pid, err := strconv.ParseUint(dirName, 10, 0)
		if err != nil {
			// Not a number, so not a PID subdir.
			continue
		}

		fdBase := procRoot + "/" + dirName + "/fd/"
		dfh, err := os.Open(fdBase)
		if err != nil {
			// Process is be gone by now, or we don't have access.
			continue
		}
		fdNames, err := dfh.Readdirnames(-1)
		dfh.Close()
		if err != nil {
			continue
		}
		for _, fdName := range fdNames {
			// We want sockets only
			stat, err := os.Stat(fdBase + fdName)
			if err != nil {
				continue
			}
			if stat.Mode()&os.ModeSocket == 0 {
				continue
			}
			sys, ok := stat.Sys().(*syscall.Stat_t)
			if !ok {
				panic("Weird result from stat.Sys()")
			}
			procmap[sys.Ino] = uint(pid)
		}
	}
	return procmap, nil
}

// transport are found in /proc/net/{tcp,udp}{,6} files
type transport struct {
	localAddress  net.IP
	localPort     uint16
	remoteAddress net.IP
	remotePort    uint16
	uid           int
	inode         uint64
}

// parseTransport parses /proc/net/{tcp,udp}{,6} files
func parseTransport(r io.Reader) []transport {
	res := []transport{}
	scanner := bufio.NewScanner(r)
	for i := 0; scanner.Scan(); i++ {
		if i == 0 {
			// Skip header
			continue
		}
		// Fields are:
		//  'sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode <more>'
		fields := strings.Fields(scanner.Text())
		if len(fields) < 10 {
			continue
		}

		localAddress, localPort, err := scanAddress(fields[1])
		if err != nil {
			continue
		}

		remoteAddress, remotePort, err := scanAddress(fields[2])
		if err != nil {
			continue
		}

		uid, err := strconv.Atoi(fields[7])
		if err != nil {
			continue
		}

		inode, err := strconv.ParseUint(fields[9], 10, 64)
		if err != nil {
			continue
		}
		t := transport{
			localAddress:  localAddress,
			localPort:     localPort,
			remoteAddress: remoteAddress,
			remotePort:    remotePort,
			uid:           uid,
			inode:         inode,
		}
		res = append(res, t)

	}
	return res
}

// scanAddress parses 'A12CF62E:E4D7' to the address and port.
// Handles IPv4 and IPv6 addresses.
// The address part are big endian 32 bit ints, hex encoded. Since net.IP is a
// byte slice we just decode the hex and flip the bytes in every group of 4.
func scanAddress(in string) (net.IP, uint16, error) {
	parts := strings.Split(in, ":")
	if len(parts) != 2 {
		return nil, 0, errors.New("invalid address:port")
	}

	// Network address is big endian. Can be either ipv4 or ipv6.
	address, err := hex.DecodeString(parts[0])
	if err != nil {
		return nil, 0, err
	}
	// reverse every 4 byte-sequence.
	for i := 0; i < len(address); i += 4 {
		address[i], address[i+3] = address[i+3], address[i]
		address[i+1], address[i+2] = address[i+2], address[i+1]
	}

	// Port number
	port, err := strconv.ParseUint(parts[1], 16, 16)
	if err != nil {
		return nil, 0, err
	}

	return net.IP(address), uint16(port), err
}

// procName does a pid->name lookup
func procName(pid uint) (string, error) {
	fh, err := os.Open(procRoot + "/" + strconv.FormatUint(uint64(pid), 10) + "/comm")
	if err != nil {
		return "", err
	}
	name := make([]byte, 1024)
	l, err := fh.Read(name)
	if err != nil {
		return "", err
	}
	if l < 2 {
		return "", nil
	}
	// drop trailing "\n"
	return string(name[:l-1]), nil
}
<commit_msg>Forgot a Close().<commit_after>package procspy

// /proc based implementation

import (
	"bufio"
	"encoding/hex"
	"errors"
	"io"
	"net"
	"os"
	"strconv"
	"strings"
	"syscall"
)

const procRoot = "/proc"

// SpyProc uses /proc directly to make the connection list.
func SpyProc() ([]ConnProc, error) {
	// A map of inode -> pid
	inodes, err := walkProcPid()
	if err != nil {
		return nil, err
	}

	res := []ConnProc{}
	for _, procFile := range []string{
		procRoot + "/net/tcp",
		procRoot + "/net/tcp6",
	} {
		fh, err := os.Open(procFile)
		if err != nil {
			// File might not be there if IPv{4,6} is not supported.
			continue
		}
		defer fh.Close()
		for _, tp := range parseTransport(fh) {
			if pid, ok := inodes[tp.inode]; ok {
				name, err := procName(pid)
				if err != nil {
					// Process might be gone by now
					continue
				}
				if tp.remoteAddress.IsUnspecified() {
					// Remote address is zero. This is a listen entry.
					continue
				}
				res = append(res, ConnProc{
					Transport:  "tcp",
					LocalAddr:  tp.localAddress.String(),
					LocalPort:  tp.localPort,
					RemoteAddr: tp.remoteAddress.String(),
					RemotePort: tp.remotePort,
					PID:        pid,
					Name:       name,
				})
			}
		}
	}
	return res, nil
}

func walkProcPid() (map[uint64]uint, error) {
	// Walk over all /proc entries (numerical ones, those are PIDs), and see if
	// their ./fd/* files are symlink to sockets.
	// Returns a map from socket id ('inode`) to PID.
	// Will return an error if /proc/ isn't there.
	fh, err := os.Open(procRoot)
	if err != nil {
		return nil, err
	}
	dirNames, err := fh.Readdirnames(-1)
	fh.Close()
	if err != nil {
		return nil, err
	}
	procmap := map[uint64]uint{}
	for _, dirName := range dirNames {
		pid, err := strconv.ParseUint(dirName, 10, 0)
		if err != nil {
			// Not a number, so not a PID subdir.
			continue
		}

		fdBase := procRoot + "/" + dirName + "/fd/"
		dfh, err := os.Open(fdBase)
		if err != nil {
			// Process is be gone by now, or we don't have access.
			continue
		}
		fdNames, err := dfh.Readdirnames(-1)
		dfh.Close()
		if err != nil {
			continue
		}
		for _, fdName := range fdNames {
			// We want sockets only
			stat, err := os.Stat(fdBase + fdName)
			if err != nil {
				continue
			}
			if stat.Mode()&os.ModeSocket == 0 {
				continue
			}
			sys, ok := stat.Sys().(*syscall.Stat_t)
			if !ok {
				panic("Weird result from stat.Sys()")
			}
			procmap[sys.Ino] = uint(pid)
		}
	}
	return procmap, nil
}

// transport are found in /proc/net/{tcp,udp}{,6} files
type transport struct {
	localAddress  net.IP
	localPort     uint16
	remoteAddress net.IP
	remotePort    uint16
	uid           int
	inode         uint64
}

// parseTransport parses /proc/net/{tcp,udp}{,6} files
func parseTransport(r io.Reader) []transport {
	res := []transport{}
	scanner := bufio.NewScanner(r)
	for i := 0; scanner.Scan(); i++ {
		if i == 0 {
			// Skip header
			continue
		}
		// Fields are:
		//  'sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode <more>'
		fields := strings.Fields(scanner.Text())
		if len(fields) < 10 {
			continue
		}

		localAddress, localPort, err := scanAddress(fields[1])
		if err != nil {
			continue
		}

		remoteAddress, remotePort, err := scanAddress(fields[2])
		if err != nil {
			continue
		}

		uid, err := strconv.Atoi(fields[7])
		if err != nil {
			continue
		}

		inode, err := strconv.ParseUint(fields[9], 10, 64)
		if err != nil {
			continue
		}
		t := transport{
			localAddress:  localAddress,
			localPort:     localPort,
			remoteAddress: remoteAddress,
			remotePort:    remotePort,
			uid:           uid,
			inode:         inode,
		}
		res = append(res, t)

	}
	return res
}

// scanAddress parses 'A12CF62E:E4D7' to the address and port.
// Handles IPv4 and IPv6 addresses.
// The address part are big endian 32 bit ints, hex encoded. Since net.IP is a
// byte slice we just decode the hex and flip the bytes in every group of 4.
func scanAddress(in string) (net.IP, uint16, error) {
	parts := strings.Split(in, ":")
	if len(parts) != 2 {
		return nil, 0, errors.New("invalid address:port")
	}

	// Network address is big endian. Can be either ipv4 or ipv6.
	address, err := hex.DecodeString(parts[0])
	if err != nil {
		return nil, 0, err
	}
	// reverse every 4 byte-sequence.
	for i := 0; i < len(address); i += 4 {
		address[i], address[i+3] = address[i+3], address[i]
		address[i+1], address[i+2] = address[i+2], address[i+1]
	}

	// Port number
	port, err := strconv.ParseUint(parts[1], 16, 16)
	if err != nil {
		return nil, 0, err
	}

	return net.IP(address), uint16(port), err
}

// procName does a pid->name lookup
func procName(pid uint) (string, error) {
	fh, err := os.Open(procRoot + "/" + strconv.FormatUint(uint64(pid), 10) + "/comm")
	if err != nil {
		return "", err
	}
	name := make([]byte, 1024)
	l, err := fh.Read(name)
	fh.Close()
	if err != nil {
		return "", err
	}
	if l < 2 {
		return "", nil
	}
	// drop trailing "\n"
	return string(name[:l-1]), nil
}
<|endoftext|>


<commit_before>package raftwrapper

import (
	"fmt"
	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/rafthttp"
	"golang.org/x/net/context"
	"strconv"
	"sync"
	"time"
)

type LogData []byte

type Node struct {
	node        raft.Node
	raftStorage *raft.MemoryStorage
	transport   *rafthttp.Transport
	peers       []string
	peerMap     map[uint64]string
	id          uint64
	raftPort    int
	cluster     int

	apiPort int

	started     bool
	initialized bool
	proposeC    chan string
	fsm         FSM

	observers     map[uint64]*Observer
	observersLock sync.RWMutex
}

type NodeConfig struct {
	FSM           FSM
	RaftPort      int
	APIPort       int
	Peers         []string
	BootstrapNode bool
}

// note: peers is only for asking to join the cluster.
// It will not be able to connect if the peers don't respond to cluster node add request
// This is because each node defines it's own uuid at startup. We must be told this UUID
// by another node.
// TODO: Look into which config options we want others to specify. For now hardcoded
// NOTE: Peers are used EXCLUSIVELY to round-robin to other nodes and attempt to add
//		ourselves to an existing cluster or bootstrap node
func NewNode(args *NodeConfig) (*Node, error) {
	rn := nonInitNode(args)

	if err := rn.attachTransport(); err != nil {
		return nil, err
	}

	return rn, nil
}

func (rn *Node) Start(httpBlock bool) error {
	if rn.started {
		return nil
	}

	var wg sync.WaitGroup

	if err := rn.transport.Start(); err != nil {
		return err
	}

	go rn.scanReady()

	if httpBlock {
		wg.Add(1)
	}
	go func(rn *Node) {
		defer wg.Done()
		rn.serveHTTP()
	}(rn)

	go rn.serveRaft()

	if err := rn.joinPeers(); err != nil {
		return err
	}
	// final step to mark node as initialized
	// TODO: Find a better place to mark initialized
	rn.initialized = true
	rn.started = true
	wg.Wait()
	return nil
}

func nonInitNode(args *NodeConfig) *Node {
	if args.BootstrapNode {
		args.Peers = nil
	}
	rn := &Node{
		proposeC:    make(chan string),
		cluster:     0x1000,
		raftStorage: raft.NewMemoryStorage(),
		peers:       args.Peers,
		id:          Uint64UUID(),
		raftPort:    args.RaftPort,
		apiPort:     args.APIPort,
		fsm:         args.FSM,
		initialized: false,
		observers:   make(map[uint64]*Observer),
		peerMap:     make(map[uint64]string),
	}

	c := &raft.Config{
		ID:              rn.id,
		ElectionTick:    10,
		HeartbeatTick:   1,
		Storage:         rn.raftStorage,
		MaxSizePerMsg:   1024 * 1024,
		MaxInflightMsgs: 256,
	}

	if args.BootstrapNode {
		rn.node = raft.StartNode(c, []raft.Peer{raft.Peer{ID: rn.id}})
	} else {
		rn.node = raft.StartNode(c, nil)
	}

	return rn
}

func (rn *Node) attachTransport() error {
	ss := &stats.ServerStats{}
	ss.Initialize()

	rn.transport = &rafthttp.Transport{
		ID:          types.ID(rn.id),
		ClusterID:   0x1000,
		Raft:        rn,
		ServerStats: ss,
		LeaderStats: stats.NewLeaderStats(strconv.FormatUint(rn.id, 10)),
		ErrorC:      make(chan error),
	}

	return nil
}

func (rn *Node) joinPeers() error {
	err := rn.requestSelfAddition()
	if err != nil {
		return err
	}
	return nil
}

func (rn *Node) proposePeerAddition(addReq *raftpb.ConfChange, async bool) error {
	addReq.Type = raftpb.ConfChangeAddNode

	observChan := make(chan Observation)
	// setup listener for node addition
	// before asking for node addition
	if !async {
		filterFn := func(o Observation) bool {

			switch o.(type) {
			case raftpb.Entry:
				entry := o.(raftpb.Entry)
				switch entry.Type {
				case raftpb.EntryConfChange:
					var cc raftpb.ConfChange
					cc.Unmarshal(entry.Data)
					rn.node.ApplyConfChange(cc)
					switch cc.Type {
					case raftpb.ConfChangeAddNode:
						// wait until we get a matching node id
						return addReq.NodeID == cc.NodeID
					default:
						return false
					}
				default:
					return false
				}
			default:
				return false
			}
		}

		observer := NewObserver(observChan, filterFn)
		rn.RegisterObserver(observer)
		defer rn.UnregisterObserver(observer)
	}

	if err := rn.node.ProposeConfChange(context.TODO(), *addReq); err != nil {
		return err
	}

	if async {
		return nil
	}

	// TODO: Do a retry here on failure for x retries
	select {
	case <-observChan:
		return nil
	case <-time.After(10 * time.Second):
		return rn.proposePeerAddition(addReq, async)

	}
}

func (rn *Node) canAddPeer() bool {
	return rn.isHealthy() && rn.initialized
}

// TODO: Define healthy
func (rn *Node) isHealthy() bool {
	return true
}

func (rn *Node) scanReady() {
	ticker := time.NewTicker(100 * time.Millisecond)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			rn.node.Tick()
		case rd := <-rn.node.Ready():
			rn.raftStorage.Append(rd.Entries)
			rn.transport.Send(rd.Messages)
			if ok := rn.publishEntries(rd.CommittedEntries); !ok {
				return
			}

			rn.node.Advance()
		}
	}
}

func (rn *Node) publishEntries(ents []raftpb.Entry) bool {
	for _, entry := range ents {
		switch entry.Type {
		case raftpb.EntryNormal:
			if len(entry.Data) == 0 {
				break
			}
			// Yes, this is probably a blocking call
			// An FSM should be responsible for being efficient
			// for high-load situations
			if err := rn.fsm.Apply(LogData(entry.Data)); err != nil {
				return false
			}

		case raftpb.EntryConfChange:
			var cc raftpb.ConfChange
			cc.Unmarshal(entry.Data)
			rn.node.ApplyConfChange(cc)

			switch cc.Type {
			case raftpb.ConfChangeAddNode:
				if len(cc.Context) > 0 {
					rn.transport.AddPeer(types.ID(cc.NodeID), []string{string(cc.Context)})
					rn.peerMap[cc.NodeID] = string(cc.Context)
				}
			case raftpb.ConfChangeRemoveNode:
				if cc.NodeID == uint64(rn.id) {
					fmt.Println("I have been removed!")
					return false
				}
				rn.transport.RemovePeer(types.ID(cc.NodeID))
			}

		}
		rn.observe(entry)
		// TODO: Add support for replay commits
		// After replaying old commits/snapshots then mark
		// this node operational
	}
	return true
}

func (rn *Node) Propose(data []byte) error {
	return rn.node.Propose(context.TODO(), data)
}

func (rn *Node) Process(ctx context.Context, m raftpb.Message) error {
	return rn.node.Step(ctx, m)
}
func (rn *Node) IsIDRemoved(id uint64) bool {
	return false
}
func (rn *Node) ReportUnreachable(id uint64)                          {}
func (rn *Node) ReportSnapshot(id uint64, status raft.SnapshotStatus) {}
<commit_msg>add exponential backoff to cluster join<commit_after>package raftwrapper

import (
	"fmt"
	"github.com/cenk/backoff"
	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/rafthttp"
	"golang.org/x/net/context"
	"log"
	"strconv"
	"sync"
	"time"
)

type LogData []byte

type Node struct {
	node        raft.Node
	raftStorage *raft.MemoryStorage
	transport   *rafthttp.Transport
	peers       []string
	peerMap     map[uint64]string
	id          uint64
	raftPort    int
	cluster     int

	apiPort int

	started     bool
	initialized bool
	proposeC    chan string
	fsm         FSM

	observers     map[uint64]*Observer
	observersLock sync.RWMutex

	initBackoffArgs *InitializationBackoffArgs
}

type NodeConfig struct {
	FSM           FSM
	RaftPort      int
	APIPort       int
	Peers         []string
	BootstrapNode bool
	InitBackoff   *InitializationBackoffArgs
}

// note: peers is only for asking to join the cluster.
// It will not be able to connect if the peers don't respond to cluster node add request
// This is because each node defines it's own uuid at startup. We must be told this UUID
// by another node.
// TODO: Look into which config options we want others to specify. For now hardcoded
// NOTE: Peers are used EXCLUSIVELY to round-robin to other nodes and attempt to add
//		ourselves to an existing cluster or bootstrap node
func NewNode(args *NodeConfig) (*Node, error) {
	rn := nonInitNode(args)

	if err := rn.attachTransport(); err != nil {
		return nil, err
	}

	return rn, nil
}

func (rn *Node) Start(httpBlock bool) error {
	if rn.started {
		return nil
	}

	var wg sync.WaitGroup

	if err := rn.transport.Start(); err != nil {
		return err
	}

	go rn.scanReady()

	if httpBlock {
		wg.Add(1)
	}
	go func(rn *Node) {
		defer wg.Done()
		rn.serveHTTP()
	}(rn)

	go rn.serveRaft()

	if err := rn.joinPeers(); err != nil {
		return err
	}
	// final step to mark node as initialized
	// TODO: Find a better place to mark initialized
	rn.initialized = true
	rn.started = true
	wg.Wait()
	return nil
}

type InitializationBackoffArgs struct {
	InitialInterval     time.Duration
	Multiplier          float64
	MaxInterval         time.Duration
	MaxElapsedTime      time.Duration
	RandomizationFactor float64
}

func (rn *Node) joinPeers() error {
	notify := func(err error, t time.Duration) {
		log.Printf("Couldn't connect to peer: %s Trying again in %v", err.Error(), t)
	}

	// TODO: Specify the backoff criteria in args
	expBackoff := backoff.NewExponentialBackOff()
	if rn.initBackoffArgs != nil {
		expBackoff.InitialInterval = rn.initBackoffArgs.InitialInterval
		expBackoff.RandomizationFactor = rn.initBackoffArgs.RandomizationFactor
		expBackoff.Multiplier = rn.initBackoffArgs.Multiplier
		expBackoff.MaxInterval = rn.initBackoffArgs.MaxInterval
		expBackoff.MaxElapsedTime = rn.initBackoffArgs.MaxElapsedTime
	} else {
		expBackoff.InitialInterval = 500 * time.Millisecond
		expBackoff.RandomizationFactor = .5
		expBackoff.Multiplier = 2
		expBackoff.MaxInterval = 5 * time.Second
		expBackoff.MaxElapsedTime = 2 * time.Minute
	}

	op := func() error {
		return rn.requestSelfAddition()
	}

	return backoff.RetryNotify(op, expBackoff, notify)
}

func nonInitNode(args *NodeConfig) *Node {
	if args.BootstrapNode {
		args.Peers = nil
	}
	rn := &Node{
		proposeC:        make(chan string),
		cluster:         0x1000,
		raftStorage:     raft.NewMemoryStorage(),
		peers:           args.Peers,
		id:              Uint64UUID(),
		raftPort:        args.RaftPort,
		apiPort:         args.APIPort,
		fsm:             args.FSM,
		initialized:     false,
		observers:       make(map[uint64]*Observer),
		peerMap:         make(map[uint64]string),
		initBackoffArgs: args.InitBackoff,
	}

	c := &raft.Config{
		ID:              rn.id,
		ElectionTick:    10,
		HeartbeatTick:   1,
		Storage:         rn.raftStorage,
		MaxSizePerMsg:   1024 * 1024,
		MaxInflightMsgs: 256,
	}

	if args.BootstrapNode {
		rn.node = raft.StartNode(c, []raft.Peer{raft.Peer{ID: rn.id}})
	} else {
		rn.node = raft.StartNode(c, nil)
	}

	return rn
}

func (rn *Node) attachTransport() error {
	ss := &stats.ServerStats{}
	ss.Initialize()

	rn.transport = &rafthttp.Transport{
		ID:          types.ID(rn.id),
		ClusterID:   0x1000,
		Raft:        rn,
		ServerStats: ss,
		LeaderStats: stats.NewLeaderStats(strconv.FormatUint(rn.id, 10)),
		ErrorC:      make(chan error),
	}

	return nil
}

func (rn *Node) proposePeerAddition(addReq *raftpb.ConfChange, async bool) error {
	addReq.Type = raftpb.ConfChangeAddNode

	observChan := make(chan Observation)
	// setup listener for node addition
	// before asking for node addition
	if !async {
		filterFn := func(o Observation) bool {

			switch o.(type) {
			case raftpb.Entry:
				entry := o.(raftpb.Entry)
				switch entry.Type {
				case raftpb.EntryConfChange:
					var cc raftpb.ConfChange
					cc.Unmarshal(entry.Data)
					rn.node.ApplyConfChange(cc)
					switch cc.Type {
					case raftpb.ConfChangeAddNode:
						// wait until we get a matching node id
						return addReq.NodeID == cc.NodeID
					default:
						return false
					}
				default:
					return false
				}
			default:
				return false
			}
		}

		observer := NewObserver(observChan, filterFn)
		rn.RegisterObserver(observer)
		defer rn.UnregisterObserver(observer)
	}

	if err := rn.node.ProposeConfChange(context.TODO(), *addReq); err != nil {
		return err
	}

	if async {
		return nil
	}

	// TODO: Do a retry here on failure for x retries
	select {
	case <-observChan:
		return nil
	case <-time.After(10 * time.Second):
		return rn.proposePeerAddition(addReq, async)

	}
}

func (rn *Node) canAddPeer() bool {
	return rn.isHealthy() && rn.initialized
}

// TODO: Define healthy
func (rn *Node) isHealthy() bool {
	return true
}

func (rn *Node) scanReady() {
	ticker := time.NewTicker(100 * time.Millisecond)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			rn.node.Tick()
		case rd := <-rn.node.Ready():
			rn.raftStorage.Append(rd.Entries)
			rn.transport.Send(rd.Messages)
			if ok := rn.publishEntries(rd.CommittedEntries); !ok {
				return
			}

			rn.node.Advance()
		}
	}
}

func (rn *Node) publishEntries(ents []raftpb.Entry) bool {
	for _, entry := range ents {
		switch entry.Type {
		case raftpb.EntryNormal:
			if len(entry.Data) == 0 {
				break
			}
			// Yes, this is probably a blocking call
			// An FSM should be responsible for being efficient
			// for high-load situations
			if err := rn.fsm.Apply(LogData(entry.Data)); err != nil {
				return false
			}

		case raftpb.EntryConfChange:
			var cc raftpb.ConfChange
			cc.Unmarshal(entry.Data)
			rn.node.ApplyConfChange(cc)

			switch cc.Type {
			case raftpb.ConfChangeAddNode:
				if len(cc.Context) > 0 {
					rn.transport.AddPeer(types.ID(cc.NodeID), []string{string(cc.Context)})
					rn.peerMap[cc.NodeID] = string(cc.Context)
				}
			case raftpb.ConfChangeRemoveNode:
				if cc.NodeID == uint64(rn.id) {
					fmt.Println("I have been removed!")
					return false
				}
				rn.transport.RemovePeer(types.ID(cc.NodeID))
			}

		}
		rn.observe(entry)
		// TODO: Add support for replay commits
		// After replaying old commits/snapshots then mark
		// this node operational
	}
	return true
}

func (rn *Node) Propose(data []byte) error {
	return rn.node.Propose(context.TODO(), data)
}

func (rn *Node) Process(ctx context.Context, m raftpb.Message) error {
	return rn.node.Step(ctx, m)
}
func (rn *Node) IsIDRemoved(id uint64) bool {
	return false
}
func (rn *Node) ReportUnreachable(id uint64)                          {}
func (rn *Node) ReportSnapshot(id uint64, status raft.SnapshotStatus) {}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"runtime/pprof"
	"syscall"

	"github.com/cenkalti/log"
	"github.com/mitchellh/go-homedir"

	"github.com/cenkalti/rain/client"
	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/torrent"
)

var (
	configPath = flag.String("config", "", "config path")
	dest       = flag.String("dest", ".", "where to download")
	port       = flag.Int("port", 0, "listen port")
	debug      = flag.Bool("debug", false, "enable debug log")
	version    = flag.Bool("version", false, "version")
	seed       = flag.Bool("seed", false, "continue seeding after dowload finishes")
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
)

func main() {
	flag.Parse()
	if *version {
		fmt.Println(torrent.Version)
		return
	}
	args := flag.Args()
	if len(args) == 0 {
		_, _ = fmt.Fprintln(os.Stderr, "Give a torrent file as first argument!")
		os.Exit(1)
	}
	if *debug {
		logger.SetLogLevel(log.DEBUG)
	}
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
	}
	cfg := client.NewConfig()
	if *configPath != "" {
		cp, err := homedir.Expand(*configPath)
		if err != nil {
			log.Fatal(err)
		}
		err = cfg.LoadFile(cp)
		if err != nil {
			log.Fatal(err)
		}
	}
	f, err := os.Open(args[0])
	if err != nil {
		log.Fatal(err)
	}
	t, err := torrent.New(f, *dest, *port)
	if err != nil {
		log.Fatal(err)
	}
	t.Start()

	sigC := make(chan os.Signal, 1)
	signal.Notify(sigC, syscall.SIGINT, syscall.SIGTERM)
LOOP:
	for {
		select {
		case <-sigC:
			break LOOP
		case <-t.CompleteNotify():
			if !*seed {
				break LOOP
			}
		}
	}
	err = t.Close()
	if err != nil {
		log.Fatal(err)
	}
	if *cpuprofile != "" {
		pprof.StopCPUProfile()
		f.Close()
	}
}
<commit_msg>cpuprofile<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"runtime/pprof"
	"syscall"

	"github.com/cenkalti/log"
	"github.com/cenkalti/rain/client"
	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/torrent"
	"github.com/mitchellh/go-homedir"
)

var (
	configPath = flag.String("config", "", "config path")
	dest       = flag.String("dest", ".", "where to download")
	port       = flag.Int("port", 0, "listen port")
	debug      = flag.Bool("debug", false, "enable debug log")
	version    = flag.Bool("version", false, "version")
	seed       = flag.Bool("seed", false, "continue seeding after dowload finishes")
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to `file`")
)

func main() {
	flag.Parse()
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal("could not create CPU profile: ", err)
		}
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatal("could not start CPU profile: ", err)
		}
		defer pprof.StopCPUProfile()
	}
	if *version {
		fmt.Println(torrent.Version)
		return
	}
	args := flag.Args()
	if len(args) == 0 {
		_, _ = fmt.Fprintln(os.Stderr, "Give a torrent file as first argument!")
		os.Exit(1)
	}
	if *debug {
		logger.SetLogLevel(log.DEBUG)
	}
	cfg := client.NewConfig()
	if *configPath != "" {
		cp, err := homedir.Expand(*configPath)
		if err != nil {
			log.Fatal(err)
		}
		err = cfg.LoadFile(cp)
		if err != nil {
			log.Fatal(err)
		}
	}
	f, err := os.Open(args[0])
	if err != nil {
		log.Fatal(err)
	}
	t, err := torrent.New(f, *dest, *port)
	if err != nil {
		log.Fatal(err)
	}
	t.Start()

	sigC := make(chan os.Signal, 1)
	signal.Notify(sigC, syscall.SIGINT, syscall.SIGTERM)
LOOP:
	for {
		select {
		case <-sigC:
			break LOOP
		case <-t.CompleteNotify():
			if !*seed {
				break LOOP
			}
		}
	}
	err = t.Close()
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zoekt

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"log"
	"sort"
)

// reader is a stateful file
type reader struct {
	r   IndexFile
	off uint32
}

func (r *reader) seek(off uint32) {
	r.off = off
}

func (r *reader) U32() (uint32, error) {
	b, err := r.r.Read(r.off, 4)
	r.off += 4
	if err != nil {
		return 0, err
	}
	return binary.BigEndian.Uint32(b), nil
}

var _ = log.Println

func (r *reader) readTOC(toc *indexTOC) error {
	sz, err := r.r.Size()
	if err != nil {
		return err
	}
	r.off = sz - 8

	var tocSection simpleSection
	if err := tocSection.read(r); err != nil {
		return err
	}

	r.seek(tocSection.off)

	sectionCount, err := r.U32()
	if err != nil {
		return err
	}

	secs := toc.sections()

	if len(secs) != int(sectionCount) {
		return fmt.Errorf("section count mismatch: got %d want %d", len(secs), sectionCount)
	}

	for _, s := range toc.sections() {
		if err := s.read(r); err != nil {
			return err
		}
	}
	return nil
}

func (r *indexData) readSectionBlob(sec simpleSection) ([]byte, error) {
	return r.file.Read(sec.off, sec.sz)
}

func (r *indexData) readSectionU32(sec simpleSection) ([]uint32, error) {
	return readSectionU32(r.file, sec)
}

func readSectionU32(f IndexFile, sec simpleSection) ([]uint32, error) {
	if sec.sz%4 != 0 {
		return nil, fmt.Errorf("barf: section size %% 4 != 0: sz %d ", sec.sz)
	}
	blob, err := f.Read(sec.off, sec.sz)
	if err != nil {
		return nil, err
	}
	arr := make([]uint32, 0, len(blob)/4)
	for len(blob) > 0 {
		arr = append(arr, binary.BigEndian.Uint32(blob))
		blob = blob[4:]
	}
	return arr, nil
}

func (r *reader) readIndexData(toc *indexTOC) (*indexData, error) {
	d := indexData{
		file:           r.r,
		ngrams:         map[ngram]simpleSection{},
		fileNameNgrams: map[ngram][]uint32{},
		branchIDs:      map[string]uint{},
		branchNames:    map[uint]string{},
	}
	blob, err := d.readSectionBlob(toc.unaryData)
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(blob, &d.unaryData); err != nil {
		return nil, err
	}

	if d.unaryData.IndexFormatVersion != IndexFormatVersion {
		return nil, fmt.Errorf("file is v%d, want v%d", d.unaryData.IndexFormatVersion, IndexFormatVersion)
	}

	d.boundaries = toc.fileContents.absoluteIndex()
	d.newlinesIndex = toc.newlines.absoluteIndex()
	d.docSectionsIndex = toc.fileSections.absoluteIndex()

	textContent, err := d.readSectionBlob(toc.ngramText)
	if err != nil {
		return nil, err
	}
	postingsIndex := toc.postings.absoluteIndex()

	for i := 0; i < len(textContent); i += ngramSize {
		j := i / ngramSize
		d.ngrams[bytesToNGram(textContent[i:i+ngramSize])] = simpleSection{
			postingsIndex[j],
			postingsIndex[j+1] - postingsIndex[j],
		}
	}

	if r := toc.fileContents.relativeIndex(); len(r) > 0 {
		d.fileEnds = r[1:]
	}
	d.fileBranchMasks, err = d.readSectionU32(toc.branchMasks)
	if err != nil {
		return nil, err
	}

	d.fileNameContent, err = d.readSectionBlob(toc.fileNames.data)
	if err != nil {
		return nil, err
	}

	d.fileNameIndex = toc.fileNames.relativeIndex()

	nameNgramText, err := d.readSectionBlob(toc.nameNgramText)
	if err != nil {
		return nil, err
	}

	fileNamePostingsData, err := d.readSectionBlob(toc.namePostings.data)
	if err != nil {
		return nil, err
	}

	fileNamePostingsIndex := toc.namePostings.relativeIndex()
	for i := 0; i < len(nameNgramText); i += ngramSize {
		j := i / ngramSize
		off := fileNamePostingsIndex[j]
		end := fileNamePostingsIndex[j+1]
		ngram := bytesToNGram(nameNgramText[i : i+ngramSize])
		d.fileNameNgrams[ngram] = fromDeltas(fileNamePostingsData[off:end], nil)
	}

	for j, br := range d.unaryData.Repository.Branches {
		id := uint(1) << uint(j)
		d.branchIDs[br.Name] = id
		d.branchNames[id] = br.Name
	}

	if blob, err := d.readSectionBlob(toc.subRepos); err != nil {
		return nil, err
	} else {
		d.subRepos = fromSizedDeltas(blob, nil)
	}

	var keys []string
	for k := range d.unaryData.SubRepoMap {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	d.subRepoPaths = keys

	if err := d.verify(); err != nil {
		return nil, err
	}

	return &d, nil
}

func (d *indexData) verify() error {
	n := len(d.fileNameIndex)
	if n == 0 {
		return nil
	}

	n--
	if len(d.fileEnds) != n {
		return fmt.Errorf("file ends %d != %d", len(d.fileEnds), n)
	}
	if len(d.boundaries) != n+1 {
		return fmt.Errorf("file name idx %d != %d", len(d.fileNameIndex), n+1)
	}
	if len(d.fileBranchMasks) != n {
		return fmt.Errorf("branch masks.")
	}
	if len(d.docSectionsIndex) != n+1 {
		return fmt.Errorf("doc sections.")
	}
	if len(d.newlinesIndex) != n+1 {
		return fmt.Errorf("nls sections.")
	}
	return nil
}

func (d *indexData) readContents(i uint32) ([]byte, error) {
	return d.readSectionBlob(simpleSection{
		off: d.boundaries[i],
		sz:  d.boundaries[i+1] - d.boundaries[i],
	})
}

func (d *indexData) readNewlines(i uint32, buf []uint32) ([]uint32, error) {
	blob, err := d.readSectionBlob(simpleSection{
		off: d.newlinesIndex[i],
		sz:  d.newlinesIndex[i+1] - d.newlinesIndex[i],
	})
	if err != nil {
		return nil, err
	}

	return fromSizedDeltas(blob, buf), nil
}

func (d *indexData) readDocSections(i uint32) ([]DocumentSection, error) {
	blob, err := d.readSectionBlob(simpleSection{
		off: d.docSectionsIndex[i],
		sz:  d.docSectionsIndex[i+1] - d.docSectionsIndex[i],
	})
	if err != nil {
		return nil, err
	}

	return unmarshalDocSections(blob), nil
}

// IndexFile is a file suitable for concurrent read access. For performance
// reasons, it allows a mmap'd implementation.
type IndexFile interface {
	Read(off uint32, sz uint32) ([]byte, error)
	Size() (uint32, error)
	Close()
	Name() string
}

// NewSearcher creates a Searcher for a single index file.
func NewSearcher(r IndexFile) (Searcher, error) {
	rd := &reader{r: r}

	var toc indexTOC
	if err := rd.readTOC(&toc); err != nil {
		return nil, err
	}
	indexData, err := rd.readIndexData(&toc)
	if err != nil {
		return nil, err
	}
	indexData.file = r
	return indexData, nil
}
<commit_msg>Apply fixes meant for previous commit.<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zoekt

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"log"
	"sort"
)

// reader is a stateful file
type reader struct {
	r   IndexFile
	off uint32
}

func (r *reader) seek(off uint32) {
	r.off = off
}

func (r *reader) U32() (uint32, error) {
	b, err := r.r.Read(r.off, 4)
	r.off += 4
	if err != nil {
		return 0, err
	}
	return binary.BigEndian.Uint32(b), nil
}

var _ = log.Println

func (r *reader) readTOC(toc *indexTOC) error {
	sz, err := r.r.Size()
	if err != nil {
		return err
	}
	r.off = sz - 8

	var tocSection simpleSection
	if err := tocSection.read(r); err != nil {
		return err
	}

	r.seek(tocSection.off)

	sectionCount, err := r.U32()
	if err != nil {
		return err
	}

	secs := toc.sections()

	if len(secs) != int(sectionCount) {
		return fmt.Errorf("section count mismatch: got %d want %d", len(secs), sectionCount)
	}

	for _, s := range toc.sections() {
		if err := s.read(r); err != nil {
			return err
		}
	}
	return nil
}

func (r *indexData) readSectionBlob(sec simpleSection) ([]byte, error) {
	return r.file.Read(sec.off, sec.sz)
}

func (r *indexData) readSectionU32(sec simpleSection) ([]uint32, error) {
	return readSectionU32(r.file, sec)
}

func readSectionU32(f IndexFile, sec simpleSection) ([]uint32, error) {
	if sec.sz%4 != 0 {
		return nil, fmt.Errorf("barf: section size %% 4 != 0: sz %d ", sec.sz)
	}
	blob, err := f.Read(sec.off, sec.sz)
	if err != nil {
		return nil, err
	}
	arr := make([]uint32, 0, len(blob)/4)
	for len(blob) > 0 {
		arr = append(arr, binary.BigEndian.Uint32(blob))
		blob = blob[4:]
	}
	return arr, nil
}

func (r *reader) readIndexData(toc *indexTOC) (*indexData, error) {
	d := indexData{
		file:           r.r,
		ngrams:         map[ngram]simpleSection{},
		fileNameNgrams: map[ngram][]uint32{},
		branchIDs:      map[string]uint{},
		branchNames:    map[uint]string{},
	}
	blob, err := d.readSectionBlob(toc.unaryData)
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(blob, &d.unaryData); err != nil {
		return nil, err
	}

	if d.unaryData.IndexFormatVersion != IndexFormatVersion {
		return nil, fmt.Errorf("file is v%d, want v%d", d.unaryData.IndexFormatVersion, IndexFormatVersion)
	}

	d.boundaries = toc.fileContents.absoluteIndex()
	d.newlinesIndex = toc.newlines.absoluteIndex()
	d.docSectionsIndex = toc.fileSections.absoluteIndex()

	textContent, err := d.readSectionBlob(toc.ngramText)
	if err != nil {
		return nil, err
	}
	postingsIndex := toc.postings.absoluteIndex()

	for i := 0; i < len(textContent); i += ngramSize {
		j := i / ngramSize
		d.ngrams[bytesToNGram(textContent[i:i+ngramSize])] = simpleSection{
			postingsIndex[j],
			postingsIndex[j+1] - postingsIndex[j],
		}
	}

	if r := toc.fileContents.relativeIndex(); len(r) > 0 {
		d.fileEnds = r[1:]
	}
	d.fileBranchMasks, err = d.readSectionU32(toc.branchMasks)
	if err != nil {
		return nil, err
	}

	d.fileNameContent, err = d.readSectionBlob(toc.fileNames.data)
	if err != nil {
		return nil, err
	}

	d.fileNameIndex = toc.fileNames.relativeIndex()

	nameNgramText, err := d.readSectionBlob(toc.nameNgramText)
	if err != nil {
		return nil, err
	}

	fileNamePostingsData, err := d.readSectionBlob(toc.namePostings.data)
	if err != nil {
		return nil, err
	}

	fileNamePostingsIndex := toc.namePostings.relativeIndex()
	for i := 0; i < len(nameNgramText); i += ngramSize {
		j := i / ngramSize
		off := fileNamePostingsIndex[j]
		end := fileNamePostingsIndex[j+1]
		ngram := bytesToNGram(nameNgramText[i : i+ngramSize])
		d.fileNameNgrams[ngram] = fromDeltas(fileNamePostingsData[off:end], nil)
	}

	for j, br := range d.unaryData.Repository.Branches {
		id := uint(1) << uint(j)
		d.branchIDs[br.Name] = id
		d.branchNames[id] = br.Name
	}

	if blob, err := d.readSectionBlob(toc.subRepos); err != nil {
		return nil, err
	} else {
		d.subRepos = fromSizedDeltas(blob, nil)
	}

	var keys []string
	for k := range d.unaryData.SubRepoMap {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	d.subRepoPaths = keys

	if err := d.verify(); err != nil {
		return nil, err
	}

	return &d, nil
}

func (d *indexData) verify() error {
	// This is not an exhaustive check: the postings can easily
	// generate OOB acccesses, and are expensive to check, but this lets us rule out
	// other sources of OOB access.
	n := len(d.fileNameIndex)
	if n == 0 {
		return nil
	}

	n--
	for what, got := range map[string]int{
		"file ends":         len(d.fileEnds),
		"boundaries":        len(d.boundaries) - 1,
		"branch masks":      len(d.fileBranchMasks),
		"doc section index": len(d.docSectionsIndex) - 1,
		"newlines index":    len(d.newlinesIndex) - 1,
	} {
		if got != n {
			return fmt.Errorf("got %s %d, want %d", what, got, n)
		}
	}
	return nil
}

func (d *indexData) readContents(i uint32) ([]byte, error) {
	return d.readSectionBlob(simpleSection{
		off: d.boundaries[i],
		sz:  d.boundaries[i+1] - d.boundaries[i],
	})
}

func (d *indexData) readNewlines(i uint32, buf []uint32) ([]uint32, error) {
	blob, err := d.readSectionBlob(simpleSection{
		off: d.newlinesIndex[i],
		sz:  d.newlinesIndex[i+1] - d.newlinesIndex[i],
	})
	if err != nil {
		return nil, err
	}

	return fromSizedDeltas(blob, buf), nil
}

func (d *indexData) readDocSections(i uint32) ([]DocumentSection, error) {
	blob, err := d.readSectionBlob(simpleSection{
		off: d.docSectionsIndex[i],
		sz:  d.docSectionsIndex[i+1] - d.docSectionsIndex[i],
	})
	if err != nil {
		return nil, err
	}

	return unmarshalDocSections(blob), nil
}

// IndexFile is a file suitable for concurrent read access. For performance
// reasons, it allows a mmap'd implementation.
type IndexFile interface {
	Read(off uint32, sz uint32) ([]byte, error)
	Size() (uint32, error)
	Close()
	Name() string
}

// NewSearcher creates a Searcher for a single index file.
func NewSearcher(r IndexFile) (Searcher, error) {
	rd := &reader{r: r}

	var toc indexTOC
	if err := rd.readTOC(&toc); err != nil {
		return nil, err
	}
	indexData, err := rd.readIndexData(&toc)
	if err != nil {
		return nil, err
	}
	indexData.file = r
	return indexData, nil
}
<|endoftext|>


<commit_before>package input

import (
	"fmt"
	"io"
	"os"
	"os/signal"
	"strings"
)

// readOptions is option for read func
type readOptions struct {
	// mask hides user input and will be matched by maskVal.
	mask    bool
	maskVal string
}

// read reads input from UI.Reader
func (i *UI) read(opts *readOptions) (string, error) {
	i.once.Do(i.setDefault)

	// sigCh is channel which is watch Interruptted signal (SIGINT)
	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, os.Interrupt)
	defer signal.Stop(sigCh)

	var resultStr string
	var resultErr error
	doneCh := make(chan struct{})

	go func() {
		defer close(doneCh)

		if opts.mask {
			f, ok := i.Reader.(*os.File)
			if !ok {
				resultErr = fmt.Errorf("reader must be a file")
				return
			}

			i.mask, i.maskVal = opts.mask, opts.maskVal
			resultStr, resultErr = i.rawRead(f)
		} else {
			line, err := i.bReader.ReadString('\n')
			if err != nil && err != io.EOF {
				resultErr = fmt.Errorf("failed to read the input: %s", err)
			}

			resultStr = strings.TrimSuffix(line, LineSep)
			// brute force for the moment
			resultStr = strings.TrimSuffix(line, "\n")
		}
	}()

	select {
	case <-sigCh:
		return "", ErrInterrupted
	case <-doneCh:
		return resultStr, resultErr
	}
}

// rawReadline tries to return a single line, not including the end-of-line
// bytes with raw Mode (without prompting nothing). Or if provided show some
// value instead of actual value.
func (i *UI) rawReadline(f *os.File) (string, error) {
	var resultBuf []byte
	for {
		var buf [1]byte
		n, err := f.Read(buf[:])
		if err != nil && err != io.EOF {
			return "", err
		}

		if n == 0 || buf[0] == '\n' || buf[0] == '\r' {
			break
		}

		if buf[0] == 3 {
			return "", ErrInterrupted
		}

		if i.mask {
			fmt.Fprintf(i.Writer, i.maskVal)
		}

		resultBuf = append(resultBuf, buf[0])
	}

	fmt.Fprintf(i.Writer, "\n")
	return string(resultBuf), nil
}
<commit_msg>Fixes trimming for windows.  \r\n are not trimmed when using TrimSuffix<commit_after>package input

import (
	"fmt"
	"io"
	"os"
	"os/signal"
	"strings"
)

// readOptions is option for read func
type readOptions struct {
	// mask hides user input and will be matched by maskVal.
	mask    bool
	maskVal string
}

// read reads input from UI.Reader
func (i *UI) read(opts *readOptions) (string, error) {
	i.once.Do(i.setDefault)

	// sigCh is channel which is watch Interruptted signal (SIGINT)
	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, os.Interrupt)
	defer signal.Stop(sigCh)

	var resultStr string
	var resultErr error
	doneCh := make(chan struct{})

	go func() {
		defer close(doneCh)

		if opts.mask {
			f, ok := i.Reader.(*os.File)
			if !ok {
				resultErr = fmt.Errorf("reader must be a file")
				return
			}

			i.mask, i.maskVal = opts.mask, opts.maskVal
			resultStr, resultErr = i.rawRead(f)
		} else {
			line, err := i.bReader.ReadString('\n')
			if err != nil && err != io.EOF {
				resultErr = fmt.Errorf("failed to read the input: %s", err)
			}

			resultStr = strings.Trim(line, LineSep)
			// brute force for the moment
			resultStr = strings.TrimSuffix(resultStr, "\n")
		}
	}()

	select {
	case <-sigCh:
		return "", ErrInterrupted
	case <-doneCh:
		return resultStr, resultErr
	}
}

// rawReadline tries to return a single line, not including the end-of-line
// bytes with raw Mode (without prompting nothing). Or if provided show some
// value instead of actual value.
func (i *UI) rawReadline(f *os.File) (string, error) {
	var resultBuf []byte
	for {
		var buf [1]byte
		n, err := f.Read(buf[:])
		if err != nil && err != io.EOF {
			return "", err
		}

		if n == 0 || buf[0] == '\n' || buf[0] == '\r' {
			break
		}

		if buf[0] == 3 {
			return "", ErrInterrupted
		}

		if i.mask {
			fmt.Fprintf(i.Writer, i.maskVal)
		}

		resultBuf = append(resultBuf, buf[0])
	}

	fmt.Fprintf(i.Writer, "\n")
	return string(resultBuf), nil
}
<|endoftext|>


<commit_before>// Package vcs provides the ability to work with varying version control systems
// (VCS),  also known as source control systems (SCM) though the same interface.
//
// This package includes a function that attempts to detect the repo type from
// the remote URL and return the proper type. For example,
//
//     remote := "https://github.com/Masterminds/vcs"
//     local, _ := ioutil.TempDir("", "go-vcs")
//     repo, err := NewRepo(remote, local)
//
// In this case repo will be a GitRepo instance. NewRepo can detect the VCS for
// numerous popular VCS and from the URL. For example, a URL ending in .git
// that's not from one of the popular VCS will be detected as a Git repo and
// the correct type will be returned.
//
// If you know the repository type and would like to create an instance of a
// specific type you can use one of constructors for a type. They are NewGitRepo,
// NewSvnRepo, NewBzrRepo, and NewHgRepo. The definition and usage is the same
// as NewRepo.
//
// Once you have an object implementing the Repo interface the operations are
// the same no matter which VCS you're using. There are some caveats. For
// example, each VCS has its own version formats that need to be respected and
// checkout out branches, if a branch is being worked with, is different in
// each VCS.
package vcs

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"
)

var (
	// ErrWrongVCS is returned when an action is tried on the wrong VCS.
	ErrWrongVCS = errors.New("Wrong VCS detected")

	// ErrCannotDetectVCS is returned when VCS cannot be detected from URI string.
	ErrCannotDetectVCS = errors.New("Cannot detect VCS")

	// ErrWrongRemote occurs when the passed in remote does not match the VCS
	// configured endpoint.
	ErrWrongRemote = errors.New("The Remote does not match the VCS endpoint")

	// ErrRevisionUnavailable happens when commit revision information is
	// unavailable.
	ErrRevisionUnavailable = errors.New("Revision unavailable")
)

// Logger is where you can provide a logger, implementing the log.Logger interface,
// where verbose output from each VCS will be written. The default logger does
// not log data. To log data supply your own logger or change the output location
// of the provided logger.
var Logger *log.Logger

func init() {
	// Initialize the logger to one that does not actually log anywhere. This is
	// to be overridden by the package user by setting vcs.Logger to a different
	// logger.
	Logger = log.New(ioutil.Discard, "go-vcs", log.LstdFlags)
}

const longForm = "2006-01-02 15:04:05 -0700"

// Type describes the type of VCS
type Type string

// VCS types
const (
	NoVCS Type = ""
	Git   Type = "git"
	Svn   Type = "svn"
	Bzr   Type = "bzr"
	Hg    Type = "hg"
)

// Repo provides an interface to work with repositories using different source
// control systems such as Git, Bzr, Mercurial, and SVN. For implementations
// of this interface see BzrRepo, GitRepo, HgRepo, and SvnRepo.
type Repo interface {

	// Vcs retrieves the underlying VCS being implemented.
	Vcs() Type

	// Remote retrieves the remote location for a repo.
	Remote() string

	// LocalPath retrieves the local file system location for a repo.
	LocalPath() string

	// Get is used to perform an initial clone/checkout of a repository.
	Get() error

	// Update performs an update to an existing checkout of a repository.
	Update() error

	// UpdateVersion sets the version of a package of a repository.
	UpdateVersion(string) error

	// Version retrieves the current version.
	Version() (string, error)

	// Date retrieves the date on the latest commit.
	Date() (time.Time, error)

	// CheckLocal verifies the local location is of the correct VCS type
	CheckLocal() bool

	// Branches returns a list of available branches on the repository.
	Branches() ([]string, error)

	// Tags returns a list of available tags on the repository.
	Tags() ([]string, error)

	// TODO: Provide a consistent manner to get reference information across
	// multiple VCS.

	// IsReference returns if a string is a reference. A reference can be a
	// commit id, branch, or tag.
	IsReference(string) bool

	// IsDirty returns if the checkout has been modified from the checked
	// out reference.
	IsDirty() bool

	// CommitInfo retrieves metadata about a commit.
	CommitInfo(string) (*CommitInfo, error)

	// Ping returns if remote location is accessible.
	Ping() bool

	// Runs a command from repo's directory.
	RunFromDir(cmd string, args ...string) ([]byte, error)
}

// NewRepo returns a Repo based on trying to detect the source control from the
// remote and local locations. The appropriate implementation will be returned
// or an ErrCannotDetectVCS if the VCS type cannot be detected.
// Note, this function may make calls to the Internet to determind help determine
// the VCS.
func NewRepo(remote, local string) (Repo, error) {
	vtype, remote, err := detectVcsFromRemote(remote)

	// From the remote URL the VCS could not be detected. See if the local
	// repo contains enough information to figure out the VCS. The reason the
	// local repo is not checked first is because of the potential for VCS type
	// switches which will be detected in each of the type builders.
	if err == ErrCannotDetectVCS {
		vtype, err = DetectVcsFromFS(local)
	}

	if err != nil {
		return nil, err
	}

	switch vtype {
	case Git:
		return NewGitRepo(remote, local)
	case Svn:
		return NewSvnRepo(remote, local)
	case Hg:
		return NewHgRepo(remote, local)
	case Bzr:
		return NewBzrRepo(remote, local)
	}

	// Should never fall through to here but just in case.
	return nil, ErrCannotDetectVCS
}

// CommitInfo contains metadata about a commit.
type CommitInfo struct {
	// The commit id
	Commit string

	// Who authored the commit
	Author string

	// Date of the commit
	Date time.Time

	// Commit message
	Message string
}

type base struct {
	remote, local string
	Logger        *log.Logger
}

func (b *base) log(v interface{}) {
	b.Logger.Printf("%s", v)
}

// Remote retrieves the remote location for a repo.
func (b *base) Remote() string {
	return b.remote
}

// LocalPath retrieves the local file system location for a repo.
func (b *base) LocalPath() string {
	return b.local
}

func (b *base) setRemote(remote string) {
	b.remote = remote
}

func (b *base) setLocalPath(local string) {
	b.local = local
}

func (b base) run(cmd string, args ...string) ([]byte, error) {
	out, err := exec.Command(cmd, args...).CombinedOutput()
	b.log(out)
	if err != nil {
		err = fmt.Errorf("%s: %s", out, err)
	}
	return out, err
}

func (b *base) RunFromDir(cmd string, args ...string) ([]byte, error) {
	c := exec.Command(cmd, args...)
	c.Dir = b.local
	c.Env = envForDir(c.Dir)
	out, err := c.CombinedOutput()
	return out, err
}

func (b *base) referenceList(c, r string) []string {
	var out []string
	re := regexp.MustCompile(r)
	for _, m := range re.FindAllStringSubmatch(c, -1) {
		out = append(out, m[1])
	}

	return out
}

func envForDir(dir string) []string {
	env := os.Environ()
	return mergeEnvLists([]string{"PWD=" + dir}, env)
}

func mergeEnvLists(in, out []string) []string {
NextVar:
	for _, inkv := range in {
		k := strings.SplitAfterN(inkv, "=", 2)[0]
		for i, outkv := range out {
			if strings.HasPrefix(outkv, k) {
				out[i] = inkv
				continue NextVar
			}
		}
		out = append(out, inkv)
	}
	return out
}
<commit_msg>Fixing interface comment per golint<commit_after>// Package vcs provides the ability to work with varying version control systems
// (VCS),  also known as source control systems (SCM) though the same interface.
//
// This package includes a function that attempts to detect the repo type from
// the remote URL and return the proper type. For example,
//
//     remote := "https://github.com/Masterminds/vcs"
//     local, _ := ioutil.TempDir("", "go-vcs")
//     repo, err := NewRepo(remote, local)
//
// In this case repo will be a GitRepo instance. NewRepo can detect the VCS for
// numerous popular VCS and from the URL. For example, a URL ending in .git
// that's not from one of the popular VCS will be detected as a Git repo and
// the correct type will be returned.
//
// If you know the repository type and would like to create an instance of a
// specific type you can use one of constructors for a type. They are NewGitRepo,
// NewSvnRepo, NewBzrRepo, and NewHgRepo. The definition and usage is the same
// as NewRepo.
//
// Once you have an object implementing the Repo interface the operations are
// the same no matter which VCS you're using. There are some caveats. For
// example, each VCS has its own version formats that need to be respected and
// checkout out branches, if a branch is being worked with, is different in
// each VCS.
package vcs

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"
)

var (
	// ErrWrongVCS is returned when an action is tried on the wrong VCS.
	ErrWrongVCS = errors.New("Wrong VCS detected")

	// ErrCannotDetectVCS is returned when VCS cannot be detected from URI string.
	ErrCannotDetectVCS = errors.New("Cannot detect VCS")

	// ErrWrongRemote occurs when the passed in remote does not match the VCS
	// configured endpoint.
	ErrWrongRemote = errors.New("The Remote does not match the VCS endpoint")

	// ErrRevisionUnavailable happens when commit revision information is
	// unavailable.
	ErrRevisionUnavailable = errors.New("Revision unavailable")
)

// Logger is where you can provide a logger, implementing the log.Logger interface,
// where verbose output from each VCS will be written. The default logger does
// not log data. To log data supply your own logger or change the output location
// of the provided logger.
var Logger *log.Logger

func init() {
	// Initialize the logger to one that does not actually log anywhere. This is
	// to be overridden by the package user by setting vcs.Logger to a different
	// logger.
	Logger = log.New(ioutil.Discard, "go-vcs", log.LstdFlags)
}

const longForm = "2006-01-02 15:04:05 -0700"

// Type describes the type of VCS
type Type string

// VCS types
const (
	NoVCS Type = ""
	Git   Type = "git"
	Svn   Type = "svn"
	Bzr   Type = "bzr"
	Hg    Type = "hg"
)

// Repo provides an interface to work with repositories using different source
// control systems such as Git, Bzr, Mercurial, and SVN. For implementations
// of this interface see BzrRepo, GitRepo, HgRepo, and SvnRepo.
type Repo interface {

	// Vcs retrieves the underlying VCS being implemented.
	Vcs() Type

	// Remote retrieves the remote location for a repo.
	Remote() string

	// LocalPath retrieves the local file system location for a repo.
	LocalPath() string

	// Get is used to perform an initial clone/checkout of a repository.
	Get() error

	// Update performs an update to an existing checkout of a repository.
	Update() error

	// UpdateVersion sets the version of a package of a repository.
	UpdateVersion(string) error

	// Version retrieves the current version.
	Version() (string, error)

	// Date retrieves the date on the latest commit.
	Date() (time.Time, error)

	// CheckLocal verifies the local location is of the correct VCS type
	CheckLocal() bool

	// Branches returns a list of available branches on the repository.
	Branches() ([]string, error)

	// Tags returns a list of available tags on the repository.
	Tags() ([]string, error)

	// TODO: Provide a consistent manner to get reference information across
	// multiple VCS.

	// IsReference returns if a string is a reference. A reference can be a
	// commit id, branch, or tag.
	IsReference(string) bool

	// IsDirty returns if the checkout has been modified from the checked
	// out reference.
	IsDirty() bool

	// CommitInfo retrieves metadata about a commit.
	CommitInfo(string) (*CommitInfo, error)

	// Ping returns if remote location is accessible.
	Ping() bool

	// RunFromDir executes a command from repo's directory.
	RunFromDir(cmd string, args ...string) ([]byte, error)
}

// NewRepo returns a Repo based on trying to detect the source control from the
// remote and local locations. The appropriate implementation will be returned
// or an ErrCannotDetectVCS if the VCS type cannot be detected.
// Note, this function may make calls to the Internet to determind help determine
// the VCS.
func NewRepo(remote, local string) (Repo, error) {
	vtype, remote, err := detectVcsFromRemote(remote)

	// From the remote URL the VCS could not be detected. See if the local
	// repo contains enough information to figure out the VCS. The reason the
	// local repo is not checked first is because of the potential for VCS type
	// switches which will be detected in each of the type builders.
	if err == ErrCannotDetectVCS {
		vtype, err = DetectVcsFromFS(local)
	}

	if err != nil {
		return nil, err
	}

	switch vtype {
	case Git:
		return NewGitRepo(remote, local)
	case Svn:
		return NewSvnRepo(remote, local)
	case Hg:
		return NewHgRepo(remote, local)
	case Bzr:
		return NewBzrRepo(remote, local)
	}

	// Should never fall through to here but just in case.
	return nil, ErrCannotDetectVCS
}

// CommitInfo contains metadata about a commit.
type CommitInfo struct {
	// The commit id
	Commit string

	// Who authored the commit
	Author string

	// Date of the commit
	Date time.Time

	// Commit message
	Message string
}

type base struct {
	remote, local string
	Logger        *log.Logger
}

func (b *base) log(v interface{}) {
	b.Logger.Printf("%s", v)
}

// Remote retrieves the remote location for a repo.
func (b *base) Remote() string {
	return b.remote
}

// LocalPath retrieves the local file system location for a repo.
func (b *base) LocalPath() string {
	return b.local
}

func (b *base) setRemote(remote string) {
	b.remote = remote
}

func (b *base) setLocalPath(local string) {
	b.local = local
}

func (b base) run(cmd string, args ...string) ([]byte, error) {
	out, err := exec.Command(cmd, args...).CombinedOutput()
	b.log(out)
	if err != nil {
		err = fmt.Errorf("%s: %s", out, err)
	}
	return out, err
}

func (b *base) RunFromDir(cmd string, args ...string) ([]byte, error) {
	c := exec.Command(cmd, args...)
	c.Dir = b.local
	c.Env = envForDir(c.Dir)
	out, err := c.CombinedOutput()
	return out, err
}

func (b *base) referenceList(c, r string) []string {
	var out []string
	re := regexp.MustCompile(r)
	for _, m := range re.FindAllStringSubmatch(c, -1) {
		out = append(out, m[1])
	}

	return out
}

func envForDir(dir string) []string {
	env := os.Environ()
	return mergeEnvLists([]string{"PWD=" + dir}, env)
}

func mergeEnvLists(in, out []string) []string {
NextVar:
	for _, inkv := range in {
		k := strings.SplitAfterN(inkv, "=", 2)[0]
		for i, outkv := range out {
			if strings.HasPrefix(outkv, k) {
				out[i] = inkv
				continue NextVar
			}
		}
		out = append(out, inkv)
	}
	return out
}
<|endoftext|>


<commit_before>//package resp provides methods to parse and format resp(redis protocal) data
package resp

import (
	"io"
	"fmt"
	"errors"
	"bytes"
	"strconv"
)

const (
	T_SimpleString	= '+'
	T_Error		= '-'
	T_Integer	= ':'
	T_BulkString	= '$'
	T_Array		= '*'
)

type Data struct {
	T byte
	str []byte
	num int64
	array []*Data
	isNil bool
}

//string\bulkString
func (d *Data) String() string {
	return string(d.str)
}

func (d *Data) Byte() []byte {
	return d.str
}

func (d *Data) Error() string {
	return string(d.str)
}

func (d *Data) Integer() int64 {
	return d.num
}

func (d *Data) Array() []*Data {
	return d.array
}

func (d *Data) IsNil() bool {
	return d.isNil == true
}

//format *Data to []byte
func FormatData(d *Data) []byte {
	ret := new(bytes.Buffer)
	ret.WriteByte(d.T)
	switch d.T {
	case T_SimpleString, T_Error:
		fmt.Fprintf(ret, "%s\r\n", d.str)
	case T_Integer:
		fmt.Fprintf(ret, "%d\r\n", d.num)
	case T_BulkString:
		fmt.Fprintf(ret, "%d\r\n%s\r\n", len(d.str), string(d.str))
	case T_Array:
		fmt.Fprintf(ret, "%d\r\n", len(d.array))
		for index := range d.array {
			ret.Write(FormatData(d.array[index]))
		}
	}
	return ret.Bytes()
}

//read from io.Reader, and parse into *Data
func ReadData(r io.Reader) (*Data, error) {

	var buf []byte
	var err error

	buf = make([]byte, 1)
	_, err = io.ReadFull(r, buf)
	if nil != err {
		return nil, err
	}

	ret := &Data{}
	switch buf[0] {
		case '+':
			ret.T = T_SimpleString
			ret.str, err = readRespLine(r)

		case '-':
			ret.T = T_Error
			ret.str, err = readRespLine(r)

		case ':':
			ret.T = T_Integer
			ret.num, err = readRespIntLine(r)

		case '$':
			var lenBulkString int64
			lenBulkString, err = readRespIntLine(r)

			ret.T = T_BulkString
			if -1 == lenBulkString {
				ret.isNil = true
			} else {
				ret.str, err = readRespN(r, lenBulkString)
				//read the followed \r\n
				_, err = readRespN(r, 2)
			}

		case '*':
			var lenArray int64
			var i int64
			lenArray, err = readRespIntLine(r)

			ret.T = T_Array
			if -1 == lenArray {
				ret.isNil = true
			} else if nil==err {
				ret.array = make([]*Data, lenArray)
				for i=0; i<lenArray; i++ {
					ret.array[i], err = ReadData(r)
				}
			}
		default:
			//Inline Commands
			tmp, err := readRespLine(r)
			if nil==err {
				tmpSlice := bytes.Fields(tmp)

				ret.T = T_Array
				ret.array = make([]*Data, len(tmpSlice))
				for index := range tmpSlice {
					t := &Data{}
					t.str = tmpSlice[index]
					t.T = T_SimpleString
					ret.array[index] = t
				}
			}

	}
	return ret, err
}

//读取当前行，并去掉最后的\r\n
func readRespLine(r io.Reader) ([]byte, error) {

	var n, i int
	var err error
	var buf []byte
	var ret *bytes.Buffer

	buf = make([]byte, 1)
	ret = &bytes.Buffer{}

	for {
		n, err = io.ReadFull(r, buf)
		if nil != err {
			return nil, err
		}

		if n==0 {
			continue
		}

		i++
		ret.WriteByte(buf[0])
		if '\n' == buf[0] {
			break
		}
	}

	return ret.Next(i-2), nil
}

//读取N个字节，并去掉最后的\r\n
func readRespN(r io.Reader, n int64) ([]byte, error) {
	var err error
	var ret []byte

	ret = make([]byte, n)
	_, err = io.ReadFull(r, ret)
	if nil!=err {
		ret = nil
	}
	return ret, err
}

//读取当前行的数字，并去掉最后的\r\n
func readRespIntLine(r io.Reader) (int64, error) {
	line, err := readRespLine(r)
	if nil!=err {
		return 0, err
	}
	return strconv.ParseInt(string(line), 10, 64)
}
<commit_msg>add NewData<commit_after>//package resp provides methods to parse and format resp(redis protocal) data
package resp

import (
	"io"
	"fmt"
	"errors"
	"bytes"
	"strconv"
)

const (
	T_SimpleString	= '+'
	T_Error		= '-'
	T_Integer	= ':'
	T_BulkString	= '$'
	T_Array		= '*'
)

type Data struct {
	T byte
	str []byte
	num int64
	array []*Data
	isNil bool
}

//string\bulkString
func (d *Data) String() string {
	return string(d.str)
}

func (d *Data) Byte() []byte {
	return d.str
}

func (d *Data) Error() string {
	return string(d.str)
}

func (d *Data) Integer() int64 {
	return d.num
}

func (d *Data) Array() []*Data {
	return d.array
}

func (d *Data) IsNil() bool {
	return d.isNil == true
}

/*
valid type: string, []byte, error, int64, []interface{}
string -> T_SimpleString
[]byte -> T_BulkString
error -> T_Error
int* -> T_Integer
[]interface{} -> array

*/
func NewData(val interface{}) (ret *Data, err error) {
	ret = new(Data)
	switch val.(type) {
		case string:
			ret.T = T_SimpleString
			ret.str = []byte(val.(string))
		case error:
			ret.T = T_Error
			ret.str = []byte(val.(error).Error())
		case int64, int8, int16, int32:
			ret.T = T_Integer
			ret.num = val.(int64)
		case []interface{}:
			ret.T = T_Array
			ret.array = make([]*Data, len(val.([]interface{})))
			for index := range val.([]interface{}) {
				ret.array[index], err = NewData(val.([]interface{}))
				if nil!=err {
					goto end
				}
			}
		default:
			return nil, errors.New("unsupported type")
	}
end:
	if nil!=err {
		ret = nil
	}
	return ret, err
}

func NewSimpleString() {}
func NewError() {}
func NewInteger() {}
func NewArray() {}

//format *Data to []byte
func FormatData(d *Data) []byte {
	ret := new(bytes.Buffer)
	ret.WriteByte(d.T)
	switch d.T {
	case T_SimpleString, T_Error:
		fmt.Fprintf(ret, "%s\r\n", d.str)
	case T_Integer:
		fmt.Fprintf(ret, "%d\r\n", d.num)
	case T_BulkString:
		fmt.Fprintf(ret, "%d\r\n%s\r\n", len(d.str), string(d.str))
	case T_Array:
		fmt.Fprintf(ret, "%d\r\n", len(d.array))
		for index := range d.array {
			ret.Write(FormatData(d.array[index]))
		}
	}
	return ret.Bytes()
}

//read from io.Reader, and parse into *Data
func ReadData(r io.Reader) (*Data, error) {

	var buf []byte
	var err error

	buf = make([]byte, 1)
	_, err = io.ReadFull(r, buf)
	if nil != err {
		return nil, err
	}

	ret := &Data{}
	switch buf[0] {
		case '+':
			ret.T = T_SimpleString
			ret.str, err = readRespLine(r)

		case '-':
			ret.T = T_Error
			ret.str, err = readRespLine(r)

		case ':':
			ret.T = T_Integer
			ret.num, err = readRespIntLine(r)

		case '$':
			var lenBulkString int64
			lenBulkString, err = readRespIntLine(r)

			ret.T = T_BulkString
			if -1 == lenBulkString {
				ret.isNil = true
			} else {
				ret.str, err = readRespN(r, lenBulkString)
				//read the followed \r\n
				_, err = readRespN(r, 2)
			}

		case '*':
			var lenArray int64
			var i int64
			lenArray, err = readRespIntLine(r)

			ret.T = T_Array
			if -1 == lenArray {
				ret.isNil = true
			} else if nil==err {
				ret.array = make([]*Data, lenArray)
				for i=0; i<lenArray; i++ {
					ret.array[i], err = ReadData(r)
				}
			}
		default:
			//Inline Commands
			tmp, err := readRespLine(r)
			if nil==err {
				tmpSlice := bytes.Fields(tmp)

				ret.T = T_Array
				ret.array = make([]*Data, len(tmpSlice))
				for index := range tmpSlice {
					t := &Data{}
					t.str = tmpSlice[index]
					t.T = T_SimpleString
					ret.array[index] = t
				}
			}

	}
	return ret, err
}

//读取当前行，并去掉最后的\r\n
func readRespLine(r io.Reader) ([]byte, error) {

	var n, i int
	var err error
	var buf []byte
	var ret *bytes.Buffer

	buf = make([]byte, 1)
	ret = &bytes.Buffer{}

	for {
		n, err = io.ReadFull(r, buf)
		if nil != err {
			return nil, err
		}

		if n==0 {
			continue
		}

		i++
		ret.WriteByte(buf[0])
		if '\n' == buf[0] {
			break
		}
	}

	return ret.Next(i-2), nil
}

//读取N个字节，并去掉最后的\r\n
func readRespN(r io.Reader, n int64) ([]byte, error) {
	var err error
	var ret []byte

	ret = make([]byte, n)
	_, err = io.ReadFull(r, ret)
	if nil!=err {
		ret = nil
	}
	return ret, err
}

//读取当前行的数字，并去掉最后的\r\n
func readRespIntLine(r io.Reader) (int64, error) {
	line, err := readRespLine(r)
	if nil!=err {
		return 0, err
	}
	return strconv.ParseInt(string(line), 10, 64)
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"log"
	"net"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/gholt/ring"
)

type msgMap struct {
	lock    sync.RWMutex
	mapping map[uint64]ring.MsgUnmarshaller
}

func newMsgMap() *msgMap {
	return &msgMap{mapping: make(map[uint64]ring.MsgUnmarshaller)}
}

func (mm *msgMap) set(t uint64, f ring.MsgUnmarshaller) ring.MsgUnmarshaller {
	mm.lock.Lock()
	p := mm.mapping[t]
	mm.mapping[t] = f
	mm.lock.Unlock()
	return p
}

func (mm *msgMap) get(t uint64) ring.MsgUnmarshaller {
	mm.lock.RLock()
	f := mm.mapping[t]
	mm.lock.RUnlock()
	return f
}

type node struct {
	id uint64
}

func (n *node) NodeID() uint64 {
	return n.id
}

func (n *node) Active() bool {
	return true
}

func (n *node) Capacity() uint32 {
	return 1
}

func (n *node) TierValues() []int {
	return nil
}

func (n *node) Address() string {
	return ""
}

type ringPipe struct {
	ring            ring.Ring
	conn            net.Conn
	lock            sync.RWMutex
	msgMap          *msgMap
	logError        *log.Logger
	logWarning      *log.Logger
	typeBytes       int
	lengthBytes     int
	writeChan       chan ring.Msg
	writingDoneChan chan struct{}
	sendDrops       uint32
}

func NewRingPipe(localNodeAddress string, c net.Conn) *ringPipe {
	b := ring.NewBuilder()
	b.SetReplicaCount(2)
	var localNodeID uint64
	n := b.AddNode(true, 1, nil, []string{"127.0.0.1:11111"}, "")
	if localNodeAddress == "127.0.0.1:11111" {
		localNodeID = n.ID()
	}
	n = b.AddNode(true, 1, nil, []string{"127.0.0.1:22222"}, "")
	if localNodeAddress == "127.0.0.1:22222" {
		localNodeID = n.ID()
	}
	n = b.AddNode(true, 1, nil, []string{"127.0.0.1:33333"}, "")
	if localNodeAddress == "127.0.0.1:33333" {
		localNodeID = n.ID()
	}
	r := b.Ring()
	r.SetLocalNode(localNodeID)
	rp := &ringPipe{
		ring:            r,
		conn:            c,
		msgMap:          newMsgMap(),
		logError:        log.New(os.Stderr, "", log.LstdFlags),
		logWarning:      log.New(os.Stderr, "", log.LstdFlags),
		typeBytes:       1,
		lengthBytes:     3,
		writeChan:       make(chan ring.Msg, 40),
		writingDoneChan: make(chan struct{}, 1),
	}
	return rp
}

func (rp *ringPipe) Ring() ring.Ring {
	return rp.ring
}

func (rp *ringPipe) Start() {
	go rp.reading()
	go rp.writing()
}

const _GLH_SEND_MSG_TIMEOUT = 1

func (rp *ringPipe) MaxMsgLength() uint64 {
	return 16 * 1024 * 1024
}

func (rp *ringPipe) SetMsgHandler(t uint64, h ring.MsgUnmarshaller) {
	rp.msgMap.set(t, h)
}

func (rp *ringPipe) MsgToNode(localNodeID uint64, m ring.Msg) {
	select {
	case rp.writeChan <- m:
	case <-time.After(_GLH_SEND_MSG_TIMEOUT * time.Second):
		atomic.AddUint32(&rp.sendDrops, 1)
	}
	m.Done()
}

func (rp *ringPipe) MsgToOtherReplicas(ringVersion int64, partition uint32, m ring.Msg) {
	// TODO: If ringVersion has changed, partition invalid, etc. return false
	select {
	case rp.writeChan <- m:
	case <-time.After(_GLH_SEND_MSG_TIMEOUT * time.Second):
		atomic.AddUint32(&rp.sendDrops, 1)
	}
	m.Done()
}

func (rp *ringPipe) reading() {
	b := make([]byte, rp.typeBytes+rp.lengthBytes)
	d := make([]byte, 65536)
	for {
		var n int
		var sn int
		var err error
		for n != len(b) {
			if err != nil {
				if n != 0 || err != io.EOF {
					rp.logError.Print("error reading msg", err)
				}
				return
			}
			sn, err = rp.conn.Read(b[n:])
			n += sn
		}
		if err != nil {
			rp.logError.Print("error reading msg start", err)
			return
		}
		var t uint64
		for i := 0; i < rp.typeBytes; i++ {
			t = (t << 8) | uint64(b[i])
		}
		var l uint64
		for i := 0; i < rp.lengthBytes; i++ {
			l = (l << 8) | uint64(b[rp.typeBytes+i])
		}
		f := rp.msgMap.get(t)
		if f != nil {
			_, err = f(rp.conn, l)
			if err != nil {
				rp.logError.Print("error reading msg content", err)
				return
			}
		} else {
			rp.logWarning.Printf("unknown msg type %d", t)
			for l > 0 {
				if err != nil {
					rp.logError.Print("err reading unknown msg content", err)
					return
				}
				if l >= uint64(len(d)) {
					sn, err = rp.conn.Read(d)
				} else {
					sn, err = rp.conn.Read(d[:l])
				}
				l -= uint64(sn)
			}
		}
	}
}

func (rp *ringPipe) writing() {
	b := make([]byte, rp.typeBytes+rp.lengthBytes)
	for {
		m := <-rp.writeChan
		if m == nil {
			break
		}
		t := m.MsgType()
		for i := rp.typeBytes - 1; i >= 0; i-- {
			b[i] = byte(t)
			t >>= 8
		}
		l := m.MsgLength()
		for i := rp.lengthBytes - 1; i >= 0; i-- {
			b[rp.typeBytes+i] = byte(l)
			l >>= 8
		}
		_, err := rp.conn.Write(b)
		if err != nil {
			rp.logError.Print("err writing msg", err)
			break
		}
		_, err = m.WriteContent(rp.conn)
		if err != nil {
			rp.logError.Print("err writing msg content", err)
			break
		}
	}
	rp.writingDoneChan <- struct{}{}
}
<commit_msg>Match tcp msg ring type and length sizes<commit_after>package main

import (
	"io"
	"log"
	"net"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/gholt/ring"
)

type msgMap struct {
	lock    sync.RWMutex
	mapping map[uint64]ring.MsgUnmarshaller
}

func newMsgMap() *msgMap {
	return &msgMap{mapping: make(map[uint64]ring.MsgUnmarshaller)}
}

func (mm *msgMap) set(t uint64, f ring.MsgUnmarshaller) ring.MsgUnmarshaller {
	mm.lock.Lock()
	p := mm.mapping[t]
	mm.mapping[t] = f
	mm.lock.Unlock()
	return p
}

func (mm *msgMap) get(t uint64) ring.MsgUnmarshaller {
	mm.lock.RLock()
	f := mm.mapping[t]
	mm.lock.RUnlock()
	return f
}

type node struct {
	id uint64
}

func (n *node) NodeID() uint64 {
	return n.id
}

func (n *node) Active() bool {
	return true
}

func (n *node) Capacity() uint32 {
	return 1
}

func (n *node) TierValues() []int {
	return nil
}

func (n *node) Address() string {
	return ""
}

type ringPipe struct {
	ring            ring.Ring
	conn            net.Conn
	lock            sync.RWMutex
	msgMap          *msgMap
	logError        *log.Logger
	logWarning      *log.Logger
	typeBytes       int
	lengthBytes     int
	writeChan       chan ring.Msg
	writingDoneChan chan struct{}
	sendDrops       uint32
}

func NewRingPipe(localNodeAddress string, c net.Conn) *ringPipe {
	b := ring.NewBuilder()
	b.SetReplicaCount(2)
	var localNodeID uint64
	n := b.AddNode(true, 1, nil, []string{"127.0.0.1:11111"}, "")
	if localNodeAddress == "127.0.0.1:11111" {
		localNodeID = n.ID()
	}
	n = b.AddNode(true, 1, nil, []string{"127.0.0.1:22222"}, "")
	if localNodeAddress == "127.0.0.1:22222" {
		localNodeID = n.ID()
	}
	n = b.AddNode(true, 1, nil, []string{"127.0.0.1:33333"}, "")
	if localNodeAddress == "127.0.0.1:33333" {
		localNodeID = n.ID()
	}
	r := b.Ring()
	r.SetLocalNode(localNodeID)
	rp := &ringPipe{
		ring:            r,
		conn:            c,
		msgMap:          newMsgMap(),
		logError:        log.New(os.Stderr, "", log.LstdFlags),
		logWarning:      log.New(os.Stderr, "", log.LstdFlags),
		typeBytes:       8,
		lengthBytes:     8,
		writeChan:       make(chan ring.Msg, 40),
		writingDoneChan: make(chan struct{}, 1),
	}
	return rp
}

func (rp *ringPipe) Ring() ring.Ring {
	return rp.ring
}

func (rp *ringPipe) Start() {
	go rp.reading()
	go rp.writing()
}

const _GLH_SEND_MSG_TIMEOUT = 1

func (rp *ringPipe) MaxMsgLength() uint64 {
	return 16 * 1024 * 1024
}

func (rp *ringPipe) SetMsgHandler(t uint64, h ring.MsgUnmarshaller) {
	rp.msgMap.set(t, h)
}

func (rp *ringPipe) MsgToNode(localNodeID uint64, m ring.Msg) {
	select {
	case rp.writeChan <- m:
	case <-time.After(_GLH_SEND_MSG_TIMEOUT * time.Second):
		atomic.AddUint32(&rp.sendDrops, 1)
	}
	m.Done()
}

func (rp *ringPipe) MsgToOtherReplicas(ringVersion int64, partition uint32, m ring.Msg) {
	// TODO: If ringVersion has changed, partition invalid, etc. return false
	select {
	case rp.writeChan <- m:
	case <-time.After(_GLH_SEND_MSG_TIMEOUT * time.Second):
		atomic.AddUint32(&rp.sendDrops, 1)
	}
	m.Done()
}

func (rp *ringPipe) reading() {
	b := make([]byte, rp.typeBytes+rp.lengthBytes)
	d := make([]byte, 65536)
	for {
		var n int
		var sn int
		var err error
		for n != len(b) {
			if err != nil {
				if n != 0 || err != io.EOF {
					rp.logError.Print("error reading msg", err)
				}
				return
			}
			sn, err = rp.conn.Read(b[n:])
			n += sn
		}
		if err != nil {
			rp.logError.Print("error reading msg start", err)
			return
		}
		var t uint64
		for i := 0; i < rp.typeBytes; i++ {
			t = (t << 8) | uint64(b[i])
		}
		var l uint64
		for i := 0; i < rp.lengthBytes; i++ {
			l = (l << 8) | uint64(b[rp.typeBytes+i])
		}
		f := rp.msgMap.get(t)
		if f != nil {
			_, err = f(rp.conn, l)
			if err != nil {
				rp.logError.Print("error reading msg content", err)
				return
			}
		} else {
			rp.logWarning.Printf("unknown msg type %d", t)
			for l > 0 {
				if err != nil {
					rp.logError.Print("err reading unknown msg content", err)
					return
				}
				if l >= uint64(len(d)) {
					sn, err = rp.conn.Read(d)
				} else {
					sn, err = rp.conn.Read(d[:l])
				}
				l -= uint64(sn)
			}
		}
	}
}

func (rp *ringPipe) writing() {
	b := make([]byte, rp.typeBytes+rp.lengthBytes)
	for {
		m := <-rp.writeChan
		if m == nil {
			break
		}
		t := m.MsgType()
		for i := rp.typeBytes - 1; i >= 0; i-- {
			b[i] = byte(t)
			t >>= 8
		}
		l := m.MsgLength()
		for i := rp.lengthBytes - 1; i >= 0; i-- {
			b[rp.typeBytes+i] = byte(l)
			l >>= 8
		}
		_, err := rp.conn.Write(b)
		if err != nil {
			rp.logError.Print("err writing msg", err)
			break
		}
		_, err = m.WriteContent(rp.conn)
		if err != nil {
			rp.logError.Print("err writing msg content", err)
			break
		}
	}
	rp.writingDoneChan <- struct{}{}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/negroni"
	"github.com/julienschmidt/httprouter"

	"github.com/wantedly/risu/registry"
	"github.com/wantedly/risu/schema"
)

func create(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	defer r.Body.Close()
	var opts schema.BuildCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		log.Fatal(err)
	}

	if opts.Dockerfile == "" {
		opts.Dockerfile = "Dockerfile"
	}

	build := schema.Build{
		ID:             uuid.NewUUID(),
		SourceRepo:     opts.SourceRepo,
		SourceRevision: opts.SourceRevision,
		Name:           opts.Name,
		Dockerfile:     opts.Dockerfile,
		CreatedAt:      time.Now(),
		UpdatedAt:      time.Now(),
	}

	reg := registry.NewRegistry("localfs", "")
	reg.Set(build)

	// debug code
	builddata, err := reg.Get(build.ID)
	fmt.Fprintln(w, builddata)
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	name := r.FormValue("name")
	fmt.Fprintf(w, "Welcome, %s!\n", name)
}

func show(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	image := ps.ByName("image")
	fmt.Fprintf(w, "Build %s!\n", image)
}

func main() {
	router := httprouter.New()
	router.GET("/", index)
	router.GET("/builds/:image", show)
	router.POST("/builds", create)

	n := negroni.Classic()
	n.UseHandler(router)
	n.Run(":8080")
}
<commit_msg>api: /builds returns build list<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/negroni"
	"github.com/julienschmidt/httprouter"

	"github.com/wantedly/risu/registry"
	"github.com/wantedly/risu/schema"
)

func create(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	defer r.Body.Close()
	var opts schema.BuildCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		log.Fatal(err)
	}

	if opts.Dockerfile == "" {
		opts.Dockerfile = "Dockerfile"
	}

	build := schema.Build{
		ID:             uuid.NewUUID(),
		SourceRepo:     opts.SourceRepo,
		SourceRevision: opts.SourceRevision,
		Name:           opts.Name,
		Dockerfile:     opts.Dockerfile,
		CreatedAt:      time.Now(),
		UpdatedAt:      time.Now(),
	}

	reg := registry.NewRegistry("localfs", "")
	reg.Set(build)

	// debug code
	builddata, err := reg.Get(build.ID)
	fmt.Fprintln(w, builddata)
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	reg := registry.NewRegistry("localfs", "")
	builds, err := reg.List()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println(builds)
}

func show(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	image := ps.ByName("image")
	fmt.Fprintf(w, "Build %s!\n", image)
}

func main() {
	router := httprouter.New()
	router.GET("/builds", index)
	router.GET("/builds/:image", show)
	router.POST("/builds", create)

	n := negroni.Classic()
	n.UseHandler(router)
	n.Run(":8080")
}
<|endoftext|>


<commit_before>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     October 10, 2022
;       related version of root zone:     2022101002
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<commit_msg>Automatic update for Thu Oct 13 08:20:57 UTC 2022<commit_after>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     October 12, 2022
;       related version of root zone:     2022101202
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<|endoftext|>


<commit_before>package main

import (
	"debug/pe"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"reflect"
	"strings"
)

type ImageResourceDirectory struct {
	Characteristics      uint32
	TimeDateStamp        uint32
	MajorVersion         uint16
	MinorVersion         uint16
	NumberOfNamedEntries uint16
	NumberOfIdEntries    uint16
}

type ImageResourceDirectoryEntry struct {
	NameOrId     uint32
	OffsetToData uint32
}

type ImageResourceDataEntry struct {
	OffsetToData uint32
	Size1        uint32
	CodePage     uint32
	Reserved     uint32
}

type RelocationEntry struct {
	RVA         uint32 // "offset within the Section's raw data where the address starts."
	SymbolIndex uint32 // "(zero based) index in the Symbol table to which the reference refers."
	Type        uint16
}

type Symbol struct {
	Name           [8]byte
	Value          uint32
	SectionNumber  uint16
	Type           uint16
	StorageClass   uint8
	AuxiliaryCount uint8
}

type StringsHeader struct {
	Length uint32
}

const (
	MASK_SUBDIRECTORY = 1 << 31
	TYPE_MANIFEST     = 24
)

var (
	STRING_RSRC = [8]byte{'.', 'r', 's', 'r', 'c', 0, 0, 0}
)

func MustGetFieldOffset(t reflect.Type, field string) uintptr {
	f, ok := t.FieldByName(field)
	if !ok {
		panic("field " + field + " not found")
	}
	return f.Offset
}

type Writer struct {
	W      io.Writer
	Offset uint32 //FIXME: uint64?
	Err    error
}

func (w *Writer) WriteLE(v interface{}) {
	if w.Err != nil {
		return
	}
	w.Err = binary.Write(w.W, binary.LittleEndian, v)
	if w.Err != nil {
		return
	}
	w.Offset += uint32(reflect.TypeOf(v).Size())
}

func main() {
	err := run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func run() error {
	//TODO: allow options:
	// -o FILENAME - output file name
	// + advanced specification of multiple resources, as a tree (json?)
	if len(os.Args) <= 1 {
		return fmt.Errorf("USAGE: %s FILE.exe.manifest\n"+
			"Generates FILE.res",
			os.Args[0])
	}

	//TODO: try to simplify by adding fake section at beginning, containing strings table in data, and characteristics saying "drop me when linking"

	fname := os.Args[1]
	suffix := ".exe.manifest"
	if !strings.HasSuffix(fname, suffix) {
		return fmt.Errorf("Filename '%s' does not end in suffix '%s'", fname, suffix)
	}
	fname = fname[:len(fname)-len(suffix)]

	manifest, err := ioutil.ReadFile(fname + suffix)
	if err != nil {
		return err
	}

	out, err := os.Create(fname + ".res")
	if err != nil {
		return err
	}
	defer out.Close()
	w := Writer{W: out}

	// precalculate some important offsets in resulting file, that we must know earlier
	rawdataoff := uint32(binary.Size(pe.FileHeader{}) + binary.Size(pe.SectionHeader32{}))
	hierarchylen := uint32(3*binary.Size(ImageResourceDirectory{}) +
		3*binary.Size(ImageResourceDirectoryEntry{}))
	rawdatalen := hierarchylen +
		uint32(1*binary.Size(ImageResourceDataEntry{})) +
		uint32(len(manifest))
	diroff := rawdataoff
	relocoff := rawdataoff + rawdatalen
	relocp := hierarchylen + uint32(MustGetFieldOffset(reflect.TypeOf(ImageResourceDataEntry{}), "OffsetToData"))
	reloclen := uint32(binary.Size(RelocationEntry{}))
	symoff := relocoff + reloclen

	coffhdr := pe.FileHeader{
		Machine:              0x014c, //FIXME: find out how to differentiate this value, or maybe not necessary for Go
		NumberOfSections:     1,      // .rsrc
		TimeDateStamp:        0,      // was also 0 in sample data from MinGW's windres.exe
		PointerToSymbolTable: uint32(symoff),
		NumberOfSymbols:      1,
		SizeOfOptionalHeader: 0,
		Characteristics:      0x0104, //FIXME: copied from windres.exe output, find out what should be here and why
	}
	w.WriteLE(coffhdr)
	if w.Err != nil {
		return fmt.Errorf("Error writing COFF header: %s", w.Err)
	}

	secthdr := pe.SectionHeader32{
		Name:                 STRING_RSRC,
		SizeOfRawData:        rawdatalen,
		PointerToRawData:     rawdataoff,
		PointerToRelocations: relocoff,
		NumberOfRelocations:  1,
		Characteristics:      0x40000040, // "INITIALIZED_DATA MEM_READ" ?
	}
	w.WriteLE(secthdr)
	if w.Err != nil {
		return fmt.Errorf("Error writing .rsrc section header: %s", w.Err)
	}

	// now, build "directory hierarchy" of .rsrc section: first type, then id/name, then language

	w.WriteLE(ImageResourceDirectory{
		NumberOfIdEntries: 1,
	})
	w.WriteLE(ImageResourceDirectoryEntry{
		NameOrId:     TYPE_MANIFEST,
		OffsetToData: MASK_SUBDIRECTORY | (w.Offset + uint32(binary.Size(ImageResourceDirectoryEntry{})) - diroff),
	})
	w.WriteLE(ImageResourceDirectory{
		NumberOfIdEntries: 1,
	})
	w.WriteLE(ImageResourceDirectoryEntry{
		NameOrId:     1, // ID
		OffsetToData: MASK_SUBDIRECTORY | (w.Offset + uint32(binary.Size(ImageResourceDirectoryEntry{})) - diroff),
	})
	w.WriteLE(ImageResourceDirectory{
		NumberOfIdEntries: 1,
	})
	w.WriteLE(ImageResourceDirectoryEntry{
		NameOrId:     0x0409, //FIXME: language; what value should be here?
		OffsetToData: w.Offset + uint32(binary.Size(ImageResourceDirectoryEntry{})) - diroff,
	})

	w.WriteLE(ImageResourceDataEntry{
		OffsetToData: w.Offset + uint32(binary.Size(ImageResourceDataEntry{})) - diroff,
		Size1:        uint32(len(manifest)),
		CodePage:     0, //FIXME: what value here? for now just tried 0
	})

	if w.Err != nil {
		return fmt.Errorf("Error writing .rsrc Directory Hierarchy: %s", w.Err)
	}

	_, err = w.W.Write(manifest)
	if err != nil {
		return fmt.Errorf("Error writing manifest contents: %s", err)
	}

	w.WriteLE(RelocationEntry{
		RVA:         relocp, // FIXME: IIUC, this resolves to value contained in ImageResourceDataEntry.OffsetToData
		SymbolIndex: 0,      // "(zero based) index in the Symbol table to which the reference refers. Once you have loaded the COFF file into memory and know where each symbol is, you find the new updated address for the given symbol and update the reference accordingly."
		Type:        7,      // according to ldpe.c, this decodes to: IMAGE_REL_I386_DIR32NB
	})

	w.WriteLE(Symbol{
		Name:           STRING_RSRC,
		Value:          0,
		SectionNumber:  1,
		Type:           0, // FIXME: wtf?
		StorageClass:   3, // FIXME: is it ok? and uint8? and what does the value mean?
		AuxiliaryCount: 0, // FIXME: wtf?
	})

	w.WriteLE(StringsHeader{
		Length: uint32(binary.Size(StringsHeader{})), // empty strings table -- but we must still show size of the table's header...
	})

	if w.Err != nil {
		return fmt.Errorf("Error writing .rsrc Symbol Table & Strings: %s", w.Err)
	}

	return nil
}
<commit_msg>add prettier commandline parsing<commit_after>package main

import (
	"debug/pe"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"reflect"
)

type ImageResourceDirectory struct {
	Characteristics      uint32
	TimeDateStamp        uint32
	MajorVersion         uint16
	MinorVersion         uint16
	NumberOfNamedEntries uint16
	NumberOfIdEntries    uint16
}

type ImageResourceDirectoryEntry struct {
	NameOrId     uint32
	OffsetToData uint32
}

type ImageResourceDataEntry struct {
	OffsetToData uint32
	Size1        uint32
	CodePage     uint32
	Reserved     uint32
}

type RelocationEntry struct {
	RVA         uint32 // "offset within the Section's raw data where the address starts."
	SymbolIndex uint32 // "(zero based) index in the Symbol table to which the reference refers."
	Type        uint16
}

type Symbol struct {
	Name           [8]byte
	Value          uint32
	SectionNumber  uint16
	Type           uint16
	StorageClass   uint8
	AuxiliaryCount uint8
}

type StringsHeader struct {
	Length uint32
}

const (
	MASK_SUBDIRECTORY = 1 << 31
	TYPE_MANIFEST     = 24
)

var (
	STRING_RSRC = [8]byte{'.', 'r', 's', 'r', 'c', 0, 0, 0}
)

func MustGetFieldOffset(t reflect.Type, field string) uintptr {
	f, ok := t.FieldByName(field)
	if !ok {
		panic("field " + field + " not found")
	}
	return f.Offset
}

type Writer struct {
	W      io.Writer
	Offset uint32 //FIXME: uint64?
	Err    error
}

func (w *Writer) WriteLE(v interface{}) {
	if w.Err != nil {
		return
	}
	w.Err = binary.Write(w.W, binary.LittleEndian, v)
	if w.Err != nil {
		return
	}
	w.Offset += uint32(reflect.TypeOf(v).Size())
}

func main() {
	//TODO: allow in options advanced specification of multiple resources, as a tree (json?)
	var fnamein, fnameout string
	flags := flag.NewFlagSet("", flag.ContinueOnError)
	flags.StringVar(&fnamein, "manifest", "", "REQUIRED: path to Windows manifest file")
	flags.StringVar(&fnameout, "o", "rsrc.syso", "name of output COFF (.res or .syso) file")
	_ = flags.Parse(os.Args[1:])
	if fnamein == "" {
		fmt.Fprintf(os.Stderr, "USAGE: %s -manifest FILE.exe.manifest [-o FILE.syso]\n"+
			"Generates a .syso file with resources in .rsrc section, for consumption by Go linker.\n"+
			"OPTIONS:\n",
			os.Args[0])
		flags.PrintDefaults()
		os.Exit(1)
	}

	err := run(fnamein, fnameout)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func run(fnamein, fnameout string) error {
	manifest, err := ioutil.ReadFile(fnamein)
	if err != nil {
		return err
	}

	out, err := os.Create(fnameout)
	if err != nil {
		return err
	}
	defer out.Close()
	w := Writer{W: out}

	// precalculate some important offsets in resulting file, that we must know earlier
	//TODO: try to simplify by adding fake section at beginning, containing strings table in data, and characteristics saying "drop me when linking"
	rawdataoff := uint32(binary.Size(pe.FileHeader{}) + binary.Size(pe.SectionHeader32{}))
	hierarchylen := uint32(3*binary.Size(ImageResourceDirectory{}) +
		3*binary.Size(ImageResourceDirectoryEntry{}))
	rawdatalen := hierarchylen +
		uint32(1*binary.Size(ImageResourceDataEntry{})) +
		uint32(len(manifest))
	diroff := rawdataoff
	relocoff := rawdataoff + rawdatalen
	relocp := hierarchylen + uint32(MustGetFieldOffset(reflect.TypeOf(ImageResourceDataEntry{}), "OffsetToData"))
	reloclen := uint32(binary.Size(RelocationEntry{}))
	symoff := relocoff + reloclen

	coffhdr := pe.FileHeader{
		Machine:              0x014c, //FIXME: find out how to differentiate this value, or maybe not necessary for Go
		NumberOfSections:     1,      // .rsrc
		TimeDateStamp:        0,      // was also 0 in sample data from MinGW's windres.exe
		PointerToSymbolTable: uint32(symoff),
		NumberOfSymbols:      1,
		SizeOfOptionalHeader: 0,
		Characteristics:      0x0104, //FIXME: copied from windres.exe output, find out what should be here and why
	}
	w.WriteLE(coffhdr)
	if w.Err != nil {
		return fmt.Errorf("Error writing COFF header: %s", w.Err)
	}

	secthdr := pe.SectionHeader32{
		Name:                 STRING_RSRC,
		SizeOfRawData:        rawdatalen,
		PointerToRawData:     rawdataoff,
		PointerToRelocations: relocoff,
		NumberOfRelocations:  1,
		Characteristics:      0x40000040, // "INITIALIZED_DATA MEM_READ" ?
	}
	w.WriteLE(secthdr)
	if w.Err != nil {
		return fmt.Errorf("Error writing .rsrc section header: %s", w.Err)
	}

	// now, build "directory hierarchy" of .rsrc section: first type, then id/name, then language

	w.WriteLE(ImageResourceDirectory{
		NumberOfIdEntries: 1,
	})
	w.WriteLE(ImageResourceDirectoryEntry{
		NameOrId:     TYPE_MANIFEST,
		OffsetToData: MASK_SUBDIRECTORY | (w.Offset + uint32(binary.Size(ImageResourceDirectoryEntry{})) - diroff),
	})
	w.WriteLE(ImageResourceDirectory{
		NumberOfIdEntries: 1,
	})
	w.WriteLE(ImageResourceDirectoryEntry{
		NameOrId:     1, // ID
		OffsetToData: MASK_SUBDIRECTORY | (w.Offset + uint32(binary.Size(ImageResourceDirectoryEntry{})) - diroff),
	})
	w.WriteLE(ImageResourceDirectory{
		NumberOfIdEntries: 1,
	})
	w.WriteLE(ImageResourceDirectoryEntry{
		NameOrId:     0x0409, //FIXME: language; what value should be here?
		OffsetToData: w.Offset + uint32(binary.Size(ImageResourceDirectoryEntry{})) - diroff,
	})

	w.WriteLE(ImageResourceDataEntry{
		OffsetToData: w.Offset + uint32(binary.Size(ImageResourceDataEntry{})) - diroff,
		Size1:        uint32(len(manifest)),
		CodePage:     0, //FIXME: what value here? for now just tried 0
	})

	if w.Err != nil {
		return fmt.Errorf("Error writing .rsrc Directory Hierarchy: %s", w.Err)
	}

	_, err = w.W.Write(manifest)
	if err != nil {
		return fmt.Errorf("Error writing manifest contents: %s", err)
	}

	w.WriteLE(RelocationEntry{
		RVA:         relocp, // FIXME: IIUC, this resolves to value contained in ImageResourceDataEntry.OffsetToData
		SymbolIndex: 0,      // "(zero based) index in the Symbol table to which the reference refers. Once you have loaded the COFF file into memory and know where each symbol is, you find the new updated address for the given symbol and update the reference accordingly."
		Type:        7,      // according to ldpe.c, this decodes to: IMAGE_REL_I386_DIR32NB
	})

	w.WriteLE(Symbol{
		Name:           STRING_RSRC,
		Value:          0,
		SectionNumber:  1,
		Type:           0, // FIXME: wtf?
		StorageClass:   3, // FIXME: is it ok? and uint8? and what does the value mean?
		AuxiliaryCount: 0, // FIXME: wtf?
	})

	w.WriteLE(StringsHeader{
		Length: uint32(binary.Size(StringsHeader{})), // empty strings table -- but we must still show size of the table's header...
	})

	if w.Err != nil {
		return fmt.Errorf("Error writing .rsrc Symbol Table & Strings: %s", w.Err)
	}

	return nil
}
<|endoftext|>