<commit_before>package bloomd

import (
	"bufio"
	"bytes"
	"crypto/sha1"
	"fmt"
	"github.com/pkg/errors"
	"golang.org/x/net/context"
	"io"
	"net"
	"strings"
	"time"
)

type Filter string

type Client interface {
	BulkSet(context.Context, Filter, ...string) ([]bool, error)
	Check(context.Context, Filter, string) (bool, error)
	Clear(context.Context, Filter) error
	Close(context.Context, Filter) error
	Create(context.Context, Filter) error
	Drop(context.Context, Filter) error
	Flush(context.Context) error
	List(context.Context) (map[string]string, error)
	MultiCheck(context.Context, Filter, ...string) ([]bool, error)
	Set(context.Context, Filter, string) (bool, error)
	Info(context.Context, Filter) (map[string]string, error)
	Ping() error
}

const (
	FLUSH           = "flush"
	LIST            = "list"
	CREATE          = "create %s"
	CREATE_CAPACITY = "%s capacity=%d"
	CREATE_PROB     = "%s prob=%f"
	CREATE_INMEM    = "%s in_memory=1"
	DROP            = "drop %s"
	CLOSE           = "close %s"
	CLEAR           = "clear %s"
	MULTI           = "m"
	BULK            = "b"
	SET             = "s %s %s"
	INFO            = "info %s"

	RESPONSE_DONE   = "Done"
	RESPONSE_EXISTS = "Exists"
	RESPONSE_YES    = "Yes"
	RESPONSE_NO     = "No"
	RESPONSE_START  = "START"
	RESPONSE_END    = "END"
)

type client struct {
	hostname string
	timeout  time.Duration
	addr     *net.TCPAddr
	attempts int
	hashKeys bool
}

func NewClient(hostname string, hashKeys bool, timeout time.Duration) (Client, error) {
	// TODO probably need to have a go routine call addr to resolve DNS periodically
	addr, err := net.ResolveTCPAddr("tcp", hostname)
	if err != nil {
		return nil, err
	}

	return &client{
		hostname: hostname,
		timeout:  timeout,
		addr:     addr,
		hashKeys: hashKeys,
	}, nil
}

// Add multi keys to the filter
func (t client) BulkSet(ctx context.Context, name Filter, keys ...string) ([]bool, error) {
	return t.sendMultiCommand(BULK, name, keys...)
}

// Check if key exists in filter
func (t client) Check(ctx context.Context, name Filter, key string) (bool, error) {
	return false, errors.New("not implemented")
}

// Clears the filter
func (t client) Clear(ctx context.Context, name Filter) error {
	return t.sendCommandDone(fmt.Sprintf(CLEAR, name))
}

// Closes the filter
func (t client) Close(ctx context.Context, name Filter) error {
	return t.sendCommandDone(fmt.Sprintf(CLOSE, name))
}

// Creates new fiter
func (t client) Create(ctx context.Context, name Filter) error {
	return t.CreateWithParams(ctx, name, 0, 0, false)
}

// Creates new fiter with additional params
func (t client) CreateWithParams(ctx context.Context, name Filter, capacity int, probability float64, inMemory bool) error {
	if probability > 0 && capacity < 1 {
		return errors.New("invalid capacity/probability")
	}

	cmd := fmt.Sprintf(CREATE, name)
	if capacity > 0 {
		cmd = fmt.Sprintf(CREATE_CAPACITY, cmd, capacity)
	}
	if probability > 0 {
		cmd = fmt.Sprintf(CREATE_PROB, cmd, probability)
	}
	if inMemory {
		cmd = fmt.Sprintf(CREATE_INMEM, cmd)
	}

	resp, err := t.sendCommand(cmd)
	if err != nil {
		return err
	}

	switch resp {
	case RESPONSE_DONE:
		return nil

	case RESPONSE_EXISTS:
		return nil

	default:
		return errors.Errorf("Invalid create resp '%s'", resp)
	}
}

// Permanently deletes filter
func (t client) Drop(ctx context.Context, name Filter) error {
	return t.sendCommandDone(fmt.Sprintf(DROP, name))
}

// Flush to disk
func (t client) Flush(ctx context.Context) error {
	return t.sendCommandDone(FLUSH)
}

func (t client) Info(ctx context.Context, name Filter) (map[string]string, error) {
	return t.sendBlockCommand(fmt.Sprintf(INFO, name))
}

// List filters
func (t client) List(ctx context.Context) (map[string]string, error) {
	return t.sendBlockCommand(LIST)
}

// Checks whether multiple keys exist in the filter
func (t client) MultiCheck(ctx context.Context, name Filter, keys ...string) ([]bool, error) {
	return t.sendMultiCommand(MULTI, name, keys...)
}

// Add new key to filter
func (t client) Set(ctx context.Context, name Filter, key string) (bool, error) {
	cmd := fmt.Sprintf(SET, name, t.hashKey(key))
	resp, err := t.sendCommand(cmd)
	if err != nil {
		return false, err
	}

	switch resp {
	case RESPONSE_YES:
		return true, nil
	case RESPONSE_NO:
		return true, nil
	default:
		return false, errors.Errorf("Invalid response '%s'", resp)
	}
}

func (t client) Ping() error {
	return errors.New("not implemented")
}

// Returns the key we should send to the server
func (t client) hashKey(key string) string {
	if t.hashKeys {
		h := sha1.New()
		s := h.Sum([]byte(key))
		return fmt.Sprintf("%x", s)
	}
	return key
}

func (t client) sendCommandDone(cmd string) error {
	resp, err := t.sendCommand(cmd)
	if err != nil {
		return errors.Wrapf(err, "bloomd: error with command '%s'", cmd)
	}

	if resp != RESPONSE_DONE {
		return errors.Errorf("bloomd: error with resp '%s' command '%s'", resp, cmd)
	}

	return nil
}

func (t client) sendMultiCommand(c string, name Filter, keys ...string) ([]bool, error) {
	cmd := t.multiCommand(c, name, keys...)

	resp, err := t.sendCommand(cmd)
	if err != nil {
		return nil, errors.Wrapf(err, "bloomd: error with multi command '%s'", cmd)
	}

	if !strings.HasPrefix(resp, RESPONSE_YES) && !strings.HasPrefix(resp, RESPONSE_NO) {
		return nil, errors.Errorf("bloomd: error with multi response '%s' command '%s'", resp, cmd)
	}

	results := make([]bool, 0, len(keys))
	responses := strings.Split(resp, " ")
	for _, r := range responses {
		results = append(results, r == RESPONSE_YES)
	}

	return results, nil
}

func (t client) multiCommand(cmd string, name Filter, keys ...string) string {
	buf := &bytes.Buffer{}
	buf.WriteString(cmd)
	buf.WriteRune(' ')
	buf.WriteString(string(name))
	for _, key := range keys {
		buf.WriteRune(' ')
		buf.WriteString(t.hashKey(key))
	}
	return buf.String()
}

func (t client) sendBlockCommand(cmd string) (map[string]string, error) {
	resp, err := t.sendCommand(LIST)
	if err != nil {
		return nil, err
	}

	if !strings.HasPrefix(resp, RESPONSE_START) {
		return nil, errors.Errorf("bloomd: invalid list start '%s' command '%s'", resp, cmd)
	}

	responses := make(map[string]string)

	split := strings.Split(resp, "\n")
	for _, line := range split {
		line := strings.TrimSpace(line)

		switch line {
		case RESPONSE_START:
		case RESPONSE_END:
		case "":
		default:
			split := strings.SplitN(line, " ", 2)
			responses[split[0]] = split[1]
		}
	}
	return responses, nil
}

func (t client) sendCommand(cmd string) (string, error) {

	conn, err := newConnection(t.addr, t.attempts)
	if err != nil {
		return "", err
	}
	defer conn.Close()

	if err := send(conn, cmd, t.attempts); err != nil {
		return "", err
	}

	line, err := recv(conn)
	if err != nil {
		return "", err
	}

	return line, nil
}

func newConnection(addr *net.TCPAddr, attempts int) (io.ReadWriteCloser, error) {
	attempted := 0

	var conn net.Conn
	var err error
	for {
		conn, err = net.DialTCP("tcp", nil, addr)
		if err != nil && attempted > attempts {
			return nil, errors.Wrap(err, "bloomd: unable to establish a connection")
		}
		attempted++
	}

	return conn, nil
}

func send(w io.Writer, cmd string, attempts int) error {
	attempted := 0

	for {
		_, err := w.Write([]byte(cmd + "\n"))
		if err != nil && attempted > attempts {
			return errors.Wrap(err, "bloomd: unable to write to connection")
		}
		attempted++
	}

	return nil
}

func recv(r io.Reader) (string, error) {
	//TODO read block
	line, err := bufio.NewReader(r).ReadString('\n')
	if err != nil && err != io.EOF {
		return line, errors.Wrap(err, "bloomd: unable to read connection")
	}
	return strings.TrimRight(line, "\r\n"), nil
}
<commit_msg>fixing read block<commit_after>package bloomd

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"github.com/pkg/errors"
	"golang.org/x/net/context"
	"io"
	"io/ioutil"
	"net"
	"strings"
	"time"
)

type Filter string

type Client interface {
	BulkSet(context.Context, Filter, ...string) ([]bool, error)
	Check(context.Context, Filter, string) (bool, error)
	Clear(context.Context, Filter) error
	Close(context.Context, Filter) error
	Create(context.Context, Filter) error
	Drop(context.Context, Filter) error
	Flush(context.Context) error
	List(context.Context) (map[string]string, error)
	MultiCheck(context.Context, Filter, ...string) ([]bool, error)
	Set(context.Context, Filter, string) (bool, error)
	Info(context.Context, Filter) (map[string]string, error)
	Ping() error
}

const (
	FLUSH           = "flush"
	LIST            = "list"
	CREATE          = "create %s"
	CREATE_CAPACITY = "%s capacity=%d"
	CREATE_PROB     = "%s prob=%f"
	CREATE_INMEM    = "%s in_memory=1"
	DROP            = "drop %s"
	CLOSE           = "close %s"
	CLEAR           = "clear %s"
	MULTI           = "m"
	BULK            = "b"
	SET             = "s %s %s"
	INFO            = "info %s"

	RESPONSE_DONE   = "Done"
	RESPONSE_EXISTS = "Exists"
	RESPONSE_YES    = "Yes"
	RESPONSE_NO     = "No"
	RESPONSE_START  = "START"
	RESPONSE_END    = "END"
)

type client struct {
	hostname string
	timeout  time.Duration
	addr     *net.TCPAddr
	attempts int
	hashKeys bool
}

func NewClient(hostname string, hashKeys bool, timeout time.Duration) (Client, error) {
	// TODO probably need to have a go routine call addr to resolve DNS periodically
	addr, err := net.ResolveTCPAddr("tcp", hostname)
	if err != nil {
		return nil, err
	}

	return &client{
		hostname: hostname,
		timeout:  timeout,
		addr:     addr,
		hashKeys: hashKeys,
	}, nil
}

// Add multi keys to the filter
func (t client) BulkSet(ctx context.Context, name Filter, keys ...string) ([]bool, error) {
	return t.sendMultiCommand(BULK, name, keys...)
}

// Check if key exists in filter
func (t client) Check(ctx context.Context, name Filter, key string) (bool, error) {
	return false, errors.New("not implemented")
}

// Clears the filter
func (t client) Clear(ctx context.Context, name Filter) error {
	return t.sendCommandDone(fmt.Sprintf(CLEAR, name))
}

// Closes the filter
func (t client) Close(ctx context.Context, name Filter) error {
	return t.sendCommandDone(fmt.Sprintf(CLOSE, name))
}

// Creates new fiter
func (t client) Create(ctx context.Context, name Filter) error {
	return t.CreateWithParams(ctx, name, 0, 0, false)
}

// Creates new fiter with additional params
func (t client) CreateWithParams(ctx context.Context, name Filter, capacity int, probability float64, inMemory bool) error {
	if probability > 0 && capacity < 1 {
		return errors.New("invalid capacity/probability")
	}

	cmd := fmt.Sprintf(CREATE, name)
	if capacity > 0 {
		cmd = fmt.Sprintf(CREATE_CAPACITY, cmd, capacity)
	}
	if probability > 0 {
		cmd = fmt.Sprintf(CREATE_PROB, cmd, probability)
	}
	if inMemory {
		cmd = fmt.Sprintf(CREATE_INMEM, cmd)
	}

	resp, err := t.sendCommand(cmd)
	if err != nil {
		return err
	}

	switch resp {
	case RESPONSE_DONE:
		return nil

	case RESPONSE_EXISTS:
		return nil

	default:
		return errors.Errorf("Invalid create resp '%s'", resp)
	}
}

// Permanently deletes filter
func (t client) Drop(ctx context.Context, name Filter) error {
	return t.sendCommandDone(fmt.Sprintf(DROP, name))
}

// Flush to disk
func (t client) Flush(ctx context.Context) error {
	return t.sendCommandDone(FLUSH)
}

func (t client) Info(ctx context.Context, name Filter) (map[string]string, error) {
	return t.sendBlockCommand(fmt.Sprintf(INFO, name))
}

// List filters
func (t client) List(ctx context.Context) (map[string]string, error) {
	return t.sendBlockCommand(LIST)
}

// Checks whether multiple keys exist in the filter
func (t client) MultiCheck(ctx context.Context, name Filter, keys ...string) ([]bool, error) {
	return t.sendMultiCommand(MULTI, name, keys...)
}

// Add new key to filter
func (t client) Set(ctx context.Context, name Filter, key string) (bool, error) {
	cmd := fmt.Sprintf(SET, name, t.hashKey(key))
	resp, err := t.sendCommand(cmd)
	if err != nil {
		return false, err
	}

	switch resp {
	case RESPONSE_YES:
		return true, nil
	case RESPONSE_NO:
		return true, nil
	default:
		return false, errors.Errorf("Invalid response '%s'", resp)
	}
}

func (t client) Ping() error {
	return errors.New("not implemented")
}

// Returns the key we should send to the server
func (t client) hashKey(key string) string {
	if t.hashKeys {
		h := sha1.New()
		s := h.Sum([]byte(key))
		return fmt.Sprintf("%x", s)
	}
	return key
}

func (t client) sendCommandDone(cmd string) error {
	resp, err := t.sendCommand(cmd)
	if err != nil {
		return errors.Wrapf(err, "bloomd: error with command '%s'", cmd)
	}

	if resp != RESPONSE_DONE {
		return errors.Errorf("bloomd: error with resp '%s' command '%s'", resp, cmd)
	}

	return nil
}

func (t client) sendMultiCommand(c string, name Filter, keys ...string) ([]bool, error) {
	cmd := t.multiCommand(c, name, keys...)

	resp, err := t.sendCommand(cmd)
	if err != nil {
		return nil, errors.Wrapf(err, "bloomd: error with multi command '%s'", cmd)
	}

	if !strings.HasPrefix(resp, RESPONSE_YES) && !strings.HasPrefix(resp, RESPONSE_NO) {
		return nil, errors.Errorf("bloomd: error with multi response '%s' command '%s'", resp, cmd)
	}

	results := make([]bool, 0, len(keys))
	responses := strings.Split(resp, " ")
	for _, r := range responses {
		results = append(results, r == RESPONSE_YES)
	}

	return results, nil
}

func (t client) multiCommand(cmd string, name Filter, keys ...string) string {
	buf := &bytes.Buffer{}
	buf.WriteString(cmd)
	buf.WriteRune(' ')
	buf.WriteString(string(name))
	for _, key := range keys {
		buf.WriteRune(' ')
		buf.WriteString(t.hashKey(key))
	}
	return buf.String()
}

func (t client) sendBlockCommand(cmd string) (map[string]string, error) {
	resp, err := t.sendCommand(LIST)
	if err != nil {
		return nil, err
	}

	if !strings.HasPrefix(resp, RESPONSE_START) {
		return nil, errors.Errorf("bloomd: invalid list start '%s' command '%s'", resp, cmd)
	}

	responses := make(map[string]string)

	split := strings.Split(resp, "\n")
	for _, line := range split {
		line := strings.TrimSpace(line)

		switch line {
		case RESPONSE_START:
		case RESPONSE_END:
		case "":
		default:
			split := strings.SplitN(line, " ", 2)
			responses[split[0]] = split[1]
		}
	}
	return responses, nil
}

func (t client) sendCommand(cmd string) (string, error) {

	conn, err := newConnection(t.addr, t.attempts)
	if err != nil {
		return "", err
	}
	defer conn.Close()

	if err := send(conn, cmd, t.attempts); err != nil {
		return "", err
	}

	line, err := recv(conn)
	if err != nil {
		return "", err
	}

	return line, nil
}

func newConnection(addr *net.TCPAddr, attempts int) (io.ReadWriteCloser, error) {
	attempted := 0

	var conn net.Conn
	var err error
	for {
		conn, err = net.DialTCP("tcp", nil, addr)
		if err != nil && attempted > attempts {
			return nil, errors.Wrap(err, "bloomd: unable to establish a connection")
		}
		attempted++
	}

	return conn, nil
}

func send(w io.Writer, cmd string, attempts int) error {
	attempted := 0

	for {
		_, err := w.Write([]byte(cmd + "\n"))
		if err != nil && attempted > attempts {
			return errors.Wrap(err, "bloomd: unable to write to connection")
		}
		attempted++
	}

	return nil
}

func recv(r io.Reader) (string, error) {
	buf, err := ioutil.ReadAll(r)
	if err != nil {
		return line, errors.Wrap(err, "bloomd: unable to read connection")
	}
	return strings.TrimRight(string(buf), "\r\n"), nil
}
<|endoftext|>


<commit_before>package weed_server

import (
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/chrislusf/raft"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/topology"
	"google.golang.org/grpc/peer"
)

func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error {
	var dn *topology.DataNode
	t := ms.Topo

	defer func() {
		if dn != nil {

			glog.V(0).Infof("unregister disconnected volume server %s:%d", dn.Ip, dn.Port)
			t.UnRegisterDataNode(dn)

			message := &master_pb.VolumeLocation{
				Url:       dn.Url(),
				PublicUrl: dn.PublicUrl,
			}
			for _, v := range dn.GetVolumes() {
				message.DeletedVids = append(message.DeletedVids, uint32(v.Id))
			}

			if len(message.DeletedVids) > 0 {
				ms.clientChansLock.RLock()
				for _, ch := range ms.clientChans {
					ch <- message
				}
				ms.clientChansLock.RUnlock()
			}

		}
	}()

	for {
		heartbeat, err := stream.Recv()
		if err != nil {
			return err
		}

		if dn == nil {
			t.Sequence.SetMax(heartbeat.MaxFileKey)
			if heartbeat.Ip == "" {
				if pr, ok := peer.FromContext(stream.Context()); ok {
					if pr.Addr != net.Addr(nil) {
						heartbeat.Ip = pr.Addr.String()[0:strings.LastIndex(pr.Addr.String(), ":")]
						glog.V(0).Infof("remote IP address is detected as %v", heartbeat.Ip)
					}
				}
			}
			dcName, rackName := t.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack)
			dc := t.GetOrCreateDataCenter(dcName)
			rack := dc.GetOrCreateRack(rackName)
			dn = rack.GetOrCreateDataNode(heartbeat.Ip,
				int(heartbeat.Port), heartbeat.PublicUrl,
				int64(heartbeat.MaxVolumeCount))
			glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort())
			if err := stream.Send(&master_pb.HeartbeatResponse{
				VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024,
			}); err != nil {
				return err
			}
		}

		glog.V(4).Infof("master received heartbeat %s", heartbeat.String())
		message := &master_pb.VolumeLocation{
			Url:       dn.Url(),
			PublicUrl: dn.PublicUrl,
		}
		if len(heartbeat.NewVolumes) > 0 || len(heartbeat.DeletedVolumes) > 0 {
			// process delta volume ids if exists for fast volume id updates
			for _, volInfo := range heartbeat.NewVolumes {
				message.NewVids = append(message.NewVids, volInfo.Id)
			}
			for _, volInfo := range heartbeat.DeletedVolumes {
				message.DeletedVids = append(message.DeletedVids, volInfo.Id)
			}
			// update master internal volume layouts
			t.IncrementalSyncDataNodeRegistration(heartbeat.NewVolumes, heartbeat.DeletedVolumes, dn)
		}

		if len(heartbeat.Volumes) > 0 {
			// process heartbeat.Volumes
			newVolumes, deletedVolumes := t.SyncDataNodeRegistration(heartbeat.Volumes, dn)

			for _, v := range newVolumes {
				glog.V(0).Infof("master see new volume %d from %s", uint32(v.Id), dn.Url())
				message.NewVids = append(message.NewVids, uint32(v.Id))
			}
			for _, v := range deletedVolumes {
				glog.V(0).Infof("master see deleted volume %d from %s", uint32(v.Id), dn.Url())
				message.DeletedVids = append(message.DeletedVids, uint32(v.Id))
			}
		}

		if len(heartbeat.NewEcShards) > 0 || len(heartbeat.DeletedEcShards) > 0 {
			// TODO send out the delta

			// update master internal volume layouts
			t.IncrementalSyncDataNodeEcShards(heartbeat.NewEcShards, heartbeat.DeletedEcShards, dn)

			for _, s := range heartbeat.NewEcShards {
				message.NewVids = append(message.NewVids, s.Id)
			}
			for _, s := range heartbeat.DeletedEcShards {
				if dn.HasVolumesById(needle.VolumeId(s.Id)) {
					continue
				}
				message.DeletedVids = append(message.DeletedVids, s.Id)
			}

		}

		if len(heartbeat.EcShards) > 0 {
			glog.V(0).Infof("master recieved ec shards from %s: %+v", dn.Url(), heartbeat.EcShards)
			newShards, deletedShards := t.SyncDataNodeEcShards(heartbeat.EcShards, dn)

			// broadcast the ec vid changes to master clients
			for _, s := range newShards {
				message.NewVids = append(message.NewVids, uint32(s.VolumeId))
			}
			for _, s := range deletedShards {
				if dn.HasVolumesById(s.VolumeId) {
					continue
				}
				message.DeletedVids = append(message.DeletedVids, uint32(s.VolumeId))
			}

		}

		if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 {
			ms.clientChansLock.RLock()
			for host, ch := range ms.clientChans {
				glog.V(0).Infof("master send to %s: %s", host, message.String())
				ch <- message
			}
			ms.clientChansLock.RUnlock()
		}

		// tell the volume servers about the leader
		newLeader, err := t.Leader()
		if err != nil {
			return err
		}
		if err := stream.Send(&master_pb.HeartbeatResponse{
			Leader: newLeader,
		}); err != nil {
			return err
		}
	}
}

// KeepConnected keep a stream gRPC call to the master. Used by clients to know the master is up.
// And clients gets the up-to-date list of volume locations
func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error {

	req, err := stream.Recv()
	if err != nil {
		return err
	}

	if !ms.Topo.IsLeader() {
		return raft.NotLeaderError
	}

	// remember client address
	ctx := stream.Context()
	// fmt.Printf("FromContext %+v\n", ctx)
	pr, ok := peer.FromContext(ctx)
	if !ok {
		glog.Error("failed to get peer from ctx")
		return fmt.Errorf("failed to get peer from ctx")
	}
	if pr.Addr == net.Addr(nil) {
		glog.Error("failed to get peer address")
		return fmt.Errorf("failed to get peer address")
	}

	clientName := req.Name + pr.Addr.String()
	glog.V(0).Infof("+ client %v", clientName)

	messageChan := make(chan *master_pb.VolumeLocation)
	stopChan := make(chan bool)

	ms.clientChansLock.Lock()
	ms.clientChans[clientName] = messageChan
	ms.clientChansLock.Unlock()

	defer func() {
		glog.V(0).Infof("- client %v", clientName)
		ms.clientChansLock.Lock()
		delete(ms.clientChans, clientName)
		ms.clientChansLock.Unlock()
	}()

	for _, message := range ms.Topo.ToVolumeLocations() {
		if err := stream.Send(message); err != nil {
			return err
		}
	}

	go func() {
		for {
			_, err := stream.Recv()
			if err != nil {
				glog.V(2).Infof("- client %v: %v", clientName, err)
				stopChan <- true
				break
			}
		}
	}()

	ticker := time.NewTicker(5 * time.Second)
	for {
		select {
		case message := <-messageChan:
			if err := stream.Send(message); err != nil {
				glog.V(0).Infof("=> client %v: %+v", clientName, message)
				return err
			}
		case <-ticker.C:
			if !ms.Topo.IsLeader() {
				return raft.NotLeaderError
			}
		case <-stopChan:
			return nil
		}
	}

	return nil
}
<commit_msg>adjust ec shard status on disconnect<commit_after>package weed_server

import (
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/chrislusf/raft"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/topology"
	"google.golang.org/grpc/peer"
)

func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error {
	var dn *topology.DataNode
	t := ms.Topo

	defer func() {
		if dn != nil {

			glog.V(0).Infof("unregister disconnected volume server %s:%d", dn.Ip, dn.Port)
			t.UnRegisterDataNode(dn)

			message := &master_pb.VolumeLocation{
				Url:       dn.Url(),
				PublicUrl: dn.PublicUrl,
			}
			for _, v := range dn.GetVolumes() {
				message.DeletedVids = append(message.DeletedVids, uint32(v.Id))
			}
			for _, s := range dn.GetEcShards() {
				message.DeletedVids = append(message.DeletedVids, uint32(s.VolumeId))
			}

			if len(message.DeletedVids) > 0 {
				ms.clientChansLock.RLock()
				for _, ch := range ms.clientChans {
					ch <- message
				}
				ms.clientChansLock.RUnlock()
			}

		}
	}()

	for {
		heartbeat, err := stream.Recv()
		if err != nil {
			return err
		}

		if dn == nil {
			t.Sequence.SetMax(heartbeat.MaxFileKey)
			if heartbeat.Ip == "" {
				if pr, ok := peer.FromContext(stream.Context()); ok {
					if pr.Addr != net.Addr(nil) {
						heartbeat.Ip = pr.Addr.String()[0:strings.LastIndex(pr.Addr.String(), ":")]
						glog.V(0).Infof("remote IP address is detected as %v", heartbeat.Ip)
					}
				}
			}
			dcName, rackName := t.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack)
			dc := t.GetOrCreateDataCenter(dcName)
			rack := dc.GetOrCreateRack(rackName)
			dn = rack.GetOrCreateDataNode(heartbeat.Ip,
				int(heartbeat.Port), heartbeat.PublicUrl,
				int64(heartbeat.MaxVolumeCount))
			glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort())
			if err := stream.Send(&master_pb.HeartbeatResponse{
				VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024,
			}); err != nil {
				return err
			}
		}

		glog.V(4).Infof("master received heartbeat %s", heartbeat.String())
		message := &master_pb.VolumeLocation{
			Url:       dn.Url(),
			PublicUrl: dn.PublicUrl,
		}
		if len(heartbeat.NewVolumes) > 0 || len(heartbeat.DeletedVolumes) > 0 {
			// process delta volume ids if exists for fast volume id updates
			for _, volInfo := range heartbeat.NewVolumes {
				message.NewVids = append(message.NewVids, volInfo.Id)
			}
			for _, volInfo := range heartbeat.DeletedVolumes {
				message.DeletedVids = append(message.DeletedVids, volInfo.Id)
			}
			// update master internal volume layouts
			t.IncrementalSyncDataNodeRegistration(heartbeat.NewVolumes, heartbeat.DeletedVolumes, dn)
		}

		if len(heartbeat.Volumes) > 0 {
			// process heartbeat.Volumes
			newVolumes, deletedVolumes := t.SyncDataNodeRegistration(heartbeat.Volumes, dn)

			for _, v := range newVolumes {
				glog.V(0).Infof("master see new volume %d from %s", uint32(v.Id), dn.Url())
				message.NewVids = append(message.NewVids, uint32(v.Id))
			}
			for _, v := range deletedVolumes {
				glog.V(0).Infof("master see deleted volume %d from %s", uint32(v.Id), dn.Url())
				message.DeletedVids = append(message.DeletedVids, uint32(v.Id))
			}
		}

		if len(heartbeat.NewEcShards) > 0 || len(heartbeat.DeletedEcShards) > 0 {
			// TODO send out the delta

			// update master internal volume layouts
			t.IncrementalSyncDataNodeEcShards(heartbeat.NewEcShards, heartbeat.DeletedEcShards, dn)

			for _, s := range heartbeat.NewEcShards {
				message.NewVids = append(message.NewVids, s.Id)
			}
			for _, s := range heartbeat.DeletedEcShards {
				if dn.HasVolumesById(needle.VolumeId(s.Id)) {
					continue
				}
				message.DeletedVids = append(message.DeletedVids, s.Id)
			}

		}

		if len(heartbeat.EcShards) > 0 {
			glog.V(0).Infof("master recieved ec shards from %s: %+v", dn.Url(), heartbeat.EcShards)
			newShards, deletedShards := t.SyncDataNodeEcShards(heartbeat.EcShards, dn)

			// broadcast the ec vid changes to master clients
			for _, s := range newShards {
				message.NewVids = append(message.NewVids, uint32(s.VolumeId))
			}
			for _, s := range deletedShards {
				if dn.HasVolumesById(s.VolumeId) {
					continue
				}
				message.DeletedVids = append(message.DeletedVids, uint32(s.VolumeId))
			}

		}

		if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 {
			ms.clientChansLock.RLock()
			for host, ch := range ms.clientChans {
				glog.V(0).Infof("master send to %s: %s", host, message.String())
				ch <- message
			}
			ms.clientChansLock.RUnlock()
		}

		// tell the volume servers about the leader
		newLeader, err := t.Leader()
		if err != nil {
			return err
		}
		if err := stream.Send(&master_pb.HeartbeatResponse{
			Leader: newLeader,
		}); err != nil {
			return err
		}
	}
}

// KeepConnected keep a stream gRPC call to the master. Used by clients to know the master is up.
// And clients gets the up-to-date list of volume locations
func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error {

	req, err := stream.Recv()
	if err != nil {
		return err
	}

	if !ms.Topo.IsLeader() {
		return raft.NotLeaderError
	}

	// remember client address
	ctx := stream.Context()
	// fmt.Printf("FromContext %+v\n", ctx)
	pr, ok := peer.FromContext(ctx)
	if !ok {
		glog.Error("failed to get peer from ctx")
		return fmt.Errorf("failed to get peer from ctx")
	}
	if pr.Addr == net.Addr(nil) {
		glog.Error("failed to get peer address")
		return fmt.Errorf("failed to get peer address")
	}

	clientName := req.Name + pr.Addr.String()
	glog.V(0).Infof("+ client %v", clientName)

	messageChan := make(chan *master_pb.VolumeLocation)
	stopChan := make(chan bool)

	ms.clientChansLock.Lock()
	ms.clientChans[clientName] = messageChan
	ms.clientChansLock.Unlock()

	defer func() {
		glog.V(0).Infof("- client %v", clientName)
		ms.clientChansLock.Lock()
		delete(ms.clientChans, clientName)
		ms.clientChansLock.Unlock()
	}()

	for _, message := range ms.Topo.ToVolumeLocations() {
		if err := stream.Send(message); err != nil {
			return err
		}
	}

	go func() {
		for {
			_, err := stream.Recv()
			if err != nil {
				glog.V(2).Infof("- client %v: %v", clientName, err)
				stopChan <- true
				break
			}
		}
	}()

	ticker := time.NewTicker(5 * time.Second)
	for {
		select {
		case message := <-messageChan:
			if err := stream.Send(message); err != nil {
				glog.V(0).Infof("=> client %v: %+v", clientName, message)
				return err
			}
		case <-ticker.C:
			if !ms.Topo.IsLeader() {
				return raft.NotLeaderError
			}
		case <-stopChan:
			return nil
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package weed_server

import (
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/chrislusf/raft"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/topology"
	"google.golang.org/grpc/peer"
)

func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error {
	var dn *topology.DataNode
	t := ms.Topo

	defer func() {
		if dn != nil {

			glog.V(0).Infof("unregister disconnected volume server %s:%d", dn.Ip, dn.Port)
			t.UnRegisterDataNode(dn)

			message := &master_pb.VolumeLocation{
				Url:       dn.Url(),
				PublicUrl: dn.PublicUrl,
			}
			for _, v := range dn.GetVolumes() {
				message.DeletedVids = append(message.DeletedVids, uint32(v.Id))
			}

			if len(message.DeletedVids) > 0 {
				ms.clientChansLock.RLock()
				for _, ch := range ms.clientChans {
					ch <- message
				}
				ms.clientChansLock.RUnlock()
			}

		}
	}()

	for {
		heartbeat, err := stream.Recv()
		if err != nil {
			return err
		}

		if dn == nil {
			t.Sequence.SetMax(heartbeat.MaxFileKey)
			if heartbeat.Ip == "" {
				if pr, ok := peer.FromContext(stream.Context()); ok {
					if pr.Addr != net.Addr(nil) {
						heartbeat.Ip = pr.Addr.String()[0:strings.LastIndex(pr.Addr.String(), ":")]
						glog.V(0).Infof("remote IP address is detected as %v", heartbeat.Ip)
					}
				}
			}
			dcName, rackName := t.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack)
			dc := t.GetOrCreateDataCenter(dcName)
			rack := dc.GetOrCreateRack(rackName)
			dn = rack.GetOrCreateDataNode(heartbeat.Ip,
				int(heartbeat.Port), heartbeat.PublicUrl,
				int64(heartbeat.MaxVolumeCount))
			glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort())
			if err := stream.Send(&master_pb.HeartbeatResponse{
				VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024,
			}); err != nil {
				return err
			}
		}

		glog.V(4).Infof("master received heartbeat %s", heartbeat.String())
		message := &master_pb.VolumeLocation{
			Url:       dn.Url(),
			PublicUrl: dn.PublicUrl,
		}
		if len(heartbeat.NewVolumes) > 0 || len(heartbeat.DeletedVolumes) > 0 {
			// process delta volume ids if exists for fast volume id updates
			for _, volInfo := range heartbeat.NewVolumes {
				message.NewVids = append(message.NewVids, volInfo.Id)
			}
			for _, volInfo := range heartbeat.DeletedVolumes {
				message.DeletedVids = append(message.DeletedVids, volInfo.Id)
			}
			// update master internal volume layouts
			t.IncrementalSyncDataNodeRegistration(heartbeat.NewVolumes, heartbeat.DeletedVolumes, dn)
		}

		if len(heartbeat.Volumes) > 0 {
			// process heartbeat.Volumes
			newVolumes, deletedVolumes := t.SyncDataNodeRegistration(heartbeat.Volumes, dn)

			for _, v := range newVolumes {
				glog.V(0).Infof("master see new volume %d from %s", uint32(v.Id), dn.Url())
				message.NewVids = append(message.NewVids, uint32(v.Id))
			}
			for _, v := range deletedVolumes {
				glog.V(0).Infof("master see deleted volume %d from %s", uint32(v.Id), dn.Url())
				message.DeletedVids = append(message.DeletedVids, uint32(v.Id))
			}
		}

		if len(heartbeat.EcShards) > 0 {
			glog.V(0).Infof("master recieved ec shards from %s: %+v", dn.Url(), heartbeat.EcShards)
		}

		if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 {
			ms.clientChansLock.RLock()
			for host, ch := range ms.clientChans {
				glog.V(0).Infof("master send to %s: %s", host, message.String())
				ch <- message
			}
			ms.clientChansLock.RUnlock()
		}

		// tell the volume servers about the leader
		newLeader, err := t.Leader()
		if err != nil {
			return err
		}
		if err := stream.Send(&master_pb.HeartbeatResponse{
			Leader: newLeader,
		}); err != nil {
			return err
		}
	}
}

// KeepConnected keep a stream gRPC call to the master. Used by clients to know the master is up.
// And clients gets the up-to-date list of volume locations
func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error {

	req, err := stream.Recv()
	if err != nil {
		return err
	}

	if !ms.Topo.IsLeader() {
		return raft.NotLeaderError
	}

	// remember client address
	ctx := stream.Context()
	// fmt.Printf("FromContext %+v\n", ctx)
	pr, ok := peer.FromContext(ctx)
	if !ok {
		glog.Error("failed to get peer from ctx")
		return fmt.Errorf("failed to get peer from ctx")
	}
	if pr.Addr == net.Addr(nil) {
		glog.Error("failed to get peer address")
		return fmt.Errorf("failed to get peer address")
	}

	clientName := req.Name + pr.Addr.String()
	glog.V(0).Infof("+ client %v", clientName)

	messageChan := make(chan *master_pb.VolumeLocation)
	stopChan := make(chan bool)

	ms.clientChansLock.Lock()
	ms.clientChans[clientName] = messageChan
	ms.clientChansLock.Unlock()

	defer func() {
		glog.V(0).Infof("- client %v", clientName)
		ms.clientChansLock.Lock()
		delete(ms.clientChans, clientName)
		ms.clientChansLock.Unlock()
	}()

	for _, message := range ms.Topo.ToVolumeLocations() {
		if err := stream.Send(message); err != nil {
			return err
		}
	}

	go func() {
		for {
			_, err := stream.Recv()
			if err != nil {
				glog.V(2).Infof("- client %v: %v", clientName, err)
				stopChan <- true
				break
			}
		}
	}()

	ticker := time.NewTicker(5 * time.Second)
	for {
		select {
		case message := <-messageChan:
			if err := stream.Send(message); err != nil {
				glog.V(0).Infof("=> client %v: %+v", clientName, message)
				return err
			}
		case <-ticker.C:
			if !ms.Topo.IsLeader() {
				return raft.NotLeaderError
			}
		case <-stopChan:
			return nil
		}
	}

	return nil
}
<commit_msg>push the ec shard info to master<commit_after>package weed_server

import (
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/chrislusf/raft"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/topology"
	"google.golang.org/grpc/peer"
)

func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error {
	var dn *topology.DataNode
	t := ms.Topo

	defer func() {
		if dn != nil {

			glog.V(0).Infof("unregister disconnected volume server %s:%d", dn.Ip, dn.Port)
			t.UnRegisterDataNode(dn)

			message := &master_pb.VolumeLocation{
				Url:       dn.Url(),
				PublicUrl: dn.PublicUrl,
			}
			for _, v := range dn.GetVolumes() {
				message.DeletedVids = append(message.DeletedVids, uint32(v.Id))
			}

			if len(message.DeletedVids) > 0 {
				ms.clientChansLock.RLock()
				for _, ch := range ms.clientChans {
					ch <- message
				}
				ms.clientChansLock.RUnlock()
			}

		}
	}()

	for {
		heartbeat, err := stream.Recv()
		if err != nil {
			return err
		}

		if dn == nil {
			t.Sequence.SetMax(heartbeat.MaxFileKey)
			if heartbeat.Ip == "" {
				if pr, ok := peer.FromContext(stream.Context()); ok {
					if pr.Addr != net.Addr(nil) {
						heartbeat.Ip = pr.Addr.String()[0:strings.LastIndex(pr.Addr.String(), ":")]
						glog.V(0).Infof("remote IP address is detected as %v", heartbeat.Ip)
					}
				}
			}
			dcName, rackName := t.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack)
			dc := t.GetOrCreateDataCenter(dcName)
			rack := dc.GetOrCreateRack(rackName)
			dn = rack.GetOrCreateDataNode(heartbeat.Ip,
				int(heartbeat.Port), heartbeat.PublicUrl,
				int64(heartbeat.MaxVolumeCount))
			glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort())
			if err := stream.Send(&master_pb.HeartbeatResponse{
				VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024,
			}); err != nil {
				return err
			}
		}

		glog.V(4).Infof("master received heartbeat %s", heartbeat.String())
		message := &master_pb.VolumeLocation{
			Url:       dn.Url(),
			PublicUrl: dn.PublicUrl,
		}
		if len(heartbeat.NewVolumes) > 0 || len(heartbeat.DeletedVolumes) > 0 {
			// process delta volume ids if exists for fast volume id updates
			for _, volInfo := range heartbeat.NewVolumes {
				message.NewVids = append(message.NewVids, volInfo.Id)
			}
			for _, volInfo := range heartbeat.DeletedVolumes {
				message.DeletedVids = append(message.DeletedVids, volInfo.Id)
			}
			// update master internal volume layouts
			t.IncrementalSyncDataNodeRegistration(heartbeat.NewVolumes, heartbeat.DeletedVolumes, dn)
		}

		if len(heartbeat.Volumes) > 0 {
			// process heartbeat.Volumes
			newVolumes, deletedVolumes := t.SyncDataNodeRegistration(heartbeat.Volumes, dn)

			for _, v := range newVolumes {
				glog.V(0).Infof("master see new volume %d from %s", uint32(v.Id), dn.Url())
				message.NewVids = append(message.NewVids, uint32(v.Id))
			}
			for _, v := range deletedVolumes {
				glog.V(0).Infof("master see deleted volume %d from %s", uint32(v.Id), dn.Url())
				message.DeletedVids = append(message.DeletedVids, uint32(v.Id))
			}
		}

		if len(heartbeat.EcShards) > 0 {
			glog.V(0).Infof("master recieved ec shards from %s: %+v", dn.Url(), heartbeat.EcShards)
			newShards, deletedShards := t.SyncDataNodeEcShards(heartbeat.EcShards, dn)

			//TODO broadcast the ec vid
			if len(newShards)>0{

			}
			if len(deletedShards)>0{

			}

		}

		if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 {
			ms.clientChansLock.RLock()
			for host, ch := range ms.clientChans {
				glog.V(0).Infof("master send to %s: %s", host, message.String())
				ch <- message
			}
			ms.clientChansLock.RUnlock()
		}

		// tell the volume servers about the leader
		newLeader, err := t.Leader()
		if err != nil {
			return err
		}
		if err := stream.Send(&master_pb.HeartbeatResponse{
			Leader: newLeader,
		}); err != nil {
			return err
		}
	}
}

// KeepConnected keep a stream gRPC call to the master. Used by clients to know the master is up.
// And clients gets the up-to-date list of volume locations
func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error {

	req, err := stream.Recv()
	if err != nil {
		return err
	}

	if !ms.Topo.IsLeader() {
		return raft.NotLeaderError
	}

	// remember client address
	ctx := stream.Context()
	// fmt.Printf("FromContext %+v\n", ctx)
	pr, ok := peer.FromContext(ctx)
	if !ok {
		glog.Error("failed to get peer from ctx")
		return fmt.Errorf("failed to get peer from ctx")
	}
	if pr.Addr == net.Addr(nil) {
		glog.Error("failed to get peer address")
		return fmt.Errorf("failed to get peer address")
	}

	clientName := req.Name + pr.Addr.String()
	glog.V(0).Infof("+ client %v", clientName)

	messageChan := make(chan *master_pb.VolumeLocation)
	stopChan := make(chan bool)

	ms.clientChansLock.Lock()
	ms.clientChans[clientName] = messageChan
	ms.clientChansLock.Unlock()

	defer func() {
		glog.V(0).Infof("- client %v", clientName)
		ms.clientChansLock.Lock()
		delete(ms.clientChans, clientName)
		ms.clientChansLock.Unlock()
	}()

	for _, message := range ms.Topo.ToVolumeLocations() {
		if err := stream.Send(message); err != nil {
			return err
		}
	}

	go func() {
		for {
			_, err := stream.Recv()
			if err != nil {
				glog.V(2).Infof("- client %v: %v", clientName, err)
				stopChan <- true
				break
			}
		}
	}()

	ticker := time.NewTicker(5 * time.Second)
	for {
		select {
		case message := <-messageChan:
			if err := stream.Send(message); err != nil {
				glog.V(0).Infof("=> client %v: %+v", clientName, message)
				return err
			}
		case <-ticker.C:
			if !ms.Topo.IsLeader() {
				return raft.NotLeaderError
			}
		case <-stopChan:
			return nil
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Peter H. Froehlich. All rights reserved.
// Use of this source code is governed by the MIT license,
// see the LICENSE.md file.

// Dupes finds duplicate files in the given paths.
//
// Run dupes as follows:
//
//	dupes path1 path2 ...
//
// Dupes will process each path. Directories will be walked
// recursively, regular files will be checked against all
// others. Dupes will print clusters of paths, separated
// by an empty line, for each duplicate it finds. Dupes will
// also print statistics about duplicates at the end.
//
// The -p option uses a "paranoid" byte-by-byte file comparison
// instead of SHA1 digests to identify duplicates.
//
// The -s option sets the minimum file size you care about;
// if defaults to 1 so empty files are ignored.
//
// The -g option sets a globbing pattern for the file names
// you care about; it defaults to * which matches all file
// names.
package main

import (
	"bytes"
	"crypto/sha1"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime/pprof"
	"sort"
)

const (
	globDefault = "*"
)

var (
	paranoid    = flag.Bool("p", false, "paranoid byte-by-byte file comparison")
	minimumSize = flag.Int64("s", 1, "minimum size (in bytes) of files to consider")
	globbing    = flag.String("g", globDefault, "glob expression for files to consider")
	cpuprofile  = flag.String("cpuprofile", "", "write cpu profile to file (development only)")
)

var (
	hashes = make(map[string]string)   // maps from digests to paths
	sizes  = make(map[int64]string)    // maps from sizes to paths
	final  = make(map[string][]string) // maps from paths to duplicate paths (collates all dupes)

	files  counter  // number of files examined
	dupes  counter  // number of duplicate files
	wasted bytesize // space (in bytes) occupied by duplicates
)

// fileContentsMatch does a byte-by-byte comparison of the files with the
// given paths
func fileContentsMatch(pa, pb string) (bool, error) {
	a, err := os.Open(pa)
	if err != nil {
		return false, err
	}
	defer a.Close()
	b, err := os.Open(pb)
	if err != nil {
		return false, err
	}
	defer b.Close()

	return fileContentsHelper(a, b)
}

func fileContentsHelper(a, b io.Reader) (bool, error) {
	bufferSize := os.Getpagesize()

	ba := make([]byte, bufferSize)
	bb := make([]byte, bufferSize)

	for {
		la, erra := a.Read(ba)
		lb, errb := b.Read(bb)

		// specification of Read() says to check returned size
		// before considering errors; who are we to disagree?
		if la > 0 || lb > 0 {
			// it's okay to use Equal here because whatever may
			// be left behind in the buffers after a short read
			// had to be Equal in the prior iteration; note that
			// we don't have to check la == lb either because if
			// they were not, Equal should fail
			if !bytes.Equal(ba, bb) {
				return false, nil
			}
		}

		// specification of Read() says that sooner or later we'll
		// see io.EOF regardless of returned size; only if both
		// files end in the same iteration (and made it past Equal
		// above) do we have a duplicate
		switch {
		case erra == io.EOF && errb == io.EOF:
			return true, nil
		case erra != nil:
			return false, erra
		case errb != nil:
			return false, errb
		}
	}
}

// checksum calculates a hash digest for the file with the given path
func checksum(path string) (string, error) {
	file, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hasher := sha1.New()
	_, err = io.Copy(hasher, file)
	sum := fmt.Sprintf("%x", hasher.Sum(nil))

	return sum, err
}

// check is called for each path we walk. It only examines regular, non-empty
// files. It first rules out duplicates by file size; for files that remain
// it calculates a checksum; if it has seen the same checksum before, it
// signals a duplicate; otherwise it remembers the checksum and the path of
// the original file before moving on; in paranoid mode it follows up with a
// byte-by-byte file comparison.
func check(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}

	size := info.Size()

	if !info.Mode().IsRegular() || size < *minimumSize {
		return nil
	}

	if *globbing != globDefault {
		matched, err := filepath.Match(*globbing, info.Name())
		if err != nil {
			return err
		}
		if !matched {
			return nil
		}
	}

	files++

	var dupe string
	var ok bool
	if dupe, ok = sizes[size]; !ok {
		sizes[size] = path
		return nil
	}

	// backpatch new file into hashes
	sum, err := checksum(dupe)
	if err != nil {
		return err
	}
	hashes[sum] = dupe

	sum, err = checksum(path)
	if err != nil {
		return err
	}

	if dupe, ok = hashes[sum]; !ok {
		hashes[sum] = path
		return nil
	}

	if *paranoid {
		same, err := fileContentsMatch(path, dupe)
		if err != nil {
			return err
		}
		if !same {
			fmt.Printf("cool: %s sha1-collides with %s!\n", path, dupe)
			return nil
		}
	}

	dupes++
	wasted += bytesize(size)

	final[dupe] = append(final[dupe], path)

	return nil
}

func sortedDupes() []string {
	var sk []string
	for k := range final {
		sk = append(sk, k)
	}
	sort.Strings(sk)
	return sk
}

func main() {
	flag.Usage = func() {
		var program = os.Args[0]
		fmt.Fprintf(os.Stderr, "Usage: %s [option...] directory...\n", program)
		flag.PrintDefaults()
	}

	flag.Parse()
	if len(flag.Args()) < 1 {
		flag.Usage()
	}

	_, err := filepath.Match(*globbing, "checking pattern syntax")
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: invalid pattern for -g (%v)\n", err)
		os.Exit(1)
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error: can't create profile (%v)\n", err)
			os.Exit(1)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	for _, root := range flag.Args() {
		err = filepath.Walk(root, check)
		if err != nil {
			fmt.Fprintf(os.Stderr, "warning: issue while walking %s (%v)\n", root, err)
		}
	}

	sk := sortedDupes()
	for _, k := range sk {
		vs := final[k]
		fmt.Println(k)
		for _, v := range vs {
			fmt.Println(v)
		}
		fmt.Println()
	}

	if len(sizes) > 0 || len(hashes) > 0 {
		fmt.Printf("%v files examined, %v duplicates found, %v wasted\n", files, dupes, wasted)
	}
}
<commit_msg>Always print the stats.<commit_after>// Copyright 2016 Peter H. Froehlich. All rights reserved.
// Use of this source code is governed by the MIT license,
// see the LICENSE.md file.

// Dupes finds duplicate files in the given paths.
//
// Run dupes as follows:
//
//	dupes path1 path2 ...
//
// Dupes will process each path. Directories will be walked
// recursively, regular files will be checked against all
// others. Dupes will print clusters of paths, separated
// by an empty line, for each duplicate it finds. Dupes will
// also print statistics about duplicates at the end.
//
// The -p option uses a "paranoid" byte-by-byte file comparison
// instead of SHA1 digests to identify duplicates.
//
// The -s option sets the minimum file size you care about;
// if defaults to 1 so empty files are ignored.
//
// The -g option sets a globbing pattern for the file names
// you care about; it defaults to * which matches all file
// names.
package main

import (
	"bytes"
	"crypto/sha1"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime/pprof"
	"sort"
)

const (
	globDefault = "*"
)

var (
	paranoid    = flag.Bool("p", false, "paranoid byte-by-byte file comparison")
	minimumSize = flag.Int64("s", 1, "minimum size (in bytes) of files to consider")
	globbing    = flag.String("g", globDefault, "glob expression for files to consider")
	cpuprofile  = flag.String("cpuprofile", "", "write cpu profile to file (development only)")
)

var (
	hashes = make(map[string]string)   // maps from digests to paths
	sizes  = make(map[int64]string)    // maps from sizes to paths
	final  = make(map[string][]string) // maps from paths to duplicate paths (collates all dupes)

	files  counter  // number of files examined
	dupes  counter  // number of duplicate files
	wasted bytesize // space (in bytes) occupied by duplicates
)

// fileContentsMatch does a byte-by-byte comparison of the files with the
// given paths
func fileContentsMatch(pa, pb string) (bool, error) {
	a, err := os.Open(pa)
	if err != nil {
		return false, err
	}
	defer a.Close()
	b, err := os.Open(pb)
	if err != nil {
		return false, err
	}
	defer b.Close()

	return fileContentsHelper(a, b)
}

func fileContentsHelper(a, b io.Reader) (bool, error) {
	bufferSize := os.Getpagesize()

	ba := make([]byte, bufferSize)
	bb := make([]byte, bufferSize)

	for {
		la, erra := a.Read(ba)
		lb, errb := b.Read(bb)

		// specification of Read() says to check returned size
		// before considering errors; who are we to disagree?
		if la > 0 || lb > 0 {
			// it's okay to use Equal here because whatever may
			// be left behind in the buffers after a short read
			// had to be Equal in the prior iteration; note that
			// we don't have to check la == lb either because if
			// they were not, Equal should fail
			if !bytes.Equal(ba, bb) {
				return false, nil
			}
		}

		// specification of Read() says that sooner or later we'll
		// see io.EOF regardless of returned size; only if both
		// files end in the same iteration (and made it past Equal
		// above) do we have a duplicate
		switch {
		case erra == io.EOF && errb == io.EOF:
			return true, nil
		case erra != nil:
			return false, erra
		case errb != nil:
			return false, errb
		}
	}
}

// checksum calculates a hash digest for the file with the given path
func checksum(path string) (string, error) {
	file, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hasher := sha1.New()
	_, err = io.Copy(hasher, file)
	sum := fmt.Sprintf("%x", hasher.Sum(nil))

	return sum, err
}

// check is called for each path we walk. It only examines regular, non-empty
// files. It first rules out duplicates by file size; for files that remain
// it calculates a checksum; if it has seen the same checksum before, it
// signals a duplicate; otherwise it remembers the checksum and the path of
// the original file before moving on; in paranoid mode it follows up with a
// byte-by-byte file comparison.
func check(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}

	size := info.Size()

	if !info.Mode().IsRegular() || size < *minimumSize {
		return nil
	}

	if *globbing != globDefault {
		matched, err := filepath.Match(*globbing, info.Name())
		if err != nil {
			return err
		}
		if !matched {
			return nil
		}
	}

	files++

	var dupe string
	var ok bool
	if dupe, ok = sizes[size]; !ok {
		sizes[size] = path
		return nil
	}

	// backpatch new file into hashes
	sum, err := checksum(dupe)
	if err != nil {
		return err
	}
	hashes[sum] = dupe

	sum, err = checksum(path)
	if err != nil {
		return err
	}

	if dupe, ok = hashes[sum]; !ok {
		hashes[sum] = path
		return nil
	}

	if *paranoid {
		same, err := fileContentsMatch(path, dupe)
		if err != nil {
			return err
		}
		if !same {
			fmt.Printf("cool: %s sha1-collides with %s!\n", path, dupe)
			return nil
		}
	}

	dupes++
	wasted += bytesize(size)

	final[dupe] = append(final[dupe], path)

	return nil
}

func sortedDupes() []string {
	var sk []string
	for k := range final {
		sk = append(sk, k)
	}
	sort.Strings(sk)
	return sk
}

func main() {
	flag.Usage = func() {
		var program = os.Args[0]
		fmt.Fprintf(os.Stderr, "Usage: %s [option...] directory...\n", program)
		flag.PrintDefaults()
	}

	flag.Parse()
	if len(flag.Args()) < 1 {
		flag.Usage()
	}

	_, err := filepath.Match(*globbing, "checking pattern syntax")
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: invalid pattern for -g (%v)\n", err)
		os.Exit(1)
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error: can't create profile (%v)\n", err)
			os.Exit(1)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	for _, root := range flag.Args() {
		err = filepath.Walk(root, check)
		if err != nil {
			fmt.Fprintf(os.Stderr, "warning: issue while walking %s (%v)\n", root, err)
		}
	}

	sk := sortedDupes()
	for _, k := range sk {
		vs := final[k]
		fmt.Println(k)
		for _, v := range vs {
			fmt.Println(v)
		}
		fmt.Println()
	}

	fmt.Printf("%v files examined, %v duplicates found, %v wasted\n", files, dupes, wasted)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"log"
	"math/rand"
	"net/http"
	"os"
	"regexp"
	"runtime"
	"time"

	"github.com/gorilla/websocket"
	sm "github.com/ifn/go-statemachine"
)

type DeskMsg struct {
	Desk [][]string `json:"desk"`
}

type PlayerMsg struct {
	Cmd  sm.EventType `json:"command"`
	Card string       `json:"card"`
}

//

var Suits []string = []string{
	"S",
	"C",
	"H",
	"D",
}

var CardValues map[string]int = map[string]int{
	"6": 6, "7": 7, "8": 8, "9": 9, "10": 10,
	"J": 11,
	"Q": 12,
	"K": 13,
	"A": 14,
}

func higher(c0, c1, t string) int {
	// c0 and c1 have the same suit
	if c0[0] == c1[0] {
		if CardValues[c0[1:]] > CardValues[c1[1:]] {
			return 1
		}
		if CardValues[c0[1:]] < CardValues[c1[1:]] {
			return -1
		}
		return 0
	}
	// c0 is trump, c1 is not
	if c0[:1] == t {
		return 1
	}
	// c1 is trump, c0 is not
	if c1[:1] == t {
		return -1
	}
	// suits are different, both are not trump
	return -2
}

var CardRE *regexp.Regexp = regexp.MustCompile(`[SCHD]([6-9JQKA]|10)`)

func isValid(c string) bool {
	return CardRE.MatchString(c)
}

//

const (
	cmdStart sm.EventType = iota
	cmdMove

	cmdCount
)

const (
	stateCollection sm.State = iota

	stateAttack
	stateDefense

	stateCount
)

type roundResult int

const (
	None roundResult = iota
	Beat
	NotBeat
)

func stateToString(s sm.State) string {
	return [...]string{
		stateCollection: "COLLECTION",
		stateDefense:    "DEFENSE",
		stateAttack:     "ATTACK",
	}[s]
}

func cmdToString(t sm.EventType) string {
	return [...]string{
		cmdStart: "START",
		cmdMove:  "MOVE",
	}[t]
}

//

type cmdArgs struct {
	conn *playerConn
	card string
}

//

func logOutOfTurn(pconn *playerConn) {
	log.Printf("out of turn: %v", pconn.conn.RemoteAddr())
}

func logWontBeat(c1, c2, t string) {
	log.Printf("%v won't bit %v, trump is ", c1, c2, t)
}

func logNoQuorum(n int) {
	log.Printf("no quorum: %v player(s)", n)
}

//

type gameState struct {
	// 1. fields that don't change during a game

	sm  *sm.StateMachine
	hub *hub

	// trump suit
	trump string

	// 2. fields that don't change during a round

	deck []string

	// attacker that started a round
	aconnStart *playerConn
	// defender
	dconn *playerConn

	// 3. fields that change during a round

	// attacker
	aconn *playerConn
	// card that should be beaten
	cardToBeat string
}

func (self *gameState) popCard() (card string) {
	if deck := self.deck; len(deck) > 0 {
		card = deck[0]
		self.deck = deck[1:]
	}
	return
}

func (self *gameState) initDeck() {
	numCards := len(Suits) * len(CardValues)
	self.deck = make([]string, 0, numCards+ /*for trump*/ 1)

	deck := make([]string, 0, numCards)
	for i := range Suits {
		for cv := range CardValues {
			deck = append(deck, Suits[i]+cv)
		}
	}

	order := rand.Perm(numCards)
	for i := range order {
		self.deck = append(self.deck, deck[order[i]])
	}
}

func (self *gameState) setTrump() {
	card := self.popCard()
	self.trump = card[:1]
	self.deck = append(self.deck, card)
}

func (self *gameState) nextPlayer(c *playerConn) *playerConn {
	return self.hub.conns.(*mapRing).Next(c).(*playerConn)
}

func (self *gameState) chooseStarting() *playerConn {
	conns := self.hub.conns.(*mapRing)

	return conns.Nth(rand.Intn(conns.Len())).(*playerConn)
}

func (self *gameState) setRoles(res roundResult) {
	switch res {
	case None:
		self.aconn = self.chooseStarting()
	case Beat:
		self.aconn = self.dconn
	case NotBeat:
		self.aconn = self.nextPlayer(self.dconn)
	}
	self.dconn = self.nextPlayer(self.aconn)
	self.aconnStart = self.aconn
}

func (self *gameState) dealCards() {
	for i := 0; i < 6; i++ {
		for pc := range self.hub.conns.Enumerate() {
			pc.(*playerConn).takeCard()
		}
	}
}

func (self *gameState) takeCards() {
	conns := self.hub.conns.(*mapRing)

	takeCards := func(pc *playerConn) {
		for len(self.deck) > 0 && len(pc.cards) < 6 {
			pc.takeCard()
		}
	}

	for pc := range conns.EnumerateFrom(self.aconnStart) {
		if pc := pc.(*playerConn); pc != self.dconn {
			takeCards(pc)
		}
	}
	takeCards(self.dconn)
}

func (self *gameState) newRound(res roundResult) {
	if res == None {
		self.initDeck()
		self.dealCards()
		self.setTrump()
	} else {
		self.takeCards()
	}
	self.setRoles(res)
}

// event handlers
// event handlers are actually transition functions.
// in case error event handler should neither change the gameState,
// nor return the state value different from passed to it as an argument.

func (self *gameState) handleStartInCollection(s sm.State, e *sm.Event) sm.State {
	if n := self.hub.conns.(*mapRing).Len(); n < 2 {
		logNoQuorum(n)
		return s
	}

	self.newRound(None)
	return stateAttack
}

func (self *gameState) handleMoveInAttack(s sm.State, e *sm.Event) sm.State {
	conn := e.Data.(cmdArgs).conn
	card := e.Data.(cmdArgs).card

	// check that it's conn's turn to move
	if conn != self.aconn {
		logOutOfTurn(conn)
		return s
	}

	// attacker sent the card
	if card != "" {
		self.cardToBeat = card
		return stateDefense
	}

	// attacker sent no card

	aconn := self.nextPlayer(self.aconn)
	if aconn == self.dconn {
		aconn = self.nextPlayer(aconn)
	}

	// check if all attackers have been polled
	if aconn == self.aconnStart {
		self.newRound(Beat)
		return stateAttack
	}

	self.aconn = aconn
	return stateAttack
}

func (self *gameState) handleMoveInDefense(s sm.State, e *sm.Event) sm.State {
	conn := e.Data.(cmdArgs).conn
	card := e.Data.(cmdArgs).card

	// check that it's conn's turn to move
	if conn != self.dconn {
		logOutOfTurn(conn)
		return s
	}

	// defender takes the cards
	if card == "" {
		self.newRound(NotBeat)
		return stateAttack
	}

	// check that the sent card is capable to beat
	if higher(card, self.cardToBeat, self.trump) != 1 {
		logWontBeat(card, self.cardToBeat, self.trump)
		return s
	}

	return stateAttack
}

func (self *gameState) showDesk(s sm.State, e *sm.Event) sm.State {
	desk, err := json.Marshal(DeskMsg{})
	if err != nil {
		log.Println(err)
		return s
	}

	self.hub.bcastChan <- desk

	return s
}

//

func NewGameState() *gameState {
	gst := new(gameState)

	gst.sm = sm.New(stateCollection, uint(stateCount), uint(cmdCount))

	gst.sm.On(cmdStart,
		[]sm.State{stateCollection},
		gst.handleStartInCollection,
	)

	gst.sm.On(cmdMove,
		[]sm.State{stateAttack},
		gst.handleMoveInAttack,
	)
	gst.sm.On(cmdMove,
		[]sm.State{stateDefense},
		gst.handleMoveInDefense,
	)

	gst.sm.On(cmdMove,
		[]sm.State{stateAttack, stateDefense},
		gst.showDesk,
	)

	gst.hub = NewHub()

	return gst
}

//

type playerConn struct {
	gst *gameState

	cards map[string]bool

	conn      *websocket.Conn
	hubToConn chan []byte
}

func (self *playerConn) takeCard() {
	if card := self.gst.popCard(); card != "" {
		self.cards[card] = true
	}
}

func (self *playerConn) write() {
	defer func() {
		err := self.conn.Close()
		if err != nil {
			log.Println(err)
		}
	}()

	for {
		select {
		case m, ok := <-self.hubToConn:
			if !ok {
				return
			}
			//TODO: text or binary?
			err := self.conn.WriteMessage(websocket.TextMessage, m)
			if err != nil {
				log.Println(err)
				return
			}
		}
	}
}

func (self *playerConn) read() {
	defer func() {
		err := self.conn.Close()
		if err != nil {
			log.Println(err)
		}
	}()

	var m PlayerMsg

	for {
		err := self.conn.ReadJSON(&m)
		if err != nil {
			log.Println(err)
			return
		}

		var event *sm.Event
		switch m.Cmd {
		case cmdStart:
			event = &sm.Event{cmdStart, nil}
		case cmdMove:
			event = &sm.Event{cmdMove, cmdArgs{self, m.Card}}
		default:
			log.Printf("unknown command: %v", m.Cmd)
			continue
		}

		err = self.gst.sm.Emit(event)
		if err != nil {
			log.Println(err)
		}
	}
}

//

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin:     func(r *http.Request) bool { return true },
}

func playerHandler(gst *gameState) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		conn, err := upgrader.Upgrade(w, r, nil)
		if err != nil {
			log.Println(err)
			return
		}

		p := &playerConn{gst, make(map[string]bool), conn, make(chan []byte)}

		gst.hub.regChan <- p
		defer func() {
			gst.hub.unregChan <- p
		}()

		go p.write()
		p.read()
	}
}

//

func startDurakSrv() error {
	gst := NewGameState()

	http.HandleFunc("/", playerHandler(gst))

	return http.ListenAndServe(":"+os.Getenv("PORT"), nil)
}

func main() {
	err := startDurakSrv()
	if err != nil {
		log.Fatal(err)
	}
}

func init() {
	rand.Seed(time.Now().UnixNano())

	runtime.GOMAXPROCS(runtime.NumCPU())
}
<commit_msg>replaced bool by empty struct in set<commit_after>package main

import (
	"encoding/json"
	"log"
	"math/rand"
	"net/http"
	"os"
	"regexp"
	"runtime"
	"time"

	"github.com/gorilla/websocket"
	sm "github.com/ifn/go-statemachine"
)

type DeskMsg struct {
	Desk [][]string `json:"desk"`
}

type PlayerMsg struct {
	Cmd  sm.EventType `json:"command"`
	Card string       `json:"card"`
}

//

var Suits []string = []string{
	"S",
	"C",
	"H",
	"D",
}

var CardValues map[string]int = map[string]int{
	"6": 6, "7": 7, "8": 8, "9": 9, "10": 10,
	"J": 11,
	"Q": 12,
	"K": 13,
	"A": 14,
}

func higher(c0, c1, t string) int {
	// c0 and c1 have the same suit
	if c0[0] == c1[0] {
		if CardValues[c0[1:]] > CardValues[c1[1:]] {
			return 1
		}
		if CardValues[c0[1:]] < CardValues[c1[1:]] {
			return -1
		}
		return 0
	}
	// c0 is trump, c1 is not
	if c0[:1] == t {
		return 1
	}
	// c1 is trump, c0 is not
	if c1[:1] == t {
		return -1
	}
	// suits are different, both are not trump
	return -2
}

var CardRE *regexp.Regexp = regexp.MustCompile(`[SCHD]([6-9JQKA]|10)`)

func isValid(c string) bool {
	return CardRE.MatchString(c)
}

//

const (
	cmdStart sm.EventType = iota
	cmdMove

	cmdCount
)

const (
	stateCollection sm.State = iota

	stateAttack
	stateDefense

	stateCount
)

type roundResult int

const (
	None roundResult = iota
	Beat
	NotBeat
)

func stateToString(s sm.State) string {
	return [...]string{
		stateCollection: "COLLECTION",
		stateDefense:    "DEFENSE",
		stateAttack:     "ATTACK",
	}[s]
}

func cmdToString(t sm.EventType) string {
	return [...]string{
		cmdStart: "START",
		cmdMove:  "MOVE",
	}[t]
}

//

type cmdArgs struct {
	conn *playerConn
	card string
}

//

func logOutOfTurn(pconn *playerConn) {
	log.Printf("out of turn: %v", pconn.conn.RemoteAddr())
}

func logWontBeat(c1, c2, t string) {
	log.Printf("%v won't bit %v, trump is ", c1, c2, t)
}

func logNoQuorum(n int) {
	log.Printf("no quorum: %v player(s)", n)
}

//

type gameState struct {
	// 1. fields that don't change during a game

	sm  *sm.StateMachine
	hub *hub

	// trump suit
	trump string

	// 2. fields that don't change during a round

	deck []string

	// attacker that started a round
	aconnStart *playerConn
	// defender
	dconn *playerConn

	// 3. fields that change during a round

	// attacker
	aconn *playerConn
	// card that should be beaten
	cardToBeat string
}

func (self *gameState) popCard() (card string) {
	if deck := self.deck; len(deck) > 0 {
		card = deck[0]
		self.deck = deck[1:]
	}
	return
}

func (self *gameState) initDeck() {
	numCards := len(Suits) * len(CardValues)
	self.deck = make([]string, 0, numCards+ /*for trump*/ 1)

	deck := make([]string, 0, numCards)
	for i := range Suits {
		for cv := range CardValues {
			deck = append(deck, Suits[i]+cv)
		}
	}

	order := rand.Perm(numCards)
	for i := range order {
		self.deck = append(self.deck, deck[order[i]])
	}
}

func (self *gameState) setTrump() {
	card := self.popCard()
	self.trump = card[:1]
	self.deck = append(self.deck, card)
}

func (self *gameState) nextPlayer(c *playerConn) *playerConn {
	return self.hub.conns.(*mapRing).Next(c).(*playerConn)
}

func (self *gameState) chooseStarting() *playerConn {
	conns := self.hub.conns.(*mapRing)

	return conns.Nth(rand.Intn(conns.Len())).(*playerConn)
}

func (self *gameState) setRoles(res roundResult) {
	switch res {
	case None:
		self.aconn = self.chooseStarting()
	case Beat:
		self.aconn = self.dconn
	case NotBeat:
		self.aconn = self.nextPlayer(self.dconn)
	}
	self.dconn = self.nextPlayer(self.aconn)
	self.aconnStart = self.aconn
}

func (self *gameState) dealCards() {
	for i := 0; i < 6; i++ {
		for pc := range self.hub.conns.Enumerate() {
			pc.(*playerConn).takeCard()
		}
	}
}

func (self *gameState) takeCards() {
	conns := self.hub.conns.(*mapRing)

	takeCards := func(pc *playerConn) {
		for len(self.deck) > 0 && len(pc.cards) < 6 {
			pc.takeCard()
		}
	}

	for pc := range conns.EnumerateFrom(self.aconnStart) {
		if pc := pc.(*playerConn); pc != self.dconn {
			takeCards(pc)
		}
	}
	takeCards(self.dconn)
}

func (self *gameState) newRound(res roundResult) {
	if res == None {
		self.initDeck()
		self.dealCards()
		self.setTrump()
	} else {
		self.takeCards()
	}
	self.setRoles(res)
}

// event handlers
// event handlers are actually transition functions.
// in case error event handler should neither change the gameState,
// nor return the state value different from passed to it as an argument.

func (self *gameState) handleStartInCollection(s sm.State, e *sm.Event) sm.State {
	if n := self.hub.conns.(*mapRing).Len(); n < 2 {
		logNoQuorum(n)
		return s
	}

	self.newRound(None)
	return stateAttack
}

func (self *gameState) handleMoveInAttack(s sm.State, e *sm.Event) sm.State {
	conn := e.Data.(cmdArgs).conn
	card := e.Data.(cmdArgs).card

	// check that it's conn's turn to move
	if conn != self.aconn {
		logOutOfTurn(conn)
		return s
	}

	// attacker sent the card
	if card != "" {
		self.cardToBeat = card
		return stateDefense
	}

	// attacker sent no card

	aconn := self.nextPlayer(self.aconn)
	if aconn == self.dconn {
		aconn = self.nextPlayer(aconn)
	}

	// check if all attackers have been polled
	if aconn == self.aconnStart {
		self.newRound(Beat)
		return stateAttack
	}

	self.aconn = aconn
	return stateAttack
}

func (self *gameState) handleMoveInDefense(s sm.State, e *sm.Event) sm.State {
	conn := e.Data.(cmdArgs).conn
	card := e.Data.(cmdArgs).card

	// check that it's conn's turn to move
	if conn != self.dconn {
		logOutOfTurn(conn)
		return s
	}

	// defender takes the cards
	if card == "" {
		self.newRound(NotBeat)
		return stateAttack
	}

	// check that the sent card is capable to beat
	if higher(card, self.cardToBeat, self.trump) != 1 {
		logWontBeat(card, self.cardToBeat, self.trump)
		return s
	}

	return stateAttack
}

func (self *gameState) showDesk(s sm.State, e *sm.Event) sm.State {
	desk, err := json.Marshal(DeskMsg{})
	if err != nil {
		log.Println(err)
		return s
	}

	self.hub.bcastChan <- desk

	return s
}

//

func NewGameState() *gameState {
	gst := new(gameState)

	gst.sm = sm.New(stateCollection, uint(stateCount), uint(cmdCount))

	gst.sm.On(cmdStart,
		[]sm.State{stateCollection},
		gst.handleStartInCollection,
	)

	gst.sm.On(cmdMove,
		[]sm.State{stateAttack},
		gst.handleMoveInAttack,
	)
	gst.sm.On(cmdMove,
		[]sm.State{stateDefense},
		gst.handleMoveInDefense,
	)

	gst.sm.On(cmdMove,
		[]sm.State{stateAttack, stateDefense},
		gst.showDesk,
	)

	gst.hub = NewHub()

	return gst
}

//

type playerConn struct {
	gst *gameState

	cards map[string]struct{}

	conn      *websocket.Conn
	hubToConn chan []byte
}

func (self *playerConn) takeCard() {
	if card := self.gst.popCard(); card != "" {
		self.cards[card] = struct{}{}
	}
}

func (self *playerConn) write() {
	defer func() {
		err := self.conn.Close()
		if err != nil {
			log.Println(err)
		}
	}()

	for {
		select {
		case m, ok := <-self.hubToConn:
			if !ok {
				return
			}
			//TODO: text or binary?
			err := self.conn.WriteMessage(websocket.TextMessage, m)
			if err != nil {
				log.Println(err)
				return
			}
		}
	}
}

func (self *playerConn) read() {
	defer func() {
		err := self.conn.Close()
		if err != nil {
			log.Println(err)
		}
	}()

	var m PlayerMsg

	for {
		err := self.conn.ReadJSON(&m)
		if err != nil {
			log.Println(err)
			return
		}

		var event *sm.Event
		switch m.Cmd {
		case cmdStart:
			event = &sm.Event{cmdStart, nil}
		case cmdMove:
			event = &sm.Event{cmdMove, cmdArgs{self, m.Card}}
		default:
			log.Printf("unknown command: %v", m.Cmd)
			continue
		}

		err = self.gst.sm.Emit(event)
		if err != nil {
			log.Println(err)
		}
	}
}

//

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin:     func(r *http.Request) bool { return true },
}

func playerHandler(gst *gameState) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		conn, err := upgrader.Upgrade(w, r, nil)
		if err != nil {
			log.Println(err)
			return
		}

		p := &playerConn{gst, make(map[string]struct{}), conn, make(chan []byte)}

		gst.hub.regChan <- p
		defer func() {
			gst.hub.unregChan <- p
		}()

		go p.write()
		p.read()
	}
}

//

func startDurakSrv() error {
	gst := NewGameState()

	http.HandleFunc("/", playerHandler(gst))

	return http.ListenAndServe(":"+os.Getenv("PORT"), nil)
}

func main() {
	err := startDurakSrv()
	if err != nil {
		log.Fatal(err)
	}
}

func init() {
	rand.Seed(time.Now().UnixNano())

	runtime.GOMAXPROCS(runtime.NumCPU())
}
<|endoftext|>


<commit_before>/*
Package embed allows for storing data resources in a virtual filesystem
that gets compiled directly into the output program, eliminating the need
to distribute data files with the application. This is especially useful for small
web servers that need to deliver content files.

The data is gzipped to save space.

An external tool for generating output files can be found at http://github.com/cratonica/embedder

Author: Clint Caywood

http://github.com/cratonica/embed
*/
package embed

import (
	"bytes"
	"compress/gzip"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

// A mapping of resource identifiers to associated data
type ResourceMap map[string][]byte

// A compressed resource map that can be used for serialization
type PackedResourceMap []byte

var byteOrder binary.ByteOrder = binary.LittleEndian

// Packs the map of resource identifiers => []byte into
// a buffer. This process is revered by calling Unpack.
func Pack(data ResourceMap) (PackedResourceMap, error) {
	var buf bytes.Buffer
	writer, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)
	if err != nil {
		return nil, err
	}
	for i, v := range data {
		if err = binary.Write(writer, byteOrder, int32(len(i))); err != nil {
			return nil, err
		}
		if _, err = writer.Write([]byte(i)); err != nil {
			return nil, err
		}
		if err = binary.Write(writer, byteOrder, int32(len(v))); err != nil {
			return nil, err
		}
		if _, err = writer.Write(v); err != nil {
		}
	}
	if err = writer.Close(); err != nil {
		return nil, err
	}
	result := buf.Bytes()
	return result, nil
}

type DeserializationError struct {
	what     string
	expected int
	actual   int
}

func (this *DeserializationError) Error() string {
	return fmt.Sprintf("Failure deserializing resource: expected %v to be %v bytes, but only read %v bytes", this.what, this.expected, this.actual)
}

func NewDeserializationError(what string, expected int, actual int) *DeserializationError {
	return &DeserializationError{what, expected, actual}
}

// Reads a buffer generated by a call to Pack, returning the original map
func Unpack(data PackedResourceMap) (ResourceMap, error) {
	result := make(map[string][]byte)
	buf := bytes.NewBuffer(data)
	reader, err := gzip.NewReader(buf)
	defer func() {
		reader.Close()
	}()
	if err != nil {
		return nil, err
	}
	for {
		var size int32
		err := binary.Read(reader, byteOrder, &size)
		if err != nil {
			break // EOF
		}
		keyBuf, err := readAll(reader, int(size))
		if err != nil {
			return nil, err
		}
		err = binary.Read(reader, byteOrder, &size)
		if err != nil {
			return nil, err
		}
		dataBuf, err := readAll(reader, int(size))
		if err != nil {
			return nil, err
		}
		result[string(keyBuf)] = dataBuf
	}
	return result, nil
}

// Recursively packs all files in the given directory into
// a resource map. Directory delimiters are converted to Unix-style /
// regardless of the host operating system.
func CreateFromFiles(path string) (ResourceMap, error) {
	result := make(map[string][]byte)
	err := crawl(filepath.Clean(path), filepath.Clean(path), result)
	if err != nil {
		return nil, err
	}
	return result, nil
}

// Generates Go code containing the given data that can be included in the target project
func GenerateGoCode(packageName string, varName string, data PackedResourceMap) string {
	template := `
package %v

import "github.com/cratonica/embed"

var %v embed.PackedResourceMap = %#v
`
	return fmt.Sprintf(template, packageName, varName, data)
}

func readAll(reader io.Reader, size int) ([]byte, error) {
	result := make([]byte, size)
	totalBytes := 0
	for totalBytes < size {
		readSize, err := reader.Read(result[totalBytes:])
		if err != nil {
			return nil, err
		}
		totalBytes += readSize
	}
	return result, nil
}

func crawl(rootPath string, currentPath string, dest map[string][]byte) error {
	files, err := ioutil.ReadDir(currentPath)
	if err != nil {
		return err
	}
	for _, file := range files {
		fullPath := filepath.Join(currentPath, file.Name())
		if file.IsDir() {
			err := crawl(rootPath, fullPath, dest)
			if err != nil {
				return err
			}
		} else {
			buf, err := ioutil.ReadFile(fullPath)
			if err != nil {
				log.Printf("Failure reading file %v: %v", fullPath, err)
			} else {
				key := strings.TrimLeft(fullPath, rootPath)
				if strings.HasPrefix(key, string(os.PathSeparator)) {
					key = key[1:]
				}
				// Convert to unix style for internal use
				key = strings.Replace(key, string(os.PathSeparator), "/", -1)
				dest[key] = buf
			}
		}
	}
	return nil
}
<commit_msg>Fix typo<commit_after>/*
Package embed allows for storing data resources in a virtual filesystem
that gets compiled directly into the output program, eliminating the need
to distribute data files with the application. This is especially useful for small
web servers that need to deliver content files.

The data is gzipped to save space.

An external tool for generating output files can be found at http://github.com/cratonica/embedder

Author: Clint Caywood

http://github.com/cratonica/embed
*/
package embed

import (
	"bytes"
	"compress/gzip"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

// A mapping of resource identifiers to associated data
type ResourceMap map[string][]byte

// A compressed resource map that can be used for serialization
type PackedResourceMap []byte

var byteOrder binary.ByteOrder = binary.LittleEndian

// Packs the map of resource identifiers => []byte into
// a buffer. This process is reversed by calling Unpack.
func Pack(data ResourceMap) (PackedResourceMap, error) {
	var buf bytes.Buffer
	writer, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)
	if err != nil {
		return nil, err
	}
	for i, v := range data {
		if err = binary.Write(writer, byteOrder, int32(len(i))); err != nil {
			return nil, err
		}
		if _, err = writer.Write([]byte(i)); err != nil {
			return nil, err
		}
		if err = binary.Write(writer, byteOrder, int32(len(v))); err != nil {
			return nil, err
		}
		if _, err = writer.Write(v); err != nil {
		}
	}
	if err = writer.Close(); err != nil {
		return nil, err
	}
	result := buf.Bytes()
	return result, nil
}

type DeserializationError struct {
	what     string
	expected int
	actual   int
}

func (this *DeserializationError) Error() string {
	return fmt.Sprintf("Failure deserializing resource: expected %v to be %v bytes, but only read %v bytes", this.what, this.expected, this.actual)
}

func NewDeserializationError(what string, expected int, actual int) *DeserializationError {
	return &DeserializationError{what, expected, actual}
}

// Reads a buffer generated by a call to Pack, returning the original map
func Unpack(data PackedResourceMap) (ResourceMap, error) {
	result := make(map[string][]byte)
	buf := bytes.NewBuffer(data)
	reader, err := gzip.NewReader(buf)
	defer func() {
		reader.Close()
	}()
	if err != nil {
		return nil, err
	}
	for {
		var size int32
		err := binary.Read(reader, byteOrder, &size)
		if err != nil {
			break // EOF
		}
		keyBuf, err := readAll(reader, int(size))
		if err != nil {
			return nil, err
		}
		err = binary.Read(reader, byteOrder, &size)
		if err != nil {
			return nil, err
		}
		dataBuf, err := readAll(reader, int(size))
		if err != nil {
			return nil, err
		}
		result[string(keyBuf)] = dataBuf
	}
	return result, nil
}

// Recursively packs all files in the given directory into
// a resource map. Directory delimiters are converted to Unix-style /
// regardless of the host operating system.
func CreateFromFiles(path string) (ResourceMap, error) {
	result := make(map[string][]byte)
	err := crawl(filepath.Clean(path), filepath.Clean(path), result)
	if err != nil {
		return nil, err
	}
	return result, nil
}

// Generates Go code containing the given data that can be included in the target project
func GenerateGoCode(packageName string, varName string, data PackedResourceMap) string {
	template := `
package %v

import "github.com/cratonica/embed"

var %v embed.PackedResourceMap = %#v
`
	return fmt.Sprintf(template, packageName, varName, data)
}

func readAll(reader io.Reader, size int) ([]byte, error) {
	result := make([]byte, size)
	totalBytes := 0
	for totalBytes < size {
		readSize, err := reader.Read(result[totalBytes:])
		if err != nil {
			return nil, err
		}
		totalBytes += readSize
	}
	return result, nil
}

func crawl(rootPath string, currentPath string, dest map[string][]byte) error {
	files, err := ioutil.ReadDir(currentPath)
	if err != nil {
		return err
	}
	for _, file := range files {
		fullPath := filepath.Join(currentPath, file.Name())
		if file.IsDir() {
			err := crawl(rootPath, fullPath, dest)
			if err != nil {
				return err
			}
		} else {
			buf, err := ioutil.ReadFile(fullPath)
			if err != nil {
				log.Printf("Failure reading file %v: %v", fullPath, err)
			} else {
				key := strings.TrimLeft(fullPath, rootPath)
				if strings.HasPrefix(key, string(os.PathSeparator)) {
					key = key[1:]
				}
				// Convert to unix style for internal use
				key = strings.Replace(key, string(os.PathSeparator), "/", -1)
				dest[key] = buf
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
The emogo package provides go bindings for emokit
(https://github.com/openyou/emokit).
*/
package emogo

import (
	"errors"
	"unsafe"
)

// #include <emokit/emokit.h>
// #include <stdint.h>
// #cgo LDFLAGS: -lemokit
import "C"

// These are defined in emokit.h and reproduced here as cgo isn't
// linking them in for some reason.
const (
	EMOKIT_VID int = 0x21a1
	EMOKIT_PID  = 0x0001
	EmokitPacketSize = 32
)

type HeadsetType uint

const (
	DeveloperHeadset HeadsetType = 0
	ConsumerHeadset HeadsetType = 1
)

// EmokitContext represents a connection to an EPOC device.
type EmokitContext struct {
	eeg *C.struct_emokit_device
}

func NewEmokitContext(t HeadsetType) (*EmokitContext, error) {
	e := new(EmokitContext)
	e.eeg = C.emokit_create()
	ret := C.emokit_open(e.eeg, C.int(EMOKIT_VID), C.int(EMOKIT_PID), C.uint(t))
	if ret != 0 {
		return nil, errors.New("Cannot access device.")
	}
	return e, nil
}

type EmokitFrame struct {
	raw []byte
	rendered C.struct_emokit_frame
}

func NewEmokitFrame() *EmokitFrame {
	f := new(EmokitFrame)
	f.raw = make([]byte, EmokitPacketSize)
	return f
}

// readData reads data from the EPOC dongle and returns 0 on success, <0
// on error.
func (e *EmokitContext) readData() error {
	n := C.emokit_read_data(e.eeg)
	if n >= 0 {
		return nil
	}
	return errors.New("emokit_read_data failed")
}

func (e *EmokitContext) getNextFrame() (*EmokitFrame, error) {
	f := NewEmokitFrame()
	f.rendered = C.emokit_get_next_frame(e.eeg)
	if f.rendered.counter == 0 {
		return nil, errors.New("Could not read raw packet.")
	}
	C.emokit_get_raw_frame(e.eeg, (*C.uchar)(unsafe.Pointer(&f.raw[0])))
	return f, nil
}

// GetFrame returns the next available EPOC frame. If there is no frame
// to be read, the error value will be EAGAIN.
func (e *EmokitContext) GetFrame() (*EmokitFrame, error) {
	err := e.readData()
	if err == nil {
		f, err := e.getNextFrame()
		if err != nil {
			return nil, err
		}
		return f, nil
	}
	return nil, err
}

func (e *EmokitContext) Count() int {
	n := C.emokit_get_count(e.eeg, C.int(EMOKIT_VID), C.int(EMOKIT_PID))
	return int(n)
}

// Raw returns the (unencrypted) raw EPOC frame.
func (f *EmokitFrame) Raw() []byte {
	return f.raw
}

// Gyro returns the current (x,y) of the frame's Gyro value.
func (f *EmokitFrame) Gyro() (int,int) {
	return int(f.rendered.gyroX), int(f.rendered.gyroY)
}
<commit_msg>Add Shutdown function<commit_after>/*
The emogo package provides go bindings for emokit
(https://github.com/openyou/emokit).
*/
package emogo

import (
	"errors"
	"unsafe"
)

// #include <emokit/emokit.h>
// #include <stdint.h>
// #cgo LDFLAGS: -lemokit
import "C"

// These are defined in emokit.h and reproduced here as cgo isn't
// linking them in for some reason.
const (
	EMOKIT_VID int = 0x21a1
	EMOKIT_PID  = 0x0001
	EmokitPacketSize = 32
)

type HeadsetType uint

const (
	DeveloperHeadset HeadsetType = 0
	ConsumerHeadset HeadsetType = 1
)

// EmokitContext represents a connection to an EPOC device.
type EmokitContext struct {
	eeg *C.struct_emokit_device
}

func NewEmokitContext(t HeadsetType) (*EmokitContext, error) {
	e := new(EmokitContext)
	e.eeg = C.emokit_create()
	ret := C.emokit_open(e.eeg, C.int(EMOKIT_VID), C.int(EMOKIT_PID), C.uint(t))
	if ret != 0 {
		return nil, errors.New("Cannot access device.")
	}
	return e, nil
}

// Shutdown closes the connection to the EPOC and frees associated
// memory.
func (c *EmokitContext) Shutdown() {
	C.emokit_close(c.eeg)
	C.emokit_delete(c.eeg)
}

type EmokitFrame struct {
	raw []byte
	rendered C.struct_emokit_frame
}

func NewEmokitFrame() *EmokitFrame {
	f := new(EmokitFrame)
	f.raw = make([]byte, EmokitPacketSize)
	return f
}

// readData reads data from the EPOC dongle and returns 0 on success, <0
// on error.
func (e *EmokitContext) readData() error {
	n := C.emokit_read_data(e.eeg)
	if n >= 0 {
		return nil
	}
	return errors.New("emokit_read_data failed")
}

func (e *EmokitContext) getNextFrame() (*EmokitFrame, error) {
	f := NewEmokitFrame()
	f.rendered = C.emokit_get_next_frame(e.eeg)
	if f.rendered.counter == 0 {
		return nil, errors.New("Could not read raw packet.")
	}
	C.emokit_get_raw_frame(e.eeg, (*C.uchar)(unsafe.Pointer(&f.raw[0])))
	return f, nil
}

// GetFrame returns the next available EPOC frame. If there is no frame
// to be read, the error value will be EAGAIN.
func (e *EmokitContext) GetFrame() (*EmokitFrame, error) {
	err := e.readData()
	if err == nil {
		f, err := e.getNextFrame()
		if err != nil {
			return nil, err
		}
		return f, nil
	}
	return nil, err
}

func (e *EmokitContext) Count() int {
	n := C.emokit_get_count(e.eeg, C.int(EMOKIT_VID), C.int(EMOKIT_PID))
	return int(n)
}

// Raw returns the (unencrypted) raw EPOC frame.
func (f *EmokitFrame) Raw() []byte {
	return f.raw
}

// Gyro returns the current (x,y) of the frame's Gyro value.
func (f *EmokitFrame) Gyro() (int,int) {
	return int(f.rendered.gyroX), int(f.rendered.gyroY)
}
<|endoftext|>


<commit_before>package logrus

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"time"
)

// Defines the key when adding errors using WithError.
var ErrorKey = "error"

// An entry is the final or intermediate Logrus logging entry. It contains all
// the fields passed with WithField{,s}. It's finally logged when Debug, Info,
// Warn, Error, Fatal or Panic is called on it. These objects can be reused and
// passed around as much as you wish to avoid field duplication.
type Entry struct {
	Logger *Logger

	// Contains all the fields set by the user.
	Data Fields

	// Time at which the log entry was created
	Time time.Time

	// Level the log entry was logged at: Debug, Info, Warn, Error, Fatal or Panic
	Level Level

	// Message passed to Debug, Info, Warn, Error, Fatal or Panic
	Message string
}

func NewEntry(logger *Logger) *Entry {
	return &Entry{
		Logger: logger,
		// Default is three fields, give a little extra room
		Data: make(Fields, 5),
	}
}

// Returns a reader for the entry, which is a proxy to the formatter.
func (entry *Entry) Reader() (*bytes.Buffer, error) {
	serialized, err := entry.Logger.Formatter.Format(entry)
	return bytes.NewBuffer(serialized), err
}

// Returns the string representation from the reader and ultimately the
// formatter.
func (entry *Entry) String() (string, error) {
	reader, err := entry.Reader()
	if err != nil {
		return "", err
	}

	return reader.String(), err
}

// Add an error as single field (using the key defined in ErrorKey) to the Entry.
func (entry *Entry) WithError(err error) *Entry {
	return entry.WithField(ErrorKey, err)
}

// Add a single field to the Entry.
func (entry *Entry) WithField(key string, value interface{}) *Entry {
	return entry.WithFields(Fields{key: value})
}

// Add a map of fields to the Entry.
func (entry *Entry) WithFields(fields Fields) *Entry {
	data := make(Fields, len(entry.Data)+len(fields))
	for k, v := range entry.Data {
		data[k] = v
	}
	for k, v := range fields {
		data[k] = v
	}
	return &Entry{Logger: entry.Logger, Data: data}
}

// This function is not declared with a pointer value because otherwise
// race conditions will occur when using multiple goroutines
func (entry Entry) log(level Level, msg string) {
	entry.Time = time.Now()
	entry.Level = level
	entry.Message = msg

	if err := entry.Logger.Hooks.Fire(level, &entry); err != nil {
		entry.Logger.mu.Lock()
		fmt.Fprintf(os.Stderr, "Failed to fire hook: %v\n", err)
		entry.Logger.mu.Unlock()
	}

	reader, err := entry.Reader()
	if err != nil {
		entry.Logger.mu.Lock()
		fmt.Fprintf(os.Stderr, "Failed to obtain reader, %v\n", err)
		entry.Logger.mu.Unlock()
	}

	entry.Logger.mu.Lock()
	defer entry.Logger.mu.Unlock()

	_, err = io.Copy(entry.Logger.Out, reader)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to write to log, %v\n", err)
	}

	// To avoid Entry#log() returning a value that only would make sense for
	// panic() to use in Entry#Panic(), we avoid the allocation by checking
	// directly here.
	if level <= PanicLevel {
		panic(&entry)
	}
}

func (entry *Entry) Debug(args ...interface{}) {
	if entry.Logger.Level >= DebugLevel {
		entry.log(DebugLevel, fmt.Sprint(args...))
	}
}

func (entry *Entry) Print(args ...interface{}) {
	entry.Info(args...)
}

func (entry *Entry) Info(args ...interface{}) {
	if entry.Logger.Level >= InfoLevel {
		entry.log(InfoLevel, fmt.Sprint(args...))
	}
}

func (entry *Entry) Warn(args ...interface{}) {
	if entry.Logger.Level >= WarnLevel {
		entry.log(WarnLevel, fmt.Sprint(args...))
	}
}

func (entry *Entry) Warning(args ...interface{}) {
	entry.Warn(args...)
}

func (entry *Entry) Error(args ...interface{}) {
	if entry.Logger.Level >= ErrorLevel {
		entry.log(ErrorLevel, fmt.Sprint(args...))
	}
}

func (entry *Entry) Fatal(args ...interface{}) {
	if entry.Logger.Level >= FatalLevel {
		entry.log(FatalLevel, fmt.Sprint(args...))
	}
	Exit(1)
}

func (entry *Entry) Panic(args ...interface{}) {
	if entry.Logger.Level >= PanicLevel {
		entry.log(PanicLevel, fmt.Sprint(args...))
	}
	panic(fmt.Sprint(args...))
}

// Entry Printf family functions

func (entry *Entry) Debugf(format string, args ...interface{}) {
	if entry.Logger.Level >= DebugLevel {
		entry.Debug(fmt.Sprintf(format, args...))
	}
}

func (entry *Entry) Infof(format string, args ...interface{}) {
	if entry.Logger.Level >= InfoLevel {
		entry.Info(fmt.Sprintf(format, args...))
	}
}

func (entry *Entry) Printf(format string, args ...interface{}) {
	entry.Infof(format, args...)
}

func (entry *Entry) Warnf(format string, args ...interface{}) {
	if entry.Logger.Level >= WarnLevel {
		entry.Warn(fmt.Sprintf(format, args...))
	}
}

func (entry *Entry) Warningf(format string, args ...interface{}) {
	entry.Warnf(format, args...)
}

func (entry *Entry) Errorf(format string, args ...interface{}) {
	if entry.Logger.Level >= ErrorLevel {
		entry.Error(fmt.Sprintf(format, args...))
	}
}

func (entry *Entry) Fatalf(format string, args ...interface{}) {
	if entry.Logger.Level >= FatalLevel {
		entry.Fatal(fmt.Sprintf(format, args...))
	}
	Exit(1)
}

func (entry *Entry) Panicf(format string, args ...interface{}) {
	if entry.Logger.Level >= PanicLevel {
		entry.Panic(fmt.Sprintf(format, args...))
	}
}

// Entry Println family functions

func (entry *Entry) Debugln(args ...interface{}) {
	if entry.Logger.Level >= DebugLevel {
		entry.Debug(entry.sprintlnn(args...))
	}
}

func (entry *Entry) Infoln(args ...interface{}) {
	if entry.Logger.Level >= InfoLevel {
		entry.Info(entry.sprintlnn(args...))
	}
}

func (entry *Entry) Println(args ...interface{}) {
	entry.Infoln(args...)
}

func (entry *Entry) Warnln(args ...interface{}) {
	if entry.Logger.Level >= WarnLevel {
		entry.Warn(entry.sprintlnn(args...))
	}
}

func (entry *Entry) Warningln(args ...interface{}) {
	entry.Warnln(args...)
}

func (entry *Entry) Errorln(args ...interface{}) {
	if entry.Logger.Level >= ErrorLevel {
		entry.Error(entry.sprintlnn(args...))
	}
}

func (entry *Entry) Fatalln(args ...interface{}) {
	if entry.Logger.Level >= FatalLevel {
		entry.Fatal(entry.sprintlnn(args...))
	}
	Exit(1)
}

func (entry *Entry) Panicln(args ...interface{}) {
	if entry.Logger.Level >= PanicLevel {
		entry.Panic(entry.sprintlnn(args...))
	}
}

// Sprintlnn => Sprint no newline. This is to get the behavior of how
// fmt.Sprintln where spaces are always added between operands, regardless of
// their type. Instead of vendoring the Sprintln implementation to spare a
// string allocation, we do the simplest thing.
func (entry *Entry) sprintlnn(args ...interface{}) string {
	msg := fmt.Sprintln(args...)
	return msg[:len(msg)-1]
}
<commit_msg>Add *Entry.Bytes() for using byte[] directly<commit_after>package logrus

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"time"
)

// Defines the key when adding errors using WithError.
var ErrorKey = "error"

// An entry is the final or intermediate Logrus logging entry. It contains all
// the fields passed with WithField{,s}. It's finally logged when Debug, Info,
// Warn, Error, Fatal or Panic is called on it. These objects can be reused and
// passed around as much as you wish to avoid field duplication.
type Entry struct {
	Logger *Logger

	// Contains all the fields set by the user.
	Data Fields

	// Time at which the log entry was created
	Time time.Time

	// Level the log entry was logged at: Debug, Info, Warn, Error, Fatal or Panic
	Level Level

	// Message passed to Debug, Info, Warn, Error, Fatal or Panic
	Message string
}

func NewEntry(logger *Logger) *Entry {
	return &Entry{
		Logger: logger,
		// Default is three fields, give a little extra room
		Data: make(Fields, 5),
	}
}

// Returns a reader for the entry, which is a proxy to the formatter.
func (entry *Entry) Reader() (*bytes.Buffer, error) {
	serialized, err := entry.Logger.Formatter.Format(entry)
	return bytes.NewBuffer(serialized), err
}

// Returns a byte[] for the entry, which is a proxy to the formatter.
func (entry *Entry) Bytes() ([]byte, error) {
	return entry.Logger.Formatter.Format(entry)
}

// Returns the string representation from the reader and ultimately the
// formatter.
func (entry *Entry) String() (string, error) {
	reader, err := entry.Reader()
	if err != nil {
		return "", err
	}

	return reader.String(), err
}

// Add an error as single field (using the key defined in ErrorKey) to the Entry.
func (entry *Entry) WithError(err error) *Entry {
	return entry.WithField(ErrorKey, err)
}

// Add a single field to the Entry.
func (entry *Entry) WithField(key string, value interface{}) *Entry {
	return entry.WithFields(Fields{key: value})
}

// Add a map of fields to the Entry.
func (entry *Entry) WithFields(fields Fields) *Entry {
	data := make(Fields, len(entry.Data)+len(fields))
	for k, v := range entry.Data {
		data[k] = v
	}
	for k, v := range fields {
		data[k] = v
	}
	return &Entry{Logger: entry.Logger, Data: data}
}

// This function is not declared with a pointer value because otherwise
// race conditions will occur when using multiple goroutines
func (entry Entry) log(level Level, msg string) {
	entry.Time = time.Now()
	entry.Level = level
	entry.Message = msg

	if err := entry.Logger.Hooks.Fire(level, &entry); err != nil {
		entry.Logger.mu.Lock()
		fmt.Fprintf(os.Stderr, "Failed to fire hook: %v\n", err)
		entry.Logger.mu.Unlock()
	}

	reader, err := entry.Reader()
	if err != nil {
		entry.Logger.mu.Lock()
		fmt.Fprintf(os.Stderr, "Failed to obtain reader, %v\n", err)
		entry.Logger.mu.Unlock()
	}

	entry.Logger.mu.Lock()
	defer entry.Logger.mu.Unlock()

	_, err = io.Copy(entry.Logger.Out, reader)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to write to log, %v\n", err)
	}

	// To avoid Entry#log() returning a value that only would make sense for
	// panic() to use in Entry#Panic(), we avoid the allocation by checking
	// directly here.
	if level <= PanicLevel {
		panic(&entry)
	}
}

func (entry *Entry) Debug(args ...interface{}) {
	if entry.Logger.Level >= DebugLevel {
		entry.log(DebugLevel, fmt.Sprint(args...))
	}
}

func (entry *Entry) Print(args ...interface{}) {
	entry.Info(args...)
}

func (entry *Entry) Info(args ...interface{}) {
	if entry.Logger.Level >= InfoLevel {
		entry.log(InfoLevel, fmt.Sprint(args...))
	}
}

func (entry *Entry) Warn(args ...interface{}) {
	if entry.Logger.Level >= WarnLevel {
		entry.log(WarnLevel, fmt.Sprint(args...))
	}
}

func (entry *Entry) Warning(args ...interface{}) {
	entry.Warn(args...)
}

func (entry *Entry) Error(args ...interface{}) {
	if entry.Logger.Level >= ErrorLevel {
		entry.log(ErrorLevel, fmt.Sprint(args...))
	}
}

func (entry *Entry) Fatal(args ...interface{}) {
	if entry.Logger.Level >= FatalLevel {
		entry.log(FatalLevel, fmt.Sprint(args...))
	}
	Exit(1)
}

func (entry *Entry) Panic(args ...interface{}) {
	if entry.Logger.Level >= PanicLevel {
		entry.log(PanicLevel, fmt.Sprint(args...))
	}
	panic(fmt.Sprint(args...))
}

// Entry Printf family functions

func (entry *Entry) Debugf(format string, args ...interface{}) {
	if entry.Logger.Level >= DebugLevel {
		entry.Debug(fmt.Sprintf(format, args...))
	}
}

func (entry *Entry) Infof(format string, args ...interface{}) {
	if entry.Logger.Level >= InfoLevel {
		entry.Info(fmt.Sprintf(format, args...))
	}
}

func (entry *Entry) Printf(format string, args ...interface{}) {
	entry.Infof(format, args...)
}

func (entry *Entry) Warnf(format string, args ...interface{}) {
	if entry.Logger.Level >= WarnLevel {
		entry.Warn(fmt.Sprintf(format, args...))
	}
}

func (entry *Entry) Warningf(format string, args ...interface{}) {
	entry.Warnf(format, args...)
}

func (entry *Entry) Errorf(format string, args ...interface{}) {
	if entry.Logger.Level >= ErrorLevel {
		entry.Error(fmt.Sprintf(format, args...))
	}
}

func (entry *Entry) Fatalf(format string, args ...interface{}) {
	if entry.Logger.Level >= FatalLevel {
		entry.Fatal(fmt.Sprintf(format, args...))
	}
	Exit(1)
}

func (entry *Entry) Panicf(format string, args ...interface{}) {
	if entry.Logger.Level >= PanicLevel {
		entry.Panic(fmt.Sprintf(format, args...))
	}
}

// Entry Println family functions

func (entry *Entry) Debugln(args ...interface{}) {
	if entry.Logger.Level >= DebugLevel {
		entry.Debug(entry.sprintlnn(args...))
	}
}

func (entry *Entry) Infoln(args ...interface{}) {
	if entry.Logger.Level >= InfoLevel {
		entry.Info(entry.sprintlnn(args...))
	}
}

func (entry *Entry) Println(args ...interface{}) {
	entry.Infoln(args...)
}

func (entry *Entry) Warnln(args ...interface{}) {
	if entry.Logger.Level >= WarnLevel {
		entry.Warn(entry.sprintlnn(args...))
	}
}

func (entry *Entry) Warningln(args ...interface{}) {
	entry.Warnln(args...)
}

func (entry *Entry) Errorln(args ...interface{}) {
	if entry.Logger.Level >= ErrorLevel {
		entry.Error(entry.sprintlnn(args...))
	}
}

func (entry *Entry) Fatalln(args ...interface{}) {
	if entry.Logger.Level >= FatalLevel {
		entry.Fatal(entry.sprintlnn(args...))
	}
	Exit(1)
}

func (entry *Entry) Panicln(args ...interface{}) {
	if entry.Logger.Level >= PanicLevel {
		entry.Panic(entry.sprintlnn(args...))
	}
}

// Sprintlnn => Sprint no newline. This is to get the behavior of how
// fmt.Sprintln where spaces are always added between operands, regardless of
// their type. Instead of vendoring the Sprintln implementation to spare a
// string allocation, we do the simplest thing.
func (entry *Entry) sprintlnn(args ...interface{}) string {
	msg := fmt.Sprintln(args...)
	return msg[:len(msg)-1]
}
<|endoftext|>