<commit_before>// Copyright (C) 2014 Andreas Klauer <Andreas.Klauer@metamorpher.de>
// License: MIT

// Package nbd uses the Linux NBD layer to emulate a block device in user space
package nbd

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"sync"
	"syscall"
)

const (
	// Defined in <linux/fs.h>:
	BLKROSET = 4701

	// Defined in <linux/nbd.h>:
	ioctlSetSock       = 43776
	ioctlSetBlockSize  = 43777
	ioctlDoIt          = 43779
	ioctlClearSock     = 43780
	ioctlClearQueue    = 43781
	ioctlSetSizeBlocks = 43783
	ioctlDisconnect    = 43784
	ioctlSetFlags      = 43786
)

const (
	cmdRead  = 0
	cmdWrite = 1
	cmdDisc  = 2
	cmdFlush = 3
	cmdTrim  = 4
)

const (
	flagSendFlush = (1 << 2) // can flush writeback cache
	flagSendTrim  = (1 << 5) // Send TRIM (discard)
	// flagHasFlags   = (1 << 0) // nbd-server supports flags
	// flagReadOnly   = (1 << 1) // device is read-only
	// flagSendFUA    = (1 << 3) // Send FUA (Force Unit Access)
	// flagRotational = (1 << 4) // Use elevator algorithm - rotational media
)

const (
	magicRequest = 0x25609513
	magicReply   = 0x67446698
	// Do *not* use magics: 0x12560953 0x96744668.
)

const (
	errIO = 5
)

// ioctl() helper function
func ioctl(a1, a2, a3 uintptr) (err error) {
	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, a1, a2, a3)
	if errno != 0 {
		err = errno
	}
	return err
}

// Device interface is a subset of os.File.
type Device interface {
	ReadAt(b []byte, off int64) (n int, err error)
	WriteAt(b []byte, off int64) (n int, err error)
	Sync() error
	Trim(off, len int64) error
}

type NBD struct {
	device    Device
	size      int64
	blocksize int64
	nbd       *os.File
	socket    int
	setsocket int
	closer    chan error
}

func Create(device Device, size int64, blocksize int64) *NBD {
	if size >= 0 {
		return &NBD{
			device:    device,
			size:      size,
			blocksize: blocksize,
			nbd:       nil,
			socket:    0,
		}
	}
	return nil
}

// return true if connected
func (nbd *NBD) IsConnected() bool {
	return nbd.nbd != nil && nbd.socket > 0
}

func (nbd *NBD) Size() int64 {
	return nbd.size
}

func (nbd *NBD) SetSize(size, blocksize int64) error {
	if err := ioctl(nbd.nbd.Fd(), ioctlSetBlockSize, uintptr(blocksize)); err != nil {
		return &os.PathError{
			Path: nbd.nbd.Name(),
			Op:   "ioctl NBD_SET_BLKSIZE",
			Err:  err,
		}
	}
	if err := ioctl(nbd.nbd.Fd(), ioctlSetSizeBlocks, uintptr(size/blocksize)); err != nil {
		return &os.PathError{
			Path: nbd.nbd.Name(),
			Op:   "ioctl NBD_SET_SIZE_BLOCKS",
			Err:  err,
		}
	}
	return nil
}

func FindDevice() (string, error) {
	// FIXME: Oh god... fixme.
	// find free nbd device
	for i := 0; ; i++ {
		dev := fmt.Sprintf("/dev/nbd%d", i)
		if _, err := os.Stat(dev); os.IsNotExist(err) {
			break // no more devices
		}
		if _, err := os.Stat(fmt.Sprintf("/sys/block/nbd%d/pid", i)); !os.IsNotExist(err) {
			continue // busy
		}
		return dev, nil
	}
	return "", errors.New("no devices available")
}

func (nbd *NBD) OpenDevice(dev string) (string, error) {
	f, err := os.Open(dev)
	if err != nil {
		return "", err
	}
	nbd.nbd = f

	// possible candidate
	ioctl(f.Fd(), BLKROSET, 0) // I'm really sorry about this
	pair, err := syscall.Socketpair(syscall.SOCK_STREAM, syscall.AF_UNIX, 0)
	if err != nil {
		return "", err
	}
	if err := ioctl(f.Fd(), ioctlSetSock, uintptr(pair[0])); err != nil {
		return "", err
	}

	nbd.setsocket = pair[0] // FIXME: We shouldn't hold on to this.
	nbd.socket = pair[1]
	return dev, nil
}

func (nbd *NBD) Serve() error {
	if err := nbd.SetSize(nbd.size, nbd.blocksize); err != nil {
		return err // already set by nbd.Size()
	}
	if err := ioctl(nbd.nbd.Fd(), ioctlSetFlags, uintptr(flagSendFlush|flagSendTrim)); err != nil {
		return &os.PathError{
			Path: nbd.nbd.Name(),
			Op:   "ioctl NBD_SET_FLAGS",
			Err:  err,
		}
	}

	c := &serverConn{
		rw: os.NewFile(uintptr(nbd.socket), "<nbd socket>"),
	}
	n := runtime.GOMAXPROCS(0)

	wg := new(sync.WaitGroup)
	wg.Add(n)
	for i := 0; i < n; i++ {
		go c.serveLoop(nbd.device, wg)
	}
	if err := waitDisconnect(nbd.nbd); err != nil {
		return err
	}
	wg.Wait()
	return nil
}

func (nbd *NBD) Close() error {
	// FIXME: Now I understand what these did, and it terrifies me.
	// syscall.Write(nbd.setsocket, make([]byte, 28))
	// syscall.Close(nbd.setsocket)
	nbd.Disconnect()
	syscall.Close(nbd.socket)
	return nbd.nbd.Close()
}

func (nbd *NBD) Disconnect() error {
	if nbd.nbd == nil {
		return nil
	}
	if err := ioctl(nbd.nbd.Fd(), ioctlDisconnect, 0); err != nil {
		return &os.PathError{
			Path: nbd.nbd.Name(),
			Op:   "ioctl NBD_DISCONNECT",
			Err:  err,
		}
	}
	return nil
}

func waitDisconnect(f *os.File) error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	// NBD_DO_IT does not return until disconnect
	if err := ioctl(f.Fd(), ioctlDoIt, 0); err != nil {
		return &os.PathError{
			Path: f.Name(),
			Op:   "ioctl NBD_DO_IT",
			Err:  err,
		}
	}
	if err := ioctl(f.Fd(), ioctlClearQueue, 0); err != nil {
		return &os.PathError{
			Path: f.Name(),
			Op:   "ioctl NBD_CLEAR_QUEUE",
			Err:  err,
		}
	}
	if err := ioctl(f.Fd(), ioctlClearSock, 0); err != nil {
		return &os.PathError{
			Path: f.Name(),
			Op:   "ioctl NBD_CLEAR_SOCK",
			Err:  err,
		}
	}
	return nil
}

type serverConn struct {
	mu sync.Mutex
	rw io.ReadWriteCloser
}

func (c *serverConn) serveLoop(dev Device, wg *sync.WaitGroup) error {
	if wg != nil {
		defer wg.Done()
	}

	hdr := new(reqHeader)
	buf := make([]byte, 16) // FIXME: Maybe this should be hdr[:16] instead?
	for {
		c.mu.Lock()

		if _, err := io.ReadFull(c.rw, hdr[:]); err != nil {
			// FIXME: Are there any valid short reads we need to handle?
			c.mu.Unlock()
			return err
		}

		if magic := hdr.magic(); magic != magicRequest {
			c.mu.Unlock()
			return fmt.Errorf("nbd: invalid magic: 0x%x", magic)
		}

		cmd, _ := hdr.command()
		if cmd == cmdWrite {
			buf = hdr.resize(buf)
			if _, err := io.ReadFull(c.rw, buf[16:]); err != nil {
				return err
			}
		}
		c.mu.Unlock()

		switch cmd {
		case cmdRead:
			buf = hdr.resize(buf)
			if _, err := dev.ReadAt(buf[16:], hdr.offset()); err != nil {
				hdr.putReplyHeader(buf, errIO)
			} else {
				hdr.putReplyHeader(buf, 0)
			}
		case cmdWrite:
			if _, err := dev.WriteAt(buf[16:], hdr.offset()); err != nil {
				hdr.putReplyHeader(buf, errIO)
			} else {
				hdr.putReplyHeader(buf, 0)
			}
			buf = buf[:16]
		case cmdTrim:
			if err := dev.Trim(hdr.offset(), int64(hdr.length())); err != nil {
				log.Printf("nbd: trim error: %s", err)
			}
			fallthrough
		case cmdFlush:
			if err := dev.Sync(); err != nil {
				log.Printf("nbd: sync error: %s", err)
			}
			hdr.putReplyHeader(buf, 0)
			buf = buf[:16]
		case cmdDisc:
			// FIXME: We're actually supposed to wait for outstanding requests to finish.
			if err := dev.Sync(); err != nil {
				log.Printf("nbd: sync error: %s", err)
			}
			return c.rw.Close()
		default:
			return errors.New("nbd: invalid command")
		}

		// FIXME: Are we sure this whole write is going to be atomic?
		if _, err := c.rw.Write(buf); err != nil {
			return err
		}
	}
}

type reqHeader [28]byte

func (h *reqHeader) command() (cmd, flags uint16) {
	return binary.BigEndian.Uint16(h[6:8]), binary.BigEndian.Uint16(h[4:6])
}

func (h *reqHeader) handle() uint64 {
	return binary.BigEndian.Uint64(h[8:16])
}

func (h *reqHeader) length() uint32 {
	return binary.BigEndian.Uint32(h[24:28])
}

func (h *reqHeader) magic() uint32 {
	return binary.BigEndian.Uint32(h[0:4])
}

func (h *reqHeader) offset() int64 {
	const maxInt64 = (^uint64(0) >> 1) + 1
	if u := binary.BigEndian.Uint64(h[16:24]); u <= maxInt64 {
		return int64(u)
	}
	panic("nbd: offset overflow")
}

func (h *reqHeader) resize(buf []byte) []byte {
	if n := int(h.length()) + 16; n > cap(buf) {
		return make([]byte, n)
	} else {
		return buf[:n]
	}
}

func (h *reqHeader) putReplyHeader(dst []byte, err uint32) {
	binary.BigEndian.PutUint32(dst[0:4], magicReply)
	binary.BigEndian.PutUint32(dst[4:8], err)
	copy(dst[8:16], h[8:16])
}
<commit_msg>add TODO for handling multithreading<commit_after>// Copyright (C) 2014 Andreas Klauer <Andreas.Klauer@metamorpher.de>
// License: MIT

// Package nbd uses the Linux NBD layer to emulate a block device in user space
package nbd

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"sync"
	"syscall"
)

const (
	// Defined in <linux/fs.h>:
	BLKROSET = 4701

	// Defined in <linux/nbd.h>:
	ioctlSetSock       = 43776
	ioctlSetBlockSize  = 43777
	ioctlDoIt          = 43779
	ioctlClearSock     = 43780
	ioctlClearQueue    = 43781
	ioctlSetSizeBlocks = 43783
	ioctlDisconnect    = 43784
	ioctlSetFlags      = 43786
)

const (
	cmdRead  = 0
	cmdWrite = 1
	cmdDisc  = 2
	cmdFlush = 3
	cmdTrim  = 4
)

const (
	flagSendFlush = (1 << 2) // can flush writeback cache
	flagSendTrim  = (1 << 5) // Send TRIM (discard)
	// flagHasFlags   = (1 << 0) // nbd-server supports flags
	// flagReadOnly   = (1 << 1) // device is read-only
	// flagSendFUA    = (1 << 3) // Send FUA (Force Unit Access)
	// flagRotational = (1 << 4) // Use elevator algorithm - rotational media
)

const (
	magicRequest = 0x25609513
	magicReply   = 0x67446698
	// Do *not* use magics: 0x12560953 0x96744668.
)

const (
	errIO = 5
)

// ioctl() helper function
func ioctl(a1, a2, a3 uintptr) (err error) {
	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, a1, a2, a3)
	if errno != 0 {
		err = errno
	}
	return err
}

// Device interface is a subset of os.File.
type Device interface {
	ReadAt(b []byte, off int64) (n int, err error)
	WriteAt(b []byte, off int64) (n int, err error)
	Sync() error
	Trim(off, len int64) error
}

type NBD struct {
	device    Device
	size      int64
	blocksize int64
	nbd       *os.File
	socket    int
	setsocket int
	closer    chan error
}

func Create(device Device, size int64, blocksize int64) *NBD {
	if size >= 0 {
		return &NBD{
			device:    device,
			size:      size,
			blocksize: blocksize,
			nbd:       nil,
			socket:    0,
		}
	}
	return nil
}

// return true if connected
func (nbd *NBD) IsConnected() bool {
	return nbd.nbd != nil && nbd.socket > 0
}

func (nbd *NBD) Size() int64 {
	return nbd.size
}

func (nbd *NBD) SetSize(size, blocksize int64) error {
	if err := ioctl(nbd.nbd.Fd(), ioctlSetBlockSize, uintptr(blocksize)); err != nil {
		return &os.PathError{
			Path: nbd.nbd.Name(),
			Op:   "ioctl NBD_SET_BLKSIZE",
			Err:  err,
		}
	}
	if err := ioctl(nbd.nbd.Fd(), ioctlSetSizeBlocks, uintptr(size/blocksize)); err != nil {
		return &os.PathError{
			Path: nbd.nbd.Name(),
			Op:   "ioctl NBD_SET_SIZE_BLOCKS",
			Err:  err,
		}
	}
	return nil
}

func FindDevice() (string, error) {
	// FIXME: Oh god... fixme.
	// find free nbd device
	for i := 0; ; i++ {
		dev := fmt.Sprintf("/dev/nbd%d", i)
		if _, err := os.Stat(dev); os.IsNotExist(err) {
			break // no more devices
		}
		if _, err := os.Stat(fmt.Sprintf("/sys/block/nbd%d/pid", i)); !os.IsNotExist(err) {
			continue // busy
		}
		return dev, nil
	}
	return "", errors.New("no devices available")
}

func (nbd *NBD) OpenDevice(dev string) (string, error) {
	f, err := os.Open(dev)
	if err != nil {
		return "", err
	}
	nbd.nbd = f

	// possible candidate
	ioctl(f.Fd(), BLKROSET, 0) // I'm really sorry about this
	pair, err := syscall.Socketpair(syscall.SOCK_STREAM, syscall.AF_UNIX, 0)
	if err != nil {
		return "", err
	}
	if err := ioctl(f.Fd(), ioctlSetSock, uintptr(pair[0])); err != nil {
		return "", err
	}

	nbd.setsocket = pair[0] // FIXME: We shouldn't hold on to this.
	nbd.socket = pair[1]
	return dev, nil
}

func (nbd *NBD) Serve() error {
	if err := nbd.SetSize(nbd.size, nbd.blocksize); err != nil {
		return err // already set by nbd.Size()
	}
	if err := ioctl(nbd.nbd.Fd(), ioctlSetFlags, uintptr(flagSendFlush|flagSendTrim)); err != nil {
		return &os.PathError{
			Path: nbd.nbd.Name(),
			Op:   "ioctl NBD_SET_FLAGS",
			Err:  err,
		}
	}

	c := &serverConn{
		rw: os.NewFile(uintptr(nbd.socket), "<nbd socket>"),
	}
	// TODO(barakmich): Scale up NBD by handling multiple requests.
	// Requires thread-safety across the block.BlockFile/agro.File
	//n := runtime.GOMAXPROCS(0) - 1
	n := 1

	wg := new(sync.WaitGroup)
	wg.Add(n)
	for i := 0; i < n; i++ {
		go c.serveLoop(nbd.device, wg)
	}
	if err := waitDisconnect(nbd.nbd); err != nil {
		return err
	}
	wg.Wait()
	return nil
}

func (nbd *NBD) Close() error {
	// FIXME: Now I understand what these did, and it terrifies me.
	// syscall.Write(nbd.setsocket, make([]byte, 28))
	// syscall.Close(nbd.setsocket)
	nbd.Disconnect()
	syscall.Close(nbd.socket)
	return nbd.nbd.Close()
}

func (nbd *NBD) Disconnect() error {
	if nbd.nbd == nil {
		return nil
	}
	if err := ioctl(nbd.nbd.Fd(), ioctlDisconnect, 0); err != nil {
		return &os.PathError{
			Path: nbd.nbd.Name(),
			Op:   "ioctl NBD_DISCONNECT",
			Err:  err,
		}
	}
	return nil
}

func waitDisconnect(f *os.File) error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	// NBD_DO_IT does not return until disconnect
	if err := ioctl(f.Fd(), ioctlDoIt, 0); err != nil {
		return &os.PathError{
			Path: f.Name(),
			Op:   "ioctl NBD_DO_IT",
			Err:  err,
		}
	}
	if err := ioctl(f.Fd(), ioctlClearQueue, 0); err != nil {
		return &os.PathError{
			Path: f.Name(),
			Op:   "ioctl NBD_CLEAR_QUEUE",
			Err:  err,
		}
	}
	if err := ioctl(f.Fd(), ioctlClearSock, 0); err != nil {
		return &os.PathError{
			Path: f.Name(),
			Op:   "ioctl NBD_CLEAR_SOCK",
			Err:  err,
		}
	}
	return nil
}

type serverConn struct {
	mu sync.Mutex
	rw io.ReadWriteCloser
}

func (c *serverConn) serveLoop(dev Device, wg *sync.WaitGroup) error {
	if wg != nil {
		defer wg.Done()
	}

	hdr := new(reqHeader)
	buf := make([]byte, 16) // FIXME: Maybe this should be hdr[:16] instead?
	for {
		c.mu.Lock()

		if _, err := io.ReadFull(c.rw, hdr[:]); err != nil {
			// FIXME: Are there any valid short reads we need to handle?
			c.mu.Unlock()
			return err
		}

		if magic := hdr.magic(); magic != magicRequest {
			c.mu.Unlock()
			return fmt.Errorf("nbd: invalid magic: 0x%x", magic)
		}

		cmd, _ := hdr.command()
		if cmd == cmdWrite {
			buf = hdr.resize(buf)
			if _, err := io.ReadFull(c.rw, buf[16:]); err != nil {
				return err
			}
		}
		c.mu.Unlock()

		switch cmd {
		case cmdRead:
			buf = hdr.resize(buf)
			if _, err := dev.ReadAt(buf[16:], hdr.offset()); err != nil {
				hdr.putReplyHeader(buf, errIO)
			} else {
				hdr.putReplyHeader(buf, 0)
			}
		case cmdWrite:
			if _, err := dev.WriteAt(buf[16:], hdr.offset()); err != nil {
				hdr.putReplyHeader(buf, errIO)
			} else {
				hdr.putReplyHeader(buf, 0)
			}
			buf = buf[:16]
		case cmdTrim:
			if err := dev.Trim(hdr.offset(), int64(hdr.length())); err != nil {
				log.Printf("nbd: trim error: %s", err)
			}
			fallthrough
		case cmdFlush:
			if err := dev.Sync(); err != nil {
				log.Printf("nbd: sync error: %s", err)
			}
			hdr.putReplyHeader(buf, 0)
			buf = buf[:16]
		case cmdDisc:
			// FIXME: We're actually supposed to wait for outstanding requests to finish.
			if err := dev.Sync(); err != nil {
				log.Printf("nbd: sync error: %s", err)
			}
			return c.rw.Close()
		default:
			return errors.New("nbd: invalid command")
		}

		// FIXME: Are we sure this whole write is going to be atomic?
		if _, err := c.rw.Write(buf); err != nil {
			return err
		}
	}
}

type reqHeader [28]byte

func (h *reqHeader) command() (cmd, flags uint16) {
	return binary.BigEndian.Uint16(h[6:8]), binary.BigEndian.Uint16(h[4:6])
}

func (h *reqHeader) handle() uint64 {
	return binary.BigEndian.Uint64(h[8:16])
}

func (h *reqHeader) length() uint32 {
	return binary.BigEndian.Uint32(h[24:28])
}

func (h *reqHeader) magic() uint32 {
	return binary.BigEndian.Uint32(h[0:4])
}

func (h *reqHeader) offset() int64 {
	const maxInt64 = (^uint64(0) >> 1) + 1
	if u := binary.BigEndian.Uint64(h[16:24]); u <= maxInt64 {
		return int64(u)
	}
	panic("nbd: offset overflow")
}

func (h *reqHeader) resize(buf []byte) []byte {
	if n := int(h.length()) + 16; n > cap(buf) {
		return make([]byte, n)
	} else {
		return buf[:n]
	}
}

func (h *reqHeader) putReplyHeader(dst []byte, err uint32) {
	binary.BigEndian.PutUint32(dst[0:4], magicReply)
	binary.BigEndian.PutUint32(dst[4:8], err)
	copy(dst[8:16], h[8:16])
}
<|endoftext|>


<commit_before>package sdp

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/pions/webrtc/pkg/ice"
)

// ICECandidateUnmarshal takes a candidate strings and returns a ice.Candidate or nil if it fails to parse
func ICECandidateUnmarshal(raw string) ice.Candidate {
	split := strings.Fields(raw)
	if len(split) < 8 {
		fmt.Printf("Attribute not long enough to be ICE candidate (%d) %s \n", len(split), raw)
		return nil
	}

	getValue := func(key string) string {
		rtrnNext := false
		for _, i := range split {
			if rtrnNext {
				return i
			} else if i == key {
				rtrnNext = true
			}
		}
		return ""
	}

	port, err := strconv.Atoi(split[5])
	if err != nil {
		return nil
	}

	// TODO verify valid address
	address := split[4]

	switch getValue("typ") {
	case "host":
		return &ice.CandidateHost{
			CandidateBase: ice.CandidateBase{
				Protocol: ice.TransportUDP,
				Address:  address,
				Port:     port,
			},
		}
	default:
		return nil
	}
}

func iceSrflxCandidateString(c *ice.CandidateSrflx, component int) string {
	return fmt.Sprintf("udpcandidate %d udp %d %s %d typ srflx raddr %s rport %d generation 0",
		component, c.CandidateBase.Priority(ice.SrflxCandidatePreference, uint16(component)), c.CandidateBase.Address, c.CandidateBase.Port, c.RemoteAddress, c.RemotePort)
}

func iceHostCandidateString(c *ice.CandidateHost, component int) string {
	return fmt.Sprintf("udpcandidate %d udp %d %s %d typ host generation 0",
		component, c.CandidateBase.Priority(ice.HostCandidatePreference, uint16(component)), c.CandidateBase.Address, c.CandidateBase.Port)
}

// ICECandidateMarshal takes a candidate and returns a string representation
func ICECandidateMarshal(c ice.Candidate) []string {
	out := make([]string, 0)

	switch c := c.(type) {
	case *ice.CandidateSrflx:
		out = append(out, iceSrflxCandidateString(c, 1))
		out = append(out, iceSrflxCandidateString(c, 2))
	case *ice.CandidateHost:
		out = append(out, iceHostCandidateString(c, 1))
		out = append(out, iceHostCandidateString(c, 2))
	}

	return out
}
<commit_msg>Parse srflx candidates in SDP<commit_after>package sdp

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/pions/webrtc/pkg/ice"
)

// ICECandidateUnmarshal takes a candidate strings and returns a ice.Candidate or nil if it fails to parse
func ICECandidateUnmarshal(raw string) ice.Candidate {
	split := strings.Fields(raw)
	if len(split) < 8 {
		fmt.Printf("Attribute not long enough to be ICE candidate (%d) %s \n", len(split), raw)
		return nil
	}

	getValue := func(key string) string {
		rtrnNext := false
		for _, i := range split {
			if rtrnNext {
				return i
			} else if i == key {
				rtrnNext = true
			}
		}
		return ""
	}

	port, err := strconv.Atoi(split[5])
	if err != nil {
		return nil
	}

	// TODO verify valid address
	address := split[4]

	switch getValue("typ") {
	case "host":
		return &ice.CandidateHost{
			CandidateBase: ice.CandidateBase{
				Protocol: ice.TransportUDP,
				Address:  address,
				Port:     port,
			},
		}
	case "srflx":
		return &ice.CandidateSrflx{
			CandidateBase: ice.CandidateBase{
				Protocol: ice.TransportUDP,
				Address:  address,
				Port:     port,
			},
		}
	default:
		return nil
	}
}

func iceSrflxCandidateString(c *ice.CandidateSrflx, component int) string {
	return fmt.Sprintf("udpcandidate %d udp %d %s %d typ srflx raddr %s rport %d generation 0",
		component, c.CandidateBase.Priority(ice.SrflxCandidatePreference, uint16(component)), c.CandidateBase.Address, c.CandidateBase.Port, c.RemoteAddress, c.RemotePort)
}

func iceHostCandidateString(c *ice.CandidateHost, component int) string {
	return fmt.Sprintf("udpcandidate %d udp %d %s %d typ host generation 0",
		component, c.CandidateBase.Priority(ice.HostCandidatePreference, uint16(component)), c.CandidateBase.Address, c.CandidateBase.Port)
}

// ICECandidateMarshal takes a candidate and returns a string representation
func ICECandidateMarshal(c ice.Candidate) []string {
	out := make([]string, 0)

	switch c := c.(type) {
	case *ice.CandidateSrflx:
		out = append(out, iceSrflxCandidateString(c, 1))
		out = append(out, iceSrflxCandidateString(c, 2))
	case *ice.CandidateHost:
		out = append(out, iceHostCandidateString(c, 1))
		out = append(out, iceHostCandidateString(c, 2))
	}

	return out
}
<|endoftext|>


<commit_before>package TF2RconWrapper

import (
	"errors"
	"regexp"
	"time"
)

const (
	// "Username<userId><steamId><Team>"
	// "1<2><3><4>" <- regex group
	logLineStart = `^"(.*)<(\d+)><(\[U:1:\d+\])><(\w+)>" `
	// "5" <- regex group
	logLineEnd = ` "(.*)"`

	logLineStartSpec = `^"(.*)<(\d+)><(\[U:1:\d+\])><(\w*)>" `
)

// regexes used in the parser
var (
	rPlayerGlobalMessage = regexp.MustCompile(logLineStart + `say` + logLineEnd)
	rPlayerChangedClass  = regexp.MustCompile(logLineStart + `changed role to` + logLineEnd)
	rPlayerTeamMessage   = regexp.MustCompile(logLineStart + `say_team` + logLineEnd)
	rPlayerChangedTeam   = regexp.MustCompile(logLineStart + `joined team` + logLineEnd)

	//Global Messages
	rPlayerConnected    = regexp.MustCompile(logLineStartSpec + `connected, address "\d+.\d+.\d+.\d+\:\d+"`)
	rPlayerDisconnected = regexp.MustCompile(logLineStartSpec + `disconnected \(reason "(.*)"\)`)
	rGameOver           = regexp.MustCompile(`^World triggered "Game_Over" reason "(.*)"`)
	rServerCvar         = regexp.MustCompile(`^server_cvar: "(.*)" "(.*)"`)
)

//LogMessage represents a log message in a TF2 server, and contains a timestamp
//and a message. The message can be a player message that contains the sender's
//username, steamid and other info or a server message.
type LogMessage struct {
	Timestamp time.Time
	Message   string
	Parsed    ParsedMsg
}

//When a player says something in the global chat, or when they join the game
type PlayerData struct {
	Username string

	SteamId string
	UserId  string

	Team    string
	NewTeam string

	Text  string
	Class string
}

type CvarData struct {
	Variable string
	Value    string
}

const (
	PlayerGlobalMessage = iota
	PlayerTeamMessage   = iota
	PlayerChangedClass  = iota
	PlayerChangedTeam   = iota

	WorldPlayerConnected    = iota
	WorldPlayerDisconnected = iota
	WorldGameOver           = iota
	ServerCvar              = iota
)

type ParsedMsg struct {
	Type int
	Data interface{}
}

var (
	ErrInvalidPacket = errors.New("Invalid Packet")
)

func getSecret(data []byte) (string, error) {
	if !(len(data) > 6) {
		return "", ErrInvalidPacket
	}

	if data[4] != 0x53 {
		return "", errors.New("Server trying to send a chat packet without a secret")
	}

	bytes := data[5:]
	var pos int

	for bytes[pos] != 0x20 {
		pos++
		if pos >= len(bytes) {
			return "", ErrInvalidPacket
		}
	}

	secret := string(bytes[:pos-1])
	if pos+1 >= len(data) {
		//No message/time data
		return "", ErrInvalidPacket
	}

	return secret, nil
}

func ParseMessage(raw RawMessage) (LogMessage, error) {
	textBytes := raw.data[0:raw.n]
	if len(textBytes) <= 24 {
		return LogMessage{}, ErrInvalidPacket
	}

	packetType := textBytes[4]

	if packetType != 0x53 {
		return LogMessage{}, errors.New("Server trying to send a chat packet without a secret")
	}

	// drop the header
	textBytes = textBytes[5:]

	pos := 0
	for textBytes[pos] != 0x20 {
		pos++
	}

	textBytes = textBytes[pos+1:]

	text := string(textBytes)

	timeText := text[:21]
	message := text[23:]

	const refTime = "01/02/2006 -  15:04:05"

	timeObj, _ := time.Parse(refTime, timeText)

	m := parse(message)

	return LogMessage{timeObj, text, m}, nil
}

func parse(message string) ParsedMsg {
	r := ParsedMsg{Type: -1}
	var m []string

	isPlayerMessage := false
	playerData := PlayerData{}

	switch {
	case rPlayerGlobalMessage.MatchString(message):
		m = rPlayerGlobalMessage.FindStringSubmatch(message)

		isPlayerMessage = true
		playerData.Text = m[5]
		r.Type = PlayerGlobalMessage

	case rPlayerTeamMessage.MatchString(message):
		m = rPlayerTeamMessage.FindStringSubmatch(message)

		isPlayerMessage = true
		playerData.Text = m[5]
		r.Type = PlayerTeamMessage

	case rPlayerChangedClass.MatchString(message):
		m = rPlayerChangedClass.FindStringSubmatch(message)

		isPlayerMessage = true
		playerData.Class = m[5]
		r.Type = PlayerChangedClass

	case rPlayerChangedTeam.MatchString(message):
		m = rPlayerChangedTeam.FindStringSubmatch(message)

		isPlayerMessage = true
		playerData.NewTeam = m[5]
		r.Type = PlayerChangedTeam

	case rPlayerConnected.MatchString(message):
		m = rPlayerConnected.FindStringSubmatch(message)

		isPlayerMessage = true
		r.Type = WorldPlayerConnected

	case rPlayerDisconnected.MatchString(message):
		m = rPlayerDisconnected.FindStringSubmatch(message)

		isPlayerMessage = true
		r.Type = WorldPlayerDisconnected

	// Non-Player Messages
	case rGameOver.MatchString(message):
		r.Type = WorldGameOver

	case rServerCvar.MatchString(message):
		m = rServerCvar.FindStringSubmatch(message)

		r.Type = ServerCvar
		r.Data = CvarData{Variable: m[1], Value: m[2]}
	}

	// fields used in all matches
	if isPlayerMessage {
		playerData.Username = m[1]
		playerData.SteamId = m[3]
		playerData.UserId = m[2]
		if r.Type != WorldPlayerDisconnected {
			playerData.Team = m[4]
		}
		r.Data = playerData
	}

	return r
}
<commit_msg>Add Event LogFileClosed<commit_after>package TF2RconWrapper

import (
	"errors"
	"regexp"
	"time"
)

const (
	// "Username<userId><steamId><Team>"
	// "1<2><3><4>" <- regex group
	logLineStart = `^"(.*)<(\d+)><(\[U:1:\d+\])><(\w+)>" `
	// "5" <- regex group
	logLineEnd = ` "(.*)"`

	logLineStartSpec = `^"(.*)<(\d+)><(\[U:1:\d+\])><(\w*)>" `
)

// regexes used in the parser
var (
	rPlayerGlobalMessage = regexp.MustCompile(logLineStart + `say` + logLineEnd)
	rPlayerChangedClass  = regexp.MustCompile(logLineStart + `changed role to` + logLineEnd)
	rPlayerTeamMessage   = regexp.MustCompile(logLineStart + `say_team` + logLineEnd)
	rPlayerChangedTeam   = regexp.MustCompile(logLineStart + `joined team` + logLineEnd)

	//Global Messages
	rPlayerConnected    = regexp.MustCompile(logLineStartSpec + `connected, address "\d+.\d+.\d+.\d+\:\d+"`)
	rPlayerDisconnected = regexp.MustCompile(logLineStartSpec + `disconnected \(reason "(.*)"\)`)
	rGameOver           = regexp.MustCompile(`^World triggered "Game_Over" reason "(.*)"`)
	rServerCvar         = regexp.MustCompile(`^server_cvar: "(.*)" "(.*)"`)

	rLogFiledClosed = regexp.MustCompile("^Log file closed.")
)

//LogMessage represents a log message in a TF2 server, and contains a timestamp
//and a message. The message can be a player message that contains the sender's
//username, steamid and other info or a server message.
type LogMessage struct {
	Timestamp time.Time
	Message   string
	Parsed    ParsedMsg
}

//When a player says something in the global chat, or when they join the game
type PlayerData struct {
	Username string

	SteamId string
	UserId  string

	Team    string
	NewTeam string

	Text  string
	Class string
}

type CvarData struct {
	Variable string
	Value    string
}

const (
	PlayerGlobalMessage = iota
	PlayerTeamMessage   = iota
	PlayerChangedClass  = iota
	PlayerChangedTeam   = iota

	WorldPlayerConnected    = iota
	WorldPlayerDisconnected = iota
	WorldGameOver           = iota
	ServerCvar              = iota

	LogFileClosed = iota
)

type ParsedMsg struct {
	Type int
	Data interface{}
}

var (
	ErrInvalidPacket = errors.New("Invalid Packet")
)

func getSecret(data []byte) (string, error) {
	if !(len(data) > 6) {
		return "", ErrInvalidPacket
	}

	if data[4] != 0x53 {
		return "", errors.New("Server trying to send a chat packet without a secret")
	}

	bytes := data[5:]
	var pos int

	for bytes[pos] != 0x20 {
		pos++
		if pos >= len(bytes) {
			return "", ErrInvalidPacket
		}
	}

	secret := string(bytes[:pos-1])
	if pos+1 >= len(data) {
		//No message/time data
		return "", ErrInvalidPacket
	}

	return secret, nil
}

func ParseMessage(raw RawMessage) (LogMessage, error) {
	textBytes := raw.data[0:raw.n]
	if len(textBytes) <= 24 {
		return LogMessage{}, ErrInvalidPacket
	}

	packetType := textBytes[4]

	if packetType != 0x53 {
		return LogMessage{}, errors.New("Server trying to send a chat packet without a secret")
	}

	// drop the header
	textBytes = textBytes[5:]

	pos := 0
	for textBytes[pos] != 0x20 {
		pos++
	}

	textBytes = textBytes[pos+1:]

	text := string(textBytes)

	timeText := text[:21]
	message := text[23:]

	const refTime = "01/02/2006 -  15:04:05"

	timeObj, _ := time.Parse(refTime, timeText)

	m := parse(message)

	return LogMessage{timeObj, text, m}, nil
}

func parse(message string) ParsedMsg {
	r := ParsedMsg{Type: -1}
	var m []string

	isPlayerMessage := false
	playerData := PlayerData{}

	switch {
	case rPlayerGlobalMessage.MatchString(message):
		m = rPlayerGlobalMessage.FindStringSubmatch(message)

		isPlayerMessage = true
		playerData.Text = m[5]
		r.Type = PlayerGlobalMessage

	case rPlayerTeamMessage.MatchString(message):
		m = rPlayerTeamMessage.FindStringSubmatch(message)

		isPlayerMessage = true
		playerData.Text = m[5]
		r.Type = PlayerTeamMessage

	case rPlayerChangedClass.MatchString(message):
		m = rPlayerChangedClass.FindStringSubmatch(message)

		isPlayerMessage = true
		playerData.Class = m[5]
		r.Type = PlayerChangedClass

	case rPlayerChangedTeam.MatchString(message):
		m = rPlayerChangedTeam.FindStringSubmatch(message)

		isPlayerMessage = true
		playerData.NewTeam = m[5]
		r.Type = PlayerChangedTeam

	case rPlayerConnected.MatchString(message):
		m = rPlayerConnected.FindStringSubmatch(message)

		isPlayerMessage = true
		r.Type = WorldPlayerConnected

	case rPlayerDisconnected.MatchString(message):
		m = rPlayerDisconnected.FindStringSubmatch(message)

		isPlayerMessage = true
		r.Type = WorldPlayerDisconnected

	// Non-Player Messages
	case rGameOver.MatchString(message):
		r.Type = WorldGameOver

	case rServerCvar.MatchString(message):
		m = rServerCvar.FindStringSubmatch(message)

		r.Type = ServerCvar
		r.Data = CvarData{Variable: m[1], Value: m[2]}

	case rLogFiledClosed.MatchString(message):
		r.Type = LogFileClosed
	}

	// fields used in all matches
	if isPlayerMessage {
		playerData.Username = m[1]
		playerData.SteamId = m[3]
		playerData.UserId = m[2]
		if r.Type != WorldPlayerDisconnected {
			playerData.Team = m[4]
		}
		r.Data = playerData
	}

	return r
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

func fib(n int) int {
	fib := []int{0, 1}
	for i := 2; i <= n; i++ {
		n := fib[i-1] + fib[i-2]
		fib = append(fib, n)
	}
	return fib[n]
}

func main() {
	// 0, 1, 1, 2, 3, 5, 8, 13, 21, 34
	fmt.Println(fib(4))
}
<commit_msg>sync Wed May 15 21:41:17 CEST 2019<commit_after>package main

import (
	"fmt"
)

func fib(n int) int {
	seq := []int{0, 1}
	for i := 2; i <= n; i++ {
		n := seq[i-1] + seq[i-2]
		seq = append(seq, n)
	}
	return seq[n]
}

func main() {
	// 0, 1, 1, 2, 3, 5, 8, 13, 21, 34
	fmt.Println(fib(4))
}
<|endoftext|>


<commit_before>package check

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
)

var (
	skipDirs     = []string{"Godeps", "vendor", "third_party"}
	skipSuffixes = []string{".pb.go", ".pb.gw.go", ".generated.go"}
)

func addSkipDirs(params []string) []string {
	for _, dir := range skipDirs {
		params = append(params, fmt.Sprintf("--skip=%s", dir))
	}
	return params
}

// GoFiles returns a slice of Go filenames
// in a given directory.
func GoFiles(dir string) ([]string, error) {
	var filenames []string
	visit := func(fp string, fi os.FileInfo, err error) error {
		for _, skip := range skipDirs {
			if strings.Contains(fp, skip) {
				return nil
			}
		}
		if err != nil {
			fmt.Println(err) // can't walk here,
			return nil       // but continue walking elsewhere
		}
		if fi.IsDir() {
			return nil // not a file.  ignore.
		}
		fiName := fi.Name()
		for _, skip := range skipSuffixes {
			if strings.HasSuffix(fiName, skip) {
				return nil
			}
		}
		ext := filepath.Ext(fiName)
		if ext == ".go" {
			filenames = append(filenames, fp)
		}
		return nil
	}

	err := filepath.Walk(dir, visit)

	return filenames, err
}

// lineCount returns the number of lines in a given file
func lineCount(filepath string) (int, error) {
	out, err := exec.Command("wc", "-l", filepath).Output()
	if err != nil {
		return 0, err
	}
	// wc output is like: 999 filename.go
	count, err := strconv.Atoi(strings.Split(strings.TrimSpace(string(out)), " ")[0])
	if err != nil {
		return 0, err
	}

	return count, nil
}

// Error contains the line number and the reason for
// an error output from a command
type Error struct {
	LineNumber  int    `json:"line_number"`
	ErrorString string `json:"error_string"`
}

// FileSummary contains the filename, location of the file
// on GitHub, and all of the errors related to the file
type FileSummary struct {
	Filename string  `json:"filename"`
	FileURL  string  `json:"file_url"`
	Errors   []Error `json:"errors"`
}

// ByFilename implements sort.Interface for []Person based on
// the Age field.
type ByFilename []FileSummary

func (a ByFilename) Len() int           { return len(a) }
func (a ByFilename) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByFilename) Less(i, j int) bool { return a[i].Filename < a[j].Filename }

func (fs *FileSummary) AddError(out string) error {
	s := strings.SplitN(out, ":", 2)
	msg := strings.SplitAfterN(s[1], ":", 3)[2]

	e := Error{ErrorString: msg}
	ls := strings.Split(s[1], ":")
	ln, err := strconv.Atoi(ls[0])
	if err != nil {
		return err
	}
	e.LineNumber = ln

	fs.Errors = append(fs.Errors, e)

	return nil
}

// GoTool runs a given go command (for example gofmt, go tool vet)
// on a directory
func GoTool(dir string, filenames, command []string) (float64, []FileSummary, error) {
	params := command[1:]
	params = addSkipDirs(params)
	params = append(params, dir+"/...")

	cmd := exec.Command(command[0], params...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return 0, []FileSummary{}, err
	}

	err = cmd.Start()
	if err != nil {
		return 0, []FileSummary{}, err
	}

	out := bufio.NewScanner(stdout)

	githubLink := strings.TrimPrefix(dir, "repos/src")

	// the same file can appear multiple times out of order
	// in the output, so we can't go line by line, have to store
	// a map of filename to FileSummary
	fsMap := map[string]FileSummary{}
	var failed = []FileSummary{}
outer:
	for out.Scan() {
		filename := strings.Split(out.Text(), ":")[0]
		filename = strings.TrimPrefix(filename, "repos/src")
		for _, skip := range skipSuffixes {
			if strings.HasSuffix(filename, skip) {
				continue outer
			}
		}
		fileURL := "https://" + strings.TrimPrefix(dir, "repos/src/") + "/blob/master" + strings.TrimPrefix(filename, githubLink)
		fs := fsMap[filename]
		if fs.Filename == "" {
			fs.Filename = filename
			fs.FileURL = fileURL
		}
		err = fs.AddError(out.Text())
		if err != nil {
			return 0, []FileSummary{}, err
		}
		fsMap[filename] = fs
	}
	if err := out.Err(); err != nil {
		return 0, []FileSummary{}, err
	}

	for _, v := range fsMap {
		failed = append(failed, v)
	}

	err = cmd.Wait()
	if exitErr, ok := err.(*exec.ExitError); ok {
		// The program has exited with an exit code != 0

		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
			// some commands exit 1 when files fail to pass (for example go vet)
			if status.ExitStatus() != 1 {
				return 0, failed, err
				// return 0, Error{}, err
			}
		}
	}

	if len(filenames) == 1 {
		lc, err := lineCount(filenames[0])
		if err != nil {
			return 0, failed, err
		}

		var errors int
		if len(failed) != 0 {
			errors = len(failed[0].Errors)
		}

		return float64(lc-errors) / float64(lc), failed, nil
	}

	return float64(len(filenames)-len(failed)) / float64(len(filenames)), failed, nil
}
<commit_msg>add comment to exported method<commit_after>package check

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
)

var (
	skipDirs     = []string{"Godeps", "vendor", "third_party"}
	skipSuffixes = []string{".pb.go", ".pb.gw.go", ".generated.go"}
)

func addSkipDirs(params []string) []string {
	for _, dir := range skipDirs {
		params = append(params, fmt.Sprintf("--skip=%s", dir))
	}
	return params
}

// GoFiles returns a slice of Go filenames
// in a given directory.
func GoFiles(dir string) ([]string, error) {
	var filenames []string
	visit := func(fp string, fi os.FileInfo, err error) error {
		for _, skip := range skipDirs {
			if strings.Contains(fp, skip) {
				return nil
			}
		}
		if err != nil {
			fmt.Println(err) // can't walk here,
			return nil       // but continue walking elsewhere
		}
		if fi.IsDir() {
			return nil // not a file.  ignore.
		}
		fiName := fi.Name()
		for _, skip := range skipSuffixes {
			if strings.HasSuffix(fiName, skip) {
				return nil
			}
		}
		ext := filepath.Ext(fiName)
		if ext == ".go" {
			filenames = append(filenames, fp)
		}
		return nil
	}

	err := filepath.Walk(dir, visit)

	return filenames, err
}

// lineCount returns the number of lines in a given file
func lineCount(filepath string) (int, error) {
	out, err := exec.Command("wc", "-l", filepath).Output()
	if err != nil {
		return 0, err
	}
	// wc output is like: 999 filename.go
	count, err := strconv.Atoi(strings.Split(strings.TrimSpace(string(out)), " ")[0])
	if err != nil {
		return 0, err
	}

	return count, nil
}

// Error contains the line number and the reason for
// an error output from a command
type Error struct {
	LineNumber  int    `json:"line_number"`
	ErrorString string `json:"error_string"`
}

// FileSummary contains the filename, location of the file
// on GitHub, and all of the errors related to the file
type FileSummary struct {
	Filename string  `json:"filename"`
	FileURL  string  `json:"file_url"`
	Errors   []Error `json:"errors"`
}

// ByFilename implements sort.Interface for []Person based on
// the Age field.
type ByFilename []FileSummary

func (a ByFilename) Len() int           { return len(a) }
func (a ByFilename) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByFilename) Less(i, j int) bool { return a[i].Filename < a[j].Filename }

// AddError adds an Error to FileSummary
func (fs *FileSummary) AddError(out string) error {
	s := strings.SplitN(out, ":", 2)
	msg := strings.SplitAfterN(s[1], ":", 3)[2]

	e := Error{ErrorString: msg}
	ls := strings.Split(s[1], ":")
	ln, err := strconv.Atoi(ls[0])
	if err != nil {
		return err
	}
	e.LineNumber = ln

	fs.Errors = append(fs.Errors, e)

	return nil
}

// GoTool runs a given go command (for example gofmt, go tool vet)
// on a directory
func GoTool(dir string, filenames, command []string) (float64, []FileSummary, error) {
	params := command[1:]
	params = addSkipDirs(params)
	params = append(params, dir+"/...")

	cmd := exec.Command(command[0], params...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return 0, []FileSummary{}, err
	}

	err = cmd.Start()
	if err != nil {
		return 0, []FileSummary{}, err
	}

	out := bufio.NewScanner(stdout)

	githubLink := strings.TrimPrefix(dir, "repos/src")

	// the same file can appear multiple times out of order
	// in the output, so we can't go line by line, have to store
	// a map of filename to FileSummary
	fsMap := map[string]FileSummary{}
	var failed = []FileSummary{}
outer:
	for out.Scan() {
		filename := strings.Split(out.Text(), ":")[0]
		filename = strings.TrimPrefix(filename, "repos/src")
		for _, skip := range skipSuffixes {
			if strings.HasSuffix(filename, skip) {
				continue outer
			}
		}
		fileURL := "https://" + strings.TrimPrefix(dir, "repos/src/") + "/blob/master" + strings.TrimPrefix(filename, githubLink)
		fs := fsMap[filename]
		if fs.Filename == "" {
			fs.Filename = filename
			fs.FileURL = fileURL
		}
		err = fs.AddError(out.Text())
		if err != nil {
			return 0, []FileSummary{}, err
		}
		fsMap[filename] = fs
	}
	if err := out.Err(); err != nil {
		return 0, []FileSummary{}, err
	}

	for _, v := range fsMap {
		failed = append(failed, v)
	}

	err = cmd.Wait()
	if exitErr, ok := err.(*exec.ExitError); ok {
		// The program has exited with an exit code != 0

		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
			// some commands exit 1 when files fail to pass (for example go vet)
			if status.ExitStatus() != 1 {
				return 0, failed, err
				// return 0, Error{}, err
			}
		}
	}

	if len(filenames) == 1 {
		lc, err := lineCount(filenames[0])
		if err != nil {
			return 0, failed, err
		}

		var errors int
		if len(failed) != 0 {
			errors = len(failed[0].Errors)
		}

		return float64(lc-errors) / float64(lc), failed, nil
	}

	return float64(len(filenames)-len(failed)) / float64(len(filenames)), failed, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tchannel

import (
	"time"

	"github.com/uber/jaeger-lib/metrics"
	"github.com/uber/tchannel-go"
	"github.com/uber/tchannel-go/thrift"
	"go.uber.org/zap"

	"github.com/jaegertracing/jaeger/pkg/discovery/peerlistmgr"
	"github.com/jaegertracing/jaeger/thrift-gen/jaeger"
	"github.com/jaegertracing/jaeger/thrift-gen/zipkincore"
)

const (
	jaegerBatches = "jaeger"
	zipkinBatches = "zipkin"
)

type batchMetrics struct {
	// Number of successful batch submissions to collector
	BatchesSubmitted metrics.Counter `metric:"batches.submitted"`

	// Number of failed batch submissions to collector
	BatchesFailures metrics.Counter `metric:"batches.failures"`

	// Number of spans in a batch submitted to collector
	BatchSize metrics.Gauge `metric:"batch_size"`

	// Number of successful span submissions to collector
	SpansSubmitted metrics.Counter `metric:"spans.submitted"`

	// Number of failed span submissions to collector
	SpansFailures metrics.Counter `metric:"spans.failures"`
}

// Reporter forwards received spans to central collector tier over TChannel.
type Reporter struct {
	channel       *tchannel.Channel
	zClient       zipkincore.TChanZipkinCollector
	jClient       jaeger.TChanCollector
	reportTimeout time.Duration
	peerListMgr   *peerlistmgr.PeerListManager
	logger        *zap.Logger
	serviceName   string
}

// New creates new TChannel-based Reporter.
func New(
	collectorServiceName string,
	channel *tchannel.Channel,
	reportTimeout time.Duration,
	peerListMgr *peerlistmgr.PeerListManager,
	zlogger *zap.Logger,
) *Reporter {
	thriftClient := thrift.NewClient(channel, collectorServiceName, nil)
	zClient := zipkincore.NewTChanZipkinCollectorClient(thriftClient)
	jClient := jaeger.NewTChanCollectorClient(thriftClient)
	return &Reporter{
		channel:       channel,
		zClient:       zClient,
		jClient:       jClient,
		reportTimeout: reportTimeout,
		peerListMgr:   peerListMgr,
		logger:        zlogger,
		serviceName:   collectorServiceName,
	}
}

// Channel returns the TChannel used by the reporter.
func (r *Reporter) Channel() *tchannel.Channel {
	return r.channel
}

// EmitZipkinBatch implements EmitZipkinBatch() of Reporter
func (r *Reporter) EmitZipkinBatch(spans []*zipkincore.Span) error {
	submissionFunc := func(ctx thrift.Context) error {
		_, err := r.zClient.SubmitZipkinBatch(ctx, spans)
		return err
	}
	return r.submitAndReport(
		submissionFunc,
		"Could not submit zipkin batch",
		int64(len(spans)),
	)
}

// EmitBatch implements EmitBatch() of Reporter
func (r *Reporter) EmitBatch(batch *jaeger.Batch) error {
	submissionFunc := func(ctx thrift.Context) error {
		_, err := r.jClient.SubmitBatches(ctx, []*jaeger.Batch{batch})
		return err
	}
	return r.submitAndReport(
		submissionFunc,
		"Could not submit jaeger batch",
		int64(len(batch.Spans)),
	)
}

func (r *Reporter) submitAndReport(submissionFunc func(ctx thrift.Context) error, errMsg string, size int64) error {
	ctx, cancel := tchannel.NewContextBuilder(r.reportTimeout).DisableTracing().Build()
	defer cancel()

	if err := submissionFunc(ctx); err != nil {
		return err
	}

	r.logger.Debug("Span batch submitted by the agent", zap.Int64("span-count", size))
	return nil
}

// CollectorServiceName returns collector service name.
func (r *Reporter) CollectorServiceName() string {
	return r.serviceName
}
<commit_msg>Remove unused code (#1199)<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tchannel

import (
	"time"

	"github.com/uber/tchannel-go"
	"github.com/uber/tchannel-go/thrift"
	"go.uber.org/zap"

	"github.com/jaegertracing/jaeger/pkg/discovery/peerlistmgr"
	"github.com/jaegertracing/jaeger/thrift-gen/jaeger"
	"github.com/jaegertracing/jaeger/thrift-gen/zipkincore"
)

// Reporter forwards received spans to central collector tier over TChannel.
type Reporter struct {
	channel       *tchannel.Channel
	zClient       zipkincore.TChanZipkinCollector
	jClient       jaeger.TChanCollector
	reportTimeout time.Duration
	peerListMgr   *peerlistmgr.PeerListManager
	logger        *zap.Logger
	serviceName   string
}

// New creates new TChannel-based Reporter.
func New(
	collectorServiceName string,
	channel *tchannel.Channel,
	reportTimeout time.Duration,
	peerListMgr *peerlistmgr.PeerListManager,
	zlogger *zap.Logger,
) *Reporter {
	thriftClient := thrift.NewClient(channel, collectorServiceName, nil)
	zClient := zipkincore.NewTChanZipkinCollectorClient(thriftClient)
	jClient := jaeger.NewTChanCollectorClient(thriftClient)
	return &Reporter{
		channel:       channel,
		zClient:       zClient,
		jClient:       jClient,
		reportTimeout: reportTimeout,
		peerListMgr:   peerListMgr,
		logger:        zlogger,
		serviceName:   collectorServiceName,
	}
}

// Channel returns the TChannel used by the reporter.
func (r *Reporter) Channel() *tchannel.Channel {
	return r.channel
}

// EmitZipkinBatch implements EmitZipkinBatch() of Reporter
func (r *Reporter) EmitZipkinBatch(spans []*zipkincore.Span) error {
	submissionFunc := func(ctx thrift.Context) error {
		_, err := r.zClient.SubmitZipkinBatch(ctx, spans)
		return err
	}
	return r.submitAndReport(
		submissionFunc,
		"Could not submit zipkin batch",
		int64(len(spans)),
	)
}

// EmitBatch implements EmitBatch() of Reporter
func (r *Reporter) EmitBatch(batch *jaeger.Batch) error {
	submissionFunc := func(ctx thrift.Context) error {
		_, err := r.jClient.SubmitBatches(ctx, []*jaeger.Batch{batch})
		return err
	}
	return r.submitAndReport(
		submissionFunc,
		"Could not submit jaeger batch",
		int64(len(batch.Spans)),
	)
}

func (r *Reporter) submitAndReport(submissionFunc func(ctx thrift.Context) error, errMsg string, size int64) error {
	ctx, cancel := tchannel.NewContextBuilder(r.reportTimeout).DisableTracing().Build()
	defer cancel()

	if err := submissionFunc(ctx); err != nil {
		return err
	}

	r.logger.Debug("Span batch submitted by the agent", zap.Int64("span-count", size))
	return nil
}

// CollectorServiceName returns collector service name.
func (r *Reporter) CollectorServiceName() string {
	return r.serviceName
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"log"
	"math"
	"math/rand"
	"net"
	"net/url"
	"strconv"
	"time"
)

const (
	connectMinResponseLength = 16
	announceMinResponseLength = 20
	connectBufferSize = 150
	announceBufferSize = 20000
	startPort = 6881
	endPort = 6889
)

type UdpTracker struct {
	*tracker
	connectionId  uint64
	transactionId uint32
	serverAddr    *net.UDPAddr
	conn          *net.UDPConn
}

type connectRequest struct {
	connectionId  uint64
	action        uint32
	transactionId uint32
}

type connectResponse struct {
	action        uint32
	transactionId uint32
	connectionId  uint64
}

type errorResponse struct {
	action        uint32
	transactionId uint32
	message       string
}

type shortString []byte

type announceRequest struct {
	connectionId  uint64
	action        uint32
	transactionId uint32
	infoHash      shortString
	peerId        shortString
	downloaded    uint64
	left          uint64
	uploaded      uint64
	event         uint32
	ipAddr        uint32
	key           uint32
	numWant       int32
	port          uint16
}

type announceResponse struct {
	action        uint32
	transactionId uint32
	interval      uint32
	leechers      uint32
	seeders       uint32
	peers         []PeerTuple
}

func (r *connectRequest) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)
	err := binary.Write(buf, binary.BigEndian, r.connectionId)
	err = binary.Write(buf, binary.BigEndian, r.action)
	err = binary.Write(buf, binary.BigEndian, r.transactionId)
	return buf.Bytes(), err
}

func (r *connectResponse) UnmarshalBinary(data []byte) error {
	buf := bytes.NewReader(data[4:])
	actionBytes := append([]byte{0, 0, 0}, data[0])
	err := binary.Read(bytes.NewReader(actionBytes), binary.BigEndian, &r.action)
	err = binary.Read(buf, binary.BigEndian, &r.transactionId)
	err = binary.Read(buf, binary.BigEndian, &r.connectionId)
	return err
}

func (r *connectResponse) MarshalBinary() ([]byte, error) {
	var b bytes.Buffer
	fmt.Fprintln(&b, r.connectionId, r.action, r.transactionId)
	return b.Bytes(), nil
}

func (r *errorResponse) UnmarshalBinary(data []byte) error {
	actionBytes := append([]byte{0, 0, 0}, data[0])
	err := binary.Read(bytes.NewReader(actionBytes), binary.BigEndian, &r.action)
	err = binary.Read(bytes.NewReader(data[4:]), binary.BigEndian, &r.transactionId)
	r.message = string(data[8:])
	return err
}

func (r *announceRequest) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)
	err := binary.Write(buf, binary.BigEndian, r.connectionId)
	err = binary.Write(buf, binary.BigEndian, r.action)
	err = binary.Write(buf, binary.BigEndian, r.transactionId)
	err = binary.Write(buf, binary.BigEndian, r.infoHash)
	err = binary.Write(buf, binary.BigEndian, r.peerId)
	err = binary.Write(buf, binary.BigEndian, r.downloaded)
	err = binary.Write(buf, binary.BigEndian, r.left)
	err = binary.Write(buf, binary.BigEndian, r.uploaded)
	err = binary.Write(buf, binary.BigEndian, r.event)
	err = binary.Write(buf, binary.BigEndian, r.ipAddr)
	err = binary.Write(buf, binary.BigEndian, r.key)
	err = binary.Write(buf, binary.BigEndian, r.numWant)
	err = binary.Write(buf, binary.BigEndian, r.port)
	return buf.Bytes(), err
}

func (r *announceResponse) UnmarshalBinary(data []byte) error {
	buf := bytes.NewReader(data[4:])
	actionBytes := append([]byte{0, 0, 0}, data[0])
	err := binary.Read(bytes.NewReader(actionBytes), binary.BigEndian, &r.action)
	err = binary.Read(buf, binary.BigEndian, &r.transactionId)
	err = binary.Read(buf, binary.BigEndian, &r.interval)
	err = binary.Read(buf, binary.BigEndian, &r.leechers)
	err = binary.Read(buf, binary.BigEndian, &r.seeders)

	if len(data) > announceMinResponseLength {
		peerBytes := data[announceMinResponseLength:]
		peers := len(peerBytes) / 6

		ipLen := 4
		portLen := 2
		for i := 0; i < peers; i++ {
			ipOffset := i*6
			portOffset := ipOffset + ipLen
			peer := PeerTuple{}
			ipBytes := peerBytes[ipOffset : ipOffset + ipLen]
			peer.IP = net.IPv4(ipBytes[0], ipBytes[1], ipBytes[2], ipBytes[3])
			binary.Read(bytes.NewReader(peerBytes[portOffset : portOffset + portLen]), binary.BigEndian, &peer.Port)
			r.peers = append(r.peers, peer)
		}
	}

	return err
}

func NewUdpTracker(key string, chans trackerPeerChans, port uint16, infoHash []byte, announce *url.URL) *UdpTracker {
	return &UdpTracker{&tracker{key: key, peerChans: chans, port: port, infoHash: infoHash, announceURL: announce}, 0, 0, &net.UDPAddr{}, &net.UDPConn{}}
}

func (tr *UdpTracker) Announce(event int) {
	err := tr.connect()
	if err != nil {
		log.Printf("Tracker : Could not connect to tracker %s: \n", tr.announceURL.String(), err)
		return
	}

	key, _ := strconv.ParseUint(tr.key, 16, 4)
	announce := &announceRequest{
		connectionId:  tr.connectionId,
		action:        1,
		transactionId: tr.transactionId,
		infoHash:      tr.infoHash,
		peerId:        PeerID[:],
		downloaded:    uint64(tr.stats.Downloaded),
		left:          uint64(tr.stats.Left),
		uploaded:      uint64(tr.stats.Uploaded),
		event:         uint32(event),
		ipAddr:        0,
		key:           uint32(key),
		numWant:       -1,
		port:          6881,
	}
	announceBytes, _ := announce.MarshalBinary()

	buff := make([]byte, announceBufferSize)
	length := tr.request(announceBytes, buff)

	if length >= announceMinResponseLength {
		var response announceResponse
		response.UnmarshalBinary(buff[:length])

		if event != Stopped {
			if response.interval != 0 {
				nextAnnounce := time.Second * time.Duration(response.interval)
				log.Println("Tracker : Announce : Scheduling next announce in", nextAnnounce)
				tr.timer = time.After(nextAnnounce)
			}

			for _, peer := range response.peers {
				tr.peerChans.peers <- peer
			}
		}
	}
}

func (tr *UdpTracker) connect() error {
	log.Printf("Tracker : Connect (%v)", tr.announceURL)

	connectReq := connectRequest{connectionId: 0x41727101980, action: 0, transactionId: tr.transactionId}
	connectBytes, _ := connectReq.MarshalBinary()

	buff := make([]byte, connectBufferSize)
	length := tr.request(connectBytes, buff)

	if length >= connectMinResponseLength {
		var response connectResponse
		response.UnmarshalBinary(buff[:length])

		if response.action == 3 || tr.transactionId != response.transactionId {
			error := errorResponse{}
			error.UnmarshalBinary(buff)
			return errors.New("Response Error: " + error.message)
		}

		tr.connectionId = response.connectionId
		return nil
	}

	return errors.New("Invalid connect response length")
}

func (tr *UdpTracker) Run() {
	log.Printf("Tracker : Run : Started (%s)\n", tr.announceURL)
	defer log.Printf("Tracker : Run : Completed (%s)\n", tr.announceURL)

	rand.Seed(time.Now().UnixNano())

	serverAddr, err := net.ResolveUDPAddr("udp", tr.announceURL.Host)
	if err != nil {
		log.Println("Tracker : Could not resolve tracker host!")
		return
	}

	var conn *net.UDPConn
	for port := startPort; err == nil && port < endPort; port++ {
		conn, err = net.ListenUDP("udp", &net.UDPAddr{Port: port})
	}
	if err != nil {
		log.Printf("Tracker : Could not bind to any port in range %d-%d\n", startPort, endPort)
		return
	}

	tr.transactionId = rand.Uint32()
	tr.serverAddr = serverAddr
	tr.conn = conn
	tr.Announce(Started)

	for {
		select {
		case <-tr.quit:
			log.Println("Tracker : Stop : Stopping")
			tr.Announce(Stopped)
			return
		case <-tr.completedCh:
			go tr.Announce(Completed)
		case <-tr.timer:
			log.Printf("Tracker : Run : Interval Timer Expired (%s)\n", tr.announceURL)
			go tr.Announce(Interval)
		case stats := <-tr.peerChans.stats:
			log.Println("read from stats", stats)
		}
	}
}

// Send a udp packet to the tracker, fill in the dest buffer with the response
func (tr *UdpTracker) request(payload []byte, dest []byte) int {
	n := 0
	totalAttempts := 0

	// notify the sender when a response is recieved
	recvChan := make(chan bool)

	// keep sending the packet and wait for the specifed time
	// before trying again, limit n to 8
	go func() {

		// initial send
		tr.conn.WriteTo(payload, tr.serverAddr)
	Listen:
		for {
			// timeout: 15 * 2 ^ n (0-8)
			timeout := time.Second * time.Duration(15 * int(math.Pow(2.0, float64(n))))
			timer := time.After(timeout)
			select {
			case <-recvChan:
				break Listen
			case <-timer:
				tr.conn.WriteTo(payload, tr.serverAddr)
				totalAttempts++
				if n < 8 { n++ }
			}
		}
	}()

	// read until some data is received
	var err error
	length := 0
	for length == 0 {
		length, _, err = tr.conn.ReadFrom(dest)

		// not sure what to do here?
		if err != nil {
			log.Println("Tracker : udp packet read error")
		}
	}

	recvChan <- true
	return length
}
<commit_msg>Removed unnecessary type<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"log"
	"math"
	"math/rand"
	"net"
	"net/url"
	"strconv"
	"time"
)

const (
	connectMinResponseLength = 16
	announceMinResponseLength = 20
	connectBufferSize = 150
	announceBufferSize = 20000
	startPort = 6881
	endPort = 6889
)

type UdpTracker struct {
	*tracker
	connectionId  uint64
	transactionId uint32
	serverAddr    *net.UDPAddr
	conn          *net.UDPConn
}

type connectRequest struct {
	connectionId  uint64
	action        uint32
	transactionId uint32
}

type connectResponse struct {
	action        uint32
	transactionId uint32
	connectionId  uint64
}

type errorResponse struct {
	action        uint32
	transactionId uint32
	message       string
}

type announceRequest struct {
	connectionId  uint64
	action        uint32
	transactionId uint32
	infoHash      []byte
	peerId        []byte
	downloaded    uint64
	left          uint64
	uploaded      uint64
	event         uint32
	ipAddr        uint32
	key           uint32
	numWant       int32
	port          uint16
}

type announceResponse struct {
	action        uint32
	transactionId uint32
	interval      uint32
	leechers      uint32
	seeders       uint32
	peers         []PeerTuple
}

func (r *connectRequest) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)
	err := binary.Write(buf, binary.BigEndian, r.connectionId)
	err = binary.Write(buf, binary.BigEndian, r.action)
	err = binary.Write(buf, binary.BigEndian, r.transactionId)
	return buf.Bytes(), err
}

func (r *connectResponse) UnmarshalBinary(data []byte) error {
	buf := bytes.NewReader(data[4:])
	actionBytes := append([]byte{0, 0, 0}, data[0])
	err := binary.Read(bytes.NewReader(actionBytes), binary.BigEndian, &r.action)
	err = binary.Read(buf, binary.BigEndian, &r.transactionId)
	err = binary.Read(buf, binary.BigEndian, &r.connectionId)
	return err
}

func (r *connectResponse) MarshalBinary() ([]byte, error) {
	var b bytes.Buffer
	fmt.Fprintln(&b, r.connectionId, r.action, r.transactionId)
	return b.Bytes(), nil
}

func (r *errorResponse) UnmarshalBinary(data []byte) error {
	actionBytes := append([]byte{0, 0, 0}, data[0])
	err := binary.Read(bytes.NewReader(actionBytes), binary.BigEndian, &r.action)
	err = binary.Read(bytes.NewReader(data[4:]), binary.BigEndian, &r.transactionId)
	r.message = string(data[8:])
	return err
}

func (r *announceRequest) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)
	err := binary.Write(buf, binary.BigEndian, r.connectionId)
	err = binary.Write(buf, binary.BigEndian, r.action)
	err = binary.Write(buf, binary.BigEndian, r.transactionId)
	err = binary.Write(buf, binary.BigEndian, r.infoHash)
	err = binary.Write(buf, binary.BigEndian, r.peerId)
	err = binary.Write(buf, binary.BigEndian, r.downloaded)
	err = binary.Write(buf, binary.BigEndian, r.left)
	err = binary.Write(buf, binary.BigEndian, r.uploaded)
	err = binary.Write(buf, binary.BigEndian, r.event)
	err = binary.Write(buf, binary.BigEndian, r.ipAddr)
	err = binary.Write(buf, binary.BigEndian, r.key)
	err = binary.Write(buf, binary.BigEndian, r.numWant)
	err = binary.Write(buf, binary.BigEndian, r.port)
	return buf.Bytes(), err
}

func (r *announceResponse) UnmarshalBinary(data []byte) error {
	buf := bytes.NewReader(data[4:])
	actionBytes := append([]byte{0, 0, 0}, data[0])
	err := binary.Read(bytes.NewReader(actionBytes), binary.BigEndian, &r.action)
	err = binary.Read(buf, binary.BigEndian, &r.transactionId)
	err = binary.Read(buf, binary.BigEndian, &r.interval)
	err = binary.Read(buf, binary.BigEndian, &r.leechers)
	err = binary.Read(buf, binary.BigEndian, &r.seeders)

	if len(data) > announceMinResponseLength {
		peerBytes := data[announceMinResponseLength:]
		peers := len(peerBytes) / 6

		ipLen := 4
		portLen := 2
		for i := 0; i < peers; i++ {
			ipOffset := i*6
			portOffset := ipOffset + ipLen
			peer := PeerTuple{}
			ipBytes := peerBytes[ipOffset : ipOffset + ipLen]
			peer.IP = net.IPv4(ipBytes[0], ipBytes[1], ipBytes[2], ipBytes[3])
			binary.Read(bytes.NewReader(peerBytes[portOffset : portOffset + portLen]), binary.BigEndian, &peer.Port)
			r.peers = append(r.peers, peer)
		}
	}

	return err
}

func NewUdpTracker(key string, chans trackerPeerChans, port uint16, infoHash []byte, announce *url.URL) *UdpTracker {
	return &UdpTracker{&tracker{key: key, peerChans: chans, port: port, infoHash: infoHash, announceURL: announce}, 0, 0, &net.UDPAddr{}, &net.UDPConn{}}
}

func (tr *UdpTracker) Announce(event int) {
	err := tr.connect()
	if err != nil {
		log.Printf("Tracker : Could not connect to tracker %s: \n", tr.announceURL.String(), err)
		return
	}

	key, _ := strconv.ParseUint(tr.key, 16, 4)
	announce := &announceRequest{
		connectionId:  tr.connectionId,
		action:        1,
		transactionId: tr.transactionId,
		infoHash:      tr.infoHash,
		peerId:        PeerID[:],
		downloaded:    uint64(tr.stats.Downloaded),
		left:          uint64(tr.stats.Left),
		uploaded:      uint64(tr.stats.Uploaded),
		event:         uint32(event),
		ipAddr:        0,
		key:           uint32(key),
		numWant:       -1,
		port:          6881,
	}
	announceBytes, _ := announce.MarshalBinary()

	buff := make([]byte, announceBufferSize)
	length := tr.request(announceBytes, buff)

	if length >= announceMinResponseLength {
		var response announceResponse
		response.UnmarshalBinary(buff[:length])

		if event != Stopped {
			if response.interval != 0 {
				nextAnnounce := time.Second * time.Duration(response.interval)
				log.Println("Tracker : Announce : Scheduling next announce in", nextAnnounce)
				tr.timer = time.After(nextAnnounce)
			}

			for _, peer := range response.peers {
				tr.peerChans.peers <- peer
			}
		}
	}
}

func (tr *UdpTracker) connect() error {
	log.Printf("Tracker : Connect (%v)", tr.announceURL)

	connectReq := connectRequest{connectionId: 0x41727101980, action: 0, transactionId: tr.transactionId}
	connectBytes, _ := connectReq.MarshalBinary()

	buff := make([]byte, connectBufferSize)
	length := tr.request(connectBytes, buff)

	if length >= connectMinResponseLength {
		var response connectResponse
		response.UnmarshalBinary(buff[:length])

		if response.action == 3 || tr.transactionId != response.transactionId {
			error := errorResponse{}
			error.UnmarshalBinary(buff)
			return errors.New("Response Error: " + error.message)
		}

		tr.connectionId = response.connectionId
		return nil
	}

	return errors.New("Invalid connect response length")
}

func (tr *UdpTracker) Run() {
	log.Printf("Tracker : Run : Started (%s)\n", tr.announceURL)
	defer log.Printf("Tracker : Run : Completed (%s)\n", tr.announceURL)

	rand.Seed(time.Now().UnixNano())

	serverAddr, err := net.ResolveUDPAddr("udp", tr.announceURL.Host)
	if err != nil {
		log.Println("Tracker : Could not resolve tracker host!")
		return
	}

	var conn *net.UDPConn
	for port := startPort; err == nil && port < endPort; port++ {
		conn, err = net.ListenUDP("udp", &net.UDPAddr{Port: port})
	}
	if err != nil {
		log.Printf("Tracker : Could not bind to any port in range %d-%d\n", startPort, endPort)
		return
	}

	tr.transactionId = rand.Uint32()
	tr.serverAddr = serverAddr
	tr.conn = conn
	tr.Announce(Started)

	for {
		select {
		case <-tr.quit:
			log.Println("Tracker : Stop : Stopping")
			tr.Announce(Stopped)
			return
		case <-tr.completedCh:
			go tr.Announce(Completed)
		case <-tr.timer:
			log.Printf("Tracker : Run : Interval Timer Expired (%s)\n", tr.announceURL)
			go tr.Announce(Interval)
		case stats := <-tr.peerChans.stats:
			log.Println("read from stats", stats)
		}
	}
}

// Send a udp packet to the tracker, fill in the dest buffer with the response
func (tr *UdpTracker) request(payload []byte, dest []byte) int {
	n := 0
	totalAttempts := 0

	// notify the sender when a response is recieved
	recvChan := make(chan bool)

	// keep sending the packet and wait for the specifed time
	// before trying again, limit n to 8
	go func() {

		// initial send
		tr.conn.WriteTo(payload, tr.serverAddr)
	Listen:
		for {
			// timeout: 15 * 2 ^ n (0-8)
			timeout := time.Second * time.Duration(15 * int(math.Pow(2.0, float64(n))))
			timer := time.After(timeout)
			select {
			case <-recvChan:
				break Listen
			case <-timer:
				tr.conn.WriteTo(payload, tr.serverAddr)
				totalAttempts++
				if n < 8 { n++ }
			}
		}
	}()

	// read until some data is received
	var err error
	length := 0
	for length == 0 {
		length, _, err = tr.conn.ReadFrom(dest)

		// not sure what to do here?
		if err != nil {
			log.Println("Tracker : udp packet read error")
		}
	}

	recvChan <- true
	return length
}
<|endoftext|>


<commit_before>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package db

import (
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	"github.com/op/go-logging"
	"github.com/spf13/viper"
	"github.com/tecbot/gorocksdb"
)

var dbLogger = logging.MustGetLogger("db")

const blockchainCF = "blockchainCF"
const stateCF = "stateCF"
const stateDeltaCF = "stateDeltaCF"
const indexesCF = "indexesCF"
const persistCF = "persistCF"

var columnfamilies = []string{
	blockchainCF,
	stateCF,
	stateDeltaCF,
	indexesCF,
	persistCF,
}

// OpenchainDB encapsulates rocksdb's structures
type OpenchainDB struct {
	DB           *gorocksdb.DB
	BlockchainCF *gorocksdb.ColumnFamilyHandle
	StateCF      *gorocksdb.ColumnFamilyHandle
	StateDeltaCF *gorocksdb.ColumnFamilyHandle
	IndexesCF    *gorocksdb.ColumnFamilyHandle
	PersistCF    *gorocksdb.ColumnFamilyHandle
}

var openchainDB *OpenchainDB
var isOpen bool

// CreateDB creates a rocks db database
func CreateDB() error {
	dbPath := getDBPath()
	dbLogger.Debug("Creating DB at [%s]", dbPath)
	missing, err := dirMissingOrEmpty(dbPath)
	if err != nil {
		return err
	}

	if !missing {
		return fmt.Errorf("db dir [%s] already exists", dbPath)
	}
	err = os.MkdirAll(path.Dir(dbPath), 0755)
	if err != nil {
		dbLogger.Error("Error calling  os.MkdirAll for directory path [%s]: %s", dbPath, err)
		return fmt.Errorf("Error making directory path [%s]: %s", dbPath, err)
	}
	opts := gorocksdb.NewDefaultOptions()
	defer opts.Destroy()
	opts.SetCreateIfMissing(true)

	db, err := gorocksdb.OpenDb(opts, dbPath)
	if err != nil {
		return err
	}

	defer db.Close()

	for _, cf := range columnfamilies {
		_, err = db.CreateColumnFamily(opts, cf)
		if err != nil {
			return err
		}
	}
	dbLogger.Debug("DB created at [%s]", dbPath)
	return nil
}

// GetDBHandle returns a handle to OpenchainDB
func GetDBHandle() *OpenchainDB {
	var err error
	if isOpen {
		return openchainDB
	}

	err = createDBIfDBPathEmpty()
	if err != nil {
		panic(fmt.Sprintf("Error while trying to create DB: %s", err))
	}

	openchainDB, err = openDB()
	if err != nil {
		panic(fmt.Sprintf("Could not open openchain db error = [%s]", err))
	}
	return openchainDB
}

// GetFromBlockchainCF get value for given key from column family - blockchainCF
func (openchainDB *OpenchainDB) GetFromBlockchainCF(key []byte) ([]byte, error) {
	return openchainDB.Get(openchainDB.BlockchainCF, key)
}

// GetFromBlockchainCFSnapshot get value for given key from column family in a DB snapshot - blockchainCF
func (openchainDB *OpenchainDB) GetFromBlockchainCFSnapshot(snapshot *gorocksdb.Snapshot, key []byte) ([]byte, error) {
	return openchainDB.getFromSnapshot(snapshot, openchainDB.BlockchainCF, key)
}

// GetFromStateCF get value for given key from column family - stateCF
func (openchainDB *OpenchainDB) GetFromStateCF(key []byte) ([]byte, error) {
	return openchainDB.Get(openchainDB.StateCF, key)
}

// GetFromStateDeltaCF get value for given key from column family - stateDeltaCF
func (openchainDB *OpenchainDB) GetFromStateDeltaCF(key []byte) ([]byte, error) {
	return openchainDB.Get(openchainDB.StateDeltaCF, key)
}

// GetFromIndexesCF get value for given key from column family - indexCF
func (openchainDB *OpenchainDB) GetFromIndexesCF(key []byte) ([]byte, error) {
	return openchainDB.Get(openchainDB.IndexesCF, key)
}

// GetBlockchainCFIterator get iterator for column family - blockchainCF
func (openchainDB *OpenchainDB) GetBlockchainCFIterator() *gorocksdb.Iterator {
	return openchainDB.getIterator(openchainDB.BlockchainCF)
}

// GetStateCFIterator get iterator for column family - stateCF
func (openchainDB *OpenchainDB) GetStateCFIterator() *gorocksdb.Iterator {
	return openchainDB.getIterator(openchainDB.StateCF)
}

// GetStateCFSnapshotIterator get iterator for column family - stateCF. This iterator
// is based on a snapshot and should be used for long running scans, such as
// reading the entire state. Remember to call iterator.Close() when you are done.
func (openchainDB *OpenchainDB) GetStateCFSnapshotIterator(snapshot *gorocksdb.Snapshot) *gorocksdb.Iterator {
	return openchainDB.getSnapshotIterator(snapshot, openchainDB.StateCF)
}

// GetStateDeltaCFIterator get iterator for column family - stateDeltaCF
func (openchainDB *OpenchainDB) GetStateDeltaCFIterator() *gorocksdb.Iterator {
	return openchainDB.getIterator(openchainDB.StateDeltaCF)
}

// GetSnapshot returns a point-in-time view of the DB. You MUST call snapshot.Release()
// when you are done with the snapshot.
func (openchainDB *OpenchainDB) GetSnapshot() *gorocksdb.Snapshot {
	return openchainDB.DB.NewSnapshot()
}

func getDBPath() string {
	dbPath := viper.GetString("peer.fileSystemPath")
	if dbPath == "" {
		panic("DB path not specified in configuration file. Please check that property 'peer.fileSystemPath' is set")
	}
	if !strings.HasSuffix(dbPath, "/") {
		dbPath = dbPath + "/"
	}
	return dbPath + "db"
}

func createDBIfDBPathEmpty() error {
	dbPath := getDBPath()
	missing, err := dirMissingOrEmpty(dbPath)
	if err != nil {
		return err
	}
	dbLogger.Debug("Is db path [%s] empty [%t]", dbPath, missing)
	if missing {
		err := CreateDB()
		if err != nil {
			return nil
		}
	}
	return nil
}

func openDB() (*OpenchainDB, error) {
	if isOpen {
		return openchainDB, nil
	}

	dbPath := getDBPath()
	opts := gorocksdb.NewDefaultOptions()
	defer opts.Destroy()

	opts.SetCreateIfMissing(false)

	cfNames := []string{"default"}
	cfNames = append(cfNames, columnfamilies...)
	var cfOpts []*gorocksdb.Options
	for _ = range cfNames {
		cfOpts = append(cfOpts, opts)
	}

	db, cfHandlers, err := gorocksdb.OpenDbColumnFamilies(opts, dbPath, cfNames, cfOpts)

	if err != nil {
		fmt.Println("Error opening DB", err)
		return nil, err
	}
	isOpen = true
	// XXX should we close cfHandlers[0]?
	return &OpenchainDB{db, cfHandlers[1], cfHandlers[2], cfHandlers[3], cfHandlers[4], cfHandlers[5]}, nil
}

// CloseDB releases all column family handles and closes rocksdb
func (openchainDB *OpenchainDB) CloseDB() {
	openchainDB.BlockchainCF.Destroy()
	openchainDB.StateCF.Destroy()
	openchainDB.StateDeltaCF.Destroy()
	openchainDB.IndexesCF.Destroy()
	openchainDB.PersistCF.Destroy()
	openchainDB.DB.Close()
	isOpen = false
}

// DeleteState delets ALL state keys/values from the DB. This is generally
// only used during state synchronization when creating a new state from
// a snapshot.
func (openchainDB *OpenchainDB) DeleteState() error {
	err := openchainDB.DB.DropColumnFamily(openchainDB.StateCF)
	if err != nil {
		dbLogger.Error("Error dropping state CF", err)
		return err
	}
	err = openchainDB.DB.DropColumnFamily(openchainDB.StateDeltaCF)
	if err != nil {
		dbLogger.Error("Error dropping state delta CF", err)
		return err
	}
	opts := gorocksdb.NewDefaultOptions()
	defer opts.Destroy()
	openchainDB.StateCF, err = openchainDB.DB.CreateColumnFamily(opts, stateCF)
	if err != nil {
		dbLogger.Error("Error creating state CF", err)
		return err
	}
	openchainDB.StateDeltaCF, err = openchainDB.DB.CreateColumnFamily(opts, stateDeltaCF)
	if err != nil {
		dbLogger.Error("Error creating state delta CF", err)
		return err
	}
	return nil
}

func (openchainDB *OpenchainDB) get(cfHandler *gorocksdb.ColumnFamilyHandle, key []byte) ([]byte, error) {
	opt := gorocksdb.NewDefaultReadOptions()
	defer opt.Destroy()
	slice, err := openchainDB.DB.GetCF(opt, cfHandler, key)
	if err != nil {
		fmt.Println("Error while trying to retrieve key:", key)
		return nil, err
	}
	defer slice.Free()
	data := append([]byte(nil), slice.Data()...)
	return data, nil
}

func (openchainDB *OpenchainDB) getFromSnapshot(snapshot *gorocksdb.Snapshot, cfHandler *gorocksdb.ColumnFamilyHandle, key []byte) ([]byte, error) {
	opt := gorocksdb.NewDefaultReadOptions()
	defer opt.Destroy()
	opt.SetSnapshot(snapshot)
	slice, err := openchainDB.DB.GetCF(opt, cfHandler, key)
	if err != nil {
		fmt.Println("Error while trying to retrieve key:", key)
		return nil, err
	}
	defer slice.Free()
	data := append([]byte(nil), slice.Data()...)
	return data, nil
}

func (openchainDB *OpenchainDB) getIterator(cfHandler *gorocksdb.ColumnFamilyHandle) *gorocksdb.Iterator {
	opt := gorocksdb.NewDefaultReadOptions()
	opt.SetFillCache(true)
	defer opt.Destroy()
	return openchainDB.DB.NewIteratorCF(opt, cfHandler)
}

func (openchainDB *OpenchainDB) getSnapshotIterator(snapshot *gorocksdb.Snapshot, cfHandler *gorocksdb.ColumnFamilyHandle) *gorocksdb.Iterator {
	opt := gorocksdb.NewDefaultReadOptions()
	defer opt.Destroy()
	opt.SetSnapshot(snapshot)
	iter := openchainDB.DB.NewIteratorCF(opt, cfHandler)
	return iter
}

func dirMissingOrEmpty(path string) (bool, error) {
	dirExists, err := dirExists(path)
	if err != nil {
		return false, err
	}
	if !dirExists {
		return true, nil
	}

	dirEmpty, err := dirEmpty(path)
	if err != nil {
		return false, err
	}
	if dirEmpty {
		return true, nil
	}
	return false, nil
}

func dirExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func dirEmpty(path string) (bool, error) {
	f, err := os.Open(path)
	if err != nil {
		return false, err
	}
	defer f.Close()

	_, err = f.Readdir(1)
	if err == io.EOF {
		return true, nil
	}
	return false, err
}
<commit_msg>core/db: make Get() public, add Put()<commit_after>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package db

import (
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	"github.com/op/go-logging"
	"github.com/spf13/viper"
	"github.com/tecbot/gorocksdb"
)

var dbLogger = logging.MustGetLogger("db")

const blockchainCF = "blockchainCF"
const stateCF = "stateCF"
const stateDeltaCF = "stateDeltaCF"
const indexesCF = "indexesCF"
const persistCF = "persistCF"

var columnfamilies = []string{
	blockchainCF,
	stateCF,
	stateDeltaCF,
	indexesCF,
	persistCF,
}

// OpenchainDB encapsulates rocksdb's structures
type OpenchainDB struct {
	DB           *gorocksdb.DB
	BlockchainCF *gorocksdb.ColumnFamilyHandle
	StateCF      *gorocksdb.ColumnFamilyHandle
	StateDeltaCF *gorocksdb.ColumnFamilyHandle
	IndexesCF    *gorocksdb.ColumnFamilyHandle
	PersistCF    *gorocksdb.ColumnFamilyHandle
}

var openchainDB *OpenchainDB
var isOpen bool

// CreateDB creates a rocks db database
func CreateDB() error {
	dbPath := getDBPath()
	dbLogger.Debug("Creating DB at [%s]", dbPath)
	missing, err := dirMissingOrEmpty(dbPath)
	if err != nil {
		return err
	}

	if !missing {
		return fmt.Errorf("db dir [%s] already exists", dbPath)
	}
	err = os.MkdirAll(path.Dir(dbPath), 0755)
	if err != nil {
		dbLogger.Error("Error calling  os.MkdirAll for directory path [%s]: %s", dbPath, err)
		return fmt.Errorf("Error making directory path [%s]: %s", dbPath, err)
	}
	opts := gorocksdb.NewDefaultOptions()
	defer opts.Destroy()
	opts.SetCreateIfMissing(true)

	db, err := gorocksdb.OpenDb(opts, dbPath)
	if err != nil {
		return err
	}

	defer db.Close()

	for _, cf := range columnfamilies {
		_, err = db.CreateColumnFamily(opts, cf)
		if err != nil {
			return err
		}
	}
	dbLogger.Debug("DB created at [%s]", dbPath)
	return nil
}

// GetDBHandle returns a handle to OpenchainDB
func GetDBHandle() *OpenchainDB {
	var err error
	if isOpen {
		return openchainDB
	}

	err = createDBIfDBPathEmpty()
	if err != nil {
		panic(fmt.Sprintf("Error while trying to create DB: %s", err))
	}

	openchainDB, err = openDB()
	if err != nil {
		panic(fmt.Sprintf("Could not open openchain db error = [%s]", err))
	}
	return openchainDB
}

// GetFromBlockchainCF get value for given key from column family - blockchainCF
func (openchainDB *OpenchainDB) GetFromBlockchainCF(key []byte) ([]byte, error) {
	return openchainDB.Get(openchainDB.BlockchainCF, key)
}

// GetFromBlockchainCFSnapshot get value for given key from column family in a DB snapshot - blockchainCF
func (openchainDB *OpenchainDB) GetFromBlockchainCFSnapshot(snapshot *gorocksdb.Snapshot, key []byte) ([]byte, error) {
	return openchainDB.getFromSnapshot(snapshot, openchainDB.BlockchainCF, key)
}

// GetFromStateCF get value for given key from column family - stateCF
func (openchainDB *OpenchainDB) GetFromStateCF(key []byte) ([]byte, error) {
	return openchainDB.Get(openchainDB.StateCF, key)
}

// GetFromStateDeltaCF get value for given key from column family - stateDeltaCF
func (openchainDB *OpenchainDB) GetFromStateDeltaCF(key []byte) ([]byte, error) {
	return openchainDB.Get(openchainDB.StateDeltaCF, key)
}

// GetFromIndexesCF get value for given key from column family - indexCF
func (openchainDB *OpenchainDB) GetFromIndexesCF(key []byte) ([]byte, error) {
	return openchainDB.Get(openchainDB.IndexesCF, key)
}

// GetBlockchainCFIterator get iterator for column family - blockchainCF
func (openchainDB *OpenchainDB) GetBlockchainCFIterator() *gorocksdb.Iterator {
	return openchainDB.getIterator(openchainDB.BlockchainCF)
}

// GetStateCFIterator get iterator for column family - stateCF
func (openchainDB *OpenchainDB) GetStateCFIterator() *gorocksdb.Iterator {
	return openchainDB.getIterator(openchainDB.StateCF)
}

// GetStateCFSnapshotIterator get iterator for column family - stateCF. This iterator
// is based on a snapshot and should be used for long running scans, such as
// reading the entire state. Remember to call iterator.Close() when you are done.
func (openchainDB *OpenchainDB) GetStateCFSnapshotIterator(snapshot *gorocksdb.Snapshot) *gorocksdb.Iterator {
	return openchainDB.getSnapshotIterator(snapshot, openchainDB.StateCF)
}

// GetStateDeltaCFIterator get iterator for column family - stateDeltaCF
func (openchainDB *OpenchainDB) GetStateDeltaCFIterator() *gorocksdb.Iterator {
	return openchainDB.getIterator(openchainDB.StateDeltaCF)
}

// GetSnapshot returns a point-in-time view of the DB. You MUST call snapshot.Release()
// when you are done with the snapshot.
func (openchainDB *OpenchainDB) GetSnapshot() *gorocksdb.Snapshot {
	return openchainDB.DB.NewSnapshot()
}

func getDBPath() string {
	dbPath := viper.GetString("peer.fileSystemPath")
	if dbPath == "" {
		panic("DB path not specified in configuration file. Please check that property 'peer.fileSystemPath' is set")
	}
	if !strings.HasSuffix(dbPath, "/") {
		dbPath = dbPath + "/"
	}
	return dbPath + "db"
}

func createDBIfDBPathEmpty() error {
	dbPath := getDBPath()
	missing, err := dirMissingOrEmpty(dbPath)
	if err != nil {
		return err
	}
	dbLogger.Debug("Is db path [%s] empty [%t]", dbPath, missing)
	if missing {
		err := CreateDB()
		if err != nil {
			return nil
		}
	}
	return nil
}

func openDB() (*OpenchainDB, error) {
	if isOpen {
		return openchainDB, nil
	}

	dbPath := getDBPath()
	opts := gorocksdb.NewDefaultOptions()
	defer opts.Destroy()

	opts.SetCreateIfMissing(false)

	cfNames := []string{"default"}
	cfNames = append(cfNames, columnfamilies...)
	var cfOpts []*gorocksdb.Options
	for _ = range cfNames {
		cfOpts = append(cfOpts, opts)
	}

	db, cfHandlers, err := gorocksdb.OpenDbColumnFamilies(opts, dbPath, cfNames, cfOpts)

	if err != nil {
		fmt.Println("Error opening DB", err)
		return nil, err
	}
	isOpen = true
	// XXX should we close cfHandlers[0]?
	return &OpenchainDB{db, cfHandlers[1], cfHandlers[2], cfHandlers[3], cfHandlers[4], cfHandlers[5]}, nil
}

// CloseDB releases all column family handles and closes rocksdb
func (openchainDB *OpenchainDB) CloseDB() {
	openchainDB.BlockchainCF.Destroy()
	openchainDB.StateCF.Destroy()
	openchainDB.StateDeltaCF.Destroy()
	openchainDB.IndexesCF.Destroy()
	openchainDB.PersistCF.Destroy()
	openchainDB.DB.Close()
	isOpen = false
}

// DeleteState delets ALL state keys/values from the DB. This is generally
// only used during state synchronization when creating a new state from
// a snapshot.
func (openchainDB *OpenchainDB) DeleteState() error {
	err := openchainDB.DB.DropColumnFamily(openchainDB.StateCF)
	if err != nil {
		dbLogger.Error("Error dropping state CF", err)
		return err
	}
	err = openchainDB.DB.DropColumnFamily(openchainDB.StateDeltaCF)
	if err != nil {
		dbLogger.Error("Error dropping state delta CF", err)
		return err
	}
	opts := gorocksdb.NewDefaultOptions()
	defer opts.Destroy()
	openchainDB.StateCF, err = openchainDB.DB.CreateColumnFamily(opts, stateCF)
	if err != nil {
		dbLogger.Error("Error creating state CF", err)
		return err
	}
	openchainDB.StateDeltaCF, err = openchainDB.DB.CreateColumnFamily(opts, stateDeltaCF)
	if err != nil {
		dbLogger.Error("Error creating state delta CF", err)
		return err
	}
	return nil
}

func (openchainDB *OpenchainDB) Get(cfHandler *gorocksdb.ColumnFamilyHandle, key []byte) ([]byte, error) {
	opt := gorocksdb.NewDefaultReadOptions()
	defer opt.Destroy()
	slice, err := openchainDB.DB.GetCF(opt, cfHandler, key)
	if err != nil {
		fmt.Println("Error while trying to retrieve key:", key)
		return nil, err
	}
	defer slice.Free()
	data := append([]byte(nil), slice.Data()...)
	return data, nil
}

func (openchainDB *OpenchainDB) Put(cfHandler *gorocksdb.ColumnFamilyHandle, key []byte, value []byte) error {
	opt := gorocksdb.NewDefaultWriteOptions()
	defer opt.Destroy()
	err := openchainDB.DB.PutCF(opt, cfHandler, key, value)
	if err != nil {
		fmt.Println("Error while trying to write key:", key)
		return err
	}
	return nil
}

func (openchainDB *OpenchainDB) getFromSnapshot(snapshot *gorocksdb.Snapshot, cfHandler *gorocksdb.ColumnFamilyHandle, key []byte) ([]byte, error) {
	opt := gorocksdb.NewDefaultReadOptions()
	defer opt.Destroy()
	opt.SetSnapshot(snapshot)
	slice, err := openchainDB.DB.GetCF(opt, cfHandler, key)
	if err != nil {
		fmt.Println("Error while trying to retrieve key:", key)
		return nil, err
	}
	defer slice.Free()
	data := append([]byte(nil), slice.Data()...)
	return data, nil
}

func (openchainDB *OpenchainDB) getIterator(cfHandler *gorocksdb.ColumnFamilyHandle) *gorocksdb.Iterator {
	opt := gorocksdb.NewDefaultReadOptions()
	opt.SetFillCache(true)
	defer opt.Destroy()
	return openchainDB.DB.NewIteratorCF(opt, cfHandler)
}

func (openchainDB *OpenchainDB) getSnapshotIterator(snapshot *gorocksdb.Snapshot, cfHandler *gorocksdb.ColumnFamilyHandle) *gorocksdb.Iterator {
	opt := gorocksdb.NewDefaultReadOptions()
	defer opt.Destroy()
	opt.SetSnapshot(snapshot)
	iter := openchainDB.DB.NewIteratorCF(opt, cfHandler)
	return iter
}

func dirMissingOrEmpty(path string) (bool, error) {
	dirExists, err := dirExists(path)
	if err != nil {
		return false, err
	}
	if !dirExists {
		return true, nil
	}

	dirEmpty, err := dirEmpty(path)
	if err != nil {
		return false, err
	}
	if dirEmpty {
		return true, nil
	}
	return false, nil
}

func dirExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func dirEmpty(path string) (bool, error) {
	f, err := os.Open(path)
	if err != nil {
		return false, err
	}
	defer f.Close()

	_, err = f.Readdir(1)
	if err == io.EOF {
		return true, nil
	}
	return false, err
}
<|endoftext|>


<commit_before>package executehelpers

import (
	"bufio"
	"bytes"
	"fmt"
	"net/http"
	"os"
	"os/exec"

	"github.com/concourse/atc"
	"github.com/concourse/fly/commands/internal/deprecated"
	"github.com/tedsuo/rata"
)

func Upload(input Input, excludeIgnored bool, atcRequester *deprecated.AtcRequester) {
	path := input.Path
	pipe := input.Pipe

	var files []string
	var err error

	if excludeIgnored {
		files, err = getGitFiles(path)
		if err != nil {
			fmt.Fprintln(os.Stderr, "could not determine ignored files:", err)
			return
		}
	} else {
		files = []string{"."}
	}

	archive, err := tarStreamFrom(path, files)
	if err != nil {
		fmt.Fprintln(os.Stderr, "could create tar stream:", err)
		return
	}

	defer archive.Close()

	uploadBits, err := atcRequester.CreateRequest(
		atc.WritePipe,
		rata.Params{"pipe_id": pipe.ID},
		archive,
	)
	if err != nil {
		panic(err)
	}

	response, err := atcRequester.HttpClient.Do(uploadBits)
	if err != nil {
		fmt.Fprintln(os.Stderr, "upload request failed:", err)
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		fmt.Fprintln(os.Stderr, badResponseError("uploading bits", response))
	}
}

func getGitFiles(dir string) ([]string, error) {
	tracked, err := gitLS(dir)
	if err != nil {
		return nil, err
	}

	untracked, err := gitLS(dir, "--others", "--exclude-standard")
	if err != nil {
		return nil, err
	}

	return append(tracked, untracked...), nil
}

func gitLS(dir string, flags ...string) ([]string, error) {
	files := []string{}

	gitLS := exec.Command("git", append([]string{"ls-files", "-z"}, flags...)...)
	gitLS.Dir = dir

	gitOut, err := gitLS.StdoutPipe()
	if err != nil {
		return nil, err
	}

	outScan := bufio.NewScanner(gitOut)
	outScan.Split(scanNull)

	err = gitLS.Start()
	if err != nil {
		return nil, err
	}

	for outScan.Scan() {
		files = append(files, outScan.Text())
	}

	err = gitLS.Wait()
	if err != nil {
		return nil, err
	}

	return files, nil
}

func scanNull(data []byte, atEOF bool) (int, []byte, error) {
	// eof, no more data; terminate
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	// look for terminating null byte
	if i := bytes.IndexByte(data, 0); i >= 0 {
		return i + 1, data[0:i], nil
	}

	// no final terminator; return what's left
	if atEOF {
		return len(data), data, nil
	}

	// request more data
	return 0, nil, nil
}
<commit_msg>Fix nil pointer dereference in upload error handling<commit_after>package executehelpers

import (
	"bufio"
	"bytes"
	"fmt"
	"net/http"
	"os"
	"os/exec"

	"github.com/concourse/atc"
	"github.com/concourse/fly/commands/internal/deprecated"
	"github.com/tedsuo/rata"
)

func Upload(input Input, excludeIgnored bool, atcRequester *deprecated.AtcRequester) {
	path := input.Path
	pipe := input.Pipe

	var files []string
	var err error

	if excludeIgnored {
		files, err = getGitFiles(path)
		if err != nil {
			fmt.Fprintln(os.Stderr, "could not determine ignored files:", err)
			return
		}
	} else {
		files = []string{"."}
	}

	archive, err := tarStreamFrom(path, files)
	if err != nil {
		fmt.Fprintln(os.Stderr, "could create tar stream:", err)
		return
	}

	defer archive.Close()

	uploadBits, err := atcRequester.CreateRequest(
		atc.WritePipe,
		rata.Params{"pipe_id": pipe.ID},
		archive,
	)
	if err != nil {
		panic(err)
	}

	response, err := atcRequester.HttpClient.Do(uploadBits)
	if err != nil {
		fmt.Fprintln(os.Stderr, "upload request failed:", err)
		return
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		fmt.Fprintln(os.Stderr, badResponseError("uploading bits", response))
	}
}

func getGitFiles(dir string) ([]string, error) {
	tracked, err := gitLS(dir)
	if err != nil {
		return nil, err
	}

	untracked, err := gitLS(dir, "--others", "--exclude-standard")
	if err != nil {
		return nil, err
	}

	return append(tracked, untracked...), nil
}

func gitLS(dir string, flags ...string) ([]string, error) {
	files := []string{}

	gitLS := exec.Command("git", append([]string{"ls-files", "-z"}, flags...)...)
	gitLS.Dir = dir

	gitOut, err := gitLS.StdoutPipe()
	if err != nil {
		return nil, err
	}

	outScan := bufio.NewScanner(gitOut)
	outScan.Split(scanNull)

	err = gitLS.Start()
	if err != nil {
		return nil, err
	}

	for outScan.Scan() {
		files = append(files, outScan.Text())
	}

	err = gitLS.Wait()
	if err != nil {
		return nil, err
	}

	return files, nil
}

func scanNull(data []byte, atEOF bool) (int, []byte, error) {
	// eof, no more data; terminate
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	// look for terminating null byte
	if i := bytes.IndexByte(data, 0); i >= 0 {
		return i + 1, data[0:i], nil
	}

	// no final terminator; return what's left
	if atEOF {
		return len(data), data, nil
	}

	// request more data
	return 0, nil, nil
}
<|endoftext|>


<commit_before>package dpn

import (
	"github.com/APTrust/bagman/bagman"
	"net/url"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

// cleanup.go cleans up files in the DPN staging area.
// This typically runs as a cron job.

type Cleanup struct {
	DPNConfig           *DPNConfig
	ProcUtil            *bagman.ProcessUtil
	LocalClient         *DPNRestClient
}

func NewCleanup(procUtil *bagman.ProcessUtil, dpnConfig *DPNConfig) (*Cleanup, error) {
	localClient, err := NewDPNRestClient(
		dpnConfig.RestClient.LocalServiceURL,
		dpnConfig.RestClient.LocalAPIRoot,
		dpnConfig.RestClient.LocalAuthToken,
		dpnConfig.LocalNode,
		dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, err
	}
	return &Cleanup {
		DPNConfig: dpnConfig,
		ProcUtil: procUtil,
		LocalClient: localClient,
	}, nil
}

func (cleanup *Cleanup) DeleteReplicatedBags() {
	cleanup.ProcUtil.MessageLog.Info("Deleting replicated bags in %s",
		cleanup.ProcUtil.Config.DPNStagingDirectory)
	files, err := ioutil.ReadDir(cleanup.ProcUtil.Config.DPNStagingDirectory)
	if err != nil {
		cleanup.ProcUtil.MessageLog.Error(err.Error())
		return
	}
	for _, finfo := range files {
		bagUUID := strings.Replace(finfo.Name(), ".tar", "", 1)
		params := &url.Values{}
		params.Set("uuid", bagUUID)
		params.Set("status", "stored") // stored == 4
		params.Set("from_node", cleanup.DPNConfig.LocalNode)
		result, err := cleanup.LocalClient.DPNReplicationListGet(params)
		if err != nil {
			cleanup.ProcUtil.MessageLog.Error("Error getting replication info for bag '%s': %v",
				bagUUID, err.Error())
		}
		tarfile := filepath.Join(cleanup.ProcUtil.Config.DPNStagingDirectory, finfo.Name())
		if result.Count >= int32(cleanup.DPNConfig.ReplicateToNumNodes) {
			cleanup.ProcUtil.MessageLog.Info("Deleting %s: %d successful replications",
				tarfile, result.Count)
			err = os.Remove(tarfile)
		} else {
			cleanup.ProcUtil.MessageLog.Info("Leaving %s: %d successful replications",
				tarfile, result.Count)
		}
	}
}
<commit_msg>Fixed PT #122207709: delete symlinks, and PT #122132433: don't delete directories<commit_after>package dpn

import (
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"net/url"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

// cleanup.go cleans up files in the DPN staging area.
// This typically runs as a cron job.

type Cleanup struct {
	DPNConfig           *DPNConfig
	ProcUtil            *bagman.ProcessUtil
	LocalClient         *DPNRestClient
}

func NewCleanup(procUtil *bagman.ProcessUtil, dpnConfig *DPNConfig) (*Cleanup, error) {
	localClient, err := NewDPNRestClient(
		dpnConfig.RestClient.LocalServiceURL,
		dpnConfig.RestClient.LocalAPIRoot,
		dpnConfig.RestClient.LocalAuthToken,
		dpnConfig.LocalNode,
		dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, err
	}
	return &Cleanup {
		DPNConfig: dpnConfig,
		ProcUtil: procUtil,
		LocalClient: localClient,
	}, nil
}

func (cleanup *Cleanup) DeleteReplicatedBags() {
	cleanup.ProcUtil.MessageLog.Info("Deleting replicated bags in %s",
		cleanup.ProcUtil.Config.DPNStagingDirectory)
	files, err := ioutil.ReadDir(cleanup.ProcUtil.Config.DPNStagingDirectory)
	if err != nil {
		cleanup.ProcUtil.MessageLog.Error(err.Error())
		return
	}
	for _, finfo := range files {
		bagUUID := strings.Replace(finfo.Name(), ".tar", "", 1)
		if !bagman.LooksLikeUUID(bagUUID) {
			continue  // Don't delete it if it's not a DPN tar file
		}
		params := &url.Values{}
		params.Set("uuid", bagUUID)
		params.Set("status", "stored") // stored == 4
		params.Set("from_node", cleanup.DPNConfig.LocalNode)
		result, err := cleanup.LocalClient.DPNReplicationListGet(params)
		if err != nil {
			cleanup.ProcUtil.MessageLog.Error("Error getting replication info for bag '%s': %v",
				bagUUID, err.Error())
			continue
		}
		tarfile := filepath.Join(cleanup.ProcUtil.Config.DPNStagingDirectory, finfo.Name())
		if result.Count >= int32(cleanup.DPNConfig.ReplicateToNumNodes) {
			cleanup.ProcUtil.MessageLog.Info("Deleting %s: %d successful replications",
				tarfile, result.Count)
			err = os.Remove(tarfile)
			for _, xfer := range result.Results {
				symlink := fmt.Sprintf("%s/dpn.%s/outbound/%s.tar",
					cleanup.ProcUtil.Config.DPNHomeDirectory, xfer.ToNode, bagUUID)
				cleanup.ProcUtil.MessageLog.Info("Deleting symlink at %s", symlink)
			}
		} else {
			cleanup.ProcUtil.MessageLog.Info("Leaving %s: %d successful replications",
				tarfile, result.Count)
		}
	}
}
<|endoftext|>


<commit_before>package rrs

import (
	"fmt"
	"testing"
	"time"

	"github.com/facebookgo/ensure"
	"github.com/teh-cmc/seq"
)

// -----------------------------------------------------------------------------

// NOTE: run these tests with `go test -race -cpu 1,8,32`

func TestRRSeq_New_BufSize(t *testing.T) {
	var rrseq *RRSeq
	var err error

	rrseq, err = NewRRSeq(testingRRSeqDefaultName, -42, testingRRServerAddrs...)
	if err != nil {
		t.Fatal(err)
	}
	ensure.DeepEqual(t, cap(rrseq.ids), 0)

	rrseq, err = NewRRSeq(testingRRSeqDefaultName, 0, testingRRServerAddrs...)
	if err != nil {
		t.Fatal(err)
	}
	ensure.DeepEqual(t, cap(rrseq.ids), 0)

	rrseq, err = NewRRSeq(testingRRSeqDefaultName, 1, testingRRServerAddrs...)
	if err != nil {
		t.Fatal(err)
	}
	ensure.DeepEqual(t, cap(rrseq.ids), 1)

	rrseq, err = NewRRSeq(testingRRSeqDefaultName, 1e6, testingRRServerAddrs...)
	if err != nil {
		t.Fatal(err)
	}
	ensure.DeepEqual(t, cap(rrseq.ids), int(1e6))
}

func TestRRSeq_FirstID(t *testing.T) {
	rrseq, err := NewRRSeq("TestRRSeq_FirstID", 1e2, testingRRServerAddrs...)
	if err != nil {
		t.Fatal(err)
	}
	ensure.DeepEqual(t, <-rrseq.GetStream(), seq.ID(1))
}

// -----------------------------------------------------------------------------

func testRRSeq_SingleClient(bufSize int, t *testing.T) {
	name := fmt.Sprintf("testRRSeq_SingleClient(%d)", bufSize)
	s, err := NewRRSeq(name, bufSize, testingRRServerAddrs...)
	if err != nil {
		t.Fatal(err)
	}
	lastID := seq.ID(0)

	go func() {
		<-time.After(time.Millisecond * 250)
		_ = s.Close()
	}()

	for id := range s.GetStream() {
		ensure.DeepEqual(t, id, lastID+1)
		lastID = id
	}
}

func TestRRSeq_BufSize0_SingleClient(t *testing.T) {
	testRRSeq_SingleClient(0, t)
}

func TestRRSeq_BufSize1_SingleClient(t *testing.T) {
	testRRSeq_SingleClient(1, t)
}

func TestRRSeq_BufSize1024_SingleClient(t *testing.T) {
	testRRSeq_SingleClient(1024, t)
}
<commit_msg>made RRSeq tests compatible with multiple -cpu values<commit_after>package rrs

import (
	"fmt"
	"runtime"
	"testing"
	"time"

	"github.com/facebookgo/ensure"
	"github.com/teh-cmc/seq"
)

// -----------------------------------------------------------------------------

// NOTE: run these tests with `go test -race -cpu 1,8,32`

func TestRRSeq_New_BufSize(t *testing.T) {
	var rrseq *RRSeq
	var err error

	name := fmt.Sprintf("TestRRSeq_New_BufSize(gomaxprocs:%d)", runtime.GOMAXPROCS(0))

	rrseq, err = NewRRSeq(name, -42, testingRRServerAddrs...)
	if err != nil {
		t.Fatal(err)
	}
	ensure.DeepEqual(t, cap(rrseq.ids), 0)

	rrseq, err = NewRRSeq(name, 0, testingRRServerAddrs...)
	if err != nil {
		t.Fatal(err)
	}
	ensure.DeepEqual(t, cap(rrseq.ids), 0)

	rrseq, err = NewRRSeq(name, 1, testingRRServerAddrs...)
	if err != nil {
		t.Fatal(err)
	}
	ensure.DeepEqual(t, cap(rrseq.ids), 1)

	rrseq, err = NewRRSeq(name, 1e6, testingRRServerAddrs...)
	if err != nil {
		t.Fatal(err)
	}
	ensure.DeepEqual(t, cap(rrseq.ids), int(1e6))
}

func TestRRSeq_FirstID(t *testing.T) {
	name := fmt.Sprintf("TestRRSeq_FirstID(gomaxprocs:%d)", runtime.GOMAXPROCS(0))
	rrseq, err := NewRRSeq(name, 1e2, testingRRServerAddrs...)
	if err != nil {
		t.Fatal(err)
	}
	ensure.DeepEqual(t, <-rrseq.GetStream(), seq.ID(1))
}

// -----------------------------------------------------------------------------

func testRRSeq_SingleClient(bufSize int, t *testing.T) {
	name := fmt.Sprintf(
		"testRRSeq_SingleClient(bufsz:%d)(gomaxprocs:%d)", bufSize, runtime.GOMAXPROCS(0),
	)
	s, err := NewRRSeq(name, bufSize, testingRRServerAddrs...)
	if err != nil {
		t.Fatal(err)
	}
	lastID := seq.ID(0)

	go func() {
		<-time.After(time.Millisecond * 500)
		_ = s.Close()
	}()

	for id := range s.GetStream() {
		ensure.DeepEqual(t, id, lastID+1)
		lastID = id
	}
}

func TestRRSeq_BufSize0_SingleClient(t *testing.T) {
	testRRSeq_SingleClient(0, t)
}

func TestRRSeq_BufSize1_SingleClient(t *testing.T) {
	testRRSeq_SingleClient(1, t)
}

func TestRRSeq_BufSize1024_SingleClient(t *testing.T) {
	testRRSeq_SingleClient(1024, t)
}
<|endoftext|>