<commit_before>package assembly

import (
	"fmt"
	"net"
	"sync"
	"time"
)

// We consider sequence #0 to be invalid... this isn't actually the case, but it
// should show up very infrequently (1/4billion), so this should be okay.
const invalidSequence = 0

// Sequence is a TCP sequence number.
type Sequence uint32

// Difference defines an ordering for comparing TCP sequences that's safe for
// roll-overs.  It returns:
//    > 0 : if t comes after s
//    < 0 : if t comes before s
//      0 : if t == s
// The number returned is the sequence difference, so 4.Difference(8) will
// return 4.
//
// It handles rollovers by considering any sequence in the first quarter of the
// uint32 space to be after any sequence in the last quarter of that space, thus
// wrapping the uint32 space.
func (s Sequence) Difference(t Sequence) int {
	if s > 0xFFFFFFFF-0xFFFFFFFF/4 && t < 0xFFFFFFFF/4 {
		t += 0xFFFFFFFF
	} else if t > 0xFFFFFFFF-0xFFFFFFFF/4 && s < 0xFFFFFFFF/4 {
		s += 0xFFFFFFFF
	}
	return int(t - s)
}

// Add adds an integer to a sequence and returns the resulting sequence.
func (s Sequence) Add(t int) Sequence {
	return s + Sequence(t)
}

// Reassembly objects are returned by the assembler in order.
type Reassembly struct {
	// Bytes is the next set of bytes in the stream.  May be empty.
	Bytes []byte
	// Seq is the current TCP sequence for this reassembly.
	Seq Sequence
	// Skip is set to true if this reassembly has skipped some number of bytes.
	// This normally occurs if packets were dropped, or if we picked up the stream
	// after it had already started sending data (IE: we start our packet capture
	// mid-stream).
	Skip bool
	// Start is set if this set of bytes has a TCP SYN accompanying it.
	Start bool
	// End is set if this set of bytes has a TCP FIN or RST accompanying it.
	End bool
}

const pageBytes = 1900

// page is used to store TCP data we're not ready for yet (out-of-order
// packets).  Unused pages are stored in and returned from a pageCache, which
// avoids memory allocation.
type page struct {
	Reassembly
	index      int
	prev, next *page
	created    time.Time
	buf        [pageBytes]byte
}

// pageCache is a concurrency-unsafe store of page objects we use to avoid
// memory allocation as much as we can.  It grows but never shrinks.
type pageCache struct {
	free       []*page
	pcSize     int
	size, used int
}

func newPageCache(pcSize int) *pageCache {
	pc := &pageCache{
		free:   make([]*page, 0, pcSize),
		pcSize: pcSize,
	}
	pc.grow()
	return pc
}

// grow exponentially increases the size of our page cache as much as necessary.
func (c *pageCache) grow() {
	pages := make([]page, c.pcSize)
	c.size += c.pcSize
	for i, _ := range pages {
		c.free = append(c.free, &pages[i])
	}
	c.pcSize *= 2
}

// next returns a clean, ready-to-use page object.
func (c *pageCache) next() (p *page) {
	if len(c.free) == 0 {
		c.grow()
	}
	i := len(c.free) - 1
	p, c.free = c.free[i], c.free[:i]
	p.prev = nil
	p.next = nil
	p.created = time.Now()
	p.Bytes = p.buf[:0]
	c.used++
	return p
}

// replace replaces a page into the pageCache.
func (c *pageCache) replace(p *page) {
	c.used--
	c.free = append(c.free, p)
}

var zeros []byte = make([]byte, 12)

// Key is a unique identifier for a TCP stream.
type Key struct {
	Version          byte // IP version, 4 or 6
	SrcIP, DstIP     [16]byte
	SrcPort, DstPort uint16
}

// Reset resets the given key with new source/destination IPs/ports.
func (k *Key) Reset(sip, dip net.IP, sp, dp uint16) {
	if len(sip) != len(dip) {
		panic("IP lengths don't match")
	}
	oldVersion := k.Version
	switch len(sip) {
	case 4:
		k.Version = 4
		copy(k.SrcIP[:4], sip)
		copy(k.DstIP[:4], dip)
		if oldVersion != 4 {
			copy(k.SrcIP[4:], zeros)
			copy(k.DstIP[4:], zeros)
		}
	case 16:
		k.Version = 6
		copy(k.SrcIP[:], sip)
		copy(k.DstIP[:], dip)
	default:
		panic("Invalid IP length")
	}
	k.SrcPort = sp
	k.DstPort = dp
}

type TCP struct {
	Key           Key
	Seq           Sequence
	SYN, FIN, RST bool
	Bytes         []byte
}

type Assembler interface {
	Assemble(t *TCP)
	Buffered() int
	FlushOlderThan(time.Time)
}

type Stream interface {
	Reassembled([]Reassembly)
	ReassemblyComplete()
}

type StreamFactory interface {
	New(k Key) Stream
}

func (a *assembler) Buffered() int {
	return a.pc.used
}

func (a *assembler) FlushOlderThan(t time.Time) {
	start := time.Now()
	fmt.Println("Flushing connections older than", t)
	a.connPool.mu.RLock()
	conns := make([]*connection, 0, len(a.connPool.conns))
	for _, conn := range a.connPool.conns {
		conns = append(conns, conn)
	}
	a.connPool.mu.RUnlock()
	closes := 0
	flushes := 0
	for _, conn := range conns {
		conn.mu.Lock()
		if (conn.first != nil && conn.first.created.Before(t)) || (conn.first == nil && conn.lastSeen.Before(t)) {
			flushes++
			a.skipFlush(conn)
			if conn.closed {
				closes++
			}
		}
		conn.mu.Unlock()
	}
	fmt.Println("Flush completed in", time.Since(start), "closed", closes, "flushed", flushes)
}

type ConnectionPool struct {
	conns   map[Key]*connection
	users   int
	mu      sync.RWMutex
	factory StreamFactory
}

func NewConnectionPool(factory StreamFactory) *ConnectionPool {
	return &ConnectionPool{
		conns:   make(map[Key]*connection),
		factory: factory,
	}
}

func NewAssembler(max, maxPer, pcSize int, pool *ConnectionPool) Assembler {
	pool.mu.Lock()
	pool.users++
	pool.mu.Unlock()
	return &assembler{
		ret:            make([]Reassembly, maxPer+1),
		pc:             newPageCache(pcSize),
		connPool:       pool,
		maxBuffered:    max,
		maxBufferedPer: maxPer,
	}
}

type connection struct {
	key               Key
	pages             int
	first, last       *page
	nextSeq           Sequence
	created, lastSeen time.Time
	stream            Stream
	closed            bool
	mu                sync.Mutex
}

type assembler struct {
	ret            []Reassembly
	pc             *pageCache
	maxBuffered    int
	maxBufferedPer int
	connPool       *ConnectionPool
}

func (p *ConnectionPool) newConnection(k *Key) *connection {
	return &connection{
		key:     *k,
		nextSeq: invalidSequence,
		created: time.Now(),
		stream:  p.factory.New(*k),
	}
}

func (p *ConnectionPool) getConnection(k *Key) *connection {
	p.mu.RLock()
	conn := p.conns[*k]
	p.mu.RUnlock()
	if conn != nil {
		return conn
	}
	conn = p.newConnection(k)
	p.mu.Lock()
	if conn2 := p.conns[*k]; conn2 != nil {
		p.mu.Unlock()
		return conn2
	}
	p.conns[*k] = conn
	p.mu.Unlock()
	return conn
}

func (a *assembler) Assemble(t *TCP) {
	a.ret = a.ret[:0]
	conn := a.connPool.getConnection(&t.Key)
	conn.mu.Lock()
	conn.lastSeen = time.Now()
	if t.SYN {
		a.ret = append(a.ret, Reassembly{
			Bytes: t.Bytes,
			Seq:   t.Seq,
			Skip:  false,
			Start: true,
		})
		conn.nextSeq = t.Seq.Add(len(t.Bytes) + 1)
	} else if conn.nextSeq == invalidSequence || conn.nextSeq.Difference(t.Seq) > 0 {
		a.insertIntoConn(t, conn)
	} else {
		span := int(t.Seq.Difference(conn.nextSeq))
		if len(t.Bytes) > span {
			a.ret = append(a.ret, Reassembly{
				Bytes: t.Bytes[span:],
				Seq:   t.Seq + Sequence(span),
				Skip:  false,
				End:   t.RST || t.FIN,
			})
			conn.nextSeq = t.Seq.Add(len(t.Bytes))
		}
	}
	if len(a.ret) > 0 {
		a.sendToConnection(conn)
	}
	conn.mu.Unlock()
}

func (a *assembler) sendToConnection(conn *connection) {
	a.addContiguous(conn)
	conn.stream.Reassembled(a.ret)
	if a.ret[len(a.ret)-1].End {
		a.close(conn)
	}
}

func (a *assembler) addContiguous(conn *connection) {
	for conn.first != nil && conn.first.Seq == conn.nextSeq {
		a.addNextFromConn(conn, false)
	}
}

func (a *assembler) skipFlush(conn *connection) {
	if conn.first == nil {
		a.close(conn)
	} else {
		a.ret = a.ret[:0]
		a.addNextFromConn(conn, true)
		a.sendToConnection(conn)
	}
}

func (a *assembler) close(conn *connection) {
	conn.stream.ReassemblyComplete()
	a.connPool.mu.Lock()
	delete(a.connPool.conns, conn.key)
	a.connPool.mu.Unlock()
	for p := conn.first; p != nil; p = p.next {
		a.pc.replace(p)
	}
	conn.closed = true
}

func (conn *connection) traverseConn(t *TCP) (prev, current *page) {
	prev = conn.last
	for prev != nil && prev.Seq.Difference(t.Seq) < 0 {
		current = prev
		prev = current.prev
	}
	return
}

func (a *assembler) insertIntoConn(t *TCP, conn *connection) {
	p := a.pageFromTcp(t)
	prev, current := conn.traverseConn(t)
	// Maintain our doubly linked list
	if current == nil || conn.last == nil {
		conn.last = p
	} else {
		p.next = current
		current.prev = p
	}
	if prev == nil || conn.first == nil {
		conn.first = p
	} else {
		p.prev = prev
		prev.next = p
	}
	conn.pages++
	if conn.pages >= a.maxBufferedPer || a.pc.used >= a.maxBuffered {
		a.addNextFromConn(conn, true)
	}
}

// pageFromTcp creates a page (or set of pages) from a TCP packet.  Note that it
// should NEVER receive a SYN packet, as it doesn't handle sequences correctly.
func (a *assembler) pageFromTcp(t *TCP) *page {
	first := a.pc.next()
	current := first
	for {
		length := min(len(t.Bytes), pageBytes)
		current.Bytes = current.buf[:length]
		copy(current.Bytes, t.Bytes)
		current.Seq = t.Seq
		t.Bytes = t.Bytes[length:]
		if len(t.Bytes) == 0 {
			break
		}
		t.Seq = t.Seq.Add(length)
		current.next = a.pc.next()
		current = current.next
	}
	current.End = t.RST || t.FIN
	return first
}

func (a *assembler) addNextFromConn(conn *connection, skip bool) {
	conn.first.Skip = skip
	a.ret = append(a.ret, conn.first.Reassembly)
	conn.nextSeq = conn.first.Seq.Add(len(conn.first.Bytes))
	a.pc.replace(conn.first)
	if conn.first == conn.last {
		conn.first = nil
		conn.last = nil
	} else {
		conn.first = conn.first.next
		conn.first.prev = nil
	}
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<commit_msg>Fix sequence number issue.<commit_after>package assembly

import (
	"fmt"
	"net"
	"sync"
	"time"
)

const invalidSequence = -1
const uint32Max = 0xFFFFFFFF

// Sequence is a TCP sequence number.
type Sequence int64

// Difference defines an ordering for comparing TCP sequences that's safe for
// roll-overs.  It returns:
//    > 0 : if t comes after s
//    < 0 : if t comes before s
//      0 : if t == s
// The number returned is the sequence difference, so 4.Difference(8) will
// return 4.
//
// It handles rollovers by considering any sequence in the first quarter of the
// uint32 space to be after any sequence in the last quarter of that space, thus
// wrapping the uint32 space.
func (s Sequence) Difference(t Sequence) int {
	if s > uint32Max-uint32Max/4 && t < uint32Max/4 {
		t += uint32Max
	} else if t > uint32Max-uint32Max/4 && s < uint32Max/4 {
		s += uint32Max
	}
	return int(t - s)
}

// Add adds an integer to a sequence and returns the resulting sequence.
func (s Sequence) Add(t int) Sequence {
	return (s + Sequence(t)) & uint32Max
}

// Reassembly objects are returned by the assembler in order.
type Reassembly struct {
	// Bytes is the next set of bytes in the stream.  May be empty.
	Bytes []byte
	// Seq is the current TCP sequence for this reassembly.
	Seq Sequence
	// Skip is set to true if this reassembly has skipped some number of bytes.
	// This normally occurs if packets were dropped, or if we picked up the stream
	// after it had already started sending data (IE: we start our packet capture
	// mid-stream).
	Skip bool
	// Start is set if this set of bytes has a TCP SYN accompanying it.
	Start bool
	// End is set if this set of bytes has a TCP FIN or RST accompanying it.
	End bool
}

const pageBytes = 1900

// page is used to store TCP data we're not ready for yet (out-of-order
// packets).  Unused pages are stored in and returned from a pageCache, which
// avoids memory allocation.
type page struct {
	Reassembly
	index      int
	prev, next *page
	created    time.Time
	buf        [pageBytes]byte
}

// pageCache is a concurrency-unsafe store of page objects we use to avoid
// memory allocation as much as we can.  It grows but never shrinks.
type pageCache struct {
	free       []*page
	pcSize     int
	size, used int
}

func newPageCache(pcSize int) *pageCache {
	pc := &pageCache{
		free:   make([]*page, 0, pcSize),
		pcSize: pcSize,
	}
	pc.grow()
	return pc
}

// grow exponentially increases the size of our page cache as much as necessary.
func (c *pageCache) grow() {
	pages := make([]page, c.pcSize)
	c.size += c.pcSize
	for i, _ := range pages {
		c.free = append(c.free, &pages[i])
	}
	c.pcSize *= 2
}

// next returns a clean, ready-to-use page object.
func (c *pageCache) next() (p *page) {
	if len(c.free) == 0 {
		c.grow()
	}
	i := len(c.free) - 1
	p, c.free = c.free[i], c.free[:i]
	p.prev = nil
	p.next = nil
	p.created = time.Now()
	p.Bytes = p.buf[:0]
	c.used++
	return p
}

// replace replaces a page into the pageCache.
func (c *pageCache) replace(p *page) {
	c.used--
	c.free = append(c.free, p)
}

var zeros []byte = make([]byte, 12)

// Key is a unique identifier for a TCP stream.
type Key struct {
	Version          byte // IP version, 4 or 6
	SrcIP, DstIP     [16]byte
	SrcPort, DstPort uint16
}

// Reset resets the given key with new source/destination IPs/ports.
func (k *Key) Reset(sip, dip net.IP, sp, dp uint16) {
	if len(sip) != len(dip) {
		panic("IP lengths don't match")
	}
	oldVersion := k.Version
	switch len(sip) {
	case 4:
		k.Version = 4
		copy(k.SrcIP[:4], sip)
		copy(k.DstIP[:4], dip)
		if oldVersion != 4 {
			copy(k.SrcIP[4:], zeros)
			copy(k.DstIP[4:], zeros)
		}
	case 16:
		k.Version = 6
		copy(k.SrcIP[:], sip)
		copy(k.DstIP[:], dip)
	default:
		panic("Invalid IP length")
	}
	k.SrcPort = sp
	k.DstPort = dp
}

type TCP struct {
	Key           Key
	Seq           Sequence
	SYN, FIN, RST bool
	Bytes         []byte
}

type Assembler interface {
	Assemble(t *TCP)
	Buffered() int
	FlushOlderThan(time.Time)
}

type Stream interface {
	Reassembled([]Reassembly)
	ReassemblyComplete()
}

type StreamFactory interface {
	New(k Key) Stream
}

func (a *assembler) Buffered() int {
	return a.pc.used
}

func (a *assembler) FlushOlderThan(t time.Time) {
	start := time.Now()
	fmt.Println("Flushing connections older than", t)
	a.connPool.mu.RLock()
	conns := make([]*connection, 0, len(a.connPool.conns))
	for _, conn := range a.connPool.conns {
		conns = append(conns, conn)
	}
	a.connPool.mu.RUnlock()
	closes := 0
	flushes := 0
	for _, conn := range conns {
		conn.mu.Lock()
		if (conn.first != nil && conn.first.created.Before(t)) || (conn.first == nil && conn.lastSeen.Before(t)) {
			flushes++
			a.skipFlush(conn)
			if conn.closed {
				closes++
			}
		}
		conn.mu.Unlock()
	}
	fmt.Println("Flush completed in", time.Since(start), "closed", closes, "flushed", flushes)
}

type ConnectionPool struct {
	conns   map[Key]*connection
	users   int
	mu      sync.RWMutex
	factory StreamFactory
}

func NewConnectionPool(factory StreamFactory) *ConnectionPool {
	return &ConnectionPool{
		conns:   make(map[Key]*connection),
		factory: factory,
	}
}

func NewAssembler(max, maxPer, pcSize int, pool *ConnectionPool) Assembler {
	pool.mu.Lock()
	pool.users++
	pool.mu.Unlock()
	return &assembler{
		ret:            make([]Reassembly, maxPer+1),
		pc:             newPageCache(pcSize),
		connPool:       pool,
		maxBuffered:    max,
		maxBufferedPer: maxPer,
	}
}

type connection struct {
	key               Key
	pages             int
	first, last       *page
	nextSeq           Sequence
	created, lastSeen time.Time
	stream            Stream
	closed            bool
	mu                sync.Mutex
}

type assembler struct {
	ret            []Reassembly
	pc             *pageCache
	maxBuffered    int
	maxBufferedPer int
	connPool       *ConnectionPool
}

func (p *ConnectionPool) newConnection(k *Key) *connection {
	return &connection{
		key:     *k,
		nextSeq: invalidSequence,
		created: time.Now(),
		stream:  p.factory.New(*k),
	}
}

func (p *ConnectionPool) getConnection(k *Key) *connection {
	p.mu.RLock()
	conn := p.conns[*k]
	p.mu.RUnlock()
	if conn != nil {
		return conn
	}
	conn = p.newConnection(k)
	p.mu.Lock()
	if conn2 := p.conns[*k]; conn2 != nil {
		p.mu.Unlock()
		return conn2
	}
	p.conns[*k] = conn
	p.mu.Unlock()
	return conn
}

func (a *assembler) Assemble(t *TCP) {
	a.ret = a.ret[:0]
	conn := a.connPool.getConnection(&t.Key)
	conn.mu.Lock()
	conn.lastSeen = time.Now()
	if t.SYN {
		a.ret = append(a.ret, Reassembly{
			Bytes: t.Bytes,
			Seq:   t.Seq,
			Skip:  false,
			Start: true,
		})
		conn.nextSeq = t.Seq.Add(len(t.Bytes) + 1)
	} else if conn.nextSeq == invalidSequence || conn.nextSeq.Difference(t.Seq) > 0 {
		a.insertIntoConn(t, conn)
	} else {
		span := int(t.Seq.Difference(conn.nextSeq))
		if len(t.Bytes) > span {
			a.ret = append(a.ret, Reassembly{
				Bytes: t.Bytes[span:],
				Seq:   t.Seq + Sequence(span),
				Skip:  false,
				End:   t.RST || t.FIN,
			})
			conn.nextSeq = t.Seq.Add(len(t.Bytes))
		}
	}
	if len(a.ret) > 0 {
		a.sendToConnection(conn)
	}
	conn.mu.Unlock()
}

func (a *assembler) sendToConnection(conn *connection) {
	a.addContiguous(conn)
	conn.stream.Reassembled(a.ret)
	if a.ret[len(a.ret)-1].End {
		a.close(conn)
	}
}

func (a *assembler) addContiguous(conn *connection) {
	for conn.first != nil && conn.first.Seq == conn.nextSeq {
		a.addNextFromConn(conn, false)
	}
}

func (a *assembler) skipFlush(conn *connection) {
	if conn.first == nil {
		a.close(conn)
	} else {
		a.ret = a.ret[:0]
		a.addNextFromConn(conn, true)
		a.sendToConnection(conn)
	}
}

func (a *assembler) close(conn *connection) {
	conn.stream.ReassemblyComplete()
	a.connPool.mu.Lock()
	delete(a.connPool.conns, conn.key)
	a.connPool.mu.Unlock()
	for p := conn.first; p != nil; p = p.next {
		a.pc.replace(p)
	}
	conn.closed = true
}

func (conn *connection) traverseConn(t *TCP) (prev, current *page) {
	prev = conn.last
	for prev != nil && prev.Seq.Difference(t.Seq) < 0 {
		current = prev
		prev = current.prev
	}
	return
}

func (a *assembler) insertIntoConn(t *TCP, conn *connection) {
	p := a.pageFromTcp(t)
	prev, current := conn.traverseConn(t)
	// Maintain our doubly linked list
	if current == nil || conn.last == nil {
		conn.last = p
	} else {
		p.next = current
		current.prev = p
	}
	if prev == nil || conn.first == nil {
		conn.first = p
	} else {
		p.prev = prev
		prev.next = p
	}
	conn.pages++
	if conn.pages >= a.maxBufferedPer || a.pc.used >= a.maxBuffered {
		a.addNextFromConn(conn, true)
	}
}

// pageFromTcp creates a page (or set of pages) from a TCP packet.  Note that it
// should NEVER receive a SYN packet, as it doesn't handle sequences correctly.
func (a *assembler) pageFromTcp(t *TCP) *page {
	first := a.pc.next()
	current := first
	for {
		length := min(len(t.Bytes), pageBytes)
		current.Bytes = current.buf[:length]
		copy(current.Bytes, t.Bytes)
		current.Seq = t.Seq
		t.Bytes = t.Bytes[length:]
		if len(t.Bytes) == 0 {
			break
		}
		t.Seq = t.Seq.Add(length)
		current.next = a.pc.next()
		current = current.next
	}
	current.End = t.RST || t.FIN
	return first
}

func (a *assembler) addNextFromConn(conn *connection, skip bool) {
	conn.first.Skip = skip
	a.ret = append(a.ret, conn.first.Reassembly)
	conn.nextSeq = conn.first.Seq.Add(len(conn.first.Bytes))
	a.pc.replace(conn.first)
	if conn.first == conn.last {
		conn.first = nil
		conn.last = nil
	} else {
		conn.first = conn.first.next
		conn.first.prev = nil
	}
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package auction_cell_rep

import (
	"errors"
	"fmt"
	"strconv"

	"github.com/cloudfoundry-incubator/auction/auctiontypes"
	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/rep"
	"github.com/cloudfoundry-incubator/rep/lrp_stopper"
	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/pivotal-golang/lager"
)

type AuctionCellRep struct {
	cellID     string
	stack      string
	lrpStopper lrp_stopper.LRPStopper
	bbs        Bbs.RepBBS
	client     executor.Client
	logger     lager.Logger
}

func New(cellID string, stack string, lrpStopper lrp_stopper.LRPStopper, bbs Bbs.RepBBS, client executor.Client, logger lager.Logger) *AuctionCellRep {
	return &AuctionCellRep{
		cellID:     cellID,
		stack:      stack,
		lrpStopper: lrpStopper,
		bbs:        bbs,
		client:     client,
		logger:     logger.Session("auction-delegate"),
	}
}

func (a *AuctionCellRep) State() (auctiontypes.CellState, error) {
	logger := a.logger.Session("auction-state")
	logger.Info("providing")

	totalResources, err := a.fetchResourcesVia(a.client.TotalResources)
	if err != nil {
		logger.Error("failed-to-get-total-resources", err)
		return auctiontypes.CellState{}, err
	}

	availableResources, err := a.fetchResourcesVia(a.client.RemainingResources)
	if err != nil {
		logger.Error("failed-to-get-remaining-resource", err)
		return auctiontypes.CellState{}, err
	}

	lrpContainers, err := a.client.ListContainers(executor.Tags{
		rep.LifecycleTag: rep.LRPLifecycle,
	})

	if err != nil {
		logger.Error("failed-to-fetch-containers", err)
		return auctiontypes.CellState{}, err
	}

	lrps := []auctiontypes.LRP{}

	for _, container := range lrpContainers {
		index, _ := strconv.Atoi(container.Tags[rep.ProcessIndexTag])
		lrp := auctiontypes.LRP{
			ProcessGuid:  container.Tags[rep.ProcessGuidTag],
			InstanceGuid: container.Guid,
			Index:        index,
			MemoryMB:     container.MemoryMB,
			DiskMB:       container.DiskMB,
		}
		lrps = append(lrps, lrp)
	}

	state := auctiontypes.CellState{
		Stack:              a.stack,
		AvailableResources: availableResources,
		TotalResources:     totalResources,
		LRPs:               lrps,
	}

	a.logger.Session("provided", lager.Data{"state": state})

	return state, nil
}

func (a *AuctionCellRep) Perform(work auctiontypes.Work) (auctiontypes.Work, error) {
	var failedWork = auctiontypes.Work{}

	logger := a.logger.Session("auction-work", lager.Data{
		"lrp-starts": len(work.LRPStarts),
		"lrp-stops":  len(work.LRPStops),
	})

	for _, stop := range work.LRPStops {
		stopLogger := logger.Session("lrp-stop-instance", lager.Data{"process-guid": stop.ProcessGuid, "instance-guid": stop.InstanceGuid, "index": stop.Index})
		stopLogger.Info("stopping")
		err := a.stopLRP(stop)
		if err != nil {
			stopLogger.Error("failed-to-stop", err)
			failedWork.LRPStops = append(failedWork.LRPStops, stop)
		} else {
			stopLogger.Info("stopped")
		}
	}

	for _, start := range work.LRPStarts {
		startLogger := logger.Session("lrp-start-instance", lager.Data{
			"process-guid":  start.DesiredLRP.ProcessGuid,
			"instance-guid": start.InstanceGuid,
			"index":         start.Index,
			"memory-mb":     start.DesiredLRP.MemoryMB,
			"disk-mb":       start.DesiredLRP.DiskMB,
		})
		startLogger.Info("starting")
		err := a.startLRP(start, startLogger)
		if err != nil {
			startLogger.Error("failed-to-start", err)
			failedWork.LRPStarts = append(failedWork.LRPStarts, start)
		} else {
			startLogger.Info("started")
		}
	}

	for _, task := range work.Tasks {
		taskLogger := logger.Session("task-start", lager.Data{
			"task-guid": task.TaskGuid,
			"memory-mb": task.MemoryMB,
			"disk-mb":   task.DiskMB,
		})
		taskLogger.Info("starting")
		err := a.startTask(task, taskLogger)
		if err != nil {
			taskLogger.Error("failed-to-start", err)
			failedWork.Tasks = append(failedWork.Tasks, task)
		} else {
			taskLogger.Info("started")
		}
	}

	return failedWork, nil
}

func (a *AuctionCellRep) startLRP(startAuction models.LRPStartAuction, logger lager.Logger) error {
	logger.Info("reserving")

	containerGuid := startAuction.InstanceGuid

	_, err := a.client.AllocateContainer(executor.Container{
		Guid: containerGuid,

		Tags: executor.Tags{
			rep.LifecycleTag:    rep.LRPLifecycle,
			rep.DomainTag:       startAuction.DesiredLRP.Domain,
			rep.ProcessGuidTag:  startAuction.DesiredLRP.ProcessGuid,
			rep.ProcessIndexTag: strconv.Itoa(startAuction.Index),
		},

		MemoryMB:     startAuction.DesiredLRP.MemoryMB,
		DiskMB:       startAuction.DesiredLRP.DiskMB,
		CPUWeight:    startAuction.DesiredLRP.CPUWeight,
		RootFSPath:   startAuction.DesiredLRP.RootFSPath,
		Ports:        a.convertPortMappings(startAuction.DesiredLRP.Ports),
		StartTimeout: startAuction.DesiredLRP.StartTimeout,

		Log: executor.LogConfig{
			Guid:       startAuction.DesiredLRP.LogGuid,
			SourceName: startAuction.DesiredLRP.LogSource,
			Index:      &startAuction.Index,
		},

		Setup:   startAuction.DesiredLRP.Setup,
		Action:  startAuction.DesiredLRP.Action,
		Monitor: startAuction.DesiredLRP.Monitor,

		Env: append([]executor.EnvironmentVariable{
			{Name: "INSTANCE_GUID", Value: startAuction.InstanceGuid},
			{Name: "INSTANCE_INDEX", Value: strconv.Itoa(startAuction.Index)},
		}, executor.EnvironmentVariablesFromModel(startAuction.DesiredLRP.EnvironmentVariables)...),
	})

	if err != nil {
		return err
	}

	logger.Info("announcing-to-bbs")
	claiming := models.NewActualLRP(startAuction.DesiredLRP.ProcessGuid,
		startAuction.InstanceGuid, a.cellID, startAuction.DesiredLRP.Domain,
		startAuction.Index, "")
	_, err = a.bbs.ClaimActualLRP(claiming)

	if err != nil {
		a.client.DeleteContainer(containerGuid)
		return err
	}

	logger.Info("running")
	err = a.client.RunContainer(containerGuid)
	if err != nil {
		a.client.DeleteContainer(containerGuid)
		a.bbs.RemoveActualLRP(claiming)
		return err
	}

	return nil
}

func (a *AuctionCellRep) stopLRP(lrp models.ActualLRP) error {
	return a.lrpStopper.StopInstance(lrp)
}

func (a *AuctionCellRep) startTask(task models.Task, logger lager.Logger) error {
	if task.Stack != a.stack {
		return errors.New(fmt.Sprintf("stack mismatch: task requested stack '%s', rep provides stack '%s'", task.Stack, a.stack))
	}

	logger.Info("allocating-container")
	_, err := a.client.AllocateContainer(executor.Container{
		Guid: task.TaskGuid,

		Tags: executor.Tags{
			rep.LifecycleTag:  rep.TaskLifecycle,
			rep.DomainTag:     task.Domain,
			rep.ResultFileTag: task.ResultFile,
		},

		DiskMB:     task.DiskMB,
		MemoryMB:   task.MemoryMB,
		CPUWeight:  task.CPUWeight,
		RootFSPath: task.RootFSPath,
		Log: executor.LogConfig{
			Guid:       task.LogGuid,
			SourceName: task.LogSource,
		},

		Action: task.Action,

		Env: executor.EnvironmentVariablesFromModel(task.EnvironmentVariables),
	})
	if err != nil {
		logger.Error("failed-to-allocate-container", err)
		return err
	}
	logger.Info("successfully-allocated-container")

	logger.Info("running-task")
	err = a.client.RunContainer(task.TaskGuid)
	if err != nil {
		logger.Error("failed-to-run-task", err)
		a.client.DeleteContainer(task.TaskGuid)
		a.markTaskAsFailed(logger, task.TaskGuid, err)
		return err
	}
	logger.Info("successfully-ran-task")

	logger.Info("starting-task")
	err = a.bbs.StartTask(task.TaskGuid, a.cellID)
	if err != nil {
		logger.Error("failed-to-mark-task-started", err)
		a.client.DeleteContainer(task.TaskGuid)
		return err
	}
	logger.Info("successfully-started-task")

	return nil
}

func (a *AuctionCellRep) convertPortMappings(containerPorts []uint32) []executor.PortMapping {
	out := []executor.PortMapping{}
	for _, port := range containerPorts {
		out = append(out, executor.PortMapping{
			ContainerPort: port,
		})
	}

	return out
}

func (a *AuctionCellRep) fetchResourcesVia(fetcher func() (executor.ExecutorResources, error)) (auctiontypes.Resources, error) {
	resources, err := fetcher()
	if err != nil {
		return auctiontypes.Resources{}, err
	}
	return auctiontypes.Resources{
		MemoryMB:   resources.MemoryMB,
		DiskMB:     resources.DiskMB,
		Containers: resources.Containers,
	}, nil
}

func (a *AuctionCellRep) markTaskAsFailed(logger lager.Logger, taskGuid string, err error) {
	logger.Info("complete-task")
	err = a.bbs.CompleteTask(taskGuid, true, "failed to run container - "+err.Error(), "")
	if err != nil {
		logger.Error("failed-to-complete-task", err)
	}
	logger.Info("successfully-completed-task")
}
<commit_msg>Include tasks in auction-work log message<commit_after>package auction_cell_rep

import (
	"errors"
	"fmt"
	"strconv"

	"github.com/cloudfoundry-incubator/auction/auctiontypes"
	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/rep"
	"github.com/cloudfoundry-incubator/rep/lrp_stopper"
	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/pivotal-golang/lager"
)

type AuctionCellRep struct {
	cellID     string
	stack      string
	lrpStopper lrp_stopper.LRPStopper
	bbs        Bbs.RepBBS
	client     executor.Client
	logger     lager.Logger
}

func New(cellID string, stack string, lrpStopper lrp_stopper.LRPStopper, bbs Bbs.RepBBS, client executor.Client, logger lager.Logger) *AuctionCellRep {
	return &AuctionCellRep{
		cellID:     cellID,
		stack:      stack,
		lrpStopper: lrpStopper,
		bbs:        bbs,
		client:     client,
		logger:     logger.Session("auction-delegate"),
	}
}

func (a *AuctionCellRep) State() (auctiontypes.CellState, error) {
	logger := a.logger.Session("auction-state")
	logger.Info("providing")

	totalResources, err := a.fetchResourcesVia(a.client.TotalResources)
	if err != nil {
		logger.Error("failed-to-get-total-resources", err)
		return auctiontypes.CellState{}, err
	}

	availableResources, err := a.fetchResourcesVia(a.client.RemainingResources)
	if err != nil {
		logger.Error("failed-to-get-remaining-resource", err)
		return auctiontypes.CellState{}, err
	}

	lrpContainers, err := a.client.ListContainers(executor.Tags{
		rep.LifecycleTag: rep.LRPLifecycle,
	})

	if err != nil {
		logger.Error("failed-to-fetch-containers", err)
		return auctiontypes.CellState{}, err
	}

	lrps := []auctiontypes.LRP{}

	for _, container := range lrpContainers {
		index, _ := strconv.Atoi(container.Tags[rep.ProcessIndexTag])
		lrp := auctiontypes.LRP{
			ProcessGuid:  container.Tags[rep.ProcessGuidTag],
			InstanceGuid: container.Guid,
			Index:        index,
			MemoryMB:     container.MemoryMB,
			DiskMB:       container.DiskMB,
		}
		lrps = append(lrps, lrp)
	}

	state := auctiontypes.CellState{
		Stack:              a.stack,
		AvailableResources: availableResources,
		TotalResources:     totalResources,
		LRPs:               lrps,
	}

	a.logger.Session("provided", lager.Data{"state": state})

	return state, nil
}

func (a *AuctionCellRep) Perform(work auctiontypes.Work) (auctiontypes.Work, error) {
	var failedWork = auctiontypes.Work{}

	logger := a.logger.Session("auction-work", lager.Data{
		"lrp-starts": len(work.LRPStarts),
		"lrp-stops":  len(work.LRPStops),
		"tasks":      len(work.Tasks),
	})

	for _, stop := range work.LRPStops {
		stopLogger := logger.Session("lrp-stop-instance", lager.Data{"process-guid": stop.ProcessGuid, "instance-guid": stop.InstanceGuid, "index": stop.Index})
		stopLogger.Info("stopping")
		err := a.stopLRP(stop)
		if err != nil {
			stopLogger.Error("failed-to-stop", err)
			failedWork.LRPStops = append(failedWork.LRPStops, stop)
		} else {
			stopLogger.Info("stopped")
		}
	}

	for _, start := range work.LRPStarts {
		startLogger := logger.Session("lrp-start-instance", lager.Data{
			"process-guid":  start.DesiredLRP.ProcessGuid,
			"instance-guid": start.InstanceGuid,
			"index":         start.Index,
			"memory-mb":     start.DesiredLRP.MemoryMB,
			"disk-mb":       start.DesiredLRP.DiskMB,
		})
		startLogger.Info("starting")
		err := a.startLRP(start, startLogger)
		if err != nil {
			startLogger.Error("failed-to-start", err)
			failedWork.LRPStarts = append(failedWork.LRPStarts, start)
		} else {
			startLogger.Info("started")
		}
	}

	for _, task := range work.Tasks {
		taskLogger := logger.Session("task-start", lager.Data{
			"task-guid": task.TaskGuid,
			"memory-mb": task.MemoryMB,
			"disk-mb":   task.DiskMB,
		})
		taskLogger.Info("starting")
		err := a.startTask(task, taskLogger)
		if err != nil {
			taskLogger.Error("failed-to-start", err)
			failedWork.Tasks = append(failedWork.Tasks, task)
		} else {
			taskLogger.Info("started")
		}
	}

	return failedWork, nil
}

func (a *AuctionCellRep) startLRP(startAuction models.LRPStartAuction, logger lager.Logger) error {
	logger.Info("reserving")

	containerGuid := startAuction.InstanceGuid

	_, err := a.client.AllocateContainer(executor.Container{
		Guid: containerGuid,

		Tags: executor.Tags{
			rep.LifecycleTag:    rep.LRPLifecycle,
			rep.DomainTag:       startAuction.DesiredLRP.Domain,
			rep.ProcessGuidTag:  startAuction.DesiredLRP.ProcessGuid,
			rep.ProcessIndexTag: strconv.Itoa(startAuction.Index),
		},

		MemoryMB:     startAuction.DesiredLRP.MemoryMB,
		DiskMB:       startAuction.DesiredLRP.DiskMB,
		CPUWeight:    startAuction.DesiredLRP.CPUWeight,
		RootFSPath:   startAuction.DesiredLRP.RootFSPath,
		Ports:        a.convertPortMappings(startAuction.DesiredLRP.Ports),
		StartTimeout: startAuction.DesiredLRP.StartTimeout,

		Log: executor.LogConfig{
			Guid:       startAuction.DesiredLRP.LogGuid,
			SourceName: startAuction.DesiredLRP.LogSource,
			Index:      &startAuction.Index,
		},

		Setup:   startAuction.DesiredLRP.Setup,
		Action:  startAuction.DesiredLRP.Action,
		Monitor: startAuction.DesiredLRP.Monitor,

		Env: append([]executor.EnvironmentVariable{
			{Name: "INSTANCE_GUID", Value: startAuction.InstanceGuid},
			{Name: "INSTANCE_INDEX", Value: strconv.Itoa(startAuction.Index)},
		}, executor.EnvironmentVariablesFromModel(startAuction.DesiredLRP.EnvironmentVariables)...),
	})

	if err != nil {
		return err
	}

	logger.Info("announcing-to-bbs")
	claiming := models.NewActualLRP(startAuction.DesiredLRP.ProcessGuid,
		startAuction.InstanceGuid, a.cellID, startAuction.DesiredLRP.Domain,
		startAuction.Index, "")
	_, err = a.bbs.ClaimActualLRP(claiming)

	if err != nil {
		a.client.DeleteContainer(containerGuid)
		return err
	}

	logger.Info("running")
	err = a.client.RunContainer(containerGuid)
	if err != nil {
		a.client.DeleteContainer(containerGuid)
		a.bbs.RemoveActualLRP(claiming)
		return err
	}

	return nil
}

func (a *AuctionCellRep) stopLRP(lrp models.ActualLRP) error {
	return a.lrpStopper.StopInstance(lrp)
}

func (a *AuctionCellRep) startTask(task models.Task, logger lager.Logger) error {
	if task.Stack != a.stack {
		return errors.New(fmt.Sprintf("stack mismatch: task requested stack '%s', rep provides stack '%s'", task.Stack, a.stack))
	}

	logger.Info("allocating-container")
	_, err := a.client.AllocateContainer(executor.Container{
		Guid: task.TaskGuid,

		Tags: executor.Tags{
			rep.LifecycleTag:  rep.TaskLifecycle,
			rep.DomainTag:     task.Domain,
			rep.ResultFileTag: task.ResultFile,
		},

		DiskMB:     task.DiskMB,
		MemoryMB:   task.MemoryMB,
		CPUWeight:  task.CPUWeight,
		RootFSPath: task.RootFSPath,
		Log: executor.LogConfig{
			Guid:       task.LogGuid,
			SourceName: task.LogSource,
		},

		Action: task.Action,

		Env: executor.EnvironmentVariablesFromModel(task.EnvironmentVariables),
	})
	if err != nil {
		logger.Error("failed-to-allocate-container", err)
		return err
	}
	logger.Info("successfully-allocated-container")

	logger.Info("running-task")
	err = a.client.RunContainer(task.TaskGuid)
	if err != nil {
		logger.Error("failed-to-run-task", err)
		a.client.DeleteContainer(task.TaskGuid)
		a.markTaskAsFailed(logger, task.TaskGuid, err)
		return err
	}
	logger.Info("successfully-ran-task")

	logger.Info("starting-task")
	err = a.bbs.StartTask(task.TaskGuid, a.cellID)
	if err != nil {
		logger.Error("failed-to-mark-task-started", err)
		a.client.DeleteContainer(task.TaskGuid)
		return err
	}
	logger.Info("successfully-started-task")

	return nil
}

func (a *AuctionCellRep) convertPortMappings(containerPorts []uint32) []executor.PortMapping {
	out := []executor.PortMapping{}
	for _, port := range containerPorts {
		out = append(out, executor.PortMapping{
			ContainerPort: port,
		})
	}

	return out
}

func (a *AuctionCellRep) fetchResourcesVia(fetcher func() (executor.ExecutorResources, error)) (auctiontypes.Resources, error) {
	resources, err := fetcher()
	if err != nil {
		return auctiontypes.Resources{}, err
	}
	return auctiontypes.Resources{
		MemoryMB:   resources.MemoryMB,
		DiskMB:     resources.DiskMB,
		Containers: resources.Containers,
	}, nil
}

func (a *AuctionCellRep) markTaskAsFailed(logger lager.Logger, taskGuid string, err error) {
	logger.Info("complete-task")
	err = a.bbs.CompleteTask(taskGuid, true, "failed to run container - "+err.Error(), "")
	if err != nil {
		logger.Error("failed-to-complete-task", err)
	}
	logger.Info("successfully-completed-task")
}
<|endoftext|>


<commit_before>package operationlock

import (
	"fmt"
	"sync"
	"time"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/shared"
)

var instanceOperationsLock sync.Mutex
var instanceOperations = make(map[string]*InstanceOperation)

// TimeoutSeconds number of seconds that the operation lock will be kept for without calling Reset().
const TimeoutSeconds = 30

// InstanceOperation operation locking.
type InstanceOperation struct {
	action       string
	chanDone     chan error
	chanReset    chan bool
	err          error
	projectName  string
	instanceName string
	reusable     bool
}

// Action returns operation's action.
func (op InstanceOperation) Action() string {
	return op.action
}

// Create creates a new operation lock for an Instance if one does not already exist and returns it.
// The lock will be released after 30s or when Done() is called, which ever occurs first.
// If reusable is set as true then future lock attempts can specify the reuse argument as true which
// will then trigger a reset of the 30s timeout on the existing lock and return it.
func Create(projectName string, instanceName string, action string, reusable bool, reuse bool) (*InstanceOperation, error) {
	if projectName == "" || instanceName == "" {
		return nil, fmt.Errorf("Invalid project or instance name")
	}

	instanceOperationsLock.Lock()
	defer instanceOperationsLock.Unlock()

	opKey := project.Instance(projectName, instanceName)

	op := instanceOperations[opKey]
	if op != nil {
		if op.reusable && reuse {
			// Reset operation timeout without releasing lock or deadlocking using Reset() function.
			op.chanReset <- true
			return op, nil
		}

		return nil, fmt.Errorf("Instance is busy running a %q operation", op.action)
	}

	op = &InstanceOperation{}
	op.projectName = projectName
	op.instanceName = instanceName
	op.action = action
	op.reusable = reusable
	op.chanDone = make(chan error, 0)
	op.chanReset = make(chan bool, 0)

	instanceOperations[opKey] = op

	go func(op *InstanceOperation) {
		for {
			select {
			case <-op.chanDone:
				return
			case <-op.chanReset:
				continue
			case <-time.After(time.Second * TimeoutSeconds):
				op.Done(fmt.Errorf("Instance %q operation timed out after %d seconds", op.action, TimeoutSeconds))
				return
			}
		}
	}(op)

	return op, nil
}

// CreateWaitGet is a weird function which does what we happen to want most of the time.
//
// If the instance has an operation of the same type and it's not reusable
// or the caller doesn't want to reuse it, the function will wait and
// indicate that it did so.
//
// If the instance has an operation of one of the alternate types, then
// the operation is returned to the user.
//
// If the instance doesn't have an operation, has an operation of a different
// type that is not in the alternate list or has the right type and is
// being reused, then this behaves as a Create call.
func CreateWaitGet(projectName string, instanceName string, action string, altActions []string, reusable bool, reuse bool) (bool, *InstanceOperation, error) {
	op := Get(projectName, instanceName)

	// No existing operation, call create.
	if op == nil {
		op, err := Create(projectName, instanceName, action, reusable, reuse)
		return false, op, err
	}

	// Operation matches and not reusable or asked to reuse, wait.
	if op.action == action && (!reuse || !op.reusable) {
		err := op.Wait()
		return true, nil, err
	}

	// Operation matches one the alternate actions, return the operation.
	if shared.StringInSlice(op.action, altActions) {
		return false, op, nil
	}

	// Send the rest to Create
	op, err := Create(projectName, instanceName, action, reusable, reuse)

	return false, op, err
}

// Get retrieves an existing lock or returns nil if no lock exists.
func Get(projectName string, instanceName string) *InstanceOperation {
	instanceOperationsLock.Lock()
	defer instanceOperationsLock.Unlock()

	opKey := project.Instance(projectName, instanceName)

	return instanceOperations[opKey]
}

// Reset resets the operation timeout to give another TimeoutSeconds seconds until it expires.
func (op *InstanceOperation) Reset() error {
	// This function can be called on a nil struct.
	if op == nil {
		return nil
	}

	instanceOperationsLock.Lock()
	defer instanceOperationsLock.Unlock()

	opKey := project.Instance(op.projectName, op.instanceName)

	// Check if already done
	runningOp, ok := instanceOperations[opKey]
	if !ok || runningOp != op {
		return fmt.Errorf("Operation is already done or expired")
	}

	op.chanReset <- true
	return nil
}

// Wait waits for an operation to finish.
func (op *InstanceOperation) Wait() error {
	// This function can be called on a nil struct.
	if op == nil {
		return nil
	}

	<-op.chanDone

	return op.err
}

// Done indicates the operation has finished.
func (op *InstanceOperation) Done(err error) {
	// This function can be called on a nil struct.
	if op == nil {
		return
	}

	instanceOperationsLock.Lock()
	defer instanceOperationsLock.Unlock()

	opKey := project.Instance(op.projectName, op.instanceName)

	// Check if already done
	runningOp, ok := instanceOperations[opKey]
	if !ok || runningOp != op {
		return
	}

	op.err = err
	delete(instanceOperations, opKey) // Delete before closing chanDone.
	close(op.chanDone)
}
<commit_msg>lxd/instance/operationlock: Add ErrNonReusuableSucceeded error and Action type and action constants<commit_after>package operationlock

import (
	"fmt"
	"sync"
	"time"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/shared"
)

// TimeoutSeconds number of seconds that the operation lock will be kept for without calling Reset().
const TimeoutSeconds = 30

// Action indicates the operation action type.
type Action string

// ActionStart for starting an instance.
const ActionStart Action = "start"

// ActionStop for stopping an instance.
const ActionStop Action = "stop"

// ActionRestart for restarting an instance.
const ActionRestart Action = "restart"

// ActionRestore for restoring an instance.
const ActionRestore Action = "restore"

// ErrNonReusuableSucceeded is returned when no operation is created due to having to wait for a matching
// non-reusuable operation that has now completed successfully.
var ErrNonReusuableSucceeded error = fmt.Errorf("A matching non-reusable operation has now succeeded")

var instanceOperationsLock sync.Mutex
var instanceOperations = make(map[string]*InstanceOperation)

// InstanceOperation operation locking.
type InstanceOperation struct {
	action       string
	chanDone     chan error
	chanReset    chan bool
	err          error
	projectName  string
	instanceName string
	reusable     bool
}

// Action returns operation's action.
func (op InstanceOperation) Action() string {
	return op.action
}

// Create creates a new operation lock for an Instance if one does not already exist and returns it.
// The lock will be released after 30s or when Done() is called, which ever occurs first.
// If reusable is set as true then future lock attempts can specify the reuse argument as true which
// will then trigger a reset of the 30s timeout on the existing lock and return it.
func Create(projectName string, instanceName string, action string, reusable bool, reuse bool) (*InstanceOperation, error) {
	if projectName == "" || instanceName == "" {
		return nil, fmt.Errorf("Invalid project or instance name")
	}

	instanceOperationsLock.Lock()
	defer instanceOperationsLock.Unlock()

	opKey := project.Instance(projectName, instanceName)

	op := instanceOperations[opKey]
	if op != nil {
		if op.reusable && reuse {
			// Reset operation timeout without releasing lock or deadlocking using Reset() function.
			op.chanReset <- true
			return op, nil
		}

		return nil, fmt.Errorf("Instance is busy running a %q operation", op.action)
	}

	op = &InstanceOperation{}
	op.projectName = projectName
	op.instanceName = instanceName
	op.action = action
	op.reusable = reusable
	op.chanDone = make(chan error, 0)
	op.chanReset = make(chan bool, 0)

	instanceOperations[opKey] = op

	go func(op *InstanceOperation) {
		for {
			select {
			case <-op.chanDone:
				return
			case <-op.chanReset:
				continue
			case <-time.After(time.Second * TimeoutSeconds):
				op.Done(fmt.Errorf("Instance %q operation timed out after %d seconds", op.action, TimeoutSeconds))
				return
			}
		}
	}(op)

	return op, nil
}

// CreateWaitGet is a weird function which does what we happen to want most of the time.
//
// If the instance has an operation of the same type and it's not reusable
// or the caller doesn't want to reuse it, the function will wait and
// indicate that it did so.
//
// If the instance has an operation of one of the alternate types, then
// the operation is returned to the user.
//
// If the instance doesn't have an operation, has an operation of a different
// type that is not in the alternate list or has the right type and is
// being reused, then this behaves as a Create call.
func CreateWaitGet(projectName string, instanceName string, action string, altActions []string, reusable bool, reuse bool) (bool, *InstanceOperation, error) {
	op := Get(projectName, instanceName)

	// No existing operation, call create.
	if op == nil {
		op, err := Create(projectName, instanceName, action, reusable, reuse)
		return false, op, err
	}

	// Operation matches and not reusable or asked to reuse, wait.
	if op.action == action && (!reuse || !op.reusable) {
		err := op.Wait()
		return true, nil, err
	}

	// Operation matches one the alternate actions, return the operation.
	if shared.StringInSlice(op.action, altActions) {
		return false, op, nil
	}

	// Send the rest to Create
	op, err := Create(projectName, instanceName, action, reusable, reuse)

	return false, op, err
}

// Get retrieves an existing lock or returns nil if no lock exists.
func Get(projectName string, instanceName string) *InstanceOperation {
	instanceOperationsLock.Lock()
	defer instanceOperationsLock.Unlock()

	opKey := project.Instance(projectName, instanceName)

	return instanceOperations[opKey]
}

// Reset resets the operation timeout to give another TimeoutSeconds seconds until it expires.
func (op *InstanceOperation) Reset() error {
	// This function can be called on a nil struct.
	if op == nil {
		return nil
	}

	instanceOperationsLock.Lock()
	defer instanceOperationsLock.Unlock()

	opKey := project.Instance(op.projectName, op.instanceName)

	// Check if already done
	runningOp, ok := instanceOperations[opKey]
	if !ok || runningOp != op {
		return fmt.Errorf("Operation is already done or expired")
	}

	op.chanReset <- true
	return nil
}

// Wait waits for an operation to finish.
func (op *InstanceOperation) Wait() error {
	// This function can be called on a nil struct.
	if op == nil {
		return nil
	}

	<-op.chanDone

	return op.err
}

// Done indicates the operation has finished.
func (op *InstanceOperation) Done(err error) {
	// This function can be called on a nil struct.
	if op == nil {
		return
	}

	instanceOperationsLock.Lock()
	defer instanceOperationsLock.Unlock()

	opKey := project.Instance(op.projectName, op.instanceName)

	// Check if already done
	runningOp, ok := instanceOperations[opKey]
	if !ok || runningOp != op {
		return
	}

	op.err = err
	delete(instanceOperations, opKey) // Delete before closing chanDone.
	close(op.chanDone)
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
)

func dataSourceAwsEc2CoipPool() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsEc2CoipPoolRead,

		Schema: map[string]*schema.Schema{
			"local_gateway_route_table_id": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},

			"pool_cidrs": {
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
				Set:      schema.HashString,
			},

			"pool_id": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},

			"tags": tagsSchemaComputed(),

			"filter": ec2CustomFiltersSchema(),
		},
	}
}

func dataSourceAwsEc2CoipPoolRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	req := &ec2.DescribeCoipPoolsInput{}

	if v, ok := d.GetOk("pool_id"); ok {
		req.PoolIds = []*string{aws.String(v.(string))}
	}

	filters := map[string]string{}

	if v, ok := d.GetOk("local_gateway_route_table_id"); ok {
		filters["coip-pool.local-gateway-route-table-id"] = v.(string)
	}

	req.Filters = buildEC2AttributeFilterList(filters)

	if tags, tagsOk := d.GetOk("tags"); tagsOk {
		req.Filters = append(req.Filters, buildEC2TagFilterList(
			keyvaluetags.New(tags.(map[string]interface{})).Ec2Tags(),
		)...)
	}

	req.Filters = append(req.Filters, buildEC2CustomFilterList(
		d.Get("filter").(*schema.Set),
	)...)
	if len(req.Filters) == 0 {
		// Don't send an empty filters list; the EC2 API won't accept it.
		req.Filters = nil
	}

	log.Printf("[DEBUG] Reading AWS COIP Pool: %s", req)
	resp, err := conn.DescribeCoipPools(req)
	if err != nil {
		return fmt.Errorf("error describing EC2 COIP Pools: %w", err)
	}
	if resp == nil || len(resp.CoipPools) == 0 {
		return fmt.Errorf("no matching COIP Pool found")
	}
	if len(resp.CoipPools) > 1 {
		return fmt.Errorf("multiple Coip Pools matched; use additional constraints to reduce matches to a single COIP Pool")
	}

	coip := resp.CoipPools[0]

	d.SetId(aws.StringValue(coip.PoolId))

	d.Set("local_gateway_route_table_id", coip.LocalGatewayRouteTableId)

	if err := d.Set("pool_cidrs", aws.StringValueSlice(coip.PoolCidrs)); err != nil {
		return fmt.Errorf("error setting pool_cidrs: %s", err)
	}

	d.Set("pool_id", coip.PoolId)

	if err := d.Set("tags", keyvaluetags.Ec2KeyValueTags(coip.Tags).IgnoreAws().Map()); err != nil {
		return fmt.Errorf("error setting tags: %s", err)
	}

	return nil
}
<commit_msg>data-source/aws_ec2_coip_pool: Include ignore tags config<commit_after>package aws

import (
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
)

func dataSourceAwsEc2CoipPool() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsEc2CoipPoolRead,

		Schema: map[string]*schema.Schema{
			"local_gateway_route_table_id": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},

			"pool_cidrs": {
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
				Set:      schema.HashString,
			},

			"pool_id": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},

			"tags": tagsSchemaComputed(),

			"filter": ec2CustomFiltersSchema(),
		},
	}
}

func dataSourceAwsEc2CoipPoolRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn
	ignoreTagsConfig := meta.(*AWSClient).IgnoreTagsConfig

	req := &ec2.DescribeCoipPoolsInput{}

	if v, ok := d.GetOk("pool_id"); ok {
		req.PoolIds = []*string{aws.String(v.(string))}
	}

	filters := map[string]string{}

	if v, ok := d.GetOk("local_gateway_route_table_id"); ok {
		filters["coip-pool.local-gateway-route-table-id"] = v.(string)
	}

	req.Filters = buildEC2AttributeFilterList(filters)

	if tags, tagsOk := d.GetOk("tags"); tagsOk {
		req.Filters = append(req.Filters, buildEC2TagFilterList(
			keyvaluetags.New(tags.(map[string]interface{})).Ec2Tags(),
		)...)
	}

	req.Filters = append(req.Filters, buildEC2CustomFilterList(
		d.Get("filter").(*schema.Set),
	)...)
	if len(req.Filters) == 0 {
		// Don't send an empty filters list; the EC2 API won't accept it.
		req.Filters = nil
	}

	log.Printf("[DEBUG] Reading AWS COIP Pool: %s", req)
	resp, err := conn.DescribeCoipPools(req)
	if err != nil {
		return fmt.Errorf("error describing EC2 COIP Pools: %w", err)
	}
	if resp == nil || len(resp.CoipPools) == 0 {
		return fmt.Errorf("no matching COIP Pool found")
	}
	if len(resp.CoipPools) > 1 {
		return fmt.Errorf("multiple Coip Pools matched; use additional constraints to reduce matches to a single COIP Pool")
	}

	coip := resp.CoipPools[0]

	d.SetId(aws.StringValue(coip.PoolId))

	d.Set("local_gateway_route_table_id", coip.LocalGatewayRouteTableId)

	if err := d.Set("pool_cidrs", aws.StringValueSlice(coip.PoolCidrs)); err != nil {
		return fmt.Errorf("error setting pool_cidrs: %s", err)
	}

	d.Set("pool_id", coip.PoolId)

	if err := d.Set("tags", keyvaluetags.Ec2KeyValueTags(coip.Tags).IgnoreAws().IgnoreConfig(ignoreTagsConfig).Map()); err != nil {
		return fmt.Errorf("error setting tags: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"debug/dwarf"
	"debug/elf"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"log"
	"math"
	"os"
	"sort"
)

const attrMIPSLinkageName dwarf.Attr = 0x2007

type rangeSizeMap map[int64]uint64

// Go's debug/dwarf package doesn't include .debug_ranges parsing support.
func parseDebugRangesFromELF(file *elf.File) (rangeSizeMap, error) {
	log.Print("parsing .debug_ranges...")
	section := file.Section(".debug_ranges")
	if section == nil {
		return nil, nil
	}

	var byteOrder binary.ByteOrder
	switch file.Data {
	case elf.ELFDATA2LSB:
		byteOrder = binary.LittleEndian
	case elf.ELFDATA2MSB:
		byteOrder = binary.BigEndian
	default:
		return nil, fmt.Errorf("%v has an unknown byte order", file)
	}

	var bytesPerAddress uint8
	switch file.Class {
	case elf.ELFCLASS32:
		bytesPerAddress = 4
	case elf.ELFCLASS64:
		bytesPerAddress = 8
	default:
		return nil, fmt.Errorf("%v has unknown class value", file)
	}

	// The .debug_ranges format is pretty simple. A DIE may use DW_AT_ranges to refer to a
	// range in the .debug_ranges section, which represents a range of non-contiguous
	// addresses. Each entry in the range is a either a range list entry, a base address
	// selection entry, or an end of list entry.
	// - A range list entry consists of a beginning address offset and an ending address
	//   offset. The beginning address offset may be 0x0, and the length of the range may be
	//   0, if the beginning and ending address offsets are equal. Range list entries may
	//   not overlap.
	// - A base address selection entry, which consists of the largest representable
	//   address, e.g. 0xffffffff for 32-bit addresses, and an address that defines the base
	//   address of subsequent entries.
	// - An end of list entry is a range list entry that has a beginning and ending address
	//   offset of 0.
	var currentOffset, pendingOffset int64
	rangeSizes := make(rangeSizeMap)
	buffer := make([]byte, 2*bytesPerAddress)
	for reader := section.Open(); ; {
		n, err := reader.Read(buffer)
		if n == 0 && err == io.EOF {
			return rangeSizes, nil
		} else if n != len(buffer) {
			return nil, fmt.Errorf("read strange number of bytes: %d", n)
		} else if err != nil {
			return nil, err
		}
		pendingOffset += int64(n)
		var begin, end uint64
		switch file.Class {
		case elf.ELFCLASS32:
			begin = uint64(byteOrder.Uint32(buffer))
			end = uint64(byteOrder.Uint32(buffer[4:]))
			if begin == math.MaxUint32 {
				continue
			}
		case elf.ELFCLASS64:
			begin = byteOrder.Uint64(buffer)
			end = byteOrder.Uint64(buffer[8:])
			if begin == math.MaxUint64 {
				continue
			}
		}
		if begin == 0 && end == 0 {
			currentOffset = pendingOffset
			continue
		}
		bytes := end - begin
		if bytes < 0 {
			return nil, fmt.Errorf("got invalid range %v", buffer)
		}
		rangeSizes[currentOffset] += bytes
	}
}

type subprogramEntry struct {
	name          string
	linkageName   string
	hasSpecOffset bool
	specOffset    dwarf.Offset
}
type subprogramMap map[dwarf.Offset]*subprogramEntry

func newSubprogramEntry(entry *dwarf.Entry) *subprogramEntry {
	subprogram := &subprogramEntry{}
	if linkageName, ok := entry.Val(attrMIPSLinkageName).(string); ok {
		subprogram.linkageName = linkageName
	}
	if specOffset, ok := entry.Val(dwarf.AttrSpecification).(dwarf.Offset); ok {
		subprogram.hasSpecOffset = true
		subprogram.specOffset = specOffset
	}
	if name, ok := entry.Val(dwarf.AttrName).(string); ok {
		subprogram.name = name
	}
	return subprogram
}

// Attempts to extract a function name from the DIE at the provided offset. Unfortunately, since
// it's C++ and DWARF, it's not just a simple matter of getting name attribute and returning it.
func nameForSubprogram(subprograms subprogramMap, offset dwarf.Offset) (string, error) {
	subprogram, ok := subprograms[offset]
	if !ok {
		return "", errors.New("couldn't find subprogram")
	}

	if subprogram.linkageName != "" {
		return subprogram.linkageName, nil
	}

	if subprogram.hasSpecOffset {
		return nameForSubprogram(subprograms, subprogram.specOffset)
	}

	if subprogram.name != "" {
		return subprogram.name, nil
	}

	return "", fmt.Errorf("subprogram 0x%x has no name, linkage name, or spec", offset)
}

func bytesForInlinedSubroutine(rangeSizes rangeSizeMap, entry *dwarf.Entry) (uint64, error) {
	// Per the DWARF spec, a DIE with associated machine code may have:
	// - A DW_AT_low_pc attribute for a snigle address (not handled)
	// - A DW_AT_low_pc and DW_AT_high_pc attribute for a single contiguous range of
	//   addresses, or
	// - A DW_AT_ranges attribute for a non-contiguous range of addresses.

	// TODO(dcheng): This tool should be able to handle either form.
	// The spec notes that DW_AT_high_pc may be either of class address or class constant.
	// In the latter case, DW_AT_high_pc is an offset from DW_AT_low_pc which gives the
	// first instruction past the last instruction associated with the DIE. This code
	// assumes the latter, since that's what Clang emits and it makes the code simpler.
	if bytes, ok := entry.Val(dwarf.AttrHighpc).(int64); ok {
		if bytes < 0 {
			return 0, fmt.Errorf("%v has negative size %d", entry, bytes)
		}
		return uint64(bytes), nil
	}

	rangeOffset, ok := entry.Val(dwarf.AttrRanges).(int64)
	if !ok {
		return 0, fmt.Errorf("%v has no valid high pc or range", entry)
	}
	bytes, ok := rangeSizes[rangeOffset]
	if !ok {
		return 0, fmt.Errorf("couldn't find range entry for %v", entry)
	}
	return bytes, nil
}

type stats struct {
	count uint64 // Number of times the function was inlined.
	bytes uint64 // Total bytes inlined for the function.
}

func analyze(file *elf.File) (map[string]*stats, error) {
	rangeSizes, err := parseDebugRangesFromELF(file)
	if err != nil {
		return nil, err
	}

	// Strictly speaking, dwarf.Data should have other debug sections too, but in practice,
	// only .debug_info is exposed.
	debugInfo, err := file.DWARF()
	if err != nil {
		return nil, err
	}

	// DIEs may refer to a DIE with a greater offset, so defer name resolution until all DIEs
	// have been read.
	infoReader := debugInfo.Reader()
	subprograms := make(subprogramMap)
	rawStats := make(map[dwarf.Offset]*stats)
	for i := 0; ; i++ {
		if i%1000000 == 0 {
			log.Printf("read %d DIEs...", i)
		}
		entry, err := infoReader.Next()
		if err != nil {
			return nil, err
		}
		if entry == nil {
			break
		}
		switch entry.Tag {
		case dwarf.TagSubprogram:
			subprograms[entry.Offset] = newSubprogramEntry(entry)
		case dwarf.TagInlinedSubroutine:
			abstractOrigin, ok := entry.Val(dwarf.AttrAbstractOrigin).(dwarf.Offset)
			if !ok {
				log.Printf("error: %v missing abstract origin", entry)
				continue
			}
			bytes, err := bytesForInlinedSubroutine(rangeSizes, entry)
			if err != nil {
				log.Printf("error: %v", err)
				continue
			}
			s, ok := rawStats[abstractOrigin]
			if !ok {
				s = &stats{}
				rawStats[abstractOrigin] = s
			}
			s.count++
			s.bytes += bytes
		}
	}

	log.Printf("resolving names for %d inlined functions", len(rawStats))
	results := make(map[string]*stats)
	for abstractOrigin, rawStat := range rawStats {
		name, err := nameForSubprogram(subprograms, abstractOrigin)
		if err != nil {
			log.Printf("error: couldn't extract name for %d: %v", abstractOrigin, err)
		}

		s, ok := results[name]
		if !ok {
			s = &stats{}
			results[name] = s
		}
		s.count += rawStat.count
		s.bytes += rawStat.bytes
	}
	return results, nil
}

type resultSorter struct {
	names   []string
	results map[string]*stats
}

func (s *resultSorter) Len() int {
	return len(s.names)
}

func (s *resultSorter) Swap(i, j int) {
	s.names[i], s.names[j] = s.names[j], s.names[i]
}

func (s *resultSorter) Less(i, j int) bool {
	return s.results[s.names[i]].bytes > s.results[s.names[j]].bytes
}

func sortAndPrintTop100(results map[string]*stats) {
	names := make([]string, 0, len(results))
	for n := range results {
		names = append(names, n)
	}
	sort.Sort(&resultSorter{names, results})
	fmt.Printf("     Count      Bytes   Name\n")
	fmt.Printf("  --------  ---------   ---------------------------------\n")
	for i, n := range names {
		if i > 100 {
			break
		}
		fmt.Printf("%10d %10d   %s\n", results[n].count, results[n].bytes, n)
	}
}

func main() {
	for _, f := range os.Args[1:] {
		log.Printf("analyzing %s...", f)
		file, err := elf.Open(f)
		if err != nil {
			log.Printf("error: couldn't open %s: %v", f, err)
			continue
		}
		results, err := analyze(file)
		if err != nil {
			log.Printf("error: couldn't analyze debug data for %s: %v", f, err)
			continue
		}
		sortAndPrintTop100(results)
	}
}
<commit_msg>Reduce memory usage by storing subprogram name information in two maps.<commit_after>package main

import (
	"debug/dwarf"
	"debug/elf"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"math"
	"os"
	"sort"
)

const attrMIPSLinkageName dwarf.Attr = 0x2007

type rangeSizeMap map[int64]uint64

// Go's debug/dwarf package doesn't include .debug_ranges parsing support.
func parseDebugRangesFromELF(file *elf.File) (rangeSizeMap, error) {
	log.Print("parsing .debug_ranges...")
	section := file.Section(".debug_ranges")
	if section == nil {
		return nil, nil
	}

	var byteOrder binary.ByteOrder
	switch file.Data {
	case elf.ELFDATA2LSB:
		byteOrder = binary.LittleEndian
	case elf.ELFDATA2MSB:
		byteOrder = binary.BigEndian
	default:
		return nil, fmt.Errorf("%v has an unknown byte order", file)
	}

	var bytesPerAddress uint8
	switch file.Class {
	case elf.ELFCLASS32:
		bytesPerAddress = 4
	case elf.ELFCLASS64:
		bytesPerAddress = 8
	default:
		return nil, fmt.Errorf("%v has unknown class value", file)
	}

	// The .debug_ranges format is pretty simple. A DIE may use DW_AT_ranges to refer to a
	// range in the .debug_ranges section, which represents a range of non-contiguous
	// addresses. Each entry in the range is a either a range list entry, a base address
	// selection entry, or an end of list entry.
	// - A range list entry consists of a beginning address offset and an ending address
	//   offset. The beginning address offset may be 0x0, and the length of the range may be
	//   0, if the beginning and ending address offsets are equal. Range list entries may
	//   not overlap.
	// - A base address selection entry, which consists of the largest representable
	//   address, e.g. 0xffffffff for 32-bit addresses, and an address that defines the base
	//   address of subsequent entries.
	// - An end of list entry is a range list entry that has a beginning and ending address
	//   offset of 0.
	var currentOffset, pendingOffset int64
	rangeSizes := make(rangeSizeMap)
	buffer := make([]byte, 2*bytesPerAddress)
	for reader := section.Open(); ; {
		n, err := reader.Read(buffer)
		if n == 0 && err == io.EOF {
			return rangeSizes, nil
		} else if n != len(buffer) {
			return nil, fmt.Errorf("read strange number of bytes: %d", n)
		} else if err != nil {
			return nil, err
		}
		pendingOffset += int64(n)
		var begin, end uint64
		switch file.Class {
		case elf.ELFCLASS32:
			begin = uint64(byteOrder.Uint32(buffer))
			end = uint64(byteOrder.Uint32(buffer[4:]))
			if begin == math.MaxUint32 {
				continue
			}
		case elf.ELFCLASS64:
			begin = byteOrder.Uint64(buffer)
			end = byteOrder.Uint64(buffer[8:])
			if begin == math.MaxUint64 {
				continue
			}
		}
		if begin == 0 && end == 0 {
			currentOffset = pendingOffset
			continue
		}
		bytes := end - begin
		if bytes < 0 {
			return nil, fmt.Errorf("got invalid range %v", buffer)
		}
		rangeSizes[currentOffset] += bytes
	}
}

type nameMap map[dwarf.Offset]string
type specMap map[dwarf.Offset]dwarf.Offset

// Attempts to extract a function name from the DIE at the provided offset. Unfortunately, since
// it's C++ and DWARF, it's not just a simple matter of getting name attribute and returning it.
func nameForSubprogram(names nameMap, specs specMap, offset dwarf.Offset) (string, error) {
	if specOffset, ok := specs[offset]; ok {
		return nameForSubprogram(names, specs, specOffset)
	}
	if name, ok := names[offset]; ok {
		return name, nil
	}
	return "", fmt.Errorf("could not find name or spec for subprogram 0x%x", offset)
}

func bytesForInlinedSubroutine(rangeSizes rangeSizeMap, entry *dwarf.Entry) (uint64, error) {
	// Per the DWARF spec, a DIE with associated machine code may have:
	// - A DW_AT_low_pc attribute for a snigle address (not handled)
	// - A DW_AT_low_pc and DW_AT_high_pc attribute for a single contiguous range of
	//   addresses, or
	// - A DW_AT_ranges attribute for a non-contiguous range of addresses.

	// TODO(dcheng): This tool should be able to handle either form.
	// The spec notes that DW_AT_high_pc may be either of class address or class constant.
	// In the latter case, DW_AT_high_pc is an offset from DW_AT_low_pc which gives the
	// first instruction past the last instruction associated with the DIE. This code
	// assumes the latter, since that's what Clang emits and it makes the code simpler.
	if bytes, ok := entry.Val(dwarf.AttrHighpc).(int64); ok {
		if bytes < 0 {
			return 0, fmt.Errorf("%v has negative size %d", entry, bytes)
		}
		return uint64(bytes), nil
	}

	rangeOffset, ok := entry.Val(dwarf.AttrRanges).(int64)
	if !ok {
		return 0, fmt.Errorf("%v has no valid high pc or range", entry)
	}
	bytes, ok := rangeSizes[rangeOffset]
	if !ok {
		return 0, fmt.Errorf("couldn't find range entry for %v", entry)
	}
	return bytes, nil
}

type stats struct {
	count uint64 // Number of times the function was inlined.
	bytes uint64 // Total bytes inlined for the function.
}

func analyze(file *elf.File) (map[string]*stats, error) {
	rangeSizes, err := parseDebugRangesFromELF(file)
	if err != nil {
		return nil, err
	}

	// Strictly speaking, dwarf.Data should have other debug sections too, but in practice,
	// only .debug_info is exposed.
	debugInfo, err := file.DWARF()
	if err != nil {
		return nil, err
	}

	// DIEs may refer to a DIE with a greater offset, so defer name resolution until all DIEs
	// have been read.
	infoReader := debugInfo.Reader()
	names := make(nameMap)
	specs := make(specMap)
	rawStats := make(map[dwarf.Offset]*stats)
	for i := 0; ; i++ {
		if i%1000000 == 0 {
			log.Printf("read %d DIEs...", i)
		}
		entry, err := infoReader.Next()
		if err != nil {
			return nil, err
		}
		if entry == nil {
			break
		}
		switch entry.Tag {
		case dwarf.TagSubprogram:
			if linkageName, ok := entry.Val(attrMIPSLinkageName).(string); ok {
				names[entry.Offset] = linkageName
				continue
			}
			if specOffset, ok := entry.Val(dwarf.AttrSpecification).(dwarf.Offset); ok {
				specs[entry.Offset] = specOffset
				continue
			}
			if name, ok := entry.Val(dwarf.AttrName).(string); ok {
				names[entry.Offset] = name
				continue
			}
		case dwarf.TagInlinedSubroutine:
			abstractOrigin, ok := entry.Val(dwarf.AttrAbstractOrigin).(dwarf.Offset)
			if !ok {
				log.Printf("error: %v missing abstract origin", entry)
				continue
			}
			bytes, err := bytesForInlinedSubroutine(rangeSizes, entry)
			if err != nil {
				log.Printf("error: %v", err)
				continue
			}
			s, ok := rawStats[abstractOrigin]
			if !ok {
				s = &stats{}
				rawStats[abstractOrigin] = s
			}
			s.count++
			s.bytes += bytes
		}
	}

	log.Printf("resolving names for %d inlined functions", len(rawStats))
	results := make(map[string]*stats)
	for abstractOrigin, rawStat := range rawStats {
		name, err := nameForSubprogram(names, specs, abstractOrigin)
		if err != nil {
			log.Printf("error: couldn't extract name for %d: %v", abstractOrigin, err)
		}

		s, ok := results[name]
		if !ok {
			s = &stats{}
			results[name] = s
		}
		s.count += rawStat.count
		s.bytes += rawStat.bytes
	}
	return results, nil
}

type resultSorter struct {
	names   []string
	results map[string]*stats
}

func (s *resultSorter) Len() int {
	return len(s.names)
}

func (s *resultSorter) Swap(i, j int) {
	s.names[i], s.names[j] = s.names[j], s.names[i]
}

func (s *resultSorter) Less(i, j int) bool {
	return s.results[s.names[i]].bytes > s.results[s.names[j]].bytes
}

func sortAndPrintTop100(results map[string]*stats) {
	names := make([]string, 0, len(results))
	for n := range results {
		names = append(names, n)
	}
	sort.Sort(&resultSorter{names, results})
	fmt.Printf("     Count      Bytes   Name\n")
	fmt.Printf("  --------  ---------   ---------------------------------\n")
	for i, n := range names {
		if i > 100 {
			break
		}
		fmt.Printf("%10d %10d   %s\n", results[n].count, results[n].bytes, n)
	}
}

func main() {
	for _, f := range os.Args[1:] {
		log.Printf("analyzing %s...", f)
		file, err := elf.Open(f)
		if err != nil {
			log.Printf("error: couldn't open %s: %v", f, err)
			continue
		}
		results, err := analyze(file)
		if err != nil {
			log.Printf("error: couldn't analyze debug data for %s: %v", f, err)
			continue
		}
		sortAndPrintTop100(results)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html/template"
	"io"
	"os"
	"os/user"
	"path/filepath"
	"runtime"

	"github.com/kardianos/osext"
	"github.com/paulrademacher/climenu"
)

const (
	def                = `default`
	appName            = `com.0xc0dedbad.kdeconnect_chrome`
	defaultExtensionID = `ofmplbbfigookafjahpeepbggpofdhbo`
)

var (
	manifestTemplate = template.Must(template.New(`manifest`).Parse(`{
  "name": "com.0xc0dedbad.kdeconnect_chrome",
  "description": "KDE Connect",
  "path": "{{.Path}}",
  "type": "stdio",
  "allowed_origins": [
    "chrome-extension://{{.ExtensionID}}/"
  ]
}`))

	// OS/browser/user/path
	installMappings map[string]map[string]map[string]string
)

type manifest struct {
	Path        string
	ExtensionID string
}

func doInstall(path, extensionID string) error {
	daemonPath := filepath.Join(path, appName)
	templatePath := filepath.Join(path, fmt.Sprintf("%s.json", appName))

	if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
		return err
	}

	exe, err := osext.Executable()
	if err != nil {
		return err
	}
	in, err := os.Open(exe)
	defer func() {
		if e := in.Close(); err != nil {
			fmt.Println(e)
			panic(e)
		}
	}()
	if err != nil {
		return err
	}
	out, err := os.OpenFile(daemonPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
	defer func() {
		if e := out.Close(); err != nil {
			fmt.Println(e)
			panic(e)
		}
	}()
	if err != nil {
		return err
	}
	//fmt.Println(`Copying daemon`, daemonPath)
	if _, err = io.Copy(out, in); err != nil {
		return err
	}

	man, err := os.OpenFile(templatePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
	defer func() {
		if e := man.Close(); err != nil {
			fmt.Println(e)
			panic(e)
		}
	}()
	if err != nil {
		return err
	}
	//fmt.Println(`Writing template`, templatePath)
	if err = manifestTemplate.Execute(man, manifest{
		Path:        daemonPath,
		ExtensionID: extensionID,
	}); err != nil {
		return err
	}

	return nil
}

func hasCustom(selection []string) bool {
	for _, s := range selection {
		if s == `custom` {
			return true
		}
	}
	return false
}

func install() error {
	u, err := user.Current()
	if err != nil {
		return err
	}

	username := u.Username
	operatingSystem := runtime.GOOS

	switch username {
	case `root`:
	default:
		username = def
	}

	switch operatingSystem {
	case `darwin`:
	default:
		operatingSystem = def
	}

	installMappings = map[string]map[string]map[string]string{
		def: {
			def: {
				def: filepath.Join(
					u.HomeDir, `/.config/google-chrome/NativeMessagingHosts`,
				),
				`root`: `/etc/opt/chrome/native-messaging-hosts`,
			},
			`vivaldi`: {
				def: filepath.Join(
					u.HomeDir, `/.config/vivaldi/NativeMessagingHosts`,
				),
				`root`: `/etc/vivaldi/native-messaging-hosts`,
			},
			`chromium`: {
				def: filepath.Join(
					u.HomeDir, `/.config/chromium/NativeMessagingHosts`,
				),
				`root`: `/etc/chromium/native-messaging-hosts`,
			},
		},
		`darwin`: {
			def: {
				def: filepath.Join(
					u.HomeDir, `/Library/Application Support/Google/Chrome/NativeMessagingHosts`,
				),
				`root`: `/Library/Google/Chrome/NativeMessagingHosts`,
			},
			`vivaldi`: {
				def: filepath.Join(
					u.HomeDir, `/Library/Application Support/Vivaldi/NativeMessagingHosts`,
				),
				`root`: `/Library/Vivaldi/NativeMessagingHosts`,
			},
			`chromium`: {
				def: filepath.Join(
					u.HomeDir, `/Library/Application Support/Chromium/NativeMessagingHosts`,
				),
				`root`: `/Library/Application Support/Chromium/NativeMessagingHosts`,
			},
		},
	}

	menu := climenu.NewCheckboxMenu(`Browser Selection`, `Select browser(s) for native host installation`, `OK`, `Cancel`)
	menu.AddMenuItem(`Chrome/Opera`, def)
	menu.AddMenuItem(`Chromium`, `chromium`)
	menu.AddMenuItem(`Vivaldi`, `vivaldi`)
	menu.AddMenuItem(`Custom`, `custom`)

	var (
		selection = make([]string, 0)
		escaped   bool
	)

	for len(selection) == 0 {
		selection, escaped = menu.Run()
		if escaped {
			os.Exit(1)
		}
	}

	if hasCustom(selection) {
		var response string
		for response == `` {
			defaultPath := installMappings[operatingSystem][username][def]
			response = climenu.GetText(`Enter the destination native messaging hosts path`, defaultPath)
		}
		selection = append(selection, response)
	}

	var extensionID string
	for extensionID == `` {
		extensionID = climenu.GetText(`Extension ID (Enter accepts default)`, defaultExtensionID)
	}

	for _, s := range selection {
		if s == `custom` {
			continue
		}
		path, ok := installMappings[operatingSystem][s][username]
		if !ok {
			// custom path
			path = s
		}
		if err := doInstall(path, extensionID); err != nil {
			return err
		}
	}

	fmt.Println(`Done.`)
	return nil
}
<commit_msg>Clarify checkbox menu behaviour in installer<commit_after>package main

import (
	"fmt"
	"html/template"
	"io"
	"os"
	"os/user"
	"path/filepath"
	"runtime"

	"github.com/kardianos/osext"
	"github.com/paulrademacher/climenu"
)

const (
	def                = `default`
	appName            = `com.0xc0dedbad.kdeconnect_chrome`
	defaultExtensionID = `ofmplbbfigookafjahpeepbggpofdhbo`
)

var (
	manifestTemplate = template.Must(template.New(`manifest`).Parse(`{
  "name": "com.0xc0dedbad.kdeconnect_chrome",
  "description": "KDE Connect",
  "path": "{{.Path}}",
  "type": "stdio",
  "allowed_origins": [
    "chrome-extension://{{.ExtensionID}}/"
  ]
}`))

	// OS/browser/user/path
	installMappings map[string]map[string]map[string]string
)

type manifest struct {
	Path        string
	ExtensionID string
}

func doInstall(path, extensionID string) error {
	daemonPath := filepath.Join(path, appName)
	templatePath := filepath.Join(path, fmt.Sprintf("%s.json", appName))

	if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
		return err
	}

	exe, err := osext.Executable()
	if err != nil {
		return err
	}
	in, err := os.Open(exe)
	defer func() {
		if e := in.Close(); err != nil {
			fmt.Println(e)
			panic(e)
		}
	}()
	if err != nil {
		return err
	}
	out, err := os.OpenFile(daemonPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
	defer func() {
		if e := out.Close(); err != nil {
			fmt.Println(e)
			panic(e)
		}
	}()
	if err != nil {
		return err
	}
	//fmt.Println(`Copying daemon`, daemonPath)
	if _, err = io.Copy(out, in); err != nil {
		return err
	}

	man, err := os.OpenFile(templatePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
	defer func() {
		if e := man.Close(); err != nil {
			fmt.Println(e)
			panic(e)
		}
	}()
	if err != nil {
		return err
	}
	//fmt.Println(`Writing template`, templatePath)
	if err = manifestTemplate.Execute(man, manifest{
		Path:        daemonPath,
		ExtensionID: extensionID,
	}); err != nil {
		return err
	}

	return nil
}

func hasCustom(selection []string) bool {
	for _, s := range selection {
		if s == `custom` {
			return true
		}
	}
	return false
}

func install() error {
	u, err := user.Current()
	if err != nil {
		return err
	}

	username := u.Username
	operatingSystem := runtime.GOOS

	switch username {
	case `root`:
	default:
		username = def
	}

	switch operatingSystem {
	case `darwin`:
	default:
		operatingSystem = def
	}

	installMappings = map[string]map[string]map[string]string{
		def: {
			def: {
				def: filepath.Join(
					u.HomeDir, `/.config/google-chrome/NativeMessagingHosts`,
				),
				`root`: `/etc/opt/chrome/native-messaging-hosts`,
			},
			`vivaldi`: {
				def: filepath.Join(
					u.HomeDir, `/.config/vivaldi/NativeMessagingHosts`,
				),
				`root`: `/etc/vivaldi/native-messaging-hosts`,
			},
			`chromium`: {
				def: filepath.Join(
					u.HomeDir, `/.config/chromium/NativeMessagingHosts`,
				),
				`root`: `/etc/chromium/native-messaging-hosts`,
			},
		},
		`darwin`: {
			def: {
				def: filepath.Join(
					u.HomeDir, `/Library/Application Support/Google/Chrome/NativeMessagingHosts`,
				),
				`root`: `/Library/Google/Chrome/NativeMessagingHosts`,
			},
			`vivaldi`: {
				def: filepath.Join(
					u.HomeDir, `/Library/Application Support/Vivaldi/NativeMessagingHosts`,
				),
				`root`: `/Library/Vivaldi/NativeMessagingHosts`,
			},
			`chromium`: {
				def: filepath.Join(
					u.HomeDir, `/Library/Application Support/Chromium/NativeMessagingHosts`,
				),
				`root`: `/Library/Application Support/Chromium/NativeMessagingHosts`,
			},
		},
	}

	menu := climenu.NewCheckboxMenu(`Browser Selection`, `Select browser(s) for native host installation (Space to select, Enter to confirm)`, `OK`, `Cancel`)
	menu.AddMenuItem(`Chrome/Opera`, def)
	menu.AddMenuItem(`Chromium`, `chromium`)
	menu.AddMenuItem(`Vivaldi`, `vivaldi`)
	menu.AddMenuItem(`Custom`, `custom`)

	var (
		selection = make([]string, 0)
		escaped   bool
	)

	for len(selection) == 0 {
		selection, escaped = menu.Run()
		if escaped {
			os.Exit(1)
		}
	}

	if hasCustom(selection) {
		var response string
		for response == `` {
			defaultPath := installMappings[operatingSystem][username][def]
			response = climenu.GetText(`Enter the destination native messaging hosts path`, defaultPath)
		}
		selection = append(selection, response)
	}

	var extensionID string
	for extensionID == `` {
		extensionID = climenu.GetText(`Extension ID (Enter accepts default)`, defaultExtensionID)
	}

	for _, s := range selection {
		if s == `custom` {
			continue
		}
		path, ok := installMappings[operatingSystem][s][username]
		if !ok {
			// custom path
			path = s
		}
		if err := doInstall(path, extensionID); err != nil {
			return err
		}
	}

	fmt.Println(`Done.`)
	return nil
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"regexp"
)

const (
	// Resource types
	RESOURCE_GROUP  = "group"
	RESOURCE_USER   = "user"
	RESOURCE_POLICY = "policy"

	// Constraints
	MAX_EXTERNAL_ID_LENGTH = 128
	MAX_NAME_LENGTH        = 128
	MAX_PATH_LENGTH        = 512

	// Actions

	// User actions
	USER_ACTION_CREATE_USER          = "iam:CreateUser"
	USER_ACTION_DELETE_USER          = "iam:DeleteUser"
	USER_ACTION_GET_USER             = "iam:GetUser"
	USER_ACTION_LIST_USERS           = "iam:ListUsers"
	USER_ACTION_UPDATE_USER          = "iam:UpdateUser"
	USER_ACTION_LIST_GROUPS_FOR_USER = "iam:ListGroupsForUser"
	USER_ACTION_LIST_ORG_USERS       = "iam:ListOrgUsers"

	// Group actions
	GROUP_ACTION_CREATE_GROUP                 = "iam:CreateGroup"
	GROUP_ACTION_DELETE_GROUP                 = "iam:DeleteGroup"
	GROUP_ACTION_GET_GROUP                    = "iam:GetGroup"
	GROUP_ACTION_LIST_GROUPS                  = "iam:ListGroups"
	GROUP_ACTION_UPDATE_GROUP                 = "iam:UpdateGroup"
	GROUP_ACTION_LIST_MEMBERS                 = "iam:ListMembers"
	GROUP_ACTION_ADD_MEMBER                   = "iam:AddMember"
	GROUP_ACTION_REMOVE_MEMBER                = "iam:RemoveMember"
	GROUP_ACTION_ATTACH_GROUP_POLICY          = "iam:AttachGroupPolicy"
	GROUP_ACTION_DETACH_GROUP_POLICY          = "iam:DetachGroupPolicy"
	GROUP_ACTION_LIST_ATTACHED_GROUP_POLICIES = "iam:ListAttachedGroupPolicies"
	GROUP_ACTION_LIST_ALL_GROUPS              = "iam:ListAllGroups"

	// Policy actions
	POLICY_ACTION_CREATE_POLICY        = "iam:CreatePolicy"
	POLICY_ACTION_DELETE_POLICY        = "iam:DeletePolicy"
	POLICY_ACTION_UPDATE_POLICY        = "iam:UpdatePolicy"
	POLICY_ACTION_GET_POLICY           = "iam:GetPolicy"
	POLICY_ACTION_LIST_ATTACHED_GROUPS = "iam:ListAttachedGroups"
	POLICY_ACTION_LIST_POLICIES        = "iam:ListPolicies"
	POLICY_ACTION_LIST_ALL_POLICIES    = "iam:ListAllPolicies"
)

func CreateUrn(org string, resource string, path string, name string) string {
	switch resource {
	case RESOURCE_USER:
		return fmt.Sprintf("urn:iws:iam:user%v%v", path, name)
	default:
		return fmt.Sprintf("urn:iws:iam:%v:%v%v%v", org, resource, path, name)
	}
}

func GetUrnPrefix(org string, resource string, path string) string {
	switch resource {
	case RESOURCE_USER:
		return fmt.Sprintf("urn:iws:iam:user%v*", path)
	default:
		return fmt.Sprintf("urn:iws:iam:%v:%v%v*", org, resource, path)
	}
}

func IsValidUserExternalID(externalID string) bool {
	r, _ := regexp.Compile(`^[\w+.@\-]+$`)
	return r.MatchString(externalID) && len(externalID) < MAX_EXTERNAL_ID_LENGTH
}

// this func validates group and policy names
func IsValidName(name string) bool {
	r, _ := regexp.Compile(`^[\w\-]+$`)
	return r.MatchString(name) && len(name) < MAX_NAME_LENGTH
}

func IsValidPath(path string) bool {
	r, _ := regexp.Compile(`^\*$|^/$|^/[\w+/\-]+\w+/$`)
	r2, _ := regexp.Compile(`[/]{2,}|[:]{2,}`)
	return r.MatchString(path) && !r2.MatchString(path) && len(path) < MAX_PATH_LENGTH
}

func IsValidEffect(effect string) error {
	if effect != "allow" && effect != "deny" {
		return &Error{
			Code:    REGEX_NO_MATCH,
			Message: fmt.Sprintf("No regex match in effect: %v", effect),
		}
	}
	return nil
}

func IsValidAction(actions []string) error {
	r, _ := regexp.Compile(`^[\w\-:]+[\w-*]+$`)
	r2, _ := regexp.Compile(`[*]{2,}|[:]{2,}`)
	for _, action := range actions {
		if !r.MatchString(action) || r2.MatchString(action) || len(action) > MAX_NAME_LENGTH {
			return &Error{
				Code:    REGEX_NO_MATCH,
				Message: fmt.Sprintf("No regex match in action: %v", action),
			}
		}
	}
	return nil
}

func IsValidResource(resources []string) error {
	r, _ := regexp.Compile(`^[*]$|^[\w+.@\-/:]+[\w+.@\-]+\*?$`)
	r2, _ := regexp.Compile(`[/]{2,}|[*]{2,}|[:]{2,}`)
	for _, resource := range resources {
		if !r.MatchString(resource) || r2.MatchString(resource) || len(resource) > MAX_PATH_LENGTH {
			return &Error{
				Code:    REGEX_NO_MATCH,
				Message: fmt.Sprintf("No regex match in resource: %v", resource),
			}
		}
	}
	return nil
}

func IsValidStatement(statements *[]Statement) error {
	for _, statement := range *statements {
		err := IsValidEffect(statement.Effect)
		if err != nil {
			return err
		}
		err = IsValidAction(statement.Action)
		if err != nil {
			return err
		}
		err = IsValidResource(statement.Resources)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Fix regex validations<commit_after>package api

import (
	"fmt"
	"regexp"
	"strings"
)

const (
	// Resource types
	RESOURCE_GROUP  = "group"
	RESOURCE_USER   = "user"
	RESOURCE_POLICY = "policy"

	// Constraints
	MAX_EXTERNAL_ID_LENGTH = 128
	MAX_NAME_LENGTH        = 128
	MAX_PATH_LENGTH        = 512

	// Actions

	// User actions
	USER_ACTION_CREATE_USER          = "iam:CreateUser"
	USER_ACTION_DELETE_USER          = "iam:DeleteUser"
	USER_ACTION_GET_USER             = "iam:GetUser"
	USER_ACTION_LIST_USERS           = "iam:ListUsers"
	USER_ACTION_UPDATE_USER          = "iam:UpdateUser"
	USER_ACTION_LIST_GROUPS_FOR_USER = "iam:ListGroupsForUser"
	USER_ACTION_LIST_ORG_USERS       = "iam:ListOrgUsers"

	// Group actions
	GROUP_ACTION_CREATE_GROUP                 = "iam:CreateGroup"
	GROUP_ACTION_DELETE_GROUP                 = "iam:DeleteGroup"
	GROUP_ACTION_GET_GROUP                    = "iam:GetGroup"
	GROUP_ACTION_LIST_GROUPS                  = "iam:ListGroups"
	GROUP_ACTION_UPDATE_GROUP                 = "iam:UpdateGroup"
	GROUP_ACTION_LIST_MEMBERS                 = "iam:ListMembers"
	GROUP_ACTION_ADD_MEMBER                   = "iam:AddMember"
	GROUP_ACTION_REMOVE_MEMBER                = "iam:RemoveMember"
	GROUP_ACTION_ATTACH_GROUP_POLICY          = "iam:AttachGroupPolicy"
	GROUP_ACTION_DETACH_GROUP_POLICY          = "iam:DetachGroupPolicy"
	GROUP_ACTION_LIST_ATTACHED_GROUP_POLICIES = "iam:ListAttachedGroupPolicies"
	GROUP_ACTION_LIST_ALL_GROUPS              = "iam:ListAllGroups"

	// Policy actions
	POLICY_ACTION_CREATE_POLICY        = "iam:CreatePolicy"
	POLICY_ACTION_DELETE_POLICY        = "iam:DeletePolicy"
	POLICY_ACTION_UPDATE_POLICY        = "iam:UpdatePolicy"
	POLICY_ACTION_GET_POLICY           = "iam:GetPolicy"
	POLICY_ACTION_LIST_ATTACHED_GROUPS = "iam:ListAttachedGroups"
	POLICY_ACTION_LIST_POLICIES        = "iam:ListPolicies"
	POLICY_ACTION_LIST_ALL_POLICIES    = "iam:ListAllPolicies"
)

func CreateUrn(org string, resource string, path string, name string) string {
	switch resource {
	case RESOURCE_USER:
		return fmt.Sprintf("urn:iws:iam:user%v%v", path, name)
	default:
		return fmt.Sprintf("urn:iws:iam:%v:%v%v%v", org, resource, path, name)
	}
}

func GetUrnPrefix(org string, resource string, path string) string {
	switch resource {
	case RESOURCE_USER:
		return fmt.Sprintf("urn:iws:iam:user%v*", path)
	default:
		return fmt.Sprintf("urn:iws:iam:%v:%v%v*", org, resource, path)
	}
}

func IsValidUserExternalID(externalID string) bool {
	r, _ := regexp.Compile(`^[\w+.@\-]+$`)
	return r.MatchString(externalID) && len(externalID) < MAX_EXTERNAL_ID_LENGTH
}

// this func validates group and policy names
func IsValidName(name string) bool {
	r, _ := regexp.Compile(`^[\w\-]+$`)
	return r.MatchString(name) && len(name) < MAX_NAME_LENGTH
}

func IsValidPath(path string) bool {
	r, _ := regexp.Compile(`^\*$|^/$|^/[\w+/\-]+\w+/$`)
	r2, _ := regexp.Compile(`[/]{2,}|[:]{2,}`)
	return r.MatchString(path) && !r2.MatchString(path) && len(path) < MAX_PATH_LENGTH
}

func IsValidEffect(effect string) error {
	if effect != "allow" && effect != "deny" {
		return &Error{
			Code:    REGEX_NO_MATCH,
			Message: fmt.Sprintf("No regex match in effect: %v", effect),
		}
	}
	return nil
}

func IsValidAction(actions []string) error {
	r, _ := regexp.Compile(`^[\w\-:]+[\w-*]+$`)
	r2, _ := regexp.Compile(`[*]{2,}|[:]{2,}`)
	for _, action := range actions {
		if !r.MatchString(action) || r2.MatchString(action) || len(action) > MAX_NAME_LENGTH {
			return &Error{
				Code:    REGEX_NO_MATCH,
				Message: fmt.Sprintf("No regex match in action: %v", action),
			}
		}
	}
	return nil
}

func IsValidResource(resources []string) error {
	//err generator helper
	errFunc := func(resource string) error {
		return &Error{
			Code:    REGEX_NO_MATCH,
			Message: fmt.Sprintf("No regex match in resource: %v", resource),
		}
	}

	wordRegex, _ := regexp.Compile(`^[\w+\-.@]+$`)
	wordPrefixRegex, _ := regexp.Compile(`^[\w+\-.@]+\*$`)

	r, _ := regexp.Compile(`^\*$|^[\w+\-@.]+\*?$|^[\w+\-@.]+\*?$|^[\w+\-@.]+(/?([\w+\-@.]+/)*([\w+\-@.]|[*])+)?$`)
	r2, _ := regexp.Compile(`[/]{2,}|[:]{2,}|[*]{2,}`)

	for _, resource := range resources {
		blocks := strings.Split(resource, ":")
		for n, block := range blocks {
			switch n {
			case 0:
				if len(blocks) < 2 { // This is the last block
					if block != "*" {
						return errFunc(resource)
					}
				} else {
					if block != "urn" {
						return errFunc(resource)
					}
				}
			case 1:
				if len(blocks) < 3 { // This is the last block
					if block != "*" && !wordPrefixRegex.MatchString(block) {
						return errFunc(resource)
					}
				} else {
					if !wordRegex.MatchString(block) {
						return errFunc(resource)
					}
				}
			case 2:
				if len(blocks) < 4 { // This is the last block
					if block != "*" && !wordPrefixRegex.MatchString(block) {
						return errFunc(resource)
					}
				} else {
					if !wordRegex.MatchString(block) {
						return errFunc(resource)
					}
				}
			case 3:
				if len(blocks) < 5 { // This is the last block
					if block != "*" && !wordPrefixRegex.MatchString(block) {
						return errFunc(resource)
					}
				} else {
					if block != "" && !wordRegex.MatchString(block) {
						return errFunc(resource)
					}
				}
			case 4:
				if !r.MatchString(block) || r2.MatchString(block) {
					return errFunc(resource)
				}
			default:
				return &Error{
					Code:    INVALID_PARAMETER_ERROR,
					Message: fmt.Sprintf("Invalid resource definition: %v", resource),
				}

			}
		}
	}
	return nil
}

func IsValidStatement(statements *[]Statement) error {
	for _, statement := range *statements {
		err := IsValidEffect(statement.Effect)
		if err != nil {
			return err
		}
		err = IsValidAction(statement.Action)
		if err != nil {
			return err
		}
		err = IsValidResource(statement.Resources)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Constantin Schomburg <me@cschomburg.com>
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package natural

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/xconstruct/stark/proto"
)

func ParseSimple(text string) (proto.Message, bool) {
	msg := proto.Message{}

	if strings.HasPrefix(text, "{") {
		if err := json.Unmarshal([]byte(text), &msg); err == nil {
			return msg, true
		}
	}

	parts := strings.Split(text, " ")
	msg.Action = parts[0]
	if msg.Action == "" {
		return msg, false
	}

	payload := make(map[string]interface{}, 0)
	for _, part := range parts[1:] {
		keyval := strings.SplitN(part, "=", 2)
		if len(keyval) == 1 {
			return msg, false
		}

		k, v := keyval[0], keyval[1]
		switch k {
		case "device":
			fallthrough
		case "destination":
			msg.Destination = v
		default:
			payload[k] = v
		}
	}
	if len(payload) > 0 {
		msg.EncodePayload(payload)
	}
	return msg, true
}

func FormatSimple(msg proto.Message) string {
	if msg.Text != "" {
		return msg.Text
	}

	return fmt.Sprintf("%s from %s.", msg.Action, msg.Source)
}
<commit_msg>Natural: Correctly set text with simple parser.<commit_after>// Copyright (C) 2014 Constantin Schomburg <me@cschomburg.com>
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package natural

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/xconstruct/stark/proto"
)

func ParseSimple(text string) (proto.Message, bool) {
	msg := proto.Message{}

	if strings.HasPrefix(text, "{") {
		if err := json.Unmarshal([]byte(text), &msg); err == nil {
			return msg, true
		}
	}

	parts := strings.Split(text, " ")
	msg.Action = parts[0]
	if msg.Action == "" {
		return msg, false
	}

	payload := make(map[string]interface{}, 0)
	for _, part := range parts[1:] {
		keyval := strings.SplitN(part, "=", 2)
		if len(keyval) == 1 {
			return msg, false
		}

		k, v := keyval[0], keyval[1]
		switch k {
		case "text":
			msg.Text = v
		case "device":
			fallthrough
		case "destination":
			msg.Destination = v
		default:
			payload[k] = v
		}
	}
	if len(payload) > 0 {
		msg.EncodePayload(payload)
	}
	return msg, true
}

func FormatSimple(msg proto.Message) string {
	if msg.Text != "" {
		return msg.Text
	}

	return fmt.Sprintf("%s from %s.", msg.Action, msg.Source)
}
<|endoftext|>


<commit_before>package zerolog

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"

	"github.com/airbrake/gobrake/v5"
	"github.com/buger/jsonparser"
	"github.com/rs/zerolog"
)

type WriteCloser struct {
	Gobrake *gobrake.Notifier
}

// Validates the WriteCloser matches the io.WriteCloser interface
var _ io.WriteCloser = (*WriteCloser)(nil)

// New creates a new WriteCloser
func New(notifier *gobrake.Notifier) (io.WriteCloser, error) {
	if notifier == nil {
		return &WriteCloser{}, errors.New("airbrake notifier not provided")
	}
	return &WriteCloser{Gobrake: notifier}, nil
}

// Write parses the log data and sends off error notices to airbrake
func (w *WriteCloser) Write(data []byte) (int, error) {
	lvl, err := jsonparser.GetUnsafeString(data, zerolog.LevelFieldName)
	if err != nil {
		return 0, fmt.Errorf("error getting zerolog level: %w", err)
	}

	if lvl != zerolog.ErrorLevel.String() {
		return len(data), nil
	}

	var logEntryData interface{}
	err = json.Unmarshal(data, &logEntryData)
	if err != nil {
		return 0, fmt.Errorf("error unmarshalling logs: %w", err)
	}
	type zeroError struct {
		message string
		error   string
	}
	var ze zeroError
	_ = jsonparser.ObjectEach(data, func(key, value []byte, vt jsonparser.ValueType, offset int) error {
		switch string(key) {
		case zerolog.MessageFieldName:
			ze.message = string(value)
		case zerolog.ErrorFieldName:
			ze.error = string(value)
		}

		return nil
	})

	// If gobrake was not setup but the writer was still used, ignore gobrake.
	if w.Gobrake == nil {
		return len(data), nil
	}

	notice := gobrake.NewNotice(ze.message, nil, 6)
	notice.Context["severity"] = lvl
	notice.Params["logEntryData"] = logEntryData
	notice.Error = errors.New(ze.error)
	w.Gobrake.SendNoticeAsync(notice)
	return len(data), nil
}

// Close flushes any remaining notices left in gobrake queue
func (w *WriteCloser) Close() error {
	w.Gobrake.Flush()
	return nil
}
<commit_msg>update zerolog integration to check for a couple special fields, and if present, move them from Notice.Params to Notice.Context<commit_after>package zerolog

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"

	"github.com/airbrake/gobrake/v5"
	"github.com/buger/jsonparser"
	"github.com/rs/zerolog"
)

type WriteCloser struct {
	Gobrake *gobrake.Notifier
}

// Validates the WriteCloser matches the io.WriteCloser interface
var _ io.WriteCloser = (*WriteCloser)(nil)

// New creates a new WriteCloser
func New(notifier *gobrake.Notifier) (io.WriteCloser, error) {
	if notifier == nil {
		return &WriteCloser{}, errors.New("airbrake notifier not provided")
	}
	return &WriteCloser{Gobrake: notifier}, nil
}

// Write parses the log data and sends off error notices to airbrake
func (w *WriteCloser) Write(data []byte) (int, error) {
	lvl, err := jsonparser.GetUnsafeString(data, zerolog.LevelFieldName)
	if err != nil {
		return 0, fmt.Errorf("error getting zerolog level: %w", err)
	}

	if lvl != zerolog.ErrorLevel.String() {
		return len(data), nil
	}

	var logEntryData interface{}
	err = json.Unmarshal(data, &logEntryData)
	if err != nil {
		return 0, fmt.Errorf("error unmarshalling logs: %w", err)
	}
	type zeroError struct {
		message string
		error   string
	}
	var ze zeroError
	_ = jsonparser.ObjectEach(data, func(key, value []byte, vt jsonparser.ValueType, offset int) error {
		switch string(key) {
		case zerolog.MessageFieldName:
			ze.message = string(value)
		case zerolog.ErrorFieldName:
			ze.error = string(value)
		}

		return nil
	})

	// If gobrake was not setup but the writer was still used, ignore gobrake.
	if w.Gobrake == nil {
		return len(data), nil
	}

	notice := gobrake.NewNotice(ze.message, nil, 6)
	notice.Context["severity"] = lvl

	// Check for the following 2 fields in logEntryData to see if they
	// can be moved to the `Notice.Context`. Doing so would automatically link
	// them in airbrake.io dashboards.
	if asMap, ok := logEntryData.(map[string]interface{}); ok {
		const HttpMethod = "httpMethod"
		const Route = "route"

		if method, ok := asMap[HttpMethod].(string); ok {
			notice.Context[HttpMethod] = method
			delete(asMap, HttpMethod)
		}

		if route, ok := asMap[Route].(string); ok {
			notice.Context[Route] = route
			delete(asMap, Route)
		}
	}

	notice.Params["logEntryData"] = logEntryData
	notice.Error = errors.New(ze.error)
	w.Gobrake.SendNoticeAsync(notice)
	return len(data), nil
}

// Close flushes any remaining notices left in gobrake queue
func (w *WriteCloser) Close() error {
	w.Gobrake.Flush()
	return nil
}
<|endoftext|>