<commit_before>// Package db implements an immutable, consistent, in-memory key/value store.
// DB uses an immutable Left-Leaning Red-Black tree (LLRB) internally.
package db

import (
	"encoding/binary"
	"reflect"
	"sync"
	"sync/atomic"
	"unsafe"

	"github.com/azmodb/db/backend"
	"github.com/azmodb/llrb"
)

const (
	// errRevisionNotFound is returned when trying to access a revision
	// that has not been created.
	errRevisionNotFound = perror("revision not found")

	// errKeyNotFound is returned when trying to access a key that has
	// not been created.
	errKeyNotFound = perror("key not found")

	// errIncompatibleValue is returned when trying create or delete a
	// value on an imcompatible key.
	errIncompatibleValue = perror("incompatible value")

	// pairDeleted is the error returned by a watcher when the
	// underlying is deleted.
	pairDeleted = perror("key/value pair deleted")

	// notifierCanceled is the error returned when the watcher is
	// canceled.
	notifierCanceled = perror("watcher shut down")

	// errInvertedRange is returned when a inverted range is supplied.
	errInvertedRange = perror("inverted range")
)

type perror string

func (e perror) Error() string { return string(e) }

// DB represents an immutable, consistent, in-memory key/value database.
// All access is performed through a transaction which can be obtained
// through the database.
type DB struct {
	writer sync.Mutex // exclusive writer transaction
	tree   unsafe.Pointer
}

type tree struct {
	root *llrb.Tree
	rev  int64
}

func newDB(t *tree) *DB {
	if t == nil {
		t = &tree{root: &llrb.Tree{}}
	}
	return &DB{tree: unsafe.Pointer(t)}
}

// New returns an immutable, consistent, in-memory key/value database.
func New() *DB { return newDB(nil) }

func (db *DB) Reload(backend backend.Backend) error {
	rev, err := backend.Last()
	if err != nil {
		return nil, err
	}

	tree := &tree{
		rev:  int64(binary.BigEndian.Uint64(rev[:])),
		root: &llrb.Tree{},
	}
	txn := tree.root.Txn()

	err = backend.Range(rev, func(key, value []byte) (err error) {
		buf := newBuffer(value)
		blocks := []block{}
		if err = decode(buf, &blocks); err != nil {
			return err
		}

		p := &pair{
			stream: &stream{},
			blocks: blocks,
		}
		p.key = make([]byte, len(key))
		copy(p.key, key)

		txn.Insert(p)
		return err
	})
	if err != nil {
		return nil, err
	}

	tree.root = txn.Commit()
	return newDB(tree), nil
}

func (db *DB) Snapshot(backend backend.Backend) error {
	tree := db.load()

	rev := [8]byte{}
	binary.BigEndian.PutUint64(rev[:], uint64(tree.rev))

	batch, err := backend.Batch(rev)
	if err != nil {
		return err
	}

	tree.root.ForEach(func(elem llrb.Element) bool {
		p := elem.(*pair)

		buf := newBuffer(nil)
		if err = encode(buf, p.blocks); err != nil {
			return true
		}

		flushed, err := batch.Put(p.key, buf.Bytes())
		if err != nil {
			return true
		}
		if flushed {
		}
		return false
	})
	if err != nil {
		batch.Close()
		return err
	}
	return batch.Close()
}

func (db *DB) store(t *tree) {
	atomic.StorePointer(&db.tree, unsafe.Pointer(t))
}

func (db *DB) load() *tree {
	return (*tree)(atomic.LoadPointer(&db.tree))
}

// Get retrieves the value for a key at revision rev. If rev <= 0 it
// returns the current value for a key. If equal is true the value
// revision must match the supplied rev.
//
// Get returns the revision of the key/value pair, the current revision
// of the database and an errors if any.
func (db *DB) Get(key []byte, rev int64, equal bool) (interface{}, int64, int64, error) {
	match := newMatcher(key)
	defer match.release()
	tree := db.load()

	if elem := tree.root.Get(match); elem != nil {
		p := elem.(*pair)
		b, found := lookup(p, rev, equal)
		if found {
			return b.Data, b.Rev, tree.rev, nil
		}
		return nil, 0, tree.rev, errRevisionNotFound
	}
	return nil, 0, tree.rev, errKeyNotFound
}

func lookup(p *pair, rev int64, equal bool) (block, bool) {
	var b block
	if rev > 0 {
		index, found := p.find(rev, equal)
		if !found {
			return b, false
		}
		b = p.at(index)
	} else {
		b = p.last()
	}
	return b, true
}

func rangeFunc(n *Notifier, rev int64, current int64, limit int32) llrb.Visitor {
	return func(elem llrb.Element) bool {
		p := elem.(*pair)
		b, found := lookup(p, rev, false)
		if found {
			return !n.send(p.key, b.Data, b.Rev, current)
		}
		return false // ignore revision not found error
	}
}

func (db *DB) get(tree *tree, key []byte, rev int64) (*Notifier, int64, error) {
	n := newNotifier(42, nil, 1)
	go func() {
		data, created, current, err := db.Get(key, rev, false)
		if err != nil {
			n.close(err)
		} else {
			n.send(key, data, created, current)
		}
	}()
	return n, tree.rev, nil
}

// Range iterates over values stored in the database in the range at rev
// over the interval [from, to] from left to right. Limit limits the
// number of keys returned. If rev <= 0 Range gets the keys at the
// current revision of the database. From/To combination:
//
//	from == nil && to == nil:
//		the request returns all keys in the database
//	from != nil && to != nil:
//		the request returns the keys in the interval
//	from != nil && to == nil:
//		the request returns the key (like Get)
//
// Range returns a notifier, the current revision of the database and an
// error if any.
func (db *DB) Range(from, to []byte, rev int64, limit int32) (*Notifier, int64, error) {
	tree := db.load()
	if compare(from, to) > 0 {
		return nil, tree.rev, errInvertedRange
	}

	if from != nil && to == nil { // simulate get request with equal == false
		return db.get(tree, from, rev)
	}

	n := newNotifier(42, nil, defaultNotifierCapacity)
	go func() {
		defer n.Cancel() // in any case cancel the infinte event queue

		if from == nil && to == nil { // foreach request
			tree.root.ForEach(rangeFunc(n, rev, tree.rev, limit))
			return
		}

		lo, hi := newMatcher(from), newMatcher(to)
		defer func() {
			lo.release()
			hi.release()
		}()
		tree.root.Range(lo, hi, rangeFunc(n, rev, tree.rev, limit))
	}()

	return n, tree.rev, nil
}

// Rev returns the current revision of the database.
func (db *DB) Rev() int64 {
	tree := db.load()
	return tree.rev
}

// Watch returns a notifier for a key. If the key does not exist it
// returns an error.
func (db *DB) Watch(key []byte) (*Notifier, int64, error) {
	match := newMatcher(key)
	defer match.release()
	tree := db.load()

	if elem := tree.root.Get(match); elem != nil {
		p := elem.(*pair)
		return p.stream.Register(), tree.rev, nil
	}
	return nil, tree.rev, errKeyNotFound
}

// Txn starts a new batch transaction. Only one batch transaction can
// be used at a time. Starting multiple batch transactions will cause
// the calls to block and be serialized until the current transaction
// finishes.
func (db *DB) Txn() *Txn {
	db.writer.Lock()
	tree := db.load()
	return &Txn{txn: tree.root.Txn(), rev: tree.rev, db: db}
}

// Txn represents a batch transaction on the database.
type Txn struct {
	txn *llrb.Txn
	rev int64
	db  *DB
}

// Updater is a function that operates on a key/value pair
type Updater func(data interface{}) interface{}

// Update updates the value for a key. If the key exists and tombstone is
// true then its previous versions will be overwritten. Supplied key
// and value must remain valid for the life of the database.
//
// It the key exists and the value data type differ it returns an error.
func (tx *Txn) Update(key []byte, up Updater, tombstone bool) (int64, error) {
	match := newMatcher(key)
	defer match.release()

	rev := tx.rev + 1
	var p *pair
	if elem := tx.txn.Get(match); elem != nil {
		p = elem.(*pair)
		last := p.last().Data
		data := up(last)
		if !typeEqual(last, data) {
			return tx.rev, errIncompatibleValue
		}
		p = p.insert(data, rev, tombstone)
	} else {
		p = newPair(key, up(nil), rev)
	}
	tx.txn.Insert(p)
	tx.rev = rev
	p.stream.Notify(p, rev)

	return tx.rev, nil
}

func noop(data interface{}) Updater {
	return func(_ interface{}) interface{} {
		return data
	}
}

// Put sets the value for a key. If the key exists and tombstone is true
// then its previous versions will be overwritten. Supplied key and
// value must remain valid for the life of the database.
//
// It the key exists and the value data type differ, it returns an error.
func (tx *Txn) Put(key []byte, data interface{}, tombstone bool) (int64, error) {
	return tx.Update(key, noop(data), tombstone)
}

// Delete removes a key/value pair and returns the current revision of the
// database.
func (tx *Txn) Delete(key []byte) int64 {
	match := newMatcher(key)
	defer match.release()

	if elem := tx.txn.Get(match); elem != nil {
		p := elem.(*pair)
		tx.txn.Delete(p)
		tx.rev++
		p.stream.Notify(p, tx.rev)
		p.stream.Cancel()
	}
	return tx.rev
}

// Commit closes the transaction and writes all changes into the
// database.
func (tx *Txn) Commit() {
	if tx.txn == nil { // already aborted or committed
		return
	}

	tree := &tree{root: tx.txn.Commit(), rev: tx.rev}
	tx.db.store(tree)
	tx.txn = nil
	tx.rev = 0
	tx.db.writer.Unlock() // release the writer lock
	tx.db = nil
}

// Rollback closes the transaction and ignores all previous updates.
func (tx *Txn) Rollback() {
	if tx.txn == nil { // already aborted or committed
		return
	}

	tx.txn = nil
	tx.db.writer.Unlock() // release the writer lock
	tx.db = nil
}

func typeEqual(a, b interface{}) bool {
	at, bt := reflect.TypeOf(a), reflect.TypeOf(b)
	ak, bk := at.Kind(), bt.Kind()
	if ak != bk {
		return false
	}
	if ak == reflect.Slice ||
		ak == reflect.Array ||
		ak == reflect.Chan ||
		ak == reflect.Map ||
		ak == reflect.Ptr {
		if at.Elem() != bt.Elem() {
			println("x")
			return false
		}
	}
	return true
}
<commit_msg>fix snapshotting<commit_after>// Package db implements an immutable, consistent, in-memory key/value store.
// DB uses an immutable Left-Leaning Red-Black tree (LLRB) internally.
package db

import (
	"encoding/binary"
	"reflect"
	"sync"
	"sync/atomic"
	"unsafe"

	"github.com/azmodb/db/backend"
	"github.com/azmodb/llrb"
)

const (
	// errRevisionNotFound is returned when trying to access a revision
	// that has not been created.
	errRevisionNotFound = perror("revision not found")

	// errKeyNotFound is returned when trying to access a key that has
	// not been created.
	errKeyNotFound = perror("key not found")

	// errIncompatibleValue is returned when trying create or delete a
	// value on an imcompatible key.
	errIncompatibleValue = perror("incompatible value")

	// pairDeleted is the error returned by a watcher when the
	// underlying is deleted.
	pairDeleted = perror("key/value pair deleted")

	// notifierCanceled is the error returned when the watcher is
	// canceled.
	notifierCanceled = perror("watcher shut down")

	// errInvertedRange is returned when a inverted range is supplied.
	errInvertedRange = perror("inverted range")
)

type perror string

func (e perror) Error() string { return string(e) }

// DB represents an immutable, consistent, in-memory key/value database.
// All access is performed through a transaction which can be obtained
// through the database.
type DB struct {
	writer sync.Mutex // exclusive writer transaction
	tree   unsafe.Pointer
}

type tree struct {
	root *llrb.Tree
	rev  int64
}

func newDB(t *tree) *DB {
	if t == nil {
		t = &tree{root: &llrb.Tree{}}
	}
	return &DB{tree: unsafe.Pointer(t)}
}

// New returns an immutable, consistent, in-memory key/value database.
func New() *DB { return newDB(nil) }

// Reload reloads the immutable, consistent, in-memory key/value database
// from the underlying backend.
func Reload(backend backend.Backend) (*DB, error) {
	rev, err := backend.Last()
	if err != nil {
		return nil, err
	}

	tree := &tree{
		rev:  int64(binary.BigEndian.Uint64(rev[:])),
		root: &llrb.Tree{},
	}
	txn := tree.root.Txn()

	err = backend.Range(rev, func(key, value []byte) (err error) {
		buf := newBuffer(value)
		blocks := []block{}
		if err = decode(buf, &blocks); err != nil {
			return err
		}

		p := &pair{
			stream: &stream{},
			blocks: blocks,
		}
		p.key = make([]byte, len(key))
		copy(p.key, key)

		txn.Insert(p)
		return err
	})
	if err != nil {
		return nil, err
	}

	tree.root = txn.Commit()
	return newDB(tree), nil
}

// Snapshot writes the entire in-memory database to the underlying
// backend.
func (db *DB) Snapshot(backend backend.Backend) error {
	tree := db.load()

	rev := [8]byte{}
	binary.BigEndian.PutUint64(rev[:], uint64(tree.rev))

	batch, err := backend.Batch(rev)
	if err != nil {
		return err
	}

	buf := newBuffer(nil)
	tree.root.ForEach(func(elem llrb.Element) bool {
		p := elem.(*pair)

		buf.Reset()
		if err = encode(buf, p.blocks); err != nil {
			return true
		}

		if err = batch.Put(p.key, buf.Bytes()); err != nil {
			return true
		}
		return false
	})
	if err != nil {
		batch.Close()
		return err
	}
	return batch.Close()
}

func (db *DB) store(t *tree) {
	atomic.StorePointer(&db.tree, unsafe.Pointer(t))
}

func (db *DB) load() *tree {
	return (*tree)(atomic.LoadPointer(&db.tree))
}

// Get retrieves the value for a key at revision rev. If rev <= 0 it
// returns the current value for a key. If equal is true the value
// revision must match the supplied rev.
//
// Get returns the revision of the key/value pair, the current revision
// of the database and an errors if any.
func (db *DB) Get(key []byte, rev int64, equal bool) (interface{}, int64, int64, error) {
	match := newMatcher(key)
	defer match.release()
	tree := db.load()

	if elem := tree.root.Get(match); elem != nil {
		p := elem.(*pair)
		b, found := lookup(p, rev, equal)
		if found {
			return b.Data, b.Rev, tree.rev, nil
		}
		return nil, 0, tree.rev, errRevisionNotFound
	}
	return nil, 0, tree.rev, errKeyNotFound
}

func lookup(p *pair, rev int64, equal bool) (block, bool) {
	var b block
	if rev > 0 {
		index, found := p.find(rev, equal)
		if !found {
			return b, false
		}
		b = p.at(index)
	} else {
		b = p.last()
	}
	return b, true
}

func rangeFunc(n *Notifier, rev int64, current int64, limit int32) llrb.Visitor {
	return func(elem llrb.Element) bool {
		p := elem.(*pair)
		b, found := lookup(p, rev, false)
		if found {
			return !n.send(p.key, b.Data, b.Rev, current)
		}
		return false // ignore revision not found error
	}
}

func (db *DB) get(tree *tree, key []byte, rev int64) (*Notifier, int64, error) {
	n := newNotifier(42, nil, 1)
	go func() {
		data, created, current, err := db.Get(key, rev, false)
		if err != nil {
			n.close(err)
		} else {
			n.send(key, data, created, current)
		}
	}()
	return n, tree.rev, nil
}

// Range iterates over values stored in the database in the range at rev
// over the interval [from, to] from left to right. Limit limits the
// number of keys returned. If rev <= 0 Range gets the keys at the
// current revision of the database. From/To combination:
//
//	from == nil && to == nil:
//		the request returns all keys in the database
//	from != nil && to != nil:
//		the request returns the keys in the interval
//	from != nil && to == nil:
//		the request returns the key (like Get)
//
// Range returns a notifier, the current revision of the database and an
// error if any.
func (db *DB) Range(from, to []byte, rev int64, limit int32) (*Notifier, int64, error) {
	tree := db.load()
	if compare(from, to) > 0 {
		return nil, tree.rev, errInvertedRange
	}

	if from != nil && to == nil { // simulate get request with equal == false
		return db.get(tree, from, rev)
	}

	n := newNotifier(42, nil, defaultNotifierCapacity)
	go func() {
		defer n.Cancel() // in any case cancel the infinte event queue

		if from == nil && to == nil { // foreach request
			tree.root.ForEach(rangeFunc(n, rev, tree.rev, limit))
			return
		}

		lo, hi := newMatcher(from), newMatcher(to)
		defer func() {
			lo.release()
			hi.release()
		}()
		tree.root.Range(lo, hi, rangeFunc(n, rev, tree.rev, limit))
	}()

	return n, tree.rev, nil
}

// Rev returns the current revision of the database.
func (db *DB) Rev() int64 {
	tree := db.load()
	return tree.rev
}

// Watch returns a notifier for a key. If the key does not exist it
// returns an error.
func (db *DB) Watch(key []byte) (*Notifier, int64, error) {
	match := newMatcher(key)
	defer match.release()
	tree := db.load()

	if elem := tree.root.Get(match); elem != nil {
		p := elem.(*pair)
		return p.stream.Register(), tree.rev, nil
	}
	return nil, tree.rev, errKeyNotFound
}

// Txn starts a new batch transaction. Only one batch transaction can
// be used at a time. Starting multiple batch transactions will cause
// the calls to block and be serialized until the current transaction
// finishes.
func (db *DB) Txn() *Txn {
	db.writer.Lock()
	tree := db.load()
	return &Txn{txn: tree.root.Txn(), rev: tree.rev, db: db}
}

// Txn represents a batch transaction on the database.
type Txn struct {
	txn *llrb.Txn
	rev int64
	db  *DB
}

// Updater is a function that operates on a key/value pair
type Updater func(data interface{}) interface{}

// Update updates the value for a key. If the key exists and tombstone is
// true then its previous versions will be overwritten. Supplied key
// and value must remain valid for the life of the database.
//
// It the key exists and the value data type differ it returns an error.
func (tx *Txn) Update(key []byte, up Updater, tombstone bool) (int64, error) {
	match := newMatcher(key)
	defer match.release()

	rev := tx.rev + 1
	var p *pair
	if elem := tx.txn.Get(match); elem != nil {
		p = elem.(*pair)
		last := p.last().Data
		data := up(last)
		if !typeEqual(last, data) {
			return tx.rev, errIncompatibleValue
		}
		p = p.insert(data, rev, tombstone)
	} else {
		p = newPair(key, up(nil), rev)
	}
	tx.txn.Insert(p)
	tx.rev = rev
	p.stream.Notify(p, rev)

	return tx.rev, nil
}

func noop(data interface{}) Updater {
	return func(_ interface{}) interface{} {
		return data
	}
}

// Put sets the value for a key. If the key exists and tombstone is true
// then its previous versions will be overwritten. Supplied key and
// value must remain valid for the life of the database.
//
// It the key exists and the value data type differ, it returns an error.
func (tx *Txn) Put(key []byte, data interface{}, tombstone bool) (int64, error) {
	return tx.Update(key, noop(data), tombstone)
}

// Delete removes a key/value pair and returns the current revision of the
// database.
func (tx *Txn) Delete(key []byte) int64 {
	match := newMatcher(key)
	defer match.release()

	if elem := tx.txn.Get(match); elem != nil {
		p := elem.(*pair)
		tx.txn.Delete(p)
		tx.rev++
		p.stream.Notify(p, tx.rev)
		p.stream.Cancel()
	}
	return tx.rev
}

// Commit closes the transaction and writes all changes into the
// database.
func (tx *Txn) Commit() {
	if tx.txn == nil { // already aborted or committed
		return
	}

	tree := &tree{root: tx.txn.Commit(), rev: tx.rev}
	tx.db.store(tree)
	tx.txn = nil
	tx.rev = 0
	tx.db.writer.Unlock() // release the writer lock
	tx.db = nil
}

// Rollback closes the transaction and ignores all previous updates.
func (tx *Txn) Rollback() {
	if tx.txn == nil { // already aborted or committed
		return
	}

	tx.txn = nil
	tx.db.writer.Unlock() // release the writer lock
	tx.db = nil
}

func typeEqual(a, b interface{}) bool {
	at, bt := reflect.TypeOf(a), reflect.TypeOf(b)
	ak, bk := at.Kind(), bt.Kind()
	if ak != bk {
		return false
	}
	if ak == reflect.Slice ||
		ak == reflect.Array ||
		ak == reflect.Chan ||
		ak == reflect.Map ||
		ak == reflect.Ptr {
		if at.Elem() != bt.Elem() {
			println("x")
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package instana

import (
	"runtime"
	"strconv"
	"time"
)

const (
	// SnapshotPeriod is the amount of time in seconds between snapshot reports.
	SnapshotPeriod = 600
)

// SnapshotS struct to hold snapshot data.
type SnapshotS struct {
	Name     string `json:"name"`
	Version  string `json:"version"`
	Root     string `json:"goroot"`
	MaxProcs int    `json:"maxprocs"`
	Compiler string `json:"compiler"`
	NumCPU   int    `json:"cpu"`
}

// MemoryS struct to hold snapshot data.
type MemoryS struct {
	Alloc         uint64  `json:"alloc"`
	TotalAlloc    uint64  `json:"total_alloc"`
	Sys           uint64  `json:"sys"`
	Lookups       uint64  `json:"lookups"`
	Mallocs       uint64  `json:"mallocs"`
	Frees         uint64  `json:"frees"`
	HeapAlloc     uint64  `json:"heap_alloc"`
	HeapSys       uint64  `json:"heap_sys"`
	HeapIdle      uint64  `json:"heap_idle"`
	HeapInuse     uint64  `json:"heap_in_use"`
	HeapReleased  uint64  `json:"heap_released"`
	HeapObjects   uint64  `json:"heap_objects"`
	PauseTotalNs  uint64  `json:"pause_total_ns"`
	PauseNs       uint64  `json:"pause_ns"`
	NumGC         uint32  `json:"num_gc"`
	GCCPUFraction float64 `json:"gc_cpu_fraction"`
}

// MetricsS struct to hold snapshot data.
type MetricsS struct {
	CgoCall   int64    `json:"cgo_call"`
	Goroutine int      `json:"goroutine"`
	Memory    *MemoryS `json:"memory"`
}

// EntityData struct to hold snapshot data.
type EntityData struct {
	PID      int        `json:"pid"`
	Snapshot *SnapshotS `json:"snapshot,omitempty"`
	Metrics  *MetricsS  `json:"metrics"`
}

type meterS struct {
	sensor            *sensorS
	numGC             uint32
	ticker            *time.Ticker
	snapshotCountdown int
}

func newMeter(sensor *sensorS) *meterS {
	sensor.logger.Debug("initializing meter")

	meter := &meterS{
		sensor: sensor,
		ticker: time.NewTicker(1 * time.Second),
	}

	go func() {
		meter.snapshotCountdown = 1
		for range meter.ticker.C {
			if meter.sensor.agent.canSend() {
				meter.snapshotCountdown--
				var s *SnapshotS
				if meter.snapshotCountdown == 0 {
					meter.snapshotCountdown = SnapshotPeriod
					s = meter.collectSnapshot()
					meter.sensor.logger.Debug("collected snapshot")
				}

				pid, _ := strconv.Atoi(meter.sensor.agent.from.PID)
				d := &EntityData{
					PID:      pid,
					Snapshot: s,
					Metrics:  meter.collectMetrics(),
				}

				go meter.send(d)
			}
		}
	}()

	return meter
}

func (r *meterS) send(d *EntityData) {
	_, err := r.sensor.agent.request(r.sensor.agent.makeURL(agentDataURL), "POST", d)

	if err != nil {
		r.sensor.agent.reset()
	}
}

func (r *meterS) collectMemoryMetrics() *MemoryS {
	var memStats runtime.MemStats
	runtime.ReadMemStats(&memStats)
	ret := &MemoryS{
		Alloc:         memStats.Alloc,
		TotalAlloc:    memStats.TotalAlloc,
		Sys:           memStats.Sys,
		Lookups:       memStats.Lookups,
		Mallocs:       memStats.Mallocs,
		Frees:         memStats.Frees,
		HeapAlloc:     memStats.HeapAlloc,
		HeapSys:       memStats.HeapSys,
		HeapIdle:      memStats.HeapIdle,
		HeapInuse:     memStats.HeapInuse,
		HeapReleased:  memStats.HeapReleased,
		HeapObjects:   memStats.HeapObjects,
		PauseTotalNs:  memStats.PauseTotalNs,
		NumGC:         memStats.NumGC,
		GCCPUFraction: memStats.GCCPUFraction}

	if r.numGC < memStats.NumGC {
		ret.PauseNs = memStats.PauseNs[(memStats.NumGC+255)%256]
		r.numGC = memStats.NumGC
	} else {
		ret.PauseNs = 0
	}

	return ret
}

func (r *meterS) collectMetrics() *MetricsS {
	return &MetricsS{
		CgoCall:   runtime.NumCgoCall(),
		Goroutine: runtime.NumGoroutine(),
		Memory:    r.collectMemoryMetrics()}
}

func (r *meterS) collectSnapshot() *SnapshotS {
	return &SnapshotS{
		Name:     r.sensor.serviceName,
		Version:  runtime.Version(),
		Root:     runtime.GOROOT(),
		MaxProcs: runtime.GOMAXPROCS(0),
		Compiler: runtime.Compiler,
		NumCPU:   runtime.NumCPU()}
}
<commit_msg>Use local ticker to gather metrics<commit_after>package instana

import (
	"runtime"
	"strconv"
	"time"
)

const (
	// SnapshotPeriod is the amount of time in seconds between snapshot reports.
	SnapshotPeriod = 600
)

// SnapshotS struct to hold snapshot data.
type SnapshotS struct {
	Name     string `json:"name"`
	Version  string `json:"version"`
	Root     string `json:"goroot"`
	MaxProcs int    `json:"maxprocs"`
	Compiler string `json:"compiler"`
	NumCPU   int    `json:"cpu"`
}

// MemoryS struct to hold snapshot data.
type MemoryS struct {
	Alloc         uint64  `json:"alloc"`
	TotalAlloc    uint64  `json:"total_alloc"`
	Sys           uint64  `json:"sys"`
	Lookups       uint64  `json:"lookups"`
	Mallocs       uint64  `json:"mallocs"`
	Frees         uint64  `json:"frees"`
	HeapAlloc     uint64  `json:"heap_alloc"`
	HeapSys       uint64  `json:"heap_sys"`
	HeapIdle      uint64  `json:"heap_idle"`
	HeapInuse     uint64  `json:"heap_in_use"`
	HeapReleased  uint64  `json:"heap_released"`
	HeapObjects   uint64  `json:"heap_objects"`
	PauseTotalNs  uint64  `json:"pause_total_ns"`
	PauseNs       uint64  `json:"pause_ns"`
	NumGC         uint32  `json:"num_gc"`
	GCCPUFraction float64 `json:"gc_cpu_fraction"`
}

// MetricsS struct to hold snapshot data.
type MetricsS struct {
	CgoCall   int64    `json:"cgo_call"`
	Goroutine int      `json:"goroutine"`
	Memory    *MemoryS `json:"memory"`
}

// EntityData struct to hold snapshot data.
type EntityData struct {
	PID      int        `json:"pid"`
	Snapshot *SnapshotS `json:"snapshot,omitempty"`
	Metrics  *MetricsS  `json:"metrics"`
}

type meterS struct {
	sensor            *sensorS
	numGC             uint32
	snapshotCountdown int
}

func newMeter(sensor *sensorS) *meterS {
	sensor.logger.Debug("initializing meter")

	meter := &meterS{
		sensor: sensor,
	}

	ticker := time.NewTicker(1 * time.Second)
	go func() {
		meter.snapshotCountdown = 1
		for range ticker.C {
			if meter.sensor.agent.canSend() {
				meter.snapshotCountdown--
				var s *SnapshotS
				if meter.snapshotCountdown == 0 {
					meter.snapshotCountdown = SnapshotPeriod
					s = meter.collectSnapshot()
					meter.sensor.logger.Debug("collected snapshot")
				}

				pid, _ := strconv.Atoi(meter.sensor.agent.from.PID)
				d := &EntityData{
					PID:      pid,
					Snapshot: s,
					Metrics:  meter.collectMetrics(),
				}

				go meter.send(d)
			}
		}
	}()

	return meter
}

func (r *meterS) send(d *EntityData) {
	_, err := r.sensor.agent.request(r.sensor.agent.makeURL(agentDataURL), "POST", d)

	if err != nil {
		r.sensor.agent.reset()
	}
}

func (r *meterS) collectMemoryMetrics() *MemoryS {
	var memStats runtime.MemStats
	runtime.ReadMemStats(&memStats)
	ret := &MemoryS{
		Alloc:         memStats.Alloc,
		TotalAlloc:    memStats.TotalAlloc,
		Sys:           memStats.Sys,
		Lookups:       memStats.Lookups,
		Mallocs:       memStats.Mallocs,
		Frees:         memStats.Frees,
		HeapAlloc:     memStats.HeapAlloc,
		HeapSys:       memStats.HeapSys,
		HeapIdle:      memStats.HeapIdle,
		HeapInuse:     memStats.HeapInuse,
		HeapReleased:  memStats.HeapReleased,
		HeapObjects:   memStats.HeapObjects,
		PauseTotalNs:  memStats.PauseTotalNs,
		NumGC:         memStats.NumGC,
		GCCPUFraction: memStats.GCCPUFraction}

	if r.numGC < memStats.NumGC {
		ret.PauseNs = memStats.PauseNs[(memStats.NumGC+255)%256]
		r.numGC = memStats.NumGC
	} else {
		ret.PauseNs = 0
	}

	return ret
}

func (r *meterS) collectMetrics() *MetricsS {
	return &MetricsS{
		CgoCall:   runtime.NumCgoCall(),
		Goroutine: runtime.NumGoroutine(),
		Memory:    r.collectMemoryMetrics()}
}

func (r *meterS) collectSnapshot() *SnapshotS {
	return &SnapshotS{
		Name:     r.sensor.serviceName,
		Version:  runtime.Version(),
		Root:     runtime.GOROOT(),
		MaxProcs: runtime.GOMAXPROCS(0),
		Compiler: runtime.Compiler,
		NumCPU:   runtime.NumCPU()}
}
<|endoftext|>


<commit_before>package gometer

import (
	"fmt"
	"io"
	"os"
	"sync"
	"time"

	"github.com/dchest/safefile"
)

// Metrics is a collection of metrics.
type Metrics interface {
	SetOutput(io.Writer)
	SetFormatter(Formatter)
	Formatter() Formatter
	Get(string) *Counter
	GetJSON(func(string) bool) []byte
	WithPrefix(string, ...interface{}) *PrefixMetrics
	SetPanicHandler(PanicHandler)
	Write() error
	StartFileWriter(FileWriterParams)
	StopFileWriter()
}

// DefaultMetrics is a default implementation of Metrics.
type DefaultMetrics struct {
	wg       sync.WaitGroup
	stopOnce sync.Once
	cancelCh chan struct{}

	mu           sync.Mutex
	out          io.Writer
	counters     map[string]*Counter
	formatter    Formatter
	panicHandler PanicHandler
}

var _ Metrics = (*DefaultMetrics)(nil)

// FileWriterParams represents a params for asynchronous file writing operation.
//
// FilePath represents a file path.
// UpdateInterval determines how often metrics data will be written to a file.
type FileWriterParams struct {
	FilePath       string
	UpdateInterval time.Duration
}

// Default is a standard metrics object.
var Default = New()

// New creates new empty collection of metrics.
func New() *DefaultMetrics {
	m := &DefaultMetrics{
		out:       os.Stderr,
		counters:  make(map[string]*Counter),
		formatter: NewFormatter("\n"),
		cancelCh:  make(chan struct{}),
	}
	return m
}

// SetOutput sets output destination for metrics.
func (m *DefaultMetrics) SetOutput(out io.Writer) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.out = out
}

// SetFormatter sets a metrics's formatter.
func (m *DefaultMetrics) SetFormatter(f Formatter) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.formatter = f
}

// Formatter returns a metrics formatter.
func (m *DefaultMetrics) Formatter() Formatter {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.formatter
}

// Get returns counter by name. If counter doesn't exist it will be created.
func (m *DefaultMetrics) Get(counterName string) *Counter {
	m.mu.Lock()
	defer m.mu.Unlock()

	if c, ok := m.counters[counterName]; ok {
		return c
	}

	c := &Counter{}
	m.counters[counterName] = c
	return c
}

// GetJSON filters counters by given predicate and returns them as a json
// marshaled map.
func (m *DefaultMetrics) GetJSON(predicate func(string) bool) []byte {
	m.mu.Lock()
	defer m.mu.Unlock()

	result := make(map[string]*Counter)
	formatter := jsonFormatter{}
	for k, v := range m.counters {
		if predicate(k) {
			result[k] = v
		}
	}

	return formatter.Format(result)
}

// SetPanicHandler sets error handler for errors that causing the panic.
func (m *DefaultMetrics) SetPanicHandler(handler PanicHandler) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.panicHandler = handler
}

// Write writes all existing metrics to output destination.
//
// Writing metrics to the file using this method will not recreate a file.
// It appends existing metrics to existing file's data.
// if you want to write metrics to clear file use WriteToFile() method.
func (m *DefaultMetrics) Write() error {
	m.mu.Lock()
	defer m.mu.Unlock()

	data := m.formatter.Format(m.counters)

	if _, err := m.out.Write(data); err != nil {
		return err
	}

	return nil
}

// StartFileWriter starts a goroutine that will periodically writes metrics to a file.
func (m *DefaultMetrics) StartFileWriter(p FileWriterParams) {
	m.wg.Add(1)

	go func() {
		defer m.wg.Done()

		ticker := time.NewTicker(p.UpdateInterval)
		defer ticker.Stop()

		for {
			select {
			case <-ticker.C:
				err := m.createAndWriteFile(p.FilePath)
				if err != nil {
					if h := m.getPanicHandler(); h != nil {
						h.Handle(err)
						return
					}
					panic(err)
				}
			case <-m.cancelCh:
				return
			}
		}
	}()
}

// StopFileWriter stops a goroutine that will periodically writes metrics to a file.
func (m *DefaultMetrics) StopFileWriter() {
	m.stopOnce.Do(func() {
		close(m.cancelCh)
	})
	m.wg.Wait()
}

// WithPrefix creates new PrefixMetrics that uses original Metrics with specified prefix.
func (m *DefaultMetrics) WithPrefix(prefix string, v ...interface{}) *PrefixMetrics {
	return &PrefixMetrics{
		Metrics: m,
		prefix:  fmt.Sprintf(prefix, v...),
	}
}

func (m *DefaultMetrics) getPanicHandler() PanicHandler {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.panicHandler
}

// These functions are used for standard metrics.

// SetOutput sets output destination for standard metrics.
func SetOutput(out io.Writer) {
	Default.mu.Lock()
	defer Default.mu.Unlock()
	Default.out = out
}

// SetFormatter sets formatter for standard metrics.
// Fore more details see DefaultMetrics.SetFormatter().
func SetFormatter(f Formatter) {
	Default.mu.Lock()
	defer Default.mu.Unlock()
	Default.formatter = f
}

// Get returns a counter by name or nil if the counter doesn't exist.
func Get(counterName string) *Counter {
	return Default.Get(counterName)
}

// GetJSON filters counters by given predicate and returns them as a json
// marshaled map.
func GetJSON(predicate func(string) bool) []byte {
	return Default.GetJSON(predicate)
}

// SetPanicHandler sets error handler for errors that causing the panic.
func SetPanicHandler(handler PanicHandler) {
	Default.mu.Lock()
	defer Default.mu.Unlock()
	Default.panicHandler = handler
}

// Write all existing metrics to an output destination.
// For more details see DefaultMetrics.Write().
func Write() error {
	return Default.Write()
}

// StartFileWriter writes all metrics to a clear file.
// For more details see DefaultMetrics.StartFileWriter().
func StartFileWriter(p FileWriterParams) {
	Default.StartFileWriter(p)
}

// StopFileWriter stops a goroutine that will periodically writes metrics to a file.
func StopFileWriter() {
	Default.StopFileWriter()
}

// WithPrefix creates new PrefixMetrics that uses original Metrics with specified prefix.
// For more details see DefaultMetrics.WithPrefix().
func WithPrefix(prefix string, v ...interface{}) *PrefixMetrics {
	return Default.WithPrefix(prefix, v...)
}

func (m *DefaultMetrics) createAndWriteFile(path string) error {
	// create an empty temporary file.
	file, err := safefile.Create(path, 0644)
	if err != nil {
		return err
	}
	defer file.Close()

	m.SetOutput(file)
	if err = m.Write(); err != nil {
		return err
	}

	// rename temporary file to existing.
	// it's necessary for atomic file rewriting.
	return file.Commit()
}
<commit_msg>fixes in comments<commit_after>package gometer

import (
	"fmt"
	"io"
	"os"
	"sync"
	"time"

	"github.com/dchest/safefile"
)

// Metrics is a collection of metrics.
type Metrics interface {
	SetOutput(io.Writer)
	SetFormatter(Formatter)
	Formatter() Formatter
	Get(string) *Counter
	GetJSON(func(string) bool) []byte
	WithPrefix(string, ...interface{}) *PrefixMetrics
	SetPanicHandler(PanicHandler)
	Write() error
	StartFileWriter(FileWriterParams)
	StopFileWriter()
}

// DefaultMetrics is a default implementation of Metrics.
type DefaultMetrics struct {
	wg       sync.WaitGroup
	stopOnce sync.Once
	cancelCh chan struct{}

	mu           sync.Mutex
	out          io.Writer
	counters     map[string]*Counter
	formatter    Formatter
	panicHandler PanicHandler
}

var _ Metrics = (*DefaultMetrics)(nil)

// FileWriterParams represents a params for asynchronous file writing operation.
//
// FilePath represents a file path.
// UpdateInterval determines how often metrics data will be written to a file.
type FileWriterParams struct {
	FilePath       string
	UpdateInterval time.Duration
}

// Default is a standard metrics object.
var Default = New()

// New creates new empty collection of metrics.
func New() *DefaultMetrics {
	m := &DefaultMetrics{
		out:       os.Stderr,
		counters:  make(map[string]*Counter),
		formatter: NewFormatter("\n"),
		cancelCh:  make(chan struct{}),
	}
	return m
}

// SetOutput sets output destination for metrics.
func (m *DefaultMetrics) SetOutput(out io.Writer) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.out = out
}

// SetFormatter sets a metrics's formatter.
func (m *DefaultMetrics) SetFormatter(f Formatter) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.formatter = f
}

// Formatter returns a metrics formatter.
func (m *DefaultMetrics) Formatter() Formatter {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.formatter
}

// Get returns counter by name. If counter doesn't exist it will be created.
func (m *DefaultMetrics) Get(counterName string) *Counter {
	m.mu.Lock()
	defer m.mu.Unlock()

	if c, ok := m.counters[counterName]; ok {
		return c
	}

	c := &Counter{}
	m.counters[counterName] = c
	return c
}

// GetJSON filters counters by given predicate and returns them as a json marshaled map.
func (m *DefaultMetrics) GetJSON(predicate func(string) bool) []byte {
	m.mu.Lock()
	defer m.mu.Unlock()

	result := make(map[string]*Counter)
	formatter := jsonFormatter{}
	for k, v := range m.counters {
		if predicate(k) {
			result[k] = v
		}
	}

	return formatter.Format(result)
}

// SetPanicHandler sets error handler for errors that causing the panic.
func (m *DefaultMetrics) SetPanicHandler(handler PanicHandler) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.panicHandler = handler
}

// Write writes all existing metrics to output destination.
//
// Writing metrics to the file using this method will not recreate a file.
// It appends existing metrics to existing file's data.
// if you want to write metrics to clear file use StartFileWriter() method.
func (m *DefaultMetrics) Write() error {
	m.mu.Lock()
	defer m.mu.Unlock()

	data := m.formatter.Format(m.counters)

	if _, err := m.out.Write(data); err != nil {
		return err
	}

	return nil
}

// StartFileWriter starts a goroutine that periodically writes metrics to a file.
func (m *DefaultMetrics) StartFileWriter(p FileWriterParams) {
	m.wg.Add(1)

	go func() {
		defer m.wg.Done()

		ticker := time.NewTicker(p.UpdateInterval)
		defer ticker.Stop()

		for {
			select {
			case <-ticker.C:
				err := m.createAndWriteFile(p.FilePath)
				if err != nil {
					if h := m.getPanicHandler(); h != nil {
						h.Handle(err)
						return
					}
					panic(err)
				}
			case <-m.cancelCh:
				return
			}
		}
	}()
}

// StopFileWriter stops a goroutine that periodically writes metrics to a file.
func (m *DefaultMetrics) StopFileWriter() {
	m.stopOnce.Do(func() {
		close(m.cancelCh)
	})
	m.wg.Wait()
}

// WithPrefix creates new PrefixMetrics that uses original Metrics with specified prefix.
func (m *DefaultMetrics) WithPrefix(prefix string, v ...interface{}) *PrefixMetrics {
	return &PrefixMetrics{
		Metrics: m,
		prefix:  fmt.Sprintf(prefix, v...),
	}
}

func (m *DefaultMetrics) getPanicHandler() PanicHandler {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.panicHandler
}

// These functions are used for standard metrics.

// SetOutput sets output destination for standard metrics.
func SetOutput(out io.Writer) {
	Default.mu.Lock()
	defer Default.mu.Unlock()
	Default.out = out
}

// SetFormatter sets formatter for standard metrics.
// Fore more details see DefaultMetrics.SetFormatter().
func SetFormatter(f Formatter) {
	Default.mu.Lock()
	defer Default.mu.Unlock()
	Default.formatter = f
}

// Get returns counter by name. If counter doesn't exist it will be created.
func Get(counterName string) *Counter {
	return Default.Get(counterName)
}

// GetJSON filters counters by given predicate and returns them as a json marshaled map.
func GetJSON(predicate func(string) bool) []byte {
	return Default.GetJSON(predicate)
}

// SetPanicHandler sets error handler for errors that causing the panic.
func SetPanicHandler(handler PanicHandler) {
	Default.mu.Lock()
	defer Default.mu.Unlock()
	Default.panicHandler = handler
}

// Write all existing metrics to an output destination.
// For more details see DefaultMetrics.Write().
func Write() error {
	return Default.Write()
}

// StartFileWriter starts a goroutine that periodically writes metrics to a file.
// For more details see DefaultMetrics.StartFileWriter().
func StartFileWriter(p FileWriterParams) {
	Default.StartFileWriter(p)
}

// StopFileWriter stops a goroutine that periodically writes metrics to a file.
func StopFileWriter() {
	Default.StopFileWriter()
}

// WithPrefix creates new PrefixMetrics that uses original Metrics with specified prefix.
// For more details see DefaultMetrics.WithPrefix().
func WithPrefix(prefix string, v ...interface{}) *PrefixMetrics {
	return Default.WithPrefix(prefix, v...)
}

func (m *DefaultMetrics) createAndWriteFile(path string) error {
	// create an empty temporary file.
	file, err := safefile.Create(path, 0644)
	if err != nil {
		return err
	}
	defer file.Close()

	m.SetOutput(file)
	if err = m.Write(); err != nil {
		return err
	}

	// rename temporary file to existing.
	// it's necessary for atomic file rewriting.
	return file.Commit()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"strings"
)

const (
	// DefaultHeight is the default height of an Image
	DefaultHeight = 500
	// DefaultWidth is the default width of an Image
	DefaultWidth = 500
)

var (
	Black = NewColor(0, 0, 0)
	White = NewColor(255, 255, 255)
)

type Color struct {
	r byte
	b byte
	g byte
}

func NewColor(r, g, b byte) Color {
	return Color{r, g, b}
}

// Image represents an image
type Image struct {
	frame  [][]Color
	height int
	width  int
}

// NewImage returns a new Image with the given height and width
func NewImage(height, width int) *Image {
	frame := make([][]Color, height)
	for i := 0; i < height; i++ {
		frame[i] = make([]Color, width)
	}
	image := &Image{
		frame:  frame,
		height: height,
		width:  width,
	}
	image.Fill(Black)
	return image
}

// DrawLines draws all lines onto the Image
func (image *Image) DrawLines(em *Matrix, c Color) error {
	if em.cols < 2 {
		return errors.New("2 or more points are required for drawing")
	}
	for i := 0; i < em.cols-1; i += 2 {
		p0 := em.GetColumn(i)
		p1 := em.GetColumn(i + 1)
		image.DrawLine(int(p0[0]), int(p0[1]), int(p1[0]), int(p1[1]), c)
	}
	return nil
}

// DrawPolygons draws all polygons onto the Image
func (image *Image) DrawPolygons(em *Matrix, c Color) error {
	if em.cols < 3 {
		return errors.New("3 or more points are required for drawing")
	}
	for i := 0; i < em.cols-2; i += 3 {
		p0 := em.GetColumn(i)
		p1 := em.GetColumn(i + 1)
		p2 := em.GetColumn(i + 2)
		if isVisible(p0, p1, p2) {
			image.DrawLine(int(p0[0]), int(p0[1]), int(p1[0]), int(p1[1]), c)
			image.DrawLine(int(p1[0]), int(p1[1]), int(p2[0]), int(p2[1]), c)
			image.DrawLine(int(p2[0]), int(p2[1]), int(p0[0]), int(p0[1]), c)
		}
	}
	return nil
}

// DrawLine draws a single line onto the Image
func (image *Image) DrawLine(x1, y1, x2, y2 int, c Color) {
	if x1 > x2 {
		x1, x2 = x2, x1
		y1, y2 = y2, y1
	}

	A := 2 * (y2 - y1)
	B := 2 * -(x2 - x1)
	m := float32(A) / float32(-B)
	if m >= 0 {
		if m <= 1 {
			image.drawOctant1(x1, y1, x2, y2, A, B, c)
		} else {
			image.drawOctant2(x1, y1, x2, y2, A, B, c)
		}
	} else {
		if m < -1 {
			image.drawOctant7(x1, y1, x2, y2, A, B, c)
		} else {
			image.drawOctant8(x1, y1, x2, y2, A, B, c)
		}
	}
}

func (image Image) drawOctant1(x1, y1, x2, y2, A, B int, c Color) {
	d := A + B/2
	for x1 <= x2 {
		image.set(x1, y1, c)
		if d > 0 {
			y1++
			d += B
		}
		x1++
		d += A
	}
}

func (image Image) drawOctant2(x1, y1, x2, y2, A, B int, c Color) {
	d := A/2 + B
	for y1 <= y2 {
		image.set(x1, y1, c)
		if d < 0 {
			x1++
			d += A
		}
		y1++
		d += B
	}
}

func (image Image) drawOctant7(x1, y1, x2, y2, A, B int, c Color) {
	d := A/2 + B
	for y1 >= y2 {
		image.set(x1, y1, c)
		if d > 0 {
			x1++
			d += A
		}
		y1--
		d -= B
	}
}

func (image Image) drawOctant8(x1, y1, x2, y2, A, B int, c Color) {
	d := A - B/2
	for x1 <= x2 {
		image.set(x1, y1, c)
		if d < 0 {
			y1--
			d -= B
		}
		x1++
		d += A
	}
}

// Fill completely fills the Image with a single color
func (image Image) Fill(c Color) {
	for y := 0; y < image.height; y++ {
		for x := 0; x < image.width; x++ {
			image.set(x, y, c)
		}
	}
}

func (image Image) set(x, y int, c Color) error {
	if x < 0 || x >= image.width {
		return errors.New("invalid x coordinate")
	}
	if y < 0 || y >= image.height {
		return errors.New("invalid y coordinate")
	}
	image.frame[y][x] = c
	return nil
}

// SavePpm will save the Image as a ppm
func (image Image) SavePpm(name string) error {
	var buffer bytes.Buffer
	buffer.WriteString(fmt.Sprintf("P3 %d %d %d\n", image.width, image.height, 255))
	for y := 0; y < image.height; y++ {
		for x := 0; x < image.width; x++ {
			color := image.frame[image.height-y-1][x]
			buffer.WriteString(fmt.Sprintf("%d %d %d\n", color.r, color.b, color.g))
		}
	}
	f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = f.WriteString(buffer.String())
	return err
}

// Save will save an Image into a given format
func (image Image) Save(name string) error {
	index := strings.Index(name, ".")
	extension := ".ppm"
	if index == -1 {
		extension = ".png"
	} else {
		extension = name[index:]
		name = name[:index]
	}

	if extension == ".ppm" {
		// save as ppm without converting
		err := image.SavePpm(fmt.Sprint(name, ".ppm"))
		return err
	}

	ppm := fmt.Sprintf("%s-tmp.ppm", name)
	err := image.SavePpm(ppm)
	if err != nil {
		return err
	}
	defer os.Remove(ppm)
	args := []string{ppm, fmt.Sprint(name, extension)}
	err = exec.Command("convert", args...).Run()
	return err
}

// Display displays the Image
func (image Image) Display() error {
	filename := "tmp.ppm"
	err := image.SavePpm(filename)
	if err != nil {
		return err
	}
	defer os.Remove(filename)

	args := []string{filename}
	err = exec.Command("display", args...).Run()
	return err
}

// MakeAnimation converts individual frames to a gif
func MakeAnimation(basename string) error {
	path := fmt.Sprintf("%s/%s*", FramesDirectory, basename)
	gif := fmt.Sprintf("%s.gif", basename)
	args := []string{"-delay", "3", path, gif}
	err := exec.Command("convert", args...).Run()
	return err
}

func isVisible(p0, p1, p2 []float64) bool {
	a := []float64{p1[0] - p0[0], p1[1] - p0[1], p1[2] - p0[2]}
	b := []float64{p2[0] - p0[0], p2[1] - p0[1], p2[2] - p0[2]}
	normal := CrossProduct(a, b)
	return normal[2] > 0
}
<commit_msg>Define image methods with pointer receivers<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"strings"
)

const (
	// DefaultHeight is the default height of an Image
	DefaultHeight = 500
	// DefaultWidth is the default width of an Image
	DefaultWidth = 500
)

var (
	Black = NewColor(0, 0, 0)
	White = NewColor(255, 255, 255)
)

type Color struct {
	r byte
	b byte
	g byte
}

func NewColor(r, g, b byte) Color {
	return Color{r, g, b}
}

// Image represents an image
type Image struct {
	frame  [][]Color
	height int
	width  int
}

// NewImage returns a new Image with the given height and width
func NewImage(height, width int) *Image {
	frame := make([][]Color, height)
	for i := 0; i < height; i++ {
		frame[i] = make([]Color, width)
	}
	image := &Image{
		frame:  frame,
		height: height,
		width:  width,
	}
	image.Fill(Black)
	return image
}

// DrawLines draws all lines onto the Image
func (image *Image) DrawLines(em *Matrix, c Color) error {
	if em.cols < 2 {
		return errors.New("2 or more points are required for drawing")
	}
	for i := 0; i < em.cols-1; i += 2 {
		p0 := em.GetColumn(i)
		p1 := em.GetColumn(i + 1)
		image.DrawLine(int(p0[0]), int(p0[1]), int(p1[0]), int(p1[1]), c)
	}
	return nil
}

// DrawPolygons draws all polygons onto the Image
func (image *Image) DrawPolygons(em *Matrix, c Color) error {
	if em.cols < 3 {
		return errors.New("3 or more points are required for drawing")
	}
	for i := 0; i < em.cols-2; i += 3 {
		p0 := em.GetColumn(i)
		p1 := em.GetColumn(i + 1)
		p2 := em.GetColumn(i + 2)
		if isVisible(p0, p1, p2) {
			image.DrawLine(int(p0[0]), int(p0[1]), int(p1[0]), int(p1[1]), c)
			image.DrawLine(int(p1[0]), int(p1[1]), int(p2[0]), int(p2[1]), c)
			image.DrawLine(int(p2[0]), int(p2[1]), int(p0[0]), int(p0[1]), c)
		}
	}
	return nil
}

// DrawLine draws a single line onto the Image
func (image *Image) DrawLine(x1, y1, x2, y2 int, c Color) {
	if x1 > x2 {
		x1, x2 = x2, x1
		y1, y2 = y2, y1
	}

	A := 2 * (y2 - y1)
	B := 2 * -(x2 - x1)
	m := float32(A) / float32(-B)
	if m >= 0 {
		if m <= 1 {
			image.drawOctant1(x1, y1, x2, y2, A, B, c)
		} else {
			image.drawOctant2(x1, y1, x2, y2, A, B, c)
		}
	} else {
		if m < -1 {
			image.drawOctant7(x1, y1, x2, y2, A, B, c)
		} else {
			image.drawOctant8(x1, y1, x2, y2, A, B, c)
		}
	}
}

func (image *Image) drawOctant1(x1, y1, x2, y2, A, B int, c Color) {
	d := A + B/2
	for x1 <= x2 {
		image.set(x1, y1, c)
		if d > 0 {
			y1++
			d += B
		}
		x1++
		d += A
	}
}

func (image *Image) drawOctant2(x1, y1, x2, y2, A, B int, c Color) {
	d := A/2 + B
	for y1 <= y2 {
		image.set(x1, y1, c)
		if d < 0 {
			x1++
			d += A
		}
		y1++
		d += B
	}
}

func (image *Image) drawOctant7(x1, y1, x2, y2, A, B int, c Color) {
	d := A/2 + B
	for y1 >= y2 {
		image.set(x1, y1, c)
		if d > 0 {
			x1++
			d += A
		}
		y1--
		d -= B
	}
}

func (image *Image) drawOctant8(x1, y1, x2, y2, A, B int, c Color) {
	d := A - B/2
	for x1 <= x2 {
		image.set(x1, y1, c)
		if d < 0 {
			y1--
			d -= B
		}
		x1++
		d += A
	}
}

// Fill completely fills the Image with a single color
func (image *Image) Fill(c Color) {
	for y := 0; y < image.height; y++ {
		for x := 0; x < image.width; x++ {
			image.set(x, y, c)
		}
	}
}

func (image *Image) set(x, y int, c Color) error {
	if x < 0 || x >= image.width {
		return errors.New("invalid x coordinate")
	}
	if y < 0 || y >= image.height {
		return errors.New("invalid y coordinate")
	}
	image.frame[y][x] = c
	return nil
}

// SavePpm will save the Image as a ppm
func (image Image) SavePpm(name string) error {
	var buffer bytes.Buffer
	buffer.WriteString(fmt.Sprintf("P3 %d %d %d\n", image.width, image.height, 255))
	for y := 0; y < image.height; y++ {
		for x := 0; x < image.width; x++ {
			color := image.frame[image.height-y-1][x]
			buffer.WriteString(fmt.Sprintf("%d %d %d\n", color.r, color.b, color.g))
		}
	}
	f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = f.WriteString(buffer.String())
	return err
}

// Save will save an Image into a given format
func (image Image) Save(name string) error {
	index := strings.Index(name, ".")
	extension := ".ppm"
	if index == -1 {
		extension = ".png"
	} else {
		extension = name[index:]
		name = name[:index]
	}

	if extension == ".ppm" {
		// save as ppm without converting
		err := image.SavePpm(fmt.Sprint(name, ".ppm"))
		return err
	}

	ppm := fmt.Sprintf("%s-tmp.ppm", name)
	err := image.SavePpm(ppm)
	if err != nil {
		return err
	}
	defer os.Remove(ppm)
	args := []string{ppm, fmt.Sprint(name, extension)}
	err = exec.Command("convert", args...).Run()
	return err
}

// Display displays the Image
func (image Image) Display() error {
	filename := "tmp.ppm"
	err := image.SavePpm(filename)
	if err != nil {
		return err
	}
	defer os.Remove(filename)

	args := []string{filename}
	err = exec.Command("display", args...).Run()
	return err
}

// MakeAnimation converts individual frames to a gif
func MakeAnimation(basename string) error {
	path := fmt.Sprintf("%s/%s*", FramesDirectory, basename)
	gif := fmt.Sprintf("%s.gif", basename)
	args := []string{"-delay", "3", path, gif}
	err := exec.Command("convert", args...).Run()
	return err
}

func isVisible(p0, p1, p2 []float64) bool {
	a := []float64{p1[0] - p0[0], p1[1] - p0[1], p1[2] - p0[2]}
	b := []float64{p2[0] - p0[0], p2[1] - p0[1], p2[2] - p0[2]}
	normal := CrossProduct(a, b)
	return normal[2] > 0
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/fsouza/go-dockerclient"
)

type images []docker.APIImages

type filter func(image docker.APIImages) bool

func (i images) Filter(f filter) images {
	ret := images{}
	for _, image := range i {
		if f(image) {
			ret = append(ret, image)
		}
	}
	return ret
}

func listImages(client *docker.Client) (images, error) {
	images := images{}
	apiImages, err := client.ListImages(false)
	if err != nil {
		return nil, err
	}
	for i := range apiImages {
		images = append(images, apiImages[i])
	}
	return apiImages, nil
}

func filterByName(name string) func(image docker.APIImages) bool {
	return func(image docker.APIImages) bool {
		for i := range image.RepoTags {
			if strings.HasPrefix(image.RepoTags[i], name) {
				return true
			}
		}
		return false
	}
}

func filterByCreatedAt(duration int) func(image docker.APIImages) bool {
	return func(image docker.APIImages) bool {
		d := time.Second * time.Duration(duration)
		return time.Since(time.Unix(image.Created, 0)) > d
	}
}

func doImage(c *cli.Context) {
	client, err := docker.NewClient(c.GlobalString("endpoint"))
	if err != nil {
		log.Fatal(err)
	}

	images, err := listImages(client)
	if err != nil {
		log.Fatal(err)
	}

	ret := images.
		Filter(filterByName(c.String("name"))).
		Filter(filterByCreatedAt(c.Int("duration")))
	for i := range ret {
		var err error
		run(!c.Bool("force"),
			func() {
				fmt.Println("dryrun: removed:", ret[i].ID, ret[i].RepoTags)
			},
			func() {
				err = client.RemoveImage(ret[i].ID)
				fmt.Println("removed:", ret[i].ID, ret[i].RepoTags)
			},
		)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s: failed to delete a image %v %s", err, ret[i].ID, ret[i].RepoTags)
		}
	}
}
<commit_msg>Fix signature<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/fsouza/go-dockerclient"
)

type images []docker.APIImages

type filter func(image docker.APIImages) bool

func (i images) Filter(f filter) images {
	ret := images{}
	for _, image := range i {
		if f(image) {
			ret = append(ret, image)
		}
	}
	return ret
}

func listImages(client *docker.Client) (images, error) {
	images := images{}
	apiImages, err := client.ListImages(false)
	if err != nil {
		return nil, err
	}
	for i := range apiImages {
		images = append(images, apiImages[i])
	}
	return apiImages, nil
}

func filterByName(name string) filter {
	return func(image docker.APIImages) bool {
		for i := range image.RepoTags {
			if strings.HasPrefix(image.RepoTags[i], name) {
				return true
			}
		}
		return false
	}
}

func filterByCreatedAt(duration int) filter {
	return func(image docker.APIImages) bool {
		d := time.Second * time.Duration(duration)
		return time.Since(time.Unix(image.Created, 0)) > d
	}
}

func doImage(c *cli.Context) {
	client, err := docker.NewClient(c.GlobalString("endpoint"))
	if err != nil {
		log.Fatal(err)
	}

	images, err := listImages(client)
	if err != nil {
		log.Fatal(err)
	}

	ret := images.
		Filter(filterByName(c.String("name"))).
		Filter(filterByCreatedAt(c.Int("duration")))
	for i := range ret {
		var err error
		run(!c.Bool("force"),
			func() {
				fmt.Println("dryrun: removed:", ret[i].ID, ret[i].RepoTags)
			},
			func() {
				err = client.RemoveImage(ret[i].ID)
				fmt.Println("removed:", ret[i].ID, ret[i].RepoTags)
			},
		)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s: failed to delete a image %v %s", err, ret[i].ID, ret[i].RepoTags)
		}
	}
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package containerd

import (
	"context"
	"fmt"

	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/platforms"
	"github.com/containerd/containerd/rootfs"
	digest "github.com/opencontainers/go-digest"
	"github.com/opencontainers/image-spec/identity"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

// Image describes an image used by containers
type Image interface {
	// Name of the image
	Name() string
	// Target descriptor for the image content
	Target() ocispec.Descriptor
	// Labels of the image
	Labels() map[string]string
	// Unpack unpacks the image's content into a snapshot
	Unpack(context.Context, string) error
	// RootFS returns the unpacked diffids that make up images rootfs.
	RootFS(ctx context.Context) ([]digest.Digest, error)
	// Size returns the total size of the image's packed resources.
	Size(ctx context.Context) (int64, error)
	// Config descriptor for the image.
	Config(ctx context.Context) (ocispec.Descriptor, error)
	// IsUnpacked returns whether or not an image is unpacked.
	IsUnpacked(context.Context, string) (bool, error)
	// ContentStore provides a content store which contains image blob data
	ContentStore() content.Store
}

var _ = (Image)(&image{})

// NewImage returns a client image object from the metadata image
func NewImage(client *Client, i images.Image) Image {
	return &image{
		client:   client,
		i:        i,
		platform: platforms.Default(),
	}
}

// NewImageWithPlatform returns a client image object from the metadata image
func NewImageWithPlatform(client *Client, i images.Image, platform platforms.MatchComparer) Image {
	return &image{
		client:   client,
		i:        i,
		platform: platform,
	}
}

type image struct {
	client *Client

	i        images.Image
	platform platforms.MatchComparer
}

func (i *image) Name() string {
	return i.i.Name
}

func (i *image) Target() ocispec.Descriptor {
	return i.i.Target
}

func (i *image) Labels() map[string]string {
	return i.i.Labels
}

func (i *image) RootFS(ctx context.Context) ([]digest.Digest, error) {
	provider := i.client.ContentStore()
	return i.i.RootFS(ctx, provider, i.platform)
}

func (i *image) Size(ctx context.Context) (int64, error) {
	provider := i.client.ContentStore()
	return i.i.Size(ctx, provider, i.platform)
}

func (i *image) Config(ctx context.Context) (ocispec.Descriptor, error) {
	provider := i.client.ContentStore()
	return i.i.Config(ctx, provider, i.platform)
}

func (i *image) IsUnpacked(ctx context.Context, snapshotterName string) (bool, error) {
	sn := i.client.SnapshotService(snapshotterName)
	cs := i.client.ContentStore()

	diffs, err := i.i.RootFS(ctx, cs, i.platform)
	if err != nil {
		return false, err
	}

	chainID := identity.ChainID(diffs)
	_, err = sn.Stat(ctx, chainID.String())
	if err == nil {
		return true, nil
	} else if !errdefs.IsNotFound(err) {
		return false, err
	}

	return false, nil
}

func (i *image) Unpack(ctx context.Context, snapshotterName string) error {
	ctx, done, err := i.client.WithLease(ctx)
	if err != nil {
		return err
	}
	defer done(ctx)

	layers, err := i.getLayers(ctx, i.platform)
	if err != nil {
		return err
	}

	var (
		sn = i.client.SnapshotService(snapshotterName)
		a  = i.client.DiffService()
		cs = i.client.ContentStore()

		chain    []digest.Digest
		unpacked bool
	)
	for _, layer := range layers {
		unpacked, err = rootfs.ApplyLayer(ctx, layer, chain, sn, a)
		if err != nil {
			return err
		}

		if unpacked {
			// Set the uncompressed label after the uncompressed
			// digest has been verified through apply.
			cinfo := content.Info{
				Digest: layer.Blob.Digest,
				Labels: map[string]string{
					"containerd.io/uncompressed": layer.Diff.Digest.String(),
				},
			}
			if _, err := cs.Update(ctx, cinfo, "labels.containerd.io/uncompressed"); err != nil {
				return err
			}
		}

		chain = append(chain, layer.Diff.Digest)
	}

	if unpacked {
		desc, err := i.i.Config(ctx, cs, i.platform)
		if err != nil {
			return err
		}

		rootfs := identity.ChainID(chain).String()

		cinfo := content.Info{
			Digest: desc.Digest,
			Labels: map[string]string{
				fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snapshotterName): rootfs,
			},
		}
		if _, err := cs.Update(ctx, cinfo, fmt.Sprintf("labels.containerd.io/gc.ref.snapshot.%s", snapshotterName)); err != nil {
			return err
		}
	}

	return nil
}

func (i *image) getLayers(ctx context.Context, platform platforms.MatchComparer) ([]rootfs.Layer, error) {
	cs := i.client.ContentStore()

	manifest, err := images.Manifest(ctx, cs, i.i.Target, platform)
	if err != nil {
		return nil, err
	}

	diffIDs, err := i.i.RootFS(ctx, cs, platform)
	if err != nil {
		return nil, errors.Wrap(err, "failed to resolve rootfs")
	}
	if len(diffIDs) != len(manifest.Layers) {
		return nil, errors.Errorf("mismatched image rootfs and manifest layers")
	}
	layers := make([]rootfs.Layer, len(diffIDs))
	for i := range diffIDs {
		layers[i].Diff = ocispec.Descriptor{
			// TODO: derive media type from compressed type
			MediaType: ocispec.MediaTypeImageLayer,
			Digest:    diffIDs[i],
		}
		layers[i].Blob = manifest.Layers[i]
	}
	return layers, nil
}

func (i *image) ContentStore() content.Store {
	return i.client.ContentStore()
}
<commit_msg>bugfix: unpack should always set the snapshot gc label<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package containerd

import (
	"context"
	"fmt"

	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/platforms"
	"github.com/containerd/containerd/rootfs"
	digest "github.com/opencontainers/go-digest"
	"github.com/opencontainers/image-spec/identity"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

// Image describes an image used by containers
type Image interface {
	// Name of the image
	Name() string
	// Target descriptor for the image content
	Target() ocispec.Descriptor
	// Labels of the image
	Labels() map[string]string
	// Unpack unpacks the image's content into a snapshot
	Unpack(context.Context, string) error
	// RootFS returns the unpacked diffids that make up images rootfs.
	RootFS(ctx context.Context) ([]digest.Digest, error)
	// Size returns the total size of the image's packed resources.
	Size(ctx context.Context) (int64, error)
	// Config descriptor for the image.
	Config(ctx context.Context) (ocispec.Descriptor, error)
	// IsUnpacked returns whether or not an image is unpacked.
	IsUnpacked(context.Context, string) (bool, error)
	// ContentStore provides a content store which contains image blob data
	ContentStore() content.Store
}

var _ = (Image)(&image{})

// NewImage returns a client image object from the metadata image
func NewImage(client *Client, i images.Image) Image {
	return &image{
		client:   client,
		i:        i,
		platform: platforms.Default(),
	}
}

// NewImageWithPlatform returns a client image object from the metadata image
func NewImageWithPlatform(client *Client, i images.Image, platform platforms.MatchComparer) Image {
	return &image{
		client:   client,
		i:        i,
		platform: platform,
	}
}

type image struct {
	client *Client

	i        images.Image
	platform platforms.MatchComparer
}

func (i *image) Name() string {
	return i.i.Name
}

func (i *image) Target() ocispec.Descriptor {
	return i.i.Target
}

func (i *image) Labels() map[string]string {
	return i.i.Labels
}

func (i *image) RootFS(ctx context.Context) ([]digest.Digest, error) {
	provider := i.client.ContentStore()
	return i.i.RootFS(ctx, provider, i.platform)
}

func (i *image) Size(ctx context.Context) (int64, error) {
	provider := i.client.ContentStore()
	return i.i.Size(ctx, provider, i.platform)
}

func (i *image) Config(ctx context.Context) (ocispec.Descriptor, error) {
	provider := i.client.ContentStore()
	return i.i.Config(ctx, provider, i.platform)
}

func (i *image) IsUnpacked(ctx context.Context, snapshotterName string) (bool, error) {
	sn := i.client.SnapshotService(snapshotterName)
	cs := i.client.ContentStore()

	diffs, err := i.i.RootFS(ctx, cs, i.platform)
	if err != nil {
		return false, err
	}

	chainID := identity.ChainID(diffs)
	_, err = sn.Stat(ctx, chainID.String())
	if err == nil {
		return true, nil
	} else if !errdefs.IsNotFound(err) {
		return false, err
	}

	return false, nil
}

func (i *image) Unpack(ctx context.Context, snapshotterName string) error {
	ctx, done, err := i.client.WithLease(ctx)
	if err != nil {
		return err
	}
	defer done(ctx)

	layers, err := i.getLayers(ctx, i.platform)
	if err != nil {
		return err
	}

	var (
		sn = i.client.SnapshotService(snapshotterName)
		a  = i.client.DiffService()
		cs = i.client.ContentStore()

		chain    []digest.Digest
		unpacked bool
	)
	for _, layer := range layers {
		unpacked, err = rootfs.ApplyLayer(ctx, layer, chain, sn, a)
		if err != nil {
			return err
		}

		if unpacked {
			// Set the uncompressed label after the uncompressed
			// digest has been verified through apply.
			cinfo := content.Info{
				Digest: layer.Blob.Digest,
				Labels: map[string]string{
					"containerd.io/uncompressed": layer.Diff.Digest.String(),
				},
			}
			if _, err := cs.Update(ctx, cinfo, "labels.containerd.io/uncompressed"); err != nil {
				return err
			}
		}

		chain = append(chain, layer.Diff.Digest)
	}

	desc, err := i.i.Config(ctx, cs, i.platform)
	if err != nil {
		return err
	}

	rootfs := identity.ChainID(chain).String()

	cinfo := content.Info{
		Digest: desc.Digest,
		Labels: map[string]string{
			fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snapshotterName): rootfs,
		},
	}

	_, err = cs.Update(ctx, cinfo, fmt.Sprintf("labels.containerd.io/gc.ref.snapshot.%s", snapshotterName))
	return err
}

func (i *image) getLayers(ctx context.Context, platform platforms.MatchComparer) ([]rootfs.Layer, error) {
	cs := i.client.ContentStore()

	manifest, err := images.Manifest(ctx, cs, i.i.Target, platform)
	if err != nil {
		return nil, err
	}

	diffIDs, err := i.i.RootFS(ctx, cs, platform)
	if err != nil {
		return nil, errors.Wrap(err, "failed to resolve rootfs")
	}
	if len(diffIDs) != len(manifest.Layers) {
		return nil, errors.Errorf("mismatched image rootfs and manifest layers")
	}
	layers := make([]rootfs.Layer, len(diffIDs))
	for i := range diffIDs {
		layers[i].Diff = ocispec.Descriptor{
			// TODO: derive media type from compressed type
			MediaType: ocispec.MediaTypeImageLayer,
			Digest:    diffIDs[i],
		}
		layers[i].Blob = manifest.Layers[i]
	}
	return layers, nil
}

func (i *image) ContentStore() content.Store {
	return i.client.ContentStore()
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"encoding/json"
	"fmt"
	"time"

	cadvisorapi "github.com/google/cadvisor/info/v1"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	mb = 1024 * 1024
	gb = 1024 * mb

	// TODO(madhusudancs): find a way to query kubelet's disk space manager to obtain this value. 256MB
	// is the default that is set today. This test might break if the default value changes. This value
	// can be configured by setting the "low-diskspace-threshold-mb" flag while starting a kubelet.
	// However, kubelets are started as part of the cluster start up, once, before any e2e test is run,
	// and remain unchanged until all the tests are run and the cluster is brought down. Changing the
	// flag value affects all the e2e tests. So we are hard-coding this value for now.
	lowDiskSpaceThreshold uint64 = 256 * mb

	nodeOODTimeOut = 5 * time.Minute

	numNodeOODPods = 3
)

// Plan:
// 1. Fill disk space on all nodes except one. One node is left out so that we can schedule pods
//    on that node. Arbitrarily choose that node to be node with index 0.  This makes this a disruptive test.
// 2. Get the CPU capacity on unfilled node.
// 3. Divide the available CPU into one less than the number of pods we want to schedule. We want
//    to schedule 3 pods, so divide CPU capacity by 2.
// 4. Request the divided CPU for each pod.
// 5. Observe that 2 of the pods schedule onto the node whose disk is not full, and the remaining
//    pod stays pending and does not schedule onto the nodes whose disks are full nor the node
//    with the other two pods, since there is not enough free CPU capacity there.
// 6. Recover disk space from one of the nodes whose disk space was previously filled. Arbritrarily
//    choose that node to be node with index 1.
// 7. Observe that the pod in pending status schedules on that node.
//
// Flaky issue #20015.  We have no clear path for how to test this functionality in a non-flaky way.
var _ = framework.KubeDescribe("NodeOutOfDisk [Serial] [Flaky] [Disruptive]", func() {
	var c clientset.Interface
	var unfilledNodeName, recoveredNodeName string
	f := framework.NewDefaultFramework("node-outofdisk")

	BeforeEach(func() {
		c = f.ClientSet

		nodelist := framework.GetReadySchedulableNodesOrDie(c)

		// Skip this test on small clusters.  No need to fail since it is not a use
		// case that any cluster of small size needs to support.
		framework.SkipUnlessNodeCountIsAtLeast(2)

		unfilledNodeName = nodelist.Items[0].Name
		for _, node := range nodelist.Items[1:] {
			fillDiskSpace(c, &node)
		}
	})

	AfterEach(func() {

		nodelist := framework.GetReadySchedulableNodesOrDie(c)
		Expect(len(nodelist.Items)).ToNot(BeZero())
		for _, node := range nodelist.Items {
			if unfilledNodeName == node.Name || recoveredNodeName == node.Name {
				continue
			}
			recoverDiskSpace(c, &node)
		}
	})

	It("runs out of disk space", func() {
		unfilledNode, err := c.Core().Nodes().Get(unfilledNodeName)
		framework.ExpectNoError(err)

		By(fmt.Sprintf("Calculating CPU availability on node %s", unfilledNode.Name))
		milliCpu, err := availCpu(c, unfilledNode)
		framework.ExpectNoError(err)

		// Per pod CPU should be just enough to fit only (numNodeOODPods - 1) pods on the given
		// node. We compute this value by dividing the available CPU capacity on the node by
		// (numNodeOODPods - 1) and subtracting ϵ from it. We arbitrarily choose ϵ to be 1%
		// of the available CPU per pod, i.e. 0.01 * milliCpu/(numNodeOODPods-1). Instead of
		// subtracting 1% from the value, we directly use 0.99 as the multiplier.
		podCPU := int64(float64(milliCpu/(numNodeOODPods-1)) * 0.99)

		ns := f.Namespace.Name
		podClient := c.Core().Pods(ns)

		By("Creating pods and waiting for all but one pods to be scheduled")

		for i := 0; i < numNodeOODPods-1; i++ {
			name := fmt.Sprintf("pod-node-outofdisk-%d", i)
			createOutOfDiskPod(c, ns, name, podCPU)

			framework.ExpectNoError(f.WaitForPodRunning(name))
			pod, err := podClient.Get(name)
			framework.ExpectNoError(err)
			Expect(pod.Spec.NodeName).To(Equal(unfilledNodeName))
		}

		pendingPodName := fmt.Sprintf("pod-node-outofdisk-%d", numNodeOODPods-1)
		createOutOfDiskPod(c, ns, pendingPodName, podCPU)

		By(fmt.Sprintf("Finding a failed scheduler event for pod %s", pendingPodName))
		wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
			selector := fields.Set{
				"involvedObject.kind":      "Pod",
				"involvedObject.name":      pendingPodName,
				"involvedObject.namespace": ns,
				"source":                   api.DefaultSchedulerName,
				"reason":                   "FailedScheduling",
			}.AsSelector()
			options := api.ListOptions{FieldSelector: selector}
			schedEvents, err := c.Core().Events(ns).List(options)
			framework.ExpectNoError(err)

			if len(schedEvents.Items) > 0 {
				return true, nil
			} else {
				return false, nil
			}
		})

		nodelist := framework.GetReadySchedulableNodesOrDie(c)
		Expect(len(nodelist.Items)).To(BeNumerically(">", 1))

		nodeToRecover := nodelist.Items[1]
		Expect(nodeToRecover.Name).ToNot(Equal(unfilledNodeName))

		recoverDiskSpace(c, &nodeToRecover)
		recoveredNodeName = nodeToRecover.Name

		By(fmt.Sprintf("Verifying that pod %s schedules on node %s", pendingPodName, recoveredNodeName))
		framework.ExpectNoError(f.WaitForPodRunning(pendingPodName))
		pendingPod, err := podClient.Get(pendingPodName)
		framework.ExpectNoError(err)
		Expect(pendingPod.Spec.NodeName).To(Equal(recoveredNodeName))
	})
})

// createOutOfDiskPod creates a pod in the given namespace with the requested amount of CPU.
func createOutOfDiskPod(c clientset.Interface, ns, name string, milliCPU int64) {
	podClient := c.Core().Pods(ns)

	pod := &api.Pod{
		ObjectMeta: api.ObjectMeta{
			Name: name,
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  "pause",
					Image: framework.GetPauseImageName(c),
					Resources: api.ResourceRequirements{
						Requests: api.ResourceList{
							// Request enough CPU to fit only two pods on a given node.
							api.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI),
						},
					},
				},
			},
		},
	}

	_, err := podClient.Create(pod)
	framework.ExpectNoError(err)
}

// availCpu calculates the available CPU on a given node by subtracting the CPU requested by
// all the pods from the total available CPU capacity on the node.
func availCpu(c clientset.Interface, node *api.Node) (int64, error) {
	podClient := c.Core().Pods(api.NamespaceAll)

	selector := fields.Set{"spec.nodeName": node.Name}.AsSelector()
	options := api.ListOptions{FieldSelector: selector}
	pods, err := podClient.List(options)
	if err != nil {
		return 0, fmt.Errorf("failed to retrieve all the pods on node %s: %v", node.Name, err)
	}
	avail := node.Status.Capacity.Cpu().MilliValue()
	for _, pod := range pods.Items {
		for _, cont := range pod.Spec.Containers {
			avail -= cont.Resources.Requests.Cpu().MilliValue()
		}
	}
	return avail, nil
}

// availSize returns the available disk space on a given node by querying node stats which
// is in turn obtained internally from cadvisor.
func availSize(c clientset.Interface, node *api.Node) (uint64, error) {
	statsResource := fmt.Sprintf("api/v1/proxy/nodes/%s/stats/", node.Name)
	framework.Logf("Querying stats for node %s using url %s", node.Name, statsResource)
	res, err := c.Core().RESTClient().Get().AbsPath(statsResource).Timeout(time.Minute).Do().Raw()
	if err != nil {
		return 0, fmt.Errorf("error querying cAdvisor API: %v", err)
	}
	ci := cadvisorapi.ContainerInfo{}
	err = json.Unmarshal(res, &ci)
	if err != nil {
		return 0, fmt.Errorf("couldn't unmarshal container info: %v", err)
	}
	return ci.Stats[len(ci.Stats)-1].Filesystem[0].Available, nil
}

// fillDiskSpace fills the available disk space on a given node by creating a large file. The disk
// space on the node is filled in such a way that the available space after filling the disk is just
// below the lowDiskSpaceThreshold mark.
func fillDiskSpace(c clientset.Interface, node *api.Node) {
	avail, err := availSize(c, node)
	framework.ExpectNoError(err, "Node %s: couldn't obtain available disk size %v", node.Name, err)

	fillSize := (avail - lowDiskSpaceThreshold + (100 * mb))

	framework.Logf("Node %s: disk space available %d bytes", node.Name, avail)
	By(fmt.Sprintf("Node %s: creating a file of size %d bytes to fill the available disk space", node.Name, fillSize))

	cmd := fmt.Sprintf("fallocate -l %d test.img", fillSize)
	framework.ExpectNoError(framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node))

	ood := framework.WaitForNodeToBe(c, node.Name, api.NodeOutOfDisk, true, nodeOODTimeOut)
	Expect(ood).To(BeTrue(), "Node %s did not run out of disk within %v", node.Name, nodeOODTimeOut)

	avail, err = availSize(c, node)
	framework.Logf("Node %s: disk space available %d bytes", node.Name, avail)
	Expect(avail < lowDiskSpaceThreshold).To(BeTrue())
}

// recoverDiskSpace recovers disk space, filled by creating a large file, on a given node.
func recoverDiskSpace(c clientset.Interface, node *api.Node) {
	By(fmt.Sprintf("Recovering disk space on node %s", node.Name))
	cmd := "rm -f test.img"
	framework.ExpectNoError(framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node))

	ood := framework.WaitForNodeToBe(c, node.Name, api.NodeOutOfDisk, false, nodeOODTimeOut)
	Expect(ood).To(BeTrue(), "Node %s's out of disk condition status did not change to false within %v", node.Name, nodeOODTimeOut)
}
<commit_msg>Marked NodeOutOfDisk test with feature label to remove it from flaky suite<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"encoding/json"
	"fmt"
	"time"

	cadvisorapi "github.com/google/cadvisor/info/v1"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	mb = 1024 * 1024
	gb = 1024 * mb

	// TODO(madhusudancs): find a way to query kubelet's disk space manager to obtain this value. 256MB
	// is the default that is set today. This test might break if the default value changes. This value
	// can be configured by setting the "low-diskspace-threshold-mb" flag while starting a kubelet.
	// However, kubelets are started as part of the cluster start up, once, before any e2e test is run,
	// and remain unchanged until all the tests are run and the cluster is brought down. Changing the
	// flag value affects all the e2e tests. So we are hard-coding this value for now.
	lowDiskSpaceThreshold uint64 = 256 * mb

	nodeOODTimeOut = 5 * time.Minute

	numNodeOODPods = 3
)

// Plan:
// 1. Fill disk space on all nodes except one. One node is left out so that we can schedule pods
//    on that node. Arbitrarily choose that node to be node with index 0.  This makes this a disruptive test.
// 2. Get the CPU capacity on unfilled node.
// 3. Divide the available CPU into one less than the number of pods we want to schedule. We want
//    to schedule 3 pods, so divide CPU capacity by 2.
// 4. Request the divided CPU for each pod.
// 5. Observe that 2 of the pods schedule onto the node whose disk is not full, and the remaining
//    pod stays pending and does not schedule onto the nodes whose disks are full nor the node
//    with the other two pods, since there is not enough free CPU capacity there.
// 6. Recover disk space from one of the nodes whose disk space was previously filled. Arbritrarily
//    choose that node to be node with index 1.
// 7. Observe that the pod in pending status schedules on that node.
//
// Flaky issue #20015.  We have no clear path for how to test this functionality in a non-flaky way.
var _ = framework.KubeDescribe("NodeOutOfDisk [Serial] [Flaky] [Disruptive] [Feature:OutOfDisk]", func() {
	var c clientset.Interface
	var unfilledNodeName, recoveredNodeName string
	f := framework.NewDefaultFramework("node-outofdisk")

	BeforeEach(func() {
		c = f.ClientSet

		nodelist := framework.GetReadySchedulableNodesOrDie(c)

		// Skip this test on small clusters.  No need to fail since it is not a use
		// case that any cluster of small size needs to support.
		framework.SkipUnlessNodeCountIsAtLeast(2)

		unfilledNodeName = nodelist.Items[0].Name
		for _, node := range nodelist.Items[1:] {
			fillDiskSpace(c, &node)
		}
	})

	AfterEach(func() {

		nodelist := framework.GetReadySchedulableNodesOrDie(c)
		Expect(len(nodelist.Items)).ToNot(BeZero())
		for _, node := range nodelist.Items {
			if unfilledNodeName == node.Name || recoveredNodeName == node.Name {
				continue
			}
			recoverDiskSpace(c, &node)
		}
	})

	It("runs out of disk space", func() {
		unfilledNode, err := c.Core().Nodes().Get(unfilledNodeName)
		framework.ExpectNoError(err)

		By(fmt.Sprintf("Calculating CPU availability on node %s", unfilledNode.Name))
		milliCpu, err := availCpu(c, unfilledNode)
		framework.ExpectNoError(err)

		// Per pod CPU should be just enough to fit only (numNodeOODPods - 1) pods on the given
		// node. We compute this value by dividing the available CPU capacity on the node by
		// (numNodeOODPods - 1) and subtracting ϵ from it. We arbitrarily choose ϵ to be 1%
		// of the available CPU per pod, i.e. 0.01 * milliCpu/(numNodeOODPods-1). Instead of
		// subtracting 1% from the value, we directly use 0.99 as the multiplier.
		podCPU := int64(float64(milliCpu/(numNodeOODPods-1)) * 0.99)

		ns := f.Namespace.Name
		podClient := c.Core().Pods(ns)

		By("Creating pods and waiting for all but one pods to be scheduled")

		for i := 0; i < numNodeOODPods-1; i++ {
			name := fmt.Sprintf("pod-node-outofdisk-%d", i)
			createOutOfDiskPod(c, ns, name, podCPU)

			framework.ExpectNoError(f.WaitForPodRunning(name))
			pod, err := podClient.Get(name)
			framework.ExpectNoError(err)
			Expect(pod.Spec.NodeName).To(Equal(unfilledNodeName))
		}

		pendingPodName := fmt.Sprintf("pod-node-outofdisk-%d", numNodeOODPods-1)
		createOutOfDiskPod(c, ns, pendingPodName, podCPU)

		By(fmt.Sprintf("Finding a failed scheduler event for pod %s", pendingPodName))
		wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
			selector := fields.Set{
				"involvedObject.kind":      "Pod",
				"involvedObject.name":      pendingPodName,
				"involvedObject.namespace": ns,
				"source":                   api.DefaultSchedulerName,
				"reason":                   "FailedScheduling",
			}.AsSelector()
			options := api.ListOptions{FieldSelector: selector}
			schedEvents, err := c.Core().Events(ns).List(options)
			framework.ExpectNoError(err)

			if len(schedEvents.Items) > 0 {
				return true, nil
			} else {
				return false, nil
			}
		})

		nodelist := framework.GetReadySchedulableNodesOrDie(c)
		Expect(len(nodelist.Items)).To(BeNumerically(">", 1))

		nodeToRecover := nodelist.Items[1]
		Expect(nodeToRecover.Name).ToNot(Equal(unfilledNodeName))

		recoverDiskSpace(c, &nodeToRecover)
		recoveredNodeName = nodeToRecover.Name

		By(fmt.Sprintf("Verifying that pod %s schedules on node %s", pendingPodName, recoveredNodeName))
		framework.ExpectNoError(f.WaitForPodRunning(pendingPodName))
		pendingPod, err := podClient.Get(pendingPodName)
		framework.ExpectNoError(err)
		Expect(pendingPod.Spec.NodeName).To(Equal(recoveredNodeName))
	})
})

// createOutOfDiskPod creates a pod in the given namespace with the requested amount of CPU.
func createOutOfDiskPod(c clientset.Interface, ns, name string, milliCPU int64) {
	podClient := c.Core().Pods(ns)

	pod := &api.Pod{
		ObjectMeta: api.ObjectMeta{
			Name: name,
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  "pause",
					Image: framework.GetPauseImageName(c),
					Resources: api.ResourceRequirements{
						Requests: api.ResourceList{
							// Request enough CPU to fit only two pods on a given node.
							api.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI),
						},
					},
				},
			},
		},
	}

	_, err := podClient.Create(pod)
	framework.ExpectNoError(err)
}

// availCpu calculates the available CPU on a given node by subtracting the CPU requested by
// all the pods from the total available CPU capacity on the node.
func availCpu(c clientset.Interface, node *api.Node) (int64, error) {
	podClient := c.Core().Pods(api.NamespaceAll)

	selector := fields.Set{"spec.nodeName": node.Name}.AsSelector()
	options := api.ListOptions{FieldSelector: selector}
	pods, err := podClient.List(options)
	if err != nil {
		return 0, fmt.Errorf("failed to retrieve all the pods on node %s: %v", node.Name, err)
	}
	avail := node.Status.Capacity.Cpu().MilliValue()
	for _, pod := range pods.Items {
		for _, cont := range pod.Spec.Containers {
			avail -= cont.Resources.Requests.Cpu().MilliValue()
		}
	}
	return avail, nil
}

// availSize returns the available disk space on a given node by querying node stats which
// is in turn obtained internally from cadvisor.
func availSize(c clientset.Interface, node *api.Node) (uint64, error) {
	statsResource := fmt.Sprintf("api/v1/proxy/nodes/%s/stats/", node.Name)
	framework.Logf("Querying stats for node %s using url %s", node.Name, statsResource)
	res, err := c.Core().RESTClient().Get().AbsPath(statsResource).Timeout(time.Minute).Do().Raw()
	if err != nil {
		return 0, fmt.Errorf("error querying cAdvisor API: %v", err)
	}
	ci := cadvisorapi.ContainerInfo{}
	err = json.Unmarshal(res, &ci)
	if err != nil {
		return 0, fmt.Errorf("couldn't unmarshal container info: %v", err)
	}
	return ci.Stats[len(ci.Stats)-1].Filesystem[0].Available, nil
}

// fillDiskSpace fills the available disk space on a given node by creating a large file. The disk
// space on the node is filled in such a way that the available space after filling the disk is just
// below the lowDiskSpaceThreshold mark.
func fillDiskSpace(c clientset.Interface, node *api.Node) {
	avail, err := availSize(c, node)
	framework.ExpectNoError(err, "Node %s: couldn't obtain available disk size %v", node.Name, err)

	fillSize := (avail - lowDiskSpaceThreshold + (100 * mb))

	framework.Logf("Node %s: disk space available %d bytes", node.Name, avail)
	By(fmt.Sprintf("Node %s: creating a file of size %d bytes to fill the available disk space", node.Name, fillSize))

	cmd := fmt.Sprintf("fallocate -l %d test.img", fillSize)
	framework.ExpectNoError(framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node))

	ood := framework.WaitForNodeToBe(c, node.Name, api.NodeOutOfDisk, true, nodeOODTimeOut)
	Expect(ood).To(BeTrue(), "Node %s did not run out of disk within %v", node.Name, nodeOODTimeOut)

	avail, err = availSize(c, node)
	framework.Logf("Node %s: disk space available %d bytes", node.Name, avail)
	Expect(avail < lowDiskSpaceThreshold).To(BeTrue())
}

// recoverDiskSpace recovers disk space, filled by creating a large file, on a given node.
func recoverDiskSpace(c clientset.Interface, node *api.Node) {
	By(fmt.Sprintf("Recovering disk space on node %s", node.Name))
	cmd := "rm -f test.img"
	framework.ExpectNoError(framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node))

	ood := framework.WaitForNodeToBe(c, node.Name, api.NodeOutOfDisk, false, nodeOODTimeOut)
	Expect(ood).To(BeTrue(), "Node %s's out of disk condition status did not change to false within %v", node.Name, nodeOODTimeOut)
}
<|endoftext|>