<commit_before>// Package tsz implements time-series compression
// it is a fork of https://github.com/dgryski/go-tsz
// which implements http://www.vldb.org/pvldb/vol8/p1816-teller.pdf
// with the following exceptions:
// * t-1 is here t0 (see devdocs)
// * we renamed the package to be more clearly about the limitation that
// you shouldn't use it for chunks longer than 4.5 hours, due to overflow
// of the first delta (14 bits), see https://github.com/grafana/metrictank/pull/1126
// * we patched a workaround to reconstruct corrupted chunks that were
//   affected by the above issue. (only works for chunks <= 9h)
package tsz

import (
	"bytes"
	"encoding/binary"
	"io"
	"math"
	"math/bits"
	"sync"
)

// Series is the basic series primitive
// you can concurrently put values, finish the stream, and create iterators
type Series struct {
	sync.Mutex

	// TODO(dgryski): timestamps in the paper are uint64
	T0  uint32
	t   uint32
	val float64

	bw       bstream
	leading  uint8
	trailing uint8
	finished bool

	tDelta uint32
}

// New series
func New(t0 uint32) *Series {
	s := Series{
		T0:      t0,
		leading: ^uint8(0),
	}

	// block header
	s.bw.writeBits(uint64(t0), 32)

	return &s

}

// Bytes value of the series stream
func (s *Series) Bytes() []byte {
	s.Lock()
	defer s.Unlock()
	return s.bw.bytes()
}

func finish(w *bstream) {
	// write an end-of-stream record
	w.writeBits(0x0f, 4)
	w.writeBits(0xffffffff, 32)
	w.writeBit(zero)
}

// Finish the series by writing an end-of-stream record
func (s *Series) Finish() {
	s.Lock()
	if !s.finished {
		finish(&s.bw)
		s.finished = true
	}
	s.Unlock()
}

// Push a timestamp and value to the series
func (s *Series) Push(t uint32, v float64) {
	s.Lock()
	defer s.Unlock()

	if s.t == 0 {
		// first point
		s.t = t
		s.val = v
		s.tDelta = t - s.T0
		s.bw.writeBits(uint64(s.tDelta), 14)
		s.bw.writeBits(math.Float64bits(v), 64)
		return
	}

	tDelta := t - s.t
	dod := int32(tDelta - s.tDelta)

	switch {
	case dod == 0:
		s.bw.writeBit(zero)
	case -63 <= dod && dod <= 64:
		s.bw.writeBits(0x02, 2) // '10'
		s.bw.writeBits(uint64(dod), 7)
	case -255 <= dod && dod <= 256:
		s.bw.writeBits(0x06, 3) // '110'
		s.bw.writeBits(uint64(dod), 9)
	case -2047 <= dod && dod <= 2048:
		s.bw.writeBits(0x0e, 4) // '1110'
		s.bw.writeBits(uint64(dod), 12)
	default:
		s.bw.writeBits(0x0f, 4) // '1111'
		s.bw.writeBits(uint64(dod), 32)
	}

	vDelta := math.Float64bits(v) ^ math.Float64bits(s.val)

	if vDelta == 0 {
		s.bw.writeBit(zero)
	} else {
		s.bw.writeBit(one)

		leading := uint8(bits.LeadingZeros64(vDelta))
		trailing := uint8(bits.TrailingZeros64(vDelta))

		// clamp number of leading zeros to avoid overflow when encoding
		if leading >= 32 {
			leading = 31
		}

		// TODO(dgryski): check if it's 'cheaper' to reset the leading/trailing bits instead
		if s.leading != ^uint8(0) && leading >= s.leading && trailing >= s.trailing {
			s.bw.writeBit(zero)
			s.bw.writeBits(vDelta>>s.trailing, 64-int(s.leading)-int(s.trailing))
		} else {
			s.leading, s.trailing = leading, trailing

			s.bw.writeBit(one)
			s.bw.writeBits(uint64(leading), 5)

			// Note that if leading == trailing == 0, then sigbits == 64.  But that value doesn't actually fit into the 6 bits we have.
			// Luckily, we never need to encode 0 significant bits, since that would put us in the other case (vdelta == 0).
			// So instead we write out a 0 and adjust it back to 64 on unpacking.
			sigbits := 64 - leading - trailing
			s.bw.writeBits(uint64(sigbits), 6)
			s.bw.writeBits(vDelta>>trailing, int(sigbits))
		}
	}

	s.tDelta = tDelta
	s.t = t
	s.val = v

}

// Iter lets you iterate over a series.  It is not concurrency-safe.
func (s *Series) Iter() *Iter {
	s.Lock()
	w := s.bw.clone()
	s.Unlock()

	finish(w)
	iter, _ := bstreamIterator(w)
	return iter
}

// Iter lets you iterate over a series.  It is not concurrency-safe.
type Iter struct {
	T0 uint32

	t   uint32
	val float64

	br       bstream
	leading  uint8
	trailing uint8

	finished bool

	tDelta uint32
	err    error
}

func bstreamIterator(br *bstream) (*Iter, error) {

	br.count = 8

	t0, err := br.readBits(32)
	if err != nil {
		return nil, err
	}

	return &Iter{
		T0: uint32(t0),
		br: *br,
	}, nil
}

// NewIterator for the series
func NewIterator(b []byte) (*Iter, error) {
	return bstreamIterator(newBReader(b))
}

func (it *Iter) dod() (int32, bool) {
	var d byte
	for i := 0; i < 4; i++ {
		d <<= 1
		bit, err := it.br.readBit()
		if err != nil {
			it.err = err
			return 0, false
		}
		if bit == zero {
			break
		}
		d |= 1
	}

	var dod int32
	var sz uint
	switch d {
	case 0x00:
		// dod == 0
	case 0x02:
		sz = 7
	case 0x06:
		sz = 9
	case 0x0e:
		sz = 12
	case 0x0f:
		bits, err := it.br.readBits(32)
		if err != nil {
			it.err = err
			return 0, false
		}

		// end of stream
		if bits == 0xffffffff {
			it.finished = true
			return 0, false
		}

		dod = int32(bits)
	}

	if sz != 0 {
		bits, err := it.br.readBits(int(sz))
		if err != nil {
			it.err = err
			return 0, false
		}
		if bits > (1 << (sz - 1)) {
			// or something
			bits = bits - (1 << sz)
		}
		dod = int32(bits)
	}

	return dod, true
}

// Next iteration of the series iterator
func (it *Iter) Next() bool {

	if it.err != nil || it.finished {
		return false
	}

	if it.t == 0 {
		// read first t and v
		tDelta, err := it.br.readBits(14)
		if err != nil {
			it.err = err
			return false
		}
		it.tDelta = uint32(tDelta)
		it.t = it.T0 + it.tDelta
		v, err := it.br.readBits(64)
		if err != nil {
			it.err = err
			return false
		}

		it.val = math.Float64frombits(v)

		// special case: read upcoming dod
		// if delta+dod <0, the delta overflowed, and should rectify it.
		// see https://github.com/grafana/metrictank/pull/1126
		// but we must take a backup of the stream because reading from the
		// stream modifies it.
		brBackup := it.br.clone()
		dod, ok := it.dod()
		if !ok {
			// in this case we can't know if the point is right or wrong.
			// long chunks with a single point in them may lead to a wrong read, but this should be rare.
			// we can't just adjust the timestamp because we don't know the length of the chunk
			// (though this could be done by having the caller pass us that information), nor whether
			// a delta that may seem low compared to chunk length was intentional or not.
			// so, nothing much to do in this case. return the possibly incorrect point.
			// and for return value, stick to normal iter semantics:
			// this read succeeded, though we already know the next one will fail
			it.br = *brBackup
			return true
		}
		if dod+int32(tDelta) < 0 {
			it.tDelta += 16384
			it.t += 16384
		}
		it.br = *brBackup
		return true
	}

	// read delta-of-delta
	dod, ok := it.dod()
	if !ok {
		return false
	}

	tDelta := it.tDelta + uint32(dod)

	it.tDelta = tDelta
	it.t = it.t + it.tDelta

	// read compressed value
	bit, err := it.br.readBit()
	if err != nil {
		it.err = err
		return false
	}

	if bit == zero {
		// it.val = it.val
	} else {
		bit, itErr := it.br.readBit()
		if itErr != nil {
			it.err = err
			return false
		}
		if bit == zero {
			// reuse leading/trailing zero bits
			// it.leading, it.trailing = it.leading, it.trailing
		} else {
			bits, err := it.br.readBits(5)
			if err != nil {
				it.err = err
				return false
			}
			it.leading = uint8(bits)

			bits, err = it.br.readBits(6)
			if err != nil {
				it.err = err
				return false
			}
			mbits := uint8(bits)
			// 0 significant bits here means we overflowed and we actually need 64; see comment in encoder
			if mbits == 0 {
				mbits = 64
			}
			it.trailing = 64 - it.leading - mbits
		}

		mbits := int(64 - it.leading - it.trailing)
		bits, err := it.br.readBits(mbits)
		if err != nil {
			it.err = err
			return false
		}
		vbits := math.Float64bits(it.val)
		vbits ^= (bits << it.trailing)
		it.val = math.Float64frombits(vbits)
	}

	return true
}

// Values at the current iterator position
func (it *Iter) Values() (uint32, float64) {
	return it.t, it.val
}

// Err error at the current iterator position
func (it *Iter) Err() error {
	return it.err
}

type errMarshal struct {
	w   io.Writer
	r   io.Reader
	err error
}

func (em *errMarshal) write(t interface{}) {
	if em.err != nil {
		return
	}
	em.err = binary.Write(em.w, binary.BigEndian, t)
}

func (em *errMarshal) read(t interface{}) {
	if em.err != nil {
		return
	}
	em.err = binary.Read(em.r, binary.BigEndian, t)
}

// MarshalBinary implements the encoding.BinaryMarshaler interface
func (s *Series) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)
	em := &errMarshal{w: buf}
	em.write(s.T0)
	em.write(s.leading)
	em.write(s.t)
	em.write(s.tDelta)
	em.write(s.trailing)
	em.write(s.val)
	bStream, err := s.bw.MarshalBinary()
	if err != nil {
		return nil, err
	}
	em.write(bStream)
	if em.err != nil {
		return nil, em.err
	}
	return buf.Bytes(), nil
}

// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
func (s *Series) UnmarshalBinary(b []byte) error {
	buf := bytes.NewReader(b)
	em := &errMarshal{r: buf}
	em.read(&s.T0)
	em.read(&s.leading)
	em.read(&s.t)
	em.read(&s.tDelta)
	em.read(&s.trailing)
	em.read(&s.val)
	outBuf := make([]byte, buf.Len())
	em.read(outBuf)
	err := s.bw.UnmarshalBinary(outBuf)
	if err != nil {
		return err
	}
	if em.err != nil {
		return em.err
	}
	return nil
}
<commit_msg>clarify<commit_after>// Package tsz implements time-series compression
// it is a fork of https://github.com/dgryski/go-tsz
// which implements http://www.vldb.org/pvldb/vol8/p1816-teller.pdf
// with the following exceptions:
// * t-1 is here t0 (see devdocs)
// * we renamed the package to be more clearly about the limitation that
// you shouldn't use it for chunks longer than 4.5 hours, due to overflow
// of the first delta (14 bits), see https://github.com/grafana/metrictank/pull/1126
// * we patched a workaround to reconstruct corrupted chunks that were
//   affected by the above issue. (only works for chunks <= 9h)
package tsz

import (
	"bytes"
	"encoding/binary"
	"io"
	"math"
	"math/bits"
	"sync"
)

// Series is the basic series primitive
// you can concurrently put values, finish the stream, and create iterators
type Series struct {
	sync.Mutex

	// TODO(dgryski): timestamps in the paper are uint64
	T0  uint32
	t   uint32
	val float64

	bw       bstream
	leading  uint8
	trailing uint8
	finished bool

	tDelta uint32
}

// New series
func New(t0 uint32) *Series {
	s := Series{
		T0:      t0,
		leading: ^uint8(0),
	}

	// block header
	s.bw.writeBits(uint64(t0), 32)

	return &s

}

// Bytes value of the series stream
func (s *Series) Bytes() []byte {
	s.Lock()
	defer s.Unlock()
	return s.bw.bytes()
}

func finish(w *bstream) {
	// write an end-of-stream record
	w.writeBits(0x0f, 4)
	w.writeBits(0xffffffff, 32)
	w.writeBit(zero)
}

// Finish the series by writing an end-of-stream record
func (s *Series) Finish() {
	s.Lock()
	if !s.finished {
		finish(&s.bw)
		s.finished = true
	}
	s.Unlock()
}

// Push a timestamp and value to the series
func (s *Series) Push(t uint32, v float64) {
	s.Lock()
	defer s.Unlock()

	if s.t == 0 {
		// first point
		s.t = t
		s.val = v
		s.tDelta = t - s.T0
		s.bw.writeBits(uint64(s.tDelta), 14)
		s.bw.writeBits(math.Float64bits(v), 64)
		return
	}

	tDelta := t - s.t
	dod := int32(tDelta - s.tDelta)

	switch {
	case dod == 0:
		s.bw.writeBit(zero)
	case -63 <= dod && dod <= 64:
		s.bw.writeBits(0x02, 2) // '10'
		s.bw.writeBits(uint64(dod), 7)
	case -255 <= dod && dod <= 256:
		s.bw.writeBits(0x06, 3) // '110'
		s.bw.writeBits(uint64(dod), 9)
	case -2047 <= dod && dod <= 2048:
		s.bw.writeBits(0x0e, 4) // '1110'
		s.bw.writeBits(uint64(dod), 12)
	default:
		s.bw.writeBits(0x0f, 4) // '1111'
		s.bw.writeBits(uint64(dod), 32)
	}

	vDelta := math.Float64bits(v) ^ math.Float64bits(s.val)

	if vDelta == 0 {
		s.bw.writeBit(zero)
	} else {
		s.bw.writeBit(one)

		leading := uint8(bits.LeadingZeros64(vDelta))
		trailing := uint8(bits.TrailingZeros64(vDelta))

		// clamp number of leading zeros to avoid overflow when encoding
		if leading >= 32 {
			leading = 31
		}

		// TODO(dgryski): check if it's 'cheaper' to reset the leading/trailing bits instead
		if s.leading != ^uint8(0) && leading >= s.leading && trailing >= s.trailing {
			s.bw.writeBit(zero)
			s.bw.writeBits(vDelta>>s.trailing, 64-int(s.leading)-int(s.trailing))
		} else {
			s.leading, s.trailing = leading, trailing

			s.bw.writeBit(one)
			s.bw.writeBits(uint64(leading), 5)

			// Note that if leading == trailing == 0, then sigbits == 64.  But that value doesn't actually fit into the 6 bits we have.
			// Luckily, we never need to encode 0 significant bits, since that would put us in the other case (vdelta == 0).
			// So instead we write out a 0 and adjust it back to 64 on unpacking.
			sigbits := 64 - leading - trailing
			s.bw.writeBits(uint64(sigbits), 6)
			s.bw.writeBits(vDelta>>trailing, int(sigbits))
		}
	}

	s.tDelta = tDelta
	s.t = t
	s.val = v

}

// Iter lets you iterate over a series.  It is not concurrency-safe.
func (s *Series) Iter() *Iter {
	s.Lock()
	w := s.bw.clone()
	s.Unlock()

	finish(w)
	iter, _ := bstreamIterator(w)
	return iter
}

// Iter lets you iterate over a series.  It is not concurrency-safe.
type Iter struct {
	T0 uint32

	t   uint32
	val float64

	br       bstream
	leading  uint8
	trailing uint8

	finished bool

	tDelta uint32
	err    error
}

func bstreamIterator(br *bstream) (*Iter, error) {

	br.count = 8

	t0, err := br.readBits(32)
	if err != nil {
		return nil, err
	}

	return &Iter{
		T0: uint32(t0),
		br: *br,
	}, nil
}

// NewIterator for the series
func NewIterator(b []byte) (*Iter, error) {
	return bstreamIterator(newBReader(b))
}

func (it *Iter) dod() (int32, bool) {
	var d byte
	for i := 0; i < 4; i++ {
		d <<= 1
		bit, err := it.br.readBit()
		if err != nil {
			it.err = err
			return 0, false
		}
		if bit == zero {
			break
		}
		d |= 1
	}

	var dod int32
	var sz uint
	switch d {
	case 0x00:
		// dod == 0
	case 0x02:
		sz = 7
	case 0x06:
		sz = 9
	case 0x0e:
		sz = 12
	case 0x0f:
		bits, err := it.br.readBits(32)
		if err != nil {
			it.err = err
			return 0, false
		}

		// end of stream
		if bits == 0xffffffff {
			it.finished = true
			return 0, false
		}

		dod = int32(bits)
	}

	if sz != 0 {
		bits, err := it.br.readBits(int(sz))
		if err != nil {
			it.err = err
			return 0, false
		}
		if bits > (1 << (sz - 1)) {
			// or something
			bits = bits - (1 << sz)
		}
		dod = int32(bits)
	}

	return dod, true
}

// Next iteration of the series iterator
func (it *Iter) Next() bool {

	if it.err != nil || it.finished {
		return false
	}

	if it.t == 0 {
		// read first t and v
		tDelta, err := it.br.readBits(14)
		if err != nil {
			it.err = err
			return false
		}
		it.tDelta = uint32(tDelta)
		it.t = it.T0 + it.tDelta
		v, err := it.br.readBits(64)
		if err != nil {
			it.err = err
			return false
		}

		it.val = math.Float64frombits(v)

		// special case: read upcoming dod
		// if delta+dod <0 (aka the upcoming delta < 0),
		// our current delta overflowed, and should rectify it.
		// see https://github.com/grafana/metrictank/pull/1126
		// but we must take a backup of the stream because reading from the
		// stream modifies it.
		brBackup := it.br.clone()
		dod, ok := it.dod()
		if !ok {
			// in this case we can't know if the point is right or wrong.
			// long chunks with a single point in them may lead to a wrong read, but this should be rare.
			// we can't just adjust the timestamp because we don't know the length of the chunk
			// (though this could be done by having the caller pass us that information), nor whether
			// a delta that may seem low compared to chunk length was intentional or not.
			// so, nothing much to do in this case. return the possibly incorrect point.
			// and for return value, stick to normal iter semantics:
			// this read succeeded, though we already know the next one will fail
			it.br = *brBackup
			return true
		}
		if dod+int32(tDelta) < 0 {
			it.tDelta += 16384
			it.t += 16384
		}
		it.br = *brBackup
		return true
	}

	// read delta-of-delta
	dod, ok := it.dod()
	if !ok {
		return false
	}

	tDelta := it.tDelta + uint32(dod)

	it.tDelta = tDelta
	it.t = it.t + it.tDelta

	// read compressed value
	bit, err := it.br.readBit()
	if err != nil {
		it.err = err
		return false
	}

	if bit == zero {
		// it.val = it.val
	} else {
		bit, itErr := it.br.readBit()
		if itErr != nil {
			it.err = err
			return false
		}
		if bit == zero {
			// reuse leading/trailing zero bits
			// it.leading, it.trailing = it.leading, it.trailing
		} else {
			bits, err := it.br.readBits(5)
			if err != nil {
				it.err = err
				return false
			}
			it.leading = uint8(bits)

			bits, err = it.br.readBits(6)
			if err != nil {
				it.err = err
				return false
			}
			mbits := uint8(bits)
			// 0 significant bits here means we overflowed and we actually need 64; see comment in encoder
			if mbits == 0 {
				mbits = 64
			}
			it.trailing = 64 - it.leading - mbits
		}

		mbits := int(64 - it.leading - it.trailing)
		bits, err := it.br.readBits(mbits)
		if err != nil {
			it.err = err
			return false
		}
		vbits := math.Float64bits(it.val)
		vbits ^= (bits << it.trailing)
		it.val = math.Float64frombits(vbits)
	}

	return true
}

// Values at the current iterator position
func (it *Iter) Values() (uint32, float64) {
	return it.t, it.val
}

// Err error at the current iterator position
func (it *Iter) Err() error {
	return it.err
}

type errMarshal struct {
	w   io.Writer
	r   io.Reader
	err error
}

func (em *errMarshal) write(t interface{}) {
	if em.err != nil {
		return
	}
	em.err = binary.Write(em.w, binary.BigEndian, t)
}

func (em *errMarshal) read(t interface{}) {
	if em.err != nil {
		return
	}
	em.err = binary.Read(em.r, binary.BigEndian, t)
}

// MarshalBinary implements the encoding.BinaryMarshaler interface
func (s *Series) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)
	em := &errMarshal{w: buf}
	em.write(s.T0)
	em.write(s.leading)
	em.write(s.t)
	em.write(s.tDelta)
	em.write(s.trailing)
	em.write(s.val)
	bStream, err := s.bw.MarshalBinary()
	if err != nil {
		return nil, err
	}
	em.write(bStream)
	if em.err != nil {
		return nil, em.err
	}
	return buf.Bytes(), nil
}

// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
func (s *Series) UnmarshalBinary(b []byte) error {
	buf := bytes.NewReader(b)
	em := &errMarshal{r: buf}
	em.read(&s.T0)
	em.read(&s.leading)
	em.read(&s.t)
	em.read(&s.tDelta)
	em.read(&s.trailing)
	em.read(&s.val)
	outBuf := make([]byte, buf.Len())
	em.read(outBuf)
	err := s.bw.UnmarshalBinary(outBuf)
	if err != nil {
		return err
	}
	if em.err != nil {
		return em.err
	}
	return nil
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"
)

// Add adds the item in the state at the given address.
//
// The item can be a ModuleState, ResourceState, or InstanceState. Depending
// on the item type, the address may or may not be valid. For example, a
// module cannot be moved to a resource address, however a resource can be
// moved to a module address (it retains the same name, under that resource).
//
// The full semantics of Add:
//
//                         ┌───────────────────────┬───────────────────────┬───────────────────────┐
//                         │    Module Address     │   Resource Address    │   Instance Address    │
// ┌───────────────────────┼───────────────────────┼───────────────────────┼───────────────────────┤
// │      ModuleState      │           ✓           │           x           │           x           │
// ├───────────────────────┼───────────────────────┼───────────────────────┼───────────────────────┤
// │     ResourceState     │           ✓           │           ✓           │        maybe*         │
// ├───────────────────────┼───────────────────────┼───────────────────────┼───────────────────────┤
// │    Instance State     │           ✓           │           ✓           │           ✓           │
// └───────────────────────┴───────────────────────┴───────────────────────┴───────────────────────┘
//
// *maybe - Resources can be added at an instance address only if the resource
//          represents a single instance (primary). Example:
//          "aws_instance.foo" can be moved to "aws_instance.bar.tainted"
//
func (s *State) Add(addrRaw string, raw interface{}) error {
	// Parse the address
	addr, err := ParseResourceAddress(addrRaw)
	if err != nil {
		return err
	}

	// Determine the types
	from := detectValueAddLoc(raw)
	to := detectAddrAddLoc(addr)

	// Find the function to do this
	fromMap, ok := stateAddFuncs[from]
	if !ok {
		return fmt.Errorf("invalid source to add to state: %T", raw)
	}
	f, ok := fromMap[to]
	if !ok {
		return fmt.Errorf("invalid destination: %s (%d)", addr, to)
	}

	// Call the migrator
	if err := f(s, addr, raw); err != nil {
		return err
	}

	// Prune the state
	s.prune()
	return nil
}

func stateAddFunc_Module_Module(s *State, addr *ResourceAddress, raw interface{}) error {
	src := raw.(*ModuleState).deepcopy()

	// If the target module exists, it is an error
	path := append([]string{"root"}, addr.Path...)
	if s.ModuleByPath(path) != nil {
		return fmt.Errorf("module target is not empty: %s", addr)
	}

	// Create it and copy our outputs and dependencies
	mod := s.AddModule(path)
	mod.Outputs = src.Outputs
	mod.Dependencies = src.Dependencies

	// Go through the resources perform an add for each of those
	for k, v := range src.Resources {
		resourceKey, err := ParseResourceStateKey(k)
		if err != nil {
			return err
		}

		// Update the resource address for this
		addrCopy := *addr
		addrCopy.Type = resourceKey.Type
		addrCopy.Name = resourceKey.Name
		addrCopy.Index = resourceKey.Index

		// Perform an add
		if err := s.Add(addrCopy.String(), v); err != nil {
			return err
		}
	}

	return nil
}

func stateAddFunc_Resource_Resource(s *State, addr *ResourceAddress, raw interface{}) error {
	src := raw.(*ResourceState)

	// Initialize the resource
	resourceRaw, exists := stateAddInitAddr(s, addr)
	if exists {
		return fmt.Errorf("resource exists and not empty: %s", addr)
	}
	resource := resourceRaw.(*ResourceState)
	resource.Type = src.Type

	// TODO: Dependencies
	// TODO: Provider?

	// Move the primary
	if src.Primary != nil {
		addrCopy := *addr
		addrCopy.InstanceType = TypePrimary
		addrCopy.InstanceTypeSet = true
		if err := s.Add(addrCopy.String(), src.Primary); err != nil {
			return err
		}
	}

	// TODO: Move all tainted
	// TODO: Move all deposed

	return nil
}

func stateAddFunc_Instance_Instance(s *State, addr *ResourceAddress, raw interface{}) error {
	src := raw.(*InstanceState).deepcopy()

	// Create the module up to this point
	path := append([]string{"root"}, addr.Path...)
	mod := s.ModuleByPath(path)
	if mod == nil {
		mod = s.AddModule(path)
	}

	// Create the resources
	resourceKey := (&ResourceStateKey{
		Name:  addr.Name,
		Type:  addr.Type,
		Index: addr.Index,
	}).String()
	resource, ok := mod.Resources[resourceKey]
	if !ok {
		resource = &ResourceState{Type: addr.Type}
		resource.init()
		mod.Resources[resourceKey] = resource
	}

	// Depending on the instance type, set it
	switch addr.InstanceType {
	case TypePrimary:
		resource.Primary = src
	default:
		return fmt.Errorf("can't move instance state to %s", addr.InstanceType)
	}

	return nil
}

// stateAddFunc is the type of function for adding an item to a state
type stateAddFunc func(s *State, addr *ResourceAddress, item interface{}) error

// stateAddFuncs has the full matrix mapping of the state adders.
var stateAddFuncs map[stateAddLoc]map[stateAddLoc]stateAddFunc

func init() {
	stateAddFuncs = map[stateAddLoc]map[stateAddLoc]stateAddFunc{
		stateAddModule: {
			stateAddModule: stateAddFunc_Module_Module,
		},
		stateAddResource: {
			stateAddResource: stateAddFunc_Resource_Resource,
		},
		stateAddInstance: {
			stateAddInstance: stateAddFunc_Instance_Instance,
		},
	}
}

// stateAddLoc is an enum to represent the location where state is being
// moved from/to. We use this for quick lookups in a function map.
type stateAddLoc uint

const (
	stateAddInvalid stateAddLoc = iota
	stateAddModule
	stateAddResource
	stateAddInstance
)

// detectAddrAddLoc detects the state type for the given address. This
// function is specifically not unit tested since we consider the State.Add
// functionality to be comprehensive enough to cover this.
func detectAddrAddLoc(addr *ResourceAddress) stateAddLoc {
	if addr.Name == "" {
		return stateAddModule
	}

	if !addr.InstanceTypeSet {
		return stateAddResource
	}

	return stateAddInstance
}

// detectValueAddLoc determines the stateAddLoc value from the raw value
// that is some State structure.
func detectValueAddLoc(raw interface{}) stateAddLoc {
	switch raw.(type) {
	case *ModuleState:
		return stateAddModule
	case *ResourceState:
		return stateAddResource
	case *InstanceState:
		return stateAddInstance
	default:
		return stateAddInvalid
	}
}

// stateAddInitAddr takes a ResourceAddress and creates the non-existing
// resources up to that point, returning the empty (or existing) interface
// at that address.
func stateAddInitAddr(s *State, addr *ResourceAddress) (interface{}, bool) {
	addType := detectAddrAddLoc(addr)

	// Get the module
	path := append([]string{"root"}, addr.Path...)
	exists := true
	mod := s.ModuleByPath(path)
	if mod == nil {
		mod = s.AddModule(path)
		exists = false
	}
	if addType == stateAddModule {
		return mod, exists
	}

	// Add the resource
	resourceKey := (&ResourceStateKey{
		Name:  addr.Name,
		Type:  addr.Type,
		Index: addr.Index,
	}).String()
	exists = true
	resource, ok := mod.Resources[resourceKey]
	if !ok {
		resource = &ResourceState{Type: addr.Type}
		resource.init()
		mod.Resources[resourceKey] = resource
		exists = false
	}
	if addType == stateAddResource {
		return resource, exists
	}

	// Get the instance
	exists = true
	var instance *InstanceState
	switch addr.InstanceType {
	case TypePrimary:
		instance = resource.Primary
	case TypeTainted:
		idx := addr.Index
		if addr.Index < 0 {
			idx = 0
		}
		if len(resource.Tainted) > idx {
			instance = resource.Tainted[idx]
		}
	case TypeDeposed:
		idx := addr.Index
		if addr.Index < 0 {
			idx = 0
		}
		if len(resource.Deposed) > idx {
			instance = resource.Deposed[idx]
		}
	}
	if instance == nil {
		instance = &InstanceState{}
		exists = false
	}

	return instance, exists
}
<commit_msg>terraform: unify on init addr<commit_after>package terraform

import (
	"fmt"
)

// Add adds the item in the state at the given address.
//
// The item can be a ModuleState, ResourceState, or InstanceState. Depending
// on the item type, the address may or may not be valid. For example, a
// module cannot be moved to a resource address, however a resource can be
// moved to a module address (it retains the same name, under that resource).
//
// The full semantics of Add:
//
//                         ┌───────────────────────┬───────────────────────┬───────────────────────┐
//                         │    Module Address     │   Resource Address    │   Instance Address    │
// ┌───────────────────────┼───────────────────────┼───────────────────────┼───────────────────────┤
// │      ModuleState      │           ✓           │           x           │           x           │
// ├───────────────────────┼───────────────────────┼───────────────────────┼───────────────────────┤
// │     ResourceState     │           ✓           │           ✓           │        maybe*         │
// ├───────────────────────┼───────────────────────┼───────────────────────┼───────────────────────┤
// │    Instance State     │           ✓           │           ✓           │           ✓           │
// └───────────────────────┴───────────────────────┴───────────────────────┴───────────────────────┘
//
// *maybe - Resources can be added at an instance address only if the resource
//          represents a single instance (primary). Example:
//          "aws_instance.foo" can be moved to "aws_instance.bar.tainted"
//
func (s *State) Add(addrRaw string, raw interface{}) error {
	// Parse the address
	addr, err := ParseResourceAddress(addrRaw)
	if err != nil {
		return err
	}

	// Determine the types
	from := detectValueAddLoc(raw)
	to := detectAddrAddLoc(addr)

	// Find the function to do this
	fromMap, ok := stateAddFuncs[from]
	if !ok {
		return fmt.Errorf("invalid source to add to state: %T", raw)
	}
	f, ok := fromMap[to]
	if !ok {
		return fmt.Errorf("invalid destination: %s (%d)", addr, to)
	}

	// Call the migrator
	if err := f(s, addr, raw); err != nil {
		return err
	}

	// Prune the state
	s.prune()
	return nil
}

func stateAddFunc_Module_Module(s *State, addr *ResourceAddress, raw interface{}) error {
	src := raw.(*ModuleState).deepcopy()

	// If the target module exists, it is an error
	path := append([]string{"root"}, addr.Path...)
	if s.ModuleByPath(path) != nil {
		return fmt.Errorf("module target is not empty: %s", addr)
	}

	// Create it and copy our outputs and dependencies
	mod := s.AddModule(path)
	mod.Outputs = src.Outputs
	mod.Dependencies = src.Dependencies

	// Go through the resources perform an add for each of those
	for k, v := range src.Resources {
		resourceKey, err := ParseResourceStateKey(k)
		if err != nil {
			return err
		}

		// Update the resource address for this
		addrCopy := *addr
		addrCopy.Type = resourceKey.Type
		addrCopy.Name = resourceKey.Name
		addrCopy.Index = resourceKey.Index

		// Perform an add
		if err := s.Add(addrCopy.String(), v); err != nil {
			return err
		}
	}

	return nil
}

func stateAddFunc_Resource_Resource(s *State, addr *ResourceAddress, raw interface{}) error {
	src := raw.(*ResourceState)

	// Initialize the resource
	resourceRaw, exists := stateAddInitAddr(s, addr)
	if exists {
		return fmt.Errorf("resource exists and not empty: %s", addr)
	}
	resource := resourceRaw.(*ResourceState)
	resource.Type = src.Type

	// TODO: Dependencies
	// TODO: Provider?

	// Move the primary
	if src.Primary != nil {
		addrCopy := *addr
		addrCopy.InstanceType = TypePrimary
		addrCopy.InstanceTypeSet = true
		if err := s.Add(addrCopy.String(), src.Primary); err != nil {
			return err
		}
	}

	// TODO: Move all tainted
	// TODO: Move all deposed

	return nil
}

func stateAddFunc_Instance_Instance(s *State, addr *ResourceAddress, raw interface{}) error {
	src := raw.(*InstanceState).deepcopy()

	// Create the instance
	instanceRaw, _ := stateAddInitAddr(s, addr)
	instance := instanceRaw.(*InstanceState)

	// Depending on the instance type, set it
	switch addr.InstanceType {
	case TypePrimary:
		*instance = *src
	default:
		return fmt.Errorf("can't move instance state to %s", addr.InstanceType)
	}

	return nil
}

// stateAddFunc is the type of function for adding an item to a state
type stateAddFunc func(s *State, addr *ResourceAddress, item interface{}) error

// stateAddFuncs has the full matrix mapping of the state adders.
var stateAddFuncs map[stateAddLoc]map[stateAddLoc]stateAddFunc

func init() {
	stateAddFuncs = map[stateAddLoc]map[stateAddLoc]stateAddFunc{
		stateAddModule: {
			stateAddModule: stateAddFunc_Module_Module,
		},
		stateAddResource: {
			stateAddResource: stateAddFunc_Resource_Resource,
		},
		stateAddInstance: {
			stateAddInstance: stateAddFunc_Instance_Instance,
		},
	}
}

// stateAddLoc is an enum to represent the location where state is being
// moved from/to. We use this for quick lookups in a function map.
type stateAddLoc uint

const (
	stateAddInvalid stateAddLoc = iota
	stateAddModule
	stateAddResource
	stateAddInstance
)

// detectAddrAddLoc detects the state type for the given address. This
// function is specifically not unit tested since we consider the State.Add
// functionality to be comprehensive enough to cover this.
func detectAddrAddLoc(addr *ResourceAddress) stateAddLoc {
	if addr.Name == "" {
		return stateAddModule
	}

	if !addr.InstanceTypeSet {
		return stateAddResource
	}

	return stateAddInstance
}

// detectValueAddLoc determines the stateAddLoc value from the raw value
// that is some State structure.
func detectValueAddLoc(raw interface{}) stateAddLoc {
	switch raw.(type) {
	case *ModuleState:
		return stateAddModule
	case *ResourceState:
		return stateAddResource
	case *InstanceState:
		return stateAddInstance
	default:
		return stateAddInvalid
	}
}

// stateAddInitAddr takes a ResourceAddress and creates the non-existing
// resources up to that point, returning the empty (or existing) interface
// at that address.
func stateAddInitAddr(s *State, addr *ResourceAddress) (interface{}, bool) {
	addType := detectAddrAddLoc(addr)

	// Get the module
	path := append([]string{"root"}, addr.Path...)
	exists := true
	mod := s.ModuleByPath(path)
	if mod == nil {
		mod = s.AddModule(path)
		exists = false
	}
	if addType == stateAddModule {
		return mod, exists
	}

	// Add the resource
	resourceKey := (&ResourceStateKey{
		Name:  addr.Name,
		Type:  addr.Type,
		Index: addr.Index,
	}).String()
	exists = true
	resource, ok := mod.Resources[resourceKey]
	if !ok {
		resource = &ResourceState{Type: addr.Type}
		resource.init()
		mod.Resources[resourceKey] = resource
		exists = false
	}
	if addType == stateAddResource {
		return resource, exists
	}

	// Get the instance
	exists = true
	var instance *InstanceState
	switch addr.InstanceType {
	case TypePrimary:
		instance = resource.Primary
	case TypeTainted:
		idx := addr.Index
		if addr.Index < 0 {
			idx = 0
		}
		if len(resource.Tainted) > idx {
			instance = resource.Tainted[idx]
		}
	case TypeDeposed:
		idx := addr.Index
		if addr.Index < 0 {
			idx = 0
		}
		if len(resource.Deposed) > idx {
			instance = resource.Deposed[idx]
		}
	}
	if instance == nil {
		instance = &InstanceState{}
		exists = false
	}

	return instance, exists
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"
	"sync"

	"github.com/hashicorp/terraform/config"
	"github.com/hashicorp/terraform/depgraph"
)

// Terraform is the primary structure that is used to interact with
// Terraform from code, and can perform operations such as returning
// all resources, a resource tree, a specific resource, etc.
type Terraform struct {
	config    *config.Config
	graph     *depgraph.Graph
	mapping   map[*config.Resource]*terraformProvider
	variables map[string]string
}

// terraformProvider contains internal state information about a resource
// provider for Terraform.
type terraformProvider struct {
	Provider ResourceProvider
	Config   *config.ProviderConfig

	sync.Once
}

// This is a function type used to implement a walker for the resource
// tree internally on the Terraform structure.
type genericWalkFunc func(*Resource) (map[string]string, error)

// Config is the configuration that must be given to instantiate
// a Terraform structure.
type Config struct {
	Config    *config.Config
	Providers map[string]ResourceProviderFactory
	Variables map[string]string

	computedPlaceholder string
}

// New creates a new Terraform structure, initializes resource providers
// for the given configuration, etc.
//
// Semantic checks of the entire configuration structure are done at this
// time, as well as richer checks such as verifying that the resource providers
// can be properly initialized, can be configured, etc.
func New(c *Config) (*Terraform, error) {
	var errs []error

	// Calculate the computed key placeholder
	c.computedPlaceholder = "tf_computed_placeholder"

	// Validate that all required variables have values
	if err := smcVariables(c); err != nil {
		errs = append(errs, err...)
	}

	// Match all the resources with a provider and initialize the providers
	mapping, err := smcProviders(c)
	if err != nil {
		errs = append(errs, err...)
	}

	// Validate all the configurations, once.
	tps := make(map[*terraformProvider]struct{})
	for _, tp := range mapping {
		if _, ok := tps[tp]; !ok {
			tps[tp] = struct{}{}
		}
	}
	for tp, _ := range tps {
		var rc *ResourceConfig
		if tp.Config != nil {
			rc = NewResourceConfig(tp.Config.RawConfig)
		}

		_, tpErrs := tp.Provider.Validate(rc)
		if len(tpErrs) > 0 {
			errs = append(errs, tpErrs...)
		}
	}

	// Build the resource graph
	graph := c.Config.Graph()
	if err := graph.Validate(); err != nil {
		errs = append(errs, fmt.Errorf(
			"Resource graph has an error: %s", err))
	}

	// If we accumulated any errors, then return them all
	if len(errs) > 0 {
		return nil, &MultiError{Errors: errs}
	}

	return &Terraform{
		config:    c.Config,
		graph:     graph,
		mapping:   mapping,
		variables: c.Variables,
	}, nil
}

func (t *Terraform) Apply(p *Plan) (*State, error) {
	result := new(State)
	err := t.graph.Walk(t.applyWalkFn(p, result))
	return result, err
}

func (t *Terraform) Plan(s *State) (*Plan, error) {
	result := new(Plan)
	err := t.graph.Walk(t.planWalkFn(s, result))
	if err != nil {
		return nil, err
	}

	return result, nil
}

func (t *Terraform) Refresh(*State) (*State, error) {
	return nil, nil
}

func (t *Terraform) applyWalkFn(
	p *Plan,
	result *State) depgraph.WalkFunc {
	var l sync.Mutex

	// Initialize the result
	result.init()

	cb := func(r *Resource) (map[string]string, error) {
		// Get the latest diff since there are no computed values anymore
		diff, err := r.Provider.Diff(r.State, r.Config)
		if err != nil {
			return nil, err
		}

		// TODO(mitchellh): we need to verify the diff doesn't change
		// anything and that the diff has no computed values (pre-computed)

		// With the completed diff, apply!
		rs, err := r.Provider.Apply(r.State, diff)
		if err != nil {
			return nil, err
		}

		// If no state was returned, then no variables were updated so
		// just return.
		if rs == nil {
			return nil, nil
		}

		var errs []error
		for ak, av := range rs.Attributes {
			// If the value is the unknown variable value, then it is an error.
			// In this case we record the error and remove it from the state
			if av == config.UnknownVariableValue {
				errs = append(errs, fmt.Errorf(
					"Attribute with unknown value: %s", ak))
				delete(rs.Attributes, ak)
			}
		}

		// Update the resulting diff
		l.Lock()
		result.Resources[r.Id] = rs
		l.Unlock()

		// Determine the new state and update variables
		vars := make(map[string]string)
		for ak, av := range rs.Attributes {
			vars[fmt.Sprintf("%s.%s", r.Id, ak)] = av
		}

		err = nil
		if len(errs) > 0 {
			err = &MultiError{Errors: errs}
		}

		return vars, err
	}

	return t.genericWalkFn(p.State, p.Diff, p.Vars, cb)
}

func (t *Terraform) planWalkFn(
	state *State, result *Plan) depgraph.WalkFunc {
	var l sync.Mutex

	// Initialize the result diff so we can write to it
	result.init()

	// Write our configuration out
	result.Config = t.config

	// Copy the variables
	result.Vars = make(map[string]string)
	for k, v := range t.variables {
		result.Vars[k] = v
	}

	cb := func(r *Resource) (map[string]string, error) {
		// Refresh the state so we're working with the latest resource info
		newState, err := r.Provider.Refresh(r.State)
		if err != nil {
			return nil, err
		}

		// Make sure the state is set to at the very least the empty state
		if newState == nil {
			newState = new(ResourceState)
		}

		// Get a diff from the newest state
		diff, err := r.Provider.Diff(newState, r.Config)
		if err != nil {
			return nil, err
		}

		l.Lock()
		if !diff.Empty() {
			result.Diff.Resources[r.Id] = diff
		}
		result.State.Resources[r.Id] = newState
		l.Unlock()

		// Determine the new state and update variables
		vars := make(map[string]string)
		rs := newState
		if !diff.Empty() {
			rs = r.State.MergeDiff(diff)
		}
		if rs != nil {
			for ak, av := range rs.Attributes {
				vars[fmt.Sprintf("%s.%s", r.Id, ak)] = av
			}
		}

		return vars, nil
	}

	return t.genericWalkFn(state, nil, t.variables, cb)
}

func (t *Terraform) genericWalkFn(
	state *State,
	diff *Diff,
	invars map[string]string,
	cb genericWalkFunc) depgraph.WalkFunc {
	var l sync.Mutex

	// Initialize the variables for application
	vars := make(map[string]string)
	for k, v := range invars {
		vars[fmt.Sprintf("var.%s", k)] = v
	}

	return func(n *depgraph.Noun) error {
		// If it is the root node, ignore
		if n.Meta == nil {
			return nil
		}

		switch n.Meta.(type) {
		case *config.ProviderConfig:
			// Ignore, we don't treat this any differently since we always
			// initialize the provider on first use and use a lock to make
			// sure we only do this once.
			return nil
		case *config.Resource:
			// Continue
		}

		r := n.Meta.(*config.Resource)
		p := t.mapping[r]
		if p == nil {
			panic(fmt.Sprintf("No provider for resource: %s", r.Id()))
		}

		// Initialize the provider if we haven't already
		if err := p.init(vars); err != nil {
			return err
		}

		// Get the resource state
		var rs *ResourceState
		if state != nil {
			rs = state.Resources[r.Id()]
		}

		// Get the resource diff
		var rd *ResourceDiff
		if diff != nil {
			rd = diff.Resources[r.Id()]
		}

		if len(vars) > 0 {
			if err := r.RawConfig.Interpolate(vars); err != nil {
				panic(fmt.Sprintf("Interpolate error: %s", err))
			}
		}

		// If we have no state, then create an empty state with the
		// type fulfilled at the least.
		if rs == nil {
			rs = new(ResourceState)
		}
		rs.Type = r.Type

		// Call the callack
		newVars, err := cb(&Resource{
			Id:       r.Id(),
			Config:   NewResourceConfig(r.RawConfig),
			Diff:     rd,
			Provider: p.Provider,
			State:    rs,
		})
		if err != nil {
			return err
		}

		if len(newVars) > 0 {
			// Acquire a lock since this function is called in parallel
			l.Lock()
			defer l.Unlock()

			// Update variables
			for k, v := range newVars {
				vars[k] = v
			}
		}

		return nil
	}
}

func (t *terraformProvider) init(vars map[string]string) (err error) {
	t.Once.Do(func() {
		var rc *ResourceConfig
		if t.Config != nil {
			if err := t.Config.RawConfig.Interpolate(vars); err != nil {
				panic(err)
			}

			rc = NewResourceConfig(t.Config.RawConfig)
		}

		err = t.Provider.Configure(rc)
	})

	return
}
<commit_msg>terraform :comments<commit_after>package terraform

import (
	"fmt"
	"sync"

	"github.com/hashicorp/terraform/config"
	"github.com/hashicorp/terraform/depgraph"
)

// Terraform is the primary structure that is used to interact with
// Terraform from code, and can perform operations such as returning
// all resources, a resource tree, a specific resource, etc.
type Terraform struct {
	config    *config.Config
	graph     *depgraph.Graph
	mapping   map[*config.Resource]*terraformProvider
	variables map[string]string
}

// terraformProvider contains internal state information about a resource
// provider for Terraform.
type terraformProvider struct {
	Provider ResourceProvider
	Config   *config.ProviderConfig

	sync.Once
}

// This is a function type used to implement a walker for the resource
// tree internally on the Terraform structure.
type genericWalkFunc func(*Resource) (map[string]string, error)

// Config is the configuration that must be given to instantiate
// a Terraform structure.
type Config struct {
	Config    *config.Config
	Providers map[string]ResourceProviderFactory
	Variables map[string]string

	computedPlaceholder string
}

// New creates a new Terraform structure, initializes resource providers
// for the given configuration, etc.
//
// Semantic checks of the entire configuration structure are done at this
// time, as well as richer checks such as verifying that the resource providers
// can be properly initialized, can be configured, etc.
func New(c *Config) (*Terraform, error) {
	var errs []error

	// Calculate the computed key placeholder
	c.computedPlaceholder = "tf_computed_placeholder"

	// Validate that all required variables have values
	if err := smcVariables(c); err != nil {
		errs = append(errs, err...)
	}

	// Match all the resources with a provider and initialize the providers
	mapping, err := smcProviders(c)
	if err != nil {
		errs = append(errs, err...)
	}

	// Validate all the configurations, once.
	tps := make(map[*terraformProvider]struct{})
	for _, tp := range mapping {
		if _, ok := tps[tp]; !ok {
			tps[tp] = struct{}{}
		}
	}
	for tp, _ := range tps {
		var rc *ResourceConfig
		if tp.Config != nil {
			rc = NewResourceConfig(tp.Config.RawConfig)
		}

		_, tpErrs := tp.Provider.Validate(rc)
		if len(tpErrs) > 0 {
			errs = append(errs, tpErrs...)
		}
	}

	// Build the resource graph
	graph := c.Config.Graph()
	if err := graph.Validate(); err != nil {
		errs = append(errs, fmt.Errorf(
			"Resource graph has an error: %s", err))
	}

	// If we accumulated any errors, then return them all
	if len(errs) > 0 {
		return nil, &MultiError{Errors: errs}
	}

	return &Terraform{
		config:    c.Config,
		graph:     graph,
		mapping:   mapping,
		variables: c.Variables,
	}, nil
}

func (t *Terraform) Apply(p *Plan) (*State, error) {
	result := new(State)
	err := t.graph.Walk(t.applyWalkFn(p, result))
	return result, err
}

func (t *Terraform) Plan(s *State) (*Plan, error) {
	result := new(Plan)
	err := t.graph.Walk(t.planWalkFn(s, result))
	if err != nil {
		return nil, err
	}

	return result, nil
}

func (t *Terraform) Refresh(*State) (*State, error) {
	return nil, nil
}

func (t *Terraform) applyWalkFn(
	p *Plan,
	result *State) depgraph.WalkFunc {
	var l sync.Mutex

	// Initialize the result
	result.init()

	cb := func(r *Resource) (map[string]string, error) {
		// Get the latest diff since there are no computed values anymore
		diff, err := r.Provider.Diff(r.State, r.Config)
		if err != nil {
			return nil, err
		}

		// TODO(mitchellh): we need to verify the diff doesn't change
		// anything and that the diff has no computed values (pre-computed)

		// With the completed diff, apply!
		rs, err := r.Provider.Apply(r.State, diff)
		if err != nil {
			return nil, err
		}

		// If no state was returned, then no variables were updated so
		// just return.
		if rs == nil {
			return nil, nil
		}

		var errs []error
		for ak, av := range rs.Attributes {
			// If the value is the unknown variable value, then it is an error.
			// In this case we record the error and remove it from the state
			if av == config.UnknownVariableValue {
				errs = append(errs, fmt.Errorf(
					"Attribute with unknown value: %s", ak))
				delete(rs.Attributes, ak)
			}
		}

		// Update the resulting diff
		l.Lock()
		result.Resources[r.Id] = rs
		l.Unlock()

		// Determine the new state and update variables
		vars := make(map[string]string)
		for ak, av := range rs.Attributes {
			vars[fmt.Sprintf("%s.%s", r.Id, ak)] = av
		}

		err = nil
		if len(errs) > 0 {
			err = &MultiError{Errors: errs}
		}

		return vars, err
	}

	return t.genericWalkFn(p.State, p.Diff, p.Vars, cb)
}

func (t *Terraform) planWalkFn(
	state *State, result *Plan) depgraph.WalkFunc {
	var l sync.Mutex

	// Initialize the result diff so we can write to it
	result.init()

	// Write our configuration out
	result.Config = t.config

	// Copy the variables
	result.Vars = make(map[string]string)
	for k, v := range t.variables {
		result.Vars[k] = v
	}

	cb := func(r *Resource) (map[string]string, error) {
		// Refresh the state so we're working with the latest resource info
		newState, err := r.Provider.Refresh(r.State)
		if err != nil {
			return nil, err
		}

		// Make sure the state is set to at the very least the empty state
		if newState == nil {
			newState = new(ResourceState)
		}

		// Set the type, the provider shouldn't modify this
		newState.Type = r.State.Type

		// Get a diff from the newest state
		diff, err := r.Provider.Diff(newState, r.Config)
		if err != nil {
			return nil, err
		}

		l.Lock()
		if !diff.Empty() {
			result.Diff.Resources[r.Id] = diff
		}
		result.State.Resources[r.Id] = newState
		l.Unlock()

		// Determine the new state and update variables
		vars := make(map[string]string)
		rs := newState
		if !diff.Empty() {
			rs = r.State.MergeDiff(diff)
		}
		if rs != nil {
			for ak, av := range rs.Attributes {
				vars[fmt.Sprintf("%s.%s", r.Id, ak)] = av
			}
		}

		return vars, nil
	}

	return t.genericWalkFn(state, nil, t.variables, cb)
}

func (t *Terraform) genericWalkFn(
	state *State,
	diff *Diff,
	invars map[string]string,
	cb genericWalkFunc) depgraph.WalkFunc {
	var l sync.Mutex

	// Initialize the variables for application
	vars := make(map[string]string)
	for k, v := range invars {
		vars[fmt.Sprintf("var.%s", k)] = v
	}

	return func(n *depgraph.Noun) error {
		// If it is the root node, ignore
		if n.Meta == nil {
			return nil
		}

		switch n.Meta.(type) {
		case *config.ProviderConfig:
			// Ignore, we don't treat this any differently since we always
			// initialize the provider on first use and use a lock to make
			// sure we only do this once.
			return nil
		case *config.Resource:
			// Continue
		}

		r := n.Meta.(*config.Resource)
		p := t.mapping[r]
		if p == nil {
			panic(fmt.Sprintf("No provider for resource: %s", r.Id()))
		}

		// Initialize the provider if we haven't already
		if err := p.init(vars); err != nil {
			return err
		}

		// Get the resource state
		var rs *ResourceState
		if state != nil {
			rs = state.Resources[r.Id()]
		}

		// Get the resource diff
		var rd *ResourceDiff
		if diff != nil {
			rd = diff.Resources[r.Id()]
		}

		if len(vars) > 0 {
			if err := r.RawConfig.Interpolate(vars); err != nil {
				panic(fmt.Sprintf("Interpolate error: %s", err))
			}
		}

		// If we have no state, then create an empty state with the
		// type fulfilled at the least.
		if rs == nil {
			rs = new(ResourceState)
		}
		rs.Type = r.Type

		// Call the callack
		newVars, err := cb(&Resource{
			Id:       r.Id(),
			Config:   NewResourceConfig(r.RawConfig),
			Diff:     rd,
			Provider: p.Provider,
			State:    rs,
		})
		if err != nil {
			return err
		}

		if len(newVars) > 0 {
			// Acquire a lock since this function is called in parallel
			l.Lock()
			defer l.Unlock()

			// Update variables
			for k, v := range newVars {
				vars[k] = v
			}
		}

		return nil
	}
}

func (t *terraformProvider) init(vars map[string]string) (err error) {
	t.Once.Do(func() {
		var rc *ResourceConfig
		if t.Config != nil {
			if err := t.Config.RawConfig.Interpolate(vars); err != nil {
				panic(err)
			}

			rc = NewResourceConfig(t.Config.RawConfig)
		}

		err = t.Provider.Configure(rc)
	})

	return
}
<|endoftext|>


<commit_before>package images // import "github.com/docker/docker/daemon/images"

import (
	"fmt"

	"github.com/containerd/containerd/platforms"
	"github.com/pkg/errors"

	"github.com/docker/distribution/reference"
	"github.com/docker/docker/errdefs"
	"github.com/docker/docker/image"
	specs "github.com/opencontainers/image-spec/specs-go/v1"
)

// ErrImageDoesNotExist is error returned when no image can be found for a reference.
type ErrImageDoesNotExist struct {
	ref reference.Reference
}

func (e ErrImageDoesNotExist) Error() string {
	ref := e.ref
	if named, ok := ref.(reference.Named); ok {
		ref = reference.TagNameOnly(named)
	}
	return fmt.Sprintf("No such image: %s", reference.FamiliarString(ref))
}

// NotFound implements the NotFound interface
func (e ErrImageDoesNotExist) NotFound() {}

// GetImage returns an image corresponding to the image referred to by refOrID.
func (i *ImageService) GetImage(refOrID string, platform *specs.Platform) (retImg *image.Image, retErr error) {
	defer func() {
		if retErr != nil || retImg == nil || platform == nil {
			return
		}

		imgPlat := specs.Platform{
			OS:           retImg.OS,
			Architecture: retImg.Architecture,
			Variant:      retImg.Variant,
		}
		p := *platform
		// Note that `platforms.Only` will fuzzy match this for us
		// For example: an armv6 image will run just fine an an armv7 CPU, without emulation or anything.
		if !platforms.Only(p).Match(imgPlat) {
			// This allows us to tell clients that we don't have the image they asked for
			// Where this gets hairy is the image store does not currently support multi-arch images, e.g.:
			//   An image `foo` may have a multi-arch manifest, but the image store only fetches the image for a specific platform
			//   The image store does not store the manifest list and image tags are assigned to architecture specific images.
			//   So we can have a `foo` image that is amd64 but the user requested armv7. If the user looks at the list of images.
			//   This may be confusing.
			//   The alternative to this is to return a errdefs.Conflict error with a helpful message, but clients will not be
			//   able to automatically tell what causes the conflict.
			retErr = errdefs.NotFound(errors.Errorf("image with reference %s was found but does not match the specified platform: wanted %s, actual: %s", refOrID, platforms.Format(p), platforms.Format(imgPlat)))
			return
		}
	}()
	ref, err := reference.ParseAnyReference(refOrID)
	if err != nil {
		return nil, errdefs.InvalidParameter(err)
	}
	namedRef, ok := ref.(reference.Named)
	if !ok {
		digested, ok := ref.(reference.Digested)
		if !ok {
			return nil, ErrImageDoesNotExist{ref}
		}
		id := image.IDFromDigest(digested.Digest())
		if img, err := i.imageStore.Get(id); err == nil {
			return img, nil
		}
		return nil, ErrImageDoesNotExist{ref}
	}

	if digest, err := i.referenceStore.Get(namedRef); err == nil {
		// Search the image stores to get the operating system, defaulting to host OS.
		id := image.IDFromDigest(digest)
		if img, err := i.imageStore.Get(id); err == nil {
			return img, nil
		}
	}

	// Search based on ID
	if id, err := i.imageStore.Search(refOrID); err == nil {
		img, err := i.imageStore.Get(id)
		if err != nil {
			return nil, ErrImageDoesNotExist{ref}
		}
		return img, nil
	}

	return nil, ErrImageDoesNotExist{ref}
}
<commit_msg>Fallback to  manifest list when no platform match<commit_after>package images // import "github.com/docker/docker/daemon/images"

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"

	"github.com/containerd/containerd/content"
	c8derrdefs "github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/leases"
	"github.com/containerd/containerd/platforms"
	"github.com/docker/distribution/reference"
	"github.com/docker/docker/errdefs"
	"github.com/docker/docker/image"
	digest "github.com/opencontainers/go-digest"
	specs "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// ErrImageDoesNotExist is error returned when no image can be found for a reference.
type ErrImageDoesNotExist struct {
	ref reference.Reference
}

func (e ErrImageDoesNotExist) Error() string {
	ref := e.ref
	if named, ok := ref.(reference.Named); ok {
		ref = reference.TagNameOnly(named)
	}
	return fmt.Sprintf("No such image: %s", reference.FamiliarString(ref))
}

// NotFound implements the NotFound interface
func (e ErrImageDoesNotExist) NotFound() {}

type manifestList struct {
	Manifests []specs.Descriptor `json:"manifests"`
}

type manifest struct {
	Config specs.Descriptor `json:"config"`
}

func (i *ImageService) manifestMatchesPlatform(img *image.Image, platform specs.Platform) bool {
	ctx := context.TODO()
	logger := logrus.WithField("image", img.ID).WithField("desiredPlatform", platforms.Format(platform))

	ls, leaseErr := i.leases.ListResources(context.TODO(), leases.Lease{ID: imageKey(img.ID().Digest())})
	if leaseErr != nil {
		logger.WithError(leaseErr).Error("Error looking up image leases")
		return false
	}

	comparer := platforms.Only(platform)

	var (
		ml manifestList
		m  manifest
	)

	makeRdr := func(ra content.ReaderAt) io.Reader {
		return io.LimitReader(io.NewSectionReader(ra, 0, ra.Size()), 1e6)
	}

	for _, r := range ls {
		logger := logger.WithField("resourceID", r.ID).WithField("resourceType", r.Type)
		logger.Debug("Checking lease resource for platform match")
		if r.Type != "content" {
			continue
		}

		ra, err := i.content.ReaderAt(ctx, specs.Descriptor{Digest: digest.Digest(r.ID)})
		if err != nil {
			if c8derrdefs.IsNotFound(err) {
				continue
			}
			logger.WithError(err).Error("Error looking up referenced manifest list for image")
			continue
		}

		data, err := ioutil.ReadAll(makeRdr(ra))
		ra.Close()

		if err != nil {
			logger.WithError(err).Error("Error reading manifest list for image")
			continue
		}

		ml.Manifests = nil

		if err := json.Unmarshal(data, &ml); err != nil {
			logger.WithError(err).Error("Error unmarshalling content")
			continue
		}

		for _, md := range ml.Manifests {
			switch md.MediaType {
			case specs.MediaTypeImageManifest, images.MediaTypeDockerSchema2Manifest:
			default:
				continue
			}

			p := specs.Platform{
				Architecture: md.Platform.Architecture,
				OS:           md.Platform.OS,
				Variant:      md.Platform.Variant,
			}
			if !comparer.Match(p) {
				logger.WithField("otherPlatform", platforms.Format(p)).Debug("Manifest is not a match")
				continue
			}

			// Here we have a platform match for the referenced manifest, let's make sure the manifest is actually for the image config we are using.

			ra, err := i.content.ReaderAt(ctx, specs.Descriptor{Digest: md.Digest})
			if err != nil {
				logger.WithField("otherDigest", md.Digest).WithError(err).Error("Could not get reader for manifest")
				continue
			}

			data, err := ioutil.ReadAll(makeRdr(ra))
			ra.Close()
			if err != nil {
				logger.WithError(err).Error("Error reading manifest for image")
				continue
			}

			if err := json.Unmarshal(data, &m); err != nil {
				logger.WithError(err).Error("Error desserializing manifest")
				continue
			}

			if m.Config.Digest == img.ID().Digest() {
				logger.WithField("manifestDigest", md.Digest).Debug("Found matching manifest for image")
				return true
			}

			logger.WithField("otherDigest", md.Digest).Debug("Skipping non-matching manifest")
		}
	}

	return false
}

// GetImage returns an image corresponding to the image referred to by refOrID.
func (i *ImageService) GetImage(refOrID string, platform *specs.Platform) (retImg *image.Image, retErr error) {
	defer func() {
		if retErr != nil || retImg == nil || platform == nil {
			return
		}

		imgPlat := specs.Platform{
			OS:           retImg.OS,
			Architecture: retImg.Architecture,
			Variant:      retImg.Variant,
		}
		p := *platform
		// Note that `platforms.Only` will fuzzy match this for us
		// For example: an armv6 image will run just fine an an armv7 CPU, without emulation or anything.
		if !platforms.Only(p).Match(imgPlat) {
			// Sometimes image variant is not populated due to legacy builders
			// We still should support falling back here.
			if imgPlat.OS == platform.OS && imgPlat.Architecture == platform.Architecture && imgPlat.Variant == "" {
				logrus.WithField("image", refOrID).WithField("platform", platforms.Format(p)).Debug("Image platform cpu variant is not populated, but otherwise it matches what was requested")
				return
			}

			// In some cases the image config can actually be wrong (e.g. classic `docker build` may not handle `--platform` correctly)
			// So we'll look up the manifest list that coresponds to this imaage to check if at least the manifest list says it is the correct image.
			if i.manifestMatchesPlatform(retImg, p) {
				return
			}

			// This allows us to tell clients that we don't have the image they asked for
			// Where this gets hairy is the image store does not currently support multi-arch images, e.g.:
			//   An image `foo` may have a multi-arch manifest, but the image store only fetches the image for a specific platform
			//   The image store does not store the manifest list and image tags are assigned to architecture specific images.
			//   So we can have a `foo` image that is amd64 but the user requested armv7. If the user looks at the list of images.
			//   This may be confusing.
			//   The alternative to this is to return a errdefs.Conflict error with a helpful message, but clients will not be
			//   able to automatically tell what causes the conflict.
			retErr = errdefs.NotFound(errors.Errorf("image with reference %s was found but does not match the specified platform: wanted %s, actual: %s", refOrID, platforms.Format(p), platforms.Format(imgPlat)))
			return
		}
	}()
	ref, err := reference.ParseAnyReference(refOrID)
	if err != nil {
		return nil, errdefs.InvalidParameter(err)
	}
	namedRef, ok := ref.(reference.Named)
	if !ok {
		digested, ok := ref.(reference.Digested)
		if !ok {
			return nil, ErrImageDoesNotExist{ref}
		}
		id := image.IDFromDigest(digested.Digest())
		if img, err := i.imageStore.Get(id); err == nil {
			return img, nil
		}
		return nil, ErrImageDoesNotExist{ref}
	}

	if digest, err := i.referenceStore.Get(namedRef); err == nil {
		// Search the image stores to get the operating system, defaulting to host OS.
		id := image.IDFromDigest(digest)
		if img, err := i.imageStore.Get(id); err == nil {
			return img, nil
		}
	}

	// Search based on ID
	if id, err := i.imageStore.Search(refOrID); err == nil {
		img, err := i.imageStore.Get(id)
		if err != nil {
			return nil, ErrImageDoesNotExist{ref}
		}
		return img, nil
	}

	return nil, ErrImageDoesNotExist{ref}
}
<|endoftext|>


<commit_before>// Copyright 2015 Matthew Collins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package render

import (
	"math"
	"sort"
	"strings"

	"github.com/thinkofdeath/steven/platform"
	"github.com/thinkofdeath/steven/platform/gl"
	"github.com/thinkofdeath/steven/type/direction"
	"github.com/thinkofdeath/steven/type/vmath"
)

var (
	chunkProgram  gl.Program
	shaderChunk   *chunkShader
	chunkProgramT gl.Program
	shaderChunkT  *chunkShader

	lastWidth, lastHeight int = -1, -1
	perspectiveMatrix         = vmath.NewMatrix4()
	cameraMatrix              = vmath.NewMatrix4()

	syncChan = make(chan func(), 500)

	glTexture    gl.Texture
	textureDepth int
)

// Start starts the renderer
func Start(debug bool) {
	if debug {
		gl.Enable(gl.DebugOutput)
		gl.DebugLog()
	}

	gl.ClearColor(122.0/255.0, 165.0/255.0, 247.0/255.0, 1.0)
	gl.Enable(gl.DepthTest)
	gl.Enable(gl.CullFaceFlag)
	gl.CullFace(gl.Back)
	gl.FrontFace(gl.ClockWise)

	chunkProgram = CreateProgram(vertex, fragment)
	shaderChunk = &chunkShader{}
	InitStruct(shaderChunk, chunkProgram)

	chunkProgramT = CreateProgram(vertex, strings.Replace(fragment, "#version 150", "#version 150\n#define alpha", 1))
	shaderChunkT = &chunkShader{}
	InitStruct(shaderChunkT, chunkProgramT)

	textureLock.Lock()
	glTexture = gl.CreateTexture()
	glTexture.Bind(gl.Texture2DArray)
	textureDepth = len(textures)
	glTexture.Image3D(0, AtlasSize, AtlasSize, len(textures), gl.RGBA, gl.UnsignedByte, make([]byte, 1))
	glTexture.Parameter(gl.TextureMagFilter, gl.Nearest)
	glTexture.Parameter(gl.TextureMinFilter, gl.Linear)
	glTexture.Parameter(gl.TextureWrapS, gl.ClampToEdge)
	glTexture.Parameter(gl.TextureWrapT, gl.ClampToEdge)
	for i, tex := range textures {
		glTexture.SubImage3D(0, 0, 0, i, AtlasSize, AtlasSize, 1, gl.RGBA, gl.UnsignedByte, tex.Buffer)
	}
	textureLock.Unlock()

	initUI()

	gl.BlendFunc(gl.SrcAlpha, gl.OneMinusSrcAlpha)
}

var (
	textureIds    []int
	frameID       uint
	nearestBuffer *ChunkBuffer
	viewVector    vmath.Vector3
)

// Draw draws a single frame
func Draw(delta float64) {
	tickAnimatedTextures(delta)
	frameID++
sync:
	for {
		select {
		case f := <-syncChan:
			f()
		default:
			break sync
		}
	}

	width, height := platform.Size()
	// Only update the viewport if the window was resized
	if lastHeight != height || lastWidth != width {
		lastWidth = width
		lastHeight = height

		perspectiveMatrix.Identity()
		perspectiveMatrix.Perspective(
			(math.Pi/180)*90,
			float32(width)/float32(height),
			0.1,
			10000.0,
		)
		gl.Viewport(0, 0, width, height)
	}

	// Textures
	textureLock.RLock()
	if textureDepth != len(textures) {
		glTexture.Bind(gl.Texture2DArray)
		textureDepth = len(textures)
		glTexture.Image3D(0, AtlasSize, AtlasSize, len(textures), gl.RGBA, gl.UnsignedByte, make([]byte, 1))
		for i := range textureDirty {
			textureDirty[i] = true
		}
	}
	for i, tex := range textures {
		if textureDirty[i] {
			textureDirty[i] = true
			glTexture.SubImage3D(0, 0, 0, i, AtlasSize, AtlasSize, 1, gl.RGBA, gl.UnsignedByte, tex.Buffer)
		}
	}
	textureLock.RUnlock()

	glTexture.Bind(gl.Texture2DArray)
	gl.ActiveTexture(0)

	gl.Clear(gl.ColorBufferBit | gl.DepthBufferBit)

	chunkProgram.Use()

	cameraMatrix.Identity()
	// +1.62 for the players height.
	// TODO(Think) Change this?
	cameraMatrix.Translate(float32(Camera.X), float32(Camera.Y), float32(-Camera.Z))
	cameraMatrix.RotateY(float32(Camera.Yaw))
	cameraMatrix.RotateX(float32(Camera.Pitch))
	cameraMatrix.Scale(-1.0, 1.0, 1.0)

	shaderChunk.PerspectiveMatrix.Matrix4(perspectiveMatrix)
	shaderChunk.CameraMatrix.Matrix4(cameraMatrix)
	shaderChunk.Texture.Int(0)

	chunkPos := position{
		X: int(Camera.X) >> 4,
		Y: int(Camera.Y) >> 4,
		Z: int(Camera.Z) >> 4,
	}
	nearestBuffer = buffers[chunkPos]

	viewVector.X = math.Cos(Camera.Yaw-math.Pi/2) * -math.Cos(Camera.Pitch)
	viewVector.Z = -math.Sin(Camera.Yaw-math.Pi/2) * -math.Cos(Camera.Pitch)
	viewVector.Y = -math.Sin(Camera.Pitch)

	for _, dir := range direction.Values {
		validDirs[dir] = viewVector.Dot(dir.AsVector()) > -0.8
	}

	renderOrder = renderOrder[:0]
	renderBuffer(nearestBuffer, chunkPos, direction.Invalid)

	chunkProgramT.Use()
	shaderChunkT.PerspectiveMatrix.Matrix4(perspectiveMatrix)
	shaderChunkT.CameraMatrix.Matrix4(cameraMatrix)
	shaderChunkT.Texture.Int(0)

	gl.Enable(gl.Blend)
	for i := range renderOrder {
		chunk := renderOrder[len(renderOrder)-1-i]
		if chunk != nil && chunk.countT > 0 {
			shaderChunkT.Offset.Float3(float32(chunk.X), float32(chunk.Y), float32(chunk.Z))

			chunk.arrayT.Bind()
			chunk.bufferT.Bind(gl.ArrayBuffer)
			data := chunk.transBuffer
			offset := 0
			sort.Sort(chunk.transInfo)
			for _, i := range chunk.transInfo {
				offset += copy(data[offset:], chunk.transData[i.Offset:i.Offset+i.Count])
			}
			chunk.bufferT.SubData(0, data)
			gl.DrawArrays(gl.Triangles, 0, chunk.countT)
		}
	}
	gl.Disable(gl.Blend)

	drawUI()
}

var (
	renderOrder []*ChunkBuffer
	validDirs   = make([]bool, len(direction.Values))
)

type renderRequest struct {
	chunk *ChunkBuffer
	pos   position
	from  direction.Type
	dist  int
}

const (
	renderQueueSize = 5000
)

var rQueue renderQueue

func renderBuffer(chunk *ChunkBuffer, pos position, from direction.Type) {
	rQueue.Append(renderRequest{chunk, pos, from, 1})
itQueue:
	for !rQueue.Empty() {
		req := rQueue.Take()
		chunk, pos, from = req.chunk, req.pos, req.from
		v := vmath.Vector3{
			float64((pos.X<<4)+8) - Camera.X,
			float64((pos.Y<<4)+8) - Camera.Y,
			float64((pos.Z<<4)+8) - Camera.Z,
		}
		if (v.LengthSquared() > 40*40 && v.Dot(viewVector) < 0) || req.dist > 20 {
			continue itQueue
		}
		if chunk == nil || chunk.renderedOn == frameID {
			continue itQueue
		}
		chunk.renderedOn = frameID
		renderOrder = append(renderOrder, chunk)

		if chunk.count > 0 {
			shaderChunk.Offset.Float3(float32(chunk.X), float32(chunk.Y), float32(chunk.Z))

			chunk.array.Bind()
			gl.DrawArrays(gl.Triangles, 0, chunk.count)
		}

		for _, dir := range direction.Values {
			if dir != from && (from == direction.Invalid || (chunk.IsVisible(from, dir) && validDirs[dir])) {
				ox, oy, oz := dir.Offset()
				pos := position{pos.X + ox, pos.Y + oy, pos.Z + oz}
				rQueue.Append(renderRequest{chunk.neighborChunks[dir], pos, dir.Opposite(), req.dist + 1})
			}
		}
	}
}

// Sync runs the passed function on the next frame on the same goroutine
// as the renderer.
func Sync(f func()) {
	syncChan <- f
}
<commit_msg>render: fix a crash with 3D texture allocation by allocating the right size buffer<commit_after>// Copyright 2015 Matthew Collins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package render

import (
	"math"
	"sort"
	"strings"

	"github.com/thinkofdeath/steven/platform"
	"github.com/thinkofdeath/steven/platform/gl"
	"github.com/thinkofdeath/steven/type/direction"
	"github.com/thinkofdeath/steven/type/vmath"
)

var (
	chunkProgram  gl.Program
	shaderChunk   *chunkShader
	chunkProgramT gl.Program
	shaderChunkT  *chunkShader

	lastWidth, lastHeight int = -1, -1
	perspectiveMatrix         = vmath.NewMatrix4()
	cameraMatrix              = vmath.NewMatrix4()

	syncChan = make(chan func(), 500)

	glTexture    gl.Texture
	textureDepth int
)

// Start starts the renderer
func Start(debug bool) {
	if debug {
		gl.Enable(gl.DebugOutput)
		gl.DebugLog()
	}

	gl.ClearColor(122.0/255.0, 165.0/255.0, 247.0/255.0, 1.0)
	gl.Enable(gl.DepthTest)
	gl.Enable(gl.CullFaceFlag)
	gl.CullFace(gl.Back)
	gl.FrontFace(gl.ClockWise)

	chunkProgram = CreateProgram(vertex, fragment)
	shaderChunk = &chunkShader{}
	InitStruct(shaderChunk, chunkProgram)

	chunkProgramT = CreateProgram(vertex, strings.Replace(fragment, "#version 150", "#version 150\n#define alpha", 1))
	shaderChunkT = &chunkShader{}
	InitStruct(shaderChunkT, chunkProgramT)

	textureLock.Lock()
	glTexture = gl.CreateTexture()
	glTexture.Bind(gl.Texture2DArray)
	textureDepth = len(textures)
	glTexture.Image3D(0, AtlasSize, AtlasSize, len(textures), gl.RGBA, gl.UnsignedByte, make([]byte, AtlasSize*AtlasSize*len(textures)*4))
	glTexture.Parameter(gl.TextureMagFilter, gl.Nearest)
	glTexture.Parameter(gl.TextureMinFilter, gl.Linear)
	glTexture.Parameter(gl.TextureWrapS, gl.ClampToEdge)
	glTexture.Parameter(gl.TextureWrapT, gl.ClampToEdge)
	for i, tex := range textures {
		glTexture.SubImage3D(0, 0, 0, i, AtlasSize, AtlasSize, 1, gl.RGBA, gl.UnsignedByte, tex.Buffer)
	}
	textureLock.Unlock()

	initUI()

	gl.BlendFunc(gl.SrcAlpha, gl.OneMinusSrcAlpha)
}

var (
	textureIds    []int
	frameID       uint
	nearestBuffer *ChunkBuffer
	viewVector    vmath.Vector3
)

// Draw draws a single frame
func Draw(delta float64) {
	tickAnimatedTextures(delta)
	frameID++
sync:
	for {
		select {
		case f := <-syncChan:
			f()
		default:
			break sync
		}
	}

	width, height := platform.Size()
	// Only update the viewport if the window was resized
	if lastHeight != height || lastWidth != width {
		lastWidth = width
		lastHeight = height

		perspectiveMatrix.Identity()
		perspectiveMatrix.Perspective(
			(math.Pi/180)*90,
			float32(width)/float32(height),
			0.1,
			10000.0,
		)
		gl.Viewport(0, 0, width, height)
	}

	// Textures
	textureLock.RLock()
	if textureDepth != len(textures) {
		glTexture.Bind(gl.Texture2DArray)
		textureDepth = len(textures)
		glTexture.Image3D(0, AtlasSize, AtlasSize, len(textures), gl.RGBA, gl.UnsignedByte, make([]byte, AtlasSize*AtlasSize*len(textures)*4))
		for i := range textureDirty {
			textureDirty[i] = true
		}
	}
	for i, tex := range textures {
		if textureDirty[i] {
			textureDirty[i] = true
			glTexture.SubImage3D(0, 0, 0, i, AtlasSize, AtlasSize, 1, gl.RGBA, gl.UnsignedByte, tex.Buffer)
		}
	}
	textureLock.RUnlock()

	glTexture.Bind(gl.Texture2DArray)
	gl.ActiveTexture(0)

	gl.Clear(gl.ColorBufferBit | gl.DepthBufferBit)

	chunkProgram.Use()

	cameraMatrix.Identity()
	// +1.62 for the players height.
	// TODO(Think) Change this?
	cameraMatrix.Translate(float32(Camera.X), float32(Camera.Y), float32(-Camera.Z))
	cameraMatrix.RotateY(float32(Camera.Yaw))
	cameraMatrix.RotateX(float32(Camera.Pitch))
	cameraMatrix.Scale(-1.0, 1.0, 1.0)

	shaderChunk.PerspectiveMatrix.Matrix4(perspectiveMatrix)
	shaderChunk.CameraMatrix.Matrix4(cameraMatrix)
	shaderChunk.Texture.Int(0)

	chunkPos := position{
		X: int(Camera.X) >> 4,
		Y: int(Camera.Y) >> 4,
		Z: int(Camera.Z) >> 4,
	}
	nearestBuffer = buffers[chunkPos]

	viewVector.X = math.Cos(Camera.Yaw-math.Pi/2) * -math.Cos(Camera.Pitch)
	viewVector.Z = -math.Sin(Camera.Yaw-math.Pi/2) * -math.Cos(Camera.Pitch)
	viewVector.Y = -math.Sin(Camera.Pitch)

	for _, dir := range direction.Values {
		validDirs[dir] = viewVector.Dot(dir.AsVector()) > -0.8
	}

	renderOrder = renderOrder[:0]
	renderBuffer(nearestBuffer, chunkPos, direction.Invalid)

	chunkProgramT.Use()
	shaderChunkT.PerspectiveMatrix.Matrix4(perspectiveMatrix)
	shaderChunkT.CameraMatrix.Matrix4(cameraMatrix)
	shaderChunkT.Texture.Int(0)

	gl.Enable(gl.Blend)
	for i := range renderOrder {
		chunk := renderOrder[len(renderOrder)-1-i]
		if chunk != nil && chunk.countT > 0 {
			shaderChunkT.Offset.Float3(float32(chunk.X), float32(chunk.Y), float32(chunk.Z))

			chunk.arrayT.Bind()
			chunk.bufferT.Bind(gl.ArrayBuffer)
			data := chunk.transBuffer
			offset := 0
			sort.Sort(chunk.transInfo)
			for _, i := range chunk.transInfo {
				offset += copy(data[offset:], chunk.transData[i.Offset:i.Offset+i.Count])
			}
			chunk.bufferT.SubData(0, data)
			gl.DrawArrays(gl.Triangles, 0, chunk.countT)
		}
	}
	gl.Disable(gl.Blend)

	drawUI()
}

var (
	renderOrder []*ChunkBuffer
	validDirs   = make([]bool, len(direction.Values))
)

type renderRequest struct {
	chunk *ChunkBuffer
	pos   position
	from  direction.Type
	dist  int
}

const (
	renderQueueSize = 5000
)

var rQueue renderQueue

func renderBuffer(chunk *ChunkBuffer, pos position, from direction.Type) {
	rQueue.Append(renderRequest{chunk, pos, from, 1})
itQueue:
	for !rQueue.Empty() {
		req := rQueue.Take()
		chunk, pos, from = req.chunk, req.pos, req.from
		v := vmath.Vector3{
			float64((pos.X<<4)+8) - Camera.X,
			float64((pos.Y<<4)+8) - Camera.Y,
			float64((pos.Z<<4)+8) - Camera.Z,
		}
		if (v.LengthSquared() > 40*40 && v.Dot(viewVector) < 0) || req.dist > 20 {
			continue itQueue
		}
		if chunk == nil || chunk.renderedOn == frameID {
			continue itQueue
		}
		chunk.renderedOn = frameID
		renderOrder = append(renderOrder, chunk)

		if chunk.count > 0 {
			shaderChunk.Offset.Float3(float32(chunk.X), float32(chunk.Y), float32(chunk.Z))

			chunk.array.Bind()
			gl.DrawArrays(gl.Triangles, 0, chunk.count)
		}

		for _, dir := range direction.Values {
			if dir != from && (from == direction.Invalid || (chunk.IsVisible(from, dir) && validDirs[dir])) {
				ox, oy, oz := dir.Offset()
				pos := position{pos.X + ox, pos.Y + oy, pos.Z + oz}
				rQueue.Append(renderRequest{chunk.neighborChunks[dir], pos, dir.Opposite(), req.dist + 1})
			}
		}
	}
}

// Sync runs the passed function on the next frame on the same goroutine
// as the renderer.
func Sync(f func()) {
	syncChan <- f
}
<|endoftext|>


<commit_before>package shh

import (
	"bytes"
	"io/ioutil"
	"time"

	"github.com/heroku/slog"
)

const (
	CONNTRACK_DATA = "/proc/sys/net/netfilter/nf_conntrack_count"
)

type Conntrack struct {
	measurements chan<- Measurement
}

func NewConntrackPoller(measurements chan<- Measurement) Conntrack {
	return Conntrack{measurements: measurements}
}

func (poller Conntrack) Poll(tick time.Time) {
	ctx := slog.Context{"poller": poller.Name(), "fn": "Poll", "tick": tick}

	count, err := ioutil.ReadFile(CONNTRACK_DATA)
	if err != nil {
		LogError(ctx, err, "reading"+CONNTRACK_DATA)
	}

	poller.measurements <- GaugeMeasurement{tick, poller.Name(), []string{"count"}, Atouint64(string(bytes.TrimSpace(count))), Connections}
}

func (poller Conntrack) Name() string {
	return "conntrack"
}

func (poller Conntrack) Exit() {}
<commit_msg>if we can't read the file, don't try to submit a measurement<commit_after>package shh

import (
	"bytes"
	"io/ioutil"
	"time"

	"github.com/heroku/slog"
)

const (
	CONNTRACK_DATA = "/proc/sys/net/netfilter/nf_conntrack_count"
)

type Conntrack struct {
	measurements chan<- Measurement
}

func NewConntrackPoller(measurements chan<- Measurement) Conntrack {
	return Conntrack{measurements: measurements}
}

func (poller Conntrack) Poll(tick time.Time) {
	ctx := slog.Context{"poller": poller.Name(), "fn": "Poll", "tick": tick}

	count, err := ioutil.ReadFile(CONNTRACK_DATA)
	if err != nil {
		LogError(ctx, err, "reading"+CONNTRACK_DATA)
		return
	}

	poller.measurements <- GaugeMeasurement{tick, poller.Name(), []string{"count"}, Atouint64(string(bytes.TrimSpace(count))), Connections}
}

func (poller Conntrack) Name() string {
	return "conntrack"
}

func (poller Conntrack) Exit() {}
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package messenger

import (
	"flag"
	"fmt"
	"reflect"
	"time"

	"github.com/gogo/protobuf/proto"
	log "github.com/golang/glog"
	mesos "github.com/mesos/mesos-go/mesosproto"
	"github.com/mesos/mesos-go/upid"
	"golang.org/x/net/context"
)

const (
	defaultQueueSize = 1024
	preparePeriod    = time.Second * 1
)

var (
	sendRoutines   int
	encodeRoutines int
	decodeRoutines int
)

func init() {
	flag.IntVar(&sendRoutines, "send-routines", 1, "Number of network sending routines")
	flag.IntVar(&encodeRoutines, "encode-routines", 1, "Number of encoding routines")
	flag.IntVar(&decodeRoutines, "decode-routines", 1, "Number of decoding routines")
}

// MessageHandler is the callback of the message. When the callback
// is invoked, the sender's upid and the message is passed to the callback.
type MessageHandler func(from *upid.UPID, pbMsg proto.Message)

// Messenger defines the interfaces that should be implemented.
type Messenger interface {
	Install(handler MessageHandler, msg proto.Message) error
	Send(ctx context.Context, upid *upid.UPID, msg proto.Message) error
	Route(ctx context.Context, from *upid.UPID, msg proto.Message) error
	Start() error
	Stop() error
	UPID() *upid.UPID
}

// MesosMessenger is an implementation of the Messenger interface.
type MesosMessenger struct {
	upid              *upid.UPID
	encodingQueue     chan *Message
	sendingQueue      chan *Message
	installedMessages map[string]reflect.Type
	installedHandlers map[string]MessageHandler
	stop              chan struct{}
	tr                Transporter
}

// NewMesosMessenger creates a new mesos messenger.
func NewHttp(upid *upid.UPID) *MesosMessenger {
	return New(upid, NewHTTPTransporter(upid))
}

func New(upid *upid.UPID, t Transporter) *MesosMessenger {
	return &MesosMessenger{
		upid:              upid,
		encodingQueue:     make(chan *Message, defaultQueueSize),
		sendingQueue:      make(chan *Message, defaultQueueSize),
		installedMessages: make(map[string]reflect.Type),
		installedHandlers: make(map[string]MessageHandler),
		tr:                t,
	}
}

/// Install installs the handler with the given message.
func (m *MesosMessenger) Install(handler MessageHandler, msg proto.Message) error {
	// Check if the message is a pointer.
	mtype := reflect.TypeOf(msg)
	if mtype.Kind() != reflect.Ptr {
		return fmt.Errorf("Message %v is not a Ptr type")
	}

	// Check if the message is already installed.
	name := getMessageName(msg)
	if _, ok := m.installedMessages[name]; ok {
		return fmt.Errorf("Message %v is already installed", name)
	}
	m.installedMessages[name] = mtype.Elem()
	m.installedHandlers[name] = handler
	m.tr.Install(name)
	return nil
}

// Send puts a message into the outgoing queue, waiting to be sent.
// With buffered channels, this will not block under moderate throughput.
// When an error is generated, the error can be communicated by placing
// a message on the incoming queue to be handled upstream.
func (m *MesosMessenger) Send(ctx context.Context, upid *upid.UPID, msg proto.Message) error {
	if upid.Equal(m.upid) {
		return fmt.Errorf("Send the message to self")
	}
	name := getMessageName(msg)
	log.V(2).Infof("Sending message %v to %v\n", name, upid)
	select {
	case <-ctx.Done():
		return ctx.Err()
	case m.encodingQueue <- &Message{upid, name, msg, nil}:
		return nil
	}
}

// Route puts a message either in the incoming or outgoing queue.
// This method is useful for:
// 1) routing internal error to callback handlers
// 2) testing components without starting remote servers.
func (m *MesosMessenger) Route(ctx context.Context, upid *upid.UPID, msg proto.Message) error {
	// if destination is not self, send to outbound.
	if !upid.Equal(m.upid) {
		return m.Send(ctx, upid, msg)
	}

	data, err := proto.Marshal(msg)
	if err != nil {
		return err
	}
	name := getMessageName(msg)
	return m.tr.Inject(ctx, &Message{upid, name, msg, data})
}

// Start starts the messenger.
func (m *MesosMessenger) Start() error {
	if err := m.tr.Listen(); err != nil {
		log.Errorf("Failed to start messenger: %v\n", err)
		return err
	}
	m.upid = m.tr.UPID()

	m.stop = make(chan struct{})
	errChan := make(chan error)
	go func() {
		if err := m.tr.Start(); err != nil {
			errChan <- err
		}
	}()

	select {
	case err := <-errChan:
		return err
	case <-time.After(preparePeriod):
	}
	for i := 0; i < sendRoutines; i++ {
		go m.sendLoop()
	}
	for i := 0; i < encodeRoutines; i++ {
		go m.encodeLoop()
	}
	for i := 0; i < decodeRoutines; i++ {
		go m.decodeLoop()
	}
	return nil
}

// Stop stops the messenger and clean up all the goroutines.
func (m *MesosMessenger) Stop() error {
	if err := m.tr.Stop(); err != nil {
		log.Errorf("Failed to stop the transporter: %v\n", err)
		return err
	}
	close(m.stop)
	return nil
}

// UPID returns the upid of the messenger.
func (m *MesosMessenger) UPID() *upid.UPID {
	return m.upid
}

func (m *MesosMessenger) encodeLoop() {
	for {
		select {
		case <-m.stop:
			return
		case msg := <-m.encodingQueue:
			e := func() error {
				//TODO(jdef) implement timeout for context
				ctx, cancel := context.WithCancel(context.TODO())
				defer cancel()

				b, err := proto.Marshal(msg.ProtoMessage)
				if err != nil {
					return err
				}
				msg.Bytes = b
				select {
				case <-ctx.Done():
					return ctx.Err()
				case m.sendingQueue <- msg:
					return nil
				}
			}()
			if e != nil {
				m.reportError(fmt.Errorf("Failed to enqueue message %v: %v", msg, e))
			}
		}
	}
}

func (m *MesosMessenger) reportError(err error) {
	log.V(2).Info(err)
	//TODO(jdef) implement timeout for context
	ctx, cancel := context.WithCancel(context.TODO())
	defer cancel()

	c := make(chan error, 1)
	go func() { c <- m.Route(ctx, m.UPID(), &mesos.FrameworkErrorMessage{Message: proto.String(err.Error())}) }()
	select {
	case <-ctx.Done():
		<-c // wait for Route to return
	case e := <-c:
		log.Errorf("failed to report error %v due to: %v", err, e)
	}
}

func (m *MesosMessenger) sendLoop() {
	for {
		select {
		case <-m.stop:
			return
		case msg := <-m.sendingQueue:
			e := func() error {
				//TODO(jdef) implement timeout for context
				ctx, cancel := context.WithCancel(context.TODO())
				defer cancel()

				c := make(chan error, 1)
				go func() { c <- m.tr.Send(ctx, msg) }()

				select {
				case <-ctx.Done():
					// Transport layer must use the context to detect cancelled requests.
					<-c // wait for Send to return
					return ctx.Err()
				case err := <-c:
					return err
				}
			}()
			if e != nil {
				m.reportError(fmt.Errorf("Failed to send message %v: %v", msg.Name, e))
			}
		}
	}
}

// Since HTTPTransporter.Recv() is already buffered, so we don't need a 'recvLoop' here.
func (m *MesosMessenger) decodeLoop() {
	for {
		select {
		case <-m.stop:
			return
		default:
		}
		msg := m.tr.Recv()
		log.V(2).Infof("Receiving message %v from %v\n", msg.Name, msg.UPID)
		msg.ProtoMessage = reflect.New(m.installedMessages[msg.Name]).Interface().(proto.Message)
		if err := proto.Unmarshal(msg.Bytes, msg.ProtoMessage); err != nil {
			log.Errorf("Failed to unmarshal message %v: %v\n", msg, err)
			continue
		}
		// TODO(yifan): Catch panic.
		m.installedHandlers[msg.Name](msg.UPID, msg.ProtoMessage)
	}
}

// getMessageName returns the name of the message in the mesos manner.
func getMessageName(msg proto.Message) string {
	return fmt.Sprintf("%v.%v", "mesos.internal", reflect.TypeOf(msg).Elem().Name())
}
<commit_msg>only log error when one occurs<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package messenger

import (
	"flag"
	"fmt"
	"reflect"
	"time"

	"github.com/gogo/protobuf/proto"
	log "github.com/golang/glog"
	mesos "github.com/mesos/mesos-go/mesosproto"
	"github.com/mesos/mesos-go/upid"
	"golang.org/x/net/context"
)

const (
	defaultQueueSize = 1024
	preparePeriod    = time.Second * 1
)

var (
	sendRoutines   int
	encodeRoutines int
	decodeRoutines int
)

func init() {
	flag.IntVar(&sendRoutines, "send-routines", 1, "Number of network sending routines")
	flag.IntVar(&encodeRoutines, "encode-routines", 1, "Number of encoding routines")
	flag.IntVar(&decodeRoutines, "decode-routines", 1, "Number of decoding routines")
}

// MessageHandler is the callback of the message. When the callback
// is invoked, the sender's upid and the message is passed to the callback.
type MessageHandler func(from *upid.UPID, pbMsg proto.Message)

// Messenger defines the interfaces that should be implemented.
type Messenger interface {
	Install(handler MessageHandler, msg proto.Message) error
	Send(ctx context.Context, upid *upid.UPID, msg proto.Message) error
	Route(ctx context.Context, from *upid.UPID, msg proto.Message) error
	Start() error
	Stop() error
	UPID() *upid.UPID
}

// MesosMessenger is an implementation of the Messenger interface.
type MesosMessenger struct {
	upid              *upid.UPID
	encodingQueue     chan *Message
	sendingQueue      chan *Message
	installedMessages map[string]reflect.Type
	installedHandlers map[string]MessageHandler
	stop              chan struct{}
	tr                Transporter
}

// NewMesosMessenger creates a new mesos messenger.
func NewHttp(upid *upid.UPID) *MesosMessenger {
	return New(upid, NewHTTPTransporter(upid))
}

func New(upid *upid.UPID, t Transporter) *MesosMessenger {
	return &MesosMessenger{
		upid:              upid,
		encodingQueue:     make(chan *Message, defaultQueueSize),
		sendingQueue:      make(chan *Message, defaultQueueSize),
		installedMessages: make(map[string]reflect.Type),
		installedHandlers: make(map[string]MessageHandler),
		tr:                t,
	}
}

/// Install installs the handler with the given message.
func (m *MesosMessenger) Install(handler MessageHandler, msg proto.Message) error {
	// Check if the message is a pointer.
	mtype := reflect.TypeOf(msg)
	if mtype.Kind() != reflect.Ptr {
		return fmt.Errorf("Message %v is not a Ptr type")
	}

	// Check if the message is already installed.
	name := getMessageName(msg)
	if _, ok := m.installedMessages[name]; ok {
		return fmt.Errorf("Message %v is already installed", name)
	}
	m.installedMessages[name] = mtype.Elem()
	m.installedHandlers[name] = handler
	m.tr.Install(name)
	return nil
}

// Send puts a message into the outgoing queue, waiting to be sent.
// With buffered channels, this will not block under moderate throughput.
// When an error is generated, the error can be communicated by placing
// a message on the incoming queue to be handled upstream.
func (m *MesosMessenger) Send(ctx context.Context, upid *upid.UPID, msg proto.Message) error {
	if upid.Equal(m.upid) {
		return fmt.Errorf("Send the message to self")
	}
	name := getMessageName(msg)
	log.V(2).Infof("Sending message %v to %v\n", name, upid)
	select {
	case <-ctx.Done():
		return ctx.Err()
	case m.encodingQueue <- &Message{upid, name, msg, nil}:
		return nil
	}
}

// Route puts a message either in the incoming or outgoing queue.
// This method is useful for:
// 1) routing internal error to callback handlers
// 2) testing components without starting remote servers.
func (m *MesosMessenger) Route(ctx context.Context, upid *upid.UPID, msg proto.Message) error {
	// if destination is not self, send to outbound.
	if !upid.Equal(m.upid) {
		return m.Send(ctx, upid, msg)
	}

	data, err := proto.Marshal(msg)
	if err != nil {
		return err
	}
	name := getMessageName(msg)
	return m.tr.Inject(ctx, &Message{upid, name, msg, data})
}

// Start starts the messenger.
func (m *MesosMessenger) Start() error {
	if err := m.tr.Listen(); err != nil {
		log.Errorf("Failed to start messenger: %v\n", err)
		return err
	}
	m.upid = m.tr.UPID()

	m.stop = make(chan struct{})
	errChan := make(chan error)
	go func() {
		if err := m.tr.Start(); err != nil {
			errChan <- err
		}
	}()

	select {
	case err := <-errChan:
		return err
	case <-time.After(preparePeriod):
	}
	for i := 0; i < sendRoutines; i++ {
		go m.sendLoop()
	}
	for i := 0; i < encodeRoutines; i++ {
		go m.encodeLoop()
	}
	for i := 0; i < decodeRoutines; i++ {
		go m.decodeLoop()
	}
	return nil
}

// Stop stops the messenger and clean up all the goroutines.
func (m *MesosMessenger) Stop() error {
	if err := m.tr.Stop(); err != nil {
		log.Errorf("Failed to stop the transporter: %v\n", err)
		return err
	}
	close(m.stop)
	return nil
}

// UPID returns the upid of the messenger.
func (m *MesosMessenger) UPID() *upid.UPID {
	return m.upid
}

func (m *MesosMessenger) encodeLoop() {
	for {
		select {
		case <-m.stop:
			return
		case msg := <-m.encodingQueue:
			e := func() error {
				//TODO(jdef) implement timeout for context
				ctx, cancel := context.WithCancel(context.TODO())
				defer cancel()

				b, err := proto.Marshal(msg.ProtoMessage)
				if err != nil {
					return err
				}
				msg.Bytes = b
				select {
				case <-ctx.Done():
					return ctx.Err()
				case m.sendingQueue <- msg:
					return nil
				}
			}()
			if e != nil {
				m.reportError(fmt.Errorf("Failed to enqueue message %v: %v", msg, e))
			}
		}
	}
}

func (m *MesosMessenger) reportError(err error) {
	log.V(2).Info(err)
	//TODO(jdef) implement timeout for context
	ctx, cancel := context.WithCancel(context.TODO())
	defer cancel()

	c := make(chan error, 1)
	go func() { c <- m.Route(ctx, m.UPID(), &mesos.FrameworkErrorMessage{Message: proto.String(err.Error())}) }()
	select {
	case <-ctx.Done():
		<-c // wait for Route to return
	case e := <-c:
		if e != nil {
			log.Errorf("failed to report error %v due to: %v", err, e)
		}
	}
}

func (m *MesosMessenger) sendLoop() {
	for {
		select {
		case <-m.stop:
			return
		case msg := <-m.sendingQueue:
			e := func() error {
				//TODO(jdef) implement timeout for context
				ctx, cancel := context.WithCancel(context.TODO())
				defer cancel()

				c := make(chan error, 1)
				go func() { c <- m.tr.Send(ctx, msg) }()

				select {
				case <-ctx.Done():
					// Transport layer must use the context to detect cancelled requests.
					<-c // wait for Send to return
					return ctx.Err()
				case err := <-c:
					return err
				}
			}()
			if e != nil {
				m.reportError(fmt.Errorf("Failed to send message %v: %v", msg.Name, e))
			}
		}
	}
}

// Since HTTPTransporter.Recv() is already buffered, so we don't need a 'recvLoop' here.
func (m *MesosMessenger) decodeLoop() {
	for {
		select {
		case <-m.stop:
			return
		default:
		}
		msg := m.tr.Recv()
		log.V(2).Infof("Receiving message %v from %v\n", msg.Name, msg.UPID)
		msg.ProtoMessage = reflect.New(m.installedMessages[msg.Name]).Interface().(proto.Message)
		if err := proto.Unmarshal(msg.Bytes, msg.ProtoMessage); err != nil {
			log.Errorf("Failed to unmarshal message %v: %v\n", msg, err)
			continue
		}
		// TODO(yifan): Catch panic.
		m.installedHandlers[msg.Name](msg.UPID, msg.ProtoMessage)
	}
}

// getMessageName returns the name of the message in the mesos manner.
func getMessageName(msg proto.Message) string {
	return fmt.Sprintf("%v.%v", "mesos.internal", reflect.TypeOf(msg).Elem().Name())
}
<|endoftext|>