<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"log"
	"math"
	"strings"
	"sync"
	"time"

	"github.com/coreos/etcd/lease"
	"github.com/coreos/etcd/storage/backend"
	"github.com/coreos/etcd/storage/storagepb"
)

const (
	// chanBufLen is the length of the buffered chan
	// for sending out watched events.
	// TODO: find a good buf value. 1024 is just a random one that
	// seems to be reasonable.
	chanBufLen = 1024
)

type (
	watcherSetByKey map[string]watcherSet
	watcherSet      map[*watcher]struct{}
)

func (w watcherSet) add(wa *watcher) {
	if _, ok := w[wa]; ok {
		panic("add watcher twice!")
	}
	w[wa] = struct{}{}
}

func (w watcherSetByKey) add(wa *watcher) {
	set := w[string(wa.key)]
	if set == nil {
		set = make(watcherSet)
		w[string(wa.key)] = set
	}
	set.add(wa)
}

func (w watcherSetByKey) getSetByKey(key string) (watcherSet, bool) {
	set, ok := w[key]
	return set, ok
}

func (w watcherSetByKey) delete(wa *watcher) bool {
	k := string(wa.key)
	if v, ok := w[k]; ok {
		if _, ok := v[wa]; ok {
			delete(v, wa)
			// if there is nothing in the set,
			// remove the set
			if len(v) == 0 {
				delete(w, k)
			}
			return true
		}
	}
	return false
}

type watchable interface {
	watch(key []byte, prefix bool, startRev int64, id WatchID, ch chan<- WatchResponse) (*watcher, cancelFunc)
	rev() int64
}

type watchableStore struct {
	mu sync.Mutex

	*store

	// contains all unsynced watchers that needs to sync with events that have happened
	unsynced watcherSetByKey

	// contains all synced watchers that are in sync with the progress of the store.
	// The key of the map is the key that the watcher watches on.
	synced watcherSetByKey

	stopc chan struct{}
	wg    sync.WaitGroup
}

// cancelFunc updates unsynced and synced maps when running
// cancel operations.
type cancelFunc func()

func newWatchableStore(b backend.Backend, le lease.Lessor) *watchableStore {
	s := &watchableStore{
		store:    NewStore(b, le),
		unsynced: make(watcherSetByKey),
		synced:   make(watcherSetByKey),
		stopc:    make(chan struct{}),
	}
	if s.le != nil {
		// use this store as the deleter so revokes trigger watch events
		s.le.SetRangeDeleter(s)
	}
	s.wg.Add(1)
	go s.syncWatchersLoop()
	return s
}

func (s *watchableStore) Put(key, value []byte, lease lease.LeaseID) (rev int64) {
	s.mu.Lock()
	defer s.mu.Unlock()

	rev = s.store.Put(key, value, lease)
	changes := s.store.getChanges()
	if len(changes) != 1 {
		log.Panicf("unexpected len(changes) != 1 after put")
	}

	ev := storagepb.Event{
		Type: storagepb.PUT,
		Kv:   &changes[0],
	}
	s.notify(rev, []storagepb.Event{ev})
	return rev
}

func (s *watchableStore) DeleteRange(key, end []byte) (n, rev int64) {
	s.mu.Lock()
	defer s.mu.Unlock()

	n, rev = s.store.DeleteRange(key, end)
	changes := s.store.getChanges()

	if len(changes) != int(n) {
		log.Panicf("unexpected len(changes) != n after deleteRange")
	}

	if n == 0 {
		return n, rev
	}

	evs := make([]storagepb.Event, n)
	for i, change := range changes {
		evs[i] = storagepb.Event{
			Type: storagepb.DELETE,
			Kv:   &change}
	}
	s.notify(rev, evs)
	return n, rev
}

func (s *watchableStore) TxnBegin() int64 {
	s.mu.Lock()
	return s.store.TxnBegin()
}

func (s *watchableStore) TxnEnd(txnID int64) error {
	err := s.store.TxnEnd(txnID)
	if err != nil {
		return err
	}

	changes := s.getChanges()
	if len(changes) == 0 {
		s.mu.Unlock()
		return nil
	}

	evs := make([]storagepb.Event, len(changes))
	for i, change := range changes {
		switch change.Value {
		case nil:
			evs[i] = storagepb.Event{
				Type: storagepb.DELETE,
				Kv:   &changes[i]}
		default:
			evs[i] = storagepb.Event{
				Type: storagepb.PUT,
				Kv:   &changes[i]}
		}
	}

	s.notify(s.store.Rev(), evs)
	s.mu.Unlock()

	return nil
}

func (s *watchableStore) Close() error {
	close(s.stopc)
	s.wg.Wait()
	return s.store.Close()
}

func (s *watchableStore) NewWatchStream() WatchStream {
	watchStreamGauge.Inc()
	return &watchStream{
		watchable: s,
		ch:        make(chan WatchResponse, chanBufLen),
		cancels:   make(map[WatchID]cancelFunc),
	}
}

func (s *watchableStore) watch(key []byte, prefix bool, startRev int64, id WatchID, ch chan<- WatchResponse) (*watcher, cancelFunc) {
	s.mu.Lock()
	defer s.mu.Unlock()

	wa := &watcher{
		key:    key,
		prefix: prefix,
		cur:    startRev,
		id:     id,
		ch:     ch,
	}

	if startRev == 0 {
		s.synced.add(wa)
	} else {
		slowWatcherGauge.Inc()
		s.unsynced.add(wa)
	}
	watcherGauge.Inc()

	cancel := cancelFunc(func() {
		s.mu.Lock()
		defer s.mu.Unlock()
		// remove references of the watcher
		if s.unsynced.delete(wa) {
			slowWatcherGauge.Dec()
			watcherGauge.Dec()
			return
		}

		if s.synced.delete(wa) {
			watcherGauge.Dec()
		}
		// If we cannot find it, it should have finished watch.
	})

	return wa, cancel
}

// syncWatchersLoop syncs the watcher in the unsynced map every 100ms.
func (s *watchableStore) syncWatchersLoop() {
	defer s.wg.Done()

	for {
		s.mu.Lock()
		s.syncWatchers()
		s.mu.Unlock()

		select {
		case <-time.After(100 * time.Millisecond):
		case <-s.stopc:
			return
		}
	}
}

// syncWatchers periodically syncs unsynced watchers by: Iterate all unsynced
// watchers to get the minimum revision within its range, skipping the
// watcher if its current revision is behind the compact revision of the
// store. And use this minimum revision to get all key-value pairs. Then send
// those events to watchers.
func (s *watchableStore) syncWatchers() {
	s.store.mu.Lock()
	defer s.store.mu.Unlock()

	if len(s.unsynced) == 0 {
		return
	}

	// in order to find key-value pairs from unsynced watchers, we need to
	// find min revision index, and these revisions can be used to
	// query the backend store of key-value pairs
	minRev := int64(math.MaxInt64)

	curRev := s.store.currentRev.main
	compactionRev := s.store.compactMainRev

	prefixes := make(map[string]struct{})
	for _, set := range s.unsynced {
		for w := range set {
			k := string(w.key)

			if w.cur > curRev {
				panic("watcher current revision should not exceed current revision")
			}

			if w.cur < compactionRev {
				select {
				case w.ch <- WatchResponse{WatchID: w.id, CompactRevision: compactionRev}:
					s.unsynced.delete(w)
				default:
					// retry next time
				}
				continue
			}

			if minRev >= w.cur {
				minRev = w.cur
			}

			if w.prefix {
				prefixes[k] = struct{}{}
			}
		}
	}

	minBytes, maxBytes := newRevBytes(), newRevBytes()
	revToBytes(revision{main: minRev}, minBytes)
	revToBytes(revision{main: curRev + 1}, maxBytes)

	// UnsafeRange returns keys and values. And in boltdb, keys are revisions.
	// values are actual key-value pairs in backend.
	tx := s.store.b.BatchTx()
	tx.Lock()
	ks, vs := tx.UnsafeRange(keyBucketName, minBytes, maxBytes, 0)

	evs := []storagepb.Event{}

	// get the list of all events from all key-value pairs
	for i, v := range vs {
		var kv storagepb.KeyValue
		if err := kv.Unmarshal(v); err != nil {
			log.Panicf("storage: cannot unmarshal event: %v", err)
		}

		k := string(kv.Key)
		if _, ok := s.unsynced.getSetByKey(k); !ok && !matchPrefix(k, prefixes) {
			continue
		}

		var ev storagepb.Event
		switch {
		case isTombstone(ks[i]):
			ev.Type = storagepb.DELETE
		default:
			ev.Type = storagepb.PUT
		}
		ev.Kv = &kv

		evs = append(evs, ev)
	}
	tx.Unlock()

	for w, es := range newWatcherToEventMap(s.unsynced, evs) {
		select {
		// s.store.Rev also uses Lock, so just return directly
		case w.ch <- WatchResponse{WatchID: w.id, Events: es, Revision: s.store.currentRev.main}:
			pendingEventsGauge.Add(float64(len(es)))
		default:
			// TODO: handle the full unsynced watchers.
			// continue to process other watchers for now, the full ones
			// will be processed next time and hopefully it will not be full.
			continue
		}
		w.cur = curRev
		s.synced.add(w)
		s.unsynced.delete(w)
	}

	slowWatcherGauge.Set(float64(len(s.unsynced)))
}

// notify notifies the fact that given event at the given rev just happened to
// watchers that watch on the key of the event.
func (s *watchableStore) notify(rev int64, evs []storagepb.Event) {
	we := newWatcherToEventMap(s.synced, evs)
	for _, wm := range s.synced {
		for w := range wm {
			es, ok := we[w]
			if !ok {
				continue
			}
			select {
			case w.ch <- WatchResponse{WatchID: w.id, Events: es, Revision: s.Rev()}:
				pendingEventsGauge.Add(float64(len(es)))
			default:
				// move slow watcher to unsynced
				w.cur = rev
				s.unsynced.add(w)
				delete(wm, w)
				slowWatcherGauge.Inc()
			}
		}
	}
}

func (s *watchableStore) rev() int64 { return s.store.Rev() }

type watcher struct {
	// the watcher key
	key []byte
	// prefix indicates if watcher is on a key or a prefix.
	// If prefix is true, the watcher is on a prefix.
	prefix bool
	// cur is the current watcher revision.
	// If cur is behind the current revision of the KV,
	// watcher is unsynced and needs to catch up.
	cur int64
	id  WatchID

	// a chan to send out the watch response.
	// The chan might be shared with other watchers.
	ch chan<- WatchResponse
}

// newWatcherToEventMap creates a map that has watcher as key and events as
// value. It enables quick events look up by watcher.
func newWatcherToEventMap(sm watcherSetByKey, evs []storagepb.Event) map[*watcher][]storagepb.Event {
	watcherToEvents := make(map[*watcher][]storagepb.Event)
	for _, ev := range evs {
		key := string(ev.Kv.Key)

		// check all prefixes of the key to notify all corresponded watchers
		for i := 0; i <= len(key); i++ {
			k := string(key[:i])

			wm, ok := sm[k]
			if !ok {
				continue
			}

			for w := range wm {
				// the watcher needs to be notified when either it watches prefix or
				// the key is exactly matched.
				if !w.prefix && i != len(ev.Kv.Key) {
					continue
				}

				if _, ok := watcherToEvents[w]; !ok {
					watcherToEvents[w] = []storagepb.Event{}
				}
				watcherToEvents[w] = append(watcherToEvents[w], ev)
			}
		}
	}

	return watcherToEvents
}

// matchPrefix returns true if key has any matching prefix
// from prefixes map.
func matchPrefix(key string, prefixes map[string]struct{}) bool {
	for p := range prefixes {
		if strings.HasPrefix(key, p) {
			return true
		}
	}
	return false
}
<commit_msg>storage: do not send outdated events to unsynced watchers<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"log"
	"math"
	"strings"
	"sync"
	"time"

	"github.com/coreos/etcd/lease"
	"github.com/coreos/etcd/storage/backend"
	"github.com/coreos/etcd/storage/storagepb"
)

const (
	// chanBufLen is the length of the buffered chan
	// for sending out watched events.
	// TODO: find a good buf value. 1024 is just a random one that
	// seems to be reasonable.
	chanBufLen = 1024
)

type (
	watcherSetByKey map[string]watcherSet
	watcherSet      map[*watcher]struct{}
)

func (w watcherSet) add(wa *watcher) {
	if _, ok := w[wa]; ok {
		panic("add watcher twice!")
	}
	w[wa] = struct{}{}
}

func (w watcherSetByKey) add(wa *watcher) {
	set := w[string(wa.key)]
	if set == nil {
		set = make(watcherSet)
		w[string(wa.key)] = set
	}
	set.add(wa)
}

func (w watcherSetByKey) getSetByKey(key string) (watcherSet, bool) {
	set, ok := w[key]
	return set, ok
}

func (w watcherSetByKey) delete(wa *watcher) bool {
	k := string(wa.key)
	if v, ok := w[k]; ok {
		if _, ok := v[wa]; ok {
			delete(v, wa)
			// if there is nothing in the set,
			// remove the set
			if len(v) == 0 {
				delete(w, k)
			}
			return true
		}
	}
	return false
}

type watchable interface {
	watch(key []byte, prefix bool, startRev int64, id WatchID, ch chan<- WatchResponse) (*watcher, cancelFunc)
	rev() int64
}

type watchableStore struct {
	mu sync.Mutex

	*store

	// contains all unsynced watchers that needs to sync with events that have happened
	unsynced watcherSetByKey

	// contains all synced watchers that are in sync with the progress of the store.
	// The key of the map is the key that the watcher watches on.
	synced watcherSetByKey

	stopc chan struct{}
	wg    sync.WaitGroup
}

// cancelFunc updates unsynced and synced maps when running
// cancel operations.
type cancelFunc func()

func newWatchableStore(b backend.Backend, le lease.Lessor) *watchableStore {
	s := &watchableStore{
		store:    NewStore(b, le),
		unsynced: make(watcherSetByKey),
		synced:   make(watcherSetByKey),
		stopc:    make(chan struct{}),
	}
	if s.le != nil {
		// use this store as the deleter so revokes trigger watch events
		s.le.SetRangeDeleter(s)
	}
	s.wg.Add(1)
	go s.syncWatchersLoop()
	return s
}

func (s *watchableStore) Put(key, value []byte, lease lease.LeaseID) (rev int64) {
	s.mu.Lock()
	defer s.mu.Unlock()

	rev = s.store.Put(key, value, lease)
	changes := s.store.getChanges()
	if len(changes) != 1 {
		log.Panicf("unexpected len(changes) != 1 after put")
	}

	ev := storagepb.Event{
		Type: storagepb.PUT,
		Kv:   &changes[0],
	}
	s.notify(rev, []storagepb.Event{ev})
	return rev
}

func (s *watchableStore) DeleteRange(key, end []byte) (n, rev int64) {
	s.mu.Lock()
	defer s.mu.Unlock()

	n, rev = s.store.DeleteRange(key, end)
	changes := s.store.getChanges()

	if len(changes) != int(n) {
		log.Panicf("unexpected len(changes) != n after deleteRange")
	}

	if n == 0 {
		return n, rev
	}

	evs := make([]storagepb.Event, n)
	for i, change := range changes {
		evs[i] = storagepb.Event{
			Type: storagepb.DELETE,
			Kv:   &change}
		evs[i].Kv.ModRevision = rev
	}
	s.notify(rev, evs)
	return n, rev
}

func (s *watchableStore) TxnBegin() int64 {
	s.mu.Lock()
	return s.store.TxnBegin()
}

func (s *watchableStore) TxnEnd(txnID int64) error {
	err := s.store.TxnEnd(txnID)
	if err != nil {
		return err
	}

	changes := s.getChanges()
	if len(changes) == 0 {
		s.mu.Unlock()
		return nil
	}

	rev := s.store.Rev()
	evs := make([]storagepb.Event, len(changes))
	for i, change := range changes {
		switch change.Value {
		case nil:
			evs[i] = storagepb.Event{
				Type: storagepb.DELETE,
				Kv:   &changes[i]}
			evs[i].Kv.ModRevision = rev
		default:
			evs[i] = storagepb.Event{
				Type: storagepb.PUT,
				Kv:   &changes[i]}
		}
	}

	s.notify(rev, evs)
	s.mu.Unlock()

	return nil
}

func (s *watchableStore) Close() error {
	close(s.stopc)
	s.wg.Wait()
	return s.store.Close()
}

func (s *watchableStore) NewWatchStream() WatchStream {
	watchStreamGauge.Inc()
	return &watchStream{
		watchable: s,
		ch:        make(chan WatchResponse, chanBufLen),
		cancels:   make(map[WatchID]cancelFunc),
	}
}

func (s *watchableStore) watch(key []byte, prefix bool, startRev int64, id WatchID, ch chan<- WatchResponse) (*watcher, cancelFunc) {
	s.mu.Lock()
	defer s.mu.Unlock()

	wa := &watcher{
		key:    key,
		prefix: prefix,
		cur:    startRev,
		id:     id,
		ch:     ch,
	}

	if startRev == 0 {
		s.synced.add(wa)
	} else {
		slowWatcherGauge.Inc()
		s.unsynced.add(wa)
	}
	watcherGauge.Inc()

	cancel := cancelFunc(func() {
		s.mu.Lock()
		defer s.mu.Unlock()
		// remove references of the watcher
		if s.unsynced.delete(wa) {
			slowWatcherGauge.Dec()
			watcherGauge.Dec()
			return
		}

		if s.synced.delete(wa) {
			watcherGauge.Dec()
		}
		// If we cannot find it, it should have finished watch.
	})

	return wa, cancel
}

// syncWatchersLoop syncs the watcher in the unsynced map every 100ms.
func (s *watchableStore) syncWatchersLoop() {
	defer s.wg.Done()

	for {
		s.mu.Lock()
		s.syncWatchers()
		s.mu.Unlock()

		select {
		case <-time.After(100 * time.Millisecond):
		case <-s.stopc:
			return
		}
	}
}

// syncWatchers periodically syncs unsynced watchers by: Iterate all unsynced
// watchers to get the minimum revision within its range, skipping the
// watcher if its current revision is behind the compact revision of the
// store. And use this minimum revision to get all key-value pairs. Then send
// those events to watchers.
func (s *watchableStore) syncWatchers() {
	s.store.mu.Lock()
	defer s.store.mu.Unlock()

	if len(s.unsynced) == 0 {
		return
	}

	// in order to find key-value pairs from unsynced watchers, we need to
	// find min revision index, and these revisions can be used to
	// query the backend store of key-value pairs
	prefixes, minRev := s.scanUnsync()
	curRev := s.store.currentRev.main
	minBytes, maxBytes := newRevBytes(), newRevBytes()
	revToBytes(revision{main: minRev}, minBytes)
	revToBytes(revision{main: curRev + 1}, maxBytes)

	// UnsafeRange returns keys and values. And in boltdb, keys are revisions.
	// values are actual key-value pairs in backend.
	tx := s.store.b.BatchTx()
	tx.Lock()
	revs, vs := tx.UnsafeRange(keyBucketName, minBytes, maxBytes, 0)
	evs := kvsToEvents(revs, vs, s.unsynced, prefixes)
	tx.Unlock()

	for w, es := range newWatcherToEventMap(s.unsynced, evs) {
		select {
		// s.store.Rev also uses Lock, so just return directly
		case w.ch <- WatchResponse{WatchID: w.id, Events: es, Revision: s.store.currentRev.main}:
			pendingEventsGauge.Add(float64(len(es)))
		default:
			// TODO: handle the full unsynced watchers.
			// continue to process other watchers for now, the full ones
			// will be processed next time and hopefully it will not be full.
			continue
		}
		w.cur = curRev
		s.synced.add(w)
		s.unsynced.delete(w)
	}

	slowWatcherGauge.Set(float64(len(s.unsynced)))
}

func (s *watchableStore) scanUnsync() (prefixes map[string]struct{}, minRev int64) {
	curRev := s.store.currentRev.main
	compactionRev := s.store.compactMainRev

	prefixes = make(map[string]struct{})
	minRev = int64(math.MaxInt64)
	for _, set := range s.unsynced {
		for w := range set {
			k := string(w.key)

			if w.cur > curRev {
				panic("watcher current revision should not exceed current revision")
			}

			if w.cur < compactionRev {
				select {
				case w.ch <- WatchResponse{WatchID: w.id, CompactRevision: compactionRev}:
					s.unsynced.delete(w)
				default:
					// retry next time
				}
				continue
			}

			if minRev > w.cur {
				minRev = w.cur
			}

			if w.prefix {
				prefixes[k] = struct{}{}
			}
		}
	}

	return prefixes, minRev
}

// kvsToEvents gets all events for the watchers from all key-value pairs
func kvsToEvents(revs, vals [][]byte, wsk watcherSetByKey, pfxs map[string]struct{}) (evs []storagepb.Event) {
	for i, v := range vals {
		var kv storagepb.KeyValue
		if err := kv.Unmarshal(v); err != nil {
			log.Panicf("storage: cannot unmarshal event: %v", err)
		}

		k := string(kv.Key)
		if _, ok := wsk.getSetByKey(k); !ok && !matchPrefix(k, pfxs) {
			continue
		}

		ty := storagepb.PUT
		if isTombstone(revs[i]) {
			ty = storagepb.DELETE
			// patch in mod revision so watchers won't skip
			kv.ModRevision = bytesToRev(revs[i]).main
		}
		evs = append(evs, storagepb.Event{Kv: &kv, Type: ty})
	}
	return evs
}

// notify notifies the fact that given event at the given rev just happened to
// watchers that watch on the key of the event.
func (s *watchableStore) notify(rev int64, evs []storagepb.Event) {
	we := newWatcherToEventMap(s.synced, evs)
	for _, wm := range s.synced {
		for w := range wm {
			es, ok := we[w]
			if !ok {
				continue
			}
			select {
			case w.ch <- WatchResponse{WatchID: w.id, Events: es, Revision: s.Rev()}:
				pendingEventsGauge.Add(float64(len(es)))
			default:
				// move slow watcher to unsynced
				w.cur = rev
				s.unsynced.add(w)
				delete(wm, w)
				slowWatcherGauge.Inc()
			}
		}
	}
}

func (s *watchableStore) rev() int64 { return s.store.Rev() }

type watcher struct {
	// the watcher key
	key []byte
	// prefix indicates if watcher is on a key or a prefix.
	// If prefix is true, the watcher is on a prefix.
	prefix bool
	// cur is the current watcher revision.
	// If cur is behind the current revision of the KV,
	// watcher is unsynced and needs to catch up.
	cur int64
	id  WatchID

	// a chan to send out the watch response.
	// The chan might be shared with other watchers.
	ch chan<- WatchResponse
}

// newWatcherToEventMap creates a map that has watcher as key and events as
// value. It enables quick events look up by watcher.
func newWatcherToEventMap(sm watcherSetByKey, evs []storagepb.Event) map[*watcher][]storagepb.Event {
	watcherToEvents := make(map[*watcher][]storagepb.Event)
	for _, ev := range evs {
		key := string(ev.Kv.Key)

		// check all prefixes of the key to notify all corresponded watchers
		for i := 0; i <= len(key); i++ {
			for w := range sm[key[:i]] {
				// don't double notify
				if ev.Kv.ModRevision < w.cur {
					continue
				}

				// the watcher needs to be notified when either it watches prefix or
				// the key is exactly matched.
				if !w.prefix && i != len(ev.Kv.Key) {
					continue
				}
				watcherToEvents[w] = append(watcherToEvents[w], ev)
			}
		}
	}

	return watcherToEvents
}

// matchPrefix returns true if key has any matching prefix
// from prefixes map.
func matchPrefix(key string, prefixes map[string]struct{}) bool {
	for p := range prefixes {
		if strings.HasPrefix(key, p) {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package beep

import "time"

// Streamer is able to stream a finite or infinite sequence of audio samples.
type Streamer interface {
	// Stream copies at most len(samples) next audio samples to the samples slice.
	//
	// The sample rate of the samples is specified by the global SampleRate variable/constant.
	// The value at samples[i][0] is the value of the left channel of the i-th sample.
	// Similarly, samples[i][1] is the value of the right channel of the i-th sample.
	//
	// Stream returns the number of streamed samples. If the Streamer is drained and no more
	// samples will be produced, it returns 0 and false. Stream must not touch any samples
	// outside samples[:n].
	//
	// There are 3 valid return pattterns of the Stream method:
	//
	//   1. n == len(samples) && ok
	//
	// Stream streamed all of the requested samples. Cases 1, 2 and 3 may occur in the following
	// calls.
	//
	//   2. 0 < n && n < len(samples) && ok
	//
	// Stream streamed n samples and drained the Streamer. Only case 3 may occur in the
	// following calls. If Err return a non-nil error, only this case is valid.
	//
	//   3. n == 0 && !ok
	//
	// The Streamer is drained and no more samples will come. Only this case may occur in the
	// following calls.
	Stream(samples [][2]float64) (n int, ok bool)

	// Err returns an error which occured during streaming. If no error occured, nil is
	// returned.
	//
	// When an error occurs, Streamer must become drained and Stream must return 0, false
	// forever.
	//
	// The reason why Stream doesn't return an error is that it dramatically simplifies
	// programming with Streamer. It's not very important to catch the error right when it
	// happens.
	Err() error
}

// StreamSeeker is a finite duration Streamer which supports seeking to an arbitrary position.
type StreamSeeker interface {
	Streamer

	// Duration returns the total duration of the Streamer.
	Duration() time.Duration

	// Position returns the current position of the Streamer. This value is between 0 and the
	// total duration.
	Position() time.Duration

	// Seek sets the position of the Streamer to the provided value.
	//
	// If an error occurs during seeking, the position remains unchanged. This error will not be
	// returned through the Streamer's Err method.
	Seek(d time.Duration) error
}

// StreamCloser is a Streamer streaming from a resource which needs to be released, such as a file
// or a network connection.
type StreamCloser interface {
	Streamer

	// Close closes the Streamer and releases it's resources. Streamer will no longer stream any
	// samples.
	Close() error
}

// StreamSeekCloser is a union of StreamSeeker and StreamCloser.
type StreamSeekCloser interface {
	Streamer
	Duration() time.Duration
	Position() time.Duration
	Seek(d time.Duration) error
	Close() error
}

// StreamerFunc is a Streamer created by simply wrapping a streaming function (usually a closure,
// which encloses a time tracking variable). This sometimes simplifies creating new streamers.
//
// Example:
//
//   noise := StreamerFunc(func(samples [][2]float64) (n int, ok bool) {
//       for i := range samples {
//           samples[i][0] = rand.Float64()*2 - 1
//           samples[i][1] = rand.Float64()*2 - 1
//       }
//       return len(samples), true
//   })
type StreamerFunc func(samples [][2]float64) (n int, ok bool)

// Stream calls the wrapped streaming function.
func (sf StreamerFunc) Stream(samples [][2]float64) (n int, ok bool) {
	return sf(samples)
}

// Err always returns nil.
func (sf StreamerFunc) Err() error {
	return nil
}
<commit_msg>update doc<commit_after>package beep

import "time"

// Streamer is able to stream a finite or infinite sequence of audio samples.
type Streamer interface {
	// Stream copies at most len(samples) next audio samples to the samples slice.
	//
	// The sample rate of the samples is unspecified in general, but should be specified for
	// each concrete Streamer.
	//
	// The value at samples[i][0] is the value of the left channel of the i-th sample.
	// Similarly, samples[i][1] is the value of the right channel of the i-th sample.
	//
	// Stream returns the number of streamed samples. If the Streamer is drained and no more
	// samples will be produced, it returns 0 and false. Stream must not touch any samples
	// outside samples[:n].
	//
	// There are 3 valid return pattterns of the Stream method:
	//
	//   1. n == len(samples) && ok
	//
	// Stream streamed all of the requested samples. Cases 1, 2 and 3 may occur in the following
	// calls.
	//
	//   2. 0 < n && n < len(samples) && ok
	//
	// Stream streamed n samples and drained the Streamer. Only case 3 may occur in the
	// following calls. If Err return a non-nil error, only this case is valid.
	//
	//   3. n == 0 && !ok
	//
	// The Streamer is drained and no more samples will come. Only this case may occur in the
	// following calls.
	Stream(samples [][2]float64) (n int, ok bool)

	// Err returns an error which occured during streaming. If no error occured, nil is
	// returned.
	//
	// When an error occurs, Streamer must become drained and Stream must return 0, false
	// forever.
	//
	// The reason why Stream doesn't return an error is that it dramatically simplifies
	// programming with Streamer. It's not very important to catch the error right when it
	// happens.
	Err() error
}

// StreamSeeker is a finite duration Streamer which supports seeking to an arbitrary position.
type StreamSeeker interface {
	Streamer

	// Duration returns the total duration of the Streamer.
	Duration() time.Duration

	// Position returns the current position of the Streamer. This value is between 0 and the
	// total duration.
	Position() time.Duration

	// Seek sets the position of the Streamer to the provided value.
	//
	// If an error occurs during seeking, the position remains unchanged. This error will not be
	// returned through the Streamer's Err method.
	Seek(d time.Duration) error
}

// StreamCloser is a Streamer streaming from a resource which needs to be released, such as a file
// or a network connection.
type StreamCloser interface {
	Streamer

	// Close closes the Streamer and releases it's resources. Streamer will no longer stream any
	// samples.
	Close() error
}

// StreamSeekCloser is a union of StreamSeeker and StreamCloser.
type StreamSeekCloser interface {
	Streamer
	Duration() time.Duration
	Position() time.Duration
	Seek(d time.Duration) error
	Close() error
}

// StreamerFunc is a Streamer created by simply wrapping a streaming function (usually a closure,
// which encloses a time tracking variable). This sometimes simplifies creating new streamers.
//
// Example:
//
//   noise := StreamerFunc(func(samples [][2]float64) (n int, ok bool) {
//       for i := range samples {
//           samples[i][0] = rand.Float64()*2 - 1
//           samples[i][1] = rand.Float64()*2 - 1
//       }
//       return len(samples), true
//   })
type StreamerFunc func(samples [][2]float64) (n int, ok bool)

// Stream calls the wrapped streaming function.
func (sf StreamerFunc) Stream(samples [][2]float64) (n int, ok bool) {
	return sf(samples)
}

// Err always returns nil.
func (sf StreamerFunc) Err() error {
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package debug

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"time"

	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/testing"
)

type DebugHooksServerSuite struct {
	testing.BaseSuite
	ctx     *HooksContext
	fakebin string
	tmpdir  string
}

var _ = gc.Suite(&DebugHooksServerSuite{})

// echocommand outputs its name and arguments to stdout for verification,
// and exits with the value of $EXIT_CODE
var echocommand = `#!/bin/bash --norc
echo $(basename $0) $@
exit $EXIT_CODE
`

var fakecommands = []string{"tmux"}

func (s *DebugHooksServerSuite) SetUpTest(c *gc.C) {
	if runtime.GOOS == "windows" {
		c.Skip("bug 1403084: Currently debug does not work on windows")
	}
	s.fakebin = c.MkDir()
	s.tmpdir = c.MkDir()
	s.PatchEnvPathPrepend(s.fakebin)
	s.PatchEnvironment("TMPDIR", s.tmpdir)
	s.PatchEnvironment("TEST_RESULT", "")
	for _, name := range fakecommands {
		err := ioutil.WriteFile(filepath.Join(s.fakebin, name), []byte(echocommand), 0777)
		c.Assert(err, jc.ErrorIsNil)
	}
	s.ctx = NewHooksContext("foo/8")
	s.ctx.FlockDir = s.tmpdir
	s.PatchEnvironment("JUJU_UNIT_NAME", s.ctx.Unit)
}

func (s *DebugHooksServerSuite) TestFindSession(c *gc.C) {
	// Test "tmux has-session" failure. The error
	// message is the output of tmux has-session.
	os.Setenv("EXIT_CODE", "1")
	session, err := s.ctx.FindSession()
	c.Assert(session, gc.IsNil)
	c.Assert(err, gc.ErrorMatches, regexp.QuoteMeta("tmux has-session -t "+s.ctx.Unit+"\n"))
	os.Setenv("EXIT_CODE", "")

	// tmux session exists, but missing debug-hooks file: error.
	session, err = s.ctx.FindSession()
	c.Assert(session, gc.IsNil)
	c.Assert(err, gc.NotNil)
	c.Assert(err, jc.Satisfies, os.IsNotExist)

	// Hooks file is present, empty.
	err = ioutil.WriteFile(s.ctx.ClientFileLock(), []byte{}, 0777)
	c.Assert(err, jc.ErrorIsNil)
	session, err = s.ctx.FindSession()
	c.Assert(session, gc.NotNil)
	c.Assert(err, jc.ErrorIsNil)
	// If session.hooks is empty, it'll match anything.
	c.Assert(session.MatchHook(""), jc.IsTrue)
	c.Assert(session.MatchHook("something"), jc.IsTrue)

	// Hooks file is present, non-empty
	err = ioutil.WriteFile(s.ctx.ClientFileLock(), []byte(`hooks: [foo, bar, baz]`), 0777)
	c.Assert(err, jc.ErrorIsNil)
	session, err = s.ctx.FindSession()
	c.Assert(session, gc.NotNil)
	c.Assert(err, jc.ErrorIsNil)
	// session should only match "foo", "bar" or "baz".
	c.Assert(session.MatchHook(""), jc.IsFalse)
	c.Assert(session.MatchHook("something"), jc.IsFalse)
	c.Assert(session.MatchHook("foo"), jc.IsTrue)
	c.Assert(session.MatchHook("bar"), jc.IsTrue)
	c.Assert(session.MatchHook("baz"), jc.IsTrue)
	c.Assert(session.MatchHook("foo bar baz"), jc.IsFalse)
}

func (s *DebugHooksServerSuite) TestRunHookExceptional(c *gc.C) {
	err := ioutil.WriteFile(s.ctx.ClientFileLock(), []byte{}, 0777)
	c.Assert(err, jc.ErrorIsNil)
	session, err := s.ctx.FindSession()
	c.Assert(session, gc.NotNil)
	c.Assert(err, jc.ErrorIsNil)

	flockAcquired := make(chan struct{}, 1)
	waitForFlock := func() {
		select {
		case <-flockAcquired:
		case <-time.After(testing.ShortWait):
			c.Fatalf("timed out waiting for hook to acquire flock")
		}
	}

	// Run the hook in debug mode with no exit flock held.
	// The exit flock will be acquired immediately, and the
	// debug-hooks server process killed.
	s.PatchValue(&waitClientExit, func(*ServerSession) {
		flockAcquired <- struct{}{}
	})
	err = session.RunHook("myhook", s.tmpdir, os.Environ())
	c.Assert(err, gc.ErrorMatches, "signal: [kK]illed")
	waitForFlock()

	// Run the hook in debug mode, simulating the holding
	// of the exit flock. This simulates the client process
	// starting but not cleanly exiting (normally the .pid
	// file is updated, and the server waits on the client
	// process' death).
	ch := make(chan bool) // acquire the flock
	var clientExited bool
	s.PatchValue(&waitClientExit, func(*ServerSession) {
		clientExited = <-ch
		flockAcquired <- struct{}{}
	})
	go func() { ch <- true }() // asynchronously release the flock
	err = session.RunHook("myhook", s.tmpdir, os.Environ())
	waitForFlock()
	c.Assert(clientExited, jc.IsTrue)
	c.Assert(err, gc.ErrorMatches, "signal: [kK]illed")
}

func (s *DebugHooksServerSuite) TestRunHook(c *gc.C) {
	err := ioutil.WriteFile(s.ctx.ClientFileLock(), []byte{}, 0777)
	c.Assert(err, jc.ErrorIsNil)
	var output bytes.Buffer
	session, err := s.ctx.FindSessionWithWriter(&output)
	c.Assert(session, gc.NotNil)
	c.Assert(err, jc.ErrorIsNil)

	const hookName = "myhook"

	// Run the hook in debug mode with the exit flock held,
	// and also create the .pid file. We'll populate it with
	// an invalid PID; this will cause the server process to
	// exit cleanly (as if the PID were real and no longer running).
	cmd := exec.Command("flock", s.ctx.ClientExitFileLock(), "-c", "sleep 5s")
	c.Assert(cmd.Start(), gc.IsNil)
	defer cmd.Process.Kill() // kill flock

	ch := make(chan error)
	go func() {
		ch <- session.RunHook(hookName, s.tmpdir, os.Environ())
	}()

	// Wait until either we find the debug dir, or the flock is released.
	ticker := time.Tick(10 * time.Millisecond)
	var debugdir os.FileInfo
	timeout := time.After(testing.LongWait)
	for debugdir == nil {
		select {
		case <-timeout:
			c.Fatal("test timed out")
		case err = <-ch:
			// flock was released before we found the debug dir.
			c.Fatalf("could not find hook.sh\nerr: %v\noutput: %s", err, output.String())
		case <-ticker:
			tmpdir, err := os.Open(s.tmpdir)
			if err != nil {
				c.Fatalf("Failed to open $TMPDIR: %s", err)
			}
			fi, err := tmpdir.Readdir(-1)
			if err != nil {
				c.Fatalf("Failed to read $TMPDIR: %s", err)
			}
			tmpdir.Close()
			for _, fi := range fi {
				if fi.IsDir() {
					hooksh := filepath.Join(s.tmpdir, fi.Name(), "hook.sh")
					if _, err = os.Stat(hooksh); err == nil {
						debugdir = fi
						break
					}
				}
			}
			if debugdir != nil {
				break
			}
			time.Sleep(10 * time.Millisecond)
		}
	}

	envsh := filepath.Join(s.tmpdir, debugdir.Name(), "env.sh")
	s.verifyEnvshFile(c, envsh, hookName)

	hookpid := filepath.Join(s.tmpdir, debugdir.Name(), "hook.pid")
	err = ioutil.WriteFile(hookpid, []byte("not a pid"), 0777)
	c.Assert(err, jc.ErrorIsNil)

	// RunHook should complete without waiting to be
	// killed, and despite the exit lock being held.
	select {
	case err = <-ch:
		c.Assert(err, jc.ErrorIsNil)
	case <-time.After(testing.LongWait):
		c.Fatal("RunHook did not complete")
	}
}

func (s *DebugHooksServerSuite) verifyEnvshFile(c *gc.C, envshPath string, hookName string) {
	data, err := ioutil.ReadFile(envshPath)
	c.Assert(err, jc.ErrorIsNil)
	contents := string(data)
	c.Assert(contents, jc.Contains, fmt.Sprintf("JUJU_UNIT_NAME=%q", s.ctx.Unit))
	c.Assert(contents, jc.Contains, fmt.Sprintf("JUJU_HOOK_NAME=%q", hookName))
	c.Assert(contents, jc.Contains, fmt.Sprintf(`PS1="%s:%s %% "`, s.ctx.Unit, hookName))
}
<commit_msg>Make DebugHook test more reliable on busy machines<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package debug

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"time"

	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/testing"
)

type DebugHooksServerSuite struct {
	testing.BaseSuite
	ctx     *HooksContext
	fakebin string
	tmpdir  string
}

var _ = gc.Suite(&DebugHooksServerSuite{})

// echocommand outputs its name and arguments to stdout for verification,
// and exits with the value of $EXIT_CODE
var echocommand = `#!/bin/bash --norc
echo $(basename $0) $@
exit $EXIT_CODE
`

var fakecommands = []string{"tmux"}

func (s *DebugHooksServerSuite) SetUpTest(c *gc.C) {
	if runtime.GOOS == "windows" {
		c.Skip("bug 1403084: Currently debug does not work on windows")
	}
	s.fakebin = c.MkDir()
	s.tmpdir = c.MkDir()
	s.PatchEnvPathPrepend(s.fakebin)
	s.PatchEnvironment("TMPDIR", s.tmpdir)
	s.PatchEnvironment("TEST_RESULT", "")
	for _, name := range fakecommands {
		err := ioutil.WriteFile(filepath.Join(s.fakebin, name), []byte(echocommand), 0777)
		c.Assert(err, jc.ErrorIsNil)
	}
	s.ctx = NewHooksContext("foo/8")
	s.ctx.FlockDir = s.tmpdir
	s.PatchEnvironment("JUJU_UNIT_NAME", s.ctx.Unit)
}

func (s *DebugHooksServerSuite) TestFindSession(c *gc.C) {
	// Test "tmux has-session" failure. The error
	// message is the output of tmux has-session.
	os.Setenv("EXIT_CODE", "1")
	session, err := s.ctx.FindSession()
	c.Assert(session, gc.IsNil)
	c.Assert(err, gc.ErrorMatches, regexp.QuoteMeta("tmux has-session -t "+s.ctx.Unit+"\n"))
	os.Setenv("EXIT_CODE", "")

	// tmux session exists, but missing debug-hooks file: error.
	session, err = s.ctx.FindSession()
	c.Assert(session, gc.IsNil)
	c.Assert(err, gc.NotNil)
	c.Assert(err, jc.Satisfies, os.IsNotExist)

	// Hooks file is present, empty.
	err = ioutil.WriteFile(s.ctx.ClientFileLock(), []byte{}, 0777)
	c.Assert(err, jc.ErrorIsNil)
	session, err = s.ctx.FindSession()
	c.Assert(session, gc.NotNil)
	c.Assert(err, jc.ErrorIsNil)
	// If session.hooks is empty, it'll match anything.
	c.Assert(session.MatchHook(""), jc.IsTrue)
	c.Assert(session.MatchHook("something"), jc.IsTrue)

	// Hooks file is present, non-empty
	err = ioutil.WriteFile(s.ctx.ClientFileLock(), []byte(`hooks: [foo, bar, baz]`), 0777)
	c.Assert(err, jc.ErrorIsNil)
	session, err = s.ctx.FindSession()
	c.Assert(session, gc.NotNil)
	c.Assert(err, jc.ErrorIsNil)
	// session should only match "foo", "bar" or "baz".
	c.Assert(session.MatchHook(""), jc.IsFalse)
	c.Assert(session.MatchHook("something"), jc.IsFalse)
	c.Assert(session.MatchHook("foo"), jc.IsTrue)
	c.Assert(session.MatchHook("bar"), jc.IsTrue)
	c.Assert(session.MatchHook("baz"), jc.IsTrue)
	c.Assert(session.MatchHook("foo bar baz"), jc.IsFalse)
}

func (s *DebugHooksServerSuite) TestRunHookExceptional(c *gc.C) {
	err := ioutil.WriteFile(s.ctx.ClientFileLock(), []byte{}, 0777)
	c.Assert(err, jc.ErrorIsNil)
	session, err := s.ctx.FindSession()
	c.Assert(session, gc.NotNil)
	c.Assert(err, jc.ErrorIsNil)

	flockAcquired := make(chan struct{}, 1)
	waitForFlock := func() {
		select {
		case <-flockAcquired:
		case <-time.After(testing.ShortWait):
			c.Fatalf("timed out waiting for hook to acquire flock")
		}
	}

	// Run the hook in debug mode with no exit flock held.
	// The exit flock will be acquired immediately, and the
	// debug-hooks server process killed.
	s.PatchValue(&waitClientExit, func(*ServerSession) {
		flockAcquired <- struct{}{}
	})
	err = session.RunHook("myhook", s.tmpdir, os.Environ())
	c.Assert(err, gc.ErrorMatches, "signal: [kK]illed")
	waitForFlock()

	// Run the hook in debug mode, simulating the holding
	// of the exit flock. This simulates the client process
	// starting but not cleanly exiting (normally the .pid
	// file is updated, and the server waits on the client
	// process' death).
	ch := make(chan bool) // acquire the flock
	var clientExited bool
	s.PatchValue(&waitClientExit, func(*ServerSession) {
		clientExited = <-ch
		flockAcquired <- struct{}{}
	})
	go func() { ch <- true }() // asynchronously release the flock
	err = session.RunHook("myhook", s.tmpdir, os.Environ())
	waitForFlock()
	c.Assert(clientExited, jc.IsTrue)
	c.Assert(err, gc.ErrorMatches, "signal: [kK]illed")
}

func (s *DebugHooksServerSuite) TestRunHook(c *gc.C) {
	err := ioutil.WriteFile(s.ctx.ClientFileLock(), []byte{}, 0777)
	c.Assert(err, jc.ErrorIsNil)
	var output bytes.Buffer
	session, err := s.ctx.FindSessionWithWriter(&output)
	c.Assert(session, gc.NotNil)
	c.Assert(err, jc.ErrorIsNil)

	const hookName = "myhook"

	// Run the hook in debug mode with the exit flock held,
	// and also create the .pid file. We'll populate it with
	// an invalid PID; this will cause the server process to
	// exit cleanly (as if the PID were real and no longer running).
	cmd := exec.Command("flock", s.ctx.ClientExitFileLock(), "-c", "sleep 10s")
	c.Assert(cmd.Start(), gc.IsNil)
	defer cmd.Process.Kill() // kill flock

	ch := make(chan error)
	go func() {
		ch <- session.RunHook(hookName, s.tmpdir, os.Environ())
	}()

	// Wait until either we find the debug dir, or the flock is released.
	ticker := time.Tick(10 * time.Millisecond)
	var debugdir os.FileInfo
	timeout := time.After(testing.LongWait)
	for debugdir == nil {
		select {
		case <-timeout:
			c.Fatal("test timed out")
		case err = <-ch:
			// flock was released before we found the debug dir.
			c.Fatalf("could not find hook.sh\nerr: %v\noutput: %s", err, output.String())
		case <-ticker:
			tmpdir, err := os.Open(s.tmpdir)
			if err != nil {
				c.Fatalf("Failed to open $TMPDIR: %s", err)
			}
			fi, err := tmpdir.Readdir(-1)
			if err != nil {
				c.Fatalf("Failed to read $TMPDIR: %s", err)
			}
			tmpdir.Close()
			for _, fi := range fi {
				if fi.IsDir() {
					hooksh := filepath.Join(s.tmpdir, fi.Name(), "hook.sh")
					if _, err = os.Stat(hooksh); err == nil {
						debugdir = fi
						break
					}
				}
			}
			if debugdir != nil {
				break
			}
			time.Sleep(10 * time.Millisecond)
		}
	}

	envsh := filepath.Join(s.tmpdir, debugdir.Name(), "env.sh")
	s.verifyEnvshFile(c, envsh, hookName)

	hookpid := filepath.Join(s.tmpdir, debugdir.Name(), "hook.pid")
	err = ioutil.WriteFile(hookpid, []byte("not a pid"), 0777)
	c.Assert(err, jc.ErrorIsNil)

	// RunHook should complete without waiting to be
	// killed, and despite the exit lock being held.
	select {
	case err = <-ch:
		c.Assert(err, jc.ErrorIsNil)
	case <-time.After(testing.LongWait):
		c.Fatal("RunHook did not complete")
	}
}

func (s *DebugHooksServerSuite) verifyEnvshFile(c *gc.C, envshPath string, hookName string) {
	data, err := ioutil.ReadFile(envshPath)
	c.Assert(err, jc.ErrorIsNil)
	contents := string(data)
	c.Assert(contents, jc.Contains, fmt.Sprintf("JUJU_UNIT_NAME=%q", s.ctx.Unit))
	c.Assert(contents, jc.Contains, fmt.Sprintf("JUJU_HOOK_NAME=%q", hookName))
	c.Assert(contents, jc.Contains, fmt.Sprintf(`PS1="%s:%s %% "`, s.ctx.Unit, hookName))
}
<|endoftext|>


<commit_before>package instructions

import (
    "jvmgo/rtda"
    "jvmgo/rtda/class"
)

// Create new object
type new_ struct {Index16Instruction}
func (self *new_) Execute(thread *rtda.Thread) {
    frame := thread.CurrentFrame()
    stack := frame.OperandStack()
    cp := frame.Method().Class().ConstantPool()
    cClass := cp.GetConstant(self.index).(class.ConstantClass)
    class := cClass.Class()
    ref := class.NewObj()
    stack.PushRef(ref)
}

// Create new array of reference
type anewarray struct {Index16Instruction}
func (self *anewarray) Execute(thread *rtda.Thread) {
    // todo
}

// Create new array
type newarray struct {
    atype uint8
}
func (self *newarray) fetchOperands(bcr *BytecodeReader) {
    self.atype = bcr.readUint8()
}
func (self *newarray) Execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    count := stack.PopInt()
    ref := class.NewArray(self.atype, count)
    stack.PushRef(ref)
}

// Create new multidimensional array
type multianewarray struct {
    index       uint16
    dimensions  uint8
}
func (self *multianewarray) fetchOperands(bcr *BytecodeReader) {
    self.index = bcr.readUint16()
    self.dimensions = bcr.readUint8()
}
func (self *multianewarray) Execute(thread *rtda.Thread) {
    // todo
}
<commit_msg>reorder code<commit_after>package instructions

import (
    "jvmgo/rtda"
    "jvmgo/rtda/class"
)

// Create new object
type new_ struct {Index16Instruction}
func (self *new_) Execute(thread *rtda.Thread) {
    frame := thread.CurrentFrame()
    stack := frame.OperandStack()
    cp := frame.Method().Class().ConstantPool()
    cClass := cp.GetConstant(self.index).(class.ConstantClass)
    class := cClass.Class()
    ref := class.NewObj()
    stack.PushRef(ref)
}

// Create new array
type newarray struct {
    atype uint8
}
func (self *newarray) fetchOperands(bcr *BytecodeReader) {
    self.atype = bcr.readUint8()
}
func (self *newarray) Execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    count := stack.PopInt()
    ref := class.NewArray(self.atype, count)
    stack.PushRef(ref)
}

// Create new array of reference
type anewarray struct {Index16Instruction}
func (self *anewarray) Execute(thread *rtda.Thread) {
    // todo
}

// Create new multidimensional array
type multianewarray struct {
    index       uint16
    dimensions  uint8
}
func (self *multianewarray) fetchOperands(bcr *BytecodeReader) {
    self.index = bcr.readUint16()
    self.dimensions = bcr.readUint8()
}
func (self *multianewarray) Execute(thread *rtda.Thread) {
    // todo
}
<|endoftext|>


<commit_before>package core

import (
	"context"

	"v2ray.com/core/common"
)

// Server is an instance of V2Ray. At any time, there must be at most one Server instance running.
// Deprecated. Use Instance directly.
type Server interface {
	common.Runnable
}

// Feature is the interface for V2Ray features. All features must implement this interface.
// All existing features have an implementation in app directory. These features can be replaced by third-party ones.
type Feature interface {
	common.Runnable
}

// Instance combines all functionalities in V2Ray.
type Instance struct {
	dnsClient     syncDNSClient
	policyManager syncPolicyManager
	dispatcher    syncDispatcher
	router        syncRouter
	ihm           syncInboundHandlerManager
	ohm           syncOutboundHandlerManager

	features []Feature
}

// New returns a new V2Ray instance based on given configuration.
// The instance is not started at this point.
// To make sure V2Ray instance works properly, the config must contain one Dispatcher, one InboundHandlerManager and one OutboundHandlerManager. Other features are optional.
func New(config *Config) (*Instance, error) {
	var server = new(Instance)

	if err := config.Transport.Apply(); err != nil {
		return nil, err
	}

	ctx := context.WithValue(context.Background(), v2rayKey, server)

	for _, appSettings := range config.App {
		settings, err := appSettings.GetInstance()
		if err != nil {
			return nil, err
		}
		if _, err := common.CreateObject(ctx, settings); err != nil {
			return nil, err
		}
	}

	for _, inbound := range config.Inbound {
		rawHandler, err := common.CreateObject(ctx, inbound)
		if err != nil {
			return nil, err
		}
		handler, ok := rawHandler.(InboundHandler)
		if !ok {
			return nil, newError("not an InboundHandler")
		}
		if err := server.InboundHandlerManager().AddHandler(ctx, handler); err != nil {
			return nil, err
		}
	}

	for _, outbound := range config.Outbound {
		rawHandler, err := common.CreateObject(ctx, outbound)
		if err != nil {
			return nil, err
		}
		handler, ok := rawHandler.(OutboundHandler)
		if !ok {
			return nil, newError("not an OutboundHandler")
		}
		if err := server.OutboundHandlerManager().AddHandler(ctx, handler); err != nil {
			return nil, err
		}
	}

	return server, nil
}

// Close shutdown the V2Ray instance.
func (s *Instance) Close() {
	for _, f := range s.features {
		f.Close()
	}
}

// Start starts the V2Ray instance, including all registered features. When Start returns error, the state of the instance is unknown.
func (s *Instance) Start() error {
	for _, f := range s.features {
		if err := f.Start(); err != nil {
			return err
		}
	}

	newError("V2Ray started").AtWarning().WriteToLog()

	return nil
}

// RegisterFeature registers the given feature into V2Ray.
// If feature is one of the following types, the corressponding feature in this Instance
// will be replaced: DNSClient, PolicyManager, Router, Dispatcher, InboundHandlerManager, OutboundHandlerManager.
func (s *Instance) RegisterFeature(feature interface{}, instance Feature) error {
	switch feature.(type) {
	case DNSClient, *DNSClient:
		s.dnsClient.Set(instance.(DNSClient))
	case PolicyManager, *PolicyManager:
		s.policyManager.Set(instance.(PolicyManager))
	case Router, *Router:
		s.router.Set(instance.(Router))
	case Dispatcher, *Dispatcher:
		s.dispatcher.Set(instance.(Dispatcher))
	case InboundHandlerManager, *InboundHandlerManager:
		s.ihm.Set(instance.(InboundHandlerManager))
	case OutboundHandlerManager, *OutboundHandlerManager:
		s.ohm.Set(instance.(OutboundHandlerManager))
	}
	s.features = append(s.features, instance)
	return nil
}

// DNSClient returns the DNSClient used by this Instance. The returned DNSClient is always functional.
func (s *Instance) DNSClient() DNSClient {
	return &(s.dnsClient)
}

// PolicyManager returns the PolicyManager used by this Instance. The returned PolicyManager is always functional.
func (s *Instance) PolicyManager() PolicyManager {
	return &(s.policyManager)
}

// Router returns the Router used by this Instance. The returned Router is always functional.
func (s *Instance) Router() Router {
	return &(s.router)
}

// Dispatcher returns the Dispatcher used by this Instance. If Dispatcher was not registered before, the returned value doesn't work, although it is not nil.
func (s *Instance) Dispatcher() Dispatcher {
	return &(s.dispatcher)
}

// InboundHandlerManager returns the InboundHandlerManager used by this Instance. If InboundHandlerManager was not registered before, the returned value doesn't work.
func (s *Instance) InboundHandlerManager() InboundHandlerManager {
	return &(s.ihm)
}

// OutboundHandlerManager returns the OutboundHandlerManager used by this Instance. If OutboundHandlerManager was not registered before, the returned value doesn't work.
func (s *Instance) OutboundHandlerManager() OutboundHandlerManager {
	return &(s.ohm)
}
<commit_msg>id of v2ray instance<commit_after>package core

import (
	"context"

	"v2ray.com/core/common"
	"v2ray.com/core/common/uuid"
)

// Server is an instance of V2Ray. At any time, there must be at most one Server instance running.
// Deprecated. Use Instance directly.
type Server interface {
	common.Runnable
}

// Feature is the interface for V2Ray features. All features must implement this interface.
// All existing features have an implementation in app directory. These features can be replaced by third-party ones.
type Feature interface {
	common.Runnable
}

// Instance combines all functionalities in V2Ray.
type Instance struct {
	dnsClient     syncDNSClient
	policyManager syncPolicyManager
	dispatcher    syncDispatcher
	router        syncRouter
	ihm           syncInboundHandlerManager
	ohm           syncOutboundHandlerManager

	features []Feature
	id       uuid.UUID
}

// New returns a new V2Ray instance based on given configuration.
// The instance is not started at this point.
// To make sure V2Ray instance works properly, the config must contain one Dispatcher, one InboundHandlerManager and one OutboundHandlerManager. Other features are optional.
func New(config *Config) (*Instance, error) {
	var server = &Instance{
		id: *(uuid.New()),
	}

	if err := config.Transport.Apply(); err != nil {
		return nil, err
	}

	ctx := context.WithValue(context.Background(), v2rayKey, server)

	for _, appSettings := range config.App {
		settings, err := appSettings.GetInstance()
		if err != nil {
			return nil, err
		}
		if _, err := common.CreateObject(ctx, settings); err != nil {
			return nil, err
		}
	}

	for _, inbound := range config.Inbound {
		rawHandler, err := common.CreateObject(ctx, inbound)
		if err != nil {
			return nil, err
		}
		handler, ok := rawHandler.(InboundHandler)
		if !ok {
			return nil, newError("not an InboundHandler")
		}
		if err := server.InboundHandlerManager().AddHandler(ctx, handler); err != nil {
			return nil, err
		}
	}

	for _, outbound := range config.Outbound {
		rawHandler, err := common.CreateObject(ctx, outbound)
		if err != nil {
			return nil, err
		}
		handler, ok := rawHandler.(OutboundHandler)
		if !ok {
			return nil, newError("not an OutboundHandler")
		}
		if err := server.OutboundHandlerManager().AddHandler(ctx, handler); err != nil {
			return nil, err
		}
	}

	return server, nil
}

// ID returns an unique ID for this V2Ray instance.
func (s *Instance) ID() uuid.UUID {
	return s.id
}

// Close shutdown the V2Ray instance.
func (s *Instance) Close() {
	for _, f := range s.features {
		f.Close()
	}
}

// Start starts the V2Ray instance, including all registered features. When Start returns error, the state of the instance is unknown.
func (s *Instance) Start() error {
	for _, f := range s.features {
		if err := f.Start(); err != nil {
			return err
		}
	}

	newError("V2Ray started").AtWarning().WriteToLog()

	return nil
}

// RegisterFeature registers the given feature into V2Ray.
// If feature is one of the following types, the corressponding feature in this Instance
// will be replaced: DNSClient, PolicyManager, Router, Dispatcher, InboundHandlerManager, OutboundHandlerManager.
func (s *Instance) RegisterFeature(feature interface{}, instance Feature) error {
	switch feature.(type) {
	case DNSClient, *DNSClient:
		s.dnsClient.Set(instance.(DNSClient))
	case PolicyManager, *PolicyManager:
		s.policyManager.Set(instance.(PolicyManager))
	case Router, *Router:
		s.router.Set(instance.(Router))
	case Dispatcher, *Dispatcher:
		s.dispatcher.Set(instance.(Dispatcher))
	case InboundHandlerManager, *InboundHandlerManager:
		s.ihm.Set(instance.(InboundHandlerManager))
	case OutboundHandlerManager, *OutboundHandlerManager:
		s.ohm.Set(instance.(OutboundHandlerManager))
	}
	s.features = append(s.features, instance)
	return nil
}

// DNSClient returns the DNSClient used by this Instance. The returned DNSClient is always functional.
func (s *Instance) DNSClient() DNSClient {
	return &(s.dnsClient)
}

// PolicyManager returns the PolicyManager used by this Instance. The returned PolicyManager is always functional.
func (s *Instance) PolicyManager() PolicyManager {
	return &(s.policyManager)
}

// Router returns the Router used by this Instance. The returned Router is always functional.
func (s *Instance) Router() Router {
	return &(s.router)
}

// Dispatcher returns the Dispatcher used by this Instance. If Dispatcher was not registered before, the returned value doesn't work, although it is not nil.
func (s *Instance) Dispatcher() Dispatcher {
	return &(s.dispatcher)
}

// InboundHandlerManager returns the InboundHandlerManager used by this Instance. If InboundHandlerManager was not registered before, the returned value doesn't work.
func (s *Instance) InboundHandlerManager() InboundHandlerManager {
	return &(s.ihm)
}

// OutboundHandlerManager returns the OutboundHandlerManager used by this Instance. If OutboundHandlerManager was not registered before, the returned value doesn't work.
func (s *Instance) OutboundHandlerManager() OutboundHandlerManager {
	return &(s.ohm)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"log"
	"net"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"git.edgecastcdn.net/vflow/packet"
	"git.edgecastcdn.net/vflow/sflow"
)

type SFServer struct {
	port        string
	addr        string
	laddr       *net.UDPAddr
	readTimeout time.Duration
	udpSize     int
	workers     int
	stop        bool
}

type UDPMsg struct {
	raddr *net.UDPAddr
	body  *bytes.Reader
}

var (
	udpChn = make(chan UDPMsg, 1000)
)

func (s *SFServer) run() {
	var (
		b  = make([]byte, s.udpSize)
		wg sync.WaitGroup
	)

	hostPort := net.JoinHostPort(s.addr, s.port)
	udpAddr, _ := net.ResolveUDPAddr("udp", hostPort)

	conn, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}

	for i := 0; i < s.workers; i++ {
		go func() {
			wg.Add(1)
			defer wg.Done()
			sFlowWorker()

		}()
	}

	for !s.stop {
		conn.SetReadDeadline(time.Now().Add(1e9))
		n, raddr, err := conn.ReadFromUDP(b)
		if err != nil {
			continue
		}
		udpChn <- UDPMsg{raddr, bytes.NewReader(b[:n])}
	}

	wg.Wait()
}

func (s *SFServer) shutdown() {
	s.stop = true
	log.Println("stopped sflow service gracefully ...")
	time.Sleep(1 * time.Second)
	log.Println("vFlow has been shutdown")
	close(udpChn)
}

func sFlowWorker() {
	var (
		msg    UDPMsg
		ok     bool
		filter = []uint32{sflow.DataCounterSample}
	)

	for {
		if msg, ok = <-udpChn; !ok {
			break
		}

		log.Println("rcvd sflow data", msg.body.Size())
		d := sflow.NewSFDecoder(msg.body, filter)
		data, err := d.SFDecode()
		if err != nil {
			log.Println(err)
		}

		switch data.(type) {
		case *packet.Packet:
			log.Printf("%#v\n", data)
		}
	}
}

func main() {
	var wg sync.WaitGroup
	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)

	sFlow := SFServer{
		port:    "6343",
		udpSize: 1500,
		workers: 10,
	}

	go func() {
		wg.Add(1)
		defer wg.Done()
		sFlow.run()
	}()

	<-signalCh
	sFlow.shutdown()
	wg.Wait()
}
<commit_msg>update vflow based on sparate sflow and adding options<commit_after>package main

import (
	"os"
	"os/signal"
	"sync"
	"syscall"
)

func main() {
	var (
		wg       sync.WaitGroup
		signalCh = make(chan os.Signal, 1)
		opts     = NewOptions()
	)

	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)

	sFlow := NewSFlow(opts)

	go func() {
		wg.Add(1)
		defer wg.Done()
		sFlow.run()
	}()

	<-signalCh
	sFlow.shutdown()
	wg.Wait()
}
<|endoftext|>


<commit_before>package golis

import (
	"errors"
	"net"
)

//Iosession
type Iosession struct {
	id        uint64
	serv      *ioserv
	conn      net.Conn
	closed    bool
	dataCh    chan interface{}
	userId    interface{}
	extraData map[string]interface{}
}

func (s *Iosession) Id() uint64 {
	return s.id
}

func (session *Iosession) SetUserId(id interface{}) {
	session.userId = id
}

func (session *Iosession) GetUserId() interface{} {
	return session.userId
}

func (session *Iosession) ExtraData(key string) (value interface{}, ok bool) {
	value, ok = session.extraData[key]
	return
}

func (session *Iosession) SetExtraData(key string, value interface{}) {
	session.extraData[key] = value
}

func (s *Iosession) Conn() net.Conn {
	return s.conn
}

func (s *Iosession) dealDataCh() {
	s.serv.wg.Add(1)
	defer s.serv.wg.Done()
	var msg interface{}
	for s.serv.runnable && !s.closed {
		select {
		case msg = <-s.dataCh:
			//fmt.Println("收到消息")
			s.serv.filterChain.msgReceived(s, msg)
		}
	}
}

func (session *Iosession) readData() {
	session.serv.wg.Add(1)
	ioBuffer := NewBuffer()
	buffer := make([]byte, 512)
	defer func() {
		if !session.closed {
			session.Close()
		}
		session.serv.wg.Done()
	}()
	var n int
	var err error
	for session.serv.runnable && !session.closed {
		n, err = session.conn.Read(buffer)
		ioBuffer.PutBytes(buffer[:n])
		if err != nil {
			session.serv.filterChain.errorCaught(session, err)
			session.Close()
			return
		}
		err = session.serv.codecer.Decode(ioBuffer, session.dataCh)
		if err != nil {
			session.serv.filterChain.errorCaught(session, err)
		}
	}
}

func (session *Iosession) Write(message interface{}) error {
	if session.serv.runnable && !session.closed {
		if msg, ok := session.serv.filterChain.msgSend(session, message); ok {
			bs, err := session.serv.codecer.Encode(msg)
			if err != nil {
				return err
			}
			_, err = session.conn.Write(bs)
			if err != nil {
				session.serv.filterChain.errorCaught(session, err)
			}
		}
		return nil
	} else {
		err := errors.New("Iosession is closed")
		session.serv.filterChain.errorCaught(session, err)
		return err
	}
}

//close iosession
func (this *Iosession) Close() {
	if !this.closed {
		this.serv.filterChain.sessionClosed(this)
		this.closed = true
	}
	this.conn.Close()
}
<commit_msg>recover panic<commit_after>package golis

import (
	"errors"
	"fmt"
	"net"
)

//Iosession
type Iosession struct {
	id        uint64
	serv      *ioserv
	conn      net.Conn
	closed    bool
	dataCh    chan interface{}
	userId    interface{}
	extraData map[string]interface{}
}

func (s *Iosession) Id() uint64 {
	return s.id
}

func (session *Iosession) SetUserId(id interface{}) {
	session.userId = id
}

func (session *Iosession) GetUserId() interface{} {
	return session.userId
}

func (session *Iosession) ExtraData(key string) (value interface{}, ok bool) {
	value, ok = session.extraData[key]
	return
}

func (session *Iosession) SetExtraData(key string, value interface{}) {
	session.extraData[key] = value
}

func (s *Iosession) Conn() net.Conn {
	return s.conn
}

func (s *Iosession) dealDataCh() {
	s.serv.wg.Add(1)
	defer s.serv.wg.Done()
	var msg interface{}
	for s.serv.runnable && !s.closed {
		select {
		case msg = <-s.dataCh:
			//fmt.Println("收到消息")
			s.serv.filterChain.msgReceived(s, msg)
		}
	}
}

func (session *Iosession) readData() {
	session.serv.wg.Add(1)
	ioBuffer := NewBuffer()
	buffer := make([]byte, 512)
	defer func() {
		if err := recover(); err != nil {
			fmt.Println(err)
		}
		if !session.closed {
			session.Close()
		}
		session.serv.wg.Done()
	}()
	var n int
	var err error
	for session.serv.runnable && !session.closed {
		n, err = session.conn.Read(buffer)
		ioBuffer.PutBytes(buffer[:n])
		if err != nil {
			session.serv.filterChain.errorCaught(session, err)
			session.Close()
			return
		}
		err = session.serv.codecer.Decode(ioBuffer, session.dataCh)
		if err != nil {
			session.serv.filterChain.errorCaught(session, err)
		}
	}
}

func (session *Iosession) Write(message interface{}) error {
	if session.serv.runnable && !session.closed {
		if msg, ok := session.serv.filterChain.msgSend(session, message); ok {
			bs, err := session.serv.codecer.Encode(msg)
			if err != nil {
				return err
			}
			_, err = session.conn.Write(bs)
			if err != nil {
				session.serv.filterChain.errorCaught(session, err)
			}
		}
		return nil
	} else {
		err := errors.New("Iosession is closed")
		session.serv.filterChain.errorCaught(session, err)
		return err
	}
}

//close iosession
func (this *Iosession) Close() {
	if !this.closed {
		this.serv.filterChain.sessionClosed(this)
		this.closed = true
	}
	this.conn.Close()
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ipv6

import "golang.org/x/net/internal/iana"

// An ICMPType represents a type of ICMP message.
type ICMPType int

func (typ ICMPType) String() string {
	s, ok := icmpTypes[typ]
	if !ok {
		return "<nil>"
	}
	return s
}

// Protocol returns the ICMPv6 protocol number.
func (typ ICMPType) Protocol() int {
	return iana.ProtocolIPv6ICMP
}

// An ICMPFilter represents an ICMP message filter for incoming
// packets. The filter belongs to a packet delivery path on a host and
// it cannot interact with forwarding packets or tunnel-outer packets.
//
// Note: RFC 2460 defines a reasonable role model. A node means a
// device that implements IP. A router means a node that forwards IP
// packets not explicitly addressed to itself, and a host means a node
// that is not a router.
type ICMPFilter struct {
	icmpv6Filter
}

// Accept accepts incoming ICMP packets including the type field value
// typ.
func (f *ICMPFilter) Accept(typ ICMPType) {
	f.accept(typ)
}

// Block blocks incoming ICMP packets including the type field value
// typ.
func (f *ICMPFilter) Block(typ ICMPType) {
	f.block(typ)
}

// SetAll sets the filter action to the filter.
func (f *ICMPFilter) SetAll(block bool) {
	f.setAll(block)
}

// WillBlock reports whether the ICMP type will be blocked.
func (f *ICMPFilter) WillBlock(typ ICMPType) bool {
	return f.willBlock(typ)
}
<commit_msg>ipv6: add missing BUGS section for Windows<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ipv6

import "golang.org/x/net/internal/iana"

// BUG(mikio): On Windows, methods related to ICMPFilter are not
// implemented.

// An ICMPType represents a type of ICMP message.
type ICMPType int

func (typ ICMPType) String() string {
	s, ok := icmpTypes[typ]
	if !ok {
		return "<nil>"
	}
	return s
}

// Protocol returns the ICMPv6 protocol number.
func (typ ICMPType) Protocol() int {
	return iana.ProtocolIPv6ICMP
}

// An ICMPFilter represents an ICMP message filter for incoming
// packets. The filter belongs to a packet delivery path on a host and
// it cannot interact with forwarding packets or tunnel-outer packets.
//
// Note: RFC 2460 defines a reasonable role model. A node means a
// device that implements IP. A router means a node that forwards IP
// packets not explicitly addressed to itself, and a host means a node
// that is not a router.
type ICMPFilter struct {
	icmpv6Filter
}

// Accept accepts incoming ICMP packets including the type field value
// typ.
func (f *ICMPFilter) Accept(typ ICMPType) {
	f.accept(typ)
}

// Block blocks incoming ICMP packets including the type field value
// typ.
func (f *ICMPFilter) Block(typ ICMPType) {
	f.block(typ)
}

// SetAll sets the filter action to the filter.
func (f *ICMPFilter) SetAll(block bool) {
	f.setAll(block)
}

// WillBlock reports whether the ICMP type will be blocked.
func (f *ICMPFilter) WillBlock(typ ICMPType) bool {
	return f.willBlock(typ)
}
<|endoftext|>


<commit_before>package etreeutils

import (
	"sort"
	"strings"

	"github.com/beevik/etree"
)

// TransformExcC14n transforms the passed element into xml-exc-c14n form.
func TransformExcC14n(el *etree.Element, inclusiveNamespacesPrefixList string) error {
	prefixes := strings.Fields(inclusiveNamespacesPrefixList)
	prefixSet := make(map[string]struct{}, len(prefixes))

	for _, prefix := range prefixes {
		prefixSet[prefix] = struct{}{}
	}

	err := transformExcC14n(DefaultNSContext, EmptyNSContext, el, prefixSet)
	if err != nil {
		return err
	}

	return nil
}

func transformExcC14n(ctx, declared NSContext, el *etree.Element, inclusiveNamespaces map[string]struct{}) error {
	scope, err := ctx.SubContext(el)
	if err != nil {
		return err
	}

	visiblyUtilizedPrefixes := map[string]struct{}{
		el.Space: struct{}{},
	}

	filteredAttrs := []etree.Attr{}

	// Filter out all namespace declarations
	for _, attr := range el.Attr {
		switch {
		case attr.Space == xmlnsPrefix:
			if _, ok := inclusiveNamespaces[attr.Key]; ok {
				visiblyUtilizedPrefixes[attr.Key] = struct{}{}
			}

		case attr.Space == defaultPrefix && attr.Key == xmlnsPrefix:
			if _, ok := inclusiveNamespaces[defaultPrefix]; ok {
				visiblyUtilizedPrefixes[defaultPrefix] = struct{}{}
			}

		default:
			if attr.Space != defaultPrefix {
				visiblyUtilizedPrefixes[attr.Space] = struct{}{}
			}

			filteredAttrs = append(filteredAttrs, attr)
		}
	}

	el.Attr = filteredAttrs

	newDeclared := declared.Copy()

	// Declare all visibly utilized prefixes that are in-scope but haven't
	// been declared in the canonicalized form yet. These might have been
	// declared on this element but then filtered out above, or they might
	// have been declared on an ancestor (before canonicalization) which
	// didn't visibly utilize and thus had them removed.
	for prefix := range visiblyUtilizedPrefixes {
		if _, ok := newDeclared.prefixes[prefix]; ok {
			continue
		}

		namespace, err := scope.LookupPrefix(prefix)
		if err != nil {
			return err
		}

		el.Attr = append(el.Attr, newDeclared.declare(prefix, namespace))
	}

	sort.Sort(SortedAttrs(el.Attr))

	// Transform child elements
	for _, child := range el.ChildElements() {
		err := transformExcC14n(scope, newDeclared, child, inclusiveNamespaces)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Fix an issue with overriding existing declarations<commit_after>package etreeutils

import (
	"sort"
	"strings"

	"github.com/beevik/etree"
)

// TransformExcC14n transforms the passed element into xml-exc-c14n form.
func TransformExcC14n(el *etree.Element, inclusiveNamespacesPrefixList string) error {
	prefixes := strings.Fields(inclusiveNamespacesPrefixList)
	prefixSet := make(map[string]struct{}, len(prefixes))

	for _, prefix := range prefixes {
		prefixSet[prefix] = struct{}{}
	}

	err := transformExcC14n(DefaultNSContext, EmptyNSContext, el, prefixSet)
	if err != nil {
		return err
	}

	return nil
}

func transformExcC14n(ctx, declared NSContext, el *etree.Element, inclusiveNamespaces map[string]struct{}) error {
	scope, err := ctx.SubContext(el)
	if err != nil {
		return err
	}

	visiblyUtilizedPrefixes := map[string]struct{}{
		el.Space: struct{}{},
	}

	filteredAttrs := []etree.Attr{}

	// Filter out all namespace declarations
	for _, attr := range el.Attr {
		switch {
		case attr.Space == xmlnsPrefix:
			if _, ok := inclusiveNamespaces[attr.Key]; ok {
				visiblyUtilizedPrefixes[attr.Key] = struct{}{}
			}

		case attr.Space == defaultPrefix && attr.Key == xmlnsPrefix:
			if _, ok := inclusiveNamespaces[defaultPrefix]; ok {
				visiblyUtilizedPrefixes[defaultPrefix] = struct{}{}
			}

		default:
			if attr.Space != defaultPrefix {
				visiblyUtilizedPrefixes[attr.Space] = struct{}{}
			}

			filteredAttrs = append(filteredAttrs, attr)
		}
	}

	el.Attr = filteredAttrs

	declared = declared.Copy()

	// Declare all visibly utilized prefixes that are in-scope but haven't
	// been declared in the canonicalized form yet. These might have been
	// declared on this element but then filtered out above, or they might
	// have been declared on an ancestor (before canonicalization) which
	// didn't visibly utilize and thus had them removed.
	for prefix := range visiblyUtilizedPrefixes {
		// Skip redundant declarations - they have to already have the same
		// value.
		if declaredNamespace, ok := declared.prefixes[prefix]; ok {
			if value, ok := scope.prefixes[prefix]; ok && declaredNamespace == value {
				continue
			}
		}

		namespace, err := scope.LookupPrefix(prefix)
		if err != nil {
			return err
		}

		el.Attr = append(el.Attr, declared.declare(prefix, namespace))
	}

	sort.Sort(SortedAttrs(el.Attr))

	// Transform child elements
	for _, child := range el.ChildElements() {
		err := transformExcC14n(scope, declared, child, inclusiveNamespaces)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// +build integrationtest

package integrationtest

import (
	"testing"

	"github.com/portworx/torpedo/drivers/scheduler"
	"github.com/stretchr/testify/require"
)

func testSnapshot(t *testing.T) {
	t.Run("simpleSnapshotTest", simpleSnapshotTest)
}

func simpleSnapshotTest(t *testing.T) {
	ctxs, err := schedulerDriver.Schedule(generateInstanceID(t, ""),
		scheduler.ScheduleOptions{AppKeys: []string{"mysql-snap-restore"}})
	require.NoError(t, err, "Error scheduling task")
	require.Equal(t, 1, len(ctxs), "Only one task should have started")

	err = schedulerDriver.WaitForRunning(ctxs[0])
	require.NoError(t, err, "Error waiting for pod to get to running state")

	scheduledNodes, err := schedulerDriver.GetNodesForApp(ctxs[0])
	require.NoError(t, err, "Error getting node for app")
	require.Equal(t, 1, len(scheduledNodes), "App should be scheduled on one node")

	err = schedulerDriver.InspectVolumes(ctxs[0])
	require.NoError(t, err, "Error waiting for volumes")
	volumeNames := getVolumeNames(t, ctxs[0])
	require.Equal(t, 3, len(volumeNames), "Should only have two volumes and a snapshot")

	snapVolInfo, err := storkVolumeDriver.InspectVolume("mysql-snapshot")
	require.NoError(t, err, "Error getting snapshot volume")
	require.NotNil(t, snapVolInfo.ParentID, "ParentID is nil for snapshot")

	parentVolInfo, err := storkVolumeDriver.InspectVolume(snapVolInfo.ParentID)
	require.NoError(t, err, "Error getting snapshot parent volume")

	parentVolName := parentVolInfo.VolumeName
	var cloneVolName string

	found := false
	for _, volume := range volumeNames {
		if volume == parentVolName {
			found = true
		} else {
			cloneVolName = volume
		}
	}
	require.True(t, found, "Parent volume (%v) not found in list of volumes: %v", parentVolName, volumeNames)

	cloneVolInfo, err := storkVolumeDriver.InspectVolume(cloneVolName)
	require.NoError(t, err, "Error getting clone volume")
	require.Equal(t, snapVolInfo.VolumeID, cloneVolInfo.ParentID, "Clone volume does not have snapshot as parent")

	verifyScheduledNode(t, scheduledNodes[0], volumeNames)

	destroyAndWait(t, ctxs)
}
<commit_msg>Pick clone volume correctly in test<commit_after>// +build integrationtest

package integrationtest

import (
	"testing"

	"github.com/portworx/torpedo/drivers/scheduler"
	"github.com/stretchr/testify/require"
)

func testSnapshot(t *testing.T) {
	t.Run("simpleSnapshotTest", simpleSnapshotTest)
}

func simpleSnapshotTest(t *testing.T) {
	ctxs, err := schedulerDriver.Schedule(generateInstanceID(t, ""),
		scheduler.ScheduleOptions{AppKeys: []string{"mysql-snap-restore"}})
	require.NoError(t, err, "Error scheduling task")
	require.Equal(t, 1, len(ctxs), "Only one task should have started")

	err = schedulerDriver.WaitForRunning(ctxs[0])
	require.NoError(t, err, "Error waiting for pod to get to running state")

	scheduledNodes, err := schedulerDriver.GetNodesForApp(ctxs[0])
	require.NoError(t, err, "Error getting node for app")
	require.Equal(t, 1, len(scheduledNodes), "App should be scheduled on one node")

	err = schedulerDriver.InspectVolumes(ctxs[0])
	require.NoError(t, err, "Error waiting for volumes")
	volumeNames := getVolumeNames(t, ctxs[0])
	require.Equal(t, 3, len(volumeNames), "Should only have two volumes and a snapshot")

	snapVolInfo, err := storkVolumeDriver.InspectVolume("mysql-snapshot")
	require.NoError(t, err, "Error getting snapshot volume")
	require.NotNil(t, snapVolInfo.ParentID, "ParentID is nil for snapshot")

	parentVolInfo, err := storkVolumeDriver.InspectVolume(snapVolInfo.ParentID)
	require.NoError(t, err, "Error getting snapshot parent volume")

	parentVolName := parentVolInfo.VolumeName
	var cloneVolName string

	found := false
	for _, volume := range volumeNames {
		if volume == parentVolName {
			found = true
		} else if volume != snapVolInfo.VolumeName {
			cloneVolName = volume
		}
	}
	require.True(t, found, "Parent volume (%v) not found in list of volumes: %v", parentVolName, volumeNames)

	cloneVolInfo, err := storkVolumeDriver.InspectVolume(cloneVolName)
	require.NoError(t, err, "Error getting clone volume")
	require.Equal(t, snapVolInfo.VolumeID, cloneVolInfo.ParentID, "Clone volume does not have snapshot as parent")

	verifyScheduledNode(t, scheduledNodes[0], volumeNames)

	destroyAndWait(t, ctxs)
}
<|endoftext|>


<commit_before>package eveConsumer

import (
	"context"
	"database/sql"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"

	"github.com/antihax/evedata/models"
	"github.com/antihax/goesi"
	"github.com/antihax/goesi/v1"
	"github.com/antihax/goesi/v3"
	"github.com/antihax/goesi/v4"
	"github.com/garyburd/redigo/redis"

	"golang.org/x/oauth2"
)

func init() {
	addConsumer("contactSync", contactSyncConsumer, "EVEDATA_contactSyncQueue")
	addTrigger("contactSync", contactSyncTrigger)
}

// Perform contact sync for wardecs
func contactSyncTrigger(c *EVEConsumer) (bool, error) {

	// Do quick maintenence to prevent errors.
	err := models.MaintContactSync()
	if err != nil {
		return false, err
	}

	// Gather characters for update. Group for optimized updating.
	rows, err := c.ctx.Db.Query(
		`SELECT S.characterID, source, group_concat(destination)
			FROM evedata.contactSyncs S
            INNER JOIN evedata.crestTokens T ON T.tokenCharacterID = destination
            WHERE lastStatus NOT LIKE "%400 Bad Request%"
		    GROUP BY source
            HAVING max(nextSync) < UTC_TIMESTAMP();`)

	if err != nil && err != sql.ErrNoRows {
		return false, err
	} else if err == sql.ErrNoRows { // Shut up warnings
		return false, nil
	}
	defer rows.Close()

	r := c.ctx.Cache.Get()
	defer r.Close()

	// Loop updatable characters
	for rows.Next() {
		var (
			characterID int64
			source      int64  // Source char
			dest        string // List of destination chars
		)

		err = rows.Scan(&characterID, &source, &dest)
		if err != nil {
			log.Printf("Contact Sync: Failed scan: %v", err)
			continue
		}

		_, err = r.Do("SADD", "EVEDATA_contactSyncQueue", fmt.Sprintf("%d:%d:%s", characterID, source, dest))
		if err != nil {
			log.Printf("Contact Sync: Failed scan: %v", err)
			continue
		}
	}
	return true, err
}

func contactSyncConsumer(c *EVEConsumer, redisPtr *redis.Conn) (bool, error) {
	r := *redisPtr
	ret, err := r.Do("SPOP", "EVEDATA_contactSyncQueue")
	if err != nil {
		return false, err
	} else if ret == nil {
		return false, nil
	}
	v, err := redis.String(ret, err)
	if err != nil {
		return false, err
	}

	// Split off characters into an array
	dest := strings.Split(v, ":")
	destinations := strings.Split(dest[2], ",")
	characterID, err := strconv.ParseInt(dest[0], 10, 64)
	source, err := strconv.ParseInt(dest[1], 10, 64)
	if err != nil {
		return false, err
	}

	var char goesiv4.GetCharactersCharacterIdOk
	for {
		var (
			r   *http.Response
			err error
		)
		// get the source character information
		char, r, err = c.ctx.ESI.V4.CharacterApi.GetCharactersCharacterId((int32)(source), nil)
		if err != nil {
			// Retry on their failure
			if r != nil && r.StatusCode >= 500 {
				continue
			}
			return false, err
		}
		break
	}

	var corp goesiv3.GetCorporationsCorporationIdOk
	for {
		var (
			r   *http.Response
			err error
		)
		corp, r, err = c.ctx.ESI.V3.CorporationApi.GetCorporationsCorporationId(char.CorporationId, nil)
		if err != nil {
			// Retry on their failure
			if r != nil && r.StatusCode >= 500 {
				continue
			}
			return false, err
		}
		break
	}

	// Find the Entity ID to search for wars.
	var searchID int32
	if corp.AllianceId > 0 {
		searchID = corp.AllianceId
	} else {
		searchID = char.CorporationId
	}

	// Map of tokens
	type characterToken struct {
		token *oauth2.TokenSource
		cid   int64
	}
	tokens := make(map[int64]characterToken)

	// Get the tokens for our destinations
	for _, cidS := range destinations {
		cid, _ := strconv.ParseInt(cidS, 10, 64)
		a, err := c.getToken(characterID, cid)
		if err != nil {
			return false, err
		}
		// Save the token.
		tokens[cid] = characterToken{token: &a, cid: cid}
	}

	// Active Wars
	activeWars, err := models.GetActiveWarsByID((int64)(searchID))
	if err != nil {
		return false, err
	}

	// Pending Wars
	pendingWars, err := models.GetPendingWarsByID((int64)(searchID))
	if err != nil {
		return false, err
	}

	// Faction Wars
	var factionWars []models.FactionWarEntities
	if corp.Faction != "" {
		factionWars, err = models.GetFactionWarEntitiesForID(models.FactionsByName[corp.Faction])
		if err != nil {
			return false, err
		}
	}

	// Loop through all the destinations
	for _, token := range tokens {
		// authentication token context for destination char
		auth := context.WithValue(context.TODO(), goesi.ContextOAuth2, *token.token)
		var (
			contacts []goesiv1.GetCharactersCharacterIdContacts200Ok
			r        *http.Response
			err      error
		)

		// Default to OK
		tokenSuccess(source, token.cid, 200, "OK")

		// Get current contacts
		for i := 1; ; i++ {
			var con []goesiv1.GetCharactersCharacterIdContacts200Ok
			con, r, err = c.ctx.ESI.V1.ContactsApi.GetCharactersCharacterIdContacts(auth, (int32)(token.cid), map[string]interface{}{"page": (int32)(i)})
			if err != nil || r.StatusCode != 200 {
				tokenError(source, token.cid, r, err)
				return false, err
			}
			if len(con) == 0 {
				break
			}
			contacts = append(contacts, con...)
		}

		// Update cache time.
		if r != nil {
			contactSync := &models.ContactSync{Source: source, Destination: token.cid}
			err := contactSync.Updated(goesi.CacheExpires(r))
			if err != nil {
				return false, err
			}
		}

		var erase []int32
		var active []int32
		var pending []int32
		var pendingMove []int32
		var activeMove []int32

		activeCheck := make(map[int32]bool)
		pendingCheck := make(map[int32]bool)

		// Build a map of active wars
		for _, war := range activeWars {
			activeCheck[(int32)(war.ID)] = true
		}

		// Add faction wars to the active list
		for _, war := range factionWars {
			activeCheck[(int32)(war.ID)] = true
		}

		// Build a map of pending wars
		for _, war := range pendingWars {
			pendingCheck[(int32)(war.ID)] = true
		}

		// Loop through all current contacts and figure out needed moves
		for _, contact := range contacts {
			// skip anything > -0.4
			if contact.Standing > -0.4 {
				continue
			}

			pend := pendingCheck[contact.ContactId]
			act := activeCheck[contact.ContactId]

			// Is this existing contact in the active list
			if !act {
				// Is this existing contact in the pending list
				if !pend { // Not in either list. delete it.
					erase = append(erase, (int32)(contact.ContactId))
				} else if pend && contact.Standing > -5.0 { // in pending list but wrong standing
					// Take it out of the active list and put into pending move.
					delete(pendingCheck, contact.ContactId)
					pendingMove = append(pendingMove, (int32)(contact.ContactId))
				} else if pend && contact.Standing == -5.0 { // Contact correct, do nothing.
					delete(pendingCheck, contact.ContactId)
				}
			} else if act && contact.Standing != -10.0 { // in active list, but wrong standing
				delete(activeCheck, contact.ContactId)
				activeMove = append(activeMove, (int32)(contact.ContactId))
			} else if act && contact.Standing == -10.0 { // Contact correct, do nothing.
				delete(activeCheck, contact.ContactId)
			}
		}

		// Build a list of active wars to add
		for con, _ := range activeCheck {
			active = append(active, con)
		}

		// Build a list of pending wars to add
		for con, _ := range pendingCheck {
			pending = append(pending, con)
		}

		// Erase contacts which have no wars.
		if len(erase) > 0 {
			for start := 0; start < len(erase); start = start + 20 {
				end := min(start+20, len(erase))
				for {
					r, err = c.ctx.ESI.V1.ContactsApi.DeleteCharactersCharacterIdContacts(auth, (int32)(token.cid), erase[start:end], nil)
					if err != nil {
						// Retry on their failure
						if r != nil && r.StatusCode >= 500 {
							continue
						}
						return false, err
					}
					break
				}
			}
		}
		// Add contacts for active wars
		if len(active) > 0 {
			for start := 0; start < len(active); start = start + 20 {
				end := min(start+20, len(active))
				for {
					_, r, err = c.ctx.ESI.V1.ContactsApi.PostCharactersCharacterIdContacts(auth, (int32)(token.cid), active[start:end], -10, nil)
					if err != nil {
						// Retry on their failure
						if r != nil && r.StatusCode >= 500 {
							continue
						}
						return false, err
					}
					break
				}
			}
		}
		// Add contacts for pending wars
		if len(pending) > 0 {
			for start := 0; start < len(pending); start = start + 20 {
				end := min(start+20, len(pending))
				for {
					_, r, err = c.ctx.ESI.V1.ContactsApi.PostCharactersCharacterIdContacts(auth, (int32)(token.cid), pending[start:end], -5, nil)
					if err != nil {
						// Retry on their failure
						if r != nil && r.StatusCode >= 500 {
							continue
						}
						return false, err
					}
					break
				}
			}
		}
		// Move contacts to active wars
		if len(activeMove) > 0 {
			for start := 0; start < len(activeMove); start = start + 20 {
				end := min(start+20, len(activeMove))
				for {
					r, err = c.ctx.ESI.V1.ContactsApi.PutCharactersCharacterIdContacts(auth, (int32)(token.cid), activeMove[start:end], -10, nil)
					if err != nil {
						// Retry on their failure
						if r != nil && r.StatusCode >= 500 {
							continue
						}
						return false, err
					}
					break
				}
			}
		}
		// Move contacts to pending wars
		if len(pendingMove) > 0 {
			for start := 0; start < len(pendingMove); start = start + 20 {
				end := min(start+20, len(pendingMove))
				for {
					r, err = c.ctx.ESI.V1.ContactsApi.PutCharactersCharacterIdContacts(auth, (int32)(token.cid), pendingMove[start:end], -5, nil)
					if err != nil {
						// Retry on their failure
						if r != nil && r.StatusCode >= 500 {
							continue
						}
						return false, err
					}
					break
				}
			}
		}
	}
	return true, err
}
<commit_msg>fail after 20 retries<commit_after>package eveConsumer

import (
	"context"
	"database/sql"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"

	"github.com/antihax/evedata/models"
	"github.com/antihax/goesi"
	"github.com/antihax/goesi/v1"
	"github.com/antihax/goesi/v3"
	"github.com/antihax/goesi/v4"
	"github.com/garyburd/redigo/redis"

	"golang.org/x/oauth2"
)

func init() {
	addConsumer("contactSync", contactSyncConsumer, "EVEDATA_contactSyncQueue")
	addTrigger("contactSync", contactSyncTrigger)
}

// Perform contact sync for wardecs
func contactSyncTrigger(c *EVEConsumer) (bool, error) {

	// Do quick maintenence to prevent errors.
	err := models.MaintContactSync()
	if err != nil {
		return false, err
	}

	// Gather characters for update. Group for optimized updating.
	rows, err := c.ctx.Db.Query(
		`SELECT S.characterID, source, group_concat(destination)
			FROM evedata.contactSyncs S
            INNER JOIN evedata.crestTokens T ON T.tokenCharacterID = destination
            WHERE lastStatus NOT LIKE "%400 Bad Request%"
		    GROUP BY source
            HAVING max(nextSync) < UTC_TIMESTAMP();`)

	if err != nil && err != sql.ErrNoRows {
		return false, err
	} else if err == sql.ErrNoRows { // Shut up warnings
		return false, nil
	}
	defer rows.Close()

	r := c.ctx.Cache.Get()
	defer r.Close()

	// Loop updatable characters
	for rows.Next() {
		var (
			characterID int64
			source      int64  // Source char
			dest        string // List of destination chars
		)

		err = rows.Scan(&characterID, &source, &dest)
		if err != nil {
			log.Printf("Contact Sync: Failed scan: %v", err)
			continue
		}

		_, err = r.Do("SADD", "EVEDATA_contactSyncQueue", fmt.Sprintf("%d:%d:%s", characterID, source, dest))
		if err != nil {
			log.Printf("Contact Sync: Failed scan: %v", err)
			continue
		}
	}
	return true, err
}

func contactSyncConsumer(c *EVEConsumer, redisPtr *redis.Conn) (bool, error) {
	r := *redisPtr
	ret, err := r.Do("SPOP", "EVEDATA_contactSyncQueue")
	if err != nil {
		return false, err
	} else if ret == nil {
		return false, nil
	}
	v, err := redis.String(ret, err)
	if err != nil {
		return false, err
	}

	// Split off characters into an array
	dest := strings.Split(v, ":")
	destinations := strings.Split(dest[2], ",")
	characterID, err := strconv.ParseInt(dest[0], 10, 64)
	source, err := strconv.ParseInt(dest[1], 10, 64)
	if err != nil {
		return false, err
	}

	var char goesiv4.GetCharactersCharacterIdOk
	for {
		var (
			r   *http.Response
			err error
		)
		// get the source character information
		char, r, err = c.ctx.ESI.V4.CharacterApi.GetCharactersCharacterId((int32)(source), nil)
		if err != nil {
			// Retry on their failure
			if r != nil && r.StatusCode >= 500 {
				continue
			}
			return false, err
		}
		break
	}

	var corp goesiv3.GetCorporationsCorporationIdOk
	for {
		var (
			r   *http.Response
			err error
		)
		corp, r, err = c.ctx.ESI.V3.CorporationApi.GetCorporationsCorporationId(char.CorporationId, nil)
		if err != nil {
			// Retry on their failure
			if r != nil && r.StatusCode >= 500 {
				continue
			}
			return false, err
		}
		break
	}

	// Find the Entity ID to search for wars.
	var searchID int32
	if corp.AllianceId > 0 {
		searchID = corp.AllianceId
	} else {
		searchID = char.CorporationId
	}

	// Map of tokens
	type characterToken struct {
		token *oauth2.TokenSource
		cid   int64
	}
	tokens := make(map[int64]characterToken)

	// Get the tokens for our destinations
	for _, cidS := range destinations {
		cid, _ := strconv.ParseInt(cidS, 10, 64)
		a, err := c.getToken(characterID, cid)
		if err != nil {
			return false, err
		}
		// Save the token.
		tokens[cid] = characterToken{token: &a, cid: cid}
	}

	// Active Wars
	activeWars, err := models.GetActiveWarsByID((int64)(searchID))
	if err != nil {
		return false, err
	}

	// Pending Wars
	pendingWars, err := models.GetPendingWarsByID((int64)(searchID))
	if err != nil {
		return false, err
	}

	// Faction Wars
	var factionWars []models.FactionWarEntities
	if corp.Faction != "" {
		factionWars, err = models.GetFactionWarEntitiesForID(models.FactionsByName[corp.Faction])
		if err != nil {
			return false, err
		}
	}

	// Loop through all the destinations
	for _, token := range tokens {
		// authentication token context for destination char
		auth := context.WithValue(context.TODO(), goesi.ContextOAuth2, *token.token)
		var (
			contacts []goesiv1.GetCharactersCharacterIdContacts200Ok
			r        *http.Response
			err      error
		)

		// Default to OK
		tokenSuccess(source, token.cid, 200, "OK")

		// Get current contacts
		for i := 1; ; i++ {
			var con []goesiv1.GetCharactersCharacterIdContacts200Ok
			con, r, err = c.ctx.ESI.V1.ContactsApi.GetCharactersCharacterIdContacts(auth, (int32)(token.cid), map[string]interface{}{"page": (int32)(i)})
			if err != nil || r.StatusCode != 200 {
				tokenError(source, token.cid, r, err)
				return false, err
			}
			if len(con) == 0 {
				break
			}
			contacts = append(contacts, con...)
		}

		// Update cache time.
		if r != nil {
			contactSync := &models.ContactSync{Source: source, Destination: token.cid}
			err := contactSync.Updated(goesi.CacheExpires(r))
			if err != nil {
				return false, err
			}
		}

		var erase []int32
		var active []int32
		var pending []int32
		var pendingMove []int32
		var activeMove []int32

		activeCheck := make(map[int32]bool)
		pendingCheck := make(map[int32]bool)

		// Build a map of active wars
		for _, war := range activeWars {
			activeCheck[(int32)(war.ID)] = true
		}

		// Add faction wars to the active list
		for _, war := range factionWars {
			activeCheck[(int32)(war.ID)] = true
		}

		// Build a map of pending wars
		for _, war := range pendingWars {
			pendingCheck[(int32)(war.ID)] = true
		}

		// Loop through all current contacts and figure out needed moves
		for _, contact := range contacts {
			// skip anything > -0.4
			if contact.Standing > -0.4 {
				continue
			}

			pend := pendingCheck[contact.ContactId]
			act := activeCheck[contact.ContactId]

			// Is this existing contact in the active list
			if !act {
				// Is this existing contact in the pending list
				if !pend { // Not in either list. delete it.
					erase = append(erase, (int32)(contact.ContactId))
				} else if pend && contact.Standing > -5.0 { // in pending list but wrong standing
					// Take it out of the active list and put into pending move.
					delete(pendingCheck, contact.ContactId)
					pendingMove = append(pendingMove, (int32)(contact.ContactId))
				} else if pend && contact.Standing == -5.0 { // Contact correct, do nothing.
					delete(pendingCheck, contact.ContactId)
				}
			} else if act && contact.Standing != -10.0 { // in active list, but wrong standing
				delete(activeCheck, contact.ContactId)
				activeMove = append(activeMove, (int32)(contact.ContactId))
			} else if act && contact.Standing == -10.0 { // Contact correct, do nothing.
				delete(activeCheck, contact.ContactId)
			}
		}

		// Build a list of active wars to add
		for con, _ := range activeCheck {
			active = append(active, con)
		}

		// Build a list of pending wars to add
		for con, _ := range pendingCheck {
			pending = append(pending, con)
		}

		// Erase contacts which have no wars.
		if len(erase) > 0 {
			for start := 0; start < len(erase); start = start + 20 {
				end := min(start+20, len(erase))
				failure:=0
				for {
					r, err = c.ctx.ESI.V1.ContactsApi.DeleteCharactersCharacterIdContacts(auth, (int32)(token.cid), erase[start:end], nil)
					if err != nil {
						// Retry on their failure
						if failure > 20 {
							break
						} else if r != nil && r.StatusCode >= 500 {
							continue
							failure++
						}
						return false, err
					}
					break
				}
			}
		}
		// Add contacts for active wars
		if len(active) > 0 {
			for start := 0; start < len(active); start = start + 20 {
				end := min(start+20, len(active))
				failure :=0
				for {
					_, r, err = c.ctx.ESI.V1.ContactsApi.PostCharactersCharacterIdContacts(auth, (int32)(token.cid), active[start:end], -10, nil)
						if err != nil {
						// Retry on their failure
						if failure > 20 {
							break
						} else if r != nil && r.StatusCode >= 500 {
							continue
							failure++
						}
						return false, err
					}
					break
				}
			}
		}
		// Add contacts for pending wars
		if len(pending) > 0 {
			for start := 0; start < len(pending); start = start + 20 {
				end := min(start+20, len(pending))
				failure :=0
				for {
					_, r, err = c.ctx.ESI.V1.ContactsApi.PostCharactersCharacterIdContacts(auth, (int32)(token.cid), pending[start:end], -5, nil)
					if err != nil {
						// Retry on their failure
						if failure > 20 {
							break
						} else if r != nil && r.StatusCode >= 500 {
							continue
							failure++
						}
						return false, err
					}
					break
				}
			}
		}
		// Move contacts to active wars
		if len(activeMove) > 0 {
			for start := 0; start < len(activeMove); start = start + 20 {
				end := min(start+20, len(activeMove))
				for {
					r, err = c.ctx.ESI.V1.ContactsApi.PutCharactersCharacterIdContacts(auth, (int32)(token.cid), activeMove[start:end], -10, nil)
					if err != nil {
						// Retry on their failure
						if r != nil && r.StatusCode >= 500 {
							continue
						}
						return false, err
					}
					break
				}
			}
		}
		// Move contacts to pending wars
		if len(pendingMove) > 0 {
			for start := 0; start < len(pendingMove); start = start + 20 {
				end := min(start+20, len(pendingMove))
				for {
					r, err = c.ctx.ESI.V1.ContactsApi.PutCharactersCharacterIdContacts(auth, (int32)(token.cid), pendingMove[start:end], -5, nil)
					if err != nil {
						// Retry on their failure
						if r != nil && r.StatusCode >= 500 {
							continue
						}
						return false, err
					}
					break
				}
			}
		}
	}
	return true, err
}
<|endoftext|>