<commit_before>// Copyright (c) 2014-2015 The Notify Authors. All rights reserved.
// Use of this source code is governed by the MIT license that can be
// found in the LICENSE file.

// +build darwin,kqueue dragonfly freebsd netbsd openbsd solaris

// watcher_trigger is used for FEN and kqueue which behave similarly:
// only files and dirs can be watched directly, but not files inside dirs.
// As a result Create events have to be generated by implementation when
// after Write event is returned for watched dir, it is rescanned and Create
// event is returned for new files and these are automatically added
// to watchlist. In case of removal of watched directory, native system returns
// events for all files, but for Rename, they also need to be generated.
// As a result native system works as something like trigger for rescan,
// but contains additional data about dir in which changes occurred. For files
// detailed data is returned.
// Usage of watcher_trigger requires:
// - trigger implementation,
// - encode func,
// - not2nat, nat2not maps.
// Required manual operations on filesystem can lead to loss of precision.

package notify

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
)

// trigger is to be implemented by platform implementation like FEN or kqueue.
type trigger interface {
	// Close closes watcher's main native file descriptor.
	Close() error
	// Stop waiting for new events.
	Stop() error
	// Create new instance of watched.
	NewWatched(string, os.FileInfo) (*watched, error)
	// Record internally new *watched instance.
	Record(*watched)
	// Del removes internal copy of *watched instance.
	Del(*watched)
	// Watched returns *watched instance and native events for native type.
	Watched(interface{}) (*watched, int64, error)
	// Init initializes native watcher call.
	Init() error
	// Watch starts watching provided file/dir.
	Watch(os.FileInfo, *watched, int64) error
	// Unwatch stops watching provided file/dir.
	Unwatch(*watched) error
	// Wait for new events.
	Wait() (interface{}, error)
	// IsStop checks if Wait finished because of request watcher's stop.
	IsStop(n interface{}, err error) bool
}

// encode Event to native representation. Implementation is to be provided by
// platform specific implementation.
var encode func(Event, bool) int64

var (
	// nat2not matches native events to notify's ones. To be initialized by
	// platform dependent implementation.
	nat2not map[Event]Event
	// not2nat matches notify's events to native ones. To be initialized by
	// platform dependent implementation.
	not2nat map[Event]Event
)

// trg is a main structure implementing watcher.
type trg struct {
	sync.Mutex
	// s is a channel used to stop monitoring.
	s chan struct{}
	// c is a channel used to pass events further.
	c chan<- EventInfo
	// pthLkp is a data structure mapping file names with data about watching
	// represented by them files/directories.
	pthLkp map[string]*watched
	// t is a platform dependent implementation of trigger.
	t trigger
}

// newWatcher returns new watcher's implementation.
func newWatcher(c chan<- EventInfo) watcher {
	t := &trg{
		s:      make(chan struct{}, 1),
		pthLkp: make(map[string]*watched, 0),
		c:      c,
	}
	t.t = newTrigger(t.pthLkp)
	if err := t.t.Init(); err != nil {
		panic(err)
	}
	go t.monitor()
	return t
}

// Close implements watcher.
func (t *trg) Close() (err error) {
	t.Lock()
	if err = t.t.Stop(); err != nil {
		t.Unlock()
		return
	}
	<-t.s
	var e error
	for _, w := range t.pthLkp {
		if e = t.unwatch(w.p, w.fi); e != nil {
			dbgprintf("trg: unwatch %q failed: %q\n", w.p, e)
			err = nonil(err, e)
		}
	}
	if e = t.t.Close(); e != nil {
		dbgprintf("trg: closing native watch failed: %q\n", e)
		err = nonil(err, e)
	}
	if remaining := len(t.pthLkp); remaining != 0 {
		err = nonil(err, fmt.Errorf("Not all watches were removed: len(t.pthLkp) == %v", len(t.pthLkp)))
	}
	t.Unlock()
	return
}

// send reported events one by one through chan.
func (t *trg) send(evn []event) {
	for i := range evn {
		t.c <- &evn[i]
	}
}

// singlewatch starts to watch given p file/directory.
func (t *trg) singlewatch(p string, e Event, direct mode, fi os.FileInfo) (err error) {
	w, ok := t.pthLkp[p]
	if !ok {
		if w, err = t.t.NewWatched(p, fi); err != nil {
			return
		}
	}
	switch direct {
	case dir:
		w.eDir |= e
	case ndir:
		w.eNonDir |= e
	case both:
		w.eDir |= e
		w.eNonDir |= e
	}
	if err = t.t.Watch(fi, w, encode(w.eDir|w.eNonDir, fi.IsDir())); err != nil {
		return
	}
	if !ok {
		t.t.Record(w)
		return nil
	}
	return errAlreadyWatched
}

// decode converts event received from native to notify.Event
// representation taking into account requested events (w).
func decode(o int64, w Event) (e Event) {
	for f, n := range nat2not {
		if o&int64(f) != 0 {
			if w&f != 0 {
				e |= f
			}
			if w&n != 0 {
				e |= n
			}
		}
	}

	return
}

func (t *trg) watch(p string, e Event, fi os.FileInfo) error {
	if err := t.singlewatch(p, e, dir, fi); err != nil {
		if err != errAlreadyWatched {
			return nil
		}
	}
	if fi.IsDir() {
		err := t.walk(p, func(fi os.FileInfo) (err error) {
			if err = t.singlewatch(filepath.Join(p, fi.Name()), e, ndir,
				fi); err != nil {
				if err != errAlreadyWatched {
					return
				}
			}
			return nil
		})
		if err != nil {
			return err
		}
	}
	return nil
}

// walk runs f func on each file/dir from p directory.
func (t *trg) walk(p string, fn func(os.FileInfo) error) error {
	fp, err := os.Open(p)
	if err != nil {
		return err
	}
	ls, err := fp.Readdir(0)
	fp.Close()
	if err != nil {
		return err
	}
	for i := range ls {
		if err := fn(ls[i]); err != nil {
			return err
		}
	}
	return nil
}

func (t *trg) unwatch(p string, fi os.FileInfo) error {
	if fi.IsDir() {
		err := t.walk(p, func(fi os.FileInfo) error {
			err := t.singleunwatch(filepath.Join(p, fi.Name()), ndir)
			if err != errNotWatched {
				return err
			}
			return nil
		})
		if err != nil {
			return err
		}
	}
	return t.singleunwatch(p, dir)
}

// Watch implements Watcher interface.
func (t *trg) Watch(p string, e Event) error {
	fi, err := os.Stat(p)
	if err != nil {
		return err
	}
	t.Lock()
	err = t.watch(p, e, fi)
	t.Unlock()
	return err
}

// Unwatch implements Watcher interface.
func (t *trg) Unwatch(p string) error {
	fi, err := os.Stat(p)
	if err != nil {
		return err
	}
	t.Lock()
	err = t.unwatch(p, fi)
	t.Unlock()
	return err
}

// Rewatch implements Watcher interface.
//
// TODO(rjeczalik): This is a naive hack. Rewrite might help.
func (t *trg) Rewatch(p string, _, e Event) error {
	fi, err := os.Stat(p)
	if err != nil {
		return err
	}
	t.Lock()
	if err = t.unwatch(p, fi); err == nil {
		// TODO(rjeczalik): If watch fails then we leave trigger in inconsistent
		// state. Handle? Panic? Native version of rewatch?
		err = t.watch(p, e, fi)
	}
	t.Unlock()
	return nil
}

func (*trg) file(w *watched, n interface{}, e Event) (evn []event) {
	evn = append(evn, event{w.p, e, w.fi.IsDir(), n})
	return
}

func (t *trg) dir(w *watched, n interface{}, e, ge Event) (evn []event) {
	// If it's dir and delete we have to send it and continue, because
	// other processing relies on opening (in this case not existing) dir.
	// Events for contents of this dir are reported by native impl.
	// However events for rename must be generated for all monitored files
	// inside of moved directory, because native impl does not report it independently
	// for each file descriptor being moved in result of move action on
	// parent directory.
	if (ge & (not2nat[Rename] | not2nat[Remove])) != 0 {
		// Write is reported also for Remove on directory. Because of that
		// we have to filter it out explicitly.
		evn = append(evn, event{w.p, e & ^Write & ^not2nat[Write], true, n})
		if ge&not2nat[Rename] != 0 {
			for p := range t.pthLkp {
				if strings.HasPrefix(p, w.p+string(os.PathSeparator)) {
					if err := t.singleunwatch(p, both); err != nil && err != errNotWatched &&
						!os.IsNotExist(err) {
						dbgprintf("trg: failed stop watching moved file (%q): %q\n",
							p, err)
					}
					if (w.eDir|w.eNonDir)&(not2nat[Rename]|Rename) != 0 {
						evn = append(evn, event{
							p, (w.eDir | w.eNonDir) & e &^ Write &^ not2nat[Write],
							w.fi.IsDir(), nil,
						})
					}
				}
			}
		}
		t.t.Del(w)
		return
	}
	if (ge & not2nat[Write]) != 0 {
		switch err := t.walk(w.p, func(fi os.FileInfo) error {
			p := filepath.Join(w.p, fi.Name())
			switch err := t.singlewatch(p, w.eDir, ndir, fi); {
			case os.IsNotExist(err) && ((w.eDir & Remove) != 0):
				evn = append(evn, event{p, Remove, fi.IsDir(), n})
			case err == errAlreadyWatched:
			case err != nil:
				dbgprintf("trg: watching %q failed: %q", p, err)
			case (w.eDir & Create) != 0:
				evn = append(evn, event{p, Create, fi.IsDir(), n})
			default:
			}
			return nil
		}); {
		case os.IsNotExist(err):
			return
		case err != nil:
			dbgprintf("trg: dir processing failed: %q", err)
		default:
		}
	}
	return
}

type mode uint

const (
	dir mode = iota
	ndir
	both
)

// unwatch stops watching p file/directory.
func (t *trg) singleunwatch(p string, direct mode) error {
	w, ok := t.pthLkp[p]
	if !ok {
		return errNotWatched
	}
	switch direct {
	case dir:
		w.eDir = 0
	case ndir:
		w.eNonDir = 0
	case both:
		w.eDir, w.eNonDir = 0, 0
	}
	if err := t.t.Unwatch(w); err != nil {
		return err
	}
	if w.eNonDir|w.eDir != 0 {
		mod := dir
		if w.eNonDir != 0 {
			mod = ndir
		}
		if err := t.singlewatch(p, w.eNonDir|w.eDir, mod,
			w.fi); err != nil && err != errAlreadyWatched {
			return err
		}
	} else {
		t.t.Del(w)
	}
	return nil
}

func (t *trg) monitor() {
	var (
		n   interface{}
		err error
	)
	for {
		switch n, err = t.t.Wait(); {
		case err == syscall.EINTR:
		case t.t.IsStop(n, err):
			t.s <- struct{}{}
			return
		case err != nil:
			dbgprintf("trg: failed to read events: %q\n", err)
		default:
			t.send(t.process(n))
		}
	}
}

// process event returned by native call.
func (t *trg) process(n interface{}) (evn []event) {
	t.Lock()
	w, ge, err := t.t.Watched(n)
	if err != nil {
		t.Unlock()
		dbgprintf("trg: %v event lookup failed: %q", Event(ge), err)
		return
	}

	e := decode(ge, w.eDir|w.eNonDir)
	if ge&int64(not2nat[Remove]|not2nat[Rename]) == 0 {
		switch fi, err := os.Stat(w.p); {
		case err != nil:
		default:
			if err = t.t.Watch(fi, w, encode(w.eDir|w.eNonDir, fi.IsDir())); err != nil {
				dbgprintf("trg: %q is no longer watched: %q", w.p, err)
				t.t.Del(w)
			}
		}
	}
	if e == Event(0) && (!w.fi.IsDir() || (ge&int64(not2nat[Write])) == 0) {
		t.Unlock()
		return
	}

	if w.fi.IsDir() {
		evn = append(evn, t.dir(w, n, e, Event(ge))...)
	} else {
		evn = append(evn, t.file(w, n, e)...)
	}
	if Event(ge)&(not2nat[Remove]|not2nat[Rename]) != 0 {
		t.t.Del(w)
	}
	t.Unlock()
	return
}
<commit_msg>trigger: typo when watching fails (return nil -> err)<commit_after>// Copyright (c) 2014-2015 The Notify Authors. All rights reserved.
// Use of this source code is governed by the MIT license that can be
// found in the LICENSE file.

// +build darwin,kqueue dragonfly freebsd netbsd openbsd solaris

// watcher_trigger is used for FEN and kqueue which behave similarly:
// only files and dirs can be watched directly, but not files inside dirs.
// As a result Create events have to be generated by implementation when
// after Write event is returned for watched dir, it is rescanned and Create
// event is returned for new files and these are automatically added
// to watchlist. In case of removal of watched directory, native system returns
// events for all files, but for Rename, they also need to be generated.
// As a result native system works as something like trigger for rescan,
// but contains additional data about dir in which changes occurred. For files
// detailed data is returned.
// Usage of watcher_trigger requires:
// - trigger implementation,
// - encode func,
// - not2nat, nat2not maps.
// Required manual operations on filesystem can lead to loss of precision.

package notify

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
)

// trigger is to be implemented by platform implementation like FEN or kqueue.
type trigger interface {
	// Close closes watcher's main native file descriptor.
	Close() error
	// Stop waiting for new events.
	Stop() error
	// Create new instance of watched.
	NewWatched(string, os.FileInfo) (*watched, error)
	// Record internally new *watched instance.
	Record(*watched)
	// Del removes internal copy of *watched instance.
	Del(*watched)
	// Watched returns *watched instance and native events for native type.
	Watched(interface{}) (*watched, int64, error)
	// Init initializes native watcher call.
	Init() error
	// Watch starts watching provided file/dir.
	Watch(os.FileInfo, *watched, int64) error
	// Unwatch stops watching provided file/dir.
	Unwatch(*watched) error
	// Wait for new events.
	Wait() (interface{}, error)
	// IsStop checks if Wait finished because of request watcher's stop.
	IsStop(n interface{}, err error) bool
}

// encode Event to native representation. Implementation is to be provided by
// platform specific implementation.
var encode func(Event, bool) int64

var (
	// nat2not matches native events to notify's ones. To be initialized by
	// platform dependent implementation.
	nat2not map[Event]Event
	// not2nat matches notify's events to native ones. To be initialized by
	// platform dependent implementation.
	not2nat map[Event]Event
)

// trg is a main structure implementing watcher.
type trg struct {
	sync.Mutex
	// s is a channel used to stop monitoring.
	s chan struct{}
	// c is a channel used to pass events further.
	c chan<- EventInfo
	// pthLkp is a data structure mapping file names with data about watching
	// represented by them files/directories.
	pthLkp map[string]*watched
	// t is a platform dependent implementation of trigger.
	t trigger
}

// newWatcher returns new watcher's implementation.
func newWatcher(c chan<- EventInfo) watcher {
	t := &trg{
		s:      make(chan struct{}, 1),
		pthLkp: make(map[string]*watched, 0),
		c:      c,
	}
	t.t = newTrigger(t.pthLkp)
	if err := t.t.Init(); err != nil {
		panic(err)
	}
	go t.monitor()
	return t
}

// Close implements watcher.
func (t *trg) Close() (err error) {
	t.Lock()
	if err = t.t.Stop(); err != nil {
		t.Unlock()
		return
	}
	<-t.s
	var e error
	for _, w := range t.pthLkp {
		if e = t.unwatch(w.p, w.fi); e != nil {
			dbgprintf("trg: unwatch %q failed: %q\n", w.p, e)
			err = nonil(err, e)
		}
	}
	if e = t.t.Close(); e != nil {
		dbgprintf("trg: closing native watch failed: %q\n", e)
		err = nonil(err, e)
	}
	if remaining := len(t.pthLkp); remaining != 0 {
		err = nonil(err, fmt.Errorf("Not all watches were removed: len(t.pthLkp) == %v", len(t.pthLkp)))
	}
	t.Unlock()
	return
}

// send reported events one by one through chan.
func (t *trg) send(evn []event) {
	for i := range evn {
		t.c <- &evn[i]
	}
}

// singlewatch starts to watch given p file/directory.
func (t *trg) singlewatch(p string, e Event, direct mode, fi os.FileInfo) (err error) {
	w, ok := t.pthLkp[p]
	if !ok {
		if w, err = t.t.NewWatched(p, fi); err != nil {
			return
		}
	}
	switch direct {
	case dir:
		w.eDir |= e
	case ndir:
		w.eNonDir |= e
	case both:
		w.eDir |= e
		w.eNonDir |= e
	}
	if err = t.t.Watch(fi, w, encode(w.eDir|w.eNonDir, fi.IsDir())); err != nil {
		return
	}
	if !ok {
		t.t.Record(w)
		return nil
	}
	return errAlreadyWatched
}

// decode converts event received from native to notify.Event
// representation taking into account requested events (w).
func decode(o int64, w Event) (e Event) {
	for f, n := range nat2not {
		if o&int64(f) != 0 {
			if w&f != 0 {
				e |= f
			}
			if w&n != 0 {
				e |= n
			}
		}
	}

	return
}

func (t *trg) watch(p string, e Event, fi os.FileInfo) error {
	if err := t.singlewatch(p, e, dir, fi); err != nil {
		if err != errAlreadyWatched {
			return err
		}
	}
	if fi.IsDir() {
		err := t.walk(p, func(fi os.FileInfo) (err error) {
			if err = t.singlewatch(filepath.Join(p, fi.Name()), e, ndir,
				fi); err != nil {
				if err != errAlreadyWatched {
					return
				}
			}
			return nil
		})
		if err != nil {
			return err
		}
	}
	return nil
}

// walk runs f func on each file/dir from p directory.
func (t *trg) walk(p string, fn func(os.FileInfo) error) error {
	fp, err := os.Open(p)
	if err != nil {
		return err
	}
	ls, err := fp.Readdir(0)
	fp.Close()
	if err != nil {
		return err
	}
	for i := range ls {
		if err := fn(ls[i]); err != nil {
			return err
		}
	}
	return nil
}

func (t *trg) unwatch(p string, fi os.FileInfo) error {
	if fi.IsDir() {
		err := t.walk(p, func(fi os.FileInfo) error {
			err := t.singleunwatch(filepath.Join(p, fi.Name()), ndir)
			if err != errNotWatched {
				return err
			}
			return nil
		})
		if err != nil {
			return err
		}
	}
	return t.singleunwatch(p, dir)
}

// Watch implements Watcher interface.
func (t *trg) Watch(p string, e Event) error {
	fi, err := os.Stat(p)
	if err != nil {
		return err
	}
	t.Lock()
	err = t.watch(p, e, fi)
	t.Unlock()
	return err
}

// Unwatch implements Watcher interface.
func (t *trg) Unwatch(p string) error {
	fi, err := os.Stat(p)
	if err != nil {
		return err
	}
	t.Lock()
	err = t.unwatch(p, fi)
	t.Unlock()
	return err
}

// Rewatch implements Watcher interface.
//
// TODO(rjeczalik): This is a naive hack. Rewrite might help.
func (t *trg) Rewatch(p string, _, e Event) error {
	fi, err := os.Stat(p)
	if err != nil {
		return err
	}
	t.Lock()
	if err = t.unwatch(p, fi); err == nil {
		// TODO(rjeczalik): If watch fails then we leave trigger in inconsistent
		// state. Handle? Panic? Native version of rewatch?
		err = t.watch(p, e, fi)
	}
	t.Unlock()
	return nil
}

func (*trg) file(w *watched, n interface{}, e Event) (evn []event) {
	evn = append(evn, event{w.p, e, w.fi.IsDir(), n})
	return
}

func (t *trg) dir(w *watched, n interface{}, e, ge Event) (evn []event) {
	// If it's dir and delete we have to send it and continue, because
	// other processing relies on opening (in this case not existing) dir.
	// Events for contents of this dir are reported by native impl.
	// However events for rename must be generated for all monitored files
	// inside of moved directory, because native impl does not report it independently
	// for each file descriptor being moved in result of move action on
	// parent directory.
	if (ge & (not2nat[Rename] | not2nat[Remove])) != 0 {
		// Write is reported also for Remove on directory. Because of that
		// we have to filter it out explicitly.
		evn = append(evn, event{w.p, e & ^Write & ^not2nat[Write], true, n})
		if ge&not2nat[Rename] != 0 {
			for p := range t.pthLkp {
				if strings.HasPrefix(p, w.p+string(os.PathSeparator)) {
					if err := t.singleunwatch(p, both); err != nil && err != errNotWatched &&
						!os.IsNotExist(err) {
						dbgprintf("trg: failed stop watching moved file (%q): %q\n",
							p, err)
					}
					if (w.eDir|w.eNonDir)&(not2nat[Rename]|Rename) != 0 {
						evn = append(evn, event{
							p, (w.eDir | w.eNonDir) & e &^ Write &^ not2nat[Write],
							w.fi.IsDir(), nil,
						})
					}
				}
			}
		}
		t.t.Del(w)
		return
	}
	if (ge & not2nat[Write]) != 0 {
		switch err := t.walk(w.p, func(fi os.FileInfo) error {
			p := filepath.Join(w.p, fi.Name())
			switch err := t.singlewatch(p, w.eDir, ndir, fi); {
			case os.IsNotExist(err) && ((w.eDir & Remove) != 0):
				evn = append(evn, event{p, Remove, fi.IsDir(), n})
			case err == errAlreadyWatched:
			case err != nil:
				dbgprintf("trg: watching %q failed: %q", p, err)
			case (w.eDir & Create) != 0:
				evn = append(evn, event{p, Create, fi.IsDir(), n})
			default:
			}
			return nil
		}); {
		case os.IsNotExist(err):
			return
		case err != nil:
			dbgprintf("trg: dir processing failed: %q", err)
		default:
		}
	}
	return
}

type mode uint

const (
	dir mode = iota
	ndir
	both
)

// unwatch stops watching p file/directory.
func (t *trg) singleunwatch(p string, direct mode) error {
	w, ok := t.pthLkp[p]
	if !ok {
		return errNotWatched
	}
	switch direct {
	case dir:
		w.eDir = 0
	case ndir:
		w.eNonDir = 0
	case both:
		w.eDir, w.eNonDir = 0, 0
	}
	if err := t.t.Unwatch(w); err != nil {
		return err
	}
	if w.eNonDir|w.eDir != 0 {
		mod := dir
		if w.eNonDir != 0 {
			mod = ndir
		}
		if err := t.singlewatch(p, w.eNonDir|w.eDir, mod,
			w.fi); err != nil && err != errAlreadyWatched {
			return err
		}
	} else {
		t.t.Del(w)
	}
	return nil
}

func (t *trg) monitor() {
	var (
		n   interface{}
		err error
	)
	for {
		switch n, err = t.t.Wait(); {
		case err == syscall.EINTR:
		case t.t.IsStop(n, err):
			t.s <- struct{}{}
			return
		case err != nil:
			dbgprintf("trg: failed to read events: %q\n", err)
		default:
			t.send(t.process(n))
		}
	}
}

// process event returned by native call.
func (t *trg) process(n interface{}) (evn []event) {
	t.Lock()
	w, ge, err := t.t.Watched(n)
	if err != nil {
		t.Unlock()
		dbgprintf("trg: %v event lookup failed: %q", Event(ge), err)
		return
	}

	e := decode(ge, w.eDir|w.eNonDir)
	if ge&int64(not2nat[Remove]|not2nat[Rename]) == 0 {
		switch fi, err := os.Stat(w.p); {
		case err != nil:
		default:
			if err = t.t.Watch(fi, w, encode(w.eDir|w.eNonDir, fi.IsDir())); err != nil {
				dbgprintf("trg: %q is no longer watched: %q", w.p, err)
				t.t.Del(w)
			}
		}
	}
	if e == Event(0) && (!w.fi.IsDir() || (ge&int64(not2nat[Write])) == 0) {
		t.Unlock()
		return
	}

	if w.fi.IsDir() {
		evn = append(evn, t.dir(w, n, e, Event(ge))...)
	} else {
		evn = append(evn, t.file(w, n, e)...)
	}
	if Event(ge)&(not2nat[Remove]|not2nat[Rename]) != 0 {
		t.t.Del(w)
	}
	t.Unlock()
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package endtoend

import (
	"flag"
	"fmt"
	"os"
	"testing"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/vttest"

	vschemapb "vitess.io/vitess/go/vt/proto/vschema"
	vttestpb "vitess.io/vitess/go/vt/proto/vttest"
)

var (
	cluster        *vttest.LocalCluster
	vtParams       mysql.ConnParams
	mysqlParams    mysql.ConnParams
	grpcAddress    string
	tabletHostName = flag.String("tablet_hostname", "", "the tablet hostname")

	schema = `
create table t1(
	id1 bigint,
	id2 bigint,
	primary key(id1)
) Engine=InnoDB;

create table t1_id2_idx(
	id2 bigint,
	keyspace_id varbinary(10),
	primary key(id2)
) Engine=InnoDB;

create table vstream_test(
	id bigint,
	val bigint,
	primary key(id)
) Engine=InnoDB;

create table aggr_test(
	id bigint,
	val1 varchar(16),
	val2 bigint,
	primary key(id)
) Engine=InnoDB;

create table t2(
	id3 bigint,
	id4 bigint,
	primary key(id3)
) Engine=InnoDB;

create table t2_id4_idx(
	id bigint not null auto_increment,
	id4 bigint,
	id3 bigint,
	primary key(id),
	key idx_id4(id4)
) Engine=InnoDB;

create table t1_last_insert_id(
	id bigint not null auto_increment,
	id1 bigint,
	primary key(id)
) Engine=InnoDB;
`

	vschema = &vschemapb.Keyspace{
		Sharded: true,
		Vindexes: map[string]*vschemapb.Vindex{
			"hash": {
				Type: "hash",
			},
			"t1_id2_vdx": {
				Type: "consistent_lookup_unique",
				Params: map[string]string{
					"table": "t1_id2_idx",
					"from":  "id2",
					"to":    "keyspace_id",
				},
				Owner: "t1",
			},
			"t2_id4_idx": {
				Type: "lookup_hash",
				Params: map[string]string{
					"table":      "t2_id4_idx",
					"from":       "id4",
					"to":         "id3",
					"autocommit": "true",
				},
				Owner: "t2",
			},
		},
		Tables: map[string]*vschemapb.Table{
			"t1": {
				ColumnVindexes: []*vschemapb.ColumnVindex{{
					Column: "id1",
					Name:   "hash",
				}, {
					Column: "id2",
					Name:   "t1_id2_vdx",
				}},
			},
			"t1_id2_idx": {
				ColumnVindexes: []*vschemapb.ColumnVindex{{
					Column: "id2",
					Name:   "hash",
				}},
			},
			"t2": {
				ColumnVindexes: []*vschemapb.ColumnVindex{{
					Column: "id3",
					Name:   "hash",
				}, {
					Column: "id4",
					Name:   "t2_id4_idx",
				}},
			},
			"t2_id4_idx": {
				ColumnVindexes: []*vschemapb.ColumnVindex{{
					Column: "id4",
					Name:   "hash",
				}},
			},
			"vstream_test": {
				ColumnVindexes: []*vschemapb.ColumnVindex{{
					Column: "id",
					Name:   "hash",
				}},
			},
			"aggr_test": {
				ColumnVindexes: []*vschemapb.ColumnVindex{{
					Column: "id",
					Name:   "hash",
				}},
				Columns: []*vschemapb.Column{{
					Name: "val1",
					Type: sqltypes.VarChar,
				}},
			},
			"t1_last_insert_id": {
				ColumnVindexes: []*vschemapb.ColumnVindex{{
					Column: "id1",
					Name:   "hash",
				}},
				Columns: []*vschemapb.Column{{
					Name: "id1",
					Type: sqltypes.Int64,
				}},
			},
		},
	}
)

func TestMain(m *testing.M) {
	flag.Parse()

	exitCode := func() int {
		var cfg vttest.Config
		cfg.Topology = &vttestpb.VTTestTopology{
			Keyspaces: []*vttestpb.Keyspace{{
				Name: "ks",
				Shards: []*vttestpb.Shard{{
					Name: "-80",
				}, {
					Name: "80-",
				}},
			}},
		}
		if err := cfg.InitSchemas("ks", schema, vschema); err != nil {
			fmt.Fprintf(os.Stderr, "%v\n", err)
			os.RemoveAll(cfg.SchemaDir)
			return 1
		}
		defer os.RemoveAll(cfg.SchemaDir)

		cfg.TabletHostName = *tabletHostName

		cluster = &vttest.LocalCluster{
			Config: cfg,
		}
		if err := cluster.Setup(); err != nil {
			fmt.Fprintf(os.Stderr, "%v\n", err)
			cluster.TearDown()
			return 1
		}
		defer cluster.TearDown()

		vtParams = mysql.ConnParams{
			Host: "localhost",
			Port: cluster.Env.PortForProtocol("vtcombo_mysql_port", ""),
		}
		mysqlParams = cluster.MySQLConnParams()
		grpcAddress = fmt.Sprintf("localhost:%d", cluster.Env.PortForProtocol("vtcombo", "grpc"))

		return m.Run()
	}()
	os.Exit(exitCode)
}
<commit_msg>Better test setup<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package endtoend

import (
	"context"
	"flag"
	"fmt"
	"os"
	"testing"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/vttest"

	vschemapb "vitess.io/vitess/go/vt/proto/vschema"
	vttestpb "vitess.io/vitess/go/vt/proto/vttest"
)

var (
	cluster        *vttest.LocalCluster
	vtParams       mysql.ConnParams
	mysqlParams    mysql.ConnParams
	grpcAddress    string
	tabletHostName = flag.String("tablet_hostname", "", "the tablet hostname")

	schema = `
create table t1(
	id1 bigint,
	id2 bigint,
	primary key(id1)
) Engine=InnoDB;

create table t1_id2_idx(
	id2 bigint,
	keyspace_id varbinary(10),
	primary key(id2)
) Engine=InnoDB;

create table vstream_test(
	id bigint,
	val bigint,
	primary key(id)
) Engine=InnoDB;

create table aggr_test(
	id bigint,
	val1 varchar(16),
	val2 bigint,
	primary key(id)
) Engine=InnoDB;

create table t2(
	id3 bigint,
	id4 bigint,
	primary key(id3)
) Engine=InnoDB;

create table t2_id4_idx(
	id bigint not null auto_increment,
	id4 bigint,
	id3 bigint,
	primary key(id),
	key idx_id4(id4)
) Engine=InnoDB;

create table t1_last_insert_id(
	id bigint not null auto_increment,
	id1 bigint,
	primary key(id)
) Engine=InnoDB;
`

	vschema = &vschemapb.Keyspace{
		Sharded: true,
		Vindexes: map[string]*vschemapb.Vindex{
			"hash": {
				Type: "hash",
			},
			"t1_id2_vdx": {
				Type: "consistent_lookup_unique",
				Params: map[string]string{
					"table": "t1_id2_idx",
					"from":  "id2",
					"to":    "keyspace_id",
				},
				Owner: "t1",
			},
			"t2_id4_idx": {
				Type: "lookup_hash",
				Params: map[string]string{
					"table":      "t2_id4_idx",
					"from":       "id4",
					"to":         "id3",
					"autocommit": "true",
				},
				Owner: "t2",
			},
		},
		Tables: map[string]*vschemapb.Table{
			"t1": {
				ColumnVindexes: []*vschemapb.ColumnVindex{{
					Column: "id1",
					Name:   "hash",
				}, {
					Column: "id2",
					Name:   "t1_id2_vdx",
				}},
			},
			"t1_id2_idx": {
				ColumnVindexes: []*vschemapb.ColumnVindex{{
					Column: "id2",
					Name:   "hash",
				}},
			},
			"t2": {
				ColumnVindexes: []*vschemapb.ColumnVindex{{
					Column: "id3",
					Name:   "hash",
				}, {
					Column: "id4",
					Name:   "t2_id4_idx",
				}},
			},
			"t2_id4_idx": {
				ColumnVindexes: []*vschemapb.ColumnVindex{{
					Column: "id4",
					Name:   "hash",
				}},
			},
			"vstream_test": {
				ColumnVindexes: []*vschemapb.ColumnVindex{{
					Column: "id",
					Name:   "hash",
				}},
			},
			"aggr_test": {
				ColumnVindexes: []*vschemapb.ColumnVindex{{
					Column: "id",
					Name:   "hash",
				}},
				Columns: []*vschemapb.Column{{
					Name: "val1",
					Type: sqltypes.VarChar,
				}},
			},
			"t1_last_insert_id": {
				ColumnVindexes: []*vschemapb.ColumnVindex{{
					Column: "id1",
					Name:   "hash",
				}},
				Columns: []*vschemapb.Column{{
					Name: "id1",
					Type: sqltypes.Int64,
				}},
			},
		},
	}
)

func TestMain(m *testing.M) {
	flag.Parse()

	exitCode := func() int {
		var cfg vttest.Config
		cfg.Topology = &vttestpb.VTTestTopology{
			Keyspaces: []*vttestpb.Keyspace{{
				Name: "ks",
				Shards: []*vttestpb.Shard{{
					Name: "-80",
				}, {
					Name: "80-",
				}},
			}},
		}
		if err := cfg.InitSchemas("ks", schema, vschema); err != nil {
			fmt.Fprintf(os.Stderr, "%v\n", err)
			os.RemoveAll(cfg.SchemaDir)
			return 1
		}
		defer os.RemoveAll(cfg.SchemaDir)

		cfg.TabletHostName = *tabletHostName

		cluster = &vttest.LocalCluster{
			Config: cfg,
		}
		if err := cluster.Setup(); err != nil {
			fmt.Fprintf(os.Stderr, "%v\n", err)
			cluster.TearDown()
			return 1
		}
		defer cluster.TearDown()

		vtParams = mysql.ConnParams{
			Host: "localhost",
			Port: cluster.Env.PortForProtocol("vtcombo_mysql_port", ""),
		}
		mysqlParams = cluster.MySQLConnParams()
		grpcAddress = fmt.Sprintf("localhost:%d", cluster.Env.PortForProtocol("vtcombo", "grpc"))

		insertStartValue()

		return m.Run()
	}()
	os.Exit(exitCode)
}

func insertStartValue() {
	ctx := context.Background()
	conn, err := mysql.Connect(ctx, &vtParams)
	if err != nil {
		panic(err)
	}
	defer conn.Close()

	// lets insert a single starting value for tests
	_, err = conn.ExecuteFetch("insert into t1_last_insert_id(id1) values(42)", 1000, true)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package platform

import (
	"fmt"
	"time"
	"unsafe"

	"golang.org/x/sys/windows"
)

const (
	whdrDone      = 0x01
	whdrPrepared  = 0x02
	whdrBeginloop = 0x04
	whdrEndloop   = 0x08
	whdrInqueue   = 0x10

	mmSysErrNoErr = 0x00

	waveFormatPCM = 0x0001

	waveMapper = 0xffffffff

	womOpen  = 0x3bb
	womClose = 0x3bc
	womDone  = 0x3bd
	wimOpen  = 0x3be
	wimClose = 0x3bf
	wimDone  = 0x3c0

	callbackNull = 0x00000
)

// AudioBuffer represents all you need to play sound.
type AudioBuffer struct {
	SamplesPerSecond uint32
	BitsPerSample    uint32
	ChannelCount     uint32
	BlockSize        uint32
	BlockCount       uint32

	currentBlock *soundBlock
	blocks       []soundBlock
	hWaveOut     uintptr
	closer chan bool
	writer chan *soundBlock
}

type soundBlock struct {
	wavehdr
	bytes []byte
	used  int
}

type wavehdr struct {
	lpData          uintptr
	dwBufferLength  uint32
	dwBytesRecorded uint32
	dwUser          uintptr
	dwFlags         uint32
	dwLoops         uint32
	lpNext          uintptr
	reserved        uintptr
}

type waveFormatEx struct {
	wFormatTag      uint16
	nChannels       uint16
	nSamplesPerSec  uint32
	nAvgBytesPerSec uint32
	nBlockAlign     uint16
	wBitsPerSample  uint16
	cbSize          uint16
}

func init() {
	winmm := windows.MustLoadDLL("Winmm.dll")
	waveOutPrepareHeader = winmm.MustFindProc("waveOutPrepareHeader")
	waveOutWrite = winmm.MustFindProc("waveOutWrite")
	waveOutOpen = winmm.MustFindProc("waveOutOpen")
	waveOutClose = winmm.MustFindProc("waveOutClose")
	waveOutUnprepareHeader = winmm.MustFindProc("waveOutUnprepareHeader")
}

// OpenAudioBuffer creates and returns a new playing buffer
func OpenAudioBuffer(blockCount, blockSize, samplesPerSecond, bitsPerSample, channelCount uint32) (*AudioBuffer, error) {
	ab := AudioBuffer{
		SamplesPerSecond: samplesPerSecond,
		BitsPerSample:    bitsPerSample,
		ChannelCount:     channelCount,
		BlockCount: blockCount,
		BlockSize: blockSize,
		writer: make(chan *soundBlock, blockCount+1),
		closer: make(chan bool),
	}
	ab.blocks = make([]soundBlock, blockCount)
	for i := range ab.blocks {
		ab.blocks[i].bytes = make([]byte, blockSize)
		ab.blocks[i].dwFlags = whdrDone
	}
	ab.currentBlock = &ab.blocks[0]

	wfx := waveFormatEx{
		wFormatTag:     waveFormatPCM,
		nSamplesPerSec: ab.SamplesPerSecond,
		wBitsPerSample: uint16(ab.BitsPerSample),
		nChannels:      uint16(ab.ChannelCount),
		nBlockAlign:    uint16(ab.BitsPerSample * ab.ChannelCount / 8),
		cbSize:         0,
	}
	wfx.nAvgBytesPerSec = uint32(wfx.nBlockAlign) * wfx.nSamplesPerSec

	if r1, r2, lastErr := waveOutOpen.Call(
		uintptr(unsafe.Pointer(&ab.hWaveOut)),
		waveMapper, uintptr(unsafe.Pointer(&wfx)),
		uintptr(0), uintptr(0), callbackNull); r1 != mmSysErrNoErr {
		return nil, fmt.Errorf("waveOutOpen error: %v, %v, %v", r1, r2, lastErr)
	}

	go ab.writerLoop()

	return &ab, nil
}

// Close closes the buffer and releases all resourses.
// It waits for all queued buffer writes to finish playing first.
func (ab *AudioBuffer) Close() error {
	for ab.BufferAvailable() / int(ab.BlockSize) < len(ab.blocks) {
		time.Sleep(5)
	}
	for i := range ab.blocks {
		block := &ab.blocks[i]
		r1, r2, lastErr := waveOutUnprepareHeader.Call(
			ab.hWaveOut, uintptr(unsafe.Pointer(&block.wavehdr)), unsafe.Sizeof(block.wavehdr))
		if r1 != 0 {
			// NOTE: try to keep going instead?
			return fmt.Errorf("waveOutUnprepareHeader error: %v, %v, %v", r1, r2, lastErr)
		}
	}
	r1, r2, lastErr := waveOutClose.Call(ab.hWaveOut)
	if r1 != 0 {
		return fmt.Errorf("waveOutClose error: %v, %v, %v", r1, r2, lastErr)
	}
	ab.closer <- true
	return nil
}

var (
	waveOutPrepareHeader   *windows.Proc
	waveOutUnprepareHeader *windows.Proc
	waveOutWrite           *windows.Proc
	waveOutOpen            *windows.Proc
	waveOutClose           *windows.Proc
)

// TODO: timeout w/ err
func (ab *AudioBuffer) waitOnFreeBlock() *soundBlock {
	for {
		for i := range ab.blocks {
			if ab.blocks[i].dwFlags&whdrDone != 0 {
				return &ab.blocks[i]
			}
		}
		time.Sleep(1)
	}
}

// BufferAvailable returns the number of bytes available
// to be filled in all the blocks not currently queued.
func (ab *AudioBuffer) BufferAvailable() int {
	freeCount := 0
	for i := range ab.blocks {
		if ab.blocks[i].dwFlags&whdrDone != 0 {
			freeCount++
		}
	}
	return freeCount * int(ab.BlockSize)
}

func (ab *AudioBuffer) BufferSize() int {
	return int(ab.BlockCount * ab.BlockSize)
}

func (ab *AudioBuffer) writerLoop() {
	for {
		select {
		case block := <-ab.writer:

			block.lpData = uintptr(unsafe.Pointer(&block.bytes[0]))

			r1, r2, lastErr := waveOutPrepareHeader.Call(
				ab.hWaveOut, uintptr(unsafe.Pointer(&block.wavehdr)), unsafe.Sizeof(block.wavehdr))
			if r1 != 0 {
				fmt.Printf("waveOutPrepareHeader error: %v, %v, %v", r1, r2, lastErr)
			}

			r1, r2, lastErr = waveOutWrite.Call(
				ab.hWaveOut, uintptr(unsafe.Pointer(&block.wavehdr)), unsafe.Sizeof(block.wavehdr))
			if r1 != 0 {
				fmt.Printf("waveOutWrite error: %v, %v, %v", r1, r2, lastErr)
			}
		case <-ab.closer:
			return
		}
	}
}

func (ab *AudioBuffer) Write(data []byte) error {
	for len(data) > 0 {

		if ab.currentBlock.dwFlags & whdrDone == 0 {
			ab.currentBlock = ab.waitOnFreeBlock()
			ab.currentBlock.used = 0
		}

		block := ab.currentBlock

		spaceLeft := len(block.bytes) - block.used

		if len(data) < spaceLeft {
			copy(block.bytes[block.used:], data)
			block.used += len(data)
			break
		}
		copy(block.bytes[block.used:], data[:spaceLeft])
		block.dwBufferLength = uint32(len(block.bytes))
		data = data[spaceLeft:]

		block.dwFlags &^= whdrDone

		// the api calls sometimes takes a few ms, so let's not wait on them
		ab.writer <- block
	}
	return nil
}
<commit_msg>Account for currentBlock.used in BufferAvailable<commit_after>package platform

import (
	"fmt"
	"time"
	"unsafe"

	"golang.org/x/sys/windows"
)

const (
	whdrDone      = 0x01
	whdrPrepared  = 0x02
	whdrBeginloop = 0x04
	whdrEndloop   = 0x08
	whdrInqueue   = 0x10

	mmSysErrNoErr = 0x00

	waveFormatPCM = 0x0001

	waveMapper = 0xffffffff

	womOpen  = 0x3bb
	womClose = 0x3bc
	womDone  = 0x3bd
	wimOpen  = 0x3be
	wimClose = 0x3bf
	wimDone  = 0x3c0

	callbackNull = 0x00000
)

// AudioBuffer represents all you need to play sound.
type AudioBuffer struct {
	SamplesPerSecond uint32
	BitsPerSample    uint32
	ChannelCount     uint32
	BlockSize        uint32
	BlockCount       uint32

	currentBlock *soundBlock
	blocks       []soundBlock
	hWaveOut     uintptr
	closer chan bool
	writer chan *soundBlock
}

type soundBlock struct {
	wavehdr
	bytes []byte
	used  int
}

type wavehdr struct {
	lpData          uintptr
	dwBufferLength  uint32
	dwBytesRecorded uint32
	dwUser          uintptr
	dwFlags         uint32
	dwLoops         uint32
	lpNext          uintptr
	reserved        uintptr
}

type waveFormatEx struct {
	wFormatTag      uint16
	nChannels       uint16
	nSamplesPerSec  uint32
	nAvgBytesPerSec uint32
	nBlockAlign     uint16
	wBitsPerSample  uint16
	cbSize          uint16
}

func init() {
	winmm := windows.MustLoadDLL("Winmm.dll")
	waveOutPrepareHeader = winmm.MustFindProc("waveOutPrepareHeader")
	waveOutWrite = winmm.MustFindProc("waveOutWrite")
	waveOutOpen = winmm.MustFindProc("waveOutOpen")
	waveOutClose = winmm.MustFindProc("waveOutClose")
	waveOutUnprepareHeader = winmm.MustFindProc("waveOutUnprepareHeader")
}

// OpenAudioBuffer creates and returns a new playing buffer
func OpenAudioBuffer(blockCount, blockSize, samplesPerSecond, bitsPerSample, channelCount uint32) (*AudioBuffer, error) {
	ab := AudioBuffer{
		SamplesPerSecond: samplesPerSecond,
		BitsPerSample:    bitsPerSample,
		ChannelCount:     channelCount,
		BlockCount: blockCount,
		BlockSize: blockSize,
		writer: make(chan *soundBlock, blockCount+1),
		closer: make(chan bool),
	}
	ab.blocks = make([]soundBlock, blockCount)
	for i := range ab.blocks {
		ab.blocks[i].bytes = make([]byte, blockSize)
		ab.blocks[i].dwFlags = whdrDone
	}
	ab.currentBlock = &ab.blocks[0]

	wfx := waveFormatEx{
		wFormatTag:     waveFormatPCM,
		nSamplesPerSec: ab.SamplesPerSecond,
		wBitsPerSample: uint16(ab.BitsPerSample),
		nChannels:      uint16(ab.ChannelCount),
		nBlockAlign:    uint16(ab.BitsPerSample * ab.ChannelCount / 8),
		cbSize:         0,
	}
	wfx.nAvgBytesPerSec = uint32(wfx.nBlockAlign) * wfx.nSamplesPerSec

	if r1, r2, lastErr := waveOutOpen.Call(
		uintptr(unsafe.Pointer(&ab.hWaveOut)),
		waveMapper, uintptr(unsafe.Pointer(&wfx)),
		uintptr(0), uintptr(0), callbackNull); r1 != mmSysErrNoErr {
		return nil, fmt.Errorf("waveOutOpen error: %v, %v, %v", r1, r2, lastErr)
	}

	go ab.writerLoop()

	return &ab, nil
}

// Close closes the buffer and releases all resourses.
// It waits for all queued buffer writes to finish playing first.
func (ab *AudioBuffer) Close() error {
	for ab.BufferAvailable() / int(ab.BlockSize) < len(ab.blocks) {
		time.Sleep(5)
	}
	for i := range ab.blocks {
		block := &ab.blocks[i]
		r1, r2, lastErr := waveOutUnprepareHeader.Call(
			ab.hWaveOut, uintptr(unsafe.Pointer(&block.wavehdr)), unsafe.Sizeof(block.wavehdr))
		if r1 != 0 {
			// NOTE: try to keep going instead?
			return fmt.Errorf("waveOutUnprepareHeader error: %v, %v, %v", r1, r2, lastErr)
		}
	}
	r1, r2, lastErr := waveOutClose.Call(ab.hWaveOut)
	if r1 != 0 {
		return fmt.Errorf("waveOutClose error: %v, %v, %v", r1, r2, lastErr)
	}
	ab.closer <- true
	return nil
}

var (
	waveOutPrepareHeader   *windows.Proc
	waveOutUnprepareHeader *windows.Proc
	waveOutWrite           *windows.Proc
	waveOutOpen            *windows.Proc
	waveOutClose           *windows.Proc
)

// TODO: timeout w/ err
func (ab *AudioBuffer) waitOnFreeBlock() *soundBlock {
	for {
		for i := range ab.blocks {
			if ab.blocks[i].dwFlags&whdrDone != 0 {
				return &ab.blocks[i]
			}
		}
		time.Sleep(1)
	}
}

// BufferAvailable returns the number of bytes available
// to be filled in all the blocks not currently queued.
func (ab *AudioBuffer) BufferAvailable() int {
	available := 0
	for i := range ab.blocks {
		if ab.blocks[i].dwFlags&whdrDone != 0 {
			block := &ab.blocks[i]
			if block == ab.currentBlock {
				available += int(ab.BlockSize) - block.used
			} else {
				available += int(ab.BlockSize)
			}
		}
	}
	return available
}

func (ab *AudioBuffer) BufferSize() int {
	return int(ab.BlockCount * ab.BlockSize)
}

func (ab *AudioBuffer) writerLoop() {
	for {
		select {
		case block := <-ab.writer:

			block.lpData = uintptr(unsafe.Pointer(&block.bytes[0]))

			r1, r2, lastErr := waveOutPrepareHeader.Call(
				ab.hWaveOut, uintptr(unsafe.Pointer(&block.wavehdr)), unsafe.Sizeof(block.wavehdr))
			if r1 != 0 {
				fmt.Printf("waveOutPrepareHeader error: %v, %v, %v", r1, r2, lastErr)
			}

			r1, r2, lastErr = waveOutWrite.Call(
				ab.hWaveOut, uintptr(unsafe.Pointer(&block.wavehdr)), unsafe.Sizeof(block.wavehdr))
			if r1 != 0 {
				fmt.Printf("waveOutWrite error: %v, %v, %v", r1, r2, lastErr)
			}
		case <-ab.closer:
			return
		}
	}
}

func (ab *AudioBuffer) Write(data []byte) error {
	for len(data) > 0 {

		if ab.currentBlock.dwFlags & whdrDone == 0 {
			ab.currentBlock = ab.waitOnFreeBlock()
			ab.currentBlock.used = 0
		}

		block := ab.currentBlock

		spaceLeft := len(block.bytes) - block.used

		if len(data) < spaceLeft {
			copy(block.bytes[block.used:], data)
			block.used += len(data)
			break
		}
		copy(block.bytes[block.used:], data[:spaceLeft])
		block.dwBufferLength = uint32(len(block.bytes))
		data = data[spaceLeft:]

		block.dwFlags &^= whdrDone

		// the api calls sometimes takes a few ms, so let's not wait on them
		ab.writer <- block
	}
	return nil
}
<|endoftext|>


<commit_before>package string_tables

import (
	"math"

	"github.com/davecgh/go-spew/spew"
	"github.com/dotabuff/d2rp/core/utils"
)

const (
	MaxNameLength  = 0x400
	KeyHistorySize = 32
)

type CSTObject interface {
	GetStringData() []byte
	GetNumEntries() int32
	GetMaxEntries() int32
	GetUserDataFixedSize() bool
	GetUserDataSizeBits() int32
}

type USTObject interface {
	GetStringData() []byte
	GetNumChangedEntries() int32
}

func ParseCST(obj CSTObject) map[int]*StringTableItem {
	return Parse(
		obj.GetStringData(),
		int(obj.GetNumEntries()),
		int(obj.GetMaxEntries()),
		int(obj.GetUserDataSizeBits()),
		obj.GetUserDataFixedSize(),
	)
}

func ParseUST(obj USTObject, meta *CacheItem) map[int]*StringTableItem {
	return Parse(
		obj.GetStringData(),
		int(obj.GetNumChangedEntries()),
		meta.MaxEntries,
		meta.Bits,
		meta.IsFixedSize,
	)
}

func Parse(data []byte, numEntries, maxEntries, dataSizeBits int, dataFixedSize bool) map[int]*StringTableItem {
	br := utils.NewBitReader(data)

	bitsPerIndex := int(math.Log(float64(maxEntries)) / math.Log(2))
	keyHistory := make([]string, 0, KeyHistorySize)
	result := map[int]*StringTableItem{}
	mysteryFlag := br.ReadBoolean()
	index := -1
	nameBuf := ""

	for len(result) < numEntries {
		if br.ReadBoolean() {
			index++
		} else {
			index = int(br.ReadUBits(bitsPerIndex))
		}
		nameBuf = ""
		if br.ReadBoolean() {
			if mysteryFlag && br.ReadBoolean() {
				panic("mysteryFlag assertion failed!")
			}
			if br.ReadBoolean() {
				basis := br.ReadUBits(5)
				length := br.ReadUBits(5)
				if int(basis) > len(keyHistory) {
					spew.Dump("Ignoring invalid history index...", keyHistory, basis, length)
					nameBuf += br.ReadStringN(MaxNameLength)
				} else {
					nameBuf += keyHistory[basis][0:length] + br.ReadStringN(int(MaxNameLength-length))
				}
			} else {
				nameBuf += br.ReadStringN(MaxNameLength)
			}
			if len(keyHistory) >= KeyHistorySize {
				copy(keyHistory[0:], keyHistory[1:])
				keyHistory[len(keyHistory)-1] = "" // or the zero value of T
				keyHistory = keyHistory[:len(keyHistory)-1]
			}
			keyHistory = append(keyHistory, nameBuf)
		}
		value := []byte{}
		if br.ReadBoolean() {
			bitLength := 0
			if dataFixedSize {
				bitLength = dataSizeBits
			} else {
				bitLength = int(br.ReadUBits(14) * 8)
			}
			value = append(value, br.ReadBitsAsBytes(bitLength)...)
		}
		result[index] = &StringTableItem{Str: nameBuf, Data: value}
	}

	return result
}
<commit_msg>make failed ward assignment less fatal and other fixes<commit_after>package string_tables

import (
	"math"

	"github.com/davecgh/go-spew/spew"
	"github.com/dotabuff/d2rp/core/utils"
)

const (
	MaxNameLength  = 0x400
	KeyHistorySize = 32
)

type CSTObject interface {
	GetStringData() []byte
	GetNumEntries() int32
	GetMaxEntries() int32
	GetUserDataFixedSize() bool
	GetUserDataSizeBits() int32
}

type USTObject interface {
	GetStringData() []byte
	GetNumChangedEntries() int32
}

func ParseCST(obj CSTObject) map[int]*StringTableItem {
	return Parse(
		obj.GetStringData(),
		int(obj.GetNumEntries()),
		int(obj.GetMaxEntries()),
		int(obj.GetUserDataSizeBits()),
		obj.GetUserDataFixedSize(),
	)
}

func ParseUST(obj USTObject, meta *CacheItem) map[int]*StringTableItem {
	return Parse(
		obj.GetStringData(),
		int(obj.GetNumChangedEntries()),
		meta.MaxEntries,
		meta.Bits,
		meta.IsFixedSize,
	)
}

func Parse(data []byte, numEntries, maxEntries, dataSizeBits int, dataFixedSize bool) map[int]*StringTableItem {
	br := utils.NewBitReader(data)

	bitsPerIndex := int(math.Log(float64(maxEntries)) / math.Log(2))
	keyHistory := make([]string, 0, KeyHistorySize)
	result := map[int]*StringTableItem{}
	mysteryFlag := br.ReadBoolean()
	index := -1
	nameBuf := ""

	for len(result) < numEntries {
		if br.ReadBoolean() {
			index++
		} else {
			index = int(br.ReadUBits(bitsPerIndex))
		}
		nameBuf = ""
		if br.ReadBoolean() {
			if mysteryFlag && br.ReadBoolean() {
				panic("mysteryFlag assertion failed!")
			}
			if br.ReadBoolean() {
				basis := br.ReadUBits(5)
				length := br.ReadUBits(5)
				if int(basis) >= len(keyHistory) {
					// spew.Dump("Ignoring invalid history index...", keyHistory, basis, length)
					nameBuf += br.ReadStringN(MaxNameLength)
				} else {
					s := keyHistory[basis]
					if int(length) > len(s) {
						spew.Dump(s, length)
						nameBuf += s[0:length] + br.ReadStringN(int(MaxNameLength-length))
					} else {
						nameBuf += s[0:length] + br.ReadStringN(int(MaxNameLength-length))
					}
				}
			} else {
				nameBuf += br.ReadStringN(MaxNameLength)
			}
			if len(keyHistory) >= KeyHistorySize {
				copy(keyHistory[0:], keyHistory[1:])
				keyHistory[len(keyHistory)-1] = "" // or the zero value of T
				keyHistory = keyHistory[:len(keyHistory)-1]
			}
			keyHistory = append(keyHistory, nameBuf)
		}
		value := []byte{}
		if br.ReadBoolean() {
			bitLength := 0
			if dataFixedSize {
				bitLength = dataSizeBits
			} else {
				bitLength = int(br.ReadUBits(14) * 8)
			}
			value = append(value, br.ReadBitsAsBytes(bitLength)...)
		}
		result[index] = &StringTableItem{Str: nameBuf, Data: value}
	}

	return result
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queuejob

import (
	"fmt"
	"sync"
	"time"

	"github.com/golang/glog"

	"k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/informers"
	coreinformers "k8s.io/client-go/informers/core/v1"
	"k8s.io/client-go/kubernetes"
	corelisters "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"

	"github.com/kubernetes-incubator/kube-arbitrator/pkg/apis/utils"
	arbv1 "github.com/kubernetes-incubator/kube-arbitrator/pkg/apis/v1"
	"github.com/kubernetes-incubator/kube-arbitrator/pkg/client"
	"github.com/kubernetes-incubator/kube-arbitrator/pkg/client/clientset"
	arbinformers "github.com/kubernetes-incubator/kube-arbitrator/pkg/client/informers"
	informersv1 "github.com/kubernetes-incubator/kube-arbitrator/pkg/client/informers/v1"
	listersv1 "github.com/kubernetes-incubator/kube-arbitrator/pkg/client/listers/v1"
)

const (
	// QueueJobLabel label string for queuejob name
	QueueJobLabel string = "queuejob.kube-arbitrator.k8s.io"
)

// Controller the QueueJob Controller type
type Controller struct {
	config           *rest.Config
	queueJobInformer informersv1.QueueJobInformer
	podInformer      coreinformers.PodInformer
	clients          *kubernetes.Clientset
	arbclients       *clientset.Clientset

	// A store of jobs
	queueJobLister listersv1.QueueJobLister
	queueJobSynced func() bool

	// A store of pods, populated by the podController
	podStore  corelisters.PodLister
	podSynced func() bool

	// eventQueue that need to sync up
	eventQueue *cache.FIFO
}

// NewQueueJobController create new QueueJob Controller
func NewQueueJobController(config *rest.Config) *Controller {
	cc := &Controller{
		config:     config,
		clients:    kubernetes.NewForConfigOrDie(config),
		arbclients: clientset.NewForConfigOrDie(config),
		eventQueue: cache.NewFIFO(eventKey),
	}

	queueJobClient, _, err := client.NewClient(cc.config)
	if err != nil {
		panic(err)
	}

	cc.queueJobInformer = arbinformers.NewSharedInformerFactory(queueJobClient, 0).QueueJob().QueueJobs()
	cc.queueJobInformer.Informer().AddEventHandler(
		cache.FilteringResourceEventHandler{
			FilterFunc: func(obj interface{}) bool {
				switch t := obj.(type) {
				case *arbv1.QueueJob:
					glog.V(4).Infof("Filter QueueJob name(%s) namespace(%s)\n", t.Name, t.Namespace)
					return true
				default:
					return false
				}
			},
			Handler: cache.ResourceEventHandlerFuncs{
				AddFunc:    cc.addQueueJob,
				UpdateFunc: cc.updateQueueJob,
				DeleteFunc: cc.deleteQueueJob,
			},
		})
	cc.queueJobLister = cc.queueJobInformer.Lister()
	cc.queueJobSynced = cc.queueJobInformer.Informer().HasSynced

	cc.podInformer = informers.NewSharedInformerFactory(cc.clients, 0).Core().V1().Pods()
	cc.podInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{
		FilterFunc: func(obj interface{}) bool {
			switch t := obj.(type) {
			case *v1.Pod:
				glog.V(4).Infof("Filter Pod name(%s) namespace(%s)\n", t.Name, t.Namespace)
				return true
			default:
				return false
			}
		},
		Handler: cache.ResourceEventHandlerFuncs{
			AddFunc:    cc.addPod,
			UpdateFunc: cc.updatePod,
			DeleteFunc: cc.deletePod,
		},
	})
	cc.podStore = cc.podInformer.Lister()
	cc.podSynced = cc.podInformer.Informer().HasSynced

	return cc
}

// Run start QueueJob Controller
func (cc *Controller) Run(stopCh chan struct{}) {
	// initialized
	createQueueJobKind(cc.config)

	go cc.queueJobInformer.Informer().Run(stopCh)
	go cc.podInformer.Informer().Run(stopCh)

	cache.WaitForCacheSync(stopCh, cc.queueJobSynced, cc.podSynced)

	go wait.Until(cc.worker, time.Second, stopCh)
}

func (cc *Controller) addQueueJob(obj interface{}) {
	qj, ok := obj.(*arbv1.QueueJob)
	if !ok {
		glog.Errorf("obj is not QueueJob")
		return
	}

	cc.enqueue(qj)
}

func (cc *Controller) updateQueueJob(oldObj, newObj interface{}) {
	newQJ, ok := newObj.(*arbv1.QueueJob)
	if !ok {
		glog.Errorf("newObj is not QueueJob")
		return
	}

	cc.enqueue(newQJ)
}

func (cc *Controller) deleteQueueJob(obj interface{}) {
	qj, ok := obj.(*arbv1.QueueJob)
	if !ok {
		glog.Errorf("obj is not QueueJob")
		return
	}

	cc.enqueue(qj)
}

func (cc *Controller) addPod(obj interface{}) {
	pod, ok := obj.(*v1.Pod)
	if !ok {
		glog.Error("Failed to convert %v to v1.Pod", obj)
		return
	}

	cc.enqueue(pod)
}

func (cc *Controller) updatePod(oldObj, newObj interface{}) {
	pod, ok := newObj.(*v1.Pod)
	if !ok {
		glog.Error("Failed to convert %v to v1.Pod", newObj)
		return
	}

	cc.enqueue(pod)
}

func (cc *Controller) deletePod(obj interface{}) {
	var pod *v1.Pod
	switch t := obj.(type) {
	case *v1.Pod:
		pod = t
	case cache.DeletedFinalStateUnknown:
		var ok bool
		pod, ok = t.Obj.(*v1.Pod)
		if !ok {
			glog.Errorf("Cannot convert to *v1.Pod: %v", t.Obj)
			return
		}
	default:
		glog.Errorf("Cannot convert to *v1.Pod: %v", t)
		return
	}

	queuejobs, err := cc.queueJobLister.List(labels.Everything())
	if err != nil {
		glog.Errorf("Failed to list QueueJobs for Pod %v/%v", pod.Namespace, pod.Name)
	}

	ctl := utils.GetController(pod)
	for _, qj := range queuejobs {
		if qj.UID == ctl {
			cc.enqueue(qj)
			break
		}
	}
}

func (cc *Controller) enqueue(obj interface{}) {
	err := cc.eventQueue.Add(obj)
	if err != nil {
		glog.Errorf("Fail to enqueue QueueJob to updateQueue, err %#v", err)
	}
}

func (cc *Controller) worker() {
	if _, err := cc.eventQueue.Pop(func(obj interface{}) error {
		var queuejob *arbv1.QueueJob
		switch v := obj.(type) {
		case *arbv1.QueueJob:
			queuejob = v
		case *v1.Pod:
			queuejobs, err := cc.queueJobLister.List(labels.Everything())
			if err != nil {
				glog.Errorf("Failed to list QueueJobs for Pod %v/%v", v.Namespace, v.Name)
			}

			ctl := utils.GetController(v)
			for _, qj := range queuejobs {
				if qj.UID == ctl {
					queuejob = qj
					break
				}
			}

		default:
			glog.Errorf("Un-supported type of %v", obj)
			return nil
		}

		if queuejob == nil {
			if acc, err := meta.Accessor(obj); err != nil {
				glog.Warningf("Failed to get QueueJob for %v/%v", acc.GetNamespace(), acc.GetName())
			}

			return nil
		}

		// sync Pods for a QueueJob
		if err := cc.syncQueueJob(queuejob); err != nil {
			glog.Errorf("Failed to sync QueueJob %s, err %#v", queuejob.Name, err)
			// If any error, requeue it.
			return err
		}

		return nil
	}); err != nil {
		glog.Errorf("Fail to pop item from updateQueue, err %#v", err)
		return
	}
}

// filterActivePods returns pods that have not terminated.
func filterActivePods(pods []*v1.Pod) []*v1.Pod {
	var result []*v1.Pod
	for _, p := range pods {
		if isPodActive(p) {
			result = append(result, p)
		} else {
			glog.V(4).Infof("Ignoring inactive pod %v/%v in state %v, deletion time %v",
				p.Namespace, p.Name, p.Status.Phase, p.DeletionTimestamp)
		}
	}
	return result
}

func isPodActive(p *v1.Pod) bool {
	return v1.PodSucceeded != p.Status.Phase &&
		v1.PodFailed != p.Status.Phase &&
		p.DeletionTimestamp == nil
}

func (cc *Controller) syncQueueJob(qj *arbv1.QueueJob) error {
	queueJob, err := cc.queueJobLister.QueueJobs(qj.Namespace).Get(qj.Name)
	if err != nil {
		if apierrors.IsNotFound(err) {
			glog.V(3).Infof("Job has been deleted: %v", qj.Name)
			return nil
		}
		return err
	}

	pods, err := cc.getPodsForQueueJob(queueJob)
	if err != nil {
		return err
	}

	return cc.manageQueueJob(queueJob, pods)
}

func (cc *Controller) getPodsForQueueJob(qj *arbv1.QueueJob) ([]*v1.Pod, error) {
	selector, err := metav1.LabelSelectorAsSelector(qj.Spec.Selector)
	if err != nil {
		return nil, fmt.Errorf("couldn't convert QueueJob selector: %v", err)
	}

	// List all pods under QueueJob
	pods, err := cc.podStore.Pods(qj.Namespace).List(selector)
	if err != nil {
		return nil, err
	}

	return pods, nil
}

// manageQueueJob is the core method responsible for managing the number of running
// pods according to what is specified in the job.Spec.
// Does NOT modify <activePods>.
func (cc *Controller) manageQueueJob(qj *arbv1.QueueJob, pods []*v1.Pod) error {
	var err error

	replicas := qj.Spec.Replicas

	running := int32(filterPods(pods, v1.PodRunning))
	pending := int32(filterPods(pods, v1.PodPending))
	succeeded := int32(filterPods(pods, v1.PodSucceeded))
	failed := int32(filterPods(pods, v1.PodFailed))

	glog.V(3).Infof("There are %d pods of QueueJob %s: replicas %d, pending %d, running %d, succeeded %d, failed %d",
		len(pods), qj.Name, pending, running, replicas, succeeded, failed)

	ss, err := cc.arbclients.ArbV1().SchedulingSpecs(qj.Namespace).List(metav1.ListOptions{
		FieldSelector: fmt.Sprintf("metadata.name=%s", qj.Name),
	})

	if len(ss.Items) == 0 {
		schedSpc := createQueueJobSchedulingSpec(qj)
		_, err := cc.arbclients.ArbV1().SchedulingSpecs(qj.Namespace).Create(schedSpc)
		if err != nil {
			glog.Errorf("Failed to create SchedulingSpec for QueueJob %v/%v: %v",
				qj.Namespace, qj.Name, err)
		}
	} else {
		glog.V(3).Infof("There's %v SchedulingSpec for QueueJob %v/%v",
			len(ss.Items), qj.Namespace, qj.Name)
	}

	// Create pod if necessary
	if diff := replicas - pending - running - succeeded; diff > 0 {
		glog.V(3).Infof("Try to create %v Pods for QueueJob %v/%v", diff, qj.Namespace, qj.Name)

		var errs []error
		wait := sync.WaitGroup{}
		wait.Add(int(diff))
		for i := int32(0); i < diff; i++ {
			go func(ix int32) {
				defer wait.Done()
				newPod := createQueueJobPod(qj, ix)
				_, err := cc.clients.Core().Pods(newPod.Namespace).Create(newPod)
				if err != nil {
					// Failed to create Pod, wait a moment and then create it again
					// This is to ensure all pods under the same QueueJob created
					// So gang-scheduling could schedule the QueueJob successfully
					glog.Errorf("Failed to create pod %s for QueueJob %s, err %#v",
						newPod.Name, qj.Name, err)
					errs = append(errs, err)
				}
			}(i)
		}
		wait.Wait()

		if len(errs) != 0 {
			return fmt.Errorf("failed to create %d pods of %d", len(errs), diff)
		}
	}

	qj.Status = arbv1.QueueJobStatus{
		Pending:      pending,
		Running:      running,
		Succeeded:    succeeded,
		Failed:       failed,
		MinAvailable: int32(qj.Spec.SchedSpec.MinAvailable),
	}

	// TODO(k82cn): replaced it with `UpdateStatus`
	if _, err := cc.arbclients.ArbV1().QueueJobs(qj.Namespace).Update(qj); err != nil {
		glog.Errorf("Failed to update status of QueueJob %v/%v: %v",
			qj.Namespace, qj.Name, err)
		return err
	}

	return err
}
<commit_msg>Updated log.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queuejob

import (
	"fmt"
	"sync"
	"time"

	"github.com/golang/glog"

	"k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/informers"
	coreinformers "k8s.io/client-go/informers/core/v1"
	"k8s.io/client-go/kubernetes"
	corelisters "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"

	"github.com/kubernetes-incubator/kube-arbitrator/pkg/apis/utils"
	arbv1 "github.com/kubernetes-incubator/kube-arbitrator/pkg/apis/v1"
	"github.com/kubernetes-incubator/kube-arbitrator/pkg/client"
	"github.com/kubernetes-incubator/kube-arbitrator/pkg/client/clientset"
	arbinformers "github.com/kubernetes-incubator/kube-arbitrator/pkg/client/informers"
	informersv1 "github.com/kubernetes-incubator/kube-arbitrator/pkg/client/informers/v1"
	listersv1 "github.com/kubernetes-incubator/kube-arbitrator/pkg/client/listers/v1"
)

const (
	// QueueJobLabel label string for queuejob name
	QueueJobLabel string = "queuejob.kube-arbitrator.k8s.io"
)

// Controller the QueueJob Controller type
type Controller struct {
	config           *rest.Config
	queueJobInformer informersv1.QueueJobInformer
	podInformer      coreinformers.PodInformer
	clients          *kubernetes.Clientset
	arbclients       *clientset.Clientset

	// A store of jobs
	queueJobLister listersv1.QueueJobLister
	queueJobSynced func() bool

	// A store of pods, populated by the podController
	podStore  corelisters.PodLister
	podSynced func() bool

	// eventQueue that need to sync up
	eventQueue *cache.FIFO
}

// NewQueueJobController create new QueueJob Controller
func NewQueueJobController(config *rest.Config) *Controller {
	cc := &Controller{
		config:     config,
		clients:    kubernetes.NewForConfigOrDie(config),
		arbclients: clientset.NewForConfigOrDie(config),
		eventQueue: cache.NewFIFO(eventKey),
	}

	queueJobClient, _, err := client.NewClient(cc.config)
	if err != nil {
		panic(err)
	}

	cc.queueJobInformer = arbinformers.NewSharedInformerFactory(queueJobClient, 0).QueueJob().QueueJobs()
	cc.queueJobInformer.Informer().AddEventHandler(
		cache.FilteringResourceEventHandler{
			FilterFunc: func(obj interface{}) bool {
				switch t := obj.(type) {
				case *arbv1.QueueJob:
					glog.V(4).Infof("Filter QueueJob name(%s) namespace(%s)\n", t.Name, t.Namespace)
					return true
				default:
					return false
				}
			},
			Handler: cache.ResourceEventHandlerFuncs{
				AddFunc:    cc.addQueueJob,
				UpdateFunc: cc.updateQueueJob,
				DeleteFunc: cc.deleteQueueJob,
			},
		})
	cc.queueJobLister = cc.queueJobInformer.Lister()
	cc.queueJobSynced = cc.queueJobInformer.Informer().HasSynced

	cc.podInformer = informers.NewSharedInformerFactory(cc.clients, 0).Core().V1().Pods()
	cc.podInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{
		FilterFunc: func(obj interface{}) bool {
			switch t := obj.(type) {
			case *v1.Pod:
				glog.V(4).Infof("Filter Pod name(%s) namespace(%s)\n", t.Name, t.Namespace)
				return true
			default:
				return false
			}
		},
		Handler: cache.ResourceEventHandlerFuncs{
			AddFunc:    cc.addPod,
			UpdateFunc: cc.updatePod,
			DeleteFunc: cc.deletePod,
		},
	})
	cc.podStore = cc.podInformer.Lister()
	cc.podSynced = cc.podInformer.Informer().HasSynced

	return cc
}

// Run start QueueJob Controller
func (cc *Controller) Run(stopCh chan struct{}) {
	// initialized
	createQueueJobKind(cc.config)

	go cc.queueJobInformer.Informer().Run(stopCh)
	go cc.podInformer.Informer().Run(stopCh)

	cache.WaitForCacheSync(stopCh, cc.queueJobSynced, cc.podSynced)

	go wait.Until(cc.worker, time.Second, stopCh)
}

func (cc *Controller) addQueueJob(obj interface{}) {
	qj, ok := obj.(*arbv1.QueueJob)
	if !ok {
		glog.Errorf("obj is not QueueJob")
		return
	}

	cc.enqueue(qj)
}

func (cc *Controller) updateQueueJob(oldObj, newObj interface{}) {
	newQJ, ok := newObj.(*arbv1.QueueJob)
	if !ok {
		glog.Errorf("newObj is not QueueJob")
		return
	}

	cc.enqueue(newQJ)
}

func (cc *Controller) deleteQueueJob(obj interface{}) {
	qj, ok := obj.(*arbv1.QueueJob)
	if !ok {
		glog.Errorf("obj is not QueueJob")
		return
	}

	cc.enqueue(qj)
}

func (cc *Controller) addPod(obj interface{}) {
	pod, ok := obj.(*v1.Pod)
	if !ok {
		glog.Error("Failed to convert %v to v1.Pod", obj)
		return
	}

	cc.enqueue(pod)
}

func (cc *Controller) updatePod(oldObj, newObj interface{}) {
	pod, ok := newObj.(*v1.Pod)
	if !ok {
		glog.Error("Failed to convert %v to v1.Pod", newObj)
		return
	}

	cc.enqueue(pod)
}

func (cc *Controller) deletePod(obj interface{}) {
	var pod *v1.Pod
	switch t := obj.(type) {
	case *v1.Pod:
		pod = t
	case cache.DeletedFinalStateUnknown:
		var ok bool
		pod, ok = t.Obj.(*v1.Pod)
		if !ok {
			glog.Errorf("Cannot convert to *v1.Pod: %v", t.Obj)
			return
		}
	default:
		glog.Errorf("Cannot convert to *v1.Pod: %v", t)
		return
	}

	queuejobs, err := cc.queueJobLister.List(labels.Everything())
	if err != nil {
		glog.Errorf("Failed to list QueueJobs for Pod %v/%v", pod.Namespace, pod.Name)
	}

	ctl := utils.GetController(pod)
	for _, qj := range queuejobs {
		if qj.UID == ctl {
			cc.enqueue(qj)
			break
		}
	}
}

func (cc *Controller) enqueue(obj interface{}) {
	err := cc.eventQueue.Add(obj)
	if err != nil {
		glog.Errorf("Fail to enqueue QueueJob to updateQueue, err %#v", err)
	}
}

func (cc *Controller) worker() {
	if _, err := cc.eventQueue.Pop(func(obj interface{}) error {
		var queuejob *arbv1.QueueJob
		switch v := obj.(type) {
		case *arbv1.QueueJob:
			queuejob = v
		case *v1.Pod:
			queuejobs, err := cc.queueJobLister.List(labels.Everything())
			if err != nil {
				glog.Errorf("Failed to list QueueJobs for Pod %v/%v", v.Namespace, v.Name)
			}

			ctl := utils.GetController(v)
			for _, qj := range queuejobs {
				if qj.UID == ctl {
					queuejob = qj
					break
				}
			}

		default:
			glog.Errorf("Un-supported type of %v", obj)
			return nil
		}

		if queuejob == nil {
			if acc, err := meta.Accessor(obj); err != nil {
				glog.Warningf("Failed to get QueueJob for %v/%v", acc.GetNamespace(), acc.GetName())
			}

			return nil
		}

		// sync Pods for a QueueJob
		if err := cc.syncQueueJob(queuejob); err != nil {
			glog.Errorf("Failed to sync QueueJob %s, err %#v", queuejob.Name, err)
			// If any error, requeue it.
			return err
		}

		return nil
	}); err != nil {
		glog.Errorf("Fail to pop item from updateQueue, err %#v", err)
		return
	}
}

// filterActivePods returns pods that have not terminated.
func filterActivePods(pods []*v1.Pod) []*v1.Pod {
	var result []*v1.Pod
	for _, p := range pods {
		if isPodActive(p) {
			result = append(result, p)
		} else {
			glog.V(4).Infof("Ignoring inactive pod %v/%v in state %v, deletion time %v",
				p.Namespace, p.Name, p.Status.Phase, p.DeletionTimestamp)
		}
	}
	return result
}

func isPodActive(p *v1.Pod) bool {
	return v1.PodSucceeded != p.Status.Phase &&
		v1.PodFailed != p.Status.Phase &&
		p.DeletionTimestamp == nil
}

func (cc *Controller) syncQueueJob(qj *arbv1.QueueJob) error {
	queueJob, err := cc.queueJobLister.QueueJobs(qj.Namespace).Get(qj.Name)
	if err != nil {
		if apierrors.IsNotFound(err) {
			glog.V(3).Infof("Job has been deleted: %v", qj.Name)
			return nil
		}
		return err
	}

	pods, err := cc.getPodsForQueueJob(queueJob)
	if err != nil {
		return err
	}

	return cc.manageQueueJob(queueJob, pods)
}

func (cc *Controller) getPodsForQueueJob(qj *arbv1.QueueJob) ([]*v1.Pod, error) {
	selector, err := metav1.LabelSelectorAsSelector(qj.Spec.Selector)
	if err != nil {
		return nil, fmt.Errorf("couldn't convert QueueJob selector: %v", err)
	}

	// List all pods under QueueJob
	pods, err := cc.podStore.Pods(qj.Namespace).List(selector)
	if err != nil {
		return nil, err
	}

	return pods, nil
}

// manageQueueJob is the core method responsible for managing the number of running
// pods according to what is specified in the job.Spec.
// Does NOT modify <activePods>.
func (cc *Controller) manageQueueJob(qj *arbv1.QueueJob, pods []*v1.Pod) error {
	var err error

	replicas := qj.Spec.Replicas

	running := int32(filterPods(pods, v1.PodRunning))
	pending := int32(filterPods(pods, v1.PodPending))
	succeeded := int32(filterPods(pods, v1.PodSucceeded))
	failed := int32(filterPods(pods, v1.PodFailed))

	glog.V(3).Infof("There are %d pods of QueueJob %s: replicas %d, pending %d, running %d, succeeded %d, failed %d",
		len(pods), qj.Name, replicas, pending, running, succeeded, failed)

	ss, err := cc.arbclients.ArbV1().SchedulingSpecs(qj.Namespace).List(metav1.ListOptions{
		FieldSelector: fmt.Sprintf("metadata.name=%s", qj.Name),
	})

	if len(ss.Items) == 0 {
		schedSpc := createQueueJobSchedulingSpec(qj)
		_, err := cc.arbclients.ArbV1().SchedulingSpecs(qj.Namespace).Create(schedSpc)
		if err != nil {
			glog.Errorf("Failed to create SchedulingSpec for QueueJob %v/%v: %v",
				qj.Namespace, qj.Name, err)
		}
	} else {
		glog.V(3).Infof("There's %v SchedulingSpec for QueueJob %v/%v",
			len(ss.Items), qj.Namespace, qj.Name)
	}

	// Create pod if necessary
	if diff := replicas - pending - running - succeeded; diff > 0 {
		glog.V(3).Infof("Try to create %v Pods for QueueJob %v/%v", diff, qj.Namespace, qj.Name)

		var errs []error
		wait := sync.WaitGroup{}
		wait.Add(int(diff))
		for i := int32(0); i < diff; i++ {
			go func(ix int32) {
				defer wait.Done()
				newPod := createQueueJobPod(qj, ix)
				_, err := cc.clients.Core().Pods(newPod.Namespace).Create(newPod)
				if err != nil {
					// Failed to create Pod, wait a moment and then create it again
					// This is to ensure all pods under the same QueueJob created
					// So gang-scheduling could schedule the QueueJob successfully
					glog.Errorf("Failed to create pod %s for QueueJob %s, err %#v",
						newPod.Name, qj.Name, err)
					errs = append(errs, err)
				}
			}(i)
		}
		wait.Wait()

		if len(errs) != 0 {
			return fmt.Errorf("failed to create %d pods of %d", len(errs), diff)
		}
	}

	qj.Status = arbv1.QueueJobStatus{
		Pending:      pending,
		Running:      running,
		Succeeded:    succeeded,
		Failed:       failed,
		MinAvailable: int32(qj.Spec.SchedSpec.MinAvailable),
	}

	// TODO(k82cn): replaced it with `UpdateStatus`
	if _, err := cc.arbclients.ArbV1().QueueJobs(qj.Namespace).Update(qj); err != nil {
		glog.Errorf("Failed to update status of QueueJob %v/%v: %v",
			qj.Namespace, qj.Name, err)
		return err
	}

	return err
}
<|endoftext|>


<commit_before>// probes.go
//
// This file implements the probe API calls

package atlas

import (
	"fmt"
	"github.com/bndr/gopencils"
	"log"
)

// GetProbe returns data for a single probe
func GetProbe(id int) (p *Probe, err error) {
	auth := WantAuth()
	api := gopencils.Api(apiEndpoint, auth)
	r, err := api.Res("probes").Id(id, &p).Get()
	if err != nil {
		err = fmt.Errorf("err: %v - r:%v\n", err, r)
		return
	}
	return
}

// ProbesList is our main answer
type ProbesList struct {
	Count    int
	Next     string
	Previous string
	Results  []Probe
}

// GetProbes returns data for a collection of probes
func GetProbes(opts map[string]string) (p []Probe, err error) {
	log.Printf("GetProbes: opts=%+v", opts)
	auth := WantAuth()
	api := gopencils.Api(apiEndpoint, auth)

	var rawlist *ProbesList

	r, err := api.Res("probes", rawlist).Get(opts)
	log.Printf("rawlist=%+v r=%+v", rawlist, r)
	if err != nil {
		err = fmt.Errorf("%v - r:%v\n", err, r)
		return
	}

	// Empty answer
	if rawlist == nil {
		return nil, fmt.Errorf("empty probe list")
	}

	if rawlist.Next != "" {
		// We have pagination

	}
	p = rawlist.Results
	fmt.Printf("r: %#v\np: %#v\n", r, p)
	return
}
<commit_msg>Investigate why call is failing.<commit_after>// probes.go
//
// This file implements the probe API calls

package atlas

import (
	"fmt"
	"github.com/bndr/gopencils"
	"log"
)

// GetProbe returns data for a single probe
func GetProbe(id int) (p *Probe, err error) {
	auth := WantAuth()
	api := gopencils.Api(apiEndpoint, auth)
	r, err := api.Res("probes").Id(id, &p).Get()
	if err != nil {
		err = fmt.Errorf("err: %v - r:%v\n", err, r)
		return
	}
	return
}

// ProbesList is our main answer
type ProbesList struct {
	Count    int
	Next     string
	Previous string
	Results  []Probe
}

// GetProbes returns data for a collection of probes
func GetProbes(opts map[string]string) (p []Probe, err error) {
	log.Printf("GetProbes: opts=%+v", opts)
	auth := WantAuth()
	api := gopencils.Api(apiEndpoint, auth)

	var rawlist ProbesList

	r, err := api.Res("probes", &rawlist).Get(opts)
	log.Printf("rawlist=%+v r=%+v", rawlist, r)
	if err != nil {
		err = fmt.Errorf("%v - r:%v\n", err, r)
		return
	}

	// Empty answer
	if rawlist.Count == 0 {
		return nil, fmt.Errorf("empty probe list")
	}

	if rawlist.Next != "" {
		// We have pagination

	}
	p = rawlist.Results
	fmt.Printf("r: %#v\np: %#v\n", r, p)
	return
}
<|endoftext|>


<commit_before>package commands

import (
	"os"

	"github.com/spf13/cobra"

	"github.com/tendermint/tendermint/privval"
	"github.com/tendermint/tmlibs/log"
)

// ResetAllCmd removes the database of this Tendermint core
// instance.
var ResetAllCmd = &cobra.Command{
	Use:   "unsafe_reset_all",
	Short: "(unsafe) Remove all the data and WAL, reset this node's validator",
	Run:   resetAll,
}

// ResetPrivValidatorCmd resets the private validator files.
var ResetPrivValidatorCmd = &cobra.Command{
	Use:   "unsafe_reset_priv_validator",
	Short: "(unsafe) Reset this node's validator",
	Run:   resetPrivValidator,
}

// ResetAll removes the privValidator files.
// Exported so other CLI tools can use it.
func ResetAll(dbDir, privValFile string, logger log.Logger) {
	resetFilePV(privValFile, logger)
	if err := os.RemoveAll(dbDir); err != nil {
		logger.Error("Error removing directory", "err", err)
		return
	}
	logger.Info("Removed all data", "dir", dbDir)
}

// XXX: this is totally unsafe.
// it's only suitable for testnets.
func resetAll(cmd *cobra.Command, args []string) {
	ResetAll(config.DBDir(), config.PrivValidatorFile(), logger)
}

// XXX: this is totally unsafe.
// it's only suitable for testnets.
func resetPrivValidator(cmd *cobra.Command, args []string) {
	resetFilePV(config.PrivValidatorFile(), logger)
}

func resetFilePV(privValFile string, logger log.Logger) {
	// Get PrivValidator
	if _, err := os.Stat(privValFile); err == nil {
		pv := privval.LoadFilePV(privValFile)
		pv.Reset()
		logger.Info("Reset PrivValidator", "file", privValFile)
	} else {
		pv := privval.GenFilePV(privValFile)
		pv.Save()
		logger.Info("Generated PrivValidator", "file", privValFile)
	}
}
<commit_msg>Change reset messages (#1699)<commit_after>package commands

import (
	"os"

	"github.com/spf13/cobra"

	"github.com/tendermint/tendermint/privval"
	"github.com/tendermint/tmlibs/log"
)

// ResetAllCmd removes the database of this Tendermint core
// instance.
var ResetAllCmd = &cobra.Command{
	Use:   "unsafe_reset_all",
	Short: "(unsafe) Remove all the data and WAL, reset this node's validator to genesis state",
	Run:   resetAll,
}

// ResetPrivValidatorCmd resets the private validator files.
var ResetPrivValidatorCmd = &cobra.Command{
	Use:   "unsafe_reset_priv_validator",
	Short: "(unsafe) Reset this node's validator to genesis state",
	Run:   resetPrivValidator,
}

// ResetAll removes the privValidator files.
// Exported so other CLI tools can use it.
func ResetAll(dbDir, privValFile string, logger log.Logger) {
	resetFilePV(privValFile, logger)
	if err := os.RemoveAll(dbDir); err != nil {
		logger.Error("Error removing directory", "err", err)
		return
	}
	logger.Info("Removed all blockchain history", "dir", dbDir)
}

// XXX: this is totally unsafe.
// it's only suitable for testnets.
func resetAll(cmd *cobra.Command, args []string) {
	ResetAll(config.DBDir(), config.PrivValidatorFile(), logger)
}

// XXX: this is totally unsafe.
// it's only suitable for testnets.
func resetPrivValidator(cmd *cobra.Command, args []string) {
	resetFilePV(config.PrivValidatorFile(), logger)
}

func resetFilePV(privValFile string, logger log.Logger) {
	// Get PrivValidator
	if _, err := os.Stat(privValFile); err == nil {
		pv := privval.LoadFilePV(privValFile)
		pv.Reset()
		logger.Info("Reset PrivValidator to genesis state", "file", privValFile)
	} else {
		pv := privval.GenFilePV(privValFile)
		pv.Save()
		logger.Info("Generated PrivValidator", "file", privValFile)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logic

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	"strings"

	"github.com/golang/glog"
	"k8s.io/api/admission/v1beta1"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/poc.autoscaling.k8s.io/v1alpha1"
)

// AdmissionServer is an admission webhook server that modifies pod resources request based on VPA recommendation
type AdmissionServer struct {
	recommendationProvider RecommendationProvider
	podPreProcessor        PodPreProcessor
}

// NewAdmissionServer constructs new AdmissionServer
func NewAdmissionServer(recommendationProvider RecommendationProvider, podPreProcessor PodPreProcessor) *AdmissionServer {
	return &AdmissionServer{recommendationProvider, podPreProcessor}
}

type patchRecord struct {
	Op    string      `json:"op,inline"`
	Path  string      `json:"path,inline"`
	Value interface{} `json:"value"`
}

func (s *AdmissionServer) getPatchesForPodResourceRequest(raw []byte, namespace string) ([]patchRecord, error) {
	pod := v1.Pod{}
	if err := json.Unmarshal(raw, &pod); err != nil {
		return nil, err
	}
	if len(pod.Name) == 0 {
		pod.Name = pod.GenerateName + "%"
		pod.Namespace = namespace
	}
	glog.V(4).Infof("Admitting pod %v", pod.ObjectMeta)
	containersResources, vpaName, err := s.recommendationProvider.GetContainersResourcesForPod(&pod)
	if err != nil {
		return nil, err
	}
	pod, err = s.podPreProcessor.Process(pod)
	if err != nil {
		return nil, err
	}
	patches := []patchRecord{}
	updatesAnnotation := []string{}
	for i, containerResources := range containersResources {

		// Add resources empty object if missing
		if pod.Spec.Containers[i].Resources.Limits == nil &&
			pod.Spec.Containers[i].Resources.Requests == nil {
			patches = append(patches, patchRecord{
				Op:    "add",
				Path:  fmt.Sprintf("/spec/containers/%d/resources", i),
				Value: v1.ResourceRequirements{},
			})
		}

		// Add request empty map if missing
		if pod.Spec.Containers[i].Resources.Requests == nil {
			patches = append(patches, patchRecord{
				Op:    "add",
				Path:  fmt.Sprintf("/spec/containers/%d/resources/requests", i),
				Value: v1.ResourceList{}})
		}

		annotations := []string{}
		for resource, request := range containerResources.Requests {
			// Set request
			patches = append(patches, patchRecord{
				Op:    "add",
				Path:  fmt.Sprintf("/spec/containers/%d/resources/requests/%s", i, resource),
				Value: request.String()})
			annotations = append(annotations, fmt.Sprintf("%s request", resource))
		}

		// Set memory limit only when user didn't specify one and we have recommendation for memory
		if _, limitSet := pod.Spec.Containers[i].Resources.Limits[v1.ResourceMemory]; !limitSet {
			limit, found := containerResources.Limits[v1.ResourceMemory]
			if found {
				// Add limits empty map if missing
				if pod.Spec.Containers[i].Resources.Limits == nil {
					patches = append(patches, patchRecord{
						Op:    "add",
						Path:  fmt.Sprintf("/spec/containers/%d/resources/limits", i),
						Value: v1.ResourceList{}})
				}
				// Set limit
				patches = append(patches, patchRecord{
					Op:    "add",
					Path:  fmt.Sprintf("/spec/containers/%d/resources/limits/%s", i, v1.ResourceMemory),
					Value: limit.String()})
				annotations = append(annotations, "memory limit")
			}
		}

		updatesAnnotation = append(updatesAnnotation, fmt.Sprintf("container %d: ", i)+strings.Join(annotations, ", "))
	}
	if len(updatesAnnotation) > 0 {
		patches = append(patches, patchRecord{
			Op:   "add",
			Path: "/metadata/annotations",
			Value: map[string]string{
				"vpaUpdates": fmt.Sprintf("Pod resources updated by %s: ", vpaName) + strings.Join(updatesAnnotation, "; ")}})
	}
	return patches, nil
}

func getPatchesForVPADefaults(raw []byte) ([]patchRecord, error) {
	vpa := vpa_types.VerticalPodAutoscaler{}
	if err := json.Unmarshal(raw, &vpa); err != nil {
		return nil, err
	}
	glog.V(4).Infof("Processing vpa: %v", vpa)
	patches := []patchRecord{}
	if vpa.Spec.UpdatePolicy == nil {
		// Sets the default updatePolicy.
		defaultUpdateMode := vpa_types.UpdateModeAuto
		patches = append(patches, patchRecord{
			Op:    "add",
			Path:  "/spec/updatePolicy",
			Value: vpa_types.PodUpdatePolicy{UpdateMode: &defaultUpdateMode}})
	}
	return patches, nil
}

// only allow pods to pull images from specific registry.
func (s *AdmissionServer) admit(data []byte) *v1beta1.AdmissionResponse {
	ar := v1beta1.AdmissionReview{}
	if err := json.Unmarshal(data, &ar); err != nil {
		glog.Error(err)
		return nil
	}
	// The externalAdmissionHookConfiguration registered via selfRegistration
	// asks the kube-apiserver only sends admission request regarding pods.
	podResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}
	vpaResource := metav1.GroupVersionResource{Group: "poc.autoscaling.k8s.io", Version: "v1alpha1", Resource: "verticalpodautoscalers"}
	var patches []patchRecord
	var err error

	switch ar.Request.Resource {
	case podResource:
		patches, err = s.getPatchesForPodResourceRequest(ar.Request.Object.Raw, ar.Request.Namespace)
	case vpaResource:
		patches, err = getPatchesForVPADefaults(ar.Request.Object.Raw)
	default:
		patches, err = nil, fmt.Errorf("expected the resource to be %v or %v", podResource, vpaResource)
	}

	if err != nil {
		glog.Error(err)
		return nil
	}
	response := v1beta1.AdmissionResponse{}
	response.Allowed = true
	if len(patches) > 0 {
		patch, err := json.Marshal(patches)
		if err != nil {
			glog.Errorf("Cannot marshal the patch %v: %v", patches, err)
			return nil
		}
		patchType := v1beta1.PatchTypeJSONPatch
		response.PatchType = &patchType
		response.Patch = patch
		glog.V(4).Infof("Sending patches: %v", patches)
	}
	return &response
}

// Serve is a handler function of AdmissionServer
func (s *AdmissionServer) Serve(w http.ResponseWriter, r *http.Request) {
	var body []byte
	if r.Body != nil {
		if data, err := ioutil.ReadAll(r.Body); err == nil {
			body = data
		}
	}

	// verify the content type is accurate
	contentType := r.Header.Get("Content-Type")
	if contentType != "application/json" {
		glog.Errorf("contentType=%s, expect application/json", contentType)
		return
	}

	reviewResponse := s.admit(body)
	ar := v1beta1.AdmissionReview{
		Response: reviewResponse,
	}

	resp, err := json.Marshal(ar)
	if err != nil {
		glog.Error(err)
	}
	if _, err := w.Write(resp); err != nil {
		glog.Error(err)
	}
}
<commit_msg>VPA: don't overwrite existing annotations when patching pod<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logic

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	"strings"

	"github.com/golang/glog"
	"k8s.io/api/admission/v1beta1"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/poc.autoscaling.k8s.io/v1alpha1"
)

// AdmissionServer is an admission webhook server that modifies pod resources request based on VPA recommendation
type AdmissionServer struct {
	recommendationProvider RecommendationProvider
	podPreProcessor        PodPreProcessor
}

// NewAdmissionServer constructs new AdmissionServer
func NewAdmissionServer(recommendationProvider RecommendationProvider, podPreProcessor PodPreProcessor) *AdmissionServer {
	return &AdmissionServer{recommendationProvider, podPreProcessor}
}

type patchRecord struct {
	Op    string      `json:"op,inline"`
	Path  string      `json:"path,inline"`
	Value interface{} `json:"value"`
}

func (s *AdmissionServer) getPatchesForPodResourceRequest(raw []byte, namespace string) ([]patchRecord, error) {
	pod := v1.Pod{}
	if err := json.Unmarshal(raw, &pod); err != nil {
		return nil, err
	}
	if len(pod.Name) == 0 {
		pod.Name = pod.GenerateName + "%"
		pod.Namespace = namespace
	}
	glog.V(4).Infof("Admitting pod %v", pod.ObjectMeta)
	containersResources, vpaName, err := s.recommendationProvider.GetContainersResourcesForPod(&pod)
	if err != nil {
		return nil, err
	}
	pod, err = s.podPreProcessor.Process(pod)
	if err != nil {
		return nil, err
	}
	patches := []patchRecord{}
	updatesAnnotation := []string{}
	for i, containerResources := range containersResources {

		// Add resources empty object if missing
		if pod.Spec.Containers[i].Resources.Limits == nil &&
			pod.Spec.Containers[i].Resources.Requests == nil {
			patches = append(patches, patchRecord{
				Op:    "add",
				Path:  fmt.Sprintf("/spec/containers/%d/resources", i),
				Value: v1.ResourceRequirements{},
			})
		}

		// Add request empty map if missing
		if pod.Spec.Containers[i].Resources.Requests == nil {
			patches = append(patches, patchRecord{
				Op:    "add",
				Path:  fmt.Sprintf("/spec/containers/%d/resources/requests", i),
				Value: v1.ResourceList{}})
		}

		annotations := []string{}
		for resource, request := range containerResources.Requests {
			// Set request
			patches = append(patches, patchRecord{
				Op:    "add",
				Path:  fmt.Sprintf("/spec/containers/%d/resources/requests/%s", i, resource),
				Value: request.String()})
			annotations = append(annotations, fmt.Sprintf("%s request", resource))
		}

		// Set memory limit only when user didn't specify one and we have recommendation for memory
		if _, limitSet := pod.Spec.Containers[i].Resources.Limits[v1.ResourceMemory]; !limitSet {
			limit, found := containerResources.Limits[v1.ResourceMemory]
			if found {
				// Add limits empty map if missing
				if pod.Spec.Containers[i].Resources.Limits == nil {
					patches = append(patches, patchRecord{
						Op:    "add",
						Path:  fmt.Sprintf("/spec/containers/%d/resources/limits", i),
						Value: v1.ResourceList{}})
				}
				// Set limit
				patches = append(patches, patchRecord{
					Op:    "add",
					Path:  fmt.Sprintf("/spec/containers/%d/resources/limits/%s", i, v1.ResourceMemory),
					Value: limit.String()})
				annotations = append(annotations, "memory limit")
			}
		}

		updatesAnnotation = append(updatesAnnotation, fmt.Sprintf("container %d: ", i)+strings.Join(annotations, ", "))
	}
	if len(updatesAnnotation) > 0 {
		if pod.Annotations == nil {
			patches = append(patches, patchRecord{
				Op:   "add",
				Path: "/metadata/annotations",
				Value: map[string]string{
					"vpaUpdates": fmt.Sprintf("Pod resources updated by %s: ", vpaName) + strings.Join(updatesAnnotation, "; ")}})
		} else {
			patches = append(patches, patchRecord{
				Op:    "add",
				Path:  "/metadata/annotations/vpaUpdates",
				Value: fmt.Sprintf("Pod resources updated by %s: ", vpaName) + strings.Join(updatesAnnotation, "; ")})
		}
	}
	return patches, nil
}

func getPatchesForVPADefaults(raw []byte) ([]patchRecord, error) {
	vpa := vpa_types.VerticalPodAutoscaler{}
	if err := json.Unmarshal(raw, &vpa); err != nil {
		return nil, err
	}
	glog.V(4).Infof("Processing vpa: %v", vpa)
	patches := []patchRecord{}
	if vpa.Spec.UpdatePolicy == nil {
		// Sets the default updatePolicy.
		defaultUpdateMode := vpa_types.UpdateModeAuto
		patches = append(patches, patchRecord{
			Op:    "add",
			Path:  "/spec/updatePolicy",
			Value: vpa_types.PodUpdatePolicy{UpdateMode: &defaultUpdateMode}})
	}
	return patches, nil
}

// only allow pods to pull images from specific registry.
func (s *AdmissionServer) admit(data []byte) *v1beta1.AdmissionResponse {
	ar := v1beta1.AdmissionReview{}
	if err := json.Unmarshal(data, &ar); err != nil {
		glog.Error(err)
		return nil
	}
	// The externalAdmissionHookConfiguration registered via selfRegistration
	// asks the kube-apiserver only sends admission request regarding pods.
	podResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}
	vpaResource := metav1.GroupVersionResource{Group: "poc.autoscaling.k8s.io", Version: "v1alpha1", Resource: "verticalpodautoscalers"}
	var patches []patchRecord
	var err error

	switch ar.Request.Resource {
	case podResource:
		patches, err = s.getPatchesForPodResourceRequest(ar.Request.Object.Raw, ar.Request.Namespace)
	case vpaResource:
		patches, err = getPatchesForVPADefaults(ar.Request.Object.Raw)
	default:
		patches, err = nil, fmt.Errorf("expected the resource to be %v or %v", podResource, vpaResource)
	}

	if err != nil {
		glog.Error(err)
		return nil
	}
	response := v1beta1.AdmissionResponse{}
	response.Allowed = true
	if len(patches) > 0 {
		patch, err := json.Marshal(patches)
		if err != nil {
			glog.Errorf("Cannot marshal the patch %v: %v", patches, err)
			return nil
		}
		patchType := v1beta1.PatchTypeJSONPatch
		response.PatchType = &patchType
		response.Patch = patch
		glog.V(4).Infof("Sending patches: %v", patches)
	}
	return &response
}

// Serve is a handler function of AdmissionServer
func (s *AdmissionServer) Serve(w http.ResponseWriter, r *http.Request) {
	var body []byte
	if r.Body != nil {
		if data, err := ioutil.ReadAll(r.Body); err == nil {
			body = data
		}
	}

	// verify the content type is accurate
	contentType := r.Header.Get("Content-Type")
	if contentType != "application/json" {
		glog.Errorf("contentType=%s, expect application/json", contentType)
		return
	}

	reviewResponse := s.admit(body)
	ar := v1beta1.AdmissionReview{
		Response: reviewResponse,
	}

	resp, err := json.Marshal(ar)
	if err != nil {
		glog.Error(err)
	}
	if _, err := w.Write(resp); err != nil {
		glog.Error(err)
	}
}
<|endoftext|>


<commit_before>package gogame

/*
#cgo pkg-config: sdl2
#include "SDL.h"

int getEventType(SDL_Event *e) {
    return e->type;
}

int getKeyCode(SDL_Event *e) {
    return e->key.keysym.sym;
}

int isKeyRepeat(SDL_Event *e) {
    return e->key.repeat;
}

int isKeyPressed(int kcode) {
    const Uint8 *state = SDL_GetKeyboardState(NULL);
    int sc = SDL_GetScancodeFromKey(kcode);
    return state[sc];
}

*/
import "C"

const (
	K_LEFT   = C.SDLK_LEFT
	K_RIGHT  = C.SDLK_RIGHT
	K_SPACE  = C.SDLK_SPACE
	K_ESC    = C.SDLK_ESCAPE
	K_RETURN = C.SDLK_RETURN
	K_P      = C.SDLK_p
	K_I      = C.SDLK_i
)

type Event interface{}

type QuitEvent interface{}

type UnknownEvent interface{}

type KeyEvent struct {
	Code int
	Down bool
}

// Poll for pending envents. Return nil if there is no event available
func PollEvent() Event {
	var cev C.SDL_Event

	for {
		if 0 == C.SDL_PollEvent(&cev) {
			return nil
		}

		switch C.getEventType(&cev) {

		case C.SDL_QUIT:
			return new(QuitEvent)

		case C.SDL_KEYDOWN:
			// Ignore repeat key events
			if C.isKeyRepeat(&cev) != 0 {
				break
			}
			kde := new(KeyEvent)
			kde.Code = int(C.getKeyCode(&cev))
			kde.Down = true
			return kde

		case C.SDL_KEYUP:
			kde := new(KeyEvent)
			kde.Code = int(C.getKeyCode(&cev))
			kde.Down = false
			return kde

		default:
			return new(UnknownEvent)
		}
	}

}

// Process events. Returns true if QuitEvent has appeared
func SlurpEvents() (quit bool) {
	quit = false
	for {
		ev := PollEvent()
		if ev == nil {
			return
		}
		switch ev.(type) {
		case *QuitEvent:
			quit = true
		}
	}
}

// Returns true if key is pressed, false otherwise
func IsKeyPressed(kcode int) bool {
	return C.isKeyPressed(C.int(kcode)) == 1
}
<commit_msg>Changed event objects names<commit_after>package gogame

/*
#cgo pkg-config: sdl2
#include "SDL.h"

int getEventType(SDL_Event *e) {
    return e->type;
}

int getKeyCode(SDL_Event *e) {
    return e->key.keysym.sym;
}

int isKeyRepeat(SDL_Event *e) {
    return e->key.repeat;
}

int isKeyPressed(int kcode) {
    const Uint8 *state = SDL_GetKeyboardState(NULL);
    int sc = SDL_GetScancodeFromKey(kcode);
    return state[sc];
}

*/
import "C"

const (
	K_LEFT   = C.SDLK_LEFT
	K_RIGHT  = C.SDLK_RIGHT
	K_SPACE  = C.SDLK_SPACE
	K_ESC    = C.SDLK_ESCAPE
	K_RETURN = C.SDLK_RETURN
	K_P      = C.SDLK_p
	K_I      = C.SDLK_i
)

type Event interface{}

type EventQuit interface{}

type EventUnknown interface{}

type EventKey struct {
	Code int
	Down bool
}

// Poll for pending envents. Return nil if there is no event available
func PollEvent() Event {
	var cev C.SDL_Event

	for {
		if 0 == C.SDL_PollEvent(&cev) {
			return nil
		}

		switch C.getEventType(&cev) {

		case C.SDL_QUIT:
			return new(EventQuit)

		case C.SDL_KEYDOWN:
			// Ignore repeat key events
			if C.isKeyRepeat(&cev) != 0 {
				break
			}
			kde := new(EventKey)
			kde.Code = int(C.getKeyCode(&cev))
			kde.Down = true
			return kde

		case C.SDL_KEYUP:
			kde := new(EventKey)
			kde.Code = int(C.getKeyCode(&cev))
			kde.Down = false
			return kde

		default:
			return new(EventUnknown)
		}
	}

}

// Process events. Returns true if EventQuit has appeared
func SlurpEvents() (quit bool) {
	quit = false
	for {
		ev := PollEvent()
		if ev == nil {
			return
		}
		switch ev.(type) {
		case *EventQuit:
			quit = true
		}
	}
}

// Returns true if key is pressed, false otherwise
func IsKeyPressed(kcode int) bool {
	return C.isKeyPressed(C.int(kcode)) == 1
}
<|endoftext|>