<commit_before>/* Copyright (C) 2017 Matthias S. Benkmann
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this file (originally named eventloop.go) and associated documentation files
 * (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is furnished
 * to do so, subject to the following conditions:
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

// Graphics related algorithms and data structures.
package gfx

import "github.com/veandco/go-sdl2/sdl"

const (
  // Time per frame.
  FrameTime time.Duration = (time.Second+30)/60

  // If this number of dropped frames is exceeded, we assume that the process
  // has been suspended for some reason and will not try to catch up.
  MaxDroppedFrames = 10
)

// Handles an sdl.Event. Used with EventLoop().
type EventHandler interface{
  // Handle the event ev. Return values:
  //   consume: true => do not pass even to other handlers in the chain
  //   quit:    true => exit the event loop (immediately if consume == true)
  Handle(event sdl.Event) (consume, quit bool)
}

// Computes and renders frames. Used with EventLoop().
type Framer interface {
  // Compute frame with number num. The EventLoop() function guarantees that
  // this function is called for each number starting with 1 with no gaps
  // in the count. Note that multiple calls to ComputeFrame() may occur with
  // no calls to RenderFrame() in between (or at all). IOW ComputeFrame() must
  // not depend on anything RenderFrame() does.
  ComputeFrame(num int64)
  // Render the most recently computed frame. Note that while calls to ComputeFrame()
  // are never dropped, calls to RenderFrame() may be. Heavy stuff should
  // be done in RenderFrame() if possible.
  // NOTE: renderer.Clear() and renderer.Present() are called by EventLoop(). Do
  // not call them here unless you have some special (weird) reason.
  RenderFrame(renderer *sdl.Renderer)
}

// An EventHandler that causes EventLoop() to quit immediately (i.e. without
// further event processing) on the following events:
//   - sdl.QuitEvent
var Quitter = &quitter{}

// Set by EventLoop().
// "lost" frames count gaps in the program's execution that are so large that
// no attempt was made to catch up by dropping frames. This happens typically
// when the program is not scheduled for a while, e.g. because of SIGSTOP. In
// this case EventLoop() counts these frames as lost and simply continues with
// the next frame as if nothing happened.
var LostFrames int64

// Set by EventLoop().
// Dropped frames are frames that the program computes without presenting
// to the user in order to catch up with the frame that's supposed to be
// showing according to execution time.
var DroppedFrames int64

// Runs an SDL event loop that processes events once per frame as determined
// by FrameTime and MaxDroppedFrames. Each event is passed to each entry in
// evhandlers in turn until one of them returns consume==true.
// After all events are handled for the frame, the ComputeFrame() function
// will be called for each of the framers. If necessary to catch up with
// the frame time the framers list will be looped through several times,
// each time increasing the number passed to ComputeFrame().
// Once all calls to ComputeFrame() are done, the framers list is walked
// through ONCE and the RenderFrame() function is called for each framer.
// Entries of evhandlers that also support the Framer interface will be
// implicitly added to the BEGINNING of the framers list. Do not manually
// add them to framers or they will be called twice.
// When EventLoop() returns (because one of evhandlers has returned quit==true),
// you can find statistics in DroppedFrames and LostFrames.
func EventLoop(renderer *sdl.Renderer, evhandlers []EventHandler, framers []Framer) {
  t0 := time.Now()

  LostFrames = 0
  DroppedFrames = 0

  var lastframe int64 = 0

  running := true
  for {
    t := time.Since(t0)
    frame := int64((t+(FrameTime>>1)) / FrameTime) - LostFrames
    if frame - lastframe > MaxDroppedFrames+1 {
      LostFrames += frame - lastframe - 1
      frame = lastframe + 1
    }

    if frame == lastframe {
      time.Sleep(time.Millisecond)
      continue
    }

    DroppedFrames += frame - lastframe - 1

    for event := sdl.PollEvent(); event != nil; event = sdl.PollEvent() {
      for _, handler := range evhandlers {
        consume,quit := handler.Handle(event)
        if quit { running = false }
        if consume { break }
      }
    }

    if !running { break }

    for lastframe < frame {
      lastframe++
      for i := range evhandlers {
        framer, ok := evhandlers[i].(Framer)
        if ok { framer.ComputeFrame(lastframe) }
      }
      for i := range framers {
        framers[i].ComputeFrame(lastframe)
      }
    }

    renderer.Clear()

    for i := range evhandlers {
      framer, ok := evhandlers[i].(Framer)
      if ok { framer.RenderFrame(renderer) }
    }
    for i := range framers {
      framers[i].RenderFrame(renderer)
    }

    renderer.Present()
  }
}

type quitter struct{}

func (*quitter) Handle(event sdl.Event) (consume, quit bool) {
  switch event.(type) {
    case *sdl.QuitEvent: return true, true
  }
  return false,false
}
<commit_msg>bugfix<commit_after>/* Copyright (C) 2017 Matthias S. Benkmann
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this file (originally named eventloop.go) and associated documentation files
 * (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is furnished
 * to do so, subject to the following conditions:
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

// Graphics related algorithms and data structures.
package gfx

import "time"
import "github.com/veandco/go-sdl2/sdl"

const (
  // Time per frame.
  FrameTime time.Duration = (time.Second+30)/60

  // If this number of dropped frames is exceeded, we assume that the process
  // has been suspended for some reason and will not try to catch up.
  MaxDroppedFrames = 10
)

// Handles an sdl.Event. Used with EventLoop().
type EventHandler interface{
  // Handle the event ev. Return values:
  //   consume: true => do not pass even to other handlers in the chain
  //   quit:    true => exit the event loop (immediately if consume == true)
  Handle(event sdl.Event) (consume, quit bool)
}

// Computes and renders frames. Used with EventLoop().
type Framer interface {
  // Compute frame with number num. The EventLoop() function guarantees that
  // this function is called for each number starting with 1 with no gaps
  // in the count. Note that multiple calls to ComputeFrame() may occur with
  // no calls to RenderFrame() in between (or at all). IOW ComputeFrame() must
  // not depend on anything RenderFrame() does.
  ComputeFrame(num int64)
  // Render the most recently computed frame. Note that while calls to ComputeFrame()
  // are never dropped, calls to RenderFrame() may be. Heavy stuff should
  // be done in RenderFrame() if possible.
  // NOTE: renderer.Clear() and renderer.Present() are called by EventLoop(). Do
  // not call them here unless you have some special (weird) reason.
  RenderFrame(renderer *sdl.Renderer)
}

// An EventHandler that causes EventLoop() to quit immediately (i.e. without
// further event processing) on the following events:
//   - sdl.QuitEvent
var Quitter = &quitter{}

// Set by EventLoop().
// "lost" frames count gaps in the program's execution that are so large that
// no attempt was made to catch up by dropping frames. This happens typically
// when the program is not scheduled for a while, e.g. because of SIGSTOP. In
// this case EventLoop() counts these frames as lost and simply continues with
// the next frame as if nothing happened.
var LostFrames int64

// Set by EventLoop().
// Dropped frames are frames that the program computes without presenting
// to the user in order to catch up with the frame that's supposed to be
// showing according to execution time.
var DroppedFrames int64

// Runs an SDL event loop that processes events once per frame as determined
// by FrameTime and MaxDroppedFrames. Each event is passed to each entry in
// evhandlers in turn until one of them returns consume==true.
// After all events are handled for the frame, the ComputeFrame() function
// will be called for each of the framers. If necessary to catch up with
// the frame time the framers list will be looped through several times,
// each time increasing the number passed to ComputeFrame().
// Once all calls to ComputeFrame() are done, the framers list is walked
// through ONCE and the RenderFrame() function is called for each framer.
// Entries of evhandlers that also support the Framer interface will be
// implicitly added to the BEGINNING of the framers list. Do not manually
// add them to framers or they will be called twice.
// When EventLoop() returns (because one of evhandlers has returned quit==true),
// you can find statistics in DroppedFrames and LostFrames.
func EventLoop(renderer *sdl.Renderer, evhandlers []EventHandler, framers []Framer) {
  t0 := time.Now()

  LostFrames = 0
  DroppedFrames = 0

  var lastframe int64 = 0

  running := true
  for {
    t := time.Since(t0)
    frame := int64((t+(FrameTime>>1)) / FrameTime) - LostFrames
    if frame - lastframe > MaxDroppedFrames+1 {
      LostFrames += frame - lastframe - 1
      frame = lastframe + 1
    }

    if frame == lastframe {
      time.Sleep(time.Millisecond)
      continue
    }

    DroppedFrames += frame - lastframe - 1

    for event := sdl.PollEvent(); event != nil; event = sdl.PollEvent() {
      for _, handler := range evhandlers {
        consume,quit := handler.Handle(event)
        if quit { running = false }
        if consume { break }
      }
    }

    if !running { break }

    for lastframe < frame {
      lastframe++
      for i := range evhandlers {
        framer, ok := evhandlers[i].(Framer)
        if ok { framer.ComputeFrame(lastframe) }
      }
      for i := range framers {
        framers[i].ComputeFrame(lastframe)
      }
    }

    renderer.Clear()

    for i := range evhandlers {
      framer, ok := evhandlers[i].(Framer)
      if ok { framer.RenderFrame(renderer) }
    }
    for i := range framers {
      framers[i].RenderFrame(renderer)
    }

    renderer.Present()
  }
}

type quitter struct{}

func (*quitter) Handle(event sdl.Event) (consume, quit bool) {
  switch event.(type) {
    case *sdl.QuitEvent: return true, true
  }
  return false,false
}
<|endoftext|>


<commit_before>package node

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"

	"github.com/c2stack/c2g/c2"
	"github.com/c2stack/c2g/meta"
)

// Value is a union of all possible values a leaf could hold.
type Value struct {
	Type       *meta.DataType
	Bool       bool
	Int        int
	UInt       uint
	Int64      int64
	UInt64     uint64
	Int64list  []int64
	UInt64list []uint64
	Str        string
	Float      float64
	Intlist    []int
	UIntlist   []uint
	Strlist    []string
	Boollist   []bool
	Floatlist  []float64
	Keys       []string
	AnyData    interface{}
}

func EncodeKey(v []*Value) string {
	var s string
	// TODO: read RFC and escape chars including commas
	for i, val := range v {
		if i > 0 {
			s = "," + val.String()
		}
		s = val.String()
	}
	return s
}

func (v *Value) Value() interface{} {
	switch v.Type.Format() {
	case meta.FMT_BOOLEAN:
		return v.Bool
	case meta.FMT_BOOLEAN_LIST:
		return v.Boollist
	case meta.FMT_INT64:
		return v.Int64
	case meta.FMT_UINT64:
		return v.UInt64
	case meta.FMT_INT64_LIST:
		return v.Int64list
	case meta.FMT_UINT64_LIST:
		return v.UInt64list
	case meta.FMT_UINT32:
		return v.UInt
	case meta.FMT_INT32, meta.FMT_ENUMERATION:
		return v.Int
	case meta.FMT_INT32_LIST, meta.FMT_ENUMERATION_LIST:
		return v.Intlist
	case meta.FMT_UINT32_LIST:
		return v.UIntlist
	case meta.FMT_STRING:
		return v.Str
	case meta.FMT_STRING_LIST:
		return v.Strlist
	case meta.FMT_ANYDATA:
		return v.AnyData
	case meta.FMT_DECIMAL64:
		return v.Float
	case meta.FMT_DECIMAL64_LIST:
		return v.Floatlist
	default:
		panic("Not implemented")
	}
}

func (a *Value) Compare(b *Value) int {
	if a == nil {
		if b == nil {
			return 0
		}
		return 1
	}
	if b == nil {
		return -1
	}
	if a.Type.Format() != b.Type.Format() {
		return int(a.Type.Format()) - int(b.Type.Format())
	}
	if meta.IsListFormat(a.Type.Format()) {
		panic("not sure how to compare arrays")
	}
	switch a.Type.Format() {
	case meta.FMT_BOOLEAN:
		if a.Bool == b.Bool {
			return 0
		}
		if a.Bool {
			return 1
		}
		return -1
	case meta.FMT_INT64:
		x := a.Int64 - b.Int64
		// we do the "long" form of comparision to avoid numeric overflow
		// where positive int64 > 2 billion is negative int32 when cast
		if x < 0 {
			return -1
		}
		if x > 0 {
			return 1
		}
		return 0
	case meta.FMT_UINT64:
		x := a.UInt64 - b.UInt64
		if x < 0 {
			return -1
		}
		if x > 0 {
			return 1
		}
		return 0
	case meta.FMT_UINT32:
		x := a.UInt - b.UInt
		if x < 0 {
			return -1
		}
		if x > 0 {
			return 1
		}
		return 0
	case meta.FMT_INT32, meta.FMT_ENUMERATION:
		return a.Int - b.Int
	case meta.FMT_STRING:
		return strings.Compare(a.Str, b.Str)
	case meta.FMT_DECIMAL64:
		x := a.Float - b.Float
		if x < 0 {
			return -1
		}
		if x > 0 {
			return 1
		}
		return 0
	default:
		panic("Not implemented")
	}
}

func (a *Value) Equal(b *Value) bool {
	if a == nil {
		if b == nil {
			return true
		}
		return false
	}
	if b == nil {
		return false
	}
	if a.Type.Format() != b.Type.Format() {
		return false
	}
	if meta.IsListFormat(a.Type.Format()) {
		return reflect.DeepEqual(a.Value(), b.Value())
	}
	return a.Value() == b.Value()
}

func (v *Value) SetEnumList(intlist []int) bool {
	strlist := make([]string, len(intlist))
	en := v.Type.Enumeration()
	for i, n := range intlist {
		if n >= len(en) {
			return false
		}
		strlist[i] = en[n]
	}
	v.Intlist = intlist
	v.Strlist = strlist
	return true
}

func NewEnumValue(t *meta.DataType, n int) *Value {
	v := &Value{Type: t}
	if !v.SetEnum(n) {
		panic(fmt.Sprintf("%d not legal value of %s", n, t.Ident))
	}
	return v
}

func (v *Value) SetEnumListByLabels(labels []string) bool {
	intlist := make([]int, len(labels))
	en := v.Type.Enumeration()
	for i, s := range labels {
		var found bool
		for j, e := range en {
			if s == e {
				found = true
				intlist[i] = j
				break
			}
		}
		if !found {
			return false
		}
	}
	v.Intlist = intlist
	v.Strlist = labels
	return true
}

func (v *Value) SetEnum(n int) bool {
	en := v.Type.Enumeration()
	if n < len(en) {
		v.Int = n
		v.Str = en[n]
		return true
	}
	return false
}

func (v *Value) SetEnumByLabel(label string) bool {
	for i, n := range v.Type.Enumeration() {
		if n == label {
			v.Int = i
			v.Str = label
			return true
		}
	}
	return false
}

func (v *Value) String() string {
	return fmt.Sprintf("%v", v.Value())
}

func SetValues(m []meta.HasDataType, objs ...interface{}) []*Value {
	var err error
	vals := make([]*Value, len(m))
	for i, obj := range objs {
		vals[i], err = SetValue(m[i].GetDataType(), obj)
		if err != nil {
			panic(err)
		}
	}
	return vals
}

// Incoming value should be of appropriate type according to given data type format
func SetValue(typ *meta.DataType, val interface{}) (*Value, error) {
	if val == nil {
		return nil, nil
	}
	reflectVal := reflect.ValueOf(val)
	v := &Value{Type: typ}
	switch typ.Format() {
	case meta.FMT_BOOLEAN:
		v.Bool = reflectVal.Bool()
	case meta.FMT_BOOLEAN_LIST:
		v.Boollist = InterfaceToBoollist(reflectVal.Interface())
	case meta.FMT_INT32_LIST:
		v.Intlist = InterfaceToIntlist(val)
	case meta.FMT_INT32:
		switch reflectVal.Kind() {
		// special case float mostly because of JSON
		case reflect.Float64:
			v.Int = int(reflectVal.Float())
		default:
			v.Int = int(reflectVal.Int())
		}
	case meta.FMT_UINT32:
		switch reflectVal.Kind() {
		case reflect.Float64:
			v.UInt = uint(reflectVal.Float())
		default:
			v.UInt = reflectVal.Interface().(uint)
		}
	case meta.FMT_DECIMAL64:
		v.Float = reflectVal.Float()
	case meta.FMT_UINT64:
		v.UInt64 = reflectVal.Interface().(uint64)
	case meta.FMT_INT64:
		switch reflectVal.Kind() {
		// special case float mostly because of JSON
		case reflect.Float64:
			v.Int64 = int64(reflectVal.Float())
		default:
			v.Int64 = reflectVal.Int()
		}
	case meta.FMT_INT64_LIST:
		v.Int64list = InterfaceToInt64list(val)
	case meta.FMT_STRING:
		v.Str = reflectVal.String()
	case meta.FMT_ENUMERATION:
		switch reflectVal.Kind() {
		case reflect.String:
			v.SetEnumByLabel(reflectVal.String())
		default:
			v.SetEnum(int(reflectVal.Int()))
		}
	case meta.FMT_ENUMERATION_LIST:
		val := reflectVal.Interface()
		strlist := InterfaceToStrlist(val)
		if len(strlist) > 0 {
			v.SetEnumListByLabels(strlist)
		} else {
			intlist := InterfaceToIntlist(val)
			v.SetEnumList(intlist)
		}
	case meta.FMT_STRING_LIST:
		v.Strlist = InterfaceToStrlist(reflectVal.Interface())
	case meta.FMT_ANYDATA:
		v.AnyData = reflectVal.Interface()
	default:
		panic(fmt.Sprintf("Format code %d not implemented", typ.Format))
	}
	return v, nil
}

func InterfaceToStrlist(o interface{}) (strlist []string) {
	switch arrayValues := o.(type) {
	case []string:
		return arrayValues
	case []interface{}:
		found := make([]string, len(arrayValues))
		for i, arrayValue := range arrayValues {
			if reflect.TypeOf(arrayValue).Kind() != reflect.String {
				return
			}
			found[i] = arrayValue.(string)
		}
		strlist = found
	}
	return
}

func InterfaceToBoollist(o interface{}) (boollist []bool) {
	switch arrayValues := o.(type) {
	case []bool:
		return arrayValues
	case []interface{}:
		boollist = make([]bool, len(arrayValues))
		for i, arrayValue := range arrayValues {
			boollist[i] = arrayValue.(bool)
		}
	}
	return
}

func InterfaceToInt64list(o interface{}) (intlist []int64) {
	switch arrayValues := o.(type) {
	case []int64:
		return arrayValues
	case []interface{}:
		intlist = make([]int64, len(arrayValues))
		for i, arrayValue := range arrayValues {
			intlist[i] = arrayValue.(int64)
		}
	}
	return
}

func InterfaceToIntlist(o interface{}) (intlist []int) {
	switch arrayValues := o.(type) {
	case []int:
		return arrayValues
	case []interface{}:
		intlist = make([]int, len(arrayValues))
		for i, arrayValue := range arrayValues {
			switch n := arrayValue.(type) {
			case int:
				intlist[i] = n
			case float32:
				intlist[i] = int(n)
			case float64:
				intlist[i] = int(n)
			case int64:
				intlist[i] = int(n)
			}
		}
	}
	return
}

func (v *Value) CoerseStrValue(s string) error {
	switch v.Type.Format() {
	case meta.FMT_BOOLEAN:
		v.Bool = s == "true"
	case meta.FMT_UINT64:
		var err error
		v.UInt64, err = strconv.ParseUint(s, 10, 64)
		if err != nil {
			return err
		}
	case meta.FMT_INT64:
		var err error
		v.Int64, err = strconv.ParseInt(s, 10, 64)
		if err != nil {
			return err
		}
	case meta.FMT_UINT32:
		i64, err := strconv.ParseUint(s, 10, 32)
		if err != nil {
			return err
		}
		v.UInt = uint(i64)
	case meta.FMT_INT32:
		var err error
		v.Int, err = strconv.Atoi(s)
		if err != nil {
			return err
		}
	case meta.FMT_STRING:
		v.Str = s
	case meta.FMT_ENUMERATION:
		eid, err := strconv.Atoi(s)
		if err == nil {
			v.SetEnum(eid)
			return nil
		}
		if !v.SetEnumByLabel(s) {
			return c2.NewErr("Not an allowed enumation: " + s)
		}
	default:
		panic(fmt.Sprintf("Coersion not supported from data format " + v.Type.Format().String()))
	}
	return nil
}
<commit_msg>add support for float point default coersion<commit_after>package node

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"

	"github.com/c2stack/c2g/c2"
	"github.com/c2stack/c2g/meta"
)

// Value is a union of all possible values a leaf could hold.
type Value struct {
	Type       *meta.DataType
	Bool       bool
	Int        int
	UInt       uint
	Int64      int64
	UInt64     uint64
	Int64list  []int64
	UInt64list []uint64
	Str        string
	Float      float64
	Intlist    []int
	UIntlist   []uint
	Strlist    []string
	Boollist   []bool
	Floatlist  []float64
	Keys       []string
	AnyData    interface{}
}

func EncodeKey(v []*Value) string {
	var s string
	// TODO: read RFC and escape chars including commas
	for i, val := range v {
		if i > 0 {
			s = "," + val.String()
		}
		s = val.String()
	}
	return s
}

func (v *Value) Value() interface{} {
	switch v.Type.Format() {
	case meta.FMT_BOOLEAN:
		return v.Bool
	case meta.FMT_BOOLEAN_LIST:
		return v.Boollist
	case meta.FMT_INT64:
		return v.Int64
	case meta.FMT_UINT64:
		return v.UInt64
	case meta.FMT_INT64_LIST:
		return v.Int64list
	case meta.FMT_UINT64_LIST:
		return v.UInt64list
	case meta.FMT_UINT32:
		return v.UInt
	case meta.FMT_INT32, meta.FMT_ENUMERATION:
		return v.Int
	case meta.FMT_INT32_LIST, meta.FMT_ENUMERATION_LIST:
		return v.Intlist
	case meta.FMT_UINT32_LIST:
		return v.UIntlist
	case meta.FMT_STRING:
		return v.Str
	case meta.FMT_STRING_LIST:
		return v.Strlist
	case meta.FMT_ANYDATA:
		return v.AnyData
	case meta.FMT_DECIMAL64:
		return v.Float
	case meta.FMT_DECIMAL64_LIST:
		return v.Floatlist
	default:
		panic("Not implemented")
	}
}

func (a *Value) Compare(b *Value) int {
	if a == nil {
		if b == nil {
			return 0
		}
		return 1
	}
	if b == nil {
		return -1
	}
	if a.Type.Format() != b.Type.Format() {
		return int(a.Type.Format()) - int(b.Type.Format())
	}
	if meta.IsListFormat(a.Type.Format()) {
		panic("not sure how to compare arrays")
	}
	switch a.Type.Format() {
	case meta.FMT_BOOLEAN:
		if a.Bool == b.Bool {
			return 0
		}
		if a.Bool {
			return 1
		}
		return -1
	case meta.FMT_INT64:
		x := a.Int64 - b.Int64
		// we do the "long" form of comparision to avoid numeric overflow
		// where positive int64 > 2 billion is negative int32 when cast
		if x < 0 {
			return -1
		}
		if x > 0 {
			return 1
		}
		return 0
	case meta.FMT_UINT64:
		x := a.UInt64 - b.UInt64
		if x < 0 {
			return -1
		}
		if x > 0 {
			return 1
		}
		return 0
	case meta.FMT_UINT32:
		x := a.UInt - b.UInt
		if x < 0 {
			return -1
		}
		if x > 0 {
			return 1
		}
		return 0
	case meta.FMT_INT32, meta.FMT_ENUMERATION:
		return a.Int - b.Int
	case meta.FMT_STRING:
		return strings.Compare(a.Str, b.Str)
	case meta.FMT_DECIMAL64:
		x := a.Float - b.Float
		if x < 0 {
			return -1
		}
		if x > 0 {
			return 1
		}
		return 0
	default:
		panic("Not implemented")
	}
}

func (a *Value) Equal(b *Value) bool {
	if a == nil {
		if b == nil {
			return true
		}
		return false
	}
	if b == nil {
		return false
	}
	if a.Type.Format() != b.Type.Format() {
		return false
	}
	if meta.IsListFormat(a.Type.Format()) {
		return reflect.DeepEqual(a.Value(), b.Value())
	}
	return a.Value() == b.Value()
}

func (v *Value) SetEnumList(intlist []int) bool {
	strlist := make([]string, len(intlist))
	en := v.Type.Enumeration()
	for i, n := range intlist {
		if n >= len(en) {
			return false
		}
		strlist[i] = en[n]
	}
	v.Intlist = intlist
	v.Strlist = strlist
	return true
}

func NewEnumValue(t *meta.DataType, n int) *Value {
	v := &Value{Type: t}
	if !v.SetEnum(n) {
		panic(fmt.Sprintf("%d not legal value of %s", n, t.Ident))
	}
	return v
}

func (v *Value) SetEnumListByLabels(labels []string) bool {
	intlist := make([]int, len(labels))
	en := v.Type.Enumeration()
	for i, s := range labels {
		var found bool
		for j, e := range en {
			if s == e {
				found = true
				intlist[i] = j
				break
			}
		}
		if !found {
			return false
		}
	}
	v.Intlist = intlist
	v.Strlist = labels
	return true
}

func (v *Value) SetEnum(n int) bool {
	en := v.Type.Enumeration()
	if n < len(en) {
		v.Int = n
		v.Str = en[n]
		return true
	}
	return false
}

func (v *Value) SetEnumByLabel(label string) bool {
	for i, n := range v.Type.Enumeration() {
		if n == label {
			v.Int = i
			v.Str = label
			return true
		}
	}
	return false
}

func (v *Value) String() string {
	return fmt.Sprintf("%v", v.Value())
}

func SetValues(m []meta.HasDataType, objs ...interface{}) []*Value {
	var err error
	vals := make([]*Value, len(m))
	for i, obj := range objs {
		vals[i], err = SetValue(m[i].GetDataType(), obj)
		if err != nil {
			panic(err)
		}
	}
	return vals
}

// Incoming value should be of appropriate type according to given data type format
func SetValue(typ *meta.DataType, val interface{}) (*Value, error) {
	if val == nil {
		return nil, nil
	}
	reflectVal := reflect.ValueOf(val)
	v := &Value{Type: typ}
	switch typ.Format() {
	case meta.FMT_BOOLEAN:
		v.Bool = reflectVal.Bool()
	case meta.FMT_BOOLEAN_LIST:
		v.Boollist = InterfaceToBoollist(reflectVal.Interface())
	case meta.FMT_INT32_LIST:
		v.Intlist = InterfaceToIntlist(val)
	case meta.FMT_INT32:
		switch reflectVal.Kind() {
		// special case float mostly because of JSON
		case reflect.Float64:
			v.Int = int(reflectVal.Float())
		default:
			v.Int = int(reflectVal.Int())
		}
	case meta.FMT_UINT32:
		switch reflectVal.Kind() {
		case reflect.Float64:
			v.UInt = uint(reflectVal.Float())
		default:
			v.UInt = reflectVal.Interface().(uint)
		}
	case meta.FMT_DECIMAL64:
		v.Float = reflectVal.Float()
	case meta.FMT_UINT64:
		v.UInt64 = reflectVal.Interface().(uint64)
	case meta.FMT_INT64:
		switch reflectVal.Kind() {
		// special case float mostly because of JSON
		case reflect.Float64:
			v.Int64 = int64(reflectVal.Float())
		default:
			v.Int64 = reflectVal.Int()
		}
	case meta.FMT_INT64_LIST:
		v.Int64list = InterfaceToInt64list(val)
	case meta.FMT_STRING:
		v.Str = reflectVal.String()
	case meta.FMT_ENUMERATION:
		switch reflectVal.Kind() {
		case reflect.String:
			v.SetEnumByLabel(reflectVal.String())
		default:
			v.SetEnum(int(reflectVal.Int()))
		}
	case meta.FMT_ENUMERATION_LIST:
		val := reflectVal.Interface()
		strlist := InterfaceToStrlist(val)
		if len(strlist) > 0 {
			v.SetEnumListByLabels(strlist)
		} else {
			intlist := InterfaceToIntlist(val)
			v.SetEnumList(intlist)
		}
	case meta.FMT_STRING_LIST:
		v.Strlist = InterfaceToStrlist(reflectVal.Interface())
	case meta.FMT_ANYDATA:
		v.AnyData = reflectVal.Interface()
	default:
		panic(fmt.Sprintf("Format code %d not implemented", typ.Format))
	}
	return v, nil
}

func InterfaceToStrlist(o interface{}) (strlist []string) {
	switch arrayValues := o.(type) {
	case []string:
		return arrayValues
	case []interface{}:
		found := make([]string, len(arrayValues))
		for i, arrayValue := range arrayValues {
			if reflect.TypeOf(arrayValue).Kind() != reflect.String {
				return
			}
			found[i] = arrayValue.(string)
		}
		strlist = found
	}
	return
}

func InterfaceToBoollist(o interface{}) (boollist []bool) {
	switch arrayValues := o.(type) {
	case []bool:
		return arrayValues
	case []interface{}:
		boollist = make([]bool, len(arrayValues))
		for i, arrayValue := range arrayValues {
			boollist[i] = arrayValue.(bool)
		}
	}
	return
}

func InterfaceToInt64list(o interface{}) (intlist []int64) {
	switch arrayValues := o.(type) {
	case []int64:
		return arrayValues
	case []interface{}:
		intlist = make([]int64, len(arrayValues))
		for i, arrayValue := range arrayValues {
			intlist[i] = arrayValue.(int64)
		}
	}
	return
}

func InterfaceToIntlist(o interface{}) (intlist []int) {
	switch arrayValues := o.(type) {
	case []int:
		return arrayValues
	case []interface{}:
		intlist = make([]int, len(arrayValues))
		for i, arrayValue := range arrayValues {
			switch n := arrayValue.(type) {
			case int:
				intlist[i] = n
			case float32:
				intlist[i] = int(n)
			case float64:
				intlist[i] = int(n)
			case int64:
				intlist[i] = int(n)
			}
		}
	}
	return
}

func (v *Value) CoerseStrValue(s string) error {
	switch v.Type.Format() {
	case meta.FMT_BOOLEAN:
		v.Bool = s == "true"
	case meta.FMT_UINT64:
		var err error
		v.UInt64, err = strconv.ParseUint(s, 10, 64)
		if err != nil {
			return err
		}
	case meta.FMT_INT64:
		var err error
		v.Int64, err = strconv.ParseInt(s, 10, 64)
		if err != nil {
			return err
		}
	case meta.FMT_UINT32:
		i64, err := strconv.ParseUint(s, 10, 32)
		if err != nil {
			return err
		}
		v.UInt = uint(i64)
	case meta.FMT_INT32:
		var err error
		v.Int, err = strconv.Atoi(s)
		if err != nil {
			return err
		}
	case meta.FMT_DECIMAL64:
		var err error
		v.Float, err = strconv.ParseFloat(s, 64)
		if err != nil {
			return err
		}
	case meta.FMT_STRING:
		v.Str = s
	case meta.FMT_ENUMERATION:
		eid, err := strconv.Atoi(s)
		if err == nil {
			v.SetEnum(eid)
			return nil
		}
		if !v.SetEnumByLabel(s) {
			return c2.NewErr("Not an allowed enumation: " + s)
		}
	default:
		panic(fmt.Sprintf("Coersion not supported from data format " + v.Type.Format().String()))
	}
	return nil
}
<|endoftext|>


<commit_before>package nomad

import (
	"strings"
	"sync/atomic"
	"time"

	log "github.com/hashicorp/go-hclog"

	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/raft"
	"github.com/hashicorp/serf/serf"
)

const (
	// StatusReap is used to update the status of a node if we
	// are handling a EventMemberReap
	StatusReap = serf.MemberStatus(-1)

	// maxPeerRetries limits how many invalidate attempts are made
	maxPeerRetries = 6

	// peerRetryBase is a baseline retry time
	peerRetryBase = 1 * time.Second
)

// serfEventHandler is used to handle events from the serf cluster
func (s *Server) serfEventHandler() {
	for {
		select {
		case e := <-s.eventCh:
			switch e.EventType() {
			case serf.EventMemberJoin:
				s.nodeJoin(e.(serf.MemberEvent))
				s.localMemberEvent(e.(serf.MemberEvent))
			case serf.EventMemberLeave, serf.EventMemberFailed:
				s.nodeFailed(e.(serf.MemberEvent))
				s.localMemberEvent(e.(serf.MemberEvent))
			case serf.EventMemberReap:
				s.localMemberEvent(e.(serf.MemberEvent))
			case serf.EventMemberUpdate, serf.EventUser, serf.EventQuery: // Ignore
			default:
				s.logger.Warn("unhandled serf event", "event", log.Fmt("%#v", e))
			}

		case <-s.shutdownCh:
			return
		}
	}
}

// nodeJoin is used to handle join events on the serf cluster
func (s *Server) nodeJoin(me serf.MemberEvent) {
	for _, m := range me.Members {
		ok, parts := isNomadServer(m)
		if !ok {
			s.logger.Warn("non-server in gossip pool", "member", m.Name)
			continue
		}
		s.logger.Info("adding server", "server", parts)

		// Check if this server is known
		found := false
		s.peerLock.Lock()
		existing := s.peers[parts.Region]
		for idx, e := range existing {
			if e.Name == parts.Name {
				existing[idx] = parts
				found = true
				break
			}
		}

		// Add ot the list if not known
		if !found {
			s.peers[parts.Region] = append(existing, parts)
		}

		// Check if a local peer
		if parts.Region == s.config.Region {
			s.localPeers[raft.ServerAddress(parts.Addr.String())] = parts
		}
		s.peerLock.Unlock()

		// If we still expecting to bootstrap, may need to handle this
		if atomic.LoadInt32(&s.config.Bootstrapped) == 0 {
			s.maybeBootstrap()
		}
	}
}

// maybeBootstrap is used to handle bootstrapping when a new server joins
func (s *Server) maybeBootstrap() {
	// Bootstrap can only be done if there are no committed logs, remove our
	// expectations of bootstrapping. This is slightly cheaper than the full
	// check that BootstrapCluster will do, so this is a good pre-filter.
	var index uint64
	var err error
	if s.raftStore != nil {
		index, err = s.raftStore.LastIndex()
	} else if s.raftInmem != nil {
		index, err = s.raftInmem.LastIndex()
	} else {
		panic("neither raftInmem or raftStore is initialized")
	}
	if err != nil {
		s.logger.Error("failed to read last raft index", "error", err)
		return
	}

	// Bootstrap can only be done if there are no committed logs,
	// remove our expectations of bootstrapping
	if index != 0 {
		atomic.StoreInt32(&s.config.Bootstrapped, 1)
		return
	}

	// Scan for all the known servers
	members := s.serf.Members()
	var servers []serverParts
	voters := 0
	for _, member := range members {
		valid, p := isNomadServer(member)
		if !valid {
			continue
		}
		if p.Region != s.config.Region {
			continue
		}
		if p.Expect != 0 && p.Expect != s.config.BootstrapExpect {
			s.logger.Error("peer has a conflicting expect value. All nodes should expect the same number", "member", member)
			return
		}
		if p.Bootstrap {
			s.logger.Error("peer has bootstrap mode. Expect disabled", "member", member)
			return
		}
		if !p.NonVoter {
			voters++
		}

		servers = append(servers, *p)
	}

	// Skip if we haven't met the minimum expect count
	if voters < s.config.BootstrapExpect {
		return
	}

	// Query each of the servers and make sure they report no Raft peers.
	req := &structs.GenericRequest{
		QueryOptions: structs.QueryOptions{
			AllowStale: true,
		},
	}
	for _, server := range servers {
		var peers []string

		// Retry with exponential backoff to get peer status from this server
		for attempt := uint(0); attempt < maxPeerRetries; attempt++ {
			if err := s.connPool.RPC(s.config.Region, server.Addr, server.MajorVersion,
				"Status.Peers", req, &peers); err != nil {
				nextRetry := (1 << attempt) * peerRetryBase
				s.logger.Error("failed to confirm peer status", "peer", server.Name, "error", err, "retry", nextRetry)
				time.Sleep(nextRetry)
			} else {
				break
			}
		}

		// Found a node with some Raft peers, stop bootstrap since there's
		// evidence of an existing cluster. We should get folded in by the
		// existing servers if that's the case, so it's cleaner to sit as a
		// candidate with no peers so we don't cause spurious elections.
		// It's OK this is racy, because even with an initial bootstrap
		// as long as one peer runs bootstrap things will work, and if we
		// have multiple peers bootstrap in the same way, that's OK. We
		// just don't want a server added much later to do a live bootstrap
		// and interfere with the cluster. This isn't required for Raft's
		// correctness because no server in the existing cluster will vote
		// for this server, but it makes things much more stable.
		if len(peers) > 0 {
			s.logger.Info("disabling bootstrap mode because existing Raft peers being reported by peer",
				"peer_name", server.Name, "peer_address", server.Addr)
			atomic.StoreInt32(&s.config.Bootstrapped, 1)
			return
		}
	}

	// Update the peer set
	// Attempt a live bootstrap!
	var configuration raft.Configuration
	var addrs []string
	minRaftVersion, err := s.autopilot.MinRaftProtocol()
	if err != nil {
		s.logger.Error("failed to read server raft versions", "error", err)
	}

	for _, server := range servers {
		addr := server.Addr.String()
		addrs = append(addrs, addr)
		var id raft.ServerID
		if minRaftVersion >= 3 {
			id = raft.ServerID(server.ID)
		} else {
			id = raft.ServerID(addr)
		}
		suffrage := raft.Voter
		if server.NonVoter {
			suffrage = raft.Nonvoter
		}
		peer := raft.Server{
			ID:       id,
			Address:  raft.ServerAddress(addr),
			Suffrage: suffrage,
		}
		configuration.Servers = append(configuration.Servers, peer)
	}
	s.logger.Info("found expected number of peers, attempting to bootstrap cluster...",
		"peers", strings.Join(addrs, ","))
	future := s.raft.BootstrapCluster(configuration)
	if err := future.Error(); err != nil {
		s.logger.Error("failed to bootstrap cluster", "error", err)
	}

	// Bootstrapping complete, or failed for some reason, don't enter this again
	atomic.StoreInt32(&s.config.Bootstrapped, 1)
}

// nodeFailed is used to handle fail events on the serf cluster
func (s *Server) nodeFailed(me serf.MemberEvent) {
	for _, m := range me.Members {
		ok, parts := isNomadServer(m)
		if !ok {
			continue
		}
		s.logger.Info("removing server", "server", parts)

		// Remove the server if known
		s.peerLock.Lock()
		existing := s.peers[parts.Region]
		n := len(existing)
		for i := 0; i < n; i++ {
			if existing[i].Name == parts.Name {
				existing[i], existing[n-1] = existing[n-1], nil
				existing = existing[:n-1]
				n--
				break
			}
		}

		// Trim the list there are no known servers in a region
		if n == 0 {
			delete(s.peers, parts.Region)
		} else {
			s.peers[parts.Region] = existing
		}

		// Check if local peer
		if parts.Region == s.config.Region {
			delete(s.localPeers, raft.ServerAddress(parts.Addr.String()))
		}
		s.peerLock.Unlock()
	}
}

// localMemberEvent is used to reconcile Serf events with the
// consistent store if we are the current leader.
func (s *Server) localMemberEvent(me serf.MemberEvent) {
	// Do nothing if we are not the leader
	if !s.IsLeader() {
		return
	}

	// Check if this is a reap event
	isReap := me.EventType() == serf.EventMemberReap

	// Queue the members for reconciliation
	for _, m := range me.Members {
		// Change the status if this is a reap event
		if isReap {
			m.Status = StatusReap
		}
		select {
		case s.reconcileCh <- m:
		default:
		}
	}
}
<commit_msg>Only bootstrap when `bootstrap_expect`<commit_after>package nomad

import (
	"strings"
	"sync/atomic"
	"time"

	log "github.com/hashicorp/go-hclog"

	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/raft"
	"github.com/hashicorp/serf/serf"
)

const (
	// StatusReap is used to update the status of a node if we
	// are handling a EventMemberReap
	StatusReap = serf.MemberStatus(-1)

	// maxPeerRetries limits how many invalidate attempts are made
	maxPeerRetries = 6

	// peerRetryBase is a baseline retry time
	peerRetryBase = 1 * time.Second
)

// serfEventHandler is used to handle events from the serf cluster
func (s *Server) serfEventHandler() {
	for {
		select {
		case e := <-s.eventCh:
			switch e.EventType() {
			case serf.EventMemberJoin:
				s.nodeJoin(e.(serf.MemberEvent))
				s.localMemberEvent(e.(serf.MemberEvent))
			case serf.EventMemberLeave, serf.EventMemberFailed:
				s.nodeFailed(e.(serf.MemberEvent))
				s.localMemberEvent(e.(serf.MemberEvent))
			case serf.EventMemberReap:
				s.localMemberEvent(e.(serf.MemberEvent))
			case serf.EventMemberUpdate, serf.EventUser, serf.EventQuery: // Ignore
			default:
				s.logger.Warn("unhandled serf event", "event", log.Fmt("%#v", e))
			}

		case <-s.shutdownCh:
			return
		}
	}
}

// nodeJoin is used to handle join events on the serf cluster
func (s *Server) nodeJoin(me serf.MemberEvent) {
	for _, m := range me.Members {
		ok, parts := isNomadServer(m)
		if !ok {
			s.logger.Warn("non-server in gossip pool", "member", m.Name)
			continue
		}
		s.logger.Info("adding server", "server", parts)

		// Check if this server is known
		found := false
		s.peerLock.Lock()
		existing := s.peers[parts.Region]
		for idx, e := range existing {
			if e.Name == parts.Name {
				existing[idx] = parts
				found = true
				break
			}
		}

		// Add ot the list if not known
		if !found {
			s.peers[parts.Region] = append(existing, parts)
		}

		// Check if a local peer
		if parts.Region == s.config.Region {
			s.localPeers[raft.ServerAddress(parts.Addr.String())] = parts
		}
		s.peerLock.Unlock()

		// If we still expecting to bootstrap, may need to handle this
		if s.config.BootstrapExpect != 0 && atomic.LoadInt32(&s.config.Bootstrapped) == 0 {
			s.maybeBootstrap()
		}
	}
}

// maybeBootstrap is used to handle bootstrapping when a new server joins
func (s *Server) maybeBootstrap() {
	// Bootstrap can only be done if there are no committed logs, remove our
	// expectations of bootstrapping. This is slightly cheaper than the full
	// check that BootstrapCluster will do, so this is a good pre-filter.
	var index uint64
	var err error
	if s.raftStore != nil {
		index, err = s.raftStore.LastIndex()
	} else if s.raftInmem != nil {
		index, err = s.raftInmem.LastIndex()
	} else {
		panic("neither raftInmem or raftStore is initialized")
	}
	if err != nil {
		s.logger.Error("failed to read last raft index", "error", err)
		return
	}

	// Bootstrap can only be done if there are no committed logs,
	// remove our expectations of bootstrapping
	if index != 0 {
		atomic.StoreInt32(&s.config.Bootstrapped, 1)
		return
	}

	// Scan for all the known servers
	members := s.serf.Members()
	var servers []serverParts
	voters := 0
	for _, member := range members {
		valid, p := isNomadServer(member)
		if !valid {
			continue
		}
		if p.Region != s.config.Region {
			continue
		}
		if p.Expect != 0 && p.Expect != s.config.BootstrapExpect {
			s.logger.Error("peer has a conflicting expect value. All nodes should expect the same number", "member", member)
			return
		}
		if p.Bootstrap {
			s.logger.Error("peer has bootstrap mode. Expect disabled", "member", member)
			return
		}
		if !p.NonVoter {
			voters++
		}

		servers = append(servers, *p)
	}

	// Skip if we haven't met the minimum expect count
	if voters < s.config.BootstrapExpect {
		return
	}

	// Query each of the servers and make sure they report no Raft peers.
	req := &structs.GenericRequest{
		QueryOptions: structs.QueryOptions{
			AllowStale: true,
		},
	}
	for _, server := range servers {
		var peers []string

		// Retry with exponential backoff to get peer status from this server
		for attempt := uint(0); attempt < maxPeerRetries; attempt++ {
			if err := s.connPool.RPC(s.config.Region, server.Addr, server.MajorVersion,
				"Status.Peers", req, &peers); err != nil {
				nextRetry := (1 << attempt) * peerRetryBase
				s.logger.Error("failed to confirm peer status", "peer", server.Name, "error", err, "retry", nextRetry)
				time.Sleep(nextRetry)
			} else {
				break
			}
		}

		// Found a node with some Raft peers, stop bootstrap since there's
		// evidence of an existing cluster. We should get folded in by the
		// existing servers if that's the case, so it's cleaner to sit as a
		// candidate with no peers so we don't cause spurious elections.
		// It's OK this is racy, because even with an initial bootstrap
		// as long as one peer runs bootstrap things will work, and if we
		// have multiple peers bootstrap in the same way, that's OK. We
		// just don't want a server added much later to do a live bootstrap
		// and interfere with the cluster. This isn't required for Raft's
		// correctness because no server in the existing cluster will vote
		// for this server, but it makes things much more stable.
		if len(peers) > 0 {
			s.logger.Info("disabling bootstrap mode because existing Raft peers being reported by peer",
				"peer_name", server.Name, "peer_address", server.Addr)
			atomic.StoreInt32(&s.config.Bootstrapped, 1)
			return
		}
	}

	// Update the peer set
	// Attempt a live bootstrap!
	var configuration raft.Configuration
	var addrs []string
	minRaftVersion, err := s.autopilot.MinRaftProtocol()
	if err != nil {
		s.logger.Error("failed to read server raft versions", "error", err)
	}

	for _, server := range servers {
		addr := server.Addr.String()
		addrs = append(addrs, addr)
		var id raft.ServerID
		if minRaftVersion >= 3 {
			id = raft.ServerID(server.ID)
		} else {
			id = raft.ServerID(addr)
		}
		suffrage := raft.Voter
		if server.NonVoter {
			suffrage = raft.Nonvoter
		}
		peer := raft.Server{
			ID:       id,
			Address:  raft.ServerAddress(addr),
			Suffrage: suffrage,
		}
		configuration.Servers = append(configuration.Servers, peer)
	}
	s.logger.Info("found expected number of peers, attempting to bootstrap cluster...",
		"peers", strings.Join(addrs, ","))
	future := s.raft.BootstrapCluster(configuration)
	if err := future.Error(); err != nil {
		s.logger.Error("failed to bootstrap cluster", "error", err)
	}

	// Bootstrapping complete, or failed for some reason, don't enter this again
	atomic.StoreInt32(&s.config.Bootstrapped, 1)
}

// nodeFailed is used to handle fail events on the serf cluster
func (s *Server) nodeFailed(me serf.MemberEvent) {
	for _, m := range me.Members {
		ok, parts := isNomadServer(m)
		if !ok {
			continue
		}
		s.logger.Info("removing server", "server", parts)

		// Remove the server if known
		s.peerLock.Lock()
		existing := s.peers[parts.Region]
		n := len(existing)
		for i := 0; i < n; i++ {
			if existing[i].Name == parts.Name {
				existing[i], existing[n-1] = existing[n-1], nil
				existing = existing[:n-1]
				n--
				break
			}
		}

		// Trim the list there are no known servers in a region
		if n == 0 {
			delete(s.peers, parts.Region)
		} else {
			s.peers[parts.Region] = existing
		}

		// Check if local peer
		if parts.Region == s.config.Region {
			delete(s.localPeers, raft.ServerAddress(parts.Addr.String()))
		}
		s.peerLock.Unlock()
	}
}

// localMemberEvent is used to reconcile Serf events with the
// consistent store if we are the current leader.
func (s *Server) localMemberEvent(me serf.MemberEvent) {
	// Do nothing if we are not the leader
	if !s.IsLeader() {
		return
	}

	// Check if this is a reap event
	isReap := me.EventType() == serf.EventMemberReap

	// Queue the members for reconciliation
	for _, m := range me.Members {
		// Change the status if this is a reap event
		if isReap {
			m.Status = StatusReap
		}
		select {
		case s.reconcileCh <- m:
		default:
		}
	}
}
<|endoftext|>


<commit_before>package fs

import (
	"io"
	"sync"
	"time"

	"github.com/chzyer/flow"
	"github.com/chzyer/logex"
)

type FlushDelegate interface {
	ReadData(off int64, n int) ([]byte, error)
	io.WriterAt

	UpdateCheckpoint(cp int64)
}

type Flusher struct {
	flow     *flow.Flow
	interval time.Duration
	offset   int64 // point to the start of partial
	delegate FlushDelegate

	flushChan   chan struct{}
	flushWaiter sync.WaitGroup

	opChan chan *flusherWriteOp
}

type FlusherConfig struct {
	Interval time.Duration
	Delegate FlushDelegate
	Offset   int64
}

func NewFlusher(f *flow.Flow, cfg *FlusherConfig) *Flusher {
	flusher := &Flusher{
		interval:  cfg.Interval,
		opChan:    make(chan *flusherWriteOp, 100),
		flushChan: make(chan struct{}, 1),
		offset:    cfg.Offset,
		delegate:  cfg.Delegate,
		flow:      f.Fork(1),
	}
	f.SetOnClose(flusher.Close)
	go flusher.loop()
	return flusher
}

func (f *Flusher) handleOpInPartialArea(dw *DiskWriter, op *flushItem) error {
	ino, idx, err := op.inoPool.RefPayloadBlock()
	if err != nil {
		return logex.Tracefmt(
			"error in fetch inode(%v): %v", op.inoPool.ino, err)
	}

	blkSize := ino.GetBlockSize(idx)
	dataAddr := ShortAddr(f.getAddr(dw.Written()))

	if blkSize > 0 {
		// ino.Offsets[idx] maybe in memory
		oldData, err := f.delegate.ReadData(int64(ino.Offsets[idx]), blkSize)
		if err != nil {
			return logex.Tracefmt(
				"error in readdata at %v(%v): %v",
				ino.Offsets[idx], blkSize, err)
		}
		dw.WriteBytes(oldData)
		Stat.Flusher.BlockCopy.AddInt(len(oldData))
	}
	length := op.data.Len()
	op.data.WriteData(dw, -1)
	ino.SetOffset(idx, dataAddr, length)

	if len(op.tmpInodes) == 0 || op.tmpInodes[len(op.tmpInodes)-1] != ino {
		op.tmpInodes = append(op.tmpInodes, ino)
	}

	return nil
}

func (f *Flusher) Flush() {
	f.flushWaiter.Add(1)
	select {
	case f.flushChan <- struct{}{}:
		f.flushWaiter.Wait()
	default:
		f.flushWaiter.Done()
	}
}

func (f *Flusher) getAddr(offset int64) Address {
	return Address(f.offset + offset)
}

func (f *Flusher) handleOpInDataArea(dw *DiskWriter, op *flushItem) error {
	var inos []*Inode
	var dataAddr ShortAddr

writePayload:
	ino, idx, err := op.inoPool.RefPayloadBlock()
	if err != nil {
		return logex.Tracefmt(
			"error in fetch inode(%v): %v", op.inoPool.ino, err)
	}

	dataAddr = ShortAddr(f.getAddr(dw.Written()))
	blkSize := ino.GetBlockSize(idx)
	if op.data.Len() < BlockSize-blkSize {
		goto exit
	}

	if blkSize > 0 {
		oldData, err := f.delegate.ReadData(int64(ino.Offsets[idx]), blkSize)
		if err != nil {
			return logex.Tracefmt(
				"error in readdata at %v: %v", ino.Offsets[idx], err)
		}
		dw.WriteBytes(oldData)
		Stat.Flusher.BlockCopy.AddInt(len(oldData))
	}

	op.data.WriteData(dw, BlockSize-blkSize)

	ino.SetOffset(idx, dataAddr, BlockSize-blkSize)
	if len(inos) == 0 || inos[len(inos)-1] != ino {
		inos = append(inos, ino)
	}
	goto writePayload

exit:
	op.tmpInodes = inos
	return nil
}

func (f *Flusher) handleOps(data []byte, ops []*flushItem) int64 {
	// p: payload, ino: inode, b: block, pp: partial payload
	// | data area | partial area            |
	// | b1 | b2   | b3                      |
	// | p1 | p2   | pp1 + pp2 + ino1 + ino2 |
	// > how to fsck ? follow a MagicEOF
	dw := NewDiskWriter(data)

	// in data area
	for idx, op := range ops {
		dw.Mark()
		if err := f.handleOpInDataArea(dw, op); err != nil {
			dw.Reset()
			op.sendDone(logex.Trace(err))
			ops[idx] = nil
			continue
		}
	}

	// in partial area
	for idx, op := range ops {
		if op == nil {
			continue
		}
		if op.data.Len() == 0 {
			continue
		}
		dw.Mark()
		if err := f.handleOpInPartialArea(dw, op); err != nil {
			dw.Reset()
			op.sendDone(logex.Trace(err))
			ops[idx] = nil
			continue
		}
	}

	// write inode
	for _, op := range ops {
		if op == nil {
			continue
		}
		for _, ino := range op.tmpInodes {
			inoAddr := f.getAddr(dw.Written())
			dw.WriteItem(ino)
			op.inoPool.OnFlush(ino, inoAddr)
		}
	}

	// send reply to ops in flush()

	dw.WriteBytes(MagicEOF)
	return dw.Written()
}

func (f *Flusher) flush(fb *flushBuffer) {
	buffer := fb.alloc()
	written := f.handleOps(buffer, fb.ops())
	buffer = buffer[:written]

	// write to disk
flush:
	_, err := f.delegate.WriteAt(buffer, f.offset)
	if err != nil {
		logex.Error("error in write data, wait 1 sec:", err)
		switch f.flow.CloseOrWait(time.Second) {
		case flow.F_CLOSED:
			for _, op := range fb.ops() {
				if op == nil {
					continue
				}
				op.sendDone(logex.Trace(err))
			}
			fb.reset()
			return
		case flow.F_TIMEOUT:
			goto flush
		}
	}

	f.offset += int64(len(buffer))
	f.delegate.UpdateCheckpoint(f.offset)
	for _, op := range fb.ops() {
		if op == nil {
			continue
		}
		op.sendDone(nil)
	}
	fb.reset()
}

func (f *Flusher) loop() {
	defer f.flow.Done()

	var (
		fb    flushBuffer
		timer = time.NewTimer(0)
	)
	fb.init()
	timer.Stop()
	wantFlush := false

loop:
	for {
		select {
		case op := <-f.opChan:
			fb.addOp(op)
			timer.Reset(f.interval)

			if !wantFlush {
			buffering:
				for {
					select {
					case <-timer.C:
						break buffering
					case <-f.flushChan:
						wantFlush = true
						break buffering
					case op := <-f.opChan:
						if !fb.addOp(op) {
							break buffering
						}
					}
				}
			}

			f.flush(&fb)
			if wantFlush {
				f.flushWaiter.Done()
				wantFlush = false
			}
		case <-f.flushChan:
			if len(f.opChan) != 0 {
				wantFlush = true
			} else {
				f.flushWaiter.Done()
			}

		case <-f.flow.IsClose():
			break loop
		}
	}
}

type flusherWriteOp struct {
	inoPool *InodePool
	done    chan error
	data    []byte
}

func (f *Flusher) WriteByInode(inoPool *InodePool, data []byte, done chan error) {
	f.opChan <- &flusherWriteOp{inoPool: inoPool, data: data, done: done}
}

func (f *Flusher) Close() {
	if !f.flow.MarkExit() {
		return
	}

	f.flow.Close()
	close(f.opChan)

	var fb flushBuffer
	fb.init()
	for op := range f.opChan {
		fb.addOp(op)
	}
	f.flush(&fb)
}

type flushItem struct {
	tmpInodes []*Inode
	inoPool   *InodePool
	done      []chan error
	data      *DataSlice
}

func (f *flushItem) sendDone(err error) {
	for _, d := range f.done {
		d <- err
	}
}

type flushBuffer struct {
	bufferingOps  []*flushItem
	bufferingSize int
	buffer        []byte
}

func (f *flushBuffer) init() {
	f.buffer = make([]byte, 4<<20)
}

func (f *flushBuffer) findOp(op *flusherWriteOp) int {
	for idx, bop := range f.bufferingOps {
		if bop.inoPool == op.inoPool {
			return idx
		}
	}
	return -1
}

// return false to flush
func (f *flushBuffer) addOp(op *flusherWriteOp) bool {
	// if multiple op belong to same file?
	idx := f.findOp(op)
	if idx >= 0 {
		bop := f.bufferingOps[idx]
		bop.data.Append(op.data)
		bop.done = append(bop.done, op.done)
	} else {
		f.bufferingOps = append(f.bufferingOps, &flushItem{
			inoPool: op.inoPool,
			done:    []chan error{op.done},
			data:    NewDataSlice(op.data),
		})
	}

	ino, err := op.inoPool.GetLastest()
	if err != nil {
		panic("can not get lastest")
	}

	// calculate the copy of partial data
	f.bufferingSize += len(op.data) +
		CalNeedInodeCnt(ino, len(op.data))*InodeSize +
		int(ino.Size)&(BlockSize-1)

	if f.bufferingSize >= 20<<20 {
		return false
	}
	return true
}

func (f *flushBuffer) alloc() []byte {
	f.bufferingSize += MagicSize // MagicEOF
	f.buffer = MakeRoom(f.buffer, f.bufferingSize)
	return f.buffer
}

func (f *flushBuffer) ops() []*flushItem {
	return f.bufferingOps
}

func (f *flushBuffer) reset() {
	f.bufferingOps = f.bufferingOps[:0]
	f.bufferingSize = 0
	f.buffer = f.buffer[:0]
}

// -----------------------------------------------------------------------------

type DataSlice struct {
	data   [][]byte
	length int
	offset int
}

func NewDataSlice(d []byte) *DataSlice {
	return &DataSlice{
		data:   [][]byte{d},
		length: len(d),
	}
}

func (f *DataSlice) Append(b []byte) {
	f.data = append(f.data, b)
	f.length += len(b)
}

func (f *DataSlice) WriteData(dw *DiskWriter, n int) {
	if n == -1 {
		n = f.length
	}

	for i := f.offset; i < len(f.data) && n > 0; i++ {
		f.offset = i
		if n > len(f.data[i]) {
			dw.WriteBytes(f.data[i])
			f.length -= len(f.data[i])
			n -= len(f.data[i])
			f.data[i] = nil
		} else {
			dw.WriteBytes(f.data[i][:n])
			f.length -= n
			f.data[i] = f.data[i][n:]
			n = 0
		}
	}
}

func (f *DataSlice) Len() int {
	return f.length
}
<commit_msg>[fs] flusher: eliminate empty flush<commit_after>package fs

import (
	"io"
	"sync"
	"time"

	"github.com/chzyer/flow"
	"github.com/chzyer/logex"
)

type FlushDelegate interface {
	ReadData(off int64, n int) ([]byte, error)
	io.WriterAt

	UpdateCheckpoint(cp int64)
}

type Flusher struct {
	flow     *flow.Flow
	interval time.Duration
	offset   int64 // point to the start of partial
	delegate FlushDelegate

	flushChan   chan struct{}
	flushWaiter sync.WaitGroup

	opChan chan *flusherWriteOp
}

type FlusherConfig struct {
	Interval time.Duration
	Delegate FlushDelegate
	Offset   int64
}

func NewFlusher(f *flow.Flow, cfg *FlusherConfig) *Flusher {
	flusher := &Flusher{
		interval:  cfg.Interval,
		opChan:    make(chan *flusherWriteOp, 100),
		flushChan: make(chan struct{}, 1),
		offset:    cfg.Offset,
		delegate:  cfg.Delegate,
		flow:      f.Fork(1),
	}
	f.SetOnClose(flusher.Close)
	go flusher.loop()
	return flusher
}

func (f *Flusher) handleOpInPartialArea(dw *DiskWriter, op *flushItem) error {
	ino, idx, err := op.inoPool.RefPayloadBlock()
	if err != nil {
		return logex.Tracefmt(
			"error in fetch inode(%v): %v", op.inoPool.ino, err)
	}

	blkSize := ino.GetBlockSize(idx)
	dataAddr := ShortAddr(f.getAddr(dw.Written()))

	if blkSize > 0 {
		// ino.Offsets[idx] maybe in memory
		oldData, err := f.delegate.ReadData(int64(ino.Offsets[idx]), blkSize)
		if err != nil {
			return logex.Tracefmt(
				"error in readdata at %v(%v): %v",
				ino.Offsets[idx], blkSize, err)
		}
		dw.WriteBytes(oldData)
		Stat.Flusher.BlockCopy.AddInt(len(oldData))
	}
	length := op.data.Len()
	op.data.WriteData(dw, -1)
	ino.SetOffset(idx, dataAddr, length)

	if len(op.tmpInodes) == 0 || op.tmpInodes[len(op.tmpInodes)-1] != ino {
		op.tmpInodes = append(op.tmpInodes, ino)
	}

	return nil
}

func (f *Flusher) Flush() {
	f.flushWaiter.Add(1)
	select {
	case f.flushChan <- struct{}{}:
		f.flushWaiter.Wait()
	default:
		f.flushWaiter.Done()
	}
}

func (f *Flusher) getAddr(offset int64) Address {
	return Address(f.offset + offset)
}

func (f *Flusher) handleOpInDataArea(dw *DiskWriter, op *flushItem) error {
	var inos []*Inode
	var dataAddr ShortAddr

writePayload:
	ino, idx, err := op.inoPool.RefPayloadBlock()
	if err != nil {
		return logex.Tracefmt(
			"error in fetch inode(%v): %v", op.inoPool.ino, err)
	}

	dataAddr = ShortAddr(f.getAddr(dw.Written()))
	blkSize := ino.GetBlockSize(idx)
	if op.data.Len() < BlockSize-blkSize {
		goto exit
	}

	if blkSize > 0 {
		oldData, err := f.delegate.ReadData(int64(ino.Offsets[idx]), blkSize)
		if err != nil {
			return logex.Tracefmt(
				"error in readdata at %v: %v", ino.Offsets[idx], err)
		}
		dw.WriteBytes(oldData)
		Stat.Flusher.BlockCopy.AddInt(len(oldData))
	}

	op.data.WriteData(dw, BlockSize-blkSize)

	ino.SetOffset(idx, dataAddr, BlockSize-blkSize)
	if len(inos) == 0 || inos[len(inos)-1] != ino {
		inos = append(inos, ino)
	}
	goto writePayload

exit:
	op.tmpInodes = inos
	return nil
}

func (f *Flusher) handleOps(data []byte, ops []*flushItem) int64 {
	// p: payload, ino: inode, b: block, pp: partial payload
	// | data area | partial area            |
	// | b1 | b2   | b3                      |
	// | p1 | p2   | pp1 + pp2 + ino1 + ino2 |
	// > how to fsck ? follow a MagicEOF
	dw := NewDiskWriter(data)

	// in data area
	for idx, op := range ops {
		dw.Mark()
		if err := f.handleOpInDataArea(dw, op); err != nil {
			dw.Reset()
			op.sendDone(logex.Trace(err))
			ops[idx] = nil
			continue
		}
	}

	// in partial area
	for idx, op := range ops {
		if op == nil {
			continue
		}
		if op.data.Len() == 0 {
			continue
		}
		dw.Mark()
		if err := f.handleOpInPartialArea(dw, op); err != nil {
			dw.Reset()
			op.sendDone(logex.Trace(err))
			ops[idx] = nil
			continue
		}
	}

	// write inode
	for _, op := range ops {
		if op == nil {
			continue
		}
		for _, ino := range op.tmpInodes {
			inoAddr := f.getAddr(dw.Written())
			dw.WriteItem(ino)
			op.inoPool.OnFlush(ino, inoAddr)
		}
	}

	// send reply to ops in flush()

	dw.WriteBytes(MagicEOF)
	return dw.Written()
}

func (f *Flusher) flush(fb *flushBuffer) {
	if len(fb.ops()) == 0 {
		return
	}
	buffer := fb.alloc()
	written := f.handleOps(buffer, fb.ops())
	buffer = buffer[:written]

	// write to disk
flush:
	_, err := f.delegate.WriteAt(buffer, f.offset)
	if err != nil {
		logex.Error("error in write data, wait 1 sec:", err)
		switch f.flow.CloseOrWait(time.Second) {
		case flow.F_CLOSED:
			for _, op := range fb.ops() {
				if op == nil {
					continue
				}
				op.sendDone(logex.Trace(err))
			}
			fb.reset()
			return
		case flow.F_TIMEOUT:
			goto flush
		}
	}

	f.offset += int64(len(buffer))
	f.delegate.UpdateCheckpoint(f.offset)
	for _, op := range fb.ops() {
		if op == nil {
			continue
		}
		op.sendDone(nil)
	}
	fb.reset()
}

func (f *Flusher) loop() {
	defer f.flow.Done()

	var (
		fb    flushBuffer
		timer = time.NewTimer(0)
	)
	fb.init()
	timer.Stop()
	wantFlush := false

loop:
	for {
		select {
		case op := <-f.opChan:
			fb.addOp(op)
			timer.Reset(f.interval)

			if !wantFlush {
			buffering:
				for {
					select {
					case <-timer.C:
						break buffering
					case <-f.flushChan:
						wantFlush = true
						break buffering
					case op := <-f.opChan:
						if !fb.addOp(op) {
							break buffering
						}
					}
				}
			}

			f.flush(&fb)
			if wantFlush {
				f.flushWaiter.Done()
				wantFlush = false
			}
		case <-f.flushChan:
			if len(f.opChan) != 0 {
				wantFlush = true
			} else {
				f.flushWaiter.Done()
			}

		case <-f.flow.IsClose():
			break loop
		}
	}
}

type flusherWriteOp struct {
	inoPool *InodePool
	done    chan error
	data    []byte
}

func (f *Flusher) WriteByInode(inoPool *InodePool, data []byte, done chan error) {
	f.opChan <- &flusherWriteOp{inoPool: inoPool, data: data, done: done}
}

func (f *Flusher) Close() {
	if !f.flow.MarkExit() {
		return
	}

	f.flow.Close()
	close(f.opChan)

	var fb flushBuffer
	fb.init()
	for op := range f.opChan {
		fb.addOp(op)
	}
	f.flush(&fb)
}

type flushItem struct {
	tmpInodes []*Inode
	inoPool   *InodePool
	done      []chan error
	data      *DataSlice
}

func (f *flushItem) sendDone(err error) {
	for _, d := range f.done {
		d <- err
	}
}

type flushBuffer struct {
	bufferingOps  []*flushItem
	bufferingSize int
	buffer        []byte
}

func (f *flushBuffer) init() {
	f.buffer = make([]byte, 4<<20)
}

func (f *flushBuffer) findOp(op *flusherWriteOp) int {
	for idx, bop := range f.bufferingOps {
		if bop.inoPool == op.inoPool {
			return idx
		}
	}
	return -1
}

// return false to flush
func (f *flushBuffer) addOp(op *flusherWriteOp) bool {
	// if multiple op belong to same file?
	idx := f.findOp(op)
	if idx >= 0 {
		bop := f.bufferingOps[idx]
		bop.data.Append(op.data)
		bop.done = append(bop.done, op.done)
	} else {
		f.bufferingOps = append(f.bufferingOps, &flushItem{
			inoPool: op.inoPool,
			done:    []chan error{op.done},
			data:    NewDataSlice(op.data),
		})
	}

	ino, err := op.inoPool.GetLastest()
	if err != nil {
		panic("can not get lastest")
	}

	// calculate the copy of partial data
	f.bufferingSize += len(op.data) +
		CalNeedInodeCnt(ino, len(op.data))*InodeSize +
		int(ino.Size)&(BlockSize-1)

	if f.bufferingSize >= 20<<20 {
		return false
	}
	return true
}

func (f *flushBuffer) alloc() []byte {
	f.bufferingSize += MagicSize // MagicEOF
	f.buffer = MakeRoom(f.buffer, f.bufferingSize)
	return f.buffer
}

func (f *flushBuffer) ops() []*flushItem {
	return f.bufferingOps
}

func (f *flushBuffer) reset() {
	f.bufferingOps = f.bufferingOps[:0]
	f.bufferingSize = 0
	f.buffer = f.buffer[:0]
}

// -----------------------------------------------------------------------------

type DataSlice struct {
	data   [][]byte
	length int
	offset int
}

func NewDataSlice(d []byte) *DataSlice {
	return &DataSlice{
		data:   [][]byte{d},
		length: len(d),
	}
}

func (f *DataSlice) Append(b []byte) {
	f.data = append(f.data, b)
	f.length += len(b)
}

func (f *DataSlice) WriteData(dw *DiskWriter, n int) {
	if n == -1 {
		n = f.length
	}

	for i := f.offset; i < len(f.data) && n > 0; i++ {
		f.offset = i
		if n > len(f.data[i]) {
			dw.WriteBytes(f.data[i])
			f.length -= len(f.data[i])
			n -= len(f.data[i])
			f.data[i] = nil
		} else {
			dw.WriteBytes(f.data[i][:n])
			f.length -= n
			f.data[i] = f.data[i][n:]
			n = 0
		}
	}
}

func (f *DataSlice) Len() int {
	return f.length
}
<|endoftext|>


<commit_before>package teams

import (
	"fmt"
	"sort"

	"golang.org/x/net/context"

	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

type statusList struct {
	Teams  []keybase1.MemberInfo `json:"teams"`
	Status libkb.AppStatus       `json:"status"`
}

func (r *statusList) GetAppStatus() *libkb.AppStatus {
	return &r.Status
}

func getTeamsListFromServer(ctx context.Context, g *libkb.GlobalContext, uid keybase1.UID, all bool) ([]keybase1.MemberInfo, error) {
	var endpoint string
	if all {
		endpoint = "team/teammates_for_user"
	} else {
		endpoint = "team/for_user"
	}
	a := libkb.NewAPIArg(endpoint)
	if uid.Exists() {
		a.Args = libkb.HTTPArgs{
			"uid": libkb.S{Val: uid.String()},
		}
	}
	a.NetContext = ctx
	a.SessionType = libkb.APISessionTypeREQUIRED
	var list statusList
	if err := g.API.GetDecode(a, &list); err != nil {
		return nil, err
	}
	return list.Teams, nil
}

func List(ctx context.Context, g *libkb.GlobalContext, arg keybase1.TeamListArg) (*keybase1.AnnotatedTeamList, error) {
	var uid keybase1.UID
	if arg.UserAssertion != "" {
		res := g.Resolver.ResolveFullExpression(ctx, arg.UserAssertion)
		if res.GetError() != nil {
			return nil, res.GetError()
		}
		uid = res.GetUID()
	}

	meUID := g.ActiveDevice.UID()

	teams, err := getTeamsListFromServer(ctx, g, uid, arg.All)
	if err != nil {
		return nil, err
	}

	teamNames := make(map[string]bool)
	upakLoader := g.GetUPAKLoader()
	annotatedTeams := make([]keybase1.AnnotatedMemberInfo, len(teams))
	administeredTeams := make(map[string]bool)
	for idx, memberInfo := range teams {
		teamNames[memberInfo.FqName] = true
		if memberInfo.UserID == meUID && (memberInfo.Role.IsAdminOrAbove() || (memberInfo.Implicit != nil && memberInfo.Implicit.Role.IsAdminOrAbove())) {
			administeredTeams[memberInfo.FqName] = true
		}

		memberuid := memberInfo.UserID
		username, err := upakLoader.LookupUsername(context.Background(), memberuid)
		if err != nil {
			return nil, err
		}
		fullName, err := engine.GetFullName(context.Background(), g, memberuid)
		if err != nil {
			return nil, err
		}
		annotatedTeams[idx] = keybase1.AnnotatedMemberInfo{
			TeamID:   memberInfo.TeamID,
			FqName:   memberInfo.FqName,
			UserID:   memberInfo.UserID,
			Role:     memberInfo.Role,
			Implicit: memberInfo.Implicit,
			Username: username.String(),
			FullName: fullName,
		}
	}

	annotatedInvites := make(map[keybase1.TeamInviteID]keybase1.AnnotatedTeamInvite)
	for teamName := range teamNames {
		_, ok := administeredTeams[teamName]
		if ok {
			t, err := GetForTeamManagementByStringName(ctx, g, teamName, true)
			if err != nil {
				g.Log.Warning("Error while getting team (%s): %v", teamName, err)
				continue
			}
			teamAnnotatedInvites, err := AnnotateInvites(ctx, g, t.chain().inner.ActiveInvites, teamName)
			if err != nil {
				return nil, err
			}
			for teamInviteID, annotatedTeamInvite := range teamAnnotatedInvites {
				annotatedInvites[teamInviteID] = annotatedTeamInvite
			}
		}
	}

	tl := keybase1.AnnotatedTeamList{
		Teams: annotatedTeams,
		AnnotatedActiveInvites: annotatedInvites,
	}

	return &tl, nil
}

func AnnotateInvites(ctx context.Context, g *libkb.GlobalContext, invites map[keybase1.TeamInviteID]keybase1.TeamInvite, teamName string) (map[keybase1.TeamInviteID]keybase1.AnnotatedTeamInvite, error) {

	annotatedInvites := make(map[keybase1.TeamInviteID]keybase1.AnnotatedTeamInvite, len(invites))
	upakLoader := g.GetUPAKLoader()
	for id, invite := range invites {
		username, err := upakLoader.LookupUsername(ctx, invite.Inviter.Uid)
		if err != nil {
			return annotatedInvites, err
		}

		name := invite.Name
		category, err := invite.Type.C()
		if err != nil {
			return nil, err
		}
		if category == keybase1.TeamInviteCategory_KEYBASE {
			// "keybase" invites (i.e. pukless users) have user version for name
			uv, err := invite.KeybaseUserVersion()
			if err != nil {
				return nil, err
			}
			normalized, err := upakLoader.LookupUsername(context.Background(), uv.Uid)
			if err != nil {
				return nil, err
			}
			name = keybase1.TeamInviteName(normalized.String())
		}
		annotatedInvites[id] = keybase1.AnnotatedTeamInvite{
			Role:            invite.Role,
			Id:              invite.Id,
			Type:            invite.Type,
			Name:            name,
			Inviter:         invite.Inviter,
			InviterUsername: username.String(),
			TeamName:        teamName,
		}
	}
	return annotatedInvites, nil
}

func TeamTree(ctx context.Context, g *libkb.GlobalContext, arg keybase1.TeamTreeArg) (res keybase1.TeamTreeResult, err error) {
	if !arg.Name.IsRootTeam() {
		return res, fmt.Errorf("cannot get tree of non-root team")
	}

	serverList, err := getTeamsListFromServer(ctx, g, "", false)
	if err != nil {
		return res, err
	}

	// Map from team name (string) to entry
	entryMap := make(map[string]keybase1.TeamTreeEntry)

	// The server might have omitted some teams, oh well.
	// Trusts the server for role.
	// Load the teams by ID to make sure they are valid and get the validated names.
	for _, info := range serverList {
		serverName, err := info.TeamName()
		if err != nil {
			return res, err
		}
		if !serverName.RootAncestorName().Eq(arg.Name) {
			// Skip those not in this tree.
			continue
		}
		team, err := Load(ctx, g, keybase1.LoadTeamArg{
			ID:          info.TeamID,
			ForceRepoll: true,
		})
		if err != nil {
			return res, err
		}
		var admin bool // true if an admin or implicit admin
		if info.Role.IsAdminOrAbove() {
			admin = true
		}
		if info.Implicit != nil && info.Implicit.Role.IsAdminOrAbove() {
			admin = true
		}
		entryMap[team.Name().String()] = keybase1.TeamTreeEntry{
			Name:  team.Name(),
			Admin: admin,
		}
	}

	// Add all parent names (recursively)
	// So that if only A.B.C is in the list, we add A.B and A as well.
	// Adding map entries while iterating is safe.
	// "If map entries are created during iteration, that entry may be produced during the iteration or may be skipped."
	for _, entry := range entryMap {
		name := entry.Name.DeepCopy()
		for name.Depth() > 0 {
			_, ok := entryMap[name.String()]
			if !ok {
				entryMap[name.String()] = keybase1.TeamTreeEntry{
					Name:  name,
					Admin: false,
				}
			}
			name, err = name.Parent()
			if err != nil {
				break
			}
		}
	}

	for _, entry := range entryMap {
		res.Entries = append(res.Entries, entry)
	}

	if len(res.Entries) == 0 {
		return res, fmt.Errorf("team not found: %v", arg.Name)
	}

	// Order into a tree order. Which happens to be alphabetical ordering.
	// Example: [a, a.b, a.b.c, a.b.d, a.e.f, a.e.g]
	sort.Slice(res.Entries, func(i, j int) bool {
		return res.Entries[i].Name.String() < res.Entries[j].Name.String()
	})

	return res, nil
}
<commit_msg>slow=false<commit_after>package teams

import (
	"fmt"
	"sort"

	"golang.org/x/net/context"

	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

type statusList struct {
	Teams  []keybase1.MemberInfo `json:"teams"`
	Status libkb.AppStatus       `json:"status"`
}

func (r *statusList) GetAppStatus() *libkb.AppStatus {
	return &r.Status
}

func getTeamsListFromServer(ctx context.Context, g *libkb.GlobalContext, uid keybase1.UID, all bool) ([]keybase1.MemberInfo, error) {
	var endpoint string
	if all {
		endpoint = "team/teammates_for_user"
	} else {
		endpoint = "team/for_user"
	}
	a := libkb.NewAPIArg(endpoint)
	if uid.Exists() {
		a.Args = libkb.HTTPArgs{
			"uid": libkb.S{Val: uid.String()},
		}
	}
	a.NetContext = ctx
	a.SessionType = libkb.APISessionTypeREQUIRED
	var list statusList
	if err := g.API.GetDecode(a, &list); err != nil {
		return nil, err
	}
	return list.Teams, nil
}

func List(ctx context.Context, g *libkb.GlobalContext, arg keybase1.TeamListArg) (*keybase1.AnnotatedTeamList, error) {
	var uid keybase1.UID
	if arg.UserAssertion != "" {
		res := g.Resolver.ResolveFullExpression(ctx, arg.UserAssertion)
		if res.GetError() != nil {
			return nil, res.GetError()
		}
		uid = res.GetUID()
	}

	meUID := g.ActiveDevice.UID()

	teams, err := getTeamsListFromServer(ctx, g, uid, arg.All)
	if err != nil {
		return nil, err
	}

	teamNames := make(map[string]bool)
	upakLoader := g.GetUPAKLoader()
	annotatedTeams := make([]keybase1.AnnotatedMemberInfo, len(teams))
	administeredTeams := make(map[string]bool)
	for idx, memberInfo := range teams {
		teamNames[memberInfo.FqName] = true
		if memberInfo.UserID == meUID && (memberInfo.Role.IsAdminOrAbove() || (memberInfo.Implicit != nil && memberInfo.Implicit.Role.IsAdminOrAbove())) {
			administeredTeams[memberInfo.FqName] = true
		}

		memberuid := memberInfo.UserID
		username, err := upakLoader.LookupUsername(context.Background(), memberuid)
		if err != nil {
			return nil, err
		}
		fullName, err := engine.GetFullName(context.Background(), g, memberuid)
		if err != nil {
			return nil, err
		}
		annotatedTeams[idx] = keybase1.AnnotatedMemberInfo{
			TeamID:   memberInfo.TeamID,
			FqName:   memberInfo.FqName,
			UserID:   memberInfo.UserID,
			Role:     memberInfo.Role,
			Implicit: memberInfo.Implicit,
			Username: username.String(),
			FullName: fullName,
		}
	}

	annotatedInvites := make(map[keybase1.TeamInviteID]keybase1.AnnotatedTeamInvite)
	for teamName := range teamNames {
		_, ok := administeredTeams[teamName]
		if ok {
			t, err := Load(ctx, g, keybase1.LoadTeamArg{
				Name:      teamName,
				NeedAdmin: true,
			})
			if err != nil {
				g.Log.Warning("Error while getting team (%s): %v", teamName, err)
				continue
			}
			teamAnnotatedInvites, err := AnnotateInvites(ctx, g, t.chain().inner.ActiveInvites, teamName)
			if err != nil {
				return nil, err
			}
			for teamInviteID, annotatedTeamInvite := range teamAnnotatedInvites {
				annotatedInvites[teamInviteID] = annotatedTeamInvite
			}
		}
	}

	tl := keybase1.AnnotatedTeamList{
		Teams: annotatedTeams,
		AnnotatedActiveInvites: annotatedInvites,
	}

	return &tl, nil
}

func AnnotateInvites(ctx context.Context, g *libkb.GlobalContext, invites map[keybase1.TeamInviteID]keybase1.TeamInvite, teamName string) (map[keybase1.TeamInviteID]keybase1.AnnotatedTeamInvite, error) {

	annotatedInvites := make(map[keybase1.TeamInviteID]keybase1.AnnotatedTeamInvite, len(invites))
	upakLoader := g.GetUPAKLoader()
	for id, invite := range invites {
		username, err := upakLoader.LookupUsername(ctx, invite.Inviter.Uid)
		if err != nil {
			return annotatedInvites, err
		}

		name := invite.Name
		category, err := invite.Type.C()
		if err != nil {
			return nil, err
		}
		if category == keybase1.TeamInviteCategory_KEYBASE {
			// "keybase" invites (i.e. pukless users) have user version for name
			uv, err := invite.KeybaseUserVersion()
			if err != nil {
				return nil, err
			}
			normalized, err := upakLoader.LookupUsername(context.Background(), uv.Uid)
			if err != nil {
				return nil, err
			}
			name = keybase1.TeamInviteName(normalized.String())
		}
		annotatedInvites[id] = keybase1.AnnotatedTeamInvite{
			Role:            invite.Role,
			Id:              invite.Id,
			Type:            invite.Type,
			Name:            name,
			Inviter:         invite.Inviter,
			InviterUsername: username.String(),
			TeamName:        teamName,
		}
	}
	return annotatedInvites, nil
}

func TeamTree(ctx context.Context, g *libkb.GlobalContext, arg keybase1.TeamTreeArg) (res keybase1.TeamTreeResult, err error) {
	if !arg.Name.IsRootTeam() {
		return res, fmt.Errorf("cannot get tree of non-root team")
	}

	serverList, err := getTeamsListFromServer(ctx, g, "", false)
	if err != nil {
		return res, err
	}

	// Map from team name (string) to entry
	entryMap := make(map[string]keybase1.TeamTreeEntry)

	// The server might have omitted some teams, oh well.
	// Trusts the server for role.
	// Load the teams by ID to make sure they are valid and get the validated names.
	for _, info := range serverList {
		serverName, err := info.TeamName()
		if err != nil {
			return res, err
		}
		if !serverName.RootAncestorName().Eq(arg.Name) {
			// Skip those not in this tree.
			continue
		}
		team, err := Load(ctx, g, keybase1.LoadTeamArg{
			ID:          info.TeamID,
			ForceRepoll: true,
		})
		if err != nil {
			return res, err
		}
		var admin bool // true if an admin or implicit admin
		if info.Role.IsAdminOrAbove() {
			admin = true
		}
		if info.Implicit != nil && info.Implicit.Role.IsAdminOrAbove() {
			admin = true
		}
		entryMap[team.Name().String()] = keybase1.TeamTreeEntry{
			Name:  team.Name(),
			Admin: admin,
		}
	}

	// Add all parent names (recursively)
	// So that if only A.B.C is in the list, we add A.B and A as well.
	// Adding map entries while iterating is safe.
	// "If map entries are created during iteration, that entry may be produced during the iteration or may be skipped."
	for _, entry := range entryMap {
		name := entry.Name.DeepCopy()
		for name.Depth() > 0 {
			_, ok := entryMap[name.String()]
			if !ok {
				entryMap[name.String()] = keybase1.TeamTreeEntry{
					Name:  name,
					Admin: false,
				}
			}
			name, err = name.Parent()
			if err != nil {
				break
			}
		}
	}

	for _, entry := range entryMap {
		res.Entries = append(res.Entries, entry)
	}

	if len(res.Entries) == 0 {
		return res, fmt.Errorf("team not found: %v", arg.Name)
	}

	// Order into a tree order. Which happens to be alphabetical ordering.
	// Example: [a, a.b, a.b.c, a.b.d, a.e.f, a.e.g]
	sort.Slice(res.Entries, func(i, j int) bool {
		return res.Entries[i].Name.String() < res.Entries[j].Name.String()
	})

	return res, nil
}
<|endoftext|>


<commit_before>package main

import (
	//"github.com/shirou/gopsutil"
	"github.com/shirou/gopsutil/mem"
	"github.com/shirou/gopsutil/cpu"
	"github.com/shirou/gopsutil/net"
	"github.com/shirou/gopsutil/load"
	"fmt"
)

func main() {
	v, _ := mem.VirtualMemory()

	// almost every return value is a struct
	//fmt.Printf("Total: %v, Free:%v, UsedPercent:%f%%\n", v.Total, v.Free, v.UsedPercent)

	// convert to JSON. String() is also implemented
	fmt.Println(v)
	fmt.Println("###########################################################################################")
	//c,_ := cpu.Info()   // cpu信息

	c,_:=cpu.Times(true)  // 目前参数没有意义，获取全部cpu的使用信息
	fmt.Println(c)
	fmt.Println("###########################################################################################")

	n,_ := net.IOCounters(true)	// 每个网卡的情况(不是网速，获取 /proc/net/dev文件数据)
	fmt.Println(n)
	fmt.Println("###########################################################################################")

	n1,_ := net.IOCounters(false)	// 全部网卡的情况
	fmt.Println(n1)
	fmt.Println("###########################################################################################")

	avg,_ := load.Avg()
	fmt.Println(avg)
	fmt.Println("###########################################################################################")

	misc,_ := load.Misc()
	fmt.Println(misc)
	fmt.Println("###########################################################################################")
}
<commit_msg>update<commit_after>package main

import (
	"time"
	//"github.com/shirou/gopsutil"
	"fmt"

	"github.com/shirou/gopsutil/cpu"
	"github.com/shirou/gopsutil/load"
	"github.com/shirou/gopsutil/mem"
	"github.com/shirou/gopsutil/net"
)

func main() {
	v, _ := mem.VirtualMemory()

	// almost every return value is a struct
	//fmt.Printf("Total: %v, Free:%v, UsedPercent:%f%%\n", v.Total, v.Free, v.UsedPercent)

	// convert to JSON. String() is also implemented
	fmt.Println(v)
	fmt.Println("###########################################################################################")
	//c,_ := cpu.Info()   // cpu信息

	c, _ := cpu.Times(true) // 目前参数没有意义，获取全部cpu的使用信息
	fmt.Println(c)
	fmt.Println("###########################################################################################")

	n, _ := net.IOCounters(true) // 每个网卡的情况(不是网速，获取 /proc/net/dev文件数据)
	fmt.Println(n)
	fmt.Println("###########################################################################################")

	n1, _ := net.IOCounters(false) // 全部网卡的情况
	fmt.Println(n1)
	fmt.Println("###########################################################################################")

	avg, _ := load.Avg()
	fmt.Println(avg)
	fmt.Println("###########################################################################################")

	misc, _ := load.Misc()
	fmt.Println(misc)
	fmt.Println("###########################################################################################")

	// http://www.blogjava.net/fjzag/articles/317773.html
	cpus, _ := cpu.Times(true) // 目前参数没有意义，获取全部cpu的使用信息

	var stat cpu.TimesStat
	for i := range cpus {
		fmt.Printf("%+v", cpus[i])
		stat.User += cpus[i].User // 从系统启动开始累计到当前时刻，处于用户态的运行时
		stat.System += cpus[i].System
		stat.Idle += cpus[i].Idle
		stat.Nice += cpus[i].Nice
		stat.Iowait += cpus[i].Iowait
		stat.Irq += cpus[i].Irq
		stat.Softirq += cpus[i].Softirq
		stat.Steal += cpus[i].Steal
		stat.Guest += cpus[i].Guest
		stat.GuestNice += cpus[i].GuestNice
		stat.Stolen += cpus[i].Stolen
	}
	fmt.Println("")
	fmt.Printf("%+v", stat)
	idle := stat.Idle / (stat.Idle + stat.System + stat.User)
	fmt.Println("")
	fmt.Printf("%+v", idle)

	fmt.Println("每个cpu的使用百分比")
	pers, _ := cpu.Percent(1*time.Second, true)
	fmt.Println(pers)

}
<|endoftext|>


<commit_before>package gorm

import (
	"database/sql"
	"database/sql/driver"
	"encoding/json"
	"fmt"
	"reflect"

	"github.com/lib/pq/hstore"
)

var hstoreType = reflect.TypeOf(Hstore{})

type Hstore map[string]*string

func (h Hstore) Value() (driver.Value, error) {
	hstore := hstore.Hstore{Map: map[string]sql.NullString{}}
	if len(h) == 0 {
		return nil, nil
	}

	for key, value := range h {
		hstore.Map[key] = sql.NullString{*value, true}
	}
	return hstore.Value()
}

func (h *Hstore) Scan(value interface{}) error {
	hstore := hstore.Hstore{}

	if err := hstore.Scan(value); err != nil {
		return err
	}

	if len(hstore.Map) == 0 {
		return nil
	}

	*h = Hstore{}
	for k := range hstore.Map {
		if hstore.Map[k].Valid {
			s := hstore.Map[k].String
			(*h)[k] = &s
		} else {
			(*h)[k] = nil
		}
	}

	return nil
}

func (h *Hstore) UnmarshalJSON(data []byte) error {
	hMap := map[string]interface{}{}
	if err := json.Unmarshal(data, &hMap); err != nil {
		return err
	}

	if len(hMap) == 0 {
		return nil
	}

	resultMap := make(map[string]*string)
	for key, value := range hMap {
		// when value is a map, then marshal it
		if reflect.ValueOf(value).Kind() == reflect.Map {
			res, err := json.Marshal(value)
			if err != nil {
				return err
			}

			s := string(res)
			resultMap[key] = &s
			continue
		}

		// for the other types convert value to string
		str := fmt.Sprintf("%v", value)
		resultMap[key] = &str
	}
	*h = resultMap

	return nil
}

func (h Hstore) MarshalJSON() ([]byte, error) {
	if len(h) == 0 {
		return json.Marshal(nil)
	}

	sqlMap := map[string]interface{}{}
	for key := range h {
		var res map[string]interface{}
		if err := json.Unmarshal([]byte(fmt.Sprintf("%v", *(h[key]))), &res); err != nil {
			sqlMap[key] = *(h[key])
		}
	}

	return json.Marshal(sqlMap)
}

type postgres struct {
}

func (s *postgres) BinVar(i int) string {
	return fmt.Sprintf("$%v", i)
}

func (s *postgres) SupportLastInsertId() bool {
	return false
}

func (d *postgres) SqlTag(value reflect.Value, size int) string {
	switch value.Kind() {
	case reflect.Bool:
		return "boolean"
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uintptr:
		return "integer"
	case reflect.Int64, reflect.Uint64:
		return "bigint"
	case reflect.Float32, reflect.Float64:
		return "numeric"
	case reflect.String:
		if size > 0 && size < 65532 {
			return fmt.Sprintf("varchar(%d)", size)
		}
		return "text"
	case reflect.Struct:
		if value.Type() == timeType {
			return "timestamp with time zone"
		}
	case reflect.Map:
		if value.Type() == hstoreType {
			return "hstore"
		}
	default:
		if _, ok := value.Interface().([]byte); ok {
			return "bytea"
		}
	}
	panic(fmt.Sprintf("invalid sql type %s (%s) for postgres", value.Type().Name(), value.Kind().String()))
}

func (s *postgres) PrimaryKeyTag(value reflect.Value, size int) string {
	switch value.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uintptr:
		return "serial PRIMARY KEY"
	case reflect.Int64, reflect.Uint64:
		return "bigserial PRIMARY KEY"
	default:
		panic("Invalid primary key type")
	}
}

func (s *postgres) ReturningStr(key string) string {
	return fmt.Sprintf("RETURNING \"%v\"", key)
}

func (s *postgres) Quote(key string) string {
	return fmt.Sprintf("\"%s\"", key)
}

func (s *postgres) HasTable(scope *Scope, tableName string) bool {
	var count int
	newScope := scope.New(nil)
	newScope.Raw(fmt.Sprintf("SELECT count(*) FROM INFORMATION_SCHEMA.tables where table_name = %v", newScope.AddToVars(tableName)))
	newScope.DB().QueryRow(newScope.Sql, newScope.SqlVars...).Scan(&count)
	return count > 0
}

func (s *postgres) HasColumn(scope *Scope, tableName string, columnName string) bool {
	var count int
	newScope := scope.New(nil)
	newScope.Raw(fmt.Sprintf("SELECT count(*) FROM information_schema.columns WHERE table_name = %v AND column_name = %v",
		newScope.AddToVars(tableName),
		newScope.AddToVars(columnName),
	))
	newScope.DB().QueryRow(newScope.Sql, newScope.SqlVars...).Scan(&count)
	return count > 0
}
<commit_msg>Social: remove Hstore custom marshaler<commit_after>package gorm

import (
	"database/sql"
	"database/sql/driver"
	"encoding/json"
	"fmt"
	"reflect"

	"github.com/lib/pq/hstore"
)

var hstoreType = reflect.TypeOf(Hstore{})

type Hstore map[string]*string

func (h Hstore) Value() (driver.Value, error) {
	hstore := hstore.Hstore{Map: map[string]sql.NullString{}}
	if len(h) == 0 {
		return nil, nil
	}

	for key, value := range h {
		hstore.Map[key] = sql.NullString{*value, true}
	}
	return hstore.Value()
}

func (h *Hstore) Scan(value interface{}) error {
	hstore := hstore.Hstore{}

	if err := hstore.Scan(value); err != nil {
		return err
	}

	if len(hstore.Map) == 0 {
		return nil
	}

	*h = Hstore{}
	for k := range hstore.Map {
		if hstore.Map[k].Valid {
			s := hstore.Map[k].String
			(*h)[k] = &s
		} else {
			(*h)[k] = nil
		}
	}

	return nil
}

func (h *Hstore) UnmarshalJSON(data []byte) error {
	hMap := map[string]interface{}{}
	if err := json.Unmarshal(data, &hMap); err != nil {
		return err
	}

	if len(hMap) == 0 {
		return nil
	}

	resultMap := make(map[string]*string)
	for key, value := range hMap {
		// when value is a map, then marshal it
		if reflect.ValueOf(value).Kind() == reflect.Map {
			res, err := json.Marshal(value)
			if err != nil {
				return err
			}

			s := string(res)
			resultMap[key] = &s
			continue
		}

		// for the other types convert value to string
		str := fmt.Sprintf("%v", value)
		resultMap[key] = &str
	}
	*h = resultMap

	return nil
}

type postgres struct {
}

func (s *postgres) BinVar(i int) string {
	return fmt.Sprintf("$%v", i)
}

func (s *postgres) SupportLastInsertId() bool {
	return false
}

func (d *postgres) SqlTag(value reflect.Value, size int) string {
	switch value.Kind() {
	case reflect.Bool:
		return "boolean"
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uintptr:
		return "integer"
	case reflect.Int64, reflect.Uint64:
		return "bigint"
	case reflect.Float32, reflect.Float64:
		return "numeric"
	case reflect.String:
		if size > 0 && size < 65532 {
			return fmt.Sprintf("varchar(%d)", size)
		}
		return "text"
	case reflect.Struct:
		if value.Type() == timeType {
			return "timestamp with time zone"
		}
	case reflect.Map:
		if value.Type() == hstoreType {
			return "hstore"
		}
	default:
		if _, ok := value.Interface().([]byte); ok {
			return "bytea"
		}
	}
	panic(fmt.Sprintf("invalid sql type %s (%s) for postgres", value.Type().Name(), value.Kind().String()))
}

func (s *postgres) PrimaryKeyTag(value reflect.Value, size int) string {
	switch value.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uintptr:
		return "serial PRIMARY KEY"
	case reflect.Int64, reflect.Uint64:
		return "bigserial PRIMARY KEY"
	default:
		panic("Invalid primary key type")
	}
}

func (s *postgres) ReturningStr(key string) string {
	return fmt.Sprintf("RETURNING \"%v\"", key)
}

func (s *postgres) Quote(key string) string {
	return fmt.Sprintf("\"%s\"", key)
}

func (s *postgres) HasTable(scope *Scope, tableName string) bool {
	var count int
	newScope := scope.New(nil)
	newScope.Raw(fmt.Sprintf("SELECT count(*) FROM INFORMATION_SCHEMA.tables where table_name = %v", newScope.AddToVars(tableName)))
	newScope.DB().QueryRow(newScope.Sql, newScope.SqlVars...).Scan(&count)
	return count > 0
}

func (s *postgres) HasColumn(scope *Scope, tableName string, columnName string) bool {
	var count int
	newScope := scope.New(nil)
	newScope.Raw(fmt.Sprintf("SELECT count(*) FROM information_schema.columns WHERE table_name = %v AND column_name = %v",
		newScope.AddToVars(tableName),
		newScope.AddToVars(columnName),
	))
	newScope.DB().QueryRow(newScope.Sql, newScope.SqlVars...).Scan(&count)
	return count > 0
}
<|endoftext|>


<commit_before>package koding

import (
	"errors"
	"time"

	"github.com/koding/kloud/eventer"
	"github.com/koding/kloud/machinestate"
	"github.com/koding/kloud/protocol"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

var (
	FreeUserTimeout = time.Minute * 30
)

// RunChecker runs the checker everny given interval time. It fetches a single
// document.
func (p *Provider) RunChecker(interval time.Duration) {
	for _ = range time.Tick(interval) {
		machine, err := p.FetchOne()
		if err != nil {
			// do not show an error if the query didn't find anything, that
			// means there is no such a document, which we don't care
			if err != mgo.ErrNotFound {
				p.Log.Error("FetchOne err: %v", err)
			}

			p.Log.Info("checker no machines available to check: %s", err.Error())

			// move one with the next one
			continue
		}

		if err := p.CheckUsage(machine); err != nil {
			p.Log.Warning("check usage of kite err: %v", err)
		}
	}
}

func (p *Provider) CheckUsage(machine *Machine) error {
	if machine == nil {
		return errors.New("machine is nil")
	}

	// release the lock from mongodb after we are done
	defer p.ResetAssignee(machine.Id.Hex())

	klient, err := p.Connect(machine.QueryString)
	if err != nil {
		return err
	}
	defer klient.Close()

	usg, err := klient.Usage()
	if err != nil {
		p.Log.Error("[%s] couldn't get usage to klient: %s", machine.Id.Hex(), err)
		return err
	}

	p.Log.Info("[%s] machine with ip %s is inactive for %s",
		machine.Id.Hex(), machine.IpAddress, usg.InactiveDuration)

	// It still have plenty of time to work, do not stop it
	if usg.InactiveDuration <= FreeUserTimeout {
		return nil
	}

	// populare a protocol.Machine instance that is needed for the Stop()
	// method
	credential := p.GetCredential(machine.Credential)

	m := &protocol.Machine{
		MachineId:   machine.Id.Hex(),
		Provider:    machine.Provider,
		Builder:     machine.Meta,
		Credential:  credential.Meta,
		State:       machine.State(),
		CurrentData: machine,
	}

	m.Builder["username"] = klient.username

	// add a fake eventer, meanse we are not reporting anyone and prevent also
	// panicing the code when someone try to call the eventer
	m.Eventer = &eventer.Events{}

	// mark it as stopping
	p.UpdateState(machine.Id.Hex(), machinestate.Stopping)

	// stop the machine
	err = p.Stop(m)
	if err != nil {
		return err
	}

	// update the state too
	return p.UpdateState(machine.Id.Hex(), machinestate.Stopped)
}

// FetchOne() fetches a single machine document from mongodb. This document is
// locked and cannot be retrieved from others anymore. After finishin work with
// this document ResetAssignee needs to be called that it's unlocked again and
// can be fetcy by others.
func (p *Provider) FetchOne() (*Machine, error) {
	machine := &Machine{}
	query := func(c *mgo.Collection) error {
		// check only machines that are running and belongs to koding provider
		// which are not assigned to anyone yet. We also check the date to not
		// pick up fresh documents. That means documents that are proccessed
		// and put into the DB will not selected until 10 seconds pass.
		egligibleMachines := bson.M{
			"provider":            "koding",
			"status.state":        "Running",
			"assignee.name":       nil,
			"assignee.assignedAt": bson.M{"$lt": time.Now().UTC().Add(-time.Second * 10)},
		}

		// once we found something, lock it by modifing the assignee.name. Also
		// create a new timestamp (assignee.assignedAt) which is needed for
		// several cases like (explained above and below)
		change := mgo.Change{
			Update: bson.M{
				"$set": bson.M{
					"assignee.name":       p.Assignee(),
					"assignee.assignedAt": time.Now().UTC(),
				},
			},
			ReturnNew: true,
		}

		// We sort according to the latest assignment date, which let's us pick
		// always the oldest one instead of random/first. Returning an error
		// means there is no document that matches our criterias.
		_, err := c.Find(egligibleMachines).Sort("assignee.assignedAt").Apply(change, &machine)
		if err != nil {
			return err
		}

		return nil
	}

	if err := p.Session.Run("jMachines", query); err != nil {
		return nil, err
	}

	return machine, nil
}
<commit_msg>kloud/queue: decrease default for local development<commit_after>package koding

import (
	"errors"
	"time"

	"github.com/koding/kloud/eventer"
	"github.com/koding/kloud/machinestate"
	"github.com/koding/kloud/protocol"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

var (
	FreeUserTimeout = time.Minute * 15
)

// RunChecker runs the checker everny given interval time. It fetches a single
// document.
func (p *Provider) RunChecker(interval time.Duration) {
	for _ = range time.Tick(interval) {
		machine, err := p.FetchOne()
		if err != nil {
			// do not show an error if the query didn't find anything, that
			// means there is no such a document, which we don't care
			if err != mgo.ErrNotFound {
				p.Log.Warning("FetchOne err: %v", err)
			}

			p.Log.Debug("checker no machines available to check: %s", err.Error())

			// move one with the next one
			continue
		}

		if err := p.CheckUsage(machine); err != nil {
			p.Log.Warning("check usage of kite err: %v", err)
		}
	}
}

func (p *Provider) CheckUsage(machine *Machine) error {
	if machine == nil {
		return errors.New("machine is nil")
	}

	// release the lock from mongodb after we are done
	defer p.ResetAssignee(machine.Id.Hex())

	klient, err := p.Connect(machine.QueryString)
	if err != nil {
		return err
	}
	defer klient.Close()

	usg, err := klient.Usage()
	if err != nil {
		p.Log.Error("[%s] couldn't get usage to klient: %s", machine.Id.Hex(), err)
		return err
	}

	p.Log.Info("[%s] machine with ip %s is inactive for %s",
		machine.Id.Hex(), machine.IpAddress, usg.InactiveDuration)

	// It still have plenty of time to work, do not stop it
	if usg.InactiveDuration <= FreeUserTimeout {
		return nil
	}

	// populare a protocol.Machine instance that is needed for the Stop()
	// method
	credential := p.GetCredential(machine.Credential)

	m := &protocol.Machine{
		MachineId:   machine.Id.Hex(),
		Provider:    machine.Provider,
		Builder:     machine.Meta,
		Credential:  credential.Meta,
		State:       machine.State(),
		CurrentData: machine,
	}

	m.Builder["username"] = klient.username

	// add a fake eventer, meanse we are not reporting anyone and prevent also
	// panicing the code when someone try to call the eventer
	m.Eventer = &eventer.Events{}

	// mark it as stopping
	p.UpdateState(machine.Id.Hex(), machinestate.Stopping)

	// stop the machine
	err = p.Stop(m)
	if err != nil {
		return err
	}

	// update the state too
	return p.UpdateState(machine.Id.Hex(), machinestate.Stopped)
}

// FetchOne() fetches a single machine document from mongodb. This document is
// locked and cannot be retrieved from others anymore. After finishin work with
// this document ResetAssignee needs to be called that it's unlocked again and
// can be fetcy by others.
func (p *Provider) FetchOne() (*Machine, error) {
	machine := &Machine{}
	query := func(c *mgo.Collection) error {
		// check only machines that are running and belongs to koding provider
		// which are not assigned to anyone yet. We also check the date to not
		// pick up fresh documents. That means documents that are proccessed
		// and put into the DB will not selected until 10 seconds pass.
		egligibleMachines := bson.M{
			"provider":            "koding",
			"status.state":        "Running",
			"assignee.name":       nil,
			"assignee.assignedAt": bson.M{"$lt": time.Now().UTC().Add(-time.Second * 10)},
		}

		// once we found something, lock it by modifing the assignee.name. Also
		// create a new timestamp (assignee.assignedAt) which is needed for
		// several cases like (explained above and below)
		change := mgo.Change{
			Update: bson.M{
				"$set": bson.M{
					"assignee.name":       p.Assignee(),
					"assignee.assignedAt": time.Now().UTC(),
				},
			},
			ReturnNew: true,
		}

		// We sort according to the latest assignment date, which let's us pick
		// always the oldest one instead of random/first. Returning an error
		// means there is no document that matches our criterias.
		_, err := c.Find(egligibleMachines).Sort("assignee.assignedAt").Apply(change, &machine)
		if err != nil {
			return err
		}

		return nil
	}

	if err := p.Session.Run("jMachines", query); err != nil {
		return nil, err
	}

	return machine, nil
}
<|endoftext|>