<commit_before>package gohistogram

import (
	"fmt"
)

type NumericHistogram struct {
	bins    []bin
	maxbins int
	total   uint64
}

// NewHistogram returns a new NumericHistogram with a maximum of n bins.
//
// There is no "optimal" bin count, but somewhere between 20 and 80 bins
// should be sufficient.
func NewHistogram(n int) *NumericHistogram {
	return &NumericHistogram{
		bins:    make([]bin, 0, n+1),
		maxbins: n,
		total:   0,
	}
}

func (h *NumericHistogram) Add(n float64) {
	defer h.trim()
	h.total++
	for i := range h.bins {
		if h.bins[i].value == n {
			h.bins[i].count++
			return
		}

		if h.bins[i].value > n {

			newbin := bin{value: n, count: 1}
			tail := h.bins[i:]
			h.bins = h.bins[0 : len(h.bins)+1]
			copy(h.bins[i+1:], tail)
			h.bins[i] = newbin

			return
		}
	}

	h.bins = append(h.bins, bin{count: 1, value: n})
}

func (h *NumericHistogram) Quantile(q float64) float64 {
	count := q * float64(h.total)
	for i := range h.bins {
		count -= float64(h.bins[i].count)

		if count <= 0 {
			return h.bins[i].value
		}
	}

	return -1
}

// CDF returns the value of the cumulative distribution function
// at x
func (h *NumericHistogram) CDF(x float64) float64 {
	count := 0.0
	for i := range h.bins {
		if h.bins[i].value <= x {
			count += float64(h.bins[i].count)
		}
	}

	return count / float64(h.total)
}

// Mean returns the sample mean of the distribution
func (h *NumericHistogram) Mean() float64 {
	if h.total == 0 {
		return 0
	}

	sum := 0.0

	for i := range h.bins {
		sum += h.bins[i].value * h.bins[i].count
	}

	return sum / float64(h.total)
}

// Variance returns the variance of the distribution
func (h *NumericHistogram) Variance() float64 {
	if h.total == 0 {
		return 0
	}

	sum := 0.0
	mean := h.Mean()

	for i := range h.bins {
		sum += (h.bins[i].count * (h.bins[i].value - mean) * (h.bins[i].value - mean))
	}

	return sum / float64(h.total)
}

func (h *NumericHistogram) Count() float64 {
	return float64(h.total)
}

// trim merges adjacent bins to decrease the bin count to the maximum value
func (h *NumericHistogram) trim() {
	for len(h.bins) > h.maxbins {
		// Find closest bins in terms of value
		minDelta := 1e99
		minDeltaIndex := 0
		for i := range h.bins {
			if i == 0 {
				continue
			}

			if delta := h.bins[i].value - h.bins[i-1].value; delta < minDelta {
				minDelta = delta
				minDeltaIndex = i
			}
		}

		// We need to merge bins minDeltaIndex-1 and minDeltaIndex
		totalCount := h.bins[minDeltaIndex-1].count + h.bins[minDeltaIndex].count
		mergedbin := bin{
			value: (h.bins[minDeltaIndex-1].value*
				h.bins[minDeltaIndex-1].count +
				h.bins[minDeltaIndex].value*
					h.bins[minDeltaIndex].count) /
				totalCount, // weighted average
			count: totalCount, // summed heights
		}
		head := h.bins[0:minDeltaIndex]
		tail := h.bins[minDeltaIndex+1:]
		head[minDeltaIndex] = mergedbin
		copy(h.bins[minDeltaIndex+1:], tail)
		h.bins = h.bins[:h.maxbins]
	}
}

// String returns a string reprentation of the histogram,
// which is useful for printing to a terminal.
func (h *NumericHistogram) String() (str string) {
	str += fmt.Sprintln("Total:", h.total)

	for i := range h.bins {
		var bar string
		for j := 0; j < int(float64(h.bins[i].count)/float64(h.total)*200); j++ {
			bar += "."
		}
		str += fmt.Sprintln(h.bins[i].value, "\t", bar)
	}

	return
}
<commit_msg>Repair trim logic<commit_after>package gohistogram

import (
	"fmt"
)

type NumericHistogram struct {
	bins    []bin
	maxbins int
	total   uint64
}

// NewHistogram returns a new NumericHistogram with a maximum of n bins.
//
// There is no "optimal" bin count, but somewhere between 20 and 80 bins
// should be sufficient.
func NewHistogram(n int) *NumericHistogram {
	return &NumericHistogram{
		bins:    make([]bin, 0, n+1),
		maxbins: n,
		total:   0,
	}
}

func (h *NumericHistogram) Add(n float64) {
	defer h.trim()
	h.total++
	for i := range h.bins {
		if h.bins[i].value == n {
			h.bins[i].count++
			return
		}

		if h.bins[i].value > n {

			newbin := bin{value: n, count: 1}
			tail := h.bins[i:]
			h.bins = h.bins[0 : len(h.bins)+1]
			copy(h.bins[i+1:], tail)
			h.bins[i] = newbin

			return
		}
	}

	h.bins = append(h.bins, bin{count: 1, value: n})
}

func (h *NumericHistogram) Quantile(q float64) float64 {
	count := q * float64(h.total)
	for i := range h.bins {
		count -= float64(h.bins[i].count)

		if count <= 0 {
			return h.bins[i].value
		}
	}

	return -1
}

// CDF returns the value of the cumulative distribution function
// at x
func (h *NumericHistogram) CDF(x float64) float64 {
	count := 0.0
	for i := range h.bins {
		if h.bins[i].value <= x {
			count += float64(h.bins[i].count)
		}
	}

	return count / float64(h.total)
}

// Mean returns the sample mean of the distribution
func (h *NumericHistogram) Mean() float64 {
	if h.total == 0 {
		return 0
	}

	sum := 0.0

	for i := range h.bins {
		sum += h.bins[i].value * h.bins[i].count
	}

	return sum / float64(h.total)
}

// Variance returns the variance of the distribution
func (h *NumericHistogram) Variance() float64 {
	if h.total == 0 {
		return 0
	}

	sum := 0.0
	mean := h.Mean()

	for i := range h.bins {
		sum += (h.bins[i].count * (h.bins[i].value - mean) * (h.bins[i].value - mean))
	}

	return sum / float64(h.total)
}

func (h *NumericHistogram) Count() float64 {
	return float64(h.total)
}

// trim merges adjacent bins to decrease the bin count to the maximum value
func (h *NumericHistogram) trim() {
	for len(h.bins) > h.maxbins {
		// Find closest bins in terms of value
		minDelta := 1e99
		minDeltaIndex := 0
		for i := range h.bins {
			if i == 0 {
				continue
			}

			if delta := h.bins[i].value - h.bins[i-1].value; delta < minDelta {
				minDelta = delta
				minDeltaIndex = i
			}
		}

		// We need to merge bins minDeltaIndex-1 and minDeltaIndex
		totalCount := h.bins[minDeltaIndex-1].count + h.bins[minDeltaIndex].count
		mergedbin := bin{
			value: (h.bins[minDeltaIndex-1].value*
				h.bins[minDeltaIndex-1].count +
				h.bins[minDeltaIndex].value*
					h.bins[minDeltaIndex].count) /
				totalCount, // weighted average
			count: totalCount, // summed heights
		}
		head := h.bins[0:minDeltaIndex]
		tail := h.bins[minDeltaIndex+1:]
		head[minDeltaIndex-1] = mergedbin
		copy(h.bins[minDeltaIndex:], tail)
		h.bins = h.bins[:h.maxbins]
	}
}

// String returns a string reprentation of the histogram,
// which is useful for printing to a terminal.
func (h *NumericHistogram) String() (str string) {
	str += fmt.Sprintln("Total:", h.total)

	for i := range h.bins {
		var bar string
		for j := 0; j < int(float64(h.bins[i].count)/float64(h.total)*200); j++ {
			bar += "."
		}
		str += fmt.Sprintln(h.bins[i].value, "\t", bar)
	}

	return
}
<|endoftext|>


<commit_before>package ccserialize

import (
	"reflect"
	"strconv"
	"strings"
)

func Serialize(input interface{}) string {
	switch reflect.ValueOf(input).Kind() {
	case reflect.Int:
		return strconv.Itoa(input.(int))
	case reflect.Int64:
		return strconv.FormatInt(input.(int64), 10)
	case reflect.Float64:
		return strconv.FormatFloat(input.(float64), 'f', 6, 64)
	case reflect.String:
		return "\"" + strings.Replace(input.(string), "\"", "\\\"", -1) + "\""
	case reflect.Map:
		t := reflect.TypeOf(input)
		v := reflect.ValueOf(input)
		it := reflect.TypeOf((*interface{})(nil)).Elem()
		m := reflect.MakeMap(reflect.MapOf(t.Key(), it))

		for _, mk := range v.MapKeys() {
			m.SetMapIndex(mk, v.MapIndex(mk))
		}

		return mapSerialize(m.Interface().(map[string]interface{}))
	case reflect.Slice:
		s := reflect.ValueOf(input)
		ret := make([]interface{}, s.Len())

		for i := 0; i < s.Len(); i++ {
			ret[i] = s.Index(i).Interface()
		}

		return arraySerialize(ret)
	case reflect.Struct:
		return structSerialize(input)
	default:
		panic("Attempt to serialize unsupported type.")
	}
}

func structSerialize(input interface{}) string {
	serializedString := "{"

	intStr := reflect.ValueOf(input)
	str := reflect.TypeOf(input)
	for i := 0; i < str.NumField(); i++ {
		f := intStr.Field(i)
		serializedString = serializedString + "[\"" + strings.ToLower(str.Field(i).Name) + "\"] = "
		serializedString = serializedString + Serialize(f.Interface()) + ", "
	}

	return serializedString + "}"
}

func arraySerialize(input []interface{}) string {
	serializedString := "{"
	for k, v := range input {
		serializedString = serializedString + "[" + strconv.Itoa(k+1) + "] = " + Serialize(v) + ", "
	}
	return serializedString + "}"
}

func mapSerialize(input map[string]interface{}) string {
	serializedString := "{"
	for k, v := range input {
		serializedString = serializedString + "[\"" + k + "\"] = " + Serialize(v) + ", "
	}
	return serializedString + "}"
}
<commit_msg>Add struct tagging for table keys under "ccserialize"<commit_after>package ccserialize

import (
	"reflect"
	"strconv"
	"strings"
)

func Serialize(input interface{}) string {
	switch reflect.ValueOf(input).Kind() {
	case reflect.Int:
		return strconv.Itoa(input.(int))
	case reflect.Int64:
		return strconv.FormatInt(input.(int64), 10)
	case reflect.Float64:
		return strconv.FormatFloat(input.(float64), 'f', 6, 64)
	case reflect.String:
		return "\"" + strings.Replace(input.(string), "\"", "\\\"", -1) + "\""
	case reflect.Map:
		t := reflect.TypeOf(input)
		v := reflect.ValueOf(input)
		it := reflect.TypeOf((*interface{})(nil)).Elem()
		m := reflect.MakeMap(reflect.MapOf(t.Key(), it))

		for _, mk := range v.MapKeys() {
			m.SetMapIndex(mk, v.MapIndex(mk))
		}

		return mapSerialize(m.Interface().(map[string]interface{}))
	case reflect.Slice:
		s := reflect.ValueOf(input)
		ret := make([]interface{}, s.Len())

		for i := 0; i < s.Len(); i++ {
			ret[i] = s.Index(i).Interface()
		}

		return arraySerialize(ret)
	case reflect.Struct:
		return structSerialize(input)
	default:
		panic("Attempt to serialize unsupported type.")
	}
}

func structSerialize(input interface{}) string {
	serializedString := "{"

	intStr := reflect.ValueOf(input)
	str := reflect.TypeOf(input)
	for i := 0; i < str.NumField(); i++ {
		f := intStr.Field(i)
		keyName := str.Field(i).Name
		if str.Field(i).Tag.Get("ccserialize") != "" {
			keyName = str.Field(i).Tag.Get("ccserialize")
		}

		serializedString = serializedString + "[\"" + strings.ToLower() + "\"] = "
		serializedString = serializedString + Serialize(f.Interface()) + ", "
	}

	return serializedString + "}"
}

func arraySerialize(input []interface{}) string {
	serializedString := "{"
	for k, v := range input {
		serializedString = serializedString + "[" + strconv.Itoa(k+1) + "] = " + Serialize(v) + ", "
	}
	return serializedString + "}"
}

func mapSerialize(input map[string]interface{}) string {
	serializedString := "{"
	for k, v := range input {
		serializedString = serializedString + "[\"" + k + "\"] = " + Serialize(v) + ", "
	}
	return serializedString + "}"
}
<|endoftext|>


<commit_before>package clusterinfo

import (
	"github.com/blang/semver"
	"time"
)

// Response is the cluster info retrievable from the / endpoint
type Response struct {
	Name        string      `json:"name"`
	ClusterName string      `json:"cluster_name"`
	ClusterUUID string      `json:"cluster_uuid"`
	Version     VersionInfo `json:"version"`
	Tagline     string      `json:"tagline"`
}

// VersionInfo is the version info retrievable from the / endpoint, embedded in Response
type VersionInfo struct {
	Number        semver.Version `json:"number"`
	BuildHash     string         `json:"build_hash"`
	BuildDate     time.Time      `json:"build_date"`
	BuildSnapshot bool           `json:"build_snapshot"`
	LuceneVersion semver.Version `json:"lucene_version"`
}
<commit_msg>parse clusterinfo.build_date as string, not time.Time. It's not evaluated as time anyways<commit_after>package clusterinfo

import (
	"github.com/blang/semver"
)

// Response is the cluster info retrievable from the / endpoint
type Response struct {
	Name        string      `json:"name"`
	ClusterName string      `json:"cluster_name"`
	ClusterUUID string      `json:"cluster_uuid"`
	Version     VersionInfo `json:"version"`
	Tagline     string      `json:"tagline"`
}

// VersionInfo is the version info retrievable from the / endpoint, embedded in Response
type VersionInfo struct {
	Number        semver.Version `json:"number"`
	BuildHash     string         `json:"build_hash"`
	BuildDate     string         `json:"build_date"`
	BuildSnapshot bool           `json:"build_snapshot"`
	LuceneVersion semver.Version `json:"lucene_version"`
}
<|endoftext|>


<commit_before>package backend

import (
	"log"
)

// MemStore is a key value store in memory.
type MemStore struct {
	Data map[string][]string
}

// Init initializes the data store
func (m *MemStore) Init() {
	log.Println(m.Data)
	m.Data = make(map[string][]string)
}

// Add inserts a key into the backend
func (m *MemStore) Add(user string, key string) (bool, error) {
	log.Println(m.Data[user])
	m.Data[user] = append(m.Data[user], key)
	return true, nil
}

// RM removes a key from the backend
func (m *MemStore) RM(user string, key string) (bool, error) {
	var idx int
	for i := 0; i < len(m.Data[user]); i++ {
		if m.Data[user][i] == key {
			idx = i
		}
	}

	m.Data[user] = append(m.Data[user][:idx], m.Data[user][idx+1:]...)

	return true, nil
}

// RMAll removes all of the keys for a given user
func (m *MemStore) RMAll(user string) (bool, error) {
	delete(m.Data, user)
	return true, nil
}

// Get queries the backend for 'user' and returns all the keys
func (m *MemStore) Get(user string) ([]string, error) {
	return m.Data[user], nil
}

// GetAll returns the entire datastructure
func (m *MemStore) GetAll() (map[string][]string, error) {
	return m.Data, nil
}

// GetKeyCount queries the number of keys a given user has
func (m *MemStore) GetKeyCount(user string) (int, error) {
	return len(m.Data[user]), nil
}

// GetCount queries the number of keys stored in a backend
func (m *MemStore) GetCount() (int, error) {
	var i int
	for j := range m.Data {
		for _ = range m.Data[j] {
			i++
		}
	}
	return i, nil
}
<commit_msg>remove debug from backend<commit_after>package backend

// MemStore is a key value store in memory.
type MemStore struct {
	Data map[string][]string
}

// Init initializes the data store
func (m *MemStore) Init() {
	m.Data = make(map[string][]string)
}

// Add inserts a key into the backend
func (m *MemStore) Add(user string, key string) (bool, error) {
	m.Data[user] = append(m.Data[user], key)
	return true, nil
}

// RM removes a key from the backend
func (m *MemStore) RM(user string, key string) (bool, error) {
	var idx int
	for i := 0; i < len(m.Data[user]); i++ {
		if m.Data[user][i] == key {
			idx = i
		}
	}

	m.Data[user] = append(m.Data[user][:idx], m.Data[user][idx+1:]...)

	return true, nil
}

// RMAll removes all of the keys for a given user
func (m *MemStore) RMAll(user string) (bool, error) {
	delete(m.Data, user)
	return true, nil
}

// Get queries the backend for 'user' and returns all the keys
func (m *MemStore) Get(user string) ([]string, error) {
	return m.Data[user], nil
}

// GetAll returns the entire datastructure
func (m *MemStore) GetAll() (map[string][]string, error) {
	return m.Data, nil
}

// GetKeyCount queries the number of keys a given user has
func (m *MemStore) GetKeyCount(user string) (int, error) {
	return len(m.Data[user]), nil
}

// GetCount queries the number of keys stored in a backend
func (m *MemStore) GetCount() (int, error) {
	var i int
	for j := range m.Data {
		for _ = range m.Data[j] {
			i++
		}
	}
	return i, nil
}
<|endoftext|>


<commit_before>package simplejson

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"log"
)

// returns the current implementation version
func Version() string {
	return "0.5.0"
}

type Json struct {
	data interface{}
}

func (j *Json) FromDB(data []byte) error {
	j.data = make(map[string]interface{})

	dec := json.NewDecoder(bytes.NewBuffer(data))
	dec.UseNumber()
	return dec.Decode(&j.data)
}

func (j *Json) ToDB() ([]byte, error) {
	return j.Encode()
}

// NewJson returns a pointer to a new `Json` object
// after unmarshaling `body` bytes
func NewJson(body []byte) (*Json, error) {
	j := new(Json)
	err := j.UnmarshalJSON(body)
	if err != nil {
		return nil, err
	}
	return j, nil
}

// New returns a pointer to a new, empty `Json` object
func New() *Json {
	return &Json{
		data: make(map[string]interface{}),
	}
}

// Interface returns the underlying data
func (j *Json) Interface() interface{} {
	return j.data
}

// Encode returns its marshaled data as `[]byte`
func (j *Json) Encode() ([]byte, error) {
	return j.MarshalJSON()
}

// EncodePretty returns its marshaled data as `[]byte` with indentation
func (j *Json) EncodePretty() ([]byte, error) {
	return json.MarshalIndent(&j.data, "", "  ")
}

// Implements the json.Marshaler interface.
func (j *Json) MarshalJSON() ([]byte, error) {
	fmt.Printf("MarshalJSON")
	return json.Marshal(&j.data)
}

// Set modifies `Json` map by `key` and `value`
// Useful for changing single key/value in a `Json` object easily.
func (j *Json) Set(key string, val interface{}) {
	m, err := j.Map()
	if err != nil {
		return
	}
	m[key] = val
}

// SetPath modifies `Json`, recursively checking/creating map keys for the supplied path,
// and then finally writing in the value
func (j *Json) SetPath(branch []string, val interface{}) {
	if len(branch) == 0 {
		j.data = val
		return
	}

	// in order to insert our branch, we need map[string]interface{}
	if _, ok := (j.data).(map[string]interface{}); !ok {
		// have to replace with something suitable
		j.data = make(map[string]interface{})
	}
	curr := j.data.(map[string]interface{})

	for i := 0; i < len(branch)-1; i++ {
		b := branch[i]
		// key exists?
		if _, ok := curr[b]; !ok {
			n := make(map[string]interface{})
			curr[b] = n
			curr = n
			continue
		}

		// make sure the value is the right sort of thing
		if _, ok := curr[b].(map[string]interface{}); !ok {
			// have to replace with something suitable
			n := make(map[string]interface{})
			curr[b] = n
		}

		curr = curr[b].(map[string]interface{})
	}

	// add remaining k/v
	curr[branch[len(branch)-1]] = val
}

// Del modifies `Json` map by deleting `key` if it is present.
func (j *Json) Del(key string) {
	m, err := j.Map()
	if err != nil {
		return
	}
	delete(m, key)
}

// Get returns a pointer to a new `Json` object
// for `key` in its `map` representation
//
// useful for chaining operations (to traverse a nested JSON):
//    js.Get("top_level").Get("dict").Get("value").Int()
func (j *Json) Get(key string) *Json {
	m, err := j.Map()
	if err == nil {
		if val, ok := m[key]; ok {
			return &Json{val}
		}
	}
	return &Json{nil}
}

// GetPath searches for the item as specified by the branch
// without the need to deep dive using Get()'s.
//
//   js.GetPath("top_level", "dict")
func (j *Json) GetPath(branch ...string) *Json {
	jin := j
	for _, p := range branch {
		jin = jin.Get(p)
	}
	return jin
}

// GetIndex returns a pointer to a new `Json` object
// for `index` in its `array` representation
//
// this is the analog to Get when accessing elements of
// a json array instead of a json object:
//    js.Get("top_level").Get("array").GetIndex(1).Get("key").Int()
func (j *Json) GetIndex(index int) *Json {
	a, err := j.Array()
	if err == nil {
		if len(a) > index {
			return &Json{a[index]}
		}
	}
	return &Json{nil}
}

// CheckGet returns a pointer to a new `Json` object and
// a `bool` identifying success or failure
//
// useful for chained operations when success is important:
//    if data, ok := js.Get("top_level").CheckGet("inner"); ok {
//        log.Println(data)
//    }
func (j *Json) CheckGet(key string) (*Json, bool) {
	m, err := j.Map()
	if err == nil {
		if val, ok := m[key]; ok {
			return &Json{val}, true
		}
	}
	return nil, false
}

// Map type asserts to `map`
func (j *Json) Map() (map[string]interface{}, error) {
	if m, ok := (j.data).(map[string]interface{}); ok {
		return m, nil
	}
	return nil, errors.New("type assertion to map[string]interface{} failed")
}

// Array type asserts to an `array`
func (j *Json) Array() ([]interface{}, error) {
	if a, ok := (j.data).([]interface{}); ok {
		return a, nil
	}
	return nil, errors.New("type assertion to []interface{} failed")
}

// Bool type asserts to `bool`
func (j *Json) Bool() (bool, error) {
	if s, ok := (j.data).(bool); ok {
		return s, nil
	}
	return false, errors.New("type assertion to bool failed")
}

// String type asserts to `string`
func (j *Json) String() (string, error) {
	if s, ok := (j.data).(string); ok {
		return s, nil
	}
	return "", errors.New("type assertion to string failed")
}

// Bytes type asserts to `[]byte`
func (j *Json) Bytes() ([]byte, error) {
	if s, ok := (j.data).(string); ok {
		return []byte(s), nil
	}
	return nil, errors.New("type assertion to []byte failed")
}

// StringArray type asserts to an `array` of `string`
func (j *Json) StringArray() ([]string, error) {
	arr, err := j.Array()
	if err != nil {
		return nil, err
	}
	retArr := make([]string, 0, len(arr))
	for _, a := range arr {
		if a == nil {
			retArr = append(retArr, "")
			continue
		}
		s, ok := a.(string)
		if !ok {
			return nil, err
		}
		retArr = append(retArr, s)
	}
	return retArr, nil
}

// MustArray guarantees the return of a `[]interface{}` (with optional default)
//
// useful when you want to interate over array values in a succinct manner:
//		for i, v := range js.Get("results").MustArray() {
//			fmt.Println(i, v)
//		}
func (j *Json) MustArray(args ...[]interface{}) []interface{} {
	var def []interface{}

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustArray() received too many arguments %d", len(args))
	}

	a, err := j.Array()
	if err == nil {
		return a
	}

	return def
}

// MustMap guarantees the return of a `map[string]interface{}` (with optional default)
//
// useful when you want to interate over map values in a succinct manner:
//		for k, v := range js.Get("dictionary").MustMap() {
//			fmt.Println(k, v)
//		}
func (j *Json) MustMap(args ...map[string]interface{}) map[string]interface{} {
	var def map[string]interface{}

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustMap() received too many arguments %d", len(args))
	}

	a, err := j.Map()
	if err == nil {
		return a
	}

	return def
}

// MustString guarantees the return of a `string` (with optional default)
//
// useful when you explicitly want a `string` in a single value return context:
//     myFunc(js.Get("param1").MustString(), js.Get("optional_param").MustString("my_default"))
func (j *Json) MustString(args ...string) string {
	var def string

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustString() received too many arguments %d", len(args))
	}

	s, err := j.String()
	if err == nil {
		return s
	}

	return def
}

// MustStringArray guarantees the return of a `[]string` (with optional default)
//
// useful when you want to interate over array values in a succinct manner:
//		for i, s := range js.Get("results").MustStringArray() {
//			fmt.Println(i, s)
//		}
func (j *Json) MustStringArray(args ...[]string) []string {
	var def []string

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustStringArray() received too many arguments %d", len(args))
	}

	a, err := j.StringArray()
	if err == nil {
		return a
	}

	return def
}

// MustInt guarantees the return of an `int` (with optional default)
//
// useful when you explicitly want an `int` in a single value return context:
//     myFunc(js.Get("param1").MustInt(), js.Get("optional_param").MustInt(5150))
func (j *Json) MustInt(args ...int) int {
	var def int

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustInt() received too many arguments %d", len(args))
	}

	i, err := j.Int()
	if err == nil {
		return i
	}

	return def
}

// MustFloat64 guarantees the return of a `float64` (with optional default)
//
// useful when you explicitly want a `float64` in a single value return context:
//     myFunc(js.Get("param1").MustFloat64(), js.Get("optional_param").MustFloat64(5.150))
func (j *Json) MustFloat64(args ...float64) float64 {
	var def float64

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustFloat64() received too many arguments %d", len(args))
	}

	f, err := j.Float64()
	if err == nil {
		return f
	}

	return def
}

// MustBool guarantees the return of a `bool` (with optional default)
//
// useful when you explicitly want a `bool` in a single value return context:
//     myFunc(js.Get("param1").MustBool(), js.Get("optional_param").MustBool(true))
func (j *Json) MustBool(args ...bool) bool {
	var def bool

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustBool() received too many arguments %d", len(args))
	}

	b, err := j.Bool()
	if err == nil {
		return b
	}

	return def
}

// MustInt64 guarantees the return of an `int64` (with optional default)
//
// useful when you explicitly want an `int64` in a single value return context:
//     myFunc(js.Get("param1").MustInt64(), js.Get("optional_param").MustInt64(5150))
func (j *Json) MustInt64(args ...int64) int64 {
	var def int64

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustInt64() received too many arguments %d", len(args))
	}

	i, err := j.Int64()
	if err == nil {
		return i
	}

	return def
}

// MustUInt64 guarantees the return of an `uint64` (with optional default)
//
// useful when you explicitly want an `uint64` in a single value return context:
//     myFunc(js.Get("param1").MustUint64(), js.Get("optional_param").MustUint64(5150))
func (j *Json) MustUint64(args ...uint64) uint64 {
	var def uint64

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustUint64() received too many arguments %d", len(args))
	}

	i, err := j.Uint64()
	if err == nil {
		return i
	}

	return def
}
<commit_msg>fix(): fixed from db -> json<commit_after>package simplejson

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"log"
)

// returns the current implementation version
func Version() string {
	return "0.5.0"
}

type Json struct {
	data interface{}
}

func (j *Json) FromDB(data []byte) error {
	j.data = make(map[string]interface{})

	dec := json.NewDecoder(bytes.NewBuffer(data))
	dec.UseNumber()
	return dec.Decode(&j.data)
}

func (j *Json) ToDB() ([]byte, error) {
	if j == nil || j.data == nil {
		return nil, nil
	}

	return j.Encode()
}

// NewJson returns a pointer to a new `Json` object
// after unmarshaling `body` bytes
func NewJson(body []byte) (*Json, error) {
	j := new(Json)
	err := j.UnmarshalJSON(body)
	if err != nil {
		return nil, err
	}
	return j, nil
}

// New returns a pointer to a new, empty `Json` object
func New() *Json {
	return &Json{
		data: make(map[string]interface{}),
	}
}

// Interface returns the underlying data
func (j *Json) Interface() interface{} {
	return j.data
}

// Encode returns its marshaled data as `[]byte`
func (j *Json) Encode() ([]byte, error) {
	return j.MarshalJSON()
}

// EncodePretty returns its marshaled data as `[]byte` with indentation
func (j *Json) EncodePretty() ([]byte, error) {
	return json.MarshalIndent(&j.data, "", "  ")
}

// Implements the json.Marshaler interface.
func (j *Json) MarshalJSON() ([]byte, error) {
	fmt.Printf("MarshalJSON")
	return json.Marshal(&j.data)
}

// Set modifies `Json` map by `key` and `value`
// Useful for changing single key/value in a `Json` object easily.
func (j *Json) Set(key string, val interface{}) {
	m, err := j.Map()
	if err != nil {
		return
	}
	m[key] = val
}

// SetPath modifies `Json`, recursively checking/creating map keys for the supplied path,
// and then finally writing in the value
func (j *Json) SetPath(branch []string, val interface{}) {
	if len(branch) == 0 {
		j.data = val
		return
	}

	// in order to insert our branch, we need map[string]interface{}
	if _, ok := (j.data).(map[string]interface{}); !ok {
		// have to replace with something suitable
		j.data = make(map[string]interface{})
	}
	curr := j.data.(map[string]interface{})

	for i := 0; i < len(branch)-1; i++ {
		b := branch[i]
		// key exists?
		if _, ok := curr[b]; !ok {
			n := make(map[string]interface{})
			curr[b] = n
			curr = n
			continue
		}

		// make sure the value is the right sort of thing
		if _, ok := curr[b].(map[string]interface{}); !ok {
			// have to replace with something suitable
			n := make(map[string]interface{})
			curr[b] = n
		}

		curr = curr[b].(map[string]interface{})
	}

	// add remaining k/v
	curr[branch[len(branch)-1]] = val
}

// Del modifies `Json` map by deleting `key` if it is present.
func (j *Json) Del(key string) {
	m, err := j.Map()
	if err != nil {
		return
	}
	delete(m, key)
}

// Get returns a pointer to a new `Json` object
// for `key` in its `map` representation
//
// useful for chaining operations (to traverse a nested JSON):
//    js.Get("top_level").Get("dict").Get("value").Int()
func (j *Json) Get(key string) *Json {
	m, err := j.Map()
	if err == nil {
		if val, ok := m[key]; ok {
			return &Json{val}
		}
	}
	return &Json{nil}
}

// GetPath searches for the item as specified by the branch
// without the need to deep dive using Get()'s.
//
//   js.GetPath("top_level", "dict")
func (j *Json) GetPath(branch ...string) *Json {
	jin := j
	for _, p := range branch {
		jin = jin.Get(p)
	}
	return jin
}

// GetIndex returns a pointer to a new `Json` object
// for `index` in its `array` representation
//
// this is the analog to Get when accessing elements of
// a json array instead of a json object:
//    js.Get("top_level").Get("array").GetIndex(1).Get("key").Int()
func (j *Json) GetIndex(index int) *Json {
	a, err := j.Array()
	if err == nil {
		if len(a) > index {
			return &Json{a[index]}
		}
	}
	return &Json{nil}
}

// CheckGet returns a pointer to a new `Json` object and
// a `bool` identifying success or failure
//
// useful for chained operations when success is important:
//    if data, ok := js.Get("top_level").CheckGet("inner"); ok {
//        log.Println(data)
//    }
func (j *Json) CheckGet(key string) (*Json, bool) {
	m, err := j.Map()
	if err == nil {
		if val, ok := m[key]; ok {
			return &Json{val}, true
		}
	}
	return nil, false
}

// Map type asserts to `map`
func (j *Json) Map() (map[string]interface{}, error) {
	if m, ok := (j.data).(map[string]interface{}); ok {
		return m, nil
	}
	return nil, errors.New("type assertion to map[string]interface{} failed")
}

// Array type asserts to an `array`
func (j *Json) Array() ([]interface{}, error) {
	if a, ok := (j.data).([]interface{}); ok {
		return a, nil
	}
	return nil, errors.New("type assertion to []interface{} failed")
}

// Bool type asserts to `bool`
func (j *Json) Bool() (bool, error) {
	if s, ok := (j.data).(bool); ok {
		return s, nil
	}
	return false, errors.New("type assertion to bool failed")
}

// String type asserts to `string`
func (j *Json) String() (string, error) {
	if s, ok := (j.data).(string); ok {
		return s, nil
	}
	return "", errors.New("type assertion to string failed")
}

// Bytes type asserts to `[]byte`
func (j *Json) Bytes() ([]byte, error) {
	if s, ok := (j.data).(string); ok {
		return []byte(s), nil
	}
	return nil, errors.New("type assertion to []byte failed")
}

// StringArray type asserts to an `array` of `string`
func (j *Json) StringArray() ([]string, error) {
	arr, err := j.Array()
	if err != nil {
		return nil, err
	}
	retArr := make([]string, 0, len(arr))
	for _, a := range arr {
		if a == nil {
			retArr = append(retArr, "")
			continue
		}
		s, ok := a.(string)
		if !ok {
			return nil, err
		}
		retArr = append(retArr, s)
	}
	return retArr, nil
}

// MustArray guarantees the return of a `[]interface{}` (with optional default)
//
// useful when you want to interate over array values in a succinct manner:
//		for i, v := range js.Get("results").MustArray() {
//			fmt.Println(i, v)
//		}
func (j *Json) MustArray(args ...[]interface{}) []interface{} {
	var def []interface{}

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustArray() received too many arguments %d", len(args))
	}

	a, err := j.Array()
	if err == nil {
		return a
	}

	return def
}

// MustMap guarantees the return of a `map[string]interface{}` (with optional default)
//
// useful when you want to interate over map values in a succinct manner:
//		for k, v := range js.Get("dictionary").MustMap() {
//			fmt.Println(k, v)
//		}
func (j *Json) MustMap(args ...map[string]interface{}) map[string]interface{} {
	var def map[string]interface{}

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustMap() received too many arguments %d", len(args))
	}

	a, err := j.Map()
	if err == nil {
		return a
	}

	return def
}

// MustString guarantees the return of a `string` (with optional default)
//
// useful when you explicitly want a `string` in a single value return context:
//     myFunc(js.Get("param1").MustString(), js.Get("optional_param").MustString("my_default"))
func (j *Json) MustString(args ...string) string {
	var def string

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustString() received too many arguments %d", len(args))
	}

	s, err := j.String()
	if err == nil {
		return s
	}

	return def
}

// MustStringArray guarantees the return of a `[]string` (with optional default)
//
// useful when you want to interate over array values in a succinct manner:
//		for i, s := range js.Get("results").MustStringArray() {
//			fmt.Println(i, s)
//		}
func (j *Json) MustStringArray(args ...[]string) []string {
	var def []string

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustStringArray() received too many arguments %d", len(args))
	}

	a, err := j.StringArray()
	if err == nil {
		return a
	}

	return def
}

// MustInt guarantees the return of an `int` (with optional default)
//
// useful when you explicitly want an `int` in a single value return context:
//     myFunc(js.Get("param1").MustInt(), js.Get("optional_param").MustInt(5150))
func (j *Json) MustInt(args ...int) int {
	var def int

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustInt() received too many arguments %d", len(args))
	}

	i, err := j.Int()
	if err == nil {
		return i
	}

	return def
}

// MustFloat64 guarantees the return of a `float64` (with optional default)
//
// useful when you explicitly want a `float64` in a single value return context:
//     myFunc(js.Get("param1").MustFloat64(), js.Get("optional_param").MustFloat64(5.150))
func (j *Json) MustFloat64(args ...float64) float64 {
	var def float64

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustFloat64() received too many arguments %d", len(args))
	}

	f, err := j.Float64()
	if err == nil {
		return f
	}

	return def
}

// MustBool guarantees the return of a `bool` (with optional default)
//
// useful when you explicitly want a `bool` in a single value return context:
//     myFunc(js.Get("param1").MustBool(), js.Get("optional_param").MustBool(true))
func (j *Json) MustBool(args ...bool) bool {
	var def bool

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustBool() received too many arguments %d", len(args))
	}

	b, err := j.Bool()
	if err == nil {
		return b
	}

	return def
}

// MustInt64 guarantees the return of an `int64` (with optional default)
//
// useful when you explicitly want an `int64` in a single value return context:
//     myFunc(js.Get("param1").MustInt64(), js.Get("optional_param").MustInt64(5150))
func (j *Json) MustInt64(args ...int64) int64 {
	var def int64

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustInt64() received too many arguments %d", len(args))
	}

	i, err := j.Int64()
	if err == nil {
		return i
	}

	return def
}

// MustUInt64 guarantees the return of an `uint64` (with optional default)
//
// useful when you explicitly want an `uint64` in a single value return context:
//     myFunc(js.Get("param1").MustUint64(), js.Get("optional_param").MustUint64(5150))
func (j *Json) MustUint64(args ...uint64) uint64 {
	var def uint64

	switch len(args) {
	case 0:
	case 1:
		def = args[0]
	default:
		log.Panicf("MustUint64() received too many arguments %d", len(args))
	}

	i, err := j.Uint64()
	if err == nil {
		return i
	}

	return def
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package node

import (
	"fmt"
	"strings"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/client/cache"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
	"k8s.io/kubernetes/pkg/client/record"
	"k8s.io/kubernetes/pkg/cloudprovider"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/kubelet/util/format"
	"k8s.io/kubernetes/pkg/types"
	utilerrors "k8s.io/kubernetes/pkg/util/errors"
	"k8s.io/kubernetes/pkg/util/node"
	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
	utilversion "k8s.io/kubernetes/pkg/util/version"

	"github.com/golang/glog"
)

const (
	// Number of Nodes that needs to be in the cluster for it to be treated as "large"
	LargeClusterThreshold = 20
)

// deletePods will delete all pods from master running on given node, and return true
// if any pods were deleted, or were found pending deletion.
func deletePods(kubeClient clientset.Interface, recorder record.EventRecorder, nodeName, nodeUID string, daemonStore cache.StoreToDaemonSetLister) (bool, error) {
	remaining := false
	selector := fields.OneTermEqualSelector(api.PodHostField, nodeName).String()
	options := v1.ListOptions{FieldSelector: selector}
	pods, err := kubeClient.Core().Pods(v1.NamespaceAll).List(options)
	var updateErrList []error

	if err != nil {
		return remaining, err
	}

	if len(pods.Items) > 0 {
		recordNodeEvent(recorder, nodeName, nodeUID, v1.EventTypeNormal, "DeletingAllPods", fmt.Sprintf("Deleting all Pods from Node %v.", nodeName))
	}

	for _, pod := range pods.Items {
		// Defensive check, also needed for tests.
		if pod.Spec.NodeName != nodeName {
			continue
		}

		// Set reason and message in the pod object.
		if _, err = setPodTerminationReason(kubeClient, &pod, nodeName); err != nil {
			if errors.IsConflict(err) {
				updateErrList = append(updateErrList,
					fmt.Errorf("update status failed for pod %q: %v", format.Pod(&pod), err))
				continue
			}
		}
		// if the pod has already been marked for deletion, we still return true that there are remaining pods.
		if pod.DeletionGracePeriodSeconds != nil {
			remaining = true
			continue
		}
		// if the pod is managed by a daemonset, ignore it
		_, err := daemonStore.GetPodDaemonSets(&pod)
		if err == nil { // No error means at least one daemonset was found
			continue
		}

		glog.V(2).Infof("Starting deletion of pod %v", pod.Name)
		recorder.Eventf(&pod, v1.EventTypeNormal, "NodeControllerEviction", "Marking for deletion Pod %s from Node %s", pod.Name, nodeName)
		if err := kubeClient.Core().Pods(pod.Namespace).Delete(pod.Name, nil); err != nil {
			return false, err
		}
		remaining = true
	}

	if len(updateErrList) > 0 {
		return false, utilerrors.NewAggregate(updateErrList)
	}
	return remaining, nil
}

// setPodTerminationReason attempts to set a reason and message in the pod status, updates it in the apiserver,
// and returns an error if it encounters one.
func setPodTerminationReason(kubeClient clientset.Interface, pod *v1.Pod, nodeName string) (*v1.Pod, error) {
	if pod.Status.Reason == node.NodeUnreachablePodReason {
		return pod, nil
	}

	pod.Status.Reason = node.NodeUnreachablePodReason
	pod.Status.Message = fmt.Sprintf(node.NodeUnreachablePodMessage, nodeName, pod.Name)

	var updatedPod *v1.Pod
	var err error
	if updatedPod, err = kubeClient.Core().Pods(pod.Namespace).UpdateStatus(pod); err != nil {
		return nil, err
	}
	return updatedPod, nil
}

func forcefullyDeletePod(c clientset.Interface, pod *v1.Pod) error {
	var zero int64
	glog.Infof("NodeController is force deleting Pod: %v:%v", pod.Namespace, pod.Name)
	err := c.Core().Pods(pod.Namespace).Delete(pod.Name, &v1.DeleteOptions{GracePeriodSeconds: &zero})
	if err == nil {
		glog.V(4).Infof("forceful deletion of %s succeeded", pod.Name)
	}
	return err
}

// forcefullyDeleteNode immediately the node. The pods on the node are cleaned
// up by the podGC.
func forcefullyDeleteNode(kubeClient clientset.Interface, nodeName string) error {
	if err := kubeClient.Core().Nodes().Delete(nodeName, nil); err != nil {
		return fmt.Errorf("unable to delete node %q: %v", nodeName, err)
	}
	return nil
}

// maybeDeleteTerminatingPod non-gracefully deletes pods that are terminating
// that should not be gracefully terminated.
func (nc *NodeController) maybeDeleteTerminatingPod(obj interface{}) {
	pod, ok := obj.(*v1.Pod)
	if !ok {
		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
		if !ok {
			glog.Errorf("Couldn't get object from tombstone %#v", obj)
			return
		}
		pod, ok = tombstone.Obj.(*v1.Pod)
		if !ok {
			glog.Errorf("Tombstone contained object that is not a Pod %#v", obj)
			return
		}
	}

	// consider only terminating pods
	if pod.DeletionTimestamp == nil {
		return
	}

	nodeObj, found, err := nc.nodeStore.Store.GetByKey(pod.Spec.NodeName)
	if err != nil {
		// this can only happen if the Store.KeyFunc has a problem creating
		// a key for the pod. If it happens once, it will happen again so
		// don't bother requeuing the pod.
		utilruntime.HandleError(err)
		return
	}

	// if there is no such node, do nothing and let the podGC clean it up.
	if !found {
		return
	}

	// delete terminating pods that have been scheduled on
	// nodes that do not support graceful termination
	// TODO(mikedanese): this can be removed when we no longer
	// guarantee backwards compatibility of master API to kubelets with
	// versions less than 1.1.0
	node := nodeObj.(*v1.Node)
	v, err := utilversion.ParseSemantic(node.Status.NodeInfo.KubeletVersion)
	if err != nil {
		glog.V(0).Infof("couldn't parse verions %q of node: %v", node.Status.NodeInfo.KubeletVersion, err)
		utilruntime.HandleError(nc.forcefullyDeletePod(pod))
		return
	}
	if v.LessThan(gracefulDeletionVersion) {
		utilruntime.HandleError(nc.forcefullyDeletePod(pod))
		return
	}
}

// update ready status of all pods running on given node from master
// return true if success
func markAllPodsNotReady(kubeClient clientset.Interface, node *v1.Node) error {
	// Don't set pods to NotReady if the kubelet is running a version that
	// doesn't understand how to correct readiness.
	// TODO: Remove this check when we no longer guarantee backward compatibility
	// with node versions < 1.2.0.
	if nodeRunningOutdatedKubelet(node) {
		return nil
	}
	nodeName := node.Name
	glog.V(2).Infof("Update ready status of pods on node [%v]", nodeName)
	opts := v1.ListOptions{FieldSelector: fields.OneTermEqualSelector(api.PodHostField, nodeName).String()}
	pods, err := kubeClient.Core().Pods(v1.NamespaceAll).List(opts)
	if err != nil {
		return err
	}

	errMsg := []string{}
	for _, pod := range pods.Items {
		// Defensive check, also needed for tests.
		if pod.Spec.NodeName != nodeName {
			continue
		}

		for i, cond := range pod.Status.Conditions {
			if cond.Type == v1.PodReady {
				pod.Status.Conditions[i].Status = v1.ConditionFalse
				glog.V(2).Infof("Updating ready status of pod %v to false", pod.Name)
				_, err := kubeClient.Core().Pods(pod.Namespace).UpdateStatus(&pod)
				if err != nil {
					glog.Warningf("Failed to update status for pod %q: %v", format.Pod(&pod), err)
					errMsg = append(errMsg, fmt.Sprintf("%v", err))
				}
				break
			}
		}
	}
	if len(errMsg) == 0 {
		return nil
	}
	return fmt.Errorf("%v", strings.Join(errMsg, "; "))
}

// nodeRunningOutdatedKubelet returns true if the kubeletVersion reported
// in the nodeInfo of the given node is "outdated", meaning < 1.2.0.
// Older versions were inflexible and modifying pod.Status directly through
// the apiserver would result in unexpected outcomes.
func nodeRunningOutdatedKubelet(node *v1.Node) bool {
	v, err := utilversion.ParseSemantic(node.Status.NodeInfo.KubeletVersion)
	if err != nil {
		glog.Errorf("couldn't parse version %q of node %v", node.Status.NodeInfo.KubeletVersion, err)
		return true
	}
	if v.LessThan(podStatusReconciliationVersion) {
		glog.Infof("Node %v running kubelet at (%v) which is less than the minimum version that allows nodecontroller to mark pods NotReady (%v).", node.Name, v, podStatusReconciliationVersion)
		return true
	}
	return false
}

func nodeExistsInCloudProvider(cloud cloudprovider.Interface, nodeName types.NodeName) (bool, error) {
	instances, ok := cloud.Instances()
	if !ok {
		return false, fmt.Errorf("%v", ErrCloudInstance)
	}
	if _, err := instances.ExternalID(nodeName); err != nil {
		if err == cloudprovider.InstanceNotFound {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

func recordNodeEvent(recorder record.EventRecorder, nodeName, nodeUID, eventtype, reason, event string) {
	ref := &v1.ObjectReference{
		Kind:      "Node",
		Name:      nodeName,
		UID:       types.UID(nodeUID),
		Namespace: "",
	}
	glog.V(2).Infof("Recording %s event message for node %s", event, nodeName)
	recorder.Eventf(ref, eventtype, reason, "Node %s event: %s", nodeName, event)
}

func recordNodeStatusChange(recorder record.EventRecorder, node *v1.Node, new_status string) {
	ref := &v1.ObjectReference{
		Kind:      "Node",
		Name:      node.Name,
		UID:       node.UID,
		Namespace: "",
	}
	glog.V(2).Infof("Recording status change %s event message for node %s", new_status, node.Name)
	// TODO: This requires a transaction, either both node status is updated
	// and event is recorded or neither should happen, see issue #6055.
	recorder.Eventf(ref, v1.EventTypeNormal, new_status, "Node %s status is now: %s", node.Name, new_status)
}
<commit_msg>Log content is wrong, modify<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package node

import (
	"fmt"
	"strings"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/client/cache"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
	"k8s.io/kubernetes/pkg/client/record"
	"k8s.io/kubernetes/pkg/cloudprovider"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/kubelet/util/format"
	"k8s.io/kubernetes/pkg/types"
	utilerrors "k8s.io/kubernetes/pkg/util/errors"
	"k8s.io/kubernetes/pkg/util/node"
	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
	utilversion "k8s.io/kubernetes/pkg/util/version"

	"github.com/golang/glog"
)

const (
	// Number of Nodes that needs to be in the cluster for it to be treated as "large"
	LargeClusterThreshold = 20
)

// deletePods will delete all pods from master running on given node, and return true
// if any pods were deleted, or were found pending deletion.
func deletePods(kubeClient clientset.Interface, recorder record.EventRecorder, nodeName, nodeUID string, daemonStore cache.StoreToDaemonSetLister) (bool, error) {
	remaining := false
	selector := fields.OneTermEqualSelector(api.PodHostField, nodeName).String()
	options := v1.ListOptions{FieldSelector: selector}
	pods, err := kubeClient.Core().Pods(v1.NamespaceAll).List(options)
	var updateErrList []error

	if err != nil {
		return remaining, err
	}

	if len(pods.Items) > 0 {
		recordNodeEvent(recorder, nodeName, nodeUID, v1.EventTypeNormal, "DeletingAllPods", fmt.Sprintf("Deleting all Pods from Node %v.", nodeName))
	}

	for _, pod := range pods.Items {
		// Defensive check, also needed for tests.
		if pod.Spec.NodeName != nodeName {
			continue
		}

		// Set reason and message in the pod object.
		if _, err = setPodTerminationReason(kubeClient, &pod, nodeName); err != nil {
			if errors.IsConflict(err) {
				updateErrList = append(updateErrList,
					fmt.Errorf("update status failed for pod %q: %v", format.Pod(&pod), err))
				continue
			}
		}
		// if the pod has already been marked for deletion, we still return true that there are remaining pods.
		if pod.DeletionGracePeriodSeconds != nil {
			remaining = true
			continue
		}
		// if the pod is managed by a daemonset, ignore it
		_, err := daemonStore.GetPodDaemonSets(&pod)
		if err == nil { // No error means at least one daemonset was found
			continue
		}

		glog.V(2).Infof("Starting deletion of pod %v", pod.Name)
		recorder.Eventf(&pod, v1.EventTypeNormal, "NodeControllerEviction", "Marking for deletion Pod %s from Node %s", pod.Name, nodeName)
		if err := kubeClient.Core().Pods(pod.Namespace).Delete(pod.Name, nil); err != nil {
			return false, err
		}
		remaining = true
	}

	if len(updateErrList) > 0 {
		return false, utilerrors.NewAggregate(updateErrList)
	}
	return remaining, nil
}

// setPodTerminationReason attempts to set a reason and message in the pod status, updates it in the apiserver,
// and returns an error if it encounters one.
func setPodTerminationReason(kubeClient clientset.Interface, pod *v1.Pod, nodeName string) (*v1.Pod, error) {
	if pod.Status.Reason == node.NodeUnreachablePodReason {
		return pod, nil
	}

	pod.Status.Reason = node.NodeUnreachablePodReason
	pod.Status.Message = fmt.Sprintf(node.NodeUnreachablePodMessage, nodeName, pod.Name)

	var updatedPod *v1.Pod
	var err error
	if updatedPod, err = kubeClient.Core().Pods(pod.Namespace).UpdateStatus(pod); err != nil {
		return nil, err
	}
	return updatedPod, nil
}

func forcefullyDeletePod(c clientset.Interface, pod *v1.Pod) error {
	var zero int64
	glog.Infof("NodeController is force deleting Pod: %v:%v", pod.Namespace, pod.Name)
	err := c.Core().Pods(pod.Namespace).Delete(pod.Name, &v1.DeleteOptions{GracePeriodSeconds: &zero})
	if err == nil {
		glog.V(4).Infof("forceful deletion of %s succeeded", pod.Name)
	}
	return err
}

// forcefullyDeleteNode immediately the node. The pods on the node are cleaned
// up by the podGC.
func forcefullyDeleteNode(kubeClient clientset.Interface, nodeName string) error {
	if err := kubeClient.Core().Nodes().Delete(nodeName, nil); err != nil {
		return fmt.Errorf("unable to delete node %q: %v", nodeName, err)
	}
	return nil
}

// maybeDeleteTerminatingPod non-gracefully deletes pods that are terminating
// that should not be gracefully terminated.
func (nc *NodeController) maybeDeleteTerminatingPod(obj interface{}) {
	pod, ok := obj.(*v1.Pod)
	if !ok {
		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
		if !ok {
			glog.Errorf("Couldn't get object from tombstone %#v", obj)
			return
		}
		pod, ok = tombstone.Obj.(*v1.Pod)
		if !ok {
			glog.Errorf("Tombstone contained object that is not a Pod %#v", obj)
			return
		}
	}

	// consider only terminating pods
	if pod.DeletionTimestamp == nil {
		return
	}

	nodeObj, found, err := nc.nodeStore.Store.GetByKey(pod.Spec.NodeName)
	if err != nil {
		// this can only happen if the Store.KeyFunc has a problem creating
		// a key for the pod. If it happens once, it will happen again so
		// don't bother requeuing the pod.
		utilruntime.HandleError(err)
		return
	}

	// if there is no such node, do nothing and let the podGC clean it up.
	if !found {
		return
	}

	// delete terminating pods that have been scheduled on
	// nodes that do not support graceful termination
	// TODO(mikedanese): this can be removed when we no longer
	// guarantee backwards compatibility of master API to kubelets with
	// versions less than 1.1.0
	node := nodeObj.(*v1.Node)
	v, err := utilversion.ParseSemantic(node.Status.NodeInfo.KubeletVersion)
	if err != nil {
		glog.V(0).Infof("Couldn't parse version %q of node: %v", node.Status.NodeInfo.KubeletVersion, err)
		utilruntime.HandleError(nc.forcefullyDeletePod(pod))
		return
	}
	if v.LessThan(gracefulDeletionVersion) {
		utilruntime.HandleError(nc.forcefullyDeletePod(pod))
		return
	}
}

// update ready status of all pods running on given node from master
// return true if success
func markAllPodsNotReady(kubeClient clientset.Interface, node *v1.Node) error {
	// Don't set pods to NotReady if the kubelet is running a version that
	// doesn't understand how to correct readiness.
	// TODO: Remove this check when we no longer guarantee backward compatibility
	// with node versions < 1.2.0.
	if nodeRunningOutdatedKubelet(node) {
		return nil
	}
	nodeName := node.Name
	glog.V(2).Infof("Update ready status of pods on node [%v]", nodeName)
	opts := v1.ListOptions{FieldSelector: fields.OneTermEqualSelector(api.PodHostField, nodeName).String()}
	pods, err := kubeClient.Core().Pods(v1.NamespaceAll).List(opts)
	if err != nil {
		return err
	}

	errMsg := []string{}
	for _, pod := range pods.Items {
		// Defensive check, also needed for tests.
		if pod.Spec.NodeName != nodeName {
			continue
		}

		for i, cond := range pod.Status.Conditions {
			if cond.Type == v1.PodReady {
				pod.Status.Conditions[i].Status = v1.ConditionFalse
				glog.V(2).Infof("Updating ready status of pod %v to false", pod.Name)
				_, err := kubeClient.Core().Pods(pod.Namespace).UpdateStatus(&pod)
				if err != nil {
					glog.Warningf("Failed to update status for pod %q: %v", format.Pod(&pod), err)
					errMsg = append(errMsg, fmt.Sprintf("%v", err))
				}
				break
			}
		}
	}
	if len(errMsg) == 0 {
		return nil
	}
	return fmt.Errorf("%v", strings.Join(errMsg, "; "))
}

// nodeRunningOutdatedKubelet returns true if the kubeletVersion reported
// in the nodeInfo of the given node is "outdated", meaning < 1.2.0.
// Older versions were inflexible and modifying pod.Status directly through
// the apiserver would result in unexpected outcomes.
func nodeRunningOutdatedKubelet(node *v1.Node) bool {
	v, err := utilversion.ParseSemantic(node.Status.NodeInfo.KubeletVersion)
	if err != nil {
		glog.Errorf("couldn't parse version %q of node %v", node.Status.NodeInfo.KubeletVersion, err)
		return true
	}
	if v.LessThan(podStatusReconciliationVersion) {
		glog.Infof("Node %v running kubelet at (%v) which is less than the minimum version that allows nodecontroller to mark pods NotReady (%v).", node.Name, v, podStatusReconciliationVersion)
		return true
	}
	return false
}

func nodeExistsInCloudProvider(cloud cloudprovider.Interface, nodeName types.NodeName) (bool, error) {
	instances, ok := cloud.Instances()
	if !ok {
		return false, fmt.Errorf("%v", ErrCloudInstance)
	}
	if _, err := instances.ExternalID(nodeName); err != nil {
		if err == cloudprovider.InstanceNotFound {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

func recordNodeEvent(recorder record.EventRecorder, nodeName, nodeUID, eventtype, reason, event string) {
	ref := &v1.ObjectReference{
		Kind:      "Node",
		Name:      nodeName,
		UID:       types.UID(nodeUID),
		Namespace: "",
	}
	glog.V(2).Infof("Recording %s event message for node %s", event, nodeName)
	recorder.Eventf(ref, eventtype, reason, "Node %s event: %s", nodeName, event)
}

func recordNodeStatusChange(recorder record.EventRecorder, node *v1.Node, new_status string) {
	ref := &v1.ObjectReference{
		Kind:      "Node",
		Name:      node.Name,
		UID:       node.UID,
		Namespace: "",
	}
	glog.V(2).Infof("Recording status change %s event message for node %s", new_status, node.Name)
	// TODO: This requires a transaction, either both node status is updated
	// and event is recorded or neither should happen, see issue #6055.
	recorder.Eventf(ref, v1.EventTypeNormal, new_status, "Node %s status is now: %s", node.Name, new_status)
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package cluster to manage a Ceph cluster.
package cluster

import (
	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
	clientcontroller "github.com/rook/rook/pkg/operator/ceph/client"
	"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
	"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
	"github.com/rook/rook/pkg/operator/ceph/object/bucket"
)

func (c *ClusterController) configureCephMonitoring(cluster *cluster, clusterInfo *cephclient.ClusterInfo) {
	var isDisabled bool
	daemons := []string{"mon", "osd", "status"}

	for _, daemon := range daemons {
		// Is the monitoring enabled for that daemon?
		isDisabled = isMonitoringDisabled(daemon, cluster.Spec)
		if health, ok := cluster.monitoringChannels[daemon]; ok {
			if health.monitoringRunning {
				// If the goroutine was running but the CR was updated to stop the monitoring we need to close the channel
				if isDisabled {
					// close the channel so the goroutine can stop
					close(cluster.monitoringChannels[daemon].stopChan)
					// Set monitoring to false since it's not running anymore
					cluster.monitoringChannels[daemon].monitoringRunning = false
				} else {
					logger.Debugf("ceph %s health go routine is already running for cluster %q", daemon, cluster.Namespace)
				}
			} else {
				// if not already running and not disabled, we run it
				if !isDisabled {
					// Run the go routine
					c.startMonitoringCheck(cluster, clusterInfo, daemon)

					// Set the flag to indicate monitoring is running
					cluster.monitoringChannels[daemon].monitoringRunning = true
				}
			}
		} else {
			// If the mon does not exist in the map, this is a first deployment or an operator restart
			// So we check the desired state from the CR and run it if necessary
			//
			// If the mon monitoring is enabled
			if !isDisabled {
				cluster.monitoringChannels[daemon] = &clusterHealth{
					stopChan:          make(chan struct{}),
					monitoringRunning: true, // Set the flag to indicate monitoring is running
				}

				// Run the go routine
				c.startMonitoringCheck(cluster, clusterInfo, daemon)
			}
		}
	}

	// Start watchers
	if cluster.watchersActivated == true {
		logger.Debugf("cluster is already being watched by bucket and client provisioner for cluster %q", cluster.Namespace)
		return
	}

	// Start client CRD watcher
	cc := clientcontroller.NewClientController(c.context, cluster.Namespace)
	cc.StartWatch(cluster.stopCh)

	// Start the object bucket provisioner
	bucketProvisioner := bucket.NewProvisioner(c.context, clusterInfo)
	// If cluster is external, pass down the user to the bucket controller

	// note: the error return below is ignored and is expected to be removed from the
	//   bucket library's `NewProvisioner` function
	bucketController, _ := bucket.NewBucketController(c.context.KubeConfig, bucketProvisioner)
	go bucketController.Run(cluster.stopCh)

	// enable the cluster watcher once
	cluster.watchersActivated = true
}

func isMonitoringDisabled(daemon string, clusterSpec *cephv1.ClusterSpec) bool {
	switch daemon {
	case "mon":
		return clusterSpec.HealthCheck.DaemonHealth.Monitor.Disabled

	case "osd":
		return clusterSpec.HealthCheck.DaemonHealth.ObjectStorageDaemon.Disabled

	case "status":
		return clusterSpec.HealthCheck.DaemonHealth.Status.Disabled
	}

	return false
}

func (c *ClusterController) startMonitoringCheck(cluster *cluster, clusterInfo *cephclient.ClusterInfo, daemon string) {
	switch daemon {
	case "mon":
		healthChecker := mon.NewHealthChecker(cluster.mons, cluster.Spec)
		logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
		go healthChecker.Check(cluster.monitoringChannels[daemon].stopChan)

	case "osd":
		c.osdChecker = osd.NewOSDHealthMonitor(c.context, clusterInfo, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Spec.HealthCheck)
		logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
		go c.osdChecker.Start(cluster.monitoringChannels[daemon].stopChan)

	case "status":
		cephChecker := newCephStatusChecker(c.context, clusterInfo, cluster.Spec.HealthCheck)
		logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
		go cephChecker.checkCephStatus(cluster.monitoringChannels[daemon].stopChan)
	}
}
<commit_msg>ceph: do not monitor osd on external mode<commit_after>/*
Copyright 2020 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package cluster to manage a Ceph cluster.
package cluster

import (
	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
	clientcontroller "github.com/rook/rook/pkg/operator/ceph/client"
	"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
	"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
	"github.com/rook/rook/pkg/operator/ceph/object/bucket"
)

func (c *ClusterController) configureCephMonitoring(cluster *cluster, clusterInfo *cephclient.ClusterInfo) {
	var isDisabled bool
	daemons := []string{"mon", "osd", "status"}

	for _, daemon := range daemons {
		// Is the monitoring enabled for that daemon?
		isDisabled = isMonitoringDisabled(daemon, cluster.Spec)
		if health, ok := cluster.monitoringChannels[daemon]; ok {
			if health.monitoringRunning {
				// If the goroutine was running but the CR was updated to stop the monitoring we need to close the channel
				if isDisabled {
					// close the channel so the goroutine can stop
					close(cluster.monitoringChannels[daemon].stopChan)
					// Set monitoring to false since it's not running anymore
					cluster.monitoringChannels[daemon].monitoringRunning = false
				} else {
					logger.Debugf("ceph %s health go routine is already running for cluster %q", daemon, cluster.Namespace)
				}
			} else {
				// if not already running and not disabled, we run it
				if !isDisabled {
					// Run the go routine
					c.startMonitoringCheck(cluster, clusterInfo, daemon)

					// Set the flag to indicate monitoring is running
					cluster.monitoringChannels[daemon].monitoringRunning = true
				}
			}
		} else {
			// If the mon does not exist in the map, this is a first deployment or an operator restart
			// So we check the desired state from the CR and run it if necessary
			//
			// If the mon monitoring is enabled
			if !isDisabled {
				cluster.monitoringChannels[daemon] = &clusterHealth{
					stopChan:          make(chan struct{}),
					monitoringRunning: true, // Set the flag to indicate monitoring is running
				}

				// Run the go routine
				c.startMonitoringCheck(cluster, clusterInfo, daemon)
			}
		}
	}

	// Start watchers
	if cluster.watchersActivated == true {
		logger.Debugf("cluster is already being watched by bucket and client provisioner for cluster %q", cluster.Namespace)
		return
	}

	// Start client CRD watcher
	cc := clientcontroller.NewClientController(c.context, cluster.Namespace)
	cc.StartWatch(cluster.stopCh)

	// Start the object bucket provisioner
	bucketProvisioner := bucket.NewProvisioner(c.context, clusterInfo)
	// If cluster is external, pass down the user to the bucket controller

	// note: the error return below is ignored and is expected to be removed from the
	//   bucket library's `NewProvisioner` function
	bucketController, _ := bucket.NewBucketController(c.context.KubeConfig, bucketProvisioner)
	go bucketController.Run(cluster.stopCh)

	// enable the cluster watcher once
	cluster.watchersActivated = true
}

func isMonitoringDisabled(daemon string, clusterSpec *cephv1.ClusterSpec) bool {
	switch daemon {
	case "mon":
		return clusterSpec.HealthCheck.DaemonHealth.Monitor.Disabled

	case "osd":
		return clusterSpec.HealthCheck.DaemonHealth.ObjectStorageDaemon.Disabled

	case "status":
		return clusterSpec.HealthCheck.DaemonHealth.Status.Disabled
	}

	return false
}

func (c *ClusterController) startMonitoringCheck(cluster *cluster, clusterInfo *cephclient.ClusterInfo, daemon string) {
	switch daemon {
	case "mon":
		healthChecker := mon.NewHealthChecker(cluster.mons, cluster.Spec)
		logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
		go healthChecker.Check(cluster.monitoringChannels[daemon].stopChan)

	case "osd":
		if !cluster.Spec.External.Enable {
			c.osdChecker = osd.NewOSDHealthMonitor(c.context, clusterInfo, cluster.Spec.RemoveOSDsIfOutAndSafeToRemove, cluster.Spec.HealthCheck)
			logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
			go c.osdChecker.Start(cluster.monitoringChannels[daemon].stopChan)
		}

	case "status":
		cephChecker := newCephStatusChecker(c.context, clusterInfo, cluster.Spec.HealthCheck)
		logger.Infof("enabling ceph %s monitoring goroutine for cluster %q", daemon, cluster.Namespace)
		go cephChecker.checkCephStatus(cluster.monitoringChannels[daemon].stopChan)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package storage

import (
	"errors"
	"testing"

	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

type Ignored struct {
	ID string
}

type IgnoredList struct {
	Items []Ignored
}

func (obj *Ignored) GetObjectKind() schema.ObjectKind     { return schema.EmptyObjectKind }
func (obj *IgnoredList) GetObjectKind() schema.ObjectKind { return schema.EmptyObjectKind }
func (obj *Ignored) DeepCopyObject() runtime.Object {
	panic("Ignored does not support DeepCopy")
}
func (obj *IgnoredList) DeepCopyObject() runtime.Object {
	panic("IgnoredList does not support DeepCopy")
}

func TestSelectionPredicate(t *testing.T) {
	table := map[string]struct {
		labelSelector, fieldSelector string
		labels                       labels.Set
		fields                       fields.Set
		err                          error
		shouldMatch                  bool
		matchSingleKey               string
	}{
		"A": {
			labelSelector: "name=foo",
			fieldSelector: "uid=12345",
			labels:        labels.Set{"name": "foo"},
			fields:        fields.Set{"uid": "12345"},
			shouldMatch:   true,
		},
		"B": {
			labelSelector: "name=foo",
			fieldSelector: "uid=12345",
			labels:        labels.Set{"name": "foo"},
			fields:        fields.Set{},
			shouldMatch:   false,
		},
		"C": {
			labelSelector: "name=foo",
			fieldSelector: "uid=12345",
			labels:        labels.Set{},
			fields:        fields.Set{"uid": "12345"},
			shouldMatch:   false,
		},
		"D": {
			fieldSelector:  "metadata.name=12345",
			labels:         labels.Set{},
			fields:         fields.Set{"metadata.name": "12345"},
			shouldMatch:    true,
			matchSingleKey: "12345",
		},
		"error": {
			labelSelector: "name=foo",
			fieldSelector: "uid=12345",
			err:           errors.New("maybe this is a 'wrong object type' error"),
			shouldMatch:   false,
		},
	}

	for name, item := range table {
		parsedLabel, err := labels.Parse(item.labelSelector)
		if err != nil {
			panic(err)
		}
		parsedField, err := fields.ParseSelector(item.fieldSelector)
		if err != nil {
			panic(err)
		}
		sp := &SelectionPredicate{
			Label: parsedLabel,
			Field: parsedField,
			GetAttrs: func(runtime.Object) (label labels.Set, field fields.Set, err error) {
				return item.labels, item.fields, item.err
			},
		}
		got, err := sp.Matches(&Ignored{})
		if e, a := item.err, err; e != a {
			t.Errorf("%v: expected %v, got %v", name, e, a)
			continue
		}
		if e, a := item.shouldMatch, got; e != a {
			t.Errorf("%v: expected %v, got %v", name, e, a)
		}
		if key := item.matchSingleKey; key != "" {
			got, ok := sp.MatchesSingle()
			if !ok {
				t.Errorf("%v: expected single match", name)
			}
			if e, a := key, got; e != a {
				t.Errorf("%v: expected %v, got %v", name, e, a)
			}
		}
	}
}
<commit_msg>Add more test code for SelectionPredicate<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package storage

import (
	"errors"
	"reflect"
	"testing"

	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

type Ignored struct {
	ID string
}

type IgnoredList struct {
	Items []Ignored
}

func (obj *Ignored) GetObjectKind() schema.ObjectKind     { return schema.EmptyObjectKind }
func (obj *IgnoredList) GetObjectKind() schema.ObjectKind { return schema.EmptyObjectKind }
func (obj *Ignored) DeepCopyObject() runtime.Object {
	panic("Ignored does not support DeepCopy")
}
func (obj *IgnoredList) DeepCopyObject() runtime.Object {
	panic("IgnoredList does not support DeepCopy")
}

func TestSelectionPredicate(t *testing.T) {
	table := map[string]struct {
		labelSelector, fieldSelector string
		labels                       labels.Set
		fields                       fields.Set
		err                          error
		shouldMatch                  bool
		matchSingleKey               string
	}{
		"A": {
			labelSelector: "name=foo",
			fieldSelector: "uid=12345",
			labels:        labels.Set{"name": "foo"},
			fields:        fields.Set{"uid": "12345"},
			shouldMatch:   true,
		},
		"B": {
			labelSelector: "name=foo",
			fieldSelector: "uid=12345",
			labels:        labels.Set{"name": "foo"},
			fields:        fields.Set{},
			shouldMatch:   false,
		},
		"C": {
			labelSelector: "name=foo",
			fieldSelector: "uid=12345",
			labels:        labels.Set{},
			fields:        fields.Set{"uid": "12345"},
			shouldMatch:   false,
		},
		"D": {
			fieldSelector:  "metadata.name=12345",
			labels:         labels.Set{},
			fields:         fields.Set{"metadata.name": "12345"},
			shouldMatch:    true,
			matchSingleKey: "12345",
		},
		"error": {
			labelSelector: "name=foo",
			fieldSelector: "uid=12345",
			err:           errors.New("maybe this is a 'wrong object type' error"),
			shouldMatch:   false,
		},
	}

	for name, item := range table {
		parsedLabel, err := labels.Parse(item.labelSelector)
		if err != nil {
			panic(err)
		}
		parsedField, err := fields.ParseSelector(item.fieldSelector)
		if err != nil {
			panic(err)
		}
		sp := &SelectionPredicate{
			Label: parsedLabel,
			Field: parsedField,
			GetAttrs: func(runtime.Object) (label labels.Set, field fields.Set, err error) {
				return item.labels, item.fields, item.err
			},
		}
		got, err := sp.Matches(&Ignored{})
		if e, a := item.err, err; e != a {
			t.Errorf("%v: expected %v, got %v", name, e, a)
			continue
		}
		if e, a := item.shouldMatch, got; e != a {
			t.Errorf("%v: expected %v, got %v", name, e, a)
		}
		got = sp.MatchesObjectAttributes(item.labels, item.fields)
		if e, a := item.shouldMatch, got; e != a {
			t.Errorf("%v: expected %v, got %v", name, e, a)
		}
		if key := item.matchSingleKey; key != "" {
			got, ok := sp.MatchesSingle()
			if !ok {
				t.Errorf("%v: expected single match", name)
			}
			if e, a := key, got; e != a {
				t.Errorf("%v: expected %v, got %v", name, e, a)
			}
		}
	}
}

func TestSelectionPredicateMatcherIndex(t *testing.T) {
	testCases := map[string]struct {
		labelSelector, fieldSelector string
		indexLabels                  []string
		indexFields                  []string
		expected                     []MatchValue
	}{
		"Match nil": {
			labelSelector: "name=foo",
			fieldSelector: "uid=12345",
			indexLabels:   []string{"bar"},
			indexFields:   []string{},
			expected:      nil,
		},
		"Match field": {
			labelSelector: "name=foo",
			fieldSelector: "uid=12345",
			indexLabels:   []string{},
			indexFields:   []string{"uid"},
			expected:      []MatchValue{{IndexName: FieldIndex("uid"), Value: "12345"}},
		},
		"Match label": {
			labelSelector: "name=foo",
			fieldSelector: "uid=12345",
			indexLabels:   []string{"name"},
			indexFields:   []string{},
			expected:      []MatchValue{{IndexName: LabelIndex("name"), Value: "foo"}},
		},
		"Match field and label": {
			labelSelector: "name=foo",
			fieldSelector: "uid=12345",
			indexLabels:   []string{"name"},
			indexFields:   []string{"uid"},
			expected:      []MatchValue{{IndexName: FieldIndex("uid"), Value: "12345"}, {IndexName: LabelIndex("name"), Value: "foo"}},
		},
		"Negative match field and label": {
			labelSelector: "name!=foo",
			fieldSelector: "uid!=12345",
			indexLabels:   []string{"name"},
			indexFields:   []string{"uid"},
			expected:      nil,
		},
		"Negative match field and match label": {
			labelSelector: "name=foo",
			fieldSelector: "uid!=12345",
			indexLabels:   []string{"name"},
			indexFields:   []string{"uid"},
			expected:      []MatchValue{{IndexName: LabelIndex("name"), Value: "foo"}},
		},
		"Negative match label and match field": {
			labelSelector: "name!=foo",
			fieldSelector: "uid=12345",
			indexLabels:   []string{"name"},
			indexFields:   []string{"uid"},
			expected:      []MatchValue{{IndexName: FieldIndex("uid"), Value: "12345"}},
		},
	}
	for name, testCase := range testCases {
		parsedLabel, err := labels.Parse(testCase.labelSelector)
		if err != nil {
			panic(err)
		}
		parsedField, err := fields.ParseSelector(testCase.fieldSelector)
		if err != nil {
			panic(err)
		}

		sp := &SelectionPredicate{
			Label:       parsedLabel,
			Field:       parsedField,
			IndexLabels: testCase.indexLabels,
			IndexFields: testCase.indexFields,
		}
		actual := sp.MatcherIndex()
		if !reflect.DeepEqual(testCase.expected, actual) {
			t.Errorf("%v: expected %v, got %v", name, testCase.expected, actual)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package plugin is a Nvim remote plugin host.
package plugin

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/neovim/go-client/nvim"
)

// Main implements the main function for a Nvim remote plugin.
//
// Plugin applications call the Main function to run the plugin. The Main
// function creates a Nvim client, calls the supplied function to register
// handlers with the plugin and then runs the server loop to handle requests
// from Nvim.
//
// Plugin applications should use the default logger in the standard log
// package for logging. If the environment variable NVIM_GO_LOG_FILE is set,
// then the default logger is configured to append to the file specified by the
// environment variable.
//
// Run the plugin application with the command line option --manifest=hostName
// to print the plugin manifest to stdout. Add the manifest manually to a
// Vimscript file. The :UpdateRemotePlugins command is not supported at this
// time.
//
// If the --manifest=host command line flag is specified, then Main prints the
// plugin manifest to stdout insead of running the application as a plugin.
func Main(registerHandlers func(p *Plugin) error) {
	pluginHost := flag.String("manifest", "", "Write plugin manifest for `host` to stdout")
	flag.Parse()

	if *pluginHost != "" {
		log.SetFlags(0)
		p := New(nil)
		if err := registerHandlers(p); err != nil {
			log.Fatal(err)
		}
		os.Stdout.Write(p.Manifest(*pluginHost))
		return
	}

	stdout := os.Stdout
	if fname := os.Getenv("NVIM_GO_LOG_FILE"); fname != "" {
		f, err := os.OpenFile(fname, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		os.Stdout = f
		os.Stderr = f
		log.SetOutput(f)
		log.SetPrefix(fmt.Sprintf("%8d ", os.Getpid()))
		log.Print("Plugin Start")
		defer log.Print("Plugin Exit")
	} else {
		log.SetFlags(0)
		os.Stdout = os.Stderr
	}
	v, err := nvim.New(os.Stdin, stdout, stdout, log.Printf)
	if err != nil {
		log.Fatal(err)
	}
	p := New(v)
	if err := registerHandlers(p); err != nil {
		log.Fatal(err)
	}
	if err := v.Serve(); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Remove log file code from nvim/plugin<commit_after>// Copyright 2016 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package plugin is a Nvim remote plugin host.
package plugin

import (
	"flag"
	"log"
	"os"

	"github.com/neovim/go-client/nvim"
)

// Main implements the main function for a Nvim remote plugin.
//
// Plugin applications call the Main function to run the plugin. The Main
// function creates a Nvim client, calls the supplied function to register
// handlers with the plugin and then runs the server loop to handle requests
// from Nvim.
//
// Applications should use the default logger in the standard log package to
// write to Nvim's log.
//
// Run the plugin application with the command line option --manifest=hostName
// to print the plugin manifest to stdout. Add the manifest manually to a
// Vimscript file. The :UpdateRemotePlugins command is not supported at this
// time.
//
// If the --manifest=host command line flag is specified, then Main prints the
// plugin manifest to stdout insead of running the application as a plugin.
func Main(registerHandlers func(p *Plugin) error) {
	pluginHost := flag.String("manifest", "", "Write plugin manifest for `host` to stdout")
	flag.Parse()

	if *pluginHost != "" {
		log.SetFlags(0)
		p := New(nil)
		if err := registerHandlers(p); err != nil {
			log.Fatal(err)
		}
		os.Stdout.Write(p.Manifest(*pluginHost))
		return
	}

	stdout := os.Stdout
	os.Stdout = os.Stderr
	log.SetFlags(0)

	v, err := nvim.New(os.Stdin, stdout, stdout, log.Printf)
	if err != nil {
		log.Fatal(err)
	}
	p := New(v)
	if err := registerHandlers(p); err != nil {
		log.Fatal(err)
	}
	if err := v.Serve(); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"os"
)

func main() {
	http.HandleFunc("/", handler)
	fmt.Println("listening...")
	err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
	if err != nil {
		panic(err)
	}
}

func handler(w http.ResponseWriter, r *http.Request) {
	// Read the Request Parameter "command"
	command := r.FormValue("command")

	if command == "/9gag" {
		fmt.Fprint(w, "Hello World")
	} else {
		fmt.Fprint(w, "I do not understand your command.")
	}
}<commit_msg>Return json response instead of text.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
)

func main() {
	http.HandleFunc("/", handler)
	fmt.Println("listening...")
	err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
	if err != nil {
		panic(err)
	}
}

func handler(w http.ResponseWriter, r *http.Request) {
	// Read the Request Parameter "command"
	command := r.FormValue("command")

	if command == "/9gag" {
		jsonResp(w, "Hello World")
	} else {
		fmt.Fprint(w, "I do not understand your command.")
	}
}

func jsonResp(w http.ResponseWriter, msg string) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	resp := map[string]string{"text": msg}
	r, err := json.Marshal(resp)
	if err != nil {
		fmt.Println("Couldn't marshal hook response:", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	w.Write(r)
}
<|endoftext|>


<commit_before>package oauth2

import (
	"encoding/json"
	"net/http"

	"golang.org/x/oauth2"
	"gopkg.in/authboss.v0"
)

const (
	googleInfoEndpoint   = `https://www.googleapis.com/userinfo/v2/me`
	facebookInfoEndpoint = `https://graph.facebook.com/me?fields=name,email`
)

type googleMeResponse struct {
	ID    string `json:"id"`
	Email string `json:"email"`
}

// testing
var clientGet = (*http.Client).Get

// Google is a callback appropriate for use with Google's OAuth2 configuration.
func Google(cfg oauth2.Config, token *oauth2.Token) (authboss.Attributes, error) {
	client := cfg.Client(oauth2.NoContext, token)
	resp, err := clientGet(client, googleInfoEndpoint)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	dec := json.NewDecoder(resp.Body)
	var jsonResp googleMeResponse
	if err = dec.Decode(&jsonResp); err != nil {
		return nil, err
	}

	return authboss.Attributes{
		authboss.StoreOAuth2UID: jsonResp.ID,
		authboss.StoreEmail:     jsonResp.Email,
	}, nil
}

type facebookMeResponse struct {
	ID    string `json:"id"`
	Email string `json:"email"`
	Name  string `json:"name"`
}

// Facebook is a callback appropriate for use with Facebook's OAuth2 configuration.
func Facebook(cfg oauth2.Config, token *oauth2.Token) (authboss.Attributes, error) {
	client := cfg.Client(oauth2.NoContext, token)
	resp, err := clientGet(client, facebookInfoEndpoint)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	dec := json.NewDecoder(resp.Body)
	var jsonResp facebookMeResponse
	if err = dec.Decode(&jsonResp); err != nil {
		return nil, err
	}

	return authboss.Attributes{
		"name":                  jsonResp.Name,
		authboss.StoreOAuth2UID: jsonResp.ID,
		authboss.StoreEmail:     jsonResp.Email,
	}, nil
}
<commit_msg>add Facebook provider for oauth2<commit_after>package oauth2

import (
	"encoding/json"
	"net/http"

	"golang.org/x/oauth2"
	"gopkg.in/authboss.v0"
)

const (
	googleInfoEndpoint   = `https://www.googleapis.com/userinfo/v2/me`
	facebookInfoEndpoint = `https://graph.facebook.com/me?fields=name,email`
)

type googleMeResponse struct {
	ID    string `json:"id"`
	Email string `json:"email"`
}

// testing
var clientGet = (*http.Client).Get

// Google is a callback appropriate for use with Google's OAuth2 configuration.
func Google(cfg oauth2.Config, token *oauth2.Token) (authboss.Attributes, error) {
	client := cfg.Client(oauth2.NoContext, token)
	resp, err := clientGet(client, googleInfoEndpoint)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	dec := json.NewDecoder(resp.Body)
	var jsonResp googleMeResponse
	if err = dec.Decode(&jsonResp); err != nil {
		return nil, err
	}

	return authboss.Attributes{
		authboss.StoreOAuth2UID: jsonResp.ID,
		authboss.StoreEmail:     jsonResp.Email,
	}, nil
}

type facebookMeResponse struct {
	ID    string `json:"id"`
	Email string `json:"email"`
	Name  string `json:"name"`
}

// Google is a callback appropriate for use with Google's OAuth2 configuration.
func Facebook(cfg oauth2.Config, token *oauth2.Token) (authboss.Attributes, error) {
	client := cfg.Client(oauth2.NoContext, token)
	resp, err := clientGet(client, facebookInfoEndpoint)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	dec := json.NewDecoder(resp.Body)
	var jsonResp facebookMeResponse
	if err = dec.Decode(&jsonResp); err != nil {
		return nil, err
	}

	return authboss.Attributes{
		"name":                  jsonResp.Name,
		authboss.StoreOAuth2UID: jsonResp.ID,
		authboss.StoreEmail:     jsonResp.Email,
	}, nil
}
<|endoftext|>


<commit_before>// Obdi - a REST interface and GUI for deploying software
// Copyright (C) 2014  Mark Clarkson
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"syscall"
	"time"
	//"encoding/json"
	//"strings"
)

func (api *Api) execCmd(job JobIn) {

	// TODO :: Put this logic in login/logout and reference count
	//defer api.Logout( )

	// Need to set the PATH to run the script from the script dir
	os.Setenv("PATH", config.ScriptDir)

	scriptfile := ""

	// Write ScriptSource to disk
	if file, err := ioutil.TempFile(os.TempDir(), "smworker_"); err != nil {
		if err := api.sendStatus(job, JobOut{
			Status:        STATUS_SYSCANCELLED,
			StatusReason:  fmt.Sprintf("TempFile error ('%s')", err.Error()),
			StatusPercent: 0,
			Errors:        0,
		}); err != nil {
			logit(fmt.Sprintf("Error: %s", err.Error()))
		}
		return
	} else {
		if _, err := file.Write(job.ScriptSource); err != nil {
			if err := api.sendStatus(job, JobOut{
				Status:        STATUS_SYSCANCELLED,
				StatusReason:  fmt.Sprintf("Write error ('%s')", err.Error()),
				StatusPercent: 0,
				Errors:        0,
			}); err != nil {
				logit(fmt.Sprintf("Error: %s", err.Error()))
			}
			return
		}
		file.Close()
		os.Chmod(file.Name(), 0755)
		scriptfile = file.Name()
	}
	defer os.Remove(scriptfile)

	// Set up command, split on spaces but preserve quoted strings
	head := scriptfile
	r := regexp.MustCompile("'.+'|\".+\"|\\S+")
	parts := r.FindAllString(job.Args, -1)
	cmd := &exec.Cmd{}
	cmd = exec.Command(head, parts...)

	// Apply the sent environment variables, split on spaces
	// but preserve quoted strings
	if len(job.EnvVars) > 0 {
		r = regexp.MustCompile("[^ ]*='[^']+'|[^ ]*=\"[^']+\"|\\S+")
		cmd.Env = r.FindAllString(job.EnvVars, -1)
		// Remove speech marks around the value of quoted strings. Matches, for
		// example, `var="val val"`, and changes to `var=val val`
		for i, j := range cmd.Env {
			r := regexp.MustCompile(`([^ ]+=)"(.*)"`)
			if r.Match([]byte(j)) {
				k := r.ReplaceAll([]byte(j), []byte(`$1$2`))
				cmd.Env[i] = string(k)
			}
		}
	} else {
		cmd.Env = []string{}
	}

	// Add the system scripts directory to Env.SYSSCRIPTDIR
	cmd.Env = append(cmd.Env, "SYSSCRIPTDIR="+config.SysScriptDir)

	cmd.Dir = os.TempDir()

	// Set up buffer for stdout
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		if err := api.sendStatus(job, JobOut{
			Status:        STATUS_SYSCANCELLED,
			StatusReason:  fmt.Sprintf("Pipe error ('%s')", err.Error()),
			StatusPercent: 0,
			Errors:        0,
		}); err != nil {
			logit(fmt.Sprintf("Error: %s", err.Error()))
		}
		return
	}
	rdr := bufio.NewReader(stdout)
	stderr, err := cmd.StderrPipe()
	if err != nil {
		if err := api.sendStatus(job, JobOut{
			Status:        STATUS_SYSCANCELLED,
			StatusReason:  fmt.Sprintf("Pipe error ('%s')", err.Error()),
			StatusPercent: 0,
			Errors:        0,
		}); err != nil {
			logit(fmt.Sprintf("Error: %s", err.Error()))
		}
		return
	}
	rdr_stderr := bufio.NewReader(stderr)

	/*
	   data := JobOut{}
	   jsondata, err := json.Marshal(data)
	*/

	// Get child processes to run in a process group so they
	// can all be killed as a group.
	cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}

	// Run command in the background (fork)
	err = cmd.Start()
	if err != nil {
		if err := api.sendStatus(job, JobOut{
			Status:        STATUS_SYSCANCELLED,
			StatusReason:  fmt.Sprintf("Fork error ('%s')", err.Error()),
			StatusPercent: 0,
			Errors:        0,
		}); err != nil {
			logit(fmt.Sprintf("Error: %s", err.Error()))
		}
		return
	}

	if err := api.sendStatus(job, JobOut{
		Status:        STATUS_INPROGRESS,
		StatusReason:  "Script started",
		StatusPercent: 0,
		Errors:        0,
	}); err != nil {
		logit(fmt.Sprintf("Error: %s", err.Error()))
	}

	// Save the pid so it can be killed
	api.SetPid(job.JobID, int64(cmd.Process.Pid))

	// Process the output
	serial := int64(1)
	line := ""
	if job.Type != 2 {
		// A user job (the default, should be type=1)
		line, err = rdr.ReadString('\n')
		api.sendOutputLine(job, line, serial)
		for err == nil {
			serial++
			a := ""
			starttime := time.Now()
			// Up to 50 lines can be read in one hit
			for i := 0; i < 50; i++ {
				if time.Duration(time.Since(starttime)) > time.Second {
					break
				}
				line, err = rdr.ReadString('\n')
				a = a + line
			}
			api.sendOutputLine(job, a, serial)
		}
	} else {
		// A system job. Send all output in a single output line
		a := ""
		for err == nil {
			line, err = rdr.ReadString('\n')
			a = a + line
		}
		api.sendOutputLine(job, a, 1)
	}

	serial++

	// Read anything in stderr, but don't send it.
	// It will be sent later if the script has non-zero exit status
	error_output := ""
	err = nil
	for err == nil {
		line, err = rdr_stderr.ReadString('\n')
		error_output = error_output + line
	}

	// Process exit status
	err = cmd.Wait()
	if err != nil {
		api.sendOutputLine(job, error_output, serial)
		status := int64(0)
		if api.UserCancel(job.JobID) == true {
			status = STATUS_USERCANCELLED
		} else {
			status = STATUS_ERROR
		}
		if err := api.sendStatus(job, JobOut{
			Status: status,
			StatusReason: fmt.Sprintf("Script, '%s', exited with error status ('%s')",
				job.ScriptName, err.Error()),
			StatusPercent: 0,
			Errors:        0,
		}); err != nil {
			logit(fmt.Sprintf("Error: (Script: '%s') %s", job.ScriptName,
				err.Error()))
		}
		api.RemoveJob(job.JobID)
		return
	}

	status := cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
	if status == 0 {
		if err := api.sendStatus(job, JobOut{
			Status:        STATUS_OK,
			StatusReason:  "Script finished successfully",
			StatusPercent: 100,
			Errors:        0,
		}); err != nil {
			logit(fmt.Sprintf("Error: %s", err.Error()))
		}
	} else {
		if err := api.sendStatus(job, JobOut{
			Status:        STATUS_ERROR,
			StatusReason:  "Non-zero exit status. Check the log.",
			StatusPercent: 100,
			Errors:        0,
		}); err != nil {
			logit(fmt.Sprintf("Error: %s", err.Error()))
		}
	}

	api.RemoveJob(int64(job.JobID))

	// logout
}

/*
func main() {
    exec_cmd ( os.Args[1:]... )
}
*/
<commit_msg>Remove outer quotes from env vars<commit_after>// Obdi - a REST interface and GUI for deploying software
// Copyright (C) 2014  Mark Clarkson
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"syscall"
	"time"
	//"encoding/json"
	//"strings"
)

func (api *Api) execCmd(job JobIn) {

	// TODO :: Put this logic in login/logout and reference count
	//defer api.Logout( )

	// Need to set the PATH to run the script from the script dir
	os.Setenv("PATH", config.ScriptDir)

	scriptfile := ""

	// Write ScriptSource to disk
	if file, err := ioutil.TempFile(os.TempDir(), "smworker_"); err != nil {
		if err := api.sendStatus(job, JobOut{
			Status:        STATUS_SYSCANCELLED,
			StatusReason:  fmt.Sprintf("TempFile error ('%s')", err.Error()),
			StatusPercent: 0,
			Errors:        0,
		}); err != nil {
			logit(fmt.Sprintf("Error: %s", err.Error()))
		}
		return
	} else {
		if _, err := file.Write(job.ScriptSource); err != nil {
			if err := api.sendStatus(job, JobOut{
				Status:        STATUS_SYSCANCELLED,
				StatusReason:  fmt.Sprintf("Write error ('%s')", err.Error()),
				StatusPercent: 0,
				Errors:        0,
			}); err != nil {
				logit(fmt.Sprintf("Error: %s", err.Error()))
			}
			return
		}
		file.Close()
		os.Chmod(file.Name(), 0755)
		scriptfile = file.Name()
	}
	defer os.Remove(scriptfile)

	// Set up command, split on spaces but preserve quoted strings
	head := scriptfile
	r := regexp.MustCompile("'.+'|\".+\"|\\S+")
	parts := r.FindAllString(job.Args, -1)
	cmd := &exec.Cmd{}
	cmd = exec.Command(head, parts...)

	// Apply the sent environment variables, split on spaces
	// but preserve quoted strings
	if len(job.EnvVars) > 0 {
		r = regexp.MustCompile("[^ ]*='[^']+'|[^ ]*=\"[^']+\"|\\S+")
		cmd.Env = r.FindAllString(job.EnvVars, -1)
		// Remove speech marks around the value of quoted strings. Matches, for
		// example, `var="val val"`, and changes to `var=val val`
		for i, j := range cmd.Env {
			r := regexp.MustCompile(`([^ ]+=)["'](.*)["']`)
			if r.Match([]byte(j)) {
				k := r.ReplaceAll([]byte(j), []byte(`$1$2`))
				cmd.Env[i] = string(k)
			}
		}
	} else {
		cmd.Env = []string{}
	}

	// Add the system scripts directory to Env.SYSSCRIPTDIR
	cmd.Env = append(cmd.Env, "SYSSCRIPTDIR="+config.SysScriptDir)

	cmd.Dir = os.TempDir()

	// Set up buffer for stdout
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		if err := api.sendStatus(job, JobOut{
			Status:        STATUS_SYSCANCELLED,
			StatusReason:  fmt.Sprintf("Pipe error ('%s')", err.Error()),
			StatusPercent: 0,
			Errors:        0,
		}); err != nil {
			logit(fmt.Sprintf("Error: %s", err.Error()))
		}
		return
	}
	rdr := bufio.NewReader(stdout)
	stderr, err := cmd.StderrPipe()
	if err != nil {
		if err := api.sendStatus(job, JobOut{
			Status:        STATUS_SYSCANCELLED,
			StatusReason:  fmt.Sprintf("Pipe error ('%s')", err.Error()),
			StatusPercent: 0,
			Errors:        0,
		}); err != nil {
			logit(fmt.Sprintf("Error: %s", err.Error()))
		}
		return
	}
	rdr_stderr := bufio.NewReader(stderr)

	/*
	   data := JobOut{}
	   jsondata, err := json.Marshal(data)
	*/

	// Get child processes to run in a process group so they
	// can all be killed as a group.
	cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}

	// Run command in the background (fork)
	err = cmd.Start()
	if err != nil {
		if err := api.sendStatus(job, JobOut{
			Status:        STATUS_SYSCANCELLED,
			StatusReason:  fmt.Sprintf("Fork error ('%s')", err.Error()),
			StatusPercent: 0,
			Errors:        0,
		}); err != nil {
			logit(fmt.Sprintf("Error: %s", err.Error()))
		}
		return
	}

	if err := api.sendStatus(job, JobOut{
		Status:        STATUS_INPROGRESS,
		StatusReason:  "Script started",
		StatusPercent: 0,
		Errors:        0,
	}); err != nil {
		logit(fmt.Sprintf("Error: %s", err.Error()))
	}

	// Save the pid so it can be killed
	api.SetPid(job.JobID, int64(cmd.Process.Pid))

	// Process the output
	serial := int64(1)
	line := ""
	if job.Type != 2 {
		// A user job (the default, should be type=1)
		line, err = rdr.ReadString('\n')
		api.sendOutputLine(job, line, serial)
		for err == nil {
			serial++
			a := ""
			starttime := time.Now()
			// Up to 50 lines can be read in one hit
			for i := 0; i < 50; i++ {
				if time.Duration(time.Since(starttime)) > time.Second {
					break
				}
				line, err = rdr.ReadString('\n')
				a = a + line
			}
			api.sendOutputLine(job, a, serial)
		}
	} else {
		// A system job. Send all output in a single output line
		a := ""
		for err == nil {
			line, err = rdr.ReadString('\n')
			a = a + line
		}
		api.sendOutputLine(job, a, 1)
	}

	serial++

	// Read anything in stderr, but don't send it.
	// It will be sent later if the script has non-zero exit status
	error_output := ""
	err = nil
	for err == nil {
		line, err = rdr_stderr.ReadString('\n')
		error_output = error_output + line
	}

	// Process exit status
	err = cmd.Wait()
	if err != nil {
		api.sendOutputLine(job, error_output, serial)
		status := int64(0)
		if api.UserCancel(job.JobID) == true {
			status = STATUS_USERCANCELLED
		} else {
			status = STATUS_ERROR
		}
		if err := api.sendStatus(job, JobOut{
			Status: status,
			StatusReason: fmt.Sprintf("Script, '%s', exited with error status ('%s')",
				job.ScriptName, err.Error()),
			StatusPercent: 0,
			Errors:        0,
		}); err != nil {
			logit(fmt.Sprintf("Error: (Script: '%s') %s", job.ScriptName,
				err.Error()))
		}
		api.RemoveJob(job.JobID)
		return
	}

	status := cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
	if status == 0 {
		if err := api.sendStatus(job, JobOut{
			Status:        STATUS_OK,
			StatusReason:  "Script finished successfully",
			StatusPercent: 100,
			Errors:        0,
		}); err != nil {
			logit(fmt.Sprintf("Error: %s", err.Error()))
		}
	} else {
		if err := api.sendStatus(job, JobOut{
			Status:        STATUS_ERROR,
			StatusReason:  "Non-zero exit status. Check the log.",
			StatusPercent: 100,
			Errors:        0,
		}); err != nil {
			logit(fmt.Sprintf("Error: %s", err.Error()))
		}
	}

	api.RemoveJob(int64(job.JobID))

	// logout
}

/*
func main() {
    exec_cmd ( os.Args[1:]... )
}
*/
<|endoftext|>