<commit_before>package models

import (
	"time"
	"github.com/lifei6671/godoc/conf"
	"github.com/astaxie/beego/orm"
)

type CommentVote struct {
	VoteId int			`orm:"column(vote_id);pk;auto;unique" json:"vote_id"`
	CommentId int			`orm:"column(comment_id);type(int);index" json:"comment_id"`
	CommentMemberId int 		`orm:"column(comment_member_id);type(int);index;default(0)" json:"comment_member_id"`
	VoteMemberId int		`orm:"column(vote_member_id);type(int);index" json:"vote_member_id"`
	VoteState int 			`orm:"column(vote_state);type(int)" json:"vote_state"`
	CreateTime time.Time		`orm:"column(create_time);type(datetime);auto_now_add" json:"create_time"`
}

// TableName 获取对应数据库表名.
func (m *CommentVote) TableName() string {
	return "comments"
}
// TableEngine 获取数据使用的引擎.
func (m *CommentVote) TableEngine() string {
	return "INNODB"
}

func (m *CommentVote) TableNameWithPrefix() string  {
	return conf.GetDatabasePrefix() + m.TableName()
}
func (u *CommentVote) TableUnique() [][]string {
	return [][]string{
		[]string{"comment_id", "vote_member_id"},
	}
}
func NewCommentVote() *CommentVote {
	return &CommentVote{}
}
func (m *CommentVote) InsertOrUpdate() (*CommentVote,error)  {
	o := orm.NewOrm()

	if m.VoteId > 0 {
		_,err := o.Update(m)
		return m,err
	}else{
		_,err := o.Insert(m)

		return m,err
	}
}<commit_msg>修改BUG<commit_after>package models

import (
	"time"
	"github.com/lifei6671/godoc/conf"
	"github.com/astaxie/beego/orm"
)

type CommentVote struct {
	VoteId int			`orm:"column(vote_id);pk;auto;unique" json:"vote_id"`
	CommentId int			`orm:"column(comment_id);type(int);index" json:"comment_id"`
	CommentMemberId int 		`orm:"column(comment_member_id);type(int);index;default(0)" json:"comment_member_id"`
	VoteMemberId int		`orm:"column(vote_member_id);type(int);index" json:"vote_member_id"`
	VoteState int 			`orm:"column(vote_state);type(int)" json:"vote_state"`
	CreateTime time.Time		`orm:"column(create_time);type(datetime);auto_now_add" json:"create_time"`
}

// TableName 获取对应数据库表名.
func (m *CommentVote) TableName() string {
	return "comment_votes"
}
// TableEngine 获取数据使用的引擎.
func (m *CommentVote) TableEngine() string {
	return "INNODB"
}

func (m *CommentVote) TableNameWithPrefix() string  {
	return conf.GetDatabasePrefix() + m.TableName()
}
func (u *CommentVote) TableUnique() [][]string {
	return [][]string{
		[]string{"comment_id", "vote_member_id"},
	}
}
func NewCommentVote() *CommentVote {
	return &CommentVote{}
}
func (m *CommentVote) InsertOrUpdate() (*CommentVote,error)  {
	o := orm.NewOrm()

	if m.VoteId > 0 {
		_,err := o.Update(m)
		return m,err
	}else{
		_,err := o.Insert(m)

		return m,err
	}
}<|endoftext|>


<commit_before>package models

import (
	"encoding/json"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestMarshaling(t *testing.T) {
	f := &Feature{
		Name:        "test",
		Value:       true,
		FeatureType: Boolean,
		Comment:     "testing",
	}

	ff := &Feature{}

	js, _ := json.Marshal(f)
	json.Unmarshal(js, &ff)

	assert.EqualValues(t, f, ff)
}

func TestTypes(t *testing.T) {
	pf := PercentileFeature("foo", 0.1, "testing")
	assert.Equal(t, Percentile, pf.FeatureType)
	assert.Equal(t, 0.1, pf.FloatValue())

	bf := BooleanFeature("foo", false, "testing")
	assert.Equal(t, Boolean, bf.FeatureType)
	assert.Equal(t, false, bf.BoolValue())

	sf := ScalarFeature("foo", 0.5, "testing")
	assert.Equal(t, Scalar, sf.FeatureType)
	assert.Equal(t, 0.5, sf.FloatValue())
}
<commit_msg>updating tests<commit_after>package models

import (
	"encoding/json"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestMarshaling(t *testing.T) {
	f := &Feature{
		Name:        "test",
		Value:       true,
		FeatureType: Boolean,
		Comment:     "testing",
	}

	ff := &Feature{}

	js, _ := json.Marshal(f)
	json.Unmarshal(js, &ff)

	assert.EqualValues(t, f, ff)
}

func TestTypes(t *testing.T) {
	pf := PercentileFeature("foo", 0.1, "testing", "me")
	assert.Equal(t, Percentile, pf.FeatureType)
	assert.Equal(t, 0.1, pf.FloatValue())

	bf := BooleanFeature("foo", false, "testing", "me")
	assert.Equal(t, Boolean, bf.FeatureType)
	assert.Equal(t, false, bf.BoolValue())
}
<|endoftext|>


<commit_before>package event

import (
	"encoding/binary"
	"strings"
	"sync"
)

const (
	OK = iota
	WARNING
	CRITICAL
)

//go:generate ffjson $GOFILE
//go:generate msgp $GOFILE

// Event represents a metric as it passes through the pipeline.
// it holds meta data about the metric, as well as methods to trace the event as it is processed
type Event struct {
	Host       string            `json:"host" msg:"host"`
	Service    string            `json:"service" msg:"service"`
	SubService string            `json:"sub_service" msg:"sub_service"`
	Metric     float64           `json:"metric" msg:"metric"`
	Tags       map[string]string `json:"tags" msg:"tags"`
	indexName  string
	wait       sync.WaitGroup
	mut        sync.Mutex
}

func (e *Event) MarshalBinary() ([]byte, error) {
	buff := make([]byte, e.Msgsize()+8)
	tmp, err := e.MarshalJSON()
	if err != nil {
		return nil, err
	}
	copy(buff[8:], tmp)

	binary.PutUvarint(buff[:8], uint64(len(tmp)))
	return buff[:8+len(tmp)], nil
}

func (e *Event) Wait() {
	e.wait.Wait()
}

// WaitDec decrements the event's waitgroup counter
func (e *Event) WaitDec() {
	e.mut.Lock()
	e.wait.Done()
	e.mut.Unlock()
}

// WaitAdd increments ot the event's waitgroup counter
func (e *Event) WaitInc() {
	e.mut.Lock()
	e.wait.Add(1)
	e.mut.Unlock()
}

// Passer provides a method for passing an event down a step in the pipeline
type Passer interface {
	Pass(e *Event)
}

func NewEvent() *Event {
	e := &Event{}
	return e
}

// Get any value on an event as a string
func (e *Event) Get(key string) string {

	// attempt to find the string values of the event
	switch strings.ToLower(key) {
	case "host":
		return e.Host
	case "service":
		return e.Service
	case "sub_service":
		return e.SubService
	}

	// if we make it to this point, assume we are looking for a tag
	if val, ok := e.Tags[key]; ok {
		return val
	}

	return ""
}

func (e *Event) IndexName() string {
	return e.Host + e.Service + e.SubService
}

func Status(code int) string {
	switch code {
	case WARNING:
		return "warning"
	case CRITICAL:
		return "critical"
	default:
		return "ok"
	}
}
<commit_msg>fixed comment<commit_after>package event

import (
	"encoding/binary"
	"strings"
	"sync"
)

const (
	OK = iota
	WARNING
	CRITICAL
)

//go:generate ffjson $GOFILE
//go:generate msgp $GOFILE

// Event represents a metric as it passes through the pipeline.
// It holds meta data about the metric, as well as methods to trace the event as it is processed
type Event struct {
	Host       string            `json:"host" msg:"host"`
	Service    string            `json:"service" msg:"service"`
	SubService string            `json:"sub_service" msg:"sub_service"`
	Metric     float64           `json:"metric" msg:"metric"`
	Tags       map[string]string `json:"tags" msg:"tags"`
	indexName  string
	wait       sync.WaitGroup
	mut        sync.Mutex
}

func (e *Event) MarshalBinary() ([]byte, error) {
	buff := make([]byte, e.Msgsize()+8)
	tmp, err := e.MarshalJSON()
	if err != nil {
		return nil, err
	}
	copy(buff[8:], tmp)

	binary.PutUvarint(buff[:8], uint64(len(tmp)))
	return buff[:8+len(tmp)], nil
}

func (e *Event) Wait() {
	e.wait.Wait()
}

// WaitDec decrements the event's waitgroup counter
func (e *Event) WaitDec() {
	e.mut.Lock()
	e.wait.Done()
	e.mut.Unlock()
}

// WaitAdd increments ot the event's waitgroup counter
func (e *Event) WaitInc() {
	e.mut.Lock()
	e.wait.Add(1)
	e.mut.Unlock()
}

// Passer provides a method for passing an event down a step in the pipeline
type Passer interface {
	Pass(e *Event)
}

func NewEvent() *Event {
	e := &Event{}
	return e
}

// Get any value on an event as a string
func (e *Event) Get(key string) string {

	// attempt to find the string values of the event
	switch strings.ToLower(key) {
	case "host":
		return e.Host
	case "service":
		return e.Service
	case "sub_service":
		return e.SubService
	}

	// if we make it to this point, assume we are looking for a tag
	if val, ok := e.Tags[key]; ok {
		return val
	}

	return ""
}

func (e *Event) IndexName() string {
	return e.Host + e.Service + e.SubService
}

func Status(code int) string {
	switch code {
	case WARNING:
		return "warning"
	case CRITICAL:
		return "critical"
	default:
		return "ok"
	}
}
<|endoftext|>


<commit_before>package util

import (
	"crypto/md5"
	"fmt"
	"io"
	"os"
	"syscall"

	"github.com/golang/glog"
)

// FileInfo describes a configuration file and is returned by fileStat.
type fileInfo struct {
	Uid  uint32
	Gid  uint32
	Mode os.FileMode
	Md5  string
}

// IsFileExist reports whether path exits.
func IsFileExist(fpath string) bool {
	if _, err := os.Stat(fpath); os.IsNotExist(err) {
		return false
	}
	return true
}

// IsSameConfig reports whether src and dest config files are equal.
// Two config files are equal when they have the same file contents and
// Unix permissions. The owner, group, and mode must match.
// It return false in other cases.
func IsSameConfig(src, dest string) (bool, error) {
	if !IsFileExist(dest) {
		return false, nil
	}
	dfi, err := getFileInfo(dest)
	if err != nil {
		return false, err
	}
	sfi, err := getFileInfo(src)
	if err != nil {
		return false, err
	}
	if dfi.Uid != sfi.Uid {
		glog.Infof("%s has UID %d should be %d", dest, dfi.Uid, sfi.Uid)
	}
	if dfi.Gid != sfi.Gid {
		glog.Infof("%s has GID %d should be %d", dest, dfi.Gid, sfi.Gid)
	}
	if dfi.Mode != sfi.Mode {
		glog.Infof("%s has mode %s should be %s", dest, os.FileMode(dfi.Mode), os.FileMode(sfi.Mode))
	}
	if dfi.Md5 != sfi.Md5 {
		glog.Infof("%s has md5sum %s should be %s", dest, dfi.Md5, sfi.Md5)
	}
	if dfi.Uid != sfi.Uid || dfi.Gid != sfi.Gid || dfi.Mode != sfi.Mode || dfi.Md5 != sfi.Md5 {
		return false, nil
	}
	return true, nil
}

// getFileInfo returns a FileInfo describing the named file.
func getFileInfo(name string) (fi fileInfo, err error) {
	if IsFileExist(name) {
		f, err := os.Open(name)
		if err != nil {
			return fi, err
		}
		defer f.Close()
		stats, _ := f.Stat()
		fi.Uid = stats.Sys().(*syscall.Stat_t).Uid
		fi.Gid = stats.Sys().(*syscall.Stat_t).Gid
		fi.Mode = stats.Mode()
		h := md5.New()
		io.Copy(h, f)
		fi.Md5 = fmt.Sprintf("%x", h.Sum(nil))
		return fi, nil
	} else {
		return fi, fmt.Errorf("File not found")
	}
}<commit_msg>be more verbose with file not found errors<commit_after>package util

import (
	"crypto/md5"
	"fmt"
	"io"
	"os"
	"syscall"

	"github.com/golang/glog"
)

// FileInfo describes a configuration file and is returned by fileStat.
type fileInfo struct {
	Uid  uint32
	Gid  uint32
	Mode os.FileMode
	Md5  string
}

// IsFileExist reports whether path exits.
func IsFileExist(fpath string) bool {
	if _, err := os.Stat(fpath); os.IsNotExist(err) {
		return false
	}
	return true
}

// IsSameConfig reports whether src and dest config files are equal.
// Two config files are equal when they have the same file contents and
// Unix permissions. The owner, group, and mode must match.
// It return false in other cases.
func IsSameConfig(src, dest string) (bool, error) {
	if !IsFileExist(dest) {
		return false, nil
	}
	dfi, err := getFileInfo(dest)
	if err != nil {
		return false, err
	}
	sfi, err := getFileInfo(src)
	if err != nil {
		return false, err
	}
	if dfi.Uid != sfi.Uid {
		glog.Infof("%s has UID %d should be %d", dest, dfi.Uid, sfi.Uid)
	}
	if dfi.Gid != sfi.Gid {
		glog.Infof("%s has GID %d should be %d", dest, dfi.Gid, sfi.Gid)
	}
	if dfi.Mode != sfi.Mode {
		glog.Infof("%s has mode %s should be %s", dest, os.FileMode(dfi.Mode), os.FileMode(sfi.Mode))
	}
	if dfi.Md5 != sfi.Md5 {
		glog.Infof("%s has md5sum %s should be %s", dest, dfi.Md5, sfi.Md5)
	}
	if dfi.Uid != sfi.Uid || dfi.Gid != sfi.Gid || dfi.Mode != sfi.Mode || dfi.Md5 != sfi.Md5 {
		return false, nil
	}
	return true, nil
}

// getFileInfo returns a FileInfo describing the named file.
func getFileInfo(name string) (fi fileInfo, err error) {
	if !IsFileExist(name) {
		return fi, fmt.Errorf("%s file not found", name)
	}

	f, err := os.Open(name)
	if err != nil {
		return fi, err
	}
	defer f.Close()

	stats, _ := f.Stat()
	fi.Uid = stats.Sys().(*syscall.Stat_t).Uid
	fi.Gid = stats.Sys().(*syscall.Stat_t).Gid
	fi.Mode = stats.Mode()

	h := md5.New()
	io.Copy(h, f)
	fi.Md5 = fmt.Sprintf("%x", h.Sum(nil))

	return fi, nil
}<|endoftext|>


<commit_before>// Copyright 2013 SteelSeries ApS. All rights reserved.
// No license is given for the use of this source code.

// This package impliments a basic LISP interpretor for embedding in a go program for scripting.
// This file defines functions for device support
package golisp

// (def-struct <name> <field>...)
// (def-field <name> <type> [<count>] [<to json translation> <from json translation>])
// (def-api <struct name> (read ) (write ))

import (
    "errors"
    //    . "github.com/steelseries/golisp"
    "fmt"
    "unsafe"
)

var CurrentDevice *Device
var CurrentStructure *DeviceStructure
var CurrentField *DeviceField

func init() {
    //    InitDeviceBuiltins()
}

func InitDeviceBuiltins() {
    MakePrimitiveFunction("def-device", -1, DefDevice)
    MakePrimitiveFunction("def-struct", -1, DefStruct)
    MakePrimitiveFunction("def-field", -1, DefField)
    MakePrimitiveFunction("range", 2, DefRange)
    MakePrimitiveFunction("values", -1, DefValues)
    MakePrimitiveFunction("deferred-validation", -1, DefDeferredValidation)
    MakePrimitiveFunction("repeat", 1, DefRepeat)
    MakePrimitiveFunction("to-json", 1, DefToJson)
    MakePrimitiveFunction("to-from", 1, DefFromJson)

    MakePrimitiveFunction("def-api", -1, DefApi)
    MakePrimitiveFunction("dump-struct", 1, DumpStructure)
    MakePrimitiveFunction("dump-expanded", 1, DumpExpanded)

    MakePrimitiveFunction("bytes-to-string", 2, BytesToString)
    MakePrimitiveFunction("string-to-bytes", 2, StringToBytes)
}

func DefDevice(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    deviceName := Car(args)
    if TypeOf(deviceName) != SymbolType {
        err = errors.New("Device name must be a symbol")
        return
    }

    if NilP(Cdr(args)) {
        err = errors.New("Device must have at least one structure")
        return
    }

    CurrentDevice := NewDeviceNamed(StringValue(deviceName))

    var structure *Data
    for c := Cdr(args); NotNilP(c); c = Cdr(c) {
        structure, err = Eval(Car(c), env)
        if err != nil {
            return
        }
        if ObjectP(structure) && TypeOfObject(structure) == "DeviceStructure" {
            CurrentDevice.AddStructure((*DeviceStructure)(ObjectValue(structure)))
        } else {
            errors.New("Expected structure declaration or api declaration")
        }
    }
    CurrentStructure = nil
    return env.BindTo(deviceName, ObjectWithTypeAndValue("Device", unsafe.Pointer(CurrentDevice))), nil
}

func DefStruct(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    structName := Car(args)
    if TypeOf(structName) != SymbolType {
        err = errors.New("Struct name must be a symbol")
        return
    }

    if NilP(Cdr(args)) {
        err = errors.New("Struct must have at least one field")
        return
    }

    structure := NewStructNamed(StringValue(structName))
    CurrentStructure = structure
    CurrentField = nil

    var field *Data
    for c := Cdr(args); NotNilP(c); c = Cdr(c) {
        field, err = Eval(Car(c), env)
        if err != nil {
            return
        }
        if ObjectP(field) && TypeOfObject(field) == "DeviceField" {
            CurrentStructure.AddField((*DeviceField)(ObjectValue(field)))
        } else {
            errors.New("Expected DeviceField")
        }
    }
    CurrentStructure = nil
    return env.BindTo(structName, ObjectWithTypeAndValue("DeviceStructure", unsafe.Pointer(structure))), nil
}

func DefField(args *Data, env *SymbolTableFrame) (field *Data, err error) {
    fieldName := Car(args)
    if TypeOf(fieldName) != SymbolType {
        err = errors.New("Field name must be a symbol")
        return
    }

    if NilP(Cdr(args)) {
        err = errors.New("Field must have at least a name and type")
        return
    }

    fieldType := Cadr(args)
    if TypeOf(fieldType) != SymbolType {
        err = errors.New("Field type must be a symbol")
        return
    }

    if !IsValidType(fieldType) {
        err = errors.New("Field type must be uint8, uint16, uint32 or a user defined device structure")
        return
    }

    CurrentField = NewField(StringValue(fieldName), StringValue(fieldType), FieldSizeOf(fieldType))

    for c := Cddr(args); NotNilP(c); c = Cdr(c) {
        field, err = Eval(Car(c), env)
        if err != nil {
            return
        }
    }

    field = ObjectWithTypeAndValue("DeviceField", unsafe.Pointer(CurrentField))
    CurrentField = nil
    return
}

func DefRange(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) != 2 {
        err = errors.New(fmt.Sprintf("range requires 2 arguments, %d found", Length(args)))
        return
    }

    if !NumberP(Car(args)) || !NumberP(Cadr(args)) {
        err = errors.New("range requires numeric arguments")
        return
    }

    CurrentField.ValidRange = &Range{Lo: uint32(NumericValue(Car(args))), Hi: uint32(NumericValue(Cadr(args)))}
    return
}

func DefValues(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) == 0 {
        err = errors.New("values requires at least 1 argument")
        return
    }

    CurrentField.ValidValues = &Values{Vals: make([]uint32, 0, 5)}

    var l *Data
    if PairP(Car(args)) {
        l, err = Eval(Car(args), env)
        if err != nil {
            return
        }
        args = l
    }

    for c := args; NotNilP(c); c = Cdr(c) {
        if !NumberP(Car(c)) {
            err = errors.New("values requires numeric arguments")
            return
        }
        CurrentField.ValidValues.AddValue(uint32(NumericValue(Car(c))))
    }
    return
}

func DefDeferredValidation(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    CurrentField.DeferredValidationCode = Car(args)
    return
}

func DefRepeat(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) != 1 {
        err = errors.New(fmt.Sprintf("repeat requires 1 argument, %d found", Length(args)))
        return
    }

    if !NumberP(Car(args)) {
        err = errors.New("repeat requires a numeric argument")
        return
    }

    CurrentField.RepeatCount = int(NumericValue(Car(args)))
    return
}

func DefToJson(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) != 1 {
        err = errors.New(fmt.Sprintf("to-json requires 1 argument, %d found", Length(args)))
        return
    }

    if !PairP(Car(args)) {
        err = errors.New("to-json requires a list argument")
        return
    }

    CurrentField.ToJsonTransform = Car(args)
    return
}

func DefFromJson(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) != 1 {
        err = errors.New(fmt.Sprintf("from-json requires 1 argument, %d found", Length(args)))
        return
    }

    if !PairP(Car(args)) {
        err = errors.New("from-json requires a list argument")
        return
    }

    CurrentField.FromJsonTransform = Car(args)
    return
}

func DefApi(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    return
}

func DumpStructure(d *Data, env *SymbolTableFrame) (result *Data, err error) {
    structObj, err := Eval(Car(d), env)
    if err != nil {
        return
    }

    if ObjectP(structObj) && TypeOfObject(structObj) == "DeviceStructure" {
        structure := ((*DeviceStructure)(ObjectValue(structObj)))
        structure.Dump()
    } else {
        err = errors.New("dump-structure expected DeviceStructure")
    }
    return
}

func DumpExpanded(d *Data, env *SymbolTableFrame) (result *Data, err error) {
    structObj, err := Eval(Car(d), env)
    if err != nil {
        return
    }

    if ObjectP(structObj) && TypeOfObject(structObj) == "DeviceStructure" {
        structure := ((*DeviceStructure)(ObjectValue(structObj)))
        structure.DumpExpanded()
    } else {
        err = errors.New("dump-expanded expected DeviceStructure")
    }
    return
}

func CharsToString(ca []uint8) string {
    s := make([]byte, len(ca))
    var lens int
    for ; lens < len(ca); lens++ {
        if ca[lens] == 0 {
            break
        }
        s[lens] = uint8(ca[lens])
    }
    return string(s[0:lens])
}

func BytesToString(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    node := Car(args)
    parent := Cadr(args)

    var bytes [16]uint8
    for c, i := node, 0; NotNilP(c); c, i = Cdr(c), i+1 {
        bytes[i] = uint8(NumericValue(Car(c)))
    }

    str := CharsToString(bytes[0:16])
    Acons(StringWithValue("name"), StringWithValue(str), parent)
    return
}

func StringToBytes(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    node := Car(args)
    parent := Cadr(args)
    var name [16]*Data
    for i, b := range []byte(StringValue(node)) {
        name[i] = NumberWithValue(uint32(b))
    }
    for j := len(StringValue(node)); j < 16; j++ {
        name[j] = NumberWithValue(0)
    }

    ary := ArrayToList(name[0:16])
    Acons(StringWithValue("name"), ary, parent)

    return
}
<commit_msg>Added Api support<commit_after>// Copyright 2013 SteelSeries ApS. All rights reserved.
// No license is given for the use of this source code.

// This package impliments a basic LISP interpretor for embedding in a go program for scripting.
// This file defines functions for device support
package golisp

// (def-struct <name> <field>...)
// (def-field <name> <type> [<count>] [<to json translation> <from json translation>])
// (def-api <struct name> (read ) (write ))

import (
    "errors"
    //    . "github.com/steelseries/golisp"
    "fmt"
    "unsafe"
)

var CurrentDevice *Device
var CurrentStructure *DeviceStructure
var CurrentField *DeviceField

func init() {
    //    InitDeviceBuiltins()
}

func InitDeviceBuiltins() {
    MakePrimitiveFunction("def-device", -1, DefDevice)
    MakePrimitiveFunction("def-struct", -1, DefStruct)
    MakePrimitiveFunction("def-field", -1, DefField)
    MakePrimitiveFunction("range", 2, DefRange)
    MakePrimitiveFunction("values", -1, DefValues)
    MakePrimitiveFunction("deferred-validation", -1, DefDeferredValidation)
    MakePrimitiveFunction("repeat", 1, DefRepeat)
    MakePrimitiveFunction("to-json", 1, DefToJson)
    MakePrimitiveFunction("to-from", 1, DefFromJson)
    MakePrimitiveFunction("def-api", -1, DefApi)
    MakePrimitiveFunction("dump-struct", 1, DumpStructure)
    MakePrimitiveFunction("dump-expanded", 1, DumpExpanded)

    MakePrimitiveFunction("bytes-to-string", 2, BytesToString)
    MakePrimitiveFunction("string-to-bytes", 2, StringToBytes)
}

func DefDevice(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    deviceName := Car(args)
    if TypeOf(deviceName) != SymbolType {
        err = errors.New("Device name must be a symbol")
        return
    }

    if NilP(Cdr(args)) {
        err = errors.New("Device must have at least one structure")
        return
    }

    CurrentDevice := NewDeviceNamed(StringValue(deviceName))

    var thing *Data
    for c := Cdr(args); NotNilP(c); c = Cdr(c) {
        thing, err = Eval(Car(c), env)
        if err != nil {
            return
        }
        if ObjectP(thing) && TypeOfObject(thing) == "DeviceStructure" {
            CurrentDevice.AddStructure((*DeviceStructure)(ObjectValue(thing)))
        } else if ObjectP(thing) && TypeOfObject(thing) == "DeviceApi" {
            CurrentDevice.AddApi((*DeviceApi)(ObjectValue(thing)))
        } else {
            errors.New("Expected structure declaration or api declaration")
        }
    }
    CurrentStructure = nil
    return env.BindTo(deviceName, ObjectWithTypeAndValue("Device", unsafe.Pointer(CurrentDevice))), nil
}

func DefStruct(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    structName := Car(args)
    if TypeOf(structName) != SymbolType {
        err = errors.New("Struct name must be a symbol")
        return
    }

    if NilP(Cdr(args)) {
        err = errors.New("Struct must have at least one field")
        return
    }

    structure := NewStructNamed(StringValue(structName))
    CurrentStructure = structure
    CurrentField = nil

    var field *Data
    for c := Cdr(args); NotNilP(c); c = Cdr(c) {
        field, err = Eval(Car(c), env)
        if err != nil {
            return
        }
        if ObjectP(field) && TypeOfObject(field) == "DeviceField" {
            CurrentStructure.AddField((*DeviceField)(ObjectValue(field)))
        } else {
            errors.New("Expected DeviceField")
        }
    }
    CurrentStructure = nil
    return env.BindTo(structName, ObjectWithTypeAndValue("DeviceStructure", unsafe.Pointer(structure))), nil
}

func DefField(args *Data, env *SymbolTableFrame) (field *Data, err error) {
    fieldName := Car(args)
    if TypeOf(fieldName) != SymbolType {
        err = errors.New("Field name must be a symbol")
        return
    }

    if NilP(Cdr(args)) {
        err = errors.New("Field must have at least a name and type")
        return
    }

    fieldType := Cadr(args)
    if TypeOf(fieldType) != SymbolType {
        err = errors.New("Field type must be a symbol")
        return
    }

    if !IsValidType(fieldType) {
        err = errors.New("Field type must be uint8, uint16, uint32 or a user defined device structure")
        return
    }

    CurrentField = NewField(StringValue(fieldName), StringValue(fieldType), FieldSizeOf(fieldType))

    for c := Cddr(args); NotNilP(c); c = Cdr(c) {
        field, err = Eval(Car(c), env)
        if err != nil {
            return
        }
    }

    field = ObjectWithTypeAndValue("DeviceField", unsafe.Pointer(CurrentField))
    CurrentField = nil
    return
}

func DefRange(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) != 2 {
        err = errors.New(fmt.Sprintf("range requires 2 arguments, %d found", Length(args)))
        return
    }

    if !NumberP(Car(args)) || !NumberP(Cadr(args)) {
        err = errors.New("range requires numeric arguments")
        return
    }

    CurrentField.ValidRange = &Range{Lo: uint32(NumericValue(Car(args))), Hi: uint32(NumericValue(Cadr(args)))}
    return
}

func DefValues(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) == 0 {
        err = errors.New("values requires at least 1 argument")
        return
    }

    CurrentField.ValidValues = &Values{Vals: make([]uint32, 0, 5)}

    var l *Data
    if PairP(Car(args)) {
        l, err = Eval(Car(args), env)
        if err != nil {
            return
        }
        args = l
    }

    for c := args; NotNilP(c); c = Cdr(c) {
        if !NumberP(Car(c)) {
            err = errors.New("values requires numeric arguments")
            return
        }
        CurrentField.ValidValues.AddValue(uint32(NumericValue(Car(c))))
    }
    return
}

func DefDeferredValidation(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    CurrentField.DeferredValidationCode = Car(args)
    return
}

func DefRepeat(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) != 1 {
        err = errors.New(fmt.Sprintf("repeat requires 1 argument, %d found", Length(args)))
        return
    }

    if !NumberP(Car(args)) {
        err = errors.New("repeat requires a numeric argument")
        return
    }

    CurrentField.RepeatCount = int(NumericValue(Car(args)))
    return
}

func DefToJson(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) != 1 {
        err = errors.New(fmt.Sprintf("to-json requires 1 argument, %d found", Length(args)))
        return
    }

    if !PairP(Car(args)) {
        err = errors.New("to-json requires a list argument")
        return
    }

    CurrentField.ToJsonTransform = Car(args)
    return
}

func DefFromJson(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) != 1 {
        err = errors.New(fmt.Sprintf("from-json requires 1 argument, %d found", Length(args)))
        return
    }

    if !PairP(Car(args)) {
        err = errors.New("from-json requires a list argument")
        return
    }

    CurrentField.FromJsonTransform = Car(args)
    return
}

func DefApi(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    return
}

func DumpStructure(d *Data, env *SymbolTableFrame) (result *Data, err error) {
    structObj, err := Eval(Car(d), env)
    if err != nil {
        return
    }

    if ObjectP(structObj) && TypeOfObject(structObj) == "DeviceStructure" {
        structure := ((*DeviceStructure)(ObjectValue(structObj)))
        structure.Dump()
    } else {
        err = errors.New("dump-structure expected DeviceStructure")
    }
    return
}

func DumpExpanded(d *Data, env *SymbolTableFrame) (result *Data, err error) {
    structObj, err := Eval(Car(d), env)
    if err != nil {
        return
    }

    if ObjectP(structObj) && TypeOfObject(structObj) == "DeviceStructure" {
        structure := ((*DeviceStructure)(ObjectValue(structObj)))
        structure.DumpExpanded()
    } else {
        err = errors.New("dump-expanded expected DeviceStructure")
    }
    return
}

func CharsToString(ca []uint8) string {
    s := make([]byte, len(ca))
    var lens int
    for ; lens < len(ca); lens++ {
        if ca[lens] == 0 {
            break
        }
        s[lens] = uint8(ca[lens])
    }
    return string(s[0:lens])
}

func BytesToString(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    node := Car(args)
    parent := Cadr(args)

    var bytes [16]uint8
    for c, i := node, 0; NotNilP(c); c, i = Cdr(c), i+1 {
        bytes[i] = uint8(NumericValue(Car(c)))
    }

    str := CharsToString(bytes[0:16])
    Acons(StringWithValue("name"), StringWithValue(str), parent)
    return
}

func StringToBytes(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    node := Car(args)
    parent := Cadr(args)
    var name [16]*Data
    for i, b := range []byte(StringValue(node)) {
        name[i] = NumberWithValue(uint32(b))
    }
    for j := len(StringValue(node)); j < 16; j++ {
        name[j] = NumberWithValue(0)
    }

    ary := ArrayToList(name[0:16])
    Acons(StringWithValue("name"), ary, parent)

    return
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package config allows a ceph config file to be stored in Kubernetes and mounted as volumes into
// Ceph daemon containers.
package config

import (
	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	"github.com/rook/rook/pkg/apis/rook.io"
	v1 "k8s.io/api/core/v1"
)

type fn func(cephv1.CephClusterHealthCheckSpec) *v1.Probe

// ConfigureLivenessProbe returns the desired liveness probe for a given daemon
func ConfigureLivenessProbe(daemon rook.KeyType, container v1.Container, healthCheck cephv1.CephClusterHealthCheckSpec) v1.Container {
	// Map of functions
	probeFnMap := map[rook.KeyType]fn{
		cephv1.KeyMon: cephv1.GetMonLivenessProbe,
		cephv1.KeyMgr: cephv1.GetMgrLivenessProbe,
		cephv1.KeyOSD: cephv1.GetOSDLivenessProbe,
		cephv1.KeyMds: cephv1.GetMdsLivenessProbe,
	}

	if _, ok := healthCheck.LivenessProbe[daemon]; ok {
		if healthCheck.LivenessProbe[daemon].Disabled {
			container.LivenessProbe = nil
		} else {
			probe := probeFnMap[daemon](healthCheck)
			// If the spec value is not empty, let's apply it along with default when some fields are not specified
			if probe != nil {
				// Set the liveness probe on the container to overwrite the default probe created by Rook
				container.LivenessProbe = GetLivenessProbeWithDefaults(probe, container.LivenessProbe)
			}
		}
	}

	return container
}

func GetLivenessProbeWithDefaults(desiredProbe, currentProbe *v1.Probe) *v1.Probe {
	// Do not replace the handler with the previous one!
	// On the first iteration, the handler appears empty and is then replaced by whatever first daemon value comes in
	// e.g: [env -i sh -c ceph --admin-daemon /run/ceph/ceph-mon.b.asok mon_status] - meaning mon b was the first picked in the list of mons
	// On the second iteration the value of mon b remains, since the pointer has been allocated
	// This means the handler is not empty anymore and not replaced by the current one which it should
	//
	// Let's always force the default handler, there is no reason to change it anyway since the underlying content is generated based on the daemon's name
	// so we can not make it generic via the spec
	desiredProbe.Handler = currentProbe.Handler

	// If the user has not specified thresholds and timeouts, set them to the same values as
	// in the default liveness probe created by Rook.
	if desiredProbe.FailureThreshold == 0 {
		desiredProbe.FailureThreshold = currentProbe.FailureThreshold
	}
	if desiredProbe.PeriodSeconds == 0 {
		desiredProbe.PeriodSeconds = currentProbe.PeriodSeconds
	}
	if desiredProbe.SuccessThreshold == 0 {
		desiredProbe.SuccessThreshold = currentProbe.SuccessThreshold
	}
	if desiredProbe.TimeoutSeconds == 0 {
		desiredProbe.TimeoutSeconds = currentProbe.TimeoutSeconds
	}
	if desiredProbe.InitialDelaySeconds == 0 {
		desiredProbe.InitialDelaySeconds = currentProbe.InitialDelaySeconds
	}

	return desiredProbe
}
<commit_msg>ceph: fix overwriting shared livenessProbe<commit_after>/*
Copyright 2020 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package config allows a ceph config file to be stored in Kubernetes and mounted as volumes into
// Ceph daemon containers.
package config

import (
	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	"github.com/rook/rook/pkg/apis/rook.io"
	v1 "k8s.io/api/core/v1"
)

type fn func(cephv1.CephClusterHealthCheckSpec) *v1.Probe

// ConfigureLivenessProbe returns the desired liveness probe for a given daemon
func ConfigureLivenessProbe(daemon rook.KeyType, container v1.Container, healthCheck cephv1.CephClusterHealthCheckSpec) v1.Container {
	// Map of functions
	probeFnMap := map[rook.KeyType]fn{
		cephv1.KeyMon: cephv1.GetMonLivenessProbe,
		cephv1.KeyMgr: cephv1.GetMgrLivenessProbe,
		cephv1.KeyOSD: cephv1.GetOSDLivenessProbe,
		cephv1.KeyMds: cephv1.GetMdsLivenessProbe,
	}

	if _, ok := healthCheck.LivenessProbe[daemon]; ok {
		if healthCheck.LivenessProbe[daemon].Disabled {
			container.LivenessProbe = nil
		} else {
			probe := probeFnMap[daemon](healthCheck)
			// If the spec value is not empty, let's apply it along with default when some fields are not specified
			if probe != nil {
				// Set the liveness probe on the container to overwrite the default probe created by Rook
				container.LivenessProbe = GetLivenessProbeWithDefaults(probe, container.LivenessProbe)
			}
		}
	}

	return container
}

func GetLivenessProbeWithDefaults(desiredProbe, currentProbe *v1.Probe) *v1.Probe {
	newProbe := *desiredProbe

	// Do not replace the handler with the previous one!
	// On the first iteration, the handler appears empty and is then replaced by whatever first daemon value comes in
	// e.g: [env -i sh -c ceph --admin-daemon /run/ceph/ceph-mon.b.asok mon_status] - meaning mon b was the first picked in the list of mons
	// On the second iteration the value of mon b remains, since the pointer has been allocated
	// This means the handler is not empty anymore and not replaced by the current one which it should
	//
	// Let's always force the default handler, there is no reason to change it anyway since the underlying content is generated based on the daemon's name
	// so we can not make it generic via the spec
	newProbe.Handler = currentProbe.Handler

	// If the user has not specified thresholds and timeouts, set them to the same values as
	// in the default liveness probe created by Rook.
	if newProbe.FailureThreshold == 0 {
		newProbe.FailureThreshold = currentProbe.FailureThreshold
	}
	if newProbe.PeriodSeconds == 0 {
		newProbe.PeriodSeconds = currentProbe.PeriodSeconds
	}
	if newProbe.SuccessThreshold == 0 {
		newProbe.SuccessThreshold = currentProbe.SuccessThreshold
	}
	if newProbe.TimeoutSeconds == 0 {
		newProbe.TimeoutSeconds = currentProbe.TimeoutSeconds
	}
	if newProbe.InitialDelaySeconds == 0 {
		newProbe.InitialDelaySeconds = currentProbe.InitialDelaySeconds
	}

	return &newProbe
}
<|endoftext|>


<commit_before>package model

import (
	"fmt"
	"github.com/KIT-MAMID/mamid/msp"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"io/ioutil"
	"log"
	"os"
	"time"
)

/*
	The structs defined in this file are stored in a database using the `gorm` package.

	Remember to
		Set primary key for a struct.
		Set constraints on specific fields where appropriate.
		Define UniqueIndexes either through a tag or through gorm.DB.AddUniqueIndex()
			for a Unique constraint over multiple fields

	Unless you have a good reason, declare attributes of a struct not null.

	Example:

		type MyType struct {
			Name string `gorm:"not null"`
		}

	Special Cases:

	Enums: 	EnumType.EnumItem => const EnumTypeEnumItem

		Structs using such 'enums' should declare appropriate constraints in the corresponding FieldTag,
		using go-sqlite3 syntax

		Example:

			type MyType struct {
				Name string `sql:"unique"`
			}

*/

type Slave struct {
	ID                   uint   `gorm:"primary_key"`
	Hostname             string `gorm:"unique_index"`
	Port                 PortNumber
	MongodPortRangeBegin PortNumber
	MongodPortRangeEnd   PortNumber
	PersistentStorage    bool
	Mongods              []*Mongod `gorm:"ForeignKey:ParentSlaveID"`
	ConfiguredState      SlaveState

	Problems []*Problem

	// Foreign keys
	RiskGroupID uint
}

type PortNumber uint16

const (
	PortNumberMin PortNumber = 1
	PortNumberMax            = 65535
)

type SlaveState uint

const (
	_                           = 0
	SlaveStateActive SlaveState = iota
	SlaveStateMaintenance
	SlaveStateDisabled
)

type ReplicaSet struct {
	ID                              uint   `gorm:"primary_key"`
	Name                            string `gorm:"unique_index"`
	PersistentMemberCount           uint
	VolatileMemberCount             uint
	ConfigureAsShardingConfigServer bool
	Mongods                         []*Mongod

	Problems []*Problem
}

type RiskGroup struct {
	ID     uint   `gorm:"primary_key"`
	Name   string `gorm:"unique_index"`
	Slaves []*Slave
}

type Mongod struct {
	// TODO missing UNIQUE constraint
	ID          uint `gorm:"primary_key"`
	Port        PortNumber
	ReplSetName string

	ObservationError   MSPError
	ObservationErrorID uint

	LastEstablishStateError   MSPError
	LastEstablishStateErrorID uint

	ParentSlave   *Slave
	ParentSlaveID uint

	ReplicaSet   *ReplicaSet
	ReplicaSetID uint

	DesiredState   MongodState
	DesiredStateID uint

	ObservedState   MongodState
	ObservedStateID uint
}

type MongodState struct {
	ID                     uint `gorm:"primary_key"`
	IsShardingConfigServer bool
	ExecutionState         MongodExecutionState
	ReplicaSetMembers      []ReplicaSetMember
}

type MongodExecutionState uint

const (
	_                                                  = 0
	MongodExecutionStateDestroyed MongodExecutionState = iota
	MongodExecutionStateNotRunning
	MongodExecutionStateRecovering // invalid for a desired MongodState
	MongodExecutionStateRunning
)

type ReplicaSetMember struct { // was ReplicaSetMember in UML
	// TODO missing primary key.
	ID       uint `gorm:"primary_key"`
	Hostname string
	Port     PortNumber

	// Foreign key to parent MongodState
	MongodStateID uint
}

type MSPError struct {
	// Union type for the different errors returned by msp
	// Necessary to decouple MSP from ORM / DB logic
	ID                 uint `gorm:"primary_key"`
	CommunicationError CommunicationError
	SlaveError         SlaveError
}

type CommunicationError struct {
	msp.CommunicationError
	ID         uint `gorm:"primary_key"`
	MSPErrorID uint
}

type SlaveError struct {
	msp.SlaveError
	ID         uint `gorm:"primary_key"`
	MSPErrorID uint
}

type ProblemType uint

const (
	_                                 = 0
	ProblemTypeConnection ProblemType = iota
	ProblemTypeMismatch
	ProblemTypeDesiredReplicaSetConstraint
	ProblemTypeObservedReplicaSetConstraint
)

type Problem struct {
	ID              uint `gorm:"primary_key"`
	Description     string
	LongDescription string
	ProblemType     ProblemType
	FirstOccurred   time.Time
	LastUpdated     time.Time

	Slave   *Slave
	SlaveID uint

	ReplicaSet   *ReplicaSet
	ReplicaSetID uint

	Mongod   *Mongod
	MongodID uint
}

func InitializeFileFromFile(path string) (db *gorm.DB, err error) {

	db, err = initializeDB(path)
	if err != nil {
		return nil, err
	}

	migrateDB(db)

	return db, nil

}

func InitializeTestDB() (db *gorm.DB, err error) {

	path := "/tmp/mamid_test.db"
	os.Remove(path)
	db, err = initializeDB(path)
	if err != nil {
		return nil, err
	}

	migrateDB(db)

	return db, nil

}

func InitializeTestDBWithSQL(sqlFilePath string) (db *gorm.DB, err error) {

	path := "/tmp/mamid_test.db"
	os.Remove(path)
	db, err = initializeDB(path)
	if err != nil {
		return nil, err
	}

	if sqlFilePath != "" {
		statements, err := ioutil.ReadFile(sqlFilePath)
		if err != nil {
			return nil, err
		}

		db.Exec(string(statements), []interface{}{})

	}

	migrateDB(db)

	return db, nil

}

func InitializeInMemoryDB(sqlFilePath string) (db *gorm.DB, err error) {

	db, err = initializeDB(":memory:")
	if err != nil {
		return
	}

	if sqlFilePath != "" {
		statements, err := ioutil.ReadFile(sqlFilePath)
		if err != nil {
			return nil, err
		}

		db.Exec(string(statements), []interface{}{})

	}

	migrateDB(db)

	return db, nil

}

func initializeDB(dsn string) (db *gorm.DB, err error) {

	db, err = gorm.Open("sqlite3", dsn)
	if err != nil {
		return nil, err
	}

	return db, nil

}

func migrateDB(db *gorm.DB) {
	db.AutoMigrate(&Slave{}, &ReplicaSet{}, &RiskGroup{}, &Mongod{}, &MongodState{}, &ReplicaSetMember{}, &Problem{}, &MSPError{}, &CommunicationError{}, &SlaveError{})
	if err := createSlaveUtilizationView(db); err != nil {
		panic(err)
	}
	if err := createReplicaSetEffectiveMembersView(db); err != nil {
		panic(err)
	}
	if err := createReplicaSetConfiguredMembersView(db); err != nil {
		panic(err)
	}
}

func createReplicaSetEffectiveMembersView(tx *gorm.DB) error {
	return tx.Exec(`
		DROP VIEW IF EXISTS replica_set_effective_members;
		CREATE VIEW replica_set_effective_members AS
		SELECT r.id as replica_set_id, m.id as mongod_id, s.persistent_storage
		FROM replica_sets r
		JOIN mongods m ON m.replica_set_id = r.id
		JOIN slaves s ON s.id = m.parent_slave_id
		JOIN mongod_states observed ON observed.id = m.observed_state_id
		JOIN mongod_states desired ON desired.id = m.desired_state_id
		WHERE
		observed.execution_state = ` + fmt.Sprintf("%d", MongodExecutionStateRunning) + `
		AND
		desired.execution_state = ` + fmt.Sprintf("%d", MongodExecutionStateRunning) + `;`).Error
}

func createSlaveUtilizationView(tx *gorm.DB) error {
	return tx.Exec(`
		DROP VIEW IF EXISTS slave_utilization;
		CREATE VIEW slave_utilization AS
		SELECT
			*,
			CASE WHEN max_mongods = 0 THEN 1 ELSE current_mongods*1.0/max_mongods END AS utilization,
			(max_mongods - current_mongods) AS free_mongods
		FROM (
			SELECT
				s.*,
				s.mongod_port_range_end - s.mongod_port_range_begin AS max_mongods,
				COUNT(DISTINCT m.id) as current_mongods
			FROM slaves s
			LEFT OUTER JOIN mongods m ON m.parent_slave_id = s.id
			GROUP BY s.id
		);`).Error
}

func createReplicaSetConfiguredMembersView(tx *gorm.DB) error {
	return tx.Exec(`
		DROP VIEW IF EXISTS replica_set_configured_members;
		CREATE VIEW replica_set_configured_members AS
		SELECT r.id as replica_set_id, m.id as mongod_id, s.persistent_storage
		FROM replica_sets r
		JOIN mongods m ON m.replica_set_id = r.id
		JOIN mongod_states desired_state ON m.desired_state_id = desired_state.id
		JOIN slaves s ON m.parent_slave_id = s.id
		WHERE
			s.configured_state != ` + fmt.Sprintf("%d", SlaveStateDisabled) + `
			AND
			desired_state.execution_state NOT IN (` +
		fmt.Sprintf("%d", MongodExecutionStateNotRunning) +
		`, ` + fmt.Sprintf("%d", MongodExecutionStateDestroyed) +
		`);`).Error
}

func RollbackOnTransactionError(tx *gorm.DB, rollbackError *error) {
	switch e := recover(); e {
	case e == gorm.ErrInvalidTransaction:
		log.Printf("ClusterAllocator: rolling back transaction after error: %v", e)
		*rollbackError = tx.Rollback().Error
		if *rollbackError != nil {
			log.Printf("ClusterAllocator: failed rolling back transaction: %v", *rollbackError)
		}
	default:
		panic(e)
	}
}
<commit_msg>ADD: master: model: foreign keys<commit_after>package model

import (
	"fmt"
	"github.com/KIT-MAMID/mamid/msp"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"io/ioutil"
	"log"
	"os"
	"time"
)

/*
	The structs defined in this file are stored in a database using the `gorm` package.

	Remember to
		Set primary key for a struct.
		Set constraints on specific fields where appropriate.
		Define UniqueIndexes either through a tag or through gorm.DB.AddUniqueIndex()
			for a Unique constraint over multiple fields

	Unless you have a good reason, declare attributes of a struct not null.

	Example:

		type MyType struct {
			Name string `gorm:"not null"`
		}

	Special Cases:

	Enums: 	EnumType.EnumItem => const EnumTypeEnumItem

		Structs using such 'enums' should declare appropriate constraints in the corresponding FieldTag,
		using go-sqlite3 syntax

		Example:

			type MyType struct {
				Name string `sql:"unique"`
			}

*/

type Slave struct {
	ID                   uint   `gorm:"primary_key"`
	Hostname             string `gorm:"unique_index"`
	Port                 PortNumber
	MongodPortRangeBegin PortNumber
	MongodPortRangeEnd   PortNumber
	PersistentStorage    bool
	Mongods              []*Mongod `gorm:"ForeignKey:ParentSlaveID"`
	ConfiguredState      SlaveState

	Problems []*Problem

	// Foreign keys
	RiskGroupID uint `sql:"type:integer REFERENCES risk_groups(id)"`
}

type PortNumber uint16

const (
	PortNumberMin PortNumber = 1
	PortNumberMax            = 65535
)

type SlaveState uint

const (
	_                           = 0
	SlaveStateActive SlaveState = iota
	SlaveStateMaintenance
	SlaveStateDisabled
)

type ReplicaSet struct {
	ID                              uint   `gorm:"primary_key"`
	Name                            string `gorm:"unique_index"`
	PersistentMemberCount           uint
	VolatileMemberCount             uint
	ConfigureAsShardingConfigServer bool
	Mongods                         []*Mongod

	Problems []*Problem
}

type RiskGroup struct {
	ID     uint   `gorm:"primary_key"`
	Name   string `gorm:"unique_index"`
	Slaves []*Slave
}

type Mongod struct {
	// TODO missing UNIQUE constraint
	ID          uint `gorm:"primary_key"`
	Port        PortNumber
	ReplSetName string

	ObservationError   MSPError
	ObservationErrorID uint `sql:"type:integer REFERENCES msp_errors(id)"`

	LastEstablishStateError   MSPError
	LastEstablishStateErrorID uint `sql:"type:integer REFERENCES msp_errors(id)"`

	ParentSlave   *Slave
	ParentSlaveID uint `sql:"type:integer REFERENCES slaves(id)"`

	ReplicaSet   *ReplicaSet
	ReplicaSetID uint `sql:"type:integer REFERENCES replica_sets(id)"`

	DesiredState   MongodState
	DesiredStateID uint `sql:"type:integer REFERENCES mongod_states(id)"`

	ObservedState   MongodState
	ObservedStateID uint `sql:"type:integer REFERENCES mongod_states(id)"`
}

type MongodState struct {
	ID                     uint `gorm:"primary_key"`
	IsShardingConfigServer bool
	ExecutionState         MongodExecutionState
	ReplicaSetMembers      []ReplicaSetMember
}

type MongodExecutionState uint

const (
	_                                                  = 0
	MongodExecutionStateDestroyed MongodExecutionState = iota
	MongodExecutionStateNotRunning
	MongodExecutionStateRecovering // invalid for a desired MongodState
	MongodExecutionStateRunning
)

type ReplicaSetMember struct { // was ReplicaSetMember in UML
	// TODO missing primary key.
	ID       uint `gorm:"primary_key"`
	Hostname string
	Port     PortNumber

	// Foreign key to parent MongodState
	MongodStateID uint `sql:"type:integer REFERENCES mongod_states(id)"`
}

type MSPError struct {
	// Union type for the different errors returned by msp
	// Necessary to decouple MSP from ORM / DB logic
	ID                 uint `gorm:"primary_key"`
	CommunicationError CommunicationError
	SlaveError         SlaveError
}

type CommunicationError struct {
	msp.CommunicationError
	ID         uint `gorm:"primary_key"`
	MSPErrorID uint `sql:"type:integer REFERENCES msp_errors(id)"`
}

type SlaveError struct {
	msp.SlaveError
	ID         uint `gorm:"primary_key"`
	MSPErrorID uint `sql:"type:integer REFERENCES msp_errors(id)"`
}

type ProblemType uint

const (
	_                                 = 0
	ProblemTypeConnection ProblemType = iota
	ProblemTypeMismatch
	ProblemTypeDesiredReplicaSetConstraint
	ProblemTypeObservedReplicaSetConstraint
)

type Problem struct {
	ID              uint `gorm:"primary_key"`
	Description     string
	LongDescription string
	ProblemType     ProblemType
	FirstOccurred   time.Time
	LastUpdated     time.Time

	Slave   *Slave
	SlaveID uint `sql:"type:integer REFERENCES slaves(id)"`

	ReplicaSet   *ReplicaSet
	ReplicaSetID uint `sql:"type:integer REFERENCES replica_sets(id)"`

	Mongod   *Mongod
	MongodID uint `sql:"type:integer REFERENCES mongods(id)"`
}

func InitializeFileFromFile(path string) (db *gorm.DB, err error) {

	db, err = initializeDB(path)
	if err != nil {
		return nil, err
	}

	migrateDB(db)

	return db, nil

}

func InitializeTestDB() (db *gorm.DB, err error) {

	path := "/tmp/mamid_test.db"
	os.Remove(path)
	db, err = initializeDB(path)
	if err != nil {
		return nil, err
	}

	migrateDB(db)

	return db, nil

}

func InitializeTestDBWithSQL(sqlFilePath string) (db *gorm.DB, err error) {

	path := "/tmp/mamid_test.db"
	os.Remove(path)
	db, err = initializeDB(path)
	if err != nil {
		return nil, err
	}

	if sqlFilePath != "" {
		statements, err := ioutil.ReadFile(sqlFilePath)
		if err != nil {
			return nil, err
		}

		db.Exec(string(statements), []interface{}{})

	}

	migrateDB(db)

	return db, nil

}

func InitializeInMemoryDB(sqlFilePath string) (db *gorm.DB, err error) {

	db, err = initializeDB(":memory:")
	if err != nil {
		return
	}

	if sqlFilePath != "" {
		statements, err := ioutil.ReadFile(sqlFilePath)
		if err != nil {
			return nil, err
		}

		db.Exec(string(statements), []interface{}{})

	}

	migrateDB(db)

	return db, nil

}

func initializeDB(dsn string) (db *gorm.DB, err error) {

	db, err = gorm.Open("sqlite3", dsn)
	if err != nil {
		return nil, err
	}

	return db, nil

}

func migrateDB(db *gorm.DB) {
	db.AutoMigrate(&Slave{}, &ReplicaSet{}, &RiskGroup{}, &Mongod{}, &MongodState{}, &ReplicaSetMember{}, &Problem{}, &MSPError{}, &CommunicationError{}, &SlaveError{})
	if err := createSlaveUtilizationView(db); err != nil {
		panic(err)
	}
	if err := createReplicaSetEffectiveMembersView(db); err != nil {
		panic(err)
	}
	if err := createReplicaSetConfiguredMembersView(db); err != nil {
		panic(err)
	}
}

func createReplicaSetEffectiveMembersView(tx *gorm.DB) error {
	return tx.Exec(`
		DROP VIEW IF EXISTS replica_set_effective_members;
		CREATE VIEW replica_set_effective_members AS
		SELECT r.id as replica_set_id, m.id as mongod_id, s.persistent_storage
		FROM replica_sets r
		JOIN mongods m ON m.replica_set_id = r.id
		JOIN slaves s ON s.id = m.parent_slave_id
		JOIN mongod_states observed ON observed.id = m.observed_state_id
		JOIN mongod_states desired ON desired.id = m.desired_state_id
		WHERE
		observed.execution_state = ` + fmt.Sprintf("%d", MongodExecutionStateRunning) + `
		AND
		desired.execution_state = ` + fmt.Sprintf("%d", MongodExecutionStateRunning) + `;`).Error
}

func createSlaveUtilizationView(tx *gorm.DB) error {
	return tx.Exec(`
		DROP VIEW IF EXISTS slave_utilization;
		CREATE VIEW slave_utilization AS
		SELECT
			*,
			CASE WHEN max_mongods = 0 THEN 1 ELSE current_mongods*1.0/max_mongods END AS utilization,
			(max_mongods - current_mongods) AS free_mongods
		FROM (
			SELECT
				s.*,
				s.mongod_port_range_end - s.mongod_port_range_begin AS max_mongods,
				COUNT(DISTINCT m.id) as current_mongods
			FROM slaves s
			LEFT OUTER JOIN mongods m ON m.parent_slave_id = s.id
			GROUP BY s.id
		);`).Error
}

func createReplicaSetConfiguredMembersView(tx *gorm.DB) error {
	return tx.Exec(`
		DROP VIEW IF EXISTS replica_set_configured_members;
		CREATE VIEW replica_set_configured_members AS
		SELECT r.id as replica_set_id, m.id as mongod_id, s.persistent_storage
		FROM replica_sets r
		JOIN mongods m ON m.replica_set_id = r.id
		JOIN mongod_states desired_state ON m.desired_state_id = desired_state.id
		JOIN slaves s ON m.parent_slave_id = s.id
		WHERE
			s.configured_state != ` + fmt.Sprintf("%d", SlaveStateDisabled) + `
			AND
			desired_state.execution_state NOT IN (` +
		fmt.Sprintf("%d", MongodExecutionStateNotRunning) +
		`, ` + fmt.Sprintf("%d", MongodExecutionStateDestroyed) +
		`);`).Error
}

func RollbackOnTransactionError(tx *gorm.DB, rollbackError *error) {
	switch e := recover(); e {
	case e == gorm.ErrInvalidTransaction:
		log.Printf("ClusterAllocator: rolling back transaction after error: %v", e)
		*rollbackError = tx.Rollback().Error
		if *rollbackError != nil {
			log.Printf("ClusterAllocator: failed rolling back transaction: %v", *rollbackError)
		}
	default:
		panic(e)
	}
}
<|endoftext|>


<commit_before>package tuikit

import (
	"unicode/utf8"

	termbox "github.com/nsf/termbox-go"
	"github.com/nsf/tulib"
	log "github.com/sgeb/go-sglog"
)

type TextInputWidget struct {
	*Canvas

	text []byte
	pos  int

	runeBytes [utf8.UTFMax]byte
}

func NewTextInputWidget() *TextInputWidget {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	return &TextInputWidget{
		Canvas: NewCanvas(0, 0),

		// will grow as needed
		text: make([]byte, 0, 64),
	}
}

func (v *TextInputWidget) HandleEvent(ev *Event) {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if ev.Type != termbox.EventKey {
		return
	}

	handled := true
	switch {
	case ev.Ch != 0:
		v.append(ev.Ch)
	case ev.Key == termbox.KeySpace:
		v.append(' ')
	case ev.Key == termbox.KeyBackspace || ev.Key == termbox.KeyBackspace2:
		v.backspace()
	case ev.Key == termbox.KeyDelete:
		v.delete()
	case ev.Key == termbox.KeyArrowLeft || ev.Key == termbox.KeyCtrlB:
		v.moveLeft()
	case ev.Key == termbox.KeyArrowRight || ev.Key == termbox.KeyCtrlF:
		v.moveRight()
	case ev.Key == termbox.KeyCtrlU:
		v.killLine()
	case ev.Key == termbox.KeyCtrlK:
		v.killToEol()
	default:
		handled = false
	}
	ev.Handled = handled
}

func (v *TextInputWidget) append(r rune) {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	log.Debug.Printf("Rune: %v (%v)", r, string(r))
	if r < utf8.RuneSelf {
		v.text = append(v.text, byte(r))
	} else {
		n := utf8.EncodeRune(v.runeBytes[0:], r)
		v.text = append(v.text, v.runeBytes[0:n]...)
	}

	v.pos++
	v.Dirty = true
}

func (v *TextInputWidget) backspace() {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if v.pos > 0 {
		v.text = append(v.text[:v.pos-1], v.text[v.pos:]...)
		v.pos--
		v.Dirty = true
	}
}

func (v *TextInputWidget) delete() {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if v.pos < len(v.text) {
		v.text = append(v.text[:v.pos], v.text[v.pos+1:]...)
		v.Dirty = true
	}
}

func (v *TextInputWidget) moveLeft() {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if v.pos > 0 {
		v.pos--
		v.Dirty = true
	}
}

func (v *TextInputWidget) moveRight() {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if v.pos < len(v.text) {
		v.pos++
		v.Dirty = true
	}
}

func (v *TextInputWidget) killLine() {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if len(v.text) > 0 {
		v.text = []byte(nil)
		v.pos = 0
		v.Dirty = true
	}
}

func (v *TextInputWidget) killToEol() {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if len(v.text) > 0 {
		v.text = v.text[:v.pos]
		v.Dirty = true
	}
}

func (v *TextInputWidget) Paint() {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if !v.Dirty {
		log.Debug.Println("Not dirty, early return")
		return
	}

	// TODO: implement scrolling
	//	start := 0
	//	pos := v.pos
	//	len := v.len
	//
	//	for pos >= v.Width {
	//		start++
	//		pos--
	//		len--
	//	}
	//	for len > v.Width {
	//		len--
	//	}

	log.Debug.Printf("Text: %v (len: %v)", string(v.text), len(v.text))

	v.Fill(v.Rect, termbox.Cell{Ch: ' '})
	v.DrawLabel(v.Rect, &tulib.DefaultLabelParams, v.text)
	v.Cursor = NewPoint(v.pos, 0)
	v.Dirty = false
}

func (v *TextInputWidget) SetSize(w, h int) {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if v.Width != w || v.Height != h {
		v.Buffer.Resize(w, h)
		v.Dirty = true
	}
}

func (v *TextInputWidget) GetCanvas() *Canvas {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	return v.Canvas
}
<commit_msg>Rename TextInputWidget to TextWidget<commit_after>package tuikit

import (
	"unicode/utf8"

	termbox "github.com/nsf/termbox-go"
	"github.com/nsf/tulib"
	log "github.com/sgeb/go-sglog"
)

type TextWidget struct {
	*Canvas

	text []byte
	pos  int

	runeBytes [utf8.UTFMax]byte
}

func NewTextWidget() *TextWidget {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	return &TextWidget{
		Canvas: NewCanvas(0, 0),

		// will grow as needed
		text: make([]byte, 0, 64),
	}
}

func (w *TextWidget) HandleEvent(ev *Event) {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if ev.Type != termbox.EventKey {
		return
	}

	handled := true
	switch {
	case ev.Ch != 0:
		w.append(ev.Ch)
	case ev.Key == termbox.KeySpace:
		w.append(' ')
	case ev.Key == termbox.KeyBackspace || ev.Key == termbox.KeyBackspace2:
		w.backspace()
	case ev.Key == termbox.KeyDelete:
		w.delete()
	case ev.Key == termbox.KeyArrowLeft || ev.Key == termbox.KeyCtrlB:
		w.moveLeft()
	case ev.Key == termbox.KeyArrowRight || ev.Key == termbox.KeyCtrlF:
		w.moveRight()
	case ev.Key == termbox.KeyCtrlU:
		w.killLine()
	case ev.Key == termbox.KeyCtrlK:
		w.killToEol()
	default:
		handled = false
	}
	ev.Handled = handled
}

func (w *TextWidget) append(r rune) {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	log.Debug.Printf("Rune: %v (%v)", r, string(r))
	if r < utf8.RuneSelf {
		w.text = append(w.text, byte(r))
	} else {
		n := utf8.EncodeRune(w.runeBytes[0:], r)
		w.text = append(w.text, w.runeBytes[0:n]...)
	}

	w.pos++
	w.Dirty = true
}

func (w *TextWidget) backspace() {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if w.pos > 0 {
		w.text = append(w.text[:w.pos-1], w.text[w.pos:]...)
		w.pos--
		w.Dirty = true
	}
}

func (w *TextWidget) delete() {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if w.pos < len(w.text) {
		w.text = append(w.text[:w.pos], w.text[w.pos+1:]...)
		w.Dirty = true
	}
}

func (w *TextWidget) moveLeft() {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if w.pos > 0 {
		w.pos--
		w.Dirty = true
	}
}

func (w *TextWidget) moveRight() {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if w.pos < len(w.text) {
		w.pos++
		w.Dirty = true
	}
}

func (w *TextWidget) killLine() {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if len(w.text) > 0 {
		w.text = []byte(nil)
		w.pos = 0
		w.Dirty = true
	}
}

func (w *TextWidget) killToEol() {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if len(w.text) > 0 {
		w.text = w.text[:w.pos]
		w.Dirty = true
	}
}

func (w *TextWidget) Paint() {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if !w.Dirty {
		log.Debug.Println("Not dirty, early return")
		return
	}

	// TODO: implement scrolling
	//	start := 0
	//	pos := w.pos
	//	len := w.len
	//
	//	for pos >= w.Width {
	//		start++
	//		pos--
	//		len--
	//	}
	//	for len > w.Width {
	//		len--
	//	}

	log.Debug.Printf("Text: %v (len: %v)", string(w.text), len(w.text))

	w.Fill(w.Rect, termbox.Cell{Ch: ' '})
	w.DrawLabel(w.Rect, &tulib.DefaultLabelParams, w.text)
	w.Cursor = NewPoint(w.pos, 0)
	w.Dirty = false
}

func (w *TextWidget) SetSize(nw, nh int) {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	if w.Width != nw || w.Height != nh {
		w.Buffer.Resize(nw, nh)
		w.Dirty = true
	}
}

func (w *TextWidget) GetCanvas() *Canvas {
	log.Trace.PrintEnter()
	defer log.Trace.PrintLeave()

	return w.Canvas
}
<|endoftext|>


<commit_before><commit_msg>add missing flyHijackTask<commit_after><|endoftext|>


<commit_before>package dragonfly

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
)

func Decode(str string) (*Job, error) {
	var job Job

	jobStr, err := decodeJobStr(str)
	if err != nil {
		return &job, err
	}
	fmt.Println(jobStr)

	jobArr, err := decodeJson(jobStr)
	if err != nil {
		return &job, err
	}

	job.Steps = make([]Step, len(jobArr))

	for i, v := range jobArr {
		var step Step
		step.Command = v[0]
		step.Args = v[1:]
		job.Steps[i] = step
	}

	return &job, err
}

func decodeJobStr(b64Str string) ([]byte, error) {
	b64Str += "=" //dragonfly trims a trailing =\n from the jobs
	jobStr, err := base64.StdEncoding.DecodeString(b64Str)

	if err != nil {
		fmt.Println(b64Str)
		b64Str += "="
		fmt.Println(b64Str)

		jobStr, err2 := base64.StdEncoding.DecodeString(b64Str)
		err = err2

		if err2 != nil {
			fmt.Println("\n******************************************************************************")
			fmt.Println(b64Str)
			fmt.Println(jobStr)
			fmt.Println("******************************************************************************\n")
		}
	}

	return jobStr, err
}

func decodeJson(jsonStr []byte) ([][]string, error) {
	var i [][]string
	err := json.Unmarshal(jsonStr, &i)
	if err != nil {
		str := string(jsonStr) + "]"
		err = json.Unmarshal([]byte(str), &i)

	}
	return i, err
}
<commit_msg>remove unnecessary puts<commit_after>package dragonfly

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
)

func Decode(str string) (*Job, error) {
	var job Job

	jobStr, err := decodeJobStr(str)
	if err != nil {
		return &job, err
	}

	jobArr, err := decodeJson(jobStr)
	if err != nil {
		return &job, err
	}

	job.Steps = make([]Step, len(jobArr))

	for i, v := range jobArr {
		var step Step
		step.Command = v[0]
		step.Args = v[1:]
		job.Steps[i] = step
	}

	return &job, err
}

func decodeJobStr(b64Str string) ([]byte, error) {
	b64Str += "=" //dragonfly trims a trailing =\n from the jobs
	jobStr, err := base64.StdEncoding.DecodeString(b64Str)

	if err != nil {
		b64Str += "="
		jobStr, err2 := base64.StdEncoding.DecodeString(b64Str)
		err = err2

		if err2 != nil {
			fmt.Println("\n******************************************************************************")
			fmt.Println(b64Str)
			fmt.Println(jobStr)
			fmt.Println("******************************************************************************\n")
		}
	}

	return jobStr, err
}

func decodeJson(jsonStr []byte) ([][]string, error) {
	var i [][]string
	err := json.Unmarshal(jsonStr, &i)
	if err != nil {
		str := string(jsonStr) + "]"
		err = json.Unmarshal([]byte(str), &i)

	}
	return i, err
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"runtime"
	"strings"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/gateway"
	"github.com/42wim/matterbridge/gateway/bridgemap"
	"github.com/google/gops/agent"
	prefixed "github.com/matterbridge/logrus-prefixed-formatter"
	"github.com/sirupsen/logrus"
)

var (
	version = "1.18.0"
	githash string

	flagConfig  = flag.String("conf", "matterbridge.toml", "config file")
	flagDebug   = flag.Bool("debug", false, "enable debug")
	flagVersion = flag.Bool("version", false, "show version")
	flagGops    = flag.Bool("gops", false, "enable gops agent")
)

func main() {
	flag.Parse()
	if *flagVersion {
		fmt.Printf("version: %s %s\n", version, githash)
		return
	}

	rootLogger := setupLogger()
	logger := rootLogger.WithFields(logrus.Fields{"prefix": "main"})

	if *flagGops {
		if err := agent.Listen(agent.Options{}); err != nil {
			logger.Errorf("Failed to start gops agent: %#v", err)
		} else {
			defer agent.Close()
		}
	}

	logger.Printf("Running version %s %s", version, githash)
	if strings.Contains(version, "-dev") {
		logger.Println("WARNING: THIS IS A DEVELOPMENT VERSION. Things may break.")
	}

	cfg := config.NewConfig(rootLogger, *flagConfig)
	cfg.BridgeValues().General.Debug = *flagDebug

	// if logging to a file, ensure it is closed when the program terminates
	// nolint:errcheck
	defer func() {
		if f, ok := rootLogger.Out.(*os.File); ok {
			f.Sync()
			f.Close()
		}
	}()

	r, err := gateway.NewRouter(rootLogger, cfg, bridgemap.FullMap)
	if err != nil {
		logger.Fatalf("Starting gateway failed: %s", err)
	}
	if err = r.Start(); err != nil {
		logger.Fatalf("Starting gateway failed: %s", err)
	}
	logger.Printf("Gateway(s) started succesfully. Now relaying messages")
	select {}
}

func setupLogger() *logrus.Logger {
	logger := &logrus.Logger{
		Out: os.Stdout,
		Formatter: &prefixed.TextFormatter{
			PrefixPadding: 13,
			DisableColors: true,
		},
		Level: logrus.InfoLevel,
	}
	if *flagDebug || os.Getenv("DEBUG") == "1" {
		logger.SetReportCaller(true)
		logger.Formatter = &prefixed.TextFormatter{
			PrefixPadding: 13,
			DisableColors: true,
			FullTimestamp: false,

			CallerFormatter: func(function, file string) string {
				return fmt.Sprintf(" [%s:%s]", function, file)
			},
			CallerPrettyfier: func(f *runtime.Frame) (string, string) {
				sp := strings.SplitAfter(f.File, "/matterbridge/")
				filename := f.File
				if len(sp) > 1 {
					filename = sp[1]
				}
				s := strings.Split(f.Function, ".")
				funcName := s[len(s)-1]
				return funcName, fmt.Sprintf("%s:%d", filename, f.Line)
			},
		}

		logger.Level = logrus.DebugLevel
		logger.WithFields(logrus.Fields{"prefix": "main"}).Info("Enabling debug logging.")
	}
	return logger
}
<commit_msg>Bump version<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"runtime"
	"strings"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/gateway"
	"github.com/42wim/matterbridge/gateway/bridgemap"
	"github.com/google/gops/agent"
	prefixed "github.com/matterbridge/logrus-prefixed-formatter"
	"github.com/sirupsen/logrus"
)

var (
	version = "1.18.1-dev"
	githash string

	flagConfig  = flag.String("conf", "matterbridge.toml", "config file")
	flagDebug   = flag.Bool("debug", false, "enable debug")
	flagVersion = flag.Bool("version", false, "show version")
	flagGops    = flag.Bool("gops", false, "enable gops agent")
)

func main() {
	flag.Parse()
	if *flagVersion {
		fmt.Printf("version: %s %s\n", version, githash)
		return
	}

	rootLogger := setupLogger()
	logger := rootLogger.WithFields(logrus.Fields{"prefix": "main"})

	if *flagGops {
		if err := agent.Listen(agent.Options{}); err != nil {
			logger.Errorf("Failed to start gops agent: %#v", err)
		} else {
			defer agent.Close()
		}
	}

	logger.Printf("Running version %s %s", version, githash)
	if strings.Contains(version, "-dev") {
		logger.Println("WARNING: THIS IS A DEVELOPMENT VERSION. Things may break.")
	}

	cfg := config.NewConfig(rootLogger, *flagConfig)
	cfg.BridgeValues().General.Debug = *flagDebug

	// if logging to a file, ensure it is closed when the program terminates
	// nolint:errcheck
	defer func() {
		if f, ok := rootLogger.Out.(*os.File); ok {
			f.Sync()
			f.Close()
		}
	}()

	r, err := gateway.NewRouter(rootLogger, cfg, bridgemap.FullMap)
	if err != nil {
		logger.Fatalf("Starting gateway failed: %s", err)
	}
	if err = r.Start(); err != nil {
		logger.Fatalf("Starting gateway failed: %s", err)
	}
	logger.Printf("Gateway(s) started succesfully. Now relaying messages")
	select {}
}

func setupLogger() *logrus.Logger {
	logger := &logrus.Logger{
		Out: os.Stdout,
		Formatter: &prefixed.TextFormatter{
			PrefixPadding: 13,
			DisableColors: true,
		},
		Level: logrus.InfoLevel,
	}
	if *flagDebug || os.Getenv("DEBUG") == "1" {
		logger.SetReportCaller(true)
		logger.Formatter = &prefixed.TextFormatter{
			PrefixPadding: 13,
			DisableColors: true,
			FullTimestamp: false,

			CallerFormatter: func(function, file string) string {
				return fmt.Sprintf(" [%s:%s]", function, file)
			},
			CallerPrettyfier: func(f *runtime.Frame) (string, string) {
				sp := strings.SplitAfter(f.File, "/matterbridge/")
				filename := f.File
				if len(sp) > 1 {
					filename = sp[1]
				}
				s := strings.Split(f.Function, ".")
				funcName := s[len(s)-1]
				return funcName, fmt.Sprintf("%s:%d", filename, f.Line)
			},
		}

		logger.Level = logrus.DebugLevel
		logger.WithFields(logrus.Fields{"prefix": "main"}).Info("Enabling debug logging.")
	}
	return logger
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/xml"
	"flag"
	irc "github.com/fluffle/goirc/client"
	_ "github.com/go-sql-driver/mysql"
	"github.com/tv42/base58"
	"html"
	"io"
	"io/ioutil"
	"log"
	"math/big"
	"math/rand"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
)

type Config struct {
	Channel      string
	DBConn       string
	Nick         string
	Ident        string
	FullName     string
	FlickrAPIKey string
	IRCPass      string
}

var (
	config             Config
	urlRegex, regexErr = regexp.Compile(`(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s` + "`" + `!()\[\]{};:'".,<>?«»“”‘’]))`)
	helpstring         = "Available commands are !hacking, !help, !haata, and !stats"
)

type Setresp struct {
	Sets []Set `xml:"collections>collection>set"`
}

type Set struct {
	Id          string `xml:"id,attr"`
	Title       string `xml:"title,attr"`
	Description string `xml:"description,attr"`
}

type Photoresp struct {
	Photos []Photo `xml:"photoset>photo"`
}

type Photo struct {
	Id        int64  `xml:"id,attr"`
	Secret    string `xml:"secret,attr"`
	Server    string `xml:"server,attr"`
	Farm      string `xml:"farm,attr"`
	Title     string `xml:"title,attr"`
	Isprimary string `xml:"isprimary,attr"`
}

func htmlfetch(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	respbody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return respbody, nil
}

func random(limit int) int {
	rand.Seed(time.Now().Unix())
	return rand.Intn(limit)
}

func sendUrl(channel, postedUrl string, conn *irc.Conn) {
	log.Println("Fetching title for " + postedUrl + " In channel " + channel)
	if !strings.HasPrefix(postedUrl, "http://") {
		if !strings.HasPrefix(postedUrl, "https://") {
			postedUrl = "http://" + postedUrl
		}
	}
	resp, err := http.Get(postedUrl)
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	buf := make([]byte, 1024)
	respbody := []byte{}
	for i := 0; i < 30; i++ {
		n, err := resp.Body.Read(buf)
		if err != nil && err != io.EOF {
			return
		}
		if n == 0 {
			break
		}
		respbody = append(respbody, buf[:n]...)
	}

	stringbody := string(respbody)
	titlestart := strings.Index(stringbody, "<title>")
	titleend := strings.Index(stringbody, "</title>")
	if titlestart != -1 && titlestart != -1 {
		title := string(respbody[titlestart+7 : titleend])
		title = strings.TrimSpace(title)
		if title != "" {
			parsedurl, err := url.Parse(postedUrl)
			if err == nil {
				postedUrl = parsedurl.Host
			}
			title = "Title: " + html.UnescapeString(title) + " (at " + postedUrl + ")"
			log.Println(title)
			conn.Privmsg(channel, title)
		}
	}
}

func dance(channel string, conn *irc.Conn) {
	conn.Privmsg(channel, ":D-<")
	time.Sleep(500 * time.Millisecond)
	conn.Privmsg(channel, ":D|<")
	time.Sleep(500 * time.Millisecond)
	conn.Privmsg(channel, ":D/<")
}

func haata(channel string, conn *irc.Conn) {
	sets, err := htmlfetch(`http://api.flickr.com/services/rest/?method=flickr.collections.getTree&api_key=` + config.FlickrAPIKey + `&user_id=57321699@N06&collection_id=57276377-72157635417889224`)
	if err != nil {
		return
	}
	var setresp Setresp
	xml.Unmarshal(sets, &setresp)
	randsetindex := random(len(setresp.Sets))
	randset := setresp.Sets[randsetindex].Id

	pics, err := htmlfetch(`http://api.flickr.com/services/rest/?method=flickr.photosets.getPhotos&api_key=` + config.FlickrAPIKey + `&photoset_id=` + randset)
	if err != nil {
		return
	}
	var photoresp Photoresp
	xml.Unmarshal(pics, &photoresp)
	randpic := random(len(photoresp.Photos))
	photostring := string(base58.EncodeBig([]byte{}, big.NewInt(photoresp.Photos[randpic].Id)))
	conn.Privmsg(channel, strings.TrimSpace(setresp.Sets[randsetindex].Title)+`: http://flic.kr/p/`+photostring)
}

func handleMessage(conn *irc.Conn, line *irc.Line) {
	urllist := []string{}
	numlinks := 0

	if strings.HasPrefix(line.Args[1], "!dance") && line.Nick == "sadbox" {
		go dance(line.Args[0], conn)
	} else if strings.HasPrefix(line.Args[1], "!audio") && line.Nick == "sadbox" {
		conn.Privmsg(line.Args[0], "https://sadbox.org/static/audiophile.html")
	} else if strings.HasPrefix(line.Args[1], "!hacking") {
		conn.Privmsg(line.Args[0], "This channel is about keyboards (not hacking), please read the topic.")
	} else if strings.HasPrefix(line.Args[1], "!help") {
		conn.Privmsg(line.Args[0], helpstring)
	} else if strings.HasPrefix(line.Args[1], "!haata") {
		go haata(line.Args[0], conn)
	} else if strings.HasPrefix(line.Args[1], "!stats") {
		conn.Privmsg(line.Args[0], "http://sadbox.org/geekhack")
	}

NextWord:
	for _, word := range strings.Split(line.Args[1], " ") {
		word = strings.TrimSpace(word)
		if urlRegex.MatchString(word) {
			for _, subUrl := range urllist {
				if subUrl == word {
					continue NextWord
				}
			}
			numlinks++
			if numlinks > 3 {
				break
			}
			urllist = append(urllist, word)
			go sendUrl(line.Args[0], word, conn)
		}

	}
	db, err := sql.Open("mysql", config.DBConn)
	if err != nil {
		log.Println(err)
	}
	defer db.Close()
	_, err = db.Exec("insert into messages (Nick, Ident, Host, Src, Cmd, Channel, Message, Time) values (?, ?, ?, ?, ?, ?, ?, ?)", line.Nick, line.Ident, line.Host, line.Src, line.Cmd, line.Args[0], line.Args[1], line.Time)
	if err != nil {
		log.Println(err)
	}
}

func init() {
	log.Println("Starting sadbot")

	flag.Parse()

	if regexErr != nil {
		log.Panic(regexErr)
	}

	xmlFile, err := ioutil.ReadFile("config.xml")
	if err != nil {
		log.Fatal(err)
	}
	xml.Unmarshal(xmlFile, &config)
}

func main() {
	log.Printf("Joining channel %s", config.Channel)
	log.Printf("Nick: %s Ident: %s FullName: %s", config.Nick, config.Ident, config.FullName)

	c := irc.SimpleClient(config.Nick, config.Ident, config.FullName)

	c.AddHandler(irc.CONNECTED,
		func(conn *irc.Conn, line *irc.Line) {
			conn.Join(config.Channel)
			conn.Privmsg("nickserv", "identify "+config.Nick+" "+config.IRCPass)
			log.Println("Connected!")
		})

	quit := make(chan bool)

	c.AddHandler(irc.DISCONNECTED,
		func(conn *irc.Conn, line *irc.Line) { quit <- true })

	c.AddHandler("PRIVMSG", handleMessage)

	if err := c.Connect("irc.freenode.net"); err != nil {
		log.Fatalln("Connection error: %s\n", err)
	}

	<-quit
}
<commit_msg>use a regex instead of strings for http matching<commit_after>package main

import (
	"database/sql"
	"encoding/xml"
	"flag"
	irc "github.com/fluffle/goirc/client"
	_ "github.com/go-sql-driver/mysql"
	"github.com/tv42/base58"
	"html"
	"io"
	"io/ioutil"
	"log"
	"math/big"
	"math/rand"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
)

type Config struct {
	Channel      string
	DBConn       string
	Nick         string
	Ident        string
	FullName     string
	FlickrAPIKey string
	IRCPass      string
}

var (
	config                  Config
	urlRegex, regexErr      = regexp.Compile(`(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s` + "`" + `!()\[\]{};:'".,<>?«»“”‘’]))`)
	httpRegex, httpRegexErr = regexp.Compile(`http(s)?://.*`)
	helpstring              = "Available commands are !hacking, !help, !haata, and !stats"
)

type Setresp struct {
	Sets []Set `xml:"collections>collection>set"`
}

type Set struct {
	Id          string `xml:"id,attr"`
	Title       string `xml:"title,attr"`
	Description string `xml:"description,attr"`
}

type Photoresp struct {
	Photos []Photo `xml:"photoset>photo"`
}

type Photo struct {
	Id        int64  `xml:"id,attr"`
	Secret    string `xml:"secret,attr"`
	Server    string `xml:"server,attr"`
	Farm      string `xml:"farm,attr"`
	Title     string `xml:"title,attr"`
	Isprimary string `xml:"isprimary,attr"`
}

func htmlfetch(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	respbody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return respbody, nil
}

func random(limit int) int {
	rand.Seed(time.Now().Unix())
	return rand.Intn(limit)
}

func sendUrl(channel, postedUrl string, conn *irc.Conn) {
	log.Println("Fetching title for " + postedUrl + " In channel " + channel)
	if !httpRegex.MatchString(postedUrl) {
		postedUrl = "http://" + postedUrl
	}

	resp, err := http.Get(postedUrl)
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	buf := make([]byte, 1024)
	respbody := []byte{}
	for i := 0; i < 30; i++ {
		n, err := resp.Body.Read(buf)
		if err != nil && err != io.EOF {
			return
		}
		if n == 0 {
			break
		}
		respbody = append(respbody, buf[:n]...)
	}

	stringbody := string(respbody)
	titlestart := strings.Index(stringbody, "<title>")
	titleend := strings.Index(stringbody, "</title>")
	if titlestart != -1 && titlestart != -1 {
		title := string(respbody[titlestart+7 : titleend])
		title = strings.TrimSpace(title)
		if title != "" {
			parsedurl, err := url.Parse(postedUrl)
			if err == nil {
				postedUrl = parsedurl.Host
			}
			title = "Title: " + html.UnescapeString(title) + " (at " + postedUrl + ")"
			log.Println(title)
			conn.Privmsg(channel, title)
		}
	}
}

func dance(channel string, conn *irc.Conn) {
	conn.Privmsg(channel, ":D-<")
	time.Sleep(500 * time.Millisecond)
	conn.Privmsg(channel, ":D|<")
	time.Sleep(500 * time.Millisecond)
	conn.Privmsg(channel, ":D/<")
}

func haata(channel string, conn *irc.Conn) {
	sets, err := htmlfetch(`http://api.flickr.com/services/rest/?method=flickr.collections.getTree&api_key=` + config.FlickrAPIKey + `&user_id=57321699@N06&collection_id=57276377-72157635417889224`)
	if err != nil {
		return
	}
	var setresp Setresp
	xml.Unmarshal(sets, &setresp)
	randsetindex := random(len(setresp.Sets))
	randset := setresp.Sets[randsetindex].Id

	pics, err := htmlfetch(`http://api.flickr.com/services/rest/?method=flickr.photosets.getPhotos&api_key=` + config.FlickrAPIKey + `&photoset_id=` + randset)
	if err != nil {
		return
	}
	var photoresp Photoresp
	xml.Unmarshal(pics, &photoresp)
	randpic := random(len(photoresp.Photos))
	photostring := string(base58.EncodeBig([]byte{}, big.NewInt(photoresp.Photos[randpic].Id)))
	conn.Privmsg(channel, strings.TrimSpace(setresp.Sets[randsetindex].Title)+`: http://flic.kr/p/`+photostring)
}

func handleMessage(conn *irc.Conn, line *irc.Line) {
	urllist := []string{}
	numlinks := 0

	if strings.HasPrefix(line.Args[1], "!dance") && line.Nick == "sadbox" {
		go dance(line.Args[0], conn)
	} else if strings.HasPrefix(line.Args[1], "!audio") && line.Nick == "sadbox" {
		conn.Privmsg(line.Args[0], "https://sadbox.org/static/audiophile.html")
	} else if strings.HasPrefix(line.Args[1], "!hacking") {
		conn.Privmsg(line.Args[0], "This channel is about keyboards (not hacking), please read the topic.")
	} else if strings.HasPrefix(line.Args[1], "!help") {
		conn.Privmsg(line.Args[0], helpstring)
	} else if strings.HasPrefix(line.Args[1], "!haata") {
		go haata(line.Args[0], conn)
	} else if strings.HasPrefix(line.Args[1], "!stats") {
		conn.Privmsg(line.Args[0], "http://sadbox.org/geekhack")
	}

NextWord:
	for _, word := range strings.Split(line.Args[1], " ") {
		word = strings.TrimSpace(word)
		if urlRegex.MatchString(word) {
			for _, subUrl := range urllist {
				if subUrl == word {
					continue NextWord
				}
			}
			numlinks++
			if numlinks > 3 {
				break
			}
			urllist = append(urllist, word)
			go sendUrl(line.Args[0], word, conn)
		}

	}
	db, err := sql.Open("mysql", config.DBConn)
	if err != nil {
		log.Println(err)
	}
	defer db.Close()
	_, err = db.Exec("insert into messages (Nick, Ident, Host, Src, Cmd, Channel, Message, Time) values (?, ?, ?, ?, ?, ?, ?, ?)", line.Nick, line.Ident, line.Host, line.Src, line.Cmd, line.Args[0], line.Args[1], line.Time)
	if err != nil {
		log.Println(err)
	}
}

func init() {
	log.Println("Starting sadbot")

	flag.Parse()

	if regexErr != nil {
		log.Panic(regexErr)
	}
	if httpRegexErr != nil {
		log.Panic(httpRegexErr)
	}

	xmlFile, err := ioutil.ReadFile("config.xml")
	if err != nil {
		log.Fatal(err)
	}
	xml.Unmarshal(xmlFile, &config)
}

func main() {
	log.Printf("Joining channel %s", config.Channel)
	log.Printf("Nick: %s Ident: %s FullName: %s", config.Nick, config.Ident, config.FullName)

	c := irc.SimpleClient(config.Nick, config.Ident, config.FullName)

	c.AddHandler(irc.CONNECTED,
		func(conn *irc.Conn, line *irc.Line) {
			conn.Join(config.Channel)
			conn.Privmsg("nickserv", "identify "+config.Nick+" "+config.IRCPass)
			log.Println("Connected!")
		})

	quit := make(chan bool)

	c.AddHandler(irc.DISCONNECTED,
		func(conn *irc.Conn, line *irc.Line) { quit <- true })

	c.AddHandler("PRIVMSG", handleMessage)

	if err := c.Connect("irc.freenode.net"); err != nil {
		log.Fatalln("Connection error: %s\n", err)
	}

	<-quit
}
<|endoftext|>


<commit_before>package marathon

import (
	"encoding/json"
	"github.com/QubitProducts/bamboo/configuration"
	"io/ioutil"
	"net/http"
	"sort"
	"strings"
)

// Describes an app process running
type Task struct {
	Host  string
	Port  int
	Ports []int
}

// A health check on the application
type HealthCheck struct {
	// One of TCP, HTTP or COMMAND
	Protocol string
	// The path (if Protocol is HTTP)
	Path string
	// The position of the port targeted in the ports array
	PortIndex int
}

// An app may have multiple processes
type App struct {
	Id              string
	EscapedId       string
	HealthCheckPath string
	HealthChecks    []HealthCheck
	Tasks           []Task
	ServicePort     int
	ServicePorts    []int
	Env             map[string]string
	Labels          map[string]string
}

type AppList []App

func (slice AppList) Len() int {
	return len(slice)
}

func (slice AppList) Less(i, j int) bool {
	return slice[i].Id < slice[j].Id
}

func (slice AppList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

type marathonTaskList []marathonTask

type marathonTasks struct {
	Tasks marathonTaskList `json:"tasks"`
}

type marathonTask struct {
	AppId        string
	Id           string
	Host         string
	Ports        []int
	ServicePorts []int
	StartedAt    string
	StagedAt     string
	Version      string
}

func (slice marathonTaskList) Len() int {
	return len(slice)
}

func (slice marathonTaskList) Less(i, j int) bool {
	return slice[i].Id < slice[j].Id
}

func (slice marathonTaskList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

type marathonApps struct {
	Apps []marathonApp `json:"apps"`
}

type marathonApp struct {
	Id           string                `json:"id"`
	HealthChecks []marathonHealthCheck `json:"healthChecks"`
	Ports        []int                 `json:"ports"`
	Env          map[string]string     `json:"env"`
	Labels       map[string]string     `json:"labels"`
}

type marathonHealthCheck struct {
	Path      string `json:"path"`
	Protocol  string `json:"protocol"`
	PortIndex int    `json:"portIndex"`
}

func fetchMarathonApps(endpoint string, conf *configuration.Configuration) (map[string]marathonApp, error) {
	client := &http.Client{}
	req, _ := http.NewRequest("GET", endpoint+"/v2/apps", nil)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")
	if len(conf.Marathon.User) > 0 && len(conf.Marathon.Password) > 0 {
		req.SetBasicAuth(conf.Marathon.User, conf.Marathon.Password)
	}
	response, err := client.Do(req)

	if err != nil {
		return nil, err
	}

	defer response.Body.Close()
	var appResponse marathonApps

	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(contents, &appResponse)
	if err != nil {
		return nil, err
	}

	dataById := map[string]marathonApp{}

	for _, appConfig := range appResponse.Apps {
		dataById[appConfig.Id] = appConfig
	}

	return dataById, nil
}

func fetchTasks(endpoint string, conf *configuration.Configuration) (map[string]marathonTaskList, error) {
	client := &http.Client{}
	req, _ := http.NewRequest("GET", endpoint+"/v2/tasks", nil)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")
	if len(conf.Marathon.User) > 0 && len(conf.Marathon.Password) > 0 {
		req.SetBasicAuth(conf.Marathon.User, conf.Marathon.Password)
	}
	response, err := client.Do(req)

	var tasks marathonTasks

	if err != nil {
		return nil, err
	}

	contents, err := ioutil.ReadAll(response.Body)
	defer response.Body.Close()
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(contents, &tasks)
	if err != nil {
		return nil, err
	}

	taskList := tasks.Tasks
	sort.Sort(taskList)

	tasksById := map[string]marathonTaskList{}
	for _, task := range taskList {
		if tasksById[task.AppId] == nil {
			tasksById[task.AppId] = marathonTaskList{}
		}
		tasksById[task.AppId] = append(tasksById[task.AppId], task)
	}

	for _, task_list := range tasksById {
		sort.Sort(task_list)
	}

	return tasksById, nil
}

func createApps(tasksById map[string]marathonTaskList, marathonApps map[string]marathonApp) AppList {

	apps := AppList{}

	for appId, mApp := range marathonApps {

		// Strip slash
		appPath := appId[1:len(appId)]

		// build App from marathonApp
		app := App{
			Id:              appPath,
			EscapedId:       strings.Replace(appId, "/", "::", -1),
			HealthCheckPath: parseHealthCheckPath(mApp.HealthChecks),
			Env:             mApp.Env,
			Labels:          mApp.Labels,
		}

		app.HealthChecks = make([]HealthCheck, 0, len(mApp.HealthChecks))
		for _, marathonCheck := range mApp.HealthChecks {
			check := HealthCheck{
				Protocol:  marathonCheck.Protocol,
				Path:      marathonCheck.Path,
				PortIndex: marathonCheck.PortIndex,
			}
			app.HealthChecks = append(app.HealthChecks, check)
		}

		if len(mApp.Ports) > 0 {
			app.ServicePort = mApp.Ports[0]
			app.ServicePorts = mApp.Ports
		}

		// build Tasks for this App
		tasks := []Task{}
		for _, mTask := range tasksById[appId] {
			if len(mTask.Ports) > 0 {
				t := Task{
					Host:  mTask.Host,
					Port:  mTask.Ports[0],
					Ports: mTask.Ports,
				}
				tasks = append(tasks, t)
			}
		}
		app.Tasks = tasks

		apps = append(apps, app)
	}
	return apps
}

func parseHealthCheckPath(checks []marathonHealthCheck) string {
	for _, check := range checks {
		if check.Protocol != "HTTP" {
			continue
		}
		return check.Path
	}
	return ""
}

/*
	Apps returns a struct that describes Marathon current app and their
	sub tasks information.

	Parameters:
		endpoint: Marathon HTTP endpoint, e.g. http://localhost:8080
*/
func FetchApps(maraconf configuration.Marathon, conf *configuration.Configuration) (AppList, error) {

	var applist AppList
	var err error

	// try all configured endpoints until one succeeds
	for _, url := range maraconf.Endpoints() {
		applist, err = _fetchApps(url, conf)
		if err == nil {
			return applist, err
		}
	}
	// return last error
	return nil, err
}

func _fetchApps(url string, conf *configuration.Configuration) (AppList, error) {
	tasks, err := fetchTasks(url, conf)
	if err != nil {
		return nil, err
	}

	marathonApps, err := fetchMarathonApps(url, conf)
	if err != nil {
		return nil, err
	}

	apps := createApps(tasks, marathonApps)
	sort.Sort(apps)
	return apps, nil
}
<commit_msg>check for string has prefix<commit_after>package marathon

import (
	"encoding/json"
	"github.com/QubitProducts/bamboo/configuration"
	"io/ioutil"
	"net/http"
	"sort"
	"strings"
)

// Describes an app process running
type Task struct {
	Host  string
	Port  int
	Ports []int
}

// A health check on the application
type HealthCheck struct {
	// One of TCP, HTTP or COMMAND
	Protocol string
	// The path (if Protocol is HTTP)
	Path string
	// The position of the port targeted in the ports array
	PortIndex int
}

// An app may have multiple processes
type App struct {
	Id              string
	EscapedId       string
	HealthCheckPath string
	HealthChecks    []HealthCheck
	Tasks           []Task
	ServicePort     int
	ServicePorts    []int
	Env             map[string]string
	Labels          map[string]string
}

type AppList []App

func (slice AppList) Len() int {
	return len(slice)
}

func (slice AppList) Less(i, j int) bool {
	return slice[i].Id < slice[j].Id
}

func (slice AppList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

type marathonTaskList []marathonTask

type marathonTasks struct {
	Tasks marathonTaskList `json:"tasks"`
}

type marathonTask struct {
	AppId        string
	Id           string
	Host         string
	Ports        []int
	ServicePorts []int
	StartedAt    string
	StagedAt     string
	Version      string
}

func (slice marathonTaskList) Len() int {
	return len(slice)
}

func (slice marathonTaskList) Less(i, j int) bool {
	return slice[i].Id < slice[j].Id
}

func (slice marathonTaskList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

type marathonApps struct {
	Apps []marathonApp `json:"apps"`
}

type marathonApp struct {
	Id           string                `json:"id"`
	HealthChecks []marathonHealthCheck `json:"healthChecks"`
	Ports        []int                 `json:"ports"`
	Env          map[string]string     `json:"env"`
	Labels       map[string]string     `json:"labels"`
}

type marathonHealthCheck struct {
	Path      string `json:"path"`
	Protocol  string `json:"protocol"`
	PortIndex int    `json:"portIndex"`
}

func fetchMarathonApps(endpoint string, conf *configuration.Configuration) (map[string]marathonApp, error) {
	client := &http.Client{}
	req, _ := http.NewRequest("GET", endpoint+"/v2/apps", nil)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")
	if len(conf.Marathon.User) > 0 && len(conf.Marathon.Password) > 0 {
		req.SetBasicAuth(conf.Marathon.User, conf.Marathon.Password)
	}
	response, err := client.Do(req)

	if err != nil {
		return nil, err
	}

	defer response.Body.Close()
	var appResponse marathonApps

	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(contents, &appResponse)
	if err != nil {
		return nil, err
	}

	dataById := map[string]marathonApp{}

	for _, appConfig := range appResponse.Apps {
		dataById[appConfig.Id] = appConfig
	}

	return dataById, nil
}

func fetchTasks(endpoint string, conf *configuration.Configuration) (map[string]marathonTaskList, error) {
	client := &http.Client{}
	req, _ := http.NewRequest("GET", endpoint+"/v2/tasks", nil)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")
	if len(conf.Marathon.User) > 0 && len(conf.Marathon.Password) > 0 {
		req.SetBasicAuth(conf.Marathon.User, conf.Marathon.Password)
	}
	response, err := client.Do(req)

	var tasks marathonTasks

	if err != nil {
		return nil, err
	}

	contents, err := ioutil.ReadAll(response.Body)
	defer response.Body.Close()
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(contents, &tasks)
	if err != nil {
		return nil, err
	}

	taskList := tasks.Tasks
	sort.Sort(taskList)

	tasksById := map[string]marathonTaskList{}
	for _, task := range taskList {
		if tasksById[task.AppId] == nil {
			tasksById[task.AppId] = marathonTaskList{}
		}
		tasksById[task.AppId] = append(tasksById[task.AppId], task)
	}

	for _, task_list := range tasksById {
		sort.Sort(task_list)
	}

	return tasksById, nil
}

func createApps(tasksById map[string]marathonTaskList, marathonApps map[string]marathonApp) AppList {

	apps := AppList{}

	for appId, mApp := range marathonApps {

        appPath := appId

		// Strip slash
		if strings.HasPrefix(appId, "/") {
		    appPath := appId[1:len(appId)]
		}

		// build App from marathonApp
		app := App{
			Id:              appPath,
			EscapedId:       strings.Replace(appId, "/", "::", -1),
			HealthCheckPath: parseHealthCheckPath(mApp.HealthChecks),
			Env:             mApp.Env,
			Labels:          mApp.Labels,
		}

		app.HealthChecks = make([]HealthCheck, 0, len(mApp.HealthChecks))
		for _, marathonCheck := range mApp.HealthChecks {
			check := HealthCheck{
				Protocol:  marathonCheck.Protocol,
				Path:      marathonCheck.Path,
				PortIndex: marathonCheck.PortIndex,
			}
			app.HealthChecks = append(app.HealthChecks, check)
		}

		if len(mApp.Ports) > 0 {
			app.ServicePort = mApp.Ports[0]
			app.ServicePorts = mApp.Ports
		}

		// build Tasks for this App
		tasks := []Task{}
		for _, mTask := range tasksById[appId] {
			if len(mTask.Ports) > 0 {
				t := Task{
					Host:  mTask.Host,
					Port:  mTask.Ports[0],
					Ports: mTask.Ports,
				}
				tasks = append(tasks, t)
			}
		}
		app.Tasks = tasks

		apps = append(apps, app)
	}
	return apps
}

func parseHealthCheckPath(checks []marathonHealthCheck) string {
	for _, check := range checks {
		if check.Protocol != "HTTP" {
			continue
		}
		return check.Path
	}
	return ""
}

/*
	Apps returns a struct that describes Marathon current app and their
	sub tasks information.

	Parameters:
		endpoint: Marathon HTTP endpoint, e.g. http://localhost:8080
*/
func FetchApps(maraconf configuration.Marathon, conf *configuration.Configuration) (AppList, error) {

	var applist AppList
	var err error

	// try all configured endpoints until one succeeds
	for _, url := range maraconf.Endpoints() {
		applist, err = _fetchApps(url, conf)
		if err == nil {
			return applist, err
		}
	}
	// return last error
	return nil, err
}

func _fetchApps(url string, conf *configuration.Configuration) (AppList, error) {
	tasks, err := fetchTasks(url, conf)
	if err != nil {
		return nil, err
	}

	marathonApps, err := fetchMarathonApps(url, conf)
	if err != nil {
		return nil, err
	}

	apps := createApps(tasks, marathonApps)
	sort.Sort(apps)
	return apps, nil
}
<|endoftext|>