<commit_before>/*
 * Copyright 2016 Xuyuan Pang
 * Author: Xuyuan Pang
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package hodor

import (
	"log"
	"net/http"
	"time"
)

// Filter interface
type Filter interface {
	Do(http.Handler) http.Handler
}

// FilterFunc type
type FilterFunc func(next http.Handler) http.Handler

// Do calls FilterFunc
func (f FilterFunc) Do(next http.Handler) http.Handler {
	return f(next)
}

var emptyFilter FilterFunc = func(next http.Handler) http.Handler {
	return next
}

// MergeFilters merges multi fileers into a single one.
func MergeFilters(filters ...Filter) Filter {
	return FilterFunc(
		func(next http.Handler) http.Handler {
			for i := len(filters) - 1; i >= 0; i-- {
				next = filters[i].Do(next)
			}
			return next
		})
}

// LogFilter new filter
func LogFilter(l *log.Logger) FilterFunc {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(
			func(w http.ResponseWriter, r *http.Request) {
				start := time.Now()
				l.Printf("%s %s", r.Method, r.RemoteAddr)
				next.ServeHTTP(w, r)
				status := w.(ResponseWriter).Status()
				l.Printf("%d %s %s", status, http.StatusText(status), time.Since(start))
			})
	}
}
<commit_msg>Log request info<commit_after>/*
 * Copyright 2016 Xuyuan Pang
 * Author: Xuyuan Pang
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package hodor

import (
	"log"
	"net/http"
	"time"
)

// Filter interface
type Filter interface {
	Do(http.Handler) http.Handler
}

// FilterFunc type
type FilterFunc func(next http.Handler) http.Handler

// Do calls FilterFunc
func (f FilterFunc) Do(next http.Handler) http.Handler {
	return f(next)
}

var emptyFilter FilterFunc = func(next http.Handler) http.Handler {
	return next
}

// MergeFilters merges multi fileers into a single one.
func MergeFilters(filters ...Filter) Filter {
	return FilterFunc(
		func(next http.Handler) http.Handler {
			for i := len(filters) - 1; i >= 0; i-- {
				next = filters[i].Do(next)
			}
			return next
		})
}

// LogFilter new filter
func LogFilter(l *log.Logger) FilterFunc {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(
			func(w http.ResponseWriter, r *http.Request) {
				addr := r.Header.Get("X-Real-IP")
				if addr == "" {
					addr = r.Header.Get("X-Forwarded-For")
					if addr == "" {
						addr = r.RemoteAddr
					}
				}
				path := r.URL.Path
				if r.URL.RawQuery != "" {
					path += "?" + r.URL.RawQuery
				}
				start := time.Now()
				l.Printf("%s %s %s", r.Method, path, addr)
				next.ServeHTTP(w, r)
				status := w.(ResponseWriter).Status()
				l.Printf("%d %s %s", status, http.StatusText(status), time.Since(start))
			})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"

	"gopkg.in/mgo.v2"
)

// Period hold the information of an activity
type Period struct {
	Type string `json:"type"` // pomodoro or rest
}

// Periods holds an array of Period type.
//
type Periods []Period

type standardResponse struct {
	Status           int    `json:"status"`
	DeveloperMessage string `json:"developerMessage"`
	UserMessage      string `json:"userMessage"`
}

func createStandardResponse(status int, devMsg, userMsg string) standardResponse {
	return standardResponse{
		Status:           status,
		DeveloperMessage: devMsg,
		UserMessage:      userMsg,
	}
}

// createPeriod handler lets you create a period obect.
//
func createPeriod(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		developerMessage := "createPeriod only POST requests are supported for this route."
		userMessage := "Oops, something went wrong, we are unable to save your data right now."
		response := createStandardResponse(400, developerMessage, userMessage)
		w.WriteHeader(http.StatusBadRequest)

		log.Println("sending response")

		if err := renderJson(w, response); err != nil {
			log.Println(err)
		}
		return
	}

	var strType string

	if strType = r.FormValue("type"); len(strType) == 0 {
		developerMessage := "createPeriod had empty type parameter, unable to create period."
		userMessage := "Oops, something went wrong, we are unable to save your data right now."
		response := createStandardResponse(400, developerMessage, userMessage)
		w.WriteHeader(http.StatusBadRequest)
		log.Println("sending response")
		if err := renderJson(w, response); err != nil {
			log.Println(err)
		}
		return
	} else {
		if strType != "pomodoro" && strType != "rest" {
			developerMessage := "createPeriod has wrong type, unable to create period."
			userMessage := "Oops, something went wrong, we are unable to save your data right now."
			response := createStandardResponse(400, developerMessage, userMessage)
			w.WriteHeader(http.StatusBadRequest)
			log.Println("sending response")
			if err := renderJson(w, response); err != nil {
				log.Println(err)
			}
			return
		}
	}

	var uri string
	uri = getMongoURI()

	log.Println("uri found: ", uri)

	var err error
	var session *mgo.Session

	log.Println("start dial")
	if session, err = mgo.Dial(uri); err != nil {
		panic(err)
	}
	log.Println("dial done")

	defer session.Close()
	log.Println("setting session mode")
	session.SetMode(mgo.Monotonic, true)

	log.Println("trying to get pomotime database")
	collection := session.DB("").C("period")

	log.Println("start insert to period entity")
	var p Period
	p = Period{strType}

	if err = collection.Insert(&p); err != nil {
		log.Fatal(err)
	}

	data := struct {
		Status           int    `json:"status"`
		DeveloperMessage string `json:"developerMessage"`
		UserMessage      string `json:"userMessage"`
	}{
		Status:           200,
		DeveloperMessage: "app correctly stored period in database.",
		UserMessage:      "your data was stored correctly.",
	}

	log.Println("sending response")
	if err := renderJson(w, data); err != nil {
		log.Println(err)
	}
}

// getPeriods handler returns a list of periods.
//
func getPeriods(w http.ResponseWriter, r *http.Request) {

	if r.Method != "GET" {
		developerMessage := "getPeriods: only GET requests are supported for this route."
		userMessage := "Oops, something went wrong, we are unable to get your data right now."
		response := createStandardResponse(400, developerMessage, userMessage)
		w.WriteHeader(http.StatusBadRequest)
		log.Println("sending response")
		if err := renderJson(w, response); err != nil {
			log.Println(err)
		}
		return
	}

	var uri string
	uri = getMongoURI()

	log.Println("uri found: ", uri)

	var err error
	var session *mgo.Session

	log.Println("start dial")
	if session, err = mgo.Dial(uri); err != nil {
		panic(err)
	}
	log.Println("dial done")

	defer session.Close()
	log.Println("setting session mode")
	session.SetMode(mgo.Monotonic, true)

	log.Println("trying database")
	collection := session.DB("").C("period")

	var results []Period

	log.Println("start search for all period objects")
	if err = collection.Find(nil).All(&results); err != nil {
		log.Fatal(err)
	}

	data := struct {
		Status           int     `json:"status"`
		DeveloperMessage string  `json:"developerMessage"`
		UserMessage      string  `json:"userMessage"`
		Periods          Periods `json:"periods"`
	}{
		Status:           200,
		DeveloperMessage: "returning all periods from database.",
		UserMessage:      "",
		Periods:          results,
	}

	log.Println("sending response")
	if err := renderJson(w, data); err != nil {
		log.Println(err)
	}
}
<commit_msg>add comments<commit_after>package main

import (
	"log"
	"net/http"

	"gopkg.in/mgo.v2"
)

// Period hold the information of an activity
type Period struct {
	Type string `json:"type"` // pomodoro or rest
}

// Periods holds an array of Period type.
//
type Periods []Period

type standardResponse struct {
	Status           int    `json:"status"`
	DeveloperMessage string `json:"developerMessage"`
	UserMessage      string `json:"userMessage"`
}

// createStandardResponse creates a standard response object with respect to the
// status, the developer message and the user message.
//
func createStandardResponse(status int, devMsg, userMsg string) standardResponse {
	return standardResponse{
		Status:           status,
		DeveloperMessage: devMsg,
		UserMessage:      userMsg,
	}
}

// createPeriod handler lets you create a period obect.
//
func createPeriod(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		developerMessage := "createPeriod only POST requests are supported for this route."
		userMessage := "Oops, something went wrong, we are unable to save your data right now."
		response := createStandardResponse(400, developerMessage, userMessage)
		w.WriteHeader(http.StatusBadRequest)
		log.Println("sending response")

		if err := renderJson(w, response); err != nil {
			log.Println(err)
		}
		return
	}

	var strType string

	if strType = r.FormValue("type"); len(strType) == 0 {
		developerMessage := "createPeriod had empty type parameter, unable to create period."
		userMessage := "Oops, something went wrong, we are unable to save your data right now."
		response := createStandardResponse(400, developerMessage, userMessage)
		w.WriteHeader(http.StatusBadRequest)
		log.Println("sending response")
		if err := renderJson(w, response); err != nil {
			log.Println(err)
		}
		return
	} else {
		if strType != "pomodoro" && strType != "rest" {
			developerMessage := "createPeriod has wrong type, unable to create period."
			userMessage := "Oops, something went wrong, we are unable to save your data right now."
			response := createStandardResponse(400, developerMessage, userMessage)
			w.WriteHeader(http.StatusBadRequest)
			log.Println("sending response")
			if err := renderJson(w, response); err != nil {
				log.Println(err)
			}
			return
		}
	}

	var uri string
	uri = getMongoURI()

	log.Println("uri found: ", uri)

	var err error
	var session *mgo.Session

	log.Println("start dial")
	if session, err = mgo.Dial(uri); err != nil {
		panic(err)
	}
	log.Println("dial done")

	defer session.Close()
	log.Println("setting session mode")
	session.SetMode(mgo.Monotonic, true)

	log.Println("trying to get pomotime database")
	collection := session.DB("").C("period")

	log.Println("start insert to period entity")
	var p Period
	p = Period{strType}

	if err = collection.Insert(&p); err != nil {
		log.Fatal(err)
	}

	data := struct {
		Status           int    `json:"status"`
		DeveloperMessage string `json:"developerMessage"`
		UserMessage      string `json:"userMessage"`
	}{
		Status:           200,
		DeveloperMessage: "app correctly stored period in database.",
		UserMessage:      "your data was stored correctly.",
	}

	log.Println("sending response")
	if err := renderJson(w, data); err != nil {
		log.Println(err)
	}
}

// getPeriods handler returns a list of periods.
//
func getPeriods(w http.ResponseWriter, r *http.Request) {

	if r.Method != "GET" {
		developerMessage := "getPeriods: only GET requests are supported for this route."
		userMessage := "Oops, something went wrong, we are unable to get your data right now."
		response := createStandardResponse(400, developerMessage, userMessage)
		w.WriteHeader(http.StatusBadRequest)
		log.Println("sending response")
		if err := renderJson(w, response); err != nil {
			log.Println(err)
		}
		return
	}

	var uri string
	uri = getMongoURI()

	log.Println("uri found: ", uri)

	var err error
	var session *mgo.Session

	log.Println("start dial")
	if session, err = mgo.Dial(uri); err != nil {
		panic(err)
	}
	log.Println("dial done")

	defer session.Close()
	log.Println("setting session mode")
	session.SetMode(mgo.Monotonic, true)

	log.Println("trying database")
	collection := session.DB("").C("period")

	var results []Period

	log.Println("start search for all period objects")
	if err = collection.Find(nil).All(&results); err != nil {
		log.Fatal(err)
	}

	data := struct {
		Status           int     `json:"status"`
		DeveloperMessage string  `json:"developerMessage"`
		UserMessage      string  `json:"userMessage"`
		Periods          Periods `json:"periods"`
	}{
		Status:           200,
		DeveloperMessage: "returning all periods from database.",
		UserMessage:      "",
		Periods:          results,
	}

	log.Println("sending response")
	if err := renderJson(w, data); err != nil {
		log.Println(err)
	}
}
<|endoftext|>


<commit_before>package fleet

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
)

const (
	unitsPath = "/units"
)

type Unit struct {
	Name         string        `json:"name,omitempty"`
	Options      []*UnitOption `json:"options,omitempty"`
	DesiredState string        `json:"desiredState,omitempty"`
	CurrentState string        `json:"currentState,omitempty"`
	MachineID    string        `json:"machineID,omitempty"`
}

type UnitOption struct {
	Section string `json:"section,omitempty"`
	Name    string `json:"name,omitempty"`
	Value   string `json:"value,omitempty"`
}

type unitResponse struct {
	Units []Unit `json:"units,omitempty"`
}

func (c *Client) Units() ([]Unit, error) {
	var unitsRes unitResponse

	req, err := http.NewRequest("GET", c.URL+basePath+unitsPath, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/json")

	httpClient := http.Client{}
	res, err := httpClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	err = json.NewDecoder(res.Body).Decode(&unitsRes)
	if err != nil {
		return nil, err
	}

	return unitsRes.Units, nil
}

func (c *Client) Submit(name string, opts []*UnitOption) error {
	unit := Unit{
		Options:      opts,
		DesiredState: "inactive",
	}

	j, err := json.Marshal(unit)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("PUT", c.URL+basePath+unitsPath+"/"+name, bytes.NewReader(j))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")

	httpClient := http.Client{}
	res, err := httpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	if res.StatusCode == 201 {
		return nil
	}

	return nil
}

func (c *Client) Load(name string) error {
	unit := Unit{
		DesiredState: "loaded",
	}

	j, err := json.Marshal(unit)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("PUT", c.URL+basePath+unitsPath+"/"+name, bytes.NewReader(j))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")

	httpClient := http.Client{}
	res, err := httpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	switch res.StatusCode {
	case 204:
		return nil
	case 400:
		// Attempting to modify an Unit with an invalid entity
		return errors.New("400 Bad Request")
	case 409:
		// Attempting to create an entity without options
		return errors.New("409 Conflict")
	default:
		message := fmt.Sprintf("%d Faild to load an unit", res.StatusCode)
		return errors.New(message)
	}
}

func (c *Client) Start(name string)   {}
func (c *Client) Stop(name string)    {}
func (c *Client) Unload(name string)  {}
func (c *Client) Destroy(name string) {}
<commit_msg>Add handling 400 and 409 response<commit_after>package fleet

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
)

const (
	unitsPath = "/units"
)

type Unit struct {
	Name         string        `json:"name,omitempty"`
	Options      []*UnitOption `json:"options,omitempty"`
	DesiredState string        `json:"desiredState,omitempty"`
	CurrentState string        `json:"currentState,omitempty"`
	MachineID    string        `json:"machineID,omitempty"`
}

type UnitOption struct {
	Section string `json:"section,omitempty"`
	Name    string `json:"name,omitempty"`
	Value   string `json:"value,omitempty"`
}

type unitResponse struct {
	Units []Unit `json:"units,omitempty"`
}

func (c *Client) Units() ([]Unit, error) {
	var unitsRes unitResponse

	req, err := http.NewRequest("GET", c.URL+basePath+unitsPath, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/json")

	httpClient := http.Client{}
	res, err := httpClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	err = json.NewDecoder(res.Body).Decode(&unitsRes)
	if err != nil {
		return nil, err
	}

	return unitsRes.Units, nil
}

func (c *Client) Submit(name string, opts []*UnitOption) error {
	unit := Unit{
		Options:      opts,
		DesiredState: "inactive",
	}

	j, err := json.Marshal(unit)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("PUT", c.URL+basePath+unitsPath+"/"+name, bytes.NewReader(j))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")

	httpClient := http.Client{}
	res, err := httpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	if res.StatusCode == 201 {
		return nil
	}

	switch res.StatusCode {
	case 201:
		return nil
	case 400:
		// Attempting to create an Unit with an invalid entity
		return errors.New("400 Bad Request")
	case 409:
		// Attempting to create an entity without options
		return errors.New("409 Conflict")
	default:
		message := fmt.Sprintf("%d Faild to load an unit", res.StatusCode)
		return errors.New(message)
	}

	return nil
}

func (c *Client) Load(name string) error {
	unit := Unit{
		DesiredState: "loaded",
	}

	j, err := json.Marshal(unit)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("PUT", c.URL+basePath+unitsPath+"/"+name, bytes.NewReader(j))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")

	httpClient := http.Client{}
	res, err := httpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	switch res.StatusCode {
	case 204:
		return nil
	case 400:
		// Attempting to modify an Unit with an invalid entity
		return errors.New("400 Bad Request")
	case 409:
		// Attempting to create an entity without options
		return errors.New("409 Conflict")
	default:
		message := fmt.Sprintf("%d Faild to load an unit", res.StatusCode)
		return errors.New(message)
	}
}

func (c *Client) Start(name string)   {}
func (c *Client) Stop(name string)    {}
func (c *Client) Unload(name string)  {}
func (c *Client) Destroy(name string) {}
<|endoftext|>


<commit_before>package flow

import (
	"fmt"
	"sync"
	"time"

	"github.com/google/gopacket"

	"github.com/redhat-cip/skydive/logging"
)

type FlowTable struct {
	lock  sync.RWMutex
	table map[string]*Flow
}

type FlowTableAsyncNotificaionUpdate interface {
	AsyncNotificaionUpdate(every time.Duration)
}

func NewFlowTable() *FlowTable {
	return &FlowTable{table: make(map[string]*Flow)}
}

func (ft *FlowTable) Update(flows []*Flow) {
	ft.lock.Lock()
	defer ft.lock.Unlock()
	for _, f := range flows {
		_, found := ft.table[f.UUID]
		if !found {
			ft.table[f.UUID] = f
		} else if f.UUID != ft.table[f.UUID].UUID {
			logging.GetLogger().Error("FlowTable Collision ", f.UUID, ft.table[f.UUID].UUID)
		}
	}
}

type ExpireFunc func(f *Flow)

func (ft *FlowTable) AsyncExpire(fn ExpireFunc, every time.Duration) {
	ticker := time.NewTicker(every)
	defer func() {
		ticker.Stop()
	}()
	for {
		select {
		case now := <-ticker.C:
			flowTableSzBefore := len(ft.table)
			expire := now.Unix() - int64((every).Seconds())

			ft.lock.Lock()
			defer ft.lock.Unlock()
			for key, f := range ft.table {
				fs := f.GetStatistics()
				if fs.Last < expire {
					duration := time.Duration(fs.Last - fs.Start)
					logging.GetLogger().Debug("%v Expire flow %s Duration %v", now, f.UUID, duration)
					/* Advise Clients */
					fn(f)
					delete(ft.table, key)
				}
			}
			logging.GetLogger().Debug("%v Expire Flow : removed %v new size %v", now, flowTableSzBefore-len(ft.table), len(ft.table))
		}
	}
}

func (ft *FlowTable) IsExist(f *Flow) bool {
	ft.lock.RLock()
	_, found := ft.table[f.UUID]
	ft.lock.RUnlock()
	return found
}

/* Must be called under ft.lock, ExpireFunc callback function are safe to call ft.Remove() */
func (ft *FlowTable) Remove(f *Flow) {
	if ft.IsExist(f) {
		logging.GetLogger().Info("FlowTable remove flow %s", f.UUID)
		delete(ft.table, f.UUID)
	} else {
		logging.GetLogger().Critical("FlowTable flow %s did not exist ...", f.UUID)
	}
}

func (ft *FlowTable) GetFlow(key string, packet *gopacket.Packet) (flow *Flow, new bool) {
	logging.GetLogger().Debug("flow key %s", key)
	ft.lock.Lock()
	flow, found := ft.table[key]
	if found == false {
		flow = &Flow{}
		ft.table[key] = flow
	}
	ft.lock.Unlock()
	return flow, !found
}

func (ft *FlowTable) JSONFlowConversationEthernetPath() string {
	str := ""
	str += "{"
	//	{"nodes":[{"name":"Myriel","group":1}, ... ],"links":[{"source":1,"target":0,"value":1},...]}

	pathMap := make(map[string]int)
	ethMap := make(map[string]int)
	for _, f := range ft.table {
		_, found := pathMap[f.LayersPath]
		if !found {
			pathMap[f.LayersPath] = len(pathMap)
		}

		ethFlow := f.GetStatistics().Endpoints[FlowEndpointType_ETHERNET.Value()]
		_, found = ethMap[ethFlow.AB.Value]
		if !found {
			ethMap[ethFlow.AB.Value] = len(ethMap)
		}
		_, found = ethMap[ethFlow.BA.Value]
		if !found {
			ethMap[ethFlow.BA.Value] = len(ethMap)
		}
	}
	var strNodes, strLinks string
	i := 0
	strNodes += "\"nodes\":["
	strLinks += "\"links\":["
	for _, f := range ft.table {
		ethFlow := f.GetStatistics().Endpoints[FlowEndpointType_ETHERNET.Value()]
		strNodes += fmt.Sprintf("{\"name\":\"%s\",\"group\":%d},", ethFlow.AB.Value, pathMap[f.LayersPath])
		strNodes += fmt.Sprintf("{\"name\":\"%s\",\"group\":%d}", ethFlow.BA.Value, pathMap[f.LayersPath])

		strLinks += fmt.Sprintf("{\"source\":%d,\"target\":%d,\"value\":%d}", ethMap[ethFlow.AB.Value], ethMap[ethFlow.BA.Value], ethFlow.AB.Bytes+ethFlow.BA.Bytes)

		if i < len(ft.table)-1 {
			strNodes += ","
			strLinks += ","
		} else {
			strNodes += "]"
			strLinks += "]"
		}
		i++
	}
	str += strNodes + "," + strLinks
	str += "}"
	return str
}

func (ft *FlowTable) NewFlowTableFromFlows(flows []*Flow) *FlowTable {
	nft := NewFlowTable()
	nft.Update(flows)
	return nft
}

/* Return a new FlowTable that contain <last> active flows */
func (ft *FlowTable) FilterLast(last time.Duration) []*Flow {
	ft.lock.RLock()
	defer ft.lock.RUnlock()
	var flows []*Flow
	selected := time.Now().Unix() - int64((last).Seconds())
	for _, f := range ft.table {
		fs := f.GetStatistics()
		if fs.Last >= selected {
			flows = append(flows, f)
		}
	}
	return flows
}
<commit_msg>[flow][table] fixup JSON Conversation output<commit_after>package flow

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/google/gopacket"

	"github.com/redhat-cip/skydive/logging"
)

type FlowTable struct {
	lock  sync.RWMutex
	table map[string]*Flow
}

type FlowTableAsyncNotificaionUpdate interface {
	AsyncNotificaionUpdate(every time.Duration)
}

func NewFlowTable() *FlowTable {
	return &FlowTable{table: make(map[string]*Flow)}
}

func (ft *FlowTable) String() string {
	return fmt.Sprintf("%d flows", len(ft.table))
}

func (ft *FlowTable) Update(flows []*Flow) {
	ft.lock.Lock()
	defer ft.lock.Unlock()
	for _, f := range flows {
		_, found := ft.table[f.UUID]
		if !found {
			ft.table[f.UUID] = f
		} else if f.UUID != ft.table[f.UUID].UUID {
			logging.GetLogger().Error("FlowTable Collision ", f.UUID, ft.table[f.UUID].UUID)
		}
	}
}

type ExpireFunc func(f *Flow)

func (ft *FlowTable) AsyncExpire(fn ExpireFunc, every time.Duration) {
	ticker := time.NewTicker(every)
	defer func() {
		ticker.Stop()
	}()
	for {
		select {
		case now := <-ticker.C:
			flowTableSzBefore := len(ft.table)
			expire := now.Unix() - int64((every).Seconds())

			ft.lock.Lock()
			defer ft.lock.Unlock()
			for key, f := range ft.table {
				fs := f.GetStatistics()
				if fs.Last < expire {
					duration := time.Duration(fs.Last - fs.Start)
					logging.GetLogger().Debug("%v Expire flow %s Duration %v", now, f.UUID, duration)
					/* Advise Clients */
					fn(f)
					delete(ft.table, key)
				}
			}
			logging.GetLogger().Debug("%v Expire Flow : removed %v new size %v", now, flowTableSzBefore-len(ft.table), len(ft.table))
		}
	}
}

func (ft *FlowTable) IsExist(f *Flow) bool {
	ft.lock.RLock()
	_, found := ft.table[f.UUID]
	ft.lock.RUnlock()
	return found
}

/* Must be called under ft.lock, ExpireFunc callback function are safe to call ft.Remove() */
func (ft *FlowTable) Remove(f *Flow) {
	if ft.IsExist(f) {
		logging.GetLogger().Info("FlowTable remove flow %s", f.UUID)
		delete(ft.table, f.UUID)
	} else {
		logging.GetLogger().Critical("FlowTable flow %s did not exist ...", f.UUID)
	}
}

func (ft *FlowTable) GetFlow(key string, packet *gopacket.Packet) (flow *Flow, new bool) {
	ft.lock.Lock()
	flow, found := ft.table[key]
	if found == false {
		flow = &Flow{}
		ft.table[key] = flow
	}
	ft.lock.Unlock()
	return flow, !found
}

func (ft *FlowTable) JSONFlowConversationEthernetPath() string {
	str := ""
	str += "{"
	//	{"nodes":[{"name":"Myriel","group":1}, ... ],"links":[{"source":1,"target":0,"value":1},...]}

	var strNodes, strLinks string
	strNodes += "\"nodes\":["
	strLinks += "\"links\":["
	pathMap := make(map[string]int)
	ethMap := make(map[string]int)
	for _, f := range ft.table {
		_, found := pathMap[f.LayersPath]
		if !found {
			pathMap[f.LayersPath] = len(pathMap)
		}

		ethFlow := f.GetStatistics().Endpoints[FlowEndpointType_ETHERNET.Value()]
		if _, found := ethMap[ethFlow.AB.Value]; !found {
			ethMap[ethFlow.AB.Value] = len(ethMap)
			strNodes += fmt.Sprintf("{\"name\":\"%s\",\"group\":%d},", ethFlow.AB.Value, pathMap[f.LayersPath])
		}
		if _, found := ethMap[ethFlow.BA.Value]; !found {
			ethMap[ethFlow.BA.Value] = len(ethMap)
			strNodes += fmt.Sprintf("{\"name\":\"%s\",\"group\":%d},", ethFlow.BA.Value, pathMap[f.LayersPath])
		}
		strLinks += fmt.Sprintf("{\"source\":%d,\"target\":%d,\"value\":%d},", ethMap[ethFlow.AB.Value], ethMap[ethFlow.BA.Value], ethFlow.AB.Bytes+ethFlow.BA.Bytes)
	}
	strNodes = strings.TrimRight(strNodes, ",")
	strNodes += "]"
	strLinks = strings.TrimRight(strLinks, ",")
	strLinks += "]"
	str += strNodes + "," + strLinks
	str += "}"
	return str
}

func (ft *FlowTable) NewFlowTableFromFlows(flows []*Flow) *FlowTable {
	nft := NewFlowTable()
	nft.Update(flows)
	return nft
}

/* Return a new FlowTable that contain <last> active flows */
func (ft *FlowTable) FilterLast(last time.Duration) []*Flow {
	ft.lock.RLock()
	defer ft.lock.RUnlock()
	var flows []*Flow
	selected := time.Now().Unix() - int64((last).Seconds())
	for _, f := range ft.table {
		fs := f.GetStatistics()
		if fs.Last >= selected {
			flows = append(flows, f)
		}
	}
	return flows
}
<|endoftext|>


<commit_before>package foundation

import (
	"errors"
	"os"

	"github.com/metacoin/flojson"
)

func RPCCall(command string, args ...interface{}) (interface{}, error) {
	// create a new command based on the command string given
	// handle errors, and return results to main function
	id := 1
	var cmd flojson.Cmd
	var err error
	var ret interface{}

	switch command {

	case "getblock":
		cmd, err = flojson.NewGetBlockCmd(id, args[0].(string))
	case "getblockhash":
		cmd, err = flojson.NewGetBlockHashCmd(id, args[0].(int))
	case "getblockcount":
		cmd, err = flojson.NewGetBlockCountCmd(id)
	case "getrawtransaction":
		cmd, err = flojson.NewGetRawTransactionCmd(id, args[0].(string))
	case "getdecoderawtransaction":
		cmd, err = flojson.NewGetRawTransactionCmd(id, args[0].(string), 1)
	case "decoderawtransaction":
		cmd, err = flojson.NewDecodeRawTransactionCmd(id, args[0].(string))
	case "getnewaddress":
		cmd, err = flojson.NewGetNewAddressCmd(id)
	case "getmininginfo":
		cmd, err = flojson.NewGetMiningInfoCmd(id)
	case "getconnectioncount":
		cmd, err = flojson.NewGetConnectionCountCmd(id)
	case "getnetworkhashps":
		cmd, err = flojson.NewGetNetworkHashPSCmd(id)
	case "settxfee":
		cmd, err = flojson.NewSetTxFeeCmd(id, args[0].(int64))
	case "sendtoaddress":
		cmd, err = flojson.NewSendToAddressCmd(id, args[0].(string), args[1].(int64), args[2], args[3], args[4])
	case "walletpassphrase":
		cmd, err = flojson.NewWalletPassphraseCmd(id, args[0].(string), args[1].(int))
	case "signmessage":
		cmd, err = flojson.NewSignMessageCmd(id, args[0].(string), args[1].(string))
	case "verifymessage":
		cmd, err = flojson.NewVerifyMessageCmd(id, args[0].(string), args[1].(string), args[2].(string))
	case "validateaddress":
		cmd, err = flojson.NewValidateAddressCmd(id, args[0].(string))
	case "listtransactions":
		cmd, err = flojson.NewListTransactionsCmd(id)
	case "getbalance":
		cmd, err = flojson.NewGetBalanceCmd(id)
	default:
		err = errors.New("command " + command + " not found")
	}

	if err != nil {
		return ret, err
	}

	if cmd != nil {
		reply, err := SendCommand(cmd)
		if err != nil {
			return ret, err
		} else {
			return reply, nil
		}
	}
	return ret, errors.New("flojson replied with empty NewCmd struct")
}

func SendCommand(cmd flojson.Cmd) (interface{}, error) {

	var ret interface{}
	ftoken := os.Getenv("F_TOKEN")
	if ftoken == "" {
		return ret, errors.New("F_TOKEN environment variable not set")
	}

	fuser := os.Getenv("F_USER")
	if ftoken == "" {
		return ret, errors.New("F_USER environment variable not set")
	}

	furi := os.Getenv("F_URI")
	if furi == "" || len(furi) < 1 {
		//return ret, errors.New("F_URI environment variable not set")
		furi = "127.0.0.1:7313"
	}

	// send command to RPC, get a response in reply
	reply, err := flojson.RpcSend(fuser, ftoken, furi, cmd)
	if err != nil {
		return ret, err
	}

	if reply.Result != nil {
		return reply.Result, nil
	} else {
		// if for some reason the result is nil, but err isn't nil, return a generic warning message
		return ret, errors.New("didn't get a response from the RPC - check auth and permission settings")
	}
}
<commit_msg>Return rpc error rather than generic error<commit_after>package foundation

import (
	"errors"
	"os"

	"github.com/metacoin/flojson"
)

func RPCCall(command string, args ...interface{}) (interface{}, error) {
	// create a new command based on the command string given
	// handle errors, and return results to main function
	id := 1
	var cmd flojson.Cmd
	var err error
	var ret interface{}

	switch command {

	case "getblock":
		cmd, err = flojson.NewGetBlockCmd(id, args[0].(string))
	case "getblockhash":
		cmd, err = flojson.NewGetBlockHashCmd(id, args[0].(int))
	case "getblockcount":
		cmd, err = flojson.NewGetBlockCountCmd(id)
	case "getrawtransaction":
		cmd, err = flojson.NewGetRawTransactionCmd(id, args[0].(string))
	case "getdecoderawtransaction":
		cmd, err = flojson.NewGetRawTransactionCmd(id, args[0].(string), 1)
	case "decoderawtransaction":
		cmd, err = flojson.NewDecodeRawTransactionCmd(id, args[0].(string))
	case "getnewaddress":
		cmd, err = flojson.NewGetNewAddressCmd(id)
	case "getmininginfo":
		cmd, err = flojson.NewGetMiningInfoCmd(id)
	case "getconnectioncount":
		cmd, err = flojson.NewGetConnectionCountCmd(id)
	case "getnetworkhashps":
		cmd, err = flojson.NewGetNetworkHashPSCmd(id)
	case "settxfee":
		cmd, err = flojson.NewSetTxFeeCmd(id, args[0].(int64))
	case "sendtoaddress":
		cmd, err = flojson.NewSendToAddressCmd(id, args[0].(string), args[1].(int64), args[2], args[3], args[4])
	case "walletpassphrase":
		cmd, err = flojson.NewWalletPassphraseCmd(id, args[0].(string), args[1].(int))
	case "signmessage":
		cmd, err = flojson.NewSignMessageCmd(id, args[0].(string), args[1].(string))
	case "verifymessage":
		cmd, err = flojson.NewVerifyMessageCmd(id, args[0].(string), args[1].(string), args[2].(string))
	case "validateaddress":
		cmd, err = flojson.NewValidateAddressCmd(id, args[0].(string))
	case "listtransactions":
		cmd, err = flojson.NewListTransactionsCmd(id)
	case "getbalance":
		cmd, err = flojson.NewGetBalanceCmd(id)
	default:
		err = errors.New("command " + command + " not found")
	}

	if err != nil {
		return ret, err
	}

	if cmd != nil {
		reply, err := SendCommand(cmd)
		if err != nil {
			return ret, err
		} else {
			return reply, nil
		}
	}
	return ret, errors.New("flojson replied with empty NewCmd struct")
}

func SendCommand(cmd flojson.Cmd) (interface{}, error) {

	var ret interface{}
	ftoken := os.Getenv("F_TOKEN")
	if ftoken == "" {
		return ret, errors.New("F_TOKEN environment variable not set")
	}

	fuser := os.Getenv("F_USER")
	if ftoken == "" {
		return ret, errors.New("F_USER environment variable not set")
	}

	furi := os.Getenv("F_URI")
	if furi == "" || len(furi) < 1 {
		//return ret, errors.New("F_URI environment variable not set")
		furi = "127.0.0.1:7313"
	}

	// send command to RPC, get a response in reply
	reply, err := flojson.RpcSend(fuser, ftoken, furi, cmd)
	if err != nil {
		return ret, err
	}

	if reply.Result != nil {
		return reply.Result, nil
	} else {
		return ret, reply.Error
	}
}
<|endoftext|>


<commit_before>package sessions

import (
	"net/http"
	"time"

	"github.com/kataras/iris/context"
)

// A Sessions manager should be responsible to Start a sesion, based
// on a Context, which should return
// a compatible Session interface, type. If the external session manager
// doesn't qualifies, then the user should code the rest of the functions with empty implementation.
//
// Sessions should be responsible to Destroy a session based
// on the Context.
type Sessions struct {
	config   Config
	provider *provider
}

// New returns a new fast, feature-rich sessions manager
// it can be adapted to an iris station
func New(cfg Config) *Sessions {
	return &Sessions{
		config:   cfg.Validate(),
		provider: newProvider(),
	}
}

// UseDatabase adds a session database to the manager's provider,
// a session db doesn't have write access
func (s *Sessions) UseDatabase(db Database) {
	s.provider.RegisterDatabase(db)
}

// updateCookie gains the ability of updating the session browser cookie to any method which wants to update it
func (s *Sessions) updateCookie(ctx context.Context, sid string, expires time.Duration) {
	cookie := &http.Cookie{}

	// The RFC makes no mention of encoding url value, so here I think to encode both sessionid key and the value using the safe(to put and to use as cookie) url-encoding
	cookie.Name = s.config.Cookie

	cookie.Value = sid
	cookie.Path = "/"
	cookie.Domain = formatCookieDomain(ctx, s.config.DisableSubdomainPersistence)
	cookie.HttpOnly = true
	// MaxAge=0 means no 'Max-Age' attribute specified.
	// MaxAge<0 means delete cookie now, equivalently 'Max-Age: 0'
	// MaxAge>0 means Max-Age attribute present and given in seconds
	if expires >= 0 {
		if expires == 0 { // unlimited life
			cookie.Expires = CookieExpireUnlimited
		} else { // > 0
			cookie.Expires = time.Now().Add(expires)
		}
		cookie.MaxAge = int(cookie.Expires.Sub(time.Now()).Seconds())
	}

	// set the cookie to secure if this is a tls wrapped request
	// and the configuration allows it.
	if ctx.Request().TLS != nil && s.config.CookieSecureTLS {
		cookie.Secure = true
	}

	// encode the session id cookie client value right before send it.
	cookie.Value = s.encodeCookieValue(cookie.Value)
	AddCookie(ctx, cookie, s.config.AllowReclaim)
}

// Start should start the session for the particular request.
func (s *Sessions) Start(ctx context.Context) *Session {
	cookieValue := s.decodeCookieValue(GetCookie(ctx, s.config.Cookie))

	if cookieValue == "" { // cookie doesn't exists, let's generate a session and add set a cookie
		sid := s.config.SessionIDGenerator()

		sess := s.provider.Init(sid, s.config.Expires)
		sess.isNew = s.provider.db.Len(sid) == 0

		s.updateCookie(ctx, sid, s.config.Expires)

		return sess
	}

	sess := s.provider.Read(cookieValue, s.config.Expires)

	return sess
}

// ShiftExpiration move the expire date of a session to a new date
// by using session default timeout configuration.
// It will return `ErrNotImplemented` if a database is used and it does not support this feature, yet.
func (s *Sessions) ShiftExpiration(ctx context.Context) error {
	return s.UpdateExpiration(ctx, s.config.Expires)
}

// UpdateExpiration change expire date of a session to a new date
// by using timeout value passed by `expires` receiver.
// It will return `ErrNotFound` when trying to update expiration on a non-existence or not valid session entry.
// It will return `ErrNotImplemented` if a database is used and it does not support this feature, yet.
func (s *Sessions) UpdateExpiration(ctx context.Context, expires time.Duration) error {
	cookieValue := s.decodeCookieValue(GetCookie(ctx, s.config.Cookie))
	if cookieValue == "" {
		return ErrNotFound
	}

	// we should also allow it to expire when the browser closed
	err := s.provider.UpdateExpiration(cookieValue, expires)
	if err == nil || expires == -1 {
		s.updateCookie(ctx, cookieValue, expires)
	}

	return err
}

// DestroyListener is the form of a destroy listener.
// Look `OnDestroy` for more.
type DestroyListener func(sid string)

// OnDestroy registers one or more destroy listeners.
// A destroy listener is fired when a session has been removed entirely from the server (the entry) and client-side (the cookie).
// Note that if a destroy listener is blocking, then the session manager will delay respectfully,
// use a goroutine inside the listener to avoid that behavior.
func (s *Sessions) OnDestroy(listeners ...DestroyListener) {
	for _, ln := range listeners {
		s.provider.registerDestroyListener(ln)
	}
}

// Destroy remove the session data and remove the associated cookie.
func (s *Sessions) Destroy(ctx context.Context) {
	cookieValue := GetCookie(ctx, s.config.Cookie)
	// decode the client's cookie value in order to find the server's session id
	// to destroy the session data.
	cookieValue = s.decodeCookieValue(cookieValue)
	if cookieValue == "" { // nothing to destroy
		return
	}
	RemoveCookie(ctx, s.config)

	s.provider.Destroy(cookieValue)
}

// DestroyByID removes the session entry
// from the server-side memory (and database if registered).
// Client's session cookie will still exist but it will be reseted on the next request.
//
// It's safe to use it even if you are not sure if a session with that id exists.
//
// Note: the sid should be the original one (i.e: fetched by a store )
// it's not decoded.
func (s *Sessions) DestroyByID(sid string) {
	s.provider.Destroy(sid)
}

// DestroyAll removes all sessions
// from the server-side memory (and database if registered).
// Client's session cookie will still exist but it will be reseted on the next request.
func (s *Sessions) DestroyAll() {
	s.provider.DestroyAll()
}

// let's keep these funcs simple, we can do it with two lines but we may add more things in the future.
func (s *Sessions) decodeCookieValue(cookieValue string) string {
	if cookieValue == "" {
		return ""
	}

	var cookieValueDecoded *string

	if decode := s.config.Decode; decode != nil {
		err := decode(s.config.Cookie, cookieValue, &cookieValueDecoded)
		if err == nil {
			cookieValue = *cookieValueDecoded
		} else {
			cookieValue = ""
		}
	}

	return cookieValue
}

func (s *Sessions) encodeCookieValue(cookieValue string) string {
	if encode := s.config.Encode; encode != nil {
		newVal, err := encode(s.config.Cookie, cookieValue)
		if err == nil {
			cookieValue = newVal
		} else {
			cookieValue = ""
		}
	}

	return cookieValue
}
<commit_msg>sessions: give ability to the end-user to modify the cookie via context.CookieOption on Start and Update/ShiftExpiration as requested at: https://github.com/kataras/iris/issues/1186, add a StartWithPath helper as well<commit_after>package sessions

import (
	"net/http"
	"time"

	"github.com/kataras/iris/context"
)

// A Sessions manager should be responsible to Start a sesion, based
// on a Context, which should return
// a compatible Session interface, type. If the external session manager
// doesn't qualifies, then the user should code the rest of the functions with empty implementation.
//
// Sessions should be responsible to Destroy a session based
// on the Context.
type Sessions struct {
	config   Config
	provider *provider
}

// New returns a new fast, feature-rich sessions manager
// it can be adapted to an iris station
func New(cfg Config) *Sessions {
	return &Sessions{
		config:   cfg.Validate(),
		provider: newProvider(),
	}
}

// UseDatabase adds a session database to the manager's provider,
// a session db doesn't have write access
func (s *Sessions) UseDatabase(db Database) {
	s.provider.RegisterDatabase(db)
}

// updateCookie gains the ability of updating the session browser cookie to any method which wants to update it
func (s *Sessions) updateCookie(ctx context.Context, sid string, expires time.Duration, options ...context.CookieOption) {
	cookie := &http.Cookie{}

	// The RFC makes no mention of encoding url value, so here I think to encode both sessionid key and the value using the safe(to put and to use as cookie) url-encoding
	cookie.Name = s.config.Cookie

	cookie.Value = sid
	cookie.Path = "/"
	cookie.Domain = formatCookieDomain(ctx, s.config.DisableSubdomainPersistence)
	cookie.HttpOnly = true
	// MaxAge=0 means no 'Max-Age' attribute specified.
	// MaxAge<0 means delete cookie now, equivalently 'Max-Age: 0'
	// MaxAge>0 means Max-Age attribute present and given in seconds
	if expires >= 0 {
		if expires == 0 { // unlimited life
			cookie.Expires = CookieExpireUnlimited
		} else { // > 0
			cookie.Expires = time.Now().Add(expires)
		}
		cookie.MaxAge = int(cookie.Expires.Sub(time.Now()).Seconds())
	}

	// set the cookie to secure if this is a tls wrapped request
	// and the configuration allows it.
	if ctx.Request().TLS != nil && s.config.CookieSecureTLS {
		cookie.Secure = true
	}

	// encode the session id cookie client value right before send it.
	cookie.Value = s.encodeCookieValue(cookie.Value)

	for _, opt := range options {
		opt(cookie)
	}

	AddCookie(ctx, cookie, s.config.AllowReclaim)
}

// Start creates or retrieves an existing session for the particular request.
func (s *Sessions) Start(ctx context.Context, cookieOptions ...context.CookieOption) *Session {
	cookieValue := s.decodeCookieValue(GetCookie(ctx, s.config.Cookie))

	if cookieValue == "" { // cookie doesn't exists, let's generate a session and add set a cookie
		sid := s.config.SessionIDGenerator()

		sess := s.provider.Init(sid, s.config.Expires)
		sess.isNew = s.provider.db.Len(sid) == 0

		s.updateCookie(ctx, sid, s.config.Expires, cookieOptions...)

		return sess
	}

	sess := s.provider.Read(cookieValue, s.config.Expires)

	return sess
}

// StartWithPath same as `Start` but it explicitly accepts the cookie path option.
func (s *Sessions) StartWithPath(ctx context.Context, path string) *Session {
	return s.Start(ctx, context.CookiePath(path))
}

// ShiftExpiration move the expire date of a session to a new date
// by using session default timeout configuration.
// It will return `ErrNotImplemented` if a database is used and it does not support this feature, yet.
func (s *Sessions) ShiftExpiration(ctx context.Context, cookieOptions ...context.CookieOption) error {
	return s.UpdateExpiration(ctx, s.config.Expires, cookieOptions...)
}

// UpdateExpiration change expire date of a session to a new date
// by using timeout value passed by `expires` receiver.
// It will return `ErrNotFound` when trying to update expiration on a non-existence or not valid session entry.
// It will return `ErrNotImplemented` if a database is used and it does not support this feature, yet.
func (s *Sessions) UpdateExpiration(ctx context.Context, expires time.Duration, cookieOptions ...context.CookieOption) error {
	cookieValue := s.decodeCookieValue(GetCookie(ctx, s.config.Cookie))
	if cookieValue == "" {
		return ErrNotFound
	}

	// we should also allow it to expire when the browser closed
	err := s.provider.UpdateExpiration(cookieValue, expires)
	if err == nil || expires == -1 {
		s.updateCookie(ctx, cookieValue, expires, cookieOptions...)
	}

	return err
}

// DestroyListener is the form of a destroy listener.
// Look `OnDestroy` for more.
type DestroyListener func(sid string)

// OnDestroy registers one or more destroy listeners.
// A destroy listener is fired when a session has been removed entirely from the server (the entry) and client-side (the cookie).
// Note that if a destroy listener is blocking, then the session manager will delay respectfully,
// use a goroutine inside the listener to avoid that behavior.
func (s *Sessions) OnDestroy(listeners ...DestroyListener) {
	for _, ln := range listeners {
		s.provider.registerDestroyListener(ln)
	}
}

// Destroy remove the session data and remove the associated cookie.
func (s *Sessions) Destroy(ctx context.Context) {
	cookieValue := GetCookie(ctx, s.config.Cookie)
	// decode the client's cookie value in order to find the server's session id
	// to destroy the session data.
	cookieValue = s.decodeCookieValue(cookieValue)
	if cookieValue == "" { // nothing to destroy
		return
	}
	RemoveCookie(ctx, s.config)

	s.provider.Destroy(cookieValue)
}

// DestroyByID removes the session entry
// from the server-side memory (and database if registered).
// Client's session cookie will still exist but it will be reseted on the next request.
//
// It's safe to use it even if you are not sure if a session with that id exists.
//
// Note: the sid should be the original one (i.e: fetched by a store )
// it's not decoded.
func (s *Sessions) DestroyByID(sid string) {
	s.provider.Destroy(sid)
}

// DestroyAll removes all sessions
// from the server-side memory (and database if registered).
// Client's session cookie will still exist but it will be reseted on the next request.
func (s *Sessions) DestroyAll() {
	s.provider.DestroyAll()
}

// let's keep these funcs simple, we can do it with two lines but we may add more things in the future.
func (s *Sessions) decodeCookieValue(cookieValue string) string {
	if cookieValue == "" {
		return ""
	}

	var cookieValueDecoded *string

	if decode := s.config.Decode; decode != nil {
		err := decode(s.config.Cookie, cookieValue, &cookieValueDecoded)
		if err == nil {
			cookieValue = *cookieValueDecoded
		} else {
			cookieValue = ""
		}
	}

	return cookieValue
}

func (s *Sessions) encodeCookieValue(cookieValue string) string {
	if encode := s.config.Encode; encode != nil {
		newVal, err := encode(s.config.Cookie, cookieValue)
		if err == nil {
			cookieValue = newVal
		} else {
			cookieValue = ""
		}
	}

	return cookieValue
}
<|endoftext|>


<commit_before>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/* This modules converts implements Snap API to become an plugin.

legend:
- metric - represents value of metric from Snap side
- fact - represents a value about a system gathered from Facter
- name - is string identifier that refers to metric from the Snap side, so name points to metric

 Implementation details:

 GetMetricTypes()
      +
      |                 +------------------+
 +----v---+  getFacts() |                  |
 | Facter +-------------> ./facter --json  |
 +----^---+             |    (goroutine)   |
      |                 +------------------+
      +
 CollectMetrics()

*/

package facter

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/control/plugin/cpolicy"
)

const (
	name       = "Intel Fact Gathering Plugin (Facter)"
	version    = 4
	pluginType = plugin.CollectorPluginType

	// parts of returned namescape
	vendor = "intel"
	prefix = "facter"
	// how long we are caching the date from external binary to prevent overuse of resources
	defaultCacheTTL = 60 * time.Second
	// how long are we going to cache available types of metrics
	defaultMetricTypesTTL = defaultCacheTTL
	// timeout we are ready to wait for external binary to gather the data
	defaultFacterTimeout = 5 * time.Second
)

func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(name, version, pluginType, []string{plugin.SnapGOBContentType}, []string{plugin.SnapGOBContentType})
}

/**********
 * Facter *
 **********/

// Facter implements API to communicate with Snap
type Facter struct {
	ttl time.Duration
	// injects implementation for getting facts - defaults to use getFacts from cmd.go
	// but allows to replace with fake during tests
	getFacts func(
		names []string,
		facterTimeout time.Duration,
		cmdConfig *cmdConfig,
	) (facts, error)
	// how much time we are ready to wait for getting result from cmd
	// is going to be passed to facterTimeout parameter in getFacts
	facterTimeout time.Duration
}

// make sure that we actually satisify requierd interface
var _ plugin.CollectorPlugin = (*Facter)(nil)

// NewFacter constructs new Facter with default values
func NewFacterCollector() *Facter {
	return &Facter{
		// injection of default implementation for gathering facts from Facter
		getFacts:      getFacts,
		facterTimeout: defaultFacterTimeout,
	}
}

// ------------ Snap plugin interface implementation --------------

// GetMetricTypes returns available metrics types
func (f *Facter) GetMetricTypes(_ plugin.PluginConfigType) ([]plugin.PluginMetricType, error) {

	// facts composed of entries
	facts, err := f.getFacts(
		nil, // ask for everything
		f.facterTimeout,
		nil, //default cmd configuration
	)
	if err != nil {
		if strings.Contains(err.Error(), "executable file not found") {
			// Facter cannot be found. Since this is called on load we should
			// not send an error as loading a plugin should not fail based on
			// whether or not a dynamic path is set.
			return []plugin.PluginMetricType{}, nil
		}
		return nil, err
	}

	// capacity - we are going to return all the facts
	metricTypes := make([]plugin.PluginMetricType, 0, len(facts))

	// create types with given namespace
	for name, _ := range facts {
		namespace := createNamespace(name)
		metricType := plugin.PluginMetricType{Namespace_: namespace}
		metricTypes = append(metricTypes, metricType)
	}

	return metricTypes, nil
}

// Collect collects metrics from external binary a returns them in form
// acceptable by Snap, only returns collects that were asked for and return nothing when asked for none
// the order of requested and received metrics isn't guaranted
func (f *Facter) CollectMetrics(metricTypes []plugin.PluginMetricType) ([]plugin.PluginMetricType, error) {

	// parse and check requested names of metrics
	names := []string{}
	for _, metricType := range metricTypes {
		namespace := metricType.Namespace()

		err := validateNamespace(namespace)
		if err != nil {
			return nil, err
		}

		// name of fact - last part of namespace
		name := namespace[2]
		names = append(names, name)
	}

	if len(names) == 0 {
		// nothing request, none returned
		// !because returned by value, it would return nil slice
		return nil, nil
	}

	// facts composed of entries
	facts, err := f.getFacts(names, f.facterTimeout, nil)
	if err != nil {
		return nil, err
	}

	// make sure that recevied len of names equals asked
	if len(facts) != len(names) {
		return nil, errors.New("assertion: getFacts returns more/less than asked!")
	}

	host, _ := os.Hostname()
	// convert facts into PluginMetrics
	metrics := make([]plugin.PluginMetricType, 0, len(facts))
	for name, value := range facts {
		namespace := createNamespace(name)
		metric := *plugin.NewPluginMetricType(namespace, time.Now(), host, nil, nil, value)
		metrics = append(metrics, metric)
	}
	return metrics, nil
}

func (f *Facter) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
	c := cpolicy.New()
	rule, _ := cpolicy.NewStringRule("name", false, "bob")
	rule2, _ := cpolicy.NewStringRule("password", true)
	p := cpolicy.NewPolicyNode()
	p.Add(rule)
	p.Add(rule2)
	c.Add([]string{"intel", "facter", "foo"}, p)
	return c, nil
}

// ------------ helper functions --------------

// validateNamespace checks namespace intel(vendor)/facter(prefix)/FACTNAME
func validateNamespace(namespace []string) error {
	if len(namespace) != 3 {
		return errors.New(fmt.Sprintf("unknown metricType %s (should containt just 3 segments)", namespace))
	}
	if namespace[0] != vendor {
		return errors.New(fmt.Sprintf("unknown metricType %s (expected vendor %s)", namespace, vendor))
	}

	if namespace[1] != prefix {
		return errors.New(fmt.Sprintf("unknown metricType %s (expected prefix %s)", namespace, prefix))
	}
	return nil
}

// namspace returns namespace slice of strings
// composed from: vendor, prefix and fact name
func createNamespace(name string) []string {
	return []string{vendor, prefix, name}

}

// helper type to deal with json values which additionally stores last update moment
type entry struct {
	value      interface{}
	lastUpdate time.Time
}
<commit_msg>Updated plugin name<commit_after>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/* This modules converts implements Snap API to become an plugin.

legend:
- metric - represents value of metric from Snap side
- fact - represents a value about a system gathered from Facter
- name - is string identifier that refers to metric from the Snap side, so name points to metric

 Implementation details:

 GetMetricTypes()
      +
      |                 +------------------+
 +----v---+  getFacts() |                  |
 | Facter +-------------> ./facter --json  |
 +----^---+             |    (goroutine)   |
      |                 +------------------+
      +
 CollectMetrics()

*/

package facter

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/control/plugin/cpolicy"
)

const (
	name       = "facter"
	version    = 5
	pluginType = plugin.CollectorPluginType

	// parts of returned namescape
	vendor = "intel"
	prefix = "facter"
	// how long we are caching the date from external binary to prevent overuse of resources
	defaultCacheTTL = 60 * time.Second
	// how long are we going to cache available types of metrics
	defaultMetricTypesTTL = defaultCacheTTL
	// timeout we are ready to wait for external binary to gather the data
	defaultFacterTimeout = 5 * time.Second
)

func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(name, version, pluginType, []string{plugin.SnapGOBContentType}, []string{plugin.SnapGOBContentType})
}

/**********
 * Facter *
 **********/

// Facter implements API to communicate with Snap
type Facter struct {
	ttl time.Duration
	// injects implementation for getting facts - defaults to use getFacts from cmd.go
	// but allows to replace with fake during tests
	getFacts func(
		names []string,
		facterTimeout time.Duration,
		cmdConfig *cmdConfig,
	) (facts, error)
	// how much time we are ready to wait for getting result from cmd
	// is going to be passed to facterTimeout parameter in getFacts
	facterTimeout time.Duration
}

// make sure that we actually satisify requierd interface
var _ plugin.CollectorPlugin = (*Facter)(nil)

// NewFacter constructs new Facter with default values
func NewFacterCollector() *Facter {
	return &Facter{
		// injection of default implementation for gathering facts from Facter
		getFacts:      getFacts,
		facterTimeout: defaultFacterTimeout,
	}
}

// ------------ Snap plugin interface implementation --------------

// GetMetricTypes returns available metrics types
func (f *Facter) GetMetricTypes(_ plugin.PluginConfigType) ([]plugin.PluginMetricType, error) {

	// facts composed of entries
	facts, err := f.getFacts(
		nil, // ask for everything
		f.facterTimeout,
		nil, //default cmd configuration
	)
	if err != nil {
		if strings.Contains(err.Error(), "executable file not found") {
			// Facter cannot be found. Since this is called on load we should
			// not send an error as loading a plugin should not fail based on
			// whether or not a dynamic path is set.
			return []plugin.PluginMetricType{}, nil
		}
		return nil, err
	}

	// capacity - we are going to return all the facts
	metricTypes := make([]plugin.PluginMetricType, 0, len(facts))

	// create types with given namespace
	for name, _ := range facts {
		namespace := createNamespace(name)
		metricType := plugin.PluginMetricType{Namespace_: namespace}
		metricTypes = append(metricTypes, metricType)
	}

	return metricTypes, nil
}

// Collect collects metrics from external binary a returns them in form
// acceptable by Snap, only returns collects that were asked for and return nothing when asked for none
// the order of requested and received metrics isn't guaranted
func (f *Facter) CollectMetrics(metricTypes []plugin.PluginMetricType) ([]plugin.PluginMetricType, error) {

	// parse and check requested names of metrics
	names := []string{}
	for _, metricType := range metricTypes {
		namespace := metricType.Namespace()

		err := validateNamespace(namespace)
		if err != nil {
			return nil, err
		}

		// name of fact - last part of namespace
		name := namespace[2]
		names = append(names, name)
	}

	if len(names) == 0 {
		// nothing request, none returned
		// !because returned by value, it would return nil slice
		return nil, nil
	}

	// facts composed of entries
	facts, err := f.getFacts(names, f.facterTimeout, nil)
	if err != nil {
		return nil, err
	}

	// make sure that recevied len of names equals asked
	if len(facts) != len(names) {
		return nil, errors.New("assertion: getFacts returns more/less than asked!")
	}

	host, _ := os.Hostname()
	// convert facts into PluginMetrics
	metrics := make([]plugin.PluginMetricType, 0, len(facts))
	for name, value := range facts {
		namespace := createNamespace(name)
		metric := *plugin.NewPluginMetricType(namespace, time.Now(), host, nil, nil, value)
		metrics = append(metrics, metric)
	}
	return metrics, nil
}

func (f *Facter) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
	c := cpolicy.New()
	rule, _ := cpolicy.NewStringRule("name", false, "bob")
	rule2, _ := cpolicy.NewStringRule("password", true)
	p := cpolicy.NewPolicyNode()
	p.Add(rule)
	p.Add(rule2)
	c.Add([]string{"intel", "facter", "foo"}, p)
	return c, nil
}

// ------------ helper functions --------------

// validateNamespace checks namespace intel(vendor)/facter(prefix)/FACTNAME
func validateNamespace(namespace []string) error {
	if len(namespace) != 3 {
		return errors.New(fmt.Sprintf("unknown metricType %s (should containt just 3 segments)", namespace))
	}
	if namespace[0] != vendor {
		return errors.New(fmt.Sprintf("unknown metricType %s (expected vendor %s)", namespace, vendor))
	}

	if namespace[1] != prefix {
		return errors.New(fmt.Sprintf("unknown metricType %s (expected prefix %s)", namespace, prefix))
	}
	return nil
}

// namspace returns namespace slice of strings
// composed from: vendor, prefix and fact name
func createNamespace(name string) []string {
	return []string{vendor, prefix, name}

}

// helper type to deal with json values which additionally stores last update moment
type entry struct {
	value      interface{}
	lastUpdate time.Time
}
<|endoftext|>


<commit_before>// Copyright © 2016 Jip J. Dekker <jip@dekker.li>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package settings

import (
	"encoding/json"
	"io/ioutil"
)

// Settings provides a structure to interact with the settings
// of a Ponder library
type Settings struct {
	Name             string   // Name of the Ponder library
	IgnoreDirs       []string // Directories to be ignored on search
	LilypondIncludes []string // Directories to be included when running the lilypond compiler
	OutputDir        string   // Directory in which all complete file are stored
}

// FromFile reads a settings file in json format and returns the Settings struct
func FromFile(path string) (*Settings, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	var s Settings
	err = json.Unmarshal(data, &s)
	if err != nil {
		return nil, err
	}
	s.AbsolutePaths(filepath.Dir(path))
	return &s, nil
}

// AbsolutePaths makes all paths in settings absolute using the given
// absolute root
func (s *settings.Settings) AbsolutePaths(root string) {
	for i := range s.IgnoreDirs {
		s.IgnoreDirs[i] := helpers.AbsolutePath(s.IgnoreDirs[i], root)
	}
	for i := range s.LilypondIncludes {
		s.LilypondIncludes[i] := helpers.AbsolutePath(s.LilypondIncludes[i], root)
	}
	s.OutputDir := helpers.AbsolutePath(s.OutputDir, root)
}
<commit_msg>Fixes method coding mistakes<commit_after>// Copyright © 2016 Jip J. Dekker <jip@dekker.li>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package settings

import (
	"encoding/json"
	"io/ioutil"
	"path/filepath"

	"github.com/jjdekker/ponder/helpers"
)

// Settings provides a structure to interact with the settings
// of a Ponder library
type Settings struct {
	Name             string   // Name of the Ponder library
	IgnoreDirs       []string // Directories to be ignored on search
	LilypondIncludes []string // Directories to be included when running the lilypond compiler
	OutputDir        string   // Directory in which all complete file are stored
}

// FromFile reads a settings file in json format and returns the Settings struct
func FromFile(path string) (*Settings, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	var s Settings
	err = json.Unmarshal(data, &s)
	if err != nil {
		return nil, err
	}
	s.AbsolutePaths(filepath.Dir(path))
	return &s, nil
}

// AbsolutePaths makes all paths in settings absolute using the given
// absolute root
func (s *Settings) AbsolutePaths(root string) {
	for i := range s.IgnoreDirs {
		s.IgnoreDirs[i] = helpers.AbsolutePath(s.IgnoreDirs[i], root)
	}
	for i := range s.LilypondIncludes {
		s.LilypondIncludes[i] = helpers.AbsolutePath(s.LilypondIncludes[i], root)
	}
	s.OutputDir = helpers.AbsolutePath(s.OutputDir, root)
}
<|endoftext|>


<commit_before>package failover

import (
	"encoding/json"
	"fmt"
	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/siddontang/go/log"
	"io"
	"net"
	"os"
	"path"
	"sync"
	"time"
)

const (
	addCmd = "add"
	delCmd = "del"
	setCmd = "set"
)

type action struct {
	Cmd     string   `json:"cmd"`
	Masters []string `json:"masters"`
}

// save mornitored master addr
type masterFSM struct {
	sync.Mutex

	masters map[string]struct{}
}

func newMasterFSM() *masterFSM {
	fsm := new(masterFSM)
	fsm.masters = make(map[string]struct{})
	return fsm
}

func (fsm *masterFSM) AddMasters(addrs []string) {
	fsm.Lock()
	defer fsm.Unlock()

	for _, addr := range addrs {
		fsm.masters[addr] = struct{}{}

	}

}

func (fsm *masterFSM) DelMasters(addrs []string) {
	fsm.Lock()
	defer fsm.Unlock()

	for _, addr := range addrs {
		delete(fsm.masters, addr)
	}
}

func (fsm *masterFSM) SetMasters(addrs []string) {
	m := make(map[string]struct{}, len(addrs))

	for _, addr := range addrs {
		m[addr] = struct{}{}
	}

	fsm.Lock()
	defer fsm.Unlock()

	fsm.masters = m
}

func (fsm *masterFSM) GetMasters() []string {
	fsm.Lock()
	defer fsm.Unlock()

	m := make([]string, 0, len(fsm.masters))
	for master, _ := range fsm.masters {
		m = append(m, master)
	}
	return m
}

func (fsm *masterFSM) IsMaster(addr string) bool {
	fsm.Lock()
	defer fsm.Unlock()

	_, ok := fsm.masters[addr]
	return ok
}

func (fsm *masterFSM) Apply(l *raft.Log) interface{} {
	var a action
	if err := json.Unmarshal(l.Data, &a); err != nil {
		log.Errorf("decode raft log err %v", err)
		return err
	}

	switch a.Cmd {
	case addCmd:
		fsm.AddMasters(a.Masters)
	case delCmd:
		fsm.DelMasters(a.Masters)
	case setCmd:
		fsm.SetMasters(a.Masters)
	default:
		log.Errorf("invalid fsm action %v", a.Cmd)
	}

	return nil
}

func (fsm *masterFSM) Snapshot() (raft.FSMSnapshot, error) {
	snap := new(masterSnapshot)
	snap.masters = make([]string, 0, len(fsm.masters))

	fsm.Lock()
	for master, _ := range fsm.masters {
		snap.masters = append(snap.masters, master)
	}
	fsm.Unlock()
	return snap, nil
}

func (fsm *masterFSM) Restore(snap io.ReadCloser) error {
	defer snap.Close()

	d := json.NewDecoder(snap)
	var masters []string

	if err := d.Decode(&masters); err != nil {
		return err
	}

	fsm.Lock()
	for _, master := range masters {
		fsm.masters[master] = struct{}{}
	}
	fsm.Unlock()

	return nil
}

type masterSnapshot struct {
	masters []string
}

func (snap *masterSnapshot) Persist(sink raft.SnapshotSink) error {
	data, _ := json.Marshal(snap.masters)
	_, err := sink.Write(data)
	if err != nil {
		sink.Cancel()
	}
	return err
}

func (snap *masterSnapshot) Release() {

}

// redis-failover uses raft to elect the cluster leader and do monitoring and failover.
type Raft struct {
	r *raft.Raft

	log       *os.File
	dbStore   *raftboltdb.BoltStore
	trans     *raft.NetworkTransport
	peerStore *raft.JSONPeers

	raftAddr string
}

func newRaft(c *Config, fsm raft.FSM) (*Raft, error) {
	r := new(Raft)

	if len(c.Raft.Addr) == 0 {
		return nil, nil
	}

	peers := make([]net.Addr, 0, len(c.Raft.Cluster))

	r.raftAddr = c.Raft.Addr

	a, err := net.ResolveTCPAddr("tcp", r.raftAddr)
	if err != nil {
		return nil, fmt.Errorf("invalid raft addr format %s, must host:port, err:%v", r.raftAddr, err)
	}

	peers = raft.AddUniquePeer(peers, a)

	for _, cluster := range c.Raft.Cluster {
		a, err = net.ResolveTCPAddr("tcp", cluster)
		if err != nil {
			return nil, fmt.Errorf("invalid cluster format %s, must host:port, err:%v", cluster, err)
		}

		peers = raft.AddUniquePeer(peers, a)
	}

	os.MkdirAll(c.Raft.DataDir, 0755)

	cfg := raft.DefaultConfig()

	if len(c.Raft.LogDir) == 0 {
		r.log = os.Stdout
	} else {
		os.MkdirAll(c.Raft.LogDir, 0755)
		logFile := path.Join(c.Raft.LogDir, "raft.log")
		f, err := os.OpenFile(logFile, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0644)
		if err != nil {
			return nil, err
		}
		r.log = f

		cfg.LogOutput = r.log
	}

	raftDBPath := path.Join(c.Raft.DataDir, "raft_db")
	r.dbStore, err = raftboltdb.NewBoltStore(raftDBPath)
	if err != nil {
		return nil, err
	}

	fileStore, err := raft.NewFileSnapshotStore(c.Raft.DataDir, 1, r.log)
	if err != nil {
		return nil, err
	}

	r.trans, err = raft.NewTCPTransport(r.raftAddr, nil, 3, 5*time.Second, r.log)
	if err != nil {
		return nil, err
	}

	r.peerStore = raft.NewJSONPeers(c.Raft.DataDir, r.trans)

	if c.Raft.ClusterState == ClusterStateNew {
		log.Infof("cluster state is new, use new cluster config")
		r.peerStore.SetPeers(peers)
	} else {
		log.Infof("cluster state is existing, use previous + new cluster config")
		ps, err := r.peerStore.Peers()
		if err != nil {
			log.Errorf("get store peers error %v", err)
			return nil, err
		}

		for _, peer := range peers {
			ps = raft.AddUniquePeer(ps, peer)
		}

		r.peerStore.SetPeers(ps)
	}

	if peers, _ := r.peerStore.Peers(); len(peers) <= 1 {
		cfg.EnableSingleNode = true
		log.Warn("raft will run in single node mode, may only be used in test")
	}

	r.r, err = raft.NewRaft(cfg, fsm, r.dbStore, r.dbStore, fileStore, r.peerStore, r.trans)

	return r, err
}

func (r *Raft) Close() {
	if r.trans != nil {
		r.trans.Close()
	}

	if r.r != nil {
		future := r.r.Shutdown()
		if err := future.Error(); err != nil {
			log.Errorf("Error shutting down raft: %v", err)
		}
	}

	if r.dbStore != nil {
		r.dbStore.Close()
	}

	if r.log != os.Stdout {
		r.log.Close()
	}
}

func (r *Raft) AddMasters(addrs []string, timeout time.Duration) error {
	var a = action{
		Cmd:     addCmd,
		Masters: addrs,
	}

	return r.apply(&a, timeout)
}

func (r *Raft) DelMasters(addrs []string, timeout time.Duration) error {
	var a = action{
		Cmd:     delCmd,
		Masters: addrs,
	}

	return r.apply(&a, timeout)
}

func (r *Raft) SetMasters(addrs []string, timeout time.Duration) error {
	var a = action{
		Cmd:     setCmd,
		Masters: addrs,
	}

	return r.apply(&a, timeout)
}

func (r *Raft) AddPeer(addr string) error {
	peer, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return err
	}

	f := r.r.AddPeer(peer)
	return f.Error()
}

func (r *Raft) DelPeer(addr string) error {
	peer, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return err
	}

	f := r.r.RemovePeer(peer)
	return f.Error()

}

func (r *Raft) SetPeers(addrs []string) error {
	peers := make([]net.Addr, 0, len(addrs))

	for _, addr := range addrs {
		peer, err := net.ResolveTCPAddr("tcp", addr)
		if err != nil {
			return err
		}
		peers = append(peers, peer)
	}

	f := r.r.SetPeers(peers)
	return f.Error()

}

func (r *Raft) GetPeers() ([]string, error) {
	peers, err := r.peerStore.Peers()
	if err != nil {
		return nil, err
	}

	addrs := make([]string, 0, len(peers))

	for _, peer := range peers {
		addrs = append(addrs, peer.String())
	}

	return addrs, nil
}

func (r *Raft) apply(a *action, timeout time.Duration) error {
	data, err := json.Marshal(a)
	if err != nil {
		return err
	}

	f := r.r.Apply(data, timeout)
	return f.Error()
}

func (r *Raft) LeaderCh() <-chan bool {
	return r.r.LeaderCh()
}

func (r *Raft) IsLeader() bool {
	addr := r.r.Leader()
	if addr == nil {
		return false
	} else {
		return addr.String() == r.raftAddr
	}
}

func (r *Raft) Leader() string {
	return r.r.Leader().String()
}

func (r *Raft) Barrier(timeout time.Duration) error {
	f := r.r.Barrier(timeout)
	return f.Error()
}
<commit_msg>fix empty address error<commit_after>package failover

import (
	"encoding/json"
	"fmt"
	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/siddontang/go/log"
	"io"
	"net"
	"os"
	"path"
	"sync"
	"time"
)

const (
	addCmd = "add"
	delCmd = "del"
	setCmd = "set"
)

type action struct {
	Cmd     string   `json:"cmd"`
	Masters []string `json:"masters"`
}

// save mornitored master addr
type masterFSM struct {
	sync.Mutex

	masters map[string]struct{}
}

func newMasterFSM() *masterFSM {
	fsm := new(masterFSM)
	fsm.masters = make(map[string]struct{})
	return fsm
}

func (fsm *masterFSM) AddMasters(addrs []string) {
	fsm.Lock()
	defer fsm.Unlock()

	for _, addr := range addrs {
		if len(addr) == 0 {
			continue
		}
		fsm.masters[addr] = struct{}{}
	}

}

func (fsm *masterFSM) DelMasters(addrs []string) {
	fsm.Lock()
	defer fsm.Unlock()

	for _, addr := range addrs {
		if len(addr) == 0 {
			continue
		}

		delete(fsm.masters, addr)
	}
}

func (fsm *masterFSM) SetMasters(addrs []string) {
	m := make(map[string]struct{}, len(addrs))

	for _, addr := range addrs {
		if len(addr) == 0 {
			continue
		}

		m[addr] = struct{}{}
	}

	fsm.Lock()
	defer fsm.Unlock()

	fsm.masters = m
}

func (fsm *masterFSM) GetMasters() []string {
	fsm.Lock()
	defer fsm.Unlock()

	m := make([]string, 0, len(fsm.masters))
	for master, _ := range fsm.masters {
		m = append(m, master)
	}

	return m
}

func (fsm *masterFSM) IsMaster(addr string) bool {
	fsm.Lock()
	defer fsm.Unlock()

	_, ok := fsm.masters[addr]
	return ok
}

func (fsm *masterFSM) Apply(l *raft.Log) interface{} {
	var a action
	if err := json.Unmarshal(l.Data, &a); err != nil {
		log.Errorf("decode raft log err %v", err)
		return err
	}

	switch a.Cmd {
	case addCmd:
		fsm.AddMasters(a.Masters)
	case delCmd:
		fsm.DelMasters(a.Masters)
	case setCmd:
		fsm.SetMasters(a.Masters)
	default:
		log.Errorf("invalid fsm action %v", a.Cmd)
	}

	return nil
}

func (fsm *masterFSM) Snapshot() (raft.FSMSnapshot, error) {
	snap := new(masterSnapshot)
	snap.masters = make([]string, 0, len(fsm.masters))

	fsm.Lock()
	for master, _ := range fsm.masters {
		snap.masters = append(snap.masters, master)
	}
	fsm.Unlock()
	return snap, nil
}

func (fsm *masterFSM) Restore(snap io.ReadCloser) error {
	defer snap.Close()

	d := json.NewDecoder(snap)
	var masters []string

	if err := d.Decode(&masters); err != nil {
		return err
	}

	fsm.Lock()
	for _, master := range masters {
		fsm.masters[master] = struct{}{}
	}
	fsm.Unlock()

	return nil
}

type masterSnapshot struct {
	masters []string
}

func (snap *masterSnapshot) Persist(sink raft.SnapshotSink) error {
	data, _ := json.Marshal(snap.masters)
	_, err := sink.Write(data)
	if err != nil {
		sink.Cancel()
	}
	return err
}

func (snap *masterSnapshot) Release() {

}

// redis-failover uses raft to elect the cluster leader and do monitoring and failover.
type Raft struct {
	r *raft.Raft

	log       *os.File
	dbStore   *raftboltdb.BoltStore
	trans     *raft.NetworkTransport
	peerStore *raft.JSONPeers

	raftAddr string
}

func newRaft(c *Config, fsm raft.FSM) (*Raft, error) {
	r := new(Raft)

	if len(c.Raft.Addr) == 0 {
		return nil, nil
	}

	peers := make([]net.Addr, 0, len(c.Raft.Cluster))

	r.raftAddr = c.Raft.Addr

	a, err := net.ResolveTCPAddr("tcp", r.raftAddr)
	if err != nil {
		return nil, fmt.Errorf("invalid raft addr format %s, must host:port, err:%v", r.raftAddr, err)
	}

	peers = raft.AddUniquePeer(peers, a)

	for _, cluster := range c.Raft.Cluster {
		a, err = net.ResolveTCPAddr("tcp", cluster)
		if err != nil {
			return nil, fmt.Errorf("invalid cluster format %s, must host:port, err:%v", cluster, err)
		}

		peers = raft.AddUniquePeer(peers, a)
	}

	os.MkdirAll(c.Raft.DataDir, 0755)

	cfg := raft.DefaultConfig()

	if len(c.Raft.LogDir) == 0 {
		r.log = os.Stdout
	} else {
		os.MkdirAll(c.Raft.LogDir, 0755)
		logFile := path.Join(c.Raft.LogDir, "raft.log")
		f, err := os.OpenFile(logFile, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0644)
		if err != nil {
			return nil, err
		}
		r.log = f

		cfg.LogOutput = r.log
	}

	raftDBPath := path.Join(c.Raft.DataDir, "raft_db")
	r.dbStore, err = raftboltdb.NewBoltStore(raftDBPath)
	if err != nil {
		return nil, err
	}

	fileStore, err := raft.NewFileSnapshotStore(c.Raft.DataDir, 1, r.log)
	if err != nil {
		return nil, err
	}

	r.trans, err = raft.NewTCPTransport(r.raftAddr, nil, 3, 5*time.Second, r.log)
	if err != nil {
		return nil, err
	}

	r.peerStore = raft.NewJSONPeers(c.Raft.DataDir, r.trans)

	if c.Raft.ClusterState == ClusterStateNew {
		log.Infof("cluster state is new, use new cluster config")
		r.peerStore.SetPeers(peers)
	} else {
		log.Infof("cluster state is existing, use previous + new cluster config")
		ps, err := r.peerStore.Peers()
		if err != nil {
			log.Errorf("get store peers error %v", err)
			return nil, err
		}

		for _, peer := range peers {
			ps = raft.AddUniquePeer(ps, peer)
		}

		r.peerStore.SetPeers(ps)
	}

	if peers, _ := r.peerStore.Peers(); len(peers) <= 1 {
		cfg.EnableSingleNode = true
		log.Warn("raft will run in single node mode, may only be used in test")
	}

	r.r, err = raft.NewRaft(cfg, fsm, r.dbStore, r.dbStore, fileStore, r.peerStore, r.trans)

	return r, err
}

func (r *Raft) Close() {
	if r.trans != nil {
		r.trans.Close()
	}

	if r.r != nil {
		future := r.r.Shutdown()
		if err := future.Error(); err != nil {
			log.Errorf("Error shutting down raft: %v", err)
		}
	}

	if r.dbStore != nil {
		r.dbStore.Close()
	}

	if r.log != os.Stdout {
		r.log.Close()
	}
}

func (r *Raft) AddMasters(addrs []string, timeout time.Duration) error {
	var a = action{
		Cmd:     addCmd,
		Masters: addrs,
	}

	return r.apply(&a, timeout)
}

func (r *Raft) DelMasters(addrs []string, timeout time.Duration) error {
	var a = action{
		Cmd:     delCmd,
		Masters: addrs,
	}

	return r.apply(&a, timeout)
}

func (r *Raft) SetMasters(addrs []string, timeout time.Duration) error {
	var a = action{
		Cmd:     setCmd,
		Masters: addrs,
	}

	return r.apply(&a, timeout)
}

func (r *Raft) AddPeer(addr string) error {
	peer, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return err
	}

	f := r.r.AddPeer(peer)
	return f.Error()
}

func (r *Raft) DelPeer(addr string) error {
	peer, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return err
	}

	f := r.r.RemovePeer(peer)
	return f.Error()

}

func (r *Raft) SetPeers(addrs []string) error {
	peers := make([]net.Addr, 0, len(addrs))

	for _, addr := range addrs {
		peer, err := net.ResolveTCPAddr("tcp", addr)
		if err != nil {
			return err
		}
		peers = append(peers, peer)
	}

	f := r.r.SetPeers(peers)
	return f.Error()

}

func (r *Raft) GetPeers() ([]string, error) {
	peers, err := r.peerStore.Peers()
	if err != nil {
		return nil, err
	}

	addrs := make([]string, 0, len(peers))

	for _, peer := range peers {
		addrs = append(addrs, peer.String())
	}

	return addrs, nil
}

func (r *Raft) apply(a *action, timeout time.Duration) error {
	data, err := json.Marshal(a)
	if err != nil {
		return err
	}

	f := r.r.Apply(data, timeout)
	return f.Error()
}

func (r *Raft) LeaderCh() <-chan bool {
	return r.r.LeaderCh()
}

func (r *Raft) IsLeader() bool {
	addr := r.r.Leader()
	if addr == nil {
		return false
	} else {
		return addr.String() == r.raftAddr
	}
}

func (r *Raft) Leader() string {
	return r.r.Leader().String()
}

func (r *Raft) Barrier(timeout time.Duration) error {
	f := r.r.Barrier(timeout)
	return f.Error()
}
<|endoftext|>


<commit_before>// Copyright 2017 The Mangos Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use file except in compliance with the License.
// You may obtain a copy of the license at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package test

// This simple test just fires off a crapton of inproc clients, to see
// how many connections we could handle.  We do this using inproc, because
// we would absolutely exhaust TCP ports before we would hit any of the
// natural limits.  The inproc transport has no such limits, so we are
// effectively just testing goroutine scalability, which is what we want.
// The intention is to demonstrate that mangos can address the C10K problem
// without breaking a sweat.

import (
	"errors"
	"math/rand"
	"sync"
	"testing"
	"time"

	"github.com/go-mangos/mangos"
	"github.com/go-mangos/mangos/protocol/rep"
	"github.com/go-mangos/mangos/protocol/req"
	"github.com/go-mangos/mangos/transport/inproc"
)

func scalabilityClient(errp *error, loops int, wg *sync.WaitGroup) {
	wg.Done()
	sock, err := req.NewSocket()
	if err != nil {
		*errp = err
		return
	}
	sock.AddTransport(inproc.NewTransport())
	defer sock.Close()
	if err := sock.Dial("inproc://atscale"); err != nil {
		*errp = err
		return
	}

	for i := 0; i < loops; i++ {
		// Inject a random sleep to avoid pounding the CPU too hard.
		time.Sleep(time.Duration(rand.Int31n(1000)) * time.Microsecond)

		msg := mangos.NewMessage(3)
		msg.Body = append(msg.Body, []byte("ping")...)
		if err := sock.SendMsg(msg); err != nil {
			*errp = err
			return
		}

		if msg, err = sock.RecvMsg(); err != nil {
			*errp = err
			return
		}
		if string(msg.Body) != "pong" {
			*errp = errors.New("response mismatch")
			return
		}
		msg.Free()
	}
}

func scalabilityServer(sock mangos.Socket) {
	defer sock.Close()
	for {
		msg, e := sock.RecvMsg()
		if e != nil {
			return
		}
		msg.Body = append(msg.Body[:0], []byte("pong")...)
		e = sock.SendMsg(msg)
		if e != nil {
			return
		}
	}
}

func TestScalability(t *testing.T) {
	// On my Mac, I can run up to 1M easily, but GC takes a long time
	// to garbage collect all the stacks.
	loops := 10
	threads := 100000

	errs := make([]error, threads)

	ssock, e := rep.NewSocket()
	if e != nil {
		t.Fatalf("Cannot make server socket: %v", e)
	}
	ssock.AddTransport(inproc.NewTransport())
	if e = ssock.Listen("inproc://atscale"); e != nil {
		t.Fatalf("Cannot listen: %v", e)
	}
	time.Sleep(time.Millisecond * 100)
	go scalabilityServer(ssock)
	wg := &sync.WaitGroup{}
	wg.Add(threads)
	for i := 0; i < threads; i++ {
		go scalabilityClient(&errs[i], loops, wg)
	}

	wg.Wait()
	for i := 0; i < threads; i++ {
		if errs[i] != nil {
			t.Fatalf("Test failed: %v", errs[i])
		}
	}
}
<commit_msg>Don't run scalability test with -race (it isn't compatible).<commit_after>// +build !race
// Copyright 2017 The Mangos Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use file except in compliance with the License.
// You may obtain a copy of the license at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package test

// This simple test just fires off a crapton of inproc clients, to see
// how many connections we could handle.  We do this using inproc, because
// we would absolutely exhaust TCP ports before we would hit any of the
// natural limits.  The inproc transport has no such limits, so we are
// effectively just testing goroutine scalability, which is what we want.
// The intention is to demonstrate that mangos can address the C10K problem
// without breaking a sweat.

import (
	"errors"
	"math/rand"
	"sync"
	"testing"
	"time"

	"github.com/go-mangos/mangos"
	"github.com/go-mangos/mangos/protocol/rep"
	"github.com/go-mangos/mangos/protocol/req"
	"github.com/go-mangos/mangos/transport/inproc"
)

func scalabilityClient(errp *error, loops int, wg *sync.WaitGroup) {
	wg.Done()
	sock, err := req.NewSocket()
	if err != nil {
		*errp = err
		return
	}
	sock.AddTransport(inproc.NewTransport())
	defer sock.Close()
	if err := sock.Dial("inproc://atscale"); err != nil {
		*errp = err
		return
	}

	for i := 0; i < loops; i++ {
		// Inject a random sleep to avoid pounding the CPU too hard.
		time.Sleep(time.Duration(rand.Int31n(1000)) * time.Microsecond)

		msg := mangos.NewMessage(3)
		msg.Body = append(msg.Body, []byte("ping")...)
		if err := sock.SendMsg(msg); err != nil {
			*errp = err
			return
		}

		if msg, err = sock.RecvMsg(); err != nil {
			*errp = err
			return
		}
		if string(msg.Body) != "pong" {
			*errp = errors.New("response mismatch")
			return
		}
		msg.Free()
	}
}

func scalabilityServer(sock mangos.Socket) {
	defer sock.Close()
	for {
		msg, e := sock.RecvMsg()
		if e != nil {
			return
		}
		msg.Body = append(msg.Body[:0], []byte("pong")...)
		e = sock.SendMsg(msg)
		if e != nil {
			return
		}
	}
}

func TestScalability(t *testing.T) {
	// On my Mac, I can run up to 1M easily, but GC takes a long time
	// to garbage collect all the stacks.
	loops := 10
	threads := 100000

	errs := make([]error, threads)

	ssock, e := rep.NewSocket()
	if e != nil {
		t.Fatalf("Cannot make server socket: %v", e)
	}
	ssock.AddTransport(inproc.NewTransport())
	if e = ssock.Listen("inproc://atscale"); e != nil {
		t.Fatalf("Cannot listen: %v", e)
	}
	time.Sleep(time.Millisecond * 100)
	go scalabilityServer(ssock)
	wg := &sync.WaitGroup{}
	wg.Add(threads)
	for i := 0; i < threads; i++ {
		go scalabilityClient(&errs[i], loops, wg)
	}

	wg.Wait()
	for i := 0; i < threads; i++ {
		if errs[i] != nil {
			t.Fatalf("Test failed: %v", errs[i])
		}
	}
}
<|endoftext|>


<commit_before>package scope

import (
	"testing"

	"github.com/gernest/ngorm/fixture"
)

func TestFieldByName(t *testing.T) {
	e := fixture.TestEngine()
	e.Parent = e
	var field fixture.CalculateField
	if f, ok := FieldByName(e, &field, "Children"); !ok || f.Relationship == nil {
		t.Errorf("Should calculate fields correctly for the first time")
	}
}
<commit_msg>[scope] Add TestQuote<commit_after>package scope

import (
	"testing"

	"github.com/gernest/ngorm/dialects/ql"
	"github.com/gernest/ngorm/fixture"
)

func TestFieldByName(t *testing.T) {
	e := fixture.TestEngine()
	e.Parent = e
	var field fixture.CalculateField
	if f, ok := FieldByName(e, &field, "Children"); !ok || f.Relationship == nil {
		t.Errorf("Should calculate fields correctly for the first time")
	}
}

func TestQuote(t *testing.T) {
	e := fixture.TestEngine()
	e.Dialect = &ql.QL{}
	e.Parent = e
	sample := []struct {
		src, expetc string
	}{
		{"quote", `"quote"`},
		{"quote.quote.quote", `"quote"."quote"."quote"`},
	}

	for _, v := range sample {
		q := Quote(e, v.src)
		if q != v.expetc {
			t.Errorf("expected %s got %s", v.expetc, q)
		}
	}
}
<|endoftext|>


<commit_before>// functions and methods related to the autoscaler

package master

import (
	"log"
	"math"
	"sort"

	"github.com/aaronang/cong-the-ripper/lib"
)

// runController runs one iteration
func (m *Master) runController() {
	err := float64(m.countRequiredSlots() - m.countTotalSlots())

	dt := m.controller.dt.Seconds()
	m.controller.integral = m.controller.integral + err*dt
	derivative := (err - m.controller.prevErr) / dt
	output := m.controller.kp*err +
		m.controller.ki*m.controller.integral +
		m.controller.kd*derivative
	m.controller.prevErr = err

	// output is the error in terms of number of resources/slots
	// we convert it to adjustment to represent the number of instances
	// NOTE the + 1 may need to be changed if lib.MaxSlotsPerInstance is not 2
	adjustment := (int(math.Ceil(output)) + 1) / lib.MaxSlotsPerInstance
	log.Printf("[autoscaler] err: %v, output: %v, adjustment: %v\n", err, output, adjustment)
	m.adjustInstanceCount(adjustment)
}

func (m *Master) adjustInstanceCount(n int) {
	if n > 0 {
		go func() {
			_, err := createSlaves(m.svc, n, "8080", m.ip, m.port)
			if err != nil {
				log.Println("[autoscaler] Failed to create slaves", err)
			} else {
				log.Printf("[autoscaler] %v instances created successfully\n", n)
				// no need to report back to the master loop
				// because it should start receiving heartbeat messages
			}
		}()
	} else {
		// negate n to represent the (positive) number of instances to kill
		n = -n
		if n == 0 {
			log.Println("[autoscaler] n is 0 in adjustInstanceCount")
			return
		}

		// kills n least loaded slaves, the killed slaves may have unfinished tasks
		// but the master should detect missing heartbeats and restart the tasks
		ips := sortInstancesByTaskCount(m.instances)[:n]
		go func() {
			_, err := terminateSlaves(m.svc, instancesFromIPs(m.svc, ips))
			if err != nil {
				log.Println("[autoscaler] Failed to terminate slaves", err)
			} else {
				log.Printf("[autoscaler] %v instances terminated successfully", n)
				// again, no need to report success/failure
				// because heartbeat messages will stop
			}
		}()
	}
}

func (m *Master) maxSlots() int {
	// TODO do we set the manually or it's a property of AWS?
	return 20 * lib.MaxSlotsPerInstance
}

func (m *Master) countRequiredSlots() int {
	cnt := len(m.scheduledTasks) + len(m.newTasks)
	if cnt > m.maxSlots() {
		return m.maxSlots()
	}
	return cnt
}

func (m *Master) countTotalSlots() int {
	cnt := 0
	for _, i := range m.instances {
		cnt += i.maxSlots
	}
	return cnt
}

func sortInstancesByTaskCount(instances map[string]slave) []string {
	pairs := make([]ipSlavePair, len(instances))
	var i int
	for k, v := range instances {
		pairs[i].slave = v
		pairs[i].ip = k
		i++
	}

	sort.Sort(byTaskCount(pairs))

	ips := make([]string, len(pairs))
	for i := range ips {
		ips[i] = pairs[i].ip
	}
	return ips
}

type ipSlavePair struct {
	ip    string
	slave slave
}

type byTaskCount []ipSlavePair

func (a byTaskCount) Len() int {
	return len(a)
}

func (a byTaskCount) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a byTaskCount) Less(i, j int) bool {
	return len(a[i].slave.tasks) < len(a[j].slave.tasks)
}
<commit_msg>Generalise instance adjustment calculation<commit_after>// functions and methods related to the autoscaler

package master

import (
	"log"
	"math"
	"sort"

	"github.com/aaronang/cong-the-ripper/lib"
)

// runController runs one iteration
func (m *Master) runController() {
	err := float64(m.countRequiredSlots() - m.countTotalSlots())

	dt := m.controller.dt.Seconds()
	m.controller.integral = m.controller.integral + err*dt
	derivative := (err - m.controller.prevErr) / dt
	output := m.controller.kp*err +
		m.controller.ki*m.controller.integral +
		m.controller.kd*derivative
	m.controller.prevErr = err

	// output is the error in terms of number of resources/slots
	// we convert it to adjustment to represent the number of instances
	adjustment := int(math.Ceil((output / float64(lib.MaxSlotsPerInstance)) - 1.11e-16))
	log.Printf("[autoscaler] err: %v, output: %v, adjustment: %v\n", err, output, adjustment)
	m.adjustInstanceCount(adjustment)
}

func (m *Master) adjustInstanceCount(n int) {
	if n > 0 {
		go func() {
			_, err := createSlaves(m.svc, n, "8080", m.ip, m.port)
			if err != nil {
				log.Println("[autoscaler] Failed to create slaves", err)
			} else {
				log.Printf("[autoscaler] %v instances created successfully\n", n)
				// no need to report back to the master loop
				// because it should start receiving heartbeat messages
			}
		}()
	} else {
		// negate n to represent the (positive) number of instances to kill
		n = -n
		if n == 0 {
			log.Println("[autoscaler] n is 0 in adjustInstanceCount")
			return
		}

		// kills n least loaded slaves, the killed slaves may have unfinished tasks
		// but the master should detect missing heartbeats and restart the tasks
		ips := sortInstancesByTaskCount(m.instances)[:n]
		go func() {
			_, err := terminateSlaves(m.svc, instancesFromIPs(m.svc, ips))
			if err != nil {
				log.Println("[autoscaler] Failed to terminate slaves", err)
			} else {
				log.Printf("[autoscaler] %v instances terminated successfully", n)
				// again, no need to report success/failure
				// because heartbeat messages will stop
			}
		}()
	}
}

func (m *Master) maxSlots() int {
	// TODO do we set the manually or it's a property of AWS?
	return 20 * lib.MaxSlotsPerInstance
}

func (m *Master) countRequiredSlots() int {
	cnt := len(m.scheduledTasks) + len(m.newTasks)
	if cnt > m.maxSlots() {
		return m.maxSlots()
	}
	return cnt
}

func (m *Master) countTotalSlots() int {
	cnt := 0
	for _, i := range m.instances {
		cnt += i.maxSlots
	}
	return cnt
}

func sortInstancesByTaskCount(instances map[string]slave) []string {
	pairs := make([]ipSlavePair, len(instances))
	var i int
	for k, v := range instances {
		pairs[i].slave = v
		pairs[i].ip = k
		i++
	}

	sort.Sort(byTaskCount(pairs))

	ips := make([]string, len(pairs))
	for i := range ips {
		ips[i] = pairs[i].ip
	}
	return ips
}

type ipSlavePair struct {
	ip    string
	slave slave
}

type byTaskCount []ipSlavePair

func (a byTaskCount) Len() int {
	return len(a)
}

func (a byTaskCount) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a byTaskCount) Less(i, j int) bool {
	return len(a[i].slave.tasks) < len(a[j].slave.tasks)
}
<|endoftext|>


<commit_before>package lib

import (
	"encoding/json"
	"errors"
	"log"
	"os"

	"github.com/garyburd/redigo/redis"
)

var StorageInitErr = errors.New("Storage is not enabled")
var StorageSaveErr = errors.New("Error trying to save to storage")

type Storage struct {
	store redis.Conn
}

func NewStorage() *Storage {
	a := &Storage{}
	redisUrl := os.Getenv("REDIS_URL")

	if redisUrl == "" {
		log.Fatal("$REDIS_URL must be set")
	}

	c, err := redis.DialURL(redisUrl)
	if err != nil {
		log.Fatal(err)
	}
	a.store = c
	return a
}

func (a *Storage) Save(userId string, n Note) error {
	serialized, err := json.Marshal(n)
	if err != nil {
		return err
	}
	_, err = a.store.Do("LPUSH", userId, serialized)
	return err
}

func (a *Storage) Get(userId string) (Notes, error) {

	count, err := redis.Int(a.store.Do("LLEN", userId))

	if err != nil {
		return nil, err
	}

	items, err := redis.Values(a.store.Do("LRANGE", userId, 0, count))

	if err != nil {
		return nil, err
	}

	var all Notes

	for i := range items {
		var n Note
		serialized, _ := redis.Bytes(items[i], nil)
		json.Unmarshal(serialized, &n)
		all = append(all, &n)
	}
	return all, nil
}
<commit_msg>use connection pool<commit_after>package lib

import (
	"encoding/json"
	"errors"
	"log"
	"os"
	"time"

	"github.com/garyburd/redigo/redis"
)

var StorageInitErr = errors.New("Storage is not enabled")
var StorageSaveErr = errors.New("Error trying to save to storage")

type Storage struct {
	store *redis.Pool
}

func NewStorage() *Storage {
	a := &Storage{}
	redisUrl := os.Getenv("REDIS_URL")

	if redisUrl == "" {
		log.Fatal("$REDIS_URL must be set")
	}

	a.store = newPool()
	return a
}

func newPool() *redis.Pool {

	redisUrl := os.Getenv("REDIS_URL")

	if redisUrl == "" {
		log.Fatal("$REDIS_URL must be set")
	}
	return &redis.Pool{
		MaxIdle:     3,
		IdleTimeout: 240 * time.Second,
		Dial: func() (redis.Conn, error) {
			c, err := redis.DialURL(redisUrl)
			if err != nil {
				return nil, err
			}
			return c, err
		},
	}
}

func (a *Storage) Save(userId string, n Note) error {

	serialized, err := json.Marshal(n)
	if err != nil {
		return err
	}
	_, err = a.store.Get().Do("LPUSH", userId, serialized)
	return err
}

func (a *Storage) Get(userId string) (Notes, error) {

	c := a.store.Get()

	count, err := redis.Int(c.Do("LLEN", userId))

	if err != nil {
		return nil, err
	}

	items, err := redis.Values(c.Do("LRANGE", userId, 0, count))

	if err != nil {
		return nil, err
	}

	var all Notes

	for i := range items {
		var n Note
		serialized, _ := redis.Bytes(items[i], nil)
		json.Unmarshal(serialized, &n)
		all = append(all, &n)
	}
	return all, nil
}
<|endoftext|>