<commit_before>// Copyright 2013 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cache

import (
	"container/list"
	"sync"
)

// Create a cache that holds the given number of items, evicting the least
// recently used item when more space is needed.
func NewLruCache(capacity uint) Cache {
	return nil
}

type lruCache struct {
	mutex    sync.RWMutex
	capacity uint

	// List of elements, with least recently used at the tail.
	elems list.List

	// Index int `elems` for lookup by key.
	index map[string]*list.Element
}

func (c *lruCache) Insert(key string, value interface{}) {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	// Make sure the key isn't already present.
	c.erase_Locked(key)

	// Add a list element and index it.
	elem := c.elems.PushFront(value)
	c.index[key] = elem
}

func (c *lruCache) erase_Locked(key string) {
	elem, ok := c.index[key]
	if !ok {
		return
	}

	delete(c.index, key)
	c.elems.Remove(elem)
}

func (c *lruCache) LookUp(key string) interface{} {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	if elem, ok := c.index[key]; ok {
		return elem.Value
	}

	return nil
}
<commit_msg>Implemented NewLruCache.<commit_after>// Copyright 2013 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cache

import (
	"container/list"
	"sync"
)

// Create a cache that holds the given number of items, evicting the least
// recently used item when more space is needed.
func NewLruCache(capacity uint) Cache {
	return &lruCache{capacity: capacity, index: make(map[string]*list.Element)}
}

type lruCache struct {
	mutex    sync.RWMutex
	capacity uint

	// List of elements, with least recently used at the tail.
	elems list.List

	// Index int `elems` for lookup by key.
	index map[string]*list.Element
}

func (c *lruCache) Insert(key string, value interface{}) {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	// Make sure the key isn't already present.
	c.erase_Locked(key)

	// Add a list element and index it.
	elem := c.elems.PushFront(value)
	c.index[key] = elem
}

func (c *lruCache) erase_Locked(key string) {
	elem, ok := c.index[key]
	if !ok {
		return
	}

	delete(c.index, key)
	c.elems.Remove(elem)
}

func (c *lruCache) LookUp(key string) interface{} {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	if elem, ok := c.index[key]; ok {
		return elem.Value
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/go.net/websocket"
	"github.com/coreos/go-etcd/etcd"
	"github.com/fsouza/go-dockerclient"
	"github.com/go-martini/martini"
)

var debugMode bool

func debug(v ...interface{}) {
	if debugMode {
		log.Println(v...)
	}
}

func assert(err error, context string) {
	if err != nil {
		log.Fatal(context+": ", err)
	}
}

func getopt(name, dfault string) string {
	value := os.Getenv(name)
	if value == "" {
		value = dfault
	}
	return value
}

type Colorizer map[string]int

// returns up to 14 color escape codes (then repeats) for each unique key
func (c Colorizer) Get(key string) string {
	i, exists := c[key]
	if !exists {
		c[key] = len(c)
		i = c[key]
	}
	bright := "1;"
	if i%14 > 6 {
		bright = ""
	}
	return "\x1b[" + bright + "3" + strconv.Itoa(7-(i%7)) + "m"
}

func syslogStreamer(target Target, types []string, logstream chan *Log) {
	typestr := "," + strings.Join(types, ",") + ","
	for logline := range logstream {
		if typestr != ",," && !strings.Contains(typestr, logline.Type) {
			continue
		}
		tag, pid := getLogName(logline.Name)
		conn, err := net.Dial("udp", target.Addr)
		assert(err, "syslog")
		// HACK: Go's syslog package hardcodes the log format, so let's send our own message
		_, err = fmt.Fprintf(conn,
			"%s %s[%s]: %s",
			time.Now().Format("2006-01-02 15:04:05"),
			tag,
			pid,
			logline.Data)
		assert(err, "syslog")
	}
}

// getLogName returns a custom tag and PID for containers that
// match Deis' specific application name format. Otherwise,
// it returns the original name and 1 as the PID.
func getLogName(name string) (string, string) {
	// example regex that should match: go_v2.web.1
	r := regexp.MustCompile(`(^[a-z0-9-]+)_(v[0-9]+)\.([a-z-_]+\.[0-9]+)$`)
	match := r.FindStringSubmatch(name)
	if match == nil {
		return name, "1"
	} else {
		return match[1], match[3]
	}
}

func websocketStreamer(w http.ResponseWriter, req *http.Request, logstream chan *Log, closer chan bool) {
	websocket.Handler(func(conn *websocket.Conn) {
		for logline := range logstream {
			if req.URL.Query().Get("type") != "" && logline.Type != req.URL.Query().Get("type") {
				continue
			}
			_, err := conn.Write(append(marshal(logline), '\n'))
			if err != nil {
				closer <- true
				return
			}
		}
	}).ServeHTTP(w, req)
}

func httpStreamer(w http.ResponseWriter, req *http.Request, logstream chan *Log, multi bool) {
	var colors Colorizer
	var usecolor, usejson bool
	nameWidth := 16
	if req.URL.Query().Get("colors") != "off" {
		colors = make(Colorizer)
		usecolor = true
	}
	if req.Header.Get("Accept") == "application/json" {
		w.Header().Add("Content-Type", "application/json")
		usejson = true
	} else {
		w.Header().Add("Content-Type", "text/plain")
	}
	for logline := range logstream {
		if req.URL.Query().Get("types") != "" && logline.Type != req.URL.Query().Get("types") {
			continue
		}
		if usejson {
			w.Write(append(marshal(logline), '\n'))
		} else {
			if multi {
				if len(logline.Name) > nameWidth {
					nameWidth = len(logline.Name)
				}
				if usecolor {
					w.Write([]byte(fmt.Sprintf(
						"%s%"+strconv.Itoa(nameWidth)+"s|%s\x1b[0m\n",
						colors.Get(logline.Name), logline.Name, logline.Data,
					)))
				} else {
					w.Write([]byte(fmt.Sprintf(
						"%"+strconv.Itoa(nameWidth)+"s|%s\n", logline.Name, logline.Data,
					)))
				}
			} else {
				w.Write(append([]byte(logline.Data), '\n'))
			}
		}
		w.(http.Flusher).Flush()
	}
}

func main() {
	debugMode = getopt("DEBUG", "") != ""
	port := getopt("PORT", "8000")
	endpoint := getopt("DOCKER_HOST", "unix:///var/run/docker.sock")
	routespath := getopt("ROUTESPATH", "/var/lib/logspout")

	client, err := docker.NewClient(endpoint)
	assert(err, "docker")
	attacher := NewAttachManager(client)
	router := NewRouteManager(attacher)

	// HACK: if we are connecting to etcd, get the logger's connection
	// details from there
	if etcdHost := os.Getenv("ETCD_HOST"); etcdHost != "" {
		connectionString := []string{"http://" + etcdHost + ":4001"}
		debug("etcd:", connectionString[0])
		etcd := etcd.NewClient(connectionString)
		etcd.SetDialTimeout(3 * time.Second)
		hostResp, err := etcd.Get("/deis/logs/host", false, false)
		assert(err, "url")
		portResp, err := etcd.Get("/deis/logs/port", false, false)
		assert(err, "url")
		host := fmt.Sprintf("%s:%s", hostResp.Node.Value, portResp.Node.Value)
		log.Println("routing all to " + host)
		router.Add(&Route{Target: Target{Type: "syslog", Addr: host}})
	}

	if len(os.Args) > 1 {
		u, err := url.Parse(os.Args[1])
		assert(err, "url")
		log.Println("routing all to " + os.Args[1])
		router.Add(&Route{Target: Target{Type: u.Scheme, Addr: u.Host}})
	}

	if _, err := os.Stat(routespath); err == nil {
		log.Println("loading and persisting routes in " + routespath)
		assert(router.Load(RouteFileStore(routespath)), "persistor")
	}

	m := martini.Classic()

	m.Get("/logs(?:/(?P<predicate>[a-zA-Z]+):(?P<value>.+))?", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		source := new(Source)
		switch {
		case params["predicate"] == "id" && params["value"] != "":
			source.ID = params["value"][:12]
		case params["predicate"] == "name" && params["value"] != "":
			source.Name = params["value"]
		case params["predicate"] == "filter" && params["value"] != "":
			source.Filter = params["value"]
		}

		if source.ID != "" && attacher.Get(source.ID) == nil {
			http.NotFound(w, req)
			return
		}

		logstream := make(chan *Log)
		defer close(logstream)

		var closer <-chan bool
		if req.Header.Get("Upgrade") == "websocket" {
			closerBi := make(chan bool)
			go websocketStreamer(w, req, logstream, closerBi)
			closer = closerBi
		} else {
			go httpStreamer(w, req, logstream, source.All() || source.Filter != "")
			closer = w.(http.CloseNotifier).CloseNotify()
		}

		attacher.Listen(source, logstream, closer)
	})

	m.Get("/routes", func(w http.ResponseWriter, req *http.Request) {
		w.Header().Add("Content-Type", "application/json")
		routes, _ := router.GetAll()
		w.Write(append(marshal(routes), '\n'))
	})

	m.Post("/routes", func(w http.ResponseWriter, req *http.Request) (int, string) {
		route := new(Route)
		if err := unmarshal(req.Body, route); err != nil {
			return http.StatusBadRequest, "Bad request: " + err.Error()
		}

		// TODO: validate?
		router.Add(route)

		w.Header().Add("Content-Type", "application/json")
		return http.StatusCreated, string(append(marshal(route), '\n'))
	})

	m.Get("/routes/:id", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		route, _ := router.Get(params["id"])
		if route == nil {
			http.NotFound(w, req)
			return
		}
		w.Write(append(marshal(route), '\n'))
	})

	m.Delete("/routes/:id", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		if ok := router.Remove(params["id"]); !ok {
			http.NotFound(w, req)
		}
	})

	log.Println("logspout serving http on :" + port)
	log.Fatal(http.ListenAndServe(":"+port, m))
}
<commit_msg>fix(logspout): add timezone to logs<commit_after>package main

import (
	"fmt"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/go.net/websocket"
	"github.com/coreos/go-etcd/etcd"
	"github.com/fsouza/go-dockerclient"
	"github.com/go-martini/martini"
)

var debugMode bool

func debug(v ...interface{}) {
	if debugMode {
		log.Println(v...)
	}
}

func assert(err error, context string) {
	if err != nil {
		log.Fatal(context+": ", err)
	}
}

func getopt(name, dfault string) string {
	value := os.Getenv(name)
	if value == "" {
		value = dfault
	}
	return value
}

type Colorizer map[string]int

// returns up to 14 color escape codes (then repeats) for each unique key
func (c Colorizer) Get(key string) string {
	i, exists := c[key]
	if !exists {
		c[key] = len(c)
		i = c[key]
	}
	bright := "1;"
	if i%14 > 6 {
		bright = ""
	}
	return "\x1b[" + bright + "3" + strconv.Itoa(7-(i%7)) + "m"
}

func syslogStreamer(target Target, types []string, logstream chan *Log) {
	typestr := "," + strings.Join(types, ",") + ","
	for logline := range logstream {
		if typestr != ",," && !strings.Contains(typestr, logline.Type) {
			continue
		}
		tag, pid := getLogName(logline.Name)
		conn, err := net.Dial("udp", target.Addr)
		assert(err, "syslog")
		// HACK: Go's syslog package hardcodes the log format, so let's send our own message
		_, err = fmt.Fprintf(conn,
			"%s %s[%s]: %s",
			time.Now().Format("2006-01-02 15:04:05 MST"),
			tag,
			pid,
			logline.Data)
		assert(err, "syslog")
	}
}

// getLogName returns a custom tag and PID for containers that
// match Deis' specific application name format. Otherwise,
// it returns the original name and 1 as the PID.
func getLogName(name string) (string, string) {
	// example regex that should match: go_v2.web.1
	r := regexp.MustCompile(`(^[a-z0-9-]+)_(v[0-9]+)\.([a-z-_]+\.[0-9]+)$`)
	match := r.FindStringSubmatch(name)
	if match == nil {
		return name, "1"
	} else {
		return match[1], match[3]
	}
}

func websocketStreamer(w http.ResponseWriter, req *http.Request, logstream chan *Log, closer chan bool) {
	websocket.Handler(func(conn *websocket.Conn) {
		for logline := range logstream {
			if req.URL.Query().Get("type") != "" && logline.Type != req.URL.Query().Get("type") {
				continue
			}
			_, err := conn.Write(append(marshal(logline), '\n'))
			if err != nil {
				closer <- true
				return
			}
		}
	}).ServeHTTP(w, req)
}

func httpStreamer(w http.ResponseWriter, req *http.Request, logstream chan *Log, multi bool) {
	var colors Colorizer
	var usecolor, usejson bool
	nameWidth := 16
	if req.URL.Query().Get("colors") != "off" {
		colors = make(Colorizer)
		usecolor = true
	}
	if req.Header.Get("Accept") == "application/json" {
		w.Header().Add("Content-Type", "application/json")
		usejson = true
	} else {
		w.Header().Add("Content-Type", "text/plain")
	}
	for logline := range logstream {
		if req.URL.Query().Get("types") != "" && logline.Type != req.URL.Query().Get("types") {
			continue
		}
		if usejson {
			w.Write(append(marshal(logline), '\n'))
		} else {
			if multi {
				if len(logline.Name) > nameWidth {
					nameWidth = len(logline.Name)
				}
				if usecolor {
					w.Write([]byte(fmt.Sprintf(
						"%s%"+strconv.Itoa(nameWidth)+"s|%s\x1b[0m\n",
						colors.Get(logline.Name), logline.Name, logline.Data,
					)))
				} else {
					w.Write([]byte(fmt.Sprintf(
						"%"+strconv.Itoa(nameWidth)+"s|%s\n", logline.Name, logline.Data,
					)))
				}
			} else {
				w.Write(append([]byte(logline.Data), '\n'))
			}
		}
		w.(http.Flusher).Flush()
	}
}

func main() {
	debugMode = getopt("DEBUG", "") != ""
	port := getopt("PORT", "8000")
	endpoint := getopt("DOCKER_HOST", "unix:///var/run/docker.sock")
	routespath := getopt("ROUTESPATH", "/var/lib/logspout")

	client, err := docker.NewClient(endpoint)
	assert(err, "docker")
	attacher := NewAttachManager(client)
	router := NewRouteManager(attacher)

	// HACK: if we are connecting to etcd, get the logger's connection
	// details from there
	if etcdHost := os.Getenv("ETCD_HOST"); etcdHost != "" {
		connectionString := []string{"http://" + etcdHost + ":4001"}
		debug("etcd:", connectionString[0])
		etcd := etcd.NewClient(connectionString)
		etcd.SetDialTimeout(3 * time.Second)
		hostResp, err := etcd.Get("/deis/logs/host", false, false)
		assert(err, "url")
		portResp, err := etcd.Get("/deis/logs/port", false, false)
		assert(err, "url")
		host := fmt.Sprintf("%s:%s", hostResp.Node.Value, portResp.Node.Value)
		log.Println("routing all to " + host)
		router.Add(&Route{Target: Target{Type: "syslog", Addr: host}})
	}

	if len(os.Args) > 1 {
		u, err := url.Parse(os.Args[1])
		assert(err, "url")
		log.Println("routing all to " + os.Args[1])
		router.Add(&Route{Target: Target{Type: u.Scheme, Addr: u.Host}})
	}

	if _, err := os.Stat(routespath); err == nil {
		log.Println("loading and persisting routes in " + routespath)
		assert(router.Load(RouteFileStore(routespath)), "persistor")
	}

	m := martini.Classic()

	m.Get("/logs(?:/(?P<predicate>[a-zA-Z]+):(?P<value>.+))?", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		source := new(Source)
		switch {
		case params["predicate"] == "id" && params["value"] != "":
			source.ID = params["value"][:12]
		case params["predicate"] == "name" && params["value"] != "":
			source.Name = params["value"]
		case params["predicate"] == "filter" && params["value"] != "":
			source.Filter = params["value"]
		}

		if source.ID != "" && attacher.Get(source.ID) == nil {
			http.NotFound(w, req)
			return
		}

		logstream := make(chan *Log)
		defer close(logstream)

		var closer <-chan bool
		if req.Header.Get("Upgrade") == "websocket" {
			closerBi := make(chan bool)
			go websocketStreamer(w, req, logstream, closerBi)
			closer = closerBi
		} else {
			go httpStreamer(w, req, logstream, source.All() || source.Filter != "")
			closer = w.(http.CloseNotifier).CloseNotify()
		}

		attacher.Listen(source, logstream, closer)
	})

	m.Get("/routes", func(w http.ResponseWriter, req *http.Request) {
		w.Header().Add("Content-Type", "application/json")
		routes, _ := router.GetAll()
		w.Write(append(marshal(routes), '\n'))
	})

	m.Post("/routes", func(w http.ResponseWriter, req *http.Request) (int, string) {
		route := new(Route)
		if err := unmarshal(req.Body, route); err != nil {
			return http.StatusBadRequest, "Bad request: " + err.Error()
		}

		// TODO: validate?
		router.Add(route)

		w.Header().Add("Content-Type", "application/json")
		return http.StatusCreated, string(append(marshal(route), '\n'))
	})

	m.Get("/routes/:id", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		route, _ := router.Get(params["id"])
		if route == nil {
			http.NotFound(w, req)
			return
		}
		w.Write(append(marshal(route), '\n'))
	})

	m.Delete("/routes/:id", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		if ok := router.Remove(params["id"]); !ok {
			http.NotFound(w, req)
		}
	})

	log.Println("logspout serving http on :" + port)
	log.Fatal(http.ListenAndServe(":"+port, m))
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gochimp

import (
	"encoding/json"
	"errors"
	"time"
)

// see https://mandrillapp.com/api/docs/messages.html
const messages_send_endpoint string = "/messages/send.json"                   // Send a new transactional message through Mandrill
const messages_send_template_endpoint string = "/messages/send-template.json" // Send a new transactional message through Mandrill using a template
const messages_search_endpoint string = "/messages/search.json"               // Search the content of recently sent messages and optionally narrow by date range, tags and senders
const messages_parse_endpoint string = "/messages/parse.json"                 // Parse the full MIME document for an email message, returning the content of the message broken into its constituent pieces
const messages_send_raw_endpoint string = "/messages/send-raw.json"           // Take a raw MIME document for a message, and send it exactly as if it were sent over the SMTP protocol

func (a *MandrillAPI) MessageSend(message Message, async bool) ([]SendResponse, error) {
	var response []SendResponse
	var params map[string]interface{} = make(map[string]interface{})
	params["message"] = message
	params["async"] = async
	err := parseMandrillJson(a, messages_send_endpoint, params, &response)
	return response, err
}

func (a *MandrillAPI) MessageSendTemplate(templateName string, templateContent []Var, message Message, async bool) ([]SendResponse, error) {
	var response []SendResponse
	if templateName == "" {
		return response, errors.New("templateName cannot be blank")
	}
	var params map[string]interface{} = make(map[string]interface{})
	params["message"] = message
	params["template_name"] = templateName
	params["async"] = async
	params["template_content"] = templateContent
	err := parseMandrillJson(a, messages_send_template_endpoint, params, &response)
	return response, err
}

func (a *MandrillAPI) MessageSearch(searchRequest SearchRequest) ([]SearchResponse, error) {
	var response []SearchResponse
	var params map[string]interface{} = make(map[string]interface{})
	//todo remove this hack
	params["query"] = searchRequest.Query
	params["date_from"] = searchRequest.DateFrom
	params["date_to"] = searchRequest.DateTo
	params["tags"] = searchRequest.Tags
	params["senders"] = searchRequest.Senders
	params["limit"] = searchRequest.Limit
	err := parseMandrillJson(a, messages_search_endpoint, params, &response)
	return response, err
}

func (a *MandrillAPI) MessageParse(rawMessage string, async bool) (Message, error) {
	var response Message
	if rawMessage == "" {
		return response, errors.New("rawMessage cannot be blank")
	}
	var params map[string]interface{} = make(map[string]interface{})
	params["raw_message"] = rawMessage
	err := parseMandrillJson(a, messages_parse_endpoint, params, &response)
	return response, err
}

// Can return oneof Invalid_Key, ValidationError or GeneralError
func (a *MandrillAPI) MessageSendRaw(rawMessage string, to []string, from Recipient, async bool) ([]SendResponse, error) {
	var response []SendResponse
	if rawMessage == "" {
		return response, errors.New("rawMessage cannot be blank")
	}
	if len(to) <= 0 {
		return response, errors.New("You need at least one recipient in the To array")
	}

	var params map[string]interface{} = make(map[string]interface{})
	params["raw_message"] = rawMessage
	params["from_email"] = from.Email
	params["from_name"] = from.Name
	params["to"] = to
	params["async"] = async
	err := parseMandrillJson(a, messages_send_raw_endpoint, params, &response)
	return response, err
}

type SearchResponse struct {
	Timestamp time.Duration       `json:"ts"`
	Id        string              `json:"_id"`
	Sender    string              `json:"sender"`
	Subject   string              `json:"subject"`
	Email     string              `json:"email"`
	Tags      []string            `json:"tags"`
	Opens     int                 `json:"opens"`
	Clicks    int                 `json:"clicks"`
	State     string              `json:"state"`
	Metadata  []map[string]string `json:"metadata"`
}

type SearchRequest struct {
	Query    string    `json:"query"`
	DateFrom time.Time `json:"date_from"`
	DateTo   time.Time `json:"date_to"`
	Tags     []string  `json:"tags"`
	Senders  []string  `json:"senders"`
	Limit    int       `json:"limit"`
}

type Message struct {
	Html                    string              `json:"html,omitempty"`
	Text                    string              `json:"text,omitempty"`
	Subject                 string              `json:"subject"`
	FromEmail               string              `json:"from_email"`
	FromName                string              `json:"from_name"`
	To                      []Recipient         `json:"to"`
	Headers                 map[string]string   `json:"headers,omitempty"`
	TrackOpens              bool                `json:"track_opens,omitempty"`
	TrackClicks             bool                `json:"track_clicks,omitempty"`
	ViewContentLink         bool                `json:"view_content_link,omitempty"`
	AutoText                bool                `json:"auto_text,omitempty"`
	AutoHtml                bool                `json:"auto_html,omitempty"`
	UrlStripQS              bool                `json:"url_strip_qs,omitempty"`
	InlineCss               bool                `json:"inline_css,omitempty"`
	PreserveRecipients      bool                `json:"preserve_recipients,omitempty"`
	Important               bool                `json:"important,omitempty"`
	BCCAddress              string              `json:"bcc_address,omitempty"`
	TrackingDomain          string              `json:"tracking_domain,omitempty"`
	SigningDomain           string              `json:"signing_domain,omitempty"`
	ReturnPathDomain        string              `json:"return_path_domain,omitempty"`
	Merge                   bool                `json:"merge,omitempty"`
	GlobalMergeVars         []Var               `json:"global_merge_vars,omitempty"`
	MergeVars               []MergeVars         `json:"merge_vars,omitempty"`
	Tags                    []string            `json:"tags,omitempty"`
	GoogleAnalyticsDomains  []string            `json:"google_analytics_domains,omitempty"`
	GoogleAnalyticsCampaign []string            `json:"google_analytics_campaign,omitempty"`
	Metadata                []map[string]string `json:"metadata,omitempty"`
	RecipientMetadata       []RecipientMetaData `json:"recipient_metadata,omitempty"`
	Attachments             []Attachment        `json:"attachments,omitempty"`
}

func (m *Message) String() string {
	b, _ := json.Marshal(m)
	return string(b)
}

func (m *Message) AddHeader(key, value string) {
	if m.Headers == nil {
		m.Headers = make(map[string]string)
	}
	m.Headers[key] = value
}

func (m *Message) AddRecipients(r ...Recipient) {
	m.To = append(m.To, r...)
}

func (m *Message) AddGlobalMergeVar(globalvars ...Var) {
	m.GlobalMergeVars = append(m.GlobalMergeVars, globalvars...)
}

func (m *Message) AddMergeVar(vars ...MergeVars) {
	m.MergeVars = append(m.MergeVars, vars...)
}

func (m *Message) AddTag(tags ...string) {
	m.Tags = append(m.Tags, tags...)
}

func (m *Message) AddGoogleAnalyticsDomains(domains ...string) {
	m.GoogleAnalyticsDomains = append(m.GoogleAnalyticsDomains, domains...)
}

func (m *Message) AddGoogleAnalyticsCampaign(campaigns ...string) {
	m.GoogleAnalyticsCampaign = append(m.GoogleAnalyticsCampaign, campaigns...)
}

func (m *Message) AddMetadata(metadata ...map[string]string) {
	m.Metadata = append(m.Metadata, metadata...)
}

func (m *Message) AddRecipientMetadata(metadata ...RecipientMetaData) {
	m.RecipientMetadata = append(m.RecipientMetadata, metadata...)
}

func (m *Message) AddAttachments(attachement ...Attachment) {
	m.Attachments = append(m.Attachments, attachement...)
}

type Attachment struct {
	Type    string `json:"type"`
	Name    string `json:"name"`
	Content string `json:"content"`
}
type RecipientMetaData struct {
	Recipient string            `json:"rcpt"`
	Vars      map[string]string `json:"values"`
}

type MergeVars struct {
	Recipient string `json:"rcpt"`
	Vars      []Var  `json:"vars"`
}

type Var struct {
	Name    string `json:"name"`
	Content string `json:"content"`
}

func NewVar(name string, content string) *Var {
	return &Var{Name: name, Content: content}
}

type Recipient struct {
	Email string `json:"email"`
	Name  string `json:"name"`
	Type  string `json:"type"`
}

type SendResponse struct {
	Email  string `json:"email"`
	Status string `json:"status"`
}
<commit_msg>Removing omitempty for TrackOpens and TrackClicks to make sure the values are transferred to mandrill as false and not omitted. This is because mandrill seems to default to true for these values, and if they are omitted there is no way to turn off click tracking.<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gochimp

import (
	"encoding/json"
	"errors"
	"time"
)

// see https://mandrillapp.com/api/docs/messages.html
const messages_send_endpoint string = "/messages/send.json"                   // Send a new transactional message through Mandrill
const messages_send_template_endpoint string = "/messages/send-template.json" // Send a new transactional message through Mandrill using a template
const messages_search_endpoint string = "/messages/search.json"               // Search the content of recently sent messages and optionally narrow by date range, tags and senders
const messages_parse_endpoint string = "/messages/parse.json"                 // Parse the full MIME document for an email message, returning the content of the message broken into its constituent pieces
const messages_send_raw_endpoint string = "/messages/send-raw.json"           // Take a raw MIME document for a message, and send it exactly as if it were sent over the SMTP protocol

func (a *MandrillAPI) MessageSend(message Message, async bool) ([]SendResponse, error) {
	var response []SendResponse
	var params map[string]interface{} = make(map[string]interface{})
	params["message"] = message
	params["async"] = async
	err := parseMandrillJson(a, messages_send_endpoint, params, &response)
	return response, err
}

func (a *MandrillAPI) MessageSendTemplate(templateName string, templateContent []Var, message Message, async bool) ([]SendResponse, error) {
	var response []SendResponse
	if templateName == "" {
		return response, errors.New("templateName cannot be blank")
	}
	var params map[string]interface{} = make(map[string]interface{})
	params["message"] = message
	params["template_name"] = templateName
	params["async"] = async
	params["template_content"] = templateContent
	err := parseMandrillJson(a, messages_send_template_endpoint, params, &response)
	return response, err
}

func (a *MandrillAPI) MessageSearch(searchRequest SearchRequest) ([]SearchResponse, error) {
	var response []SearchResponse
	var params map[string]interface{} = make(map[string]interface{})
	//todo remove this hack
	params["query"] = searchRequest.Query
	params["date_from"] = searchRequest.DateFrom
	params["date_to"] = searchRequest.DateTo
	params["tags"] = searchRequest.Tags
	params["senders"] = searchRequest.Senders
	params["limit"] = searchRequest.Limit
	err := parseMandrillJson(a, messages_search_endpoint, params, &response)
	return response, err
}

func (a *MandrillAPI) MessageParse(rawMessage string, async bool) (Message, error) {
	var response Message
	if rawMessage == "" {
		return response, errors.New("rawMessage cannot be blank")
	}
	var params map[string]interface{} = make(map[string]interface{})
	params["raw_message"] = rawMessage
	err := parseMandrillJson(a, messages_parse_endpoint, params, &response)
	return response, err
}

// Can return oneof Invalid_Key, ValidationError or GeneralError
func (a *MandrillAPI) MessageSendRaw(rawMessage string, to []string, from Recipient, async bool) ([]SendResponse, error) {
	var response []SendResponse
	if rawMessage == "" {
		return response, errors.New("rawMessage cannot be blank")
	}
	if len(to) <= 0 {
		return response, errors.New("You need at least one recipient in the To array")
	}

	var params map[string]interface{} = make(map[string]interface{})
	params["raw_message"] = rawMessage
	params["from_email"] = from.Email
	params["from_name"] = from.Name
	params["to"] = to
	params["async"] = async
	err := parseMandrillJson(a, messages_send_raw_endpoint, params, &response)
	return response, err
}

type SearchResponse struct {
	Timestamp time.Duration       `json:"ts"`
	Id        string              `json:"_id"`
	Sender    string              `json:"sender"`
	Subject   string              `json:"subject"`
	Email     string              `json:"email"`
	Tags      []string            `json:"tags"`
	Opens     int                 `json:"opens"`
	Clicks    int                 `json:"clicks"`
	State     string              `json:"state"`
	Metadata  []map[string]string `json:"metadata"`
}

type SearchRequest struct {
	Query    string    `json:"query"`
	DateFrom time.Time `json:"date_from"`
	DateTo   time.Time `json:"date_to"`
	Tags     []string  `json:"tags"`
	Senders  []string  `json:"senders"`
	Limit    int       `json:"limit"`
}

type Message struct {
	Html                    string              `json:"html,omitempty"`
	Text                    string              `json:"text,omitempty"`
	Subject                 string              `json:"subject"`
	FromEmail               string              `json:"from_email"`
	FromName                string              `json:"from_name"`
	To                      []Recipient         `json:"to"`
	Headers                 map[string]string   `json:"headers,omitempty"`
	TrackOpens              bool                `json:"track_opens"`
	TrackClicks             bool                `json:"track_clicks"`
	ViewContentLink         bool                `json:"view_content_link,omitempty"`
	AutoText                bool                `json:"auto_text,omitempty"`
	AutoHtml                bool                `json:"auto_html,omitempty"`
	UrlStripQS              bool                `json:"url_strip_qs,omitempty"`
	InlineCss               bool                `json:"inline_css,omitempty"`
	PreserveRecipients      bool                `json:"preserve_recipients,omitempty"`
	Important               bool                `json:"important,omitempty"`
	BCCAddress              string              `json:"bcc_address,omitempty"`
	TrackingDomain          string              `json:"tracking_domain,omitempty"`
	SigningDomain           string              `json:"signing_domain,omitempty"`
	ReturnPathDomain        string              `json:"return_path_domain,omitempty"`
	Merge                   bool                `json:"merge,omitempty"`
	GlobalMergeVars         []Var               `json:"global_merge_vars,omitempty"`
	MergeVars               []MergeVars         `json:"merge_vars,omitempty"`
	Tags                    []string            `json:"tags,omitempty"`
	GoogleAnalyticsDomains  []string            `json:"google_analytics_domains,omitempty"`
	GoogleAnalyticsCampaign []string            `json:"google_analytics_campaign,omitempty"`
	Metadata                []map[string]string `json:"metadata,omitempty"`
	RecipientMetadata       []RecipientMetaData `json:"recipient_metadata,omitempty"`
	Attachments             []Attachment        `json:"attachments,omitempty"`
}

func (m *Message) String() string {
	b, _ := json.Marshal(m)
	return string(b)
}

func (m *Message) AddHeader(key, value string) {
	if m.Headers == nil {
		m.Headers = make(map[string]string)
	}
	m.Headers[key] = value
}

func (m *Message) AddRecipients(r ...Recipient) {
	m.To = append(m.To, r...)
}

func (m *Message) AddGlobalMergeVar(globalvars ...Var) {
	m.GlobalMergeVars = append(m.GlobalMergeVars, globalvars...)
}

func (m *Message) AddMergeVar(vars ...MergeVars) {
	m.MergeVars = append(m.MergeVars, vars...)
}

func (m *Message) AddTag(tags ...string) {
	m.Tags = append(m.Tags, tags...)
}

func (m *Message) AddGoogleAnalyticsDomains(domains ...string) {
	m.GoogleAnalyticsDomains = append(m.GoogleAnalyticsDomains, domains...)
}

func (m *Message) AddGoogleAnalyticsCampaign(campaigns ...string) {
	m.GoogleAnalyticsCampaign = append(m.GoogleAnalyticsCampaign, campaigns...)
}

func (m *Message) AddMetadata(metadata ...map[string]string) {
	m.Metadata = append(m.Metadata, metadata...)
}

func (m *Message) AddRecipientMetadata(metadata ...RecipientMetaData) {
	m.RecipientMetadata = append(m.RecipientMetadata, metadata...)
}

func (m *Message) AddAttachments(attachement ...Attachment) {
	m.Attachments = append(m.Attachments, attachement...)
}

type Attachment struct {
	Type    string `json:"type"`
	Name    string `json:"name"`
	Content string `json:"content"`
}
type RecipientMetaData struct {
	Recipient string            `json:"rcpt"`
	Vars      map[string]string `json:"values"`
}

type MergeVars struct {
	Recipient string `json:"rcpt"`
	Vars      []Var  `json:"vars"`
}

type Var struct {
	Name    string `json:"name"`
	Content string `json:"content"`
}

func NewVar(name string, content string) *Var {
	return &Var{Name: name, Content: content}
}

type Recipient struct {
	Email string `json:"email"`
	Name  string `json:"name"`
	Type  string `json:"type"`
}

type SendResponse struct {
	Email  string `json:"email"`
	Status string `json:"status"`
}
<|endoftext|>


<commit_before>package util

import (
	"io/ioutil"
	"os"
	"path"
	"strings"
)

func FileExist(name string) bool {
	_, err := os.Stat(name)
	return err == nil
}

func BaseName(s, suffix string) string {
	base := path.Base(s)
	if suffix != "" {
		base = strings.TrimSuffix(base, suffix)
	}
	return base
}

func TempDir() (string, error) {
	return ioutil.TempDir("", "chef-runner-")
}
<commit_msg>Document util package<commit_after>// Package util provides various utility functions.
package util

import (
	"io/ioutil"
	"os"
	"path"
	"strings"
)

// FileExist reports whether a file or directory exists.
func FileExist(name string) bool {
	_, err := os.Stat(name)
	return err == nil
}

// BaseName, as the basename Unix tool, deletes any prefix ending with the last
// slash character present in a string, and a suffix, if given.
func BaseName(s, suffix string) string {
	base := path.Base(s)
	if suffix != "" {
		base = strings.TrimSuffix(base, suffix)
	}
	return base
}

// TempDir creates a new temporary directory to be used by chef-runner.
func TempDir() (string, error) {
	return ioutil.TempDir("", "chef-runner-")
}
<|endoftext|>


<commit_before>package util

import (
	"io"
	"log"
	"math/rand"
	"os"
	"time"
)

func QuickSort(a []int) {
	sort(a, 0, len(a))
}

func sort(a []int, from, to int) {
	if to-from <= 1 {
		return
	}
	p := partition(a, from, to)
	if p > from {
		sort(a, from, p)
	}
	if p < to {
		sort(a, p+1, to)
	}
}

func partition(a []int, from, to int) int {
	m := (from + to) >> 1
	if a[from] > a[m] {
		a[from], a[m] = a[m], a[from]
	}
	i := from + 1
	for j := from + 1; j < to; j++ {
		if a[j] < a[from] {
			if i != j {
				a[i], a[j] = a[j], a[i]
			}
			i++
		}
	}
	if from != i-1 {
		a[from], a[i-1] = a[i-1], a[from]
	}
	return i - 1
}

func Shuffle(a []int64) {
	rand.Seed(time.Now().UnixNano())
	for i := range a {
		j := rand.Intn(i + 1)
		a[i], a[j] = a[j], a[i]
	}
}

func GoNonBlocking(f func() interface{}) (ch chan interface{}) {
	go func() {
		select {
		case ch <- f():
		default:
		}
	}()
	return
}

var Lt *log.Logger
var Li *log.Logger
var Lw *log.Logger
var Le *log.Logger
var Lf *log.Logger
var Lc *log.Logger

func init() {
	Lt = log.New(os.Stderr, "T ", log.LstdFlags|log.Lshortfile)
	Li = log.New(os.Stderr, "I ", log.LstdFlags|log.Lshortfile)
	Lw = log.New(os.Stderr, "W ", log.LstdFlags|log.Lshortfile)
	Le = log.New(os.Stderr, "E ", log.LstdFlags|log.Lshortfile)
	Lf = log.New(os.Stderr, "F ", log.LstdFlags|log.Lshortfile)
	Lc = log.New(os.Stderr, "C ", log.LstdFlags|log.Lshortfile)
}

func InitLoggers(w io.Writer) {
	Lt = log.New(w, "T ", log.LstdFlags|log.Lshortfile)
	Li = log.New(w, "I ", log.LstdFlags|log.Lshortfile)
	Lw = log.New(w, "W ", log.LstdFlags|log.Lshortfile)
	Le = log.New(w, "E ", log.LstdFlags|log.Lshortfile)
	Lf = log.New(w, "F ", log.LstdFlags|log.Lshortfile)
	Lc = log.New(w, "C ", log.LstdFlags|log.Lshortfile)
}

type Cue struct{}

func LeadingZeros(d interface{}) int {
	switch t := d.(type) {
	case uint8:
		return lzs(uint64(t), 8)
	case int8:
		return lzs(uint64(t), 8)
	case int16:
		return lzs(uint64(t), 16)
	case uint16:
		return lzs(uint64(t), 16)
	case int32:
		return lzs(uint64(t), 32)
	case uint32:
		return lzs(uint64(t), 32)
	case int64:
		return lzs(uint64(t), 64)
	case uint64:
		return lzs(uint64(t), 64)
	}
	return -1 // unreachable code
}

func lzs(x uint64, s uint) int {
	u := uint64(1 << (s - 1))
	for i := uint(0); i < s; i++ {
		if (u>>i)&x > 0 {
			return int(i)
		}
	}
	return int(s)
}

func Size(d interface{}) int {
	switch d.(type) {
	case uint8, int8: // also applied to byte
		return 8
	case uint16, int16:
		return 16
	case int32, uint32: // also applied to rune
		return 32
	case int64, uint64:
		return 64
	default:
		return -1
	}
}
<commit_msg>add randomness to quick sort<commit_after>package util

import (
	"io"
	"log"
	"math/rand"
	"os"
	"time"
)

func QuickSort(a []int) {
	sort(a, 0, len(a))
}

func sort(a []int, from, to int) {
	if to-from <= 1 {
		return
	}
	p := partition(a, from, to)
	if p > from {
		sort(a, from, p)
	}
	if p < to {
		sort(a, p+1, to)
	}
}

func partition(a []int, from, to int) int {
	m := from + rand.Intn(to-from)
	a[from], a[m] = a[m], a[from]
	i := from + 1
	for j := from + 1; j < to; j++ {
		if a[j] < a[from] {
			if i != j {
				a[i], a[j] = a[j], a[i]
			}
			i++
		}
	}
	if from != i-1 {
		a[from], a[i-1] = a[i-1], a[from]
	}
	return i - 1
}

func Shuffle(a []int64) {
	rand.Seed(time.Now().UnixNano())
	for i := range a {
		j := rand.Intn(i + 1)
		a[i], a[j] = a[j], a[i]
	}
}

func GoNonBlocking(f func() interface{}) (ch chan interface{}) {
	go func() {
		select {
		case ch <- f():
		default:
		}
	}()
	return
}

var Lt *log.Logger
var Li *log.Logger
var Lw *log.Logger
var Le *log.Logger
var Lf *log.Logger
var Lc *log.Logger

func init() {
	Lt = log.New(os.Stderr, "T ", log.LstdFlags|log.Lshortfile)
	Li = log.New(os.Stderr, "I ", log.LstdFlags|log.Lshortfile)
	Lw = log.New(os.Stderr, "W ", log.LstdFlags|log.Lshortfile)
	Le = log.New(os.Stderr, "E ", log.LstdFlags|log.Lshortfile)
	Lf = log.New(os.Stderr, "F ", log.LstdFlags|log.Lshortfile)
	Lc = log.New(os.Stderr, "C ", log.LstdFlags|log.Lshortfile)
}

func InitLoggers(w io.Writer) {
	Lt = log.New(w, "T ", log.LstdFlags|log.Lshortfile)
	Li = log.New(w, "I ", log.LstdFlags|log.Lshortfile)
	Lw = log.New(w, "W ", log.LstdFlags|log.Lshortfile)
	Le = log.New(w, "E ", log.LstdFlags|log.Lshortfile)
	Lf = log.New(w, "F ", log.LstdFlags|log.Lshortfile)
	Lc = log.New(w, "C ", log.LstdFlags|log.Lshortfile)
}

type Cue struct{}

func LeadingZeros(d interface{}) int {
	switch t := d.(type) {
	case uint8:
		return lzs(uint64(t), 8)
	case int8:
		return lzs(uint64(t), 8)
	case int16:
		return lzs(uint64(t), 16)
	case uint16:
		return lzs(uint64(t), 16)
	case int32:
		return lzs(uint64(t), 32)
	case uint32:
		return lzs(uint64(t), 32)
	case int64:
		return lzs(uint64(t), 64)
	case uint64:
		return lzs(uint64(t), 64)
	}
	return -1 // unreachable code
}

func lzs(x uint64, s uint) int {
	u := uint64(1 << (s - 1))
	for i := uint(0); i < s; i++ {
		if (u>>i)&x > 0 {
			return int(i)
		}
	}
	return int(s)
}

func Size(d interface{}) int {
	switch d.(type) {
	case uint8, int8: // also applied to byte
		return 8
	case uint16, int16:
		return 16
	case int32, uint32: // also applied to rune
		return 32
	case int64, uint64:
		return 64
	default:
		return -1
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package util

import (
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

const (
	millisecondsInSecond = 1000
	secondsInMinute      = 60
	minutesInHour        = 60
)

func ParseDuration(duration int) (seconds, minutes, hours int) {
	seconds = duration / millisecondsInSecond
	if seconds >= secondsInMinute {
		minutes = seconds / secondsInMinute
		seconds -= minutes * secondsInMinute
	}
	if minutes >= minutesInHour {
		hours = minutes / minutesInHour
		minutes -= hours * minutesInHour
	}
	return
}

func DurationString(seconds, minutes, hours int) (duration string) {
	duration = formatNumber(minutes) + ":" + formatNumber(seconds)
	if hours > 0 {
		duration = formatNumber(hours) + ":" + duration
	}
	return
}

func formatNumber(num int) (formatted string) {
	if num < 10 {
		formatted += "0"
	}
	formatted += strconv.Itoa(num)
	return
}

func SanitizePath(path string) string {
	if strings.HasPrefix(path, "~") {
		path = strings.Replace(path, "~", os.Getenv("HOME"), 1)
	}
	return filepath.Clean(path)
}
<commit_msg>Rename duration arg to milliseconds in ParseDuration<commit_after>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package util

import (
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

const (
	millisecondsInSecond = 1000
	secondsInMinute      = 60
	minutesInHour        = 60
)

func ParseDuration(milliseconds int) (seconds, minutes, hours int) {
	seconds = milliseconds / millisecondsInSecond
	if seconds >= secondsInMinute {
		minutes = seconds / secondsInMinute
		seconds -= minutes * secondsInMinute
	}
	if minutes >= minutesInHour {
		hours = minutes / minutesInHour
		minutes -= hours * minutesInHour
	}
	return
}

func DurationString(seconds, minutes, hours int) (duration string) {
	duration = formatNumber(minutes) + ":" + formatNumber(seconds)
	if hours > 0 {
		duration = formatNumber(hours) + ":" + duration
	}
	return
}

func formatNumber(num int) (formatted string) {
	if num < 10 {
		formatted += "0"
	}
	formatted += strconv.Itoa(num)
	return
}

func SanitizePath(path string) string {
	if strings.HasPrefix(path, "~") {
		path = strings.Replace(path, "~", os.Getenv("HOME"), 1)
	}
	return filepath.Clean(path)
}
<|endoftext|>


<commit_before>package overlay

import (
	"encoding/json"
	"fmt"
	"net"
	"sync"
	"syscall"

	"github.com/Sirupsen/logrus"
	"github.com/docker/libnetwork/datastore"
	"github.com/docker/libnetwork/ipallocator"
	"github.com/docker/libnetwork/osl"
	"github.com/vishvananda/netlink"
	"github.com/vishvananda/netlink/nl"
)

type networkTable map[string]*network

type network struct {
	id          string
	vni         uint32
	dbIndex     uint64
	dbExists    bool
	sbox        osl.Sandbox
	endpoints   endpointTable
	ipAllocator *ipallocator.IPAllocator
	gw          net.IP
	vxlanName   string
	driver      *driver
	joinCnt     int
	once        *sync.Once
	initEpoch   int
	initErr     error
	sync.Mutex
}

func (d *driver) CreateNetwork(id string, option map[string]interface{}) error {
	if id == "" {
		return fmt.Errorf("invalid network id")
	}

	n := &network{
		id:        id,
		driver:    d,
		endpoints: endpointTable{},
		once:      &sync.Once{},
	}

	n.gw = bridgeIP.IP

	d.addNetwork(n)

	if err := n.obtainVxlanID(); err != nil {
		return err
	}

	return nil
}

func (d *driver) DeleteNetwork(nid string) error {
	if nid == "" {
		return fmt.Errorf("invalid network id")
	}

	n := d.network(nid)
	if n == nil {
		return fmt.Errorf("could not find network with id %s", nid)
	}

	d.deleteNetwork(nid)

	return n.releaseVxlanID()
}

func (n *network) joinSandbox() error {
	n.Lock()
	if n.joinCnt != 0 {
		n.joinCnt++
		n.Unlock()
		return nil
	}
	n.Unlock()

	// If there is a race between two go routines here only one will win
	// the other will wait.
	n.once.Do(func() {
		// save the error status of initSandbox in n.initErr so that
		// all the racing go routines are able to know the status.
		n.initErr = n.initSandbox()
	})

	// Increment joinCnt in all the goroutines only when the one time initSandbox
	// was a success.
	n.Lock()
	if n.initErr == nil {
		n.joinCnt++
	}
	err := n.initErr
	n.Unlock()

	return err
}

func (n *network) leaveSandbox() {
	n.Lock()
	n.joinCnt--
	if n.joinCnt != 0 {
		n.Unlock()
		return
	}

	// We are about to destroy sandbox since the container is leaving the network
	// Reinitialize the once variable so that we will be able to trigger one time
	// sandbox initialization(again) when another container joins subsequently.
	n.once = &sync.Once{}
	n.Unlock()

	n.destroySandbox()
}

func (n *network) destroySandbox() {
	sbox := n.sandbox()
	if sbox != nil {
		for _, iface := range sbox.Info().Interfaces() {
			iface.Remove()
		}

		if err := deleteVxlan(n.vxlanName); err != nil {
			logrus.Warnf("could not cleanup sandbox properly: %v", err)
		}

		sbox.Destroy()
	}
}

func (n *network) initSandbox() error {
	n.Lock()
	n.initEpoch++
	n.Unlock()

	sbox, err := osl.NewSandbox(
		osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), true)
	if err != nil {
		return fmt.Errorf("could not create network sandbox: %v", err)
	}

	// Add a bridge inside the namespace
	if err := sbox.AddInterface("bridge1", "br",
		sbox.InterfaceOptions().Address(bridgeIP),
		sbox.InterfaceOptions().Bridge(true)); err != nil {
		return fmt.Errorf("could not create bridge inside the network sandbox: %v", err)
	}

	vxlanName, err := createVxlan(n.vxlanID())
	if err != nil {
		return err
	}

	if err := sbox.AddInterface(vxlanName, "vxlan",
		sbox.InterfaceOptions().Master("bridge1")); err != nil {
		return fmt.Errorf("could not add vxlan interface inside the network sandbox: %v",
			err)
	}

	n.vxlanName = vxlanName

	n.setSandbox(sbox)

	n.driver.peerDbUpdateSandbox(n.id)

	var nlSock *nl.NetlinkSocket
	sbox.InvokeFunc(func() {
		nlSock, err = nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_NEIGH)
		if err != nil {
			err = fmt.Errorf("failed to subscribe to neighbor group netlink messages")
		}
	})

	go n.watchMiss(nlSock)

	return nil
}

func (n *network) watchMiss(nlSock *nl.NetlinkSocket) {
	for {
		msgs, err := nlSock.Receive()
		if err != nil {
			logrus.Errorf("Failed to receive from netlink: %v ", err)
			continue
		}

		for _, msg := range msgs {
			if msg.Header.Type != syscall.RTM_GETNEIGH && msg.Header.Type != syscall.RTM_NEWNEIGH {
				continue
			}

			neigh, err := netlink.NeighDeserialize(msg.Data)
			if err != nil {
				logrus.Errorf("Failed to deserialize netlink ndmsg: %v", err)
				continue
			}

			if neigh.IP.To16() != nil {
				continue
			}

			if neigh.State&(netlink.NUD_STALE|netlink.NUD_INCOMPLETE) == 0 {
				continue
			}

			mac, vtep, err := n.driver.resolvePeer(n.id, neigh.IP)
			if err != nil {
				logrus.Errorf("could not resolve peer %q: %v", neigh.IP, err)
				continue
			}

			if err := n.driver.peerAdd(n.id, "dummy", neigh.IP, mac, vtep, true); err != nil {
				logrus.Errorf("could not add neighbor entry for missed peer: %v", err)
			}
		}
	}
}

func (d *driver) addNetwork(n *network) {
	d.Lock()
	d.networks[n.id] = n
	d.Unlock()
}

func (d *driver) deleteNetwork(nid string) {
	d.Lock()
	delete(d.networks, nid)
	d.Unlock()
}

func (d *driver) network(nid string) *network {
	d.Lock()
	defer d.Unlock()

	return d.networks[nid]
}

func (n *network) sandbox() osl.Sandbox {
	n.Lock()
	defer n.Unlock()

	return n.sbox
}

func (n *network) setSandbox(sbox osl.Sandbox) {
	n.Lock()
	n.sbox = sbox
	n.Unlock()
}

func (n *network) vxlanID() uint32 {
	n.Lock()
	defer n.Unlock()

	return n.vni
}

func (n *network) setVxlanID(vni uint32) {
	n.Lock()
	n.vni = vni
	n.Unlock()
}

func (n *network) Key() []string {
	return []string{"overlay", "network", n.id}
}

func (n *network) KeyPrefix() []string {
	return []string{"overlay", "network"}
}

func (n *network) Value() []byte {
	b, err := json.Marshal(n.vxlanID())
	if err != nil {
		return []byte{}
	}

	return b
}

func (n *network) Index() uint64 {
	return n.dbIndex
}

func (n *network) SetIndex(index uint64) {
	n.dbIndex = index
	n.dbExists = true
}

func (n *network) Exists() bool {
	return n.dbExists
}

func (n *network) SetValue(value []byte) error {
	var vni uint32
	err := json.Unmarshal(value, &vni)
	if err == nil {
		n.setVxlanID(vni)
	}
	return err
}

func (n *network) writeToStore() error {
	return n.driver.store.PutObjectAtomic(n)
}

func (n *network) releaseVxlanID() error {
	if n.driver.store == nil {
		return fmt.Errorf("no datastore configured. cannot release vxlan id")
	}

	if n.vxlanID() == 0 {
		return nil
	}

	if err := n.driver.store.DeleteObjectAtomic(n); err != nil {
		if err == datastore.ErrKeyModified || err == datastore.ErrKeyNotFound {
			// In both the above cases we can safely assume that the key has been removed by some other
			// instance and so simply get out of here
			return nil
		}

		return fmt.Errorf("failed to delete network to vxlan id map: %v", err)
	}

	n.driver.vxlanIdm.Release(n.vxlanID())
	n.setVxlanID(0)
	return nil
}

func (n *network) obtainVxlanID() error {
	if n.driver.store == nil {
		return fmt.Errorf("no datastore configured. cannot obtain vxlan id")
	}

	for {
		var vxlanID uint32
		if err := n.driver.store.GetObject(datastore.Key(n.Key()...), n); err != nil {
			if err == datastore.ErrKeyNotFound {
				vxlanID, err = n.driver.vxlanIdm.GetID()
				if err != nil {
					return fmt.Errorf("failed to allocate vxlan id: %v", err)
				}

				n.setVxlanID(vxlanID)
				if err := n.writeToStore(); err != nil {
					n.driver.vxlanIdm.Release(n.vxlanID())
					n.setVxlanID(0)
					if err == datastore.ErrKeyModified {
						continue
					}
					return fmt.Errorf("failed to update data store with vxlan id: %v", err)
				}
				return nil
			}
			return fmt.Errorf("failed to obtain vxlan id from data store: %v", err)
		}
		return nil
	}
}
<commit_msg>updated for testing purpose 13:20<commit_after>package overlay

import (
	"encoding/json"
	"fmt"
	"net"
	"sync"
	"syscall"

	"github.com/Sirupsen/logrus"
	"github.com/docker/libnetwork/datastore"
	"github.com/docker/libnetwork/ipallocator"
	"github.com/docker/libnetwork/osl"
	"github.com/vishvananda/netlink"
	"github.com/vishvananda/netlink/nl"
)

type networkTable map[string]*network

type network struct {
	id          string
	vni         uint32
	dbIndex     uint64
	dbExists    bool
	sbox        osl.Sandbox
	endpoints   endpointTable
	ipAllocator *ipallocator.IPAllocator
	gw          net.IP
	vxlanName   string
	driver      *driver
	joinCnt     int
	once        *sync.Once
	initEpoch   int
	initErr     error
	sync.Mutex
}

func (d *driver) CreateNetwork(id string, option map[string]interface{}) error {
	logrus.Debugf("create network call recieved at overlay driver")
	logrus.WarnF("create network call recieved at overlay driver")
	if id == "" {
		return fmt.Errorf("invalid network id")
	}

	n := &network{
		id:        id,
		driver:    d,
		endpoints: endpointTable{},
		once:      &sync.Once{},
	}

	n.gw = bridgeIP.IP

	d.addNetwork(n)

	if err := n.obtainVxlanID(); err != nil {
		return err
	}

	return nil
}

func (d *driver) DeleteNetwork(nid string) error {
	if nid == "" {
		return fmt.Errorf("invalid network id")
	}

	n := d.network(nid)
	if n == nil {
		return fmt.Errorf("could not find network with id %s", nid)
	}

	d.deleteNetwork(nid)

	return n.releaseVxlanID()
}

func (n *network) joinSandbox() error {
	n.Lock()
	if n.joinCnt != 0 {
		n.joinCnt++
		n.Unlock()
		return nil
	}
	n.Unlock()

	// If there is a race between two go routines here only one will win
	// the other will wait.
	n.once.Do(func() {
		// save the error status of initSandbox in n.initErr so that
		// all the racing go routines are able to know the status.
		n.initErr = n.initSandbox()
	})

	// Increment joinCnt in all the goroutines only when the one time initSandbox
	// was a success.
	n.Lock()
	if n.initErr == nil {
		n.joinCnt++
	}
	err := n.initErr
	n.Unlock()

	return err
}

func (n *network) leaveSandbox() {
	n.Lock()
	n.joinCnt--
	if n.joinCnt != 0 {
		n.Unlock()
		return
	}

	// We are about to destroy sandbox since the container is leaving the network
	// Reinitialize the once variable so that we will be able to trigger one time
	// sandbox initialization(again) when another container joins subsequently.
	n.once = &sync.Once{}
	n.Unlock()

	n.destroySandbox()
}

func (n *network) destroySandbox() {
	sbox := n.sandbox()
	if sbox != nil {
		for _, iface := range sbox.Info().Interfaces() {
			iface.Remove()
		}

		if err := deleteVxlan(n.vxlanName); err != nil {
			logrus.Warnf("could not cleanup sandbox properly: %v", err)
		}

		sbox.Destroy()
	}
}

func (n *network) initSandbox() error {
	n.Lock()
	n.initEpoch++
	n.Unlock()

	sbox, err := osl.NewSandbox(
		osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), true)
	if err != nil {
		return fmt.Errorf("could not create network sandbox: %v", err)
	}

	// Add a bridge inside the namespace
	if err := sbox.AddInterface("bridge1", "br",
		sbox.InterfaceOptions().Address(bridgeIP),
		sbox.InterfaceOptions().Bridge(true)); err != nil {
		return fmt.Errorf("could not create bridge inside the network sandbox: %v", err)
	}

	vxlanName, err := createVxlan(n.vxlanID())
	if err != nil {
		return err
	}

	if err := sbox.AddInterface(vxlanName, "vxlan",
		sbox.InterfaceOptions().Master("bridge1")); err != nil {
		return fmt.Errorf("could not add vxlan interface inside the network sandbox: %v",
			err)
	}

	n.vxlanName = vxlanName

	n.setSandbox(sbox)

	n.driver.peerDbUpdateSandbox(n.id)

	var nlSock *nl.NetlinkSocket
	sbox.InvokeFunc(func() {
		nlSock, err = nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_NEIGH)
		if err != nil {
			err = fmt.Errorf("failed to subscribe to neighbor group netlink messages")
		}
	})

	go n.watchMiss(nlSock)

	return nil
}

func (n *network) watchMiss(nlSock *nl.NetlinkSocket) {
	for {
		msgs, err := nlSock.Receive()
		if err != nil {
			logrus.Errorf("Failed to receive from netlink: %v ", err)
			continue
		}

		for _, msg := range msgs {
			if msg.Header.Type != syscall.RTM_GETNEIGH && msg.Header.Type != syscall.RTM_NEWNEIGH {
				continue
			}

			neigh, err := netlink.NeighDeserialize(msg.Data)
			if err != nil {
				logrus.Errorf("Failed to deserialize netlink ndmsg: %v", err)
				continue
			}

			if neigh.IP.To16() != nil {
				continue
			}

			if neigh.State&(netlink.NUD_STALE|netlink.NUD_INCOMPLETE) == 0 {
				continue
			}

			mac, vtep, err := n.driver.resolvePeer(n.id, neigh.IP)
			if err != nil {
				logrus.Errorf("could not resolve peer %q: %v", neigh.IP, err)
				continue
			}

			if err := n.driver.peerAdd(n.id, "dummy", neigh.IP, mac, vtep, true); err != nil {
				logrus.Errorf("could not add neighbor entry for missed peer: %v", err)
			}
		}
	}
}

func (d *driver) addNetwork(n *network) {
	d.Lock()
	d.networks[n.id] = n
	d.Unlock()
}

func (d *driver) deleteNetwork(nid string) {
	d.Lock()
	delete(d.networks, nid)
	d.Unlock()
}

func (d *driver) network(nid string) *network {
	d.Lock()
	defer d.Unlock()

	return d.networks[nid]
}

func (n *network) sandbox() osl.Sandbox {
	n.Lock()
	defer n.Unlock()

	return n.sbox
}

func (n *network) setSandbox(sbox osl.Sandbox) {
	n.Lock()
	n.sbox = sbox
	n.Unlock()
}

func (n *network) vxlanID() uint32 {
	n.Lock()
	defer n.Unlock()

	return n.vni
}

func (n *network) setVxlanID(vni uint32) {
	n.Lock()
	n.vni = vni
	n.Unlock()
}

func (n *network) Key() []string {
	return []string{"overlay", "network", n.id}
}

func (n *network) KeyPrefix() []string {
	return []string{"overlay", "network"}
}

func (n *network) Value() []byte {
	b, err := json.Marshal(n.vxlanID())
	if err != nil {
		return []byte{}
	}

	return b
}

func (n *network) Index() uint64 {
	return n.dbIndex
}

func (n *network) SetIndex(index uint64) {
	n.dbIndex = index
	n.dbExists = true
}

func (n *network) Exists() bool {
	return n.dbExists
}

func (n *network) SetValue(value []byte) error {
	var vni uint32
	err := json.Unmarshal(value, &vni)
	if err == nil {
		n.setVxlanID(vni)
	}
	return err
}

func (n *network) writeToStore() error {
	return n.driver.store.PutObjectAtomic(n)
}

func (n *network) releaseVxlanID() error {
	if n.driver.store == nil {
		return fmt.Errorf("no datastore configured. cannot release vxlan id")
	}

	if n.vxlanID() == 0 {
		return nil
	}

	if err := n.driver.store.DeleteObjectAtomic(n); err != nil {
		if err == datastore.ErrKeyModified || err == datastore.ErrKeyNotFound {
			// In both the above cases we can safely assume that the key has been removed by some other
			// instance and so simply get out of here
			return nil
		}

		return fmt.Errorf("failed to delete network to vxlan id map: %v", err)
	}

	n.driver.vxlanIdm.Release(n.vxlanID())
	n.setVxlanID(0)
	return nil
}

func (n *network) obtainVxlanID() error {
	if n.driver.store == nil {
		return fmt.Errorf("no datastore configured. cannot obtain vxlan id")
	}

	for {
		var vxlanID uint32
		if err := n.driver.store.GetObject(datastore.Key(n.Key()...), n); err != nil {
			if err == datastore.ErrKeyNotFound {
				vxlanID, err = n.driver.vxlanIdm.GetID()
				if err != nil {
					return fmt.Errorf("failed to allocate vxlan id: %v", err)
				}

				n.setVxlanID(vxlanID)
				if err := n.writeToStore(); err != nil {
					n.driver.vxlanIdm.Release(n.vxlanID())
					n.setVxlanID(0)
					if err == datastore.ErrKeyModified {
						continue
					}
					return fmt.Errorf("failed to update data store with vxlan id: %v", err)
				}
				return nil
			}
			return fmt.Errorf("failed to obtain vxlan id from data store: %v", err)
		}
		return nil
	}
}
<|endoftext|>


<commit_before>package memory

import (
	"runtime"
	"sync"
	"time"
)

// Forked from https://github.com/patrickmn/go-cache

// CacheWrapper is used to ensure that the underlying cleaner goroutine used to clean expired keys will not prevent
// Cache from being garbage collected.
type CacheWrapper struct {
	*Cache
}

// A cleaner will periodically delete expired keys from cache.
type cleaner struct {
	interval time.Duration
	stop     chan bool
}

// Run will periodically delete expired keys from given cache until GC notify that it should stop.
func (cleaner *cleaner) Run(cache *Cache) {
	ticker := time.NewTicker(cleaner.interval)
	for {
		select {
		case <-ticker.C:
			cache.Clean()
		case <-cleaner.stop:
			ticker.Stop()
			return
		}
	}
}

// stopCleaner is a callback from GC used to stop cleaner goroutine.
func stopCleaner(wrapper *CacheWrapper) {
	wrapper.cleaner.stop <- true
}

// startCleaner will start a cleaner goroutine for given cache.
func startCleaner(cache *Cache, interval time.Duration) {
	cleaner := &cleaner{
		interval: interval,
		stop:     make(chan bool),
	}

	cache.cleaner = cleaner
	go cleaner.Run(cache)
}

// Counter is a simple counter with an optional expiration.
type Counter struct {
	Value      int64
	Expiration int64
}

// Expired returns true if the counter has expired.
func (counter Counter) Expired() bool {
	if counter.Expiration == 0 {
		return false
	}
	return time.Now().UnixNano() > counter.Expiration
}

// Cache contains a collection of counters.
type Cache struct {
	mutex    sync.RWMutex
	counters map[string]Counter
	cleaner  *cleaner
}

// NewCache returns a new cache.
func NewCache(cleanInterval time.Duration) *CacheWrapper {

	cache := &Cache{
		counters: map[string]Counter{},
	}

	wrapper := &CacheWrapper{Cache: cache}

	if cleanInterval > 0 {
		startCleaner(cache, cleanInterval)
		runtime.SetFinalizer(wrapper, stopCleaner)
	}

	return wrapper
}

// Increment increments given value on key.
// If key is undefined or expired, it will create it.
func (cache *Cache) Increment(key string, value int64, duration time.Duration) (int64, time.Time) {
	cache.mutex.Lock()

	counter, ok := cache.counters[key]
	if !ok || counter.Expired() {
		expiration := time.Now().Add(duration).UnixNano()
		counter = Counter{
			Value:      value,
			Expiration: expiration,
		}

		cache.counters[key] = counter
		cache.mutex.Unlock()

		return value, time.Unix(0, expiration)
	}

	value = counter.Value + value
	counter.Value = value
	expiration := counter.Expiration

	cache.counters[key] = counter
	cache.mutex.Unlock()

	return value, time.Unix(0, expiration)
}

// Get returns key's value and expiration.
func (cache *Cache) Get(key string, duration time.Duration) (int64, time.Time) {
	cache.mutex.RLock()

	counter, ok := cache.counters[key]
	if !ok || counter.Expired() {
		expiration := time.Now().Add(duration).UnixNano()
		cache.mutex.RUnlock()
		return 0, time.Unix(0, expiration)
	}

	value := counter.Value
	expiration := counter.Expiration
	cache.mutex.RUnlock()

	return value, time.Unix(0, expiration)
}

// Clean will deleted any expired keys.
func (cache *Cache) Clean() {
	now := time.Now().UnixNano()

	cache.mutex.Lock()
	for key, counter := range cache.counters {
		if now > counter.Expiration {
			delete(cache.counters, key)
		}
	}
	cache.mutex.Unlock()
}

// Reset changes the key's value and resets the expiration.
func (cache *Cache) Reset(key string, duration time.Duration) (int64, time.Time) {
	cache.mutex.Lock()
	delete(cache.counters, key)
	cache.mutex.Unlock()

	expiration := time.Now().Add(duration).UnixNano()
	return 0, time.Unix(0, expiration)
}
<commit_msg>fix(store/memory): Try an experiment to uses sync.Map to store counters<commit_after>package memory

import (
	"runtime"
	"sync"
	"time"
)

// Forked from https://github.com/patrickmn/go-cache

// CacheWrapper is used to ensure that the underlying cleaner goroutine used to clean expired keys will not prevent
// Cache from being garbage collected.
type CacheWrapper struct {
	*Cache
}

// A cleaner will periodically delete expired keys from cache.
type cleaner struct {
	interval time.Duration
	stop     chan bool
}

// Run will periodically delete expired keys from given cache until GC notify that it should stop.
func (cleaner *cleaner) Run(cache *Cache) {
	ticker := time.NewTicker(cleaner.interval)
	for {
		select {
		case <-ticker.C:
			cache.Clean()
		case <-cleaner.stop:
			ticker.Stop()
			return
		}
	}
}

// stopCleaner is a callback from GC used to stop cleaner goroutine.
func stopCleaner(wrapper *CacheWrapper) {
	wrapper.cleaner.stop <- true
	wrapper.cleaner = nil
}

// startCleaner will start a cleaner goroutine for given cache.
func startCleaner(cache *Cache, interval time.Duration) {
	cleaner := &cleaner{
		interval: interval,
		stop:     make(chan bool),
	}

	cache.cleaner = cleaner
	go cleaner.Run(cache)
}

// Counter is a simple counter with an expiration.
type Counter struct {
	mutex      sync.RWMutex
	value      int64
	expiration int64
}

// Expired returns true if the counter has expired.
func (counter *Counter) Expired() bool {
	counter.mutex.RLock()
	defer counter.mutex.RUnlock()

	if counter.expiration == 0 {
		return true
	}
	return time.Now().UnixNano() > counter.expiration
}

// Load returns the value and the expiration of this counter.
// If the counter is expired, it will use the given expiration.
func (counter *Counter) Load(expiration int64) (int64, int64) {
	counter.mutex.RLock()
	defer counter.mutex.RUnlock()

	if counter.expiration == 0 || time.Now().UnixNano() > counter.expiration {
		return 0, expiration
	}

	return counter.value, counter.expiration
}

// Increment increments given value on this counter.
// If the counter is expired, it will use the given expiration.
// It returns its current value and expiration.
func (counter *Counter) Increment(value int64, expiration int64) (int64, int64) {
	counter.mutex.Lock()
	defer counter.mutex.Unlock()

	if counter.expiration == 0 || time.Now().UnixNano() > counter.expiration {
		counter.value = value
		counter.expiration = expiration
		return counter.value, counter.expiration
	}

	counter.value += value
	return counter.value, counter.expiration
}

// Cache contains a collection of counters.
type Cache struct {
	counters sync.Map
	cleaner  *cleaner
}

// NewCache returns a new cache.
func NewCache(cleanInterval time.Duration) *CacheWrapper {

	cache := &Cache{}
	wrapper := &CacheWrapper{Cache: cache}

	if cleanInterval > 0 {
		startCleaner(cache, cleanInterval)
		runtime.SetFinalizer(wrapper, stopCleaner)
	}

	return wrapper
}

// LoadOrStore returns the existing counter for the key if present.
// Otherwise, it stores and returns the given counter.
// The loaded result is true if the counter was loaded, false if stored.
func (cache *Cache) LoadOrStore(key string, counter *Counter) (*Counter, bool) {
	val, loaded := cache.counters.LoadOrStore(key, counter)
	if val == nil {
		return counter, false
	}

	actual := val.(*Counter)
	return actual, loaded
}

// Load returns the counter stored in the map for a key, or nil if no counter is present.
// The ok result indicates whether counter was found in the map.
func (cache *Cache) Load(key string) (*Counter, bool) {
	val, ok := cache.counters.Load(key)
	if val == nil || !ok {
		return nil, false
	}
	actual := val.(*Counter)
	return actual, true
}

// Store sets the counter for a key.
func (cache *Cache) Store(key string, counter *Counter) {
	cache.counters.Store(key, counter)
}

// Delete deletes the value for a key.
func (cache *Cache) Delete(key string) {
	cache.counters.Delete(key)
}

// Range calls handler sequentially for each key and value present in the cache.
// If handler returns false, range stops the iteration.
func (cache *Cache) Range(handler func(key string, counter *Counter)) {
	cache.counters.Range(func(k interface{}, v interface{}) bool {
		if v == nil {
			return true
		}

		key := k.(string)
		counter := v.(*Counter)

		handler(key, counter)

		return true
	})
}

// Increment increments given value on key.
// If key is undefined or expired, it will create it.
func (cache *Cache) Increment(key string, value int64, duration time.Duration) (int64, time.Time) {
	expiration := time.Now().Add(duration).UnixNano()

	counter, loaded := cache.LoadOrStore(key, &Counter{
		mutex:      sync.RWMutex{},
		value:      value,
		expiration: expiration,
	})

	if !loaded {
		return value, time.Unix(0, expiration)
	}

	value, expiration = counter.Increment(value, expiration)
	cache.Store(key, counter)
	return value, time.Unix(0, expiration)
}

// Get returns key's value and expiration.
func (cache *Cache) Get(key string, duration time.Duration) (int64, time.Time) {
	expiration := time.Now().Add(duration).UnixNano()

	counter, ok := cache.Load(key)
	if !ok {
		return 0, time.Unix(0, expiration)
	}

	value, expiration := counter.Load(expiration)

	return value, time.Unix(0, expiration)
}

// Clean will deleted any expired keys.
func (cache *Cache) Clean() {
	cache.Range(func(key string, counter *Counter) {
		if counter.Expired() {
			cache.counters.Delete(key)
		}
	})
}

// Reset changes the key's value and resets the expiration.
func (cache *Cache) Reset(key string, duration time.Duration) (int64, time.Time) {
	cache.counters.Delete(key)

	expiration := time.Now().Add(duration).UnixNano()
	return 0, time.Unix(0, expiration)
}
<|endoftext|>


<commit_before>package server

import (
	"os"
	"text/template"

	"github.com/urfave/cli"

	"github.com/drone/drone-cli/drone/internal"
)

var serverCreateCmd = cli.Command{
	Name:   "create",
	Usage:  "crate a new server",
	Action: serverCreate,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:   "format",
			Usage:  "format output",
			Value:  tmplServerCreate,
			Hidden: true,
		},
	},
}

func serverCreate(c *cli.Context) error {
	client, err := internal.NewAutoscaleClient(c)
	if err != nil {
		return err
	}

	server, err := client.ServerCreate()
	if err != nil {
		return err
	}

	tmpl, err := template.New("_").Parse(c.String("format") + "\n")
	if err != nil {
		return err
	}
	return tmpl.Execute(os.Stdout, server)
}

var tmplServerCreate = `Name: {{ .Name }}
State: {{ .State }}
`
<commit_msg>fix create usage spelling<commit_after>package server

import (
	"os"
	"text/template"

	"github.com/urfave/cli"

	"github.com/drone/drone-cli/drone/internal"
)

var serverCreateCmd = cli.Command{
	Name:   "create",
	Usage:  "create a new server",
	Action: serverCreate,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:   "format",
			Usage:  "format output",
			Value:  tmplServerCreate,
			Hidden: true,
		},
	},
}

func serverCreate(c *cli.Context) error {
	client, err := internal.NewAutoscaleClient(c)
	if err != nil {
		return err
	}

	server, err := client.ServerCreate()
	if err != nil {
		return err
	}

	tmpl, err := template.New("_").Parse(c.String("format") + "\n")
	if err != nil {
		return err
	}
	return tmpl.Execute(os.Stdout, server)
}

var tmplServerCreate = `Name: {{ .Name }}
State: {{ .State }}
`
<|endoftext|>


<commit_before>package common

import "github.com/couchbase/indexing/secondary/dcp"
import "github.com/couchbase/indexing/secondary/logging"
import "errors"
import "fmt"
import "time"
import "net"
import "net/url"
import "sync"

var (
	ErrInvalidNodeId       = errors.New("Invalid NodeId")
	ErrInvalidService      = errors.New("Invalid service")
	ErrNodeNotBucketMember = errors.New("Node is not a member of bucket")
)

const (
	INDEX_ADMIN_SERVICE = "indexAdmin"
	INDEX_SCAN_SERVICE  = "indexScan"
	INDEX_HTTP_SERVICE  = "indexHttp"
)

// Helper object for fetching cluster information
// Can be used by services running on a cluster node to connect with
// local management service for obtaining cluster information.
// Info cache can be updated by using Refresh() method.
type ClusterInfoCache struct {
	sync.Mutex
	url       string
	poolName  string
	logPrefix string
	retries   int

	client  couchbase.Client
	pool    couchbase.Pool
	nodes   []couchbase.Node
	nodesvs []couchbase.NodeServices
}

type ServiceAddressProvider interface {
	GetLocalServiceAddress(srvc string) (string, error)
	GetLocalServicePort(srvc string) (string, error)
	GetLocalServiceHost(srvc string) (string, error)
	GetLocalHostAddress() (string, error)
}

type NodeId int

func NewClusterInfoCache(clusterUrl string, pool string) (*ClusterInfoCache, error) {
	c := &ClusterInfoCache{
		url:      clusterUrl,
		poolName: pool,
		retries:  0,
	}

	return c, nil
}

func (c *ClusterInfoCache) SetLogPrefix(p string) {
	c.logPrefix = p
}

func (c *ClusterInfoCache) SetMaxRetries(r int) {
	c.retries = r
}

func (c *ClusterInfoCache) Fetch() error {

	fn := func(r int, err error) error {
		if r > 0 {
			logging.Infof("%vError occured during cluster info update (%v) .. Retrying(%d)",
				c.logPrefix, err, r)
		}

		c.client, err = couchbase.Connect(c.url)
		if err != nil {
			return err
		}

		c.pool, err = c.client.GetPool(c.poolName)
		if err != nil {
			return err
		}

		var nodes []couchbase.Node
		for _, n := range c.pool.Nodes {
			if n.ClusterMembership == "active" {
				nodes = append(nodes, n)
			}
		}
		c.nodes = nodes

		found := false
		for _, node := range c.nodes {
			if node.ThisNode {
				found = true
			}
		}

		if !found {
			return errors.New("Current node's cluster membership is not active")
		}

		var poolServs couchbase.PoolServices
		poolServs, err = c.client.GetPoolServices(c.poolName)
		if err != nil {
			return err
		}
		c.nodesvs = poolServs.NodesExt

		return nil
	}

	rh := NewRetryHelper(c.retries, time.Second, 1, fn)
	return rh.Run()
}

func (c ClusterInfoCache) GetNodesByServiceType(srvc string) (nids []NodeId) {
	for i, svs := range c.nodesvs {
		if _, ok := svs.Services[srvc]; ok {
			nids = append(nids, NodeId(i))
		}
	}

	return
}

func (c ClusterInfoCache) GetNodesByBucket(bucket string) (nids []NodeId, err error) {
	b, berr := c.pool.GetBucket(bucket)
	if berr != nil {
		err = berr
		return
	}
	defer b.Close()

	for i, _ := range c.nodes {
		nid := NodeId(i)
		if _, ok := c.findVBServerIndex(b, nid); ok {
			nids = append(nids, nid)
		}
	}

	return
}

func (c ClusterInfoCache) GetCurrentNode() NodeId {
	for i, node := range c.nodes {
		if node.ThisNode {
			return NodeId(i)
		}
	}
	// TODO: can we avoid this panic ?
	panic("Current node is not in active membership")
}

func (c ClusterInfoCache) GetServiceAddress(nid NodeId, srvc string) (addr string, err error) {
	var port int
	var ok bool

	if int(nid) >= len(c.nodesvs) {
		err = ErrInvalidNodeId
		return
	}

	node := c.nodesvs[nid]
	if port, ok = node.Services[srvc]; !ok {
		err = ErrInvalidService
		return
	}

	// For current node, hostname might be empty
	// Insert hostname used to connect to the cluster
	cUrl, err := url.Parse(c.url)
	if err != nil {
		return "", errors.New("Unable to parse cluster url - " + err.Error())
	}
	h, _, _ := net.SplitHostPort(cUrl.Host)
	if node.Hostname == "" {
		node.Hostname = h
	}

	addr = net.JoinHostPort(node.Hostname, fmt.Sprint(port))
	return
}

func (c ClusterInfoCache) GetVBuckets(nid NodeId, bucket string) (vbs []uint32, err error) {
	b, berr := c.pool.GetBucket(bucket)
	if berr != nil {
		err = berr
		return
	}
	defer b.Close()

	idx, ok := c.findVBServerIndex(b, nid)
	if !ok {
		err = ErrNodeNotBucketMember
		return
	}

	vbmap := b.VBServerMap()

	for vb, idxs := range vbmap.VBucketMap {
		if idxs[0] == idx {
			vbs = append(vbs, uint32(vb))
		}
	}

	return
}

func (c ClusterInfoCache) findVBServerIndex(b *couchbase.Bucket, nid NodeId) (int, bool) {
	bnodes := b.Nodes()

	for idx, n := range bnodes {
		if c.sameNode(n, c.nodes[nid]) {
			return idx, true
		}
	}

	return 0, false
}

func (c ClusterInfoCache) sameNode(n1 couchbase.Node, n2 couchbase.Node) bool {
	return n1.Hostname == n2.Hostname
}

func (c ClusterInfoCache) GetLocalServiceAddress(srvc string) (string, error) {
	node := c.GetCurrentNode()
	return c.GetServiceAddress(node, srvc)
}

func (c ClusterInfoCache) GetLocalServicePort(srvc string) (string, error) {
	addr, err := c.GetLocalServiceAddress(srvc)
	if err != nil {
		return addr, err
	}

	_, p, e := net.SplitHostPort(addr)
	if e != nil {
		return p, e
	}

	return net.JoinHostPort("", p), nil
}

func (c ClusterInfoCache) GetLocalServiceHost(srvc string) (string, error) {

	addr, err := c.GetLocalServiceAddress(srvc)
	if err != nil {
		return addr, err
	}

	h, _, err := net.SplitHostPort(addr)
	if err != nil {
		return "", err
	}

	return h, nil
}

func (c ClusterInfoCache) GetLocalHostAddress() (string, error) {

	cUrl, err := url.Parse(c.url)
	if err != nil {
		return "", errors.New("Unable to parse cluster url - " + err.Error())
	}
	_, p, _ := net.SplitHostPort(cUrl.Host)

	h, err := c.GetLocalServiceHost(INDEX_ADMIN_SERVICE)
	if err != nil {
		return "", err
	}

	return net.JoinHostPort(h, p), nil
}
<commit_msg>cluster_info: Add IsNodeHealthy() API<commit_after>package common

import "github.com/couchbase/indexing/secondary/dcp"
import "github.com/couchbase/indexing/secondary/logging"
import "errors"
import "fmt"
import "time"
import "net"
import "net/url"
import "sync"

var (
	ErrInvalidNodeId       = errors.New("Invalid NodeId")
	ErrInvalidService      = errors.New("Invalid service")
	ErrNodeNotBucketMember = errors.New("Node is not a member of bucket")
)

const (
	INDEX_ADMIN_SERVICE = "indexAdmin"
	INDEX_SCAN_SERVICE  = "indexScan"
	INDEX_HTTP_SERVICE  = "indexHttp"
)

// Helper object for fetching cluster information
// Can be used by services running on a cluster node to connect with
// local management service for obtaining cluster information.
// Info cache can be updated by using Refresh() method.
type ClusterInfoCache struct {
	sync.Mutex
	url       string
	poolName  string
	logPrefix string
	retries   int

	client  couchbase.Client
	pool    couchbase.Pool
	nodes   []couchbase.Node
	nodesvs []couchbase.NodeServices
}

type ServiceAddressProvider interface {
	GetLocalServiceAddress(srvc string) (string, error)
	GetLocalServicePort(srvc string) (string, error)
	GetLocalServiceHost(srvc string) (string, error)
	GetLocalHostAddress() (string, error)
}

type NodeId int

func NewClusterInfoCache(clusterUrl string, pool string) (*ClusterInfoCache, error) {
	c := &ClusterInfoCache{
		url:      clusterUrl,
		poolName: pool,
		retries:  0,
	}

	return c, nil
}

func (c *ClusterInfoCache) SetLogPrefix(p string) {
	c.logPrefix = p
}

func (c *ClusterInfoCache) SetMaxRetries(r int) {
	c.retries = r
}

func (c *ClusterInfoCache) Fetch() error {

	fn := func(r int, err error) error {
		if r > 0 {
			logging.Infof("%vError occured during cluster info update (%v) .. Retrying(%d)",
				c.logPrefix, err, r)
		}

		c.client, err = couchbase.Connect(c.url)
		if err != nil {
			return err
		}

		c.pool, err = c.client.GetPool(c.poolName)
		if err != nil {
			return err
		}

		var nodes []couchbase.Node
		for _, n := range c.pool.Nodes {
			if n.ClusterMembership == "active" {
				nodes = append(nodes, n)
			}
		}
		c.nodes = nodes

		found := false
		for _, node := range c.nodes {
			if node.ThisNode {
				found = true
			}
		}

		if !found {
			return errors.New("Current node's cluster membership is not active")
		}

		var poolServs couchbase.PoolServices
		poolServs, err = c.client.GetPoolServices(c.poolName)
		if err != nil {
			return err
		}
		c.nodesvs = poolServs.NodesExt

		return nil
	}

	rh := NewRetryHelper(c.retries, time.Second, 1, fn)
	return rh.Run()
}

func (c ClusterInfoCache) GetNodesByServiceType(srvc string) (nids []NodeId) {
	for i, svs := range c.nodesvs {
		if _, ok := svs.Services[srvc]; ok {
			nids = append(nids, NodeId(i))
		}
	}

	return
}

func (c ClusterInfoCache) GetNodesByBucket(bucket string) (nids []NodeId, err error) {
	b, berr := c.pool.GetBucket(bucket)
	if berr != nil {
		err = berr
		return
	}
	defer b.Close()

	for i, _ := range c.nodes {
		nid := NodeId(i)
		if _, ok := c.findVBServerIndex(b, nid); ok {
			nids = append(nids, nid)
		}
	}

	return
}

func (c ClusterInfoCache) GetCurrentNode() NodeId {
	for i, node := range c.nodes {
		if node.ThisNode {
			return NodeId(i)
		}
	}
	// TODO: can we avoid this panic ?
	panic("Current node is not in active membership")
}

func (c ClusterInfoCache) IsNodeHealthy(nid NodeId) (bool, error) {
	if int(nid) >= len(c.nodes) {
		return false, ErrInvalidNodeId
	}

	return c.nodes[nid].Status == "healthy", nil
}

func (c ClusterInfoCache) GetServiceAddress(nid NodeId, srvc string) (addr string, err error) {
	var port int
	var ok bool

	if int(nid) >= len(c.nodesvs) {
		err = ErrInvalidNodeId
		return
	}

	node := c.nodesvs[nid]
	if port, ok = node.Services[srvc]; !ok {
		err = ErrInvalidService
		return
	}

	// For current node, hostname might be empty
	// Insert hostname used to connect to the cluster
	cUrl, err := url.Parse(c.url)
	if err != nil {
		return "", errors.New("Unable to parse cluster url - " + err.Error())
	}
	h, _, _ := net.SplitHostPort(cUrl.Host)
	if node.Hostname == "" {
		node.Hostname = h
	}

	addr = net.JoinHostPort(node.Hostname, fmt.Sprint(port))
	return
}

func (c ClusterInfoCache) GetVBuckets(nid NodeId, bucket string) (vbs []uint32, err error) {
	b, berr := c.pool.GetBucket(bucket)
	if berr != nil {
		err = berr
		return
	}
	defer b.Close()

	idx, ok := c.findVBServerIndex(b, nid)
	if !ok {
		err = ErrNodeNotBucketMember
		return
	}

	vbmap := b.VBServerMap()

	for vb, idxs := range vbmap.VBucketMap {
		if idxs[0] == idx {
			vbs = append(vbs, uint32(vb))
		}
	}

	return
}

func (c ClusterInfoCache) findVBServerIndex(b *couchbase.Bucket, nid NodeId) (int, bool) {
	bnodes := b.Nodes()

	for idx, n := range bnodes {
		if c.sameNode(n, c.nodes[nid]) {
			return idx, true
		}
	}

	return 0, false
}

func (c ClusterInfoCache) sameNode(n1 couchbase.Node, n2 couchbase.Node) bool {
	return n1.Hostname == n2.Hostname
}

func (c ClusterInfoCache) GetLocalServiceAddress(srvc string) (string, error) {
	node := c.GetCurrentNode()
	return c.GetServiceAddress(node, srvc)
}

func (c ClusterInfoCache) GetLocalServicePort(srvc string) (string, error) {
	addr, err := c.GetLocalServiceAddress(srvc)
	if err != nil {
		return addr, err
	}

	_, p, e := net.SplitHostPort(addr)
	if e != nil {
		return p, e
	}

	return net.JoinHostPort("", p), nil
}

func (c ClusterInfoCache) GetLocalServiceHost(srvc string) (string, error) {

	addr, err := c.GetLocalServiceAddress(srvc)
	if err != nil {
		return addr, err
	}

	h, _, err := net.SplitHostPort(addr)
	if err != nil {
		return "", err
	}

	return h, nil
}

func (c ClusterInfoCache) GetLocalHostAddress() (string, error) {

	cUrl, err := url.Parse(c.url)
	if err != nil {
		return "", errors.New("Unable to parse cluster url - " + err.Error())
	}
	_, p, _ := net.SplitHostPort(cUrl.Host)

	h, err := c.GetLocalServiceHost(INDEX_ADMIN_SERVICE)
	if err != nil {
		return "", err
	}

	return net.JoinHostPort(h, p), nil
}
<|endoftext|>