<commit_before>// Copyright 2016 Afshin Darian. All rights reserved.
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package sleuth

import (
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/ursiform/logger"
	"github.com/zeromq/gyre"
)

type listener struct {
	*sync.Mutex
	handles map[string]chan *http.Response
}

type notifier struct {
	*sync.Mutex
	notify chan struct{}
}

// Client is the peer on the sleuth network that makes requests and, if a
// handler has been provided, responds to peer requests.
type Client struct {
	// Timeout is the duration to wait before an outstanding request times out.
	// By default, it is set to 500ms.
	Timeout time.Duration

	additions *notifier
	group     string
	handle    int64
	handler   http.Handler
	listener  *listener
	log       *logger.Logger
	node      *gyre.Gyre

	directory map[string]string   // map[node-name]service-type
	services  map[string]*workers // map[service-type]service-workers
}

func (c *Client) add(group, name, node, service, version string) error {
	if group != c.group {
		c.log.Debug("sleuth: no group header for %s, client-only", name)
		return nil
	}
	// Node and service are required. Version is optional.
	if len(node) == 0 || len(service) == 0 {
		return newError(errAdd, "failed to add %s node?=%t, type?=%t",
			name, len(node) > 0, len(service) > 0)
	}
	// Associate the node name with its service in the directory.
	c.directory[name] = service
	// Create a service workers collection if necessary.
	if c.services[service] == nil {
		c.services[service] = newWorkers()
	}
	// Add peer to the service workers.
	p := &peer{name: name, node: node, service: service, version: version}
	c.services[service].add(p)
	// If necessary, notify the additions channel that a peer has been added.
	if c.additions.notify != nil {
		c.additions.notify <- struct{}{}
	}
	c.log.Info("sleuth: add %s/%s %s to %s", service, version, name, c.group)
	return nil
}

// Returns true if it had to block and false if it returns immediately.
func (c *Client) block(services ...string) bool {
	// Block until the required services are available to the client.
	c.additions.Lock()
	defer c.additions.Unlock()
	// Even though the client may have just checked to see if services exist,
	// the check is performed here in case there was a delay waiting for the
	// additions mutex to become available.
	if c.has(services...) {
		return false
	}
	c.log.Blocked("sleuth: waiting for client to find %s", services)
	c.additions.notify = make(chan struct{})
	for range c.additions.notify {
		if c.has(services...) {
			break
		}
	}
	c.additions.notify = nil
	c.log.Unblocked("sleuth: client found %s", services)
	return true
}

// Close leaves the sleuth network and stops the Gyre node.
func (c *Client) Close() error {
	c.log.Info("%s leaving %s...", c.node.Name(), c.group)
	if err := c.node.Leave(c.group); err != nil {
		return newError(errLeave, err.Error())
	}
	if err := c.node.Stop(); err != nil {
		c.log.Warn("sleuth: %s %s [%d]",
			c.node.Name(), err.Error(), warnClose)
	}
	return nil
}

func (c *Client) dispatch(payload []byte) error {
	// Returned responses (RECV command) and outstanding requests (REPL command)
	// have these headers, respectively: SLEUTH-V0RECV and SLEUTH-V0REPL
	groupLength := len(c.group)
	dispatchLength := 4
	headerLength := groupLength + dispatchLength
	// If the message header does not match the group, bail.
	if len(payload) < headerLength || string(payload[0:groupLength]) != c.group {
		return newError(errDispatchHeader, "bad dispatch header")
	}
	action := string(payload[groupLength : groupLength+dispatchLength])
	switch action {
	case recv:
		return c.receive(payload[headerLength:])
	case repl:
		return c.reply(payload[headerLength:])
	default:
		return newError(errDispatchAction, "bad dispatch action: %s", action)
	}
}

// Do sends an HTTP request to a service and returns and HTTP response. The URL
// for requests needs to use the following format:
// 	sleuth://service-name/requested-path
// For example, a request to the path /bar?baz=qux of a service called
// foo-service would have the URL:
// 	sleuth://foo-service/bar?baz=qux
func (c *Client) Do(req *http.Request) (*http.Response, error) {
	to := req.URL.Host
	// Handles are hexadecimal strings that are incremented by one.
	handle := strconv.FormatInt(c.handle, 16)
	c.handle++
	if req.URL.Scheme != scheme {
		err := newError(errScheme,
			"URL scheme must be \"%s\" in %s", scheme, req.URL.String())
		return nil, err
	}
	services, ok := c.services[to]
	if !ok {
		return nil, newError(errUnknownService, "%s is an unknown service", to)
	}
	p := services.next()
	payload, err := reqMarshal(c.group, c.node.UUID(), handle, req)
	if err != nil {
		return nil, err.(*Error).escalate(errRequest)
	}
	c.log.Debug("sleuth: %s %s://%s@%s%s",
		req.Method, scheme, to, p.name, req.URL.String())
	if err = c.node.Whisper(p.node, payload); err != nil {
		return nil, newError(errReqWhisper, err.Error())
	}
	listener := make(chan *http.Response, 1)
	c.listen(handle, listener)
	response := <-listener
	if response != nil {
		return response, nil
	}
	return nil, newError(errTimeout,
		"%s {%s}%s timed out", req.Method, to, req.URL.String())
}

func (c *Client) has(services ...string) bool {
	// Check to see if required services are already registered.
	verified := make(map[string]bool)
	available := 0
	for _, service := range services {
		verified[service] = false
	}
	total := len(verified)
	for service := range verified {
		if workers, ok := c.services[service]; ok && workers.available() {
			verified[service] = true
			available += 1
		}
	}
	return available == total
}

func (c *Client) listen(handle string, listener chan *http.Response) {
	c.listener.Lock()
	defer c.listener.Unlock()
	c.listener.handles[handle] = listener
	go c.timeout(handle)
}

func (c *Client) receive(payload []byte) error {
	handle, res, err := resUnmarshal(payload)
	if err != nil {
		return err.(*Error).escalate(errRECV)
	}
	c.listener.Lock()
	defer c.listener.Unlock()
	if listener, ok := c.listener.handles[handle]; ok {
		listener <- res
		delete(c.listener.handles, handle)
		return nil
	}
	return newError(errRECV, "unknown handle %d", handle)
}

func (c *Client) remove(name string) {
	if service, ok := c.directory[name]; ok {
		remaining, _ := c.services[service].remove(name)
		if remaining == 0 {
			delete(c.services, service)
		}
		delete(c.directory, name)
		c.log.Info("sleuth: remove %s:%s", service, name)
		return
	}
}

func (c *Client) reply(payload []byte) error {
	dest, req, err := reqUnmarshal(c.group, payload)
	if err != nil {
		return err.(*Error).escalate(errREPL)
	}
	c.handler.ServeHTTP(newWriter(c.node, dest), req)
	return nil
}

func (c *Client) timeout(handle string) {
	<-time.After(c.Timeout)
	c.listener.Lock()
	defer c.listener.Unlock()
	if listener, ok := c.listener.handles[handle]; ok {
		listener <- nil
		delete(c.listener.handles, handle)
	}
}

// WaitFor blocks until the required services are available to the client.
func (c *Client) WaitFor(services ...string) {
	if !c.has(services...) {
		c.block(services...)
	}
}

func newClient(group string, node *gyre.Gyre, out *logger.Logger) *Client {
	return &Client{
		additions: &notifier{Mutex: new(sync.Mutex)},
		directory: make(map[string]string),
		group:     group,
		listener: &listener{
			new(sync.Mutex),
			make(map[string]chan *http.Response)},
		log:      out,
		node:     node,
		Timeout:  time.Millisecond * 500,
		services: make(map[string]*workers)}
}
<commit_msg>Remove superfluous return.<commit_after>// Copyright 2016 Afshin Darian. All rights reserved.
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package sleuth

import (
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/ursiform/logger"
	"github.com/zeromq/gyre"
)

type listener struct {
	*sync.Mutex
	handles map[string]chan *http.Response
}

type notifier struct {
	*sync.Mutex
	notify chan struct{}
}

// Client is the peer on the sleuth network that makes requests and, if a
// handler has been provided, responds to peer requests.
type Client struct {
	// Timeout is the duration to wait before an outstanding request times out.
	// By default, it is set to 500ms.
	Timeout time.Duration

	additions *notifier
	group     string
	handle    int64
	handler   http.Handler
	listener  *listener
	log       *logger.Logger
	node      *gyre.Gyre

	directory map[string]string   // map[node-name]service-type
	services  map[string]*workers // map[service-type]service-workers
}

func (c *Client) add(group, name, node, service, version string) error {
	if group != c.group {
		c.log.Debug("sleuth: no group header for %s, client-only", name)
		return nil
	}
	// Node and service are required. Version is optional.
	if len(node) == 0 || len(service) == 0 {
		return newError(errAdd, "failed to add %s node?=%t, type?=%t",
			name, len(node) > 0, len(service) > 0)
	}
	// Associate the node name with its service in the directory.
	c.directory[name] = service
	// Create a service workers collection if necessary.
	if c.services[service] == nil {
		c.services[service] = newWorkers()
	}
	// Add peer to the service workers.
	p := &peer{name: name, node: node, service: service, version: version}
	c.services[service].add(p)
	// If necessary, notify the additions channel that a peer has been added.
	if c.additions.notify != nil {
		c.additions.notify <- struct{}{}
	}
	c.log.Info("sleuth: add %s/%s %s to %s", service, version, name, c.group)
	return nil
}

// Returns true if it had to block and false if it returns immediately.
func (c *Client) block(services ...string) bool {
	// Block until the required services are available to the client.
	c.additions.Lock()
	defer c.additions.Unlock()
	// Even though the client may have just checked to see if services exist,
	// the check is performed here in case there was a delay waiting for the
	// additions mutex to become available.
	if c.has(services...) {
		return false
	}
	c.log.Blocked("sleuth: waiting for client to find %s", services)
	c.additions.notify = make(chan struct{})
	for range c.additions.notify {
		if c.has(services...) {
			break
		}
	}
	c.additions.notify = nil
	c.log.Unblocked("sleuth: client found %s", services)
	return true
}

// Close leaves the sleuth network and stops the Gyre node.
func (c *Client) Close() error {
	c.log.Info("%s leaving %s...", c.node.Name(), c.group)
	if err := c.node.Leave(c.group); err != nil {
		return newError(errLeave, err.Error())
	}
	if err := c.node.Stop(); err != nil {
		c.log.Warn("sleuth: %s %s [%d]",
			c.node.Name(), err.Error(), warnClose)
	}
	return nil
}

func (c *Client) dispatch(payload []byte) error {
	// Returned responses (RECV command) and outstanding requests (REPL command)
	// have these headers, respectively: SLEUTH-V0RECV and SLEUTH-V0REPL
	groupLength := len(c.group)
	dispatchLength := 4
	headerLength := groupLength + dispatchLength
	// If the message header does not match the group, bail.
	if len(payload) < headerLength || string(payload[0:groupLength]) != c.group {
		return newError(errDispatchHeader, "bad dispatch header")
	}
	action := string(payload[groupLength : groupLength+dispatchLength])
	switch action {
	case recv:
		return c.receive(payload[headerLength:])
	case repl:
		return c.reply(payload[headerLength:])
	default:
		return newError(errDispatchAction, "bad dispatch action: %s", action)
	}
}

// Do sends an HTTP request to a service and returns and HTTP response. The URL
// for requests needs to use the following format:
// 	sleuth://service-name/requested-path
// For example, a request to the path /bar?baz=qux of a service called
// foo-service would have the URL:
// 	sleuth://foo-service/bar?baz=qux
func (c *Client) Do(req *http.Request) (*http.Response, error) {
	to := req.URL.Host
	// Handles are hexadecimal strings that are incremented by one.
	handle := strconv.FormatInt(c.handle, 16)
	c.handle++
	if req.URL.Scheme != scheme {
		err := newError(errScheme,
			"URL scheme must be \"%s\" in %s", scheme, req.URL.String())
		return nil, err
	}
	services, ok := c.services[to]
	if !ok {
		return nil, newError(errUnknownService, "%s is an unknown service", to)
	}
	p := services.next()
	payload, err := reqMarshal(c.group, c.node.UUID(), handle, req)
	if err != nil {
		return nil, err.(*Error).escalate(errRequest)
	}
	c.log.Debug("sleuth: %s %s://%s@%s%s",
		req.Method, scheme, to, p.name, req.URL.String())
	if err = c.node.Whisper(p.node, payload); err != nil {
		return nil, newError(errReqWhisper, err.Error())
	}
	listener := make(chan *http.Response, 1)
	c.listen(handle, listener)
	response := <-listener
	if response != nil {
		return response, nil
	}
	return nil, newError(errTimeout,
		"%s {%s}%s timed out", req.Method, to, req.URL.String())
}

func (c *Client) has(services ...string) bool {
	// Check to see if required services are already registered.
	verified := make(map[string]bool)
	available := 0
	for _, service := range services {
		verified[service] = false
	}
	total := len(verified)
	for service := range verified {
		if workers, ok := c.services[service]; ok && workers.available() {
			verified[service] = true
			available += 1
		}
	}
	return available == total
}

func (c *Client) listen(handle string, listener chan *http.Response) {
	c.listener.Lock()
	defer c.listener.Unlock()
	c.listener.handles[handle] = listener
	go c.timeout(handle)
}

func (c *Client) receive(payload []byte) error {
	handle, res, err := resUnmarshal(payload)
	if err != nil {
		return err.(*Error).escalate(errRECV)
	}
	c.listener.Lock()
	defer c.listener.Unlock()
	if listener, ok := c.listener.handles[handle]; ok {
		listener <- res
		delete(c.listener.handles, handle)
		return nil
	}
	return newError(errRECV, "unknown handle %d", handle)
}

func (c *Client) remove(name string) {
	if service, ok := c.directory[name]; ok {
		remaining, _ := c.services[service].remove(name)
		if remaining == 0 {
			delete(c.services, service)
		}
		delete(c.directory, name)
		c.log.Info("sleuth: remove %s:%s", service, name)
	}
}

func (c *Client) reply(payload []byte) error {
	dest, req, err := reqUnmarshal(c.group, payload)
	if err != nil {
		return err.(*Error).escalate(errREPL)
	}
	c.handler.ServeHTTP(newWriter(c.node, dest), req)
	return nil
}

func (c *Client) timeout(handle string) {
	<-time.After(c.Timeout)
	c.listener.Lock()
	defer c.listener.Unlock()
	if listener, ok := c.listener.handles[handle]; ok {
		listener <- nil
		delete(c.listener.handles, handle)
	}
}

// WaitFor blocks until the required services are available to the client.
func (c *Client) WaitFor(services ...string) {
	if !c.has(services...) {
		c.block(services...)
	}
}

func newClient(group string, node *gyre.Gyre, out *logger.Logger) *Client {
	return &Client{
		additions: &notifier{Mutex: new(sync.Mutex)},
		directory: make(map[string]string),
		group:     group,
		listener: &listener{
			new(sync.Mutex),
			make(map[string]chan *http.Response)},
		log:      out,
		node:     node,
		Timeout:  time.Millisecond * 500,
		services: make(map[string]*workers)}
}
<|endoftext|>


<commit_before>package wsevent

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"log"
	"net/http"
	"sync"
	"time"

	ws "github.com/gorilla/websocket"
)

//Client
type Client struct {
	//Session ID
	id string

	conn     *ws.Conn
	connLock *sync.RWMutex
	request  *http.Request
}

type request struct {
	Id   string
	Data json.RawMessage
}

type reply struct {
	Id   string `json:"id"`
	Data string `json:"data,string"`
}

var (
	reqPool   = &sync.Pool{New: func() interface{} { return request{} }}
	replyPool = &sync.Pool{New: func() interface{} { return reply{} }}
)

func genID() string {
	bytes := make([]byte, 32)
	rand.Read(bytes)

	return base64.URLEncoding.EncodeToString(bytes)
}

//Returns the client's unique session ID
func (c *Client) Id() string {
	return c.id
}

// Returns the first http request when established connection.
func (c *Client) Request() *http.Request {
	return c.request
}

func (s *Server) NewClientWithID(upgrader ws.Upgrader, w http.ResponseWriter, r *http.Request, id string) (*Client, error) {
	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		return nil, err
	}

	client := &Client{
		id:       id,
		conn:     conn,
		connLock: new(sync.RWMutex),
		request:  r,
	}
	s.newClient <- client

	return client, nil
}

func (s *Server) NewClient(upgrader ws.Upgrader, w http.ResponseWriter, r *http.Request) (*Client, error) {
	return s.NewClientWithID(upgrader, w, r, genID())
}

func (c *Client) Close() error {
	c.connLock.Lock()
	defer c.connLock.Unlock()
	return c.conn.Close()
}

//A thread-safe variant of WriteMessage
func (c *Client) Emit(data string) error {
	c.connLock.Lock()
	defer c.connLock.Unlock()
	return c.conn.WriteMessage(ws.TextMessage, []byte(data))
}

type emitJS struct {
	Id   int         `json:"id"`
	Data interface{} `json:"data"`
}

var emitPool = &sync.Pool{New: func() interface{} { return emitJS{} }}

//A thread-safe variant of EmitJSON
func (c *Client) EmitJSON(v interface{}) error {
	c.connLock.Lock()
	defer c.connLock.Unlock()

	js := emitPool.Get().(emitJS)
	defer emitPool.Put(js)

	js.Id = -1
	js.Data = v

	return c.conn.WriteJSON(js)
}

func (c *Client) cleanup(s *Server) {
	c.conn.Close()

	s.joinedRoomsLock.RLock()
	for _, room := range s.joinedRooms[c.id] {
		//log.Println(room)
		index := -1

		s.roomsLock.Lock()
		for i, client := range s.rooms[room] {
			if client.id == c.id {
				index = i
			}
		}

		s.rooms[room] = append(s.rooms[room][:index], s.rooms[room][index+1:]...)
		s.roomsLock.Unlock()
	}
	s.joinedRoomsLock.RUnlock()

	s.joinedRoomsLock.Lock()
	delete(s.joinedRooms, c.id)
	s.joinedRoomsLock.Unlock()

	if s.OnDisconnect != nil {
		s.OnDisconnect(c.id)
	}
}

func (c *Client) listener(s *Server) {
	throttle := time.NewTicker(time.Millisecond * 10)
	defer throttle.Stop()
	for {
		<-throttle.C
		mtype, data, err := c.conn.ReadMessage()
		if err != nil {
			c.cleanup(s)
			return
		}
		if mtype != ws.TextMessage {
			c.conn.Close()
			return
		}

		js := reqPool.Get().(request)

		if err := json.Unmarshal(data, &js); err != nil {
			continue
		}

		callName := s.Extractor(js.Data)

		s.handlersLock.RLock()
		f, ok := s.handlers[callName]
		s.handlersLock.RUnlock()

		if !ok {
			if s.DefaultHandler != nil {
				f = s.DefaultHandler
				goto call
			}
			continue
		}
	call:
		go func() {
			rtrn := f(s, c, js.Data)
			replyJs := replyPool.Get().(reply)
			replyJs.Id = js.Id
			replyJs.Data = string(rtrn)

			bytes, _ := json.Marshal(replyJs)
			c.Emit(string(bytes))

			reqPool.Put(js)
			replyPool.Put(replyJs)
		}()
	}
}
<commit_msg>remove log<commit_after>package wsevent

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"net/http"
	"sync"
	"time"

	ws "github.com/gorilla/websocket"
)

//Client
type Client struct {
	//Session ID
	id string

	conn     *ws.Conn
	connLock *sync.RWMutex
	request  *http.Request
}

type request struct {
	Id   string
	Data json.RawMessage
}

type reply struct {
	Id   string `json:"id"`
	Data string `json:"data,string"`
}

var (
	reqPool   = &sync.Pool{New: func() interface{} { return request{} }}
	replyPool = &sync.Pool{New: func() interface{} { return reply{} }}
)

func genID() string {
	bytes := make([]byte, 32)
	rand.Read(bytes)

	return base64.URLEncoding.EncodeToString(bytes)
}

//Returns the client's unique session ID
func (c *Client) Id() string {
	return c.id
}

// Returns the first http request when established connection.
func (c *Client) Request() *http.Request {
	return c.request
}

func (s *Server) NewClientWithID(upgrader ws.Upgrader, w http.ResponseWriter, r *http.Request, id string) (*Client, error) {
	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		return nil, err
	}

	client := &Client{
		id:       id,
		conn:     conn,
		connLock: new(sync.RWMutex),
		request:  r,
	}
	s.newClient <- client

	return client, nil
}

func (s *Server) NewClient(upgrader ws.Upgrader, w http.ResponseWriter, r *http.Request) (*Client, error) {
	return s.NewClientWithID(upgrader, w, r, genID())
}

func (c *Client) Close() error {
	c.connLock.Lock()
	defer c.connLock.Unlock()
	return c.conn.Close()
}

//A thread-safe variant of WriteMessage
func (c *Client) Emit(data string) error {
	c.connLock.Lock()
	defer c.connLock.Unlock()
	return c.conn.WriteMessage(ws.TextMessage, []byte(data))
}

type emitJS struct {
	Id   int         `json:"id"`
	Data interface{} `json:"data"`
}

var emitPool = &sync.Pool{New: func() interface{} { return emitJS{} }}

//A thread-safe variant of EmitJSON
func (c *Client) EmitJSON(v interface{}) error {
	c.connLock.Lock()
	defer c.connLock.Unlock()

	js := emitPool.Get().(emitJS)
	defer emitPool.Put(js)

	js.Id = -1
	js.Data = v

	return c.conn.WriteJSON(js)
}

func (c *Client) cleanup(s *Server) {
	c.conn.Close()

	s.joinedRoomsLock.RLock()
	for _, room := range s.joinedRooms[c.id] {
		//log.Println(room)
		index := -1

		s.roomsLock.Lock()
		for i, client := range s.rooms[room] {
			if client.id == c.id {
				index = i
			}
		}

		s.rooms[room] = append(s.rooms[room][:index], s.rooms[room][index+1:]...)
		s.roomsLock.Unlock()
	}
	s.joinedRoomsLock.RUnlock()

	s.joinedRoomsLock.Lock()
	delete(s.joinedRooms, c.id)
	s.joinedRoomsLock.Unlock()

	if s.OnDisconnect != nil {
		s.OnDisconnect(c.id)
	}
}

func (c *Client) listener(s *Server) {
	throttle := time.NewTicker(time.Millisecond * 10)
	defer throttle.Stop()
	for {
		<-throttle.C
		mtype, data, err := c.conn.ReadMessage()
		if err != nil {
			c.cleanup(s)
			return
		}
		if mtype != ws.TextMessage {
			c.conn.Close()
			return
		}

		js := reqPool.Get().(request)

		if err := json.Unmarshal(data, &js); err != nil {
			continue
		}

		callName := s.Extractor(js.Data)

		s.handlersLock.RLock()
		f, ok := s.handlers[callName]
		s.handlersLock.RUnlock()

		if !ok {
			if s.DefaultHandler != nil {
				f = s.DefaultHandler
				goto call
			}
			continue
		}
	call:
		go func() {
			rtrn := f(s, c, js.Data)
			replyJs := replyPool.Get().(reply)
			replyJs.Id = js.Id
			replyJs.Data = string(rtrn)

			bytes, _ := json.Marshal(replyJs)
			c.Emit(string(bytes))

			reqPool.Put(js)
			replyPool.Put(replyJs)
		}()
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 The SkyDNS Authors. All rights reserved.
// Use of this source code is governed by The MIT License (MIT) that can be
// found in the LICENSE file.

package main

import (
	"log"
	"net/url"
	"strings"

	"github.com/coreos/go-etcd/etcd"
)

func NewClient(machines []string) (client *etcd.Client) {
	// set default if not specified in env
	if len(machines) == 1 && machines[0] == "" {
		machines[0] = "http://127.0.0.1:4001"

	}
	// override if we have a commandline flag as well
	if machine != "" {
		machines = strings.Split(machine, ",")
	}
	if strings.HasPrefix(machines[0], "https://") {
		var err error
		if client, err = etcd.NewTLSClient(machines, tlspem, tlskey, cacert); err != nil {
			// TODO(miek): would be nice if this wan't a fatal error
			log.Fatalf("failure to connect: %s\n", err)
		}
		client.SyncCluster()
	} else {
		client = etcd.NewClient(machines)
		client.SyncCluster()
	}
	return client
}

// updateClient updates the client with the machines found in v2/_etcd/machines.
func (s *server) UpdateClient(resp *etcd.Response) {
	machines := make([]string, 0, 3)
	for _, m := range resp.Node.Nodes {
		u, e := url.Parse(m.Value)
		if e != nil {
			continue
		}
		// etcd=bla&raft=bliep
		// TODO(miek): surely there is a better way to do this
		ms := strings.Split(u.String(), "&")
		if len(ms) == 0 {
			continue
		}
		if len(ms[0]) < 5 {
			continue
		}
		machines = append(machines, ms[0][5:])
	}
	s.config.log.Infof("setting new etcd cluster to %v", machines)
	c := NewClient(machines)
	// This is our RCU, switch the pointer, old readers get the old
	// one, new reader get the new one.
	s.client = c
}
<commit_msg>removed left over code<commit_after>// Copyright (c) 2014 The SkyDNS Authors. All rights reserved.
// Use of this source code is governed by The MIT License (MIT) that can be
// found in the LICENSE file.

package main

import (
	"log"
	"net/url"
	"strings"

	"github.com/coreos/go-etcd/etcd"
)

func NewClient(machines []string) (client *etcd.Client) {
	// set default if not specified in env
	if len(machines) == 1 && machines[0] == "" {
		machines[0] = "http://127.0.0.1:4001"
	}
	if strings.HasPrefix(machines[0], "https://") {
		var err error
		if client, err = etcd.NewTLSClient(machines, tlspem, tlskey, cacert); err != nil {
			// TODO(miek): would be nice if this wan't a fatal error
			log.Fatalf("failure to connect: %s\n", err)
		}
		client.SyncCluster()
	} else {
		client = etcd.NewClient(machines)
		client.SyncCluster()
	}
	return client
}

// updateClient updates the client with the machines found in v2/_etcd/machines.
func (s *server) UpdateClient(resp *etcd.Response) {
	machines := make([]string, 0, 3)
	for _, m := range resp.Node.Nodes {
		u, e := url.Parse(m.Value)
		if e != nil {
			continue
		}
		// etcd=bla&raft=bliep
		// TODO(miek): surely there is a better way to do this
		ms := strings.Split(u.String(), "&")
		if len(ms) == 0 {
			continue
		}
		if len(ms[0]) < 5 {
			continue
		}
		machines = append(machines, ms[0][5:])
	}
	s.config.log.Infof("setting new etcd cluster to %v", machines)
	c := NewClient(machines)
	// This is our RCU, switch the pointer, old readers get the old
	// one, new reader get the new one.
	s.client = c
}
<|endoftext|>


<commit_before>package kinesis

import (
	"net"
	"net/http"
	"time"
)

const AWSSecurityTokenHeader = "X-Amz-Security-Token"

// Client is like http.Client, but signs all requests using Auth.
type Client struct {
	// Auth holds the credentials for this client instance
	auth Auth
	// The http client to make requests with. If nil, http.DefaultClient is used.
	client *http.Client
}

// NewClient creates a new Client that uses the credentials in the specified
// Auth object.
//
// This function assumes the Auth object has been sanely initialized. If you
// wish to infer auth credentials from the environment, refer to NewAuth
func NewClient(auth Auth) *Client {
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		DisableKeepAlives:   true,
	}

	client := &http.Client{
		Transport: transport,
	}

	return &Client{auth: auth, client: client}
}

// NewClientWithHTTPClient creates a client with a non-default http client
// ie. a timeout could be set on the HTTP client to timeout if Kinesis doesn't
// response in a timely manner like after the 5 minute mark where the current
// shard iterator expires
func NewClientWithHTTPClient(auth Auth, httpClient *http.Client) *Client {
	return &Client{auth: auth, client: httpClient}
}

// Do some request, but sign it before sending
func (c *Client) Do(req *http.Request) (*http.Response, error) {
	err := Sign(c.auth, req)
	if err != nil {
		return nil, err
	}

	if c.auth.HasExpiration() && time.Now().After(c.auth.GetExpiration()) {
		if err = c.auth.Renew(); err != nil { // TODO: (see auth.go#Renew) may be slow
			return nil, err
		}
	}

	if c.auth.GetToken() != "" {
		req.Header.Add(AWSSecurityTokenHeader, c.auth.GetToken())
	}

	return c.client.Do(req)
}
<commit_msg>Add HTTP request timeout.<commit_after>package kinesis

import (
	"net"
	"net/http"
	"time"
)

const AWSSecurityTokenHeader = "X-Amz-Security-Token"

// Client is like http.Client, but signs all requests using Auth.
type Client struct {
	// Auth holds the credentials for this client instance
	auth Auth
	// The http client to make requests with. If nil, http.DefaultClient is used.
	client *http.Client
}

// NewClient creates a new Client that uses the credentials in the specified
// Auth object.
//
// This function assumes the Auth object has been sanely initialized. If you
// wish to infer auth credentials from the environment, refer to NewAuth
func NewClient(auth Auth) *Client {
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout: 5 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 5 * time.Second,
		DisableKeepAlives:   true,
	}

	client := &http.Client{
		Transport: transport,
		Timeout:   500 * time.Millisecond,
	}

	return &Client{auth: auth, client: client}
}

// NewClientWithHTTPClient creates a client with a non-default http client
// ie. a timeout could be set on the HTTP client to timeout if Kinesis doesn't
// response in a timely manner like after the 5 minute mark where the current
// shard iterator expires
func NewClientWithHTTPClient(auth Auth, httpClient *http.Client) *Client {
	return &Client{auth: auth, client: httpClient}
}

// Do some request, but sign it before sending
func (c *Client) Do(req *http.Request) (*http.Response, error) {
	err := Sign(c.auth, req)
	if err != nil {
		return nil, err
	}

	if c.auth.HasExpiration() && time.Now().After(c.auth.GetExpiration()) {
		if err = c.auth.Renew(); err != nil { // TODO: (see auth.go#Renew) may be slow
			return nil, err
		}
	}

	if c.auth.GetToken() != "" {
		req.Header.Add(AWSSecurityTokenHeader, c.auth.GetToken())
	}

	return c.client.Do(req)
}
<|endoftext|>


<commit_before>package getter

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	urlhelper "github.com/hashicorp/go-getter/helper/url"
	"github.com/hashicorp/go-safetemp"
)

// Client is a client for downloading things.
//
// Top-level functions such as Get are shortcuts for interacting with a client.
// Using a client directly allows more fine-grained control over how downloading
// is done, as well as customizing the protocols supported.
type Client struct {
	// Src is the source URL to get.
	//
	// Dst is the path to save the downloaded thing as. If Dir is set to
	// true, then this should be a directory. If the directory doesn't exist,
	// it will be created for you.
	//
	// Pwd is the working directory for detection. If this isn't set, some
	// detection may fail. Client will not default pwd to the current
	// working directory for security reasons.
	Src string
	Dst string
	Pwd string

	// Mode is the method of download the client will use. See ClientMode
	// for documentation.
	Mode ClientMode

	// Detectors is the list of detectors that are tried on the source.
	// If this is nil, then the default Detectors will be used.
	Detectors []Detector

	// Decompressors is the map of decompressors supported by this client.
	// If this is nil, then the default value is the Decompressors global.
	Decompressors map[string]Decompressor

	// Getters is the map of protocols supported by this client. If this
	// is nil, then the default Getters variable will be used.
	Getters map[string]Getter

	// Dir, if true, tells the Client it is downloading a directory (versus
	// a single file). This distinction is necessary since filenames and
	// directory names follow the same format so disambiguating is impossible
	// without knowing ahead of time.
	//
	// WARNING: deprecated. If Mode is set, that will take precedence.
	Dir bool
}

// Get downloads the configured source to the destination.
func (c *Client) Get() error {
	// Store this locally since there are cases we swap this
	mode := c.Mode
	if mode == ClientModeInvalid {
		if c.Dir {
			mode = ClientModeDir
		} else {
			mode = ClientModeFile
		}
	}

	// Default decompressor value
	decompressors := c.Decompressors
	if decompressors == nil {
		decompressors = Decompressors
	}

	// Detect the URL. This is safe if it is already detected.
	detectors := c.Detectors
	if detectors == nil {
		detectors = Detectors
	}
	src, err := Detect(c.Src, c.Pwd, detectors)
	if err != nil {
		return err
	}

	// Determine if we have a forced protocol, i.e. "git::http://..."
	force, src := getForcedGetter(src)

	// If there is a subdir component, then we download the root separately
	// and then copy over the proper subdir.
	var realDst string
	dst := c.Dst
	src, subDir := SourceDirSubdir(src)
	if subDir != "" {
		td, tdcloser, err := safetemp.Dir("", "getter")
		if err != nil {
			return err
		}
		defer tdcloser.Close()

		realDst = dst
		dst = td
	}

	u, err := urlhelper.Parse(src)
	if err != nil {
		return err
	}
	if force == "" {
		force = u.Scheme
	}

	getters := c.Getters
	if getters == nil {
		getters = Getters
	}

	g, ok := getters[force]
	if !ok {
		return fmt.Errorf(
			"download not supported for scheme '%s'", force)
	}

	// We have magic query parameters that we use to signal different features
	q := u.Query()

	// Determine if we have an archive type
	archiveV := q.Get("archive")
	if archiveV != "" {
		// Delete the paramter since it is a magic parameter we don't
		// want to pass on to the Getter
		q.Del("archive")
		u.RawQuery = q.Encode()

		// If we can parse the value as a bool and it is false, then
		// set the archive to "-" which should never map to a decompressor
		if b, err := strconv.ParseBool(archiveV); err == nil && !b {
			archiveV = "-"
		}
	}
	if archiveV == "" {
		// We don't appear to... but is it part of the filename?
		matchingLen := 0
		for k, _ := range decompressors {
			if strings.HasSuffix(u.Path, "."+k) && len(k) > matchingLen {
				archiveV = k
				matchingLen = len(k)
			}
		}
	}

	// If we have a decompressor, then we need to change the destination
	// to download to a temporary path. We unarchive this into the final,
	// real path.
	var decompressDst string
	var decompressDir bool
	decompressor := decompressors[archiveV]
	if decompressor != nil {
		// Create a temporary directory to store our archive. We delete
		// this at the end of everything.
		td, err := ioutil.TempDir("", "getter")
		if err != nil {
			return fmt.Errorf(
				"Error creating temporary directory for archive: %s", err)
		}
		defer os.RemoveAll(td)

		// Swap the download directory to be our temporary path and
		// store the old values.
		decompressDst = dst
		decompressDir = mode != ClientModeFile
		dst = filepath.Join(td, "archive")
		mode = ClientModeFile
	}

	// Determine if we have a checksum
	var checksumHash hash.Hash
	var checksumValue []byte
	if v := q.Get("checksum"); v != "" {
		// Delete the query parameter if we have it.
		q.Del("checksum")
		u.RawQuery = q.Encode()

		// Determine the checksum hash type
		checksumType := ""
		idx := strings.Index(v, ":")
		if idx > -1 {
			checksumType = v[:idx]
		}
		switch checksumType {
		case "md5":
			checksumHash = md5.New()
		case "sha1":
			checksumHash = sha1.New()
		case "sha256":
			checksumHash = sha256.New()
		case "sha512":
			checksumHash = sha512.New()
		default:
			return fmt.Errorf(
				"unsupported checksum type: %s", checksumType)
		}

		// Get the remainder of the value and parse it into bytes
		b, err := hex.DecodeString(v[idx+1:])
		if err != nil {
			return fmt.Errorf("invalid checksum: %s", err)
		}

		// Set our value
		checksumValue = b
	}

	if mode == ClientModeAny {
		// Ask the getter which client mode to use
		mode, err = g.ClientMode(u)
		if err != nil {
			return err
		}

		// Destination is the base name of the URL path in "any" mode when
		// a file source is detected.
		if mode == ClientModeFile {
			filename := filepath.Base(u.Path)

			// Determine if we have a custom file name
			if v := q.Get("filename"); v != "" {
				// Delete the query parameter if we have it.
				q.Del("filename")
				u.RawQuery = q.Encode()

				filename = v
			}

			dst = filepath.Join(dst, filename)
		}
	}

	// If we're not downloading a directory, then just download the file
	// and return.
	if mode == ClientModeFile {
		err := g.GetFile(dst, u)
		if err != nil {
			return err
		}

		if checksumHash != nil {
			if err := checksum(dst, checksumHash, checksumValue); err != nil {
				return err
			}
		}

		if decompressor != nil {
			// We have a decompressor, so decompress the current destination
			// into the final destination with the proper mode.
			err := decompressor.Decompress(decompressDst, dst, decompressDir)
			if err != nil {
				return err
			}

			// Swap the information back
			dst = decompressDst
			if decompressDir {
				mode = ClientModeAny
			} else {
				mode = ClientModeFile
			}
		}

		// We check the dir value again because it can be switched back
		// if we were unarchiving. If we're still only Get-ing a file, then
		// we're done.
		if mode == ClientModeFile {
			return nil
		}
	}

	// If we're at this point we're either downloading a directory or we've
	// downloaded and unarchived a directory and we're just checking subdir.
	// In the case we have a decompressor we don't Get because it was Get
	// above.
	if decompressor == nil {
		// If we're getting a directory, then this is an error. You cannot
		// checksum a directory. TODO: test
		if checksumHash != nil {
			return fmt.Errorf(
				"checksum cannot be specified for directory download")
		}

		// We're downloading a directory, which might require a bit more work
		// if we're specifying a subdir.
		err := g.Get(dst, u)
		if err != nil {
			err = fmt.Errorf("error downloading '%s': %s", src, err)
			return err
		}
	}

	// If we have a subdir, copy that over
	if subDir != "" {
		if err := os.RemoveAll(realDst); err != nil {
			return err
		}
		if err := os.MkdirAll(realDst, 0755); err != nil {
			return err
		}

		// Process any globs
		subDir, err := SubdirGlob(dst, subDir)
		if err != nil {
			return err
		}

		return copyDir(realDst, subDir, false)
	}

	return nil
}

// checksum is a simple method to compute the checksum of a source file
// and compare it to the given expected value.
func checksum(source string, h hash.Hash, v []byte) error {
	f, err := os.Open(source)
	if err != nil {
		return fmt.Errorf("Failed to open file for checksum: %s", err)
	}
	defer f.Close()

	if _, err := io.Copy(h, f); err != nil {
		return fmt.Errorf("Failed to hash: %s", err)
	}

	if actual := h.Sum(nil); !bytes.Equal(actual, v) {
		return fmt.Errorf(
			"Checksums did not match.\nExpected: %s\nGot: %s",
			hex.EncodeToString(v),
			hex.EncodeToString(actual))
	}

	return nil
}
<commit_msg>don't get file when checksum of destination is valid<commit_after>package getter

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	urlhelper "github.com/hashicorp/go-getter/helper/url"
	"github.com/hashicorp/go-safetemp"
)

// Client is a client for downloading things.
//
// Top-level functions such as Get are shortcuts for interacting with a client.
// Using a client directly allows more fine-grained control over how downloading
// is done, as well as customizing the protocols supported.
type Client struct {
	// Src is the source URL to get.
	//
	// Dst is the path to save the downloaded thing as. If Dir is set to
	// true, then this should be a directory. If the directory doesn't exist,
	// it will be created for you.
	//
	// Pwd is the working directory for detection. If this isn't set, some
	// detection may fail. Client will not default pwd to the current
	// working directory for security reasons.
	Src string
	Dst string
	Pwd string

	// Mode is the method of download the client will use. See ClientMode
	// for documentation.
	Mode ClientMode

	// Detectors is the list of detectors that are tried on the source.
	// If this is nil, then the default Detectors will be used.
	Detectors []Detector

	// Decompressors is the map of decompressors supported by this client.
	// If this is nil, then the default value is the Decompressors global.
	Decompressors map[string]Decompressor

	// Getters is the map of protocols supported by this client. If this
	// is nil, then the default Getters variable will be used.
	Getters map[string]Getter

	// Dir, if true, tells the Client it is downloading a directory (versus
	// a single file). This distinction is necessary since filenames and
	// directory names follow the same format so disambiguating is impossible
	// without knowing ahead of time.
	//
	// WARNING: deprecated. If Mode is set, that will take precedence.
	Dir bool
}

// Get downloads the configured source to the destination.
func (c *Client) Get() error {
	// Store this locally since there are cases we swap this
	mode := c.Mode
	if mode == ClientModeInvalid {
		if c.Dir {
			mode = ClientModeDir
		} else {
			mode = ClientModeFile
		}
	}

	// Default decompressor value
	decompressors := c.Decompressors
	if decompressors == nil {
		decompressors = Decompressors
	}

	// Detect the URL. This is safe if it is already detected.
	detectors := c.Detectors
	if detectors == nil {
		detectors = Detectors
	}
	src, err := Detect(c.Src, c.Pwd, detectors)
	if err != nil {
		return err
	}

	// Determine if we have a forced protocol, i.e. "git::http://..."
	force, src := getForcedGetter(src)

	// If there is a subdir component, then we download the root separately
	// and then copy over the proper subdir.
	var realDst string
	dst := c.Dst
	src, subDir := SourceDirSubdir(src)
	if subDir != "" {
		td, tdcloser, err := safetemp.Dir("", "getter")
		if err != nil {
			return err
		}
		defer tdcloser.Close()

		realDst = dst
		dst = td
	}

	u, err := urlhelper.Parse(src)
	if err != nil {
		return err
	}
	if force == "" {
		force = u.Scheme
	}

	getters := c.Getters
	if getters == nil {
		getters = Getters
	}

	g, ok := getters[force]
	if !ok {
		return fmt.Errorf(
			"download not supported for scheme '%s'", force)
	}

	// We have magic query parameters that we use to signal different features
	q := u.Query()

	// Determine if we have an archive type
	archiveV := q.Get("archive")
	if archiveV != "" {
		// Delete the paramter since it is a magic parameter we don't
		// want to pass on to the Getter
		q.Del("archive")
		u.RawQuery = q.Encode()

		// If we can parse the value as a bool and it is false, then
		// set the archive to "-" which should never map to a decompressor
		if b, err := strconv.ParseBool(archiveV); err == nil && !b {
			archiveV = "-"
		}
	}
	if archiveV == "" {
		// We don't appear to... but is it part of the filename?
		matchingLen := 0
		for k, _ := range decompressors {
			if strings.HasSuffix(u.Path, "."+k) && len(k) > matchingLen {
				archiveV = k
				matchingLen = len(k)
			}
		}
	}

	// If we have a decompressor, then we need to change the destination
	// to download to a temporary path. We unarchive this into the final,
	// real path.
	var decompressDst string
	var decompressDir bool
	decompressor := decompressors[archiveV]
	if decompressor != nil {
		// Create a temporary directory to store our archive. We delete
		// this at the end of everything.
		td, err := ioutil.TempDir("", "getter")
		if err != nil {
			return fmt.Errorf(
				"Error creating temporary directory for archive: %s", err)
		}
		defer os.RemoveAll(td)

		// Swap the download directory to be our temporary path and
		// store the old values.
		decompressDst = dst
		decompressDir = mode != ClientModeFile
		dst = filepath.Join(td, "archive")
		mode = ClientModeFile
	}

	// Determine if we have a checksum
	var checksumHash hash.Hash
	var checksumValue []byte
	if v := q.Get("checksum"); v != "" {
		// Delete the query parameter if we have it.
		q.Del("checksum")
		u.RawQuery = q.Encode()

		// Determine the checksum hash type
		checksumType := ""
		idx := strings.Index(v, ":")
		if idx > -1 {
			checksumType = v[:idx]
		}
		switch checksumType {
		case "md5":
			checksumHash = md5.New()
		case "sha1":
			checksumHash = sha1.New()
		case "sha256":
			checksumHash = sha256.New()
		case "sha512":
			checksumHash = sha512.New()
		default:
			return fmt.Errorf(
				"unsupported checksum type: %s", checksumType)
		}

		// Get the remainder of the value and parse it into bytes
		b, err := hex.DecodeString(v[idx+1:])
		if err != nil {
			return fmt.Errorf("invalid checksum: %s", err)
		}

		// Set our value
		checksumValue = b
	}

	if mode == ClientModeAny {
		// Ask the getter which client mode to use
		mode, err = g.ClientMode(u)
		if err != nil {
			return err
		}

		// Destination is the base name of the URL path in "any" mode when
		// a file source is detected.
		if mode == ClientModeFile {
			filename := filepath.Base(u.Path)

			// Determine if we have a custom file name
			if v := q.Get("filename"); v != "" {
				// Delete the query parameter if we have it.
				q.Del("filename")
				u.RawQuery = q.Encode()

				filename = v
			}

			dst = filepath.Join(dst, filename)
		}
	}

	// If we're not downloading a directory, then just download the file
	// and return.
	if mode == ClientModeFile {
		getFile := true
		if checksumHash != nil {
			if err := checksum(dst, checksumHash, checksumValue); err == nil {
				// don't get the file if the checksum of dst is correct
				getFile = false
			}
		}
		if getFile {
			err := g.GetFile(dst, u)
			if err != nil {
				return err
			}

			if checksumHash != nil {
				if err := checksum(dst, checksumHash, checksumValue); err != nil {
					return err
				}
			}
		}

		if decompressor != nil {
			// We have a decompressor, so decompress the current destination
			// into the final destination with the proper mode.
			err := decompressor.Decompress(decompressDst, dst, decompressDir)
			if err != nil {
				return err
			}

			// Swap the information back
			dst = decompressDst
			if decompressDir {
				mode = ClientModeAny
			} else {
				mode = ClientModeFile
			}
		}

		// We check the dir value again because it can be switched back
		// if we were unarchiving. If we're still only Get-ing a file, then
		// we're done.
		if mode == ClientModeFile {
			return nil
		}
	}

	// If we're at this point we're either downloading a directory or we've
	// downloaded and unarchived a directory and we're just checking subdir.
	// In the case we have a decompressor we don't Get because it was Get
	// above.
	if decompressor == nil {
		// If we're getting a directory, then this is an error. You cannot
		// checksum a directory. TODO: test
		if checksumHash != nil {
			return fmt.Errorf(
				"checksum cannot be specified for directory download")
		}

		// We're downloading a directory, which might require a bit more work
		// if we're specifying a subdir.
		err := g.Get(dst, u)
		if err != nil {
			err = fmt.Errorf("error downloading '%s': %s", src, err)
			return err
		}
	}

	// If we have a subdir, copy that over
	if subDir != "" {
		if err := os.RemoveAll(realDst); err != nil {
			return err
		}
		if err := os.MkdirAll(realDst, 0755); err != nil {
			return err
		}

		// Process any globs
		subDir, err := SubdirGlob(dst, subDir)
		if err != nil {
			return err
		}

		return copyDir(realDst, subDir, false)
	}

	return nil
}

// checksum is a simple method to compute the checksum of a source file
// and compare it to the given expected value.
func checksum(source string, h hash.Hash, v []byte) error {
	f, err := os.Open(source)
	if err != nil {
		return fmt.Errorf("Failed to open file for checksum: %s", err)
	}
	defer f.Close()

	if _, err := io.Copy(h, f); err != nil {
		return fmt.Errorf("Failed to hash: %s", err)
	}

	if actual := h.Sum(nil); !bytes.Equal(actual, v) {
		return fmt.Errorf(
			"Checksums did not match.\nExpected: %s\nGot: %s",
			hex.EncodeToString(v),
			hex.EncodeToString(actual))
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Jason Goecke
// client.go

package wit

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net/http"
	"os"
)

const (
	USERAGENT = "WIT (Go net/http)"
)

// Represents a client for the Wit API (https://wit.ai/docs/api)
type WitClient struct {
	ApiBase string
}

// Represents the HTTP parameters to pass along to the Wit API
type HttpParams struct {
	Verb        string
	Resource    string
	ContentType string
	Data        []byte
}

// Stores the ApiKey for the Wit API
var ApiKey string

// Creates a NewClient for the Wit API
//
//		client := wit.NewClient("<ACCESS-TOKEN>")
func NewClient(apiKey string) *WitClient {
	client := &WitClient{}
	client.ApiBase = "https://api.wit.ai"
	ApiKey = apiKey
	return client
}

// Provides a common facility for doing a DELETE on a Wit resource
//
//		result, err := delete("https://api.wit.ai/entities", "favorite_city")
func delete(resource string, id string) ([]byte, error) {
	httpParams := &HttpParams{}
	httpParams.Resource = resource + "/" + id
	httpParams.Verb = "DELETE"
	return processRequest(httpParams)
}

// Provides a common facility for doing a GET on a Wit resource
//
//		result, err := get("https://api.wit.ai/entities/favorite_city")
func get(resource string) ([]byte, error) {
	httpParams := &HttpParams{}
	httpParams.Resource = resource
	httpParams.Verb = "GET"
	return processRequest(httpParams)
}

// Provides a common facility for doing a POST on a Wit resource. Takes
// JSON []byte for the data argument.
//
//		result, err := post("https://api.wit.ai/entities", entity)
func post(resource string, data []byte) ([]byte, error) {
	httpParams := &HttpParams{"POST", resource, "application/json", data}
	return processRequest(httpParams)
}

// Provides a common facility for doing a POST with a file on a Wit resource.
//
//		result, err := postFile("https://api.wit.ai/messages", message)
func postFile(resource string, request *MessageRequest) ([]byte, error) {
	if request.File != "" {
		file, err := os.Open(request.File)
		if err != nil {
			return nil, err
		}
		defer file.Close()
		stats, statsErr := file.Stat()
		if statsErr != nil {
			return nil, statsErr
		}
		var size int64 = stats.Size()
		data := make([]byte, size)
		file.Read(data)
		httpParams := &HttpParams{"POST", resource, request.ContentType, data}
		return processRequest(httpParams)
	} else {
		if request.FileContents != nil {
			httpParams := &HttpParams{"POST", resource, request.ContentType, request.FileContents}
			return processRequest(httpParams)
		} else {
			return nil, errors.New("Must provide a filename or contents")
		}
	}
}

// Provides a common facility for doing a PUT on a Wit resource.
//
//		result, err := put("https://api.wit.ai/entities", entity)
func put(resource string, data []byte) ([]byte, error) {
	httpParams := &HttpParams{"PUT", resource, "application/json", data}
	return processRequest(httpParams)
}

// Processes an HTTP request to the Wit API
func processRequest(httpParams *HttpParams) ([]byte, error) {
	reader := bytes.NewReader(httpParams.Data)
	httpClient := &http.Client{}
	req, err := http.NewRequest(httpParams.Verb, httpParams.Resource, reader)
	if err != nil {
		return nil, err
	}
	setHeaders(req, httpParams.ContentType)
	result, err := httpClient.Do(req)
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(result.Body)
	result.Body.Close()
	if result.StatusCode != 200 {
		return nil, errors.New(http.StatusText(result.StatusCode))
	}
	return body, nil
}

// Sets the custom headers required for the Wit.ai API
//
//		setHeaders(req, httpParams.ContentType)
func setHeaders(req *http.Request, contentType string) {
	req.Header.Add("Authorization", "Bearer "+ApiKey)
	req.Header.Set("Content-Type", contentType)
	req.Header.Set("Accept", "application/json")
}
<commit_msg>Now supports the mandatory versioning of the Wit API of v=20140401.<commit_after>// Copyright (c) 2014 Jason Goecke
// client.go

package wit

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
)

const (
	USERAGENT   = "WIT (Go net/http)"
	API_VERSION = "v=20140401"
)

// Represents a client for the Wit API (https://wit.ai/docs/api)
type WitClient struct {
	ApiBase string
}

// Represents the HTTP parameters to pass along to the Wit API
type HttpParams struct {
	Verb        string
	Resource    string
	ContentType string
	Data        []byte
}

// Stores the ApiKey for the Wit API
var ApiKey string

// Creates a NewClient for the Wit API
//
//		client := wit.NewClient("<ACCESS-TOKEN>")
func NewClient(apiKey string) *WitClient {
	client := &WitClient{}
	client.ApiBase = "https://api.wit.ai"
	ApiKey = apiKey
	return client
}

// Provides a common facility for doing a DELETE on a Wit resource
//
//		result, err := delete("https://api.wit.ai/entities", "favorite_city")
func delete(resource string, id string) ([]byte, error) {
	httpParams := &HttpParams{}
	httpParams.Resource = resource + "/" + id
	httpParams.Verb = "DELETE"
	return processRequest(httpParams)
}

// Provides a common facility for doing a GET on a Wit resource
//
//		result, err := get("https://api.wit.ai/entities/favorite_city")
func get(resource string) ([]byte, error) {
	httpParams := &HttpParams{}
	httpParams.Resource = resource
	httpParams.Verb = "GET"
	return processRequest(httpParams)
}

// Provides a common facility for doing a POST on a Wit resource. Takes
// JSON []byte for the data argument.
//
//		result, err := post("https://api.wit.ai/entities", entity)
func post(resource string, data []byte) ([]byte, error) {
	httpParams := &HttpParams{"POST", resource, "application/json", data}
	return processRequest(httpParams)
}

// Provides a common facility for doing a POST with a file on a Wit resource.
//
//		result, err := postFile("https://api.wit.ai/messages", message)
func postFile(resource string, request *MessageRequest) ([]byte, error) {
	if request.File != "" {
		file, err := os.Open(request.File)
		if err != nil {
			return nil, err
		}
		defer file.Close()
		stats, statsErr := file.Stat()
		if statsErr != nil {
			return nil, statsErr
		}
		var size int64 = stats.Size()
		data := make([]byte, size)
		file.Read(data)
		httpParams := &HttpParams{"POST", resource, request.ContentType, data}
		return processRequest(httpParams)
	} else {
		if request.FileContents != nil {
			httpParams := &HttpParams{"POST", resource, request.ContentType, request.FileContents}
			return processRequest(httpParams)
		} else {
			return nil, errors.New("Must provide a filename or contents")
		}
	}
}

// Provides a common facility for doing a PUT on a Wit resource.
//
//		result, err := put("https://api.wit.ai/entities", entity)
func put(resource string, data []byte) ([]byte, error) {
	httpParams := &HttpParams{"PUT", resource, "application/json", data}
	return processRequest(httpParams)
}

// Processes an HTTP request to the Wit API
func processRequest(httpParams *HttpParams) ([]byte, error) {
	regex := regexp.MustCompile(`\?`)
	if regex.MatchString(httpParams.Resource) {
		httpParams.Resource += "&" + API_VERSION
	} else {
		httpParams.Resource += "?" + API_VERSION
	}
	reader := bytes.NewReader(httpParams.Data)
	httpClient := &http.Client{}
	req, err := http.NewRequest(httpParams.Verb, httpParams.Resource, reader)
	if err != nil {
		return nil, err
	}
	setHeaders(req, httpParams.ContentType)
	result, err := httpClient.Do(req)
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(result.Body)
	result.Body.Close()
	if result.StatusCode != 200 {
		return nil, errors.New(http.StatusText(result.StatusCode))
	}
	return body, nil
}

// Sets the custom headers required for the Wit.ai API
//
//		setHeaders(req, httpParams.ContentType)
func setHeaders(req *http.Request, contentType string) {
	req.Header.Add("Authorization", "Bearer "+ApiKey)
	req.Header.Set("Content-Type", contentType)
	req.Header.Set("Accept", "application/json")
}
<|endoftext|>


<commit_before>package whois

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"syscall"
	"time"
)

const (
	// DefaultTimeout sets the maximum lifetime of whois requests.
	DefaultTimeout = 30 * time.Second

	// DefaultReadLimit sets the maximum bytes a client will attempt to read from a connection.
	DefaultReadLimit = 1 << 20 // 1 MB
)

// Client represents a whois client. It contains an http.Client, for executing
// some whois Requests.
type Client struct {
	HTTPClient *http.Client
	timeout    time.Duration
}

// DefaultClient represents a shared whois client with a default timeout, HTTP
// transport, and dialer.
var DefaultClient = NewClient(DefaultTimeout)

// NewClient creates and initializes a new Client with the specified timeout.
func NewClient(timeout time.Duration) *Client {
	client := &Client{timeout: timeout}
	transport := &http.Transport{
		Dial:                  client.Dial,
		Proxy:                 http.ProxyFromEnvironment,
		TLSHandshakeTimeout:   timeout,
		ResponseHeaderTimeout: timeout,
	}
	client.HTTPClient = &http.Client{Transport: transport}
	return client
}

// Dial implements the Dial interface, strictly enforcing that cumulative dial +
// read time is limited to timeout. It applies to both whois and HTTP connections.
func (c *Client) Dial(network, address string) (net.Conn, error) {
	deadline := time.Now().Add(c.timeout)
	conn, err := net.DialTimeout(network, address, c.timeout)
	if err != nil {
		return nil, err
	}
	conn.SetDeadline(deadline)
	return conn, nil
}

// Fetch sends the Request to a whois server.
func (c *Client) Fetch(req *Request) (*Response, error) {
	if req.URL != "" {
		return c.fetchHTTP(req)
	}
	return c.fetchWhois(req)
}

func (c *Client) fetchWhois(req *Request) (*Response, error) {
	conn, err := c.Dial("tcp", req.Host+":43")
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	if _, err = conn.Write(req.Body); err != nil {
		logError(err)
		return nil, err
	}
	res := NewResponse(req.Query, req.Host)
	if res.Body, err = ioutil.ReadAll(io.LimitReader(conn, DefaultReadLimit)); err != nil {
		logError(err)
		return nil, err
	}
	res.DetectContentType("")
	return res, nil
}

func (c *Client) fetchHTTP(req *Request) (*Response, error) {
	hreq, err := httpRequest(req)
	if err != nil {
		return nil, err
	}
	hres, err := c.HTTPClient.Do(hreq)
	if err != nil {
		return nil, err
	}
	res := NewResponse(req.Query, req.Host)
	if res.Body, err = ioutil.ReadAll(io.LimitReader(hres.Body, DefaultReadLimit)); err != nil {
		logError(err)
		return nil, err
	}
	res.DetectContentType(hres.Header.Get("Content-Type"))
	return res, nil
}

func httpRequest(req *Request) (*http.Request, error) {
	var hreq *http.Request
	var err error
	// POST if non-zero Request.Body
	if len(req.Body) > 0 {
		hreq, err = http.NewRequest("POST", req.URL, bytes.NewReader(req.Body))
	} else {
		hreq, err = http.NewRequest("GET", req.URL, nil)
	}
	if err != nil {
		return nil, err
	}
	// Some web whois servers require a Referer header
	hreq.Header.Add("Referer", req.URL)
	return hreq, nil
}

func logError(err error) {
	switch t := err.(type) {
	case syscall.Errno:
		fmt.Fprintf(os.Stderr, "syscall.Errno %d: %s\n", t, err.Error())
	case net.Error:
		fmt.Fprintf(os.Stderr, "net.Error timeout=%t, temp=%t: %s\n", t.Timeout(), t.Temporary(), err.Error())
	default:
		fmt.Fprintf(os.Stderr, "Unknown error %v: %s\n", t, err.Error())
	}
}
<commit_msg>whois: expose Client.Dial and HTTPClient to allow proxying or other modification of network requests<commit_after>package whois

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"syscall"
	"time"
)

const (
	// DefaultTimeout sets the maximum lifetime of whois requests.
	DefaultTimeout = 30 * time.Second

	// DefaultReadLimit sets the maximum bytes a client will attempt to read from a connection.
	DefaultReadLimit = 1 << 20 // 1 MB
)

// Client represents a whois client. It contains an http.Client, for executing
// some whois Requests.
type Client struct {
	Dial       func(string, string) (net.Conn, error)
	HTTPClient *http.Client
}

// DefaultClient represents a shared whois client with a default timeout, HTTP
// transport, and dialer.
var DefaultClient = NewClient(DefaultTimeout)

// NewClient creates and initializes a new Client with the specified timeout.
func NewClient(timeout time.Duration) *Client {
	dial := func(network, address string) (net.Conn, error) {
		deadline := time.Now().Add(timeout)
		conn, err := net.DialTimeout(network, address, timeout)
		if err != nil {
			return nil, err
		}
		conn.SetDeadline(deadline)
		return conn, nil
	}
	c := &Client{
		Dial:       dial,
		HTTPClient: &http.Client{},
	}
	c.HTTPClient.Transport = &http.Transport{
		Dial:                  c.dial,
		Proxy:                 http.ProxyFromEnvironment,
		TLSHandshakeTimeout:   timeout,
		ResponseHeaderTimeout: timeout,
	}
	return c
}

func (c *Client) dial(network, address string) (net.Conn, error) {
	return c.Dial(network, address)
}

// Fetch sends the Request to a whois server.
func (c *Client) Fetch(req *Request) (*Response, error) {
	if req.URL != "" {
		return c.fetchHTTP(req)
	}
	return c.fetchWhois(req)
}

func (c *Client) fetchWhois(req *Request) (*Response, error) {
	conn, err := c.Dial("tcp", req.Host+":43")
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	if _, err = conn.Write(req.Body); err != nil {
		logError(err)
		return nil, err
	}
	res := NewResponse(req.Query, req.Host)
	if res.Body, err = ioutil.ReadAll(io.LimitReader(conn, DefaultReadLimit)); err != nil {
		logError(err)
		return nil, err
	}
	res.DetectContentType("")
	return res, nil
}

func (c *Client) fetchHTTP(req *Request) (*Response, error) {
	hreq, err := httpRequest(req)
	if err != nil {
		return nil, err
	}
	hres, err := c.HTTPClient.Do(hreq)
	if err != nil {
		return nil, err
	}
	res := NewResponse(req.Query, req.Host)
	if res.Body, err = ioutil.ReadAll(io.LimitReader(hres.Body, DefaultReadLimit)); err != nil {
		logError(err)
		return nil, err
	}
	res.DetectContentType(hres.Header.Get("Content-Type"))
	return res, nil
}

func httpRequest(req *Request) (*http.Request, error) {
	var hreq *http.Request
	var err error
	// POST if non-zero Request.Body
	if len(req.Body) > 0 {
		hreq, err = http.NewRequest("POST", req.URL, bytes.NewReader(req.Body))
	} else {
		hreq, err = http.NewRequest("GET", req.URL, nil)
	}
	if err != nil {
		return nil, err
	}
	// Some web whois servers require a Referer header
	hreq.Header.Add("Referer", req.URL)
	return hreq, nil
}

func logError(err error) {
	switch t := err.(type) {
	case syscall.Errno:
		fmt.Fprintf(os.Stderr, "syscall.Errno %d: %s\n", t, err.Error())
	case net.Error:
		fmt.Fprintf(os.Stderr, "net.Error timeout=%t, temp=%t: %s\n", t.Timeout(), t.Temporary(), err.Error())
	default:
		fmt.Fprintf(os.Stderr, "Unknown error %v: %s\n", t, err.Error())
	}
}
<|endoftext|>


<commit_before>package updown

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
)

const (
	libraryVersion = "0.0.1"
	defaultBaseURL = "https://updown.io/api/"
	userAgent      = "Go Updown v" + libraryVersion
	mediaType      = "application/json"
)

// An ErrorResponse reports the error caused by an API request
type ErrorResponse struct {
	// HTTP response that caused this error
	Response *http.Response

	// Error message
	Message string
}

func (r *ErrorResponse) Error() string {
	return fmt.Sprintf("%v %v: %d %v",
		r.Response.Request.Method, r.Response.Request.URL, r.Response.StatusCode, r.Message)
}

// Client manages communication the API
type Client struct {
	// HTTP client used to communicate with the API
	client *http.Client

	// Base URL for API requests
	BaseURL *url.URL

	// User agent for client
	UserAgent string

	// APIKey to use for the API
	APIKey string

	// Services used for communications with the API
	Check    CheckService
	Downtime DowntimeService
	Metric   MetricService
}

// NewClient returns a new DigitalOcean API client.
func NewClient(apiKey string, httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}

	baseURL, _ := url.Parse(defaultBaseURL)

	c := &Client{
		client:    httpClient,
		BaseURL:   baseURL,
		UserAgent: userAgent,
		APIKey:    apiKey,
	}
	c.Check = CheckService{client: c, cache: NewMemoryCache()}
	c.Downtime = DowntimeService{client: c}
	c.Metric = MetricService{client: c}

	return c
}

// NewRequest creates an API request. A relative URL can be provided in urlStr, which will be resolved to the
// BaseURL of the Client. Relative URLS should always be specified without a preceding slash.
// If specified, the value pointed to by body is JSON encoded and included in as the request body.
func (c *Client) NewRequest(method, urlStr string, body interface{}) (*http.Request, error) {
	rel, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}

	u := c.BaseURL.ResolveReference(rel)

	buf := new(bytes.Buffer)
	if body != nil {
		err := json.NewEncoder(buf).Encode(body)
		if err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", mediaType)
	req.Header.Add("Accept", mediaType)
	req.Header.Add("User-Agent", userAgent)
	req.Header.Add("X-API-KEY", c.APIKey)
	return req, nil
}

// Do sends an API request and returns the API response. The API response is JSON decoded and stored in the value
// pointed to by v, or returned as an error if an API error has occurred. If v implements the io.Writer interface,
// the raw response will be written to v, without attempting to decode it.
func (c *Client) Do(req *http.Request, v interface{}) (*http.Response, error) {
	response, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer func() {
		if rerr := response.Body.Close(); err == nil {
			err = rerr
		}
	}()

	err = CheckResponse(response)
	if err != nil {
		return response, err
	}

	if v != nil {
		if w, ok := v.(io.Writer); ok {
			_, err := io.Copy(w, response.Body)
			if err != nil {
				return nil, err
			}
		} else {
			err := json.NewDecoder(response.Body).Decode(v)
			if err != nil {
				return nil, err
			}
		}
	}

	return response, err
}

// CheckResponse checks the API response for errors, and returns them if present. A response is considered an
// error if it has a status code outside the 200 range. API error responses are expected to have either no response
// body, or a JSON response body that maps to ErrorResponse. Any other response body will be silently ignored.
func CheckResponse(r *http.Response) error {
	if c := r.StatusCode; c >= 200 && c <= 299 {
		return nil
	}

	errorResponse := &ErrorResponse{Response: r}
	data, err := ioutil.ReadAll(r.Body)
	if err == nil && len(data) > 0 {
		err := json.Unmarshal(data, errorResponse)
		if err != nil {
			return err
		}
	}

	return errorResponse
}
<commit_msg>Release 0.1<commit_after>package updown

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
)

const (
	libraryVersion = "0.1"
	defaultBaseURL = "https://updown.io/api/"
	userAgent      = "Go Updown v" + libraryVersion
	mediaType      = "application/json"
)

// An ErrorResponse reports the error caused by an API request
type ErrorResponse struct {
	// HTTP response that caused this error
	Response *http.Response

	// Error message
	Message string
}

func (r *ErrorResponse) Error() string {
	return fmt.Sprintf("%v %v: %d %v",
		r.Response.Request.Method, r.Response.Request.URL, r.Response.StatusCode, r.Message)
}

// Client manages communication the API
type Client struct {
	// HTTP client used to communicate with the API
	client *http.Client

	// Base URL for API requests
	BaseURL *url.URL

	// User agent for client
	UserAgent string

	// APIKey to use for the API
	APIKey string

	// Services used for communications with the API
	Check    CheckService
	Downtime DowntimeService
	Metric   MetricService
}

// NewClient returns a new DigitalOcean API client.
func NewClient(apiKey string, httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}

	baseURL, _ := url.Parse(defaultBaseURL)

	c := &Client{
		client:    httpClient,
		BaseURL:   baseURL,
		UserAgent: userAgent,
		APIKey:    apiKey,
	}
	c.Check = CheckService{client: c, cache: NewMemoryCache()}
	c.Downtime = DowntimeService{client: c}
	c.Metric = MetricService{client: c}

	return c
}

// NewRequest creates an API request. A relative URL can be provided in urlStr, which will be resolved to the
// BaseURL of the Client. Relative URLS should always be specified without a preceding slash.
// If specified, the value pointed to by body is JSON encoded and included in as the request body.
func (c *Client) NewRequest(method, urlStr string, body interface{}) (*http.Request, error) {
	rel, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}

	u := c.BaseURL.ResolveReference(rel)

	buf := new(bytes.Buffer)
	if body != nil {
		err := json.NewEncoder(buf).Encode(body)
		if err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", mediaType)
	req.Header.Add("Accept", mediaType)
	req.Header.Add("User-Agent", userAgent)
	req.Header.Add("X-API-KEY", c.APIKey)
	return req, nil
}

// Do sends an API request and returns the API response. The API response is JSON decoded and stored in the value
// pointed to by v, or returned as an error if an API error has occurred. If v implements the io.Writer interface,
// the raw response will be written to v, without attempting to decode it.
func (c *Client) Do(req *http.Request, v interface{}) (*http.Response, error) {
	response, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer func() {
		if rerr := response.Body.Close(); err == nil {
			err = rerr
		}
	}()

	err = CheckResponse(response)
	if err != nil {
		return response, err
	}

	if v != nil {
		if w, ok := v.(io.Writer); ok {
			_, err := io.Copy(w, response.Body)
			if err != nil {
				return nil, err
			}
		} else {
			err := json.NewDecoder(response.Body).Decode(v)
			if err != nil {
				return nil, err
			}
		}
	}

	return response, err
}

// CheckResponse checks the API response for errors, and returns them if present. A response is considered an
// error if it has a status code outside the 200 range. API error responses are expected to have either no response
// body, or a JSON response body that maps to ErrorResponse. Any other response body will be silently ignored.
func CheckResponse(r *http.Response) error {
	if c := r.StatusCode; c >= 200 && c <= 299 {
		return nil
	}

	errorResponse := &ErrorResponse{Response: r}
	data, err := ioutil.ReadAll(r.Body)
	if err == nil && len(data) > 0 {
		err := json.Unmarshal(data, errorResponse)
		if err != nil {
			return err
		}
	}

	return errorResponse
}
<|endoftext|>


<commit_before><commit_msg>Send 404 if no mappings found in config.<commit_after><|endoftext|>


<commit_before><commit_msg>Fix Home variable not being initialized<commit_after><|endoftext|>


<commit_before>package stats

import (
	"fmt"
	"net"
	"os"
	"regexp"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/ehazlett/interlock"
	"github.com/ehazlett/interlock/plugins"
	"github.com/samalba/dockerclient"
)

const (
	defaultImageNameRegex = ".*"
)

var (
	errorChan = make(chan error)
)

type StatsPlugin struct {
	interlockConfig *interlock.Config
	pluginConfig    *PluginConfig
	client          *dockerclient.DockerClient
}

func init() {
	plugins.Register(
		pluginInfo.Name,
		&plugins.RegisteredPlugin{
			New: NewPlugin,
			Info: func() *interlock.PluginInfo {
				return pluginInfo
			},
		})
}

func loadPluginConfig() (*PluginConfig, error) {
	defaultImageNameFilter := regexp.MustCompile(defaultImageNameRegex)

	cfg := &PluginConfig{
		CarbonAddress:   "",
		StatsPrefix:     "docker.stats",
		ImageNameFilter: defaultImageNameFilter,
	}

	// load custom config via environment
	carbonAddress := os.Getenv("STATS_CARBON_ADDRESS")
	if carbonAddress != "" {
		cfg.CarbonAddress = carbonAddress
	}

	statsPrefix := os.Getenv("STATS_PREFIX")
	if statsPrefix != "" {
		cfg.StatsPrefix = statsPrefix
	}

	imageNameFilter := os.Getenv("STATS_IMAGE_NAME_FILTER")
	if imageNameFilter != "" {
		// validate regex
		r, err := regexp.Compile(imageNameFilter)
		if err != nil {
			return nil, err
		}
		cfg.ImageNameFilter = r
	}

	return cfg, nil
}

func NewPlugin(interlockConfig *interlock.Config, client *dockerclient.DockerClient) (interlock.Plugin, error) {
	p := StatsPlugin{interlockConfig: interlockConfig, client: client}
	cfg, err := loadPluginConfig()
	if err != nil {
		return nil, err
	}
	p.pluginConfig = cfg
	return p, nil
}

func (p StatsPlugin) handleStats(id string, cb dockerclient.StatCallback, ec chan error, args ...interface{}) {
	go p.client.StartMonitorStats(id, cb, ec, args)
}

func (p StatsPlugin) Info() *interlock.PluginInfo {
	return pluginInfo
}

func (p StatsPlugin) sendStat(path string, value interface{}, t *time.Time) error {
	conn, err := net.Dial("tcp", p.pluginConfig.CarbonAddress)
	if err != nil {
		return err
	}
	defer conn.Close()

	timestamp := t.Unix()
	v := fmt.Sprintf("%s %v %d", path, value, timestamp)
	plugins.Log(pluginInfo.Name, log.DebugLevel,
		fmt.Sprintf("sending to carbon: %v", v),
	)
	fmt.Fprintf(conn, "%s\n", v)

	return nil
}

func (p StatsPlugin) sendEventStats(id string, stats *dockerclient.Stats, ec chan error, args ...interface{}) {
	if len(id) > 12 {
		id = id[:12]
	}
	statBasePath := p.pluginConfig.StatsPrefix + ".containers." + id
	type containerStat struct {
		Key   string
		Value interface{}
	}
	statData := []containerStat{
		{
			Key:   "cpu.usage.totalUsage",
			Value: stats.CpuStats.CpuUsage.TotalUsage,
		},
		{
			Key:   "cpu.usage.usageInKernelmode",
			Value: stats.CpuStats.CpuUsage.UsageInKernelmode,
		},
		{
			Key:   "cpu.usage.usageInUsermode",
			Value: stats.CpuStats.CpuUsage.UsageInUsermode,
		},
		{
			Key:   "memory.stats.usage",
			Value: stats.MemoryStats.Usage,
		},
		{
			Key:   "memory.stats.maxUsage",
			Value: stats.MemoryStats.MaxUsage,
		},
		{
			Key:   "memory.stats.failcnt",
			Value: stats.MemoryStats.Failcnt,
		},
		{
			Key:   "memory.stats.limit",
			Value: stats.MemoryStats.Limit,
		},
		{
			Key:   "network.rxBytes",
			Value: stats.Network.RxBytes,
		},
		{
			Key:   "network.rxPackets",
			Value: stats.Network.RxPackets,
		},
		{
			Key:   "network.rxErrors",
			Value: stats.Network.RxErrors,
		},
		{
			Key:   "network.rxDropped",
			Value: stats.Network.RxDropped,
		},
		{
			Key:   "network.txBytes",
			Value: stats.Network.TxBytes,
		},
		{
			Key:   "network.txPackets",
			Value: stats.Network.TxPackets,
		},
		{
			Key:   "network.txErrors",
			Value: stats.Network.TxErrors,
		},
		{
			Key:   "network.txDropped",
			Value: stats.Network.TxDropped,
		},
	}

	timestamp := time.Now()
	for _, s := range statData {
		plugins.Log(pluginInfo.Name,
			log.DebugLevel,
			fmt.Sprintf("stat t=%d id=%s key=%s value=%v",
				timestamp,
				id,
				s.Key,
				s.Value,
			),
		)
		m := fmt.Sprintf("%s.%s", statBasePath, s.Key)
		plugins.Log(pluginInfo.Name, log.DebugLevel, m)
		if err := p.sendStat(m, s.Value, &timestamp); err != nil {
			ec <- err
		}
	}

	return
}

func (p StatsPlugin) HandleEvent(event *dockerclient.Event) error {
	t := time.Now()
	if err := p.sendStat(p.pluginConfig.StatsPrefix+".all.events", 1, &t); err != nil {
		plugins.Log(pluginInfo.Name, log.ErrorLevel, err.Error())
	}
	if event.Status == "start" {
		// get container info for event
		c, err := p.client.InspectContainer(event.Id)
		if err != nil {
			return err
		}

		fmt.Println(c.Config.Image)
		// match regex to start monitoring
		if p.pluginConfig.ImageNameFilter.MatchString(c.Config.Image) {
			plugins.Log(pluginInfo.Name, log.DebugLevel,
				fmt.Sprintf("gathering stats: image=%s id=%s", c.Image, c.Id[:12]))
			go p.handleStats(event.Id, p.sendEventStats, errorChan, nil)
		}
	}
	return nil
}
<commit_msg>stats: start stats for containers on launch<commit_after>package stats

import (
	"fmt"
	"net"
	"os"
	"regexp"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/ehazlett/interlock"
	"github.com/ehazlett/interlock/plugins"
	"github.com/samalba/dockerclient"
)

const (
	defaultImageNameRegex = ".*"
)

var (
	errorChan = make(chan error)
)

type StatsPlugin struct {
	interlockConfig *interlock.Config
	pluginConfig    *PluginConfig
	client          *dockerclient.DockerClient
}

func init() {
	plugins.Register(
		pluginInfo.Name,
		&plugins.RegisteredPlugin{
			New: NewPlugin,
			Info: func() *interlock.PluginInfo {
				return pluginInfo
			},
		})
}

func loadPluginConfig() (*PluginConfig, error) {
	defaultImageNameFilter := regexp.MustCompile(defaultImageNameRegex)

	cfg := &PluginConfig{
		CarbonAddress:   "",
		StatsPrefix:     "docker.stats",
		ImageNameFilter: defaultImageNameFilter,
	}

	// load custom config via environment
	carbonAddress := os.Getenv("STATS_CARBON_ADDRESS")
	if carbonAddress != "" {
		cfg.CarbonAddress = carbonAddress
	}

	statsPrefix := os.Getenv("STATS_PREFIX")
	if statsPrefix != "" {
		cfg.StatsPrefix = statsPrefix
	}

	imageNameFilter := os.Getenv("STATS_IMAGE_NAME_FILTER")
	if imageNameFilter != "" {
		// validate regex
		r, err := regexp.Compile(imageNameFilter)
		if err != nil {
			return nil, err
		}
		cfg.ImageNameFilter = r
	}

	return cfg, nil
}

func NewPlugin(interlockConfig *interlock.Config, client *dockerclient.DockerClient) (interlock.Plugin, error) {
	p := StatsPlugin{interlockConfig: interlockConfig, client: client}
	cfg, err := loadPluginConfig()
	if err != nil {
		return nil, err
	}
	p.pluginConfig = cfg

	// check all containers to see if stats are needed
	if err := p.initialize(); err != nil {
		return nil, err
	}

	// handle errorChan
	go func() {
		for {
			err := <-errorChan
			plugins.Log(pluginInfo.Name,
				log.ErrorLevel,
				err.Error(),
			)
		}
	}()

	return p, nil
}

func (p StatsPlugin) initialize() error {
	containers, err := p.client.ListContainers(false, false, "")
	if err != nil {
		return err
	}

	for _, c := range containers {
		if err := p.startStats(c.Id); err != nil {
			errorChan <- err
		}
	}

	return nil
}

func (p StatsPlugin) handleStats(id string, cb dockerclient.StatCallback, ec chan error, args ...interface{}) {
	go p.client.StartMonitorStats(id, cb, ec, args)
}

func (p StatsPlugin) Info() *interlock.PluginInfo {
	return pluginInfo
}

func (p StatsPlugin) sendStat(path string, value interface{}, t *time.Time) error {
	conn, err := net.Dial("tcp", p.pluginConfig.CarbonAddress)
	if err != nil {
		return err
	}
	defer conn.Close()

	timestamp := t.Unix()
	v := fmt.Sprintf("%s %v %d", path, value, timestamp)
	plugins.Log(pluginInfo.Name, log.DebugLevel,
		fmt.Sprintf("sending to carbon: %v", v),
	)
	fmt.Fprintf(conn, "%s\n", v)

	return nil
}

func (p StatsPlugin) sendEventStats(id string, stats *dockerclient.Stats, ec chan error, args ...interface{}) {
	if len(id) > 12 {
		id = id[:12]
	}
	statBasePath := p.pluginConfig.StatsPrefix + ".containers." + id
	type containerStat struct {
		Key   string
		Value interface{}
	}
	statData := []containerStat{
		{
			Key:   "cpu.usage.totalUsage",
			Value: stats.CpuStats.CpuUsage.TotalUsage,
		},
		{
			Key:   "cpu.usage.usageInKernelmode",
			Value: stats.CpuStats.CpuUsage.UsageInKernelmode,
		},
		{
			Key:   "cpu.usage.usageInUsermode",
			Value: stats.CpuStats.CpuUsage.UsageInUsermode,
		},
		{
			Key:   "memory.stats.usage",
			Value: stats.MemoryStats.Usage,
		},
		{
			Key:   "memory.stats.maxUsage",
			Value: stats.MemoryStats.MaxUsage,
		},
		{
			Key:   "memory.stats.failcnt",
			Value: stats.MemoryStats.Failcnt,
		},
		{
			Key:   "memory.stats.limit",
			Value: stats.MemoryStats.Limit,
		},
		{
			Key:   "network.rxBytes",
			Value: stats.Network.RxBytes,
		},
		{
			Key:   "network.rxPackets",
			Value: stats.Network.RxPackets,
		},
		{
			Key:   "network.rxErrors",
			Value: stats.Network.RxErrors,
		},
		{
			Key:   "network.rxDropped",
			Value: stats.Network.RxDropped,
		},
		{
			Key:   "network.txBytes",
			Value: stats.Network.TxBytes,
		},
		{
			Key:   "network.txPackets",
			Value: stats.Network.TxPackets,
		},
		{
			Key:   "network.txErrors",
			Value: stats.Network.TxErrors,
		},
		{
			Key:   "network.txDropped",
			Value: stats.Network.TxDropped,
		},
	}

	timestamp := time.Now()
	for _, s := range statData {
		plugins.Log(pluginInfo.Name,
			log.DebugLevel,
			fmt.Sprintf("stat t=%d id=%s key=%s value=%v",
				timestamp.UnixNano(),
				id,
				s.Key,
				s.Value,
			),
		)
		m := fmt.Sprintf("%s.%s", statBasePath, s.Key)
		if err := p.sendStat(m, s.Value, &timestamp); err != nil {
			ec <- err
		}
	}

	return
}

func (p StatsPlugin) startStats(id string) error {
	// get container info for event
	c, err := p.client.InspectContainer(id)
	if err != nil {
		return err
	}
	// match regex to start monitoring
	if p.pluginConfig.ImageNameFilter.MatchString(c.Config.Image) {
		plugins.Log(pluginInfo.Name, log.DebugLevel,
			fmt.Sprintf("gathering stats: image=%s id=%s", c.Image, c.Id[:12]))
		go p.handleStats(id, p.sendEventStats, errorChan, nil)
	}

	return nil
}

func (p StatsPlugin) HandleEvent(event *dockerclient.Event) error {
	t := time.Now()
	if err := p.sendStat(p.pluginConfig.StatsPrefix+".all.events", 1, &t); err != nil {
		plugins.Log(pluginInfo.Name, log.ErrorLevel, err.Error())
	}
	if event.Status == "start" {
		if err := p.startStats(event.Id); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package phases

import (
	"fmt"

	"github.com/pkg/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/klog"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
	patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
	"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
	"k8s.io/kubernetes/pkg/util/normalizer"
)

var (
	uploadKubeadmConfigLongDesc = fmt.Sprintf(normalizer.LongDesc(`
		Uploads the kubeadm ClusterConfiguration to a ConfigMap called %s in the %s namespace.
		This enables correct configuration of system components and a seamless user experience when upgrading.

		Alternatively, you can use kubeadm config.
		`), kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem)

	uploadKubeadmConfigExample = normalizer.Examples(`
		# uploads the configuration of your cluster
		kubeadm alpha phase upload-config --config=myConfig.yaml
		`)

	uploadKubeletConfigLongDesc = normalizer.LongDesc(`
		Uploads kubelet configuration extracted from the kubeadm InitConfiguration object to a ConfigMap
		of the form kubelet-config-1.X in the cluster, where X is the minor version of the current (API Server) Kubernetes version.
		`)

	uploadKubeletConfigExample = normalizer.Examples(`
		# Uploads the kubelet configuration from the kubeadm Config file to a ConfigMap in the cluster.
		kubeadm init phase upload-config kubelet --config kubeadm.yaml
		`)
)

type uploadConfigData interface {
	Cfg() *kubeadmapi.InitConfiguration
	Client() (clientset.Interface, error)
}

// NewUploadConfigPhase returns the phase to uploadConfig
func NewUploadConfigPhase() workflow.Phase {
	return workflow.Phase{
		Name:    "upload-config",
		Aliases: []string{"uploadconfig"},
		Short:   "Uploads the kubeadm and kubelet configuration to a ConfigMap",
		Long:    cmdutil.MacroCommandLongDescription,
		Phases: []workflow.Phase{
			{
				Name:     "kubeadm",
				Short:    "Uploads the kubeadm ClusterConfiguration to a ConfigMap",
				Long:     uploadKubeadmConfigLongDesc,
				Example:  uploadKubeadmConfigExample,
				Run:      runUploadKubeadmConfig,
				CmdFlags: getUploadConfigPhaseFlags(),
			},
			{
				Name:     "kubelet",
				Short:    "Uploads the kubelet component config to a ConfigMap",
				Long:     uploadKubeletConfigLongDesc,
				Example:  uploadKubeletConfigExample,
				Run:      runUploadKubeletConfig,
				CmdFlags: getUploadConfigPhaseFlags(),
			},
		},
	}
}

func getUploadConfigPhaseFlags() []string {
	return []string{
		options.CfgPath,
		options.KubeconfigPath,
	}
}

// runUploadKubeadmConfig uploads the kubeadm configuration to a ConfigMap
func runUploadKubeadmConfig(c workflow.RunData) error {
	cfg, client, err := getUploadConfigData(c)
	if err != nil {
		return err
	}

	klog.V(1).Infof("[upload-config] Uploading the kubeadm ClusterConfiguration to a ConfigMap")
	if err := uploadconfig.UploadConfiguration(cfg, client); err != nil {
		return errors.Wrap(err, "error uploading the kubeadm ClusterConfiguration")
	}
	return nil
}

// runUploadKubeletConfig uploads the kubelet configuration to a ConfigMap
func runUploadKubeletConfig(c workflow.RunData) error {
	cfg, client, err := getUploadConfigData(c)
	if err != nil {
		return err
	}

	klog.V(1).Infof("[upload-config] Uploading the kubelet component config to a ConfigMap")
	if err = kubeletphase.CreateConfigMap(cfg, client); err != nil {
		return errors.Wrap(err, "error creating kubelet configuration ConfigMap")
	}

	klog.V(1).Infof("[upload-config] Preserving the CRISocket information for the control-plane node")
	if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil {
		return errors.Wrap(err, "Error writing Crisocket information for the control-plane node")
	}
	return nil
}

func getUploadConfigData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, error) {
	data, ok := c.(uploadConfigData)
	if !ok {
		return nil, nil, errors.New("upload-config phase invoked with an invalid data struct")
	}
	cfg := data.Cfg()
	client, err := data.Client()
	if err != nil {
		return nil, nil, err
	}
	return cfg, client, err
}
<commit_msg>Fix kubeadm upload-config kubeadm example<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package phases

import (
	"fmt"

	"github.com/pkg/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/klog"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
	patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
	"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
	"k8s.io/kubernetes/pkg/util/normalizer"
)

var (
	uploadKubeadmConfigLongDesc = fmt.Sprintf(normalizer.LongDesc(`
		Uploads the kubeadm ClusterConfiguration to a ConfigMap called %s in the %s namespace.
		This enables correct configuration of system components and a seamless user experience when upgrading.

		Alternatively, you can use kubeadm config.
		`), kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem)

	uploadKubeadmConfigExample = normalizer.Examples(`
		# uploads the configuration of your cluster
		kubeadm init phase upload-config --config=myConfig.yaml
		`)

	uploadKubeletConfigLongDesc = normalizer.LongDesc(`
		Uploads kubelet configuration extracted from the kubeadm InitConfiguration object to a ConfigMap
		of the form kubelet-config-1.X in the cluster, where X is the minor version of the current (API Server) Kubernetes version.
		`)

	uploadKubeletConfigExample = normalizer.Examples(`
		# Uploads the kubelet configuration from the kubeadm Config file to a ConfigMap in the cluster.
		kubeadm init phase upload-config kubelet --config kubeadm.yaml
		`)
)

type uploadConfigData interface {
	Cfg() *kubeadmapi.InitConfiguration
	Client() (clientset.Interface, error)
}

// NewUploadConfigPhase returns the phase to uploadConfig
func NewUploadConfigPhase() workflow.Phase {
	return workflow.Phase{
		Name:    "upload-config",
		Aliases: []string{"uploadconfig"},
		Short:   "Uploads the kubeadm and kubelet configuration to a ConfigMap",
		Long:    cmdutil.MacroCommandLongDescription,
		Phases: []workflow.Phase{
			{
				Name:     "kubeadm",
				Short:    "Uploads the kubeadm ClusterConfiguration to a ConfigMap",
				Long:     uploadKubeadmConfigLongDesc,
				Example:  uploadKubeadmConfigExample,
				Run:      runUploadKubeadmConfig,
				CmdFlags: getUploadConfigPhaseFlags(),
			},
			{
				Name:     "kubelet",
				Short:    "Uploads the kubelet component config to a ConfigMap",
				Long:     uploadKubeletConfigLongDesc,
				Example:  uploadKubeletConfigExample,
				Run:      runUploadKubeletConfig,
				CmdFlags: getUploadConfigPhaseFlags(),
			},
		},
	}
}

func getUploadConfigPhaseFlags() []string {
	return []string{
		options.CfgPath,
		options.KubeconfigPath,
	}
}

// runUploadKubeadmConfig uploads the kubeadm configuration to a ConfigMap
func runUploadKubeadmConfig(c workflow.RunData) error {
	cfg, client, err := getUploadConfigData(c)
	if err != nil {
		return err
	}

	klog.V(1).Infof("[upload-config] Uploading the kubeadm ClusterConfiguration to a ConfigMap")
	if err := uploadconfig.UploadConfiguration(cfg, client); err != nil {
		return errors.Wrap(err, "error uploading the kubeadm ClusterConfiguration")
	}
	return nil
}

// runUploadKubeletConfig uploads the kubelet configuration to a ConfigMap
func runUploadKubeletConfig(c workflow.RunData) error {
	cfg, client, err := getUploadConfigData(c)
	if err != nil {
		return err
	}

	klog.V(1).Infof("[upload-config] Uploading the kubelet component config to a ConfigMap")
	if err = kubeletphase.CreateConfigMap(cfg, client); err != nil {
		return errors.Wrap(err, "error creating kubelet configuration ConfigMap")
	}

	klog.V(1).Infof("[upload-config] Preserving the CRISocket information for the control-plane node")
	if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil {
		return errors.Wrap(err, "Error writing Crisocket information for the control-plane node")
	}
	return nil
}

func getUploadConfigData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, error) {
	data, ok := c.(uploadConfigData)
	if !ok {
		return nil, nil, errors.New("upload-config phase invoked with an invalid data struct")
	}
	cfg := data.Cfg()
	client, err := data.Client()
	if err != nil {
		return nil, nil, err
	}
	return cfg, client, err
}
<|endoftext|>


<commit_before><commit_msg>ucdn ok<commit_after><|endoftext|>


<commit_before><commit_msg>error-out on bad juju<commit_after><|endoftext|>


<commit_before><commit_msg>emitting event from plugins<commit_after><|endoftext|>