<commit_before>package servicebroker

import (
	"cf/api"
	"cf/configuration"
	"cf/errors"
	"cf/requirements"
	"cf/terminal"
	"github.com/codegangsta/cli"
)

type DeleteServiceBroker struct {
	ui     terminal.UI
	config configuration.Reader
	repo   api.ServiceBrokerRepository
}

func NewDeleteServiceBroker(ui terminal.UI, config configuration.Reader, repo api.ServiceBrokerRepository) (cmd DeleteServiceBroker) {
	cmd.ui = ui
	cmd.config = config
	cmd.repo = repo
	return
}

func (cmd DeleteServiceBroker) GetRequirements(reqFactory requirements.Factory, c *cli.Context) (reqs []requirements.Requirement, err error) {
	if len(c.Args()) != 1 {
		err = errors.New("Incorrect Usage")
		cmd.ui.FailWithUsage(c, "delete-service-broker")
		return
	}

	reqs = append(reqs, reqFactory.NewLoginRequirement())

	return
}
func (cmd DeleteServiceBroker) Run(c *cli.Context) {
	brokerName := c.Args()[0]
	if !c.Bool("f") && !cmd.ui.ConfirmDelete("service-broker", brokerName) {
		return
	}

	cmd.ui.Say("Deleting service broker %s as %s...",
		terminal.EntityNameColor(brokerName),
		terminal.EntityNameColor(cmd.config.Username()),
	)

	broker, apiErr := cmd.repo.FindByName(brokerName)

	switch apiErr.(type) {
	case nil:
	case *errors.ModelNotFoundError:
		cmd.ui.Ok()
		cmd.ui.Warn("Service Broker %s does not exist.", brokerName)
		return
	default:
		cmd.ui.Failed(apiErr.Error())
		return
	}

	apiErr = cmd.repo.Delete(broker.Guid)
	if apiErr != nil {
		cmd.ui.Failed(apiErr.Error())
		return
	}

	cmd.ui.Ok()
	return
}
<commit_msg>Formatting<commit_after>package servicebroker

import (
	"cf/api"
	"cf/configuration"
	"cf/errors"
	"cf/requirements"
	"cf/terminal"
	"github.com/codegangsta/cli"
)

type DeleteServiceBroker struct {
	ui     terminal.UI
	config configuration.Reader
	repo   api.ServiceBrokerRepository
}

func NewDeleteServiceBroker(ui terminal.UI, config configuration.Reader, repo api.ServiceBrokerRepository) (cmd DeleteServiceBroker) {
	cmd.ui = ui
	cmd.config = config
	cmd.repo = repo
	return
}

func (cmd DeleteServiceBroker) GetRequirements(reqFactory requirements.Factory, c *cli.Context) (reqs []requirements.Requirement, err error) {
	if len(c.Args()) != 1 {
		err = errors.New("Incorrect Usage")
		cmd.ui.FailWithUsage(c, "delete-service-broker")
		return
	}

	reqs = append(reqs, reqFactory.NewLoginRequirement())
	return
}

func (cmd DeleteServiceBroker) Run(c *cli.Context) {
	brokerName := c.Args()[0]
	if !c.Bool("f") && !cmd.ui.ConfirmDelete("service-broker", brokerName) {
		return
	}

	cmd.ui.Say("Deleting service broker %s as %s...",
		terminal.EntityNameColor(brokerName),
		terminal.EntityNameColor(cmd.config.Username()),
	)

	broker, apiErr := cmd.repo.FindByName(brokerName)

	switch apiErr.(type) {
	case nil:
	case *errors.ModelNotFoundError:
		cmd.ui.Ok()
		cmd.ui.Warn("Service Broker %s does not exist.", brokerName)
		return
	default:
		cmd.ui.Failed(apiErr.Error())
		return
	}

	apiErr = cmd.repo.Delete(broker.Guid)
	if apiErr != nil {
		cmd.ui.Failed(apiErr.Error())
		return
	}

	cmd.ui.Ok()
	return
}
<|endoftext|>


<commit_before>// THIS IS A WORK IN PROGRESS
// based on http://zguide.zeromq.org/php:chapter8#Detecting-Disappearances

// The Peer will listen on gomsg.Client and call on gomsg.Server

package grapevine

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"io"
	"net"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/imdario/mergo"
	"github.com/quintans/gomsg"
	tk "github.com/quintans/toolkit"
	"github.com/quintans/toolkit/faults"
	"github.com/quintans/toolkit/log"
)

// defaults
const (
	UuidSize       = 16
	PING           = "PING"
	PeerAddressKey = "peer.address"
)

type node struct {
	uuid            string
	client          *gomsg.Client
	debouncer       *tk.Debouncer
	beaconLastTime  time.Time
	beaconCountdown int
}

type Config struct {
	Uuid                  tk.UUID
	BeaconAddr            string
	BeaconName            string
	BeaconMaxDatagramSize int
	BeaconInterval        time.Duration
	BeaconMaxInterval     time.Duration
	// BeaconCountdown is the number of consecutive UDP pings inside the ping window
	// to reactivate the UDP health check
	BeaconCountdown int
}

type Peer struct {
	sync.RWMutex
	*gomsg.Server

	cfg     Config
	tcpAddr string
	// peers that will be used to listen
	peers map[string]*node
	// the server will be used to send
	udpConn      *net.UDPConn
	handlers     map[string][]interface{}
	beaconTicker *tk.Ticker
}

func NewPeer(cfg Config) *Peer {
	if cfg.Uuid.IsZero() {
		cfg.Uuid = tk.NewUUID()
	}

	// apply defaults
	var defaultConfig = Config{
		BeaconMaxDatagramSize: 1024,
		BeaconAddr:            "224.0.0.1:9999",
		BeaconName:            "Beacon",
		BeaconInterval:        time.Second,
		BeaconMaxInterval:     time.Second * 2,
		BeaconCountdown:       3,
	}
	mergo.Merge(&cfg, defaultConfig)

	peer := &Peer{
		Server:   gomsg.NewServer(),
		cfg:      cfg,
		peers:    make(map[string]*node),
		handlers: make(map[string][]interface{}),
	}

	return peer
}

func (peer *Peer) Config() Config {
	return peer.cfg
}

func (peer *Peer) Bind(tcpAddr string) <-chan error {
	peer.tcpAddr = tcpAddr
	peer.Logger().Infof("Binding Cluster=%s, UUID=%s at %s", peer.cfg.BeaconName, peer.cfg.Uuid, tcpAddr)

	// special case where we receive a targeted request
	// when a peer tries to check if I exist
	// because it did not received the beacon in time
	peer.Server.Handle(PING, func(uuid []byte) bool {
		return bytes.Compare(uuid, peer.cfg.Uuid.Bytes()) == 0
	})

	var err = peer.serveUDP(peer.beaconHandler)
	if err != nil {
		var cherr = make(chan error, 1)
		cherr <- err
		return cherr
	}
	peer.Server.AddBindListeners(func(l net.Listener) {
		peer.startBeacon()
	})

	return peer.Server.Listen(tcpAddr)
}

func (peer *Peer) checkPeer(uuid string, addr string) {
	peer.Lock()
	var connect = false
	if n := peer.peers[uuid]; n != nil {
		if addr != n.client.Address() {
			delete(peer.peers, uuid)
			peer.Logger().Infof("%s - OLD peer at %s", peer.tcpAddr, addr)
			// client reconnected with another address
			n.client.Destroy()
			n.debouncer.Kill()
			connect = true
		} else {
			peer.checkBeacon(n)
		}
	} else {
		peer.Logger().Infof("%s - NEW peer at %s", peer.tcpAddr, addr)
		connect = true
	}
	peer.Unlock()

	if connect {
		var err = peer.connectPeer(uuid, addr)
		if err != nil {
			peer.Logger().Errorf("Unable to connect to peer at %s: %+v", addr, err)
		}
	}
}

func (peer *Peer) checkBeacon(n *node) {
	if n.beaconCountdown == 0 {
		// this debouncer is only for UDP beacon when beaconCountdown == 0
		n.debouncer.Delay(nil)
	} else {
		var now = time.Now()
		if now.Sub(n.beaconLastTime) < peer.cfg.BeaconMaxInterval {
			n.beaconCountdown--
		} else {
			n.beaconCountdown = peer.cfg.BeaconCountdown
		}
		if n.beaconCountdown == 0 {
			// the client responded, switching to UDP
			peer.Logger().Infof("%s - Peer at %s responded. Switching to UDP listening", peer.tcpAddr, n.client.Address())
			// kill the TCP health check
			n.debouncer.Kill()
			peer.healthCheckByUDP(n)
		}
		n.beaconLastTime = now
	}
}

func (peer *Peer) connectPeer(uuid string, addr string) error {
	var cli = gomsg.NewClient()
	// register peer public IP:Port
	var ip, err = gomsg.IP()
	if err != nil {
		return err
	}
	cli.Metadata()[PeerAddressKey] = ip + ":" + strconv.Itoa(peer.BindPort())

	cli.SetLogger(log.Wrap{peer.Logger().CallerAt(2), "{Client@" + addr + "}"})
	var n = &node{
		uuid:   uuid,
		client: cli,
	}
	peer.healthCheckByUDP(n)

	peer.Lock()
	peer.peers[uuid] = n
	peer.Unlock()

	// when it connects it will be already in peers
	var e = <-cli.Connect(addr)
	if e != nil {
		peer.Lock()
		delete(peer.peers, uuid)
		peer.Unlock()
		return e
	}

	peer.RLock()
	// apply all handlers
	for k, v := range peer.handlers {
		cli.Handle(k, v...)
	}
	peer.RUnlock()

	peer.listPeers(addr)

	return nil
}

func (peer *Peer) dropPeer(n *node) {
	peer.Lock()

	peer.Logger().Infof("%s - Droping unresponsive peer at %s", peer.tcpAddr, n.client.Address())
	n.client.Destroy()
	// no need to kill the debouncer. If this was called, it means it fired.
	n.debouncer = nil
	delete(peer.peers, n.uuid)
	peer.Unlock()

	peer.listPeers("")
}

func (peer *Peer) listPeers(addr string) {
	if peer.Logger().IsActive(log.INFO) {
		peer.RLock()
		var arr = make([]*node, len(peer.peers))
		var i = 0
		for _, n := range peer.peers {
			arr[i] = n
			i++
		}
		peer.RUnlock()

		sort.Slice(arr, func(i, j int) bool {
			return strings.Compare(arr[i].client.Address(), arr[j].client.Address()) < 0
		})

		var buf bytes.Buffer
		buf.WriteString(" {\n    ")
		var peerAddr string
		var ip, err = gomsg.IP()
		if err == nil {
			peerAddr = fmt.Sprintf("%s:%d", ip, peer.BindPort())
		} else {
			peerAddr = peer.BindAddress().String()
		}
		buf.WriteString(peerAddr)
		buf.WriteString(" (this)\n")
		for _, n := range arr {
			buf.WriteString("    ")
			var a = n.client.Address()
			buf.WriteString(a)
			if a == addr {
				buf.WriteString(" (NEW)")
			}
			buf.WriteString("\n")
		}
		buf.WriteString(" }\n")

		peer.Logger().Infof("Peers of %s\n%s", peer.cfg.BeaconName, buf.String())
	}
}

// healthCheckByIP is the client that checks actively the remote peer
func (peer *Peer) healthCheckByTCP(n *node) {
	var uuid = n.client.RemoteUuid().Bytes()
	var ticker = tk.NewTicker(peer.cfg.BeaconInterval, func(t time.Time) {
		<-n.client.RequestTimeout(PING, uuid, func(ok bool) {
			if ok {
				n.debouncer.Delay(nil)
			} else {
				peer.Logger().Debugf("Peer with UUID %x at %s no longer valid.", uuid, n.client.Address())
				n.debouncer.Kill()
			}
		}, peer.cfg.BeaconInterval)
	})
	n.debouncer = tk.NewDebounce(peer.cfg.BeaconMaxInterval, func(o interface{}) {
		peer.Logger().Debugf("%s - Failed to contact by TCP peer at %s.", peer.tcpAddr, n.client.Address())
		peer.dropPeer(n)
	})
	n.debouncer.OnExit = func() {
		ticker.Stop()
	}
}

func (peer *Peer) healthCheckByUDP(n *node) {
	n.debouncer = tk.NewDebounce(peer.cfg.BeaconMaxInterval, func(o interface{}) {
		// the client did not responded, switching to TCP
		peer.Logger().Debugf("%s - Silent peer at %s. Switching to TCP ping", peer.tcpAddr, n.client.Address())
		n.beaconCountdown = peer.cfg.BeaconCountdown
		peer.healthCheckByTCP(n)
	})
}

func (peer *Peer) beaconHandler(src *net.UDPAddr, n int, b []byte) {
	// starts with tag
	if bytes.HasPrefix(b, []byte(peer.cfg.BeaconName)) {
		var r = bytes.NewReader(b)
		r.Seek(int64(len(peer.cfg.BeaconName)), io.SeekStart)
		var uuid = make([]byte, UuidSize)
		r.Read(uuid)
		// ignore self
		if bytes.Compare(uuid, peer.cfg.Uuid.Bytes()) != 0 {
			var buf16 = make([]byte, 2)
			r.Read(buf16)
			var port = int(binary.LittleEndian.Uint16(buf16))
			peer.checkPeer(hex.EncodeToString(uuid), src.IP.String()+":"+strconv.Itoa(port))
		}
	}
}

func (peer *Peer) startBeacon() error {
	addr, err := net.ResolveUDPAddr("udp", peer.cfg.BeaconAddr)
	if err != nil {
		return faults.Wrap(err)
	}
	c, err := net.DialUDP("udp", nil, addr)
	if err != nil {
		return faults.Wrap(err)
	}
	var buf16 = make([]byte, 2)
	var port = uint16(peer.Server.BindPort())
	binary.LittleEndian.PutUint16(buf16, port)

	var buf bytes.Buffer
	buf.WriteString(peer.cfg.BeaconName)
	buf.Write(peer.cfg.Uuid.Bytes())
	buf.Write(buf16)

	var data = buf.Bytes()
	if peer.beaconTicker != nil {
		peer.beaconTicker.Stop()
	}
	peer.Logger().Infof("Heartbeat on %s every %s",
		peer.cfg.BeaconAddr,
		peer.cfg.BeaconInterval,
	)
	peer.beaconTicker = tk.NewDelayedTicker(0, peer.cfg.BeaconInterval, func(t time.Time) {
		var _, err = c.Write(data)
		if err != nil {
			peer.Logger().Errorf("Unable to write to UDP %s. Error: %+v", peer.cfg.BeaconAddr, faults.Wrap(err))
		}
	})

	return nil
}

func (peer *Peer) serveUDP(hnd func(*net.UDPAddr, int, []byte)) error {
	addr, err := net.ResolveUDPAddr("udp", peer.cfg.BeaconAddr)
	if err != nil {
		return faults.Wrap(err)
	}

	l, err := net.ListenUDP("udp", addr)
	if err != nil {
		return faults.Wrap(err)
	}
	l.SetReadBuffer(peer.cfg.BeaconMaxDatagramSize)
	peer.udpConn = l
	go func() {
		var payload = make([]byte, peer.cfg.BeaconMaxDatagramSize)
		for {
			n, src, err := l.ReadFromUDP(payload)
			if peer.udpConn == nil {
				return
			} else if err != nil {
				peer.Logger().Errorf("%s - ReadFromUDP failed: %+v", peer.tcpAddr, faults.Wrap(err))
				return
			}
			hnd(src, n, payload)
		}
	}()
	return nil
}

func (peer *Peer) Handle(name string, hnd ...interface{}) {
	peer.Logger().Infof("Registering handler for %s", name)

	peer.Lock()
	peer.handlers[name] = hnd
	for _, v := range peer.peers {
		v.client.Handle(name, hnd)
	}
	peer.Unlock()
}

func (peer *Peer) Cancel(name string) {
	peer.Logger().Infof("Canceling handler for %s", name)

	peer.Lock()
	delete(peer.handlers, name)
	for _, v := range peer.peers {
		v.client.Cancel(name)
	}
	peer.Unlock()
}

func (peer *Peer) Destroy() {
	peer.Server.Destroy()
	var conn = peer.udpConn
	peer.udpConn = nil
	if conn != nil {
		conn.Close()
	}
	peer.Lock()
	for _, v := range peer.peers {
		v.debouncer.Kill()
		v.client.Destroy()
	}
	peer.peers = make(map[string]*node)
	peer.Unlock()

	if peer.beaconTicker != nil {
		peer.beaconTicker.Stop()
	}
	peer.beaconTicker = nil
}
<commit_msg>added peers listeners<commit_after>// THIS IS A WORK IN PROGRESS
// based on http://zguide.zeromq.org/php:chapter8#Detecting-Disappearances

// The Peer will listen on gomsg.Client and call on gomsg.Server

package grapevine

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"io"
	"net"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/imdario/mergo"
	"github.com/quintans/gomsg"
	tk "github.com/quintans/toolkit"
	"github.com/quintans/toolkit/faults"
	"github.com/quintans/toolkit/log"
)

// defaults
const (
	UuidSize       = 16
	PING           = "PING"
	PeerAddressKey = "peer.address"
)

type PeerListener func(*gomsg.Client)

type node struct {
	uuid            string
	client          *gomsg.Client
	debouncer       *tk.Debouncer
	beaconLastTime  time.Time
	beaconCountdown int
}

type Config struct {
	Uuid                  tk.UUID
	BeaconAddr            string
	BeaconName            string
	BeaconMaxDatagramSize int
	BeaconInterval        time.Duration
	BeaconMaxInterval     time.Duration
	// BeaconCountdown is the number of consecutive UDP pings inside the ping window
	// to reactivate the UDP health check
	BeaconCountdown int
}

type Peer struct {
	sync.RWMutex
	*gomsg.Server

	cfg     Config
	tcpAddr string
	// peers that will be used to listen
	peers map[string]*node
	// the server will be used to send
	udpConn           *net.UDPConn
	handlers          map[string][]interface{}
	beaconTicker      *tk.Ticker
	newPeerListeners  map[uint64]PeerListener
	dropPeerListeners map[uint64]PeerListener
	listenersIdx      uint64
}

func NewPeer(cfg Config) *Peer {
	if cfg.Uuid.IsZero() {
		cfg.Uuid = tk.NewUUID()
	}

	// apply defaults
	var defaultConfig = Config{
		BeaconMaxDatagramSize: 1024,
		BeaconAddr:            "224.0.0.1:9999",
		BeaconName:            "Beacon",
		BeaconInterval:        time.Second,
		BeaconMaxInterval:     time.Second * 2,
		BeaconCountdown:       3,
	}
	mergo.Merge(&cfg, defaultConfig)

	peer := &Peer{
		Server: gomsg.NewServer(),
		cfg:    cfg,
	}

	peer.reset()

	return peer
}

func (this *Peer) reset() {
	this.peers = make(map[string]*node)
	this.handlers = make(map[string][]interface{})
	this.newPeerListeners = make(map[uint64]PeerListener)
	this.dropPeerListeners = make(map[uint64]PeerListener)
	this.listenersIdx = 0
}

func (this *Peer) AddNewPeerListener(listener PeerListener) uint64 {
	this.Lock()
	defer this.Unlock()

	this.listenersIdx++
	this.newPeerListeners[this.listenersIdx] = listener
	return this.listenersIdx
}

// RemoveSendListener removes a previously added listener on send messages
func (this *Peer) RemoveNewPeerListener(idx uint64) {
	this.Lock()
	delete(this.newPeerListeners, idx)
	this.Unlock()
}

func (this *Peer) fireNewPeerListener(cli *gomsg.Client) {
	for _, listener := range clonePeerListeners(this.RWMutex, this.newPeerListeners) {
		listener(cli)
	}
}

func (this *Peer) AddDropPeerListener(listener PeerListener) uint64 {
	this.Lock()
	defer this.Unlock()

	this.listenersIdx++
	this.dropPeerListeners[this.listenersIdx] = listener
	return this.listenersIdx
}

// RemoveSendListener removes a previously added listener on send messages
func (this *Peer) RemoveDropPeerListener(idx uint64) {
	this.Lock()
	defer this.Unlock()

	delete(this.dropPeerListeners, idx)
}

func (this *Peer) fireDropPeerListener(cli *gomsg.Client) {
	for _, listener := range clonePeerListeners(this.RWMutex, this.dropPeerListeners) {
		listener(cli)
	}
}

func clonePeerListeners(mu sync.RWMutex, m map[uint64]PeerListener) []PeerListener {
	mu.RLock()
	arr := make([]PeerListener, len(m))
	var i = 0
	for _, v := range m {
		arr[i] = v
		i++
	}
	mu.RUnlock()

	return arr
}

func (peer *Peer) Config() Config {
	return peer.cfg
}

func (peer *Peer) Bind(tcpAddr string) <-chan error {
	peer.tcpAddr = tcpAddr
	peer.Logger().Infof("Binding Cluster=%s, UUID=%s at %s", peer.cfg.BeaconName, peer.cfg.Uuid, tcpAddr)

	// special case where we receive a targeted request
	// when a peer tries to check if I exist
	// because it did not received the beacon in time
	peer.Server.Handle(PING, func(uuid []byte) bool {
		return bytes.Compare(uuid, peer.cfg.Uuid.Bytes()) == 0
	})

	var err = peer.serveUDP(peer.beaconHandler)
	if err != nil {
		var cherr = make(chan error, 1)
		cherr <- err
		return cherr
	}
	peer.Server.AddBindListeners(func(l net.Listener) {
		peer.startBeacon()
	})

	return peer.Server.Listen(tcpAddr)
}

func (peer *Peer) checkPeer(uuid string, addr string) {
	peer.Lock()
	var connect = false
	if n := peer.peers[uuid]; n != nil {
		if addr != n.client.Address() {
			delete(peer.peers, uuid)
			peer.Logger().Infof("%s - OLD peer at %s", peer.tcpAddr, addr)
			// client reconnected with another address
			n.client.Destroy()
			n.debouncer.Kill()
			connect = true
		} else {
			peer.checkBeacon(n)
		}
	} else {
		peer.Logger().Infof("%s - NEW peer at %s", peer.tcpAddr, addr)
		connect = true
	}
	peer.Unlock()

	if connect {
		var err = peer.connectPeer(uuid, addr)
		if err != nil {
			peer.Logger().Errorf("Unable to connect to peer at %s: %+v", addr, err)
		}
	}
}

func (peer *Peer) checkBeacon(n *node) {
	if n.beaconCountdown == 0 {
		// this debouncer is only for UDP beacon when beaconCountdown == 0
		n.debouncer.Delay(nil)
	} else {
		var now = time.Now()
		if now.Sub(n.beaconLastTime) < peer.cfg.BeaconMaxInterval {
			n.beaconCountdown--
		} else {
			n.beaconCountdown = peer.cfg.BeaconCountdown
		}
		if n.beaconCountdown == 0 {
			// the client responded, switching to UDP
			peer.Logger().Infof("%s - Peer at %s responded. Switching to UDP listening", peer.tcpAddr, n.client.Address())
			// kill the TCP health check
			n.debouncer.Kill()
			peer.healthCheckByUDP(n)
		}
		n.beaconLastTime = now
	}
}

func (peer *Peer) connectPeer(uuid string, addr string) error {
	var cli = gomsg.NewClient()
	// register peer public IP:Port
	var ip, err = gomsg.IP()
	if err != nil {
		return err
	}
	// copy metada
	var md = cli.Metadata()
	for k, v := range peer.Metadata() {
		md[k] = v
	}
	cli.Metadata()[PeerAddressKey] = ip + ":" + strconv.Itoa(peer.BindPort())

	cli.SetLogger(log.Wrap{peer.Logger().CallerAt(2), "{Client@" + addr + "}"})
	var n = &node{
		uuid:   uuid,
		client: cli,
	}
	peer.healthCheckByUDP(n)

	peer.Lock()
	peer.peers[uuid] = n
	peer.Unlock()

	// when it connects it will be already in peers
	var e = <-cli.Connect(addr)
	if e != nil {
		peer.Lock()
		delete(peer.peers, uuid)
		peer.Unlock()
		return e
	}

	peer.RLock()
	// apply all handlers
	for k, v := range peer.handlers {
		cli.Handle(k, v...)
	}
	peer.RUnlock()

	peer.listPeers(addr)

	peer.fireNewPeerListener(cli)

	return nil
}

func (peer *Peer) dropPeer(n *node) {
	peer.fireDropPeerListener(n.client)

	peer.Lock()
	peer.Logger().Infof("%s - Droping unresponsive peer at %s", peer.tcpAddr, n.client.Address())
	n.client.Destroy()
	// no need to kill the debouncer. If this was called, it means it fired.
	n.debouncer = nil
	delete(peer.peers, n.uuid)
	peer.Unlock()

	peer.listPeers("")

}

func (peer *Peer) listPeers(addr string) {
	if peer.Logger().IsActive(log.INFO) {
		peer.RLock()
		var arr = make([]*node, len(peer.peers))
		var i = 0
		for _, n := range peer.peers {
			arr[i] = n
			i++
		}
		peer.RUnlock()

		sort.Slice(arr, func(i, j int) bool {
			return strings.Compare(arr[i].client.Address(), arr[j].client.Address()) < 0
		})

		var buf bytes.Buffer
		buf.WriteString(" {\n    ")
		var peerAddr string
		var ip, err = gomsg.IP()
		if err == nil {
			peerAddr = fmt.Sprintf("%s:%d", ip, peer.BindPort())
		} else {
			peerAddr = peer.BindAddress().String()
		}
		buf.WriteString(peerAddr)
		buf.WriteString(" (this)\n")
		for _, n := range arr {
			buf.WriteString("    ")
			var a = n.client.Address()
			buf.WriteString(a)
			if a == addr {
				buf.WriteString(" (NEW)")
			}
			buf.WriteString("\n")
		}
		buf.WriteString(" }\n")

		peer.Logger().Infof("Peers of %s\n%s", peer.cfg.BeaconName, buf.String())
	}
}

// healthCheckByIP is the client that checks actively the remote peer
func (peer *Peer) healthCheckByTCP(n *node) {
	var uuid = n.client.RemoteUuid().Bytes()
	var ticker = tk.NewTicker(peer.cfg.BeaconInterval, func(t time.Time) {
		<-n.client.RequestTimeout(PING, uuid, func(ok bool) {
			if ok {
				n.debouncer.Delay(nil)
			} else {
				peer.Logger().Debugf("Peer with UUID %x at %s no longer valid.", uuid, n.client.Address())
				n.debouncer.Kill()
			}
		}, peer.cfg.BeaconInterval)
	})
	n.debouncer = tk.NewDebounce(peer.cfg.BeaconMaxInterval, func(o interface{}) {
		peer.Logger().Debugf("%s - Failed to contact by TCP peer at %s.", peer.tcpAddr, n.client.Address())
		peer.dropPeer(n)
	})
	n.debouncer.OnExit = func() {
		ticker.Stop()
	}
}

func (peer *Peer) healthCheckByUDP(n *node) {
	n.debouncer = tk.NewDebounce(peer.cfg.BeaconMaxInterval, func(o interface{}) {
		// the client did not responded, switching to TCP
		peer.Logger().Debugf("%s - Silent peer at %s. Switching to TCP ping", peer.tcpAddr, n.client.Address())
		n.beaconCountdown = peer.cfg.BeaconCountdown
		peer.healthCheckByTCP(n)
	})
}

func (peer *Peer) beaconHandler(src *net.UDPAddr, n int, b []byte) {
	// starts with tag
	if bytes.HasPrefix(b, []byte(peer.cfg.BeaconName)) {
		var r = bytes.NewReader(b)
		r.Seek(int64(len(peer.cfg.BeaconName)), io.SeekStart)
		var uuid = make([]byte, UuidSize)
		r.Read(uuid)
		// ignore self
		if bytes.Compare(uuid, peer.cfg.Uuid.Bytes()) != 0 {
			var buf16 = make([]byte, 2)
			r.Read(buf16)
			var port = int(binary.LittleEndian.Uint16(buf16))
			peer.checkPeer(hex.EncodeToString(uuid), src.IP.String()+":"+strconv.Itoa(port))
		}
	}
}

func (peer *Peer) startBeacon() error {
	addr, err := net.ResolveUDPAddr("udp", peer.cfg.BeaconAddr)
	if err != nil {
		return faults.Wrap(err)
	}
	c, err := net.DialUDP("udp", nil, addr)
	if err != nil {
		return faults.Wrap(err)
	}
	var buf16 = make([]byte, 2)
	var port = uint16(peer.Server.BindPort())
	binary.LittleEndian.PutUint16(buf16, port)

	var buf bytes.Buffer
	buf.WriteString(peer.cfg.BeaconName)
	buf.Write(peer.cfg.Uuid.Bytes())
	buf.Write(buf16)

	var data = buf.Bytes()
	if peer.beaconTicker != nil {
		peer.beaconTicker.Stop()
	}
	peer.Logger().Infof("Heartbeat on %s every %s",
		peer.cfg.BeaconAddr,
		peer.cfg.BeaconInterval,
	)
	peer.beaconTicker = tk.NewDelayedTicker(0, peer.cfg.BeaconInterval, func(t time.Time) {
		var _, err = c.Write(data)
		if err != nil {
			peer.Logger().Errorf("Unable to write to UDP %s. Error: %+v", peer.cfg.BeaconAddr, faults.Wrap(err))
		}
	})

	return nil
}

func (peer *Peer) serveUDP(hnd func(*net.UDPAddr, int, []byte)) error {
	addr, err := net.ResolveUDPAddr("udp", peer.cfg.BeaconAddr)
	if err != nil {
		return faults.Wrap(err)
	}

	l, err := net.ListenUDP("udp", addr)
	if err != nil {
		return faults.Wrap(err)
	}
	l.SetReadBuffer(peer.cfg.BeaconMaxDatagramSize)
	peer.udpConn = l
	go func() {
		var payload = make([]byte, peer.cfg.BeaconMaxDatagramSize)
		for {
			n, src, err := l.ReadFromUDP(payload)
			if peer.udpConn == nil {
				return
			} else if err != nil {
				peer.Logger().Errorf("%s - ReadFromUDP failed: %+v", peer.tcpAddr, faults.Wrap(err))
				return
			}
			hnd(src, n, payload)
		}
	}()
	return nil
}

func (peer *Peer) Handle(name string, hnd ...interface{}) {
	peer.Logger().Infof("Registering handler for %s", name)

	peer.Lock()
	peer.handlers[name] = hnd
	for _, v := range peer.peers {
		v.client.Handle(name, hnd)
	}
	peer.Unlock()
}

func (peer *Peer) Cancel(name string) {
	peer.Logger().Infof("Canceling handler for %s", name)

	peer.Lock()
	delete(peer.handlers, name)
	for _, v := range peer.peers {
		v.client.Cancel(name)
	}
	peer.Unlock()
}

func (peer *Peer) Destroy() {
	peer.Server.Destroy()
	var conn = peer.udpConn
	peer.udpConn = nil
	if conn != nil {
		conn.Close()
	}
	if peer.beaconTicker != nil {
		peer.beaconTicker.Stop()
	}
	peer.beaconTicker = nil

	peer.Lock()
	for _, v := range peer.peers {
		v.debouncer.Kill()
		v.client.Destroy()
	}
	peer.reset()
	peer.Unlock()
}
<|endoftext|>


<commit_before>// Copyright © 2014 Steve Francia <spf@spf13.com>.
// Copyright 2013 tsuru authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package afero

import (
	"bytes"
	"io"
	"os"
	"sync"
	"sync/atomic"
)

import "time"

type MemDir interface {
	Len() int
	Names() []string
	Files() []File
	Add(File)
	Remove(File)
}

type InMemoryFile struct {
	sync.Mutex
	at      int64
	name    string
	data    []byte
	memDir  MemDir
	dir     bool
	closed  bool
	mode    os.FileMode
	modtime time.Time
}

func MemFileCreate(name string) *InMemoryFile {
	return &InMemoryFile{name: name, mode: os.ModeTemporary, modtime: time.Now()}
}

func (f *InMemoryFile) Open() error {
	atomic.StoreInt64(&f.at, 0)
	f.Lock()
	f.closed = false
	f.Unlock()
	return nil
}

func (f *InMemoryFile) Close() error {
	atomic.StoreInt64(&f.at, 0)
	f.Lock()
	f.closed = true
	f.Unlock()
	return nil
}

func (f *InMemoryFile) Name() string {
	return f.name
}

func (f *InMemoryFile) Stat() (os.FileInfo, error) {
	return &InMemoryFileInfo{f}, nil
}

func (f *InMemoryFile) Readdir(count int) (res []os.FileInfo, err error) {
	files := f.memDir.Files()
	limit := len(files)

	if len(files) == 0 {
		return
	}

	if count > 0 {
		limit = count
	}

	if len(files) < limit {
		err = io.EOF
	}

	res = make([]os.FileInfo, f.memDir.Len())

	i := 0
	for _, file := range f.memDir.Files() {
		res[i], _ = file.Stat()
		i++
	}
	return res, nil
}

func (f *InMemoryFile) Readdirnames(n int) (names []string, err error) {
	fi, err := f.Readdir(n)
	names = make([]string, len(fi))
	for i, f := range fi {
		names[i] = f.Name()
	}
	return names, err
}

func (f *InMemoryFile) Read(b []byte) (n int, err error) {
	f.Lock()
	defer f.Unlock()
	if f.closed == true {
		return 0, ErrFileClosed
	}
	if len(b) > 0 && int(f.at) == len(f.data) {
		return 0, io.EOF
	}
	if len(f.data)-int(f.at) >= len(b) {
		n = len(b)
	} else {
		n = len(f.data) - int(f.at)
	}
	copy(b, f.data[f.at:f.at+int64(n)])
	atomic.AddInt64(&f.at, int64(n))
	return
}

func (f *InMemoryFile) ReadAt(b []byte, off int64) (n int, err error) {
	atomic.StoreInt64(&f.at, off)
	return f.Read(b)
}

func (f *InMemoryFile) Truncate(size int64) error {
	if f.closed == true {
		return ErrFileClosed
	}
	if size < 0 {
		return ErrOutOfRange
	}
	if size > int64(len(f.data)) {
		diff := size - int64(len(f.data))
		f.data = append(f.data, bytes.Repeat([]byte{00}, int(diff))...)
	} else {
		f.data = f.data[0:size]
	}
	return nil
}

func (f *InMemoryFile) Seek(offset int64, whence int) (int64, error) {
	if f.closed == true {
		return 0, ErrFileClosed
	}
	switch whence {
	case 0:
		atomic.StoreInt64(&f.at, offset)
	case 1:
		atomic.AddInt64(&f.at, int64(offset))
	case 2:
		atomic.StoreInt64(&f.at, int64(len(f.data))+offset)
	}
	return f.at, nil
}

func (f *InMemoryFile) Write(b []byte) (n int, err error) {
	n = len(b)
	cur := atomic.LoadInt64(&f.at)
	f.Lock()
	defer f.Unlock()
	diff := cur - int64(len(f.data))
	var tail []byte
	if n+int(cur) < len(f.data) {
		tail = f.data[n+int(cur):]
	}
	if diff > 0 {
		f.data = append(bytes.Repeat([]byte{00}, int(diff)), b...)
		f.data = append(f.data, tail...)
	} else {
		f.data = append(f.data[:cur], b...)
		f.data = append(f.data, tail...)
	}

	atomic.StoreInt64(&f.at, int64(len(f.data)))
	return
}

func (f *InMemoryFile) WriteAt(b []byte, off int64) (n int, err error) {
	atomic.StoreInt64(&f.at, off)
	return f.Write(b)
}

func (f *InMemoryFile) WriteString(s string) (ret int, err error) {
	return f.Write([]byte(s))
}

func (f *InMemoryFile) Info() *InMemoryFileInfo {
	return &InMemoryFileInfo{file: f}
}

type InMemoryFileInfo struct {
	file *InMemoryFile
}

// Implements os.FileInfo
func (s *InMemoryFileInfo) Name() string       { return s.file.Name() }
func (s *InMemoryFileInfo) Mode() os.FileMode  { return s.file.mode }
func (s *InMemoryFileInfo) ModTime() time.Time { return s.file.modtime }
func (s *InMemoryFileInfo) IsDir() bool        { return s.file.dir }
func (s *InMemoryFileInfo) Sys() interface{}   { return nil }
func (s *InMemoryFileInfo) Size() int64 {
	if s.IsDir() {
		return int64(42)
	}
	return int64(len(s.file.data))
}
<commit_msg>fix InMemoryFileInfo.Name() and File.Readdirnames() to only return the file name and not the full path to make it conform with the behaviour of the os package<commit_after>// Copyright © 2014 Steve Francia <spf@spf13.com>.
// Copyright 2013 tsuru authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package afero

import (
	"bytes"
	"io"
	"os"
	"path"
	"sync"
	"sync/atomic"
)

import "time"

type MemDir interface {
	Len() int
	Names() []string
	Files() []File
	Add(File)
	Remove(File)
}

type InMemoryFile struct {
	sync.Mutex
	at      int64
	name    string
	data    []byte
	memDir  MemDir
	dir     bool
	closed  bool
	mode    os.FileMode
	modtime time.Time
}

func MemFileCreate(name string) *InMemoryFile {
	return &InMemoryFile{name: name, mode: os.ModeTemporary, modtime: time.Now()}
}

func (f *InMemoryFile) Open() error {
	atomic.StoreInt64(&f.at, 0)
	f.Lock()
	f.closed = false
	f.Unlock()
	return nil
}

func (f *InMemoryFile) Close() error {
	atomic.StoreInt64(&f.at, 0)
	f.Lock()
	f.closed = true
	f.Unlock()
	return nil
}

func (f *InMemoryFile) Name() string {
	return f.name
}

func (f *InMemoryFile) Stat() (os.FileInfo, error) {
	return &InMemoryFileInfo{f}, nil
}

func (f *InMemoryFile) Readdir(count int) (res []os.FileInfo, err error) {
	files := f.memDir.Files()
	limit := len(files)

	if len(files) == 0 {
		return
	}

	if count > 0 {
		limit = count
	}

	if len(files) < limit {
		err = io.EOF
	}

	res = make([]os.FileInfo, f.memDir.Len())

	i := 0
	for _, file := range f.memDir.Files() {
		res[i], _ = file.Stat()
		i++
	}
	return res, nil
}

func (f *InMemoryFile) Readdirnames(n int) (names []string, err error) {
	fi, err := f.Readdir(n)
	names = make([]string, len(fi))
	for i, f := range fi {
		_, names[i] = path.Split(f.Name())
	}
	return names, err
}

func (f *InMemoryFile) Read(b []byte) (n int, err error) {
	f.Lock()
	defer f.Unlock()
	if f.closed == true {
		return 0, ErrFileClosed
	}
	if len(b) > 0 && int(f.at) == len(f.data) {
		return 0, io.EOF
	}
	if len(f.data)-int(f.at) >= len(b) {
		n = len(b)
	} else {
		n = len(f.data) - int(f.at)
	}
	copy(b, f.data[f.at:f.at+int64(n)])
	atomic.AddInt64(&f.at, int64(n))
	return
}

func (f *InMemoryFile) ReadAt(b []byte, off int64) (n int, err error) {
	atomic.StoreInt64(&f.at, off)
	return f.Read(b)
}

func (f *InMemoryFile) Truncate(size int64) error {
	if f.closed == true {
		return ErrFileClosed
	}
	if size < 0 {
		return ErrOutOfRange
	}
	if size > int64(len(f.data)) {
		diff := size - int64(len(f.data))
		f.data = append(f.data, bytes.Repeat([]byte{00}, int(diff))...)
	} else {
		f.data = f.data[0:size]
	}
	return nil
}

func (f *InMemoryFile) Seek(offset int64, whence int) (int64, error) {
	if f.closed == true {
		return 0, ErrFileClosed
	}
	switch whence {
	case 0:
		atomic.StoreInt64(&f.at, offset)
	case 1:
		atomic.AddInt64(&f.at, int64(offset))
	case 2:
		atomic.StoreInt64(&f.at, int64(len(f.data))+offset)
	}
	return f.at, nil
}

func (f *InMemoryFile) Write(b []byte) (n int, err error) {
	n = len(b)
	cur := atomic.LoadInt64(&f.at)
	f.Lock()
	defer f.Unlock()
	diff := cur - int64(len(f.data))
	var tail []byte
	if n+int(cur) < len(f.data) {
		tail = f.data[n+int(cur):]
	}
	if diff > 0 {
		f.data = append(bytes.Repeat([]byte{00}, int(diff)), b...)
		f.data = append(f.data, tail...)
	} else {
		f.data = append(f.data[:cur], b...)
		f.data = append(f.data, tail...)
	}

	atomic.StoreInt64(&f.at, int64(len(f.data)))
	return
}

func (f *InMemoryFile) WriteAt(b []byte, off int64) (n int, err error) {
	atomic.StoreInt64(&f.at, off)
	return f.Write(b)
}

func (f *InMemoryFile) WriteString(s string) (ret int, err error) {
	return f.Write([]byte(s))
}

func (f *InMemoryFile) Info() *InMemoryFileInfo {
	return &InMemoryFileInfo{file: f}
}

type InMemoryFileInfo struct {
	file *InMemoryFile
}

// Implements os.FileInfo
func (s *InMemoryFileInfo) Name() string {
	_, name := path.Split(s.file.Name())
	return name
}
func (s *InMemoryFileInfo) Mode() os.FileMode  { return s.file.mode }
func (s *InMemoryFileInfo) ModTime() time.Time { return s.file.modtime }
func (s *InMemoryFileInfo) IsDir() bool        { return s.file.dir }
func (s *InMemoryFileInfo) Sys() interface{}   { return nil }
func (s *InMemoryFileInfo) Size() int64 {
	if s.IsDir() {
		return int64(42)
	}
	return int64(len(s.file.data))
}
<|endoftext|>


<commit_before>package route53

import (
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/StackExchange/dnscontrol/models"
	"github.com/StackExchange/dnscontrol/providers"
	"github.com/StackExchange/dnscontrol/providers/diff"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/private/waiter"
	r53 "github.com/aws/aws-sdk-go/service/route53"
	r53d "github.com/aws/aws-sdk-go/service/route53domains"
	"github.com/pkg/errors"
)

type route53Provider struct {
	client    *r53.Route53
	registrar *r53d.Route53Domains
	zones     map[string]*r53.HostedZone
}

func newRoute53Reg(conf map[string]string) (providers.Registrar, error) {
	return newRoute53(conf, nil)
}

func newRoute53Dsp(conf map[string]string, metadata json.RawMessage) (providers.DNSServiceProvider, error) {
	return newRoute53(conf, metadata)
}

func newRoute53(m map[string]string, metadata json.RawMessage) (*route53Provider, error) {
	keyId, secretKey := m["KeyId"], m["SecretKey"]

	// Route53 uses a global endpoint and route53domains
	// currently only has a single regional endpoint in us-east-1
	// http://docs.aws.amazon.com/general/latest/gr/rande.html#r53_region
	config := &aws.Config{
		Region: aws.String("us-east-1"),
	}

	if keyId != "" || secretKey != "" {
		config.Credentials = credentials.NewStaticCredentials(keyId, secretKey, "")
	}
	sess := session.New(config)

	api := &route53Provider{client: r53.New(sess), registrar: r53d.New(sess)}
	err := api.getZones()
	if err != nil {
		return nil, err
	}
	return api, nil
}

func init() {
	providers.RegisterDomainServiceProviderType("ROUTE53", newRoute53Dsp, providers.CanUsePTR, providers.CanUseSRV)
	providers.RegisterRegistrarType("ROUTE53", newRoute53Reg)
}

func sPtr(s string) *string {
	return &s
}

func (r *route53Provider) getZones() error {
	var nextMarker *string
	r.zones = make(map[string]*r53.HostedZone)
	for {
		if nextMarker != nil {
			fmt.Println(*nextMarker)
		}
		inp := &r53.ListHostedZonesInput{Marker: nextMarker}
		out, err := r.client.ListHostedZones(inp)
		if err != nil && strings.Contains(err.Error(), "is not authorized") {
			return errors.New("Check your credentials, your not authorized to perform actions on Route 53 AWS Service")
		} else if err != nil {
			return err
		}
		for _, z := range out.HostedZones {
			domain := strings.TrimSuffix(*z.Name, ".")
			r.zones[domain] = z
		}
		if out.NextMarker != nil {
			nextMarker = out.NextMarker
		} else {
			break
		}
	}
	return nil
}

//map key for grouping records
type key struct {
	Name, Type string
}

func getKey(r *models.RecordConfig) key {
	return key{r.NameFQDN, r.Type}
}

type errNoExist struct {
	domain string
}

func (e errNoExist) Error() string {
	return fmt.Sprintf("Domain %s not found in your route 53 account", e.domain)
}

func (r *route53Provider) GetNameservers(domain string) ([]*models.Nameserver, error) {

	zone, ok := r.zones[domain]
	if !ok {
		return nil, errNoExist{domain}
	}
	z, err := r.client.GetHostedZone(&r53.GetHostedZoneInput{Id: zone.Id})
	if err != nil {
		return nil, err
	}
	ns := []*models.Nameserver{}
	if z.DelegationSet != nil {
		for _, nsPtr := range z.DelegationSet.NameServers {
			ns = append(ns, &models.Nameserver{Name: *nsPtr})
		}
	}
	return ns, nil
}

func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	dc.Punycode()

	var corrections = []*models.Correction{}
	zone, ok := r.zones[dc.Name]
	// add zone if it doesn't exist
	if !ok {
		return nil, errNoExist{dc.Name}
	}

	records, err := r.fetchRecordSets(zone.Id)
	if err != nil {
		return nil, err
	}

	var existingRecords = []*models.RecordConfig{}
	for _, set := range records {
		for _, rec := range set.ResourceRecords {
			if *set.Type == "SOA" {
				continue
			}
			r := &models.RecordConfig{
				NameFQDN:       unescape(set.Name),
				Type:           *set.Type,
				Target:         *rec.Value,
				TTL:            uint32(*set.TTL),
				CombinedTarget: true,
			}
			existingRecords = append(existingRecords, r)
		}
	}
	for _, want := range dc.Records {
		want.MergeToTarget()
	}

	//diff
	differ := diff.New(dc)
	_, create, delete, modify := differ.IncrementalDiff(existingRecords)

	namesToUpdate := map[key][]string{}
	for _, c := range create {
		namesToUpdate[getKey(c.Desired)] = append(namesToUpdate[getKey(c.Desired)], c.String())
	}
	for _, d := range delete {
		namesToUpdate[getKey(d.Existing)] = append(namesToUpdate[getKey(d.Existing)], d.String())
	}
	for _, m := range modify {
		namesToUpdate[getKey(m.Desired)] = append(namesToUpdate[getKey(m.Desired)], m.String())
	}

	if len(namesToUpdate) == 0 {
		return nil, nil
	}

	updates := map[key][]*models.RecordConfig{}
	//for each name we need to update, collect relevant records from dc
	for k := range namesToUpdate {
		updates[k] = nil
		for _, rc := range dc.Records {
			if getKey(rc) == k {
				updates[k] = append(updates[k], rc)
			}
		}
	}

	dels := []*r53.Change{}
	changes := []*r53.Change{}
	changeDesc := ""
	delDesc := ""
	for k, recs := range updates {
		chg := &r53.Change{}
		var rrset *r53.ResourceRecordSet
		if len(recs) == 0 {
			dels = append(dels, chg)
			chg.Action = sPtr("DELETE")
			delDesc += strings.Join(namesToUpdate[k], "\n") + "\n"
			// on delete just submit the original resource set we got from r53.
			for _, r := range records {
				if *r.Name == k.Name+"." && *r.Type == k.Type {
					rrset = r
					break
				}
			}
		} else {
			changes = append(changes, chg)
			changeDesc += strings.Join(namesToUpdate[k], "\n") + "\n"
			//on change or create, just build a new record set from our desired state
			chg.Action = sPtr("UPSERT")
			rrset = &r53.ResourceRecordSet{
				Name:            sPtr(k.Name),
				Type:            sPtr(k.Type),
				ResourceRecords: []*r53.ResourceRecord{},
			}
			for _, r := range recs {
				val := r.Target
				rr := &r53.ResourceRecord{
					Value: &val,
				}
				rrset.ResourceRecords = append(rrset.ResourceRecords, rr)
				i := int64(r.TTL)
				rrset.TTL = &i //TODO: make sure that ttls are consistent within a set
			}
		}
		chg.ResourceRecordSet = rrset
	}

	changeReq := &r53.ChangeResourceRecordSetsInput{
		ChangeBatch: &r53.ChangeBatch{Changes: changes},
	}

	delReq := &r53.ChangeResourceRecordSetsInput{
		ChangeBatch: &r53.ChangeBatch{Changes: dels},
	}

	addCorrection := func(msg string, req *r53.ChangeResourceRecordSetsInput) {
		corrections = append(corrections,
			&models.Correction{
				Msg: msg,
				F: func() error {
					req.HostedZoneId = zone.Id
					_, err := r.client.ChangeResourceRecordSets(req)
					return err
				},
			})
	}

	if len(dels) > 0 {
		addCorrection(delDesc, delReq)
	}

	if len(changes) > 0 {
		addCorrection(changeDesc, changeReq)
	}

	return corrections, nil

}

func (r *route53Provider) GetRegistrarCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	corrections := []*models.Correction{}
	actualSet, err := r.getRegistrarNameservers(&dc.Name)
	if err != nil {
		return nil, err
	}
	sort.Strings(actualSet)
	actual := strings.Join(actualSet, ",")

	expectedSet := []string{}
	for _, ns := range dc.Nameservers {
		expectedSet = append(expectedSet, ns.Name)
	}
	sort.Strings(expectedSet)
	expected := strings.Join(expectedSet, ",")

	if actual != expected {
		return []*models.Correction{
			{
				Msg: fmt.Sprintf("Update nameservers %s -> %s", actual, expected),
				F: func() error {
					operationId, err := r.updateRegistrarNameservers(dc.Name, expectedSet)
					if err != nil {
						return err
					}

					return r.waitUntilNameserversUpdate(operationId)
				},
			},
		}, nil
	}

	return corrections, nil
}

func (r *route53Provider) getRegistrarNameservers(domainName *string) ([]string, error) {
	domainDetail, err := r.registrar.GetDomainDetail(&r53d.GetDomainDetailInput{DomainName: domainName})
	if err != nil {
		return nil, err
	}

	nameservers := []string{}
	for _, ns := range domainDetail.Nameservers {
		nameservers = append(nameservers, *ns.Name)
	}

	return nameservers, nil
}

func (r *route53Provider) updateRegistrarNameservers(domainName string, nameservers []string) (*string, error) {
	servers := []*r53d.Nameserver{}
	for i := range nameservers {
		servers = append(servers, &r53d.Nameserver{Name: &nameservers[i]})
	}

	domainUpdate, err := r.registrar.UpdateDomainNameservers(&r53d.UpdateDomainNameserversInput{DomainName: &domainName, Nameservers: servers})
	if err != nil {
		return nil, err
	}

	return domainUpdate.OperationId, nil
}

func (r *route53Provider) fetchRecordSets(zoneID *string) ([]*r53.ResourceRecordSet, error) {
	if zoneID == nil || *zoneID == "" {
		return nil, nil
	}
	var next *string
	var nextType *string
	var records []*r53.ResourceRecordSet
	for {
		listInput := &r53.ListResourceRecordSetsInput{
			HostedZoneId:    zoneID,
			StartRecordName: next,
			StartRecordType: nextType,
			MaxItems:        sPtr("100"),
		}
		list, err := r.client.ListResourceRecordSets(listInput)
		if err != nil {
			return nil, err
		}
		records = append(records, list.ResourceRecordSets...)
		if list.NextRecordName != nil {
			next = list.NextRecordName
			nextType = list.NextRecordType
		} else {
			break
		}
	}
	return records, nil
}

//we have to process names from route53 to match what we expect and to remove their odd octal encoding
func unescape(s *string) string {
	if s == nil {
		return ""
	}
	name := strings.TrimSuffix(*s, ".")
	name = strings.Replace(name, `\052`, "*", -1) //TODO: escape all octal sequences
	return name
}

func (r *route53Provider) EnsureDomainExists(domain string) error {
	if _, ok := r.zones[domain]; ok {
		return nil
	}
	fmt.Printf("Adding zone for %s to route 53 account\n", domain)
	in := &r53.CreateHostedZoneInput{
		Name:            &domain,
		CallerReference: sPtr(fmt.Sprint(time.Now().UnixNano())),
	}
	_, err := r.client.CreateHostedZone(in)
	return err

}

func (r *route53Provider) waitUntilNameserversUpdate(operationId *string) error {
	fmt.Print("Waiting for registrar update to complete...")

	waiterCfg := waiter.Config{
		Operation:   "GetOperationDetail",
		Delay:       30,
		MaxAttempts: 10,
		Acceptors: []waiter.WaitAcceptor{
			{
				State:    "success",
				Matcher:  "path",
				Argument: "Status",
				Expected: "SUCCESSFUL",
			},
		},
	}

	w := waiter.Waiter{
		Client: r.registrar,
		Input: &r53d.GetOperationDetailInput{
			OperationId: operationId,
		},
		Config: waiterCfg,
	}

	return w.Wait()
}
<commit_msg>Route 53 provider now supports CAA (#193)<commit_after>package route53

import (
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/StackExchange/dnscontrol/models"
	"github.com/StackExchange/dnscontrol/providers"
	"github.com/StackExchange/dnscontrol/providers/diff"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/private/waiter"
	r53 "github.com/aws/aws-sdk-go/service/route53"
	r53d "github.com/aws/aws-sdk-go/service/route53domains"
	"github.com/pkg/errors"
)

type route53Provider struct {
	client    *r53.Route53
	registrar *r53d.Route53Domains
	zones     map[string]*r53.HostedZone
}

func newRoute53Reg(conf map[string]string) (providers.Registrar, error) {
	return newRoute53(conf, nil)
}

func newRoute53Dsp(conf map[string]string, metadata json.RawMessage) (providers.DNSServiceProvider, error) {
	return newRoute53(conf, metadata)
}

func newRoute53(m map[string]string, metadata json.RawMessage) (*route53Provider, error) {
	keyId, secretKey := m["KeyId"], m["SecretKey"]

	// Route53 uses a global endpoint and route53domains
	// currently only has a single regional endpoint in us-east-1
	// http://docs.aws.amazon.com/general/latest/gr/rande.html#r53_region
	config := &aws.Config{
		Region: aws.String("us-east-1"),
	}

	if keyId != "" || secretKey != "" {
		config.Credentials = credentials.NewStaticCredentials(keyId, secretKey, "")
	}
	sess := session.New(config)

	api := &route53Provider{client: r53.New(sess), registrar: r53d.New(sess)}
	err := api.getZones()
	if err != nil {
		return nil, err
	}
	return api, nil
}

func init() {
	providers.RegisterDomainServiceProviderType("ROUTE53", newRoute53Dsp, providers.CanUsePTR, providers.CanUseSRV, providers.CanUseCAA)
	providers.RegisterRegistrarType("ROUTE53", newRoute53Reg)
}

func sPtr(s string) *string {
	return &s
}

func (r *route53Provider) getZones() error {
	var nextMarker *string
	r.zones = make(map[string]*r53.HostedZone)
	for {
		if nextMarker != nil {
			fmt.Println(*nextMarker)
		}
		inp := &r53.ListHostedZonesInput{Marker: nextMarker}
		out, err := r.client.ListHostedZones(inp)
		if err != nil && strings.Contains(err.Error(), "is not authorized") {
			return errors.New("Check your credentials, your not authorized to perform actions on Route 53 AWS Service")
		} else if err != nil {
			return err
		}
		for _, z := range out.HostedZones {
			domain := strings.TrimSuffix(*z.Name, ".")
			r.zones[domain] = z
		}
		if out.NextMarker != nil {
			nextMarker = out.NextMarker
		} else {
			break
		}
	}
	return nil
}

//map key for grouping records
type key struct {
	Name, Type string
}

func getKey(r *models.RecordConfig) key {
	return key{r.NameFQDN, r.Type}
}

type errNoExist struct {
	domain string
}

func (e errNoExist) Error() string {
	return fmt.Sprintf("Domain %s not found in your route 53 account", e.domain)
}

func (r *route53Provider) GetNameservers(domain string) ([]*models.Nameserver, error) {

	zone, ok := r.zones[domain]
	if !ok {
		return nil, errNoExist{domain}
	}
	z, err := r.client.GetHostedZone(&r53.GetHostedZoneInput{Id: zone.Id})
	if err != nil {
		return nil, err
	}
	ns := []*models.Nameserver{}
	if z.DelegationSet != nil {
		for _, nsPtr := range z.DelegationSet.NameServers {
			ns = append(ns, &models.Nameserver{Name: *nsPtr})
		}
	}
	return ns, nil
}

func (r *route53Provider) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	dc.Punycode()

	var corrections = []*models.Correction{}
	zone, ok := r.zones[dc.Name]
	// add zone if it doesn't exist
	if !ok {
		return nil, errNoExist{dc.Name}
	}

	records, err := r.fetchRecordSets(zone.Id)
	if err != nil {
		return nil, err
	}

	var existingRecords = []*models.RecordConfig{}
	for _, set := range records {
		for _, rec := range set.ResourceRecords {
			if *set.Type == "SOA" {
				continue
			}
			r := &models.RecordConfig{
				NameFQDN:       unescape(set.Name),
				Type:           *set.Type,
				Target:         *rec.Value,
				TTL:            uint32(*set.TTL),
				CombinedTarget: true,
			}
			existingRecords = append(existingRecords, r)
		}
	}
	for _, want := range dc.Records {
		want.MergeToTarget()
	}

	//diff
	differ := diff.New(dc)
	_, create, delete, modify := differ.IncrementalDiff(existingRecords)

	namesToUpdate := map[key][]string{}
	for _, c := range create {
		namesToUpdate[getKey(c.Desired)] = append(namesToUpdate[getKey(c.Desired)], c.String())
	}
	for _, d := range delete {
		namesToUpdate[getKey(d.Existing)] = append(namesToUpdate[getKey(d.Existing)], d.String())
	}
	for _, m := range modify {
		namesToUpdate[getKey(m.Desired)] = append(namesToUpdate[getKey(m.Desired)], m.String())
	}

	if len(namesToUpdate) == 0 {
		return nil, nil
	}

	updates := map[key][]*models.RecordConfig{}
	//for each name we need to update, collect relevant records from dc
	for k := range namesToUpdate {
		updates[k] = nil
		for _, rc := range dc.Records {
			if getKey(rc) == k {
				updates[k] = append(updates[k], rc)
			}
		}
	}

	dels := []*r53.Change{}
	changes := []*r53.Change{}
	changeDesc := ""
	delDesc := ""
	for k, recs := range updates {
		chg := &r53.Change{}
		var rrset *r53.ResourceRecordSet
		if len(recs) == 0 {
			dels = append(dels, chg)
			chg.Action = sPtr("DELETE")
			delDesc += strings.Join(namesToUpdate[k], "\n") + "\n"
			// on delete just submit the original resource set we got from r53.
			for _, r := range records {
				if *r.Name == k.Name+"." && *r.Type == k.Type {
					rrset = r
					break
				}
			}
		} else {
			changes = append(changes, chg)
			changeDesc += strings.Join(namesToUpdate[k], "\n") + "\n"
			//on change or create, just build a new record set from our desired state
			chg.Action = sPtr("UPSERT")
			rrset = &r53.ResourceRecordSet{
				Name:            sPtr(k.Name),
				Type:            sPtr(k.Type),
				ResourceRecords: []*r53.ResourceRecord{},
			}
			for _, r := range recs {
				val := r.Target
				rr := &r53.ResourceRecord{
					Value: &val,
				}
				rrset.ResourceRecords = append(rrset.ResourceRecords, rr)
				i := int64(r.TTL)
				rrset.TTL = &i //TODO: make sure that ttls are consistent within a set
			}
		}
		chg.ResourceRecordSet = rrset
	}

	changeReq := &r53.ChangeResourceRecordSetsInput{
		ChangeBatch: &r53.ChangeBatch{Changes: changes},
	}

	delReq := &r53.ChangeResourceRecordSetsInput{
		ChangeBatch: &r53.ChangeBatch{Changes: dels},
	}

	addCorrection := func(msg string, req *r53.ChangeResourceRecordSetsInput) {
		corrections = append(corrections,
			&models.Correction{
				Msg: msg,
				F: func() error {
					req.HostedZoneId = zone.Id
					_, err := r.client.ChangeResourceRecordSets(req)
					return err
				},
			})
	}

	if len(dels) > 0 {
		addCorrection(delDesc, delReq)
	}

	if len(changes) > 0 {
		addCorrection(changeDesc, changeReq)
	}

	return corrections, nil

}

func (r *route53Provider) GetRegistrarCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	corrections := []*models.Correction{}
	actualSet, err := r.getRegistrarNameservers(&dc.Name)
	if err != nil {
		return nil, err
	}
	sort.Strings(actualSet)
	actual := strings.Join(actualSet, ",")

	expectedSet := []string{}
	for _, ns := range dc.Nameservers {
		expectedSet = append(expectedSet, ns.Name)
	}
	sort.Strings(expectedSet)
	expected := strings.Join(expectedSet, ",")

	if actual != expected {
		return []*models.Correction{
			{
				Msg: fmt.Sprintf("Update nameservers %s -> %s", actual, expected),
				F: func() error {
					operationId, err := r.updateRegistrarNameservers(dc.Name, expectedSet)
					if err != nil {
						return err
					}

					return r.waitUntilNameserversUpdate(operationId)
				},
			},
		}, nil
	}

	return corrections, nil
}

func (r *route53Provider) getRegistrarNameservers(domainName *string) ([]string, error) {
	domainDetail, err := r.registrar.GetDomainDetail(&r53d.GetDomainDetailInput{DomainName: domainName})
	if err != nil {
		return nil, err
	}

	nameservers := []string{}
	for _, ns := range domainDetail.Nameservers {
		nameservers = append(nameservers, *ns.Name)
	}

	return nameservers, nil
}

func (r *route53Provider) updateRegistrarNameservers(domainName string, nameservers []string) (*string, error) {
	servers := []*r53d.Nameserver{}
	for i := range nameservers {
		servers = append(servers, &r53d.Nameserver{Name: &nameservers[i]})
	}

	domainUpdate, err := r.registrar.UpdateDomainNameservers(&r53d.UpdateDomainNameserversInput{DomainName: &domainName, Nameservers: servers})
	if err != nil {
		return nil, err
	}

	return domainUpdate.OperationId, nil
}

func (r *route53Provider) fetchRecordSets(zoneID *string) ([]*r53.ResourceRecordSet, error) {
	if zoneID == nil || *zoneID == "" {
		return nil, nil
	}
	var next *string
	var nextType *string
	var records []*r53.ResourceRecordSet
	for {
		listInput := &r53.ListResourceRecordSetsInput{
			HostedZoneId:    zoneID,
			StartRecordName: next,
			StartRecordType: nextType,
			MaxItems:        sPtr("100"),
		}
		list, err := r.client.ListResourceRecordSets(listInput)
		if err != nil {
			return nil, err
		}
		records = append(records, list.ResourceRecordSets...)
		if list.NextRecordName != nil {
			next = list.NextRecordName
			nextType = list.NextRecordType
		} else {
			break
		}
	}
	return records, nil
}

//we have to process names from route53 to match what we expect and to remove their odd octal encoding
func unescape(s *string) string {
	if s == nil {
		return ""
	}
	name := strings.TrimSuffix(*s, ".")
	name = strings.Replace(name, `\052`, "*", -1) //TODO: escape all octal sequences
	return name
}

func (r *route53Provider) EnsureDomainExists(domain string) error {
	if _, ok := r.zones[domain]; ok {
		return nil
	}
	fmt.Printf("Adding zone for %s to route 53 account\n", domain)
	in := &r53.CreateHostedZoneInput{
		Name:            &domain,
		CallerReference: sPtr(fmt.Sprint(time.Now().UnixNano())),
	}
	_, err := r.client.CreateHostedZone(in)
	return err

}

func (r *route53Provider) waitUntilNameserversUpdate(operationId *string) error {
	fmt.Print("Waiting for registrar update to complete...")

	waiterCfg := waiter.Config{
		Operation:   "GetOperationDetail",
		Delay:       30,
		MaxAttempts: 10,
		Acceptors: []waiter.WaitAcceptor{
			{
				State:    "success",
				Matcher:  "path",
				Argument: "Status",
				Expected: "SUCCESSFUL",
			},
		},
	}

	w := waiter.Waiter{
		Client: r.registrar,
		Input: &r53d.GetOperationDetailInput{
			OperationId: operationId,
		},
		Config: waiterCfg,
	}

	return w.Wait()
}
<|endoftext|>


<commit_before>package models

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"time"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"gopkg.in/yaml.v2"
)

// MPages is the map containing all the articles. The key is the slug of the article
// for low complexity access when querying a slug.
var MPages map[string]*Page

// SPages is a sorted slice of pages. Sorted by date, it is used to render all
// the pages on the index page.
var SPages pageSlice

// Page is the main struct. It contains everything needed to render the article.
type Page struct {
	Raw         string
	Markdown    template.HTML
	Title       string
	Description string
	Author      string
	Date        time.Time
	DateFmt     string
	Tags        []string
	File        string
	Slug        string
}

// NewPageFromFile parses a file, inserts it in the map and slice, and returns a *Page instance
func NewPageFromFile(fn string) (*Page, error) {
	var err error
	p := new(Page)

	if err = p.ParseFile(fn); err != nil {
		return nil, errors.Wrap(err, "NewPageFromFile")
	}
	if err = p.Insert(false); err != nil {
		return nil, errors.Wrap(err, "NewPageFromFile")
	}

	logrus.WithFields(logrus.Fields{
		"file": filepath.Base(fn),
		"url":  fmt.Sprintf("/post/%s", p.Slug),
	}).Info("New file added")

	return p, nil
}

type pageSlice []*Page

// Len is part of sort.Interface.
func (p pageSlice) Len() int {
	return len(p)
}

// Swap is part of sort.Interface.
func (p pageSlice) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

// Less is part of sort.Interface. We use count as the value to sort by
func (p pageSlice) Less(i, j int) bool {
	return p[i].Date.After(p[j].Date)
}

// ParseFile parses a whole file and fills the Page struct.
func (p *Page) ParseFile(fn string) error {
	var err error

	p.File = filepath.Base(fn)
	h, b, err := SplitFile(fn)
	if err != nil {
		return err
	}
	p.Raw = string(b)
	p.ParseMarkdown(b)
	if err = p.ParseMetadata(h); err != nil {
		return err
	}
	return nil
}

// UpdateFromFile parses the file once more
func (p *Page) UpdateFromFile(fn string) error {
	var err error
	old := p.Slug
	if err = p.ParseFile(fn); err != nil {
		return errors.Wrap(err, "UpdateFromFile")
	}
	if _, ok := MPages[p.Slug]; !ok {
		delete(MPages, old)
		MPages[p.Slug] = p
	}
	sort.Sort(SPages)

	logrus.WithFields(logrus.Fields{
		"file": filepath.Base(fn),
		"url":  fmt.Sprintf("/post/%s", p.Slug),
	}).Info("Modified")

	return nil
}

// Pop removes a Page (in case the file is deleted for example)
func (p *Page) Pop() {
	for i, n := range SPages {
		if n == p {
			SPages = append(SPages[:i], SPages[i+1:]...)
		}
	}
	delete(MPages, p.Slug)
	logrus.WithFields(logrus.Fields{
		"file": filepath.Base(p.File),
		"url":  fmt.Sprintf("/post/%s", p.Slug),
	}).Info("Removed")
}

// ParseMetadata parses the metadata on top of the markdown files. It will also
// raise errors when mandatory fields aren't present, or some slugs are duplicates.
func (p *Page) ParseMetadata(h []byte) error {
	var err error
	var t time.Time

	m := meta{}
	if err = yaml.Unmarshal(h, &m); err != nil {
		return err
	}
	if err = m.Validate(); err != nil {
		return err
	}
	if t, err = time.Parse("2006-01-02 15:04:05", m.Date); err != nil {
		return err
	}
	p.Slug = m.GenerateSlug()
	p.Description = m.Description
	p.Date = t
	p.DateFmt = t.Format("2006/01/02 15:04")
	p.Tags = m.Tags
	p.Author = m.Author
	p.Title = m.Title
	return nil
}

// ParseMarkdown will simply parse the markdown b and put it inside the Page structure.
func (p *Page) ParseMarkdown(b []byte) {
	p.Markdown = template.HTML(string(render(b)))
}

// Insert will try to insert the file in the MPages map and SPages slice. It will
// also validates that no pages have the same slug, and sort the SPages slice in case
// it's not a batch insertion. (A batch insertion means after all the inserts,
// SPages will be sorted manually)
func (p *Page) Insert(batch bool) error {
	if val, ok := MPages[p.Slug]; ok {
		return fmt.Errorf("Two pages have the same slug : %s and %s both have %s", p.File, val.File, p.Slug)
	}
	MPages[p.Slug] = p
	SPages = append(SPages, p)
	if !batch {
		sort.Sort(SPages)
	}
	return nil
}

// ParseDir cycles through a directory and parses each file one by one.
func ParseDir(dir string) error {
	var err error
	var files []os.FileInfo
	start := time.Now()

	if files, err = ioutil.ReadDir(dir); err != nil {
		return err
	}

	MPages = make(map[string]*Page, len(files))
	SPages = make(pageSlice, 0, len(files))

	for _, f := range files {
		s := time.Now()
		p := Page{}
		if err = p.ParseFile(path.Join(dir, f.Name())); err != nil {
			logrus.WithFields(logrus.Fields{
				"took":  time.Since(s),
				"file":  p.File,
				"state": "ignored",
			}).WithError(err).Error("Couldn't process file")
			continue
		}
		if err = p.Insert(true); err != nil {
			logrus.WithFields(logrus.Fields{
				"took":  time.Since(s),
				"file":  p.File,
				"state": "ignored",
			}).WithError(err).Error("Couldn't insert file")
			continue
		}
		logrus.WithFields(logrus.Fields{
			"took": time.Since(s),
			"file": p.File,
			"url":  fmt.Sprintf("/post/%s", p.Slug),
		}).Info("Generated page successfully")
	}
	sort.Sort(SPages)
	logrus.WithFields(logrus.Fields{
		"files": len(files),
		"took":  time.Since(start),
	}).Info("Generated files")
	return nil
}
<commit_msg>Fixing complaining linter<commit_after>package models

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"time"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"gopkg.in/yaml.v2"
)

// MPages is the map containing all the articles. The key is the slug of the article
// for low complexity access when querying a slug.
var MPages map[string]*Page

// SPages is a sorted slice of pages. Sorted by date, it is used to render all
// the pages on the index page.
var SPages pageSlice

// Page is the main struct. It contains everything needed to render the article.
type Page struct {
	Raw         string
	Markdown    template.HTML
	Title       string
	Description string
	Author      string
	Date        time.Time
	DateFmt     string
	Tags        []string
	File        string
	Slug        string
}

// NewPageFromFile parses a file, inserts it in the map and slice, and returns a *Page instance
func NewPageFromFile(fn string) (*Page, error) {
	var err error
	p := new(Page)

	if err = p.ParseFile(fn); err != nil {
		return nil, errors.Wrap(err, "NewPageFromFile")
	}
	if err = p.Insert(false); err != nil {
		return nil, errors.Wrap(err, "NewPageFromFile")
	}

	logrus.WithFields(logrus.Fields{
		"file": filepath.Base(fn),
		"url":  fmt.Sprintf("/post/%s", p.Slug),
	}).Info("New file added")

	return p, nil
}

type pageSlice []*Page

// Len is part of sort.Interface.
func (p pageSlice) Len() int {
	return len(p)
}

// Swap is part of sort.Interface.
func (p pageSlice) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

// Less is part of sort.Interface. We use count as the value to sort by
func (p pageSlice) Less(i, j int) bool {
	return p[i].Date.After(p[j].Date)
}

// ParseFile parses a whole file and fills the Page struct.
func (p *Page) ParseFile(fn string) error {
	var err error

	p.File = filepath.Base(fn)
	h, b, err := SplitFile(fn)
	if err != nil {
		return err
	}
	p.Raw = string(b)
	p.ParseMarkdown(b)
	return p.ParseMetadata(h)
}

// UpdateFromFile parses the file once more
func (p *Page) UpdateFromFile(fn string) error {
	var err error
	old := p.Slug
	if err = p.ParseFile(fn); err != nil {
		return errors.Wrap(err, "UpdateFromFile")
	}
	if _, ok := MPages[p.Slug]; !ok {
		delete(MPages, old)
		MPages[p.Slug] = p
	}
	sort.Sort(SPages)

	logrus.WithFields(logrus.Fields{
		"file": filepath.Base(fn),
		"url":  fmt.Sprintf("/post/%s", p.Slug),
	}).Info("Modified")

	return nil
}

// Pop removes a Page (in case the file is deleted for example)
func (p *Page) Pop() {
	for i, n := range SPages {
		if n == p {
			SPages = append(SPages[:i], SPages[i+1:]...)
		}
	}
	delete(MPages, p.Slug)
	logrus.WithFields(logrus.Fields{
		"file": filepath.Base(p.File),
		"url":  fmt.Sprintf("/post/%s", p.Slug),
	}).Info("Removed")
}

// ParseMetadata parses the metadata on top of the markdown files. It will also
// raise errors when mandatory fields aren't present, or some slugs are duplicates.
func (p *Page) ParseMetadata(h []byte) error {
	var err error
	var t time.Time

	m := meta{}
	if err = yaml.Unmarshal(h, &m); err != nil {
		return err
	}
	if err = m.Validate(); err != nil {
		return err
	}
	if t, err = time.Parse("2006-01-02 15:04:05", m.Date); err != nil {
		return err
	}
	p.Slug = m.GenerateSlug()
	p.Description = m.Description
	p.Date = t
	p.DateFmt = t.Format("2006/01/02 15:04")
	p.Tags = m.Tags
	p.Author = m.Author
	p.Title = m.Title
	return nil
}

// ParseMarkdown will simply parse the markdown b and put it inside the Page structure.
func (p *Page) ParseMarkdown(b []byte) {
	p.Markdown = template.HTML(string(render(b)))
}

// Insert will try to insert the file in the MPages map and SPages slice. It will
// also validates that no pages have the same slug, and sort the SPages slice in case
// it's not a batch insertion. (A batch insertion means after all the inserts,
// SPages will be sorted manually)
func (p *Page) Insert(batch bool) error {
	if val, ok := MPages[p.Slug]; ok {
		return fmt.Errorf("Two pages have the same slug : %s and %s both have %s", p.File, val.File, p.Slug)
	}
	MPages[p.Slug] = p
	SPages = append(SPages, p)
	if !batch {
		sort.Sort(SPages)
	}
	return nil
}

// ParseDir cycles through a directory and parses each file one by one.
func ParseDir(dir string) error {
	var err error
	var files []os.FileInfo
	start := time.Now()

	if files, err = ioutil.ReadDir(dir); err != nil {
		return err
	}

	MPages = make(map[string]*Page, len(files))
	SPages = make(pageSlice, 0, len(files))

	for _, f := range files {
		s := time.Now()
		p := Page{}
		if err = p.ParseFile(path.Join(dir, f.Name())); err != nil {
			logrus.WithFields(logrus.Fields{
				"took":  time.Since(s),
				"file":  p.File,
				"state": "ignored",
			}).WithError(err).Error("Couldn't process file")
			continue
		}
		if err = p.Insert(true); err != nil {
			logrus.WithFields(logrus.Fields{
				"took":  time.Since(s),
				"file":  p.File,
				"state": "ignored",
			}).WithError(err).Error("Couldn't insert file")
			continue
		}
		logrus.WithFields(logrus.Fields{
			"took": time.Since(s),
			"file": p.File,
			"url":  fmt.Sprintf("/post/%s", p.Slug),
		}).Info("Generated page successfully")
	}
	sort.Sort(SPages)
	logrus.WithFields(logrus.Fields{
		"files": len(files),
		"took":  time.Since(start),
	}).Info("Generated files")
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"time"

	"github.com/replaygaming/eventsource"
)

type LogMonitoring struct {
	prefix string
	connections int
}

func NewMetrics(prefix string) (LogMonitoring, error) {
	monitor := LogMonitoring{prefix: prefix}

	return monitor, nil
}

func (monitor LogMonitoring) ClientCount(count int) {
	if count != monitor.connections {
		monitor.connections = count
		log.Printf("[METRIC] %sconnections: %d\n", monitor.prefix, count)
	}
}

func (monitor LogMonitoring) EventDone(event eventsource.Event, duration time.Duration, eventdurations []time.Duration) {
	var sum int64
	var count int64
	var avg float64

	for _, d := range eventdurations {
		if d > 0 {
			sum += d.Nanoseconds()
		}
	}

	count = int64(len(eventdurations))

	if count > 0 {
		avg = float64(sum) / float64(count)
	}

	log.Printf("[METRIC] %s.event_distributed.clients: %d\n", monitor.prefix, count)
	log.Printf("[METRIC] %s.event_distributed.avg_time: %dns\n", monitor.prefix, avg)
}
<commit_msg>Go fmt<commit_after>package main

import (
	"log"
	"time"

	"github.com/replaygaming/eventsource"
)

type LogMonitoring struct {
	prefix      string
	connections int
}

func NewMetrics(prefix string) (LogMonitoring, error) {
	monitor := LogMonitoring{prefix: prefix}

	return monitor, nil
}

func (monitor LogMonitoring) ClientCount(count int) {
	if count != monitor.connections {
		monitor.connections = count
		log.Printf("[METRIC] %sconnections: %d\n", monitor.prefix, count)
	}
}

func (monitor LogMonitoring) EventDone(event eventsource.Event, duration time.Duration, eventdurations []time.Duration) {
	var sum int64
	var count int64
	var avg float64

	for _, d := range eventdurations {
		if d > 0 {
			sum += d.Nanoseconds()
		}
	}

	count = int64(len(eventdurations))

	if count > 0 {
		avg = float64(sum) / float64(count)
	}

	log.Printf("[METRIC] %s.event_distributed.clients: %d\n", monitor.prefix, count)
	log.Printf("[METRIC] %s.event_distributed.avg_time: %dns\n", monitor.prefix, avg)
}
<|endoftext|>


<commit_before>package subtitles

import (
	"fmt"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

// CleanupSub parses .srt or .ssa, performs cleanup and renders to a .srt, returning a string. caller is responsible for passing UTF8 string
func CleanupSub(utf8 string, filterName string, keepAds bool, sync int) (string, error) {

	var captions []caption

	if looksLikeSrt(utf8) {
		captions = parseSrt(utf8)
	} else {
		// falls back on .ssa decoding, for now
		captions = parseSsa(utf8)
	}

	if !keepAds {
		captions = removeAds(captions)
	}

	if sync != 0 {
		captions = resyncSubs(captions, sync)
	}

	captions = filterSubs(captions, filterName)

	out := renderSrt(captions)

	return out, nil
}

func resyncSubs(subs []caption, sync int) []caption {

	//	var res []caption
	fmt.Printf("resyncing with %d\n", sync)

	for i := range subs {
		subs[i].Start = subs[i].Start.Add(time.Duration(sync) * time.Millisecond)
		subs[i].End = subs[i].End.Add(time.Duration(sync) * time.Millisecond)
	}

	return subs
}

// RemoveAds removes advertisement from the subtitles
func removeAds(subs []caption) []caption {

	var res []caption

	ads := []string{

		// english:
		"captions paid for by",
		"english subtitles",
		"subtitles:", "subtitles by",
		"subtitles downloaded",
		"captioning by", "captions by",
		"transcript :", "transcript:", "transcript by",
		"sync, corrected", "synced and corrected",
		"sync and corrected", "sync & corrections",
		"sync and corrections",
		"traduction:", "transcript par",
		"corrections by",
		"sync by n17t01",
		"sync,", "synchro :", "synchro:", "synced by", "synchronized by",
		"synchronization by", "synchronisation:",
		"resynchronization:",
		"resync:", "resynchro", "resync by",
		"translation by",
		"encoded by",
		"web-dl",
		"subscene",
		"seriessub",
		"addic7ed", "addicted.com", "vaioholics",
		"sdimedia", "sdi media",
		"allsubs.org", "hdbits.org", "bierdopje.com", "subcentral",
		"cssubs", "tvsub", "uksubtitles",
		"ragbear.com", "ydy.com", "yyets.net", "indivx.net", "sub-way.fr",
		"forom.com", "forom. com", "facebook.com", "hdvietnam.com",
		"@gmail.com", "@hotmail.com", "@hotmail.fr",
		"napisy.org", "1000fr.com",
		"opensubtitles", "open subtitles", "s u b t i t l e",
		"sous-titres.eu", "300mbfilms.com", "put.io", "subtitulos.es", "osdb.link",
		"simail.si", "sf.net", "vitac.com",
		"yify-torrents", "yify torrents",
		"thepiratebay", "anoxmous", "verdikt", "la fisher team", "red bee media",
		"mkv player", "best watched using", "advertise your product", "remove all ads",
		"memoryonsmells", "1st-booking",

		// swedish:
		"swedish subtitles", "svenska undertexter", "internationella undertexter",
		"svensktextning", "(c) sveriges televisionit",
		"undertexter.se", "undertexter. se", "swesub.nu", "divxsweden",
		"undertext av", "översatt av", "översättning:", "översättning av", "rättad av",
		"synkad av", "synkat av", "synk:", "synkning:", "redigerad av", "textning:",
		"svensk text", "text av", "text:", "omsynk:", "omsynkad",
		"transkribering:", "piratpartiet.se",
		"korrektur:", "korrekturläst", "texter på nätet", "text hämtad från",
		"din filmsajt på nätet", "din största filmsajt på nätet",
		"alltid nya texter",
		"senaste undertexter på",
		"programtextning", "översättargrupp",
		"mediatextgruppen", "visiontext", "scandinavian text service",
		"jhs International", "svensk medietext",

		// french:
		"relecture et corrections finales:",
	}

	seq := 1
	for orgSeq, sub := range subs {

		isAd := false

		for _, line := range sub.Text {
			x := strings.ToLower(line)
			for _, adLine := range ads {
				if !isAd && strings.Contains(x, adLine) {
					isAd = true
					log.Println("[ads]", (orgSeq + 1), sub.Text, "matched", adLine)
					break
				}
			}
		}

		if !isAd {
			sub.Seq = seq
			res = append(res, sub)
			seq++
		}
	}

	return res
}
<commit_msg>cleaner: match 'Shared by http://DJJ.HOME.SAPO.PT/'<commit_after>package subtitles

import (
	"fmt"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

// CleanupSub parses .srt or .ssa, performs cleanup and renders to a .srt, returning a string. caller is responsible for passing UTF8 string
func CleanupSub(utf8 string, filterName string, keepAds bool, sync int) (string, error) {

	var captions []caption

	if looksLikeSrt(utf8) {
		captions = parseSrt(utf8)
	} else {
		// falls back on .ssa decoding, for now
		captions = parseSsa(utf8)
	}

	if !keepAds {
		captions = removeAds(captions)
	}

	if sync != 0 {
		captions = resyncSubs(captions, sync)
	}

	captions = filterSubs(captions, filterName)

	out := renderSrt(captions)

	return out, nil
}

func resyncSubs(subs []caption, sync int) []caption {

	//	var res []caption
	fmt.Printf("resyncing with %d\n", sync)

	for i := range subs {
		subs[i].Start = subs[i].Start.Add(time.Duration(sync) * time.Millisecond)
		subs[i].End = subs[i].End.Add(time.Duration(sync) * time.Millisecond)
	}

	return subs
}

// RemoveAds removes advertisement from the subtitles
func removeAds(subs []caption) []caption {

	var res []caption

	ads := []string{

		// english:
		"captions paid for by",
		"english subtitles",
		"subtitles:", "subtitles by",
		"subtitles downloaded",
		"captioning by", "captions by",
		"transcript :", "transcript:", "transcript by",
		"sync, corrected", "synced and corrected",
		"sync and corrected", "sync & corrections",
		"sync and corrections",
		"traduction:", "transcript par",
		"corrections by",
		"sync by n17t01",
		"sync,", "synchro :", "synchro:", "synced by", "synchronized by",
		"synchronization by", "synchronisation:",
		"resynchronization:",
		"resync:", "resynchro", "resync by",
		"translation by",
		"encoded by",
		"web-dl",
		"subscene",
		"seriessub",
		"addic7ed", "addicted.com", "vaioholics",
		"sdimedia", "sdi media",
		"allsubs.org", "hdbits.org", "bierdopje.com", "subcentral",
		"cssubs", "tvsub", "uksubtitles",
		"ragbear.com", "ydy.com", "yyets.net", "indivx.net", "sub-way.fr",
		"forom.com", "forom. com", "facebook.com", "hdvietnam.com", "sapo.pt",
		"@gmail.com", "@hotmail.com", "@hotmail.fr",
		"napisy.org", "1000fr.com",
		"opensubtitles", "open subtitles", "s u b t i t l e",
		"sous-titres.eu", "300mbfilms.com", "put.io", "subtitulos.es", "osdb.link",
		"simail.si", "sf.net", "vitac.com",
		"yify-torrents", "yify torrents",
		"thepiratebay", "anoxmous", "verdikt", "la fisher team", "red bee media",
		"mkv player", "best watched using", "advertise your product", "remove all ads",
		"memoryonsmells", "1st-booking",

		// swedish:
		"swedish subtitles", "svenska undertexter", "internationella undertexter",
		"svensktextning", "(c) sveriges televisionit",
		"undertexter.se", "undertexter. se", "swesub.nu", "divxsweden",
		"undertext av", "översatt av", "översättning:", "översättning av", "rättad av",
		"synkad av", "synkat av", "synk:", "synkning:", "redigerad av", "textning:",
		"svensk text", "text av", "text:", "omsynk:", "omsynkad",
		"transkribering:", "piratpartiet.se",
		"korrektur:", "korrekturläst", "texter på nätet", "text hämtad från",
		"din filmsajt på nätet", "din största filmsajt på nätet",
		"alltid nya texter",
		"senaste undertexter på",
		"programtextning", "översättargrupp",
		"mediatextgruppen", "visiontext", "scandinavian text service",
		"jhs International", "svensk medietext",

		// french:
		"relecture et corrections finales:",
	}

	seq := 1
	for orgSeq, sub := range subs {

		isAd := false

		for _, line := range sub.Text {
			x := strings.ToLower(line)
			for _, adLine := range ads {
				if !isAd && strings.Contains(x, adLine) {
					isAd = true
					log.Println("[ads]", (orgSeq + 1), sub.Text, "matched", adLine)
					break
				}
			}
		}

		if !isAd {
			sub.Seq = seq
			res = append(res, sub)
			seq++
		}
	}

	return res
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/andelf/go-curl"
	"github.com/nu7hatch/gouuid"
)

type Config struct {
	Location         string
	ChecksUrl        string
	MeasurementsUrl  string
	MeasurementsUser string
	MeasurementsPass string
	MeasurerCount    int
	RecorderCount    int
}

type Check struct {
	Id  string `json:"id"`
	Url string `json:"url"`
}

type Measurement struct {
	Check             Check   `json:"check"`
	Id                string  `json:"id"`
	Location          string  `json:"location"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	LocalIp           string  `json:"local_ip,omitempty"`
	PrimaryIp         string  `json:"primary_ip,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	HttpStatus        int     `json:"http_status,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
}

func (c *Check) Measure(config Config) Measurement {
	var m Measurement

	id, _ := uuid.NewV4()
	m.Id = id.String()
	m.Check = *c
	m.Location = config.Location

	easy := curl.EasyInit()
	defer easy.Cleanup()

	easy.Setopt(curl.OPT_URL, c.Url)

	// dummy func for curl output
	noOut := func(buf []byte, userdata interface{}) bool {
		return true
	}

	easy.Setopt(curl.OPT_WRITEFUNCTION, noOut)
	easy.Setopt(curl.OPT_CONNECTTIMEOUT, 10)
	easy.Setopt(curl.OPT_TIMEOUT, 10)

	now := time.Now()
	m.T = int(now.Unix())

	if err := easy.Perform(); err != nil {
		if e, ok := err.(curl.CurlError); ok {
			m.ExitStatus = (int(e))
			return m
		}
		os.Exit(1)
	}

	m.ExitStatus = 0
	http_status, _ := easy.Getinfo(curl.INFO_RESPONSE_CODE)
	m.HttpStatus = http_status.(int)

	connect_time, _ := easy.Getinfo(curl.INFO_CONNECT_TIME)
	m.ConnectTime = connect_time.(float64)

	namelookup_time, _ := easy.Getinfo(curl.INFO_NAMELOOKUP_TIME)
	m.NameLookupTime = namelookup_time.(float64)

	starttransfer_time, _ := easy.Getinfo(curl.INFO_STARTTRANSFER_TIME)
	m.StartTransferTime = starttransfer_time.(float64)

	total_time, _ := easy.Getinfo(curl.INFO_TOTAL_TIME)
	m.TotalTime = total_time.(float64)

	local_ip, _ := easy.Getinfo(curl.INFO_LOCAL_IP)
	m.LocalIp = local_ip.(string)

	primary_ip, _ := easy.Getinfo(curl.INFO_PRIMARY_IP)
	m.PrimaryIp = primary_ip.(string)

	return m
}

func measurer(config Config, checks chan Check, measurements chan Measurement) {
	for {
		c := <-checks
		m := c.Measure(config)

		measurements <- m
	}
}

func record(config Config, payload []Measurement) {
	s, err := json.Marshal(&payload)
	if err != nil {
		panic(err)
	}

	body := bytes.NewBuffer(s)
	req, err := http.NewRequest("POST", config.MeasurementsUrl, body)
	if err != nil {
		panic(err)
	}

	req.Header.Add("Content-Type", "application/json")

	if config.MeasurementsUser != "" {
		req.SetBasicAuth(config.MeasurementsUser, config.MeasurementsPass)
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}

	fmt.Printf("fn=Record http_code=%d\n", resp.StatusCode)
	resp.Body.Close()
}

func recorder(config Config, measurements chan Measurement) {
	tickChan := time.NewTicker(time.Millisecond * 1000).C
	payload := make([]Measurement, 0, 100)

	for {
		select {
		case m := <-measurements:
			payload = append(payload, m)
		case <-tickChan:
			l := len(payload)
			fmt.Printf("fn=RecordLoop payload_size=%d\n", l)

			if l > 0 {
				record(config, payload)
				payload = make([]Measurement, 0, 100)
			}
		}
	}
}

func getChecks(config Config) []Check {
	url := config.ChecksUrl

	res, err := http.Get(url)
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		panic(err)
	}

	var checks []Check
	err = json.Unmarshal(body, &checks)
	if err != nil {
		panic(err)
	}

	return checks
}

func scheduler(check Check, checks chan Check) {
	for {
		checks <- check
		time.Sleep(1000 * time.Millisecond)
	}
}

func main() {
	config := Config{}
	flag.StringVar(&config.Location, "location", "undefined", "location of this sensor")
	flag.StringVar(&config.ChecksUrl, "checks_url", "https://s3.amazonaws.com/canary-public-data/checks.json", "URL for check data")
	flag.StringVar(&config.MeasurementsUrl, "measurements_url", "http://localhost:5000/measurements", "URL to POST measurements to")
	flag.IntVar(&config.MeasurerCount, "measurer_count", 1, "number of measurers to run")
	flag.IntVar(&config.RecorderCount, "recorder_count", 1, "number of recorders to run")
	flag.Parse()

	u, err := url.Parse(config.MeasurementsUrl)
	if err != nil {
		panic(err)
	}

	if u.User != nil {
		config.MeasurementsUser = u.User.Username()
		config.MeasurementsPass, _ = u.User.Password()
	}

	check_list := getChecks(config)

	toMeasurer := make(chan Check)
	toRecorder := make(chan Measurement)

	for i := 0; i < config.MeasurerCount; i++ {
		go measurer(config, toMeasurer, toRecorder)
	}

	for i := 0; i < config.RecorderCount; i++ {
		go recorder(config, toRecorder)
	}

	for _, c := range check_list {
		go scheduler(c, toMeasurer)
	}

	select {}
}
<commit_msg>adjust channel names in funcs, too<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/andelf/go-curl"
	"github.com/nu7hatch/gouuid"
)

type Config struct {
	Location         string
	ChecksUrl        string
	MeasurementsUrl  string
	MeasurementsUser string
	MeasurementsPass string
	MeasurerCount    int
	RecorderCount    int
}

type Check struct {
	Id  string `json:"id"`
	Url string `json:"url"`
}

type Measurement struct {
	Check             Check   `json:"check"`
	Id                string  `json:"id"`
	Location          string  `json:"location"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	LocalIp           string  `json:"local_ip,omitempty"`
	PrimaryIp         string  `json:"primary_ip,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	HttpStatus        int     `json:"http_status,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
}

func (c *Check) Measure(config Config) Measurement {
	var m Measurement

	id, _ := uuid.NewV4()
	m.Id = id.String()
	m.Check = *c
	m.Location = config.Location

	easy := curl.EasyInit()
	defer easy.Cleanup()

	easy.Setopt(curl.OPT_URL, c.Url)

	// dummy func for curl output
	noOut := func(buf []byte, userdata interface{}) bool {
		return true
	}

	easy.Setopt(curl.OPT_WRITEFUNCTION, noOut)
	easy.Setopt(curl.OPT_CONNECTTIMEOUT, 10)
	easy.Setopt(curl.OPT_TIMEOUT, 10)

	now := time.Now()
	m.T = int(now.Unix())

	if err := easy.Perform(); err != nil {
		if e, ok := err.(curl.CurlError); ok {
			m.ExitStatus = (int(e))
			return m
		}
		os.Exit(1)
	}

	m.ExitStatus = 0
	http_status, _ := easy.Getinfo(curl.INFO_RESPONSE_CODE)
	m.HttpStatus = http_status.(int)

	connect_time, _ := easy.Getinfo(curl.INFO_CONNECT_TIME)
	m.ConnectTime = connect_time.(float64)

	namelookup_time, _ := easy.Getinfo(curl.INFO_NAMELOOKUP_TIME)
	m.NameLookupTime = namelookup_time.(float64)

	starttransfer_time, _ := easy.Getinfo(curl.INFO_STARTTRANSFER_TIME)
	m.StartTransferTime = starttransfer_time.(float64)

	total_time, _ := easy.Getinfo(curl.INFO_TOTAL_TIME)
	m.TotalTime = total_time.(float64)

	local_ip, _ := easy.Getinfo(curl.INFO_LOCAL_IP)
	m.LocalIp = local_ip.(string)

	primary_ip, _ := easy.Getinfo(curl.INFO_PRIMARY_IP)
	m.PrimaryIp = primary_ip.(string)

	return m
}

func measurer(config Config, toMeasurer chan Check, toRecorder chan Measurement) {
	for {
		c := <-toMeasurer
		m := c.Measure(config)

		toRecorder <- m
	}
}

func record(config Config, payload []Measurement) {
	s, err := json.Marshal(&payload)
	if err != nil {
		panic(err)
	}

	body := bytes.NewBuffer(s)
	req, err := http.NewRequest("POST", config.MeasurementsUrl, body)
	if err != nil {
		panic(err)
	}

	req.Header.Add("Content-Type", "application/json")

	if config.MeasurementsUser != "" {
		req.SetBasicAuth(config.MeasurementsUser, config.MeasurementsPass)
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}

	fmt.Printf("fn=Record http_code=%d\n", resp.StatusCode)
	resp.Body.Close()
}

func recorder(config Config, toRecorder chan Measurement) {
	tickChan := time.NewTicker(time.Millisecond * 1000).C
	payload := make([]Measurement, 0, 100)

	for {
		select {
		case m := <-toRecorder:
			payload = append(payload, m)
		case <-tickChan:
			l := len(payload)
			fmt.Printf("fn=RecordLoop payload_size=%d\n", l)

			if l > 0 {
				record(config, payload)
				payload = make([]Measurement, 0, 100)
			}
		}
	}
}

func getChecks(config Config) []Check {
	url := config.ChecksUrl

	res, err := http.Get(url)
	if err != nil {
		panic(err)
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		panic(err)
	}

	var checks []Check
	err = json.Unmarshal(body, &checks)
	if err != nil {
		panic(err)
	}

	return checks
}

func scheduler(check Check, toMeasurer chan Check) {
	for {
		toMeasurer <- check
		time.Sleep(1000 * time.Millisecond)
	}
}

func main() {
	config := Config{}
	flag.StringVar(&config.Location, "location", "undefined", "location of this sensor")
	flag.StringVar(&config.ChecksUrl, "checks_url", "https://s3.amazonaws.com/canary-public-data/checks.json", "URL for check data")
	flag.StringVar(&config.MeasurementsUrl, "measurements_url", "http://localhost:5000/measurements", "URL to POST measurements to")
	flag.IntVar(&config.MeasurerCount, "measurer_count", 1, "number of measurers to run")
	flag.IntVar(&config.RecorderCount, "recorder_count", 1, "number of recorders to run")
	flag.Parse()

	u, err := url.Parse(config.MeasurementsUrl)
	if err != nil {
		panic(err)
	}

	if u.User != nil {
		config.MeasurementsUser = u.User.Username()
		config.MeasurementsPass, _ = u.User.Password()
	}

	check_list := getChecks(config)

	toMeasurer := make(chan Check)
	toRecorder := make(chan Measurement)

	for i := 0; i < config.MeasurerCount; i++ {
		go measurer(config, toMeasurer, toRecorder)
	}

	for i := 0; i < config.RecorderCount; i++ {
		go recorder(config, toRecorder)
	}

	for _, c := range check_list {
		go scheduler(c, toMeasurer)
	}

	select {}
}
<|endoftext|>


<commit_before>package sprok

/*
 * Copyright 2014 Albert P. Tobey <atobey@datastax.com> @AlTobey
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"fmt"
	"log"
	"os"
	"strings"
	"syscall"
)

type Process struct {
	Env    map[string]string `"env"`
	Argv   []string          `"argv"`
	Stdin  string            `"stdin"`
	Stdout string            `"stdout"`
	Stderr string            `"stderr"`
}

// NewProcess returns a Process struct with the env map and argv allocated
// and all stdio pointed at /dev/null.
func NewProcess() Process {
	return Process{
		Env:    map[string]string{},
		Argv:   make([]string, 1),
		Stdin:  "/dev/null",
		Stdout: "/dev/null",
		Stderr: "/dev/null",
	}
}

// Exec executes Argv with environment Env and file descriptors 1, 2, and 3 open on the
// files specified in Stdin, Stdout, and Stderr. They all default to /dev/null if left unspecified
// in the config or for empty string "".
func (p *Process) Exec() error {
	stdin := openFile(p.Stdin, os.O_RDONLY)
	stdout := openFile(p.Stdout, os.O_WRONLY|os.O_CREATE|os.O_APPEND)
	stderr := openFile(p.Stderr, os.O_WRONLY|os.O_CREATE|os.O_APPEND)

	err := syscall.Dup2(int(stdin.Fd()), int(os.Stdin.Fd()))
	if err != nil {
		log.Fatalf("Failed to redirect stdin: %s\n", err)
	}

	err = syscall.Dup2(int(stdout.Fd()), int(os.Stdout.Fd()))
	if err != nil {
		log.Fatalf("Failed to redirect stdout: %s\n", err)
	}

	err = syscall.Dup2(int(stderr.Fd()), int(os.Stderr.Fd()))
	if err != nil {
		log.Fatalf("Failed to redirect stderr: %s\n", err)
	}

	return syscall.Exec(p.Argv[0], p.Argv[1:], p.envPairs())
}

func (p *Process) String() string {
	env := strings.Join(p.envPairs(), " ")
	cmd := strings.Join(p.Argv, " ")
	// FOO=BAR cmd -arg1 -arg2 foo < /dev/null 1>/dev/null 2>/dev/null
	return fmt.Sprintf("%s %s <%s 1>%s 2>%s", env, cmd, p.Stdin, p.Stdout, p.Stderr)
}

func (p *Process) envPairs() []string {
	env := make([]string, len(p.Env))
	i := 0
	for key, value := range p.Env {
		env[i] = fmt.Sprintf("%s=%s", key, value)
		i++
	}
	return env
}

func openFile(name string, flags int) *os.File {
	if name == "/dev/null" {
		name = os.DevNull
	}

	f, err := os.OpenFile(name, flags, 0644)
	if err != nil {
		log.Fatalf("Could not open file '%s': %s\n", name, err)
	}

	return f
}
<commit_msg>support 2>&1 style output to the same file<commit_after>package sprok

/*
 * Copyright 2014 Albert P. Tobey <atobey@datastax.com> @AlTobey
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"fmt"
	"log"
	"os"
	"strings"
	"syscall"
)

type Process struct {
	Env    map[string]string `"env"`
	Argv   []string          `"argv"`
	Stdin  string            `"stdin"`
	Stdout string            `"stdout"`
	Stderr string            `"stderr"`
}

// NewProcess returns a Process struct with the env map and argv allocated
// and all stdio pointed at /dev/null.
func NewProcess() Process {
	return Process{
		Env:    map[string]string{},
		Argv:   make([]string, 1),
		Stdin:  "/dev/null",
		Stdout: "/dev/null",
		Stderr: "/dev/null",
	}
}

// Exec executes Argv with environment Env and file descriptors 1, 2, and 3 open on the
// files specified in Stdin, Stdout, and Stderr. They all default to /dev/null if left unspecified
// in the config or for empty string "".
func (p *Process) Exec() error {
	stdin := openFile(p.Stdin, os.O_RDONLY)
	stdout := openFile(p.Stdout, os.O_WRONLY|os.O_CREATE|os.O_APPEND)

	var stderr *os.File
	if p.Stdout == p.Stderr {
		stderr = stdout
	} else {
		stderr = openFile(p.Stderr, os.O_WRONLY|os.O_CREATE|os.O_APPEND)
	}

	err := syscall.Dup2(int(stdin.Fd()), int(os.Stdin.Fd()))
	if err != nil {
		log.Fatalf("Failed to redirect stdin: %s\n", err)
	}

	err = syscall.Dup2(int(stdout.Fd()), int(os.Stdout.Fd()))
	if err != nil {
		log.Fatalf("Failed to redirect stdout: %s\n", err)
	}

	err = syscall.Dup2(int(stderr.Fd()), int(os.Stderr.Fd()))
	if err != nil {
		log.Fatalf("Failed to redirect stderr: %s\n", err)
	}

	return syscall.Exec(p.Argv[0], p.Argv[1:], p.envPairs())
}

func (p *Process) String() string {
	env := strings.Join(p.envPairs(), " ")
	cmd := strings.Join(p.Argv, " ")
	// FOO=BAR cmd -arg1 -arg2 foo < /dev/null 1>/dev/null 2>/dev/null
	return fmt.Sprintf("%s %s <%s 1>%s 2>%s", env, cmd, p.Stdin, p.Stdout, p.Stderr)
}

func (p *Process) envPairs() []string {
	env := make([]string, len(p.Env))
	i := 0
	for key, value := range p.Env {
		env[i] = fmt.Sprintf("%s=%s", key, value)
		i++
	}
	return env
}

func openFile(name string, flags int) *os.File {
	if name == "/dev/null" {
		name = os.DevNull
	}

	f, err := os.OpenFile(name, flags, 0644)
	if err != nil {
		log.Fatalf("Could not open file '%s': %s\n", name, err)
	}

	return f
}
<|endoftext|>