<commit_before>//
// message.go
//
package srnd

import (
  "encoding/base64"
  "fmt"
  "io"
  "log"
  "mime"
  "mime/multipart"
  "net/textproto"
  "strings"
  "time"
)

type ArticleHeaders map[string][]string

func (self ArticleHeaders) Has(key string) bool {
  _, ok := self[key]
  return ok
}

func (self ArticleHeaders) Set(key, val string) {
  self[key] = []string{val}
}

func (self ArticleHeaders) Get(key, fallback string) string {
  val , ok := self[key]
  if ok {
    return val[0]
  } else {
    return fallback
  }
}

type NNTPMessage interface {
  // this message's messsge id
  MessageID() string
  // the parent message's messageid if it's specified
  Reference() string
  // the newsgroup this post is in
  Newsgroup() string
  // the name of the poster
  Name() string
  // any email address associated with the post
  Email() string
  // the subject of the post
  Subject() string
  // when this was posted
  Posted() int64
  // the path header
  Path() string
  // append something to path
  // return message with new path
  AppendPath(part string) NNTPMessage
  // the type of this message usually a mimetype
  ContentType() string
  // was this post a sage?
  Sage() bool
  // was this post a root post?
  OP() bool
  // all attachments
  Attachments() []NNTPAttachment
  // all headers
  Headers() ArticleHeaders
  // write out body
  WriteBody(wr io.Writer) error
  // attach a file
  Attach(att NNTPAttachment) NNTPMessage
  // get the plaintext message if it exists
  Message() string
  // pack the whole message and prepare for write
  Pack()
}

type MessageReader interface {
  // read a message from a reader
  ReadMessage(r io.Reader) (NNTPMessage, error)
}

type MessageWriter interface {
  // write a message to a writer
  WriteMessage(nntp NNTPMessage, wr io.Writer) error
}

type MessageSigner interface {
  // sign a message, return a new message that is signed
  SignMessage(nntp NNTPMessage) (NNTPMessage, error)
}

type MessageVerifier interface {
  // verify a message, return the message that is verified
  VerifyMessage(nntp NNTPMessage) (NNTPMessage, error)
}

type nntpArticle struct {
  headers ArticleHeaders
  boundary string
  message nntpAttachment
  attachments []NNTPAttachment
}

// create a simple plaintext nntp message
func newPlaintextArticle(message, email, subject, name, instance, newsgroup string) NNTPMessage {
  nntp := nntpArticle{
    headers: make(ArticleHeaders),
  }
  nntp.headers.Set("From", fmt.Sprintf("%s <%s>", name, email))
  nntp.headers.Set("Subject", subject)
  if isSage(subject) {
    nntp.headers.Set("X-Sage", "1")
  }
  nntp.headers.Set("Path", instance)
  nntp.headers.Set("Message-ID", genMessageID(instance))
  // posted now
  nntp.headers.Set("Date", timeNowStr())
  nntp.headers.Set("Newsgroups", newsgroup)
  nntp.message = createPlaintextAttachment(message)
  nntp.Pack()
  return nntp
}

func (self nntpArticle) MessageID() string {
  return self.headers.Get("Message-ID", self.headers.Get("Messageid", self.headers.Get("MessageID", self.headers.Get("Message-Id", ""))))
}

func (self nntpArticle) Pack() {
  if len(self.boundary) == 0 {
    self.boundary = randStr(24)
    self.headers.Set("Mime-Version", "1.0")
    self.headers.Set("Content-Type", fmt.Sprintf("multitype/mixed; boundary=%s", self.boundary))
    log.Println("pack article boundary is", self.boundary)
  }

}

func (self nntpArticle) Reference() string {
  return self.headers.Get("Reference", "")
}

func (self nntpArticle) Newsgroup() string {
  return self.headers.Get("Newsgroups", "")
}

func (self nntpArticle) Name() string {
  from := self.headers.Get("From", "anonymous <a@no.n>")
  idx := strings.Index(from, " ")
  if idx > 1 {
    return from[:idx]
  }
  return "[Invalid From header]"
}

func (self nntpArticle) Email() string {
  from := self.headers.Get("From", "anonymous <a@no.n>")
  idx := strings.Index(from, " ")
  if idx > 1 {
    idx_1 := strings.Index(from[:idx], "<")
    idx_2 := strings.Index(from[:idx], ">")
    if idx_2 > 0 && idx_1 > 0 && idx_2 > idx_1 {
      return from[1+idx+idx_1:idx+idx_2]
    }
  }
  return "[Invalid From header]"

}

func (self nntpArticle) Subject() string {
  return self.headers.Get("Subject", "")
}

func (self nntpArticle) Posted() int64 {
  posted := self.headers.Get("Date", "")
  t, err := time.Parse(time.RFC1123Z, posted)
  if err == nil {
    return t.Unix()
  }
  return 0
}

func (self nntpArticle) Message() string {
  return self.message.body.String()
}

func (self nntpArticle) Path() string {
  return self.headers.Get("Path", "unspecified")
}

func (self nntpArticle) Headers() ArticleHeaders {
  return self.headers
}

func (self nntpArticle) AppendPath(part string) NNTPMessage {
  if self.headers.Has("Path") {
    self.headers.Set("Path", part + "!" + self.Path())
  } else {
    self.headers.Set("Path", part)
  }
  return self
}
func (self nntpArticle) ContentType() string {
  // assumes text/plain if unspecified
  return self.headers.Get("Content-Type", "text/plain")
}

func (self nntpArticle) Sage() bool {
  return self.headers.Has("X-Sage")
}

func (self nntpArticle) OP() bool {
  return ! self.headers.Has("Reference")
}

func (self nntpArticle) Attachments() []NNTPAttachment {
  return self.attachments
}

func (self nntpArticle) Attach(att NNTPAttachment) NNTPMessage {
  log.Println("attaching file", att.Filename())
  self.attachments = append(self.attachments, att)
  return self
}

func (self nntpArticle) WriteBody(wr io.Writer) (err error) {
  log.Printf("writing %d attachments", len(self.attachments))
  content_type := self.ContentType()
  _, params, err := mime.ParseMediaType(content_type)
  if err != nil {
    log.Println("failed to parse media type", err)
    return err
  }

  boundary, ok := params["boundary"]
  if ok {
    w := multipart.NewWriter(wr)

    err = w.SetBoundary(boundary)
    if err == nil {
      attachments := []NNTPAttachment{self.message}
      attachments = append(attachments, self.attachments...)
      log.Println("writing nntp body")
      for _ , att := range(attachments) {
        hdr := att.Header()
        if hdr == nil {
          hdr = make(textproto.MIMEHeader)
        }
        hdr.Set("Content-Transfer-Encoding", "base64")
        part, err := w.CreatePart(hdr)
        if err != nil {
          log.Println("error writing part", err)
        }
        if strings.HasPrefix(att.Mime(), "text/plain") {
          err = att.WriteTo(part)
        } else {
          enc := base64.NewEncoder(base64.StdEncoding, part)
          err = att.WriteTo(enc)
          enc.Close()
        }
        if err != nil {
          break
        }
      }
    }
    if err != nil {
      log.Println(err)
    }
    err = w.Close()
  } else {
    _, err = self.message.body.WriteTo(wr)
  }
  return err
}
<commit_msg>multipart<commit_after>//
// message.go
//
package srnd

import (
  "encoding/base64"
  "fmt"
  "io"
  "log"
  "mime"
  "mime/multipart"
  "net/textproto"
  "strings"
  "time"
)

type ArticleHeaders map[string][]string

func (self ArticleHeaders) Has(key string) bool {
  _, ok := self[key]
  return ok
}

func (self ArticleHeaders) Set(key, val string) {
  self[key] = []string{val}
}

func (self ArticleHeaders) Get(key, fallback string) string {
  val , ok := self[key]
  if ok {
    return val[0]
  } else {
    return fallback
  }
}

type NNTPMessage interface {
  // this message's messsge id
  MessageID() string
  // the parent message's messageid if it's specified
  Reference() string
  // the newsgroup this post is in
  Newsgroup() string
  // the name of the poster
  Name() string
  // any email address associated with the post
  Email() string
  // the subject of the post
  Subject() string
  // when this was posted
  Posted() int64
  // the path header
  Path() string
  // append something to path
  // return message with new path
  AppendPath(part string) NNTPMessage
  // the type of this message usually a mimetype
  ContentType() string
  // was this post a sage?
  Sage() bool
  // was this post a root post?
  OP() bool
  // all attachments
  Attachments() []NNTPAttachment
  // all headers
  Headers() ArticleHeaders
  // write out body
  WriteBody(wr io.Writer) error
  // attach a file
  Attach(att NNTPAttachment) NNTPMessage
  // get the plaintext message if it exists
  Message() string
  // pack the whole message and prepare for write
  Pack()
}

type MessageReader interface {
  // read a message from a reader
  ReadMessage(r io.Reader) (NNTPMessage, error)
}

type MessageWriter interface {
  // write a message to a writer
  WriteMessage(nntp NNTPMessage, wr io.Writer) error
}

type MessageSigner interface {
  // sign a message, return a new message that is signed
  SignMessage(nntp NNTPMessage) (NNTPMessage, error)
}

type MessageVerifier interface {
  // verify a message, return the message that is verified
  VerifyMessage(nntp NNTPMessage) (NNTPMessage, error)
}

type nntpArticle struct {
  headers ArticleHeaders
  boundary string
  message nntpAttachment
  attachments []NNTPAttachment
}

// create a simple plaintext nntp message
func newPlaintextArticle(message, email, subject, name, instance, newsgroup string) NNTPMessage {
  nntp := nntpArticle{
    headers: make(ArticleHeaders),
  }
  nntp.headers.Set("From", fmt.Sprintf("%s <%s>", name, email))
  nntp.headers.Set("Subject", subject)
  if isSage(subject) {
    nntp.headers.Set("X-Sage", "1")
  }
  nntp.headers.Set("Path", instance)
  nntp.headers.Set("Message-ID", genMessageID(instance))
  // posted now
  nntp.headers.Set("Date", timeNowStr())
  nntp.headers.Set("Newsgroups", newsgroup)
  nntp.message = createPlaintextAttachment(message)
  nntp.Pack()
  return nntp
}

func (self nntpArticle) MessageID() string {
  return self.headers.Get("Message-ID", self.headers.Get("Messageid", self.headers.Get("MessageID", self.headers.Get("Message-Id", ""))))
}

func (self nntpArticle) Pack() {
  if len(self.boundary) == 0 {
    self.boundary = randStr(24)
    self.headers.Set("Mime-Version", "1.0")
    self.headers.Set("Content-Type", fmt.Sprintf("multipart/mixed; boundary=%s", self.boundary))
    log.Println("pack article boundary is", self.boundary)
  }

}

func (self nntpArticle) Reference() string {
  return self.headers.Get("Reference", "")
}

func (self nntpArticle) Newsgroup() string {
  return self.headers.Get("Newsgroups", "")
}

func (self nntpArticle) Name() string {
  from := self.headers.Get("From", "anonymous <a@no.n>")
  idx := strings.Index(from, " ")
  if idx > 1 {
    return from[:idx]
  }
  return "[Invalid From header]"
}

func (self nntpArticle) Email() string {
  from := self.headers.Get("From", "anonymous <a@no.n>")
  idx := strings.Index(from, " ")
  if idx > 1 {
    idx_1 := strings.Index(from[:idx], "<")
    idx_2 := strings.Index(from[:idx], ">")
    if idx_2 > 0 && idx_1 > 0 && idx_2 > idx_1 {
      return from[1+idx+idx_1:idx+idx_2]
    }
  }
  return "[Invalid From header]"

}

func (self nntpArticle) Subject() string {
  return self.headers.Get("Subject", "")
}

func (self nntpArticle) Posted() int64 {
  posted := self.headers.Get("Date", "")
  t, err := time.Parse(time.RFC1123Z, posted)
  if err == nil {
    return t.Unix()
  }
  return 0
}

func (self nntpArticle) Message() string {
  return self.message.body.String()
}

func (self nntpArticle) Path() string {
  return self.headers.Get("Path", "unspecified")
}

func (self nntpArticle) Headers() ArticleHeaders {
  return self.headers
}

func (self nntpArticle) AppendPath(part string) NNTPMessage {
  if self.headers.Has("Path") {
    self.headers.Set("Path", part + "!" + self.Path())
  } else {
    self.headers.Set("Path", part)
  }
  return self
}
func (self nntpArticle) ContentType() string {
  // assumes text/plain if unspecified
  return self.headers.Get("Content-Type", "text/plain")
}

func (self nntpArticle) Sage() bool {
  return self.headers.Has("X-Sage")
}

func (self nntpArticle) OP() bool {
  return ! self.headers.Has("Reference")
}

func (self nntpArticle) Attachments() []NNTPAttachment {
  return self.attachments
}

func (self nntpArticle) Attach(att NNTPAttachment) NNTPMessage {
  log.Println("attaching file", att.Filename())
  self.attachments = append(self.attachments, att)
  return self
}

func (self nntpArticle) WriteBody(wr io.Writer) (err error) {
  log.Printf("writing %d attachments", len(self.attachments))
  content_type := self.ContentType()
  _, params, err := mime.ParseMediaType(content_type)
  if err != nil {
    log.Println("failed to parse media type", err)
    return err
  }

  boundary, ok := params["boundary"]
  if ok {
    w := multipart.NewWriter(wr)

    err = w.SetBoundary(boundary)
    if err == nil {
      attachments := []NNTPAttachment{self.message}
      attachments = append(attachments, self.attachments...)
      log.Println("writing nntp body")
      for _ , att := range(attachments) {
        hdr := att.Header()
        if hdr == nil {
          hdr = make(textproto.MIMEHeader)
        }
        hdr.Set("Content-Transfer-Encoding", "base64")
        part, err := w.CreatePart(hdr)
        if err != nil {
          log.Println("error writing part", err)
        }
        if strings.HasPrefix(att.Mime(), "text/plain") {
          err = att.WriteTo(part)
        } else {
          enc := base64.NewEncoder(base64.StdEncoding, part)
          err = att.WriteTo(enc)
          enc.Close()
        }
        if err != nil {
          break
        }
      }
    }
    if err != nil {
      log.Println(err)
    }
    err = w.Close()
  } else {
    _, err = self.message.body.WriteTo(wr)
  }
  return err
}
<|endoftext|>


<commit_before>/*
memberlist is a library that manages cluster
membership and member failure detection using a gossip based protocol.

The use cases for such a library are far-reaching: all distributed systems
require membership, and memberlist is a re-usable solution to managing
cluster membership and node failure detection.

memberlist is eventually consistent but converges quickly on average.
The speed at which it converges can be heavily tuned via various knobs
on the protocol. Node failures are detected and network partitions are partially
tolerated by attempting to communicate to potentially dead nodes through
multiple routes.
*/
package memberlist

import (
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"sync"
	"time"
)

type Config struct {
	// The name of this node. This must be unique in the cluster.
	Name string

	// Configuration related to what address to bind to and ports to
	// listen on. The ports used must match every node in the cluster,
	// since they'll be used for connecting as well as listening.
	BindAddr string
	UDPPort  int
	TCPPort  int

	// TCPTimeout is the timeout for establishing a TCP connection with
	// a remote node for a full state sync.
	TCPTimeout time.Duration

	// IndirectChecks is the number of nodes that will be asked to perform
	// an indirect probe of a node in the case a direct probe fails. Memberlist
	// waits for an ack from any single indirect node, so increasing this
	// number will increase the likelihood that an indirect probe will succeed
	// at the expense of bandwidth.
	IndirectChecks int

	// RetransmitMult is the multiplier for the number of retransmissions
	// that are attempted for messages broadcasted over gossip. The actual
	// count of retransmissions is calculated using the formula:
	//
	//   Retransmits = RetransmitMult * log(N+1)
	//
	// This allows the retransmits to scale properly with cluster size. The
	// higher the multiplier, the more likely a failed broadcast is to converge
	// at the expense of increased bandwidth.
	RetransmitMult int

	// SuspicionMult is the multiplier for determining the time an
	// inaccessible node is considered suspect before declaring it dead.
	// The actual timeout is calculated using the formula:
	//
	//   SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
	//
	// This allows the timeout to scale properly with expected propagation
	// delay with a larger cluster size. The higher the multiplier, the longer
	// an inaccessible node is considered part of the cluster before declaring
	// it dead, giving that suspect node more time to refute if it is indeed
	// still alive.
	SuspicionMult int

	// PushPullInterval is the interval between complete state syncs.
	// Complete state syncs are done with a single node over TCP and are
	// quite expensive relative to standard gossiped messages. Setting this
	// to zero will disable state push/pull syncs completely.
	//
	// Setting this interval lower (more frequent) will increase convergence
	// speeds across larger clusters at the expense of increased bandwidth
	// usage.
	PushPullInterval time.Duration

	// ProbeInterval and ProbeTimeout are used to configure probing
	// behavior for memberlist.
	//
	// ProbeInterval is the interval between random node probes. Setting
	// this lower (more frequent) will cause the memberlist cluster to detect
	// failed nodes more quickly at the expense of increased bandwidth usage.
	//
	// ProbeTimeout is the timeout to wait for an ack from a probed node
	// before assuming it is unhealthy. This should be set to 99-percentile
	// of RTT (round-trip time) on your network.
	ProbeInterval time.Duration
	ProbeTimeout  time.Duration

	// GossipInterval and GossipNodes are used to configure the gossip
	// behavior of memberlist.
	//
	// GossipInterval is the interval between sending messages that need
	// to be gossiped that haven't been able to piggyback on probing messages.
	// If this is set to zero, non-piggyback gossip is disabled. By lowering
	// this value (more frequent) gossip messages are propagated across
	// the cluster more quickly at the expense of increased bandwidth.
	//
	// GossipNodes is the number of random nodes to send gossip messages to
	// per GossipInterval. Increasing this number causes the gossip messages
	// to propagate across the cluster more quickly at the expense of
	// increased bandwidth.
	GossipInterval time.Duration
	GossipNodes    int

	// Delegate and Events are delegates for receiving and providing
	// data to memberlist via callback mechanisms. For Delegate, see
	// the Delegate interface. For Events, see the EventDelegate interface.
	Delegate Delegate
	Events   EventDelegate

	// LogOutput is the writer where logs should be sent. If this is not
	// set, logging will go to stderr by default.
	LogOutput io.Writer
}

type Memberlist struct {
	config         *Config
	shutdown       bool
	leave          bool
	leaveBroadcast chan struct{}

	udpListener *net.UDPConn
	tcpListener *net.TCPListener

	sequenceNum uint32 // Local sequence number
	incarnation uint32 // Local incarnation number

	nodeLock sync.RWMutex
	nodes    []*nodeState          // Known nodes
	nodeMap  map[string]*nodeState // Maps Addr.String() -> NodeState

	tickerLock sync.Mutex
	tickers    []*time.Ticker
	stopTick   chan struct{}
	probeIndex int

	ackLock     sync.Mutex
	ackHandlers map[uint32]*ackHandler

	broadcasts *TransmitLimitedQueue

	startStopLock sync.Mutex

	logger *log.Logger
}

// DefaultConfig returns a sane set of configurations for Memberlist.
// It uses the hostname as the node name, and otherwise sets very conservative
// values that are sane for most LAN environments. The default configuration
// errs on the side on the side of caution, choosing values that are optimized
// for higher convergence at the cost of higher bandwidth usage. Regardless,
// these values are a good starting point when getting started with memberlist.
func DefaultConfig() *Config {
	hostname, _ := os.Hostname()
	return &Config{
		Name:             hostname,
		BindAddr:         "0.0.0.0",
		UDPPort:          7946,
		TCPPort:          7946,
		TCPTimeout:       10 * time.Second,       // Timeout after 10 seconds
		IndirectChecks:   3,                      // Use 3 nodes for the indirect ping
		RetransmitMult:   4,                      // Retransmit a message 4 * log(N+1) nodes
		SuspicionMult:    5,                      // Suspect a node for 5 * log(N+1) * Interval
		PushPullInterval: 30 * time.Second,       // Low frequency
		ProbeTimeout:     500 * time.Millisecond, // Reasonable RTT time for LAN
		ProbeInterval:    1 * time.Second,        // Failure check every second

		GossipNodes:    3,                      // Gossip to 3 nodes
		GossipInterval: 200 * time.Millisecond, // Gossip more rapidly
	}
}

// newMemberlist creates the network listeners.
// Does not schedule execution of background maintenence.
func newMemberlist(conf *Config) (*Memberlist, error) {
	tcpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.TCPPort)
	tcpLn, err := net.Listen("tcp", tcpAddr)
	if err != nil {
		return nil, fmt.Errorf("Failed to start TCP listener. Err: %s", err)
	}

	udpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.UDPPort)
	udpLn, err := net.ListenPacket("udp", udpAddr)
	if err != nil {
		tcpLn.Close()
		return nil, fmt.Errorf("Failed to start UDP listener. Err: %s", err)
	}

	// Set the UDP receive window size
	setUDPRecvBuf(udpLn.(*net.UDPConn))

	if conf.LogOutput == nil {
		conf.LogOutput = os.Stderr
	}

	m := &Memberlist{
		config:         conf,
		leaveBroadcast: make(chan struct{}, 1),
		udpListener:    udpLn.(*net.UDPConn),
		tcpListener:    tcpLn.(*net.TCPListener),
		nodeMap:        make(map[string]*nodeState),
		ackHandlers:    make(map[uint32]*ackHandler),
		broadcasts:     &TransmitLimitedQueue{RetransmitMult: conf.RetransmitMult},
		logger:         log.New(conf.LogOutput, "", log.LstdFlags),
	}
	m.broadcasts.NumNodes = func() int { return len(m.nodes) }
	go m.tcpListen()
	go m.udpListen()
	return m, nil
}

// Create will create a new Memberlist using the given configuration.
// This will not connect to any other node (see Join) yet, but will start
// all the listeners to allow other nodes to join this memberlist.
// After creating a Memberlist, the configuration given should not be
// modified by the user anymore.
func Create(conf *Config) (*Memberlist, error) {
	m, err := newMemberlist(conf)
	if err != nil {
		return nil, err
	}
	if err := m.setAlive(); err != nil {
		m.Shutdown()
		return nil, err
	}
	m.schedule()
	return m, nil
}

// Join is used to take an existing Memberlist and attempt to join a cluster
// by contacting all the given hosts and performing a state sync. Initially,
// the Memberlist only contains our own state, so doing this will cause
// remote nodes to become aware of the existence of this node, effectively
// joining the cluster.
//
// This returns the number of hosts successfully contacted and an error if
// none could be reached. If an error is returned, the node did not successfully
// join the cluster.
func (m *Memberlist) Join(existing []string) (int, error) {
	// Attempt to join any of them
	numSuccess := 0
	var retErr error
	for _, exist := range existing {
		addr, err := net.ResolveIPAddr("ip", exist)
		if err != nil {
			retErr = err
			continue
		}

		if err := m.pushPullNode(addr.IP); err != nil {
			retErr = err
			continue
		}

		numSuccess++
	}

	if numSuccess > 0 {
		retErr = nil
	}

	return numSuccess, retErr
}

// setAlive is used to mark this node as being alive. This is the same
// as if we received an alive notification our own network channel for
// ourself.
func (m *Memberlist) setAlive() error {
	// Pick a private IP address
	var ipAddr []byte
	if m.config.BindAddr == "0.0.0.0" {
		// We're not bound to a specific IP, so let's list the interfaces
		// on this machine and use the first private IP we find.
		addresses, err := net.InterfaceAddrs()
		if err != nil {
			return fmt.Errorf("Failed to get interface addresses! Err: %vn", err)
		}

		// Find private IPv4 address
		for _, addr := range addresses {
			ip, ok := addr.(*net.IPNet)
			if !ok {
				continue
			}
			if ip.IP.To4() == nil {
				continue
			}
			if !isPrivateIP(ip.IP.String()) {
				continue
			}
			ipAddr = ip.IP
			break
		}

		// Failed to find private IP, use loopback
		if ipAddr == nil {
			ipAddr = []byte{127, 0, 0, 1}
		}
	} else {
		// Use the IP that we're bound to.
		addr := m.tcpListener.Addr().(*net.TCPAddr)
		ipAddr = addr.IP
	}

	// Get the node meta data
	var meta []byte
	if m.config.Delegate != nil {
		meta = m.config.Delegate.NodeMeta(metaMaxSize)
		if len(meta) > metaMaxSize {
			panic("Node meta data provided is longer than the limit")
		}
	}

	a := alive{
		Incarnation: m.nextIncarnation(),
		Node:        m.config.Name,
		Addr:        ipAddr,
		Meta:        meta,
	}
	m.aliveNode(&a)

	return nil
}

// Members returns a list of all known live nodes. The node structures
// returned must not be modified. If you wish to modify a Node, make a
// copy first.
func (m *Memberlist) Members() []*Node {
	m.nodeLock.RLock()
	defer m.nodeLock.RUnlock()

	nodes := make([]*Node, 0, len(m.nodes))
	for _, n := range m.nodes {
		if n.State != stateDead {
			nodes = append(nodes, &n.Node)
		}
	}

	return nodes
}

// NumMembers returns the number of alive nodes currently known. Between
// the time of calling this and calling Members, the number of alive nodes
// may have changed, so this shouldn't be used to determine how many
// members will be returned by Members.
func (m *Memberlist) NumMembers() (alive int) {
	m.nodeLock.RLock()
	defer m.nodeLock.RUnlock()

	for _, n := range m.nodes {
		if n.State != stateDead {
			alive++
		}
	}

	return
}

// Leave will broadcast a leave message but will not shutdown the background
// listeners, meaning the node will continue participating in gossip and state
// updates.
//
// This will block until the leave message is successfully broadcasted to
// a member of the cluster, if any exist or until a specified timeout
// is reached.
//
// This method is safe to call multiple times, but must not be called
// after the cluster is already shut down.
func (m *Memberlist) Leave(timeout time.Duration) error {
	m.startStopLock.Lock()
	defer m.startStopLock.Unlock()

	if m.shutdown {
		panic("leave after shutdown")
	}

	if !m.leave {
		m.leave = true

		state, ok := m.nodeMap[m.config.Name]
		if !ok {
			m.logger.Println("[WARN] Leave but we're not in the node map.")
			return nil
		}

		d := dead{
			Incarnation: state.Incarnation,
			Node:        state.Name,
		}
		m.deadNode(&d)

		// Check for any other alive node
		anyAlive := false
		for _, n := range m.nodes {
			if n.State != stateDead {
				anyAlive = true
				break
			}
		}

		// Block until the broadcast goes out
		if anyAlive {
			var timeoutCh <-chan time.Time
			if timeout > 0 {
				timeoutCh = time.After(timeout)
			}
			select {
			case <-m.leaveBroadcast:
			case <-timeoutCh:
				return fmt.Errorf("timeout waiting for leave broadcast")
			}
		}
	}

	return nil
}

// Shutdown will stop any background maintanence of network activity
// for this memberlist, causing it to appear "dead". A leave message
// will not be broadcasted prior, so the cluster being left will have
// to detect this node's shutdown using probing. If you wish to more
// gracefully exit the cluster, call Leave prior to shutting down.
//
// This method is safe to call multiple times.
func (m *Memberlist) Shutdown() error {
	m.startStopLock.Lock()
	defer m.startStopLock.Unlock()

	if !m.shutdown {
		m.shutdown = true
		m.deschedule()
		m.udpListener.Close()
		m.tcpListener.Close()
	}

	return nil
}
<commit_msg>adding configuration for encryption and compression<commit_after>/*
memberlist is a library that manages cluster
membership and member failure detection using a gossip based protocol.

The use cases for such a library are far-reaching: all distributed systems
require membership, and memberlist is a re-usable solution to managing
cluster membership and node failure detection.

memberlist is eventually consistent but converges quickly on average.
The speed at which it converges can be heavily tuned via various knobs
on the protocol. Node failures are detected and network partitions are partially
tolerated by attempting to communicate to potentially dead nodes through
multiple routes.
*/
package memberlist

import (
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"sync"
	"time"
)

type Config struct {
	// The name of this node. This must be unique in the cluster.
	Name string

	// Configuration related to what address to bind to and ports to
	// listen on. The ports used must match every node in the cluster,
	// since they'll be used for connecting as well as listening.
	BindAddr string
	UDPPort  int
	TCPPort  int

	// TCPTimeout is the timeout for establishing a TCP connection with
	// a remote node for a full state sync.
	TCPTimeout time.Duration

	// IndirectChecks is the number of nodes that will be asked to perform
	// an indirect probe of a node in the case a direct probe fails. Memberlist
	// waits for an ack from any single indirect node, so increasing this
	// number will increase the likelihood that an indirect probe will succeed
	// at the expense of bandwidth.
	IndirectChecks int

	// RetransmitMult is the multiplier for the number of retransmissions
	// that are attempted for messages broadcasted over gossip. The actual
	// count of retransmissions is calculated using the formula:
	//
	//   Retransmits = RetransmitMult * log(N+1)
	//
	// This allows the retransmits to scale properly with cluster size. The
	// higher the multiplier, the more likely a failed broadcast is to converge
	// at the expense of increased bandwidth.
	RetransmitMult int

	// SuspicionMult is the multiplier for determining the time an
	// inaccessible node is considered suspect before declaring it dead.
	// The actual timeout is calculated using the formula:
	//
	//   SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
	//
	// This allows the timeout to scale properly with expected propagation
	// delay with a larger cluster size. The higher the multiplier, the longer
	// an inaccessible node is considered part of the cluster before declaring
	// it dead, giving that suspect node more time to refute if it is indeed
	// still alive.
	SuspicionMult int

	// PushPullInterval is the interval between complete state syncs.
	// Complete state syncs are done with a single node over TCP and are
	// quite expensive relative to standard gossiped messages. Setting this
	// to zero will disable state push/pull syncs completely.
	//
	// Setting this interval lower (more frequent) will increase convergence
	// speeds across larger clusters at the expense of increased bandwidth
	// usage.
	PushPullInterval time.Duration

	// ProbeInterval and ProbeTimeout are used to configure probing
	// behavior for memberlist.
	//
	// ProbeInterval is the interval between random node probes. Setting
	// this lower (more frequent) will cause the memberlist cluster to detect
	// failed nodes more quickly at the expense of increased bandwidth usage.
	//
	// ProbeTimeout is the timeout to wait for an ack from a probed node
	// before assuming it is unhealthy. This should be set to 99-percentile
	// of RTT (round-trip time) on your network.
	ProbeInterval time.Duration
	ProbeTimeout  time.Duration

	// GossipInterval and GossipNodes are used to configure the gossip
	// behavior of memberlist.
	//
	// GossipInterval is the interval between sending messages that need
	// to be gossiped that haven't been able to piggyback on probing messages.
	// If this is set to zero, non-piggyback gossip is disabled. By lowering
	// this value (more frequent) gossip messages are propagated across
	// the cluster more quickly at the expense of increased bandwidth.
	//
	// GossipNodes is the number of random nodes to send gossip messages to
	// per GossipInterval. Increasing this number causes the gossip messages
	// to propagate across the cluster more quickly at the expense of
	// increased bandwidth.
	GossipInterval time.Duration
	GossipNodes    int

	// EnableCompression is used to control message compression. This can
	// be used to reduce bandwidth usage at the cost of slightly more CPU
	// utilization.
	EnableCompression bool

	// SecretKey is provided if message level encryption and verification
	// are to be used. The key is passed through a Key Derivation Function
	// (PBKDF2) to ensure suitability. This key is also used for an HMAC-SHA1
	// to provide message integrity.
	SecretKey string

	// Delegate and Events are delegates for receiving and providing
	// data to memberlist via callback mechanisms. For Delegate, see
	// the Delegate interface. For Events, see the EventDelegate interface.
	Delegate Delegate
	Events   EventDelegate

	// LogOutput is the writer where logs should be sent. If this is not
	// set, logging will go to stderr by default.
	LogOutput io.Writer
}

type Memberlist struct {
	config         *Config
	shutdown       bool
	leave          bool
	leaveBroadcast chan struct{}

	udpListener *net.UDPConn
	tcpListener *net.TCPListener

	sequenceNum uint32 // Local sequence number
	incarnation uint32 // Local incarnation number

	nodeLock sync.RWMutex
	nodes    []*nodeState          // Known nodes
	nodeMap  map[string]*nodeState // Maps Addr.String() -> NodeState

	tickerLock sync.Mutex
	tickers    []*time.Ticker
	stopTick   chan struct{}
	probeIndex int

	ackLock     sync.Mutex
	ackHandlers map[uint32]*ackHandler

	broadcasts *TransmitLimitedQueue

	startStopLock sync.Mutex

	logger *log.Logger
}

// DefaultConfig returns a sane set of configurations for Memberlist.
// It uses the hostname as the node name, and otherwise sets very conservative
// values that are sane for most LAN environments. The default configuration
// errs on the side on the side of caution, choosing values that are optimized
// for higher convergence at the cost of higher bandwidth usage. Regardless,
// these values are a good starting point when getting started with memberlist.
func DefaultConfig() *Config {
	hostname, _ := os.Hostname()
	return &Config{
		Name:             hostname,
		BindAddr:         "0.0.0.0",
		UDPPort:          7946,
		TCPPort:          7946,
		TCPTimeout:       10 * time.Second,       // Timeout after 10 seconds
		IndirectChecks:   3,                      // Use 3 nodes for the indirect ping
		RetransmitMult:   4,                      // Retransmit a message 4 * log(N+1) nodes
		SuspicionMult:    5,                      // Suspect a node for 5 * log(N+1) * Interval
		PushPullInterval: 30 * time.Second,       // Low frequency
		ProbeTimeout:     500 * time.Millisecond, // Reasonable RTT time for LAN
		ProbeInterval:    1 * time.Second,        // Failure check every second

		GossipNodes:    3,                      // Gossip to 3 nodes
		GossipInterval: 200 * time.Millisecond, // Gossip more rapidly

		EnableCompression: true, // Enable compression by default
		SecretKey:         "",
	}
}

// newMemberlist creates the network listeners.
// Does not schedule execution of background maintenence.
func newMemberlist(conf *Config) (*Memberlist, error) {
	tcpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.TCPPort)
	tcpLn, err := net.Listen("tcp", tcpAddr)
	if err != nil {
		return nil, fmt.Errorf("Failed to start TCP listener. Err: %s", err)
	}

	udpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.UDPPort)
	udpLn, err := net.ListenPacket("udp", udpAddr)
	if err != nil {
		tcpLn.Close()
		return nil, fmt.Errorf("Failed to start UDP listener. Err: %s", err)
	}

	// Set the UDP receive window size
	setUDPRecvBuf(udpLn.(*net.UDPConn))

	if conf.LogOutput == nil {
		conf.LogOutput = os.Stderr
	}

	m := &Memberlist{
		config:         conf,
		leaveBroadcast: make(chan struct{}, 1),
		udpListener:    udpLn.(*net.UDPConn),
		tcpListener:    tcpLn.(*net.TCPListener),
		nodeMap:        make(map[string]*nodeState),
		ackHandlers:    make(map[uint32]*ackHandler),
		broadcasts:     &TransmitLimitedQueue{RetransmitMult: conf.RetransmitMult},
		logger:         log.New(conf.LogOutput, "", log.LstdFlags),
	}
	m.broadcasts.NumNodes = func() int { return len(m.nodes) }
	go m.tcpListen()
	go m.udpListen()
	return m, nil
}

// Create will create a new Memberlist using the given configuration.
// This will not connect to any other node (see Join) yet, but will start
// all the listeners to allow other nodes to join this memberlist.
// After creating a Memberlist, the configuration given should not be
// modified by the user anymore.
func Create(conf *Config) (*Memberlist, error) {
	m, err := newMemberlist(conf)
	if err != nil {
		return nil, err
	}
	if err := m.setAlive(); err != nil {
		m.Shutdown()
		return nil, err
	}
	m.schedule()
	return m, nil
}

// Join is used to take an existing Memberlist and attempt to join a cluster
// by contacting all the given hosts and performing a state sync. Initially,
// the Memberlist only contains our own state, so doing this will cause
// remote nodes to become aware of the existence of this node, effectively
// joining the cluster.
//
// This returns the number of hosts successfully contacted and an error if
// none could be reached. If an error is returned, the node did not successfully
// join the cluster.
func (m *Memberlist) Join(existing []string) (int, error) {
	// Attempt to join any of them
	numSuccess := 0
	var retErr error
	for _, exist := range existing {
		addr, err := net.ResolveIPAddr("ip", exist)
		if err != nil {
			retErr = err
			continue
		}

		if err := m.pushPullNode(addr.IP); err != nil {
			retErr = err
			continue
		}

		numSuccess++
	}

	if numSuccess > 0 {
		retErr = nil
	}

	return numSuccess, retErr
}

// setAlive is used to mark this node as being alive. This is the same
// as if we received an alive notification our own network channel for
// ourself.
func (m *Memberlist) setAlive() error {
	// Pick a private IP address
	var ipAddr []byte
	if m.config.BindAddr == "0.0.0.0" {
		// We're not bound to a specific IP, so let's list the interfaces
		// on this machine and use the first private IP we find.
		addresses, err := net.InterfaceAddrs()
		if err != nil {
			return fmt.Errorf("Failed to get interface addresses! Err: %vn", err)
		}

		// Find private IPv4 address
		for _, addr := range addresses {
			ip, ok := addr.(*net.IPNet)
			if !ok {
				continue
			}
			if ip.IP.To4() == nil {
				continue
			}
			if !isPrivateIP(ip.IP.String()) {
				continue
			}
			ipAddr = ip.IP
			break
		}

		// Failed to find private IP, use loopback
		if ipAddr == nil {
			ipAddr = []byte{127, 0, 0, 1}
		}
	} else {
		// Use the IP that we're bound to.
		addr := m.tcpListener.Addr().(*net.TCPAddr)
		ipAddr = addr.IP
	}

	// Get the node meta data
	var meta []byte
	if m.config.Delegate != nil {
		meta = m.config.Delegate.NodeMeta(metaMaxSize)
		if len(meta) > metaMaxSize {
			panic("Node meta data provided is longer than the limit")
		}
	}

	a := alive{
		Incarnation: m.nextIncarnation(),
		Node:        m.config.Name,
		Addr:        ipAddr,
		Meta:        meta,
	}
	m.aliveNode(&a)

	return nil
}

// Members returns a list of all known live nodes. The node structures
// returned must not be modified. If you wish to modify a Node, make a
// copy first.
func (m *Memberlist) Members() []*Node {
	m.nodeLock.RLock()
	defer m.nodeLock.RUnlock()

	nodes := make([]*Node, 0, len(m.nodes))
	for _, n := range m.nodes {
		if n.State != stateDead {
			nodes = append(nodes, &n.Node)
		}
	}

	return nodes
}

// NumMembers returns the number of alive nodes currently known. Between
// the time of calling this and calling Members, the number of alive nodes
// may have changed, so this shouldn't be used to determine how many
// members will be returned by Members.
func (m *Memberlist) NumMembers() (alive int) {
	m.nodeLock.RLock()
	defer m.nodeLock.RUnlock()

	for _, n := range m.nodes {
		if n.State != stateDead {
			alive++
		}
	}

	return
}

// Leave will broadcast a leave message but will not shutdown the background
// listeners, meaning the node will continue participating in gossip and state
// updates.
//
// This will block until the leave message is successfully broadcasted to
// a member of the cluster, if any exist or until a specified timeout
// is reached.
//
// This method is safe to call multiple times, but must not be called
// after the cluster is already shut down.
func (m *Memberlist) Leave(timeout time.Duration) error {
	m.startStopLock.Lock()
	defer m.startStopLock.Unlock()

	if m.shutdown {
		panic("leave after shutdown")
	}

	if !m.leave {
		m.leave = true

		state, ok := m.nodeMap[m.config.Name]
		if !ok {
			m.logger.Println("[WARN] Leave but we're not in the node map.")
			return nil
		}

		d := dead{
			Incarnation: state.Incarnation,
			Node:        state.Name,
		}
		m.deadNode(&d)

		// Check for any other alive node
		anyAlive := false
		for _, n := range m.nodes {
			if n.State != stateDead {
				anyAlive = true
				break
			}
		}

		// Block until the broadcast goes out
		if anyAlive {
			var timeoutCh <-chan time.Time
			if timeout > 0 {
				timeoutCh = time.After(timeout)
			}
			select {
			case <-m.leaveBroadcast:
			case <-timeoutCh:
				return fmt.Errorf("timeout waiting for leave broadcast")
			}
		}
	}

	return nil
}

// Shutdown will stop any background maintanence of network activity
// for this memberlist, causing it to appear "dead". A leave message
// will not be broadcasted prior, so the cluster being left will have
// to detect this node's shutdown using probing. If you wish to more
// gracefully exit the cluster, call Leave prior to shutting down.
//
// This method is safe to call multiple times.
func (m *Memberlist) Shutdown() error {
	m.startStopLock.Lock()
	defer m.startStopLock.Unlock()

	if !m.shutdown {
		m.shutdown = true
		m.deschedule()
		m.udpListener.Close()
		m.tcpListener.Close()
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The Neugram Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package syntax_test

import (
	"io/ioutil"
	"path/filepath"
	"strings"
	"testing"

	"neugram.io/ng/parser"
	"neugram.io/ng/syntax"
)

func TestWalk(t *testing.T) {
	files, err := filepath.Glob("../eval/testdata/*.ng")
	if err != nil {
		t.Fatal(err)
	}
	if len(files) == 0 {
		t.Fatal("cannot find testdata")
	}

	for _, file := range files {
		file := file
		test := file[len("testdata/") : len(file)-3]
		t.Run(test, func(t *testing.T) {
			source, err := ioutil.ReadFile(file)
			if err != nil {
				t.Fatal(err)
			}
			p := parser.New(file)
			f, err := p.Parse(source)
			if err != nil {
				if strings.HasSuffix(test, "_error") {
					return // probably a parser test
				}
				t.Fatal(err)
			}

			preCount, postCount := 0, 0
			preFn := func(c *syntax.Cursor) bool {
				preCount++
				if c.Name == "" {
					t.Errorf("cursor has no name on %v", c.Node)
				}
				if c.Parent == nil {
					t.Errorf("cursor has no parent on %v", c.Node)
				}
				return true
			}
			postFn := func(c *syntax.Cursor) bool {
				postCount++
				return true
			}
			if res := syntax.Walk(f, preFn, postFn); res != f {
				t.Errorf("Walk returned %v, not original %v", res, f)
			}
			if preCount != postCount {
				t.Errorf("Walk has imbalanced pre/post counts: %d/%d", preCount, postCount)
			}
			if preCount == 0 {
				t.Errorf("Walk didn't visit anything")
			}
		})
	}
}
<commit_msg>syntax: display correct test name<commit_after>// Copyright 2017 The Neugram Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package syntax_test

import (
	"io/ioutil"
	"path/filepath"
	"strings"
	"testing"

	"neugram.io/ng/parser"
	"neugram.io/ng/syntax"
)

func TestWalk(t *testing.T) {
	files, err := filepath.Glob("../eval/testdata/*.ng")
	if err != nil {
		t.Fatal(err)
	}
	if len(files) == 0 {
		t.Fatal("cannot find testdata")
	}

	for _, file := range files {
		file := file
		test := file[len("testdata") : len(file)-3]
		t.Run(test, func(t *testing.T) {
			source, err := ioutil.ReadFile(file)
			if err != nil {
				t.Fatal(err)
			}
			p := parser.New(file)
			f, err := p.Parse(source)
			if err != nil {
				if strings.HasSuffix(test, "_error") {
					return // probably a parser test
				}
				t.Fatal(err)
			}

			preCount, postCount := 0, 0
			preFn := func(c *syntax.Cursor) bool {
				preCount++
				if c.Name == "" {
					t.Errorf("cursor has no name on %v", c.Node)
				}
				if c.Parent == nil {
					t.Errorf("cursor has no parent on %v", c.Node)
				}
				return true
			}
			postFn := func(c *syntax.Cursor) bool {
				postCount++
				return true
			}
			if res := syntax.Walk(f, preFn, postFn); res != f {
				t.Errorf("Walk returned %v, not original %v", res, f)
			}
			if preCount != postCount {
				t.Errorf("Walk has imbalanced pre/post counts: %d/%d", preCount, postCount)
			}
			if preCount == 0 {
				t.Errorf("Walk didn't visit anything")
			}
		})
	}
}
<|endoftext|>


<commit_before>package chatroom

import (
	"container/list"
	"time"
)

type Event struct {
	Type      string // "join", "leave", or "message"
	User      string
	Timestamp int    // Unix timestmap (secs)
	Text      string // What the user said (if Type == "say")
}

type Subscription struct {
	Archive []Event      // All the events from the archive.
	New     <-chan Event // New events coming in.
}

// Owner of a subscription must cancel it when they stop listening to events.
func (s Subscription) Cancel() {
	unsubscribe <- s.New // Unsubscribe the channel.
	drain(s.New)         // Drain it, just in case there was a pending publish.
}

func newEvent(typ, user, msg string) Event {
	return Event{typ, user, int(time.Now().Unix()), msg}
}

func Subscribe() Subscription {
	resp := make(chan Subscription)
	subscribe <- resp
	return <-resp
}

func Join(user string) {
	publish <- newEvent("join", user, "")
}

func Say(user, message string) {
	publish <- newEvent("message", user, message)
}

func Leave(user string) {
	publish <- newEvent("leave", user, "")
}

const archiveSize = 10

var (
	// Send a channel here to get room events back.  It will send the entire
	// archive initially, and then new messages as they come in.
	subscribe = make(chan (chan<- Subscription), 10)
	// Send a channel here to unsubscribe.
	unsubscribe = make(chan (<-chan Event), 10)
	// Send events here to publish them.
	publish = make(chan Event, 10)
)

// This function loops forever, handling the chat room pubsub
func chatroom() {
	archive := list.New()
	subscribers := list.New()

	for {
		select {
		case ch := <-subscribe:
			var events []Event
			for e := archive.Front(); e != nil; e = e.Next() {
				events = append(events, e.Value.(Event))
			}
			subscriber := make(chan Event, 10)
			subscribers.PushBack(subscriber)
			ch <- Subscription{events, subscriber}

		case event := <-publish:
			for ch := subscribers.Front(); ch != nil; ch = ch.Next() {
				ch.Value.(chan Event) <- event
			}
			if archive.Len() >= archiveSize {
				archive.Remove(archive.Front())
			}
			archive.PushBack(event)

		case unsub := <-unsubscribe:
			for ch := subscribers.Front(); ch != nil; ch = ch.Next() {
				if ch.Value.(chan Event) == unsub {
					subscribers.Remove(ch)
					break
				}
			}
		}
	}
}

func init() {
	go chatroom()
}

// Helpers

// Drains a given channel of any messages.
func drain(ch <-chan Event) {
	for {
		select {
		case _, ok := <-ch:
			if !ok {
				return
			}
		default:
			return
		}
	}
}
<commit_msg>Fixes typo in chatroom module comment<commit_after>package chatroom

import (
	"container/list"
	"time"
)

type Event struct {
	Type      string // "join", "leave", or "message"
	User      string
	Timestamp int    // Unix timestamp (secs)
	Text      string // What the user said (if Type == "say")
}

type Subscription struct {
	Archive []Event      // All the events from the archive.
	New     <-chan Event // New events coming in.
}

// Owner of a subscription must cancel it when they stop listening to events.
func (s Subscription) Cancel() {
	unsubscribe <- s.New // Unsubscribe the channel.
	drain(s.New)         // Drain it, just in case there was a pending publish.
}

func newEvent(typ, user, msg string) Event {
	return Event{typ, user, int(time.Now().Unix()), msg}
}

func Subscribe() Subscription {
	resp := make(chan Subscription)
	subscribe <- resp
	return <-resp
}

func Join(user string) {
	publish <- newEvent("join", user, "")
}

func Say(user, message string) {
	publish <- newEvent("message", user, message)
}

func Leave(user string) {
	publish <- newEvent("leave", user, "")
}

const archiveSize = 10

var (
	// Send a channel here to get room events back.  It will send the entire
	// archive initially, and then new messages as they come in.
	subscribe = make(chan (chan<- Subscription), 10)
	// Send a channel here to unsubscribe.
	unsubscribe = make(chan (<-chan Event), 10)
	// Send events here to publish them.
	publish = make(chan Event, 10)
)

// This function loops forever, handling the chat room pubsub
func chatroom() {
	archive := list.New()
	subscribers := list.New()

	for {
		select {
		case ch := <-subscribe:
			var events []Event
			for e := archive.Front(); e != nil; e = e.Next() {
				events = append(events, e.Value.(Event))
			}
			subscriber := make(chan Event, 10)
			subscribers.PushBack(subscriber)
			ch <- Subscription{events, subscriber}

		case event := <-publish:
			for ch := subscribers.Front(); ch != nil; ch = ch.Next() {
				ch.Value.(chan Event) <- event
			}
			if archive.Len() >= archiveSize {
				archive.Remove(archive.Front())
			}
			archive.PushBack(event)

		case unsub := <-unsubscribe:
			for ch := subscribers.Front(); ch != nil; ch = ch.Next() {
				if ch.Value.(chan Event) == unsub {
					subscribers.Remove(ch)
					break
				}
			}
		}
	}
}

func init() {
	go chatroom()
}

// Helpers

// Drains a given channel of any messages.
func drain(ch <-chan Event) {
	for {
		select {
		case _, ok := <-ch:
			if !ok {
				return
			}
		default:
			return
		}
	}
}
<|endoftext|>


<commit_before>package ircmsg

import (
	"fmt"
	"reflect"
	"testing"
)

type testcase struct {
	escaped   string
	unescaped string
}

var tests = []testcase{
	{"te\\nst", "te\nst"},
	{"tes\\\\st", "tes\\st"},
	{"te😃st", "te😃st"},
}

var unescapeTests = []testcase{
	{"te\\n\\kst", "te\nkst"},
	{"te\\n\\kst\\", "te\nkst"},
	{"te\\\\nst", "te\\nst"},
	{"te😃st", "te😃st"},
	{"0\\n1\\n2\\n3\\n4\\n5\\n6\\n\\", "0\n1\n2\n3\n4\n5\n6\n"},
	{"test\\", "test"},
	{"te\\:st\\", "te;st"},
	{"te\\:\\st\\", "te; t"},
	{"\\\\te\\:\\st", "\\te; t"},
	{"test\\", "test"},
	{"\\", ""},
	{"", ""},
}

func TestEscape(t *testing.T) {
	for _, pair := range tests {
		val := EscapeTagValue(pair.unescaped)

		if val != pair.escaped {
			t.Error(
				"For", pair.unescaped,
				"expected", pair.escaped,
				"got", val,
			)
		}
	}
}

func TestUnescape(t *testing.T) {
	for _, pair := range tests {
		val := UnescapeTagValue(pair.escaped)

		if val != pair.unescaped {
			t.Error(
				"For", pair.escaped,
				"expected", pair.unescaped,
				"got", val,
			)
		}
	}
	for _, pair := range unescapeTests {
		val := UnescapeTagValue(pair.escaped)

		if val != pair.unescaped {
			t.Error(
				"For", pair.escaped,
				"expected", pair.unescaped,
				"got", val,
			)
		}
	}
}

// tag string tests
type testtags struct {
	raw  string
	tags map[string]string
}

var tagdecodetests = []testtags{
	{"", map[string]string{}},
	{"time=12732;re", map[string]string{"time": "12732", "re": ""}},
	{"time=12732;re=;asdf=5678", map[string]string{"time": "12732", "re": "", "asdf": "5678"}},
}

func parseTags(rawTags string) (map[string]string, error) {
	message, err := ParseLineStrict(fmt.Sprintf("@%s :shivaram TAGMSG #darwin\r\n", rawTags), true, 0)
	return message.AllTags(), err
}

func TestDecodeTags(t *testing.T) {
	for _, pair := range tagdecodetests {
		tags, err := parseTags(pair.raw)
		if err != nil {
			t.Error(
				"For", pair.raw,
				"Failed to parse line:", err,
			)
		}

		if !reflect.DeepEqual(tags, pair.tags) {
			t.Error(
				"For", pair.raw,
				"expected", pair.tags,
				"got", tags,
			)
		}
	}
}
<commit_msg>add additional test from review feedback<commit_after>package ircmsg

import (
	"fmt"
	"reflect"
	"testing"
)

type testcase struct {
	escaped   string
	unescaped string
}

var tests = []testcase{
	{"te\\nst", "te\nst"},
	{"tes\\\\st", "tes\\st"},
	{"te😃st", "te😃st"},
}

var unescapeTests = []testcase{
	{"te\\n\\kst", "te\nkst"},
	{"te\\n\\kst\\", "te\nkst"},
	{"te\\\\nst", "te\\nst"},
	{"te😃st", "te😃st"},
	{"0\\n1\\n2\\n3\\n4\\n5\\n6\\n\\", "0\n1\n2\n3\n4\n5\n6\n"},
	{"test\\", "test"},
	{"te\\:st\\", "te;st"},
	{"te\\:\\st\\", "te; t"},
	{"\\\\te\\:\\st", "\\te; t"},
	{"test\\", "test"},
	{"\\", ""},
	{"", ""},
}

func TestEscape(t *testing.T) {
	for _, pair := range tests {
		val := EscapeTagValue(pair.unescaped)

		if val != pair.escaped {
			t.Error(
				"For", pair.unescaped,
				"expected", pair.escaped,
				"got", val,
			)
		}
	}
}

func TestUnescape(t *testing.T) {
	for _, pair := range tests {
		val := UnescapeTagValue(pair.escaped)

		if val != pair.unescaped {
			t.Error(
				"For", pair.escaped,
				"expected", pair.unescaped,
				"got", val,
			)
		}
	}
	for _, pair := range unescapeTests {
		val := UnescapeTagValue(pair.escaped)

		if val != pair.unescaped {
			t.Error(
				"For", pair.escaped,
				"expected", pair.unescaped,
				"got", val,
			)
		}
	}
}

// tag string tests
type testtags struct {
	raw  string
	tags map[string]string
}

var tagdecodetests = []testtags{
	{"", map[string]string{}},
	{"time=12732;re", map[string]string{"time": "12732", "re": ""}},
	{"time=12732;re=;asdf=5678", map[string]string{"time": "12732", "re": "", "asdf": "5678"}},
	{"=these;time=12732;=shouldbe;re=;asdf=5678;=ignored", map[string]string{"time": "12732", "re": "", "asdf": "5678"}},
}

func parseTags(rawTags string) (map[string]string, error) {
	message, err := ParseLineStrict(fmt.Sprintf("@%s :shivaram TAGMSG #darwin\r\n", rawTags), true, 0)
	return message.AllTags(), err
}

func TestDecodeTags(t *testing.T) {
	for _, pair := range tagdecodetests {
		tags, err := parseTags(pair.raw)
		if err != nil {
			t.Error(
				"For", pair.raw,
				"Failed to parse line:", err,
			)
		}

		if !reflect.DeepEqual(tags, pair.tags) {
			t.Error(
				"For", pair.raw,
				"expected", pair.tags,
				"got", tags,
			)
		}
	}
}
<|endoftext|>


<commit_before>package system

import (
	"fmt"
	"path"
	"testing"

	"github.com/goesd/support/assert"
)

const (
	fixturePath = "fixtures"
)

func TestLoadTGFF(t *testing.T) {
	plat, app, err := LoadTGFF(findFixture("002_040"))

	assert.Success(err, t)
	assert.Equal(len(plat.Cores), 2, t)
	assert.Equal(len(app.Tasks), 40, t)

	tasks := []struct {
		children []uint16
	}{
		{[]uint16{1}},
		{[]uint16{2, 3, 23, 32}},
		{[]uint16{4, 10, 11, 12}},
		{[]uint16{9, 10, 26}},
		{[]uint16{5, 6, 7, 8}},

		{[]uint16{10, 11, 27}},
		{nil},
		{[]uint16{9}},
		{[]uint16{9}},
		{[]uint16{13, 14, 15}},

		{[]uint16{12}},
		{[]uint16{12, 31}},
		{[]uint16{17, 18, 24}},
		{[]uint16{16, 25}},
		{[]uint16{36}},

		{[]uint16{19, 20, 23, 29}},
		{[]uint16{31}},
		{[]uint16{22, 35}},
		{[]uint16{21, 23}},
		{[]uint16{20, 24}},

		{[]uint16{24, 26}},
		{[]uint16{27}},
		{[]uint16{28, 29}},
		{[]uint16{31}},
		{[]uint16{30}},

		{[]uint16{37, 38}},
		{nil},
		{[]uint16{33, 34}},
		{[]uint16{29}},
		{nil},

		{nil},
		{[]uint16{36}},
		{nil},
		{nil},
		{nil},

		{nil},
		{nil},
		{nil},
		{[]uint16{39}},
		{nil},
	}

	for i, task := range tasks {
		assert.Equal(app.Tasks[i].Children, task.children, t)
	}
}

func TestExtractTaskNumber(t *testing.T) {
	scenarios := []struct {
		name    string
		total   uint16
		result  uint16
		success bool
	}{
		{"t0_0", 50, 0, true},
		{"t0_42", 50, 42, true},
		{"t0_42", 43, 42, true},
		{"t0_42", 42, 0, false},
		{"t1_42", 50, 0, false},
		{"t0_-2", 50, 0, false},
	}

	for _, s := range scenarios {
		result, err := extractTaskID(s.name, s.total)

		if s.success {
			assert.Success(err, t)
		} else {
			assert.Failure(err, t)
		}

		assert.Equal(result, s.result, t)
	}
}

func BenchmarkLoadTGFF(b *testing.B) {
	path := findFixture("002_040")

	for i := 0; i < b.N; i++ {
		LoadTGFF(path)
	}
}

func findFixture(name string) string {
	return path.Join(fixturePath, fmt.Sprintf("%s.tgff", name))
}
<commit_msg>A minor change in system<commit_after>package system

import (
	"fmt"
	"path"
	"testing"

	"github.com/goesd/support/assert"
)

const (
	fixturePath = "fixtures"
)

func TestLoadTGFF(t *testing.T) {
	plat, app, err := LoadTGFF(findFixture("002_040"))

	assert.Success(err, t)
	assert.Equal(len(plat.Cores), 2, t)
	assert.Equal(len(app.Tasks), 40, t)

	tasks := []struct {
		children []uint16
	}{
		{[]uint16{1}},
		{[]uint16{2, 3, 23, 32}},
		{[]uint16{4, 10, 11, 12}},
		{[]uint16{9, 10, 26}},
		{[]uint16{5, 6, 7, 8}},

		{[]uint16{10, 11, 27}},
		{nil},
		{[]uint16{9}},
		{[]uint16{9}},
		{[]uint16{13, 14, 15}},

		{[]uint16{12}},
		{[]uint16{12, 31}},
		{[]uint16{17, 18, 24}},
		{[]uint16{16, 25}},
		{[]uint16{36}},

		{[]uint16{19, 20, 23, 29}},
		{[]uint16{31}},
		{[]uint16{22, 35}},
		{[]uint16{21, 23}},
		{[]uint16{20, 24}},

		{[]uint16{24, 26}},
		{[]uint16{27}},
		{[]uint16{28, 29}},
		{[]uint16{31}},
		{[]uint16{30}},

		{[]uint16{37, 38}},
		{nil},
		{[]uint16{33, 34}},
		{[]uint16{29}},
		{nil},

		{nil},
		{[]uint16{36}},
		{nil},
		{nil},
		{nil},

		{nil},
		{nil},
		{nil},
		{[]uint16{39}},
		{nil},
	}

	for i, task := range tasks {
		assert.Equal(app.Tasks[i].Children, task.children, t)
	}
}

func TestExtractTaskNumber(t *testing.T) {
	scenarios := []struct {
		name    string
		total   uint16
		result  uint16
		success bool
	}{
		{"t0_0", 50, 0, true},
		{"t0_42", 50, 42, true},
		{"t0_42", 43, 42, true},
		{"t0_42", 42, 0, false},
		{"t1_42", 50, 0, false},
		{"t0_-2", 50, 0, false},
	}

	for _, s := range scenarios {
		result, err := extractTaskID(s.name, s.total)

		if s.success {
			assert.Success(err, t)
		} else {
			assert.Failure(err, t)
		}

		assert.Equal(result, s.result, t)
	}
}

func BenchmarkLoadTGFF_002_040(b *testing.B) {
	path := findFixture("002_040")

	for i := 0; i < b.N; i++ {
		_, _, err := LoadTGFF(path)
		if err != nil {
			b.Fatal(err)
		}
	}
}

func findFixture(name string) string {
	return path.Join(fixturePath, fmt.Sprintf("%s.tgff", name))
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Rohith All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package marathon

import (
	"encoding/json"
	"fmt"
	"github.com/donovanhide/eventsource"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strings"
	"time"
)

// Subscriptions is a collection to urls that marathon is implementing a callback on
type Subscriptions struct {
	CallbackURLs []string `json:"callbackUrls"`
}

// Subscriptions retrieves a list of registered subscriptions
func (r *marathonClient) Subscriptions() (*Subscriptions, error) {
	subscriptions := new(Subscriptions)
	if err := r.apiGet(marathonAPISubscription, nil, subscriptions); err != nil {
		return nil, err
	}

	return subscriptions, nil
}

// AddEventsListener adds your self as a listener to events from Marathon
//		channel:	a EventsChannel used to receive event on
func (r *marathonClient) AddEventsListener(channel EventsChannel, filter int) error {
	r.Lock()
	defer r.Unlock()

	// step: someone has asked to start listening to event, we need to register for events
	// if we haven't done so already
	if err := r.registerSubscription(); err != nil {
		return err
	}

	if _, found := r.listeners[channel]; !found {
		log.Printf("adding a watch for events: %d, channel: %v", filter, channel)
		r.listeners[channel] = filter
	}
	return nil
}

// RemoveEventsListener removes the channel from the events listeners
//		channel:			the channel you are removing
func (r *marathonClient) RemoveEventsListener(channel EventsChannel) {
	r.Lock()
	defer r.Unlock()

	if _, found := r.listeners[channel]; found {
		delete(r.listeners, channel)
		// step: if there is no one else listening, let's remove ourselves
		// from the events callback
		if r.config.EventsTransport == EventsTransportCallback && len(r.listeners) == 0 {
			r.Unsubscribe(r.SubscriptionURL())
		}
	}
}

// SubscriptionURL retrieves the subscription callback URL used when registering
func (r *marathonClient) SubscriptionURL() string {
	if r.config.CallbackURL != "" {
		return fmt.Sprintf("%s%s", r.config.CallbackURL, defaultEventsURL)
	}

	return fmt.Sprintf("http://%s:%d%s", r.ipAddress, r.config.EventsPort, defaultEventsURL)
}

// RegisterSubscription registers ourselves with Marathon to receive events from configured transport facility
func (r *marathonClient) registerSubscription() error {
	switch r.config.EventsTransport {
	case EventsTransportCallback:
		return r.registerCallbackSubscription()
	case EventsTransportSSE:
		return r.registerSSESubscription()
	default:
		return fmt.Errorf("the events transport: %d is not supported", r.config.EventsTransport)
	}
}

func (r *marathonClient) registerCallbackSubscription() error {
	if r.eventsHTTP == nil {
		ipAddress, err := getInterfaceAddress(r.config.EventsInterface)
		if err != nil {
			return fmt.Errorf("Unable to get the ip address from the interface: %s, error: %s",
				r.config.EventsInterface, err)
		}

		// step: set the ip address
		r.ipAddress = ipAddress
		binding := fmt.Sprintf("%s:%d", ipAddress, r.config.EventsPort)
		// step: register the handler
		http.HandleFunc(defaultEventsURL, r.handleCallbackEvent)
		// step: create the http server
		r.eventsHTTP = &http.Server{
			Addr:           binding,
			Handler:        nil,
			ReadTimeout:    10 * time.Second,
			WriteTimeout:   10 * time.Second,
			MaxHeaderBytes: 1 << 20,
		}

		// @todo need to add a timeout value here
		listener, err := net.Listen("tcp", binding)
		if err != nil {
			return nil
		}

		go func() {
			for {
				r.eventsHTTP.Serve(listener)
			}
		}()
	}

	// step: get the callback url
	callback := r.SubscriptionURL()

	// step: check if the callback is registered
	found, err := r.HasSubscription(callback)
	if err != nil {
		return err
	}
	if !found {
		// step: we need to register ourselves
		uri := fmt.Sprintf("%s?callbackUrl=%s", marathonAPISubscription, callback)
		if err := r.apiPost(uri, "", nil); err != nil {
			return err
		}
	}

	return nil
}

func (r *marathonClient) registerSSESubscription() error {
	// Prevent multiple SSE subscriptions
	if r.subscribedToSSE {
		return nil
	}

	var stream *eventsource.Stream

	// Try to connect to Marathon until succeed or
	// the whole custer is down
	for {
		// Get a member from the cluster
		marathon, err := r.cluster.GetMember()
		if err != nil {
			return err
		}
		url := fmt.Sprintf("%s/%s", marathon, marathonAPIEventStream)

		// Try to connect to stream
		stream, err = eventsource.Subscribe(url, "")
		if err == nil {
			break
		}

		log.Printf("failed to connect to Marathon event stream, error: %s", err)
		r.cluster.MarkDown()
	}

	go func() {
		for {
			select {
			case ev := <-stream.Events:
				r.handleEvent(ev.Data())
			case err := <-stream.Errors:
				log.Printf("failed to receive event, error: %s", err)
			}
		}
	}()

	r.subscribedToSSE = true
	return nil
}

// Unsubscribe removes ourselves from Marathon's callback facility
//	url		: the url you wish to unsubscribe
func (r *marathonClient) Unsubscribe(callbackURL string) error {
	// step: remove from the list of subscriptions
	return r.apiDelete(fmt.Sprintf("%s?callbackUrl=%s", marathonAPISubscription, callbackURL), nil, nil)
}

// HasSubscription checks to see a subscription already exists with Marathon
//		callback:			the url of the callback
func (r *marathonClient) HasSubscription(callback string) (bool, error) {
	// step: generate our events callback
	subscriptions, err := r.Subscriptions()
	if err != nil {
		return false, err
	}

	for _, subscription := range subscriptions.CallbackURLs {
		if callback == subscription {
			return true, nil
		}
	}

	return false, nil
}

func (r *marathonClient) handleEvent(content string) {
	// step: process and decode the event
	eventType := new(EventType)
	err := json.NewDecoder(strings.NewReader(content)).Decode(eventType)
	if err != nil {
		log.Printf("failed to decode the event type, content: %s, error: %s", content, err)
		return
	}

	// step: check whether event type is handled
	event, err := GetEvent(eventType.EventType)
	if err != nil {
		log.Printf("unable to handle event, type: %s, error: %s", eventType.EventType, err)
		return
	}

	// step: let's decode message
	err = json.NewDecoder(strings.NewReader(content)).Decode(event.Event)
	if err != nil {
		log.Printf("failed to decode the event, id: %d, error: %s", event.ID, err)
		return
	}

	r.RLock()
	defer r.RUnlock()

	// step: check if anyone is listen for this event
	for channel, filter := range r.listeners {
		// step: check if this listener wants this event type
		if event.ID&filter != 0 {
			go func(ch EventsChannel, e *Event) {
				ch <- e
			}(channel, event)
		}
	}
}

func (r *marathonClient) handleCallbackEvent(writer http.ResponseWriter, request *http.Request) {
	body, err := ioutil.ReadAll(request.Body)
	if err != nil {
		log.Printf("failed to read request body, error: %s", err)
		return
	}

	r.handleEvent(string(body[:]))
}
<commit_msg>remove this log, it feels random just to log adding and nothing else<commit_after>/*
Copyright 2014 Rohith All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package marathon

import (
	"encoding/json"
	"fmt"
	"github.com/donovanhide/eventsource"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strings"
	"time"
)

// Subscriptions is a collection to urls that marathon is implementing a callback on
type Subscriptions struct {
	CallbackURLs []string `json:"callbackUrls"`
}

// Subscriptions retrieves a list of registered subscriptions
func (r *marathonClient) Subscriptions() (*Subscriptions, error) {
	subscriptions := new(Subscriptions)
	if err := r.apiGet(marathonAPISubscription, nil, subscriptions); err != nil {
		return nil, err
	}

	return subscriptions, nil
}

// AddEventsListener adds your self as a listener to events from Marathon
//		channel:	a EventsChannel used to receive event on
func (r *marathonClient) AddEventsListener(channel EventsChannel, filter int) error {
	r.Lock()
	defer r.Unlock()

	// step: someone has asked to start listening to event, we need to register for events
	// if we haven't done so already
	if err := r.registerSubscription(); err != nil {
		return err
	}

	if _, found := r.listeners[channel]; !found {
		r.listeners[channel] = filter
	}
	return nil
}

// RemoveEventsListener removes the channel from the events listeners
//		channel:			the channel you are removing
func (r *marathonClient) RemoveEventsListener(channel EventsChannel) {
	r.Lock()
	defer r.Unlock()

	if _, found := r.listeners[channel]; found {
		delete(r.listeners, channel)
		// step: if there is no one else listening, let's remove ourselves
		// from the events callback
		if r.config.EventsTransport == EventsTransportCallback && len(r.listeners) == 0 {
			r.Unsubscribe(r.SubscriptionURL())
		}
	}
}

// SubscriptionURL retrieves the subscription callback URL used when registering
func (r *marathonClient) SubscriptionURL() string {
	if r.config.CallbackURL != "" {
		return fmt.Sprintf("%s%s", r.config.CallbackURL, defaultEventsURL)
	}

	return fmt.Sprintf("http://%s:%d%s", r.ipAddress, r.config.EventsPort, defaultEventsURL)
}

// RegisterSubscription registers ourselves with Marathon to receive events from configured transport facility
func (r *marathonClient) registerSubscription() error {
	switch r.config.EventsTransport {
	case EventsTransportCallback:
		return r.registerCallbackSubscription()
	case EventsTransportSSE:
		return r.registerSSESubscription()
	default:
		return fmt.Errorf("the events transport: %d is not supported", r.config.EventsTransport)
	}
}

func (r *marathonClient) registerCallbackSubscription() error {
	if r.eventsHTTP == nil {
		ipAddress, err := getInterfaceAddress(r.config.EventsInterface)
		if err != nil {
			return fmt.Errorf("Unable to get the ip address from the interface: %s, error: %s",
				r.config.EventsInterface, err)
		}

		// step: set the ip address
		r.ipAddress = ipAddress
		binding := fmt.Sprintf("%s:%d", ipAddress, r.config.EventsPort)
		// step: register the handler
		http.HandleFunc(defaultEventsURL, r.handleCallbackEvent)
		// step: create the http server
		r.eventsHTTP = &http.Server{
			Addr:           binding,
			Handler:        nil,
			ReadTimeout:    10 * time.Second,
			WriteTimeout:   10 * time.Second,
			MaxHeaderBytes: 1 << 20,
		}

		// @todo need to add a timeout value here
		listener, err := net.Listen("tcp", binding)
		if err != nil {
			return nil
		}

		go func() {
			for {
				r.eventsHTTP.Serve(listener)
			}
		}()
	}

	// step: get the callback url
	callback := r.SubscriptionURL()

	// step: check if the callback is registered
	found, err := r.HasSubscription(callback)
	if err != nil {
		return err
	}
	if !found {
		// step: we need to register ourselves
		uri := fmt.Sprintf("%s?callbackUrl=%s", marathonAPISubscription, callback)
		if err := r.apiPost(uri, "", nil); err != nil {
			return err
		}
	}

	return nil
}

func (r *marathonClient) registerSSESubscription() error {
	// Prevent multiple SSE subscriptions
	if r.subscribedToSSE {
		return nil
	}

	var stream *eventsource.Stream

	// Try to connect to Marathon until succeed or
	// the whole custer is down
	for {
		// Get a member from the cluster
		marathon, err := r.cluster.GetMember()
		if err != nil {
			return err
		}
		url := fmt.Sprintf("%s/%s", marathon, marathonAPIEventStream)

		// Try to connect to stream
		stream, err = eventsource.Subscribe(url, "")
		if err == nil {
			break
		}

		log.Printf("failed to connect to Marathon event stream, error: %s", err)
		r.cluster.MarkDown()
	}

	go func() {
		for {
			select {
			case ev := <-stream.Events:
				r.handleEvent(ev.Data())
			case err := <-stream.Errors:
				log.Printf("failed to receive event, error: %s", err)
			}
		}
	}()

	r.subscribedToSSE = true
	return nil
}

// Unsubscribe removes ourselves from Marathon's callback facility
//	url		: the url you wish to unsubscribe
func (r *marathonClient) Unsubscribe(callbackURL string) error {
	// step: remove from the list of subscriptions
	return r.apiDelete(fmt.Sprintf("%s?callbackUrl=%s", marathonAPISubscription, callbackURL), nil, nil)
}

// HasSubscription checks to see a subscription already exists with Marathon
//		callback:			the url of the callback
func (r *marathonClient) HasSubscription(callback string) (bool, error) {
	// step: generate our events callback
	subscriptions, err := r.Subscriptions()
	if err != nil {
		return false, err
	}

	for _, subscription := range subscriptions.CallbackURLs {
		if callback == subscription {
			return true, nil
		}
	}

	return false, nil
}

func (r *marathonClient) handleEvent(content string) {
	// step: process and decode the event
	eventType := new(EventType)
	err := json.NewDecoder(strings.NewReader(content)).Decode(eventType)
	if err != nil {
		log.Printf("failed to decode the event type, content: %s, error: %s", content, err)
		return
	}

	// step: check whether event type is handled
	event, err := GetEvent(eventType.EventType)
	if err != nil {
		log.Printf("unable to handle event, type: %s, error: %s", eventType.EventType, err)
		return
	}

	// step: let's decode message
	err = json.NewDecoder(strings.NewReader(content)).Decode(event.Event)
	if err != nil {
		log.Printf("failed to decode the event, id: %d, error: %s", event.ID, err)
		return
	}

	r.RLock()
	defer r.RUnlock()

	// step: check if anyone is listen for this event
	for channel, filter := range r.listeners {
		// step: check if this listener wants this event type
		if event.ID&filter != 0 {
			go func(ch EventsChannel, e *Event) {
				ch <- e
			}(channel, event)
		}
	}
}

func (r *marathonClient) handleCallbackEvent(writer http.ResponseWriter, request *http.Request) {
	body, err := ioutil.ReadAll(request.Body)
	if err != nil {
		log.Printf("failed to read request body, error: %s", err)
		return
	}

	r.handleEvent(string(body[:]))
}
<|endoftext|>


<commit_before>package main

import (
	"sort"
	"time"

	. "github.com/bytbox/go-mail"
)

type Threaded struct {
	Message
	Parent   *Threaded
	Children []*Threaded

	parId    []string
}

func (tm *Threaded) addChild(c *Threaded) {
	tm.Children = append(tm.Children, c)
}

func (tm *Threaded) modified() time.Time {
	return tm.Date
}

type Sortable []*Threaded

func (s Sortable) Len() int {
	return len(s)
}

func (s Sortable) Less(i, j int) bool {
	return s[i].Date.Before(s[j].Date)
}

func (s Sortable) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func Thread(msgs []Message) []*Threaded {
	tmap := map[string]*Threaded{}
	for _, msg := range msgs {
		mid := msg.MessageId
		tm := &Threaded{msg, nil, nil, nil}
		// we just use the first parent
		if len(msg.References) > 0 {
			tm.parId = msg.References
		}
		tmap[mid] = tm
	}

	for _, tm := range tmap {
		if tm.parId != nil {
			// The parent should be set using the last id we have
			// registered.
			for _, id := range tm.parId {
				p, ok := tmap[id]
				if ok {
					tm.Parent = p
				}
			}
			if tm.Parent != nil {
				tm.Parent.addChild(tm)
			}
		}
	}

	threaded := []*Threaded{}
	for _, tm := range tmap {
		if tm.Parent == nil {
			threaded = append(threaded, tm)
		}
	}

	sort.Sort(Sortable(threaded))
	return threaded
}
<commit_msg>Sorting by thread<commit_after>package main

import (
	"sort"
	"time"

	. "github.com/bytbox/go-mail"
)

type Threaded struct {
	Message
	Parent   *Threaded
	Children []*Threaded

	parId    []string
}

func (tm *Threaded) addChild(c *Threaded) {
	tm.Children = append(tm.Children, c)
}

func (tm *Threaded) modified() time.Time {
	d := tm.Date
	for _, c := range tm.Children {
		if d.Before(c.Date) {
			d = c.Date
		}
	}
	return d
}

type Sortable []*Threaded

func (s Sortable) Len() int {
	return len(s)
}

func (s Sortable) Less(i, j int) bool {
	return s[i].Date.Before(s[j].Date)
}

func (s Sortable) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func Thread(msgs []Message) []*Threaded {
	tmap := map[string]*Threaded{}
	for _, msg := range msgs {
		mid := msg.MessageId
		tm := &Threaded{msg, nil, nil, nil}
		// we just use the first parent
		if len(msg.References) > 0 {
			tm.parId = msg.References
		}
		tmap[mid] = tm
	}

	for _, tm := range tmap {
		if tm.parId != nil {
			// The parent should be set using the last id we have
			// registered.
			for _, id := range tm.parId {
				p, ok := tmap[id]
				if ok {
					tm.Parent = p
				}
			}
			if tm.Parent != nil {
				tm.Parent.addChild(tm)
			}
		}
	}

	threaded := []*Threaded{}
	for _, tm := range tmap {
		if tm.Parent == nil {
			threaded = append(threaded, tm)
		}
	}

	sort.Sort(Sortable(threaded))
	return threaded
}
<|endoftext|>


<commit_before>package fly

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"time"

	"github.com/EngineerBetter/concourse-up/config"
	"github.com/EngineerBetter/concourse-up/util"
)

// DarwinBinaryURL is a compile-time variable set with -ldflags
var DarwinBinaryURL = "COMPILE_TIME_VARIABLE_fly_darwin_binary_url"

// LinuxBinaryURL is a compile-time variable set with -ldflags
var LinuxBinaryURL = "COMPILE_TIME_VARIABLE_fly_linux_binary_url"

// WindowsBinaryURL is a compile-time variable set with -ldflags
var WindowsBinaryURL = "COMPILE_TIME_VARIABLE_fly_windows_binary_url"

// IClient represents an interface for a client
type IClient interface {
	CanConnect() (bool, error)
	SetDefaultPipeline(deployArgs *config.DeployArgs, config *config.Config) error
	Cleanup() error
}

// Client represents a low-level wrapper for fly
type Client struct {
	tempDir *util.TempDir
	creds   Credentials
	stdout  io.Writer
	stderr  io.Writer
}

// Credentials represents credentials needed to connect to concourse
type Credentials struct {
	Target   string
	API      string
	Username string
	Password string
	CACert   string
}

// New returns a new fly client
func New(creds Credentials, stdout, stderr io.Writer) (IClient, error) {
	tempDir, err := util.NewTempDir()
	if err != nil {
		return nil, err
	}

	fileHandler, err := os.Create(tempDir.Path("fly"))
	if err != nil {
		return nil, err
	}
	defer fileHandler.Close()

	url, err := getFlyURL()
	if err != nil {
		return nil, err
	}

	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if _, err := io.Copy(fileHandler, resp.Body); err != nil {
		return nil, err
	}

	if err := fileHandler.Sync(); err != nil {
		return nil, err
	}

	if err := os.Chmod(fileHandler.Name(), 0700); err != nil {
		return nil, err
	}

	return &Client{
		tempDir,
		creds,
		stdout,
		stderr,
	}, nil
}

// CanConnect returns true if it can connect to the concourse
func (client *Client) CanConnect() (bool, error) {
	cmd := exec.Command(
		client.tempDir.Path("fly"),
		"--target",
		client.creds.Target,
		"login",
		"--insecure",
		"--concourse-url",
		client.creds.API,
		"--username",
		client.creds.Username,
		"--password",
		client.creds.Password,
	)

	stderr := bytes.NewBuffer(nil)
	cmd.Stdout = client.stdout
	cmd.Stderr = stderr

	runErr := cmd.Run()
	if runErr == nil {
		return true, nil
	}

	stderrBytes, err := ioutil.ReadAll(stderr)
	if err != nil {
		return false, err
	}

	if strings.Contains(string(stderrBytes), "could not reach the Concourse server") {
		return false, nil
	}

	// if there is a legitimate error, copy it to stderr for debugging
	if _, err := client.stderr.Write(stderrBytes); err != nil {
		return false, err
	}

	return false, runErr
}

// SetDefaultPipeline sets the default pipeline against a given concourse
func (client *Client) SetDefaultPipeline(deployArgs *config.DeployArgs, config *config.Config) error {
	if err := client.login(); err != nil {
		return err
	}

	pipelinePath := client.tempDir.Path("default-pipeline.yml")
	pipelineName := "concourse-up-self-update"

	if err := client.writePipelineConfig(pipelinePath, deployArgs, config); err != nil {
		return err
	}

	if err := client.run("set-pipeline", "--pipeline", pipelineName, "--config", pipelinePath, "--non-interactive"); err != nil {
		return err
	}

	if err := os.Remove(pipelinePath); err != nil {
		return err
	}

	return nil
}

func (client *Client) writePipelineConfig(pipelinePath string, deployArgs *config.DeployArgs, config *config.Config) error {
	fileHandler, err := os.Create(pipelinePath)
	if err != nil {
		return err
	}
	defer fileHandler.Close()

	params, err := client.buildDefaultPipelineParams(deployArgs, config)
	if err != nil {
		return err
	}

	pipelineConfig, err := util.RenderTemplate(defaultPipelineTemplate, params)
	if err != nil {
		return err
	}

	if _, err := fileHandler.Write(pipelineConfig); err != nil {
		return err
	}

	if err := fileHandler.Sync(); err != nil {
		return err
	}

	return nil
}

// Cleanup removes tempfiles
func (client *Client) Cleanup() error {
	return client.tempDir.Cleanup()
}

func (client *Client) login() error {
	attempts := 100

	if _, err := client.stdout.Write([]byte("Waiting for Concourse ATC to start... \n")); err != nil {
		return err
	}

	for i := 0; i < attempts; i++ {
		canConnect, err := client.CanConnect()
		if err != nil {
			return err
		}
		if canConnect {
			return nil
		}

		time.Sleep(time.Second)
	}

	return fmt.Errorf("failed to log in to %s after %d attempts", client.creds.API, attempts)
}

func (client *Client) run(args ...string) error {
	args = append([]string{"--target", client.creds.Target}, args...)
	cmd := exec.Command(client.tempDir.Path("fly"), args...)
	cmd.Stdout = client.stdout
	cmd.Stderr = client.stderr
	return cmd.Run()
}

func getFlyURL() (string, error) {
	os := runtime.GOOS
	if os == "darwin" {
		return DarwinBinaryURL, nil
	} else if os == "linux" {
		return LinuxBinaryURL, nil
	} else if os == "windows" {
		return WindowsBinaryURL, nil
	}
	return "", fmt.Errorf("unknown os: `%s`", os)
}

func (client *Client) buildDefaultPipelineParams(deployArgs *config.DeployArgs, config *config.Config) (*defaultPipelineParams, error) {
	awsAccessKeyID := os.Getenv("AWS_ACCESS_KEY_ID")
	if awsAccessKeyID == "" {
		return nil, errors.New("env var AWS_ACCESS_KEY_ID not found")
	}

	awsSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
	if awsSecretAccessKey == "" {
		return nil, errors.New("env var AWS_SECRET_ACCESS_KEY not found")
	}

	return &defaultPipelineParams{
		FlagAWSRegion:      deployArgs.AWSRegion,
		FlagDomain:         deployArgs.Domain,
		FlagTLSCert:        deployArgs.TLSCert,
		FlagTLSKey:         deployArgs.TLSKey,
		FlagWorkers:        deployArgs.WorkerCount,
		FlagWorkerSize:     deployArgs.WorkerSize,
		Deployment:         strings.TrimPrefix(config.Deployment, "concourse-up-"),
		AWSAccessKeyID:     awsAccessKeyID,
		AWSSecretAccessKey: awsSecretAccessKey,
	}, nil
}

type defaultPipelineParams struct {
	FlagAWSRegion      string
	FlagDomain         string
	FlagTLSCert        string
	FlagTLSKey         string
	FlagWorkers        int
	FlagWorkerSize     string
	Deployment         string
	AWSAccessKeyID     string
	AWSSecretAccessKey string
	AWSDefaultRegion   string
}

// Indent is a helper function to indent the field a given number of spaces
func (params defaultPipelineParams) Indent(countStr, field string) string {
	return util.Indent(countStr, field)
}

const defaultPipelineTemplate = `
---
resources:
- name: concourse-up-release
  type: github-release
  source:
    user: engineerbetter
    repository: concourse-up
    pre_release: true

jobs:
- name: self-update
  serial: true
  plan:
  - get: concourse-up-release
    trigger: true
  - task: update
    params:
      AWS_REGION: "<% .FlagAWSRegion %>"
      DOMAIN: "<% .FlagDomain %>"
      TLS_CERT: |-
        <% .Indent "8" .FlagTLSCert %>
      TLS_KEY: |-
        <% .Indent "8" .FlagTLSKey %>
      WORKERS: "<% .FlagWorkers %>"
      WORKER_SIZE: "<% .FlagWorkerSize %>"
      DEPLOYMENT: "<% .Deployment %>"
      AWS_ACCESS_KEY_ID: "<% .AWSAccessKeyID %>"
      AWS_SECRET_ACCESS_KEY: "<% .AWSSecretAccessKey %>"
      SELF_UPDATE: true
    config:
      platform: linux
      image_resource:
        type: docker-image
        source:
          repository: engineerbetter/cup-image
      inputs:
      - name: concourse-up-release
      run:
        path: bash
        args:
        - -c
        - |
          set -eux

          cd concourse-up-release
          chmod +x concourse-up-linux-amd64
          ./concourse-up-linux-amd64 deploy $DEPLOYMENT
`
<commit_msg>increase time waiting for ATC to wake up<commit_after>package fly

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"time"

	"github.com/EngineerBetter/concourse-up/config"
	"github.com/EngineerBetter/concourse-up/util"
)

// DarwinBinaryURL is a compile-time variable set with -ldflags
var DarwinBinaryURL = "COMPILE_TIME_VARIABLE_fly_darwin_binary_url"

// LinuxBinaryURL is a compile-time variable set with -ldflags
var LinuxBinaryURL = "COMPILE_TIME_VARIABLE_fly_linux_binary_url"

// WindowsBinaryURL is a compile-time variable set with -ldflags
var WindowsBinaryURL = "COMPILE_TIME_VARIABLE_fly_windows_binary_url"

// IClient represents an interface for a client
type IClient interface {
	CanConnect() (bool, error)
	SetDefaultPipeline(deployArgs *config.DeployArgs, config *config.Config) error
	Cleanup() error
}

// Client represents a low-level wrapper for fly
type Client struct {
	tempDir *util.TempDir
	creds   Credentials
	stdout  io.Writer
	stderr  io.Writer
}

// Credentials represents credentials needed to connect to concourse
type Credentials struct {
	Target   string
	API      string
	Username string
	Password string
	CACert   string
}

// New returns a new fly client
func New(creds Credentials, stdout, stderr io.Writer) (IClient, error) {
	tempDir, err := util.NewTempDir()
	if err != nil {
		return nil, err
	}

	fileHandler, err := os.Create(tempDir.Path("fly"))
	if err != nil {
		return nil, err
	}
	defer fileHandler.Close()

	url, err := getFlyURL()
	if err != nil {
		return nil, err
	}

	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if _, err := io.Copy(fileHandler, resp.Body); err != nil {
		return nil, err
	}

	if err := fileHandler.Sync(); err != nil {
		return nil, err
	}

	if err := os.Chmod(fileHandler.Name(), 0700); err != nil {
		return nil, err
	}

	return &Client{
		tempDir,
		creds,
		stdout,
		stderr,
	}, nil
}

// CanConnect returns true if it can connect to the concourse
func (client *Client) CanConnect() (bool, error) {
	cmd := exec.Command(
		client.tempDir.Path("fly"),
		"--target",
		client.creds.Target,
		"login",
		"--insecure",
		"--concourse-url",
		client.creds.API,
		"--username",
		client.creds.Username,
		"--password",
		client.creds.Password,
	)

	stderr := bytes.NewBuffer(nil)
	cmd.Stdout = client.stdout
	cmd.Stderr = stderr

	runErr := cmd.Run()
	if runErr == nil {
		return true, nil
	}

	stderrBytes, err := ioutil.ReadAll(stderr)
	if err != nil {
		return false, err
	}

	if strings.Contains(string(stderrBytes), "could not reach the Concourse server") {
		return false, nil
	}

	// if there is a legitimate error, copy it to stderr for debugging
	if _, err := client.stderr.Write(stderrBytes); err != nil {
		return false, err
	}

	return false, runErr
}

// SetDefaultPipeline sets the default pipeline against a given concourse
func (client *Client) SetDefaultPipeline(deployArgs *config.DeployArgs, config *config.Config) error {
	if err := client.login(); err != nil {
		return err
	}

	pipelinePath := client.tempDir.Path("default-pipeline.yml")
	pipelineName := "concourse-up-self-update"

	if err := client.writePipelineConfig(pipelinePath, deployArgs, config); err != nil {
		return err
	}

	if err := client.run("set-pipeline", "--pipeline", pipelineName, "--config", pipelinePath, "--non-interactive"); err != nil {
		return err
	}

	if err := os.Remove(pipelinePath); err != nil {
		return err
	}

	return nil
}

func (client *Client) writePipelineConfig(pipelinePath string, deployArgs *config.DeployArgs, config *config.Config) error {
	fileHandler, err := os.Create(pipelinePath)
	if err != nil {
		return err
	}
	defer fileHandler.Close()

	params, err := client.buildDefaultPipelineParams(deployArgs, config)
	if err != nil {
		return err
	}

	pipelineConfig, err := util.RenderTemplate(defaultPipelineTemplate, params)
	if err != nil {
		return err
	}

	if _, err := fileHandler.Write(pipelineConfig); err != nil {
		return err
	}

	if err := fileHandler.Sync(); err != nil {
		return err
	}

	return nil
}

// Cleanup removes tempfiles
func (client *Client) Cleanup() error {
	return client.tempDir.Cleanup()
}

func (client *Client) login() error {
	attempts := 50
	secondsBetweenAttempts := 4

	if _, err := client.stdout.Write([]byte("Waiting for Concourse ATC to start... \n")); err != nil {
		return err
	}

	for i := 0; i < attempts; i++ {
		canConnect, err := client.CanConnect()
		if err != nil {
			return err
		}
		if canConnect {
			return nil
		}

		time.Sleep(time.Second * time.Duration(secondsBetweenAttempts))
	}

	return fmt.Errorf("failed to log in to %s after %d seconds", client.creds.API, attempts*secondsBetweenAttempts)
}

func (client *Client) run(args ...string) error {
	args = append([]string{"--target", client.creds.Target}, args...)
	cmd := exec.Command(client.tempDir.Path("fly"), args...)
	cmd.Stdout = client.stdout
	cmd.Stderr = client.stderr
	return cmd.Run()
}

func getFlyURL() (string, error) {
	os := runtime.GOOS
	if os == "darwin" {
		return DarwinBinaryURL, nil
	} else if os == "linux" {
		return LinuxBinaryURL, nil
	} else if os == "windows" {
		return WindowsBinaryURL, nil
	}
	return "", fmt.Errorf("unknown os: `%s`", os)
}

func (client *Client) buildDefaultPipelineParams(deployArgs *config.DeployArgs, config *config.Config) (*defaultPipelineParams, error) {
	awsAccessKeyID := os.Getenv("AWS_ACCESS_KEY_ID")
	if awsAccessKeyID == "" {
		return nil, errors.New("env var AWS_ACCESS_KEY_ID not found")
	}

	awsSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
	if awsSecretAccessKey == "" {
		return nil, errors.New("env var AWS_SECRET_ACCESS_KEY not found")
	}

	return &defaultPipelineParams{
		FlagAWSRegion:      deployArgs.AWSRegion,
		FlagDomain:         deployArgs.Domain,
		FlagTLSCert:        deployArgs.TLSCert,
		FlagTLSKey:         deployArgs.TLSKey,
		FlagWorkers:        deployArgs.WorkerCount,
		FlagWorkerSize:     deployArgs.WorkerSize,
		Deployment:         strings.TrimPrefix(config.Deployment, "concourse-up-"),
		AWSAccessKeyID:     awsAccessKeyID,
		AWSSecretAccessKey: awsSecretAccessKey,
	}, nil
}

type defaultPipelineParams struct {
	FlagAWSRegion      string
	FlagDomain         string
	FlagTLSCert        string
	FlagTLSKey         string
	FlagWorkers        int
	FlagWorkerSize     string
	Deployment         string
	AWSAccessKeyID     string
	AWSSecretAccessKey string
	AWSDefaultRegion   string
}

// Indent is a helper function to indent the field a given number of spaces
func (params defaultPipelineParams) Indent(countStr, field string) string {
	return util.Indent(countStr, field)
}

const defaultPipelineTemplate = `
---
resources:
- name: concourse-up-release
  type: github-release
  source:
    user: engineerbetter
    repository: concourse-up
    pre_release: true

jobs:
- name: self-update
  serial: true
  plan:
  - get: concourse-up-release
    trigger: true
  - task: update
    params:
      AWS_REGION: "<% .FlagAWSRegion %>"
      DOMAIN: "<% .FlagDomain %>"
      TLS_CERT: |-
        <% .Indent "8" .FlagTLSCert %>
      TLS_KEY: |-
        <% .Indent "8" .FlagTLSKey %>
      WORKERS: "<% .FlagWorkers %>"
      WORKER_SIZE: "<% .FlagWorkerSize %>"
      DEPLOYMENT: "<% .Deployment %>"
      AWS_ACCESS_KEY_ID: "<% .AWSAccessKeyID %>"
      AWS_SECRET_ACCESS_KEY: "<% .AWSSecretAccessKey %>"
      SELF_UPDATE: true
    config:
      platform: linux
      image_resource:
        type: docker-image
        source:
          repository: engineerbetter/cup-image
      inputs:
      - name: concourse-up-release
      run:
        path: bash
        args:
        - -c
        - |
          set -eux

          cd concourse-up-release
          chmod +x concourse-up-linux-amd64
          ./concourse-up-linux-amd64 deploy $DEPLOYMENT
`
<|endoftext|>