<commit_before>/*
Package stats currentelly only implements a t-test for importance score analysis in CloudForest.

Some functions are ports or functions from rf-ace's math.cpp
*/
package stats

import (
	"math"
)

//MeanAndVar returns the sample mean, variance and count as float64's.
func MeanAndVar(X *[]float64) (m, v, n float64) {
	for _, x := range *X {
		m += x
		v += x * x
	}
	n = float64(len(*X))
	m /= n
	v -= n * m * m
	v /= (n - 1.0)
	return
}

//Ttest performs a Welch's t test and returns p-value, t-value and the degrees of freedom.
//The p value based on the hypothesis that mean(B)>mean(A).
//Based on code from rf-ace.
func Ttest(A, B *[]float64) (p, t, v float64) {

	// Calculate means and variances for each of two samples.
	Am, Av, An := MeanAndVar(A)
	Bm, Bv, Bn := MeanAndVar(B)

	//Welch's t test
	As := Av / An
	Bs := Bv / Bn
	s := As + Bs
	t = (Am - Bm) / math.Sqrt(s)

	// Degree's Freedom for Welch's
	v = s * s / (As*As/(An-1) + Bs*Bs/(Bn-1))

	// Find the tail probability  of t

	// Transformed t-test statistic
	ttrans := v / (t*t + v)

	// This variable will store the integral of the tail of the t-distribution
	integral := 0.0

	// When ttrans > 0.9, we need to recast the integration in order to retain
	// accuracy. In other words we make use of the following identity:
	//
	// I(x,a,b) = 1 - I(1-x,b,a)
	if ttrans > 0.9 {
		// Calculate I(x,a,b) as 1 - I(1-x,b,a)
		integral = 1 - regularizedIncompleteBeta(1-ttrans, 0.5, v/2)

	} else {
		// Calculate I(x,a,b) directly
		integral = regularizedIncompleteBeta(ttrans, v/2, 0.5)
	}

	// We need to be careful about which way to calculate the integral so that it represents
	// the tail of the t-distribution. The sign of the tvalue hints which way to integrate
	if t > 0.0 {
		p = (integral / 2)
	} else {
		p = (1 - integral/2)
	}
	return
}

func regularizedIncompleteBeta(x, a, b float64) float64 {
	i := 50
	continuedFraction := 1.0
	m := 0.0

	for i > 0 {
		m = float64(i)
		continuedFraction = 1.0 + dE(m, x, a, b)/(1+dO(m, x, a, b)/continuedFraction)
		i--
	}
	return (math.Pow(x, a) * math.Pow(1-x, b) / (a * beta(a, b) * (1 + dO(0, x, a, b)/continuedFraction)))
}

func dO(m, x, a, b float64) float64 {
	return (-1.0 * (a + m) * (a + b + m) * x / ((a + 2*m) * (a + 2*m + 1)))
}

/**
  Even factors for the infinite continued fraction representation of the
  regularized incomplete beta function
*/
func dE(m, x, a, b float64) float64 {
	return (m * (b - m) * x / ((a + 2*m - 1) * (a + 2*m)))
}

func lgamma(x float64) float64 {
	v, _ := math.Lgamma(x)
	//v := math.Log(math.Abs(math.Gamma(x)))
	return v
}

func beta(a, b float64) float64 {
	return (math.Exp(lgamma(a) + lgamma(b) - lgamma(a+b)))
}
<commit_msg>Clean up stats documentation<commit_after>/*
Package stats currentelly only implements a welch's t-test for importance score analysis
in CloudForest.
*/
package stats

import (
	"math"
)

//MeanAndVar returns the sample mean, variance and count as float64's.
func MeanAndVar(X *[]float64) (m, v, n float64) {
	for _, x := range *X {
		m += x
		v += x * x
	}
	n = float64(len(*X))
	m /= n
	v -= n * m * m
	v /= (n - 1.0)
	return
}

//Ttest performs a Welch's t test and returns p-value, t-value and the degrees of freedom.
//The p value based on the hypothesis that mean(B)>mean(A).
//Based on similar code in rf-ace (Apache 2.0, Timo Erkkilä)
func Ttest(A, B *[]float64) (p, t, v float64) {

	// Calculate means and variances for each of two samples.
	Am, Av, An := MeanAndVar(A)
	Bm, Bv, Bn := MeanAndVar(B)

	//Welch's t test
	As := Av / An
	Bs := Bv / Bn
	s := As + Bs
	t = (Am - Bm) / math.Sqrt(s)

	// Degree's Freedom for Welch's
	v = s * s / (As*As/(An-1) + Bs*Bs/(Bn-1))

	// Find the tail probability  of t

	// Transformed t-test statistic
	ttrans := v / (t*t + v)

	// This variable will store the integral of the tail of the t-distribution
	integral := 0.0

	// Comment from rf-ace:
	// When ttrans > 0.9, we need to recast the integration in order to retain
	// accuracy. In other words we make use of the following identity:
	//
	// I(x,a,b) = 1 - I(1-x,b,a)
	if ttrans > 0.9 {
		// Calculate I(x,a,b) as 1 - I(1-x,b,a)
		integral = 1 - regularizedIncompleteBeta(1-ttrans, 0.5, v/2)

	} else {
		// Calculate I(x,a,b) directly
		integral = regularizedIncompleteBeta(ttrans, v/2, 0.5)
	}

	// Comment from rf-ace:
	// We need to be careful about which way to calculate the integral so that it represents
	// the tail of the t-distribution. The sign of the tvalue hints which way to integrate
	if t > 0.0 {
		p = (integral / 2)
	} else {
		p = (1 - integral/2)
	}
	return
}

//Based on similar code in rf-ace (Apache 2.0, Timo Erkkilä)
func regularizedIncompleteBeta(x, a, b float64) float64 {
	i := 50
	continuedFraction := 1.0
	m := 0.0

	for i > 0 {
		m = float64(i)
		continuedFraction = 1.0 + dE(m, x, a, b)/(1+dO(m, x, a, b)/continuedFraction)
		i--
	}
	return (math.Pow(x, a) * math.Pow(1-x, b) / (a * beta(a, b) * (1 + dO(0, x, a, b)/continuedFraction)))
}

/*
  Even and odd factors for the infinite continued fraction representation of the
  regularized incomplete beta function.
  Based on similar code in rf-ace (Apache 2.0, Timo Erkkilä)
*/
func dO(m, x, a, b float64) float64 {
	return (-1.0 * (a + m) * (a + b + m) * x / ((a + 2*m) * (a + 2*m + 1)))
}

func dE(m, x, a, b float64) float64 {
	return (m * (b - m) * x / ((a + 2*m - 1) * (a + 2*m)))
}

//Based on similar code in rf-ace (Apache 2.0, Timo Erkkilä)
func lgamma(x float64) float64 {
	v, _ := math.Lgamma(x)
	//v := math.Log(math.Abs(math.Gamma(x)))
	return v
}

//Based on similar code in rf-ace (Apache 2.0, Timo Erkkilä)
func beta(a, b float64) float64 {
	return (math.Exp(lgamma(a) + lgamma(b) - lgamma(a+b)))
}
<|endoftext|>


<commit_before>package statsd

import (
	"bytes"
	"errors"
	"math/rand"
	"net"
	"strconv"
	"sync"
	"time"
)

type Statter interface {
	Inc(string, int64, float32) error
	Dec(string, int64, float32) error
	Gauge(string, int64, float32) error
	GaugeDelta(string, int64, float32) error
	Timing(string, int64, float32) error
	TimingDuration(string, time.Duration, float32) error
	Set(string, string, float32) error
	SetInt(string, int64, float32) error
	Raw(string, string, float32) error
	SetPrefix(string)
	Close() error
}

type Sender interface {
	Send(data []byte) (int, error)
	Close() error
}

type Client struct {
	// prefix for statsd name
	prefix string
	// sync.Pool of bytes.Buffer
	bytePool *sync.Pool
	// packet sender
	sender Sender
}

// Close closes the connection and cleans up.
func (s *Client) Close() error {
	if s == nil {
		return nil
	}
	err := s.sender.Close()
	return err
}

// Increments a statsd count type.
// stat is a string name for the metric.
// value is the integer value
// rate is the sample rate (0.0 to 1.0)
func (s *Client) Inc(stat string, value int64, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	dap := strconv.FormatInt(value, 10)
	return s.submit(stat, dap, "|c", rate)
}

// Decrements a statsd count type.
// stat is a string name for the metric.
// value is the integer value.
// rate is the sample rate (0.0 to 1.0).
func (s *Client) Dec(stat string, value int64, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	dap := strconv.FormatInt(-value, 10)
	return s.submit(stat, dap, "|c", rate)
}

// Submits/Updates a statsd gauge type.
// stat is a string name for the metric.
// value is the integer value.
// rate is the sample rate (0.0 to 1.0).
func (s *Client) Gauge(stat string, value int64, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	dap := strconv.FormatInt(value, 10)
	return s.submit(stat, dap, "|g", rate)
}

// Submits a delta to a statsd gauge.
// stat is the string name for the metric.
// value is the (positive or negative) change.
// rate is the sample rate (0.0 to 1.0).
func (s *Client) GaugeDelta(stat string, value int64, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}

	prefix := ""
	if value >= 0 {
		prefix = "+"
	}
	dap := prefix + strconv.FormatInt(value, 10)
	return s.submit(stat, dap, "|g", rate)
}

// Submits a statsd timing type.
// stat is a string name for the metric.
// delta is the time duration value in milliseconds
// rate is the sample rate (0.0 to 1.0).
func (s *Client) Timing(stat string, delta int64, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	dap := strconv.FormatInt(delta, 10)
	return s.submit(stat, dap, "|ms", rate)
}

// Submits a statsd timing type.
// stat is a string name for the metric.
// delta is the timing value as time.Duration
// rate is the sample rate (0.0 to 1.0).
func (s *Client) TimingDuration(stat string, delta time.Duration, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	ms := float64(delta) / float64(time.Millisecond)
	//dap := fmt.Sprintf("%.02f|ms", ms)
	dap := strconv.FormatFloat(ms, 'f', -1, 64)
	return s.submit(stat, dap, "|ms", rate)
}

// Submits a stats set type
// stat is a string name for the metric.
// value is the string value
// rate is the sample rate (0.0 to 1.0).
func (s *Client) Set(stat string, value string, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	return s.submit(stat, value, "|s", rate)
}

// Submits a number as a stats set type.
// stat is a string name for the metric.
// value is the integer value
// rate is the sample rate (0.0 to 1.0).
func (s *Client) SetInt(stat string, value int64, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	dap := strconv.FormatInt(value, 10)
	return s.submit(stat, dap, "|s", rate)
}

// Raw submits a preformatted value.
// stat is the string name for the metric.
// value is a preformatted "raw" value string.
// rate is the sample rate (0.0 to 1.0).
func (s *Client) Raw(stat string, value string, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	return s.submit(stat, value, "", rate)
}

func (s *Client) getBuffer() *bytes.Buffer {
	if s.bytePool == nil {
		return &bytes.Buffer{}
	}
	buf := s.bytePool.Get().(*bytes.Buffer)
	buf.Reset()
	return buf
}

func (s *Client) putBuffer(buf *bytes.Buffer) {
	if s.bytePool == nil {
		return
	}
	s.bytePool.Put(buf)
	return
}

// submit an already sampled raw stat
func (s *Client) submit(stat, value, suffix string, rate float32) error {
	if s == nil {
		return nil
	}

	data := s.getBuffer()
	if s.prefix != "" {
		data.WriteString(s.prefix)
		data.WriteString(".")
	}
	data.WriteString(stat)
	data.WriteString(":")
	data.WriteString(value)
	if suffix != "" {
		data.WriteString(suffix)
	}

	if rate < 1 {
		data.WriteString("|@")
		data.WriteString(strconv.FormatFloat(float64(rate), 'f', 6, 32))
	}

	d := data.Bytes()
	_, err := s.sender.Send(d)
	if err != nil {
		return err
	}
	return nil
}

// check for nil client, and perform sampling calculation
func (s *Client) includeStat(rate float32) bool {
	if s == nil {
		return false
	}

	if rate < 1 {
		if rand.Float32() < rate {
			return true
		}
		return false
	}
	return true
}

// Sets/Updates the statsd client prefix.
func (s *Client) SetPrefix(prefix string) {
	if s == nil {
		return
	}
	s.prefix = prefix
}

// SimpleSender provides a socket send interface.
type SimpleSender struct {
	// underlying connection
	c net.PacketConn
	// resolved udp address
	ra *net.UDPAddr
}

// Send sends the data to the server endpoint.
func (s *SimpleSender) Send(data []byte) (int, error) {
	// no need for locking here, as the underlying fdNet
	// already serialized writes
	n, err := s.c.(*net.UDPConn).WriteToUDP(data, s.ra)
	if err != nil {
		return 0, err
	}
	if n == 0 {
		return n, errors.New("Wrote no bytes")
	}
	return n, nil
}

// Closes SimpleSender
func (s *SimpleSender) Close() error {
	err := s.c.Close()
	return err
}

// Returns a new SimpleSender for sending to the supplied addresss.
//
// addr is a string of the format "hostname:port", and must be parsable by
// net.ResolveUDPAddr.
func NewSimpleSender(addr string) (Sender, error) {
	c, err := net.ListenPacket("udp", ":0")
	if err != nil {
		return nil, err
	}

	ra, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		return nil, err
	}

	sender := &SimpleSender{
		c:  c,
		ra: ra,
	}

	return sender, nil
}

// Returns a pointer to a new Client, and an error.
//
// addr is a string of the format "hostname:port", and must be parsable by
// net.ResolveUDPAddr.
//
// prefix is the statsd client prefix. Can be "" if no prefix is desired.
func NewClient(addr, prefix string) (Statter, error) {
	sender, err := NewSimpleSender(addr)
	if err != nil {
		return nil, err
	}

	client := &Client{
		prefix:   prefix,
		bytePool: &sync.Pool{New: func() interface{} { return &bytes.Buffer{} }},
		sender:   sender,
	}

	return client, nil
}

// Compatibility alias
var Dial = New
var New = NewClient
<commit_msg>use buffer pool properly now that there is no longer raciness in bufferedSender<commit_after>package statsd

import (
	"bytes"
	"errors"
	"math/rand"
	"net"
	"strconv"
	"sync"
	"time"
)

type Statter interface {
	Inc(string, int64, float32) error
	Dec(string, int64, float32) error
	Gauge(string, int64, float32) error
	GaugeDelta(string, int64, float32) error
	Timing(string, int64, float32) error
	TimingDuration(string, time.Duration, float32) error
	Set(string, string, float32) error
	SetInt(string, int64, float32) error
	Raw(string, string, float32) error
	SetPrefix(string)
	Close() error
}

type Sender interface {
	Send(data []byte) (int, error)
	Close() error
}

type Client struct {
	// prefix for statsd name
	prefix string
	// sync.Pool of bytes.Buffer
	bytePool *sync.Pool
	// packet sender
	sender Sender
}

// Close closes the connection and cleans up.
func (s *Client) Close() error {
	if s == nil {
		return nil
	}
	err := s.sender.Close()
	return err
}

// Increments a statsd count type.
// stat is a string name for the metric.
// value is the integer value
// rate is the sample rate (0.0 to 1.0)
func (s *Client) Inc(stat string, value int64, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	dap := strconv.FormatInt(value, 10)
	return s.submit(stat, dap, "|c", rate)
}

// Decrements a statsd count type.
// stat is a string name for the metric.
// value is the integer value.
// rate is the sample rate (0.0 to 1.0).
func (s *Client) Dec(stat string, value int64, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	dap := strconv.FormatInt(-value, 10)
	return s.submit(stat, dap, "|c", rate)
}

// Submits/Updates a statsd gauge type.
// stat is a string name for the metric.
// value is the integer value.
// rate is the sample rate (0.0 to 1.0).
func (s *Client) Gauge(stat string, value int64, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	dap := strconv.FormatInt(value, 10)
	return s.submit(stat, dap, "|g", rate)
}

// Submits a delta to a statsd gauge.
// stat is the string name for the metric.
// value is the (positive or negative) change.
// rate is the sample rate (0.0 to 1.0).
func (s *Client) GaugeDelta(stat string, value int64, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}

	prefix := ""
	if value >= 0 {
		prefix = "+"
	}
	dap := prefix + strconv.FormatInt(value, 10)
	return s.submit(stat, dap, "|g", rate)
}

// Submits a statsd timing type.
// stat is a string name for the metric.
// delta is the time duration value in milliseconds
// rate is the sample rate (0.0 to 1.0).
func (s *Client) Timing(stat string, delta int64, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	dap := strconv.FormatInt(delta, 10)
	return s.submit(stat, dap, "|ms", rate)
}

// Submits a statsd timing type.
// stat is a string name for the metric.
// delta is the timing value as time.Duration
// rate is the sample rate (0.0 to 1.0).
func (s *Client) TimingDuration(stat string, delta time.Duration, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	ms := float64(delta) / float64(time.Millisecond)
	//dap := fmt.Sprintf("%.02f|ms", ms)
	dap := strconv.FormatFloat(ms, 'f', -1, 64)
	return s.submit(stat, dap, "|ms", rate)
}

// Submits a stats set type
// stat is a string name for the metric.
// value is the string value
// rate is the sample rate (0.0 to 1.0).
func (s *Client) Set(stat string, value string, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	return s.submit(stat, value, "|s", rate)
}

// Submits a number as a stats set type.
// stat is a string name for the metric.
// value is the integer value
// rate is the sample rate (0.0 to 1.0).
func (s *Client) SetInt(stat string, value int64, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	dap := strconv.FormatInt(value, 10)
	return s.submit(stat, dap, "|s", rate)
}

// Raw submits a preformatted value.
// stat is the string name for the metric.
// value is a preformatted "raw" value string.
// rate is the sample rate (0.0 to 1.0).
func (s *Client) Raw(stat string, value string, rate float32) error {
	if !s.includeStat(rate) {
		return nil
	}
	return s.submit(stat, value, "", rate)
}

func (s *Client) getBuffer() *bytes.Buffer {
	if s.bytePool == nil {
		return &bytes.Buffer{}
	}
	buf := s.bytePool.Get().(*bytes.Buffer)
	return buf
}

func (s *Client) putBuffer(buf *bytes.Buffer) {
	if s.bytePool == nil {
		return
	}
	buf.Reset()
	s.bytePool.Put(buf)
	return
}

// submit an already sampled raw stat
func (s *Client) submit(stat, value, suffix string, rate float32) error {
	if s == nil {
		return nil
	}

	data := s.getBuffer()
	defer s.putBuffer(data)
	if s.prefix != "" {
		data.WriteString(s.prefix)
		data.WriteString(".")
	}
	data.WriteString(stat)
	data.WriteString(":")
	data.WriteString(value)
	if suffix != "" {
		data.WriteString(suffix)
	}

	if rate < 1 {
		data.WriteString("|@")
		data.WriteString(strconv.FormatFloat(float64(rate), 'f', 6, 32))
	}

	_, err := s.sender.Send(data.Bytes())
	if err != nil {
		return err
	}
	return nil
}

// check for nil client, and perform sampling calculation
func (s *Client) includeStat(rate float32) bool {
	if s == nil {
		return false
	}

	if rate < 1 {
		if rand.Float32() < rate {
			return true
		}
		return false
	}
	return true
}

// Sets/Updates the statsd client prefix.
func (s *Client) SetPrefix(prefix string) {
	if s == nil {
		return
	}
	s.prefix = prefix
}

// SimpleSender provides a socket send interface.
type SimpleSender struct {
	// underlying connection
	c net.PacketConn
	// resolved udp address
	ra *net.UDPAddr
}

// Send sends the data to the server endpoint.
func (s *SimpleSender) Send(data []byte) (int, error) {
	// no need for locking here, as the underlying fdNet
	// already serialized writes
	n, err := s.c.(*net.UDPConn).WriteToUDP(data, s.ra)
	if err != nil {
		return 0, err
	}
	if n == 0 {
		return n, errors.New("Wrote no bytes")
	}
	return n, nil
}

// Closes SimpleSender
func (s *SimpleSender) Close() error {
	err := s.c.Close()
	return err
}

// Returns a new SimpleSender for sending to the supplied addresss.
//
// addr is a string of the format "hostname:port", and must be parsable by
// net.ResolveUDPAddr.
func NewSimpleSender(addr string) (Sender, error) {
	c, err := net.ListenPacket("udp", ":0")
	if err != nil {
		return nil, err
	}

	ra, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		return nil, err
	}

	sender := &SimpleSender{
		c:  c,
		ra: ra,
	}

	return sender, nil
}

// Returns a pointer to a new Client, and an error.
//
// addr is a string of the format "hostname:port", and must be parsable by
// net.ResolveUDPAddr.
//
// prefix is the statsd client prefix. Can be "" if no prefix is desired.
func NewClient(addr, prefix string) (Statter, error) {
	sender, err := NewSimpleSender(addr)
	if err != nil {
		return nil, err
	}

	client := &Client{
		prefix:   prefix,
		bytePool: &sync.Pool{New: func() interface{} { return &bytes.Buffer{} }},
		sender:   sender,
	}

	return client, nil
}

// Compatibility alias
var Dial = New
var New = NewClient
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package atomic

import (
	"runtime"
	"sync"
	"testing"
)

const (
	_parallelism = 4
	_iterations  = 1000
)

func runStress(f func()) {
	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(_parallelism))

	var wg sync.WaitGroup
	wg.Add(_parallelism)
	for i := 0; i < _parallelism; i++ {
		go func() {
			defer wg.Done()
			for j := 0; j < _iterations; j++ {
				f()
			}
		}()
	}

	wg.Wait()
}

func TestStressInt32(t *testing.T) {
	var atom Int32
	runStress(func() {
		atom.Load()
		atom.Add(1)
		atom.Sub(2)
		atom.Inc()
		atom.Dec()
		atom.CAS(1, 0)
		atom.Swap(5)
		atom.Store(1)
	})
}

func TestStressInt64(t *testing.T) {
	var atom Int64
	runStress(func() {
		atom.Load()
		atom.Add(1)
		atom.Sub(2)
		atom.Inc()
		atom.Dec()
		atom.CAS(1, 0)
		atom.Swap(5)
		atom.Store(1)

	})
}

func TestStressUint32(t *testing.T) {
	var atom Uint32
	runStress(func() {
		atom.Load()
		atom.Add(1)
		atom.Sub(2)
		atom.Inc()
		atom.Dec()
		atom.CAS(1, 0)
		atom.Swap(5)
		atom.Store(1)
	})
}

func TestStressUint64(t *testing.T) {
	var atom Uint64
	runStress(func() {
		atom.Load()
		atom.Add(1)
		atom.Sub(2)
		atom.Inc()
		atom.Dec()
		atom.CAS(1, 0)
		atom.Swap(5)
		atom.Store(1)
	})
}

func TestStressFloat64(t *testing.T) {
	var atom Float64
	runStress(func() {
		atom.Load()
		atom.CAS(1.0, 0.1)
		atom.Add(1.1)
		atom.Sub(0.2)
		atom.Store(1.0)
	})
}

func TestStressBool(t *testing.T) {
	var atom Bool
	runStress(func() {
		atom.Load()
		atom.Store(false)
		atom.Swap(true)
		atom.CAS(true, false)
		atom.CAS(true, false)
		atom.Load()
		atom.Toggle()
		atom.Toggle()
	})
}

func TestStressString(t *testing.T) {
	var atom String
	runStress(func() {
		atom.Load()
		atom.Store("abc")
		atom.Load()
		atom.Store("def")

	})
}
<commit_msg>Use stress tests for benchmarks<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package atomic

import (
	"runtime"
	"sync"
	"testing"
)

const (
	_parallelism = 4
	_iterations  = 1000
)

var _stressTests = map[string]func(){
	"i32":    stressInt32,
	"i64":    stressInt64,
	"u32":    stressUint32,
	"u64":    stressUint64,
	"f64":    stressFloat64,
	"bool":   stressBool,
	"string": stressString,
}

func TestStress(t *testing.T) {
	for name, f := range _stressTests {
		t.Run(name, func(t *testing.T) {
			defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(_parallelism))

			start := make(chan struct{})
			var wg sync.WaitGroup
			wg.Add(_parallelism)
			for i := 0; i < _parallelism; i++ {
				go func() {
					defer wg.Done()
					<-start
					for j := 0; j < _iterations; j++ {
						f()
					}
				}()
			}
			close(start)
			wg.Wait()
		})
	}
}

func BenchmarkStress(b *testing.B) {
	for name, f := range _stressTests {
		b.Run(name, func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				f()
			}
		})
	}
}

func stressInt32() {
	var atom Int32
	atom.Load()
	atom.Add(1)
	atom.Sub(2)
	atom.Inc()
	atom.Dec()
	atom.CAS(1, 0)
	atom.Swap(5)
	atom.Store(1)
}

func stressInt64() {
	var atom Int64
	atom.Load()
	atom.Add(1)
	atom.Sub(2)
	atom.Inc()
	atom.Dec()
	atom.CAS(1, 0)
	atom.Swap(5)
	atom.Store(1)
}

func stressUint32() {
	var atom Uint32
	atom.Load()
	atom.Add(1)
	atom.Sub(2)
	atom.Inc()
	atom.Dec()
	atom.CAS(1, 0)
	atom.Swap(5)
	atom.Store(1)
}

func stressUint64() {
	var atom Uint64
	atom.Load()
	atom.Add(1)
	atom.Sub(2)
	atom.Inc()
	atom.Dec()
	atom.CAS(1, 0)
	atom.Swap(5)
	atom.Store(1)
}

func stressFloat64() {
	var atom Float64
	atom.Load()
	atom.CAS(1.0, 0.1)
	atom.Add(1.1)
	atom.Sub(0.2)
	atom.Store(1.0)
}

func stressBool() {
	var atom Bool
	atom.Load()
	atom.Store(false)
	atom.Swap(true)
	atom.CAS(true, false)
	atom.CAS(true, false)
	atom.Load()
	atom.Toggle()
	atom.Toggle()
}

func stressString() {
	var atom String
	atom.Load()
	atom.Store("abc")
	atom.Load()
	atom.Store("def")
	atom.Load()
	atom.Store("")
}
<|endoftext|>


<commit_before>package gofakeit

import (
	"fmt"
	"testing"
)

type Basic struct {
	s string
	S string
}

type Nested struct {
	A   string
	B   *Basic
	bar *Basic
}

type BuiltIn struct {
	Uint8   *uint8
	Uint16  *uint16
	Uint32  *uint32
	Uint64  *uint64
	Int     *int
	Int8    *int8
	Int16   *int16
	Int32   *int32
	Int64   *int64
	Float32 *float32
	Float64 *float64
	Bool    *bool
}

type Template struct {
	Number *string `fake:"#"`
	Name   *string `fake:"{firstname}"`
}

type StructArray struct {
	Bars   []*Basic
	Builds []BuiltIn
	Skips  []*Basic    `fake:"skip"`
	Empty  []*Basic    `fake:"0"`
	Multy  []*Template `fake:"3"`
}

type NestedArray struct {
	NA []StructArray `fake:"2"`
}

func TestStructBasic(t *testing.T) {
	var basic Basic
	Struct(&basic)
	if basic.s != "" {
		t.Error("unexported field is not populated")
	}
	if basic.S == "" {
		t.Error("exported field is populated")
	}
}

func TestStructNested(t *testing.T) {
	var nested Nested
	Struct(&nested)
	if nested.A == "" {
		t.Error("exported string field is populated")
	}
	if nested.B == nil {
		t.Error("exported struct field is populated")
	}
	if nested.B.S == "" {
		t.Error("nested struct string field is not populated")
	}
	if nested.bar != nil {
		t.Error("nested struct bar should be be populated due to unexported variable")
	}
}

func TestStructBuiltInTypes(t *testing.T) {
	var builtIn BuiltIn
	Struct(&builtIn)
	if builtIn.Uint8 == nil {
		t.Error("builtIn Uint8 was not set")
	}
	if builtIn.Uint16 == nil {
		t.Error("builtIn Uint16 was not set")
	}
	if builtIn.Uint32 == nil {
		t.Error("builtIn Uint32 was not set")
	}
	if builtIn.Uint64 == nil {
		t.Error("builtIn Uint64 was not set")
	}
	if builtIn.Int == nil {
		t.Error("builtIn int was not set")
	}
	if builtIn.Int8 == nil {
		t.Error("builtIn int8 was not set")
	}
	if builtIn.Int16 == nil {
		t.Error("builtIn int16 was not set")
	}
	if builtIn.Int32 == nil {
		t.Error("builtIn int32 was not set")
	}
	if builtIn.Int64 == nil {
		t.Error("builtIn int64 was not set")
	}
	if builtIn.Float32 == nil {
		t.Error("builtIn float32 was not set")
	}
	if builtIn.Float64 == nil {
		t.Error("builtIn float64 was not set")
	}
	if builtIn.Bool == nil {
		t.Error("builtIn bool was not set")
	}
}

func TestStructWithTemplate(t *testing.T) {
	var template Template
	Struct(&template)
	if *template.Number == "" {
		t.Error("template number should set to number value")
	}
	if *template.Name == "" {
		t.Error("template number should set to number value")
	}
}

func TestStructArray(t *testing.T) {
	var sa StructArray
	Struct(&sa)
	if len(sa.Bars) != 1 {
		t.Error("sa slice Bars is not populated")
	}
	if len(sa.Builds) != 1 {
		t.Error("sa slice Builds is not populated")
	}
	if len(sa.Empty) != 0 {
		t.Error("sa slice Empty is populated")
	}
	if len(sa.Skips) != 0 {
		t.Error("sa slice Empty is populated")
	}
	if len(sa.Multy) != 3 {
		t.Error("sa slice Multy is not fully populated")
	}
}

func TestStructNestedArray(t *testing.T) {
	var na NestedArray
	Struct(&na)
	if len(na.NA) != 2 {
		t.Error("na slice NA is not populated")
	}
	for _, elem := range na.NA {
		if len(elem.Builds) == 0 {
			t.Error("a slice Builds is not populated")
		}
		if len(elem.Empty) != 0 {
			t.Error("a slice Empty is populated")
		}
		if len(elem.Skips) != 0 {
			t.Error("a slice Empty is populated")
		}
		if len(elem.Multy) != 3 {
			t.Error("a slice Multy is not fully populated")
		}
	}
}

func Example_struct() {
	Seed(11)

	type Foo struct {
		Bar     string
		Int     int
		Pointer *int
		Name    string  `fake:"{firstname}"`
		Number  string  `fake:"{number:1,10}"`
		Skip    *string `fake:"skip"`
	}

	var f Foo
	Struct(&f)

	fmt.Printf("%s\n", f.Bar)
	fmt.Printf("%d\n", f.Int)
	fmt.Printf("%d\n", *f.Pointer)
	fmt.Printf("%v\n", f.Name)
	fmt.Printf("%v\n", f.Number)
	fmt.Printf("%v\n", f.Skip)

	// Output: bRMaRxHki
	// -8576773003117070818
	// -7054675846543980602
	// Enrique
	// 4
	// <nil>
}

func Example_array() {
	Seed(11)

	type Foo struct {
		Bar     string
		Int     int
		Pointer *int
		Name    string  `fake:"{firstname}"`
		Number  string  `fake:"{number:1,10}"`
		Skip    *string `fake:"skip"`
	}

	type FooMany struct {
		Foos []Foo `fake:"2"`
	}

	var fm FooMany
	Struct(&fm)

	for _, f := range fm.Foos {
		fmt.Printf("%s\n", f.Bar)
		fmt.Printf("%d\n", f.Int)
		fmt.Printf("%d\n", *f.Pointer)
		fmt.Printf("%v\n", f.Name)
		fmt.Printf("%v\n", f.Number)
		fmt.Printf("%v\n", f.Skip)
	}

	// Output: brmarxhki
	// -8576773003117070818
	// -7054675846543980602
	// Enrique
	// 4
	// <nil>
	// pwyj
	// -3430133205295092491
	// -2330884613995904932
	// Fred
	// 1
	// <nil>
}
<commit_msg>struct test - added seeds and updated output<commit_after>package gofakeit

import (
	"fmt"
	"testing"
)

type Basic struct {
	s string
	S string
}

type Nested struct {
	A   string
	B   *Basic
	bar *Basic
}

type BuiltIn struct {
	Uint8   *uint8
	Uint16  *uint16
	Uint32  *uint32
	Uint64  *uint64
	Int     *int
	Int8    *int8
	Int16   *int16
	Int32   *int32
	Int64   *int64
	Float32 *float32
	Float64 *float64
	Bool    *bool
}

type Template struct {
	Number *string `fake:"#"`
	Name   *string `fake:"{firstname}"`
}

type StructArray struct {
	Bars   []*Basic
	Builds []BuiltIn
	Skips  []*Basic    `fake:"skip"`
	Empty  []*Basic    `fake:"0"`
	Multy  []*Template `fake:"3"`
}

type NestedArray struct {
	NA []StructArray `fake:"2"`
}

func TestStructBasic(t *testing.T) {
	Seed(11)

	var basic Basic
	Struct(&basic)
	if basic.s != "" {
		t.Error("unexported field is not populated")
	}
	if basic.S == "" {
		t.Error("exported field is populated")
	}
}

func TestStructNested(t *testing.T) {
	Seed(11)

	var nested Nested
	Struct(&nested)
	if nested.A == "" {
		t.Error("exported string field is populated")
	}
	if nested.B == nil {
		t.Error("exported struct field is populated")
	}
	if nested.B.S == "" {
		t.Error("nested struct string field is not populated")
	}
	if nested.bar != nil {
		t.Error("nested struct bar should be be populated due to unexported variable")
	}
}

func TestStructBuiltInTypes(t *testing.T) {
	Seed(11)

	var builtIn BuiltIn
	Struct(&builtIn)
	if builtIn.Uint8 == nil {
		t.Error("builtIn Uint8 was not set")
	}
	if builtIn.Uint16 == nil {
		t.Error("builtIn Uint16 was not set")
	}
	if builtIn.Uint32 == nil {
		t.Error("builtIn Uint32 was not set")
	}
	if builtIn.Uint64 == nil {
		t.Error("builtIn Uint64 was not set")
	}
	if builtIn.Int == nil {
		t.Error("builtIn int was not set")
	}
	if builtIn.Int8 == nil {
		t.Error("builtIn int8 was not set")
	}
	if builtIn.Int16 == nil {
		t.Error("builtIn int16 was not set")
	}
	if builtIn.Int32 == nil {
		t.Error("builtIn int32 was not set")
	}
	if builtIn.Int64 == nil {
		t.Error("builtIn int64 was not set")
	}
	if builtIn.Float32 == nil {
		t.Error("builtIn float32 was not set")
	}
	if builtIn.Float64 == nil {
		t.Error("builtIn float64 was not set")
	}
	if builtIn.Bool == nil {
		t.Error("builtIn bool was not set")
	}
}

func TestStructWithTemplate(t *testing.T) {
	Seed(11)

	var template Template
	Struct(&template)
	if *template.Number == "" {
		t.Error("template number should set to number value")
	}
	if *template.Name == "" {
		t.Error("template number should set to number value")
	}
}

func TestStructArray(t *testing.T) {
	Seed(11)

	var sa StructArray
	Struct(&sa)
	if len(sa.Bars) != 1 {
		t.Error("sa slice Bars is not populated")
	}
	if len(sa.Builds) != 1 {
		t.Error("sa slice Builds is not populated")
	}
	if len(sa.Empty) != 0 {
		t.Error("sa slice Empty is populated")
	}
	if len(sa.Skips) != 0 {
		t.Error("sa slice Empty is populated")
	}
	if len(sa.Multy) != 3 {
		t.Error("sa slice Multy is not fully populated")
	}
}

func TestStructNestedArray(t *testing.T) {
	Seed(11)

	var na NestedArray
	Struct(&na)
	if len(na.NA) != 2 {
		t.Error("na slice NA is not populated")
	}
	for _, elem := range na.NA {
		if len(elem.Builds) == 0 {
			t.Error("a slice Builds is not populated")
		}
		if len(elem.Empty) != 0 {
			t.Error("a slice Empty is populated")
		}
		if len(elem.Skips) != 0 {
			t.Error("a slice Empty is populated")
		}
		if len(elem.Multy) != 3 {
			t.Error("a slice Multy is not fully populated")
		}
	}
}

func Example_struct() {
	Seed(11)

	type Foo struct {
		Bar     string
		Int     int
		Pointer *int
		Name    string  `fake:"{firstname}"`
		Number  string  `fake:"{number:1,10}"`
		Skip    *string `fake:"skip"`
	}

	var f Foo
	Struct(&f)

	fmt.Printf("%s\n", f.Bar)
	fmt.Printf("%d\n", f.Int)
	fmt.Printf("%d\n", *f.Pointer)
	fmt.Printf("%v\n", f.Name)
	fmt.Printf("%v\n", f.Number)
	fmt.Printf("%v\n", f.Skip)

	// Output: bRMaRxHki
	// -8576773003117070818
	// -7054675846543980602
	// Enrique
	// 4
	// <nil>
}

func Example_array() {
	Seed(11)

	type Foo struct {
		Bar     string
		Int     int
		Pointer *int
		Name    string  `fake:"{firstname}"`
		Number  string  `fake:"{number:1,10}"`
		Skip    *string `fake:"skip"`
	}

	type FooMany struct {
		Foos []Foo `fake:"2"`
	}

	var fm FooMany
	Struct(&fm)

	for _, f := range fm.Foos {
		fmt.Printf("%s\n", f.Bar)
		fmt.Printf("%d\n", f.Int)
		fmt.Printf("%d\n", *f.Pointer)
		fmt.Printf("%v\n", f.Name)
		fmt.Printf("%v\n", f.Number)
		fmt.Printf("%v\n", f.Skip)
	}

	// Output: bRMaRxHki
	// -8576773003117070818
	// -7054675846543980602
	// Enrique
	// 4
	// <nil>
	// pWYJ
	// -3430133205295092491
	// -2330884613995904932
	// Fred
	// 1
	// <nil>
}
<|endoftext|>


<commit_before>package php

import (
	"strings"
	"unicode"
)

const shortPHPBegin = "<?"
const longPHPBegin = "<?php"
const phpEnd = "?>"

const eof = -1

// lexHTML consumes and emits an html item until it
// finds a php begin
func lexHTML(l *lexer) stateFn {
	for {
		if strings.HasPrefix(l.input[l.pos:], shortPHPBegin) {
			if l.pos > l.start {
				l.emit(itemHTML)
			}
			return lexPHPBegin
		}
		if l.next() == eof {
			break
		}
	}
	if l.pos > l.start {
		l.emit(itemHTML)
	}
	l.emit(itemEOF)
	return nil
}

func lexPHPBegin(l *lexer) stateFn {
	if strings.HasPrefix(l.input[l.pos:], longPHPBegin) {
		l.pos += len(longPHPBegin)
	}
	if strings.HasPrefix(l.input[l.pos:], shortPHPBegin) {
		l.pos += len(shortPHPBegin)
	}
	l.emit(itemPHPBegin)
	return lexPHP
}

func lexPHP(l *lexer) stateFn {
	l.skipSpace()

	if r := l.peek(); unicode.IsDigit(r) {
		return lexNumberLiteral
	} else if r == '.' {
		l.next() // must advance because we only peeked before
		secondR := l.peek()
		l.backup()
		if unicode.IsDigit(secondR) {
			return lexNumberLiteral
		}
	}

	if strings.HasPrefix(l.input[l.pos:], "?>") {
		return lexPHPEnd
	}

	if strings.HasPrefix(l.input[l.pos:], "#") {
		return lexLineComment
	}

	if strings.HasPrefix(l.input[l.pos:], "//") {
		return lexLineComment
	}

	if strings.HasPrefix(l.input[l.pos:], "/*") {
		return lexBlockComment
	}

	if l.peek() == eof {
		l.emit(itemEOF)
		return nil
	}

	if l.peek() == '\'' {
		return lexSingleQuotedStringLiteral
	}

	if l.peek() == '"' {
		return lexDoubleQuotedStringLiteral
	}

	for _, token := range tokenList {
		item := tokenMap[token]
		if strings.HasPrefix(strings.ToLower(l.input[l.pos:]), token) {
			l.pos += len(token)
			if isKeyword(item) && l.accept(alphabet+underscore+digits) {
				l.backup() // to account for the character consumed by accept
				l.pos -= len(token)
				break
			}
			l.emit(item)
			return lexPHP
		}
	}

	l.acceptRun(alphabet + underscore + digits + "\\")
	l.emit(itemIdentifier)
	return lexPHP
}

func lexNumberLiteral(l *lexer) stateFn {
	// is decimal?
	l.acceptRun(digits)
	if l.accept(".") {
		l.acceptRun(digits)
	}

	l.emit(itemNumberLiteral)
	return lexPHP
}

func lexSingleQuotedStringLiteral(l *lexer) stateFn {
	l.next()
	for {
		switch l.next() {
		case '\\':
			l.next()
			continue
		case '\'':
			l.emit(itemStringLiteral)
			return lexPHP
		}
	}
}

func lexDoubleQuotedStringLiteral(l *lexer) stateFn {
	l.next()
	for {
		switch l.next() {
		case '\\':
			l.next()
			continue
		case '"':
			l.emit(itemStringLiteral)
			return lexPHP
		}
	}
}

const alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const digits = "0123456789"
const underscore = "_"

func lexIdentifier(l *lexer) stateFn {
	l.accept("$")
	l.accept(underscore + alphabet)
	l.acceptRun(underscore + alphabet + digits)
	l.emit(itemVariableOperator)
	return lexPHP
}

// lexPHPEnd lexes the end of a PHP section returning the context to HTML
func lexPHPEnd(l *lexer) stateFn {
	l.pos += len(phpEnd)
	l.emit(itemPHPEnd)
	return lexHTML
}

func lexLineComment(l *lexer) stateFn {
	lineLength := strings.Index(l.input[l.pos:], "\n") + 1
	if lineLength == 0 {
		// this is the last line, so lex until the end
		lineLength = len(l.input[l.pos:])
	}
	// don't lex php end
	if phpEndLength := strings.Index(l.input[l.pos:l.pos+lineLength], phpEnd); phpEndLength >= 0 && phpEndLength < lineLength {
		lineLength = phpEndLength
	}
	l.pos += lineLength
	l.ignore()
	return lexPHP
}

func lexBlockComment(l *lexer) stateFn {
	commentLength := strings.Index(l.input[l.pos:], "*/") + 2
	if commentLength == 1 {
		// the file ends before we find */
		commentLength = len(l.input[l.pos:])
	}
	if phpEndLength := strings.Index(l.input[l.pos:l.pos+commentLength], phpEnd); phpEndLength >= 0 {
		commentLength = phpEndLength
	}
	l.pos += commentLength
	l.ignore()
	return lexPHP
}
<commit_msg>Improved case insensitivity handling of keywords<commit_after>package php

import (
	"strings"
	"unicode"
)

const shortPHPBegin = "<?"
const longPHPBegin = "<?php"
const phpEnd = "?>"

const eof = -1

// lexHTML consumes and emits an html item until it
// finds a php begin
func lexHTML(l *lexer) stateFn {
	for {
		if strings.HasPrefix(l.input[l.pos:], shortPHPBegin) {
			if l.pos > l.start {
				l.emit(itemHTML)
			}
			return lexPHPBegin
		}
		if l.next() == eof {
			break
		}
	}
	if l.pos > l.start {
		l.emit(itemHTML)
	}
	l.emit(itemEOF)
	return nil
}

func lexPHPBegin(l *lexer) stateFn {
	if strings.HasPrefix(l.input[l.pos:], longPHPBegin) {
		l.pos += len(longPHPBegin)
	}
	if strings.HasPrefix(l.input[l.pos:], shortPHPBegin) {
		l.pos += len(shortPHPBegin)
	}
	l.emit(itemPHPBegin)
	return lexPHP
}

func lexPHP(l *lexer) stateFn {
	l.skipSpace()

	if r := l.peek(); unicode.IsDigit(r) {
		return lexNumberLiteral
	} else if r == '.' {
		l.next() // must advance because we only peeked before
		secondR := l.peek()
		l.backup()
		if unicode.IsDigit(secondR) {
			return lexNumberLiteral
		}
	}

	if strings.HasPrefix(l.input[l.pos:], "?>") {
		return lexPHPEnd
	}

	if strings.HasPrefix(l.input[l.pos:], "#") {
		return lexLineComment
	}

	if strings.HasPrefix(l.input[l.pos:], "//") {
		return lexLineComment
	}

	if strings.HasPrefix(l.input[l.pos:], "/*") {
		return lexBlockComment
	}

	if l.peek() == eof {
		l.emit(itemEOF)
		return nil
	}

	if l.peek() == '\'' {
		return lexSingleQuotedStringLiteral
	}

	if l.peek() == '"' {
		return lexDoubleQuotedStringLiteral
	}

	for _, token := range tokenList {
		item := tokenMap[token]
		potentialToken := l.input[l.pos:]
		if len(potentialToken) > len(token) {
			potentialToken = potentialToken[:len(token)]
		}
		if strings.HasPrefix(strings.ToLower(potentialToken), token) {
			l.pos += len(token)
			if isKeyword(item) && l.accept(alphabet+underscore+digits) {
				l.backup() // to account for the character consumed by accept
				l.pos -= len(token)
				break
			}
			l.emit(item)
			return lexPHP
		}
	}

	l.acceptRun(alphabet + underscore + digits + "\\")
	l.emit(itemIdentifier)
	return lexPHP
}

func lexNumberLiteral(l *lexer) stateFn {
	// is decimal?
	l.acceptRun(digits)
	if l.accept(".") {
		l.acceptRun(digits)
	}

	l.emit(itemNumberLiteral)
	return lexPHP
}

func lexSingleQuotedStringLiteral(l *lexer) stateFn {
	l.next()
	for {
		switch l.next() {
		case '\\':
			l.next()
			continue
		case '\'':
			l.emit(itemStringLiteral)
			return lexPHP
		}
	}
}

func lexDoubleQuotedStringLiteral(l *lexer) stateFn {
	l.next()
	for {
		switch l.next() {
		case '\\':
			l.next()
			continue
		case '"':
			l.emit(itemStringLiteral)
			return lexPHP
		}
	}
}

const alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const digits = "0123456789"
const underscore = "_"

func lexIdentifier(l *lexer) stateFn {
	l.accept("$")
	l.accept(underscore + alphabet)
	l.acceptRun(underscore + alphabet + digits)
	l.emit(itemVariableOperator)
	return lexPHP
}

// lexPHPEnd lexes the end of a PHP section returning the context to HTML
func lexPHPEnd(l *lexer) stateFn {
	l.pos += len(phpEnd)
	l.emit(itemPHPEnd)
	return lexHTML
}

func lexLineComment(l *lexer) stateFn {
	lineLength := strings.Index(l.input[l.pos:], "\n") + 1
	if lineLength == 0 {
		// this is the last line, so lex until the end
		lineLength = len(l.input[l.pos:])
	}
	// don't lex php end
	if phpEndLength := strings.Index(l.input[l.pos:l.pos+lineLength], phpEnd); phpEndLength >= 0 && phpEndLength < lineLength {
		lineLength = phpEndLength
	}
	l.pos += lineLength
	l.ignore()
	return lexPHP
}

func lexBlockComment(l *lexer) stateFn {
	commentLength := strings.Index(l.input[l.pos:], "*/") + 2
	if commentLength == 1 {
		// the file ends before we find */
		commentLength = len(l.input[l.pos:])
	}
	if phpEndLength := strings.Index(l.input[l.pos:l.pos+commentLength], phpEnd); phpEndLength >= 0 {
		commentLength = phpEndLength
	}
	l.pos += commentLength
	l.ignore()
	return lexPHP
}
<|endoftext|>


<commit_before>package swarm

import (
	"errors"
	"fmt"
	"net"
	"sync"

	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	ident "github.com/jbenet/go-ipfs/identify"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"
)

var ErrAlreadyOpen = errors.New("Error: Connection to this peer already open.")

// Message represents a packet of information sent to or received from a
// particular Peer.
type Message struct {
	// To or from, depending on direction.
	Peer *peer.Peer

	// Opaque data
	Data []byte
}

// Cleaner looking helper function to make a new message struct
func NewMessage(p *peer.Peer, data proto.Message) *Message {
	bytes, err := proto.Marshal(data)
	if err != nil {
		u.PErr("%v\n", err.Error())
		return nil
	}
	return &Message{
		Peer: p,
		Data: bytes,
	}
}

// Chan is a swarm channel, which provides duplex communication and errors.
type Chan struct {
	Outgoing chan *Message
	Incoming chan *Message
	Errors   chan error
	Close    chan bool
}

// NewChan constructs a Chan instance, with given buffer size bufsize.
func NewChan(bufsize int) *Chan {
	return &Chan{
		Outgoing: make(chan *Message, bufsize),
		Incoming: make(chan *Message, bufsize),
		Errors:   make(chan error, bufsize),
		Close:    make(chan bool, bufsize),
	}
}

// Contains a set of errors mapping to each of the swarms addresses
// that were listened on
type SwarmListenErr struct {
	Errors []error
}

func (se *SwarmListenErr) Error() string {
	if se == nil {
		return "<nil error>"
	}
	var out string
	for i, v := range se.Errors {
		if v != nil {
			out += fmt.Sprintf("%d: %s\n", i, v)
		}
	}
	return out
}

// Swarm is a connection muxer, allowing connections to other peers to
// be opened and closed, while still using the same Chan for all
// communication. The Chan sends/receives Messages, which note the
// destination or source Peer.
type Swarm struct {
	Chan      *Chan
	conns     ConnMap
	connsLock sync.RWMutex

	filterChans map[PBWrapper_MessageType]*Chan
	toFilter    chan *Message
	newFilters  chan *newFilterInfo

	local     *peer.Peer
	listeners []net.Listener
	haltroute chan struct{}
}

// NewSwarm constructs a Swarm, with a Chan.
func NewSwarm(local *peer.Peer) *Swarm {
	s := &Swarm{
		Chan:        NewChan(10),
		conns:       ConnMap{},
		local:       local,
		filterChans: make(map[PBWrapper_MessageType]*Chan),
		toFilter:    make(chan *Message, 32),
		newFilters:  make(chan *newFilterInfo),
		haltroute:   make(chan struct{}),
	}
	go s.routeMessages()
	go s.fanOut()
	return s
}

// Open listeners for each network the swarm should listen on
func (s *Swarm) Listen() error {
	var ret_err *SwarmListenErr
	for i, addr := range s.local.Addresses {
		err := s.connListen(addr)
		if err != nil {
			if ret_err == nil {
				ret_err = new(SwarmListenErr)
				ret_err.Errors = make([]error, len(s.local.Addresses))
			}
			ret_err.Errors[i] = err
			u.PErr("Failed to listen on: %s [%s]", addr, err)
		}
	}
	if ret_err == nil {
		return nil
	}
	return ret_err
}

// Listen for new connections on the given multiaddr
func (s *Swarm) connListen(maddr *ma.Multiaddr) error {
	netstr, addr, err := maddr.DialArgs()
	if err != nil {
		return err
	}

	list, err := net.Listen(netstr, addr)
	if err != nil {
		return err
	}

	// NOTE: this may require a lock around it later. currently, only run on setup
	s.listeners = append(s.listeners, list)

	// Accept and handle new connections on this listener until it errors
	go func() {
		for {
			nconn, err := list.Accept()
			if err != nil {
				e := fmt.Errorf("Failed to accept connection: %s - %s [%s]",
					netstr, addr, err)
				go func() { s.Chan.Errors <- e }()
				return
			}
			go s.handleNewConn(nconn)
		}
	}()

	return nil
}

// Handle getting ID from this peer and adding it into the map
func (s *Swarm) handleNewConn(nconn net.Conn) {
	p := new(peer.Peer)

	conn := &Conn{
		Peer: p,
		Addr: nil,
		Conn: nconn,
	}
	newConnChans(conn)

	sin, sout, err := ident.Handshake(s.local, p, conn.Incoming.MsgChan, conn.Outgoing.MsgChan)
	if err != nil {
		u.PErr("%v\n", err.Error())
		conn.Close()
		return
	}

	// Get address to contact remote peer from
	addr := <-sin
	maddr, err := ma.NewMultiaddr(string(addr))
	if err != nil {
		u.PErr("Got invalid address from peer.")
		s.Error(err)
		return
	}
	p.AddAddress(maddr)

	conn.secIn = sin
	conn.secOut = sout

	err = s.StartConn(conn)
	if err != nil {
		s.Error(err)
	}
}

// Close closes a swarm.
func (s *Swarm) Close() {
	s.connsLock.RLock()
	l := len(s.conns)
	s.connsLock.RUnlock()

	for i := 0; i < l; i++ {
		s.Chan.Close <- true // fan ins
	}
	s.Chan.Close <- true // fan out
	s.Chan.Close <- true // listener

	for _, list := range s.listeners {
		list.Close()
	}

	s.haltroute <- struct{}{}

	for _, filter := range s.filterChans {
		filter.Close <- true
	}
}

// Dial connects to a peer.
//
// The idea is that the client of Swarm does not need to know what network
// the connection will happen over. Swarm can use whichever it choses.
// This allows us to use various transport protocols, do NAT traversal/relay,
// etc. to achive connection.
//
// For now, Dial uses only TCP. This will be extended.
func (s *Swarm) Dial(peer *peer.Peer) (*Conn, error, bool) {
	k := peer.Key()

	// check if we already have an open connection first
	s.connsLock.RLock()
	conn, found := s.conns[k]
	s.connsLock.RUnlock()
	if found {
		return conn, nil, true
	}

	// open connection to peer
	conn, err := Dial("tcp", peer)
	if err != nil {
		return nil, err, false
	}

	return conn, nil, false
}

// StartConn adds the passed in connection to its peerMap and starts
// the fanIn routine for that connection
func (s *Swarm) StartConn(conn *Conn) error {
	if conn == nil {
		return errors.New("Tried to start nil connection.")
	}

	u.DOut("Starting connection: %s\n", conn.Peer.Key().Pretty())
	// add to conns
	s.connsLock.Lock()
	if _, ok := s.conns[conn.Peer.Key()]; ok {
		s.connsLock.Unlock()
		return ErrAlreadyOpen
	}
	s.conns[conn.Peer.Key()] = conn
	s.connsLock.Unlock()

	// kick off reader goroutine
	go s.fanIn(conn)
	return nil
}

// Handles the unwrapping + sending of messages to the right connection.
func (s *Swarm) fanOut() {
	for {
		select {
		case <-s.Chan.Close:
			return // told to close.
		case msg, ok := <-s.Chan.Outgoing:
			if !ok {
				return
			}

			if len(msg.Data) > MaxMessageSize {
				s.Error(fmt.Errorf("Exceeded max message size! (tried to send len = %d)", len(msg.Data)))
			}

			s.connsLock.RLock()
			conn, found := s.conns[msg.Peer.Key()]
			s.connsLock.RUnlock()

			if !found {
				e := fmt.Errorf("Sent msg to peer without open conn: %v",
					msg.Peer)
				s.Chan.Errors <- e
				continue
			}

			// queue it in the connection's buffer
			conn.secOut <- msg.Data
		}
	}
}

// Handles the receiving + wrapping of messages, per conn.
// Consider using reflect.Select with one goroutine instead of n.
func (s *Swarm) fanIn(conn *Conn) {
	for {
		select {
		case <-s.Chan.Close:
			// close Conn.
			conn.Close()
			goto out

		case <-conn.Closed:
			goto out

		case data, ok := <-conn.secIn:
			if !ok {
				e := fmt.Errorf("Error retrieving from conn: %v", conn.Peer.Key().Pretty())
				s.Chan.Errors <- e
				goto out
			}

			msg := &Message{Peer: conn.Peer, Data: data}
			s.toFilter <- msg
		}
	}
out:

	s.connsLock.Lock()
	delete(s.conns, conn.Peer.Key())
	s.connsLock.Unlock()
}

type newFilterInfo struct {
	Type PBWrapper_MessageType
	resp chan *Chan
}

func (s *Swarm) routeMessages() {
	for {
		select {
		case mes, ok := <-s.toFilter:
			if !ok {
				return
			}
			wrapper, err := Unwrap(mes.Data)
			if err != nil {
				u.PErr("error in route messages: %s\n", err)
			}

			ch, ok := s.filterChans[PBWrapper_MessageType(wrapper.GetType())]
			if !ok {
				u.PErr("Received message with invalid type: %d\n", wrapper.GetType())
				continue
			}

			mes.Data = wrapper.GetMessage()
			ch.Incoming <- mes
		case gchan := <-s.newFilters:
			nch, ok := s.filterChans[gchan.Type]
			if !ok {
				nch = NewChan(16)
				s.filterChans[gchan.Type] = nch
				go s.muxChan(nch, gchan.Type)
			}
			gchan.resp <- nch
		case <-s.haltroute:
			return
		}
	}
}

func (s *Swarm) muxChan(ch *Chan, typ PBWrapper_MessageType) {
	for {
		select {
		case <-ch.Close:
			return
		case mes := <-ch.Outgoing:
			data, err := Wrap(mes.Data, typ)
			if err != nil {
				u.PErr("muxChan error: %s\n", err)
				continue
			}
			mes.Data = data
			s.Chan.Outgoing <- mes
		}
	}
}

func (s *Swarm) Find(key u.Key) *peer.Peer {
	s.connsLock.RLock()
	defer s.connsLock.RUnlock()
	conn, found := s.conns[key]
	if !found {
		return nil
	}
	return conn.Peer
}

// GetConnection will check if we are already connected to the peer in question
// and only open a new connection if we arent already
func (s *Swarm) GetConnection(id peer.ID, addr *ma.Multiaddr) (*peer.Peer, error) {
	p := &peer.Peer{
		ID:        id,
		Addresses: []*ma.Multiaddr{addr},
	}

	if id.Equal(s.local.ID) {
		panic("Attempted connection to self!")
	}

	conn, err, reused := s.Dial(p)
	if err != nil {
		return nil, err
	}

	if reused {
		return p, nil
	}

	err = s.handleDialedCon(conn)
	return conn.Peer, err
}

// Handle performing a handshake on a new connection and ensuring proper forward communication
func (s *Swarm) handleDialedCon(conn *Conn) error {
	sin, sout, err := ident.Handshake(s.local, conn.Peer, conn.Incoming.MsgChan, conn.Outgoing.MsgChan)
	if err != nil {
		return err
	}

	// Send node an address that you can be reached on
	myaddr := s.local.NetAddress("tcp")
	mastr, err := myaddr.String()
	if err != nil {
		return errors.New("No local address to send to peer.")
	}

	sout <- []byte(mastr)

	conn.secIn = sin
	conn.secOut = sout

	s.StartConn(conn)

	return nil
}

// ConnectNew is for connecting to a peer when you dont know their ID,
// Should only be used when you are sure that you arent already connected to peer in question
func (s *Swarm) ConnectNew(addr *ma.Multiaddr) (*peer.Peer, error) {
	if addr == nil {
		return nil, errors.New("nil Multiaddr passed to swarm.Connect()")
	}
	npeer := new(peer.Peer)
	npeer.AddAddress(addr)

	conn, err := Dial("tcp", npeer)
	if err != nil {
		return nil, err
	}

	err = s.handleDialedCon(conn)
	return npeer, err
}

// Removes a given peer from the swarm and closes connections to it
func (s *Swarm) Drop(p *peer.Peer) error {
	u.DOut("Dropping peer: [%s]\n", p.ID.Pretty())
	s.connsLock.RLock()
	conn, found := s.conns[u.Key(p.ID)]
	s.connsLock.RUnlock()
	if !found {
		return u.ErrNotFound
	}

	s.connsLock.Lock()
	delete(s.conns, u.Key(p.ID))
	s.connsLock.Unlock()

	return conn.Close()
}

func (s *Swarm) Error(e error) {
	s.Chan.Errors <- e
}

func (s *Swarm) GetErrChan() chan error {
	return s.Chan.Errors
}

func (s *Swarm) GetChannel(typ PBWrapper_MessageType) *Chan {
	nfi := &newFilterInfo{
		Type: typ,
		resp: make(chan *Chan),
	}
	s.newFilters <- nfi

	return <-nfi.resp
}

// Temporary to ensure that the Swarm always matches the Network interface as we are changing it
var _ Network = &Swarm{}
<commit_msg>panic -> error<commit_after>package swarm

import (
	"errors"
	"fmt"
	"net"
	"sync"

	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	ident "github.com/jbenet/go-ipfs/identify"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"
)

var ErrAlreadyOpen = errors.New("Error: Connection to this peer already open.")

// Message represents a packet of information sent to or received from a
// particular Peer.
type Message struct {
	// To or from, depending on direction.
	Peer *peer.Peer

	// Opaque data
	Data []byte
}

// Cleaner looking helper function to make a new message struct
func NewMessage(p *peer.Peer, data proto.Message) *Message {
	bytes, err := proto.Marshal(data)
	if err != nil {
		u.PErr("%v\n", err.Error())
		return nil
	}
	return &Message{
		Peer: p,
		Data: bytes,
	}
}

// Chan is a swarm channel, which provides duplex communication and errors.
type Chan struct {
	Outgoing chan *Message
	Incoming chan *Message
	Errors   chan error
	Close    chan bool
}

// NewChan constructs a Chan instance, with given buffer size bufsize.
func NewChan(bufsize int) *Chan {
	return &Chan{
		Outgoing: make(chan *Message, bufsize),
		Incoming: make(chan *Message, bufsize),
		Errors:   make(chan error, bufsize),
		Close:    make(chan bool, bufsize),
	}
}

// Contains a set of errors mapping to each of the swarms addresses
// that were listened on
type SwarmListenErr struct {
	Errors []error
}

func (se *SwarmListenErr) Error() string {
	if se == nil {
		return "<nil error>"
	}
	var out string
	for i, v := range se.Errors {
		if v != nil {
			out += fmt.Sprintf("%d: %s\n", i, v)
		}
	}
	return out
}

// Swarm is a connection muxer, allowing connections to other peers to
// be opened and closed, while still using the same Chan for all
// communication. The Chan sends/receives Messages, which note the
// destination or source Peer.
type Swarm struct {
	Chan      *Chan
	conns     ConnMap
	connsLock sync.RWMutex

	filterChans map[PBWrapper_MessageType]*Chan
	toFilter    chan *Message
	newFilters  chan *newFilterInfo

	local     *peer.Peer
	listeners []net.Listener
	haltroute chan struct{}
}

// NewSwarm constructs a Swarm, with a Chan.
func NewSwarm(local *peer.Peer) *Swarm {
	s := &Swarm{
		Chan:        NewChan(10),
		conns:       ConnMap{},
		local:       local,
		filterChans: make(map[PBWrapper_MessageType]*Chan),
		toFilter:    make(chan *Message, 32),
		newFilters:  make(chan *newFilterInfo),
		haltroute:   make(chan struct{}),
	}
	go s.routeMessages()
	go s.fanOut()
	return s
}

// Open listeners for each network the swarm should listen on
func (s *Swarm) Listen() error {
	var ret_err *SwarmListenErr
	for i, addr := range s.local.Addresses {
		err := s.connListen(addr)
		if err != nil {
			if ret_err == nil {
				ret_err = new(SwarmListenErr)
				ret_err.Errors = make([]error, len(s.local.Addresses))
			}
			ret_err.Errors[i] = err
			u.PErr("Failed to listen on: %s [%s]", addr, err)
		}
	}
	if ret_err == nil {
		return nil
	}
	return ret_err
}

// Listen for new connections on the given multiaddr
func (s *Swarm) connListen(maddr *ma.Multiaddr) error {
	netstr, addr, err := maddr.DialArgs()
	if err != nil {
		return err
	}

	list, err := net.Listen(netstr, addr)
	if err != nil {
		return err
	}

	// NOTE: this may require a lock around it later. currently, only run on setup
	s.listeners = append(s.listeners, list)

	// Accept and handle new connections on this listener until it errors
	go func() {
		for {
			nconn, err := list.Accept()
			if err != nil {
				e := fmt.Errorf("Failed to accept connection: %s - %s [%s]",
					netstr, addr, err)
				go func() { s.Chan.Errors <- e }()
				return
			}
			go s.handleNewConn(nconn)
		}
	}()

	return nil
}

// Handle getting ID from this peer and adding it into the map
func (s *Swarm) handleNewConn(nconn net.Conn) {
	p := new(peer.Peer)

	conn := &Conn{
		Peer: p,
		Addr: nil,
		Conn: nconn,
	}
	newConnChans(conn)

	sin, sout, err := ident.Handshake(s.local, p, conn.Incoming.MsgChan, conn.Outgoing.MsgChan)
	if err != nil {
		u.PErr("%v\n", err.Error())
		conn.Close()
		return
	}

	// Get address to contact remote peer from
	addr := <-sin
	maddr, err := ma.NewMultiaddr(string(addr))
	if err != nil {
		u.PErr("Got invalid address from peer.")
		s.Error(err)
		return
	}
	p.AddAddress(maddr)

	conn.secIn = sin
	conn.secOut = sout

	err = s.StartConn(conn)
	if err != nil {
		s.Error(err)
	}
}

// Close closes a swarm.
func (s *Swarm) Close() {
	s.connsLock.RLock()
	l := len(s.conns)
	s.connsLock.RUnlock()

	for i := 0; i < l; i++ {
		s.Chan.Close <- true // fan ins
	}
	s.Chan.Close <- true // fan out
	s.Chan.Close <- true // listener

	for _, list := range s.listeners {
		list.Close()
	}

	s.haltroute <- struct{}{}

	for _, filter := range s.filterChans {
		filter.Close <- true
	}
}

// Dial connects to a peer.
//
// The idea is that the client of Swarm does not need to know what network
// the connection will happen over. Swarm can use whichever it choses.
// This allows us to use various transport protocols, do NAT traversal/relay,
// etc. to achive connection.
//
// For now, Dial uses only TCP. This will be extended.
func (s *Swarm) Dial(peer *peer.Peer) (*Conn, error, bool) {
	k := peer.Key()

	// check if we already have an open connection first
	s.connsLock.RLock()
	conn, found := s.conns[k]
	s.connsLock.RUnlock()
	if found {
		return conn, nil, true
	}

	// open connection to peer
	conn, err := Dial("tcp", peer)
	if err != nil {
		return nil, err, false
	}

	return conn, nil, false
}

// StartConn adds the passed in connection to its peerMap and starts
// the fanIn routine for that connection
func (s *Swarm) StartConn(conn *Conn) error {
	if conn == nil {
		return errors.New("Tried to start nil connection.")
	}

	u.DOut("Starting connection: %s\n", conn.Peer.Key().Pretty())
	// add to conns
	s.connsLock.Lock()
	if _, ok := s.conns[conn.Peer.Key()]; ok {
		s.connsLock.Unlock()
		return ErrAlreadyOpen
	}
	s.conns[conn.Peer.Key()] = conn
	s.connsLock.Unlock()

	// kick off reader goroutine
	go s.fanIn(conn)
	return nil
}

// Handles the unwrapping + sending of messages to the right connection.
func (s *Swarm) fanOut() {
	for {
		select {
		case <-s.Chan.Close:
			return // told to close.
		case msg, ok := <-s.Chan.Outgoing:
			if !ok {
				return
			}

			if len(msg.Data) > MaxMessageSize {
				s.Error(fmt.Errorf("Exceeded max message size! (tried to send len = %d)", len(msg.Data)))
			}

			s.connsLock.RLock()
			conn, found := s.conns[msg.Peer.Key()]
			s.connsLock.RUnlock()

			if !found {
				e := fmt.Errorf("Sent msg to peer without open conn: %v",
					msg.Peer)
				s.Chan.Errors <- e
				continue
			}

			// queue it in the connection's buffer
			conn.secOut <- msg.Data
		}
	}
}

// Handles the receiving + wrapping of messages, per conn.
// Consider using reflect.Select with one goroutine instead of n.
func (s *Swarm) fanIn(conn *Conn) {
	for {
		select {
		case <-s.Chan.Close:
			// close Conn.
			conn.Close()
			goto out

		case <-conn.Closed:
			goto out

		case data, ok := <-conn.secIn:
			if !ok {
				e := fmt.Errorf("Error retrieving from conn: %v", conn.Peer.Key().Pretty())
				s.Chan.Errors <- e
				goto out
			}

			msg := &Message{Peer: conn.Peer, Data: data}
			s.toFilter <- msg
		}
	}
out:

	s.connsLock.Lock()
	delete(s.conns, conn.Peer.Key())
	s.connsLock.Unlock()
}

type newFilterInfo struct {
	Type PBWrapper_MessageType
	resp chan *Chan
}

func (s *Swarm) routeMessages() {
	for {
		select {
		case mes, ok := <-s.toFilter:
			if !ok {
				return
			}
			wrapper, err := Unwrap(mes.Data)
			if err != nil {
				u.PErr("error in route messages: %s\n", err)
			}

			ch, ok := s.filterChans[PBWrapper_MessageType(wrapper.GetType())]
			if !ok {
				u.PErr("Received message with invalid type: %d\n", wrapper.GetType())
				continue
			}

			mes.Data = wrapper.GetMessage()
			ch.Incoming <- mes
		case gchan := <-s.newFilters:
			nch, ok := s.filterChans[gchan.Type]
			if !ok {
				nch = NewChan(16)
				s.filterChans[gchan.Type] = nch
				go s.muxChan(nch, gchan.Type)
			}
			gchan.resp <- nch
		case <-s.haltroute:
			return
		}
	}
}

func (s *Swarm) muxChan(ch *Chan, typ PBWrapper_MessageType) {
	for {
		select {
		case <-ch.Close:
			return
		case mes := <-ch.Outgoing:
			data, err := Wrap(mes.Data, typ)
			if err != nil {
				u.PErr("muxChan error: %s\n", err)
				continue
			}
			mes.Data = data
			s.Chan.Outgoing <- mes
		}
	}
}

func (s *Swarm) Find(key u.Key) *peer.Peer {
	s.connsLock.RLock()
	defer s.connsLock.RUnlock()
	conn, found := s.conns[key]
	if !found {
		return nil
	}
	return conn.Peer
}

// GetConnection will check if we are already connected to the peer in question
// and only open a new connection if we arent already
func (s *Swarm) GetConnection(id peer.ID, addr *ma.Multiaddr) (*peer.Peer, error) {
	p := &peer.Peer{
		ID:        id,
		Addresses: []*ma.Multiaddr{addr},
	}

	if id.Equal(s.local.ID) {
		return nil, errors.New("Attempted connection to self!")
	}

	conn, err, reused := s.Dial(p)
	if err != nil {
		return nil, err
	}

	if reused {
		return p, nil
	}

	err = s.handleDialedCon(conn)
	return conn.Peer, err
}

// Handle performing a handshake on a new connection and ensuring proper forward communication
func (s *Swarm) handleDialedCon(conn *Conn) error {
	sin, sout, err := ident.Handshake(s.local, conn.Peer, conn.Incoming.MsgChan, conn.Outgoing.MsgChan)
	if err != nil {
		return err
	}

	// Send node an address that you can be reached on
	myaddr := s.local.NetAddress("tcp")
	mastr, err := myaddr.String()
	if err != nil {
		return errors.New("No local address to send to peer.")
	}

	sout <- []byte(mastr)

	conn.secIn = sin
	conn.secOut = sout

	s.StartConn(conn)

	return nil
}

// ConnectNew is for connecting to a peer when you dont know their ID,
// Should only be used when you are sure that you arent already connected to peer in question
func (s *Swarm) ConnectNew(addr *ma.Multiaddr) (*peer.Peer, error) {
	if addr == nil {
		return nil, errors.New("nil Multiaddr passed to swarm.Connect()")
	}
	npeer := new(peer.Peer)
	npeer.AddAddress(addr)

	conn, err := Dial("tcp", npeer)
	if err != nil {
		return nil, err
	}

	err = s.handleDialedCon(conn)
	return npeer, err
}

// Removes a given peer from the swarm and closes connections to it
func (s *Swarm) Drop(p *peer.Peer) error {
	u.DOut("Dropping peer: [%s]\n", p.ID.Pretty())
	s.connsLock.RLock()
	conn, found := s.conns[u.Key(p.ID)]
	s.connsLock.RUnlock()
	if !found {
		return u.ErrNotFound
	}

	s.connsLock.Lock()
	delete(s.conns, u.Key(p.ID))
	s.connsLock.Unlock()

	return conn.Close()
}

func (s *Swarm) Error(e error) {
	s.Chan.Errors <- e
}

func (s *Swarm) GetErrChan() chan error {
	return s.Chan.Errors
}

func (s *Swarm) GetChannel(typ PBWrapper_MessageType) *Chan {
	nfi := &newFilterInfo{
		Type: typ,
		resp: make(chan *Chan),
	}
	s.newFilters <- nfi

	return <-nfi.resp
}

// Temporary to ensure that the Swarm always matches the Network interface as we are changing it
var _ Network = &Swarm{}
<|endoftext|>


<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/control-center/serviced/coordinator/client"
	"github.com/control-center/serviced/domain/host"
	"github.com/zenoss/glog"
)

const monitorSubDir string = "/monitor"

// Monitor monitors the exporting of a file system to clients.
type Monitor struct {
	driver StorageDriver

	monitoredHosts     map[string]bool
	monitoredHostsLock sync.RWMutex

	shouldRestart   bool
	monitorInterval time.Duration
	previousRestart time.Time

	conn               client.Connection
	storageClientsPath string
}

// NewMonitor returns a Monitor object to monitor the exported file system
func NewMonitor(driver StorageDriver, monitorInterval time.Duration) (*Monitor, error) {
	m := &Monitor{
		driver:          driver,
		monitoredHosts:  make(map[string]bool),
		shouldRestart:   getShouldRestartDFSOnFailure(),
		monitorInterval: monitorInterval,
	}

	return m, nil
}

// UpdateRemoteMonitorFile is used by remote clients to write a tiny file to the DFS volume at the given cycle
func UpdateRemoteMonitorFile(localPath string, writeInterval time.Duration, ipAddr string, shutdown <-chan interface{}) {
	remoteFile := path.Join(localPath, monitorSubDir, ipAddr)
	glog.Infof("updating DFS volume monitor file %s at write interval: %s", remoteFile, writeInterval)

	for {
		glog.V(2).Infof("writing DFS file %s", remoteFile)
		if err := ioutil.WriteFile(remoteFile, []byte(ipAddr), 0600); err != nil {
			glog.Warningf("unable to write DFS file %s: %s", remoteFile, err)
		}

		// wait for next cycle or shutdown
		select {
		case <-time.After(writeInterval):

		case <-shutdown:
			glog.Infof("no longer writing remote monitor status for DFS volume %s to %s", localPath, remoteFile)
			return
		}
	}
}

// SetMonitorStorageClients sets the monitored remote IPs that are active
func (m *Monitor) SetMonitorStorageClients(conn client.Connection, storageClientsPath string) {
	m.conn = conn
	m.storageClientsPath = storageClientsPath
	m.SetMonitorRemoteHosts(getActiveRemoteHosts(conn, storageClientsPath, m.monitorInterval)...)
}

// SetMonitorRemoteHosts sets the remote hosts to monitor
func (m *Monitor) SetMonitorRemoteHosts(ipAddrs ...string) {
	if len(ipAddrs) == 0 {
		glog.Warningf("disabled DFS volume monitoring - no remote IPs to monitor")
	} else {
		glog.V(2).Infof("enabled DFS volume monitoring for remote IPs: %+v", ipAddrs)
	}
	m.monitoredHostsLock.Lock()
	defer m.monitoredHostsLock.Unlock()
	for _, ipAddr := range ipAddrs {
		m.monitoredHosts[ipAddr] = true
	}
}

// setMonitorRemoteHost updates the status of the monitored remote
func (m *Monitor) setMonitorRemoteHost(ipAddr string, hasUpdated bool) {
	m.monitoredHostsLock.Lock()
	defer m.monitoredHostsLock.Unlock()
	m.monitoredHosts[ipAddr] = hasUpdated
}

// getMonitorRemoteHosts returns the statuses of the monitored remotes
func (m *Monitor) getMonitorRemoteHosts() map[string]bool {
	m.monitoredHostsLock.RLock()
	defer m.monitoredHostsLock.RUnlock()
	return m.monitoredHosts
}

// ProcessDFSVolumePollUpdateFunc is called by MonitorDFSVolume when updates are detected
type DFSVolumeMonitorPollUpdateFunc func(mountpoint, remoteIP string, isUpdated bool)

// DFSVolumeMonitorPollUpdateFunc restarts nfs based on status of monitored remotes
func (m *Monitor) DFSVolumeMonitorPollUpdateFunc(mountpoint, remoteIP string, hasUpdatedFile bool) {
	// monitor dfs; log warnings each cycle; restart dfs if needed

	if hasUpdatedFile {
		return
	} else if len(m.getMonitorRemoteHosts()) == 0 {
		return
	}

	glog.Warningf("DFS NFS volume %s is not seen by remoteIP:%s - further action may be needed i.e: restart nfs", mountpoint, remoteIP)
	now := time.Now()
	since := now.Sub(m.previousRestart)
	if !m.shouldRestart {
		glog.Warningf("Not restarting DFS NFS service due to configuration setting: SERVICED_MONITOR_DFS_MASTER_RESTART=0")
		return
	} else if since < m.monitorInterval {
		glog.Warningf("Not restarting DFS NFS service - have not surpassed interval: %s since last restart", m.monitorInterval)
		return
	}

	m.previousRestart = now
	if err := m.driver.Restart(); err != nil {
		glog.Errorf("Error restarting DFS NFS service: %s", err)
	}
}

// MonitorVolume monitors the DFS volume - logs on failure and calls pollUpdateFunc
func (m *Monitor) MonitorDFSVolume(mountpoint string, shutdown <-chan interface{}, pollUpdateFunc DFSVolumeMonitorPollUpdateFunc) {
	glog.Infof("monitoring DFS export info for DFS volume %s at polling interval: %s", mountpoint, m.monitorInterval)

	m.previousRestart = time.Now()

	monitorPath := path.Join(mountpoint, monitorSubDir)
	os.RemoveAll(monitorPath)
	if err := os.MkdirAll(monitorPath, 0755); err != nil {
		glog.Errorf("no longer monitoring status for DFS volume %s - unable to mkdir %+v: %s", mountpoint, monitorPath, err)
		return
	}

	for {
		// check all active remotes
		remotes := m.getMonitorRemoteHosts()
		remoteIPs := make([]string, 0, len(remotes))
		for k := range remotes {
			remoteIPs = append(remoteIPs, k)
		}

		activeRemotes := remoteIPs
		if m.conn != nil && len(m.storageClientsPath) != 0 {
			activeRemotes = getActiveRemoteHosts(m.conn, m.storageClientsPath, m.monitorInterval)
			glog.V(2).Infof("DFS active remotes: %s", activeRemotes)
			m.SetMonitorRemoteHosts(activeRemotes...)
		}

		for _, remoteIP := range activeRemotes {
			remoteFile := path.Join(monitorPath, remoteIP)
			sinceModified, err := fileSinceModified(remoteFile)
			hasUpdatedFile := false
			if err != nil {
				glog.Warningf("remote agent %s is not synced on DFS volume %s for file %s: %s", remoteIP, mountpoint, remoteFile, err)
			} else if sinceModified > m.monitorInterval {
				glog.Warningf("remote agent %s is not synced DFS volume %s for file %s (since modified: %s)", remoteIP, mountpoint, remoteFile, sinceModified)
			} else {
				glog.V(4).Infof("remote agent %s is synced on DFS volume %s for file %s (since modified: %s)", remoteIP, mountpoint, remoteFile, sinceModified)
				hasUpdatedFile = true
			}

			m.setMonitorRemoteHost(remoteIP, hasUpdatedFile)
			pollUpdateFunc(mountpoint, remoteIP, hasUpdatedFile)
		}

		// wait for next cycle or shutdown
		select {
		case <-time.After(m.monitorInterval):

		case <-shutdown:
			glog.Infof("no longer monitoring status for DFS volume %s", mountpoint)
			return
		}
	}
}

// fileSinceModified returns the duration of the file since last modified
func fileSinceModified(filename string) (time.Duration, error) {
	fileinfo, err := os.Stat(filename)
	if err != nil {
		glog.V(2).Infof("unable to stat file: %s %s", filename, err)
		return 0, err
	}

	timeSince := time.Now().Sub(fileinfo.ModTime())
	glog.V(4).Infof("file %s was modified %s ago", filename, timeSince)
	return timeSince, nil
}

// Get the variable determining if we will restart NFS when we detect a failure
func getShouldRestartDFSOnFailure() bool {
	shouldRestart := os.Getenv("SERVICED_MONITOR_DFS_MASTER_RESTART")
	if len(shouldRestart) == 0 {
		glog.Infof("defaulting SERVICED_MONITOR_DFS_MASTER_RESTART to true")
		return true
	} else if shouldRestart == "1" {
		return true
	} else {
		return false
	}
}

// getDefaultDFSMonitorRemoteInterval returns the duration specified in seconds for the env var SERVICED_DFS_MONITOR_REMOTE_UPDATE_INTERVAL
func getDefaultDFSMonitorRemoteInterval() time.Duration {
	return getEnvMinDuration("SERVICED_MONITOR_DFS_REMOTE_UPDATE_INTERVAL", 60, 60)
}

func getDefaultNFSMonitorMasterInterval() time.Duration {
	min := getDefaultDFSMonitorRemoteInterval()
	return getEnvMinDuration("SERVICED_MONITOR_DFS_MASTER_INTERVAL", 900, int32(min.Seconds())*2)
}

// getEnvMinDuration returns the time.Duration env var meeting minimum and default duration
func getEnvMinDuration(envvar string, def, min int32) time.Duration {
	duration := def
	envval := os.Getenv(envvar)
	if len(strings.TrimSpace(envval)) == 0 {
		// ignore unset envvar
	} else if intVal, intErr := strconv.ParseInt(envval, 0, 32); intErr != nil {
		glog.Warningf("ignoring invalid %s of '%s': %s", envvar, envval, intErr)
		duration = min
	} else if int32(intVal) < min {
		glog.Warningf("ignoring invalid %s of '%s' < minimum:%v seconds", envvar, envval, min)
	} else {
		duration = int32(intVal)
	}

	return time.Duration(duration) * time.Second
}

// StorageClientHostNode is the zookeeper node for a host
type StorageClientHostNode struct {
	host.Host
	version interface{}
}

// Version is an implementation of client.Node
func (v *StorageClientHostNode) Version() interface{} { return v.version }

// SetVersion is an implementation of client.Node
func (v *StorageClientHostNode) SetVersion(version interface{}) { v.version = version }

// getActiveRemoteHosts returns a slice of activeClientIPs
func getActiveRemoteHosts(conn client.Connection, storageClientsPath string, monitorInterval time.Duration) []string {
	// clients is not full list of remotes when called from server.go - retrieve our own list from zookeeper
	var err error
	remoteIPs, err := getAllRemoteHostsFromZookeeper(conn, storageClientsPath)
	if err != nil {
		return []string{}
	}
	glog.V(2).Infof("DFS remote IPs: %+v", remoteIPs)

	// determine active hosts
	var activeClientIPs []string
	now := time.Now()
	for _, clnt := range remoteIPs {
		cp := path.Join(storageClientsPath, clnt)
		glog.V(2).Infof("retrieving info for DFS for remoteIP %s at zookeeper node %s", clnt, cp)

		hnode := StorageClientHostNode{}
		err := conn.Get(cp, &hnode)
		if err != nil && err != client.ErrEmptyNode {
			glog.Errorf("DFS could not get remote host zookeeper node %s: %s", cp, err)
			continue
		}
		if hnode.Host.UpdatedAt.IsZero() {
			glog.Infof("DFS not monitoring non-active host %+v:  HostID:%v  UpdatedAt:%+v", clnt, hnode.ID, hnode.UpdatedAt)
			continue
		}

		elapsed := now.Sub(hnode.Host.UpdatedAt)
		glog.V(2).Infof("retrieved info for DFS for remoteIP %s  UpdatedAt:%s  elapsed:%s  monitorInterval:%s", clnt, hnode.Host.UpdatedAt, elapsed, monitorInterval)
		if elapsed > monitorInterval {
			glog.Infof("DFS not monitoring non-active host %+v:  HostID:%v  UpdatedAt:%+v  lastseen:%s ago", clnt, hnode.ID, hnode.UpdatedAt, elapsed)
			continue
		}

		activeClientIPs = append(activeClientIPs, clnt)
	}

	glog.V(2).Infof("DFS remote active IPs: %+v", activeClientIPs)
	return activeClientIPs
}

// getAllRemoteHostsFromZookeeper retrieves a list of remote storage clients from zookeeper
func getAllRemoteHostsFromZookeeper(conn client.Connection, storageClientsPath string) ([]string, error) {
	clients, err := conn.Children(storageClientsPath)
	if err != nil {
		glog.Errorf("unable to retrieve list of DFS remote hosts: %s", err)
		return []string{}, err
	}

	glog.V(4).Infof("DFS remote IPs: %+v", clients)
	return clients, nil
}
<commit_msg>show changes to DFS active remotes to be monitored<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"io/ioutil"
	"os"
	"path"
	"reflect"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/control-center/serviced/coordinator/client"
	"github.com/control-center/serviced/domain/host"
	"github.com/zenoss/glog"
)

const monitorSubDir string = "/monitor"

// Monitor monitors the exporting of a file system to clients.
type Monitor struct {
	driver StorageDriver

	monitoredHosts     map[string]bool
	monitoredHostsLock sync.RWMutex

	shouldRestart   bool
	monitorInterval time.Duration
	previousRestart time.Time

	conn               client.Connection
	storageClientsPath string
}

// NewMonitor returns a Monitor object to monitor the exported file system
func NewMonitor(driver StorageDriver, monitorInterval time.Duration) (*Monitor, error) {
	m := &Monitor{
		driver:          driver,
		monitoredHosts:  make(map[string]bool),
		shouldRestart:   getShouldRestartDFSOnFailure(),
		monitorInterval: monitorInterval,
	}

	return m, nil
}

// UpdateRemoteMonitorFile is used by remote clients to write a tiny file to the DFS volume at the given cycle
func UpdateRemoteMonitorFile(localPath string, writeInterval time.Duration, ipAddr string, shutdown <-chan interface{}) {
	remoteFile := path.Join(localPath, monitorSubDir, ipAddr)
	glog.Infof("updating DFS volume monitor file %s at write interval: %s", remoteFile, writeInterval)

	for {
		glog.V(2).Infof("writing DFS file %s", remoteFile)
		if err := ioutil.WriteFile(remoteFile, []byte(ipAddr), 0600); err != nil {
			glog.Warningf("unable to write DFS file %s: %s", remoteFile, err)
		}

		// wait for next cycle or shutdown
		select {
		case <-time.After(writeInterval):

		case <-shutdown:
			glog.Infof("no longer writing remote monitor status for DFS volume %s to %s", localPath, remoteFile)
			return
		}
	}
}

// SetMonitorStorageClients sets the monitored remote IPs that are active
func (m *Monitor) SetMonitorStorageClients(conn client.Connection, storageClientsPath string) {
	m.conn = conn
	m.storageClientsPath = storageClientsPath
	m.SetMonitorRemoteHosts(getActiveRemoteHosts(conn, storageClientsPath, m.monitorInterval)...)
}

// SetMonitorRemoteHosts sets the remote hosts to monitor
func (m *Monitor) SetMonitorRemoteHosts(ipAddrs ...string) {
	if len(ipAddrs) == 0 {
		glog.Warningf("disabled DFS volume monitoring - no remote IPs to monitor")
	} else {
		glog.V(2).Infof("enabled DFS volume monitoring for remote IPs: %+v", ipAddrs)
	}
	m.monitoredHostsLock.Lock()
	defer m.monitoredHostsLock.Unlock()
	for _, ipAddr := range ipAddrs {
		m.monitoredHosts[ipAddr] = true
	}
}

// setMonitorRemoteHost updates the status of the monitored remote
func (m *Monitor) setMonitorRemoteHost(ipAddr string, hasUpdated bool) {
	m.monitoredHostsLock.Lock()
	defer m.monitoredHostsLock.Unlock()
	m.monitoredHosts[ipAddr] = hasUpdated
}

// getMonitorRemoteHosts returns the statuses of the monitored remotes
func (m *Monitor) getMonitorRemoteHosts() map[string]bool {
	m.monitoredHostsLock.RLock()
	defer m.monitoredHostsLock.RUnlock()
	return m.monitoredHosts
}

// ProcessDFSVolumePollUpdateFunc is called by MonitorDFSVolume when updates are detected
type DFSVolumeMonitorPollUpdateFunc func(mountpoint, remoteIP string, isUpdated bool)

// DFSVolumeMonitorPollUpdateFunc restarts nfs based on status of monitored remotes
func (m *Monitor) DFSVolumeMonitorPollUpdateFunc(mountpoint, remoteIP string, hasUpdatedFile bool) {
	// monitor dfs; log warnings each cycle; restart dfs if needed

	if hasUpdatedFile {
		return
	} else if len(m.getMonitorRemoteHosts()) == 0 {
		return
	}

	glog.Warningf("DFS NFS volume %s is not seen by remoteIP:%s - further action may be needed i.e: restart nfs", mountpoint, remoteIP)
	now := time.Now()
	since := now.Sub(m.previousRestart)
	if !m.shouldRestart {
		glog.Warningf("Not restarting DFS NFS service due to configuration setting: SERVICED_MONITOR_DFS_MASTER_RESTART=0")
		return
	} else if since < m.monitorInterval {
		glog.Warningf("Not restarting DFS NFS service - have not surpassed interval: %s since last restart", m.monitorInterval)
		return
	}

	m.previousRestart = now
	if err := m.driver.Restart(); err != nil {
		glog.Errorf("Error restarting DFS NFS service: %s", err)
	}
}

// MonitorVolume monitors the DFS volume - logs on failure and calls pollUpdateFunc
func (m *Monitor) MonitorDFSVolume(mountpoint string, shutdown <-chan interface{}, pollUpdateFunc DFSVolumeMonitorPollUpdateFunc) {
	glog.Infof("monitoring DFS export info for DFS volume %s at polling interval: %s", mountpoint, m.monitorInterval)

	m.previousRestart = time.Now()

	monitorPath := path.Join(mountpoint, monitorSubDir)
	os.RemoveAll(monitorPath)
	if err := os.MkdirAll(monitorPath, 0755); err != nil {
		glog.Errorf("no longer monitoring status for DFS volume %s - unable to mkdir %+v: %s", mountpoint, monitorPath, err)
		return
	}

	for {
		// check all active remotes
		remotes := m.getMonitorRemoteHosts()
		remoteIPs := make([]string, 0, len(remotes))
		for k := range remotes {
			remoteIPs = append(remoteIPs, k)
		}

		activeRemotes := remoteIPs
		if m.conn != nil && len(m.storageClientsPath) != 0 {
			activeRemotes = getActiveRemoteHosts(m.conn, m.storageClientsPath, m.monitorInterval)
			sort.Sort(sort.StringSlice(activeRemotes))
			sort.Sort(sort.StringSlice(remoteIPs))
			if !reflect.DeepEqual(activeRemotes, remoteIPs) {
				glog.Infof("DFS active remotes to be monitored has changed to: %+v", activeRemotes)
			}
			m.SetMonitorRemoteHosts(activeRemotes...)
		}

		for _, remoteIP := range activeRemotes {
			remoteFile := path.Join(monitorPath, remoteIP)
			sinceModified, err := fileSinceModified(remoteFile)
			hasUpdatedFile := false
			if err != nil {
				glog.Warningf("remote agent %s is not synced on DFS volume %s for file %s: %s", remoteIP, mountpoint, remoteFile, err)
			} else if sinceModified > m.monitorInterval {
				glog.Warningf("remote agent %s is not synced DFS volume %s for file %s (since modified: %s)", remoteIP, mountpoint, remoteFile, sinceModified)
			} else {
				glog.V(4).Infof("remote agent %s is synced on DFS volume %s for file %s (since modified: %s)", remoteIP, mountpoint, remoteFile, sinceModified)
				hasUpdatedFile = true
			}

			m.setMonitorRemoteHost(remoteIP, hasUpdatedFile)
			pollUpdateFunc(mountpoint, remoteIP, hasUpdatedFile)
		}

		// wait for next cycle or shutdown
		select {
		case <-time.After(m.monitorInterval):

		case <-shutdown:
			glog.Infof("no longer monitoring status for DFS volume %s", mountpoint)
			return
		}
	}
}

// fileSinceModified returns the duration of the file since last modified
func fileSinceModified(filename string) (time.Duration, error) {
	fileinfo, err := os.Stat(filename)
	if err != nil {
		glog.V(2).Infof("unable to stat file: %s %s", filename, err)
		return 0, err
	}

	timeSince := time.Now().Sub(fileinfo.ModTime())
	glog.V(4).Infof("file %s was modified %s ago", filename, timeSince)
	return timeSince, nil
}

// Get the variable determining if we will restart NFS when we detect a failure
func getShouldRestartDFSOnFailure() bool {
	shouldRestart := os.Getenv("SERVICED_MONITOR_DFS_MASTER_RESTART")
	if len(shouldRestart) == 0 {
		glog.Infof("defaulting SERVICED_MONITOR_DFS_MASTER_RESTART to true")
		return true
	} else if shouldRestart == "1" {
		return true
	} else {
		return false
	}
}

// getDefaultDFSMonitorRemoteInterval returns the duration specified in seconds for the env var SERVICED_DFS_MONITOR_REMOTE_UPDATE_INTERVAL
func getDefaultDFSMonitorRemoteInterval() time.Duration {
	return getEnvMinDuration("SERVICED_MONITOR_DFS_REMOTE_UPDATE_INTERVAL", 60, 60)
}

func getDefaultNFSMonitorMasterInterval() time.Duration {
	min := getDefaultDFSMonitorRemoteInterval()
	return getEnvMinDuration("SERVICED_MONITOR_DFS_MASTER_INTERVAL", 900, int32(min.Seconds())*2)
}

// getEnvMinDuration returns the time.Duration env var meeting minimum and default duration
func getEnvMinDuration(envvar string, def, min int32) time.Duration {
	duration := def
	envval := os.Getenv(envvar)
	if len(strings.TrimSpace(envval)) == 0 {
		// ignore unset envvar
	} else if intVal, intErr := strconv.ParseInt(envval, 0, 32); intErr != nil {
		glog.Warningf("ignoring invalid %s of '%s': %s", envvar, envval, intErr)
		duration = min
	} else if int32(intVal) < min {
		glog.Warningf("ignoring invalid %s of '%s' < minimum:%v seconds", envvar, envval, min)
	} else {
		duration = int32(intVal)
	}

	return time.Duration(duration) * time.Second
}

// StorageClientHostNode is the zookeeper node for a host
type StorageClientHostNode struct {
	host.Host
	version interface{}
}

// Version is an implementation of client.Node
func (v *StorageClientHostNode) Version() interface{} { return v.version }

// SetVersion is an implementation of client.Node
func (v *StorageClientHostNode) SetVersion(version interface{}) { v.version = version }

// getActiveRemoteHosts returns a slice of activeClientIPs
func getActiveRemoteHosts(conn client.Connection, storageClientsPath string, monitorInterval time.Duration) []string {
	// clients is not full list of remotes when called from server.go - retrieve our own list from zookeeper
	var err error
	remoteIPs, err := getAllRemoteHostsFromZookeeper(conn, storageClientsPath)
	if err != nil {
		return []string{}
	}
	glog.V(2).Infof("DFS remote IPs: %+v", remoteIPs)

	// determine active hosts
	var activeClientIPs []string
	now := time.Now()
	for _, clnt := range remoteIPs {
		cp := path.Join(storageClientsPath, clnt)
		glog.V(2).Infof("retrieving info for DFS for remoteIP %s at zookeeper node %s", clnt, cp)

		hnode := StorageClientHostNode{}
		err := conn.Get(cp, &hnode)
		if err != nil && err != client.ErrEmptyNode {
			glog.Errorf("DFS could not get remote host zookeeper node %s: %s", cp, err)
			continue
		}
		if hnode.Host.UpdatedAt.IsZero() {
			glog.Infof("DFS not monitoring non-active host %+v:  HostID:%v  UpdatedAt:%+v", clnt, hnode.ID, hnode.UpdatedAt)
			continue
		}

		elapsed := now.Sub(hnode.Host.UpdatedAt)
		glog.V(2).Infof("retrieved info for DFS for remoteIP %s  UpdatedAt:%s  elapsed:%s  monitorInterval:%s", clnt, hnode.Host.UpdatedAt, elapsed, monitorInterval)
		if elapsed > monitorInterval {
			glog.Infof("DFS not monitoring non-active host %+v:  HostID:%v  UpdatedAt:%+v  lastseen:%s ago", clnt, hnode.ID, hnode.UpdatedAt, elapsed)
			continue
		}

		activeClientIPs = append(activeClientIPs, clnt)
	}

	glog.V(2).Infof("DFS remote active IPs: %+v", activeClientIPs)
	return activeClientIPs
}

// getAllRemoteHostsFromZookeeper retrieves a list of remote storage clients from zookeeper
func getAllRemoteHostsFromZookeeper(conn client.Connection, storageClientsPath string) ([]string, error) {
	clients, err := conn.Children(storageClientsPath)
	if err != nil {
		glog.Errorf("unable to retrieve list of DFS remote hosts: %s", err)
		return []string{}, err
	}

	glog.V(4).Infof("DFS remote IPs: %+v", clients)
	return clients, nil
}
<|endoftext|>