<commit_before>package core

import (
	"errors"
	. "github.com/smartystreets/goconvey/convey"
	"pfi/sensorbee/sensorbee/data"
	"testing"
	"time"
)

func TestRewindableSource(t *testing.T) {
	Convey("Given a default topology", t, func() {
		t := NewDefaultTopology(NewContext(nil), "dt1")
		Reset(func() {
			t.Stop()
		})

		so := NewTupleEmitterSource(freshTuples())
		son, err := t.AddSource("source", NewRewindableSource(so), &SourceConfig{
			PausedOnStartup: true,
		})
		So(err, ShouldBeNil)
		son.State().Wait(TSPaused)

		b := &BlockingForwardBox{cnt: 1000}
		bn, err := t.AddBox("box", b, nil)
		So(err, ShouldBeNil)
		bn.State().Wait(TSRunning)
		So(bn.Input("source", &BoxInputConfig{
			Capacity: 1, // (almost) blocking channel
		}), ShouldBeNil)

		si := NewTupleCollectorSink()
		sin, err := t.AddSink("sink", si, nil)
		So(sin.Input("box", nil), ShouldBeNil)
		sin.State().Wait(TSRunning)

		Convey("When emitting all tuples", func() {
			So(son.Resume(), ShouldBeNil)
			si.Wait(8)

			Convey("Then the source shouldn't stop", func() {
				So(son.State().Get(), ShouldEqual, TSRunning)
			})

			Convey("Then status should show that it's waiting for rewind", func() {
				waitForWaitingForRewind(son)
				st := son.Status()
				v, _ := st.Get(data.MustCompilePath("source.waiting_for_rewind"))
				So(v, ShouldEqual, data.True)
			})
		})

		Convey("When rewinding before sending any tuple", func() {
			So(son.Rewind(), ShouldBeNil)
			So(son.Resume(), ShouldBeNil)
			si.Wait(8)

			Convey("Then the sink should receive all tuples", func() {
				So(si.len(), ShouldEqual, 8)
			})
		})

		Convey("When rewinding after sending some tuples", func() {
			// emit 2 tuples and then blocks. So, 2 tuples went to the sink,
			// 1 tuple is blocked in the box, and 2 tuples is blocked in the
			// channel between the source and box because its capacity is 1
			// (1 tuple is in the queue and the other one is blocked at sending
			// operation). So, 5 tuples in total were emitted from the source.
			b.setCnt(2)
			So(son.Resume(), ShouldBeNil)
			si.Wait(2)
			So(son.Pause(), ShouldBeNil)
			b.EmitTuples(1000)
			So(son.Rewind(), ShouldBeNil)
			So(son.Resume(), ShouldBeNil)

			Convey("Then all tuple should be able to be sent again", func() {
				si.Wait(12)

				// Due to concurrency, the number of tuples arriving to the sink
				// can be either 12 or 13. It could be 11 but very rare.
				So(si.len(), ShouldBeGreaterThanOrEqualTo, 12)
				So(si.len(), ShouldBeLessThanOrEqualTo, 13)
			})
		})

		Convey("When sending all tuples", func() {
			So(son.Resume(), ShouldBeNil)
			si.Wait(8)

			Convey("The source should be able to be rewound", func() {
				So(son.Rewind(), ShouldBeNil)
				si.Wait(16)
				So(si.len(), ShouldEqual, 16)
			})

			Convey("Then source should stop without rewinding", func() {
				So(son.Stop(), ShouldBeNil)
			})
		})

		Convey("When rewinding the paused source", func() {
			So(son.Rewind(), ShouldBeNil)

			Convey("Then it shouldn't be resumed", func() {
				So(son.State().Get(), ShouldEqual, TSPaused)
			})

			Convey("Then it should show that it isn't waiting for rewind", func() {
				// It's still generating tuples, just the process is being paused.
				st := son.Status()
				v, _ := st.Get(data.MustCompilePath("source.waiting_for_rewind"))
				So(v, ShouldEqual, data.False)
			})
		})

		Convey("When calling Rewind on non-rewindable source", func() {
			so2 := NewTupleEmitterSource(freshTuples())
			son2, err := t.AddSource("source2", so2, nil)
			So(err, ShouldBeNil)

			Convey("Then it should fail", func() {
				So(son2.Rewind(), ShouldNotBeNil)
			})
		})

		Convey("When calling Rewind on the stopped source", func() {
			So(son.Stop(), ShouldBeNil)

			Convey("Then it should fail", func() {
				So(son.Rewind(), ShouldNotBeNil)
			})
		})
	})
}

type dummyNonstoppableSource struct {
	stopped bool
}

func (d *dummyNonstoppableSource) GenerateStream(ctx *Context, w Writer) error {
	defer func() {
		d.stopped = true
	}()
	for {
		if err := w.Write(ctx, NewTuple(data.Map{"a": data.True})); err != nil {
			return err
		}
	}
}

func (d *dummyNonstoppableSource) Stop(ctx *Context) error {
	return nil
}

func TestImplementSourceStop(t *testing.T) {
	Convey("Given a stoppable source via ImplementSourceStop", t, func() {
		ctx := NewContext(nil)
		s := ImplementSourceStop(NewTupleEmitterSource(freshTuples()))
		Reset(func() {
			s.Stop(ctx)
		})

		ch := make(chan error, 1)
		go func() {
			ch <- s.GenerateStream(ctx, WriterFunc(func(ctx *Context, t *Tuple) error {
				return nil
			}))
		}()

		Convey("When waiting for the source to be stopped", func() {
			err := <-ch

			Convey("Then it should stop without explicitly calling Stop method", func() {
				So(err, ShouldBeNil)
			})
		})

		Convey("When converting it to RewindableSource", func() {
			_, ok := s.(RewindableSource)

			Convey("Then it should fail", func() {
				So(ok, ShouldBeFalse)
			})
		})
	})

	Convey("Given a non stoppable source via ImplementSourceStop", t, func() {
		ctx := NewContext(nil)
		ns := &dummyNonstoppableSource{}
		s := ImplementSourceStop(ns)
		Reset(func() {
			s.Stop(ctx)
		})

		ch := make(chan error, 1)
		go func() {
			ch <- s.GenerateStream(ctx, WriterFunc(func(ctx *Context, t *Tuple) error {
				return nil
			}))
		}()

		Convey("When stopping the source", func() {
			So(s.Stop(ctx), ShouldBeNil)

			Convey("Then the original source should stop", func() {
				So(ns.stopped, ShouldBeTrue)
			})

			Convey("Then GenerateStream should return", func() {
				So(<-ch, ShouldBeNil)
			})
		})

		Convey("When converting it to RewindableSource", func() {
			_, ok := s.(RewindableSource)

			Convey("Then it should fail", func() {
				So(ok, ShouldBeFalse)
			})
		})
	})
}

type dummyBlockingSource struct {
}

func (d *dummyBlockingSource) GenerateStream(ctx *Context, w Writer) error {
	// This sleep is only for avoiding a goroutine leak and all tests assume
	// that they finish before this method returns.
	time.Sleep(10 * time.Second)
	return nil
}

func (d *dummyBlockingSource) Stop(ctx *Context) error {
	return errors.New("cannot stop blocking source")
}

func TestRewindableSourceForceStop(t *testing.T) {
	Convey("Given a source whose GenerateStream will never return and whose Stop fails", t, func() {
		ctx := NewContext(nil)
		s := ImplementSourceStop(&dummyBlockingSource{})
		Reset(func() {
			s.Stop(ctx)
		})

		ch := make(chan error, 1)
		go func() {
			ch <- s.GenerateStream(ctx, WriterFunc(func(ctx *Context, t *Tuple) error {
				return nil
			}))
		}()

		Convey("When stopping the source", func() {
			err := s.Stop(ctx)

			Convey("Then it should fail", func() {
				So(err, ShouldNotBeNil)
				So(err.Error(), ShouldEqual, "cannot stop blocking source")
			})

			Convey("Then GenerateStream should stop", func() {
				var err error
				select {
				case err = <-ch:
				default:
				}

				So(err, ShouldNotBeNil)
			})
		})
	})
}

func waitForWaitingForRewind(son SourceNode) {
	so := son.Source()
	rso := so.(*rewindableSource)
	for {
		rso.rwm.RLock()
		if rso.waitingForRewind {
			rso.rwm.RUnlock()
			return
		}
		rso.rwm.RUnlock()
	}
}
<commit_msg>Fix deadlock on Linux.<commit_after>package core

import (
	"errors"
	. "github.com/smartystreets/goconvey/convey"
	"pfi/sensorbee/sensorbee/data"
	"testing"
	"time"
)

func TestRewindableSource(t *testing.T) {
	Convey("Given a default topology", t, func() {
		t := NewDefaultTopology(NewContext(nil), "dt1")
		Reset(func() {
			t.Stop()
		})

		fts := freshTuples()
		so := NewTupleEmitterSource(fts)
		son, err := t.AddSource("source", NewRewindableSource(so), &SourceConfig{
			PausedOnStartup: true,
		})
		So(err, ShouldBeNil)
		son.State().Wait(TSPaused)

		b := &BlockingForwardBox{cnt: 1000}
		bn, err := t.AddBox("box", b, nil)
		So(err, ShouldBeNil)
		bn.State().Wait(TSRunning)
		So(bn.Input("source", &BoxInputConfig{
			Capacity: 1, // (almost) blocking channel
		}), ShouldBeNil)

		si := NewTupleCollectorSink()
		sin, err := t.AddSink("sink", si, nil)
		So(sin.Input("box", nil), ShouldBeNil)
		sin.State().Wait(TSRunning)

		Convey("When emitting all tuples", func() {
			So(son.Resume(), ShouldBeNil)
			si.Wait(8)

			Convey("Then the source shouldn't stop", func() {
				So(son.State().Get(), ShouldEqual, TSRunning)
			})

			Convey("Then status should show that it's waiting for rewind", func() {
				waitForWaitingForRewind(son)
				st := son.Status()
				v, _ := st.Get(data.MustCompilePath("source.waiting_for_rewind"))
				So(v, ShouldEqual, data.True)
			})
		})

		Convey("When rewinding before sending any tuple", func() {
			So(son.Rewind(), ShouldBeNil)
			So(son.Resume(), ShouldBeNil)
			si.Wait(8)

			Convey("Then the sink should receive all tuples", func() {
				So(si.len(), ShouldEqual, 8)
			})
		})

		Convey("When rewinding after sending some tuples", func() {
			// emit 2 tuples and then blocks. So, 2 tuples went to the sink,
			// 1 tuple is blocked in the box, and 2 tuples is blocked in the
			// channel between the source and box because its capacity is 1
			// (1 tuple is in the queue and the other one is blocked at sending
			// operation). So, 5 tuples in total could be emitted from the source.
			b.setCnt(2)
			So(son.Resume(), ShouldBeNil)
			si.Wait(2)
			So(son.Pause(), ShouldBeNil)
			b.EmitTuples(1000)
			So(son.Rewind(), ShouldBeNil)
			So(son.Resume(), ShouldBeNil)

			Convey("Then all tuple should be able to be sent again", func() {
				waitForLastTuple(si, fts[len(fts)-1])

				// Due to cuncurrency, the number of tuples arriving to the sink
				// is not constant.
				offset := si.len() - len(fts)
				for i := range fts {
					So(si.get(offset+i), ShouldResemble, fts[i])
				}
			})
		})

		Convey("When sending all tuples", func() {
			So(son.Resume(), ShouldBeNil)
			si.Wait(8)

			Convey("The source should be able to be rewound", func() {
				So(son.Rewind(), ShouldBeNil)
				si.Wait(16)
				So(si.len(), ShouldEqual, 16)
			})

			Convey("Then source should stop without rewinding", func() {
				So(son.Stop(), ShouldBeNil)
			})
		})

		Convey("When rewinding the paused source", func() {
			So(son.Rewind(), ShouldBeNil)

			Convey("Then it shouldn't be resumed", func() {
				So(son.State().Get(), ShouldEqual, TSPaused)
			})

			Convey("Then it should show that it isn't waiting for rewind", func() {
				// It's still generating tuples, just the process is being paused.
				st := son.Status()
				v, _ := st.Get(data.MustCompilePath("source.waiting_for_rewind"))
				So(v, ShouldEqual, data.False)
			})
		})

		Convey("When calling Rewind on non-rewindable source", func() {
			so2 := NewTupleEmitterSource(freshTuples())
			son2, err := t.AddSource("source2", so2, nil)
			So(err, ShouldBeNil)

			Convey("Then it should fail", func() {
				So(son2.Rewind(), ShouldNotBeNil)
			})
		})

		Convey("When calling Rewind on the stopped source", func() {
			So(son.Stop(), ShouldBeNil)

			Convey("Then it should fail", func() {
				So(son.Rewind(), ShouldNotBeNil)
			})
		})
	})
}

type dummyNonstoppableSource struct {
	stopped bool
}

func (d *dummyNonstoppableSource) GenerateStream(ctx *Context, w Writer) error {
	defer func() {
		d.stopped = true
	}()
	for {
		if err := w.Write(ctx, NewTuple(data.Map{"a": data.True})); err != nil {
			return err
		}
	}
}

func (d *dummyNonstoppableSource) Stop(ctx *Context) error {
	return nil
}

func TestImplementSourceStop(t *testing.T) {
	Convey("Given a stoppable source via ImplementSourceStop", t, func() {
		ctx := NewContext(nil)
		s := ImplementSourceStop(NewTupleEmitterSource(freshTuples()))
		Reset(func() {
			s.Stop(ctx)
		})

		ch := make(chan error, 1)
		go func() {
			ch <- s.GenerateStream(ctx, WriterFunc(func(ctx *Context, t *Tuple) error {
				return nil
			}))
		}()

		Convey("When waiting for the source to be stopped", func() {
			err := <-ch

			Convey("Then it should stop without explicitly calling Stop method", func() {
				So(err, ShouldBeNil)
			})
		})

		Convey("When converting it to RewindableSource", func() {
			_, ok := s.(RewindableSource)

			Convey("Then it should fail", func() {
				So(ok, ShouldBeFalse)
			})
		})
	})

	Convey("Given a non stoppable source via ImplementSourceStop", t, func() {
		ctx := NewContext(nil)
		ns := &dummyNonstoppableSource{}
		s := ImplementSourceStop(ns)
		Reset(func() {
			s.Stop(ctx)
		})

		ch := make(chan error, 1)
		go func() {
			ch <- s.GenerateStream(ctx, WriterFunc(func(ctx *Context, t *Tuple) error {
				return nil
			}))
		}()

		Convey("When stopping the source", func() {
			So(s.Stop(ctx), ShouldBeNil)

			Convey("Then the original source should stop", func() {
				So(ns.stopped, ShouldBeTrue)
			})

			Convey("Then GenerateStream should return", func() {
				So(<-ch, ShouldBeNil)
			})
		})

		Convey("When converting it to RewindableSource", func() {
			_, ok := s.(RewindableSource)

			Convey("Then it should fail", func() {
				So(ok, ShouldBeFalse)
			})
		})
	})
}

type dummyBlockingSource struct {
}

func (d *dummyBlockingSource) GenerateStream(ctx *Context, w Writer) error {
	// This sleep is only for avoiding a goroutine leak and all tests assume
	// that they finish before this method returns.
	time.Sleep(10 * time.Second)
	return nil
}

func (d *dummyBlockingSource) Stop(ctx *Context) error {
	return errors.New("cannot stop blocking source")
}

func TestRewindableSourceForceStop(t *testing.T) {
	Convey("Given a source whose GenerateStream will never return and whose Stop fails", t, func() {
		ctx := NewContext(nil)
		s := ImplementSourceStop(&dummyBlockingSource{})
		Reset(func() {
			s.Stop(ctx)
		})

		ch := make(chan error, 1)
		go func() {
			ch <- s.GenerateStream(ctx, WriterFunc(func(ctx *Context, t *Tuple) error {
				return nil
			}))
		}()

		Convey("When stopping the source", func() {
			err := s.Stop(ctx)

			Convey("Then it should fail", func() {
				So(err, ShouldNotBeNil)
				So(err.Error(), ShouldEqual, "cannot stop blocking source")
			})

			Convey("Then GenerateStream should stop", func() {
				var err error
				select {
				case err = <-ch:
				default:
				}

				So(err, ShouldNotBeNil)
			})
		})
	})
}

func waitForWaitingForRewind(son SourceNode) {
	so := son.Source()
	rso := so.(*rewindableSource)
	for {
		rso.rwm.RLock()
		if rso.waitingForRewind {
			rso.rwm.RUnlock()
			return
		}
		rso.rwm.RUnlock()
	}
}

func waitForLastTuple(si *TupleCollectorSink, t *Tuple) {
	for si.get(si.len()-1) != t {
		time.Sleep(time.Nanosecond)
	}
}
<|endoftext|>


<commit_before>package golangdb

/* Defines a type of database engine */
type DatabaseEngine int

/* Constants used to specify a type of database connection */
const (
	NONE DatabaseEngine = iota
	SQLITE
	MYSQL
	MSSQL
	TESTDB
)

var databaseEngineNames = map[DatabaseEngine]string{
	NONE:   "None",
	SQLITE: "SQLite",
	MYSQL:  "MySQL",
	MSSQL:  "MSSQL",
	TESTDB: "Test (fake) DB",
}

func GetDatabaseEngineFromName(name string) DatabaseEngine {
	for k, v := range databaseEngineNames {
		if v == name {
			return k
		}
	}

	return NONE
}

func (this *DatabaseEngine) ToString() string {
	return databaseEngineNames[*this]
}<commit_msg>Comparing string names in lowercase<commit_after>package golangdb

import (
	"strings"
)

/* Defines a type of database engine */
type DatabaseEngine int

/* Constants used to specify a type of database connection */
const (
	NONE DatabaseEngine = iota
	SQLITE
	MYSQL
	MSSQL
	TESTDB
)

var databaseEngineNames = map[DatabaseEngine]string{
	NONE:   "None",
	SQLITE: "SQLite",
	MYSQL:  "MySQL",
	MSSQL:  "MSSQL",
	TESTDB: "Test (fake) DB",
}

func GetDatabaseEngineFromName(name string) DatabaseEngine {
	for k, v := range databaseEngineNames {
		if strings.ToLower(v) == strings.ToLower(name) {
			return k
		}
	}

	return NONE
}

func (this *DatabaseEngine) ToString() string {
	return databaseEngineNames[*this]
}
<|endoftext|>


<commit_before>// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package stat

import (
	"github.com/gonum/matrix/mat64"
)

// CovarianceMatrix calculates a covariance matrix (also known as a
// variance-covariance matrix) from a matrix of data.
func CovarianceMatrix(x mat64.Matrix) *mat64.Dense {

	// matrix version of the two pass algorithm.  This doesn't use
	// the correction found in the Covariance and Variance functions.

	r, _ := x.Dims()
	b := ones(1, r)
	b.Mul(b, x)
	b.Scale(1/float64(r), b)

	// todo: avoid unneeded memory expansion here.
	mu := new(mat64.Dense)
	mu.Mul(ones(r,1),b)

	// this could also be done with a clone & row viewer
	xc := mat64.DenseCopyOf(x)
	xc.Sub(xc, mu)

	// todo: avoid matrix copy
	xt := new(mat64.Dense)
	xt.TCopy(xc)

	ss := new(mat64.Dense)
	ss.Mul(xt, xc)
	ss.Scale(1/float64(r-1), ss)
	return ss
}

// ones is a matrix of all ones.
func ones(r, c int) *mat64.Dense {
	x := make([]float64, r*c)
	for i := range x {
		x[i] = 1
	}
	return mat64.NewDense(r, c, x)
}
<commit_msg>run go fmt<commit_after>// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package stat

import (
	"github.com/gonum/matrix/mat64"
)

// CovarianceMatrix calculates a covariance matrix (also known as a
// variance-covariance matrix) from a matrix of data.
func CovarianceMatrix(x mat64.Matrix) *mat64.Dense {

	// matrix version of the two pass algorithm.  This doesn't use
	// the correction found in the Covariance and Variance functions.

	r, _ := x.Dims()
	b := ones(1, r)
	b.Mul(b, x)
	b.Scale(1/float64(r), b)

	// todo: avoid unneeded memory expansion here.
	mu := new(mat64.Dense)
	mu.Mul(ones(r, 1), b)

	// this could also be done with a clone & row viewer
	xc := mat64.DenseCopyOf(x)
	xc.Sub(xc, mu)

	// todo: avoid matrix copy
	xt := new(mat64.Dense)
	xt.TCopy(xc)

	ss := new(mat64.Dense)
	ss.Mul(xt, xc)
	ss.Scale(1/float64(r-1), ss)
	return ss
}

// ones is a matrix of all ones.
func ones(r, c int) *mat64.Dense {
	x := make([]float64, r*c)
	for i := range x {
		x[i] = 1
	}
	return mat64.NewDense(r, c, x)
}
<|endoftext|>


<commit_before>// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package stat

import (
	"sync"

	"github.com/gonum/matrix/mat64"
)

// CovarianceMatrix calculates a covariance matrix (also known as a
// variance-covariance matrix) from a matrix of data, using a two-pass
// algorithm.  It will have better performance if a BLAS engine is
// registered in gonum/matrix/mat64.
//
// The matrix returned will be symmetric, square, and positive-semidefinite.
func CovarianceMatrix(x mat64.Matrix) *mat64.Dense {

	// matrix version of the two pass algorithm.  This doesn't use
	// the correction found in the Covariance and Variance functions.
	if mat64.Registered() == nil {
		// implementation that doesn't rely on a blasEngine
		return covarianceMatrixWithoutBLAS(x)
	}
	r, _ := x.Dims()

	// determine the mean of each of the columns
	b := ones(1, r)
	b.Mul(b, x)
	b.Scale(1/float64(r), b)
	mu := b.RowView(0)

	// subtract the mean from the data
	xc := mat64.DenseCopyOf(x)
	for i := 0; i < r; i++ {
		rv := xc.RowView(i)
		for j, mean := range mu {
			rv[j] -= mean
		}
	}

	var xt mat64.Dense
	xt.TCopy(xc)

	// TODO: indicate that the resulting matrix is symmetric, which
	// should improve performance.
	var ss mat64.Dense
	ss.Mul(&xt, xc)
	ss.Scale(1/float64(r-1), &ss)
	return &ss
}

type covMatSlice struct {
	i, j int
	x, y []float64
}

func covarianceMatrixWithoutBLAS(x mat64.Matrix) *mat64.Dense {
	r, c := x.Dims()

	// split out the matrix into columns
	cols := make([][]float64, c)
	for j := range cols {
		cols[j] = make([]float64, r)
	}

	if xRaw, ok := x.(mat64.RawMatrixer); ok {
		for k, v := range xRaw.RawMatrix().Data {
			i := k / c
			j := k % c
			cols[j][i] = v
		}
	} else {
		for j := 0; j < c; j++ {
			for i := 0; i < r; i++ {
				cols[j][i] = x.At(i, j)
			}
		}
	}

	// center the columns
	for j := range cols {
		mean := Mean(cols[j], nil)
		for i := range cols[j] {
			cols[j][i] -= mean
		}
	}

	blockSize := 1024
	if blockSize > c {
		blockSize = c
	}
	var wg sync.WaitGroup
	wg.Add(blockSize)
	colCh := make(chan covMatSlice, blockSize)

	m := mat64.NewDense(c, c, nil)
	for i := 0; i < blockSize; i++ {
		go func(in <-chan covMatSlice) {
			for {
				xy, more := <-in
				if !more {
					wg.Done()
					return
				}

				if xy.i == xy.j {
					m.Set(xy.i, xy.j, centeredVariance(xy.x))
					continue
				}
				v := centeredCovariance(xy.x, xy.y)
				m.Set(xy.i, xy.j, v)
				m.Set(xy.j, xy.i, v)
			}
		}(colCh)
	}
	go func(out chan<- covMatSlice) {
		for i := 0; i < c; i++ {
			for j := 0; j <= i; j++ {
				out <- covMatSlice{
					i: i,
					j: j,
					x: cols[i],
					y: cols[j],
				}
			}
		}
		close(out)
	}(colCh)
	// create the output matrix
	wg.Wait()
	return m
}

// ones is a matrix of all ones.
func ones(r, c int) *mat64.Dense {
	x := make([]float64, r*c)
	for i := range x {
		x[i] = 1
	}
	return mat64.NewDense(r, c, x)
}

// centeredVariance calculates the sum of squares of a single
// series, for calculating variance.
func centeredVariance(x []float64) float64 {
	var ss float64
	for _, xv := range x {
		ss += xv * xv
	}
	return ss / float64(len(x)-1)
}

// centeredCovariance calculates the sum of squares of two
// series, for calculating variance.  The input lengths are
// assumed to be identical.
func centeredCovariance(x, y []float64) float64 {
	var ss float64
	for i, xv := range x {
		ss += xv * y[i]
	}
	return ss / float64(len(x)-1)
}
<commit_msg>go fmt<commit_after>// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package stat

import (
	"sync"

	"github.com/gonum/matrix/mat64"
)

// CovarianceMatrix calculates a covariance matrix (also known as a
// variance-covariance matrix) from a matrix of data, using a two-pass
// algorithm.  It will have better performance if a BLAS engine is
// registered in gonum/matrix/mat64.
//
// The matrix returned will be symmetric, square, and positive-semidefinite.
func CovarianceMatrix(x mat64.Matrix) *mat64.Dense {

	// matrix version of the two pass algorithm.  This doesn't use
	// the correction found in the Covariance and Variance functions.
	if mat64.Registered() == nil {
		// implementation that doesn't rely on a blasEngine
		return covarianceMatrixWithoutBLAS(x)
	}
	r, _ := x.Dims()

	// determine the mean of each of the columns
	b := ones(1, r)
	b.Mul(b, x)
	b.Scale(1/float64(r), b)
	mu := b.RowView(0)

	// subtract the mean from the data
	xc := mat64.DenseCopyOf(x)
	for i := 0; i < r; i++ {
		rv := xc.RowView(i)
		for j, mean := range mu {
			rv[j] -= mean
		}
	}

	var xt mat64.Dense
	xt.TCopy(xc)

	// TODO: indicate that the resulting matrix is symmetric, which
	// should improve performance.
	var ss mat64.Dense
	ss.Mul(&xt, xc)
	ss.Scale(1/float64(r-1), &ss)
	return &ss
}

type covMatSlice struct {
	i, j int
	x, y []float64
}

func covarianceMatrixWithoutBLAS(x mat64.Matrix) *mat64.Dense {
	r, c := x.Dims()

	// split out the matrix into columns
	cols := make([][]float64, c)
	for j := range cols {
		cols[j] = make([]float64, r)
	}

	if xRaw, ok := x.(mat64.RawMatrixer); ok {
		for k, v := range xRaw.RawMatrix().Data {
			i := k / c
			j := k % c
			cols[j][i] = v
		}
	} else {
		for j := 0; j < c; j++ {
			for i := 0; i < r; i++ {
				cols[j][i] = x.At(i, j)
			}
		}
	}

	// center the columns
	for j := range cols {
		mean := Mean(cols[j], nil)
		for i := range cols[j] {
			cols[j][i] -= mean
		}
	}

	blockSize := 1024
	if blockSize > c {
		blockSize = c
	}
	var wg sync.WaitGroup
	wg.Add(blockSize)
	colCh := make(chan covMatSlice, blockSize)

	m := mat64.NewDense(c, c, nil)
	for i := 0; i < blockSize; i++ {
		go func(in <-chan covMatSlice) {
			for {
				xy, more := <-in
				if !more {
					wg.Done()
					return
				}

				if xy.i == xy.j {
					m.Set(xy.i, xy.j, centeredVariance(xy.x))
					continue
				}
				v := centeredCovariance(xy.x, xy.y)
				m.Set(xy.i, xy.j, v)
				m.Set(xy.j, xy.i, v)
			}
		}(colCh)
	}
	go func(out chan<- covMatSlice) {
		for i := 0; i < c; i++ {
			for j := 0; j <= i; j++ {
				out <- covMatSlice{
					i: i,
					j: j,
					x: cols[i],
					y: cols[j],
				}
			}
		}
		close(out)
	}(colCh)
	// create the output matrix
	wg.Wait()
	return m
}

// ones is a matrix of all ones.
func ones(r, c int) *mat64.Dense {
	x := make([]float64, r*c)
	for i := range x {
		x[i] = 1
	}
	return mat64.NewDense(r, c, x)
}

// centeredVariance calculates the sum of squares of a single
// series, for calculating variance.
func centeredVariance(x []float64) float64 {
	var ss float64
	for _, xv := range x {
		ss += xv * xv
	}
	return ss / float64(len(x)-1)
}

// centeredCovariance calculates the sum of squares of two
// series, for calculating variance.  The input lengths are
// assumed to be identical.
func centeredCovariance(x, y []float64) float64 {
	var ss float64
	for i, xv := range x {
		ss += xv * y[i]
	}
	return ss / float64(len(x)-1)
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package gce

import (
	"code.google.com/p/google-api-go-client/compute/v1"
	"github.com/juju/errors"

	"github.com/juju/juju/constraints"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/environs/instances"
	"github.com/juju/juju/environs/simplestreams"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/provider/common"
	"github.com/juju/juju/state/multiwatcher"
	"github.com/juju/juju/tools"
)

func (env *environ) StartInstance(args environs.StartInstanceParams) (*environs.StartInstanceResult, error) {
	// Please note that in order to fulfil the demands made of Instances and
	// AllInstances, it is imperative that some environment feature be used to
	// keep track of which instances were actually started by juju.
	env = env.getSnapshot()

	// Start a new raw instance.

	if args.MachineConfig.HasNetworks() {
		return nil, errors.New("starting instances with networks is not supported yet")
	}

	spec, err := env.finishMachineConfig(args)
	if err != nil {
		return nil, errors.Trace(err)
	}

	raw, err := env.newRawInstance(args, spec)
	if err != nil {
		return nil, errors.Trace(err)
	}

	inst := &environInstance{
		id:   instance.Id(raw.Name),
		env:  env,
		zone: raw.Zone,
	}
	inst.update(env, raw)
	logger.Infof("started instance %q in %q", inst.Id(), raw.Zone)

	// Handle the new instance.

	env.handleStateMachine(args, raw)

	// Build the result.

	hwc := env.getHardwareCharacteristics(spec, raw)

	result := environs.StartInstanceResult{
		Instance: inst,
		Hardware: hwc,
	}
	return &result, nil
}

func (env *environ) finishMachineConfig(args environs.StartInstanceParams) (*instances.InstanceSpec, error) {
	arches := args.Tools.Arches()
	series := args.Tools.OneSeries()
	spec, err := env.findInstanceSpec(env.Config().ImageStream(), &instances.InstanceConstraint{
		Region:      env.ecfg.region(),
		Series:      series,
		Arches:      arches,
		Constraints: args.Constraints,
		// TODO(ericsnow) Is this right?
		Storage: []string{storageScratch, storagePersistent},
	})
	if err != nil {
		return nil, errors.Trace(err)
	}

	envTools, err := args.Tools.Match(tools.Filter{Arch: spec.Image.Arch})
	if err != nil {
		return nil, errors.Errorf("chosen architecture %v not present in %v", spec.Image.Arch, arches)
	}

	args.MachineConfig.Tools = envTools[0]
	err = environs.FinishMachineConfig(args.MachineConfig, env.Config())
	return spec, errors.Trace(err)
}

func (env *environ) findInstanceSpec(stream string, ic *instances.InstanceConstraint) (*instances.InstanceSpec, error) {
	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, errors.Trace(err)
	}

	cloudSpec, err := env.cloudSpec(ic.Region)
	if err != nil {
		return nil, errors.Trace(err)
	}

	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: cloudSpec,
		Series:    []string{ic.Series},
		Arches:    ic.Arches,
		Stream:    stream,
	})

	matchingImages, _, err := imagemetadata.Fetch(sources, imageConstraint, signedImageDataOnly)
	if err != nil {
		return nil, errors.Trace(err)
	}

	images := instances.ImageMetadataToImages(matchingImages)
	spec, err := instances.FindInstanceSpec(images, ic, allInstanceTypes)
	return spec, errors.Trace(err)
}

func (env *environ) newRawInstance(args environs.StartInstanceParams, spec *instances.InstanceSpec) (*compute.Instance, error) {
	userData, err := environs.ComposeUserData(args.MachineConfig, nil)
	if err != nil {
		return nil, errors.Annotate(err, "cannot make user data")
	}
	logger.Debugf("GCE user data; %d bytes", len(userData))
	machineID := common.MachineFullName(env, args.MachineConfig.MachineId)
	disks := getDisks(spec, args.Constraints)
	instance := &compute.Instance{
		// TODO(ericsnow) populate/verify these values.
		Name:  machineID,
		Disks: disks,
		// TODO(ericsnow) Do we really need this?
		Metadata: &compute.Metadata{Items: []*compute.MetadataItems{{
			Key:   "metadata.cloud-init:user-data",
			Value: string(userData),
		}}},
	}

	availabilityZones, err := env.parseAvailabilityZones(args)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if err := env.gce.newInstance(instance, spec.InstanceType.Name, availabilityZones); err != nil {
		return nil, errors.Trace(err)
	}
	return instance, nil
}

func getDisks(spec *instances.InstanceSpec, cons constraints.Value) []*compute.AttachedDisk {
	// TODO(ericsnow) Are we passing the right image value?
	rootDisk, size := diskSpec(cons.RootDisk, spec.Image.Id, true)
	if cons.RootDisk != nil && size == minDiskSize {
		msg := "Ignoring root-disk constraint of %dM because it is smaller than the GCE image size of %dM"
		logger.Infof(msg, *cons.RootDisk, minDiskSize)
	}
	return []*compute.AttachedDisk{rootDisk}
}

func (env *environ) handleStateMachine(args environs.StartInstanceParams, raw *compute.Instance) {
	if multiwatcher.AnyJobNeedsState(args.MachineConfig.Jobs...) {
		err := common.AddStateInstance(env.Storage(), instance.Id(raw.Name))
		if err != nil {
			logger.Errorf("could not record instance in provider-state: %v", err)
		}
	}
}

func (env *environ) getHardwareCharacteristics(spec *instances.InstanceSpec, raw *compute.Instance) *instance.HardwareCharacteristics {
	rawSize := raw.Disks[0].InitializeParams.DiskSizeGb
	rootDiskSize := uint64(rawSize) * 1024
	hwc := instance.HardwareCharacteristics{
		Arch:     &spec.Image.Arch,
		Mem:      &spec.InstanceType.Mem,
		CpuCores: &spec.InstanceType.CpuCores,
		CpuPower: spec.InstanceType.CpuPower,
		RootDisk: &rootDiskSize,
		// TODO(ericsnow) Add Tags here?
		// Tags *compute.Tags
		AvailabilityZone: &raw.Zone,
	}
	return &hwc
}

func (env *environ) AllInstances() ([]instance.Instance, error) {
	instances, err := env.instances()
	return instances, errors.Trace(err)
}

func (env *environ) StopInstances(instances ...instance.Id) error {
	env = env.getSnapshot()

	var ids []string
	for _, id := range instances {
		ids = append(ids, string(id))
	}
	if err := env.gce.removeInstances(env, ids...); err != nil {
		return errors.Trace(err)
	}

	return common.RemoveStateInstances(env.Storage(), instances...)
}
<commit_msg>Add comments in newRawInstance.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package gce

import (
	"code.google.com/p/google-api-go-client/compute/v1"
	"github.com/juju/errors"

	"github.com/juju/juju/constraints"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/environs/instances"
	"github.com/juju/juju/environs/simplestreams"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/provider/common"
	"github.com/juju/juju/state/multiwatcher"
	"github.com/juju/juju/tools"
)

func (env *environ) StartInstance(args environs.StartInstanceParams) (*environs.StartInstanceResult, error) {
	// Please note that in order to fulfil the demands made of Instances and
	// AllInstances, it is imperative that some environment feature be used to
	// keep track of which instances were actually started by juju.
	env = env.getSnapshot()

	// Start a new raw instance.

	if args.MachineConfig.HasNetworks() {
		return nil, errors.New("starting instances with networks is not supported yet")
	}

	spec, err := env.finishMachineConfig(args)
	if err != nil {
		return nil, errors.Trace(err)
	}

	raw, err := env.newRawInstance(args, spec)
	if err != nil {
		return nil, errors.Trace(err)
	}

	inst := &environInstance{
		id:   instance.Id(raw.Name),
		env:  env,
		zone: raw.Zone,
	}
	inst.update(env, raw)
	logger.Infof("started instance %q in %q", inst.Id(), raw.Zone)

	// Handle the new instance.

	env.handleStateMachine(args, raw)

	// Build the result.

	hwc := env.getHardwareCharacteristics(spec, raw)

	result := environs.StartInstanceResult{
		Instance: inst,
		Hardware: hwc,
	}
	return &result, nil
}

func (env *environ) finishMachineConfig(args environs.StartInstanceParams) (*instances.InstanceSpec, error) {
	arches := args.Tools.Arches()
	series := args.Tools.OneSeries()
	spec, err := env.findInstanceSpec(env.Config().ImageStream(), &instances.InstanceConstraint{
		Region:      env.ecfg.region(),
		Series:      series,
		Arches:      arches,
		Constraints: args.Constraints,
		// TODO(ericsnow) Is this right?
		Storage: []string{storageScratch, storagePersistent},
	})
	if err != nil {
		return nil, errors.Trace(err)
	}

	envTools, err := args.Tools.Match(tools.Filter{Arch: spec.Image.Arch})
	if err != nil {
		return nil, errors.Errorf("chosen architecture %v not present in %v", spec.Image.Arch, arches)
	}

	args.MachineConfig.Tools = envTools[0]
	err = environs.FinishMachineConfig(args.MachineConfig, env.Config())
	return spec, errors.Trace(err)
}

func (env *environ) findInstanceSpec(stream string, ic *instances.InstanceConstraint) (*instances.InstanceSpec, error) {
	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, errors.Trace(err)
	}

	cloudSpec, err := env.cloudSpec(ic.Region)
	if err != nil {
		return nil, errors.Trace(err)
	}

	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: cloudSpec,
		Series:    []string{ic.Series},
		Arches:    ic.Arches,
		Stream:    stream,
	})

	matchingImages, _, err := imagemetadata.Fetch(sources, imageConstraint, signedImageDataOnly)
	if err != nil {
		return nil, errors.Trace(err)
	}

	images := instances.ImageMetadataToImages(matchingImages)
	spec, err := instances.FindInstanceSpec(images, ic, allInstanceTypes)
	return spec, errors.Trace(err)
}

func (env *environ) newRawInstance(args environs.StartInstanceParams, spec *instances.InstanceSpec) (*compute.Instance, error) {
	userData, err := environs.ComposeUserData(args.MachineConfig, nil)
	if err != nil {
		return nil, errors.Annotate(err, "cannot make user data")
	}
	logger.Debugf("GCE user data; %d bytes", len(userData))
	machineID := common.MachineFullName(env, args.MachineConfig.MachineId)
	disks := getDisks(spec, args.Constraints)
	instance := &compute.Instance{
		Name: machineID,
		// MachineType is set in the env.gce.newInstance call.
		Disks: disks,
		// We don't set NetworkInterfaces (we use the default).
		// We store a snapshot of what information was used to create
		// this instance. It is only informational.
		Metadata: &compute.Metadata{Items: []*compute.MetadataItems{{
			Key:   "metadata.cloud-init:user-data",
			Value: string(userData),
		}}},
	}

	availabilityZones, err := env.parseAvailabilityZones(args)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if err := env.gce.newInstance(instance, spec.InstanceType.Name, availabilityZones); err != nil {
		return nil, errors.Trace(err)
	}
	return instance, nil
}

func getDisks(spec *instances.InstanceSpec, cons constraints.Value) []*compute.AttachedDisk {
	// TODO(ericsnow) Are we passing the right image value?
	boot := true
	rootDisk, size := diskSpec(cons.RootDisk, spec.Image.Id, boot)
	if cons.RootDisk != nil && size == minDiskSize {
		msg := "Ignoring root-disk constraint of %dM because it is smaller than the GCE image size of %dM"
		logger.Infof(msg, *cons.RootDisk, minDiskSize)
	}
	return []*compute.AttachedDisk{rootDisk}
}

func (env *environ) handleStateMachine(args environs.StartInstanceParams, raw *compute.Instance) {
	if multiwatcher.AnyJobNeedsState(args.MachineConfig.Jobs...) {
		err := common.AddStateInstance(env.Storage(), instance.Id(raw.Name))
		if err != nil {
			logger.Errorf("could not record instance in provider-state: %v", err)
		}
	}
}

func (env *environ) getHardwareCharacteristics(spec *instances.InstanceSpec, raw *compute.Instance) *instance.HardwareCharacteristics {
	rawSize := raw.Disks[0].InitializeParams.DiskSizeGb
	rootDiskSize := uint64(rawSize) * 1024
	hwc := instance.HardwareCharacteristics{
		Arch:     &spec.Image.Arch,
		Mem:      &spec.InstanceType.Mem,
		CpuCores: &spec.InstanceType.CpuCores,
		CpuPower: spec.InstanceType.CpuPower,
		RootDisk: &rootDiskSize,
		// TODO(ericsnow) Add Tags here?
		// Tags *compute.Tags
		AvailabilityZone: &raw.Zone,
	}
	return &hwc
}

func (env *environ) AllInstances() ([]instance.Instance, error) {
	instances, err := env.instances()
	return instances, errors.Trace(err)
}

func (env *environ) StopInstances(instances ...instance.Id) error {
	env = env.getSnapshot()

	var ids []string
	for _, id := range instances {
		ids = append(ids, string(id))
	}
	if err := env.gce.removeInstances(env, ids...); err != nil {
		return errors.Trace(err)
	}

	return common.RemoveStateInstances(env.Storage(), instances...)
}
<|endoftext|>


<commit_before>package vom2

// TODO(toddw): This file contains hacky stuff for the vom->vom2 transition.  It
// should be removed when the transition is complete.

import "os"

var enabled = false

func init() {
	// The env var must start with VEYRON_, in order for the device manager
	// to pass it through, for
	// veyron.io/veyron/veyron/services/mgmt/device/impl test.
	if os.Getenv("VEYRON_VOM2") != "" {
		enabled = true
	}
}

// IsEnabled returns true iff the vom2 transition is enabled.  We check the
// VEYRON_VOM2 environment variable in an init function, and if it's set to
// anything other than the empty string, the transition is enabled.
func IsEnabled() bool {
	return enabled
}

// SetEnabled explicitly enables/disables the vom2 transition.  The passed-in
// value overrides the value set in VEYRON_VOM2 environment variable.
func SetEnabled(val bool) {
	enabled = val
}
<commit_msg>veyron: Enable vom2 by default<commit_after>package vom2

// TODO(toddw): This file contains hacky stuff for the vom->vom2 transition.  It
// should be removed when the transition is complete.

import "os"

var enabled = true

func init() {
	// The env var must start with VEYRON_, in order for the device manager
	// to pass it through, for
	// veyron.io/veyron/veyron/services/mgmt/device/impl test.
	if os.Getenv("VEYRON_VOM2") != "" {
		enabled = true
	}
}

// IsEnabled returns true iff the vom2 transition is enabled.  We check the
// VEYRON_VOM2 environment variable in an init function, and if it's set to
// anything other than the empty string, the transition is enabled.
func IsEnabled() bool {
	return enabled
}

// SetEnabled explicitly enables/disables the vom2 transition.  The passed-in
// value overrides the value set in VEYRON_VOM2 environment variable.
func SetEnabled(val bool) {
	enabled = val
}
<|endoftext|>


<commit_before>package aranGoDriver

type TestSession struct {
}

func NewTestSession() *TestSession {
	return &TestSession{}
}

func (session TestSession) Connect(username string, password string) {

}

func (session TestSession) CreateDB(dbname string) {

}
<commit_msg>fit testsession<commit_after>package aranGoDriver

import (
	"fmt"
)

type TestSession struct {
	database []string
}

func NewTestSession() *TestSession {
	return &TestSession{}
}

// Connect test
func (session TestSession) Connect(username string, password string) {
	fmt.Println("Connect to DB")
}

// CreateDB test create a db
func (session *TestSession) CreateDB(dbname string) {
	session.database = append(session.database, dbname)
}
<|endoftext|>


<commit_before>package main

import (
	"cpollet.net/itinerants/cmd"
	"cpollet.net/itinerants/helpers"
	"os"
)

func main() {
	program, args := helpers.Pop(os.Args)
	command, args := helpers.Pop(args)

	switch command {
	case "login":
		cmd.Login(program, args)
		break
	case "events":
		cmd.Events(program, args)
		break
	case "help":
		helpers.Die("CSV format: event;day;month:year;time (hh:mm);people")
	default:
		usage(program)
	}
}

func usage(program string) {
	helpers.Dief("Usage: %s [help,login,events]", program)
}
<commit_msg>fixed help<commit_after>package main

import (
	"cpollet.net/itinerants/cmd"
	"cpollet.net/itinerants/helpers"
	"os"
)

func main() {
	program, args := helpers.Pop(os.Args)
	command, args := helpers.Pop(args)

	switch command {
	case "login":
		cmd.Login(program, args)
		break
	case "events":
		cmd.Events(program, args)
		break
	case "help":
		helpers.Die("CSV format: event,day,month,year,time (hh:mm),people")
	default:
		usage(program)
	}
}

func usage(program string) {
	helpers.Dief("Usage: %s [help,login,events]", program)
}
<|endoftext|>


<commit_before>package kateway

import (
	"sync"
	"time"

	"github.com/funkygao/gafka/cmd/kguard/watchers"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
)

var _ watchers.Watcher = &WatchKateway{}

// WatchKateway monitors aliveness of kateway cluster.
type WatchKateway struct {
	Zkzone *zk.ZkZone
	Stop   chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup
}

func (this *WatchKateway) Init() {}

func (this *WatchKateway) Run() {
	defer this.Wg.Done()

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	liveKateways := metrics.NewRegisteredGauge("kateway.live", nil)
	for {
		select {
		case <-this.Stop:
			return

		case <-ticker.C:
			kws, _ := this.Zkzone.KatewayInfos()
			liveKateways.Update(int64(len(kws)))
		}
	}
}
<commit_msg>on startup, report the correct live kateway instances<commit_after>package kateway

import (
	"sync"
	"time"

	"github.com/funkygao/gafka/cmd/kguard/watchers"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
)

var _ watchers.Watcher = &WatchKateway{}

// WatchKateway monitors aliveness of kateway cluster.
type WatchKateway struct {
	Zkzone *zk.ZkZone
	Stop   chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup
}

func (this *WatchKateway) Init() {}

func (this *WatchKateway) Run() {
	defer this.Wg.Done()

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	liveKateways := metrics.NewRegisteredGauge("kateway.live", nil)

	// warmup
	kws, _ := this.Zkzone.KatewayInfos()
	liveKateways.Update(int64(len(kws)))

	for {
		select {
		case <-this.Stop:
			return

		case <-ticker.C:
			kws, _ := this.Zkzone.KatewayInfos()
			liveKateways.Update(int64(len(kws)))
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Juniper Networks, Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"time"

	"github.com/golang/glog"
	flag "github.com/spf13/pflag"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/controller/framework"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"

	"github.com/Juniper/contrail-kubernetes/pkg/network"
)

type Config struct {
	KubeUrl      string
	ResyncPeriod time.Duration
}

type NetworkManager struct {
	config Config

	Client     *client.Client
	Controller network.NetworkController

	PodStore    cache.Store
	PodInformer *framework.Controller

	NamespaceStore    cache.Store
	NamespaceInformer *framework.Controller

	RCStore    cache.Store
	RCInformer *framework.Controller

	ServiceStore    cache.Store
	ServiceInformer *framework.Controller

	Shutdown chan struct{}
}

func NewNetworkManager() *NetworkManager {
	manager := new(NetworkManager)
	manager.config = Config{
		KubeUrl:      "http://localhost:8080",
		ResyncPeriod: time.Minute,
	}
	manager.Shutdown = make(chan struct{})
	return manager
}

func (m *NetworkManager) AddFlags(fs *flag.FlagSet) {
	fs.StringVar(&m.config.KubeUrl, "master", m.config.KubeUrl,
		"Kubernetes API endpoint")
}

func (m *NetworkManager) start(args []string) {
	config := &client.Config{
		Host: m.config.KubeUrl,
	}
	var err error
	m.Client, err = client.New(config)
	if err != nil {
		glog.Fatalf("Invalid API configuratin: %v", err)
	}

	m.Controller = network.NewNetworkFactory().Create(m.Client, args)

	m.PodStore, m.PodInformer = framework.NewInformer(
		cache.NewListWatchFromClient(
			m.Client,
			string(api.ResourcePods),
			api.NamespaceAll,
			fields.Everything(),
		),
		&api.Pod{},
		m.config.ResyncPeriod,
		framework.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				m.Controller.AddPod(obj.(*api.Pod))
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
				m.Controller.UpdatePod(
					oldObj.(*api.Pod), newObj.(*api.Pod))
			},
			DeleteFunc: func(obj interface{}) {
				if pod, ok := obj.(*api.Pod); ok {
					m.Controller.DeletePod(pod)
				}
			},
		},
	)

	// m.NamespaceStore, m.NamespaceInformer = framework.NewInformer(
	// 	cache.NewListWatchFromClient(
	// 		m.Client,
	// 		"namespaces",
	// 		api.NamespaceAll,
	// 		fields.Everything(),
	// 	),
	// 	&api.Namespace{},
	// 	m.config.ResyncPeriod,
	// 	framework.ResourceEventHandlerFuncs{
	// 		AddFunc: func(obj interface{}) {
	// 			m.Controller.AddNamespace(
	// 				obj.(*api.Namespace))
	// 		},
	// 		UpdateFunc: func(oldObj, newObj interface{}) {
	// 			m.Controller.UpdateNamespace(
	// 				oldObj.(*api.Namespace),
	// 				newObj.(*api.Namespace))
	// 		},
	// 		DeleteFunc: func(obj interface{}) {
	// 			if namespace, ok := obj.(*api.Namespace); ok {
	// 				m.Controller.DeleteNamespace(namespace)
	// 			}
	// 		},
	// 	},
	// )

	m.RCStore, m.RCInformer = framework.NewInformer(
		cache.NewListWatchFromClient(
			m.Client,
			string(api.ResourceReplicationControllers),
			api.NamespaceAll,
			fields.Everything(),
		),
		&api.ReplicationController{},
		m.config.ResyncPeriod,
		framework.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				m.Controller.AddReplicationController(
					obj.(*api.ReplicationController))
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
				m.Controller.UpdateReplicationController(
					oldObj.(*api.ReplicationController),
					newObj.(*api.ReplicationController))
			},
			DeleteFunc: func(obj interface{}) {
				if rc, ok := obj.(*api.ReplicationController); ok {
					m.Controller.DeleteReplicationController(rc)
				}
			},
		},
	)

	m.ServiceStore, m.ServiceInformer = framework.NewInformer(
		cache.NewListWatchFromClient(
			m.Client,
			string(api.ResourceServices),
			api.NamespaceAll,
			fields.Everything(),
		),
		&api.Service{},
		m.config.ResyncPeriod,
		framework.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				m.Controller.AddService(
					obj.(*api.Service))
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
				m.Controller.UpdateService(
					oldObj.(*api.Service),
					newObj.(*api.Service))
			},
			DeleteFunc: func(obj interface{}) {
				if service, ok := obj.(*api.Service); ok {
					m.Controller.DeleteService(service)
				}
			},
		},
	)

	m.Controller.SetPodStore(m.PodStore)
	m.Controller.SetNamespaceStore(m.NamespaceStore)
	m.Controller.SetReplicationControllerStore(m.RCStore)
	m.Controller.SetServiceStore(m.ServiceStore)
}

func (m *NetworkManager) Run(args []string) error {
	m.start(args)
	go m.PodInformer.Run(m.Shutdown)
	go m.NamespaceInformer.Run(m.Shutdown)
	go m.RCInformer.Run(m.Shutdown)
	go m.ServiceInformer.Run(m.Shutdown)
	go m.Controller.Run(m.Shutdown)
	select {}
}
<commit_msg>Comment out namespace cache<commit_after>/*
Copyright 2015 Juniper Networks, Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"time"

	"github.com/golang/glog"
	flag "github.com/spf13/pflag"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/controller/framework"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"

	"github.com/Juniper/contrail-kubernetes/pkg/network"
)

type Config struct {
	KubeUrl      string
	ResyncPeriod time.Duration
}

type NetworkManager struct {
	config Config

	Client     *client.Client
	Controller network.NetworkController

	PodStore    cache.Store
	PodInformer *framework.Controller

	NamespaceStore    cache.Store
	NamespaceInformer *framework.Controller

	RCStore    cache.Store
	RCInformer *framework.Controller

	ServiceStore    cache.Store
	ServiceInformer *framework.Controller

	Shutdown chan struct{}
}

func NewNetworkManager() *NetworkManager {
	manager := new(NetworkManager)
	manager.config = Config{
		KubeUrl:      "http://localhost:8080",
		ResyncPeriod: time.Minute,
	}
	manager.Shutdown = make(chan struct{})
	return manager
}

func (m *NetworkManager) AddFlags(fs *flag.FlagSet) {
	fs.StringVar(&m.config.KubeUrl, "master", m.config.KubeUrl,
		"Kubernetes API endpoint")
}

func (m *NetworkManager) start(args []string) {
	config := &client.Config{
		Host: m.config.KubeUrl,
	}
	var err error
	m.Client, err = client.New(config)
	if err != nil {
		glog.Fatalf("Invalid API configuratin: %v", err)
	}

	m.Controller = network.NewNetworkFactory().Create(m.Client, args)

	m.PodStore, m.PodInformer = framework.NewInformer(
		cache.NewListWatchFromClient(
			m.Client,
			string(api.ResourcePods),
			api.NamespaceAll,
			fields.Everything(),
		),
		&api.Pod{},
		m.config.ResyncPeriod,
		framework.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				m.Controller.AddPod(obj.(*api.Pod))
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
				m.Controller.UpdatePod(
					oldObj.(*api.Pod), newObj.(*api.Pod))
			},
			DeleteFunc: func(obj interface{}) {
				if pod, ok := obj.(*api.Pod); ok {
					m.Controller.DeletePod(pod)
				}
			},
		},
	)

	// m.NamespaceStore, m.NamespaceInformer = framework.NewInformer(
	// 	cache.NewListWatchFromClient(
	// 		m.Client,
	// 		"namespaces",
	// 		api.NamespaceAll,
	// 		fields.Everything(),
	// 	),
	// 	&api.Namespace{},
	// 	m.config.ResyncPeriod,
	// 	framework.ResourceEventHandlerFuncs{
	// 		AddFunc: func(obj interface{}) {
	// 			m.Controller.AddNamespace(
	// 				obj.(*api.Namespace))
	// 		},
	// 		UpdateFunc: func(oldObj, newObj interface{}) {
	// 			m.Controller.UpdateNamespace(
	// 				oldObj.(*api.Namespace),
	// 				newObj.(*api.Namespace))
	// 		},
	// 		DeleteFunc: func(obj interface{}) {
	// 			if namespace, ok := obj.(*api.Namespace); ok {
	// 				m.Controller.DeleteNamespace(namespace)
	// 			}
	// 		},
	// 	},
	// )

	m.RCStore, m.RCInformer = framework.NewInformer(
		cache.NewListWatchFromClient(
			m.Client,
			string(api.ResourceReplicationControllers),
			api.NamespaceAll,
			fields.Everything(),
		),
		&api.ReplicationController{},
		m.config.ResyncPeriod,
		framework.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				m.Controller.AddReplicationController(
					obj.(*api.ReplicationController))
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
				m.Controller.UpdateReplicationController(
					oldObj.(*api.ReplicationController),
					newObj.(*api.ReplicationController))
			},
			DeleteFunc: func(obj interface{}) {
				if rc, ok := obj.(*api.ReplicationController); ok {
					m.Controller.DeleteReplicationController(rc)
				}
			},
		},
	)

	m.ServiceStore, m.ServiceInformer = framework.NewInformer(
		cache.NewListWatchFromClient(
			m.Client,
			string(api.ResourceServices),
			api.NamespaceAll,
			fields.Everything(),
		),
		&api.Service{},
		m.config.ResyncPeriod,
		framework.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				m.Controller.AddService(
					obj.(*api.Service))
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
				m.Controller.UpdateService(
					oldObj.(*api.Service),
					newObj.(*api.Service))
			},
			DeleteFunc: func(obj interface{}) {
				if service, ok := obj.(*api.Service); ok {
					m.Controller.DeleteService(service)
				}
			},
		},
	)

	m.Controller.SetPodStore(m.PodStore)
	m.Controller.SetNamespaceStore(m.NamespaceStore)
	m.Controller.SetReplicationControllerStore(m.RCStore)
	m.Controller.SetServiceStore(m.ServiceStore)
}

func (m *NetworkManager) Run(args []string) error {
	m.start(args)
	go m.PodInformer.Run(m.Shutdown)
	// go m.NamespaceInformer.Run(m.Shutdown)
	go m.RCInformer.Run(m.Shutdown)
	go m.ServiceInformer.Run(m.Shutdown)
	go m.Controller.Run(m.Shutdown)
	select {}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/kisielk/whisper-go/whisper"
	"github.com/raintank/metrictank/api"
	"github.com/raintank/metrictank/conf"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/metrictank/mdata/chunk/archive"
	"gopkg.in/raintank/schema.v1"
)

var (
	httpEndpoint = flag.String(
		"http-endpoint",
		"http://127.0.0.1:8080/chunks",
		"The http endpoint to send the data to",
	)
	namePrefix = flag.String(
		"name-prefix",
		"",
		"Prefix to prepend before every metric name, should include the '.' if necessary",
	)
	threads = flag.Int(
		"threads",
		10,
		"Number of workers threads to process and convert .wsp files",
	)
	writeUnfinishedChunks = flag.Bool(
		"write-unfinished-chunks",
		false,
		"Defines if chunks that have not completed their chunk span should be written",
	)
	orgId = flag.Int(
		"orgid",
		1,
		"Organization ID the data belongs to ",
	)
	insecureSSL = flag.Bool(
		"insecure-ssl",
		false,
		"Disables ssl certificate verification",
	)
	whisperDirectory = flag.String(
		"whisper-directory",
		"/opt/graphite/storage/whisper",
		"The directory that contains the whisper file structure",
	)
	httpAuth = flag.String(
		"http-auth",
		"",
		"The credentials used to authenticate in the format \"user:password\"",
	)
	dstSchemas = flag.String(
		"dst-schemas",
		"",
		"The filename of the output schemas definition file",
	)
	nameFilterPattern = flag.String(
		"name-filter",
		"",
		"A regex pattern to be applied to all metric names, only matching ones will be imported",
	)
	importUpTo = flag.Uint(
		"import-up-to",
		math.MaxUint32,
		"Only import up to the specified timestamp",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"More detailed logging",
	)
	schemas        conf.Schemas
	nameFilter     *regexp.Regexp
	processedCount uint32
	skippedCount   uint32
)

func main() {
	var err error
	flag.Parse()
	if *verbose {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}

	nameFilter = regexp.MustCompile(*nameFilterPattern)
	schemas, err = conf.ReadSchemas(*dstSchemas)
	if err != nil {
		panic(fmt.Sprintf("Error when parsing schemas file: %q", err))
	}

	fileChan := make(chan string)

	wg := &sync.WaitGroup{}
	wg.Add(*threads)
	for i := 0; i < *threads; i++ {
		go processFromChan(fileChan, wg)
	}

	getFileListIntoChan(fileChan)
	wg.Wait()
}

func processFromChan(files chan string, wg *sync.WaitGroup) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: *insecureSSL},
	}
	client := &http.Client{Transport: tr}

	for file := range files {
		fd, err := os.Open(file)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}
		w, err := whisper.OpenWhisper(fd)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}

		name := getMetricName(file)
		log.Debugf("Processing file %s (%s)", file, name)
		mets, err := getMetrics(w, file, name)
		if err != nil {
			log.Errorf("Failed to get metric: %q", err)
			continue
		}

		for _, met := range mets {
			b, err := met.MarshalCompressed()
			if err != nil {
				log.Errorf("Failed to encode metric: %q", err)
				continue
			}
			size := b.Len()

			req, err := http.NewRequest("POST", *httpEndpoint, io.Reader(b))
			if err != nil {
				log.Fatal(fmt.Sprintf("Cannot construct request to http endpoint %q: %q", *httpEndpoint, err))
			}
			req.Header.Set("Content-Type", "application/json")
			req.Header.Set("Content-Encoding", "gzip")

			if len(*httpAuth) > 0 {
				req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(*httpAuth)))
			}

			success := false
			attempts := 0
			for !success {
				pre := time.Now()
				resp, err := client.Do(req)
				passed := time.Now().Sub(pre).Seconds()
				if err != nil || resp.StatusCode-(resp.StatusCode%100) != 200 {
					if err != nil {
						log.Warningf("Error posting %s (%d bytes), to endpoint %q status %d (attempt %d/%fs, retrying): %s", name, size, *httpEndpoint, resp.StatusCode, attempts, passed, err)
					} else {
						log.Warningf("Error posting %s (%d bytes) to endpoint %q status %d (attempt %d/%fs, retrying)", name, size, *httpEndpoint, resp.StatusCode, attempts, passed)
					}
					attempts++
				} else {
					log.Debugf("Posted %s (%d bytes) to endpoint %q in %f seconds", name, size, *httpEndpoint, passed)
					success = true
				}
				io.Copy(ioutil.Discard, resp.Body)
				resp.Body.Close()
			}

			processed := atomic.AddUint32(&processedCount, 1)
			if processed%100 == 0 {
				skipped := atomic.LoadUint32(&skippedCount)
				log.Infof("Processed %d files, %d skipped", processed, skipped)
			}
		}
	}
	wg.Done()
}

// generate the metric name based on the file name and given prefix
func getMetricName(file string) string {
	// remove all leading '/' from file name
	for file[0] == '/' {
		file = file[1:]
	}

	return *namePrefix + strings.Replace(strings.TrimSuffix(file, ".wsp"), "/", ".", -1)
}

// pointSorter sorts points by timestamp
type pointSorter []whisper.Point

func (a pointSorter) Len() int           { return len(a) }
func (a pointSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp }

// the whisper archives are organized like a ringbuffer. since we need to
// insert the points into the chunks in order we first need to sort them
func sortPoints(points pointSorter) pointSorter {
	sort.Sort(points)
	return points
}

func shortAggMethodString(aggMethod whisper.AggregationMethod) (string, error) {
	switch aggMethod {
	case whisper.AggregationAverage:
		return "avg", nil
	case whisper.AggregationSum:
		return "sum", nil
	case whisper.AggregationMin:
		return "min", nil
	case whisper.AggregationMax:
		return "max", nil
	case whisper.AggregationLast:
		return "lst", nil
	default:
		return "", fmt.Errorf("Unknown aggregation method %d", aggMethod)
	}
}

func getMetrics(w *whisper.Whisper, file, name string) ([]archive.Metric, error) {
	var res []archive.Metric
	if len(w.Header.Archives) == 0 {
		return res, fmt.Errorf("Whisper file contains no archives: %q", file)
	}

	method, err := shortAggMethodString(w.Header.Metadata.AggregationMethod)
	if err != nil {
		return res, err
	}

	md := schema.MetricData{
		Name:     name,
		Metric:   name,
		Interval: int(w.Header.Archives[0].SecondsPerPoint),
		Value:    0,
		Unit:     "unknown",
		Time:     0,
		Mtype:    "gauge",
		Tags:     []string{},
		OrgId:    *orgId,
	}
	md.SetId()
	_, schema := schemas.Match(md.Name, 0)

	points := make(map[int][]whisper.Point)
	for i := range w.Header.Archives {
		p, err := w.DumpArchive(i)
		if err != nil {
			return res, fmt.Errorf("Failed to dump archive %d from whisper file %s", i, file)
		}
		points[i] = p
	}

	conversion := newConversion(w.Header.Archives, points, method)
	for retIdx, retention := range schema.Retentions {
		metric := archive.Metric{
			AggregationMethod: uint32(w.Header.Metadata.AggregationMethod),
			MetricData:        md,
		}
		convertedPoints := conversion.getPoints(retIdx, uint32(retention.SecondsPerPoint), uint32(retention.NumberOfPoints))
		for m, p := range convertedPoints {
			log.Debugf("First point %d last point %d", p[0].Timestamp, p[len(p)-1].Timestamp)
			rowKey := getRowKey(retIdx, md.Id, m, retention.SecondsPerPoint)
			encodedChunks := encodedChunksFromPoints(p, uint32(retention.SecondsPerPoint), retention.ChunkSpan)
			log.Debugf("Archive %d Method %s got %d points = %d chunks at a span of %d", retIdx, m, len(p), len(encodedChunks), retention.ChunkSpan)
			metric.Archives = append(metric.Archives, archive.Archive{
				SecondsPerPoint: uint32(retention.SecondsPerPoint),
				Points:          uint32(retention.NumberOfPoints),
				Chunks:          encodedChunks,
				RowKey:          rowKey,
			})
			if int64(p[len(p)-1].Timestamp) > md.Time {
				md.Time = int64(p[len(p)-1].Timestamp)
			}
		}
		res = append(res, metric)
	}

	return res, nil
}

func getRowKey(retIdx int, id, meth string, secondsPerPoint int) string {
	if retIdx == 0 {
		return id
	} else {
		return api.AggMetricKey(
			id,
			meth,
			uint32(secondsPerPoint),
		)
	}
}

func encodedChunksFromPoints(points []whisper.Point, intervalIn, chunkSpan uint32) []chunk.IterGen {
	var point whisper.Point
	var t0, prevT0 uint32
	var c *chunk.Chunk
	var encodedChunks []chunk.IterGen

	for _, point = range points {
		// this shouldn't happen, but if it would we better catch it here because Metrictank wouldn't handle it well:
		// https://github.com/raintank/metrictank/blob/f1868cccfb92fc82cd853914af958f6d187c5f74/mdata/aggmetric.go#L378
		if point.Timestamp == 0 {
			continue
		}

		t0 = point.Timestamp - (point.Timestamp % chunkSpan)
		if prevT0 == 0 {
			c = chunk.New(t0)
			prevT0 = t0
		} else if prevT0 != t0 {
			c.Finish()

			encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))

			c = chunk.New(t0)
			prevT0 = t0
		}

		err := c.Push(point.Timestamp, point.Value)
		if err != nil {
			panic(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %q", t0, err))
		}
	}

	// if the last written point was also the last one of the current chunk,
	// or if writeUnfinishedChunks is on, we close the chunk and push it
	if point.Timestamp == t0+chunkSpan-intervalIn || *writeUnfinishedChunks {
		c.Finish()
		encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))
	}

	return encodedChunks
}

// scan a directory and feed the list of whisper files relative to base into the given channel
func getFileListIntoChan(fileChan chan string) {
	filepath.Walk(
		*whisperDirectory,
		func(path string, info os.FileInfo, err error) error {
			name := getMetricName(path)
			if !nameFilter.Match([]byte(getMetricName(name))) {
				log.Debugf("Skipping file %s with name %s", path, name)
				atomic.AddUint32(&skippedCount, 1)
				return nil
			}
			if len(path) >= 4 && path[len(path)-4:] == ".wsp" {
				fileChan <- path
			}
			return nil
		},
	)

	close(fileChan)
}
<commit_msg>consolidate multiple archives into one payload<commit_after>package main

import (
	"crypto/tls"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/kisielk/whisper-go/whisper"
	"github.com/raintank/metrictank/api"
	"github.com/raintank/metrictank/conf"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/metrictank/mdata/chunk/archive"
	"gopkg.in/raintank/schema.v1"
)

var (
	httpEndpoint = flag.String(
		"http-endpoint",
		"http://127.0.0.1:8080/chunks",
		"The http endpoint to send the data to",
	)
	namePrefix = flag.String(
		"name-prefix",
		"",
		"Prefix to prepend before every metric name, should include the '.' if necessary",
	)
	threads = flag.Int(
		"threads",
		10,
		"Number of workers threads to process and convert .wsp files",
	)
	writeUnfinishedChunks = flag.Bool(
		"write-unfinished-chunks",
		false,
		"Defines if chunks that have not completed their chunk span should be written",
	)
	orgId = flag.Int(
		"orgid",
		1,
		"Organization ID the data belongs to ",
	)
	insecureSSL = flag.Bool(
		"insecure-ssl",
		false,
		"Disables ssl certificate verification",
	)
	whisperDirectory = flag.String(
		"whisper-directory",
		"/opt/graphite/storage/whisper",
		"The directory that contains the whisper file structure",
	)
	httpAuth = flag.String(
		"http-auth",
		"",
		"The credentials used to authenticate in the format \"user:password\"",
	)
	dstSchemas = flag.String(
		"dst-schemas",
		"",
		"The filename of the output schemas definition file",
	)
	nameFilterPattern = flag.String(
		"name-filter",
		"",
		"A regex pattern to be applied to all metric names, only matching ones will be imported",
	)
	importUpTo = flag.Uint(
		"import-up-to",
		math.MaxUint32,
		"Only import up to the specified timestamp",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"More detailed logging",
	)
	schemas        conf.Schemas
	nameFilter     *regexp.Regexp
	processedCount uint32
	skippedCount   uint32
)

func main() {
	var err error
	flag.Parse()
	if *verbose {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}

	nameFilter = regexp.MustCompile(*nameFilterPattern)
	schemas, err = conf.ReadSchemas(*dstSchemas)
	if err != nil {
		panic(fmt.Sprintf("Error when parsing schemas file: %q", err))
	}

	fileChan := make(chan string)

	wg := &sync.WaitGroup{}
	wg.Add(*threads)
	for i := 0; i < *threads; i++ {
		go processFromChan(fileChan, wg)
	}

	getFileListIntoChan(fileChan)
	wg.Wait()
}

func processFromChan(files chan string, wg *sync.WaitGroup) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: *insecureSSL},
	}
	client := &http.Client{Transport: tr}

	for file := range files {
		fd, err := os.Open(file)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}
		w, err := whisper.OpenWhisper(fd)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}

		name := getMetricName(file)
		log.Debugf("Processing file %s (%s)", file, name)
		met, err := getMetric(w, file, name)
		if err != nil {
			log.Errorf("Failed to get metric: %q", err)
			continue
		}

		b, err := met.MarshalCompressed()
		if err != nil {
			log.Errorf("Failed to encode metric: %q", err)
			continue
		}
		size := b.Len()

		req, err := http.NewRequest("POST", *httpEndpoint, io.Reader(b))
		if err != nil {
			log.Fatal(fmt.Sprintf("Cannot construct request to http endpoint %q: %q", *httpEndpoint, err))
		}
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Content-Encoding", "gzip")

		if len(*httpAuth) > 0 {
			req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(*httpAuth)))
		}

		success := false
		attempts := 0
		for !success {
			pre := time.Now()
			resp, err := client.Do(req)
			passed := time.Now().Sub(pre).Seconds()
			if err != nil || resp.StatusCode-(resp.StatusCode%100) != 200 {
				if err != nil {
					log.Warningf("Error posting %s (%d bytes), to endpoint %q status %d (attempt %d/%fs, retrying): %s", name, size, *httpEndpoint, resp.StatusCode, attempts, passed, err)
				} else {
					log.Warningf("Error posting %s (%d bytes) to endpoint %q status %d (attempt %d/%fs, retrying)", name, size, *httpEndpoint, resp.StatusCode, attempts, passed)
				}
				attempts++
			} else {
				log.Debugf("Posted %s (%d bytes) to endpoint %q in %f seconds", name, size, *httpEndpoint, passed)
				success = true
			}
			io.Copy(ioutil.Discard, resp.Body)
			resp.Body.Close()
		}

		processed := atomic.AddUint32(&processedCount, 1)
		if processed%100 == 0 {
			skipped := atomic.LoadUint32(&skippedCount)
			log.Infof("Processed %d files, %d skipped", processed, skipped)
		}
	}
	wg.Done()
}

// generate the metric name based on the file name and given prefix
func getMetricName(file string) string {
	// remove all leading '/' from file name
	for file[0] == '/' {
		file = file[1:]
	}

	return *namePrefix + strings.Replace(strings.TrimSuffix(file, ".wsp"), "/", ".", -1)
}

// pointSorter sorts points by timestamp
type pointSorter []whisper.Point

func (a pointSorter) Len() int           { return len(a) }
func (a pointSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp }

// the whisper archives are organized like a ringbuffer. since we need to
// insert the points into the chunks in order we first need to sort them
func sortPoints(points pointSorter) pointSorter {
	sort.Sort(points)
	return points
}

func shortAggMethodString(aggMethod whisper.AggregationMethod) (string, error) {
	switch aggMethod {
	case whisper.AggregationAverage:
		return "avg", nil
	case whisper.AggregationSum:
		return "sum", nil
	case whisper.AggregationMin:
		return "min", nil
	case whisper.AggregationMax:
		return "max", nil
	case whisper.AggregationLast:
		return "lst", nil
	default:
		return "", fmt.Errorf("Unknown aggregation method %d", aggMethod)
	}
}

func getMetric(w *whisper.Whisper, file, name string) (archive.Metric, error) {
	res := archive.Metric{
		AggregationMethod: uint32(w.Header.Metadata.AggregationMethod),
	}
	if len(w.Header.Archives) == 0 {
		return res, fmt.Errorf("Whisper file contains no archives: %q", file)
	}

	method, err := shortAggMethodString(w.Header.Metadata.AggregationMethod)
	if err != nil {
		return res, err
	}

	md := schema.MetricData{
		Name:     name,
		Metric:   name,
		Interval: int(w.Header.Archives[0].SecondsPerPoint),
		Value:    0,
		Unit:     "unknown",
		Time:     0,
		Mtype:    "gauge",
		Tags:     []string{},
		OrgId:    *orgId,
	}
	md.SetId()
	_, schema := schemas.Match(md.Name, 0)

	points := make(map[int][]whisper.Point)
	for i := range w.Header.Archives {
		p, err := w.DumpArchive(i)
		if err != nil {
			return res, fmt.Errorf("Failed to dump archive %d from whisper file %s", i, file)
		}
		points[i] = p
	}

	conversion := newConversion(w.Header.Archives, points, method)
	for retIdx, retention := range schema.Retentions {
		convertedPoints := conversion.getPoints(retIdx, uint32(retention.SecondsPerPoint), uint32(retention.NumberOfPoints))
		for m, p := range convertedPoints {
			rowKey := getRowKey(retIdx, md.Id, m, retention.SecondsPerPoint)
			encodedChunks := encodedChunksFromPoints(p, uint32(retention.SecondsPerPoint), retention.ChunkSpan)
			log.Debugf("Archive %d Method %s got %d points = %d chunks at a span of %d", retIdx, m, len(p), len(encodedChunks), retention.ChunkSpan)
			res.Archives = append(res.Archives, archive.Archive{
				SecondsPerPoint: uint32(retention.SecondsPerPoint),
				Points:          uint32(retention.NumberOfPoints),
				Chunks:          encodedChunks,
				RowKey:          rowKey,
			})
			if int64(p[len(p)-1].Timestamp) > md.Time {
				md.Time = int64(p[len(p)-1].Timestamp)
			}
		}
	}
	res.MetricData = md

	return res, nil
}

func getRowKey(retIdx int, id, meth string, secondsPerPoint int) string {
	if retIdx == 0 {
		return id
	} else {
		return api.AggMetricKey(
			id,
			meth,
			uint32(secondsPerPoint),
		)
	}
}

func encodedChunksFromPoints(points []whisper.Point, intervalIn, chunkSpan uint32) []chunk.IterGen {
	var point whisper.Point
	var t0, prevT0 uint32
	var c *chunk.Chunk
	var encodedChunks []chunk.IterGen

	for _, point = range points {
		// this shouldn't happen, but if it would we better catch it here because Metrictank wouldn't handle it well:
		// https://github.com/raintank/metrictank/blob/f1868cccfb92fc82cd853914af958f6d187c5f74/mdata/aggmetric.go#L378
		if point.Timestamp == 0 {
			continue
		}

		t0 = point.Timestamp - (point.Timestamp % chunkSpan)
		if prevT0 == 0 {
			c = chunk.New(t0)
			prevT0 = t0
		} else if prevT0 != t0 {
			c.Finish()

			encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))

			c = chunk.New(t0)
			prevT0 = t0
		}

		err := c.Push(point.Timestamp, point.Value)
		if err != nil {
			panic(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %q", t0, err))
		}
	}

	// if the last written point was also the last one of the current chunk,
	// or if writeUnfinishedChunks is on, we close the chunk and push it
	if point.Timestamp == t0+chunkSpan-intervalIn || *writeUnfinishedChunks {
		c.Finish()
		encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))
	}

	return encodedChunks
}

// scan a directory and feed the list of whisper files relative to base into the given channel
func getFileListIntoChan(fileChan chan string) {
	filepath.Walk(
		*whisperDirectory,
		func(path string, info os.FileInfo, err error) error {
			name := getMetricName(path)
			if !nameFilter.Match([]byte(getMetricName(name))) {
				log.Debugf("Skipping file %s with name %s", path, name)
				atomic.AddUint32(&skippedCount, 1)
				return nil
			}
			if len(path) >= 4 && path[len(path)-4:] == ".wsp" {
				fileChan <- path
			}
			return nil
		},
	)

	close(fileChan)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/grafana/metrictank/api"
	"github.com/grafana/metrictank/conf"
	"github.com/grafana/metrictank/mdata/chunk"
	"github.com/grafana/metrictank/mdata/chunk/archive"
	"github.com/kisielk/whisper-go/whisper"
	"gopkg.in/raintank/schema.v1"
)

var (
	httpEndpoint = flag.String(
		"http-endpoint",
		"http://127.0.0.1:8080/chunks",
		"The http endpoint to send the data to",
	)
	namePrefix = flag.String(
		"name-prefix",
		"",
		"Prefix to prepend before every metric name, should include the '.' if necessary",
	)
	threads = flag.Int(
		"threads",
		10,
		"Number of workers threads to process and convert .wsp files",
	)
	writeUnfinishedChunks = flag.Bool(
		"write-unfinished-chunks",
		false,
		"Defines if chunks that have not completed their chunk span should be written",
	)
	orgId = flag.Int(
		"orgid",
		1,
		"Organization ID the data belongs to ",
	)
	insecureSSL = flag.Bool(
		"insecure-ssl",
		false,
		"Disables ssl certificate verification",
	)
	whisperDirectory = flag.String(
		"whisper-directory",
		"/opt/graphite/storage/whisper",
		"The directory that contains the whisper file structure",
	)
	httpAuth = flag.String(
		"http-auth",
		"",
		"The credentials used to authenticate in the format \"user:password\"",
	)
	dstSchemas = flag.String(
		"dst-schemas",
		"",
		"The filename of the output schemas definition file",
	)
	nameFilterPattern = flag.String(
		"name-filter",
		"",
		"A regex pattern to be applied to all metric names, only matching ones will be imported",
	)
	importUpTo = flag.Uint(
		"import-up-to",
		math.MaxUint32,
		"Only import up to the specified timestamp",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"More detailed logging",
	)
	schemas        conf.Schemas
	nameFilter     *regexp.Regexp
	processedCount uint32
	skippedCount   uint32
)

func main() {
	var err error
	flag.Parse()
	if *verbose {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}

	nameFilter = regexp.MustCompile(*nameFilterPattern)
	schemas, err = conf.ReadSchemas(*dstSchemas)
	if err != nil {
		panic(fmt.Sprintf("Error when parsing schemas file: %q", err))
	}

	fileChan := make(chan string)

	wg := &sync.WaitGroup{}
	wg.Add(*threads)
	for i := 0; i < *threads; i++ {
		go processFromChan(fileChan, wg)
	}

	getFileListIntoChan(fileChan)
	wg.Wait()
}

func processFromChan(files chan string, wg *sync.WaitGroup) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: *insecureSSL},
	}
	client := &http.Client{Transport: tr}

	for file := range files {
		fd, err := os.Open(file)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}
		w, err := whisper.OpenWhisper(fd)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}

		name := getMetricName(file)
		log.Debugf("Processing file %s (%s)", file, name)
		met, err := getMetric(w, file, name)
		if err != nil {
			log.Errorf("Failed to get metric: %q", err)
			continue
		}

		success := false
		attempts := 0
		for !success {
			b, err := met.MarshalCompressed()
			if err != nil {
				log.Errorf("Failed to encode metric: %q", err)
				continue
			}
			size := b.Len()

			req, err := http.NewRequest("POST", *httpEndpoint, io.Reader(b))
			if err != nil {
				log.Fatal(fmt.Sprintf("Cannot construct request to http endpoint %q: %q", *httpEndpoint, err))
			}

			req.Header.Set("Content-Type", "application/json")
			req.Header.Set("Content-Encoding", "gzip")

			if len(*httpAuth) > 0 {
				req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(*httpAuth)))
			}

			pre := time.Now()
			resp, err := client.Do(req)
			passed := time.Now().Sub(pre).Seconds()
			if err != nil || resp.StatusCode >= 300 {
				if err != nil {
					log.Warningf("Error posting %s (%d bytes), to endpoint %q (attempt %d/%fs, retrying): %s", name, size, *httpEndpoint, attempts, passed, err)
					attempts++
					continue
				} else {
					log.Warningf("Error posting %s (%d bytes) to endpoint %q status %d (attempt %d/%fs, retrying)", name, size, *httpEndpoint, resp.StatusCode, attempts, passed)
				}
				attempts++
			} else {
				log.Debugf("Posted %s (%d bytes) to endpoint %q in %f seconds", name, size, *httpEndpoint, passed)
				success = true
			}
			io.Copy(ioutil.Discard, resp.Body)
			resp.Body.Close()
		}

		processed := atomic.AddUint32(&processedCount, 1)
		if processed%100 == 0 {
			skipped := atomic.LoadUint32(&skippedCount)
			log.Infof("Processed %d files, %d skipped", processed, skipped)
		}
	}
	wg.Done()
}

// generate the metric name based on the file name and given prefix
func getMetricName(file string) string {
	// remove all leading '/' from file name
	file = strings.TrimPrefix(file, *whisperDirectory)
	for file[0] == '/' {
		file = file[1:]
	}

	return *namePrefix + strings.Replace(strings.TrimSuffix(file, ".wsp"), "/", ".", -1)
}

// pointSorter sorts points by timestamp
type pointSorter []whisper.Point

func (a pointSorter) Len() int           { return len(a) }
func (a pointSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp }

// the whisper archives are organized like a ringbuffer. since we need to
// insert the points into the chunks in order we first need to sort them
func sortPoints(points pointSorter) pointSorter {
	sort.Sort(points)
	return points
}

func shortAggMethodString(aggMethod whisper.AggregationMethod) (string, error) {
	switch aggMethod {
	case whisper.AggregationAverage:
		return "avg", nil
	case whisper.AggregationSum:
		return "sum", nil
	case whisper.AggregationMin:
		return "min", nil
	case whisper.AggregationMax:
		return "max", nil
	case whisper.AggregationLast:
		return "lst", nil
	default:
		return "", fmt.Errorf("Unknown aggregation method %d", aggMethod)
	}
}

func getMetric(w *whisper.Whisper, file, name string) (archive.Metric, error) {
	res := archive.Metric{
		AggregationMethod: uint32(w.Header.Metadata.AggregationMethod),
	}
	if len(w.Header.Archives) == 0 {
		return res, fmt.Errorf("Whisper file contains no archives: %q", file)
	}

	method, err := shortAggMethodString(w.Header.Metadata.AggregationMethod)
	if err != nil {
		return res, err
	}

	md := schema.MetricData{
		Name:     name,
		Metric:   name,
		Interval: int(w.Header.Archives[0].SecondsPerPoint),
		Value:    0,
		Unit:     "unknown",
		Time:     0,
		Mtype:    "gauge",
		Tags:     []string{},
		OrgId:    *orgId,
	}
	md.SetId()
	_, schema := schemas.Match(md.Name, w.Header.Archives[0].SecondsPerPoint)

	points := make(map[int][]whisper.Point)
	for i := range w.Header.Archives {
		p, err := w.DumpArchive(i)
		if err != nil {
			return res, fmt.Errorf("Failed to dump archive %d from whisper file %s", i, file)
		}
		points[i] = p
	}

	conversion := newConversion(w.Header.Archives, points, method)
	for retIdx, retention := range schema.Retentions {
		convertedPoints := conversion.getPoints(retIdx, uint32(retention.SecondsPerPoint), uint32(retention.NumberOfPoints))
		for m, p := range convertedPoints {
			if len(p) == 0 {
				continue
			}
			rowKey := getRowKey(retIdx, md.Id, m, retention.SecondsPerPoint)
			encodedChunks := encodedChunksFromPoints(p, uint32(retention.SecondsPerPoint), retention.ChunkSpan)
			log.Debugf("Archive %d Method %s got %d points = %d chunks at a span of %d", retIdx, m, len(p), len(encodedChunks), retention.ChunkSpan)
			res.Archives = append(res.Archives, archive.Archive{
				SecondsPerPoint: uint32(retention.SecondsPerPoint),
				Points:          uint32(retention.NumberOfPoints),
				Chunks:          encodedChunks,
				RowKey:          rowKey,
			})
			if int64(p[len(p)-1].Timestamp) > md.Time {
				md.Time = int64(p[len(p)-1].Timestamp)
			}
		}
	}
	res.MetricData = md

	return res, nil
}

func getRowKey(retIdx int, id, meth string, secondsPerPoint int) string {
	if retIdx == 0 {
		return id
	} else {
		return api.AggMetricKey(
			id,
			meth,
			uint32(secondsPerPoint),
		)
	}
}

func encodedChunksFromPoints(points []whisper.Point, intervalIn, chunkSpan uint32) []chunk.IterGen {
	var point whisper.Point
	var t0, prevT0 uint32
	var c *chunk.Chunk
	var encodedChunks []chunk.IterGen

	for _, point = range points {
		// this shouldn't happen, but if it would we better catch it here because Metrictank wouldn't handle it well:
		// https://github.com/grafana/metrictank/blob/f1868cccfb92fc82cd853914af958f6d187c5f74/mdata/aggmetric.go#L378
		if point.Timestamp == 0 {
			continue
		}

		t0 = point.Timestamp - (point.Timestamp % chunkSpan)
		if prevT0 == 0 {
			c = chunk.New(t0)
			prevT0 = t0
		} else if prevT0 != t0 {
			c.Finish()

			encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))

			c = chunk.New(t0)
			prevT0 = t0
		}

		err := c.Push(point.Timestamp, point.Value)
		if err != nil {
			panic(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %q", t0, err))
		}
	}

	// if the last written point was also the last one of the current chunk,
	// or if writeUnfinishedChunks is on, we close the chunk and push it
	if point.Timestamp == t0+chunkSpan-intervalIn || *writeUnfinishedChunks {
		c.Finish()
		encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))
	}

	return encodedChunks
}

// scan a directory and feed the list of whisper files relative to base into the given channel
func getFileListIntoChan(fileChan chan string) {
	filepath.Walk(
		*whisperDirectory,
		func(path string, info os.FileInfo, err error) error {
			name := getMetricName(path)
			if !nameFilter.Match([]byte(getMetricName(name))) {
				log.Debugf("Skipping file %s with name %s", path, name)
				atomic.AddUint32(&skippedCount, 1)
				return nil
			}
			if len(path) >= 4 && path[len(path)-4:] == ".wsp" {
				fileChan <- path
			}
			return nil
		},
	)

	close(fileChan)
}
<commit_msg>fix index error when processing root directory<commit_after>package main

import (
	"crypto/tls"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/grafana/metrictank/api"
	"github.com/grafana/metrictank/conf"
	"github.com/grafana/metrictank/mdata/chunk"
	"github.com/grafana/metrictank/mdata/chunk/archive"
	"github.com/kisielk/whisper-go/whisper"
	"gopkg.in/raintank/schema.v1"
)

var (
	httpEndpoint = flag.String(
		"http-endpoint",
		"http://127.0.0.1:8080/chunks",
		"The http endpoint to send the data to",
	)
	namePrefix = flag.String(
		"name-prefix",
		"",
		"Prefix to prepend before every metric name, should include the '.' if necessary",
	)
	threads = flag.Int(
		"threads",
		10,
		"Number of workers threads to process and convert .wsp files",
	)
	writeUnfinishedChunks = flag.Bool(
		"write-unfinished-chunks",
		false,
		"Defines if chunks that have not completed their chunk span should be written",
	)
	orgId = flag.Int(
		"orgid",
		1,
		"Organization ID the data belongs to ",
	)
	insecureSSL = flag.Bool(
		"insecure-ssl",
		false,
		"Disables ssl certificate verification",
	)
	whisperDirectory = flag.String(
		"whisper-directory",
		"/opt/graphite/storage/whisper",
		"The directory that contains the whisper file structure",
	)
	httpAuth = flag.String(
		"http-auth",
		"",
		"The credentials used to authenticate in the format \"user:password\"",
	)
	dstSchemas = flag.String(
		"dst-schemas",
		"",
		"The filename of the output schemas definition file",
	)
	nameFilterPattern = flag.String(
		"name-filter",
		"",
		"A regex pattern to be applied to all metric names, only matching ones will be imported",
	)
	importUpTo = flag.Uint(
		"import-up-to",
		math.MaxUint32,
		"Only import up to the specified timestamp",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"More detailed logging",
	)
	schemas        conf.Schemas
	nameFilter     *regexp.Regexp
	processedCount uint32
	skippedCount   uint32
)

func main() {
	var err error
	flag.Parse()
	if *verbose {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}

	nameFilter = regexp.MustCompile(*nameFilterPattern)
	schemas, err = conf.ReadSchemas(*dstSchemas)
	if err != nil {
		panic(fmt.Sprintf("Error when parsing schemas file: %q", err))
	}

	fileChan := make(chan string)

	wg := &sync.WaitGroup{}
	wg.Add(*threads)
	for i := 0; i < *threads; i++ {
		go processFromChan(fileChan, wg)
	}

	getFileListIntoChan(fileChan)
	wg.Wait()
}

func processFromChan(files chan string, wg *sync.WaitGroup) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: *insecureSSL},
	}
	client := &http.Client{Transport: tr}

	for file := range files {
		fd, err := os.Open(file)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}
		w, err := whisper.OpenWhisper(fd)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}

		name := getMetricName(file)
		log.Debugf("Processing file %s (%s)", file, name)
		met, err := getMetric(w, file, name)
		if err != nil {
			log.Errorf("Failed to get metric: %q", err)
			continue
		}

		success := false
		attempts := 0
		for !success {
			b, err := met.MarshalCompressed()
			if err != nil {
				log.Errorf("Failed to encode metric: %q", err)
				continue
			}
			size := b.Len()

			req, err := http.NewRequest("POST", *httpEndpoint, io.Reader(b))
			if err != nil {
				log.Fatal(fmt.Sprintf("Cannot construct request to http endpoint %q: %q", *httpEndpoint, err))
			}

			req.Header.Set("Content-Type", "application/json")
			req.Header.Set("Content-Encoding", "gzip")

			if len(*httpAuth) > 0 {
				req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(*httpAuth)))
			}

			pre := time.Now()
			resp, err := client.Do(req)
			passed := time.Now().Sub(pre).Seconds()
			if err != nil || resp.StatusCode >= 300 {
				if err != nil {
					log.Warningf("Error posting %s (%d bytes), to endpoint %q (attempt %d/%fs, retrying): %s", name, size, *httpEndpoint, attempts, passed, err)
					attempts++
					continue
				} else {
					log.Warningf("Error posting %s (%d bytes) to endpoint %q status %d (attempt %d/%fs, retrying)", name, size, *httpEndpoint, resp.StatusCode, attempts, passed)
				}
				attempts++
			} else {
				log.Debugf("Posted %s (%d bytes) to endpoint %q in %f seconds", name, size, *httpEndpoint, passed)
				success = true
			}
			io.Copy(ioutil.Discard, resp.Body)
			resp.Body.Close()
		}

		processed := atomic.AddUint32(&processedCount, 1)
		if processed%100 == 0 {
			skipped := atomic.LoadUint32(&skippedCount)
			log.Infof("Processed %d files, %d skipped", processed, skipped)
		}
	}
	wg.Done()
}

// generate the metric name based on the file name and given prefix
func getMetricName(file string) string {
	// remove all leading '/' from file name
	file = strings.TrimPrefix(file, *whisperDirectory)
	for file[0] == '/' {
		file = file[1:]
	}

	return *namePrefix + strings.Replace(strings.TrimSuffix(file, ".wsp"), "/", ".", -1)
}

// pointSorter sorts points by timestamp
type pointSorter []whisper.Point

func (a pointSorter) Len() int           { return len(a) }
func (a pointSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp }

// the whisper archives are organized like a ringbuffer. since we need to
// insert the points into the chunks in order we first need to sort them
func sortPoints(points pointSorter) pointSorter {
	sort.Sort(points)
	return points
}

func shortAggMethodString(aggMethod whisper.AggregationMethod) (string, error) {
	switch aggMethod {
	case whisper.AggregationAverage:
		return "avg", nil
	case whisper.AggregationSum:
		return "sum", nil
	case whisper.AggregationMin:
		return "min", nil
	case whisper.AggregationMax:
		return "max", nil
	case whisper.AggregationLast:
		return "lst", nil
	default:
		return "", fmt.Errorf("Unknown aggregation method %d", aggMethod)
	}
}

func getMetric(w *whisper.Whisper, file, name string) (archive.Metric, error) {
	res := archive.Metric{
		AggregationMethod: uint32(w.Header.Metadata.AggregationMethod),
	}
	if len(w.Header.Archives) == 0 {
		return res, fmt.Errorf("Whisper file contains no archives: %q", file)
	}

	method, err := shortAggMethodString(w.Header.Metadata.AggregationMethod)
	if err != nil {
		return res, err
	}

	md := schema.MetricData{
		Name:     name,
		Metric:   name,
		Interval: int(w.Header.Archives[0].SecondsPerPoint),
		Value:    0,
		Unit:     "unknown",
		Time:     0,
		Mtype:    "gauge",
		Tags:     []string{},
		OrgId:    *orgId,
	}
	md.SetId()
	_, schema := schemas.Match(md.Name, int(w.Header.Archives[0].SecondsPerPoint))

	points := make(map[int][]whisper.Point)
	for i := range w.Header.Archives {
		p, err := w.DumpArchive(i)
		if err != nil {
			return res, fmt.Errorf("Failed to dump archive %d from whisper file %s", i, file)
		}
		points[i] = p
	}

	conversion := newConversion(w.Header.Archives, points, method)
	for retIdx, retention := range schema.Retentions {
		convertedPoints := conversion.getPoints(retIdx, uint32(retention.SecondsPerPoint), uint32(retention.NumberOfPoints))
		for m, p := range convertedPoints {
			if len(p) == 0 {
				continue
			}
			rowKey := getRowKey(retIdx, md.Id, m, retention.SecondsPerPoint)
			encodedChunks := encodedChunksFromPoints(p, uint32(retention.SecondsPerPoint), retention.ChunkSpan)
			log.Debugf("Archive %d Method %s got %d points = %d chunks at a span of %d", retIdx, m, len(p), len(encodedChunks), retention.ChunkSpan)
			res.Archives = append(res.Archives, archive.Archive{
				SecondsPerPoint: uint32(retention.SecondsPerPoint),
				Points:          uint32(retention.NumberOfPoints),
				Chunks:          encodedChunks,
				RowKey:          rowKey,
			})
			if int64(p[len(p)-1].Timestamp) > md.Time {
				md.Time = int64(p[len(p)-1].Timestamp)
			}
		}
	}
	res.MetricData = md

	return res, nil
}

func getRowKey(retIdx int, id, meth string, secondsPerPoint int) string {
	if retIdx == 0 {
		return id
	} else {
		return api.AggMetricKey(
			id,
			meth,
			uint32(secondsPerPoint),
		)
	}
}

func encodedChunksFromPoints(points []whisper.Point, intervalIn, chunkSpan uint32) []chunk.IterGen {
	var point whisper.Point
	var t0, prevT0 uint32
	var c *chunk.Chunk
	var encodedChunks []chunk.IterGen

	for _, point = range points {
		// this shouldn't happen, but if it would we better catch it here because Metrictank wouldn't handle it well:
		// https://github.com/grafana/metrictank/blob/f1868cccfb92fc82cd853914af958f6d187c5f74/mdata/aggmetric.go#L378
		if point.Timestamp == 0 {
			continue
		}

		t0 = point.Timestamp - (point.Timestamp % chunkSpan)
		if prevT0 == 0 {
			c = chunk.New(t0)
			prevT0 = t0
		} else if prevT0 != t0 {
			c.Finish()

			encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))

			c = chunk.New(t0)
			prevT0 = t0
		}

		err := c.Push(point.Timestamp, point.Value)
		if err != nil {
			panic(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %q", t0, err))
		}
	}

	// if the last written point was also the last one of the current chunk,
	// or if writeUnfinishedChunks is on, we close the chunk and push it
	if point.Timestamp == t0+chunkSpan-intervalIn || *writeUnfinishedChunks {
		c.Finish()
		encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))
	}

	return encodedChunks
}

// scan a directory and feed the list of whisper files relative to base into the given channel
func getFileListIntoChan(fileChan chan string) {
	filepath.Walk(
		*whisperDirectory,
		func(path string, info os.FileInfo, err error) error {
			if path == *whisperDirectory {
				return nil
			}
			name := getMetricName(path)
			if !nameFilter.Match([]byte(getMetricName(name))) {
				log.Debugf("Skipping file %s with name %s", path, name)
				atomic.AddUint32(&skippedCount, 1)
				return nil
			}
			if len(path) >= 4 && path[len(path)-4:] == ".wsp" {
				fileChan <- path
			}
			return nil
		},
	)

	close(fileChan)
}
<|endoftext|>