<commit_before>/*
Package supervisor provides supervisor trees for Go applications.

This package is a clean reimplementation of github.com/thejerf/suture, aiming
to be more Go idiomatic, thus less Erlang-like.

It is built on top of context package, with all of its advantages, namely the
possibility trickle down context-related values and cancellation signals.

TheJerf's blog post about Suture is a very good and helpful read to understand
how this package has been implemented.

http://www.jerf.org/iri/post/2930
*/
package supervisor // import "cirello.io/supervisor"

import (
	"fmt"
	"log"
	"math"
	"sync"
	"time"

	"golang.org/x/net/context"
)

// Service is the public interface expected by a Supervisor.
//
// This will be internally named after the result of fmt.Stringer, if available.
// Otherwise it will going to use an internal representation for the service
// name.
type Service interface {
	// Serve is called by a Supervisor to start the service. It expects the
	// service to honor the passed context and its lifetime. Observe
	// <-ctx.Done() and ctx.Err(). If the service is stopped by anything
	// but the Supervisor, it will get started again. Be careful with shared
	// state among restarts.
	Serve(ctx context.Context)
}

// Supervisor is the basic datastructure responsible for offering a supervisor
// tree. It implements Service, therefore it can be nested if necessary. When
// passing the Supervisor around, remind to do it as reference (&supervisor).
type Supervisor struct {
	// Name for this supervisor tree, used for logging.
	Name string

	// FailureDecay is the timespan on which the current failure count will
	// be halved.
	FailureDecay float64

	// FailureThreshold is the maximum accepted number of failures, after
	// decay adjustment, that shall trigger the back-off wait.
	FailureThreshold float64

	// Backoff is the wait duration when hit threshold.
	Backoff time.Duration

	// Log is a replaceable function used for overall logging
	Log func(string)

	ready sync.Once

	addedService    chan struct{}
	startedServices chan struct{}

	runningServices sync.WaitGroup

	servicesMu sync.Mutex
	services   map[string]Service

	cancelationsMu sync.Mutex
	cancelations   map[string]context.CancelFunc

	backoffMu sync.Mutex
	backoff   map[string]*backoff

	singleflight singleflight
}

func (s *Supervisor) String() string {
	return s.Name
}

func (s *Supervisor) prepare() {
	s.ready.Do(func() {
		if s.Name == "" {
			s.Name = "supervisor"
		}
		s.addedService = make(chan struct{}, 1)
		s.backoff = make(map[string]*backoff)
		s.cancelations = make(map[string]context.CancelFunc)
		s.services = make(map[string]Service)
		s.startedServices = make(chan struct{}, 1)

		if s.Log == nil {
			s.Log = func(str string) {
				log.Println(s.Name, ":", str)
			}
		}
		if s.FailureDecay == 0 {
			s.FailureDecay = 30
		}
		if s.FailureThreshold == 0 {
			s.FailureThreshold = 5
		}
		if s.Backoff == 0 {
			s.Backoff = 15 * time.Second
		}
	})
}

// Add inserts into the Supervisor tree a new service. If the Supervisor is
// already started, it will start it automatically.
func (s *Supervisor) Add(service Service) {
	s.prepare()

	name := fmt.Sprintf("%s", service)

	s.servicesMu.Lock()
	s.backoffMu.Lock()
	s.backoff[name] = &backoff{}
	s.services[name] = service
	s.backoffMu.Unlock()
	s.servicesMu.Unlock()

	select {
	case s.addedService <- struct{}{}:
	default:
	}
}

// Remove stops the service in the Supervisor tree and remove from it.
func (s *Supervisor) Remove(name string) {
	s.prepare()

	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()
	if _, ok := s.services[name]; !ok {
		return
	}

	s.cancelationsMu.Lock()
	defer s.cancelationsMu.Unlock()
	if c, ok := s.cancelations[name]; ok {
		delete(s.cancelations, name)
		c()
	}
}

// Serve starts the Supervisor tree. It can be started only once at a time. If
// stopped (canceled), it can be restarted. It will discard context of
// successive calls to supervisor when still running, and hold them until
// original call is complete.
func (s *Supervisor) Serve(ctx context.Context) {
	s.prepare()

	s.singleflight.do(func() {
		s.serve(ctx)
	})
}

// Services return a list of services
func (s *Supervisor) Services() map[string]Service {
	svclist := make(map[string]Service)
	s.servicesMu.Lock()
	for k, v := range s.services {
		svclist[k] = v
	}
	s.servicesMu.Unlock()
	return svclist
}

// Cancelations return a list of services names and their cancellation calls
func (s *Supervisor) Cancelations() map[string]context.CancelFunc {
	svclist := make(map[string]context.CancelFunc)
	s.cancelationsMu.Lock()
	for k, v := range s.cancelations {
		svclist[k] = v
	}
	s.cancelationsMu.Unlock()
	return svclist
}

func (s *Supervisor) serve(ctx context.Context) {
	select {
	case s.addedService <- struct{}{}:
	default:
	}
	go func(ctx context.Context) {
		for {
			select {
			case <-s.addedService:
				s.startServices(ctx)
				select {
				case s.startedServices <- struct{}{}:
				default:
				}

			case <-ctx.Done():
				return
			}
		}
	}(ctx)
	<-ctx.Done()

	s.runningServices.Wait()

	s.cancelationsMu.Lock()
	s.cancelations = make(map[string]context.CancelFunc)
	s.cancelationsMu.Unlock()
	return
}

func (s *Supervisor) startServices(ctx context.Context) {
	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()

	var wg sync.WaitGroup

	for name, svc := range s.services {
		s.cancelationsMu.Lock()
		_, ok := s.cancelations[name]
		if ok {
			s.cancelationsMu.Unlock()
			continue
		}

		wg.Add(1)
		s.runningServices.Add(1)
		go func(name string, svc Service) {
			wg.Done()
			for {
				retry := func() (retry bool) {
					select {
					case <-ctx.Done():
						return false
					default:
					}

					defer func() {
						if r := recover(); r != nil {
							s.Log(fmt.Sprint("trapped panic:", r))
							retry = true
						}
					}()

					c, cancel := context.WithCancel(ctx)
					s.cancelationsMu.Lock()
					s.cancelations[name] = cancel
					s.cancelationsMu.Unlock()
					svc.Serve(c)

					select {
					case <-ctx.Done():
						return false
					default:
						return true
					}
				}()
				if retry {
					s.Log(fmt.Sprintf("restarting %s", name))
					s.backoffMu.Lock()
					b := s.backoff[name]
					s.backoffMu.Unlock()
					b.wait(s.FailureDecay, s.FailureThreshold, s.Backoff)
					continue
				}
				break
			}
			s.runningServices.Done()
		}(name, svc)
		s.cancelationsMu.Unlock()
	}

	wg.Wait()
}

type backoff struct {
	lastfail time.Time
	failures float64
}

func (b *backoff) wait(failureDecay float64, threshold float64, backoffDur time.Duration) {
	if b.lastfail.IsZero() {
		b.lastfail = time.Now()
		b.failures = 1.0
	} else {
		b.failures++
		intervals := time.Since(b.lastfail).Seconds() / failureDecay
		b.failures = b.failures*math.Pow(.5, intervals) + 1
	}

	if b.failures > threshold {
		time.Sleep(backoffDur)
	}
}

// Based from groupcache's singleflight
type singleflight struct {
	wg     sync.WaitGroup
	mu     sync.Mutex
	flying bool
}

func (g *singleflight) do(fn func()) {
	g.mu.Lock()
	if g.flying {
		g.mu.Unlock()
		g.wg.Wait()
		return
	}

	g.wg.Add(1)
	g.flying = true
	g.mu.Unlock()

	fn()
	g.wg.Done()

	g.mu.Lock()
	g.flying = false
	g.mu.Unlock()

	return
}
<commit_msg>do not hold concurrent calls to the supervisor<commit_after>/*
Package supervisor provides supervisor trees for Go applications.

This package is a clean reimplementation of github.com/thejerf/suture, aiming
to be more Go idiomatic, thus less Erlang-like.

It is built on top of context package, with all of its advantages, namely the
possibility trickle down context-related values and cancellation signals.

TheJerf's blog post about Suture is a very good and helpful read to understand
how this package has been implemented.

http://www.jerf.org/iri/post/2930
*/
package supervisor // import "cirello.io/supervisor"

import (
	"fmt"
	"log"
	"math"
	"sync"
	"time"

	"golang.org/x/net/context"
)

// Service is the public interface expected by a Supervisor.
//
// This will be internally named after the result of fmt.Stringer, if available.
// Otherwise it will going to use an internal representation for the service
// name.
type Service interface {
	// Serve is called by a Supervisor to start the service. It expects the
	// service to honor the passed context and its lifetime. Observe
	// <-ctx.Done() and ctx.Err(). If the service is stopped by anything
	// but the Supervisor, it will get started again. Be careful with shared
	// state among restarts.
	Serve(ctx context.Context)
}

// Supervisor is the basic datastructure responsible for offering a supervisor
// tree. It implements Service, therefore it can be nested if necessary. When
// passing the Supervisor around, remind to do it as reference (&supervisor).
type Supervisor struct {
	// Name for this supervisor tree, used for logging.
	Name string

	// FailureDecay is the timespan on which the current failure count will
	// be halved.
	FailureDecay float64

	// FailureThreshold is the maximum accepted number of failures, after
	// decay adjustment, that shall trigger the back-off wait.
	FailureThreshold float64

	// Backoff is the wait duration when hit threshold.
	Backoff time.Duration

	// Log is a replaceable function used for overall logging
	Log func(string)

	ready sync.Once

	addedService    chan struct{}
	startedServices chan struct{}

	runningServices sync.WaitGroup

	servicesMu sync.Mutex
	services   map[string]Service

	cancelationsMu sync.Mutex
	cancelations   map[string]context.CancelFunc

	backoffMu sync.Mutex
	backoff   map[string]*backoff

	singleflight singleflight
}

func (s *Supervisor) String() string {
	return s.Name
}

func (s *Supervisor) prepare() {
	s.ready.Do(func() {
		if s.Name == "" {
			s.Name = "supervisor"
		}
		s.addedService = make(chan struct{}, 1)
		s.backoff = make(map[string]*backoff)
		s.cancelations = make(map[string]context.CancelFunc)
		s.services = make(map[string]Service)
		s.startedServices = make(chan struct{}, 1)

		if s.Log == nil {
			s.Log = func(str string) {
				log.Println(s.Name, ":", str)
			}
		}
		if s.FailureDecay == 0 {
			s.FailureDecay = 30
		}
		if s.FailureThreshold == 0 {
			s.FailureThreshold = 5
		}
		if s.Backoff == 0 {
			s.Backoff = 15 * time.Second
		}
	})
}

// Add inserts into the Supervisor tree a new service. If the Supervisor is
// already started, it will start it automatically.
func (s *Supervisor) Add(service Service) {
	s.prepare()

	name := fmt.Sprintf("%s", service)

	s.servicesMu.Lock()
	s.backoffMu.Lock()
	s.backoff[name] = &backoff{}
	s.services[name] = service
	s.backoffMu.Unlock()
	s.servicesMu.Unlock()

	select {
	case s.addedService <- struct{}{}:
	default:
	}
}

// Remove stops the service in the Supervisor tree and remove from it.
func (s *Supervisor) Remove(name string) {
	s.prepare()

	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()
	if _, ok := s.services[name]; !ok {
		return
	}

	s.cancelationsMu.Lock()
	defer s.cancelationsMu.Unlock()
	if c, ok := s.cancelations[name]; ok {
		delete(s.cancelations, name)
		c()
	}
}

// Serve starts the Supervisor tree. It can be started only once at a time. If
// stopped (canceled), it can be restarted. It will discard context of
// successive calls to supervisor when still running.
func (s *Supervisor) Serve(ctx context.Context) {
	s.prepare()

	s.singleflight.do(func() {
		s.serve(ctx)
	})
}

// Services return a list of services
func (s *Supervisor) Services() map[string]Service {
	svclist := make(map[string]Service)
	s.servicesMu.Lock()
	for k, v := range s.services {
		svclist[k] = v
	}
	s.servicesMu.Unlock()
	return svclist
}

// Cancelations return a list of services names and their cancellation calls
func (s *Supervisor) Cancelations() map[string]context.CancelFunc {
	svclist := make(map[string]context.CancelFunc)
	s.cancelationsMu.Lock()
	for k, v := range s.cancelations {
		svclist[k] = v
	}
	s.cancelationsMu.Unlock()
	return svclist
}

func (s *Supervisor) serve(ctx context.Context) {
	select {
	case s.addedService <- struct{}{}:
	default:
	}
	go func(ctx context.Context) {
		for {
			select {
			case <-s.addedService:
				s.startServices(ctx)
				select {
				case s.startedServices <- struct{}{}:
				default:
				}

			case <-ctx.Done():
				return
			}
		}
	}(ctx)
	<-ctx.Done()

	s.runningServices.Wait()

	s.cancelationsMu.Lock()
	s.cancelations = make(map[string]context.CancelFunc)
	s.cancelationsMu.Unlock()
	return
}

func (s *Supervisor) startServices(ctx context.Context) {
	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()

	var wg sync.WaitGroup

	for name, svc := range s.services {
		s.cancelationsMu.Lock()
		_, ok := s.cancelations[name]
		if ok {
			s.cancelationsMu.Unlock()
			continue
		}

		wg.Add(1)
		s.runningServices.Add(1)
		go func(name string, svc Service) {
			wg.Done()
			for {
				retry := func() (retry bool) {
					select {
					case <-ctx.Done():
						return false
					default:
					}

					defer func() {
						if r := recover(); r != nil {
							s.Log(fmt.Sprint("trapped panic:", r))
							retry = true
						}
					}()

					c, cancel := context.WithCancel(ctx)
					s.cancelationsMu.Lock()
					s.cancelations[name] = cancel
					s.cancelationsMu.Unlock()
					svc.Serve(c)

					select {
					case <-ctx.Done():
						return false
					default:
						return true
					}
				}()
				if retry {
					s.Log(fmt.Sprintf("restarting %s", name))
					s.backoffMu.Lock()
					b := s.backoff[name]
					s.backoffMu.Unlock()
					b.wait(s.FailureDecay, s.FailureThreshold, s.Backoff)
					continue
				}
				break
			}
			s.runningServices.Done()
		}(name, svc)
		s.cancelationsMu.Unlock()
	}

	wg.Wait()
}

type backoff struct {
	lastfail time.Time
	failures float64
}

func (b *backoff) wait(failureDecay float64, threshold float64, backoffDur time.Duration) {
	if b.lastfail.IsZero() {
		b.lastfail = time.Now()
		b.failures = 1.0
	} else {
		b.failures++
		intervals := time.Since(b.lastfail).Seconds() / failureDecay
		b.failures = b.failures*math.Pow(.5, intervals) + 1
	}

	if b.failures > threshold {
		time.Sleep(backoffDur)
	}
}

// Based from groupcache's singleflight
type singleflight struct {
	mu     sync.Mutex
	flying bool
}

func (g *singleflight) do(fn func()) {
	g.mu.Lock()
	if g.flying {
		g.mu.Unlock()
		return
	}

	g.flying = true
	g.mu.Unlock()

	fn()

	g.mu.Lock()
	g.flying = false
	g.mu.Unlock()

	return
}
<|endoftext|>


<commit_before>package graphs

import (
	"testing"
)

func TestTopologicalSort_SimpleGraph(t *testing.T) {
	graph := NewDigraph()

	graph.AddEdge(1, 3, 0)
	graph.AddEdge(1, 2, 0)
	topOrder, topClasses, noDAGError := TopologicalSort(graph)
	if noDAGError != nil {
		t.Error("Error was thrown: %v", noDAGError)
	}
	if topOrder.Len() != 3 {
		t.Log("Topological order should have 3 items")
		t.Error("Topological order should have 3 items")
	}

	// check if the topological class assigned to each node is correct
	if len(topClasses) != 3 {
		t.Log("We don't have information about each node.")
		t.Error("We don't have information about each node.")
	}
	if topClasses[1] != 0 {
		t.Error("Node 1 has wrong class")
	}
	if topClasses[2] != 1 {
		t.Error("Node 2 has wrong class")
	}
	if topClasses[3] != 1 {
		t.Error("Node 3 has wrong class")
	}

	// check if the topological ordering is correct	e := topOrder.Front()
	e := topOrder.Front()
	if e.Value.(int) != 1 {
		t.Error("First node shuld be: 1")
	}
	e = e.Next()
	if e.Value.(int) != 2 && e.Value.(int) != 3 {
		t.Error("Second node shuld be: 2 or 3")
	}
	e = e.Next()
	if e.Value.(int) != 2 && e.Value.(int) != 3 {
		t.Error("Third node shuld be: 2 or 3")
	}
}

func TestTopologicalSort_Case2(t *testing.T) {
	graph := NewDigraph()

	graph.AddEdge(1, 2, 0)
	graph.AddEdge(1, 3, 0)
	graph.AddEdge(2, 4, 0)
	graph.AddEdge(3, 4, 0)
	topOrder, topClasses, noDAGError := TopologicalSort(graph)
	if noDAGError != nil {
		t.Error("Error was thrown: %v", noDAGError)
	}
	if topOrder.Len() != 4 {
		t.Log("Topological order should have 4 items")
		t.Error("Topological order should have 4 items")
	}
	// check if the topological class assigned to each node is correct
	if len(topClasses) != 4 {
		t.Log("We don't have information about each node.")
		t.Error("We don't have information about each node.")
	}
	if topClasses[1] != 0 {
		t.Error("Node 1 has wrong class")
	}
	if topClasses[2] != 1 {
		t.Error("Node 2 has wrong class")
	}
	if topClasses[3] != 1 {
		t.Error("Node 3 has wrong class")
	}
	if topClasses[4] != 2 {
		t.Error("Node 4 has wrong class")
	}

	// check if the topological ordering is correct
	e := topOrder.Front()
	if e.Value.(int) != 1 {
		t.Error("First node should be: 1")
	}
	e = e.Next()
	if e.Value.(int) != 2 && e.Value.(int) != 3 {
		t.Error("Second node should be: 2 or 3")
	}
	e = e.Next()
	if e.Value.(int) != 2 && e.Value.(int) != 3 {
		t.Error("Third node should be: 2 or 3")
	}
	e = e.Next()
	if e.Value.(int) != 4 {
		t.Error("Fourth node should be: 4")
	}
}

func TestTopologicalSort_NotDAG(t *testing.T) {
	graph := NewDigraph()

	graph.AddEdge(1, 2, 0)
	graph.AddEdge(1, 3, 0)
	graph.AddEdge(2, 1, 0)
	_, _, noDagError := TopologicalSort(graph)
	if noDagError != ErrNoDAG {
		t.Error("This graph is not a DAG. TopologicalSort should have returned an Error.")
	}
}
<commit_msg>Fix go vet issues<commit_after>package graphs

import (
	"testing"
)

func TestTopologicalSort_SimpleGraph(t *testing.T) {
	graph := NewDigraph()

	graph.AddEdge(1, 3, 0)
	graph.AddEdge(1, 2, 0)
	topOrder, topClasses, noDAGError := TopologicalSort(graph)
	if noDAGError != nil {
		t.Errorf("Error was thrown: %v", noDAGError)
	}
	if topOrder.Len() != 3 {
		t.Log("Topological order should have 3 items")
		t.Error("Topological order should have 3 items")
	}

	// check if the topological class assigned to each node is correct
	if len(topClasses) != 3 {
		t.Log("We don't have information about each node.")
		t.Error("We don't have information about each node.")
	}
	if topClasses[1] != 0 {
		t.Error("Node 1 has wrong class")
	}
	if topClasses[2] != 1 {
		t.Error("Node 2 has wrong class")
	}
	if topClasses[3] != 1 {
		t.Error("Node 3 has wrong class")
	}

	// check if the topological ordering is correct	e := topOrder.Front()
	e := topOrder.Front()
	if e.Value.(int) != 1 {
		t.Error("First node shuld be: 1")
	}
	e = e.Next()
	if e.Value.(int) != 2 && e.Value.(int) != 3 {
		t.Error("Second node shuld be: 2 or 3")
	}
	e = e.Next()
	if e.Value.(int) != 2 && e.Value.(int) != 3 {
		t.Error("Third node shuld be: 2 or 3")
	}
}

func TestTopologicalSort_Case2(t *testing.T) {
	graph := NewDigraph()

	graph.AddEdge(1, 2, 0)
	graph.AddEdge(1, 3, 0)
	graph.AddEdge(2, 4, 0)
	graph.AddEdge(3, 4, 0)
	topOrder, topClasses, noDAGError := TopologicalSort(graph)
	if noDAGError != nil {
		t.Errorf("Error was thrown: %v", noDAGError)
	}
	if topOrder.Len() != 4 {
		t.Log("Topological order should have 4 items")
		t.Error("Topological order should have 4 items")
	}
	// check if the topological class assigned to each node is correct
	if len(topClasses) != 4 {
		t.Log("We don't have information about each node.")
		t.Error("We don't have information about each node.")
	}
	if topClasses[1] != 0 {
		t.Error("Node 1 has wrong class")
	}
	if topClasses[2] != 1 {
		t.Error("Node 2 has wrong class")
	}
	if topClasses[3] != 1 {
		t.Error("Node 3 has wrong class")
	}
	if topClasses[4] != 2 {
		t.Error("Node 4 has wrong class")
	}

	// check if the topological ordering is correct
	e := topOrder.Front()
	if e.Value.(int) != 1 {
		t.Error("First node should be: 1")
	}
	e = e.Next()
	if e.Value.(int) != 2 && e.Value.(int) != 3 {
		t.Error("Second node should be: 2 or 3")
	}
	e = e.Next()
	if e.Value.(int) != 2 && e.Value.(int) != 3 {
		t.Error("Third node should be: 2 or 3")
	}
	e = e.Next()
	if e.Value.(int) != 4 {
		t.Error("Fourth node should be: 4")
	}
}

func TestTopologicalSort_NotDAG(t *testing.T) {
	graph := NewDigraph()

	graph.AddEdge(1, 2, 0)
	graph.AddEdge(1, 3, 0)
	graph.AddEdge(2, 1, 0)
	_, _, noDagError := TopologicalSort(graph)
	if noDagError != ErrNoDAG {
		t.Error("This graph is not a DAG. TopologicalSort should have returned an Error.")
	}
}
<|endoftext|>


<commit_before>/*
   Copyright (c) 2014, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package mysql_test

import (
	"database/sql"
	"os"
	"testing"
	"time"

	_ "github.com/arnehormann/mysql"
	"github.com/percona/cloud-protocol/proto"
	"github.com/percona/percona-agent/mm"
	"github.com/percona/percona-agent/mm/mysql"
	mysqlConn "github.com/percona/percona-agent/mysql"
	"github.com/percona/percona-agent/pct"
	"github.com/percona/percona-agent/test"
	"github.com/percona/percona-agent/test/mock"
	. "gopkg.in/check.v1"
)

/**
 * This must be set, else all tests will fail.
 */
var dsn = os.Getenv("PCT_TEST_MYSQL_DSN")

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { TestingT(t) }

type TestSuite struct {
	db             *sql.DB
	logChan        chan *proto.LogEntry
	logger         *pct.Logger
	tickChan       chan time.Time
	collectionChan chan *mm.Collection
	name           string
}

var _ = Suite(&TestSuite{})

func (s *TestSuite) SetUpSuite(t *C) {
	if dsn == "" {
		t.Fatal("PCT_TEST_MYSQL_DSN is not set")
	}

	// Get our own connection to MySQL.
	db, err := test.ConnectMySQL(dsn)
	if err != nil {
		t.Fatal(err)
	}
	s.db = db

	s.logChan = make(chan *proto.LogEntry, 10)
	s.logger = pct.NewLogger(s.logChan, "mm-manager-test")
	s.tickChan = make(chan time.Time)
	s.collectionChan = make(chan *mm.Collection, 1)
	s.name = "mm-mysql-db1"
}

func (s *TestSuite) TearDownSuite(t *C) {
	if s.db != nil {
		s.db.Close()
	}
}

// --------------------------------------------------------------------------

func (s *TestSuite) TestStartCollectStop(t *C) {
	/**
	 * The mm manager uses a mm monitor factory to create monitors.  This is
	 * what the factory does...
	 */

	// First, monitors monitor an instance of some service (MySQL, RabbitMQ, etc.)
	// So the instance name and id are given.  Second, every monitor has its own
	// specific config info which is sent as the proto.Cmd.Data.  This config
	// embed a mm.Config which embed an instance.Config:
	config := &mysql.Config{
		Config: mm.Config{
			ServiceInstance: proto.ServiceInstance{
				Service:    "mysql",
				InstanceId: 1,
			},
			Collect: 1,
			Report:  60,
		},
		Status: map[string]string{
			"threads_connected": "gauge",
			"threads_running":   "gauge",
		},
	}

	// From the config, the factory determine's the monitor's name based on
	// the service instance it's monitoring, and it creates a mysql.Connector
	// for the DSN for that service (since it's a MySQL monitor in this case).
	// It creates the monitor with these args:
	m := mysql.NewMonitor(s.name, config, s.logger, mysqlConn.NewConnection(dsn))
	if m == nil {
		t.Fatal("Make new mysql.Monitor")
	}

	// The factory returns the monitor to the manager which starts it with
	// the necessary channels:
	err := m.Start(s.tickChan, s.collectionChan)
	if err != nil {
		t.Fatalf("Start monitor without error, got %s", err)
	}

	// monitor=Ready once it has successfully connected to MySQL.  This may
	// take a few seconds (hopefully < 5) on a slow test machine.
	if ok := test.WaitStatus(5, m, s.name+"-mysql", "Connected"); !ok {
		t.Fatal("Monitor is ready")
	}

	// The monitor should only collect and send metrics on ticks; we haven't ticked yet.
	got := test.WaitCollection(s.collectionChan, 0)
	if len(got) > 0 {
		t.Fatal("No tick, no collection; got %+v", got)
	}

	// Now tick.  This should make monitor collect.
	now := time.Now()
	s.tickChan <- now
	got = test.WaitCollection(s.collectionChan, 1)
	if len(got) == 0 {
		t.Fatal("Got a collection after tick")
	}
	c := got[0]

	if c.Ts != now.Unix() {
		t.Error("Collection.Ts set to %s; got %s", now.Unix(), c.Ts)
	}

	// Only two metrics should be reported, from the config ^: threads_connected,
	// threads_running.  Their values (from MySQL) are variable, but we know they
	// should be > 1 because we're a thread connected and running.
	if len(c.Metrics) != 2 {
		t.Fatal("Collected only configured metrics; got %+v", c.Metrics)
	}
	if c.Metrics[0].Name != "mysql/threads_connected" {
		t.Error("First metric is ", "mysql/threads_connected; got", c.Metrics[0].Name)
	}
	if c.Metrics[0].Number < 1 {
		t.Error("mysql/threads_connected > 1; got", c.Metrics[0].Number)
	}
	if c.Metrics[1].Name != "mysql/threads_running" {
		t.Error("Second metric is ", "mysql/threads_running got", c.Metrics[1].Name)
	}
	if c.Metrics[1].Number < 1 {
		t.Error("mysql/threads_running > 1; got", c.Metrics[0].Number)
	}

	/**
	 * Stop the monitor.
	 */

	m.Stop()

	if ok := test.WaitStatus(5, m, s.name, "Stopped"); !ok {
		t.Fatal("Monitor has stopped")
	}
}

func (s *TestSuite) TestCollectInnoDBStats(t *C) {
	/**
	 * Disable and reset InnoDB metrics so we can test that the monitor enables and sets them.
	 */
	if _, err := s.db.Exec("set global innodb_monitor_disable = '%'"); err != nil {
		t.Fatal(err)
	}
	if _, err := s.db.Exec("set global innodb_monitor_reset_all = '%'"); err != nil {
		t.Fatal(err)
	}

	s.db.Exec("drop database if exists test_pct")
	s.db.Exec("create database test_pct")
	s.db.Exec("create table test_pct.t (i int) engine=innodb")
	defer s.db.Exec("drop database if exists test_pct")

	config := &mysql.Config{
		Config: mm.Config{
			ServiceInstance: proto.ServiceInstance{
				Service:    "mysql",
				InstanceId: 1,
			},
			Collect: 1,
			Report:  60,
		},
		Status: map[string]string{},
		InnoDB: []string{"dml_%"}, // same as above ^
	}

	// Start a monitor with InnoDB metrics.
	// See TestStartCollectStop() for description of these steps.
	cc := mock.NewSlowMySQL(dsn)
	cc.SetGlobalDelay(0 * time.Second)

	m := mysql.NewMonitor(s.name, config, s.logger, mysqlConn.NewConnection(dsn))
	if m == nil {
		t.Fatal("Make new mysql.Monitor")
	}

	err := m.Start(s.tickChan, s.collectionChan)
	if err != nil {
		t.Fatalf("Start monitor without error, got %s", err)
	}

	if ok := test.WaitStatus(5, m, s.name+"-mysql", "Connected"); !ok {
		t.Fatal("Monitor is ready")
	}

	// Do INSERT to increment dml_inserts before monitor collects.  If it enabled
	// the InnoDB metrics and collects them, we should get dml_inserts=1 this later..
	s.db.Exec("insert into test_pct.t (i) values (42)")

	s.tickChan <- time.Now()
	got := test.WaitCollection(s.collectionChan, 1)
	if len(got) == 0 {
		t.Fatal("Got a collection after tick")
	}
	c := got[0]

	/**
	 * ...monitor should have collected the InnoDB metrics:
	 *
	 * mysql> SELECT NAME, SUBSYSTEM, COUNT, TYPE FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE STATUS='enabled';
	 * +-------------+-----------+-------+----------------+
	 * | NAME        | SUBSYSTEM | COUNT | TYPE           |
	 * +-------------+-----------+-------+----------------+
	 * | dml_reads   | dml       |     0 | status_counter |
	 * | dml_inserts | dml       |     1 | status_counter |
	 * | dml_deletes | dml       |     0 | status_counter |
	 * | dml_updates | dml       |     0 | status_counter |
	 * +-------------+-----------+-------+----------------+
	 */
	if len(c.Metrics) != 4 {
		t.Fatal("Collect 4 InnoDB metrics; got %+v", c.Metrics)
	}
	expect := []mm.Metric{
		{Name: "mysql/innodb/dml/dml_reads", Type: "counter", Number: 0},
		{Name: "mysql/innodb/dml/dml_inserts", Type: "counter", Number: 1}, // <-- our INSERT
		{Name: "mysql/innodb/dml/dml_deletes", Type: "counter", Number: 0},
		{Name: "mysql/innodb/dml/dml_updates", Type: "counter", Number: 0},
	}
	if ok, diff := test.IsDeeply(c.Metrics, expect); !ok {
		t.Error(diff)
	}

	// Stop montior, clean up.
	m.Stop()
}

func (s *TestSuite) TestCollectUserstats(t *C) {
	/**
	 * Disable and reset user stats.
	 */
	if _, err := s.db.Exec("set global userstat = off"); err != nil {
		t.Fatal(err)
	}
	if _, err := s.db.Exec("flush user_statistics"); err != nil {
		t.Fatal(err)
	}
	if _, err := s.db.Exec("flush index_statistics"); err != nil {
		t.Fatal(err)
	}

	config := &mysql.Config{
		Config: mm.Config{
			ServiceInstance: proto.ServiceInstance{
				Service:    "mysql",
				InstanceId: 1,
			},
			Collect: 1,
			Report:  60,
		},
		UserStats: true,
	}

	m := mysql.NewMonitor(s.name, config, s.logger, mysqlConn.NewConnection(dsn))
	if m == nil {
		t.Fatal("Make new mysql.Monitor")
	}

	err := m.Start(s.tickChan, s.collectionChan)
	if err != nil {
		t.Fatalf("Start monitor without error, got %s", err)
	}

	if ok := test.WaitStatus(5, m, s.name+"-mysql", "Connected"); !ok {
		t.Fatal("Monitor is ready")
	}

	var user, host string
	err = s.db.QueryRow("SELECT SUBSTRING_INDEX(CURRENT_USER(),'@',1) AS 'user', SUBSTRING_INDEX(CURRENT_USER(),'@',-1) AS host").Scan(&user, &host)
	if err != nil {
		t.Fatal(err)
	}

	// To get index stats, we need to use an index: mysq.user PK <host, user>
	rows, err := s.db.Query("select * from mysql.user where host=? and user=?", host, user)
	if err != nil {
		t.Fatal(err)
	}
	defer rows.Close()

	s.tickChan <- time.Now()
	got := test.WaitCollection(s.collectionChan, 1)
	if len(got) == 0 {
		t.Fatal("Got a collection after tick")
	}
	c := got[0]

	/**
	 * Monitor should have collected the user stats: just table and index.
	 * Values vary a little, but there should be a table metric for mysql.user
	 * because login uses this table.
	 */
	if len(c.Metrics) < 1 {
		t.Fatalf("Collect at least 1 user stat metric; got %+v", c.Metrics)
	}

	var tblStat mm.Metric
	var idxStat mm.Metric
	for _, m := range c.Metrics {
		switch m.Name {
		case "mysql/db.mysql/t.user/rows_read":
			tblStat = m
		case "mysql/db.mysql/t.user/idx.PRIMARY/rows_read":
			idxStat = m
		}
	}

	// At least 2 rows should have been read from mysql.user:
	//   1: our db connection
	//   2: the monitor's db connection
	if tblStat.Number < 2 {
		t.Errorf("mysql/db.mysql/t.user/rows_read >= 2, got %+v", tblStat)
	}

	// At least 1 index read on mysql.user PK due to our SELECT ^.
	if idxStat.Number < 1 {
		t.Errorf("mysql/db.mysql/t.user/idx.PRIMARY/rows_read >= 1, got %+v", idxStat)
	}

	// Stop montior, clean up.
	m.Stop()
}

func (s *TestSuite) TestSlowResponse(t *C) {

	config := &mysql.Config{
		Config: mm.Config{
			ServiceInstance: proto.ServiceInstance{
				Service:    "mysql",
				InstanceId: 1,
			},
			Collect: 1,
			Report:  60,
		},
		UserStats: true,
	}

	slowCon := mock.NewSlowMySQL(dsn)
	slowCon.SetGlobalDelay((config.Collect + 1) * time.Second)
	m := mysql.NewMonitor(s.name, config, s.logger, slowCon)
	if m == nil {
		t.Fatal("Make new mysql.Monitor")
	}

	err := m.Start(s.tickChan, s.collectionChan)
	if err != nil {
		t.Fatalf("Start monitor without error, got %s", err)
	}

	if ok := test.WaitStatus(5, m, s.name+"-mysql", "Connected"); !ok {
		t.Fatal("Monitor is ready")
	}

	s.tickChan <- time.Now()
	got := test.WaitCollection(s.collectionChan, 1)
	// If it took more than 10% of the tik interval, the monitor must
	// discard those metrics -> len(got) == 0
	t.Check(got, HasLen, 0)

	m.Stop()
}
<commit_msg>PCT-565 Fixed error in test<commit_after>/*
   Copyright (c) 2014, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package mysql_test

import (
	"database/sql"
	"os"
	"testing"
	"time"

	_ "github.com/arnehormann/mysql"
	"github.com/percona/cloud-protocol/proto"
	"github.com/percona/percona-agent/mm"
	"github.com/percona/percona-agent/mm/mysql"
	mysqlConn "github.com/percona/percona-agent/mysql"
	"github.com/percona/percona-agent/pct"
	"github.com/percona/percona-agent/test"
	"github.com/percona/percona-agent/test/mock"
	. "gopkg.in/check.v1"
)

/**
 * This must be set, else all tests will fail.
 */
var dsn = os.Getenv("PCT_TEST_MYSQL_DSN")

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { TestingT(t) }

type TestSuite struct {
	db             *sql.DB
	logChan        chan *proto.LogEntry
	logger         *pct.Logger
	tickChan       chan time.Time
	collectionChan chan *mm.Collection
	name           string
}

var _ = Suite(&TestSuite{})

func (s *TestSuite) SetUpSuite(t *C) {
	if dsn == "" {
		t.Fatal("PCT_TEST_MYSQL_DSN is not set")
	}

	// Get our own connection to MySQL.
	db, err := test.ConnectMySQL(dsn)
	if err != nil {
		t.Fatal(err)
	}
	s.db = db

	s.logChan = make(chan *proto.LogEntry, 10)
	s.logger = pct.NewLogger(s.logChan, "mm-manager-test")
	s.tickChan = make(chan time.Time)
	s.collectionChan = make(chan *mm.Collection, 1)
	s.name = "mm-mysql-db1"
}

func (s *TestSuite) TearDownSuite(t *C) {
	if s.db != nil {
		s.db.Close()
	}
}

// --------------------------------------------------------------------------

func (s *TestSuite) TestStartCollectStop(t *C) {
	/**
	 * The mm manager uses a mm monitor factory to create monitors.  This is
	 * what the factory does...
	 */

	// First, monitors monitor an instance of some service (MySQL, RabbitMQ, etc.)
	// So the instance name and id are given.  Second, every monitor has its own
	// specific config info which is sent as the proto.Cmd.Data.  This config
	// embed a mm.Config which embed an instance.Config:
	config := &mysql.Config{
		Config: mm.Config{
			ServiceInstance: proto.ServiceInstance{
				Service:    "mysql",
				InstanceId: 1,
			},
			Collect: 1,
			Report:  60,
		},
		Status: map[string]string{
			"threads_connected": "gauge",
			"threads_running":   "gauge",
		},
	}

	// From the config, the factory determine's the monitor's name based on
	// the service instance it's monitoring, and it creates a mysql.Connector
	// for the DSN for that service (since it's a MySQL monitor in this case).
	// It creates the monitor with these args:
	m := mysql.NewMonitor(s.name, config, s.logger, mysqlConn.NewConnection(dsn))
	if m == nil {
		t.Fatal("Make new mysql.Monitor")
	}

	// The factory returns the monitor to the manager which starts it with
	// the necessary channels:
	err := m.Start(s.tickChan, s.collectionChan)
	if err != nil {
		t.Fatalf("Start monitor without error, got %s", err)
	}

	// monitor=Ready once it has successfully connected to MySQL.  This may
	// take a few seconds (hopefully < 5) on a slow test machine.
	if ok := test.WaitStatus(5, m, s.name+"-mysql", "Connected"); !ok {
		t.Fatal("Monitor is ready")
	}

	// The monitor should only collect and send metrics on ticks; we haven't ticked yet.
	got := test.WaitCollection(s.collectionChan, 0)
	if len(got) > 0 {
		t.Fatal("No tick, no collection; got %+v", got)
	}

	// Now tick.  This should make monitor collect.
	now := time.Now()
	s.tickChan <- now
	got = test.WaitCollection(s.collectionChan, 1)
	if len(got) == 0 {
		t.Fatal("Got a collection after tick")
	}
	c := got[0]

	if c.Ts != now.Unix() {
		t.Error("Collection.Ts set to %s; got %s", now.Unix(), c.Ts)
	}

	// Only two metrics should be reported, from the config ^: threads_connected,
	// threads_running.  Their values (from MySQL) are variable, but we know they
	// should be > 1 because we're a thread connected and running.
	if len(c.Metrics) != 2 {
		t.Fatal("Collected only configured metrics; got %+v", c.Metrics)
	}
	if c.Metrics[0].Name != "mysql/threads_connected" {
		t.Error("First metric is ", "mysql/threads_connected; got", c.Metrics[0].Name)
	}
	if c.Metrics[0].Number < 1 {
		t.Error("mysql/threads_connected > 1; got", c.Metrics[0].Number)
	}
	if c.Metrics[1].Name != "mysql/threads_running" {
		t.Error("Second metric is ", "mysql/threads_running got", c.Metrics[1].Name)
	}
	if c.Metrics[1].Number < 1 {
		t.Error("mysql/threads_running > 1; got", c.Metrics[0].Number)
	}

	/**
	 * Stop the monitor.
	 */

	m.Stop()

	if ok := test.WaitStatus(5, m, s.name, "Stopped"); !ok {
		t.Fatal("Monitor has stopped")
	}
}

func (s *TestSuite) TestCollectInnoDBStats(t *C) {
	/**
	 * Disable and reset InnoDB metrics so we can test that the monitor enables and sets them.
	 */
	if _, err := s.db.Exec("set global innodb_monitor_disable = '%'"); err != nil {
		t.Fatal(err)
	}
	if _, err := s.db.Exec("set global innodb_monitor_reset_all = '%'"); err != nil {
		t.Fatal(err)
	}

	s.db.Exec("drop database if exists test_pct")
	s.db.Exec("create database test_pct")
	s.db.Exec("create table test_pct.t (i int) engine=innodb")
	defer s.db.Exec("drop database if exists test_pct")

	config := &mysql.Config{
		Config: mm.Config{
			ServiceInstance: proto.ServiceInstance{
				Service:    "mysql",
				InstanceId: 1,
			},
			Collect: 1,
			Report:  60,
		},
		Status: map[string]string{},
		InnoDB: []string{"dml_%"}, // same as above ^
	}

	// Start a monitor with InnoDB metrics.
	// See TestStartCollectStop() for description of these steps.
	cc := mock.NewSlowMySQL(dsn)
	cc.SetGlobalDelay(0 * time.Second)

	m := mysql.NewMonitor(s.name, config, s.logger, mysqlConn.NewConnection(dsn))
	if m == nil {
		t.Fatal("Make new mysql.Monitor")
	}

	err := m.Start(s.tickChan, s.collectionChan)
	if err != nil {
		t.Fatalf("Start monitor without error, got %s", err)
	}

	if ok := test.WaitStatus(5, m, s.name+"-mysql", "Connected"); !ok {
		t.Fatal("Monitor is ready")
	}

	// Do INSERT to increment dml_inserts before monitor collects.  If it enabled
	// the InnoDB metrics and collects them, we should get dml_inserts=1 this later..
	s.db.Exec("insert into test_pct.t (i) values (42)")

	s.tickChan <- time.Now()
	got := test.WaitCollection(s.collectionChan, 1)
	if len(got) == 0 {
		t.Fatal("Got a collection after tick")
	}
	c := got[0]

	/**
	 * ...monitor should have collected the InnoDB metrics:
	 *
	 * mysql> SELECT NAME, SUBSYSTEM, COUNT, TYPE FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE STATUS='enabled';
	 * +-------------+-----------+-------+----------------+
	 * | NAME        | SUBSYSTEM | COUNT | TYPE           |
	 * +-------------+-----------+-------+----------------+
	 * | dml_reads   | dml       |     0 | status_counter |
	 * | dml_inserts | dml       |     1 | status_counter |
	 * | dml_deletes | dml       |     0 | status_counter |
	 * | dml_updates | dml       |     0 | status_counter |
	 * +-------------+-----------+-------+----------------+
	 */
	if len(c.Metrics) != 4 {
		t.Fatal("Collect 4 InnoDB metrics; got %+v", c.Metrics)
	}
	expect := []mm.Metric{
		{Name: "mysql/innodb/dml/dml_reads", Type: "counter", Number: 0},
		{Name: "mysql/innodb/dml/dml_inserts", Type: "counter", Number: 1}, // <-- our INSERT
		{Name: "mysql/innodb/dml/dml_deletes", Type: "counter", Number: 0},
		{Name: "mysql/innodb/dml/dml_updates", Type: "counter", Number: 0},
	}
	if ok, diff := test.IsDeeply(c.Metrics, expect); !ok {
		t.Error(diff)
	}

	// Stop montior, clean up.
	m.Stop()
}

func (s *TestSuite) TestCollectUserstats(t *C) {
	/**
	 * Disable and reset user stats.
	 */
	if _, err := s.db.Exec("set global userstat = off"); err != nil {
		t.Fatal(err)
	}
	if _, err := s.db.Exec("flush user_statistics"); err != nil {
		t.Fatal(err)
	}
	if _, err := s.db.Exec("flush index_statistics"); err != nil {
		t.Fatal(err)
	}

	config := &mysql.Config{
		Config: mm.Config{
			ServiceInstance: proto.ServiceInstance{
				Service:    "mysql",
				InstanceId: 1,
			},
			Collect: 1,
			Report:  60,
		},
		UserStats: true,
	}

	m := mysql.NewMonitor(s.name, config, s.logger, mysqlConn.NewConnection(dsn))
	if m == nil {
		t.Fatal("Make new mysql.Monitor")
	}

	err := m.Start(s.tickChan, s.collectionChan)
	if err != nil {
		t.Fatalf("Start monitor without error, got %s", err)
	}

	if ok := test.WaitStatus(5, m, s.name+"-mysql", "Connected"); !ok {
		t.Fatal("Monitor is ready")
	}

	var user, host string
	err = s.db.QueryRow("SELECT SUBSTRING_INDEX(CURRENT_USER(),'@',1) AS 'user', SUBSTRING_INDEX(CURRENT_USER(),'@',-1) AS host").Scan(&user, &host)
	if err != nil {
		t.Fatal(err)
	}

	// To get index stats, we need to use an index: mysq.user PK <host, user>
	rows, err := s.db.Query("select * from mysql.user where host=? and user=?", host, user)
	if err != nil {
		t.Fatal(err)
	}
	defer rows.Close()

	s.tickChan <- time.Now()
	got := test.WaitCollection(s.collectionChan, 1)
	if len(got) == 0 {
		t.Fatal("Got a collection after tick")
	}
	c := got[0]

	/**
	 * Monitor should have collected the user stats: just table and index.
	 * Values vary a little, but there should be a table metric for mysql.user
	 * because login uses this table.
	 */
	if len(c.Metrics) < 1 {
		t.Fatalf("Collect at least 1 user stat metric; got %+v", c.Metrics)
	}

	var tblStat mm.Metric
	var idxStat mm.Metric
	for _, m := range c.Metrics {
		switch m.Name {
		case "mysql/db.mysql/t.user/rows_read":
			tblStat = m
		case "mysql/db.mysql/t.user/idx.PRIMARY/rows_read":
			idxStat = m
		}
	}

	// At least 2 rows should have been read from mysql.user:
	//   1: our db connection
	//   2: the monitor's db connection
	if tblStat.Number < 2 {
		t.Errorf("mysql/db.mysql/t.user/rows_read >= 2, got %+v", tblStat)
	}

	// At least 1 index read on mysql.user PK due to our SELECT ^.
	if idxStat.Number < 1 {
		t.Errorf("mysql/db.mysql/t.user/idx.PRIMARY/rows_read >= 1, got %+v", idxStat)
	}

	// Stop montior, clean up.
	m.Stop()
}

func (s *TestSuite) TestSlowResponse(t *C) {

	config := &mysql.Config{
		Config: mm.Config{
			ServiceInstance: proto.ServiceInstance{
				Service:    "mysql",
				InstanceId: 1,
			},
			Collect: 1,
			Report:  60,
		},
		UserStats: true,
	}

	slowCon := mock.NewSlowMySQL(dsn)
	slowCon.SetGlobalDelay(time.Duration(config.Collect+1) * time.Second)
	m := mysql.NewMonitor(s.name, config, s.logger, slowCon)
	if m == nil {
		t.Fatal("Make new mysql.Monitor")
	}

	err := m.Start(s.tickChan, s.collectionChan)
	if err != nil {
		t.Fatalf("Start monitor without error, got %s", err)
	}

	if ok := test.WaitStatus(5, m, s.name+"-mysql", "Connected"); !ok {
		t.Fatal("Monitor is ready")
	}

	s.tickChan <- time.Now()
	got := test.WaitCollection(s.collectionChan, 1)
	// If it took more than 10% of the tik interval, the monitor must
	// discard those metrics -> len(got) == 0
	t.Check(got, HasLen, 0)

	m.Stop()
}
<|endoftext|>


<commit_before>package reference

import (
	"regexp"

	// "opencontainers/go-digest" requires us to load the algorithms that we
	// want to use into the binary (it calls .Available).
	_ "crypto/sha256"

	distreference "github.com/docker/distribution/reference"
	"github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
)

const (
	// XDefaultTag defines the default tag used when performing images related actions and no tag or digest is specified
	XDefaultTag = "latest"
	// XDefaultHostname is the default built-in hostname
	XDefaultHostname = "docker.io"
	// XLegacyDefaultHostname is automatically converted to DefaultHostname
	XLegacyDefaultHostname = "index.docker.io"
	// XDefaultRepoPrefix is the prefix used for default repositories in default host
	XDefaultRepoPrefix = "library/"
)

// XParseNamed parses s and returns a syntactically valid reference implementing
// the Named interface. The reference must have a name, otherwise an error is
// returned.
// If an error was encountered it is returned, along with a nil Reference.
func XParseNamed(s string) (distreference.Named, error) {
	named, err := distreference.ParseNormalizedNamed(s)
	if err != nil {
		return nil, errors.Wrapf(err, "Error parsing reference: %q is not a valid repository/tag", s)
	}
	r, err := distreference.ParseNormalizedNamed(named.Name())
	if err != nil {
		return nil, err
	}
	if canonical, isCanonical := named.(distreference.Canonical); isCanonical {
		return distreference.WithDigest(r, canonical.Digest())
	}
	if tagged, isTagged := named.(distreference.NamedTagged); isTagged {
		return distreference.WithTag(r, tagged.Tag())
	}
	return r, nil
}

// XParseIDOrReference parses string for an image ID or a reference. ID can be
// without a default prefix.
func XParseIDOrReference(idOrRef string) (digest.Digest, distreference.Named, error) {
	if err := validateID(idOrRef); err == nil {
		idOrRef = "sha256:" + idOrRef
	}
	if dgst, err := digest.Parse(idOrRef); err == nil {
		return dgst, nil, nil
	}
	ref, err := XParseNamed(idOrRef)
	return "", ref, err
}

var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)

func validateID(id string) error {
	if ok := validHex.MatchString(id); !ok {
		return errors.Errorf("image ID %q is invalid", id)
	}
	return nil
}
<commit_msg>Do not normalize the input string twice in reference.XParseNamed<commit_after>package reference

import (
	"regexp"

	// "opencontainers/go-digest" requires us to load the algorithms that we
	// want to use into the binary (it calls .Available).
	_ "crypto/sha256"

	distreference "github.com/docker/distribution/reference"
	"github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
)

const (
	// XDefaultTag defines the default tag used when performing images related actions and no tag or digest is specified
	XDefaultTag = "latest"
	// XDefaultHostname is the default built-in hostname
	XDefaultHostname = "docker.io"
	// XLegacyDefaultHostname is automatically converted to DefaultHostname
	XLegacyDefaultHostname = "index.docker.io"
	// XDefaultRepoPrefix is the prefix used for default repositories in default host
	XDefaultRepoPrefix = "library/"
)

// XParseNamed parses s and returns a syntactically valid reference implementing
// the Named interface. The reference must have a name, otherwise an error is
// returned.
// If an error was encountered it is returned, along with a nil Reference.
func XParseNamed(s string) (distreference.Named, error) {
	named, err := distreference.ParseNormalizedNamed(s)
	if err != nil {
		return nil, errors.Wrapf(err, "Error parsing reference: %q is not a valid repository/tag", s)
	}
	r, err := distreference.WithName(named.Name())
	if err != nil {
		return nil, err
	}
	if canonical, isCanonical := named.(distreference.Canonical); isCanonical {
		return distreference.WithDigest(r, canonical.Digest())
	}
	if tagged, isTagged := named.(distreference.NamedTagged); isTagged {
		return distreference.WithTag(r, tagged.Tag())
	}
	return r, nil
}

// XParseIDOrReference parses string for an image ID or a reference. ID can be
// without a default prefix.
func XParseIDOrReference(idOrRef string) (digest.Digest, distreference.Named, error) {
	if err := validateID(idOrRef); err == nil {
		idOrRef = "sha256:" + idOrRef
	}
	if dgst, err := digest.Parse(idOrRef); err == nil {
		return dgst, nil, nil
	}
	ref, err := XParseNamed(idOrRef)
	return "", ref, err
}

var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)

func validateID(id string) error {
	if ok := validHex.MatchString(id); !ok {
		return errors.Errorf("image ID %q is invalid", id)
	}
	return nil
}
<|endoftext|>


<commit_before>// Package twse - Fetch stock data from TWSE, OTC
// 擷取台灣股市上市、上櫃股票資訊
//
package twse

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/toomore/gogrs/utils"
)

type unixMapData map[int64][][]string

// Data start with stock no, date.
type Data struct {
	No          string
	Date        time.Time
	RawData     [][]string
	UnixMapData unixMapData
	exchange    string
	openList    []float64
	priceList   []float64
	rangeList   []float64
	volumeList  []uint64
}

// NewTWSE 建立一個 TWSE 上市股票
func NewTWSE(No string, Date time.Time) *Data {
	return &Data{
		No:       No,
		Date:     Date,
		exchange: "tse",
	}
}

// NewOTC 建立一個 OTC 上櫃股票
func NewOTC(No string, Date time.Time) *Data {
	return &Data{
		No:       No,
		Date:     Date,
		exchange: "otc",
	}
}

// URL return stock csv url path.
func (d Data) URL() string {
	var path string
	var host string

	if d.exchange == "tse" {
		path = fmt.Sprintf(utils.TWSECSV, d.Date.Year(), d.Date.Month(), d.Date.Year(), d.Date.Month(), d.No, utils.RandInt())
		host = utils.TWSEHOST
	} else if d.exchange == "otc" {
		path = fmt.Sprintf(utils.OTCCSV, d.Date.Year()-1911, d.Date.Month(), d.No, utils.RandInt())
		host = utils.OTCHOST
	}

	return fmt.Sprintf("%s%s", host, path)
}

// Round will do sub one month.
func (d *Data) Round() {
	year, month, _ := d.Date.Date()
	d.Date = time.Date(year, month-1, 1, 0, 0, 0, 0, d.Date.Location())
}

// PlusData will do Round() and Get().
func (d *Data) PlusData() {
	d.Round()
	d.Get()
}

func (d *Data) clearCache() {
	d.rangeList = nil
	d.openList = nil
	d.priceList = nil
	d.volumeList = nil
}

// Get return csv data in array.
func (d *Data) Get() ([][]string, error) {
	if d.UnixMapData == nil {
		d.UnixMapData = make(unixMapData)
	}
	if len(d.UnixMapData[d.Date.Unix()]) == 0 {
		csvFiles, err := http.Get(d.URL())
		if err != nil {
			return nil, fmt.Errorf("Network fail: %s", err)
		}
		defer csvFiles.Body.Close()
		data, _ := ioutil.ReadAll(csvFiles.Body)
		csvArrayContent := strings.Split(string(data), "\n")
		for i := range csvArrayContent {
			csvArrayContent[i] = strings.TrimSpace(csvArrayContent[i])
		}
		var csvReader *csv.Reader
		if (d.exchange == "tse" && len(csvArrayContent) > 2) || (d.exchange == "otc" && len(csvArrayContent) > 5) {
			if d.exchange == "tse" {
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[2:], "\n")))
			} else if d.exchange == "otc" {
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[5:len(csvArrayContent)-1], "\n")))
			}
			allData, err := csvReader.ReadAll()
			d.RawData = append(allData, d.RawData...)
			d.UnixMapData[d.Date.Unix()] = allData
			d.clearCache()
			return allData, err
		}
		return nil, errors.New("Not enough data.")
	}
	return d.UnixMapData[d.Date.Unix()], nil
}

// GetByTimeMap return a map by key of time.Time
func (d Data) GetByTimeMap() map[time.Time]interface{} {
	data := make(map[time.Time]interface{})
	dailyData, _ := d.Get()
	for _, v := range dailyData {
		data[utils.ParseDate(v[0])] = v
	}
	return data
}

func (d Data) getColsList(colsNo int) []string {
	var result []string
	result = make([]string, len(d.RawData))
	for i, value := range d.RawData {
		result[i] = value[colsNo]
	}
	return result
}

func (d Data) getColsListFloat64(colsNo int) []float64 {
	var result []float64
	result = make([]float64, len(d.RawData))
	for i, v := range d.getColsList(colsNo) {
		result[i], _ = strconv.ParseFloat(v, 64)
	}
	return result
}

// GetVolumeList 取得 成交股數 序列
func (d *Data) GetVolumeList() []uint64 {
	if d.volumeList == nil {
		var result []uint64
		result = make([]uint64, len(d.RawData))
		for i, v := range d.getColsList(1) {
			result[i], _ = strconv.ParseUint(strings.Replace(v, ",", "", -1), 10, 64)
		}
		d.volumeList = result
	}
	return d.volumeList
}

// GetOpenList 取得 開盤價 序列
func (d *Data) GetOpenList() []float64 {
	if d.openList == nil {
		d.openList = d.getColsListFloat64(3)
	}
	return d.openList
}

// GetPriceList 取得 收盤價 序列
func (d *Data) GetPriceList() []float64 {
	if d.priceList == nil {
		d.priceList = d.getColsListFloat64(6)
	}
	return d.priceList
}

// GetRangeList 取得 漲跌價差 序列
func (d *Data) GetRangeList() []float64 {
	if d.rangeList == nil {
		d.rangeList = d.getColsListFloat64(7)
	}
	return d.rangeList
}

// MA 計算 收盤價 的移動平均
func (d Data) MA(days int) []float64 {
	var result []float64
	var priceList = d.GetPriceList()
	result = make([]float64, len(priceList)-days+1)
	for i := range priceList[days-1:] {
		result[i] = utils.AvgFloat64(priceList[i : i+days])
	}
	return result
}

// MABR 計算 收盤價移動平均 的乖離
func (d Data) MABR(days1, days2 int) []float64 {
	return utils.CalDiffFloat(d.MA(days1), d.MA(days2))
}

// MAV 計算 成交股數 的移動平均
func (d Data) MAV(days int) []uint64 {
	var result []uint64
	var volumeList = d.GetVolumeList()
	result = make([]uint64, len(volumeList)-days+1)
	for i := range volumeList[days-1:] {
		result[i] = utils.AvgUint64(volumeList[i : i+days])
	}
	return result
}

// MAVBR 計算 成交股數移動平均 的乖離
func (d Data) MAVBR(days1, days2 int) []int64 {
	MAV1 := d.MAV(days1)
	MAV2 := d.MAV(days2)
	result := make([]int64, len(MAV2))
	for i := 1; i <= len(MAV2); i++ {
		result[len(MAV2)-i] = int64(MAV1[len(MAV1)-i]) - int64(MAV2[len(MAV2)-i])
	}
	return result
}

// IsRed 計算是否收紅 K
func (d Data) IsRed() bool {
	var rangeList = d.GetRangeList()
	return rangeList[len(rangeList)-1] > 0
}

// FmtData is struct for daily data format.
type FmtData struct {
	Date       time.Time
	Volume     uint64  //成交股數
	TotalPrice uint64  //成交金額
	Open       float64 //開盤價
	High       float64 //最高價
	Low        float64 //最低價
	Price      float64 //收盤價
	Range      float64 //漲跌價差
	Totalsale  uint64  //成交筆數
}

// FormatData is format daily data.
func (d Data) FormatData() []FmtData {
	result := make([]FmtData, len(d.RawData))
	var loopd FmtData
	for i, v := range d.RawData {
		loopd.Date = utils.ParseDate(v[0])
		loopd.Volume, _ = strconv.ParseUint(strings.Replace(v[1], ",", "", -1), 10, 32)
		loopd.TotalPrice, _ = strconv.ParseUint(strings.Replace(v[2], ",", "", -1), 10, 32)
		loopd.Open, _ = strconv.ParseFloat(v[3], 64)
		loopd.High, _ = strconv.ParseFloat(v[4], 64)
		loopd.Low, _ = strconv.ParseFloat(v[5], 64)
		loopd.Price, _ = strconv.ParseFloat(v[6], 64)
		loopd.Range, _ = strconv.ParseFloat(v[7], 64)
		loopd.Totalsale, _ = strconv.ParseUint(strings.Replace(v[8], ",", "", -1), 10, 64)
		result[i] = loopd
	}
	return result
}
<commit_msg>Add `CalDiffInt64` into twse.<commit_after>// Package twse - Fetch stock data from TWSE, OTC
// 擷取台灣股市上市、上櫃股票資訊
//
package twse

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/toomore/gogrs/utils"
)

type unixMapData map[int64][][]string

// Data start with stock no, date.
type Data struct {
	No          string
	Date        time.Time
	RawData     [][]string
	UnixMapData unixMapData
	exchange    string
	openList    []float64
	priceList   []float64
	rangeList   []float64
	volumeList  []uint64
}

// NewTWSE 建立一個 TWSE 上市股票
func NewTWSE(No string, Date time.Time) *Data {
	return &Data{
		No:       No,
		Date:     Date,
		exchange: "tse",
	}
}

// NewOTC 建立一個 OTC 上櫃股票
func NewOTC(No string, Date time.Time) *Data {
	return &Data{
		No:       No,
		Date:     Date,
		exchange: "otc",
	}
}

// URL return stock csv url path.
func (d Data) URL() string {
	var path string
	var host string

	if d.exchange == "tse" {
		path = fmt.Sprintf(utils.TWSECSV, d.Date.Year(), d.Date.Month(), d.Date.Year(), d.Date.Month(), d.No, utils.RandInt())
		host = utils.TWSEHOST
	} else if d.exchange == "otc" {
		path = fmt.Sprintf(utils.OTCCSV, d.Date.Year()-1911, d.Date.Month(), d.No, utils.RandInt())
		host = utils.OTCHOST
	}

	return fmt.Sprintf("%s%s", host, path)
}

// Round will do sub one month.
func (d *Data) Round() {
	year, month, _ := d.Date.Date()
	d.Date = time.Date(year, month-1, 1, 0, 0, 0, 0, d.Date.Location())
}

// PlusData will do Round() and Get().
func (d *Data) PlusData() {
	d.Round()
	d.Get()
}

func (d *Data) clearCache() {
	d.rangeList = nil
	d.openList = nil
	d.priceList = nil
	d.volumeList = nil
}

// Get return csv data in array.
func (d *Data) Get() ([][]string, error) {
	if d.UnixMapData == nil {
		d.UnixMapData = make(unixMapData)
	}
	if len(d.UnixMapData[d.Date.Unix()]) == 0 {
		csvFiles, err := http.Get(d.URL())
		if err != nil {
			return nil, fmt.Errorf("Network fail: %s", err)
		}
		defer csvFiles.Body.Close()
		data, _ := ioutil.ReadAll(csvFiles.Body)
		csvArrayContent := strings.Split(string(data), "\n")
		for i := range csvArrayContent {
			csvArrayContent[i] = strings.TrimSpace(csvArrayContent[i])
		}
		var csvReader *csv.Reader
		if (d.exchange == "tse" && len(csvArrayContent) > 2) || (d.exchange == "otc" && len(csvArrayContent) > 5) {
			if d.exchange == "tse" {
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[2:], "\n")))
			} else if d.exchange == "otc" {
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[5:len(csvArrayContent)-1], "\n")))
			}
			allData, err := csvReader.ReadAll()
			d.RawData = append(allData, d.RawData...)
			d.UnixMapData[d.Date.Unix()] = allData
			d.clearCache()
			return allData, err
		}
		return nil, errors.New("Not enough data.")
	}
	return d.UnixMapData[d.Date.Unix()], nil
}

// GetByTimeMap return a map by key of time.Time
func (d Data) GetByTimeMap() map[time.Time]interface{} {
	data := make(map[time.Time]interface{})
	dailyData, _ := d.Get()
	for _, v := range dailyData {
		data[utils.ParseDate(v[0])] = v
	}
	return data
}

func (d Data) getColsList(colsNo int) []string {
	var result []string
	result = make([]string, len(d.RawData))
	for i, value := range d.RawData {
		result[i] = value[colsNo]
	}
	return result
}

func (d Data) getColsListFloat64(colsNo int) []float64 {
	var result []float64
	result = make([]float64, len(d.RawData))
	for i, v := range d.getColsList(colsNo) {
		result[i], _ = strconv.ParseFloat(v, 64)
	}
	return result
}

// GetVolumeList 取得 成交股數 序列
func (d *Data) GetVolumeList() []uint64 {
	if d.volumeList == nil {
		var result []uint64
		result = make([]uint64, len(d.RawData))
		for i, v := range d.getColsList(1) {
			result[i], _ = strconv.ParseUint(strings.Replace(v, ",", "", -1), 10, 64)
		}
		d.volumeList = result
	}
	return d.volumeList
}

// GetOpenList 取得 開盤價 序列
func (d *Data) GetOpenList() []float64 {
	if d.openList == nil {
		d.openList = d.getColsListFloat64(3)
	}
	return d.openList
}

// GetPriceList 取得 收盤價 序列
func (d *Data) GetPriceList() []float64 {
	if d.priceList == nil {
		d.priceList = d.getColsListFloat64(6)
	}
	return d.priceList
}

// GetRangeList 取得 漲跌價差 序列
func (d *Data) GetRangeList() []float64 {
	if d.rangeList == nil {
		d.rangeList = d.getColsListFloat64(7)
	}
	return d.rangeList
}

// MA 計算 收盤價 的移動平均
func (d Data) MA(days int) []float64 {
	var result []float64
	var priceList = d.GetPriceList()
	result = make([]float64, len(priceList)-days+1)
	for i := range priceList[days-1:] {
		result[i] = utils.AvgFloat64(priceList[i : i+days])
	}
	return result
}

// MABR 計算 收盤價移動平均 的乖離
func (d Data) MABR(days1, days2 int) []float64 {
	return utils.CalDiffFloat(d.MA(days1), d.MA(days2))
}

// MAV 計算 成交股數 的移動平均
func (d Data) MAV(days int) []uint64 {
	var result []uint64
	var volumeList = d.GetVolumeList()
	result = make([]uint64, len(volumeList)-days+1)
	for i := range volumeList[days-1:] {
		result[i] = utils.AvgUint64(volumeList[i : i+days])
	}
	return result
}

// MAVBR 計算 成交股數移動平均 的乖離
func (d Data) MAVBR(days1, days2 int) []int64 {
	MAV1 := d.MAV(days1)
	MAV2 := d.MAV(days2)
	var length int
	if len(MAV1) <= len(MAV2) {
		length = len(MAV1)
	} else {
		length = len(MAV2)
	}
	result1 := make([]int64, length)
	result2 := make([]int64, length)
	for i := 1; i <= length; i++ {
		result1[length-i] = int64(MAV1[len(MAV1)-i])
		result2[length-i] = int64(MAV2[len(MAV2)-i])
	}

	return utils.CalDiffInt64(result1, result2)
}

// IsRed 計算是否收紅 K
func (d Data) IsRed() bool {
	var rangeList = d.GetRangeList()
	return rangeList[len(rangeList)-1] > 0
}

// FmtData is struct for daily data format.
type FmtData struct {
	Date       time.Time
	Volume     uint64  //成交股數
	TotalPrice uint64  //成交金額
	Open       float64 //開盤價
	High       float64 //最高價
	Low        float64 //最低價
	Price      float64 //收盤價
	Range      float64 //漲跌價差
	Totalsale  uint64  //成交筆數
}

// FormatData is format daily data.
func (d Data) FormatData() []FmtData {
	result := make([]FmtData, len(d.RawData))
	var loopd FmtData
	for i, v := range d.RawData {
		loopd.Date = utils.ParseDate(v[0])
		loopd.Volume, _ = strconv.ParseUint(strings.Replace(v[1], ",", "", -1), 10, 32)
		loopd.TotalPrice, _ = strconv.ParseUint(strings.Replace(v[2], ",", "", -1), 10, 32)
		loopd.Open, _ = strconv.ParseFloat(v[3], 64)
		loopd.High, _ = strconv.ParseFloat(v[4], 64)
		loopd.Low, _ = strconv.ParseFloat(v[5], 64)
		loopd.Price, _ = strconv.ParseFloat(v[6], 64)
		loopd.Range, _ = strconv.ParseFloat(v[7], 64)
		loopd.Totalsale, _ = strconv.ParseUint(strings.Replace(v[8], ",", "", -1), 10, 64)
		result[i] = loopd
	}
	return result
}
<|endoftext|>


<commit_before>// Copyright 2016 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import (
	"strconv"
	"strings"
)

var version string = "0.20.0"

func Full() string {
	return version
}

func getSubVersion(v string, position int) int64 {
	arr := strings.Split(v, ".")
	if len(arr) < 3 {
		return 0
	}
	res, _ := strconv.ParseInt(arr[position], 10, 64)
	return res
}

func Proto(v string) int64 {
	return getSubVersion(v, 0)
}

func Major(v string) int64 {
	return getSubVersion(v, 1)
}

func Minor(v string) int64 {
	return getSubVersion(v, 2)
}

// add every case there if server will not accept client's protocol and return false
func Compat(client string) (ok bool, msg string) {
	if LessThan(client, "0.18.0") {
		return false, "Please upgrade your frpc version to at least 0.18.0"
	}
	return true, ""
}

func LessThan(client string, server string) bool {
	vc := Proto(client)
	vs := Proto(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}

	vc = Major(client)
	vs = Major(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}

	vc = Minor(client)
	vs = Minor(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}
	return false
}
<commit_msg>version to v0.21.0<commit_after>// Copyright 2016 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import (
	"strconv"
	"strings"
)

var version string = "0.21.0"

func Full() string {
	return version
}

func getSubVersion(v string, position int) int64 {
	arr := strings.Split(v, ".")
	if len(arr) < 3 {
		return 0
	}
	res, _ := strconv.ParseInt(arr[position], 10, 64)
	return res
}

func Proto(v string) int64 {
	return getSubVersion(v, 0)
}

func Major(v string) int64 {
	return getSubVersion(v, 1)
}

func Minor(v string) int64 {
	return getSubVersion(v, 2)
}

// add every case there if server will not accept client's protocol and return false
func Compat(client string) (ok bool, msg string) {
	if LessThan(client, "0.18.0") {
		return false, "Please upgrade your frpc version to at least 0.18.0"
	}
	return true, ""
}

func LessThan(client string, server string) bool {
	vc := Proto(client)
	vs := Proto(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}

	vc = Major(client)
	vs = Major(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}

	vc = Minor(client)
	vs = Minor(server)
	if vc > vs {
		return false
	} else if vc < vs {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package hostdb

// scan.go contians the functions which periodically scan the list of all hosts
// to see which hosts are online or offline, and to get any updates to the
// settings of the hosts.

import (
	"crypto/rand"
	"math/big"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	DefaultScanSleep = 2*time.Hour + 18*time.Minute
	MaxScanSleep     = 6 * time.Hour
	MinScanSleep     = 1 * time.Hour

	MaxActiveHosts              = 500
	InactiveHostCheckupQuantity = 250

	maxSettingsLen = 1024

	hostRequestTimeout = 15 * time.Second

	// scanningThreads is the number of threads that will be probing hosts for
	// their settings and checking for reliability.
	scanningThreads = 4
)

var (
	MaxReliability     = types.NewCurrency64(50) // Given the scanning defaults, about 1 week of survival.
	DefaultReliability = types.NewCurrency64(20) // Given the scanning defaults, about 3 days of survival.
	UnreachablePenalty = types.NewCurrency64(1)
)

// decrementReliability reduces the reliability of a node, moving it out of the
// set of active hosts or deleting it entirely if necessary.
func (hdb *HostDB) decrementReliability(addr modules.NetAddress, penalty types.Currency) {
	// Look up the entry and decrement the reliability.
	entry, exists := hdb.allHosts[addr]
	if !exists {
		return
	}
	entry.reliability = entry.reliability.Sub(penalty)

	// If the entry is in the active database, remove it from the active
	// database.
	node, exists := hdb.activeHosts[addr]
	if exists {
		delete(hdb.activeHosts, entry.IPAddress)
		node.removeNode()
		hdb.notifySubscribers()
	}

	// If the reliability has fallen to 0, remove the host from the
	// database entirely.
	if entry.reliability.IsZero() {
		delete(hdb.allHosts, addr)
	}
}

// threadedProbeHost tries to fetch the settings of a host. If successful, the
// host is put in the set of active hosts. If unsuccessful, the host id deleted
// from the set of active hosts.
func (hdb *HostDB) threadedProbeHosts() {
	for hostEntry := range hdb.scanPool {
		// Request settings from the queue'd host entry.
		var settings modules.HostSettings
		err := func() error {
			conn, err := net.DialTimeout("tcp", string(hostEntry.IPAddress), hostRequestTimeout)
			if err != nil {
				return err
			}
			defer conn.Close()
			err = encoding.WriteObject(conn, [8]byte{'S', 'e', 't', 't', 'i', 'n', 'g', 's'})
			if err != nil {
				return err
			}
			return encoding.ReadObject(conn, &settings, maxSettingsLen)
		}()

		// Now that network communication is done, lock the hostdb to modify the
		// host entry.
		id := hdb.mu.Lock()
		defer hdb.mu.Unlock(id)
		if err != nil {
			hdb.decrementReliability(hostEntry.IPAddress, UnreachablePenalty)
			return
		}

		// Update the host settings, reliability, and weight. The old IPAddress
		// must be preserved.
		settings.IPAddress = hostEntry.HostSettings.IPAddress
		hostEntry.HostSettings = settings
		hostEntry.reliability = MaxReliability
		hostEntry.weight = hdb.hostWeight(*hostEntry)

		// If the host is not already in the database and 'MaxActiveHosts' has not
		// been reached, add the host to the database.
		_, exists1 := hdb.activeHosts[hostEntry.IPAddress]
		_, exists2 := hdb.allHosts[hostEntry.IPAddress]
		if !exists1 && exists2 && len(hdb.activeHosts) < MaxActiveHosts {
			hdb.insertNode(hostEntry)
			hdb.notifySubscribers()
		}
	}
}

// threadedScan is an ongoing function which will query the full set of hosts
// every few hours to see who is online and available for uploading.
func (hdb *HostDB) threadedScan() {
	for {
		// Determine who to scan. At most 'MaxActiveHosts' will be scanned,
		// starting with the active hosts followed by a random selection of the
		// inactive hosts.
		id := hdb.mu.Lock()
		{
			// Scan all active hosts.
			for _, host := range hdb.activeHosts {
				go func() {
					hdb.scanPool <- host.hostEntry
				}()
			}

			// Assemble all of the inactive hosts into a single array.
			var random []*hostEntry
			for _, entry := range hdb.allHosts {
				entry2, exists := hdb.activeHosts[entry.IPAddress]
				if !exists {
					random = append(random, entry)
				} else {
					if build.DEBUG {
						if entry2.hostEntry != entry {
							panic("allHosts + activeHosts mismatch!")
						}
					}
				}
			}

			// Randomize the slice by swapping each element with an element
			// that hasn't been visited yet.
			for i := 0; i < len(random); i++ {
				N, err := rand.Int(rand.Reader, big.NewInt(int64(len(random)-i)))
				if err != nil {
					if build.DEBUG {
						panic(err)
					}
				} else {
					break
				}

				n := int(N.Int64()) + i
				tmp := random[i]
				random[i] = random[n]
				random[n] = tmp
			}

			// Select the first InactiveHostCheckupQuantity hosts from the
			// shuffled list and scan them.
			n := InactiveHostCheckupQuantity
			if len(random) < InactiveHostCheckupQuantity {
				n = len(random)
			}
			for i := 0; i < n; i++ {
				go func() {
					hdb.scanPool <- random[i]
				}()
			}
		}
		hdb.mu.Unlock(id)

		// Sleep for a random amount of time before doing another round of
		// scanning. The minimums and maximums keep the scan time reasonable,
		// while the randomness prevents the scanning from always happening at
		// the same time of day or week.
		maxBig := big.NewInt(int64(MaxScanSleep))
		minBig := big.NewInt(int64(MinScanSleep))
		randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
		if err != nil {
			if build.DEBUG {
				panic(err)
			} else {
				// If there's an error, sleep for the default amount of time.
				defaultBig := big.NewInt(int64(DefaultScanSleep))
				randSleep = defaultBig.Sub(defaultBig, minBig)
			}
		}
		time.Sleep(time.Duration(randSleep.Int64()) + MinScanSleep) // this means the MaxScanSleep is actual Max+Min.
	}
}
<commit_msg>fix deadlocks<commit_after>package hostdb

// scan.go contians the functions which periodically scan the list of all hosts
// to see which hosts are online or offline, and to get any updates to the
// settings of the hosts.

import (
	"crypto/rand"
	"math/big"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	DefaultScanSleep = 2*time.Hour + 18*time.Minute
	MaxScanSleep     = 6 * time.Hour
	MinScanSleep     = 1 * time.Hour

	MaxActiveHosts              = 500
	InactiveHostCheckupQuantity = 250

	maxSettingsLen = 1024

	hostRequestTimeout = 15 * time.Second

	// scanningThreads is the number of threads that will be probing hosts for
	// their settings and checking for reliability.
	scanningThreads = 4
)

var (
	MaxReliability     = types.NewCurrency64(50) // Given the scanning defaults, about 1 week of survival.
	DefaultReliability = types.NewCurrency64(20) // Given the scanning defaults, about 3 days of survival.
	UnreachablePenalty = types.NewCurrency64(1)
)

// decrementReliability reduces the reliability of a node, moving it out of the
// set of active hosts or deleting it entirely if necessary.
func (hdb *HostDB) decrementReliability(addr modules.NetAddress, penalty types.Currency) {
	// Look up the entry and decrement the reliability.
	entry, exists := hdb.allHosts[addr]
	if !exists {
		return
	}
	entry.reliability = entry.reliability.Sub(penalty)

	// If the entry is in the active database, remove it from the active
	// database.
	node, exists := hdb.activeHosts[addr]
	if exists {
		delete(hdb.activeHosts, entry.IPAddress)
		node.removeNode()
		hdb.notifySubscribers()
	}

	// If the reliability has fallen to 0, remove the host from the
	// database entirely.
	if entry.reliability.IsZero() {
		delete(hdb.allHosts, addr)
	}
}

// threadedProbeHost tries to fetch the settings of a host. If successful, the
// host is put in the set of active hosts. If unsuccessful, the host id deleted
// from the set of active hosts.
func (hdb *HostDB) threadedProbeHosts() {
	for hostEntry := range hdb.scanPool {
		// Request settings from the queue'd host entry.
		var settings modules.HostSettings
		err := func() error {
			conn, err := net.DialTimeout("tcp", string(hostEntry.IPAddress), hostRequestTimeout)
			if err != nil {
				return err
			}
			defer conn.Close()
			err = encoding.WriteObject(conn, [8]byte{'S', 'e', 't', 't', 'i', 'n', 'g', 's'})
			if err != nil {
				return err
			}
			return encoding.ReadObject(conn, &settings, maxSettingsLen)
		}()

		// Now that network communication is done, lock the hostdb to modify the
		// host entry.
		id := hdb.mu.Lock()
		{
			if err != nil {
				hdb.decrementReliability(hostEntry.IPAddress, UnreachablePenalty)
				hdb.mu.Unlock(id)
				continue
			}

			// Update the host settings, reliability, and weight. The old IPAddress
			// must be preserved.
			settings.IPAddress = hostEntry.HostSettings.IPAddress
			hostEntry.HostSettings = settings
			hostEntry.reliability = MaxReliability
			hostEntry.weight = hdb.hostWeight(*hostEntry)

			// If the host is not already in the database and 'MaxActiveHosts' has not
			// been reached, add the host to the database.
			_, exists1 := hdb.activeHosts[hostEntry.IPAddress]
			_, exists2 := hdb.allHosts[hostEntry.IPAddress]
			if !exists1 && exists2 && len(hdb.activeHosts) < MaxActiveHosts {
				hdb.insertNode(hostEntry)
				hdb.notifySubscribers()
			}
		}
		hdb.mu.Unlock(id)
	}
}

// threadedScan is an ongoing function which will query the full set of hosts
// every few hours to see who is online and available for uploading.
func (hdb *HostDB) threadedScan() {
	for {
		// Determine who to scan. At most 'MaxActiveHosts' will be scanned,
		// starting with the active hosts followed by a random selection of the
		// inactive hosts.
		id := hdb.mu.Lock()
		{
			// Scan all active hosts.
			for _, host := range hdb.activeHosts {
				go func() {
					hdb.scanPool <- host.hostEntry
				}()
			}

			// Assemble all of the inactive hosts into a single array.
			var random []*hostEntry
			for _, entry := range hdb.allHosts {
				entry2, exists := hdb.activeHosts[entry.IPAddress]
				if !exists {
					random = append(random, entry)
				} else {
					if build.DEBUG {
						if entry2.hostEntry != entry {
							panic("allHosts + activeHosts mismatch!")
						}
					}
				}
			}

			// Randomize the slice by swapping each element with an element
			// that hasn't been visited yet.
			for i := 0; i < len(random); i++ {
				N, err := rand.Int(rand.Reader, big.NewInt(int64(len(random)-i)))
				if err != nil {
					if build.DEBUG {
						panic(err)
					}
				} else {
					break
				}

				n := int(N.Int64()) + i
				tmp := random[i]
				random[i] = random[n]
				random[n] = tmp
			}

			// Select the first InactiveHostCheckupQuantity hosts from the
			// shuffled list and scan them.
			n := InactiveHostCheckupQuantity
			if len(random) < InactiveHostCheckupQuantity {
				n = len(random)
			}
			for i := 0; i < n; i++ {
				go func() {
					hdb.scanPool <- random[i]
				}()
			}
		}
		hdb.mu.Unlock(id)

		// Sleep for a random amount of time before doing another round of
		// scanning. The minimums and maximums keep the scan time reasonable,
		// while the randomness prevents the scanning from always happening at
		// the same time of day or week.
		maxBig := big.NewInt(int64(MaxScanSleep))
		minBig := big.NewInt(int64(MinScanSleep))
		randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
		if err != nil {
			if build.DEBUG {
				panic(err)
			} else {
				// If there's an error, sleep for the default amount of time.
				defaultBig := big.NewInt(int64(DefaultScanSleep))
				randSleep = defaultBig.Sub(defaultBig, minBig)
			}
		}
		time.Sleep(time.Duration(randSleep.Int64()) + MinScanSleep) // this means the MaxScanSleep is actual Max+Min.
	}
}
<|endoftext|>


<commit_before>package model

import (
	"strconv"
	"time"
)

type Model interface {
	StopAreas() StopAreas
	StopVisits() StopVisits
	VehicleJourneys() VehicleJourneys
	Lines() Lines
	GetDate() time.Time
	SetDate(string) time.Time
	// ...
}

type MemoryModel struct {
	stopAreas       StopAreas
	stopVisits      StopVisits
	vehicleJourneys VehicleJourneys
	lines           Lines
	date            time.Time
}

func NewMemoryModel() *MemoryModel {
	model := &MemoryModel{}

	stopAreas := NewMemoryStopAreas()
	stopAreas.model = model
	model.stopAreas = stopAreas

	stopVisits := NewMemoryStopVisits()
	stopVisits.model = model
	model.stopVisits = stopVisits

	vehicleJourneys := NewMemoryVehicleJourneys()
	vehicleJourneys.model = model
	model.vehicleJourneys = vehicleJourneys

	lines := NewMemoryLines()
	lines.model = model
	model.lines = lines

	return model
}

func (model *MemoryModel) SetDate(reloadHour string) time.Time {
	hour, minute := 4, 0
	if len(reloadHour) == 5 {
		hour, _ = strconv.Atoi(reloadHour[0:2])
		minute, _ = strconv.Atoi(reloadHour[3:5])
	}
	loc_cet, _ := time.LoadLocation("CET")
	now := time.Now().In(loc_cet)
	model.date = time.Date(now.Year(), now.Month(), now.Day()+1, hour, minute, 0, 0, loc_cet)
	return model.date
}

func (model *MemoryModel) GetDate() time.Time {
	return model.date
}

func (model *MemoryModel) StopAreas() StopAreas {
	return model.stopAreas
}

func (model *MemoryModel) StopVisits() StopVisits {
	return model.stopVisits
}

func (model *MemoryModel) VehicleJourneys() VehicleJourneys {
	return model.vehicleJourneys
}

func (model *MemoryModel) Lines() Lines {
	return model.lines
}

func (model *MemoryModel) NewTransaction() *Transaction {
	return NewTransaction(model)
}
<commit_msg>Clock and test error message change<commit_after>package model

import (
	"strconv"
	"time"
)

type Model interface {
	StopAreas() StopAreas
	StopVisits() StopVisits
	VehicleJourneys() VehicleJourneys
	Lines() Lines
	GetDate() time.Time
	SetDate(string) time.Time
	// ...
}

type MemoryModel struct {
	stopAreas       StopAreas
	stopVisits      StopVisits
	vehicleJourneys VehicleJourneys
	lines           Lines
	date            time.Time
}

func NewMemoryModel() *MemoryModel {
	model := &MemoryModel{}

	stopAreas := NewMemoryStopAreas()
	stopAreas.model = model
	model.stopAreas = stopAreas

	stopVisits := NewMemoryStopVisits()
	stopVisits.model = model
	model.stopVisits = stopVisits

	vehicleJourneys := NewMemoryVehicleJourneys()
	vehicleJourneys.model = model
	model.vehicleJourneys = vehicleJourneys

	lines := NewMemoryLines()
	lines.model = model
	model.lines = lines

	return model
}

func (model *MemoryModel) SetDate(reloadHour string) time.Time {
	hour, minute := 4, 0
	if len(reloadHour) == 5 {
		hour, _ = strconv.Atoi(reloadHour[0:2])
		minute, _ = strconv.Atoi(reloadHour[3:5])
	}
	loc_cet, _ := time.LoadLocation("CET")
	now := DefaultClock().Now().In(loc_cet)
	model.date = time.Date(now.Year(), now.Month(), now.Day()+1, hour, minute, 0, 0, loc_cet)
	return model.date
}

func (model *MemoryModel) GetDate() time.Time {
	return model.date
}

func (model *MemoryModel) StopAreas() StopAreas {
	return model.stopAreas
}

func (model *MemoryModel) StopVisits() StopVisits {
	return model.stopVisits
}

func (model *MemoryModel) VehicleJourneys() VehicleJourneys {
	return model.vehicleJourneys
}

func (model *MemoryModel) Lines() Lines {
	return model.lines
}

func (model *MemoryModel) NewTransaction() *Transaction {
	return NewTransaction(model)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// THIS FILE IS DEPRECATED!!!
// Please edit the new "jiri" tool in release/go/src/v.io/jiri.

package main

import (
	"log"
	"os"
	"os/exec"
)

// main calls "jiri" tool with whatever arguments it was called with.
func main() {
	args := os.Args[1:]
	cmd := exec.Command("jiri", args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	if err := cmd.Run(); err != nil {
		log.Fatalf("Error running %v %v: %v", cmd.Path, args, err)
	}
}
<commit_msg>TBR: Don't print an error when the jiri command fails, since it will have printed its own error.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// THIS FILE IS DEPRECATED!!!
// Please edit the new "jiri" tool in release/go/src/v.io/jiri.

package main

import (
	"os"
	"os/exec"
)

// main calls "jiri" tool with whatever arguments it was called with.
func main() {
	args := os.Args[1:]
	cmd := exec.Command("jiri", args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	if err := cmd.Run(); err != nil {
		// The jiri tool should have reported an error in its output.  Don't
		// print an error here because it can be confusing and makes it harder
		// to spot the real error.
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/BurntSushi/toml"
	"github.com/brentp/irelate"
	"github.com/brentp/vcfgo"
)

// anno holds information about the annotation files parsed from the toml config.
type anno struct {
	File    string
	Ops     []string
	Fields  []string
	Columns []int
	// the names in the output.
	Names []string
}

func (a *anno) isNumber(idx int) bool {
	return a.Ops[idx] == "mean" || a.Ops[idx] == "max" || a.Ops[idx] == "min" || a.Ops[idx] == "count"
}

type Annotations struct {
	Annotation []anno
}

// can get a bam without an op. default it to 'count'
func fixBam(as []anno, j int) anno {
	a := as[j]
	if strings.HasSuffix(a.File, ".bam") {
		as[j].Columns = []int{0}
		as[j].Ops = []string{"count"}
	}
	return as[j]
}

// updateHeader adds a new info item to the header for each new annotation
func updateHeader(files []anno, j int, query *vcfgo.Reader) {
	cfg := files[j]
	for i, name := range cfg.Names {
		ntype := "Character"
		if strings.HasSuffix(cfg.File, ".bam") || cfg.isNumber(i) {
			ntype = "Float"
		}
		var desc string
		// write the VCF header.
		if strings.HasSuffix(cfg.File, ".bam") {
			cfg = fixBam(files, j)
			desc = fmt.Sprintf("calculated by coverage from %s", cfg.File)
		} else if cfg.Fields != nil {
			desc = fmt.Sprintf("calculated by %s of overlapping values in field %s from %s", cfg.Ops[i], cfg.Fields[i], cfg.File)
		} else {
			desc = fmt.Sprintf("calculated by %s of overlapping values in column %d from %s", cfg.Ops[i], cfg.Columns[i], cfg.File)
		}
		query.Header.Infos[name] = &vcfgo.Info{Id: name, Number: "1", Type: ntype, Description: desc}

	}
}

// Anno takes a query vcf and a set of annotations, and writes to outw.
// If ends is specified, then the query is annotated for start, end and the interval itself.
// if strict is true a variant is only annotated with another variant if they share the same
// position, the same ref allele, and at least 1 alt allele.
func Anno(queryVCF string, configs Annotations, outw io.Writer, ends bool, strict bool) {

	files := configs.Annotation

	streams := make([]irelate.RelatableChannel, 0)
	query := irelate.Vopen(queryVCF)

	streams = append(streams, irelate.StreamVCF(query))

	for j, cfg := range files {
		if cfg.Names == nil {
			if cfg.Fields == nil {
				log.Fatal("must specify either fields or names")
			}
			cfg.Names = cfg.Fields
			files[j].Names = cfg.Fields
		}
		updateHeader(files, j, query)
		if strings.HasSuffix(cfg.File, ".vcf.gz") || strings.HasSuffix(cfg.File, ".vcf") {
			v := irelate.Vopen(cfg.File)
			streams = append(streams, irelate.StreamVCF(v))
		} else {
			streams = append(streams, irelate.Streamer(cfg.File))
		}
	}
	out, err := vcfgo.NewWriter(outw, query.Header)
	if err != nil {
		panic(err)
	}

	annotateEnds := INTERVAL
	if ends {
		annotateEnds = BOTH
	}
	// the *Prefix functions let 'chr1' == '1'
	for interval := range irelate.IRelate(irelate.CheckOverlapPrefix, 0, irelate.LessPrefix, streams...) {
		variant := interval.(*irelate.Variant)
		if len(variant.Related()) > 0 {
			sep := Partition(variant, len(streams)-1)
			updateInfo(variant, sep, files, annotateEnds, strict)

		}
		fmt.Fprintln(out, variant)
	}
}

const LEFT = "left__"
const RIGHT = "right__"
const BOTH = "both__"
const INTERVAL = ""

func updateInfo(iv *irelate.Variant, sep [][]irelate.Relatable, files []anno, ends string, strict bool) {
	for i, cfg := range files {

		v := iv.Variant
		var valsByFld [][]interface{}
		if ends == BOTH {
			// we want to annotate ends of the interval independently.
			// note that we automatically set strict to false.
			updateInfo(iv, sep, files, INTERVAL, false)
			updateInfo(iv, sep, files, LEFT, false)
			updateInfo(iv, sep, files, RIGHT, false)
			return
		} else if ends == "" {
			valsByFld = Collect(iv, sep[i], cfg, strict)
		} else if ends == LEFT {
			// hack. We know end() is calculated as length of ref. so we set it to have len 1 temporarily.
			ref := v.Ref
			alt := v.Alt
			v.Ref = "A"
			v.Alt = []string{"T"}
			valsByFld = Collect(iv, sep[i], cfg, strict)
			v.Ref, v.Alt = ref, alt
		} else if ends == RIGHT {
			// artificially set end to be the right end of the interval.
			pos, ref, alt := v.Pos, v.Ref, v.Alt
			v.Pos = uint64(v.End())
			v.Ref, v.Alt = "A", []string{"T"}
			valsByFld = Collect(iv, sep[i], cfg, strict)
			v.Pos, v.Ref, v.Alt = pos, ref, alt

		}

		for i, vals := range valsByFld {
			// currently we don't do anything without overlaps.
			if len(vals) == 0 {
				continue
			}
			v.Info.Add(ends+cfg.Names[i], Reducers[cfg.Ops[i]](vals))
		}
	}
}

func checkAnno(a anno) error {
	if a.Fields == nil {
		// Columns: BED/BAM
		if a.Columns == nil {
			return fmt.Errorf("must specify either 'fields' or 'columns' for %s", a.File)
		}
		if len(a.Ops) != len(a.Columns) && !strings.HasSuffix(a.File, ".bam") {
			return fmt.Errorf("must specify same # of 'columns' as 'ops' for %s", a.File)
		}
		if len(a.Names) != len(a.Columns) && !strings.HasSuffix(a.File, ".bam") {
			return fmt.Errorf("must specify same # of 'names' as 'ops' for %s", a.File)
		}
	}
	// Fields: VCF
	if a.Columns != nil {
		return fmt.Errorf("specify only 'fields' or 'columns' not both %s", a.File)
	}
	if len(a.Ops) != len(a.Fields) {
		return fmt.Errorf("must specify same # of 'fields' as 'ops' for %s", a.File)
	}
	return nil
}

func main() {

	ends := flag.Bool("ends", false, "annotate the start and end as well as the interval itself.")
	notstrict := flag.Bool("permissive-overlap", false, "allow variants to be annotated by another even if the don't"+
		"share the same ref and alt alleles. Default is to require exact match between variants.")
	flag.Parse()
	inFiles := flag.Args()
	if len(inFiles) != 2 {
		fmt.Printf(`Usage:
%s config.toml intput.vcf > annotated.vcf

`, os.Args[0])
		flag.PrintDefaults()
		return
	}

	var config Annotations
	if _, err := toml.DecodeFile(inFiles[0], &config); err != nil {
		panic(err)
	}
	for _, a := range config.Annotation {
		checkAnno(a)
	}
	strict := !*notstrict
	Anno(inFiles[1], config, os.Stdout, *ends, strict)
}
<commit_msg>only annotate both ends of variants with length > 1<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/BurntSushi/toml"
	"github.com/brentp/irelate"
	"github.com/brentp/vcfgo"
)

// anno holds information about the annotation files parsed from the toml config.
type anno struct {
	File    string
	Ops     []string
	Fields  []string
	Columns []int
	// the names in the output.
	Names []string
}

func (a *anno) isNumber(idx int) bool {
	return a.Ops[idx] == "mean" || a.Ops[idx] == "max" || a.Ops[idx] == "min" || a.Ops[idx] == "count"
}

type Annotations struct {
	Annotation []anno
}

// can get a bam without an op. default it to 'count'
func fixBam(as []anno, j int) anno {
	a := as[j]
	if strings.HasSuffix(a.File, ".bam") {
		as[j].Columns = []int{0}
		as[j].Ops = []string{"count"}
	}
	return as[j]
}

// updateHeader adds a new info item to the header for each new annotation
func updateHeader(files []anno, j int, query *vcfgo.Reader) {
	cfg := files[j]
	for i, name := range cfg.Names {
		ntype := "Character"
		if strings.HasSuffix(cfg.File, ".bam") || cfg.isNumber(i) {
			ntype = "Float"
		}
		var desc string
		// write the VCF header.
		if strings.HasSuffix(cfg.File, ".bam") {
			cfg = fixBam(files, j)
			desc = fmt.Sprintf("calculated by coverage from %s", cfg.File)
		} else if cfg.Fields != nil {
			desc = fmt.Sprintf("calculated by %s of overlapping values in field %s from %s", cfg.Ops[i], cfg.Fields[i], cfg.File)
		} else {
			desc = fmt.Sprintf("calculated by %s of overlapping values in column %d from %s", cfg.Ops[i], cfg.Columns[i], cfg.File)
		}
		query.Header.Infos[name] = &vcfgo.Info{Id: name, Number: "1", Type: ntype, Description: desc}

	}
}

// Anno takes a query vcf and a set of annotations, and writes to outw.
// If ends is specified, then the query is annotated for start, end and the interval itself.
// if strict is true a variant is only annotated with another variant if they share the same
// position, the same ref allele, and at least 1 alt allele.
func Anno(queryVCF string, configs Annotations, outw io.Writer, ends bool, strict bool) {

	files := configs.Annotation

	streams := make([]irelate.RelatableChannel, 0)
	query := irelate.Vopen(queryVCF)

	streams = append(streams, irelate.StreamVCF(query))

	for j, cfg := range files {
		if cfg.Names == nil {
			if cfg.Fields == nil {
				log.Fatal("must specify either fields or names")
			}
			cfg.Names = cfg.Fields
			files[j].Names = cfg.Fields
		}
		updateHeader(files, j, query)
		if strings.HasSuffix(cfg.File, ".vcf.gz") || strings.HasSuffix(cfg.File, ".vcf") {
			v := irelate.Vopen(cfg.File)
			streams = append(streams, irelate.StreamVCF(v))
		} else {
			streams = append(streams, irelate.Streamer(cfg.File))
		}
	}
	out, err := vcfgo.NewWriter(outw, query.Header)
	if err != nil {
		panic(err)
	}

	annotateEnds := INTERVAL
	if ends {
		annotateEnds = BOTH
	}
	// the *Prefix functions let 'chr1' == '1'
	for interval := range irelate.IRelate(irelate.CheckOverlapPrefix, 0, irelate.LessPrefix, streams...) {
		variant := interval.(*irelate.Variant)
		if len(variant.Related()) > 0 {
			sep := Partition(variant, len(streams)-1)
			updateInfo(variant, sep, files, annotateEnds, strict)

		}
		fmt.Fprintln(out, variant)
	}
}

const LEFT = "left__"
const RIGHT = "right__"
const BOTH = "both__"
const INTERVAL = ""

func updateInfo(iv *irelate.Variant, sep [][]irelate.Relatable, files []anno, ends string, strict bool) {
	for i, cfg := range files {

		v := iv.Variant
		var valsByFld [][]interface{}
		if ends == BOTH {
			// we want to annotate ends of the interval independently.
			// note that we automatically set strict to false.
			updateInfo(iv, sep, files, INTERVAL, false)
			// only do this if the variant is longer than 1 base.
			if iv.End()-iv.Start() > 1 {
				//log.Println("interval:", v.Start(), v.End())
				updateInfo(iv, sep, files, LEFT, false)
				updateInfo(iv, sep, files, RIGHT, false)
			}
			return
		} else if ends == INTERVAL {
			valsByFld = Collect(iv, sep[i], cfg, strict)
		} else if ends == LEFT {
			// hack. We know end() is calculated as length of ref. so we set it to have len 1 temporarily.
			ref := v.Ref
			alt := v.Alt
			v.Ref = "A"
			v.Alt = []string{"T"}
			//log.Println("left:", v.Start(), v.End())
			valsByFld = Collect(iv, sep[i], cfg, strict)
			v.Ref, v.Alt = ref, alt
		} else if ends == RIGHT {
			// artificially set end to be the right end of the interval.
			pos, ref, alt := v.Pos, v.Ref, v.Alt
			v.Pos = uint64(v.End())
			v.Ref, v.Alt = "A", []string{"T"}
			//log.Println("right:", v.Start(), v.End())
			valsByFld = Collect(iv, sep[i], cfg, strict)
			v.Pos, v.Ref, v.Alt = pos, ref, alt

		}

		for i, vals := range valsByFld {
			// currently we don't do anything without overlaps.
			if len(vals) == 0 {
				continue
			}
			v.Info.Add(ends+cfg.Names[i], Reducers[cfg.Ops[i]](vals))
		}
	}
}

func checkAnno(a anno) error {
	if a.Fields == nil {
		// Columns: BED/BAM
		if a.Columns == nil {
			return fmt.Errorf("must specify either 'fields' or 'columns' for %s", a.File)
		}
		if len(a.Ops) != len(a.Columns) && !strings.HasSuffix(a.File, ".bam") {
			return fmt.Errorf("must specify same # of 'columns' as 'ops' for %s", a.File)
		}
		if len(a.Names) != len(a.Columns) && !strings.HasSuffix(a.File, ".bam") {
			return fmt.Errorf("must specify same # of 'names' as 'ops' for %s", a.File)
		}
	}
	// Fields: VCF
	if a.Columns != nil {
		return fmt.Errorf("specify only 'fields' or 'columns' not both %s", a.File)
	}
	if len(a.Ops) != len(a.Fields) {
		return fmt.Errorf("must specify same # of 'fields' as 'ops' for %s", a.File)
	}
	return nil
}

func main() {

	ends := flag.Bool("ends", false, "annotate the start and end as well as the interval itself.")
	notstrict := flag.Bool("permissive-overlap", false, "allow variants to be annotated by another even if the don't"+
		"share the same ref and alt alleles. Default is to require exact match between variants.")
	flag.Parse()
	inFiles := flag.Args()
	if len(inFiles) != 2 {
		fmt.Printf(`Usage:
%s config.toml intput.vcf > annotated.vcf

`, os.Args[0])
		flag.PrintDefaults()
		return
	}

	var config Annotations
	if _, err := toml.DecodeFile(inFiles[0], &config); err != nil {
		panic(err)
	}
	for _, a := range config.Annotation {
		checkAnno(a)
	}
	strict := !*notstrict
	Anno(inFiles[1], config, os.Stdout, *ends, strict)
}
<|endoftext|>


<commit_before>package main

import "gopkg.in/h2non/bimg.v1"

// Version stores the current package semantic version
const Version = "1.0.18"

// Versions represents the used versions for several significant dependencies
type Versions struct {
	ImaginaryVersion string `json:"imaginary"`
	BimgVersion      string `json:"bimg"`
	VipsVersion      string `json:"libvips"`
}

// CurrentVersions stores the current runtime system version metadata
var CurrentVersions = Versions{Version, bimg.Version, bimg.VipsVersion}
<commit_msg>bumping to 1.1.0<commit_after>package main

import "gopkg.in/h2non/bimg.v1"

// Version stores the current package semantic version
const Version = "1.1.0"

// Versions represents the used versions for several significant dependencies
type Versions struct {
	ImaginaryVersion string `json:"imaginary"`
	BimgVersion      string `json:"bimg"`
	VipsVersion      string `json:"libvips"`
}

// CurrentVersions stores the current runtime system version metadata
var CurrentVersions = Versions{Version, bimg.Version, bimg.VipsVersion}
<|endoftext|>


<commit_before>package gf

const VERSION = "v1.6.6"
const AUTHORS = "john<john@goframe.org>"
<commit_msg>version updates<commit_after>package gf

const VERSION = "v1.6.7"
const AUTHORS = "john<john@goframe.org>"
<|endoftext|>


<commit_before>package main

// VERSION - snagsby version
const VERSION = "0.3.0-rc.2"
<commit_msg>v0.3.0-rc.3<commit_after>package main

// VERSION - snagsby version
const VERSION = "0.3.0-rc.3"
<|endoftext|>


<commit_before>package gock

// Version defines the current package semantic version.
const Version = "1.0.1"
<commit_msg>feat(version): bump to 1.0.2<commit_after>package gock

// Version defines the current package semantic version.
const Version = "1.0.2"
<|endoftext|>


<commit_before>package main

// VERSION - snagsby version
const VERSION = "0.3.0"
<commit_msg>v0.3.0-rc.1<commit_after>package main

// VERSION - snagsby version
const VERSION = "0.3.0-rc.1"
<|endoftext|>


<commit_before>package dns

import "fmt"

// Version is current version of this library.
var Version = V{1, 0, 14}

// V holds the version of this library.
type V struct {
	Major, Minor, Patch int
}

func (v V) String() string {
	return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
}
<commit_msg>Release 1.0.15<commit_after>package dns

import "fmt"

// Version is current version of this library.
var Version = V{1, 0, 15}

// V holds the version of this library.
type V struct {
	Major, Minor, Patch int
}

func (v V) String() string {
	return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
}
<|endoftext|>


<commit_before>package main

const (
	// Release is the asset version of the site. Bump when any assets are
	// updated to blow away any browser caches.
	Release = "77"
)
<commit_msg>Bump assets version<commit_after>package main

const (
	// Release is the asset version of the site. Bump when any assets are
	// updated to blow away any browser caches.
	Release = "78"
)
<|endoftext|>


<commit_before>package main

const VERSION = "0.6.3"
<commit_msg>:+1: Bump up the version to 0.6.4-alpha1<commit_after>package main

const VERSION = "0.6.4-alpha1"
<|endoftext|>


<commit_before>package main

const VERSION = "2.4.1"
<commit_msg>bump version to 2.4.2-beta<commit_after>package main

const VERSION = "2.4.2-beta"
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2018 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package grpc

// Version is the current grpc version.
const Version = "1.20.0-dev"
<commit_msg>Change version to 1.21.0-dev (#2759)<commit_after>/*
 *
 * Copyright 2018 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package grpc

// Version is the current grpc version.
const Version = "1.21.0-dev"
<|endoftext|>