<commit_before>package flux

import (
	"errors"
	"io"
	"sync"
	"sync/atomic"
)

/* Reactors define an the idea of continous, reactive change which is a revised implementation of FRP principles with a golang view and approach. Reactor are like a reactive queue where each reactor builds off a previous reactor to allow a simple top-down flow of data.
This approach lends itself from very simple streaming operations to complex stream processing systems.
Due to the use of unbuffered channels, Reactor require that the next keep the rule of the channel contract
.i.e a reactor channel must have someone to collect/listen/retrieve the data within it and
ensure a continouse operation else close and end the reactor
*/

// ErrFailedBind represent a failure in binding two Reactors
var ErrFailedBind = errors.New("Failed to Bind Reactors")

// ErrReactorClosed returned when reactor is closed
var ErrReactorClosed = errors.New("Reactor is Closed")

// SignalMuxHandler provides a signal function type:
/*
  It takes three arguments:
		- reactor:(Reactor) the reactor itself for reply processing
		- failure:(error) the current error being returned when a data is nil
		- data:(interface{}) the current data being returned,nil when theres an error
*/
type SignalMuxHandler func(reactor Reactor, failure error, signal interface{})

// Reactor provides an interface definition for the reactor type to allow compatibility by future extenders when composing with other structs.
type Reactor interface {
	io.Closer
	Connector
	Sender
	Replier
	Detacher

	UseRoot(Reactor)
	Manage()
}

// ReactiveStack provides a concrete implementation
type ReactiveStack struct {
	ps                    *PressureStream
	op                    SignalMuxHandler
	root                  Reactor
	branch, enders, roots *mapReact
	wg                    sync.WaitGroup
	ro                    sync.Mutex
	bit                   int64
}

//ReactIdentityProcessor provides the processor for a ReactIdentity
func ReactIdentityProcessor() SignalMuxHandler {
	return func(r Reactor, err error, data interface{}) {
		if err != nil {
			r.ReplyError(err)
			return
		}
		r.Reply(data)
	}
}

//ReactIdentity returns a reactor that passes on its request to its listeners if any without modification
func ReactIdentity() Reactor {
	return Reactive(ReactIdentityProcessor())
}

// Reactive returns a ReactiveStacks
func Reactive(fx SignalMuxHandler) *ReactiveStack {
	r := BuildReactive(fx)
	go r.Manage()
	return r
}

// BuildReactive returns a Reactor without calling the Manager processor,this is to allow a more control managment of the operation of the Reactor e.g pass the process up to a Work Pool
func BuildReactive(fx SignalMuxHandler) *ReactiveStack {
	data := make(chan interface{})
	errs := make(chan interface{})

	r := ReactiveStack{
		ps:     BuildPressureStream(data, errs),
		branch: NewMapReact(),
		enders: NewMapReact(),
		roots:  NewMapReact(),
		op:     fx,
	}

	return &r
}

// Close ends signaling operation to the next stack its closing
func (r *ReactiveStack) Close() error {
	if atomic.LoadInt64(&r.bit) > 0 {
		return ErrReactorClosed
	}

	r.roots.Do(func(rm SenderDetachCloser) {
		rm.Detach(r)
	})

	r.wg.Wait()
	atomic.StoreInt64(&r.bit, 1)

	r.ps.Close()
	// r.roots.Clean()
	// r.branch.Clean()
	return nil
}

//UseRoot adds a new root as part of the reactor roots
func (r *ReactiveStack) UseRoot(rx Reactor) {
	r.roots.Add(rx)
}

// Manage manages the operations of reactor
func (r *ReactiveStack) Manage() {
	defer func() {
		r.enders.Close()
		r.roots.Clean()
		r.branch.Clean()
	}()

	for {
		select {
		case err, ok := <-r.ps.Errors:
			if !ok {
				return
			}
			r.wg.Done()
			// go r.op(r, err.(error), nil)
			r.op(r, err.(error), nil)
		case signal, ok := <-r.ps.Signals:
			if !ok {
				return
			}
			r.wg.Done()
			// go r.op(r, nil, signal)
			r.op(r, nil, signal)
		}
	}
}

//Detacher details the detach interface used by the Reactor
type Detacher interface {
	Detach(Reactor)
}

// Detach nullifies the next link of this Reactor
func (r *ReactiveStack) Detach(rm Reactor) {
	r.enders.Disable(rm)
	r.branch.Disable(rm)
}

// Sender defines the delivery methods used to deliver data into Reactor process
type Sender interface {
	Send(v interface{})
	SendError(v error)
}

// SendError returns the in-put pipe
func (r *ReactiveStack) SendError(e error) {
	r.wg.Add(1)
	r.ps.SendError(e)
}

// Send returns the in-put pipe
func (r *ReactiveStack) Send(d interface{}) {
	r.wg.Add(1)
	r.ps.SendSignal(d)
}

//DistributeSignals provide a function that takes a React and other multiple Reactor and distribute the data from the first reactor to others
func DistributeSignals(rs Reactor, ms ...Sender) Reactor {
	return rs.React(func(r Reactor, err error, data interface{}) {
		for _, mi := range ms {
			go func(rec Sender) {
				if err != nil {
					rec.SendError(err)
					return
				}
				rec.Send(data)
			}(mi)
		}
	}, true)
}

// Replier defines reply methods to reply to requests
type Replier interface {
	Reply(v interface{})
	ReplyError(v error)
}

// Reply returns the out-put pipe
func (r *ReactiveStack) Reply(d interface{}) {
	r.branch.Deliver(nil, d)
}

// ReplyError returns the out-put pipe
func (r *ReactiveStack) ReplyError(err error) {
	r.branch.Deliver(err, nil)
}

// Connector defines the core connecting methods used for binding with a Reactor
type Connector interface {
	//Bind provides a convenient way of binding 2 reactors
	Bind(r Reactor, closeAlong bool)
	//React generates a reactor based off its caller
	React(s SignalMuxHandler, closeAlong bool) Reactor
	BindControl(r Reactor, fx func())
}

// Bind connects a reactor to this reactor as an alternative to a connection by the React() approach
func (r *ReactiveStack) Bind(rx Reactor, closeAlong bool) {

	r.branch.Add(rx)
	rx.UseRoot(r)

	if closeAlong {
		r.enders.Add(rx)
	}
}

// BindControl provides a means of adding an extra control layer for binding,it runs a go-routine that waits till the root is closed to run the supplied function
func (r *ReactiveStack) BindControl(rx Reactor, fx func()) {
	r.branch.Add(rx)
	// r.enders.Add(rx)
	rx.UseRoot(r)

	go func(ro sync.WaitGroup) {
		ro.Wait()
		fx()
	}(r.wg)
}

// React creates a reactivestack from this current one with a boolean value
func (r *ReactiveStack) React(fx SignalMuxHandler, closeAlong bool) Reactor {
	nx := Reactive(fx)
	nx.UseRoot(r)

	r.branch.Add(nx)
	if closeAlong {
		r.enders.Add(nx)
	}
	return nx
}

// SendBinder defines the combination of the Sender and Binding interfaces
type SendBinder interface {
	Sender
	Connector
}

// MergeReactors merges data from a set of Senders into a new reactor stream
func MergeReactors(rs ...SendBinder) Reactor {
	ms := ReactIdentity()
	for _, si := range rs {
		func(so SendBinder) {
			var cu int
			size := len(rs)
			so.BindControl(ms, func() {
				if cu == size {
					ms.Close()
					return
				}
				cu++
			})
		}(si)
	}
	return ms
}

//ReplierCloser provides an interface that combines Replier and Closer interfaces
type ReplierCloser interface {
	Replier
	io.Closer
}

//SenderCloser provides an interface that combines Sender and Closer interfaces
type SenderCloser interface {
	Sender
	io.Closer
}

//SenderDetachCloser provides an interface that combines Sender and Closer interfaces
type SenderDetachCloser interface {
	Sender
	Detacher
	io.Closer
}

// MapReact provides a nice way of adding multiple reacts into a reactor reply list.
type mapReact struct {
	ro sync.RWMutex
	ma map[SenderDetachCloser]bool
}

// NewMapReact returns a new MapReact store
func NewMapReact() *mapReact {
	ma := mapReact{ma: make(map[SenderDetachCloser]bool)}
	return &ma
}

func (m *mapReact) Clean() {
	m.ro.Lock()
	m.ma = make(map[SenderDetachCloser]bool)
	m.ro.Unlock()
}

func (m *mapReact) Deliver(err error, data interface{}) {
	m.ro.RLock()
	if m.ma != nil {
		for ms, ok := range m.ma {
			if !ok {
				continue
			}

			if err != nil {
				ms.SendError(err)
				continue
			}

			ms.Send(data)
		}
	}
	m.ro.RUnlock()
}

func (m *mapReact) Add(r SenderDetachCloser) {
	m.ro.Lock()
	if m.ma != nil {
		if !m.ma[r] {
			m.ma[r] = true
		}
	}
	m.ro.Unlock()
}

func (m *mapReact) Disable(r SenderDetachCloser) {
	m.ro.Lock()
	if m.ma != nil {
		if _, ok := m.ma[r]; ok {
			m.ma[r] = false
		}
	}
	m.ro.Unlock()
}

func (m *mapReact) Length() int {
	var l int
	m.ro.RLock()
	if m.ma != nil {
		l = len(m.ma)
	}
	m.ro.RUnlock()
	return l
}

func (m *mapReact) Do(fx func(SenderDetachCloser)) {
	m.ro.RLock()
	if m.ma != nil {
		for ms := range m.ma {
			fx(ms)
		}
	}
	m.ro.RUnlock()
}

func (m *mapReact) DisableAll() {
	m.ro.Lock()
	if m.ma != nil {
		for ms := range m.ma {
			m.ma[ms] = false
		}
	}
	m.ro.Unlock()
}

func (m *mapReact) Close() {
	m.ro.RLock()
	if m.ma != nil {
		for ms, ok := range m.ma {
			if !ok {
				continue
			}
			ms.Close()
		}
	}
	m.ro.RUnlock()
}

// ChannelStream provides a simple struct for exposing outputs from Reactor to outside
type ChannelStream struct {
	Data   chan interface{}
	Errors chan error
}

// NewChannelStream returns a new channel stream instance
func NewChannelStream() *ChannelStream {
	return &ChannelStream{
		Data:   make(chan interface{}),
		Errors: make(chan error),
	}
}
<commit_msg>adding fx work<commit_after>package flux

import (
	"errors"
	"io"
	"sync"
	"sync/atomic"
)

/* Reactors define an the idea of continous, reactive change which is a revised implementation of FRP principles with a golang view and approach. Reactor are like a reactive queue where each reactor builds off a previous reactor to allow a simple top-down flow of data.
This approach lends itself from very simple streaming operations to complex stream processing systems.
Due to the use of unbuffered channels, Reactor require that the next keep the rule of the channel contract
.i.e a reactor channel must have someone to collect/listen/retrieve the data within it and
ensure a continouse operation else close and end the reactor
*/

// ErrFailedBind represent a failure in binding two Reactors
var ErrFailedBind = errors.New("Failed to Bind Reactors")

// ErrReactorClosed returned when reactor is closed
var ErrReactorClosed = errors.New("Reactor is Closed")

// SignalMuxHandler provides a signal function type:
/*
  It takes three arguments:
		- reactor:(Reactor) the reactor itself for reply processing
		- failure:(error) the current error being returned when a data is nil
		- data:(interface{}) the current data being returned,nil when theres an error
*/
type SignalMuxHandler func(reactor Reactor, failure error, signal interface{})

// Reactor provides an interface definition for the reactor type to allow compatibility by future extenders when composing with other structs.
type Reactor interface {
	io.Closer
	Connector
	Sender
	Replier
	Detacher

	UseRoot(Reactor)
	Manage()
}

// ReactiveStack provides a concrete implementation
type ReactiveStack struct {
	ps                    *PressureStream
	op                    SignalMuxHandler
	root                  Reactor
	branch, enders, roots *mapReact
	wg                    sync.WaitGroup
	ro                    sync.Mutex
	bit                   int64
}

//ReactIdentityProcessor provides the processor for a ReactIdentity
func ReactIdentityProcessor() SignalMuxHandler {
	return func(r Reactor, err error, data interface{}) {
		if err != nil {
			r.ReplyError(err)
			return
		}
		r.Reply(data)
	}
}

//ReactIdentity returns a reactor that passes on its request to its listeners if any without modification
func ReactIdentity() Reactor {
	return Reactive(ReactIdentityProcessor())
}

// Reactive returns a ReactiveStacks
func Reactive(fx SignalMuxHandler) *ReactiveStack {
	r := BuildReactive(fx)
	go r.Manage()
	return r
}

// BuildReactive returns a Reactor without calling the Manager processor,this is to allow a more control managment of the operation of the Reactor e.g pass the process up to a Work Pool
func BuildReactive(fx SignalMuxHandler) *ReactiveStack {
	data := make(chan interface{})
	errs := make(chan interface{})

	r := ReactiveStack{
		ps:     BuildPressureStream(data, errs),
		branch: NewMapReact(),
		enders: NewMapReact(),
		roots:  NewMapReact(),
		op:     fx,
	}

	return &r
}

// Close ends signaling operation to the next stack its closing
func (r *ReactiveStack) Close() error {
	if atomic.LoadInt64(&r.bit) > 0 {
		return ErrReactorClosed
	}

	r.roots.Do(func(rm SenderDetachCloser) {
		rm.Detach(r)
	})

	r.wg.Wait()
	atomic.StoreInt64(&r.bit, 1)

	r.ps.Close()
	// r.roots.Clean()
	// r.branch.Clean()
	return nil
}

//UseRoot adds a new root as part of the reactor roots
func (r *ReactiveStack) UseRoot(rx Reactor) {
	r.roots.Add(rx)
}

// Manage manages the operations of reactor
func (r *ReactiveStack) Manage() {
	defer func() {
		r.enders.Close()
		r.roots.Clean()
		r.branch.Clean()
	}()

	for {
		select {
		case err, ok := <-r.ps.Errors:
			if !ok {
				return
			}
			r.wg.Done()
			// go r.op(r, err.(error), nil)
			r.op(r, err.(error), nil)
		case signal, ok := <-r.ps.Signals:
			if !ok {
				return
			}
			r.wg.Done()
			// go r.op(r, nil, signal)
			r.op(r, nil, signal)
		}
	}
}

//Detacher details the detach interface used by the Reactor
type Detacher interface {
	Detach(Reactor)
}

// Detach nullifies the next link of this Reactor
func (r *ReactiveStack) Detach(rm Reactor) {
	r.enders.Disable(rm)
	r.branch.Disable(rm)
}

// Sender defines the delivery methods used to deliver data into Reactor process
type Sender interface {
	Send(v interface{})
	SendError(v error)
}

// SendError returns the in-put pipe
func (r *ReactiveStack) SendError(e error) {
	r.wg.Add(1)
	r.ps.SendError(e)
}

// Send returns the in-put pipe
func (r *ReactiveStack) Send(d interface{}) {
	r.wg.Add(1)
	r.ps.SendSignal(d)
}

//DistributeSignals provide a function that takes a React and other multiple Reactor and distribute the data from the first reactor to others
func DistributeSignals(rs Reactor, ms ...Sender) Reactor {
	return rs.React(func(r Reactor, err error, data interface{}) {
		for _, mi := range ms {
			go func(rec Sender) {
				if err != nil {
					rec.SendError(err)
					return
				}
				rec.Send(data)
			}(mi)
		}
	}, true)
}

// Replier defines reply methods to reply to requests
type Replier interface {
	Reply(v interface{})
	ReplyError(v error)
}

// Reply returns the out-put pipe
func (r *ReactiveStack) Reply(d interface{}) {
	r.branch.Deliver(nil, d)
}

// ReplyError returns the out-put pipe
func (r *ReactiveStack) ReplyError(err error) {
	r.branch.Deliver(err, nil)
}

// Connector defines the core connecting methods used for binding with a Reactor
type Connector interface {
	//Bind provides a convenient way of binding 2 reactors
	Bind(r Reactor, closeAlong bool)
	//React generates a reactor based off its caller
	React(s SignalMuxHandler, closeAlong bool) Reactor
	BindControl(r Reactor, fx func())
}

// Bind connects a reactor to this reactor as an alternative to a connection by the React() approach
func (r *ReactiveStack) Bind(rx Reactor, closeAlong bool) {

	r.branch.Add(rx)
	rx.UseRoot(r)

	if closeAlong {
		r.enders.Add(rx)
	}
}

// BindControl provides a means of adding an extra control layer for binding,it runs a go-routine that waits till the root is closed to run the supplied function
func (r *ReactiveStack) BindControl(rx Reactor, fx func()) {
	r.branch.Add(rx)
	// r.enders.Add(rx)
	rx.UseRoot(r)

	go func(ro sync.WaitGroup) {
		ro.Wait()
		fx()
	}(r.wg)
}

// React creates a reactivestack from this current one with a boolean value
func (r *ReactiveStack) React(fx SignalMuxHandler, closeAlong bool) Reactor {
	nx := Reactive(fx)
	nx.UseRoot(r)

	r.branch.Add(nx)
	if closeAlong {
		r.enders.Add(nx)
	}
	return nx
}

// SendBinder defines the combination of the Sender and Binding interfaces
type SendBinder interface {
	Sender
	Connector
}

// MergeReactors merges data from a set of Senders into a new reactor stream
func MergeReactors(rs ...SendBinder) Reactor {
	ms := ReactIdentity()
	for _, si := range rs {
		func(so SendBinder) {
			var cu int
			size := len(rs)
			so.BindControl(ms, func() {
				if cu == size {
					ms.Close()
					return
				}
				cu++
			})
		}(si)
	}
	return ms
}

//ReplierCloser provides an interface that combines Replier and Closer interfaces
type ReplierCloser interface {
	Replier
	io.Closer
}

//SenderCloser provides an interface that combines Sender and Closer interfaces
type SenderCloser interface {
	Sender
	io.Closer
}

//SenderDetachCloser provides an interface that combines Sender and Closer interfaces
type SenderDetachCloser interface {
	Sender
	Detacher
	io.Closer
}

// MapReact provides a nice way of adding multiple reacts into a reactor reply list.
type mapReact struct {
	ro sync.RWMutex
	ma map[SenderDetachCloser]bool
}

// NewMapReact returns a new MapReact store
func NewMapReact() *mapReact {
	ma := mapReact{ma: make(map[SenderDetachCloser]bool)}
	return &ma
}

func (m *mapReact) Clean() {
	m.ro.Lock()
	m.ma = make(map[SenderDetachCloser]bool)
	m.ro.Unlock()
}

func (m *mapReact) Deliver(err error, data interface{}) {
	m.ro.RLock()
	for ms, ok := range m.ma {
		if !ok {
			continue
		}

		if err != nil {
			ms.SendError(err)
			continue
		}

		ms.Send(data)
	}
	m.ro.RUnlock()
}

func (m *mapReact) Add(r SenderDetachCloser) {
	m.ro.Lock()
	if !m.ma[r] {
		m.ma[r] = true
	}
	m.ro.Unlock()
}

func (m *mapReact) Disable(r SenderDetachCloser) {
	m.ro.Lock()
	if _, ok := m.ma[r]; ok {
		m.ma[r] = false
	}
	m.ro.Unlock()
}

func (m *mapReact) Length() int {
	var l int
	m.ro.RLock()
	l = len(m.ma)
	m.ro.RUnlock()
	return l
}

func (m *mapReact) Do(fx func(SenderDetachCloser)) {
	m.ro.RLock()
	for ms := range m.ma {
		fx(ms)
	}
	m.ro.RUnlock()
}

func (m *mapReact) DisableAll() {
	m.ro.Lock()
	for ms := range m.ma {
		m.ma[ms] = false
	}
	m.ro.Unlock()
}

func (m *mapReact) Close() {
	m.ro.RLock()
	for ms, ok := range m.ma {
		if !ok {
			continue
		}
		ms.Close()
	}
	m.ro.RUnlock()
}

// ChannelStream provides a simple struct for exposing outputs from Reactor to outside
type ChannelStream struct {
	Data   chan interface{}
	Errors chan error
}

// NewChannelStream returns a new channel stream instance
func NewChannelStream() *ChannelStream {
	return &ChannelStream{
		Data:   make(chan interface{}),
		Errors: make(chan error),
	}
}
<|endoftext|>


<commit_before>package karmabot

import (
	"fmt"
	"regexp"
	"strings"
)

type karmaRegex struct {
	user, autocomplete, explicitAutocomplete, points, reason string
}

var karmaReg = &karmaRegex{
	user:                 `@??((?:<@)??\w[A-Za-z0-9_\-@<>]+?)`,
	autocomplete:         `:?? ??`,
	explicitAutocomplete: `(?:: )??`,
	points:               `([\+]{2,}|[\-]{2,})`,
	reason:               `(?:(?: for)? +(.*))?`,
}

func (r *karmaRegex) GetGive() *regexp.Regexp {
	expression := fmt.Sprintf(
		"(?:%s)|(?:%s)",
		strings.Join(
			[]string{
				"^",
				r.user,
				r.autocomplete,
				r.points,
				r.reason,
				"$",
			},
			"",
		),
		strings.Join(
			[]string{
				`\s+`,
				r.user,
				r.explicitAutocomplete,
				r.points,
				r.reason,
				"$",
			},
			"",
		),
	)

	return regexp.MustCompile(expression)
}

func (r *karmaRegex) GetMotivate() *regexp.Regexp {
	expression := strings.Join(
		[]string{
			`^(?:\?|!)m +`,
			r.user,
			r.autocomplete,
			"$",
		},
		"",
	)

	return regexp.MustCompile(expression)
}

func (r *karmaRegex) GetQuery() *regexp.Regexp {
	expression := strings.Join(
		[]string{
			`^`,
			r.user,
			r.autocomplete,
			"==",
			"$",
		},
		"",
	)

	return regexp.MustCompile(expression)
}
<commit_msg>fix regex, oops<commit_after>package karmabot

import (
	"fmt"
	"regexp"
	"strings"
)

type karmaRegex struct {
	user, autocomplete, explicitAutocomplete, points, reason string
}

var karmaReg = &karmaRegex{
	user:                 `@??((?:<@)??[A-Za-z0-9_\-@<>]+?)`,
	autocomplete:         `:?? ??`,
	explicitAutocomplete: `(?:: )??`,
	points:               `([\+]{2,}|[\-]{2,})`,
	reason:               `(?:(?: for)? +(.*))?`,
}

func (r *karmaRegex) GetGive() *regexp.Regexp {
	expression := fmt.Sprintf(
		"(?:%s)|(?:%s)",
		strings.Join(
			[]string{
				"^",
				r.user,
				r.autocomplete,
				r.points,
				r.reason,
				"$",
			},
			"",
		),
		strings.Join(
			[]string{
				`\s+`,
				r.user,
				r.explicitAutocomplete,
				r.points,
				r.reason,
				"$",
			},
			"",
		),
	)

	return regexp.MustCompile(expression)
}

func (r *karmaRegex) GetMotivate() *regexp.Regexp {
	expression := strings.Join(
		[]string{
			`^(?:\?|!)m +`,
			r.user,
			r.autocomplete,
			"$",
		},
		"",
	)

	return regexp.MustCompile(expression)
}

func (r *karmaRegex) GetQuery() *regexp.Regexp {
	expression := strings.Join(
		[]string{
			`^`,
			r.user,
			r.autocomplete,
			"==",
			"$",
		},
		"",
	)

	return regexp.MustCompile(expression)
}
<|endoftext|>


<commit_before>package annotations

import (
	"encoding/json"
	"errors"
	"fmt"
	"time"

	"regexp"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

var uuidExtractRegex = regexp.MustCompile(".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$")

// Service interface. Compatible with the baserwftapp service EXCEPT for
// 1) the Write function, which has signature Write(thing interface{}) error...
// 2) the DecodeJson function, which has signature DecodeJSON(*json.Decoder) (thing interface{}, identity string, err error)
// The problem is that we have a list of things, and the uuid is for a related OTHER thing
// TODO - move to implement a shared defined Service interface?
type Service interface {
	Write(contentUUID string, thing interface{}) (err error)
	Read(contentUUID string) (thing interface{}, found bool, err error)
	Delete(contentUUID string) (found bool, err error)
	Check() (err error)
	DecodeJSON(*json.Decoder) (thing interface{}, err error)
	Count() (int, error)
	Initialise() error
}

//holds the Neo4j-specific information
type service struct {
	cypherRunner    neoutils.CypherRunner
	indexManager    neoutils.IndexManager
	platformVersion string
}

//NewAnnotationsService instantiate driver
func NewAnnotationsService(cypherRunner neoutils.CypherRunner, indexManager neoutils.IndexManager, platformVersion string) service {
	if platformVersion == "" {
		log.Fatalf("PlatformVersion was not specified!")
	}
	return service{cypherRunner, indexManager, platformVersion}
}

// DecodeJSON decodes to a list of annotations, for ease of use this is a struct itself
func (s service) DecodeJSON(dec *json.Decoder) (interface{}, error) {
	a := annotations{}
	err := dec.Decode(&a)
	return a, err
}

func (s service) Read(contentUUID string) (thing interface{}, found bool, err error) {
	results := []annotation{}

	//TODO shouldn't return Provenances if none of the scores, agentRole or atTime are set
	statementTemplate := `
					MATCH (c:Thing{uuid:{contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing)
					WITH c, cc, rel, {id:cc.uuid,prefLabel:cc.prefLabel,types:labels(cc),predicate:type(rel)} as thing,
					collect(
						{scores:[
							{scoringSystem:'%s', value:rel.relevanceScore},
							{scoringSystem:'%s', value:rel.confidenceScore}],
						agentRole:rel.annotatedBy,
						atTime:rel.annotatedDate}) as provenances
					RETURN thing, provenances ORDER BY thing.id
									`
	statement := fmt.Sprintf(statementTemplate, relevanceScoringSystem, confidenceScoringSystem)

	query := &neoism.CypherQuery{
		Statement:  statement,
		Parameters: neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion},
		Result:     &results,
	}
	err = s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up uuid %s with query %s from neoism: %+v", contentUUID, query.Statement, err)
		return annotations{}, false, fmt.Errorf("Error accessing Annotations datastore for uuid: %s", contentUUID)
	}
	log.Debugf("CypherResult Read Annotations for uuid: %s was: %+v", contentUUID, results)
	if (len(results)) == 0 {
		return annotations{}, false, nil
	}

	for idx := range results {
		mapToResponseFormat(&results[idx])
	}

	return results, true, nil
}

//Delete removes all the annotations for this content. Ignore the nodes on either end -
//may leave nodes that are only 'things' inserted by this writer: clean up
//as a result of this will need to happen externally if required
func (s service) Delete(contentUUID string) (bool, error) {

	query := &neoism.CypherQuery{
		Statement:    `MATCH (c:Thing{uuid: {contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing) DELETE rel`,
		Parameters:   neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion},
		IncludeStats: true,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	stats, err := query.Stats()
	if err != nil {
		return false, err
	}

	var found bool
	if stats.ContainsUpdates {
		found = true
	}

	return found, err
}

//Write a set of annotations associated with a piece of content. Any annotations
//already there will be removed
func (s service) Write(contentUUID string, thing interface{}) (err error) {
	annotationsToWrite := thing.(annotations)

	if contentUUID == "" {
		return errors.New("Content uuid is required")
	}
	if err := validateAnnotations(&annotationsToWrite); err != nil {
		log.Warnf("Validation of supplied annotations failed")
		return err
	}

	if len(annotationsToWrite) == 0 {
		log.Warnf("No new annotations supplied for content uuid: %s", contentUUID)
	}

	queries := append([]*neoism.CypherQuery{}, dropAllAnnotationsQuery(contentUUID, s.platformVersion))

	var statements = []string{}
	for _, annotationToWrite := range annotationsToWrite {
		query, err := createAnnotationQuery(contentUUID, annotationToWrite, s.platformVersion)
		if err != nil {
			return err
		}
		statements = append(statements, query.Statement)
		queries = append(queries, query)
	}
	log.Infof("Updated Annotations for content uuid: %s", contentUUID)
	log.Debugf("For update, ran statements: %+v", statements)

	return s.cypherRunner.CypherBatch(queries)
}

// Check tests neo4j by running a simple cypher query
func (s service) Check() error {
	return neoutils.Check(s.cypherRunner)
}

func (s service) Count() (int, error) {
	results := []struct {
		Count int `json:"c"`
	}{}

	query := &neoism.CypherQuery{
		Statement:  `MATCH ()-[r{platformVersion:{platformVersion}}]->() RETURN count(r) as c`,
		Parameters: neoism.Props{"platformVersion": s.platformVersion},
		Result:     &results,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

func (s service) Initialise() error {
	return nil // No constraints need to be set up
}

func createAnnotationRelationship(relation string) (statement string) {
	stmt := `
                MERGE (content:Thing{uuid:{contentID}})
                MERGE (concept:Thing{uuid:{conceptID}})
                MERGE (content)-[pred:%s{platformVersion:{platformVersion}}]->(concept)
                SET pred={annProps}
                `
	statement = fmt.Sprintf(stmt, relation)
	return statement
}

func getRelationshipFromPredicate(predicate string) (relation string) {
	if predicate != "" {
		relation = relations[predicate]
	} else {
		relation = relations["mentions"]
	}
	return relation
}

func createAnnotationQuery(contentUUID string, ann annotation, platformVersion string) (*neoism.CypherQuery, error) {
	query := neoism.CypherQuery{}
	thingID, err := extractUUIDFromURI(ann.Thing.ID)
	if err != nil {
		return nil, err
	}

	//todo temporary chnage to deal with multiple provenances
	/*if len(ann.Provenances) > 1 {
		return nil, errors.New("Cannot insert a MENTIONS annotation with multiple provenances")
	}*/

	var prov provenance
	params := map[string]interface{}{}
	params["platformVersion"] = platformVersion

	if len(ann.Provenances) >= 1 {
		prov = ann.Provenances[0]
		annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, supplied, err := extractDataFromProvenance(&prov)

		if err != nil {
			log.Infof("ERROR=%s", err)
			return nil, err
		}

		if supplied == true {
			if annotatedBy != "" {
				params["annotatedBy"] = annotatedBy
			}
			if prov.AtTime != "" {
				params["annotatedDateEpoch"] = annotatedDateEpoch
				params["annotatedDate"] = prov.AtTime
			}
			params["relevanceScore"] = relevanceScore
			params["confidenceScore"] = confidenceScore
		}
	}

	relation := getRelationshipFromPredicate(ann.Thing.Predicate)
	query.Statement = createAnnotationRelationship(relation)
	query.Parameters = map[string]interface{}{
		"contentID":       contentUUID,
		"conceptID":       thingID,
		"platformVersion": platformVersion,
		"annProps":        params,
	}
	return &query, nil
}

func extractDataFromProvenance(prov *provenance) (string, int64, float64, float64, bool, error) {
	if len(prov.Scores) == 0 {
		return "", -1, -1, -1, false, nil
	}
	var annotatedBy string
	var annotatedDateEpoch int64
	var confidenceScore, relevanceScore float64
	var err error
	if prov.AgentRole != "" {
		annotatedBy, err = extractUUIDFromURI(prov.AgentRole)
	}
	if prov.AtTime != "" {
		annotatedDateEpoch, err = convertAnnotatedDateToEpoch(prov.AtTime)
	}
	relevanceScore, confidenceScore, err = extractScores(prov.Scores)

	if err != nil {
		return "", -1, -1, -1, true, err
	}
	return annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, true, nil
}

func extractUUIDFromURI(uri string) (string, error) {
	result := uuidExtractRegex.FindStringSubmatch(uri)
	if len(result) == 2 {
		return result[1], nil
	}
	return "", fmt.Errorf("Couldn't extract uuid from uri %s", uri)
}

func convertAnnotatedDateToEpoch(annotatedDateString string) (int64, error) {
	datetimeEpoch, err := time.Parse(time.RFC3339, annotatedDateString)

	if err != nil {
		return 0, err
	}

	return datetimeEpoch.Unix(), nil
}

func extractScores(scores []score) (float64, float64, error) {
	var relevanceScore, confidenceScore float64
	for _, score := range scores {
		scoringSystem := score.ScoringSystem
		value := score.Value
		switch scoringSystem {
		case relevanceScoringSystem:
			relevanceScore = value
		case confidenceScoringSystem:
			confidenceScore = value
		}
	}
	return relevanceScore, confidenceScore, nil
}

func dropAllAnnotationsQuery(contentUUID string, platformVersion string) *neoism.CypherQuery {
	matchStmtTemplate := `optional match (:Thing{uuid:{contentID}})-[r{platformVersion:{platformVersion}}]->(:Thing)
                        delete r`

	query := neoism.CypherQuery{}
	query.Statement = matchStmtTemplate
	query.Parameters = neoism.Props{"contentID": contentUUID, "platformVersion": platformVersion}
	return &query
}

func validateAnnotations(annotations *annotations) error {
	//TODO - for consistency, we should probably just not create the annotation?
	for _, annotation := range *annotations {
		if annotation.Thing.ID == "" {
			return ValidationError{fmt.Sprintf("Concept uuid missing for annotation %+v", annotation)}
		}
	}
	return nil
}

//ValidationError is thrown when the annotations are not valid because mandatory information is missing
type ValidationError struct {
	Msg string
}

func (v ValidationError) Error() string {
	return v.Msg
}

func mapToResponseFormat(ann *annotation) {
	ann.Thing.ID = mapper.IDURL(ann.Thing.ID)
	// We expect only ONE provenance - provenance value is considered valid even if the AgentRole is not specified. See: v1 - isClassifiedBy
	for idx := range ann.Provenances {
		if ann.Provenances[idx].AgentRole != "" {
			ann.Provenances[idx].AgentRole = mapper.IDURL(ann.Provenances[idx].AgentRole)
		}
	}
}
<commit_msg>Both delete and pre-write delete no longer remove content/brand relationships<commit_after>package annotations

import (
	"encoding/json"
	"errors"
	"fmt"
	"time"

	"regexp"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

var uuidExtractRegex = regexp.MustCompile(".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$")

// Service interface. Compatible with the baserwftapp service EXCEPT for
// 1) the Write function, which has signature Write(thing interface{}) error...
// 2) the DecodeJson function, which has signature DecodeJSON(*json.Decoder) (thing interface{}, identity string, err error)
// The problem is that we have a list of things, and the uuid is for a related OTHER thing
// TODO - move to implement a shared defined Service interface?
type Service interface {
	Write(contentUUID string, thing interface{}) (err error)
	Read(contentUUID string) (thing interface{}, found bool, err error)
	Delete(contentUUID string) (found bool, err error)
	Check() (err error)
	DecodeJSON(*json.Decoder) (thing interface{}, err error)
	Count() (int, error)
	Initialise() error
}

//holds the Neo4j-specific information
type service struct {
	cypherRunner    neoutils.CypherRunner
	indexManager    neoutils.IndexManager
	platformVersion string
}

//NewAnnotationsService instantiate driver
func NewAnnotationsService(cypherRunner neoutils.CypherRunner, indexManager neoutils.IndexManager, platformVersion string) service {
	if platformVersion == "" {
		log.Fatalf("PlatformVersion was not specified!")
	}
	return service{cypherRunner, indexManager, platformVersion}
}

// DecodeJSON decodes to a list of annotations, for ease of use this is a struct itself
func (s service) DecodeJSON(dec *json.Decoder) (interface{}, error) {
	a := annotations{}
	err := dec.Decode(&a)
	return a, err
}

func (s service) Read(contentUUID string) (thing interface{}, found bool, err error) {
	results := []annotation{}

	//TODO shouldn't return Provenances if none of the scores, agentRole or atTime are set
	statementTemplate := `
					MATCH (c:Thing{uuid:{contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing)
					WITH c, cc, rel, {id:cc.uuid,prefLabel:cc.prefLabel,types:labels(cc),predicate:type(rel)} as thing,
					collect(
						{scores:[
							{scoringSystem:'%s', value:rel.relevanceScore},
							{scoringSystem:'%s', value:rel.confidenceScore}],
						agentRole:rel.annotatedBy,
						atTime:rel.annotatedDate}) as provenances
					RETURN thing, provenances ORDER BY thing.id
									`
	statement := fmt.Sprintf(statementTemplate, relevanceScoringSystem, confidenceScoringSystem)

	query := &neoism.CypherQuery{
		Statement:  statement,
		Parameters: neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion},
		Result:     &results,
	}
	err = s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up uuid %s with query %s from neoism: %+v", contentUUID, query.Statement, err)
		return annotations{}, false, fmt.Errorf("Error accessing Annotations datastore for uuid: %s", contentUUID)
	}
	log.Debugf("CypherResult Read Annotations for uuid: %s was: %+v", contentUUID, results)
	if (len(results)) == 0 {
		return annotations{}, false, nil
	}

	for idx := range results {
		mapToResponseFormat(&results[idx])
	}

	return results, true, nil
}

//Delete removes all the annotations for this content. Ignore the nodes on either end -
//may leave nodes that are only 'things' inserted by this writer: clean up
//as a result of this will need to happen externally if required
func (s service) Delete(contentUUID string) (bool, error) {

	query := &neoism.CypherQuery{
		Statement:    `MATCH (c:Thing{uuid: {contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing) WHERE NOT cc:Brand DELETE rel`,
		Parameters:   neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion},
		IncludeStats: true,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	stats, err := query.Stats()
	if err != nil {
		return false, err
	}

	var found bool
	if stats.ContainsUpdates {
		found = true
	}

	return found, err
}

//Write a set of annotations associated with a piece of content. Any annotations
//already there will be removed
func (s service) Write(contentUUID string, thing interface{}) (err error) {
	annotationsToWrite := thing.(annotations)

	if contentUUID == "" {
		return errors.New("Content uuid is required")
	}
	if err := validateAnnotations(&annotationsToWrite); err != nil {
		log.Warnf("Validation of supplied annotations failed")
		return err
	}

	if len(annotationsToWrite) == 0 {
		log.Warnf("No new annotations supplied for content uuid: %s", contentUUID)
	}

	queries := append([]*neoism.CypherQuery{}, dropAllAnnotationsQuery(contentUUID, s.platformVersion))

	var statements = []string{}
	for _, annotationToWrite := range annotationsToWrite {
		query, err := createAnnotationQuery(contentUUID, annotationToWrite, s.platformVersion)
		if err != nil {
			return err
		}
		statements = append(statements, query.Statement)
		queries = append(queries, query)
	}
	log.Infof("Updated Annotations for content uuid: %s", contentUUID)
	log.Debugf("For update, ran statements: %+v", statements)

	return s.cypherRunner.CypherBatch(queries)
}

// Check tests neo4j by running a simple cypher query
func (s service) Check() error {
	return neoutils.Check(s.cypherRunner)
}

func (s service) Count() (int, error) {
	results := []struct {
		Count int `json:"c"`
	}{}

	query := &neoism.CypherQuery{
		Statement:  `MATCH ()-[r{platformVersion:{platformVersion}}]->() RETURN count(r) as c`,
		Parameters: neoism.Props{"platformVersion": s.platformVersion},
		Result:     &results,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

func (s service) Initialise() error {
	return nil // No constraints need to be set up
}

func createAnnotationRelationship(relation string) (statement string) {
	stmt := `
                MERGE (content:Thing{uuid:{contentID}})
                MERGE (concept:Thing{uuid:{conceptID}})
                MERGE (content)-[pred:%s{platformVersion:{platformVersion}}]->(concept)
                SET pred={annProps}
                `
	statement = fmt.Sprintf(stmt, relation)
	return statement
}

func getRelationshipFromPredicate(predicate string) (relation string) {
	if predicate != "" {
		relation = relations[predicate]
	} else {
		relation = relations["mentions"]
	}
	return relation
}

func createAnnotationQuery(contentUUID string, ann annotation, platformVersion string) (*neoism.CypherQuery, error) {
	query := neoism.CypherQuery{}
	thingID, err := extractUUIDFromURI(ann.Thing.ID)
	if err != nil {
		return nil, err
	}

	//todo temporary chnage to deal with multiple provenances
	/*if len(ann.Provenances) > 1 {
		return nil, errors.New("Cannot insert a MENTIONS annotation with multiple provenances")
	}*/

	var prov provenance
	params := map[string]interface{}{}
	params["platformVersion"] = platformVersion

	if len(ann.Provenances) >= 1 {
		prov = ann.Provenances[0]
		annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, supplied, err := extractDataFromProvenance(&prov)

		if err != nil {
			log.Infof("ERROR=%s", err)
			return nil, err
		}

		if supplied == true {
			if annotatedBy != "" {
				params["annotatedBy"] = annotatedBy
			}
			if prov.AtTime != "" {
				params["annotatedDateEpoch"] = annotatedDateEpoch
				params["annotatedDate"] = prov.AtTime
			}
			params["relevanceScore"] = relevanceScore
			params["confidenceScore"] = confidenceScore
		}
	}

	relation := getRelationshipFromPredicate(ann.Thing.Predicate)
	query.Statement = createAnnotationRelationship(relation)
	query.Parameters = map[string]interface{}{
		"contentID":       contentUUID,
		"conceptID":       thingID,
		"platformVersion": platformVersion,
		"annProps":        params,
	}
	return &query, nil
}

func extractDataFromProvenance(prov *provenance) (string, int64, float64, float64, bool, error) {
	if len(prov.Scores) == 0 {
		return "", -1, -1, -1, false, nil
	}
	var annotatedBy string
	var annotatedDateEpoch int64
	var confidenceScore, relevanceScore float64
	var err error
	if prov.AgentRole != "" {
		annotatedBy, err = extractUUIDFromURI(prov.AgentRole)
	}
	if prov.AtTime != "" {
		annotatedDateEpoch, err = convertAnnotatedDateToEpoch(prov.AtTime)
	}
	relevanceScore, confidenceScore, err = extractScores(prov.Scores)

	if err != nil {
		return "", -1, -1, -1, true, err
	}
	return annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, true, nil
}

func extractUUIDFromURI(uri string) (string, error) {
	result := uuidExtractRegex.FindStringSubmatch(uri)
	if len(result) == 2 {
		return result[1], nil
	}
	return "", fmt.Errorf("Couldn't extract uuid from uri %s", uri)
}

func convertAnnotatedDateToEpoch(annotatedDateString string) (int64, error) {
	datetimeEpoch, err := time.Parse(time.RFC3339, annotatedDateString)

	if err != nil {
		return 0, err
	}

	return datetimeEpoch.Unix(), nil
}

func extractScores(scores []score) (float64, float64, error) {
	var relevanceScore, confidenceScore float64
	for _, score := range scores {
		scoringSystem := score.ScoringSystem
		value := score.Value
		switch scoringSystem {
		case relevanceScoringSystem:
			relevanceScore = value
		case confidenceScoringSystem:
			confidenceScore = value
		}
	}
	return relevanceScore, confidenceScore, nil
}

func dropAllAnnotationsQuery(contentUUID string, platformVersion string) *neoism.CypherQuery {
	matchStmtTemplate := `OPTIONAL MATCH (:Thing{uuid:{contentID}})-[r{platformVersion:{platformVersion}}]->(t:Thing)
												where NOT t:Brand
                        DELETE r`

	query := neoism.CypherQuery{}
	query.Statement = matchStmtTemplate
	query.Parameters = neoism.Props{"contentID": contentUUID, "platformVersion": platformVersion}
	return &query
}

func validateAnnotations(annotations *annotations) error {
	//TODO - for consistency, we should probably just not create the annotation?
	for _, annotation := range *annotations {
		if annotation.Thing.ID == "" {
			return ValidationError{fmt.Sprintf("Concept uuid missing for annotation %+v", annotation)}
		}
	}
	return nil
}

//ValidationError is thrown when the annotations are not valid because mandatory information is missing
type ValidationError struct {
	Msg string
}

func (v ValidationError) Error() string {
	return v.Msg
}

func mapToResponseFormat(ann *annotation) {
	ann.Thing.ID = mapper.IDURL(ann.Thing.ID)
	// We expect only ONE provenance - provenance value is considered valid even if the AgentRole is not specified. See: v1 - isClassifiedBy
	for idx := range ann.Provenances {
		if ann.Provenances[idx].AgentRole != "" {
			ann.Provenances[idx].AgentRole = mapper.IDURL(ann.Provenances[idx].AgentRole)
		}
	}
}
<|endoftext|>


<commit_before>package nomad

import (
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/testutil"
)

func TestWorker_dequeueEvaluation(t *testing.T) {
	s1 := testServer(t, func(c *Config) {
		c.NumSchedulers = 0
		c.EnabledSchedulers = []string{structs.JobTypeService}
	})
	defer s1.Shutdown()
	testutil.WaitForLeader(t, s1.RPC)

	// Create the evaluation
	eval1 := mockEval()
	s1.evalBroker.Enqueue(eval1)

	// Create a worker
	w := &Worker{srv: s1, logger: s1.logger}

	// Attempt dequeue
	eval, shutdown := w.dequeueEvaluation(10 * time.Millisecond)
	if shutdown {
		t.Fatalf("should not shutdown")
	}

	// Ensure we get a sane eval
	if !reflect.DeepEqual(eval, eval1) {
		t.Fatalf("bad: %#v %#v", eval, eval1)
	}
}

func TestWorker_dequeueEvaluation_shutdown(t *testing.T) {
	s1 := testServer(t, func(c *Config) {
		c.NumSchedulers = 0
		c.EnabledSchedulers = []string{structs.JobTypeService}
	})
	defer s1.Shutdown()
	testutil.WaitForLeader(t, s1.RPC)

	// Create a worker
	w := &Worker{srv: s1, logger: s1.logger}

	go func() {
		time.Sleep(10 * time.Millisecond)
		s1.Shutdown()
	}()

	// Attempt dequeue
	eval, shutdown := w.dequeueEvaluation(10 * time.Millisecond)
	if !shutdown {
		t.Fatalf("should not shutdown")
	}

	// Ensure we get a sane eval
	if eval != nil {
		t.Fatalf("bad: %#v", eval)
	}
}

func TestWorker_sendAck(t *testing.T) {
	s1 := testServer(t, func(c *Config) {
		c.NumSchedulers = 0
		c.EnabledSchedulers = []string{structs.JobTypeService}
	})
	defer s1.Shutdown()
	testutil.WaitForLeader(t, s1.RPC)

	// Create the evaluation
	eval1 := mockEval()
	s1.evalBroker.Enqueue(eval1)

	// Create a worker
	w := &Worker{srv: s1, logger: s1.logger}

	// Attempt dequeue
	eval, _ := w.dequeueEvaluation(10 * time.Millisecond)

	// Check the depth is 0, 1 unacked
	stats := s1.evalBroker.Stats()
	if stats.TotalReady != 0 && stats.TotalUnacked != 1 {
		t.Fatalf("bad: %#v", stats)
	}

	// Send the Nack
	w.sendAck(eval.ID, false)

	// Check the depth is 1, nothing unacked
	stats = s1.evalBroker.Stats()
	if stats.TotalReady != 1 && stats.TotalUnacked != 0 {
		t.Fatalf("bad: %#v", stats)
	}

	// Attempt dequeue
	eval, _ = w.dequeueEvaluation(10 * time.Millisecond)

	// Send the Ack
	w.sendAck(eval.ID, true)

	// Check the depth is 0
	stats = s1.evalBroker.Stats()
	if stats.TotalReady != 0 && stats.TotalUnacked != 0 {
		t.Fatalf("bad: %#v", stats)
	}
}

func TestWorker_waitForIndex(t *testing.T) {
	s1 := testServer(t, func(c *Config) {
		c.NumSchedulers = 0
		c.EnabledSchedulers = []string{structs.JobTypeService}
	})
	defer s1.Shutdown()
	testutil.WaitForLeader(t, s1.RPC)

	// Get the current index
	index := s1.raft.AppliedIndex()

	// Cause an increment
	go func() {
		time.Sleep(10 * time.Millisecond)
		s1.raft.Barrier(0)
	}()

	// Wait for a future index
	w := &Worker{srv: s1, logger: s1.logger}
	err := w.waitForIndex(index+1, time.Second)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Cause a timeout
	err = w.waitForIndex(index+100, 10*time.Millisecond)
	if err == nil || !strings.Contains(err.Error(), "timeout") {
		t.Fatalf("err: %v", err)
	}
}
<commit_msg>nomad: more worker testing<commit_after>package nomad

import (
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/scheduler"
	"github.com/hashicorp/nomad/testutil"
)

type NoopScheduler struct {
	state   scheduler.State
	planner scheduler.Planner
	eval    *structs.Evaluation
	err     error
}

func (n *NoopScheduler) Process(eval *structs.Evaluation) error {
	if n.state == nil {
		panic("missing state")
	}
	if n.planner == nil {
		panic("missing planner")
	}
	n.eval = eval
	return n.err
}

func init() {
	scheduler.BuiltinSchedulers["noop"] = func(s scheduler.State, p scheduler.Planner) scheduler.Scheduler {
		n := &NoopScheduler{
			state:   s,
			planner: p,
		}
		return n
	}
}

func TestWorker_dequeueEvaluation(t *testing.T) {
	s1 := testServer(t, func(c *Config) {
		c.NumSchedulers = 0
		c.EnabledSchedulers = []string{structs.JobTypeService}
	})
	defer s1.Shutdown()
	testutil.WaitForLeader(t, s1.RPC)

	// Create the evaluation
	eval1 := mockEval()
	s1.evalBroker.Enqueue(eval1)

	// Create a worker
	w := &Worker{srv: s1, logger: s1.logger}

	// Attempt dequeue
	eval, shutdown := w.dequeueEvaluation(10 * time.Millisecond)
	if shutdown {
		t.Fatalf("should not shutdown")
	}

	// Ensure we get a sane eval
	if !reflect.DeepEqual(eval, eval1) {
		t.Fatalf("bad: %#v %#v", eval, eval1)
	}
}

func TestWorker_dequeueEvaluation_shutdown(t *testing.T) {
	s1 := testServer(t, func(c *Config) {
		c.NumSchedulers = 0
		c.EnabledSchedulers = []string{structs.JobTypeService}
	})
	defer s1.Shutdown()
	testutil.WaitForLeader(t, s1.RPC)

	// Create a worker
	w := &Worker{srv: s1, logger: s1.logger}

	go func() {
		time.Sleep(10 * time.Millisecond)
		s1.Shutdown()
	}()

	// Attempt dequeue
	eval, shutdown := w.dequeueEvaluation(10 * time.Millisecond)
	if !shutdown {
		t.Fatalf("should not shutdown")
	}

	// Ensure we get a sane eval
	if eval != nil {
		t.Fatalf("bad: %#v", eval)
	}
}

func TestWorker_sendAck(t *testing.T) {
	s1 := testServer(t, func(c *Config) {
		c.NumSchedulers = 0
		c.EnabledSchedulers = []string{structs.JobTypeService}
	})
	defer s1.Shutdown()
	testutil.WaitForLeader(t, s1.RPC)

	// Create the evaluation
	eval1 := mockEval()
	s1.evalBroker.Enqueue(eval1)

	// Create a worker
	w := &Worker{srv: s1, logger: s1.logger}

	// Attempt dequeue
	eval, _ := w.dequeueEvaluation(10 * time.Millisecond)

	// Check the depth is 0, 1 unacked
	stats := s1.evalBroker.Stats()
	if stats.TotalReady != 0 && stats.TotalUnacked != 1 {
		t.Fatalf("bad: %#v", stats)
	}

	// Send the Nack
	w.sendAck(eval.ID, false)

	// Check the depth is 1, nothing unacked
	stats = s1.evalBroker.Stats()
	if stats.TotalReady != 1 && stats.TotalUnacked != 0 {
		t.Fatalf("bad: %#v", stats)
	}

	// Attempt dequeue
	eval, _ = w.dequeueEvaluation(10 * time.Millisecond)

	// Send the Ack
	w.sendAck(eval.ID, true)

	// Check the depth is 0
	stats = s1.evalBroker.Stats()
	if stats.TotalReady != 0 && stats.TotalUnacked != 0 {
		t.Fatalf("bad: %#v", stats)
	}
}

func TestWorker_waitForIndex(t *testing.T) {
	s1 := testServer(t, func(c *Config) {
		c.NumSchedulers = 0
		c.EnabledSchedulers = []string{structs.JobTypeService}
	})
	defer s1.Shutdown()
	testutil.WaitForLeader(t, s1.RPC)

	// Get the current index
	index := s1.raft.AppliedIndex()

	// Cause an increment
	go func() {
		time.Sleep(10 * time.Millisecond)
		s1.raft.Barrier(0)
	}()

	// Wait for a future index
	w := &Worker{srv: s1, logger: s1.logger}
	err := w.waitForIndex(index+1, time.Second)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Cause a timeout
	err = w.waitForIndex(index+100, 10*time.Millisecond)
	if err == nil || !strings.Contains(err.Error(), "timeout") {
		t.Fatalf("err: %v", err)
	}
}

func TestWorker_invokeScheduler(t *testing.T) {
	s1 := testServer(t, func(c *Config) {
		c.NumSchedulers = 0
		c.EnabledSchedulers = []string{structs.JobTypeService}
	})
	defer s1.Shutdown()

	w := &Worker{srv: s1, logger: s1.logger}
	eval := mockEval()
	eval.Type = "noop"

	err := w.invokeScheduler(eval)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
}

func TestWorker_SubmitPlan(t *testing.T) {
	s1 := testServer(t, func(c *Config) {
		c.NumSchedulers = 0
		c.EnabledSchedulers = []string{structs.JobTypeService}
	})
	defer s1.Shutdown()
	testutil.WaitForLeader(t, s1.RPC)

	// TODO: This requires the plan apply to work
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package config

import (
	"fmt"

	"github.com/syncthing/syncthing/lib/util"
)

type OptionsConfiguration struct {
	ListenAddresses         []string `xml:"listenAddress" json:"listenAddresses" default:"default"`
	GlobalAnnServers        []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"default" restart:"true"`
	GlobalAnnEnabled        bool     `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true" restart:"true"`
	LocalAnnEnabled         bool     `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true" restart:"true"`
	LocalAnnPort            int      `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027" restart:"true"`
	LocalAnnMCAddr          string   `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff12::8384]:21027" restart:"true"`
	MaxSendKbps             int      `xml:"maxSendKbps" json:"maxSendKbps"`
	MaxRecvKbps             int      `xml:"maxRecvKbps" json:"maxRecvKbps"`
	ReconnectIntervalS      int      `xml:"reconnectionIntervalS" json:"reconnectionIntervalS" default:"60"`
	RelaysEnabled           bool     `xml:"relaysEnabled" json:"relaysEnabled" default:"true"`
	RelayReconnectIntervalM int      `xml:"relayReconnectIntervalM" json:"relayReconnectIntervalM" default:"10"`
	StartBrowser            bool     `xml:"startBrowser" json:"startBrowser" default:"true"`
	NATEnabled              bool     `xml:"natEnabled" json:"natEnabled" default:"true"`
	NATLeaseM               int      `xml:"natLeaseMinutes" json:"natLeaseMinutes" default:"60"`
	NATRenewalM             int      `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
	NATTimeoutS             int      `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
	URAccepted              int      `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
	URSeen                  int      `xml:"urSeen" json:"urSeen"`         // Report which the user has been prompted for.
	URUniqueID              string   `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
	URURL                   string   `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
	URPostInsecurely        bool     `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing
	URInitialDelayS         int      `xml:"urInitialDelayS" json:"urInitialDelayS" default:"1800"`
	RestartOnWakeup         bool     `xml:"restartOnWakeup" json:"restartOnWakeup" default:"true" restart:"true"`
	AutoUpgradeIntervalH    int      `xml:"autoUpgradeIntervalH" json:"autoUpgradeIntervalH" default:"12" restart:"true"` // 0 for off
	UpgradeToPreReleases    bool     `xml:"upgradeToPreReleases" json:"upgradeToPreReleases" restart:"true"`              // when auto upgrades are enabled
	KeepTemporariesH        int      `xml:"keepTemporariesH" json:"keepTemporariesH" default:"24"`                        // 0 for off
	CacheIgnoredFiles       bool     `xml:"cacheIgnoredFiles" json:"cacheIgnoredFiles" default:"false" restart:"true"`
	ProgressUpdateIntervalS int      `xml:"progressUpdateIntervalS" json:"progressUpdateIntervalS" default:"5"`
	LimitBandwidthInLan     bool     `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
	MinHomeDiskFree         Size     `xml:"minHomeDiskFree" json:"minHomeDiskFree" default:"1 %"`
	ReleasesURL             string   `xml:"releasesURL" json:"releasesURL" default:"https://upgrades.syncthing.net/meta.json" restart:"true"`
	AlwaysLocalNets         []string `xml:"alwaysLocalNet" json:"alwaysLocalNets"`
	OverwriteRemoteDevNames bool     `xml:"overwriteRemoteDeviceNamesOnConnect" json:"overwriteRemoteDeviceNamesOnConnect" default:"false"`
	TempIndexMinBlocks      int      `xml:"tempIndexMinBlocks" json:"tempIndexMinBlocks" default:"10"`
	UnackedNotificationIDs  []string `xml:"unackedNotificationID" json:"unackedNotificationIDs"`
	TrafficClass            int      `xml:"trafficClass" json:"trafficClass"`
	DefaultFolderPath       string   `xml:"defaultFolderPath" json:"defaultFolderPath" default:"~"`
	SetLowPriority          bool     `xml:"setLowPriority" json:"setLowPriority" default:"true"`
	MaxConcurrentScans      int      `xml:"maxConcurrentScans" json:"maxConcurrentScans"`

	DeprecatedUPnPEnabled        bool     `xml:"upnpEnabled,omitempty" json:"-"`
	DeprecatedUPnPLeaseM         int      `xml:"upnpLeaseMinutes,omitempty" json:"-"`
	DeprecatedUPnPRenewalM       int      `xml:"upnpRenewalMinutes,omitempty" json:"-"`
	DeprecatedUPnPTimeoutS       int      `xml:"upnpTimeoutSeconds,omitempty" json:"-"`
	DeprecatedRelayServers       []string `xml:"relayServer,omitempty" json:"-"`
	DeprecatedMinHomeDiskFreePct float64  `xml:"minHomeDiskFreePct" json:"-"`
}

func (orig OptionsConfiguration) Copy() OptionsConfiguration {
	c := orig
	c.ListenAddresses = make([]string, len(orig.ListenAddresses))
	copy(c.ListenAddresses, orig.ListenAddresses)
	c.GlobalAnnServers = make([]string, len(orig.GlobalAnnServers))
	copy(c.GlobalAnnServers, orig.GlobalAnnServers)
	c.AlwaysLocalNets = make([]string, len(orig.AlwaysLocalNets))
	copy(c.AlwaysLocalNets, orig.AlwaysLocalNets)
	c.UnackedNotificationIDs = make([]string, len(orig.UnackedNotificationIDs))
	copy(c.UnackedNotificationIDs, orig.UnackedNotificationIDs)
	return c
}

// RequiresRestartOnly returns a copy with only the attributes that require
// restart on change.
func (orig OptionsConfiguration) RequiresRestartOnly() OptionsConfiguration {
	copy := orig
	blank := OptionsConfiguration{}
	util.CopyMatchingTag(&blank, &copy, "restart", func(v string) bool {
		if len(v) > 0 && v != "true" {
			panic(fmt.Sprintf(`unexpected tag value: %s. expected untagged or "true"`, v))
		}
		return v != "true"
	})
	return copy
}
<commit_msg>lib/config: Add omitempty to DeprecatedMinHomeDiskFreePct (fixes #5482) (#5484)<commit_after>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package config

import (
	"fmt"

	"github.com/syncthing/syncthing/lib/util"
)

type OptionsConfiguration struct {
	ListenAddresses         []string `xml:"listenAddress" json:"listenAddresses" default:"default"`
	GlobalAnnServers        []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"default" restart:"true"`
	GlobalAnnEnabled        bool     `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true" restart:"true"`
	LocalAnnEnabled         bool     `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true" restart:"true"`
	LocalAnnPort            int      `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027" restart:"true"`
	LocalAnnMCAddr          string   `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff12::8384]:21027" restart:"true"`
	MaxSendKbps             int      `xml:"maxSendKbps" json:"maxSendKbps"`
	MaxRecvKbps             int      `xml:"maxRecvKbps" json:"maxRecvKbps"`
	ReconnectIntervalS      int      `xml:"reconnectionIntervalS" json:"reconnectionIntervalS" default:"60"`
	RelaysEnabled           bool     `xml:"relaysEnabled" json:"relaysEnabled" default:"true"`
	RelayReconnectIntervalM int      `xml:"relayReconnectIntervalM" json:"relayReconnectIntervalM" default:"10"`
	StartBrowser            bool     `xml:"startBrowser" json:"startBrowser" default:"true"`
	NATEnabled              bool     `xml:"natEnabled" json:"natEnabled" default:"true"`
	NATLeaseM               int      `xml:"natLeaseMinutes" json:"natLeaseMinutes" default:"60"`
	NATRenewalM             int      `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
	NATTimeoutS             int      `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
	URAccepted              int      `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
	URSeen                  int      `xml:"urSeen" json:"urSeen"`         // Report which the user has been prompted for.
	URUniqueID              string   `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
	URURL                   string   `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
	URPostInsecurely        bool     `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing
	URInitialDelayS         int      `xml:"urInitialDelayS" json:"urInitialDelayS" default:"1800"`
	RestartOnWakeup         bool     `xml:"restartOnWakeup" json:"restartOnWakeup" default:"true" restart:"true"`
	AutoUpgradeIntervalH    int      `xml:"autoUpgradeIntervalH" json:"autoUpgradeIntervalH" default:"12" restart:"true"` // 0 for off
	UpgradeToPreReleases    bool     `xml:"upgradeToPreReleases" json:"upgradeToPreReleases" restart:"true"`              // when auto upgrades are enabled
	KeepTemporariesH        int      `xml:"keepTemporariesH" json:"keepTemporariesH" default:"24"`                        // 0 for off
	CacheIgnoredFiles       bool     `xml:"cacheIgnoredFiles" json:"cacheIgnoredFiles" default:"false" restart:"true"`
	ProgressUpdateIntervalS int      `xml:"progressUpdateIntervalS" json:"progressUpdateIntervalS" default:"5"`
	LimitBandwidthInLan     bool     `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
	MinHomeDiskFree         Size     `xml:"minHomeDiskFree" json:"minHomeDiskFree" default:"1 %"`
	ReleasesURL             string   `xml:"releasesURL" json:"releasesURL" default:"https://upgrades.syncthing.net/meta.json" restart:"true"`
	AlwaysLocalNets         []string `xml:"alwaysLocalNet" json:"alwaysLocalNets"`
	OverwriteRemoteDevNames bool     `xml:"overwriteRemoteDeviceNamesOnConnect" json:"overwriteRemoteDeviceNamesOnConnect" default:"false"`
	TempIndexMinBlocks      int      `xml:"tempIndexMinBlocks" json:"tempIndexMinBlocks" default:"10"`
	UnackedNotificationIDs  []string `xml:"unackedNotificationID" json:"unackedNotificationIDs"`
	TrafficClass            int      `xml:"trafficClass" json:"trafficClass"`
	DefaultFolderPath       string   `xml:"defaultFolderPath" json:"defaultFolderPath" default:"~"`
	SetLowPriority          bool     `xml:"setLowPriority" json:"setLowPriority" default:"true"`
	MaxConcurrentScans      int      `xml:"maxConcurrentScans" json:"maxConcurrentScans"`

	DeprecatedUPnPEnabled        bool     `xml:"upnpEnabled,omitempty" json:"-"`
	DeprecatedUPnPLeaseM         int      `xml:"upnpLeaseMinutes,omitempty" json:"-"`
	DeprecatedUPnPRenewalM       int      `xml:"upnpRenewalMinutes,omitempty" json:"-"`
	DeprecatedUPnPTimeoutS       int      `xml:"upnpTimeoutSeconds,omitempty" json:"-"`
	DeprecatedRelayServers       []string `xml:"relayServer,omitempty" json:"-"`
	DeprecatedMinHomeDiskFreePct float64  `xml:"minHomeDiskFreePct,omitempty" json:"-"`
}

func (orig OptionsConfiguration) Copy() OptionsConfiguration {
	c := orig
	c.ListenAddresses = make([]string, len(orig.ListenAddresses))
	copy(c.ListenAddresses, orig.ListenAddresses)
	c.GlobalAnnServers = make([]string, len(orig.GlobalAnnServers))
	copy(c.GlobalAnnServers, orig.GlobalAnnServers)
	c.AlwaysLocalNets = make([]string, len(orig.AlwaysLocalNets))
	copy(c.AlwaysLocalNets, orig.AlwaysLocalNets)
	c.UnackedNotificationIDs = make([]string, len(orig.UnackedNotificationIDs))
	copy(c.UnackedNotificationIDs, orig.UnackedNotificationIDs)
	return c
}

// RequiresRestartOnly returns a copy with only the attributes that require
// restart on change.
func (orig OptionsConfiguration) RequiresRestartOnly() OptionsConfiguration {
	copy := orig
	blank := OptionsConfiguration{}
	util.CopyMatchingTag(&blank, &copy, "restart", func(v string) bool {
		if len(v) > 0 && v != "true" {
			panic(fmt.Sprintf(`unexpected tag value: %s. expected untagged or "true"`, v))
		}
		return v != "true"
	})
	return copy
}
<|endoftext|>


<commit_before>// Copyright 2013 The rerun AUTHORS. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/howeyc/fsnotify"
	"go/build"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
)

var (
	do_tests  = flag.Bool("test", false, "Run tests before running program.")
	test_only = flag.Bool("test-only", false, "Only run tests.")
)

func install(buildpath, lastError string) (installed bool, errorOutput string, err error) {
	cmdline := []string{"go", "get", buildpath}

	// setup the build command, use a shared buffer for both stdOut and stdErr
	cmd := exec.Command("go", cmdline[1:]...)
	buf := bytes.NewBuffer([]byte{})
	cmd.Stdout = buf
	cmd.Stderr = buf

	err = cmd.Run()

	// when there is any output, the go command failed.
	if buf.Len() > 0 {
		errorOutput = buf.String()
		if errorOutput != lastError {
			fmt.Print(errorOutput)
		}
		err = errors.New("compile error")
		return
	}

	// all seems fine
	installed = true
	return
}

func test(buildpath string) (passed bool, err error) {
	cmdline := []string{"go", "test", "-v", buildpath}

	// setup the build command, use a shared buffer for both stdOut and stdErr
	cmd := exec.Command("go", cmdline[1:]...)
	buf := bytes.NewBuffer([]byte{})
	cmd.Stdout = buf
	cmd.Stderr = buf

	err = cmd.Run()
	passed = err == nil

	if !passed {
		fmt.Println(buf)
	} else {
		log.Println("tests passed")
	}

	return
}

func run(binName, binPath string, args []string) (runch chan bool) {
	runch = make(chan bool)
	go func() {
		cmdline := append([]string{binName}, args...)
		var proc *os.Process
		for relaunch := range runch {
			if proc != nil {
				proc.Kill()
			}
			if !relaunch {
				continue
			}
			cmd := exec.Command(binPath, args...)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			log.Print(cmdline)
			err := cmd.Start()
			if err != nil {
				log.Printf("error on starting process: '%s'\n", err)
			}
			proc = cmd.Process
		}
	}()
	return
}

func getWatcher(buildpath string) (watcher *fsnotify.Watcher, err error) {
	watcher, err = fsnotify.NewWatcher()
	addToWatcher(watcher, buildpath, map[string]bool{})
	return
}

func addToWatcher(watcher *fsnotify.Watcher, importpath string, watching map[string]bool) {
	pkg, err := build.Import(importpath, "", 0)
	if err != nil {
		return
	}
	if pkg.Goroot {
		return
	}
	watcher.Watch(pkg.Dir)
	watching[importpath] = true
	for _, imp := range pkg.Imports {
		if !watching[imp] {
			addToWatcher(watcher, imp, watching)
		}
	}
}

func rerun(buildpath string, args []string) (err error) {
	log.Printf("setting up %s %v", buildpath, args)

	pkg, err := build.Import(buildpath, "", 0)
	if err != nil {
		return
	}

	if pkg.Name != "main" {
		err = errors.New(fmt.Sprintf("expected package %q, got %q", "main", pkg.Name))
		return
	}

	_, binName := path.Split(buildpath)
	var binPath string
	if gobin := os.Getenv("GOBIN"); gobin != "" {
		binPath = filepath.Join(gobin, binName)
	} else {
		binPath = filepath.Join(pkg.BinDir, binName)
	}

	var runch chan bool
	if !(*test_only) {
		runch = run(binName, binPath, args)
	}

	no_run := false
	if *do_tests {
		passed, _ := test(buildpath)
		if !passed {
			no_run = true
		}
	}

	var errorOutput string
	_, errorOutput, ierr := install(buildpath, errorOutput)
	if !no_run && !(*test_only) && ierr == nil {
		runch <- true
	}

	var watcher *fsnotify.Watcher
	watcher, err = getWatcher(buildpath)
	if err != nil {
		return
	}

	for {
		// read event from the watcher
		we, _ := <-watcher.Event

		log.Print(we.Name)

		// close the watcher
		watcher.Close()
		// to clean things up: read events from the watcher until events chan is closed.
		go func(events chan *fsnotify.FileEvent) {
			for _ = range events {

			}
		}(watcher.Event)
		// create a new watcher
		log.Println("rescanning")
		watcher, err = getWatcher(buildpath)
		if err != nil {
			return
		}

		// we don't need the errors from the new watcher.
		// we continiously discard them from the channel to avoid a deadlock.
		go func(errors chan error) {
			for _ = range errors {

			}
		}(watcher.Error)

		var installed bool
		// rebuild
		installed, errorOutput, _ = install(buildpath, errorOutput)
		if !installed {
			continue
		}

		if *do_tests {
			passed, _ := test(buildpath)
			if !passed {
				continue
			}
		}

		// rerun. if we're only testing, sending
		if !(*test_only) {
			runch <- true
		}
	}
	return
}

func main() {
	flag.Parse()
	if *test_only {
		*do_tests = true
	}

	if len(flag.Args()) < 1 {
		log.Fatal("Usage: rerun [--test] [--test-only] <import path> [arg]*")
	}

	buildpath := flag.Args()[0]
	args := flag.Args()[1:]
	err := rerun(buildpath, args)
	if err != nil {
		log.Print(err)
	}
}
<commit_msg>filter out non-.go files from watch events<commit_after>// Copyright 2013 The rerun AUTHORS. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/howeyc/fsnotify"
	"go/build"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
)

var (
	do_tests  = flag.Bool("test", false, "Run tests before running program.")
	test_only = flag.Bool("test-only", false, "Only run tests.")
)

func install(buildpath, lastError string) (installed bool, errorOutput string, err error) {
	cmdline := []string{"go", "get", buildpath}

	// setup the build command, use a shared buffer for both stdOut and stdErr
	cmd := exec.Command("go", cmdline[1:]...)
	buf := bytes.NewBuffer([]byte{})
	cmd.Stdout = buf
	cmd.Stderr = buf

	err = cmd.Run()

	// when there is any output, the go command failed.
	if buf.Len() > 0 {
		errorOutput = buf.String()
		if errorOutput != lastError {
			fmt.Print(errorOutput)
		}
		err = errors.New("compile error")
		return
	}

	// all seems fine
	installed = true
	return
}

func test(buildpath string) (passed bool, err error) {
	cmdline := []string{"go", "test", "-v", buildpath}

	// setup the build command, use a shared buffer for both stdOut and stdErr
	cmd := exec.Command("go", cmdline[1:]...)
	buf := bytes.NewBuffer([]byte{})
	cmd.Stdout = buf
	cmd.Stderr = buf

	err = cmd.Run()
	passed = err == nil

	if !passed {
		fmt.Println(buf)
	} else {
		log.Println("tests passed")
	}

	return
}

func run(binName, binPath string, args []string) (runch chan bool) {
	runch = make(chan bool)
	go func() {
		cmdline := append([]string{binName}, args...)
		var proc *os.Process
		for relaunch := range runch {
			if proc != nil {
				proc.Kill()
			}
			if !relaunch {
				continue
			}
			cmd := exec.Command(binPath, args...)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			log.Print(cmdline)
			err := cmd.Start()
			if err != nil {
				log.Printf("error on starting process: '%s'\n", err)
			}
			proc = cmd.Process
		}
	}()
	return
}

func getWatcher(buildpath string) (watcher *fsnotify.Watcher, err error) {
	watcher, err = fsnotify.NewWatcher()
	addToWatcher(watcher, buildpath, map[string]bool{})
	return
}

func addToWatcher(watcher *fsnotify.Watcher, importpath string, watching map[string]bool) {
	pkg, err := build.Import(importpath, "", 0)
	if err != nil {
		return
	}
	if pkg.Goroot {
		return
	}
	watcher.Watch(pkg.Dir)
	watching[importpath] = true
	for _, imp := range pkg.Imports {
		if !watching[imp] {
			addToWatcher(watcher, imp, watching)
		}
	}
}

func rerun(buildpath string, args []string) (err error) {
	log.Printf("setting up %s %v", buildpath, args)

	pkg, err := build.Import(buildpath, "", 0)
	if err != nil {
		return
	}

	if pkg.Name != "main" {
		err = errors.New(fmt.Sprintf("expected package %q, got %q", "main", pkg.Name))
		return
	}

	_, binName := path.Split(buildpath)
	var binPath string
	if gobin := os.Getenv("GOBIN"); gobin != "" {
		binPath = filepath.Join(gobin, binName)
	} else {
		binPath = filepath.Join(pkg.BinDir, binName)
	}

	var runch chan bool
	if !(*test_only) {
		runch = run(binName, binPath, args)
	}

	no_run := false
	if *do_tests {
		passed, _ := test(buildpath)
		if !passed {
			no_run = true
		}
	}

	var errorOutput string
	_, errorOutput, ierr := install(buildpath, errorOutput)
	if !no_run && !(*test_only) && ierr == nil {
		runch <- true
	}

	var watcher *fsnotify.Watcher
	watcher, err = getWatcher(buildpath)
	if err != nil {
		return
	}

	for {
		// read event from the watcher
		we, _ := <-watcher.Event
		// other files in the directory don't count - we watch the whole thing in case new .go files appear.
		if filepath.Ext(we.Name) != ".go" {
			continue
		}

		log.Print(we.Name)

		// close the watcher
		watcher.Close()
		// to clean things up: read events from the watcher until events chan is closed.
		go func(events chan *fsnotify.FileEvent) {
			for _ = range events {

			}
		}(watcher.Event)
		// create a new watcher
		log.Println("rescanning")
		watcher, err = getWatcher(buildpath)
		if err != nil {
			return
		}

		// we don't need the errors from the new watcher.
		// we continiously discard them from the channel to avoid a deadlock.
		go func(errors chan error) {
			for _ = range errors {

			}
		}(watcher.Error)

		var installed bool
		// rebuild
		installed, errorOutput, _ = install(buildpath, errorOutput)
		if !installed {
			continue
		}

		if *do_tests {
			passed, _ := test(buildpath)
			if !passed {
				continue
			}
		}

		// rerun. if we're only testing, sending
		if !(*test_only) {
			runch <- true
		}
	}
	return
}

func main() {
	flag.Parse()
	if *test_only {
		*do_tests = true
	}

	if len(flag.Args()) < 1 {
		log.Fatal("Usage: rerun [--test] [--test-only] <import path> [arg]*")
	}

	buildpath := flag.Args()[0]
	args := flag.Args()[1:]
	err := rerun(buildpath, args)
	if err != nil {
		log.Print(err)
	}
}
<|endoftext|>


<commit_before>package hal

import (
	"log"
	"os"
	"os/signal"
	"syscall"
)

// Robot receives messages from an adapter and sends them to listeners
type Robot struct {
	*Config
	Name       string
	Alias      string
	Adapter    Adapter
	Logger     *log.Logger
	handlers   []Handler
	signalChan chan os.Signal

	// Listeners map[string][]Handler
}

// Handlers returns the robot's handlers
func (robot *Robot) Handlers() []Handler {
	return robot.handlers
}

const (
	RobotRegex = `^(?:(?:h|hubot)[:,]?)\s+(?:(do .+))`
)

// NewRobot returns a new Robot instance
func NewRobot() (*Robot, error) {
	robot := &Robot{}
	config := NewConfig()

	adapter, err := NewAdapter(config.AdapterName)
	if err != nil {
		log.Println(err)
		return nil, err
	}
	adapter.SetRobot(robot)

	robot.Name = config.Name
	robot.Logger = config.Logger
	robot.Adapter = adapter
	robot.signalChan = make(chan os.Signal, 1)

	return robot, nil
}

// Handle registers a new handler with the robot
func (robot *Robot) Handle(handlers ...Handler) {
	robot.handlers = append(robot.handlers, handlers...)
}

// Receive dispatches messages to our handlers
func (robot *Robot) Receive(msg *Message) error {
	for _, handler := range robot.handlers {
		response := NewResponse(robot, msg)

		err := handler.Handle(response)
		if err != nil {
			return err
		}
	}
	return nil
}

// Stop initiates the shutdown process
func (robot *Robot) Stop() error {
	robot.Adapter.Stop()
	log.Println("Shutting down...")

	return nil
}

// Run starts up the robot
func (robot *Robot) Run() error {
	defer robot.Stop()

	stop := false

	go robot.Adapter.Run()

	signal.Notify(robot.signalChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)

	for !stop {
		select {
		case sig := <-robot.signalChan:
			switch sig {
			case syscall.SIGHUP:
				log.Println("Reloading...")
			case syscall.SIGINT, syscall.SIGTERM:
				stop = true
			}
		}
	}

	return nil
}

func (robot *Robot) RespondRegex(pattern string) string {
	str := `^(?:`
	if robot.Alias != "" {
		str += `(?:` + robot.Alias + `|` + robot.Name + `)`
	} else {
		str += robot.Name
	}
	str += `[:,]?)\s+(?:` + pattern + `)`
	return str
	// return `^(?:(?:h|hubot)[:,]?)\s+(?:` + pattern + `)`
}
<commit_msg>code cleanup<commit_after>package hal

import (
	"log"
	"os"
	"os/signal"
	"syscall"
)

// Robot receives messages from an adapter and sends them to listeners
type Robot struct {
	*Config
	Name       string
	Alias      string
	Adapter    Adapter
	Logger     *log.Logger
	handlers   []Handler
	signalChan chan os.Signal

	// Listeners map[string][]Handler
}

// Handlers returns the robot's handlers
func (robot *Robot) Handlers() []Handler {
	return robot.handlers
}

// NewRobot returns a new Robot instance
func NewRobot() (*Robot, error) {
	robot := &Robot{}
	config := NewConfig()

	adapter, err := NewAdapter(config.AdapterName)
	if err != nil {
		log.Println(err)
		return nil, err
	}
	adapter.SetRobot(robot)

	robot.Name = config.Name
	robot.Logger = config.Logger
	robot.Adapter = adapter
	robot.signalChan = make(chan os.Signal, 1)

	return robot, nil
}

// Handle registers a new handler with the robot
func (robot *Robot) Handle(handlers ...Handler) {
	robot.handlers = append(robot.handlers, handlers...)
}

// Receive dispatches messages to our handlers
func (robot *Robot) Receive(msg *Message) error {
	for _, handler := range robot.handlers {
		response := NewResponse(robot, msg)

		err := handler.Handle(response)
		if err != nil {
			return err
		}
	}
	return nil
}

// Stop initiates the shutdown process
func (robot *Robot) Stop() error {
	robot.Adapter.Stop()
	log.Println("Shutting down...")

	return nil
}

// Run starts up the robot
func (robot *Robot) Run() error {
	defer robot.Stop()

	stop := false

	go robot.Adapter.Run()

	signal.Notify(robot.signalChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)

	for !stop {
		select {
		case sig := <-robot.signalChan:
			switch sig {
			case syscall.SIGHUP:
				log.Println("Reloading...")
			case syscall.SIGINT, syscall.SIGTERM:
				stop = true
			}
		}
	}

	return nil
}

func (robot *Robot) RespondRegex(pattern string) string {
	str := `^(?:`
	if robot.Alias != "" {
		str += `(?:` + robot.Alias + `|` + robot.Name + `)`
	} else {
		str += robot.Name
	}
	str += `[:,]?)\s+(?:` + pattern + `)`
	return str
	// return `^(?:(?:h|hubot)[:,]?)\s+(?:` + pattern + `)`
}
<|endoftext|>


<commit_before>package node

import (
	"fmt"
	"os"
	"strconv"
	"time"

	"golang.org/x/net/context"

	client "github.com/coreos/etcd/clientv3"

	"sunteng/commons/log"
	"sunteng/commons/util"
	"sunteng/cronsun/conf"
	"sunteng/cronsun/models"
	"sunteng/cronsun/node/cron"
)

// Node 执行 cron 命令服务的结构体
type Node struct {
	*models.Client
	*models.Node
	*cron.Cron

	jobs   Jobs
	groups Groups
	cmds   map[string]*models.Cmd

	link
	// 删除的 job id，用于 group 更新
	delIDs map[string]bool

	ttl int64

	lID client.LeaseID // lease id
	lch <-chan *client.LeaseKeepAliveResponse

	done chan struct{}
}

func NewNode(cfg *conf.Conf) (n *Node, err error) {
	ip, err := util.GetLocalIP()
	if err != nil {
		return
	}

	n = &Node{
		Client: models.DefalutClient,
		Node: &models.Node{
			ID:  ip.String(),
			PID: strconv.Itoa(os.Getpid()),
		},
		Cron: cron.New(),

		cmds: make(map[string]*models.Cmd),

		ttl:  cfg.Ttl,
		done: make(chan struct{}),
	}
	return
}

// 注册到 /cronsun/proc/xx
func (n *Node) Register() (err error) {
	pid, err := n.Node.Exist()
	if err != nil {
		return
	}

	if pid != -1 {
		return fmt.Errorf("node[%s] pid[%d] exist", n.Node.ID, pid)
	}

	resp, err := n.Client.Grant(context.TODO(), n.ttl)
	if err != nil {
		return
	}

	if _, err = n.Node.Put(client.WithLease(resp.ID)); err != nil {
		return
	}

	ch, err := n.Client.KeepAlive(context.TODO(), resp.ID)
	if err != nil {
		return
	}

	n.lID, n.lch = resp.ID, ch
	return
}

func (n *Node) loadJobs() (err error) {
	if n.groups, err = models.GetGroups(""); err != nil {
		return
	}

	jobs, err := models.GetJobs()
	if err != nil {
		return
	}

	n.jobs, n.link = make(Jobs, len(jobs)), newLink(len(n.groups))
	if len(jobs) == 0 {
		return
	}

	for _, job := range jobs {
		n.addJob(job, false)
	}

	return
}

func (n *Node) addJob(job *models.Job, notice bool) {
	n.link.addJob(job)

	cmds := job.Cmds(n.ID, n.groups)
	if len(cmds) == 0 {
		return
	}

	n.jobs[job.ID] = job
	for _, cmd := range cmds {
		n.addCmd(cmd, notice)
	}
	return
}

func (n *Node) delJob(id string) {
	job, ok := n.jobs[id]
	// 之前此任务没有在当前结点执行
	if !ok {
		n.delIDs[id] = true
		return
	}

	delete(n.jobs, id)
	n.link.delJob(job)

	cmds := job.Cmds(n.ID, n.groups)
	if len(cmds) == 0 {
		return
	}

	for _, cmd := range cmds {
		n.delCmd(cmd)
	}
	return
}

func (n *Node) modJob(job *models.Job) {
	oJob, ok := n.jobs[job.ID]
	// 之前此任务没有在当前结点执行，直接增加任务
	if !ok {
		n.addJob(job, true)
		return
	}

	n.link.delJob(oJob)
	prevCmds := oJob.Cmds(n.ID, n.groups)
	*oJob = *job
	cmds := oJob.Cmds(n.ID, n.groups)

	for id, cmd := range cmds {
		n.addCmd(cmd, true)
		delete(prevCmds, id)
	}

	for _, cmd := range prevCmds {
		n.delCmd(cmd)
	}

	n.link.addJob(oJob)
}

func (n *Node) addCmd(cmd *models.Cmd, notice bool) {
	c, ok := n.cmds[cmd.GetID()]
	if ok {
		sch := c.Schedule
		*c = *cmd

		// 节点执行时间不变，不用更新 cron
		if c.Schedule == sch {
			return
		}
	} else {
		c = cmd
	}

	if err := n.Cron.AddJob(c.Schedule, c); err != nil {
		msg := fmt.Sprintf("job[%s] rule[%s] timer[%s] parse err: %s", c.Job.ID, c.JobRule.ID, c.Schedule, err.Error())
		log.Warn(msg)
		c.Fail(time.Now(), msg)
		return
	}

	if !ok {
		n.cmds[c.GetID()] = c
	}

	if notice {
		log.Noticef("job[%s] rule[%s] timer[%s] has added", c.Job.ID, c.JobRule.ID, c.Schedule)
	}
	return
}

func (n *Node) delCmd(cmd *models.Cmd) {
	delete(n.cmds, cmd.GetID())
	n.Cron.DelJob(cmd)
	log.Noticef("job[%s] rule[%s] timer[%s] has deleted", cmd.Job.ID, cmd.JobRule.ID, cmd.Schedule)
}

func (n *Node) addGroup(g *models.Group) {
	n.groups[g.ID] = g
}

func (n *Node) delGroup(id string) {
	delete(n.groups, id)
	n.link.delGroup(id)

	job, ok := n.jobs[id]
	// 之前此任务没有在当前结点执行
	if !ok {
		return
	}

	cmds := job.Cmds(n.ID, n.groups)
	if len(cmds) == 0 {
		return
	}

	for _, cmd := range cmds {
		n.delCmd(cmd)
	}
	return
}

func (n *Node) modGroup(g *models.Group) {
	oGroup, ok := n.groups[g.ID]
	if !ok {
		n.addGroup(g)
		return
	}

	// 都包含/都不包含当前节点，对当前节点任务无影响
	if (oGroup.Included(n.ID) && g.Included(n.ID)) || (!oGroup.Included(n.ID) && !g.Included(n.ID)) {
		*oGroup = *g
		return
	}

	// 增加当前节点
	if !oGroup.Included(n.ID) && g.Included(n.ID) {
		n.groups[g.ID] = g
		jls := n.link[g.ID]
		if len(jls) == 0 {
			return
		}

		var err error
		for jid, jl := range jls {
			job, ok := n.jobs[jid]
			if !ok {
				// job 已删除
				if n.delIDs[jid] {
					n.link.delGroupJob(g.ID, jid)
					continue
				}

				if job, err = models.GetJob(jl.gname, jid); err != nil {
					log.Warnf("get job[%s][%s] err: %s", jl.gname, jid, err.Error())
					n.link.delGroupJob(g.ID, jid)
					continue
				}
			}

			cmds := job.Cmds(n.ID, n.groups)
			for _, cmd := range cmds {
				n.addCmd(cmd, true)
			}
		}
		return
	}

	// 移除当前节点
	jls := n.link[g.ID]
	if len(jls) == 0 {
		n.groups[g.ID] = g
		return
	}

	for jid, _ := range jls {
		job, ok := n.jobs[jid]
		if !ok {
			// 数据出错
			log.Warnf("WTF! group[%s] job[%s]", g.ID, jid)
			n.link.delGroupJob(g.ID, jid)
			continue
		}

		n.groups[oGroup.ID] = oGroup
		prevCmds := job.Cmds(n.ID, n.groups)
		n.groups[g.ID] = g
		cmds := job.Cmds(n.ID, n.groups)

		for id, cmd := range cmds {
			n.addCmd(cmd, true)
			delete(prevCmds, id)
		}

		for _, cmd := range prevCmds {
			n.delCmd(cmd)
		}
	}

	n.groups[g.ID] = g
	return
}

func (n *Node) watchJobs() {
	rch := models.WatchJobs()
	for wresp := range rch {
		for _, ev := range wresp.Events {
			switch {
			case ev.IsCreate():
				job, err := models.GetJobFromKv(ev.Kv)
				if err != nil {
					log.Warnf(err.Error())
					continue
				}

				n.addJob(job, true)
			case ev.IsModify():
				job, err := models.GetJobFromKv(ev.Kv)
				if err != nil {
					log.Warnf(err.Error())
					continue
				}

				n.modJob(job)
			case ev.Type == client.EventTypeDelete:
				n.delJob(models.GetIDFromKey(string(ev.Kv.Key)))
			default:
				log.Warnf("unknown event type[%v] from job[%s]", ev.Type, string(ev.Kv.Key))
			}
		}
	}
}

func (n *Node) watchGroups() {
	rch := models.WatchGroups()
	for wresp := range rch {
		for _, ev := range wresp.Events {
			switch {
			case ev.IsCreate():
				g, err := models.GetGroupFromKv(ev.Kv)
				if err != nil {
					log.Warnf(err.Error())
					continue
				}

				n.addGroup(g)
			case ev.IsModify():
				g, err := models.GetGroupFromKv(ev.Kv)
				if err != nil {
					log.Warnf(err.Error())
					continue
				}

				n.modGroup(g)
			case ev.Type == client.EventTypeDelete:
				n.delGroup(models.GetIDFromKey(string(ev.Kv.Key)))
			default:
				log.Warnf("unknown event type[%v] from group[%s]", ev.Type, string(ev.Kv.Key))
			}
		}
	}
}

// 启动服务
func (n *Node) Run() (err error) {
	go n.keepAlive()

	defer func() {
		if err != nil {
			n.Stop(nil)
		}
	}()

	if err = n.loadJobs(); err != nil {
		return
	}

	n.Cron.Start()
	go n.watchJobs()
	go n.watchGroups()
	n.Node.On()
	return
}

// 断网掉线重新注册
func (n *Node) keepAlive() {
	for {
		for _ = range n.lch {
		}

		select {
		case <-n.done:
			return
		default:
		}
		time.Sleep(time.Duration(n.ttl+1) * time.Second)

		log.Noticef("%s has dropped, try to reconnect...", n.String())
		if err := n.Register(); err != nil {
			log.Warn(err.Error())
		} else {
			log.Noticef("%s reconnected", n.String())
		}
	}
}

// 停止服务
func (n *Node) Stop(i interface{}) {
	n.Node.Down()
	close(n.done)
	n.Node.Del()
	n.Client.Close()
	n.Cron.Stop()
}
<commit_msg>node: 提取 group 处理函数<commit_after>package node

import (
	"fmt"
	"os"
	"strconv"
	"time"

	"golang.org/x/net/context"

	client "github.com/coreos/etcd/clientv3"

	"sunteng/commons/log"
	"sunteng/commons/util"
	"sunteng/cronsun/conf"
	"sunteng/cronsun/models"
	"sunteng/cronsun/node/cron"
)

// Node 执行 cron 命令服务的结构体
type Node struct {
	*models.Client
	*models.Node
	*cron.Cron

	jobs   Jobs
	groups Groups
	cmds   map[string]*models.Cmd

	link
	// 删除的 job id，用于 group 更新
	delIDs map[string]bool

	ttl int64

	lID client.LeaseID // lease id
	lch <-chan *client.LeaseKeepAliveResponse

	done chan struct{}
}

func NewNode(cfg *conf.Conf) (n *Node, err error) {
	ip, err := util.GetLocalIP()
	if err != nil {
		return
	}

	n = &Node{
		Client: models.DefalutClient,
		Node: &models.Node{
			ID:  ip.String(),
			PID: strconv.Itoa(os.Getpid()),
		},
		Cron: cron.New(),

		cmds: make(map[string]*models.Cmd),

		ttl:  cfg.Ttl,
		done: make(chan struct{}),
	}
	return
}

// 注册到 /cronsun/proc/xx
func (n *Node) Register() (err error) {
	pid, err := n.Node.Exist()
	if err != nil {
		return
	}

	if pid != -1 {
		return fmt.Errorf("node[%s] pid[%d] exist", n.Node.ID, pid)
	}

	resp, err := n.Client.Grant(context.TODO(), n.ttl)
	if err != nil {
		return
	}

	if _, err = n.Node.Put(client.WithLease(resp.ID)); err != nil {
		return
	}

	ch, err := n.Client.KeepAlive(context.TODO(), resp.ID)
	if err != nil {
		return
	}

	n.lID, n.lch = resp.ID, ch
	return
}

func (n *Node) loadJobs() (err error) {
	if n.groups, err = models.GetGroups(""); err != nil {
		return
	}

	jobs, err := models.GetJobs()
	if err != nil {
		return
	}

	n.jobs, n.link = make(Jobs, len(jobs)), newLink(len(n.groups))
	if len(jobs) == 0 {
		return
	}

	for _, job := range jobs {
		n.addJob(job, false)
	}

	return
}

func (n *Node) addJob(job *models.Job, notice bool) {
	n.link.addJob(job)

	cmds := job.Cmds(n.ID, n.groups)
	if len(cmds) == 0 {
		return
	}

	n.jobs[job.ID] = job
	for _, cmd := range cmds {
		n.addCmd(cmd, notice)
	}
	return
}

func (n *Node) delJob(id string) {
	job, ok := n.jobs[id]
	// 之前此任务没有在当前结点执行
	if !ok {
		n.delIDs[id] = true
		return
	}

	delete(n.jobs, id)
	n.link.delJob(job)

	cmds := job.Cmds(n.ID, n.groups)
	if len(cmds) == 0 {
		return
	}

	for _, cmd := range cmds {
		n.delCmd(cmd)
	}
	return
}

func (n *Node) modJob(job *models.Job) {
	oJob, ok := n.jobs[job.ID]
	// 之前此任务没有在当前结点执行，直接增加任务
	if !ok {
		n.addJob(job, true)
		return
	}

	n.link.delJob(oJob)
	prevCmds := oJob.Cmds(n.ID, n.groups)
	*oJob = *job
	cmds := oJob.Cmds(n.ID, n.groups)

	for id, cmd := range cmds {
		n.addCmd(cmd, true)
		delete(prevCmds, id)
	}

	for _, cmd := range prevCmds {
		n.delCmd(cmd)
	}

	n.link.addJob(oJob)
}

func (n *Node) addCmd(cmd *models.Cmd, notice bool) {
	c, ok := n.cmds[cmd.GetID()]
	if ok {
		sch := c.Schedule
		*c = *cmd

		// 节点执行时间不变，不用更新 cron
		if c.Schedule == sch {
			return
		}
	} else {
		c = cmd
	}

	if err := n.Cron.AddJob(c.Schedule, c); err != nil {
		msg := fmt.Sprintf("job[%s] rule[%s] timer[%s] parse err: %s", c.Job.ID, c.JobRule.ID, c.Schedule, err.Error())
		log.Warn(msg)
		c.Fail(time.Now(), msg)
		return
	}

	if !ok {
		n.cmds[c.GetID()] = c
	}

	if notice {
		log.Noticef("job[%s] rule[%s] timer[%s] has added", c.Job.ID, c.JobRule.ID, c.Schedule)
	}
	return
}

func (n *Node) delCmd(cmd *models.Cmd) {
	delete(n.cmds, cmd.GetID())
	n.Cron.DelJob(cmd)
	log.Noticef("job[%s] rule[%s] timer[%s] has deleted", cmd.Job.ID, cmd.JobRule.ID, cmd.Schedule)
}

func (n *Node) addGroup(g *models.Group) {
	n.groups[g.ID] = g
}

func (n *Node) delGroup(id string) {
	delete(n.groups, id)
	n.link.delGroup(id)

	job, ok := n.jobs[id]
	// 之前此任务没有在当前结点执行
	if !ok {
		return
	}

	cmds := job.Cmds(n.ID, n.groups)
	if len(cmds) == 0 {
		return
	}

	for _, cmd := range cmds {
		n.delCmd(cmd)
	}
	return
}

func (n *Node) modGroup(g *models.Group) {
	oGroup, ok := n.groups[g.ID]
	if !ok {
		n.addGroup(g)
		return
	}

	// 都包含/都不包含当前节点，对当前节点任务无影响
	if (oGroup.Included(n.ID) && g.Included(n.ID)) || (!oGroup.Included(n.ID) && !g.Included(n.ID)) {
		*oGroup = *g
		return
	}

	// 增加当前节点
	if !oGroup.Included(n.ID) && g.Included(n.ID) {
		n.groupAddNode(g)
		return
	}

	// 移除当前节点
	n.groupRmNode(g, oGroup)
	return
}

func (n *Node) groupAddNode(g *models.Group) {
	n.groups[g.ID] = g
	jls := n.link[g.ID]
	if len(jls) == 0 {
		return
	}

	var err error
	for jid, jl := range jls {
		job, ok := n.jobs[jid]
		if !ok {
			// job 已删除
			if n.delIDs[jid] {
				n.link.delGroupJob(g.ID, jid)
				continue
			}

			if job, err = models.GetJob(jl.gname, jid); err != nil {
				log.Warnf("get job[%s][%s] err: %s", jl.gname, jid, err.Error())
				n.link.delGroupJob(g.ID, jid)
				continue
			}
		}

		cmds := job.Cmds(n.ID, n.groups)
		for _, cmd := range cmds {
			n.addCmd(cmd, true)
		}
	}
	return
}

func (n *Node) groupRmNode(g, og *models.Group) {
	jls := n.link[g.ID]
	if len(jls) == 0 {
		n.groups[g.ID] = g
		return
	}

	for jid, _ := range jls {
		job, ok := n.jobs[jid]
		if !ok {
			// 数据出错
			log.Warnf("WTF! group[%s] job[%s]", g.ID, jid)
			n.link.delGroupJob(g.ID, jid)
			continue
		}

		n.groups[og.ID] = og
		prevCmds := job.Cmds(n.ID, n.groups)
		n.groups[g.ID] = g
		cmds := job.Cmds(n.ID, n.groups)

		for id, cmd := range cmds {
			n.addCmd(cmd, true)
			delete(prevCmds, id)
		}

		for _, cmd := range prevCmds {
			n.delCmd(cmd)
		}
	}

	n.groups[g.ID] = g
}

func (n *Node) watchJobs() {
	rch := models.WatchJobs()
	for wresp := range rch {
		for _, ev := range wresp.Events {
			switch {
			case ev.IsCreate():
				job, err := models.GetJobFromKv(ev.Kv)
				if err != nil {
					log.Warnf(err.Error())
					continue
				}

				n.addJob(job, true)
			case ev.IsModify():
				job, err := models.GetJobFromKv(ev.Kv)
				if err != nil {
					log.Warnf(err.Error())
					continue
				}

				n.modJob(job)
			case ev.Type == client.EventTypeDelete:
				n.delJob(models.GetIDFromKey(string(ev.Kv.Key)))
			default:
				log.Warnf("unknown event type[%v] from job[%s]", ev.Type, string(ev.Kv.Key))
			}
		}
	}
}

func (n *Node) watchGroups() {
	rch := models.WatchGroups()
	for wresp := range rch {
		for _, ev := range wresp.Events {
			switch {
			case ev.IsCreate():
				g, err := models.GetGroupFromKv(ev.Kv)
				if err != nil {
					log.Warnf(err.Error())
					continue
				}

				n.addGroup(g)
			case ev.IsModify():
				g, err := models.GetGroupFromKv(ev.Kv)
				if err != nil {
					log.Warnf(err.Error())
					continue
				}

				n.modGroup(g)
			case ev.Type == client.EventTypeDelete:
				n.delGroup(models.GetIDFromKey(string(ev.Kv.Key)))
			default:
				log.Warnf("unknown event type[%v] from group[%s]", ev.Type, string(ev.Kv.Key))
			}
		}
	}
}

// 启动服务
func (n *Node) Run() (err error) {
	go n.keepAlive()

	defer func() {
		if err != nil {
			n.Stop(nil)
		}
	}()

	if err = n.loadJobs(); err != nil {
		return
	}

	n.Cron.Start()
	go n.watchJobs()
	go n.watchGroups()
	n.Node.On()
	return
}

// 断网掉线重新注册
func (n *Node) keepAlive() {
	for {
		for _ = range n.lch {
		}

		select {
		case <-n.done:
			return
		default:
		}
		time.Sleep(time.Duration(n.ttl+1) * time.Second)

		log.Noticef("%s has dropped, try to reconnect...", n.String())
		if err := n.Register(); err != nil {
			log.Warn(err.Error())
		} else {
			log.Noticef("%s reconnected", n.String())
		}
	}
}

// 停止服务
func (n *Node) Stop(i interface{}) {
	n.Node.Down()
	close(n.done)
	n.Node.Del()
	n.Client.Close()
	n.Cron.Stop()
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/voxelbrain/goptions"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
	"log"
	"mime"
	"os"
	"path/filepath"
	"sync"
)

const (
	VERSION = "0.1.1"
)

type Item struct {
	Prefix string
	Path   string
	os.FileInfo
}

func main() {
	options := struct {
		AccessKey   string `goptions:"-k, --access-key, obligatory, description='AWS Access Key ID'"`
		SecretKey   string `goptions:"-s, --secret-key, obligatory, description='AWS Secret Access Key'"`
		Region      string `goptions:"-r, --region, description='API Region name (default: us-west-1)'"`
		Bucket      string `goptions:"-b, --bucket, obligatory, description='Bucket to push to'"`
		Prefix      string `goptions:"-p, --prefix, description='Prefix to prepend to the items'"`
		Concurrency int    `goptions:"-c, --concurrency, description='Number of coroutines (default: 10)'"`
		goptions.Remainder
	}{
		Concurrency: 10,
		Region:      aws.USWest.Name,
	}

	err := goptions.Parse(&options)
	if err != nil || len(options.Remainder) <= 0 {
		if err != goptions.ErrHelpRequest {
			log.Printf("Error: %s", err)
		}
		goptions.PrintHelp()
		return
	}

	c := make(chan Item)
	go func() {
		for _, prefix := range options.Remainder {
			newprefix, err := filepath.Abs(prefix)
			if err != nil {
				log.Printf("Path %s could not be made absolute: %s", prefix, err)
				continue
			}
			log.Printf("Traversing %s...", newprefix)
			filepath.Walk(newprefix, func(path string, info os.FileInfo, err error) error {
				if info.IsDir() {
					return nil
				}
				c <- Item{
					Prefix:   newprefix,
					Path:     path,
					FileInfo: info,
				}
				return nil
			})
		}
		close(c)
	}()

	auth := aws.Auth{
		AccessKey: options.AccessKey,
		SecretKey: options.SecretKey,
	}

	region, ok := aws.Regions[options.Region]
	if !ok {
		log.Fatalf("Invalid region name %s", options.Region)
	}

	s3i := s3.New(auth, region)
	bucket := s3i.Bucket(options.Bucket)

	var wg sync.WaitGroup
	wg.Add(options.Concurrency)
	for i := 0; i < options.Concurrency; i++ {
		go func() {
			for item := range c {
				func() {
					f, err := os.Open(item.Path)
					if err != nil {
						log.Printf("Pushing %s failed: %s", item.Path, err)
					}
					defer f.Close()

					path := item.Path[len(item.Prefix)+1:]
					err = bucket.PutReader(options.Prefix+path, f, item.FileInfo.Size(), mime.TypeByExtension(filepath.Ext(item.Path)), s3.BucketOwnerFull)
					if err != nil {
						log.Printf("Uploading %s failed: %s", path, err)
						return
					}
					log.Printf("Uploading %s done", path)
				}()
			}
			wg.Done()
		}()
	}
	wg.Wait()
}
<commit_msg>Implement get<commit_after>package main

import (
	"github.com/voxelbrain/goptions"
	"io"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
	"log"
	"mime"
	"os"
	"path/filepath"
	"sync"
)

const (
	VERSION = "0.1.1"
)

type Item struct {
	Prefix string
	Path   string
	os.FileInfo
}

var (
	options = struct {
		AccessKey   string `goptions:"-k, --access-key, obligatory, description='AWS Access Key ID'"`
		SecretKey   string `goptions:"-s, --secret-key, obligatory, description='AWS Secret Access Key'"`
		Region      string `goptions:"-r, --region, description='API Region name (default: us-west-1)'"`
		Bucket      string `goptions:"-b, --bucket, obligatory, description='Bucket to push to'"`
		Concurrency int    `goptions:"-c, --concurrency, description='Number of coroutines (default: 10)'"`
		goptions.Remainder
		goptions.Verbs
		Put struct {
			Prefix string `goptions:"-p, --prefix, description='Prefix to prepend to the items'"`
		} `goptions:"put"`
		Get struct {
			Prefix string `goptions:"-p, --prefix, description='Only get items starting with prefix'"`
		} `goptions:"get"`
	}{
		Concurrency: 10,
		Region:      aws.USWest.Name,
	}
)

func init() {
	err := goptions.Parse(&options)
	if err != nil || len(options.Remainder) <= 0 || len(options.Verbs) <= 0 {
		if err != goptions.ErrHelpRequest && err != nil {
			log.Printf("Error: %s", err)
		}
		goptions.PrintHelp()
		os.Exit(1)
	}

}

func main() {
	auth := aws.Auth{
		AccessKey: options.AccessKey,
		SecretKey: options.SecretKey,
	}

	region, ok := aws.Regions[options.Region]
	if !ok {
		log.Fatalf("Invalid region name %s", options.Region)
	}

	s3i := s3.New(auth, region)
	bucket := s3i.Bucket(options.Bucket)

	switch options.Verbs {
	case "put":
		c := listLocalFiles(options.Remainder...)
		putFiles(bucket, c)
	case "get":
		c := listBucketFiles(bucket)
		getFiles(bucket, c)
	}
}

func listLocalFiles(path ...string) <-chan *Item {
	c := make(chan *Item)
	go func() {
		for _, prefix := range options.Remainder {
			newprefix, err := filepath.Abs(prefix)
			if err != nil {
				log.Printf("Path %s could not be made absolute: %s. Skipping...", prefix, err)
				continue
			}
			log.Printf("Traversing %s...", newprefix)
			filepath.Walk(newprefix, func(path string, info os.FileInfo, err error) error {
				if info.IsDir() {
					return nil
				}
				c <- &Item{
					Prefix:   newprefix,
					Path:     path,
					FileInfo: info,
				}
				return nil
			})
		}
		close(c)
	}()
	return c
}

func listBucketFiles(bucket *s3.Bucket) <-chan *Item {
	c := make(chan *Item)
	go func() {
		resp, err := bucket.List(options.Get.Prefix, "", "", 1000000)
		if err != nil {
			log.Printf("Could not list items in bucket: %s", err)
		}
		for _, item := range resp.Contents {
			c <- &Item{
				Prefix:   options.Get.Prefix,
				Path:     item.Key,
				FileInfo: nil,
			}
		}
		close(c)
	}()
	return c
}

func putFiles(bucket *s3.Bucket, c <-chan *Item) {
	var wg sync.WaitGroup
	wg.Add(options.Concurrency)
	for i := 0; i < options.Concurrency; i++ {
		go func() {
			for item := range c {
				func() {
					f, err := os.Open(item.Path)
					if err != nil {
						log.Printf("Pushing %s failed: %s", item.Path, err)
					}
					defer f.Close()

					path := item.Path[len(item.Prefix)+1:]
					err = bucket.PutReader(options.Put.Prefix+path, f, item.FileInfo.Size(), mime.TypeByExtension(filepath.Ext(item.Path)), s3.BucketOwnerFull)
					if err != nil {
						log.Printf("Uploading %s failed: %s", path, err)
						return
					}
					log.Printf("Uploading %s done", path)
				}()
			}
			wg.Done()
		}()
	}
	wg.Wait()
}

func getFiles(bucket *s3.Bucket, c <-chan *Item) {
	var wg sync.WaitGroup
	wg.Add(options.Concurrency)
	for i := 0; i < options.Concurrency; i++ {
		go func() {
			for item := range c {
				func() {
					itempath := item.Path[len(item.Prefix):]
					dirname, fname := filepath.Split(itempath)
					dirname = filepath.Join(options.Remainder[0], dirname)

					os.MkdirAll(dirname, os.FileMode(0755))
					f, err := os.Create(filepath.Join(dirname, fname))
					if err != nil {
						log.Printf("Opening %s failed: %s", item.Path, err)
					}
					defer f.Close()

					rc, err := bucket.GetReader(item.Path)
					if err != nil {
						log.Printf("Downloading %s failed: %s", item.Path, err)
						return
					}
					defer rc.Close()
					io.Copy(f, rc)
					log.Printf("Downloading %s done", item.Path)
				}()
			}
			wg.Done()
		}()
	}
	wg.Wait()
}
<|endoftext|>