<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package sd

import (
	"context"
	"errors"
	"fmt"
	"sync"
	"time"

	"github.com/apache/servicecomb-service-center/datasource/sdcommon"
	"github.com/apache/servicecomb-service-center/pkg/backoff"
	"github.com/apache/servicecomb-service-center/pkg/gopool"
	"github.com/apache/servicecomb-service-center/pkg/log"
	"github.com/apache/servicecomb-service-center/pkg/util"
	rmodel "github.com/go-chassis/cari/discovery"
)

// MongoCacher manages mongo cache.
// To updateOp cache, MongoCacher watch mongo event and pull data periodly from mongo.
// When the cache data changes, MongoCacher creates events and notifies it's
// subscribers.
// Use Options to set it's behaviors.
type MongoCacher struct {
	Options     *Options
	reListCount int
	isFirstTime bool
	cache       *MongoCache
	ready       chan struct{}
	lw          sdcommon.ListWatch
	mux         sync.Mutex
	once        sync.Once
	goroutine   *gopool.Pool
}

func (c *MongoCacher) Cache() *MongoCache {
	return c.cache
}

func (c *MongoCacher) Run() {
	c.once.Do(func() {
		c.goroutine.Do(c.refresh)
	})
}

func (c *MongoCacher) Stop() {
	c.goroutine.Close(true)

	util.SafeCloseChan(c.ready)
}

func (c *MongoCacher) Ready() <-chan struct{} {
	return c.ready
}

func (c *MongoCacher) IsReady() bool {
	select {
	case <-c.ready:
		return true
	default:
		return false
	}
}

func (c *MongoCacher) needList() bool {
	if c.isFirstTime {
		c.isFirstTime = false
		return true
	}
	c.reListCount++
	if c.reListCount < sdcommon.DefaultForceListInterval {
		return false
	}
	c.reListCount = 0
	return true
}

func (c *MongoCacher) doList(cfg sdcommon.ListWatchConfig) error {
	resp, err := c.lw.List(cfg)
	if err != nil {
		return err
	}

	resources := resp.Resources

	defer log.Debug(fmt.Sprintf("finish to cache key %s, %d items",
		c.Options.Key, len(resources)))

	//just reset the cacher if cache marked dirty
	if c.cache.Dirty() {
		c.reset(resources)
		log.Warn(fmt.Sprintf("Cache[%s] is reset!", c.cache.Name()))
		return nil
	}

	// calc and return the diff between cache and mongodb
	events := c.filter(resources)

	//notify the subscribers
	c.sync(events)
	return nil
}

func (c *MongoCacher) reset(infos []*sdcommon.Resource) {
	// clear cache before Set is safe, because the watch operation is stop,
	// but here will make all API requests go to MONGO directly.
	c.cache.Clear()
	// do not notify when cacher is dirty status,
	// otherwise, too many events will notify to downstream.
	c.buildCache(c.filter(infos))
}

func (c *MongoCacher) doWatch(cfg sdcommon.ListWatchConfig) error {
	if eventbus := c.lw.EventBus(cfg); eventbus != nil {
		return c.handleEventBus(eventbus)
	}
	return fmt.Errorf("handle a nil watcher")
}

func (c *MongoCacher) ListAndWatch(ctx context.Context) error {
	c.mux.Lock()
	defer c.mux.Unlock()
	defer log.Recover() // ensure ListAndWatch never raise panic

	cfg := sdcommon.ListWatchConfig{
		Timeout: c.Options.Timeout,
		Context: ctx,
	}

	// first time should initial cache, set watch timeout less
	if c.isFirstTime {
		cfg.Timeout = FirstTimeout
	}

	err := c.doWatch(cfg)
	if err != nil {
		log.Error("doWatch err", err)
	}

	// the scenario need to list mongo:
	// 1. Initial: cache is building, the lister's is first time to run.
	// 2. Runtime: error occurs in previous watch operation, the lister's status is set to error.
	// 3. Runtime: watch operation timed out over DEFAULT_FORCE_LIST_INTERVAL times.
	if c.needList() {
		// recover timeout for list
		if c.isFirstTime {
			cfg.Timeout = c.Options.Timeout
		}

		if err := c.doList(cfg); err != nil && (!c.IsReady()) {
			log.Error("doList error", err)
			return err // do retry to list mongo
		}
		// keep going to next step:
		// 1. doList return OK.
		// 2. some traps in mongo client
	}

	util.SafeCloseChan(c.ready)

	return nil
}

func (c *MongoCacher) handleEventBus(eventbus *sdcommon.EventBus) error {
	defer eventbus.Stop()

	if eventbus.Bus == nil {
		return nil
	}

	for resp := range eventbus.ResourceEventBus() {
		events := make([]MongoEvent, 0)

		if resp == nil {
			return errors.New("handle watcher error")
		}

		for _, resource := range resp.Resources {
			action := resp.Action
			var event MongoEvent
			switch action {
			case sdcommon.ActionCreate:
				event = NewMongoEventByResource(resource, rmodel.EVT_CREATE)
			case sdcommon.ActionUpdate:
				event = NewMongoEventByResource(resource, rmodel.EVT_UPDATE)
			case sdcommon.ActionDelete:
				resource.Key = c.cache.GetKeyByDocumentID(resource.DocumentID)
				resource.Value = c.cache.Get(resource.Key)
				event = NewMongoEventByResource(resource, rmodel.EVT_DELETE)
			}
			events = append(events, event)
		}

		c.sync(events)
		log.Debug(fmt.Sprintf("finish to handle %d events, table: %s", len(events), c.Options.Key))
	}

	return nil
}

func (c *MongoCacher) refresh(ctx context.Context) {
	log.Debug(fmt.Sprintf("start to list and watch %s", c.Options))
	retries := 0

	timer := time.NewTimer(sdcommon.MinWaitInterval)
	defer timer.Stop()
	for {
		nextPeriod := sdcommon.MinWaitInterval
		if err := c.ListAndWatch(ctx); err != nil {
			retries++
			nextPeriod = backoff.GetBackoff().Delay(retries)
		} else {
			retries = 0
		}

		select {
		case <-ctx.Done():
			log.Debug(fmt.Sprintf("stop to list and watch %s", c.Options))
			return
		case <-timer.C:
			timer.Reset(nextPeriod)
		}
	}
}

// keep the evts valID when call sync
func (c *MongoCacher) sync(evts []MongoEvent) {
	if len(evts) == 0 {
		return
	}

	c.onEvents(evts)
}

func (c *MongoCacher) filter(infos []*sdcommon.Resource) []MongoEvent {
	nc := len(infos)
	newStore := make(map[string]interface{}, nc)
	documentIDRecord := make(map[string]string, nc)
	indexRecord := make(map[string]string, nc)

	for _, info := range infos {
		event := NewMongoEventByResource(info, rmodel.EVT_CREATE)
		newStore[event.ResourceID] = info.Value
		documentIDRecord[event.ResourceID] = info.DocumentID
		indexRecord[event.ResourceID] = info.Index
	}

	filterStopCh := make(chan struct{})
	eventsCh := make(chan [sdcommon.EventBlockSize]MongoEvent, 2)

	go c.filterDelete(newStore, eventsCh, filterStopCh)

	go c.filterCreateOrUpdate(newStore, documentIDRecord, indexRecord, eventsCh, filterStopCh)

	events := make([]MongoEvent, 0, nc)
	for block := range eventsCh {
		for _, e := range block {
			if e.Value == nil {
				break
			}
			events = append(events, e)
		}
	}
	return events
}

func (c *MongoCacher) filterDelete(newStore map[string]interface{},
	eventsCh chan [sdcommon.EventBlockSize]MongoEvent, filterStopCh chan struct{}) {
	var block [sdcommon.EventBlockSize]MongoEvent
	i := 0

	c.cache.ForEach(func(k string, v interface{}) (next bool) {
		next = true

		_, ok := newStore[k]
		if ok {
			// k in store, also in new store, is not deleted, return
			return
		}

		// k in store but not in new store, it means k is deleted
		if i >= sdcommon.EventBlockSize {
			eventsCh <- block
			block = [sdcommon.EventBlockSize]MongoEvent{}
			i = 0
		}

		documentID := c.cache.GetDocumentIDByBussinessID(k)
		index := c.cache.GetIndexByBussinessID(k)
		block[i] = NewMongoEvent(k, documentID, index, rmodel.EVT_DELETE, v)
		i++
		return
	})

	if i > 0 {
		eventsCh <- block
	}

	close(filterStopCh)
}

func (c *MongoCacher) filterCreateOrUpdate(newStore map[string]interface{}, newDocumentStore map[string]string, indexRecord map[string]string,
	eventsCh chan [sdcommon.EventBlockSize]MongoEvent, filterStopCh chan struct{}) {
	var block [sdcommon.EventBlockSize]MongoEvent
	i := 0

	for k, v := range newStore {
		ov := c.cache.Get(k)
		if ov == nil {
			if i >= sdcommon.EventBlockSize {
				eventsCh <- block
				block = [sdcommon.EventBlockSize]MongoEvent{}
				i = 0
			}

			block[i] = NewMongoEvent(k, newDocumentStore[k], indexRecord[k], rmodel.EVT_CREATE, v)
			i++

			continue
		}

		if c.isValueNotUpdated(v, ov) {
			continue
		}

		log.Debug(fmt.Sprintf("value is updateOp of key:%s, old value is:%s, new value is:%s", k, ov, v))

		if i >= sdcommon.EventBlockSize {
			eventsCh <- block
			block = [sdcommon.EventBlockSize]MongoEvent{}
			i = 0
		}

		block[i] = NewMongoEvent(k, newDocumentStore[k], indexRecord[k], rmodel.EVT_UPDATE, v)
		i++
	}

	if i > 0 {
		eventsCh <- block
	}

	<-filterStopCh

	close(eventsCh)
}

func (c *MongoCacher) isValueNotUpdated(value interface{}, newValue interface{}) bool {
	var modTime string
	var newModTime string

	switch c.Options.Key {
	case instance:
		instance := value.(Instance)
		newInstance := newValue.(Instance)
		if instance.Instance == nil || newInstance.Instance == nil {
			return true
		}
		modTime = instance.Instance.ModTimestamp
		newModTime = newInstance.Instance.ModTimestamp
	case service:
		service := value.(Service)
		newService := newValue.(Service)
		if service.Service == nil || newService.Service == nil {
			return true
		}
		modTime = service.Service.ModTimestamp
		newModTime = newService.Service.ModTimestamp
	}

	if newModTime == "" || modTime == newModTime {
		return true
	}

	return false
}

func (c *MongoCacher) onEvents(events []MongoEvent) {
	c.buildCache(events)

	c.notify(events)
}

func (c *MongoCacher) buildCache(events []MongoEvent) {
	for i, evt := range events {
		key := evt.ResourceID
		value := c.cache.Get(key)
		ok := value != nil

		switch evt.Type {
		case rmodel.EVT_CREATE, rmodel.EVT_UPDATE:
			switch {
			case !c.IsReady():
				evt.Type = rmodel.EVT_INIT
			case !ok && evt.Type != rmodel.EVT_CREATE:
				log.Warn(fmt.Sprintf("unexpected %s event! it should be %s key %s",
					evt.Type, rmodel.EVT_CREATE, key))
				evt.Type = rmodel.EVT_CREATE
			case ok && evt.Type != rmodel.EVT_UPDATE:
				log.Warn(fmt.Sprintf("unexpected %s event! it should be %s key %s",
					evt.Type, rmodel.EVT_UPDATE, key))
				evt.Type = rmodel.EVT_UPDATE
			}

			c.cache.Put(key, evt.Value)
			c.cache.PutDocumentID(key, evt.DocumentID)
			c.cache.PutIndex(evt.Index, evt.ResourceID)

			events[i] = evt
		case rmodel.EVT_DELETE:
			if !ok {
				log.Warn(fmt.Sprintf("unexpected %s event! key %s does not cache",
					evt.Type, key))
			} else {
				evt.Value = value

				c.cache.Remove(key)
				c.cache.RemoveDocumentID(evt.DocumentID)
				c.cache.RemoveIndex(evt.Index, evt.ResourceID)
			}
			events[i] = evt
		}
	}
}

func (c *MongoCacher) notify(evts []MongoEvent) {
	eventProxy := EventProxy(c.Options.Key)

	if eventProxy == nil {
		return
	}

	defer log.Recover()

	for _, evt := range evts {
		eventProxy.OnEvent(evt)
	}
}

func NewMongoCacher(options *Options, cache *MongoCache) *MongoCacher {
	return &MongoCacher{
		Options:     options,
		isFirstTime: true,
		cache:       cache,
		ready:       make(chan struct{}),
		lw: &mongoListWatch{
			Key: options.Key,
		},
		goroutine: gopool.New(context.Background()),
	}
}
<commit_msg>mongo listwatch bug fix (#848)<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package sd

import (
	"context"
	"errors"
	"fmt"
	"sync"
	"time"

	"github.com/apache/servicecomb-service-center/datasource/sdcommon"
	"github.com/apache/servicecomb-service-center/pkg/backoff"
	"github.com/apache/servicecomb-service-center/pkg/gopool"
	"github.com/apache/servicecomb-service-center/pkg/log"
	"github.com/apache/servicecomb-service-center/pkg/util"
	rmodel "github.com/go-chassis/cari/discovery"
)

// MongoCacher manages mongo cache.
// To updateOp cache, MongoCacher watch mongo event and pull data periodly from mongo.
// When the cache data changes, MongoCacher creates events and notifies it's
// subscribers.
// Use Options to set it's behaviors.
type MongoCacher struct {
	Options     *Options
	reListCount int
	isFirstTime bool
	cache       *MongoCache
	ready       chan struct{}
	lw          sdcommon.ListWatch
	mux         sync.Mutex
	once        sync.Once
	goroutine   *gopool.Pool
}

func (c *MongoCacher) Cache() *MongoCache {
	return c.cache
}

func (c *MongoCacher) Run() {
	c.once.Do(func() {
		c.goroutine.Do(c.refresh)
	})
}

func (c *MongoCacher) Stop() {
	c.goroutine.Close(true)

	util.SafeCloseChan(c.ready)
}

func (c *MongoCacher) Ready() <-chan struct{} {
	return c.ready
}

func (c *MongoCacher) IsReady() bool {
	select {
	case <-c.ready:
		return true
	default:
		return false
	}
}

func (c *MongoCacher) needList() bool {
	if c.isFirstTime {
		c.isFirstTime = false
		return true
	}
	c.reListCount++
	if c.reListCount < sdcommon.DefaultForceListInterval {
		return false
	}
	c.reListCount = 0
	return true
}

func (c *MongoCacher) doList(cfg sdcommon.ListWatchConfig) error {
	resp, err := c.lw.List(cfg)
	if err != nil {
		return err
	}

	resources := resp.Resources

	defer log.Debug(fmt.Sprintf("finish to cache key %s, %d items",
		c.Options.Key, len(resources)))

	//just reset the cacher if cache marked dirty
	if c.cache.Dirty() {
		c.reset(resources)
		log.Warn(fmt.Sprintf("Cache[%s] is reset!", c.cache.Name()))
		return nil
	}

	// calc and return the diff between cache and mongodb
	events := c.filter(resources)

	//notify the subscribers
	c.sync(events)
	return nil
}

func (c *MongoCacher) reset(infos []*sdcommon.Resource) {
	// clear cache before Set is safe, because the watch operation is stop,
	// but here will make all API requests go to MONGO directly.
	c.cache.Clear()
	// do not notify when cacher is dirty status,
	// otherwise, too many events will notify to downstream.
	c.buildCache(c.filter(infos))
}

func (c *MongoCacher) doWatch(cfg sdcommon.ListWatchConfig) error {
	if eventbus := c.lw.EventBus(cfg); eventbus != nil {
		return c.handleEventBus(eventbus)
	}
	return fmt.Errorf("handle a nil watcher")
}

func (c *MongoCacher) ListAndWatch(ctx context.Context) error {
	c.mux.Lock()
	defer c.mux.Unlock()
	defer log.Recover() // ensure ListAndWatch never raise panic

	cfg := sdcommon.ListWatchConfig{
		Timeout: c.Options.Timeout,
		Context: ctx,
	}

	// first time should initial cache, set watch timeout less
	if c.isFirstTime {
		cfg.Timeout = FirstTimeout
	}

	err := c.doWatch(cfg)
	if err != nil {
		log.Error("doWatch err", err)
	}

	// the scenario need to list mongo:
	// 1. Initial: cache is building, the lister's is first time to run.
	// 2. Runtime: error occurs in previous watch operation, the lister's status is set to error.
	// 3. Runtime: watch operation timed out over DEFAULT_FORCE_LIST_INTERVAL times.
	if c.needList() {
		// recover timeout for list
		if c.isFirstTime {
			cfg.Timeout = c.Options.Timeout
		}

		if err := c.doList(cfg); err != nil && (!c.IsReady()) {
			log.Error("doList error", err)
			return err // do retry to list mongo
		}
		// keep going to next step:
		// 1. doList return OK.
		// 2. some traps in mongo client
	}

	util.SafeCloseChan(c.ready)

	return nil
}

func (c *MongoCacher) handleEventBus(eventbus *sdcommon.EventBus) error {
	defer eventbus.Stop()

	if eventbus.Bus == nil {
		return nil
	}

	for resp := range eventbus.ResourceEventBus() {
		events := make([]MongoEvent, 0)

		if resp == nil {
			return errors.New("handle watcher error")
		}

		for _, resource := range resp.Resources {
			action := resp.Action
			var event MongoEvent
			switch action {
			case sdcommon.ActionCreate:
				event = NewMongoEventByResource(resource, rmodel.EVT_CREATE)
			case sdcommon.ActionUpdate:
				event = NewMongoEventByResource(resource, rmodel.EVT_UPDATE)
			case sdcommon.ActionDelete:
				resource.Key = c.cache.GetKeyByDocumentID(resource.DocumentID)
				resource.Value = c.cache.Get(resource.Key)
				event = NewMongoEventByResource(resource, rmodel.EVT_DELETE)
			}
			events = append(events, event)
		}

		c.sync(events)
		log.Debug(fmt.Sprintf("finish to handle %d events, table: %s", len(events), c.Options.Key))
	}

	return nil
}

func (c *MongoCacher) refresh(ctx context.Context) {
	log.Debug(fmt.Sprintf("start to list and watch %s", c.Options))
	retries := 0

	timer := time.NewTimer(sdcommon.MinWaitInterval)
	defer timer.Stop()
	for {
		nextPeriod := sdcommon.MinWaitInterval
		if err := c.ListAndWatch(ctx); err != nil {
			retries++
			nextPeriod = backoff.GetBackoff().Delay(retries)
		} else {
			retries = 0
		}

		select {
		case <-ctx.Done():
			log.Debug(fmt.Sprintf("stop to list and watch %s", c.Options))
			return
		case <-timer.C:
			timer.Reset(nextPeriod)
		}
	}
}

// keep the evts valID when call sync
func (c *MongoCacher) sync(evts []MongoEvent) {
	if len(evts) == 0 {
		return
	}

	c.onEvents(evts)
}

func (c *MongoCacher) filter(infos []*sdcommon.Resource) []MongoEvent {
	nc := len(infos)
	newStore := make(map[string]interface{}, nc)
	documentIDRecord := make(map[string]string, nc)
	indexRecord := make(map[string]string, nc)

	for _, info := range infos {
		event := NewMongoEventByResource(info, rmodel.EVT_CREATE)
		newStore[event.ResourceID] = info.Value
		documentIDRecord[event.ResourceID] = info.DocumentID
		indexRecord[event.ResourceID] = info.Index
	}

	filterStopCh := make(chan struct{})
	eventsCh := make(chan [sdcommon.EventBlockSize]MongoEvent, 2)

	go c.filterDelete(newStore, eventsCh, filterStopCh)

	go c.filterCreateOrUpdate(newStore, documentIDRecord, indexRecord, eventsCh, filterStopCh)

	events := make([]MongoEvent, 0, nc)
	for block := range eventsCh {
		for _, e := range block {
			if e.Value == nil {
				break
			}
			events = append(events, e)
		}
	}
	return events
}

func (c *MongoCacher) filterDelete(newStore map[string]interface{},
	eventsCh chan [sdcommon.EventBlockSize]MongoEvent, filterStopCh chan struct{}) {
	var block [sdcommon.EventBlockSize]MongoEvent
	i := 0

	c.cache.ForEach(func(k string, v interface{}) (next bool) {
		next = true

		_, ok := newStore[k]
		if ok {
			// k in store, also in new store, is not deleted, return
			return
		}

		// k in store but not in new store, it means k is deleted
		if i >= sdcommon.EventBlockSize {
			eventsCh <- block
			block = [sdcommon.EventBlockSize]MongoEvent{}
			i = 0
		}

		documentID := c.cache.GetDocumentIDByBussinessID(k)
		index := c.cache.GetIndexByBussinessID(k)
		block[i] = NewMongoEvent(k, documentID, index, rmodel.EVT_DELETE, v)
		i++
		return
	})

	if i > 0 {
		eventsCh <- block
	}

	close(filterStopCh)
}

func (c *MongoCacher) filterCreateOrUpdate(newStore map[string]interface{}, newDocumentStore map[string]string, indexRecord map[string]string,
	eventsCh chan [sdcommon.EventBlockSize]MongoEvent, filterStopCh chan struct{}) {
	var block [sdcommon.EventBlockSize]MongoEvent
	i := 0

	for k, v := range newStore {
		ov := c.cache.Get(k)
		if ov == nil {
			if i >= sdcommon.EventBlockSize {
				eventsCh <- block
				block = [sdcommon.EventBlockSize]MongoEvent{}
				i = 0
			}

			block[i] = NewMongoEvent(k, newDocumentStore[k], indexRecord[k], rmodel.EVT_CREATE, v)
			i++

			continue
		}

		if c.isValueNotUpdated(v, ov) {
			continue
		}

		log.Debug(fmt.Sprintf("value is updateOp of key:%s, old value is:%s, new value is:%s", k, ov, v))

		if i >= sdcommon.EventBlockSize {
			eventsCh <- block
			block = [sdcommon.EventBlockSize]MongoEvent{}
			i = 0
		}

		block[i] = NewMongoEvent(k, newDocumentStore[k], indexRecord[k], rmodel.EVT_UPDATE, v)
		i++
	}

	if i > 0 {
		eventsCh <- block
	}

	<-filterStopCh

	close(eventsCh)
}

func (c *MongoCacher) isValueNotUpdated(value interface{}, newValue interface{}) bool {
	var modTime string
	var newModTime string

	switch c.Options.Key {
	case instance:
		instance := value.(Instance)
		newInstance := newValue.(Instance)
		if instance.Instance == nil || newInstance.Instance == nil {
			return true
		}
		modTime = instance.Instance.ModTimestamp
		newModTime = newInstance.Instance.ModTimestamp
	case service:
		service := value.(Service)
		newService := newValue.(Service)
		if service.Service == nil || newService.Service == nil {
			return true
		}
		modTime = service.Service.ModTimestamp
		newModTime = newService.Service.ModTimestamp
	}

	if newModTime == "" || modTime == newModTime {
		return true
	}

	return false
}

func (c *MongoCacher) onEvents(events []MongoEvent) {
	c.buildCache(events)

	c.notify(events)
}

func (c *MongoCacher) buildCache(events []MongoEvent) {
	for i, evt := range events {
		key := evt.ResourceID
		value := c.cache.Get(key)
		ok := value != nil

		switch evt.Type {
		case rmodel.EVT_CREATE, rmodel.EVT_UPDATE:
			switch {
			case !c.IsReady():
				evt.Type = rmodel.EVT_INIT
			case !ok && evt.Type != rmodel.EVT_CREATE:
				log.Warn(fmt.Sprintf("unexpected %s event! it should be %s key %s",
					evt.Type, rmodel.EVT_CREATE, key))
				evt.Type = rmodel.EVT_CREATE
			case ok && evt.Type != rmodel.EVT_UPDATE:
				log.Warn(fmt.Sprintf("unexpected %s event! it should be %s key %s",
					evt.Type, rmodel.EVT_UPDATE, key))
				evt.Type = rmodel.EVT_UPDATE
			}

			c.cache.Put(key, evt.Value)
			c.cache.PutDocumentID(key, evt.DocumentID)
			c.cache.PutIndex(evt.Index, evt.ResourceID)

			events[i] = evt
		case rmodel.EVT_DELETE:
			if !ok {
				log.Warn(fmt.Sprintf("unexpected %s event! key %s does not cache",
					evt.Type, key))
			} else {
				evt.Value = value

				c.cache.Remove(key)
				c.cache.RemoveDocumentID(evt.DocumentID)
				c.cache.RemoveIndex(evt.Index, evt.ResourceID)
			}
			events[i] = evt
		}
	}
}

func (c *MongoCacher) notify(evts []MongoEvent) {
	eventProxy := EventProxy(c.Options.Key)

	if eventProxy == nil {
		return
	}

	defer log.Recover()

	for _, evt := range evts {
		if evt.Type == rmodel.EVT_DELETE && evt.Value == nil {
			log.Warn(fmt.Sprintf("caught delete event:%s, but value can't get from caches, it may be deleted by last list", evt.ResourceID))
			continue
		}
		eventProxy.OnEvent(evt)
	}
}

func NewMongoCacher(options *Options, cache *MongoCache) *MongoCacher {
	return &MongoCacher{
		Options:     options,
		isFirstTime: true,
		cache:       cache,
		ready:       make(chan struct{}),
		lw: &mongoListWatch{
			Key: options.Key,
		},
		goroutine: gopool.New(context.Background()),
	}
}
<|endoftext|>


<commit_before>package securitygroup

import (
	"encoding/json"
	"fmt"
	"strings"

	cfclient "github.com/cloudfoundry-community/go-cfclient"
	"github.com/pivotalservices/cf-mgmt/config"
	"github.com/pivotalservices/cf-mgmt/space"
	"github.com/pkg/errors"
	"github.com/xchapter7x/lo"
)

//NewManager -
func NewManager(client CFClient, spaceMgr space.Manager, cfg config.Reader, peek bool) Manager {
	return &DefaultManager{
		Cfg:          cfg,
		Client:       client,
		SpaceManager: spaceMgr,
		Peek:         peek,
	}
}

//DefaultSecurityGroupManager -
type DefaultManager struct {
	Cfg          config.Reader
	SpaceManager space.Manager
	Client       CFClient
	Peek         bool
}

//CreateApplicationSecurityGroups -
func (m *DefaultManager) CreateApplicationSecurityGroups() error {
	spaceConfigs, err := m.Cfg.GetSpaceConfigs()
	if err != nil {
		return err
	}
	sgs, err := m.ListNonDefaultSecurityGroups()
	if err != nil {
		return err
	}

	for _, input := range spaceConfigs {
		space, err := m.SpaceManager.FindSpace(input.Org, input.Space)
		if err != nil {
			return err
		}

		existingSpaceSecurityGroups, err := m.ListSpaceSecurityGroups(space.Guid)
		if err != nil {
			return errors.Wrapf(err, "Unabled to list existing space security groups for org/space [%s/%s]", input.Org, input.Space)
		}
		lo.G.Debugf("Existing space security groups %+v", existingSpaceSecurityGroups)
		// iterate through and assign named security groups to the space - ensuring that they are up to date is
		// done elsewhere.
		for _, securityGroupName := range input.ASGs {
			if sgInfo, ok := sgs[securityGroupName]; ok {
				if _, ok := existingSpaceSecurityGroups[securityGroupName]; !ok {
					err := m.AssignSecurityGroupToSpace(space, sgInfo)
					if err != nil {
						return err
					}
				} else {
					delete(existingSpaceSecurityGroups, securityGroupName)
				}
			} else {
				return fmt.Errorf("Security group [%s] does not exist", securityGroupName)
			}
		}

		spaceSecurityGroupName := fmt.Sprintf("%s-%s", input.Org, input.Space)
		if input.EnableSecurityGroup {
			var sgInfo cfclient.SecGroup
			var ok bool
			if sgInfo, ok = sgs[spaceSecurityGroupName]; ok {
				changed, err := m.hasSecurityGroupChanged(sgInfo, input.GetSecurityGroupContents())
				if err != nil {
					return errors.Wrapf(err, "Checking if security group %s has changed", spaceSecurityGroupName)
				}
				if changed {
					if err := m.UpdateSecurityGroup(sgInfo, input.GetSecurityGroupContents()); err != nil {
						return err
					}
				}
			} else {
				securityGroup, err := m.CreateSecurityGroup(spaceSecurityGroupName, input.GetSecurityGroupContents())
				if err != nil {
					return err
				}
				sgInfo = *securityGroup
			}
			if _, ok := existingSpaceSecurityGroups[spaceSecurityGroupName]; !ok {
				err := m.AssignSecurityGroupToSpace(space, sgInfo)
				if err != nil {
					return err
				}
			} else {
				delete(existingSpaceSecurityGroups, spaceSecurityGroupName)
			}
		}

		if input.EnableUnassignSecurityGroup {
			lo.G.Debugf("Existing space security groups after %+v", existingSpaceSecurityGroups)
			for sgName, _ := range existingSpaceSecurityGroups {
				if sgInfo, ok := sgs[sgName]; ok {
					err := m.UnassignSecurityGroupToSpace(space, sgInfo)
					if err != nil {
						return err
					}
				} else {
					return fmt.Errorf("Security group [%s] does not exist", sgName)
				}
			}
		}
	}
	return nil
}

func (m *DefaultManager) ListSecurityGroups() (map[string]cfclient.SecGroup, error) {
	securityGroups := make(map[string]cfclient.SecGroup)
	secGroups, err := m.Client.ListSecGroups()
	if err != nil {
		return securityGroups, err
	}
	lo.G.Debug("Total security groups returned :", len(secGroups))
	for _, sg := range secGroups {
		securityGroups[sg.Name] = sg
	}
	return securityGroups, nil
}

//CreateGlobalSecurityGroups -
func (m *DefaultManager) CreateGlobalSecurityGroups() error {
	sgs, err := m.ListSecurityGroups()
	if err != nil {
		return err
	}
	securityGroupConfigs, err := m.Cfg.GetASGConfigs()
	if err != nil {
		return err
	}
	defaultSecurityGroupConfigs, err := m.Cfg.GetDefaultASGConfigs()
	if err != nil {
		return err
	}
	err = m.processSecurityGroups(securityGroupConfigs, sgs)
	if err != nil {
		return err
	}
	err = m.processSecurityGroups(defaultSecurityGroupConfigs, sgs)
	if err != nil {
		return err
	}

	return nil
}

//AssignDefaultSecurityGroups -
func (m *DefaultManager) AssignDefaultSecurityGroups() error {
	sgs, err := m.ListSecurityGroups()
	if err != nil {
		return err
	}
	globalConfig, err := m.Cfg.GetGlobalConfig()
	if err != nil {
		return err
	}

	for _, runningGroup := range globalConfig.RunningSecurityGroups {
		if group, ok := sgs[runningGroup]; ok {
			if !group.Running {
				err = m.AssignRunningSecurityGroup(group)
				if err != nil {
					return err
				}
			}
		} else {
			if !m.Peek {
				return fmt.Errorf("Running security group [%s] does not exist", runningGroup)
			} else {
				lo.G.Infof("[dry-run]: assigning sg %s as running security group", runningGroup)
			}
		}
	}

	for _, stagingGroup := range globalConfig.StagingSecurityGroups {
		if group, ok := sgs[stagingGroup]; ok {
			if !group.Staging {
				err = m.AssignStagingSecurityGroup(group)
				if err != nil {
					return err
				}
			}
		} else {
			if !m.Peek {
				return fmt.Errorf("Staging security group [%s] does not exist", stagingGroup)
			} else {
				lo.G.Infof("[dry-run]: assigning sg %s as staging security group", stagingGroup)
			}
		}
	}

	if globalConfig.EnableUnassignSecurityGroups {
		for groupName, group := range sgs {
			if group.Running && !m.contains(globalConfig.RunningSecurityGroups, groupName) {
				err = m.UnassignRunningSecurityGroup(group)
				if err != nil {
					return err
				}
			}
			if group.Staging && !m.contains(globalConfig.StagingSecurityGroups, groupName) {
				err = m.UnassignStagingSecurityGroup(group)
				if err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func (m *DefaultManager) contains(list []string, groupName string) bool {
	groupNameToUpper := strings.ToUpper(groupName)
	for _, v := range list {
		if strings.ToUpper(v) == groupNameToUpper {
			return true
		}
	}
	return false
}

func (m *DefaultManager) processSecurityGroups(securityGroupConfigs []config.ASGConfig, sgs map[string]cfclient.SecGroup) error {
	for _, input := range securityGroupConfigs {
		sgName := input.Name

		// For every named security group
		// Check if it's a new group or Update
		if sgInfo, ok := sgs[sgName]; ok {
			changed, err := m.hasSecurityGroupChanged(sgInfo, input.Rules)
			if err != nil {
				return err
			}
			if changed {
				if err := m.UpdateSecurityGroup(sgInfo, input.Rules); err != nil {
					return err
				}
			}
		} else {
			if _, err := m.CreateSecurityGroup(sgName, input.Rules); err != nil {
				return err
			}
		}
	}

	return nil
}

func (m *DefaultManager) hasSecurityGroupChanged(sgInfo cfclient.SecGroup, rules string) (bool, error) {
	jsonBytes, err := json.Marshal(sgInfo.Rules)
	if err != nil {
		return false, err
	}
	secRules := []cfclient.SecGroupRule{}
	err = json.Unmarshal([]byte(rules), &secRules)
	if err != nil {
		return false, err
	}
	jsonBytesToCompare, err := json.Marshal(secRules)
	if err != nil {
		return false, err
	}
	match, err := DoesJsonMatch(string(jsonBytes), string(jsonBytesToCompare))
	if err != nil {
		return false, err
	}
	return !match, nil
}

func (m *DefaultManager) AssignSecurityGroupToSpace(space cfclient.Space, secGroup cfclient.SecGroup) error {
	for _, configuredSpace := range secGroup.SpacesData {
		if configuredSpace.Entity.Guid == space.Guid {
			return nil
		}
	}
	if m.Peek {
		lo.G.Infof("[dry-run]: assigning security group %s to space %s", secGroup.Name, space.Name)
		return nil
	}
	lo.G.Infof("assigning security group %s to space %s", secGroup.Name, space.Name)
	return m.Client.BindSecGroup(secGroup.Guid, space.Guid)
}

func (m *DefaultManager) UnassignSecurityGroupToSpace(space cfclient.Space, secGroup cfclient.SecGroup) error {
	for _, configuredSpace := range secGroup.SpacesData {
		if configuredSpace.Entity.Guid == space.Guid {
			return nil
		}
	}
	if m.Peek {
		lo.G.Infof("[dry-run]: unassigning security group %s to space %s", secGroup.Name, space.Name)
		return nil
	}
	lo.G.Infof("unassigning security group %s to space %s", secGroup.Name, space.Name)
	return m.Client.UnbindSecGroup(secGroup.Guid, space.Guid)
}

func (m *DefaultManager) CreateSecurityGroup(sgName, contents string) (*cfclient.SecGroup, error) {
	if m.Peek {
		lo.G.Infof("[dry-run]: creating securityGroup %s with contents %s", sgName, contents)
		return &cfclient.SecGroup{Name: "dry-run-name", Guid: "dry-run-guid"}, nil
	}
	securityGroupRules := []cfclient.SecGroupRule{}
	err := json.Unmarshal([]byte(contents), &securityGroupRules)
	if err != nil {
		return nil, err
	}
	lo.G.Infof("creating securityGroup %s with contents %s", sgName, contents)
	return m.Client.CreateSecGroup(sgName, securityGroupRules, nil)
}

func (m *DefaultManager) UpdateSecurityGroup(sg cfclient.SecGroup, contents string) error {
	if m.Peek {
		lo.G.Infof("[dry-run]: updating securityGroup %s with contents %s", sg.Name, contents)
		return nil
	}
	securityGroupRules := []cfclient.SecGroupRule{}
	err := json.Unmarshal([]byte(contents), &securityGroupRules)
	if err != nil {
		return err
	}
	lo.G.Infof("[dry-run]: updating securityGroup %s with contents %s", sg.Name, contents)
	_, err = m.Client.UpdateSecGroup(sg.Guid, sg.Name, securityGroupRules, nil)
	return err
}
func (m *DefaultManager) ListNonDefaultSecurityGroups() (map[string]cfclient.SecGroup, error) {
	securityGroups := make(map[string]cfclient.SecGroup)
	groupMap, err := m.ListSecurityGroups()
	if err != nil {
		return nil, err
	}
	for key, groupMap := range groupMap {
		if groupMap.Running == false && groupMap.Staging == false {
			securityGroups[key] = groupMap
		}
	}
	return securityGroups, nil
}

func (m *DefaultManager) ListDefaultSecurityGroups() (map[string]cfclient.SecGroup, error) {
	securityGroups := make(map[string]cfclient.SecGroup)
	groupMap, err := m.ListSecurityGroups()
	if err != nil {
		return nil, err
	}
	for key, groupMap := range groupMap {
		if groupMap.Running == true || groupMap.Staging == true {
			securityGroups[key] = groupMap
		}
	}
	return securityGroups, nil
}

func (m *DefaultManager) AssignRunningSecurityGroup(sg cfclient.SecGroup) error {
	if m.Peek {
		lo.G.Infof("[dry-run]: assigning sg %s as running security group", sg.Name)
		return nil
	}
	lo.G.Infof("assigning sg %s as running security group", sg.Name)
	return m.Client.BindRunningSecGroup(sg.Guid)
}
func (m *DefaultManager) AssignStagingSecurityGroup(sg cfclient.SecGroup) error {
	if m.Peek {
		lo.G.Infof("[dry-run]: assigning sg %s as staging security group", sg.Name)
		return nil
	}
	lo.G.Infof("assigning sg %s as staging security group", sg.Name)
	return m.Client.BindStagingSecGroup(sg.Guid)
}
func (m *DefaultManager) UnassignRunningSecurityGroup(sg cfclient.SecGroup) error {
	if m.Peek {
		lo.G.Infof("[dry-run]: unassinging sg %s as running security group", sg.Name)
		return nil
	}
	lo.G.Infof("unassinging sg %s as running security group", sg.Name)
	return m.Client.UnbindRunningSecGroup(sg.Guid)
}
func (m *DefaultManager) UnassignStagingSecurityGroup(sg cfclient.SecGroup) error {
	if m.Peek {
		lo.G.Infof("[dry-run]: unassigning sg %s as staging security group", sg.Name)
		return nil
	}
	lo.G.Infof("unassigning sg %s as staging security group", sg.Name)
	return m.Client.UnbindStagingSecGroup(sg.Guid)
}

func (m *DefaultManager) GetSecurityGroupRules(sgGUID string) ([]byte, error) {
	secGroup, err := m.Client.GetSecGroup(sgGUID)
	if err != nil {
		return nil, err
	}
	return json.MarshalIndent(secGroup.Rules, "", "\t")
}

func (m *DefaultManager) ListSpaceSecurityGroups(spaceGUID string) (map[string]string, error) {
	secGroups, err := m.Client.ListSpaceSecGroups(spaceGUID)
	if err != nil {
		return nil, err
	}
	lo.G.Debug("Total security groups returned :", len(secGroups))
	names := make(map[string]string)
	for _, sg := range secGroups {
		if sg.Running == false && sg.Staging == false {
			names[sg.Name] = sg.Guid
		}
	}
	return names, nil
}
<commit_msg>adding logic to allow peek to work even if security group had not been created - https://github.com/pivotalservices/cf-mgmt/issues/184<commit_after>package securitygroup

import (
	"encoding/json"
	"fmt"
	"strings"

	cfclient "github.com/cloudfoundry-community/go-cfclient"
	"github.com/pivotalservices/cf-mgmt/config"
	"github.com/pivotalservices/cf-mgmt/space"
	"github.com/pkg/errors"
	"github.com/xchapter7x/lo"
)

//NewManager -
func NewManager(client CFClient, spaceMgr space.Manager, cfg config.Reader, peek bool) Manager {
	return &DefaultManager{
		Cfg:          cfg,
		Client:       client,
		SpaceManager: spaceMgr,
		Peek:         peek,
	}
}

//DefaultSecurityGroupManager -
type DefaultManager struct {
	Cfg          config.Reader
	SpaceManager space.Manager
	Client       CFClient
	Peek         bool
}

//CreateApplicationSecurityGroups -
func (m *DefaultManager) CreateApplicationSecurityGroups() error {
	spaceConfigs, err := m.Cfg.GetSpaceConfigs()
	if err != nil {
		return err
	}
	sgs, err := m.ListNonDefaultSecurityGroups()
	if err != nil {
		return err
	}

	for _, input := range spaceConfigs {
		space, err := m.SpaceManager.FindSpace(input.Org, input.Space)
		if err != nil {
			return err
		}

		existingSpaceSecurityGroups, err := m.ListSpaceSecurityGroups(space.Guid)
		if err != nil {
			return errors.Wrapf(err, "Unabled to list existing space security groups for org/space [%s/%s]", input.Org, input.Space)
		}
		lo.G.Debugf("Existing space security groups %+v", existingSpaceSecurityGroups)
		// iterate through and assign named security groups to the space - ensuring that they are up to date is
		// done elsewhere.
		for _, securityGroupName := range input.ASGs {
			if sgInfo, ok := sgs[securityGroupName]; ok {
				if _, ok := existingSpaceSecurityGroups[securityGroupName]; !ok {
					err := m.AssignSecurityGroupToSpace(space, sgInfo)
					if err != nil {
						return err
					}
				} else {
					delete(existingSpaceSecurityGroups, securityGroupName)
				}
			} else {
				return fmt.Errorf("Security group [%s] does not exist", securityGroupName)
			}
		}

		spaceSecurityGroupName := fmt.Sprintf("%s-%s", input.Org, input.Space)
		if input.EnableSecurityGroup {
			var sgInfo cfclient.SecGroup
			var ok bool
			if sgInfo, ok = sgs[spaceSecurityGroupName]; ok {
				changed, err := m.hasSecurityGroupChanged(sgInfo, input.GetSecurityGroupContents())
				if err != nil {
					return errors.Wrapf(err, "Checking if security group %s has changed", spaceSecurityGroupName)
				}
				if changed {
					if err := m.UpdateSecurityGroup(sgInfo, input.GetSecurityGroupContents()); err != nil {
						return err
					}
				}
			} else {
				securityGroup, err := m.CreateSecurityGroup(spaceSecurityGroupName, input.GetSecurityGroupContents())
				if err != nil {
					return err
				}
				sgInfo = *securityGroup
			}
			if _, ok := existingSpaceSecurityGroups[spaceSecurityGroupName]; !ok {
				err := m.AssignSecurityGroupToSpace(space, sgInfo)
				if err != nil {
					return err
				}
			} else {
				delete(existingSpaceSecurityGroups, spaceSecurityGroupName)
			}
		}

		if input.EnableUnassignSecurityGroup {
			lo.G.Debugf("Existing space security groups after %+v", existingSpaceSecurityGroups)
			for sgName, _ := range existingSpaceSecurityGroups {
				if sgInfo, ok := sgs[sgName]; ok {
					err := m.UnassignSecurityGroupToSpace(space, sgInfo)
					if err != nil {
						return err
					}
				} else {
					return fmt.Errorf("Security group [%s] does not exist", sgName)
				}
			}
		}
	}
	return nil
}

func (m *DefaultManager) ListSecurityGroups() (map[string]cfclient.SecGroup, error) {
	securityGroups := make(map[string]cfclient.SecGroup)
	secGroups, err := m.Client.ListSecGroups()
	if err != nil {
		return securityGroups, err
	}
	lo.G.Debug("Total security groups returned :", len(secGroups))
	for _, sg := range secGroups {
		securityGroups[sg.Name] = sg
	}
	return securityGroups, nil
}

//CreateGlobalSecurityGroups -
func (m *DefaultManager) CreateGlobalSecurityGroups() error {
	sgs, err := m.ListSecurityGroups()
	if err != nil {
		return err
	}
	securityGroupConfigs, err := m.Cfg.GetASGConfigs()
	if err != nil {
		return err
	}
	defaultSecurityGroupConfigs, err := m.Cfg.GetDefaultASGConfigs()
	if err != nil {
		return err
	}
	err = m.processSecurityGroups(securityGroupConfigs, sgs)
	if err != nil {
		return err
	}
	err = m.processSecurityGroups(defaultSecurityGroupConfigs, sgs)
	if err != nil {
		return err
	}

	return nil
}

//AssignDefaultSecurityGroups -
func (m *DefaultManager) AssignDefaultSecurityGroups() error {
	sgs, err := m.ListSecurityGroups()
	if err != nil {
		return err
	}
	globalConfig, err := m.Cfg.GetGlobalConfig()
	if err != nil {
		return err
	}

	for _, runningGroup := range globalConfig.RunningSecurityGroups {
		if group, ok := sgs[runningGroup]; ok {
			if !group.Running {
				err = m.AssignRunningSecurityGroup(group)
				if err != nil {
					return err
				}
			}
		} else {
			if !m.Peek {
				return fmt.Errorf("Running security group [%s] does not exist", runningGroup)
			} else {
				lo.G.Infof("[dry-run]: assigning yet to be created sg %s as running security group", runningGroup)
			}
		}
	}

	for _, stagingGroup := range globalConfig.StagingSecurityGroups {
		if group, ok := sgs[stagingGroup]; ok {
			if !group.Staging {
				err = m.AssignStagingSecurityGroup(group)
				if err != nil {
					return err
				}
			}
		} else {
			if !m.Peek {
				return fmt.Errorf("Staging security group [%s] does not exist", stagingGroup)
			} else {
				lo.G.Infof("[dry-run]: assigning yet to be created sg %s as staging security group", stagingGroup)
			}
		}
	}

	if globalConfig.EnableUnassignSecurityGroups {
		for groupName, group := range sgs {
			if group.Running && !m.contains(globalConfig.RunningSecurityGroups, groupName) {
				err = m.UnassignRunningSecurityGroup(group)
				if err != nil {
					return err
				}
			}
			if group.Staging && !m.contains(globalConfig.StagingSecurityGroups, groupName) {
				err = m.UnassignStagingSecurityGroup(group)
				if err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func (m *DefaultManager) contains(list []string, groupName string) bool {
	groupNameToUpper := strings.ToUpper(groupName)
	for _, v := range list {
		if strings.ToUpper(v) == groupNameToUpper {
			return true
		}
	}
	return false
}

func (m *DefaultManager) processSecurityGroups(securityGroupConfigs []config.ASGConfig, sgs map[string]cfclient.SecGroup) error {
	for _, input := range securityGroupConfigs {
		sgName := input.Name

		// For every named security group
		// Check if it's a new group or Update
		if sgInfo, ok := sgs[sgName]; ok {
			changed, err := m.hasSecurityGroupChanged(sgInfo, input.Rules)
			if err != nil {
				return err
			}
			if changed {
				if err := m.UpdateSecurityGroup(sgInfo, input.Rules); err != nil {
					return err
				}
			}
		} else {
			if _, err := m.CreateSecurityGroup(sgName, input.Rules); err != nil {
				return err
			}
		}
	}

	return nil
}

func (m *DefaultManager) hasSecurityGroupChanged(sgInfo cfclient.SecGroup, rules string) (bool, error) {
	jsonBytes, err := json.Marshal(sgInfo.Rules)
	if err != nil {
		return false, err
	}
	secRules := []cfclient.SecGroupRule{}
	err = json.Unmarshal([]byte(rules), &secRules)
	if err != nil {
		return false, err
	}
	jsonBytesToCompare, err := json.Marshal(secRules)
	if err != nil {
		return false, err
	}
	match, err := DoesJsonMatch(string(jsonBytes), string(jsonBytesToCompare))
	if err != nil {
		return false, err
	}
	return !match, nil
}

func (m *DefaultManager) AssignSecurityGroupToSpace(space cfclient.Space, secGroup cfclient.SecGroup) error {
	for _, configuredSpace := range secGroup.SpacesData {
		if configuredSpace.Entity.Guid == space.Guid {
			return nil
		}
	}
	if m.Peek {
		lo.G.Infof("[dry-run]: assigning security group %s to space %s", secGroup.Name, space.Name)
		return nil
	}
	lo.G.Infof("assigning security group %s to space %s", secGroup.Name, space.Name)
	return m.Client.BindSecGroup(secGroup.Guid, space.Guid)
}

func (m *DefaultManager) UnassignSecurityGroupToSpace(space cfclient.Space, secGroup cfclient.SecGroup) error {
	for _, configuredSpace := range secGroup.SpacesData {
		if configuredSpace.Entity.Guid == space.Guid {
			return nil
		}
	}
	if m.Peek {
		lo.G.Infof("[dry-run]: unassigning security group %s to space %s", secGroup.Name, space.Name)
		return nil
	}
	lo.G.Infof("unassigning security group %s to space %s", secGroup.Name, space.Name)
	return m.Client.UnbindSecGroup(secGroup.Guid, space.Guid)
}

func (m *DefaultManager) CreateSecurityGroup(sgName, contents string) (*cfclient.SecGroup, error) {
	if m.Peek {
		lo.G.Infof("[dry-run]: creating securityGroup %s with contents %s", sgName, contents)
		return &cfclient.SecGroup{Name: "dry-run-name", Guid: "dry-run-guid"}, nil
	}
	securityGroupRules := []cfclient.SecGroupRule{}
	err := json.Unmarshal([]byte(contents), &securityGroupRules)
	if err != nil {
		return nil, err
	}
	lo.G.Infof("creating securityGroup %s with contents %s", sgName, contents)
	return m.Client.CreateSecGroup(sgName, securityGroupRules, nil)
}

func (m *DefaultManager) UpdateSecurityGroup(sg cfclient.SecGroup, contents string) error {
	if m.Peek {
		lo.G.Infof("[dry-run]: updating securityGroup %s with contents %s", sg.Name, contents)
		return nil
	}
	securityGroupRules := []cfclient.SecGroupRule{}
	err := json.Unmarshal([]byte(contents), &securityGroupRules)
	if err != nil {
		return err
	}
	lo.G.Infof("[dry-run]: updating securityGroup %s with contents %s", sg.Name, contents)
	_, err = m.Client.UpdateSecGroup(sg.Guid, sg.Name, securityGroupRules, nil)
	return err
}
func (m *DefaultManager) ListNonDefaultSecurityGroups() (map[string]cfclient.SecGroup, error) {
	securityGroups := make(map[string]cfclient.SecGroup)
	groupMap, err := m.ListSecurityGroups()
	if err != nil {
		return nil, err
	}
	for key, groupMap := range groupMap {
		if groupMap.Running == false && groupMap.Staging == false {
			securityGroups[key] = groupMap
		}
	}
	return securityGroups, nil
}

func (m *DefaultManager) ListDefaultSecurityGroups() (map[string]cfclient.SecGroup, error) {
	securityGroups := make(map[string]cfclient.SecGroup)
	groupMap, err := m.ListSecurityGroups()
	if err != nil {
		return nil, err
	}
	for key, groupMap := range groupMap {
		if groupMap.Running == true || groupMap.Staging == true {
			securityGroups[key] = groupMap
		}
	}
	return securityGroups, nil
}

func (m *DefaultManager) AssignRunningSecurityGroup(sg cfclient.SecGroup) error {
	if m.Peek {
		lo.G.Infof("[dry-run]: assigning sg %s as running security group", sg.Name)
		return nil
	}
	lo.G.Infof("assigning sg %s as running security group", sg.Name)
	return m.Client.BindRunningSecGroup(sg.Guid)
}
func (m *DefaultManager) AssignStagingSecurityGroup(sg cfclient.SecGroup) error {
	if m.Peek {
		lo.G.Infof("[dry-run]: assigning sg %s as staging security group", sg.Name)
		return nil
	}
	lo.G.Infof("assigning sg %s as staging security group", sg.Name)
	return m.Client.BindStagingSecGroup(sg.Guid)
}
func (m *DefaultManager) UnassignRunningSecurityGroup(sg cfclient.SecGroup) error {
	if m.Peek {
		lo.G.Infof("[dry-run]: unassinging sg %s as running security group", sg.Name)
		return nil
	}
	lo.G.Infof("unassinging sg %s as running security group", sg.Name)
	return m.Client.UnbindRunningSecGroup(sg.Guid)
}
func (m *DefaultManager) UnassignStagingSecurityGroup(sg cfclient.SecGroup) error {
	if m.Peek {
		lo.G.Infof("[dry-run]: unassigning sg %s as staging security group", sg.Name)
		return nil
	}
	lo.G.Infof("unassigning sg %s as staging security group", sg.Name)
	return m.Client.UnbindStagingSecGroup(sg.Guid)
}

func (m *DefaultManager) GetSecurityGroupRules(sgGUID string) ([]byte, error) {
	secGroup, err := m.Client.GetSecGroup(sgGUID)
	if err != nil {
		return nil, err
	}
	return json.MarshalIndent(secGroup.Rules, "", "\t")
}

func (m *DefaultManager) ListSpaceSecurityGroups(spaceGUID string) (map[string]string, error) {
	secGroups, err := m.Client.ListSpaceSecGroups(spaceGUID)
	if err != nil {
		return nil, err
	}
	lo.G.Debug("Total security groups returned :", len(secGroups))
	names := make(map[string]string)
	for _, sg := range secGroups {
		if sg.Running == false && sg.Staging == false {
			names[sg.Name] = sg.Guid
		}
	}
	return names, nil
}
<|endoftext|>


<commit_before>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package provision provides interfaces that need to be satisfied in order to
// implement a new provisioner on tsuru.
package provision

import (
	"fmt"
	"io"
)

const (
	StatusStarted    = "started"
	StatusPending    = "pending"
	StatusDown       = "down"
	StatusError      = "error"
	StatusInstalling = "installing"
	StatusCreating   = "creating"
)

// Unit represents a provision unit. Can be a machine, container or anything
// IP-addressable.
type Unit struct {
	Name    string
	AppName string
	Type    string
	Machine int
	Ip      string
	Status  string
}

// AppUnit represents a unit in an app.
type AppUnit interface {
	// Returns the name of the unit.
	GetName() string

	// Returns the number of the unit.
	GetMachine() int
}

// App represents a tsuru app.
//
// It contains only relevant information for provisioning.
type App interface {
	// Log should be used to log messages in the app.
	Log(message, source string) error

	// GetName returns the name of the app.
	GetName() string

	// GetFramework returns the framework (type) of the app. It is equivalent
	// to the Unit `Type` field.
	GetFramework() string

	// GetUnits returns all units of the app, in a slice.
	ProvisionUnits() []AppUnit
}

// Provisioner is the basic interface of this package.
//
// Any tsuru provisioner must implement this interface in order to provision
// tsuru apps.
//
// Tsuru comes with a default provisioner: juju. One can add other provisioners
// by satisfying this interface and registering it using the function Register.
type Provisioner interface {
	// Provision is called when tsuru is creating the app.
	Provision(App) error

	// Destroy is called when tsuru is destroying the app.
	Destroy(App) error

	// ExecuteCommand runs a command in all units of the app.
	ExecuteCommand(stdout, stderr io.Writer, app App, cmd string, args ...string) error

	// CollectStatus returns information about all provisioned units. It's used
	// by tsuru collector when updating the status of apps in the database.
	CollectStatus() ([]Unit, error)
}

var provisioners = make(map[string]Provisioner)

// Register registers a new provisioner in the Provisioner registry.
func Register(name string, p Provisioner) {
	provisioners[name] = p
}

// Get gets the named provisioner from the registry.
func Get(name string) (Provisioner, error) {
	p, ok := provisioners[name]
	if !ok {
		return nil, fmt.Errorf("Unknown provisioner: %q.", name)
	}
	return p, nil
}

type Error struct {
	Reason string
	Err    error
}

func (e *Error) Error() string {
	var err string
	if e.Err != nil {
		err = e.Err.Error() + ": " + e.Reason
	} else {
		err = e.Reason
	}
	return err
}
<commit_msg>provision: introduce Status type<commit_after>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package provision provides interfaces that need to be satisfied in order to
// implement a new provisioner on tsuru.
package provision

import (
	"fmt"
	"io"
)

type Status string

const (
	StatusStarted    = Status("started")
	StatusPending    = Status("pending")
	StatusDown       = Status("down")
	StatusError      = Status("error")
	StatusInstalling = Status("installing")
	StatusCreating   = Status("creating")
)

// Unit represents a provision unit. Can be a machine, container or anything
// IP-addressable.
type Unit struct {
	Name    string
	AppName string
	Type    string
	Machine int
	Ip      string
	Status  Status
}

// AppUnit represents a unit in an app.
type AppUnit interface {
	// Returns the name of the unit.
	GetName() string

	// Returns the number of the unit.
	GetMachine() int

	// Returns the status of the unit.
	GetStatus() Status
}

// App represents a tsuru app.
//
// It contains only relevant information for provisioning.
type App interface {
	// Log should be used to log messages in the app.
	Log(message, source string) error

	// GetName returns the name of the app.
	GetName() string

	// GetFramework returns the framework (type) of the app. It is equivalent
	// to the Unit `Type` field.
	GetFramework() string

	// GetUnits returns all units of the app, in a slice.
	ProvisionUnits() []AppUnit
}

// Provisioner is the basic interface of this package.
//
// Any tsuru provisioner must implement this interface in order to provision
// tsuru apps.
//
// Tsuru comes with a default provisioner: juju. One can add other provisioners
// by satisfying this interface and registering it using the function Register.
type Provisioner interface {
	// Provision is called when tsuru is creating the app.
	Provision(App) error

	// Destroy is called when tsuru is destroying the app.
	Destroy(App) error

	// ExecuteCommand runs a command in all units of the app.
	ExecuteCommand(stdout, stderr io.Writer, app App, cmd string, args ...string) error

	// CollectStatus returns information about all provisioned units. It's used
	// by tsuru collector when updating the status of apps in the database.
	CollectStatus() ([]Unit, error)
}

var provisioners = make(map[string]Provisioner)

// Register registers a new provisioner in the Provisioner registry.
func Register(name string, p Provisioner) {
	provisioners[name] = p
}

// Get gets the named provisioner from the registry.
func Get(name string) (Provisioner, error) {
	p, ok := provisioners[name]
	if !ok {
		return nil, fmt.Errorf("Unknown provisioner: %q.", name)
	}
	return p, nil
}

type Error struct {
	Reason string
	Err    error
}

func (e *Error) Error() string {
	var err string
	if e.Err != nil {
		err = e.Err.Error() + ": " + e.Reason
	} else {
		err = e.Reason
	}
	return err
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tide

import (
	"context"
	"fmt"
	"testing"

	"github.com/Sirupsen/logrus"
	"github.com/shurcooL/githubql"

	"k8s.io/test-infra/prow/kube"
)

func testPullsMatchList(t *testing.T, test string, actual []pullRequest, expected []int) {
	if len(actual) != len(expected) {
		t.Errorf("Wrong size for case %s. Got PRs %+v, wanted numbers %v.", test, actual, expected)
		return
	}
	for _, pr := range actual {
		var found bool
		n1 := int(pr.Number)
		for _, n2 := range expected {
			if n1 == n2 {
				found = true
			}
		}
		if !found {
			t.Errorf("For case %s, found PR %d but shouldn't have.", test, n1)
		}
	}
}

type prowjob struct {
	prNumber int
	job      string
	state    kube.ProwJobState
}

func TestAccumulate(t *testing.T) {
	tests := []struct {
		presubmits   []string
		pullRequests []int
		prowJobs     []prowjob

		successes []int
		pendings  []int
		none      []int
	}{
		{
			presubmits:   []string{"job1", "job2"},
			pullRequests: []int{1, 2, 3, 4, 5, 6, 7},
			prowJobs: []prowjob{
				{2, "job1", kube.PendingState},
				{3, "job1", kube.PendingState},
				{3, "job2", kube.TriggeredState},
				{4, "job1", kube.FailureState},
				{4, "job2", kube.PendingState},
				{5, "job1", kube.PendingState},
				{5, "job2", kube.FailureState},
				{5, "job2", kube.PendingState},
				{6, "job1", kube.SuccessState},
				{6, "job2", kube.PendingState},
				{7, "job1", kube.SuccessState},
				{7, "job2", kube.SuccessState},
				{7, "job1", kube.FailureState},
			},

			successes: []int{7},
			pendings:  []int{3, 5, 6},
			none:      []int{1, 2, 4},
		},
	}

	for i, test := range tests {
		var pulls []pullRequest
		for _, p := range test.pullRequests {
			pulls = append(pulls, pullRequest{Number: githubql.Int(p)})
		}
		var pjs []kube.ProwJob
		for _, pj := range test.prowJobs {
			pjs = append(pjs, kube.ProwJob{
				Spec:   kube.ProwJobSpec{Job: pj.job, Refs: kube.Refs{Pulls: []kube.Pull{{Number: pj.prNumber}}}},
				Status: kube.ProwJobStatus{State: pj.state},
			})
		}

		successes, pendings, nones := accumulate(test.presubmits, pulls, pjs)

		t.Logf("test run %d", i)
		testPullsMatchList(t, "successes", successes, test.successes)
		testPullsMatchList(t, "pendings", pendings, test.pendings)
		testPullsMatchList(t, "nones", nones, test.none)
	}
}

type fgc struct {
	refs map[string]string
}

func (f *fgc) GetRef(o, r, ref string) (string, error) {
	return f.refs[o+"/"+r+" "+ref], nil
}

func (f *fgc) Query(ctx context.Context, q interface{}, vars map[string]interface{}) error {
	return nil
}

// TestDividePool ensures that subpools returned by dividePool satisfy a few
// important invariants.
func TestDividePool(t *testing.T) {
	testPulls := []struct {
		org    string
		repo   string
		number int
		branch string
	}{
		{
			org:    "k",
			repo:   "t-i",
			number: 5,
			branch: "master",
		},
		{
			org:    "k",
			repo:   "t-i",
			number: 6,
			branch: "master",
		},
		{
			org:    "k",
			repo:   "k",
			number: 123,
			branch: "master",
		},
		{
			org:    "k",
			repo:   "k",
			number: 1000,
			branch: "release-1.6",
		},
	}
	testPJs := []struct {
		jobType kube.ProwJobType
		org     string
		repo    string
		baseRef string
		baseSHA string
	}{
		{
			jobType: kube.PresubmitJob,
			org:     "k",
			repo:    "t-i",
			baseRef: "master",
			baseSHA: "123",
		},
		{
			jobType: kube.BatchJob,
			org:     "k",
			repo:    "t-i",
			baseRef: "master",
			baseSHA: "123",
		},
		{
			jobType: kube.PeriodicJob,
		},
		{
			jobType: kube.PresubmitJob,
			org:     "k",
			repo:    "t-i",
			baseRef: "patch",
			baseSHA: "123",
		},
		{
			jobType: kube.PresubmitJob,
			org:     "k",
			repo:    "t-i",
			baseRef: "master",
			baseSHA: "abc",
		},
		{
			jobType: kube.PresubmitJob,
			org:     "o",
			repo:    "t-i",
			baseRef: "master",
			baseSHA: "123",
		},
		{
			jobType: kube.PresubmitJob,
			org:     "k",
			repo:    "other",
			baseRef: "master",
			baseSHA: "123",
		},
	}
	fc := &fgc{
		refs: map[string]string{"k/t-i heads/master": "123"},
	}
	c := &Controller{
		log: logrus.NewEntry(logrus.StandardLogger()),
		ghc: fc,
	}
	var pulls []pullRequest
	for _, p := range testPulls {
		npr := pullRequest{Number: githubql.Int(p.number)}
		npr.BaseRef.Name = githubql.String(p.branch)
		npr.BaseRef.Prefix = "refs/heads/"
		npr.Repository.Name = githubql.String(p.repo)
		npr.Repository.Owner.Login = githubql.String(p.org)
		pulls = append(pulls, npr)
	}
	var pjs []kube.ProwJob
	for _, pj := range testPJs {
		pjs = append(pjs, kube.ProwJob{
			Spec: kube.ProwJobSpec{
				Type: pj.jobType,
				Refs: kube.Refs{
					Org:     pj.org,
					Repo:    pj.repo,
					BaseRef: pj.baseRef,
					BaseSHA: pj.baseSHA,
				},
			},
		})
	}
	sps, err := c.dividePool(pulls, pjs)
	if err != nil {
		t.Fatalf("Error dividing pool: %v", err)
	}
	if len(sps) == 0 {
		t.Error("No subpools.")
	}
	for _, sp := range sps {
		name := fmt.Sprintf("%s/%s %s", sp.org, sp.repo, sp.branch)
		sha := fc.refs[sp.org+"/"+sp.repo+" heads/"+sp.branch]
		if sp.sha != sha {
			t.Errorf("For subpool %s, got sha %s, expected %s.", name, sp.sha, sha)
		}
		if len(sp.prs) == 0 {
			t.Errorf("Subpool %s has no PRs.", name)
		}
		for _, pr := range sp.prs {
			if string(pr.Repository.Owner.Login) != sp.org || string(pr.Repository.Name) != sp.repo || string(pr.BaseRef.Name) != sp.branch {
				t.Errorf("PR in wrong subpool. Got PR %+v in subpool %s.", pr, name)
			}
		}
		for _, pj := range sp.pjs {
			if pj.Spec.Type != kube.PresubmitJob && pj.Spec.Type != kube.BatchJob {
				t.Errorf("PJ with bad type in subpool %s: %+v", name, pj)
			}
			if pj.Spec.Refs.Org != sp.org || pj.Spec.Refs.Repo != sp.repo || pj.Spec.Refs.BaseRef != sp.branch || pj.Spec.Refs.BaseSHA != sp.sha {
				t.Errorf("PJ in wrong subpool. Got PJ %+v in subpool %s.", pj, name)
			}
		}
	}
}
<commit_msg>Add more test cases in accumulate<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tide

import (
	"context"
	"fmt"
	"testing"

	"github.com/Sirupsen/logrus"
	"github.com/shurcooL/githubql"

	"k8s.io/test-infra/prow/kube"
)

func testPullsMatchList(t *testing.T, test string, actual []pullRequest, expected []int) {
	if len(actual) != len(expected) {
		t.Errorf("Wrong size for case %s. Got PRs %+v, wanted numbers %v.", test, actual, expected)
		return
	}
	for _, pr := range actual {
		var found bool
		n1 := int(pr.Number)
		for _, n2 := range expected {
			if n1 == n2 {
				found = true
			}
		}
		if !found {
			t.Errorf("For case %s, found PR %d but shouldn't have.", test, n1)
		}
	}
}

type prowjob struct {
	prNumber int
	job      string
	state    kube.ProwJobState
}

func TestAccumulate(t *testing.T) {
	tests := []struct {
		presubmits   []string
		pullRequests []int
		prowJobs     []prowjob

		successes []int
		pendings  []int
		none      []int
	}{
		{
			presubmits:   []string{"job1", "job2"},
			pullRequests: []int{1, 2, 3, 4, 5, 6, 7},
			prowJobs: []prowjob{
				{2, "job1", kube.PendingState},
				{3, "job1", kube.PendingState},
				{3, "job2", kube.TriggeredState},
				{4, "job1", kube.FailureState},
				{4, "job2", kube.PendingState},
				{5, "job1", kube.PendingState},
				{5, "job2", kube.FailureState},
				{5, "job2", kube.PendingState},
				{6, "job1", kube.SuccessState},
				{6, "job2", kube.PendingState},
				{7, "job1", kube.SuccessState},
				{7, "job2", kube.SuccessState},
				{7, "job1", kube.FailureState},
			},

			successes: []int{7},
			pendings:  []int{3, 5, 6},
			none:      []int{1, 2, 4},
		},
		{
			presubmits:   []string{"job1", "job2", "job3", "job4"},
			pullRequests: []int{7},
			prowJobs: []prowjob{
				{7, "job1", kube.SuccessState},
				{7, "job2", kube.FailureState},
				{7, "job3", kube.FailureState},
				{7, "job4", kube.FailureState},
				{7, "job3", kube.FailureState},
				{7, "job4", kube.FailureState},
				{7, "job2", kube.SuccessState},
				{7, "job3", kube.SuccessState},
				{7, "job4", kube.FailureState},
			},

			successes: []int{},
			pendings:  []int{},
			none:      []int{7},
		},
		{
			presubmits:   []string{"job1", "job2", "job3", "job4"},
			pullRequests: []int{7},
			prowJobs: []prowjob{
				{7, "job1", kube.FailureState},
				{7, "job2", kube.FailureState},
				{7, "job3", kube.FailureState},
				{7, "job4", kube.FailureState},
				{7, "job3", kube.FailureState},
				{7, "job4", kube.FailureState},
				{7, "job2", kube.FailureState},
				{7, "job3", kube.FailureState},
				{7, "job4", kube.FailureState},
			},

			successes: []int{},
			pendings:  []int{},
			none:      []int{7},
		},
		{
			presubmits:   []string{"job1", "job2", "job3", "job4"},
			pullRequests: []int{7},
			prowJobs: []prowjob{
				{7, "job1", kube.SuccessState},
				{7, "job2", kube.FailureState},
				{7, "job3", kube.FailureState},
				{7, "job4", kube.FailureState},
				{7, "job3", kube.FailureState},
				{7, "job4", kube.FailureState},
				{7, "job2", kube.SuccessState},
				{7, "job3", kube.SuccessState},
				{7, "job4", kube.SuccessState},
				{7, "job1", kube.FailureState},
			},

			successes: []int{7},
			pendings:  []int{},
			none:      []int{},
		},
		{
			presubmits:   []string{"job1", "job2", "job3", "job4"},
			pullRequests: []int{7},
			prowJobs: []prowjob{
				{7, "job1", kube.SuccessState},
				{7, "job2", kube.FailureState},
				{7, "job3", kube.FailureState},
				{7, "job4", kube.FailureState},
				{7, "job3", kube.FailureState},
				{7, "job4", kube.FailureState},
				{7, "job2", kube.SuccessState},
				{7, "job3", kube.SuccessState},
				{7, "job4", kube.PendingState},
				{7, "job1", kube.FailureState},
			},

			successes: []int{},
			pendings:  []int{7},
			none:      []int{},
		},
	}

	for i, test := range tests {
		var pulls []pullRequest
		for _, p := range test.pullRequests {
			pulls = append(pulls, pullRequest{Number: githubql.Int(p)})
		}
		var pjs []kube.ProwJob
		for _, pj := range test.prowJobs {
			pjs = append(pjs, kube.ProwJob{
				Spec:   kube.ProwJobSpec{Job: pj.job, Refs: kube.Refs{Pulls: []kube.Pull{{Number: pj.prNumber}}}},
				Status: kube.ProwJobStatus{State: pj.state},
			})
		}

		successes, pendings, nones := accumulate(test.presubmits, pulls, pjs)

		t.Logf("test run %d", i)
		testPullsMatchList(t, "successes", successes, test.successes)
		testPullsMatchList(t, "pendings", pendings, test.pendings)
		testPullsMatchList(t, "nones", nones, test.none)
	}
}

type fgc struct {
	refs map[string]string
}

func (f *fgc) GetRef(o, r, ref string) (string, error) {
	return f.refs[o+"/"+r+" "+ref], nil
}

func (f *fgc) Query(ctx context.Context, q interface{}, vars map[string]interface{}) error {
	return nil
}

// TestDividePool ensures that subpools returned by dividePool satisfy a few
// important invariants.
func TestDividePool(t *testing.T) {
	testPulls := []struct {
		org    string
		repo   string
		number int
		branch string
	}{
		{
			org:    "k",
			repo:   "t-i",
			number: 5,
			branch: "master",
		},
		{
			org:    "k",
			repo:   "t-i",
			number: 6,
			branch: "master",
		},
		{
			org:    "k",
			repo:   "k",
			number: 123,
			branch: "master",
		},
		{
			org:    "k",
			repo:   "k",
			number: 1000,
			branch: "release-1.6",
		},
	}
	testPJs := []struct {
		jobType kube.ProwJobType
		org     string
		repo    string
		baseRef string
		baseSHA string
	}{
		{
			jobType: kube.PresubmitJob,
			org:     "k",
			repo:    "t-i",
			baseRef: "master",
			baseSHA: "123",
		},
		{
			jobType: kube.BatchJob,
			org:     "k",
			repo:    "t-i",
			baseRef: "master",
			baseSHA: "123",
		},
		{
			jobType: kube.PeriodicJob,
		},
		{
			jobType: kube.PresubmitJob,
			org:     "k",
			repo:    "t-i",
			baseRef: "patch",
			baseSHA: "123",
		},
		{
			jobType: kube.PresubmitJob,
			org:     "k",
			repo:    "t-i",
			baseRef: "master",
			baseSHA: "abc",
		},
		{
			jobType: kube.PresubmitJob,
			org:     "o",
			repo:    "t-i",
			baseRef: "master",
			baseSHA: "123",
		},
		{
			jobType: kube.PresubmitJob,
			org:     "k",
			repo:    "other",
			baseRef: "master",
			baseSHA: "123",
		},
	}
	fc := &fgc{
		refs: map[string]string{"k/t-i heads/master": "123"},
	}
	c := &Controller{
		log: logrus.NewEntry(logrus.StandardLogger()),
		ghc: fc,
	}
	var pulls []pullRequest
	for _, p := range testPulls {
		npr := pullRequest{Number: githubql.Int(p.number)}
		npr.BaseRef.Name = githubql.String(p.branch)
		npr.BaseRef.Prefix = "refs/heads/"
		npr.Repository.Name = githubql.String(p.repo)
		npr.Repository.Owner.Login = githubql.String(p.org)
		pulls = append(pulls, npr)
	}
	var pjs []kube.ProwJob
	for _, pj := range testPJs {
		pjs = append(pjs, kube.ProwJob{
			Spec: kube.ProwJobSpec{
				Type: pj.jobType,
				Refs: kube.Refs{
					Org:     pj.org,
					Repo:    pj.repo,
					BaseRef: pj.baseRef,
					BaseSHA: pj.baseSHA,
				},
			},
		})
	}
	sps, err := c.dividePool(pulls, pjs)
	if err != nil {
		t.Fatalf("Error dividing pool: %v", err)
	}
	if len(sps) == 0 {
		t.Error("No subpools.")
	}
	for _, sp := range sps {
		name := fmt.Sprintf("%s/%s %s", sp.org, sp.repo, sp.branch)
		sha := fc.refs[sp.org+"/"+sp.repo+" heads/"+sp.branch]
		if sp.sha != sha {
			t.Errorf("For subpool %s, got sha %s, expected %s.", name, sp.sha, sha)
		}
		if len(sp.prs) == 0 {
			t.Errorf("Subpool %s has no PRs.", name)
		}
		for _, pr := range sp.prs {
			if string(pr.Repository.Owner.Login) != sp.org || string(pr.Repository.Name) != sp.repo || string(pr.BaseRef.Name) != sp.branch {
				t.Errorf("PR in wrong subpool. Got PR %+v in subpool %s.", pr, name)
			}
		}
		for _, pj := range sp.pjs {
			if pj.Spec.Type != kube.PresubmitJob && pj.Spec.Type != kube.BatchJob {
				t.Errorf("PJ with bad type in subpool %s: %+v", name, pj)
			}
			if pj.Spec.Refs.Org != sp.org || pj.Spec.Refs.Repo != sp.repo || pj.Spec.Refs.BaseRef != sp.branch || pj.Spec.Refs.BaseSHA != sp.sha {
				t.Errorf("PJ in wrong subpool. Got PJ %+v in subpool %s.", pj, name)
			}
		}
	}
}
<|endoftext|>


<commit_before>package strumt

import (
	"io"
)

// Prompter defines a generic common prompt.
//
// ID returns a string id to identify prompter and to let other prompter call it.
//
// PromptString returns a string to be diplayed as prompt.
//
// NextOnError is triggered when an error occurred during
// prompt sequence, it must returns the id of the prompt
// to be called when an error occured, most of the time it would
// be the id of the current prompt to loop on it
type Prompter interface {
	ID() string
	PromptString() string
	NextOnError(error) string
}

// LinePrompter defines a one line prompter
// that will ask only for one user input.
//
// NextOnSuccess must returns the id of the next prompt
// to be called. To mark prompter as the last prompter
// NextOnSucces must returns an empty string
type LinePrompter interface {
	Prompter
	NextOnSuccess(string) string
	Parse(string) error
}

// MultilinePrompter defines a mutiline prompter
// that will let the possibility to the user to
// provide several input, result is provided as
// an input slice.
//
// NextOnSuccess must returns the id of the next prompt
// to be called. To mark prompter as the last prompter
// NextOnSucces must returns an empty string
type MultilinePrompter interface {
	Prompter
	NextOnSuccess([]string) string
	Parse([]string) error
}

// PromptRenderer can be implemented to customize
// the way prompt is rendered, PromptString result is given as parameter
type PromptRenderer interface {
	PrintPrompt(io.Writer, string)
}

// ErrorRenderer can be implemented to customize
// the way an error returned by Parse is rendered
type ErrorRenderer interface {
	PrintError(io.Writer, error)
}

// SeparatorRenderer can be implemented to customize
// the way a prompt is separated from another, default
// is to add a new line
type SeparatorRenderer interface {
	PrintSeparator(io.Writer)
}
<commit_msg>Fix typo<commit_after>package strumt

import (
	"io"
)

// Prompter defines a generic common prompt.
//
// ID returns a string id to identify prompter and to let other prompter call it.
//
// PromptString returns a string to be diplayed as prompt.
//
// NextOnError is triggered when an error occurred during
// prompt sequence, it must returns the id of the prompt
// to be called when an error occurred, most of the time it would
// be the id of the current prompt to loop on it
type Prompter interface {
	ID() string
	PromptString() string
	NextOnError(error) string
}

// LinePrompter defines a one line prompter
// that will ask only for one user input.
//
// NextOnSuccess must returns the id of the next prompt
// to be called. To mark prompter as the last prompter
// NextOnSucces must returns an empty string
type LinePrompter interface {
	Prompter
	NextOnSuccess(string) string
	Parse(string) error
}

// MultilinePrompter defines a mutiline prompter
// that will let the possibility to the user to
// provide several input, result is provided as
// an input slice.
//
// NextOnSuccess must returns the id of the next prompt
// to be called. To mark prompter as the last prompter
// NextOnSucces must returns an empty string
type MultilinePrompter interface {
	Prompter
	NextOnSuccess([]string) string
	Parse([]string) error
}

// PromptRenderer can be implemented to customize
// the way prompt is rendered, PromptString result is given as parameter
type PromptRenderer interface {
	PrintPrompt(io.Writer, string)
}

// ErrorRenderer can be implemented to customize
// the way an error returned by Parse is rendered
type ErrorRenderer interface {
	PrintError(io.Writer, error)
}

// SeparatorRenderer can be implemented to customize
// the way a prompt is separated from another, default
// is to add a new line
type SeparatorRenderer interface {
	PrintSeparator(io.Writer)
}
<|endoftext|>


<commit_before>package go_config

type Config interface {
	Fields

	UseSource(sources ...Source)
	Unmarshal(v interface{}, prefix string) error

	SetDefault(key string, val interface{})
}

type Fields interface {
	Get(key string) interface{}

	Bool(key string) bool
	Float(key string) float64
	Int(key string) int
	UInt(key string) uint
	Slice(key, delimiter string) []interface{}
	String(key string) string
	StringMap(key string) map[string]interface{}

	Sub(key string) Fields

	IsSet(key string) bool
}

type Source interface {
	Get(key string) interface{}

	Bool(key string) (bool, error)
	Float(key string) (float64, error)
	Int(key string) (int, error)
	UInt(key string) (uint, error)
	Slice(key, delimiter string) ([]interface{}, error)
	String(key string) (string, error)
	StringMap(key string) map[string]interface{}

	IsSet(key string) bool
}
<commit_msg>updated interfaces<commit_after>package go_config

type Config interface {
	Fields

	UseSource(sources ...Source)
	SetDefault(key string, val interface{})
}

type Fields interface {
	Bool(key string) bool
	Get(key string) interface{}
	Float(key string) float64
	Int(key string) int
	IsSet(key string) bool
	Slice(key, delimiter string) []interface{}
	String(key string) string
	StringMap(key string) map[string]interface{}
	Sub(key string) Fields
	UInt(key string) uint
	Unmarshal(v interface{}, prefix string) error
}

type Source interface {
	Get(key string) interface{}

	Bool(key string) (bool, error)
	Float(key string) (float64, error)
	Int(key string) (int, error)
	UInt(key string) (uint, error)
	Slice(key, delimiter string) ([]interface{}, error)
	String(key string) (string, error)
	StringMap(key string) map[string]interface{}

	IsSet(key string) bool
}
<|endoftext|>


<commit_before>package eventhub

type DataBackend interface {
	Save(e *Event) error
	GetById(id int) (*Event, error)
	FilterBy(m map[string]interface{}) ([]*Event, error)
}

type EventFeed interface {
	Updates() <-chan Event
	Close() error
}

type Broadcaster interface {
	Register(client int)
	Constrict(client int, parameter, value string) //only broadcast certain events
	Listen() error
	Stop() error
}
<commit_msg>data backend now needs to implement eventfeed<commit_after>package eventhub

type EventFeed interface {
	Updates() <-chan Event
	Close() error
}

//Queryable Data store
type DataBackend interface {
	EventFeed
	Save(e *Event) error
	GetById(id int) (*Event, error)
	FilterBy(m map[string]interface{}) ([]*Event, error)
}

type Broadcaster interface {
	Register(client int)
	Constrict(client int, parameter, value string) //only broadcast certain events
	Listen() error
	Stop() error
}
<|endoftext|>


<commit_before>/*
	Copyright 2015-2016 the original author or authors.

	Licensed under the Apache License, Version 2.0 (the "License");
	you may not use this file except in compliance with the License.
	You may obtain a copy of the License at

		http://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing, software
	distributed under the License is distributed on an "AS IS" BASIS,
	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	See the License for the specific language governing permissions and
	limitations under the License.
 */
package io

import (
	"bufio"
	"encoding/xml"
	"fmt"
	"github.com/nano-projects/nanogo/models"
	"gopkg.in/yaml.v2"
	"log"
	"os"
	"strings"
)

func GeneralDefaultWebapp(_arg *models.Argument) {
	schema := models.Schema{}
	yml := GeneralWebapp(_arg)
	if err := yaml.Unmarshal([]byte(yml), &schema); err != nil {
		log.Fatalf("error: %v", err)
		return
	}

	generalDefault(_arg, &schema, &yml)
}

func GeneralDefaultScheduler(_arg *models.Argument) {
	schema := models.Schema{}
	yml := GeneralScheduler(_arg)
	if err := yaml.Unmarshal([]byte(yml), &schema); err != nil {
		log.Fatalf("error: %v", err)
		return
	}

	generalDefault(_arg, &schema, &yml)
}

func generalDefault(arg *models.Argument, schema *models.Schema, yml *string) {
	for module, project := range (*schema).Projects {
		project.Xmlns = XMLNS
		project.XmlnsXsi = XMLNS_XSI
		project.XsiSchemaLocation = XSI_SCHEMA_LOCATION

		if value, err := xml.MarshalIndent(project, "", "    "); err != nil {
			log.Fatalf("error: %v", err)
			return
		} else {
			if project.ArtifactId == *(*arg).ArtifactId {
				mkdirBase(arg, (*arg).ArtifactId)
				absolutePath := *(*arg).Path + *(*arg).ArtifactId
				writeFile(absolutePath+"/src/yml/nanogo.yml", PROPERTIES_LICENSE+*yml)
				writeFile(absolutePath+"/pom.xml", xml.Header+XML_LICENSE+string(value))
			} else {
				var moduleType string
				if project.Packaging == "war" || project.Packaging == "ear" || project.ModuleType == "web" {
					moduleType = "web"
				}

				absolutePath := *(*arg).Path + *(*arg).ArtifactId + "/" + module
				mkdir(&absolutePath, arg, (*arg).GroupId, (*arg).ArtifactId, &moduleType)
				writeFile(*(*arg).Path+*(*arg).ArtifactId+"/"+module+"/pom.xml", xml.Header+XML_LICENSE+string(value))
			}
		}
	}
}

func mkdirBase(arg *models.Argument, artifactId *string) {
	absolutePath := *(*arg).Path + *artifactId

	os.MkdirAll(absolutePath, 0755)
	os.MkdirAll(absolutePath+"/src/eclipse", 0755)
	os.MkdirAll(absolutePath+"/src/mvn", 0755)
	os.MkdirAll(absolutePath+"/src/yml", 0755)

	writeFile(absolutePath+"/src/eclipse/eclipse-code-template.xml", GeneralCodeTemplates())
	writeFile(absolutePath+"/src/eclipse/eclipse-formatter.xml", GeneralEclipseCheckstyle())
	writeFile(absolutePath+"/src/mvn/settings.xml", GeneralMavenSettings())

	writeFile(absolutePath+"/.gitignore", GeneralGitIgnore())
	if !*(*arg).Findbugs {
		writeFile(absolutePath+"/findbugs-rules.xml", GeneralFindBugs())
	}

	if !*(*arg).Checkstyle {
		writeFile(absolutePath+"/checkstyle-rules.xml", GeneralCheckstyleRules())
		writeFile(absolutePath+"/checkstyle-suppressions.xml", GeneralCheckstyleSuppressions())
	}

	if !*(*arg).License {
		os.MkdirAll(absolutePath+"/src/licensing", 0755)
		writeFile(absolutePath+"/src/licensing/header-definitions.xml", GeneralLicenseHeaderDefinitions())
		writeFile(absolutePath+"/src/licensing/header.txt", GeneralLicenseHeader())
	}
}

func mkdir(absolutePath *string, arg *models.Argument, groupId *string, artifactId *string, moduleType *string) {
	pack := "/" + strings.Replace(strings.Replace(*groupId, ".", "/", -1), "-", "/", -1) + "/" + strings.Replace(*artifactId, "-", "/", -1)
	os.MkdirAll(*absolutePath, 0755)
	os.MkdirAll(*absolutePath+"/src/main/java"+pack, 0755)
	os.MkdirAll(*absolutePath+"/src/main/resources", 0755)
	os.MkdirAll(*absolutePath+"/src/test/java"+pack, 0755)
	os.MkdirAll(*absolutePath+"/src/test/resources", 0755)

	writeFile(*absolutePath+"/src/main/java"+pack+"/.gitkeep", "")
	writeFile(*absolutePath+"/src/main/resources/.gitkeep", "")
	writeFile(*absolutePath+"/src/test/java"+pack+"/.gitkeep", "")
	writeFile(*absolutePath+"/src/test/resources/.gitkeep", "")

	if *moduleType == "web" {
		os.MkdirAll(*absolutePath+"/src/main/webapp/WEB-INF", 0755)
		os.MkdirAll(*absolutePath+"/bin", 0755)
		os.MkdirAll(*absolutePath+"/configure/public", 0755)
		os.MkdirAll(*absolutePath+"/configure/sit", 0755)
		os.MkdirAll(*absolutePath+"/configure/uat", 0755)
		os.MkdirAll(*absolutePath+"/configure/release", 0755)

		writeFile(*absolutePath+"/configure/public/.gitkeep", "")
		writeFile(*absolutePath+"/configure/sit/.gitkeep", "")
		writeFile(*absolutePath+"/configure/uat/.gitkeep", "")
		writeFile(*absolutePath+"/configure/release/.gitkeep", "")

		newGroupId := strings.Replace(*groupId, "-", ".", -1)
		newArtifactId := strings.Replace(*artifactId, "-", ".", -1)

		writeFile(*absolutePath+"/bin/bootstrap.sh", GeneralBootstrapShell(newGroupId, newArtifactId))
		writeFile(*absolutePath+"/src/main/java"+pack+"/Bootstrap.java", GeneralBootstrap(newGroupId, newArtifactId))
		writeFile(*absolutePath+"/src/main/resources/assembly.xml", GeneralAssembly())

		if *(*arg).NewWebapp {
			writeFile(*absolutePath+"/src/main/resources/context.properties", GeneralWebappContext(newGroupId, newArtifactId))
		}

		if *(*arg).NewScheduler {
			writeFile(*absolutePath+"/src/main/resources/context.properties", GeneralSchedulerContext(newGroupId, newArtifactId))
		}

		writeFile(*absolutePath+"/src/main/webapp/WEB-INF/jetty.xml", GeneralJettyXml(*(*arg).Port))
		writeFile(*absolutePath+"/src/main/webapp/WEB-INF/web.xml", GeneralWebXml(*artifactId))
		writeFile(*absolutePath+"/src/main/webapp/WEB-INF/webdefault.xml", GeneralWebDefaultXml())
		writeFile(*absolutePath+"/src/main/webapp/index.jsp", GeneralIndexJsp())
	}
}

func writeFile(fileName, data string) {
	fmt.Println("create file: ", fileName)
	outputFile, outputError := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE, 0755)
	if outputError != nil {
		fmt.Printf("An error occurred with file opening or creation\n")
		return
	}
	defer outputFile.Close()

	outputWriter := bufio.NewWriter(outputFile)
	outputWriter.WriteString(data)
	outputWriter.Flush()
}
<commit_msg>update README.md<commit_after>/*
	Copyright 2015-2016 the original author or authors.

	Licensed under the Apache License, Version 2.0 (the "License");
	you may not use this file except in compliance with the License.
	You may obtain a copy of the License at

		http://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing, software
	distributed under the License is distributed on an "AS IS" BASIS,
	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	See the License for the specific language governing permissions and
	limitations under the License.
 */
package io

import (
	"bufio"
	"encoding/xml"
	"fmt"
	"github.com/nano-projects/nanogo/models"
	"gopkg.in/yaml.v2"
	"log"
	"os"
	"strings"
)

const (
	FILE_MODE = 0755
)

func GeneralDefaultWebapp(_arg *models.Argument) {
	schema := models.Schema{}
	yml := GeneralWebapp(_arg)
	if err := yaml.Unmarshal([]byte(yml), &schema); err != nil {
		log.Fatalf("error: %v", err)
		return
	}

	generalDefault(_arg, &schema, &yml)
}

func GeneralDefaultScheduler(_arg *models.Argument) {
	schema := models.Schema{}
	yml := GeneralScheduler(_arg)
	if err := yaml.Unmarshal([]byte(yml), &schema); err != nil {
		log.Fatalf("error: %v", err)
		return
	}

	generalDefault(_arg, &schema, &yml)
}

func generalDefault(arg *models.Argument, schema *models.Schema, yml *string) {
	for module, project := range (*schema).Projects {
		project.Xmlns = XMLNS
		project.XmlnsXsi = XMLNS_XSI
		project.XsiSchemaLocation = XSI_SCHEMA_LOCATION

		if value, err := xml.MarshalIndent(project, "", "    "); err != nil {
			log.Fatalf("error: %v", err)
			return
		} else {
			if project.ArtifactId == *(*arg).ArtifactId {
				mkdirBase(arg, (*arg).ArtifactId)
				absolutePath := *(*arg).Path + *(*arg).ArtifactId
				writeFile(absolutePath+"/src/yml/nanogo.yml", PROPERTIES_LICENSE+*yml)
				writeFile(absolutePath+"/pom.xml", xml.Header+XML_LICENSE+string(value))
			} else {
				var moduleType string
				if project.Packaging == "war" || project.Packaging == "ear" || project.ModuleType == "web" {
					moduleType = "web"
				}

				absolutePath := *(*arg).Path + *(*arg).ArtifactId + "/" + module
				mkdir(&absolutePath, arg, (*arg).GroupId, (*arg).ArtifactId, &moduleType)
				writeFile(*(*arg).Path+*(*arg).ArtifactId+"/"+module+"/pom.xml", xml.Header+XML_LICENSE+string(value))
			}
		}
	}
}

func mkdirBase(arg *models.Argument, artifactId *string) {
	absolutePath := *(*arg).Path + *artifactId

	os.MkdirAll(absolutePath, FILE_MODE)
	os.MkdirAll(absolutePath+"/src/eclipse", FILE_MODE)
	os.MkdirAll(absolutePath+"/src/mvn", FILE_MODE)
	os.MkdirAll(absolutePath+"/src/yml", FILE_MODE)

	writeFile(absolutePath+"/src/eclipse/eclipse-code-template.xml", GeneralCodeTemplates())
	writeFile(absolutePath+"/src/eclipse/eclipse-formatter.xml", GeneralEclipseCheckstyle())
	writeFile(absolutePath+"/src/mvn/settings.xml", GeneralMavenSettings())

	writeFile(absolutePath+"/.gitignore", GeneralGitIgnore())
	if !*(*arg).Findbugs {
		writeFile(absolutePath+"/findbugs-rules.xml", GeneralFindBugs())
	}

	if !*(*arg).Checkstyle {
		writeFile(absolutePath+"/checkstyle-rules.xml", GeneralCheckstyleRules())
		writeFile(absolutePath+"/checkstyle-suppressions.xml", GeneralCheckstyleSuppressions())
	}

	if !*(*arg).License {
		os.MkdirAll(absolutePath+"/src/licensing", FILE_MODE)
		writeFile(absolutePath+"/src/licensing/header-definitions.xml", GeneralLicenseHeaderDefinitions())
		writeFile(absolutePath+"/src/licensing/header.txt", GeneralLicenseHeader())
	}
}

func mkdir(absolutePath *string, arg *models.Argument, groupId *string, artifactId *string, moduleType *string) {
	pack := "/" + strings.Replace(strings.Replace(*groupId, ".", "/", -1), "-", "/", -1) + "/" + strings.Replace(*artifactId, "-", "/", -1)
	os.MkdirAll(*absolutePath, FILE_MODE)
	os.MkdirAll(*absolutePath+"/src/main/java"+pack, FILE_MODE)
	os.MkdirAll(*absolutePath+"/src/main/resources", FILE_MODE)
	os.MkdirAll(*absolutePath+"/src/test/java"+pack, FILE_MODE)
	os.MkdirAll(*absolutePath+"/src/test/resources", FILE_MODE)

	writeFile(*absolutePath+"/src/main/java"+pack+"/.gitkeep", "")
	writeFile(*absolutePath+"/src/main/resources/.gitkeep", "")
	writeFile(*absolutePath+"/src/test/java"+pack+"/.gitkeep", "")
	writeFile(*absolutePath+"/src/test/resources/.gitkeep", "")

	if *moduleType == "web" {
		os.MkdirAll(*absolutePath+"/src/main/webapp/WEB-INF", FILE_MODE)
		os.MkdirAll(*absolutePath+"/bin", FILE_MODE)
		os.MkdirAll(*absolutePath+"/configure/public", FILE_MODE)
		os.MkdirAll(*absolutePath+"/configure/sit", FILE_MODE)
		os.MkdirAll(*absolutePath+"/configure/uat", FILE_MODE)
		os.MkdirAll(*absolutePath+"/configure/release", FILE_MODE)

		writeFile(*absolutePath+"/configure/public/.gitkeep", "")
		writeFile(*absolutePath+"/configure/sit/.gitkeep", "")
		writeFile(*absolutePath+"/configure/uat/.gitkeep", "")
		writeFile(*absolutePath+"/configure/release/.gitkeep", "")

		newGroupId := strings.Replace(*groupId, "-", ".", -1)
		newArtifactId := strings.Replace(*artifactId, "-", ".", -1)

		writeFile(*absolutePath+"/bin/bootstrap.sh", GeneralBootstrapShell(newGroupId, newArtifactId))
		writeFile(*absolutePath+"/src/main/java"+pack+"/Bootstrap.java", GeneralBootstrap(newGroupId, newArtifactId))
		writeFile(*absolutePath+"/src/main/resources/assembly.xml", GeneralAssembly())

		if *(*arg).NewWebapp {
			writeFile(*absolutePath+"/src/main/resources/context.properties", GeneralWebappContext(newGroupId, newArtifactId))
		}

		if *(*arg).NewScheduler {
			writeFile(*absolutePath+"/src/main/resources/context.properties", GeneralSchedulerContext(newGroupId, newArtifactId))
		}

		writeFile(*absolutePath+"/src/main/webapp/WEB-INF/jetty.xml", GeneralJettyXml(*(*arg).Port))
		writeFile(*absolutePath+"/src/main/webapp/WEB-INF/web.xml", GeneralWebXml(*artifactId))
		writeFile(*absolutePath+"/src/main/webapp/WEB-INF/webdefault.xml", GeneralWebDefaultXml())
		writeFile(*absolutePath+"/src/main/webapp/index.jsp", GeneralIndexJsp())
	}
}

func writeFile(fileName, data string) {
	fmt.Println("create file: ", fileName)
	outputFile, outputError := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE, FILE_MODE)
	if outputError != nil {
		fmt.Printf("An error occurred with file opening or creation\n")
		return
	}
	defer outputFile.Close()

	outputWriter := bufio.NewWriter(outputFile)
	outputWriter.WriteString(data)
	outputWriter.Flush()
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/fsouza/go-dockerclient"
	"os"
	"path/filepath"
	"text/template"
)

func usage() {
	println("Usage: docker-log template.file")
}

func generateFile(templatePath string, containers []docker.APIContainers) {
	tmpl, err := template.ParseFiles(templatePath)
	if err != nil {
		panic(err)
	}

	err = tmpl.ExecuteTemplate(os.Stdout, filepath.Base(templatePath), containers)
}

func main() {

	if len(os.Args) != 2 {
		usage()
		os.Exit(1)
	}

	endpoint := "unix:///var/run/docker.sock"
	client, err := docker.NewClient(endpoint)

	if err != nil {
		panic(err)
	}

	containers, err := client.ListContainers(docker.ListContainersOptions{
		All: false,
	})
	if err != nil {
		panic(err)
	}

	generateFile(os.Args[1], containers)
}
<commit_msg>Watch docker container events<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/fsouza/go-dockerclient"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"text/template"
)

var watch bool

type Event struct {
	ContainerId string `json:"id"`
	Status      string `json:"status"`
	Image       string `json:"from"`
}

func usage() {
	println("Usage: docker-gen [-watch] <template> [<dest>]")
}

func generateFile(templatePath string, containers []docker.APIContainers) {
	tmpl, err := template.ParseFiles(templatePath)
	if err != nil {
		panic(err)
	}

	dest := os.Stdout
	if flag.NArg() == 2 {
		dest, err = os.Create(flag.Arg(1))
		if err != nil {
			fmt.Println("unable to create dest file %s: %s", flag.Arg(1), err)
			os.Exit(1)
		}
	}

	err = tmpl.ExecuteTemplate(dest, filepath.Base(templatePath), containers)
}

func newConn() (*httputil.ClientConn, error) {
	conn, err := net.Dial("unix", "/var/run/docker.sock")
	if err != nil {
		return nil, err
	}
	return httputil.NewClientConn(conn, nil), nil
}

func getEvents() chan *Event {
	eventChan := make(chan *Event, 100)
	go func() {
		defer close(eventChan)

		c, err := newConn()
		if err != nil {
			fmt.Printf("cannot connect to docker: %s", err)
			return
		}
		defer c.Close()

		req, err := http.NewRequest("GET", "/events", nil)
		if err != nil {
			fmt.Printf("bad request for events: %s", err)
			return
		}

		resp, err := c.Do(req)
		if err != nil {
			fmt.Printf("cannot connect to events endpoint: %s", err)
			return
		}
		defer resp.Body.Close()

		// handle signals to stop the socket
		sigChan := make(chan os.Signal, 1)
		signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
		go func() {
			for sig := range sigChan {
				fmt.Printf("received signal '%v', exiting\n", sig)

				c.Close()
				close(eventChan)
				os.Exit(0)
			}
		}()

		dec := json.NewDecoder(resp.Body)
		for {
			var event *Event
			if err := dec.Decode(&event); err != nil {
				if err == io.EOF {
					break
				}
				fmt.Printf("cannot decode json: %s", err)
				continue
			}
			eventChan <- event
		}
		fmt.Printf("closing event channel")
	}()
	return eventChan
}

func generateFromContainers(client *docker.Client) {
	containers, err := client.ListContainers(docker.ListContainersOptions{
		All: false,
	})
	if err != nil {
		fmt.Printf("error listing containers: %s", err)
		return
	}

	generateFile(flag.Arg(0), containers)
}

func main() {

	flag.BoolVar(&watch, "watch", false, "watch for container changes")
	flag.Parse()

	if flag.NArg() < 1 {
		usage()
		os.Exit(1)
	}

	endpoint := "unix:///var/run/docker.sock"
	client, err := docker.NewClient(endpoint)

	if err != nil {
		panic(err)
	}

	generateFromContainers(client)
	if !watch {
		return
	}

	eventChan := getEvents()
	for {
		event := <-eventChan
		if event.Status == "start" || event.Status == "stop" {
			generateFromContainers(client)
		}
	}

}
<|endoftext|>


<commit_before>package server

import (
	"github.com/golang/glog"
	"github.com/golang/protobuf/proto"
	"github.com/oikomi/FishChatServer2/common/ecode"
	"github.com/oikomi/FishChatServer2/conf_discovery/etcd"
	"github.com/oikomi/FishChatServer2/libnet"
	"github.com/oikomi/FishChatServer2/protocol/external"
	"github.com/oikomi/FishChatServer2/server/access/client"
	"github.com/oikomi/FishChatServer2/server/access/conf"
	"github.com/oikomi/FishChatServer2/server/access/rpc"
)

type Server struct {
	Server    *libnet.Server
	RPCServer *rpc.RPCServer
}

func New() (s *Server) {
	s = &Server{}
	go rpc.RPCServerInit()
	return
}

func (s *Server) sessionLoop(client *client.Client) {
	for {
		reqData, err := client.Session.Receive()
		if err != nil {
			glog.Error(err)
		}
		if reqData != nil {
			baseCMD := &external.Base{}
			if err = proto.Unmarshal(reqData, baseCMD); err != nil {
				if err = client.Session.Send(&external.Error{
					Cmd:     external.ErrServerCMD,
					ErrCode: ecode.ServerErr.Uint32(),
					ErrStr:  ecode.ServerErr.String(),
				}); err != nil {
					glog.Error(err)
				}
				continue
			}
			if err = client.Parse(baseCMD.Cmd, reqData); err != nil {
				glog.Error(err)
				continue
			}
		}
	}
}

func (s *Server) Loop(rpcClient *rpc.RPCClient) {
	for {
		session, err := s.Server.Accept()
		if err != nil {
			glog.Error(err)
		}
		glog.Info("a new client ", session.ID())
		c := client.New(session, rpcClient)
		go s.sessionLoop(c)
	}
}

func (s *Server) SDHeart() {
	glog.Info("SDHeart")
	work1 := etcd.NewWorker(conf.Conf.ConfDiscovery.Gateway.Name, conf.Conf.Server.Addr, conf.Conf.ConfDiscovery.Gateway.Root,
		conf.Conf.ConfDiscovery.Gateway.Addrs)
	go work1.HeartBeat()
	work2 := etcd.NewWorker(conf.Conf.ConfDiscovery.MsgJob.Name, conf.Conf.RPCServer.Addr, conf.Conf.ConfDiscovery.MsgJob.Root,
		conf.Conf.ConfDiscovery.MsgJob.Addrs)
	go work2.HeartBeat()
	glog.Info(conf.Conf.ConfDiscovery.Notify)
	work3 := etcd.NewWorker(conf.Conf.ConfDiscovery.Notify.Name, conf.Conf.RPCServer.Addr, conf.Conf.ConfDiscovery.Notify.Root,
		conf.Conf.ConfDiscovery.Notify.Addrs)
	go work3.HeartBeat()
	work4 := etcd.NewWorker(conf.Conf.ConfDiscovery.Logic.Name, conf.Conf.RPCServer.Addr, conf.Conf.ConfDiscovery.Logic.Root,
		conf.Conf.ConfDiscovery.Logic.Addrs)
	go work4.HeartBeat()
}
<commit_msg>Create server.go<commit_after>package server

import (
	"github.com/golang/glog"
	"github.com/golang/protobuf/proto"
	"github.com/liberalman/im/common/ecode"
	"github.com/liberalman/im/conf_discovery/etcd"
	"github.com/liberalman/im/libnet"
	"github.com/liberalman/im/protocol/external"
	"github.com/liberalman/im/server/access/client"
	"github.com/liberalman/im/server/access/conf"
	"github.com/liberalman/im/server/access/rpc"
)

type Server struct {
	Server    *libnet.Server
	RPCServer *rpc.RPCServer
}

func New() (s *Server) {
	s = &Server{}
	go rpc.RPCServerInit()
	return
}

func (s *Server) sessionLoop(client *client.Client) {
	for {
		reqData, err := client.Session.Receive()
		if err != nil {
			glog.Error(err)
		}
		if reqData != nil {
			baseCMD := &external.Base{}
			if err = proto.Unmarshal(reqData, baseCMD); err != nil {
				if err = client.Session.Send(&external.Error{
					Cmd:     external.ErrServerCMD,
					ErrCode: ecode.ServerErr.Uint32(),
					ErrStr:  ecode.ServerErr.String(),
				}); err != nil {
					glog.Error(err)
				}
				continue
			}
			if err = client.Parse(baseCMD.Cmd, reqData); err != nil {
				glog.Error(err)
				continue
			}
		}
	}
}

func (s *Server) Loop(rpcClient *rpc.RPCClient) {
	for {
		session, err := s.Server.Accept()
		if err != nil {
			glog.Error(err)
		}
		glog.Info("a new client ", session.ID())
		c := client.New(session, rpcClient)
		go s.sessionLoop(c)
	}
}

func (s *Server) SDHeart() {
	glog.Info("SDHeart")
	work1 := etcd.NewWorker(conf.Conf.ConfDiscovery.Gateway.Name, conf.Conf.Server.Addr, conf.Conf.ConfDiscovery.Gateway.Root,
		conf.Conf.ConfDiscovery.Gateway.Addrs)
	go work1.HeartBeat()
	work2 := etcd.NewWorker(conf.Conf.ConfDiscovery.MsgJob.Name, conf.Conf.RPCServer.Addr, conf.Conf.ConfDiscovery.MsgJob.Root,
		conf.Conf.ConfDiscovery.MsgJob.Addrs)
	go work2.HeartBeat()
	glog.Info(conf.Conf.ConfDiscovery.Notify)
	work3 := etcd.NewWorker(conf.Conf.ConfDiscovery.Notify.Name, conf.Conf.RPCServer.Addr, conf.Conf.ConfDiscovery.Notify.Root,
		conf.Conf.ConfDiscovery.Notify.Addrs)
	go work3.HeartBeat()
	work4 := etcd.NewWorker(conf.Conf.ConfDiscovery.Logic.Name, conf.Conf.RPCServer.Addr, conf.Conf.ConfDiscovery.Logic.Root,
		conf.Conf.ConfDiscovery.Logic.Addrs)
	go work4.HeartBeat()
}
<|endoftext|>


<commit_before>// +build darwin freebsd openbsd netbsd dragonfly
// +build !appengine

package isatty

import (
	"syscall"
	"unsafe"
)

const ioctlReadTermios = syscall.TIOCGETA

// IsTerminal return true if the file descriptor is terminal.
func IsTerminal(fd uintptr) bool {
	var termios syscall.Termios
	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0)
	return err == 0
}

// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2
// terminal. This is also always false on this environment.
func IsCygwinTerminal(fd uintptr) bool {
	return false
}
<commit_msg>Use golang.org/x/sys/unix for IsTerminal on *BSD<commit_after>// +build darwin freebsd openbsd netbsd dragonfly
// +build !appengine

package isatty

import "golang.org/x/sys/unix"

// IsTerminal return true if the file descriptor is terminal.
func IsTerminal(fd uintptr) bool {
	_, err := unix.IoctlGetTermios(int(fd), unix.TIOCGETA)
	return err == nil
}

// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2
// terminal. This is also always false on this environment.
func IsCygwinTerminal(fd uintptr) bool {
	return false
}
<|endoftext|>