<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/globalsign/mgo"
	"github.com/globalsign/mgo/bson"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/api/shutdown"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/log"
	"golang.org/x/time/rate"
)

var (
	bulkMaxWaitMongoTime = 1 * time.Second
	bulkMaxNumberMsgs    = 1000
	bulkQueueMaxSize     = 10000

	rateLimitWarningInterval = 5 * time.Second

	buckets = append([]float64{0.1, 0.5}, prometheus.ExponentialBuckets(1, 1.6, 15)...)

	logsInQueue = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_queue_current",
		Help: "The current number of log entries in dispatcher queue.",
	})

	logsInAppQueues = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "tsuru_logs_app_queues_current",
		Help: "The current number of log entries in app queues.",
	}, []string{"app"})

	logsQueueBlockedTotal = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_queue_blocked_seconds_total",
		Help: "The total time spent blocked trying to add log to queue.",
	})

	logsQueueSize = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_dispatcher_queue_size",
		Help: "The max number of log entries in a dispatcher queue.",
	})

	logsEnqueued = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_enqueued_total",
		Help: "The number of log entries enqueued for processing.",
	}, []string{"app"})

	logsWritten = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_write_total",
		Help: "The number of log entries written to mongo.",
	}, []string{"app"})

	logsDropped = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_dropped_total",
		Help: "The number of log entries dropped due to full buffers.",
	}, []string{"app"})

	logsDroppedRateLimit = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_dropped_rate_limit_total",
		Help: "The number of log entries dropped due to rate limit exceeded.",
	}, []string{"app"})

	logsMongoFullLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_full_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})

	logsMongoLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})
)

func init() {
	prometheus.MustRegister(logsInQueue)
	prometheus.MustRegister(logsInAppQueues)
	prometheus.MustRegister(logsQueueSize)
	prometheus.MustRegister(logsEnqueued)
	prometheus.MustRegister(logsWritten)
	prometheus.MustRegister(logsDropped)
	prometheus.MustRegister(logsDroppedRateLimit)
	prometheus.MustRegister(logsQueueBlockedTotal)
	prometheus.MustRegister(logsMongoFullLatency)
	prometheus.MustRegister(logsMongoLatency)
}

type LogListener struct {
	c       <-chan Applog
	logConn *db.LogStorage
	quit    chan struct{}
}

func isCappedPositionLost(err error) bool {
	if err == nil {
		return false
	}
	return strings.Contains(err.Error(), "CappedPositionLost")
}

func isSessionClosed(r interface{}) bool {
	return fmt.Sprintf("%v", r) == "Session already closed"
}

func NewLogListener(a *App, filterLog Applog) (*LogListener, error) {
	conn, err := db.LogConn()
	if err != nil {
		return nil, err
	}
	c := make(chan Applog, 10)
	quit := make(chan struct{})
	coll := conn.AppLogCollection(a.Name)
	var lastLog Applog
	err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	if err == mgo.ErrNotFound {
		// Tail cursors do not work correctly if the collection is empty (the
		// Next() call wouldn't block). So if the collection is empty we insert
		// the very first log line in it. This is quite rare in the real world
		// though so the impact of this extra log message is really small.
		err = a.Log("Logs initialization", "tsuru", "")
		if err != nil {
			return nil, err
		}
		err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	}
	if err != nil {
		return nil, err
	}
	lastId := lastLog.MongoID
	mkQuery := func() bson.M {
		m := bson.M{
			"_id": bson.M{"$gt": lastId},
		}
		if filterLog.Source != "" {
			m["source"] = filterLog.Source
		}
		if filterLog.Unit != "" {
			m["unit"] = filterLog.Unit
		}
		return m
	}
	query := coll.Find(mkQuery())
	tailTimeout := 10 * time.Second
	iter := query.Sort("$natural").Tail(tailTimeout)
	go func() {
		defer close(c)
		defer func() {
			if r := recover(); r != nil {
				if isSessionClosed(r) {
					return
				}
				panic(err)
			}
		}()
		for {
			var applog Applog
			for iter.Next(&applog) {
				lastId = applog.MongoID
				select {
				case c <- applog:
				case <-quit:
					iter.Close()
					return
				}
			}
			if iter.Timeout() {
				continue
			}
			if err := iter.Err(); err != nil {
				if !isCappedPositionLost(err) {
					log.Errorf("error tailing logs: %v", err)
					iter.Close()
					return
				}
			}
			iter.Close()
			query = coll.Find(mkQuery())
			iter = query.Sort("$natural").Tail(tailTimeout)
		}
	}()
	l := LogListener{c: c, logConn: conn, quit: quit}
	return &l, nil
}

func (l *LogListener) ListenChan() <-chan Applog {
	return l.c
}

func (l *LogListener) Close() {
	l.logConn.Close()
	if l.quit != nil {
		close(l.quit)
		l.quit = nil
	}
}

type LogDispatcher struct {
	mu             sync.RWMutex
	dispatchers    map[string]*appLogDispatcher
	msgCh          chan *msgWithTS
	shuttingDown   int32
	doneProcessing chan struct{}
}

type msgWithTS struct {
	msg        *Applog
	arriveTime time.Time
}

func NewlogDispatcher(chanSize int) *LogDispatcher {
	d := &LogDispatcher{
		dispatchers:    make(map[string]*appLogDispatcher),
		msgCh:          make(chan *msgWithTS, chanSize),
		doneProcessing: make(chan struct{}),
	}
	go d.runWriter()
	shutdown.Register(d)
	logsQueueSize.Set(float64(chanSize))
	return d
}

func (d *LogDispatcher) getMessageDispatcher(msg *Applog) *appLogDispatcher {
	appName := msg.AppName
	d.mu.RLock()
	appD, ok := d.dispatchers[appName]
	if !ok {
		d.mu.RUnlock()
		d.mu.Lock()
		appD, ok = d.dispatchers[appName]
		if !ok {
			appD = newAppLogDispatcher(appName)
			d.dispatchers[appName] = appD
		}
		d.mu.Unlock()
	} else {
		d.mu.RUnlock()
	}
	return appD
}

func (d *LogDispatcher) runWriter() {
	defer close(d.doneProcessing)
	for msgExtra := range d.msgCh {
		if msgExtra == nil {
			break
		}
		logsInQueue.Dec()
		appD := d.getMessageDispatcher(msgExtra.msg)
		appD.send(msgExtra)
	}
}

func (d *LogDispatcher) Send(msg *Applog) error {
	if atomic.LoadInt32(&d.shuttingDown) == 1 {
		return errors.New("log dispatcher is shutting down")
	}
	logsInQueue.Inc()
	logsEnqueued.WithLabelValues(msg.AppName).Inc()
	msgExtra := &msgWithTS{msg: msg, arriveTime: time.Now()}
	select {
	case d.msgCh <- msgExtra:
	default:
		t0 := time.Now()
		d.msgCh <- msgExtra
		logsQueueBlockedTotal.Add(time.Since(t0).Seconds())
	}
	return nil
}

func (a *LogDispatcher) String() string {
	return "log dispatcher"
}

func (d *LogDispatcher) Shutdown(ctx context.Context) error {
	atomic.StoreInt32(&d.shuttingDown, 1)
	d.msgCh <- nil
	<-d.doneProcessing
	logsInQueue.Set(0)
	for _, appD := range d.dispatchers {
		appD.stopWait()
	}
	return nil
}

type appLogDispatcher struct {
	appName string
	*bulkProcessor
}

func newAppLogDispatcher(appName string) *appLogDispatcher {
	d := &appLogDispatcher{
		bulkProcessor: initBulkProcessor(bulkMaxWaitMongoTime, bulkMaxNumberMsgs, appName),
		appName:       appName,
	}
	d.flushable = d
	go d.run()
	return d
}

func (d *appLogDispatcher) flush(msgs []interface{}, lastMessage *msgWithTS) bool {
	conn, err := db.LogConn()
	if err != nil {
		log.Errorf("[log flusher] unable to connect to mongodb: %s", err)
		return false
	}
	coll := conn.AppLogCollection(d.appName)
	err = coll.Insert(msgs...)
	coll.Close()
	if err != nil {
		log.Errorf("[log flusher] unable to insert logs: %s", err)
		return false
	}
	if lastMessage != nil {
		logsMongoLatency.Observe(time.Since(lastMessage.arriveTime).Seconds())
		logsMongoFullLatency.Observe(time.Since(lastMessage.msg.Date).Seconds())
	}
	logsWritten.WithLabelValues(d.appName).Add(float64(len(msgs)))
	return true
}

type bulkProcessor struct {
	appName     string
	maxWaitTime time.Duration
	bulkSize    int
	finished    chan struct{}
	ch          chan *msgWithTS
	nextNotify  *time.Timer
	flushable   interface {
		flush([]interface{}, *msgWithTS) bool
	}
}

func initBulkProcessor(maxWait time.Duration, bulkSize int, appName string) *bulkProcessor {
	queueSize, err := config.GetInt("log:queue-size")
	if err != nil || queueSize == 0 {
		queueSize = bulkQueueMaxSize
	}
	return &bulkProcessor{
		appName:     appName,
		maxWaitTime: maxWait,
		bulkSize:    bulkSize,
		finished:    make(chan struct{}),
		ch:          make(chan *msgWithTS, queueSize),
		nextNotify:  time.NewTimer(0),
	}
}

func (p *bulkProcessor) send(msg *msgWithTS) {
	select {
	case p.ch <- msg:
		logsInAppQueues.WithLabelValues(p.appName).Set(float64(len(p.ch)))
	default:
		logsDropped.WithLabelValues(p.appName).Inc()
		select {
		case <-p.nextNotify.C:
			log.Errorf("dropping log messages to mongodb due to full channel buffer. app: %q, len: %d", msg.msg.AppName, len(p.ch))
			p.nextNotify.Reset(time.Minute)
		default:
		}
	}
}

func (p *bulkProcessor) stopWait() {
	p.ch <- nil
	<-p.finished
}

func (p *bulkProcessor) rateLimitWarning(rateLimit int) *Applog {
	return &Applog{
		AppName: p.appName,
		Date:    time.Now(),
		Message: fmt.Sprintf("Log messages dropped due to exceeded rate limit. Limit: %v logs/s.", rateLimit),
		Source:  "tsuru",
		Unit:    "api",
	}
}

func updateLogRateLimiter(rateLimiter *rate.Limiter) *rate.Limiter {
	rateLimit, _ := config.GetInt("log:app-log-rate-limit")
	if rateLimit <= 0 {
		return nil
	}
	if rateLimiter == nil || rateLimiter.Burst() != rateLimit {
		return rate.NewLimiter(rate.Limit(rateLimit), rateLimit)
	}
	return rateLimiter
}

func (p *bulkProcessor) run() {
	defer close(p.finished)
	t := time.NewTimer(p.maxWaitTime)
	pos := 0
	bulkBuffer := make([]interface{}, p.bulkSize)
	shouldReturn := false
	var lastMessage *msgWithTS
	var lastRateNotice time.Time
	logsInAppQueue := logsInAppQueues.WithLabelValues(p.appName)
	logsDropped := logsDroppedRateLimit.WithLabelValues(p.appName)
	rateLimiter := updateLogRateLimiter(nil)
	for {
		var flush bool
		select {
		case msgExtra := <-p.ch:
			logsInAppQueue.Set(float64(len(p.ch)))
			if msgExtra == nil {
				flush = true
				shouldReturn = true
				break
			}
			if pos == p.bulkSize {
				flush = true
				break
			}

			if rateLimiter != nil && !rateLimiter.Allow() {
				logsDropped.Inc()
				if time.Since(lastRateNotice) > rateLimitWarningInterval {
					lastRateNotice = time.Now()
					bulkBuffer[pos] = p.rateLimitWarning(rateLimiter.Burst())
					pos++
				}
				flush = p.bulkSize == pos
				break
			}

			lastMessage = msgExtra
			bulkBuffer[pos] = msgExtra.msg
			pos++
			flush = p.bulkSize == pos
		case <-t.C:
			flush = true
			t.Reset(p.maxWaitTime)
		}
		if flush && pos > 0 {
			if p.flushable.flush(bulkBuffer[:pos], lastMessage) {
				lastMessage = nil
				pos = 0
			}
			rateLimiter = updateLogRateLimiter(rateLimiter)
		}
		if shouldReturn {
			return
		}
	}
}
<commit_msg>app: fix leaked log db connections on log listener errors<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/globalsign/mgo"
	"github.com/globalsign/mgo/bson"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/api/shutdown"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/log"
	"golang.org/x/time/rate"
)

var (
	bulkMaxWaitMongoTime = 1 * time.Second
	bulkMaxNumberMsgs    = 1000
	bulkQueueMaxSize     = 10000

	rateLimitWarningInterval = 5 * time.Second

	buckets = append([]float64{0.1, 0.5}, prometheus.ExponentialBuckets(1, 1.6, 15)...)

	logsInQueue = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_queue_current",
		Help: "The current number of log entries in dispatcher queue.",
	})

	logsInAppQueues = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "tsuru_logs_app_queues_current",
		Help: "The current number of log entries in app queues.",
	}, []string{"app"})

	logsQueueBlockedTotal = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_queue_blocked_seconds_total",
		Help: "The total time spent blocked trying to add log to queue.",
	})

	logsQueueSize = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_dispatcher_queue_size",
		Help: "The max number of log entries in a dispatcher queue.",
	})

	logsEnqueued = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_enqueued_total",
		Help: "The number of log entries enqueued for processing.",
	}, []string{"app"})

	logsWritten = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_write_total",
		Help: "The number of log entries written to mongo.",
	}, []string{"app"})

	logsDropped = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_dropped_total",
		Help: "The number of log entries dropped due to full buffers.",
	}, []string{"app"})

	logsDroppedRateLimit = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "tsuru_logs_dropped_rate_limit_total",
		Help: "The number of log entries dropped due to rate limit exceeded.",
	}, []string{"app"})

	logsMongoFullLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_full_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})

	logsMongoLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})
)

func init() {
	prometheus.MustRegister(logsInQueue)
	prometheus.MustRegister(logsInAppQueues)
	prometheus.MustRegister(logsQueueSize)
	prometheus.MustRegister(logsEnqueued)
	prometheus.MustRegister(logsWritten)
	prometheus.MustRegister(logsDropped)
	prometheus.MustRegister(logsDroppedRateLimit)
	prometheus.MustRegister(logsQueueBlockedTotal)
	prometheus.MustRegister(logsMongoFullLatency)
	prometheus.MustRegister(logsMongoLatency)
}

type LogListener struct {
	c       <-chan Applog
	logConn *db.LogStorage
	quit    chan struct{}
}

func isCappedPositionLost(err error) bool {
	if err == nil {
		return false
	}
	return strings.Contains(err.Error(), "CappedPositionLost")
}

func isSessionClosed(r interface{}) bool {
	return fmt.Sprintf("%v", r) == "Session already closed"
}

func NewLogListener(a *App, filterLog Applog) (*LogListener, error) {
	conn, err := db.LogConn()
	if err != nil {
		return nil, err
	}
	c := make(chan Applog, 10)
	quit := make(chan struct{})
	coll := conn.AppLogCollection(a.Name)
	var lastLog Applog
	err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	if err == mgo.ErrNotFound {
		// Tail cursors do not work correctly if the collection is empty (the
		// Next() call wouldn't block). So if the collection is empty we insert
		// the very first log line in it. This is quite rare in the real world
		// though so the impact of this extra log message is really small.
		err = a.Log("Logs initialization", "tsuru", "")
		if err != nil {
			conn.Close()
			return nil, err
		}
		err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	}
	if err != nil {
		conn.Close()
		return nil, err
	}
	lastId := lastLog.MongoID
	mkQuery := func() bson.M {
		m := bson.M{
			"_id": bson.M{"$gt": lastId},
		}
		if filterLog.Source != "" {
			m["source"] = filterLog.Source
		}
		if filterLog.Unit != "" {
			m["unit"] = filterLog.Unit
		}
		return m
	}
	query := coll.Find(mkQuery())
	tailTimeout := 10 * time.Second
	iter := query.Sort("$natural").Tail(tailTimeout)
	go func() {
		defer close(c)
		defer func() {
			if r := recover(); r != nil {
				if isSessionClosed(r) {
					return
				}
				panic(err)
			}
		}()
		for {
			var applog Applog
			for iter.Next(&applog) {
				lastId = applog.MongoID
				select {
				case c <- applog:
				case <-quit:
					iter.Close()
					return
				}
			}
			if iter.Timeout() {
				continue
			}
			if err := iter.Err(); err != nil {
				if !isCappedPositionLost(err) {
					log.Errorf("error tailing logs: %v", err)
					iter.Close()
					return
				}
			}
			iter.Close()
			query = coll.Find(mkQuery())
			iter = query.Sort("$natural").Tail(tailTimeout)
		}
	}()
	l := LogListener{c: c, logConn: conn, quit: quit}
	return &l, nil
}

func (l *LogListener) ListenChan() <-chan Applog {
	return l.c
}

func (l *LogListener) Close() {
	l.logConn.Close()
	if l.quit != nil {
		close(l.quit)
		l.quit = nil
	}
}

type LogDispatcher struct {
	mu             sync.RWMutex
	dispatchers    map[string]*appLogDispatcher
	msgCh          chan *msgWithTS
	shuttingDown   int32
	doneProcessing chan struct{}
}

type msgWithTS struct {
	msg        *Applog
	arriveTime time.Time
}

func NewlogDispatcher(chanSize int) *LogDispatcher {
	d := &LogDispatcher{
		dispatchers:    make(map[string]*appLogDispatcher),
		msgCh:          make(chan *msgWithTS, chanSize),
		doneProcessing: make(chan struct{}),
	}
	go d.runWriter()
	shutdown.Register(d)
	logsQueueSize.Set(float64(chanSize))
	return d
}

func (d *LogDispatcher) getMessageDispatcher(msg *Applog) *appLogDispatcher {
	appName := msg.AppName
	d.mu.RLock()
	appD, ok := d.dispatchers[appName]
	if !ok {
		d.mu.RUnlock()
		d.mu.Lock()
		appD, ok = d.dispatchers[appName]
		if !ok {
			appD = newAppLogDispatcher(appName)
			d.dispatchers[appName] = appD
		}
		d.mu.Unlock()
	} else {
		d.mu.RUnlock()
	}
	return appD
}

func (d *LogDispatcher) runWriter() {
	defer close(d.doneProcessing)
	for msgExtra := range d.msgCh {
		if msgExtra == nil {
			break
		}
		logsInQueue.Dec()
		appD := d.getMessageDispatcher(msgExtra.msg)
		appD.send(msgExtra)
	}
}

func (d *LogDispatcher) Send(msg *Applog) error {
	if atomic.LoadInt32(&d.shuttingDown) == 1 {
		return errors.New("log dispatcher is shutting down")
	}
	logsInQueue.Inc()
	logsEnqueued.WithLabelValues(msg.AppName).Inc()
	msgExtra := &msgWithTS{msg: msg, arriveTime: time.Now()}
	select {
	case d.msgCh <- msgExtra:
	default:
		t0 := time.Now()
		d.msgCh <- msgExtra
		logsQueueBlockedTotal.Add(time.Since(t0).Seconds())
	}
	return nil
}

func (a *LogDispatcher) String() string {
	return "log dispatcher"
}

func (d *LogDispatcher) Shutdown(ctx context.Context) error {
	atomic.StoreInt32(&d.shuttingDown, 1)
	d.msgCh <- nil
	<-d.doneProcessing
	logsInQueue.Set(0)
	for _, appD := range d.dispatchers {
		appD.stopWait()
	}
	return nil
}

type appLogDispatcher struct {
	appName string
	*bulkProcessor
}

func newAppLogDispatcher(appName string) *appLogDispatcher {
	d := &appLogDispatcher{
		bulkProcessor: initBulkProcessor(bulkMaxWaitMongoTime, bulkMaxNumberMsgs, appName),
		appName:       appName,
	}
	d.flushable = d
	go d.run()
	return d
}

func (d *appLogDispatcher) flush(msgs []interface{}, lastMessage *msgWithTS) bool {
	conn, err := db.LogConn()
	if err != nil {
		log.Errorf("[log flusher] unable to connect to mongodb: %s", err)
		return false
	}
	coll := conn.AppLogCollection(d.appName)
	err = coll.Insert(msgs...)
	coll.Close()
	if err != nil {
		log.Errorf("[log flusher] unable to insert logs: %s", err)
		return false
	}
	if lastMessage != nil {
		logsMongoLatency.Observe(time.Since(lastMessage.arriveTime).Seconds())
		logsMongoFullLatency.Observe(time.Since(lastMessage.msg.Date).Seconds())
	}
	logsWritten.WithLabelValues(d.appName).Add(float64(len(msgs)))
	return true
}

type bulkProcessor struct {
	appName     string
	maxWaitTime time.Duration
	bulkSize    int
	finished    chan struct{}
	ch          chan *msgWithTS
	nextNotify  *time.Timer
	flushable   interface {
		flush([]interface{}, *msgWithTS) bool
	}
}

func initBulkProcessor(maxWait time.Duration, bulkSize int, appName string) *bulkProcessor {
	queueSize, err := config.GetInt("log:queue-size")
	if err != nil || queueSize == 0 {
		queueSize = bulkQueueMaxSize
	}
	return &bulkProcessor{
		appName:     appName,
		maxWaitTime: maxWait,
		bulkSize:    bulkSize,
		finished:    make(chan struct{}),
		ch:          make(chan *msgWithTS, queueSize),
		nextNotify:  time.NewTimer(0),
	}
}

func (p *bulkProcessor) send(msg *msgWithTS) {
	select {
	case p.ch <- msg:
		logsInAppQueues.WithLabelValues(p.appName).Set(float64(len(p.ch)))
	default:
		logsDropped.WithLabelValues(p.appName).Inc()
		select {
		case <-p.nextNotify.C:
			log.Errorf("dropping log messages to mongodb due to full channel buffer. app: %q, len: %d", msg.msg.AppName, len(p.ch))
			p.nextNotify.Reset(time.Minute)
		default:
		}
	}
}

func (p *bulkProcessor) stopWait() {
	p.ch <- nil
	<-p.finished
}

func (p *bulkProcessor) rateLimitWarning(rateLimit int) *Applog {
	return &Applog{
		AppName: p.appName,
		Date:    time.Now(),
		Message: fmt.Sprintf("Log messages dropped due to exceeded rate limit. Limit: %v logs/s.", rateLimit),
		Source:  "tsuru",
		Unit:    "api",
	}
}

func updateLogRateLimiter(rateLimiter *rate.Limiter) *rate.Limiter {
	rateLimit, _ := config.GetInt("log:app-log-rate-limit")
	if rateLimit <= 0 {
		return nil
	}
	if rateLimiter == nil || rateLimiter.Burst() != rateLimit {
		return rate.NewLimiter(rate.Limit(rateLimit), rateLimit)
	}
	return rateLimiter
}

func (p *bulkProcessor) run() {
	defer close(p.finished)
	t := time.NewTimer(p.maxWaitTime)
	pos := 0
	bulkBuffer := make([]interface{}, p.bulkSize)
	shouldReturn := false
	var lastMessage *msgWithTS
	var lastRateNotice time.Time
	logsInAppQueue := logsInAppQueues.WithLabelValues(p.appName)
	logsDropped := logsDroppedRateLimit.WithLabelValues(p.appName)
	rateLimiter := updateLogRateLimiter(nil)
	for {
		var flush bool
		select {
		case msgExtra := <-p.ch:
			logsInAppQueue.Set(float64(len(p.ch)))
			if msgExtra == nil {
				flush = true
				shouldReturn = true
				break
			}
			if pos == p.bulkSize {
				flush = true
				break
			}

			if rateLimiter != nil && !rateLimiter.Allow() {
				logsDropped.Inc()
				if time.Since(lastRateNotice) > rateLimitWarningInterval {
					lastRateNotice = time.Now()
					bulkBuffer[pos] = p.rateLimitWarning(rateLimiter.Burst())
					pos++
				}
				flush = p.bulkSize == pos
				break
			}

			lastMessage = msgExtra
			bulkBuffer[pos] = msgExtra.msg
			pos++
			flush = p.bulkSize == pos
		case <-t.C:
			flush = true
			t.Reset(p.maxWaitTime)
		}
		if flush && pos > 0 {
			if p.flushable.flush(bulkBuffer[:pos], lastMessage) {
				lastMessage = nil
				pos = 0
			}
			rateLimiter = updateLogRateLimiter(rateLimiter)
		}
		if shouldReturn {
			return
		}
	}
}
<|endoftext|>


<commit_before>package cryptoengine

import (
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
)

const (
	keyPath     = "keys"
	testKeyPath = "test_keys"
)

var (
	osSeparator, _             = strconv.Unquote(strconv.QuoteRuneToASCII(os.PathSeparator))
	keysFolderPrefixFormat     = fmt.Sprintf("%s%s", keyPath, osSeparator) + "%s"
	testKeysFolderPrefixFormat = fmt.Sprintf("%s%s", testKeyPath, osSeparator) + "%s"
)

// create the keys folder if it does not exist, with the proper permission
func init() {
	if err := createBaseKeyFolder(keyPath); err != nil {
		log.Println(err)
	}
}

// Check if a file exists
func fileExists(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil
}

// Check if a key file exists
func keyFileExists(filename string) bool {
	_, err := os.Stat(fmt.Sprintf(keysFolderPrefixFormat, filename))
	return err == nil
}

// Read the full file into a byte slice
func readFile(filename string) ([]byte, error) {
	return ioutil.ReadFile(filename)
}

// Writes a file with read only permissions
// If the file already exists then it returns the specific error: os.ErrExist
// This is thanks to the flag O_CREATE
func writeFile(filename string, data []byte) error {

	if fileExists(filename) {
		return os.ErrExist
	}

	file, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0400)
	if err != nil {
		log.Println(err)
		return err
	}

	_, err = file.Write(data)
	return err

}

// Read the key file into a 32 byte array
func readKey(filename, pathFormat string) ([keySize]byte, error) {
	var data32 [keySize]byte

	// read the data back
	data, err := readFile(fmt.Sprintf(pathFormat, filename))
	if err != nil {
		return data32, err
	}
	// decode from hex
	dst := make([]byte, len(data))
	_, err = hex.Decode(dst, data) //.StdEncoding.Decode(dst, data)
	if err != nil {
		return data32, err
	}
	// fill in the 32 byte array witht he data and return it
	copy(data32[:], dst[:keySize])
	return data32, err
}

// Write the key file hex encoded
func writeKey(filename, pathFormat string, data []byte) error {
	dst := make([]byte, hex.EncodedLen(len(data))) //StdEncoding.EncodedLen(len(data)))
	hex.Encode(dst, data)                          // StdEncoding.Encode(dst, data)
	filePath := fmt.Sprintf(pathFormat, filename)
	return writeFile(filePath, dst)
}

// Check if the file or directory exists and then deletes it
func deleteFile(filename string) error {
	if fileExists(filename) {
		return os.Remove(filename)
	}
	return nil
}

func createBaseKeyFolder(path string) error {
	if fileExists(path) {
		return nil
	}
	return os.MkdirAll(path, 0700)
}

func removeFolder(path string) error {
	return os.RemoveAll(path)
}
<commit_msg>Allow the ability to optionally set keypath with SEC51_KEYPATH env var<commit_after>package cryptoengine

import (
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
)

const (
	testKeyPath = "test_keys"
)

var (
	keyPath                    string
	keysFolderPrefixFormat     string
	testKeysFolderPrefixFormat string
)

// create the keys folder if it does not exist, with the proper permission
func init() {
	if os.Getenv("SEC51_KEYPATH") != "" {
		keyPath = os.Getenv("SEC51_KEYPATH")
	} else {
		keyPath = "keys"
	}

	keysFolderPrefixFormat = filepath.Join(keyPath, "%s")
	testKeysFolderPrefixFormat = filepath.Join(testKeyPath, "%s")
	if err := createBaseKeyFolder(keyPath); err != nil {
		log.Println(err)
	}
}

// Check if a file exists
func fileExists(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil
}

// Check if a key file exists
func keyFileExists(filename string) bool {
	_, err := os.Stat(fmt.Sprintf(keysFolderPrefixFormat, filename))
	return err == nil
}

// Read the full file into a byte slice
func readFile(filename string) ([]byte, error) {
	return ioutil.ReadFile(filename)
}

// Writes a file with read only permissions
// If the file already exists then it returns the specific error: os.ErrExist
// This is thanks to the flag O_CREATE
func writeFile(filename string, data []byte) error {

	if fileExists(filename) {
		return os.ErrExist
	}

	file, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0400)
	if err != nil {
		log.Println(err)
		return err
	}

	_, err = file.Write(data)
	return err

}

// Read the key file into a 32 byte array
func readKey(filename, pathFormat string) ([keySize]byte, error) {
	var data32 [keySize]byte

	// read the data back
	data, err := readFile(fmt.Sprintf(pathFormat, filename))
	if err != nil {
		return data32, err
	}
	// decode from hex
	dst := make([]byte, len(data))
	_, err = hex.Decode(dst, data) //.StdEncoding.Decode(dst, data)
	if err != nil {
		return data32, err
	}
	// fill in the 32 byte array witht he data and return it
	copy(data32[:], dst[:keySize])
	return data32, err
}

// Write the key file hex encoded
func writeKey(filename, pathFormat string, data []byte) error {
	dst := make([]byte, hex.EncodedLen(len(data))) //StdEncoding.EncodedLen(len(data)))
	hex.Encode(dst, data)                          // StdEncoding.Encode(dst, data)
	filePath := fmt.Sprintf(pathFormat, filename)
	return writeFile(filePath, dst)
}

// Check if the file or directory exists and then deletes it
func deleteFile(filename string) error {
	if fileExists(filename) {
		return os.Remove(filename)
	}
	return nil
}

func createBaseKeyFolder(path string) error {
	if fileExists(path) {
		return nil
	}
	return os.MkdirAll(path, 0700)
}

func removeFolder(path string) error {
	return os.RemoveAll(path)
}
<|endoftext|>


<commit_before>// Copyright 2012 The go-config Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Author Nelson

// config package privide:
// 		atomic storage object
//		file configuration save, load and watcher
//		xml and Json encoder, decoder interface
package config

import (
	"errors"
	"io/ioutil"
	"reflect"
	"sync"
	"time"

	"github.com/howeyc/fsnotify"
)

type NotifyAction int

const (
	NA_Continue = iota
	NA_Restart
	NA_Stop
)

type NotifyHandler func(Sender *FileConfig, event *fsnotify.FileEvent) NotifyAction

var (
	implError     = errors.New("imtemplment of coder is wrong, cant return nil when not error")
	coderNilError = errors.New("config Coder is nil")
	saveNilError  = errors.New("can not load or save nil configuration value")
)

// Coder interface define encode and decode function
// Encode encodes configuration object to byte slice
// Decode decodes byte slice to configuration object
type Coder interface {
	Encode(v interface{}) ([]byte, error)
	Decode(data []byte, v interface{}) error
}

// A FileConfig which is used for both reading and writing configration file.
// base is Config object which provide storage. Writing and reading simultaneously are safety.
// The coder is Coder interface which encode and decode configration object
type FileConfig struct {
	base          Config
	fileName      string
	coder         Coder
	lk            sync.RWMutex
	done          chan bool
	watcherStared bool
}

// NewFileConfig creates FileConfig object
func NewFileConfig(fileName string, coder Coder) *FileConfig {
	fc := new(FileConfig)
	fc.fileName = fileName
	fc.coder = coder
	fc.done = make(chan bool)
	return fc
}

// WatcherStared is a perperty which shows the file watcher is started or not
func (fc *FileConfig) WatcherStared() bool {
	fc.lk.RLock()
	defer fc.lk.RUnlock()
	return fc.watcherStared
}

// StopWatcher stop the file watcher
// the StopWatcher should be call when you no longer need fc object if WatcherStared
func (fc *FileConfig) StopWatcher() {
	fc.lk.Lock()
	defer fc.lk.Unlock()

	if fc.watcherStared == true {
		fc.done <- true
		select {
		case <-fc.done:
		case <-time.After(2 * time.Second):
		}
	}
	return
}

// StartWatcher start a file watcher to notify file changed
// StartWatcher call the handle once the file changed
func (fc *FileConfig) StartWatcher(handler NotifyHandler) error {
	fc.lk.Lock()
	defer fc.lk.Unlock()

	// access instence var directly
	if fc.watcherStared {
		return nil
	}

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}

	// access instance var directly
	err = watcher.Watch(fc.fileName)
	if err != nil {
		watcher.Close()
		return err
	}

	go func() {
		fc.watcherStared = true
		defer func() {
			fc.watcherStared = false
		}()
		defer watcher.Close()
		for {
			select {
			case ev := <-watcher.Event:
				if handler != nil {
					action := handler(fc, ev)
					switch action {
					case NA_Stop:
						return
					case NA_Restart:
						watcher.RemoveWatch(fc.FileName())
						watcher.Watch(fc.FileName())
					}
				}
			case <-watcher.Error:
				return
			case <-fc.done:
				fc.done <- true
				return
			}
		}
	}()

	return nil
}

// LoadFromFile load content from file and decodes it to object
// save the decoded object to internal storage
// v: decoded object receiveer, must be a pointer to object
// You should call fc function after file changed to refrash the data;
// otherwise you can call ConfValue to get object from internal storage
func (fc *FileConfig) LoadFromFile(v interface{}) error {
	fc.lk.RLock()
	defer fc.lk.RUnlock()

	if v == nil {
		return saveNilError
	}

	if fc.coder == nil {
		return coderNilError
	}

	data, err := ioutil.ReadFile(fc.fileName)
	if err != nil {
		return err
	}

	err = fc.coder.Decode(data, v)
	if err != nil {
		return err
	}
	if v == nil {
		return implError
	}

	fc.base.Set(reflect.ValueOf(v).Elem().Interface())
	return nil
}

// SaveToFile encode conf object to byte slice and save encoded object to file
// encoded object save to internal storage simultaneously.
// writes encoded data to a file named by FileName property.
// If the file does not exist, SaveToFile creates it; otherwise truncates it before writing.
func (fc *FileConfig) SaveToFile(v interface{}) error {
	fc.lk.Lock()
	defer fc.lk.Unlock()

	if v == nil {
		return saveNilError
	}

	data, err := fc.coder.Encode(v)
	if err != nil {
		return err
	}
	fc.base.Set(v)

	return ioutil.WriteFile(fc.fileName, data, 0666)
}

// FileName property getter
func (fc *FileConfig) FileName() string {
	fc.lk.RLock()
	defer fc.lk.RUnlock()
	return fc.fileName
}

// SetFileName set config filename
func (fc *FileConfig) SetFileName(name string) {
	fc.lk.Lock()
	defer fc.lk.Unlock()

	if fc.fileName != name {
		fc.fileName = name
	}
}

// Value get config value from internal storage
func (fc *FileConfig) Value() interface{} {
	return fc.base.Get()
}
<commit_msg>Change commnets<commit_after>// Copyright 2012 The go-config Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Author Nelson

// config package privide:
// 		atomic storage object
//		file configuration save, load and watcher
//		xml and Json encoder, decoder interface
package config

import (
	"errors"
	"io/ioutil"
	"reflect"
	"sync"
	"time"

	"github.com/howeyc/fsnotify"
)

type NotifyAction int

const (
	NA_Continue = iota
	NA_Restart
	NA_Stop
)

type NotifyHandler func(Sender *FileConfig, event *fsnotify.FileEvent) NotifyAction

var (
	implError     = errors.New("imtemplment of coder is wrong, cant return nil when not error")
	coderNilError = errors.New("config Coder is nil")
	saveNilError  = errors.New("can not load or save nil configuration value")
)

// Coder interface define encode and decode function
// Encode encodes configuration object to byte slice
// Decode decodes byte slice to configuration object
type Coder interface {
	Encode(v interface{}) ([]byte, error)
	Decode(data []byte, v interface{}) error
}

// A FileConfig which is used for both reading and writing configration file.
// base is Config object which provide storage. Writing and reading simultaneously are safety.
// The coder is Coder interface which encode and decode configration object
type FileConfig struct {
	base          Config
	fileName      string
	coder         Coder
	lk            sync.RWMutex
	done          chan bool
	watcherStared bool
}

// NewFileConfig creates FileConfig object
func NewFileConfig(fileName string, coder Coder) *FileConfig {
	fc := new(FileConfig)
	fc.fileName = fileName
	fc.coder = coder
	fc.done = make(chan bool)
	return fc
}

// WatcherStared is a perperty which shows the file watcher is started or not
func (fc *FileConfig) WatcherStared() bool {
	fc.lk.RLock()
	defer fc.lk.RUnlock()
	return fc.watcherStared
}

// StopWatcher stop the file watcher
// the StopWatcher should be call when you no longer need fc object if WatcherStared
func (fc *FileConfig) StopWatcher() {
	fc.lk.Lock()
	defer fc.lk.Unlock()

	if fc.watcherStared == true {
		fc.done <- true
		select {
		case <-fc.done:
		case <-time.After(2 * time.Second):
		}
	}
	return
}

// StartWatcher start a file watcher to notify file changed
// StartWatcher call the handle once the file changed
func (fc *FileConfig) StartWatcher(handler NotifyHandler) error {
	fc.lk.Lock()
	defer fc.lk.Unlock()

	// access instence var directly
	if fc.watcherStared {
		return nil
	}

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}

	// access instance var directly
	err = watcher.Watch(fc.fileName)
	if err != nil {
		watcher.Close()
		return err
	}

	go func() {
		fc.watcherStared = true
		defer func() {
			fc.watcherStared = false
		}()
		defer watcher.Close()
		for {
			select {
			case ev := <-watcher.Event:
				if handler != nil {
					action := handler(fc, ev)
					switch action {
					case NA_Stop:
						return
					case NA_Restart:
						watcher.RemoveWatch(fc.FileName())
						watcher.Watch(fc.FileName())
					}
				}
			case <-watcher.Error:
				return
			case <-fc.done:
				fc.done <- true
				return
			}
		}
	}()

	return nil
}

// LoadFromFile load content from file and decodes it to object
// save the decoded object to internal storage
// v: decoded object receiveer, must be a pointer to object
// You should call fc function after file changed to refrash the data;
// otherwise you can call ConfValue to get object from internal storage
func (fc *FileConfig) LoadFromFile(v interface{}) error {
	fc.lk.RLock()
	defer fc.lk.RUnlock()

	if v == nil {
		return saveNilError
	}

	if fc.coder == nil {
		return coderNilError
	}

	data, err := ioutil.ReadFile(fc.fileName)
	if err != nil {
		return err
	}

	err = fc.coder.Decode(data, v)
	if err != nil {
		return err
	}
	if v == nil {
		return implError
	}

	fc.base.Set(reflect.ValueOf(v).Elem().Interface())
	return nil
}

// SaveToFile encode v object to byte slice and save encoded object to file
// encoded object save to internal storage simultaneously.
// writes encoded data to a file named by FileName property.
// If the file does not exist, SaveToFile creates it; otherwise truncates it before writing.
func (fc *FileConfig) SaveToFile(v interface{}) error {
	fc.lk.Lock()
	defer fc.lk.Unlock()

	if v == nil {
		return saveNilError
	}

	data, err := fc.coder.Encode(v)
	if err != nil {
		return err
	}
	fc.base.Set(v)

	return ioutil.WriteFile(fc.fileName, data, 0666)
}

// FileName property getter
func (fc *FileConfig) FileName() string {
	fc.lk.RLock()
	defer fc.lk.RUnlock()
	return fc.fileName
}

// SetFileName set config filename
func (fc *FileConfig) SetFileName(name string) {
	fc.lk.Lock()
	defer fc.lk.Unlock()

	if fc.fileName != name {
		fc.fileName = name
	}
}

// Value get config value from internal storage
func (fc *FileConfig) Value() interface{} {
	return fc.base.Get()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"log"
	"os"

	"github.com/flowbase/flowbase"
)

// --------------------------------------------------------------------------------
// FileReader
// --------------------------------------------------------------------------------

type FileReader struct {
	InFileName chan string
	OutLine    chan string
}

func NewFileReader() *FileReader {
	return &FileReader{
		InFileName: make(chan string, BUFSIZE),
		OutLine:    make(chan string, BUFSIZE),
	}
}

func (p *FileReader) Run() {
	defer close(p.OutLine)

	flowbase.Debug.Println("Starting loop")
	for fileName := range p.InFileName {
		flowbase.Debug.Printf("Starting processing file %s\n", fileName)
		fh, err := os.Open(fileName)
		if err != nil {
			log.Fatal(err)
		}
		defer fh.Close()

		sc := bufio.NewScanner(fh)
		for sc.Scan() {
			if err := sc.Err(); err != nil {
				log.Fatal(err)
			}
			p.OutLine <- sc.Text()
		}
	}
}
<commit_msg>Add comments to FileReader process<commit_after>package main

import (
	"bufio"
	"log"
	"os"

	"github.com/flowbase/flowbase"
)

// --------------------------------------------------------------------------------
// FileReader
// --------------------------------------------------------------------------------

// FileReader is a process that reads files, based on filenames it receives on the
// FileReader.InFileName port / channel, and writes out the output line by line
// as strings on the FileReader.OutLine port / channel.
type FileReader struct {
	InFileName chan string
	OutLine    chan string
}

// NewFileReader returns an initialized FileReader.
func NewFileReader() *FileReader {
	return &FileReader{
		InFileName: make(chan string, BUFSIZE),
		OutLine:    make(chan string, BUFSIZE),
	}
}

// Run runs the FileReader process.
func (p *FileReader) Run() {
	defer close(p.OutLine)

	flowbase.Debug.Println("Starting loop")
	for fileName := range p.InFileName {
		flowbase.Debug.Printf("Starting processing file %s\n", fileName)
		fh, err := os.Open(fileName)
		if err != nil {
			log.Fatal(err)
		}
		defer fh.Close()

		sc := bufio.NewScanner(fh)
		for sc.Scan() {
			if err := sc.Err(); err != nil {
				log.Fatal(err)
			}
			p.OutLine <- sc.Text()
		}
	}
}
<|endoftext|>


<commit_before>package demo

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	newappengine "google.golang.org/appengine"
	"google.golang.org/appengine/urlfetch"

	"appengine"
	"appengine/datastore"
)

func init() {
	http.HandleFunc("/export", handleContacts)

	http.HandleFunc("/contacts", handleContacts)
	http.HandleFunc("/contacts/export", handleContactsExport)
}

func isUrlOnGoogleApp(writer http.ResponseWriter, request *http.Request, url string) bool {
	ctx := newappengine.NewContext(request)
	client := urlfetch.Client(ctx)

	uri := fmt.Sprintf("https://www.google.com/a/%s/ServiceLogin", url)
	resp, err := client.Get(uri)

	if err != nil {
		http.Error(writer, err.Error(), http.StatusInternalServerError)
		return false
	}

	body, err := ioutil.ReadAll(resp.Body)
	responseBody := string(body[:])

	// Check if the Google Apps login URL is valid
	return strings.Contains(responseBody, "https://www.google.com/accounts/AccountChooser")
}

func handleContacts(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Redirect(w, r, "/?error=noDirectAccess", http.StatusTemporaryRedirect)
		return
	}

	url, err := getProperDomainNameFromUrl(r.FormValue("url"))
	if err != nil {
		http.Redirect(w, r, "/?error=badUrl", http.StatusTemporaryRedirect)
		return
	}

	if !isUrlOnGoogleApp(w, r, url) {
		http.Redirect(w, r, "/?error=notOnGoogleApps", http.StatusTemporaryRedirect)
		return
	}

	ctx := appengine.NewContext(r)
	config.RedirectURL = fmt.Sprintf(`http://%s/contacts/export`, r.Host)

	x := AppState{url}
	url = config.AuthCodeURL(x.encodeState())
	ctx.Infof("Auth: %v", url)

	http.Redirect(w, r, url, http.StatusTemporaryRedirect)
}

func handleContactsExport(w http.ResponseWriter, r *http.Request) {
	y := r.FormValue("state")

	state := new(AppState)
	state.decodeState(y)

	w.Header().Set(`Content-Type`, `application/csv`)
	w.Header().Set(`Content-Disposition`, `attachment; filename="`+state.Domain+`-contacts-export.csv"`)

	ctx := appengine.NewContext(r)
	buf := loadFullFeed(state.Domain, ctx, r)

	ctx.Infof("%v", buf.String())
	writeCSV(ctx, w, buf.Bytes())
}

func handleExport(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)

	d := &ImportData{}
	k := datastore.NewKey(ctx, "ImportData", "ImportedFeed", 0, nil)

	w.Header().Set("Content-Type", "application/csv")
	if err := datastore.Get(ctx, k, d); err != nil {
		ctx.Errorf("get: %v", err)
	} else {
		writeCSV(ctx, w, d.Data)
	}
}

func getProperDomainNameFromUrl(u string) (string, error) {
	uri, err := url.Parse(u)
	if err != nil {
		return ``, fmt.Errorf("The URL seems to be invalid")
	}

	p := strings.Split(uri.Host, ".")

	if len(p) < 2 {
		return ``, fmt.Errorf("The URL seems to be invalid")
	}

	p = p[len(p)-2 : len(p)]

	return strings.Join(p, "."), nil
}
<commit_msg>add copyright notice header<commit_after>// Copyright 2016 Takbok, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package demo

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	newappengine "google.golang.org/appengine"
	"google.golang.org/appengine/urlfetch"

	"appengine"
	"appengine/datastore"
)

func init() {
	http.HandleFunc("/export", handleContacts)

	http.HandleFunc("/contacts", handleContacts)
	http.HandleFunc("/contacts/export", handleContactsExport)
}

func isUrlOnGoogleApp(writer http.ResponseWriter, request *http.Request, url string) bool {
	ctx := newappengine.NewContext(request)
	client := urlfetch.Client(ctx)

	uri := fmt.Sprintf("https://www.google.com/a/%s/ServiceLogin", url)
	resp, err := client.Get(uri)

	if err != nil {
		http.Error(writer, err.Error(), http.StatusInternalServerError)
		return false
	}

	body, err := ioutil.ReadAll(resp.Body)
	responseBody := string(body[:])

	// Check if the Google Apps login URL is valid
	return strings.Contains(responseBody, "https://www.google.com/accounts/AccountChooser")
}

func handleContacts(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Redirect(w, r, "/?error=noDirectAccess", http.StatusTemporaryRedirect)
		return
	}

	url, err := getProperDomainNameFromUrl(r.FormValue("url"))
	if err != nil {
		http.Redirect(w, r, "/?error=badUrl", http.StatusTemporaryRedirect)
		return
	}

	if !isUrlOnGoogleApp(w, r, url) {
		http.Redirect(w, r, "/?error=notOnGoogleApps", http.StatusTemporaryRedirect)
		return
	}

	ctx := appengine.NewContext(r)
	config.RedirectURL = fmt.Sprintf(`http://%s/contacts/export`, r.Host)

	x := AppState{url}
	url = config.AuthCodeURL(x.encodeState())
	ctx.Infof("Auth: %v", url)

	http.Redirect(w, r, url, http.StatusTemporaryRedirect)
}

func handleContactsExport(w http.ResponseWriter, r *http.Request) {
	y := r.FormValue("state")

	state := new(AppState)
	state.decodeState(y)

	w.Header().Set(`Content-Type`, `application/csv`)
	w.Header().Set(`Content-Disposition`, `attachment; filename="`+state.Domain+`-contacts-export.csv"`)

	ctx := appengine.NewContext(r)
	buf := loadFullFeed(state.Domain, ctx, r)

	ctx.Infof("%v", buf.String())
	writeCSV(ctx, w, buf.Bytes())
}

func handleExport(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)

	d := &ImportData{}
	k := datastore.NewKey(ctx, "ImportData", "ImportedFeed", 0, nil)

	w.Header().Set("Content-Type", "application/csv")
	if err := datastore.Get(ctx, k, d); err != nil {
		ctx.Errorf("get: %v", err)
	} else {
		writeCSV(ctx, w, d.Data)
	}
}

func getProperDomainNameFromUrl(u string) (string, error) {
	uri, err := url.Parse(u)
	if err != nil {
		return ``, fmt.Errorf("The URL seems to be invalid")
	}

	p := strings.Split(uri.Host, ".")

	if len(p) < 2 {
		return ``, fmt.Errorf("The URL seems to be invalid")
	}

	p = p[len(p)-2 : len(p)]

	return strings.Join(p, "."), nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metric

import (
	"context"
	"encoding/binary"
	"fmt"
	"hash/fnv"
	"io"
	"sync"
	"time"

	monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"istio.io/istio/mixer/pkg/adapter"
)

// Abstracts over the specific impl for testing.
type bufferedClient interface {
	io.Closer

	Record([]*monitoringpb.TimeSeries)
}

// A wrapper around the stackdriver client SDK that handles batching data before sending.
type buffered struct {
	project     string
	pushMetrics pushFunc
	l           adapter.Logger

	closeMe io.Closer

	// Guards buffer
	m      sync.Mutex
	buffer []*monitoringpb.TimeSeries

	// Guards merged time series
	tsm          sync.Mutex
	mergeTrigger int
	mergedTS     map[uint64]*monitoringpb.TimeSeries

	// retryBuffer holds timeseries that fail in push.
	retryBuffer []*monitoringpb.TimeSeries
	// retryCounter maps a timeseries to the attempts that it has been retried.
	// Key is the hash of timeseries' metric name, monitored resource and start time.
	retryCounter map[uint64]int

	timeSeriesBatchSize int
	retryLimit          int
	pushInterval        time.Duration

	env adapter.Env
}

// batchTimeSeries slices the given time series array with respect to the given batch size limit.
func batchTimeSeries(series []*monitoringpb.TimeSeries, tsLimit int) [][]*monitoringpb.TimeSeries {
	var batches [][]*monitoringpb.TimeSeries
	for i := 0; i < len(series); i += tsLimit {
		e := i + tsLimit
		if e > len(series) {
			e = len(series)
		}
		batches = append(batches, series[i:e])
	}
	return batches
}

func (b *buffered) start(env adapter.Env, ticker *time.Ticker, quit chan struct{}) {
	env.ScheduleDaemon(func() {
		select {
		case <-ticker.C:
			b.mergeTimeSeries()
			b.Send()
		case <-quit:
			return
		}
	})
}

func (b *buffered) Record(toSend []*monitoringpb.TimeSeries) {
	b.m.Lock()
	b.buffer = append(b.buffer, toSend...)
	if len(b.buffer) >= b.mergeTrigger {
		// pull the trigger to merge buffered time series
		b.env.ScheduleWork(func() {
			b.mergeTimeSeries()
		})
	}
	// TODO: gauge metric reporting how many bytes/timeseries we're holding right now
	b.m.Unlock()
}

func (b *buffered) mergeTimeSeries() {
	b.m.Lock()
	temp := b.buffer
	b.buffer = make([]*monitoringpb.TimeSeries, 0, len(temp))
	b.m.Unlock()

	b.tsm.Lock()
	for _, ts := range temp {
		k := toKey(ts.Metric, ts.Resource)
		if p, ok := b.mergedTS[k]; !ok {
			b.mergedTS[k] = ts
		} else {
			if n, err := mergePoints(p, ts); err != nil {
				b.l.Warningf("failed to merge time series %v and %v: %v", p, ts, err)
			} else {
				b.mergedTS[k] = n
			}
		}
	}
	b.tsm.Unlock()
}

func (b *buffered) Send() {
	b.tsm.Lock()
	if len(b.mergedTS) == 0 && len(b.retryBuffer) == 0 {
		b.tsm.Unlock()
		b.l.Debugf("No data to send to Stackdriver.")
		return
	}

	tmpBuffer := b.mergedTS
	b.mergedTS = make(map[uint64]*monitoringpb.TimeSeries)
	b.tsm.Unlock()

	toSend := make([]*monitoringpb.TimeSeries, 0, len(tmpBuffer))
	for _, v := range tmpBuffer {
		toSend = append(toSend, v)
	}
	merged := append(b.retryBuffer, toSend...)
	b.retryBuffer = make([]*monitoringpb.TimeSeries, 0, len(b.retryBuffer))
	batches := batchTimeSeries(merged, b.timeSeriesBatchSize)
	// Spread monitoring API calls evenly over the push interval.
	ns := b.pushInterval.Nanoseconds() / int64(len(batches))
	t := time.NewTicker(time.Duration(ns))
	defer t.Stop()
	for i, timeSeries := range batches {
		err := b.pushMetrics(context.Background(),
			&monitoringpb.CreateTimeSeriesRequest{
				Name:       "projects/" + b.project,
				TimeSeries: timeSeries,
			})

		// TODO: this is executed in a daemon, so we can't get out info about errors other than logging.
		// We need to build framework level support for these kinds of async tasks. Perhaps a generic batching adapter
		// can handle some of this complexity?
		if err != nil {
			ets := handleError(err, timeSeries)
			b.updateRetryBuffer(ets)
			_ = b.l.Errorf("Stackdriver returned: %v\nGiven data: %v", err, timeSeries)
		} else {
			b.l.Debugf("Successfully sent data to Stackdriver.")
		}
		if i+1 != len(batches) {
			// Do not block and wait after the last batch pushed. No need to wait any more.
			// This also leaves portion of push interval for other operations in Send().
			<-t.C
		}
	}
}

func (b *buffered) Close() error {
	b.l.Infof("Sending last data before shutting down")
	b.Send()
	return b.closeMe.Close()
}

// handleError extract out timeseries that fails to create from response status.
// If no sepecific timeseries listed in error response, retry all time series in batch.
func handleError(err error, tsSent []*monitoringpb.TimeSeries) []*monitoringpb.TimeSeries {
	errorTS := make([]*monitoringpb.TimeSeries, 0)
	retryAll := true
	s, ok := status.FromError(err)
	if !ok {
		return errorTS
	}
	sd := s.Details()
	for _, i := range sd {
		if t, ok := i.(*monitoringpb.CreateTimeSeriesError); ok {
			retryAll = false
			if !isRetryable(codes.Code(t.GetStatus().Code)) {
				continue
			}
			errorTS = append(errorTS, t.GetTimeSeries())
		}
	}
	if isRetryable(status.Code(err)) && retryAll {
		errorTS = append(errorTS, tsSent...)
	}
	return errorTS
}

func (b *buffered) updateRetryBuffer(errorTS []*monitoringpb.TimeSeries) {
	retryCounter := map[uint64]int{}
	for _, ts := range errorTS {
		k, err := toRetryKey(ts)
		if err != nil {
			b.l.Debugf("cannot generate retry key for timeseries %v: %v", ts, err)
			continue
		}
		if c, ok := b.retryCounter[k]; ok {
			attempt := c + 1
			if attempt >= b.retryLimit {
				continue
			}
			retryCounter[k] = attempt
		} else {
			retryCounter[k] = 0
		}
		b.retryBuffer = append(b.retryBuffer, ts)
	}
	b.retryCounter = retryCounter
}

func isRetryable(c codes.Code) bool {
	switch c {
	case codes.Canceled, codes.DeadlineExceeded, codes.ResourceExhausted, codes.Aborted, codes.Internal, codes.Unavailable:
		return true
	}
	return false
}

func toRetryKey(ts *monitoringpb.TimeSeries) (uint64, error) {
	hash := fnv.New64()
	if len(ts.Points) != 1 {
		return 0, fmt.Errorf("timeseries has to contain exactly one point")
	}

	buf := make([]byte, 8)
	mKey := toKey(ts.Metric, ts.Resource)
	binary.BigEndian.PutUint64(buf, mKey)
	_, _ = hash.Write(buf)

	s := ts.Points[0].Interval.StartTime.GetSeconds()
	binary.BigEndian.PutUint64(buf, uint64(s))
	_, _ = hash.Write(buf)
	return hash.Sum64(), nil
}
<commit_msg>fix stackdriver adapter getting shutdown (#15612)<commit_after>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metric

import (
	"context"
	"encoding/binary"
	"fmt"
	"hash/fnv"
	"io"
	"sync"
	"time"

	monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"istio.io/istio/mixer/pkg/adapter"
)

// Abstracts over the specific impl for testing.
type bufferedClient interface {
	io.Closer

	Record([]*monitoringpb.TimeSeries)
}

// A wrapper around the stackdriver client SDK that handles batching data before sending.
type buffered struct {
	project     string
	pushMetrics pushFunc
	l           adapter.Logger

	closeMe io.Closer

	// Guards buffer
	m      sync.Mutex
	buffer []*monitoringpb.TimeSeries

	// Guards merged time series
	tsm          sync.Mutex
	mergeTrigger int
	mergedTS     map[uint64]*monitoringpb.TimeSeries

	// retryBuffer holds timeseries that fail in push.
	retryBuffer []*monitoringpb.TimeSeries
	// retryCounter maps a timeseries to the attempts that it has been retried.
	// Key is the hash of timeseries' metric name, monitored resource and start time.
	retryCounter map[uint64]int

	timeSeriesBatchSize int
	retryLimit          int
	pushInterval        time.Duration

	env adapter.Env
}

// batchTimeSeries slices the given time series array with respect to the given batch size limit.
func batchTimeSeries(series []*monitoringpb.TimeSeries, tsLimit int) [][]*monitoringpb.TimeSeries {
	var batches [][]*monitoringpb.TimeSeries
	for i := 0; i < len(series); i += tsLimit {
		e := i + tsLimit
		if e > len(series) {
			e = len(series)
		}
		batches = append(batches, series[i:e])
	}
	return batches
}

func (b *buffered) start(env adapter.Env, ticker *time.Ticker, quit chan struct{}) {
	env.ScheduleDaemon(func() {
		for {
			select {
			case <-ticker.C:
				b.mergeTimeSeries()
				b.Send()
			case <-quit:
				return
			}
		}
	})
}

func (b *buffered) Record(toSend []*monitoringpb.TimeSeries) {
	b.m.Lock()
	b.buffer = append(b.buffer, toSend...)
	if len(b.buffer) >= b.mergeTrigger {
		// pull the trigger to merge buffered time series
		b.env.ScheduleWork(func() {
			b.mergeTimeSeries()
		})
	}
	// TODO: gauge metric reporting how many bytes/timeseries we're holding right now
	b.m.Unlock()
}

func (b *buffered) mergeTimeSeries() {
	b.m.Lock()
	temp := b.buffer
	b.buffer = make([]*monitoringpb.TimeSeries, 0, len(temp))
	b.m.Unlock()

	b.tsm.Lock()
	for _, ts := range temp {
		k := toKey(ts.Metric, ts.Resource)
		if p, ok := b.mergedTS[k]; !ok {
			b.mergedTS[k] = ts
		} else {
			if n, err := mergePoints(p, ts); err != nil {
				b.l.Warningf("failed to merge time series %v and %v: %v", p, ts, err)
			} else {
				b.mergedTS[k] = n
			}
		}
	}
	b.tsm.Unlock()
}

func (b *buffered) Send() {
	b.tsm.Lock()
	if len(b.mergedTS) == 0 && len(b.retryBuffer) == 0 {
		b.tsm.Unlock()
		b.l.Debugf("No data to send to Stackdriver.")
		return
	}

	tmpBuffer := b.mergedTS
	b.mergedTS = make(map[uint64]*monitoringpb.TimeSeries)
	b.tsm.Unlock()

	toSend := make([]*monitoringpb.TimeSeries, 0, len(tmpBuffer))
	for _, v := range tmpBuffer {
		toSend = append(toSend, v)
	}
	merged := append(b.retryBuffer, toSend...)
	b.retryBuffer = make([]*monitoringpb.TimeSeries, 0, len(b.retryBuffer))
	batches := batchTimeSeries(merged, b.timeSeriesBatchSize)
	// Spread monitoring API calls evenly over the push interval.
	ns := b.pushInterval.Nanoseconds() / int64(len(batches))
	t := time.NewTicker(time.Duration(ns))
	defer t.Stop()
	for i, timeSeries := range batches {
		err := b.pushMetrics(context.Background(),
			&monitoringpb.CreateTimeSeriesRequest{
				Name:       "projects/" + b.project,
				TimeSeries: timeSeries,
			})

		// TODO: this is executed in a daemon, so we can't get out info about errors other than logging.
		// We need to build framework level support for these kinds of async tasks. Perhaps a generic batching adapter
		// can handle some of this complexity?
		if err != nil {
			ets := handleError(err, timeSeries)
			b.updateRetryBuffer(ets)
			_ = b.l.Errorf("Stackdriver returned: %v\nGiven data: %v", err, timeSeries)
		} else {
			b.l.Debugf("Successfully sent data to Stackdriver.")
		}
		if i+1 != len(batches) {
			// Do not block and wait after the last batch pushed. No need to wait any more.
			// This also leaves portion of push interval for other operations in Send().
			<-t.C
		}
	}
}

func (b *buffered) Close() error {
	b.l.Infof("Sending last data before shutting down")
	b.Send()
	return b.closeMe.Close()
}

// handleError extract out timeseries that fails to create from response status.
// If no sepecific timeseries listed in error response, retry all time series in batch.
func handleError(err error, tsSent []*monitoringpb.TimeSeries) []*monitoringpb.TimeSeries {
	errorTS := make([]*monitoringpb.TimeSeries, 0)
	retryAll := true
	s, ok := status.FromError(err)
	if !ok {
		return errorTS
	}
	sd := s.Details()
	for _, i := range sd {
		if t, ok := i.(*monitoringpb.CreateTimeSeriesError); ok {
			retryAll = false
			if !isRetryable(codes.Code(t.GetStatus().Code)) {
				continue
			}
			errorTS = append(errorTS, t.GetTimeSeries())
		}
	}
	if isRetryable(status.Code(err)) && retryAll {
		errorTS = append(errorTS, tsSent...)
	}
	return errorTS
}

func (b *buffered) updateRetryBuffer(errorTS []*monitoringpb.TimeSeries) {
	retryCounter := map[uint64]int{}
	for _, ts := range errorTS {
		k, err := toRetryKey(ts)
		if err != nil {
			b.l.Debugf("cannot generate retry key for timeseries %v: %v", ts, err)
			continue
		}
		if c, ok := b.retryCounter[k]; ok {
			attempt := c + 1
			if attempt >= b.retryLimit {
				continue
			}
			retryCounter[k] = attempt
		} else {
			retryCounter[k] = 0
		}
		b.retryBuffer = append(b.retryBuffer, ts)
	}
	b.retryCounter = retryCounter
}

func isRetryable(c codes.Code) bool {
	switch c {
	case codes.Canceled, codes.DeadlineExceeded, codes.ResourceExhausted, codes.Aborted, codes.Internal, codes.Unavailable:
		return true
	}
	return false
}

func toRetryKey(ts *monitoringpb.TimeSeries) (uint64, error) {
	hash := fnv.New64()
	if len(ts.Points) != 1 {
		return 0, fmt.Errorf("timeseries has to contain exactly one point")
	}

	buf := make([]byte, 8)
	mKey := toKey(ts.Metric, ts.Resource)
	binary.BigEndian.PutUint64(buf, mKey)
	_, _ = hash.Write(buf)

	s := ts.Points[0].Interval.StartTime.GetSeconds()
	binary.BigEndian.PutUint64(buf, uint64(s))
	_, _ = hash.Write(buf)
	return hash.Sum64(), nil
}
<|endoftext|>


<commit_before>package main

import ()

func init() {
	h := NewHandler()
	h.CommandName = "help"
	h.CommandPattern = "(help)( )(.*)"
	h.HandlerFunc = func(cmd *Command) string {
		cmdList := ""

		for k, _ := range HandlerRegistry {
			cmdList += k + "\n"
		}

		return `
Help
----
Commands:
` + cmdList + `
eg:
host localhost
port 9200
index movies
get _search?q=title:thx1138
`
	}
	HandlerRegistry[h.CommandName] = h
}
<commit_msg>Alphabetised command listing.<commit_after>package main

import (
	"sort"
)

func init() {
	h := NewHandler()
	h.CommandName = "help"
	h.CommandPattern = "(help)( )(.*)"
	h.Usage = "help"
	h.HandlerFunc = func(cmd *Command) string {
		cmdList := ""
		keys := []string{}
		for k, _ := range HandlerRegistry {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for _, s := range keys {
			cmdList += s + ": " + HandlerRegistry[s].Usage + "\n"
		}
		return `
	Help
	----

Commands
--------
` + cmdList + `
`
	}
	HandlerRegistry[h.CommandName] = h
}
<|endoftext|>


<commit_before>package lib

import (
	"fmt"
	"hash"
	"hash/crc64"
	"crypto/md5"
	"encoding/base64"
	"strings"
	"io"
	"os"
)

var specChineseHash = SpecText{

	synopsisText: "计算本地文件的crc64或md5",

	paramText: "file_url [options]",

	syntaxText: `
    ossutil hash file_url [--type=hashtype]
`,

	detailHelpText: `
    该命令计算本地文件的crc64值或md5/content-md5值, 可以通过--type选项来控制计算的类型，
    可选类型值为crc64或md5, 默认为` + DefaultHashType +`。

    注意：oss文件的crc64和content-md5值一般可通过stat命令查看到，参考` + StatCRC64 + `
    字段和` + StatContentMD5 + `字段。若文件在oss支持crc64功能之前上传，则stat命令不支持查看crc64值；
    对于append和multipart类型的文件，stat命令不支持查看content-md5值。

    crc64的计算标准参考ECMA-182标准(http://www.ecma-international.org/publications/standards/Ecma-182.htm)。

    计算类型为md5时，会同时输出文件的md5以及content-md5值。content-md5值其实是先计算md5
    值获得128比特位数字，然后对该数字进行base64编码得到的值。关于content-md5的更多信息，
    请参考https://tools.ietf.org/html/rfc1864。

用法:

    ossutil hash file_url [--type=hashtype]
`,

    sampleText: `
    1) 计算本地文件的crc64:
        ossutil hash test.txt 或
        ossutil hash test.txt --type=crc64

        输出:
        CRC64-ECMA                  : 295992936743767023

    2) 计算本地文件的md5:
        ossutil hash test.txt --type=md5

        输出:
        MD5                         : 01C3C45C03B2AF225EFAD9F911A33D73
        Content-MD5                 : AcPEXAOyryJe+tn5EaM9cw==
`,
}

var specEnglishHash = SpecText{

	synopsisText: "Get crc64 or md5 of local file",

	paramText: "file_url [options]",

	syntaxText: `
    ossutil hash file_url [--type=hashtype]
`,

	detailHelpText: `
    The command calculate crc64 or md5/content-md5 value of the specified local file,
    specify the hashtype by --type, default hashtype is ` + DefaultHashType +`.

    Warning: user can use stat command to check the crc64 or md5/content-md5 value of
    normal oss object, see the ` + StatCRC64 + ` and ` + StatContentMD5 + ` field. If the object
    was uploaded to oss before oss support crc64 feature, stat result will not show
    ` + StatCRC64 + `, if the object is append file type or multipart, stat result
    will not show ` + StatContentMD5 + `.

    Crc64 is calcuated according to ECMA-182(http://www.ecma-international.org/publications/standards/Ecma-182.htm).

    When hashtype is md5, it will output both md5 and content-md5 of local file.
    Content-md5 is base64 encoded string of md5. For more detial about content-md5,
    please refer to https://tools.ietf.org/html/rfc1864.

Usage:

    ossutil hash file_url [--type=hashtype]
`,

	sampleText: `
    1) Get crc64 of local file:
        ossutil hash test.txt or
        ossutil hash test.txt --type=crc64

        output:
        CRC64-ECMA                  : 295992936743767023

    2) Get md5  of local file:
        ossutil hash test.txt --type=md5

        output:
        MD5                         : 01C3C45C03B2AF225EFAD9F911A33D73
        Content-MD5                 : AcPEXAOyryJe+tn5EaM9cw==
`,
}

// HashCommand is the command to get crc64/md5 of local file
type HashCommand struct {
	command Command
}

var hashCommand = HashCommand{
	command: Command{
		name:             "hash",
		nameAlias:        []string{""},
		minArgc:          1,
		maxArgc:          1,
		specChinese:      specChineseHash,
		specEnglish:      specEnglishHash,
		group:            GroupTypeAdditionalCommand,
		validOptionNames: []string{
            OptionHashType,
        },
	},
}
// function for FormatHelper interface
func (cc *HashCommand) formatHelpForWhole() string {
	return cc.command.formatHelpForWhole()
}

func (cc *HashCommand) formatIndependHelp() string {
	return cc.command.formatIndependHelp()
}

// Init simulate inheritance, and polymorphism
func (cc *HashCommand) Init(args []string, options OptionMapType) error {
	return cc.command.Init(args, options, cc)
}

// RunCommand simulate inheritance, and polymorphism
func (cc *HashCommand) RunCommand() error {
    hashType, _ := GetString(OptionHashType, cc.command.options)
    path := cc.command.args[0]

    f, err := os.Open(path)
    if err != nil {
        return err
    }
    defer f.Close()
    f.Seek(0, os.SEEK_SET)

    switch strings.ToLower(hashType) {
    case MD5HashType:
        return hashMD5(f)
    default:
        return hashCRC64(f)
    }
}

func hashMD5(f io.Reader) error {
    md5Ins := md5.New()
    w, _ := md5Ins.(hash.Hash)
    if _, err := io.Copy(w, f); err != nil {
        return err
    }

    result := md5Ins.Sum(nil)
    fmt.Printf("%-28s: %X\n", HashMD5, result)

    encoded := base64.StdEncoding.EncodeToString(result)
    fmt.Printf("%-28s: %s\n", HashContentMD5, encoded)
    return nil
}

func hashCRC64(f io.Reader) error {
    crc64Ins := crc64.New(crc64.MakeTable(crc64.ECMA))
    w, _ := crc64Ins.(hash.Hash)
    if _, err := io.Copy(w, f); err != nil {
        return err
    }

    result := crc64Ins.Sum64()
    fmt.Printf("%-28s: %d\n", HashCRC64, result)
    return nil
}
<commit_msg>fix hash bug that need config file<commit_after>package lib

import (
	"fmt"
	"hash"
	"hash/crc64"
	"crypto/md5"
	"encoding/base64"
	"strings"
	"io"
	"os"
)

var specChineseHash = SpecText{

	synopsisText: "计算本地文件的crc64或md5",

	paramText: "file_url [options]",

	syntaxText: `
    ossutil hash file_url [--type=hashtype]
`,

	detailHelpText: `
    该命令计算本地文件的crc64值或md5/content-md5值, 可以通过--type选项来控制计算的类型，
    可选类型值为crc64或md5, 默认为` + DefaultHashType +`。

    注意：oss文件的crc64和content-md5值一般可通过stat命令查看到，参考` + StatCRC64 + `
    字段和` + StatContentMD5 + `字段。若文件在oss支持crc64功能之前上传，则stat命令不支持查看crc64值；
    对于append和multipart类型的文件，stat命令不支持查看content-md5值。

    crc64的计算标准参考ECMA-182标准(http://www.ecma-international.org/publications/standards/Ecma-182.htm)。

    计算类型为md5时，会同时输出文件的md5以及content-md5值。content-md5值其实是先计算md5
    值获得128比特位数字，然后对该数字进行base64编码得到的值。关于content-md5的更多信息，
    请参考https://tools.ietf.org/html/rfc1864。

用法:

    ossutil hash file_url [--type=hashtype]
`,

    sampleText: `
    1) 计算本地文件的crc64:
        ossutil hash test.txt 或
        ossutil hash test.txt --type=crc64

        输出:
        CRC64-ECMA                  : 295992936743767023

    2) 计算本地文件的md5:
        ossutil hash test.txt --type=md5

        输出:
        MD5                         : 01C3C45C03B2AF225EFAD9F911A33D73
        Content-MD5                 : AcPEXAOyryJe+tn5EaM9cw==
`,
}

var specEnglishHash = SpecText{

	synopsisText: "Get crc64 or md5 of local file",

	paramText: "file_url [options]",

	syntaxText: `
    ossutil hash file_url [--type=hashtype]
`,

	detailHelpText: `
    The command calculate crc64 or md5/content-md5 value of the specified local file,
    specify the hashtype by --type, default hashtype is ` + DefaultHashType +`.

    Warning: user can use stat command to check the crc64 or md5/content-md5 value of
    normal oss object, see the ` + StatCRC64 + ` and ` + StatContentMD5 + ` field. If the object
    was uploaded to oss before oss support crc64 feature, stat result will not show
    ` + StatCRC64 + `, if the object is append file type or multipart, stat result
    will not show ` + StatContentMD5 + `.

    Crc64 is calcuated according to ECMA-182(http://www.ecma-international.org/publications/standards/Ecma-182.htm).

    When hashtype is md5, it will output both md5 and content-md5 of local file.
    Content-md5 is base64 encoded string of md5. For more detial about content-md5,
    please refer to https://tools.ietf.org/html/rfc1864.

Usage:

    ossutil hash file_url [--type=hashtype]
`,

	sampleText: `
    1) Get crc64 of local file:
        ossutil hash test.txt or
        ossutil hash test.txt --type=crc64

        output:
        CRC64-ECMA                  : 295992936743767023

    2) Get md5  of local file:
        ossutil hash test.txt --type=md5

        output:
        MD5                         : 01C3C45C03B2AF225EFAD9F911A33D73
        Content-MD5                 : AcPEXAOyryJe+tn5EaM9cw==
`,
}

// HashCommand is the command to get crc64/md5 of local file
type HashCommand struct {
	command Command
}

var hashCommand = HashCommand{
	command: Command{
		name:             "hash",
		nameAlias:        []string{""},
		minArgc:          1,
		maxArgc:          1,
		specChinese:      specChineseHash,
		specEnglish:      specEnglishHash,
		group:            GroupTypeAdditionalCommand,
		validOptionNames: []string{
            OptionHashType,
        },
	},
}

// function for RewriteLoadConfiger interface
func (hc *HashCommand) rewriteLoadConfig(configFile string) error {
    // read config file, if error exist, do not print error
    var err error
    if hc.command.configOptions, err = LoadConfig(configFile); err != nil {
        hc.command.configOptions = OptionMapType{}
    }
	return nil
}

// function for FormatHelper interface
func (hc *HashCommand) formatHelpForWhole() string {
	return hc.command.formatHelpForWhole()
}

func (hc *HashCommand) formatIndependHelp() string {
	return hc.command.formatIndependHelp()
}

// Init simulate inheritance, and polymorphism
func (hc *HashCommand) Init(args []string, options OptionMapType) error {
	return hc.command.Init(args, options, hc)
}

// RunCommand simulate inheritance, and polymorphism
func (hc *HashCommand) RunCommand() error {
    hashType, _ := GetString(OptionHashType, hc.command.options)
    path := hc.command.args[0]

    f, err := os.Open(path)
    if err != nil {
        return err
    }
    defer f.Close()
    f.Seek(0, os.SEEK_SET)

    switch strings.ToLower(hashType) {
    case MD5HashType:
        return hashMD5(f)
    default:
        return hashCRC64(f)
    }
}

func hashMD5(f io.Reader) error {
    md5Ins := md5.New()
    w, _ := md5Ins.(hash.Hash)
    if _, err := io.Copy(w, f); err != nil {
        return err
    }

    result := md5Ins.Sum(nil)
    fmt.Printf("%-28s: %X\n", HashMD5, result)

    encoded := base64.StdEncoding.EncodeToString(result)
    fmt.Printf("%-28s: %s\n", HashContentMD5, encoded)
    return nil
}

func hashCRC64(f io.Reader) error {
    crc64Ins := crc64.New(crc64.MakeTable(crc64.ECMA))
    w, _ := crc64Ins.(hash.Hash)
    if _, err := io.Copy(w, f); err != nil {
        return err
    }

    result := crc64Ins.Sum64()
    fmt.Printf("%-28s: %d\n", HashCRC64, result)
    return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

// Glue code between BTCD code & Factom.

package btcd

import (
	"errors"
	"fmt"
	"os"

	//	"github.com/FactomProject/btcd/chaincfg"
	//	"github.com/FactomProject/btcutil"

	cp "github.com/FactomProject/FactomCode/controlpanel"
	"github.com/FactomProject/FactomCode/database"
	"github.com/FactomProject/btcd/wire"
	"github.com/davecgh/go-spew/spew"
)

var _ = fmt.Printf

var (
	local_Server *server
	db           database.Db              // database
	inMsgQueue   chan wire.FtmInternalMsg //incoming message queue for factom application messages
	outMsgQueue  chan wire.FtmInternalMsg //outgoing message queue for factom application messages

	inCtlMsgQueue  chan wire.FtmInternalMsg //incoming message queue for factom control messages
	outCtlMsgQueue chan wire.FtmInternalMsg //outgoing message queue for factom control messages
)

// start up Factom queue(s) managers/processors
// this is to be called within the btcd's main code
func factomForkInit(s *server) {
	// tweak some config options
	cfg.DisableCheckpoints = true

	local_Server = s // local copy of our server pointer

	// Write outgoing factom messages into P2P network
	go func() {
		for msg := range outMsgQueue {
			switch msg.(type) {
			case *wire.MsgInt_DirBlock:
				dirBlock, _ := msg.(*wire.MsgInt_DirBlock)
				iv := wire.NewInvVect(wire.InvTypeFactomDirBlock, dirBlock.ShaHash)
				s.RelayInventory(iv, nil)

			case wire.Message:
				wireMsg, _ := msg.(wire.Message)
				//s.BroadcastMessage(wireMsg)
				if ClientOnly {
					//fmt.Println("broadcasting from client.")
					s.BroadcastMessage(wireMsg)
				} else {
					if _, ok := msg.(*wire.MsgAcknowledgement); ok {
						//fmt.Println("broadcasting from server.")
						s.BroadcastMessage(wireMsg)
					}
				}

			case *wire.MsgInt_ReSyncup:
				msg, _ := msg.(*wire.MsgInt_ReSyncup)
				fmt.Println("MsgInt_ReSyncup: ", spew.Sdump(msg))
				p := local_Server.SyncPeer()
				if p != nil {
					fmt.Println("start re-syncup: ")
					locator := DirBlockLocatorFromHash(msg.StartHash)
					p.PushGetDirBlocksMsg(locator, &zeroHash)
				}
				fmt.Println("sync peer is nil, no syncup renewed.")

			default:
				panic(fmt.Sprintf("bad outMsgQueue message received: %v", msg))
			}
		}
	}()

	go func() {
		for msg := range outCtlMsgQueue {

			fmt.Printf("in range outCtlMsgQueue, msg:%+v\n", msg)

			msgEom, _ := msg.(*wire.MsgInt_EOM)

			//			switch msgEom.Command() {
			switch msg.Command() {

			case wire.CmdInt_EOM:

				switch msgEom.EOM_Type {

				case wire.END_MINUTE_10:
					panic(errors.New("unhandled END_MINUTE_10"))

				default:
					panic(errors.New("unhandled EOM type"))
				}

			default:
				panic(errors.New("unhandled CmdInt_EOM"))
			}

			/*
				switch msg.EOM_Type {

				case wire.END_MINUTE_10:
					//util.Trace("EOM10")
				default:
					//util.Trace("default")
				}
			*/

			/*
				switch msg.Command() {
				case factomwire.CmdTx:
					InMsgQueue <- msg //    for testing
					server.blockManager.QueueTx(msg.(*factomwire.MsgTx), nil)
				case factomwire.CmdConfirmation:
					server.blockManager.QueueConf(msg.(*factomwire.MsgConfirmation), nil)

				default:
					inMsgQueue <- msg
					outMsgQueue <- msg
				}
			*/
		}
	}()
}

func Start_btcd(
	ldb database.Db,
	inMsgQ chan wire.FtmInternalMsg,
	outMsgQ chan wire.FtmInternalMsg,
	inCtlMsgQ chan wire.FtmInternalMsg,
	outCtlMsgQ chan wire.FtmInternalMsg,
	user, pass string, clientMode bool) {

	factomdUser = user
	factomdPass = pass

	ClientOnly = clientMode

	if ClientOnly {
		cp.CP.AddUpdate(
			"FactomMode", // tag
			"system",     // Category
			"Factom Mode: Full Node (Client)", // Title
			"", // Message
			0)
		fmt.Println("\n\n>>>>>>>>>>>>>>>>>  CLIENT MODE <<<<<<<<<<<<<<<<<<<<<<<\n\n")
	} else {
		cp.CP.AddUpdate(
			"FactomMode",                    // tag
			"system",                        // Category
			"Factom Mode: Federated Server", // Title
			"", // Message
			0)
		fmt.Println("\n\n>>>>>>>>>>>>>>>>>  SERVER MODE <<<<<<<<<<<<<<<<<<<<<<<\n\n")
	}

	db = ldb

	inMsgQueue = inMsgQ
	outMsgQueue = outMsgQ

	inCtlMsgQueue = inCtlMsgQ
	outCtlMsgQueue = outCtlMsgQ

	// Use all processor cores.
	//runtime.GOMAXPROCS(runtime.NumCPU())

	// Up some limits.
	//if err := limits.SetLimits(); err != nil {
	//	os.Exit(1)
	//}

	// Call serviceMain on Windows to handle running as a service.  When
	// the return isService flag is true, exit now since we ran as a
	// service.  Otherwise, just fall through to normal operation.
	/*if runtime.GOOS == "windows" {
		isService, err := winServiceMain()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		if isService {
			os.Exit(0)
		}
	}
	*/

	// Work around defer not working after os.Exit()
	if err := btcdMain(nil); err != nil {
		os.Exit(1)
	}
}

// Handle factom app imcoming msg
func (p *peer) handleCommitChainMsg(msg *wire.MsgCommitChain) {
	// Add the msg to inbound msg queue
	if !ClientOnly {
		inMsgQueue <- msg
	}
}

// Handle factom app imcoming msg
func (p *peer) handleRevealChainMsg(msg *wire.MsgRevealChain) {
	// Add the msg to inbound msg queue
	//inMsgQueue <- msg
	if !ClientOnly {
		inMsgQueue <- msg
	}
}

// Handle factom app imcoming msg
func (p *peer) handleCommitEntryMsg(msg *wire.MsgCommitEntry) {
	// Add the msg to inbound msg queue
	if !ClientOnly {
		inMsgQueue <- msg
	}
}

// Handle factom app imcoming msg
func (p *peer) handleRevealEntryMsg(msg *wire.MsgRevealEntry) {
	// Add the msg to inbound msg queue
	if !ClientOnly {
		inMsgQueue <- msg
	}
}

// Handle factom app imcoming msg
func (p *peer) handleAcknoledgementMsg(msg *wire.MsgAcknowledgement) {
	// Add the msg to inbound msg queue
	if !ClientOnly {
		inMsgQueue <- msg
	}
}

// returns true if the message should be relayed, false otherwise
func (p *peer) shallRelay(msg interface{}) bool {
	hash, _ := wire.NewShaHashFromStruct(msg)
	iv := wire.NewInvVect(wire.InvTypeFactomRaw, hash)

	if !p.isKnownInventory(iv) {
		p.AddKnownInventory(iv)

		return true
	}

	fmt.Println("******************* SHALL NOT RELAY !!!!!!!!!!! ******************")

	return false
}

// Call FactomRelay to relay/broadcast a Factom message (to your peers).
// The intent is to call this function after certain 'processor' checks been done.
func (p *peer) FactomRelay(msg wire.Message) {

	// broadcast/relay only if hadn't been done for this peer
	if p.shallRelay(msg) {
		//		p.server.BroadcastMessage(msg, p)
		local_Server.BroadcastMessage(msg)
	}
}

/*
// func (pl *ProcessList) AddFtmTxToProcessList(msg wire.Message, msgHash *wire.ShaHash) error {
func fakehook1(msg wire.Message, msgHash *wire.ShaHash) error {
	return nil
}

func factom_PL_hook(tx *btcutil.Tx, label string) error {
	//util.Trace("label= " + label)

	_ = fakehook1(tx.MsgTx(), tx.Sha())

	return nil
}

// for Jack
func global_DeleteMemPoolEntry(hash *wire.ShaHash) {
	// TODO: ensure mutex-protection
}

// check a few btcd-related flags for sanity in our fork
func (b *blockManager) factomChecks() {
	//util.Trace()

	if cfg.AddrIndex {
		panic(errors.New("AddrIndex must be disabled and it is NOT !!!"))
	}

	// DisableCheckpoints should always be set
	if !cfg.DisableCheckpoints {
		panic(errors.New("checkpoints must be disabled and they are NOT !!!"))
	}

	if cfg.RegressionTest || cfg.SimNet || cfg.Generate {
		panic(100)
	}

	if cfg.TestNet3 {
		panic(errors.New("TestNet mode is NOT SUPPORTED (remove the option from the command line or from the .conf file)!"))
	}

	//util.Trace()
}

// feed all incoming Txs to the inner Factom code (for Jack)
// TODO: do this after proper mempool/orphanpool/validity triangulation & checks
func factomIngressTx_hook(tx *wire.MsgTx) error {
	//util.Trace()

	ecmap := make(map[wire.ShaHash]uint64)

	txid, _ := tx.TxSha()
	hash, _ := wire.NewShaHash(wire.Sha256(txid.Bytes()))

	ecmap[*hash] = 1

	// Use wallet's public key to add EC??
	sig := wallet.SignData(nil)
	hash2 := new(wire.ShaHash)
	hash2.SetBytes((*sig.Pub.Key)[:])

	ecmap[*hash2] = 100

	txHash, _ := tx.TxSha()
	fo := &wire.MsgInt_FactoidObj{tx, &txHash, ecmap}

	fmt.Println("ecmap len =", len(ecmap))

	inMsgQueue <- fo

	return nil
}

func factomIngressBlock_hook(hash *wire.ShaHash) error {
	//util.Trace(fmt.Sprintf("hash: %s", hash))

	fbo := &wire.MsgInt_FactoidBlock{
		ShaHash: *hash}

	doneFBlockQueue <- fbo

	return nil
}
*/

/*
func ExtractPkScriptAddrs(pkScript []byte, chainParams *chaincfg.Params) ([]btcutil.Address, int, error) {
	oldWay := false

	//util.Trace("bytes= " + spew.Sdump(pkScript))

	var addrs []btcutil.Address
	var requiredSigs int

	if oldWay {
		// A pay-to-pubkey script is of the form:
		//  <pubkey> OP_CHECKSIG
		// Therefore the pubkey is the first item on the stack.
		// Skip the pubkey if it's invalid for some reason.
		requiredSigs = 1
		addr, err := btcutil.NewAddressPubKey(pkScript, chainParams)
		if err == nil {
			addrs = append(addrs, addr)
		}

	} else {

		// A pay-to-pubkey-hash script is of the form:
		//  OP_DUP OP_HASH160 <hash> OP_EQUALVERIFY OP_CHECKSIG
		// Therefore the pubkey hash is the 3rd item on the stack.
		// Skip the pubkey hash if it's invalid for some reason.
		requiredSigs = 1
		//	addr, err := btcutil.NewAddressPubKeyHash(pops[2].data,
		addr, err := btcutil.NewAddressPubKeyHash(pkScript, chainParams)
		if err == nil {
			addrs = append(addrs, addr)
		}
	}

	//util.Trace("addrs= " + spew.Sdump(addrs))

	return addrs, requiredSigs, nil
}

// PayToAddrScript creates a new script to pay a transaction output to a the
// specified address.
func PayToAddrScript(addr btcutil.Address) ([]byte, error) {
	scrAddr := addr.ScriptAddress()

	//util.Trace("scrAddr= " + spew.Sdump(scrAddr))

	return scrAddr, nil

	//	panic(errors.New("PayToAddrScript -- NOT IMPLEMENTED !!!"))

	//	return payToPubKeyHashScript(addr.ScriptAddress())
}
*/
<commit_msg>removed synced print msgs<commit_after>// Copyright 2015 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

// Glue code between BTCD code & Factom.

package btcd

import (
	"errors"
	"fmt"
	"os"

	//	"github.com/FactomProject/btcd/chaincfg"
	//	"github.com/FactomProject/btcutil"

	cp "github.com/FactomProject/FactomCode/controlpanel"
	"github.com/FactomProject/FactomCode/database"
	"github.com/FactomProject/btcd/wire"
	"github.com/davecgh/go-spew/spew"
)

var _ = fmt.Printf

var (
	local_Server *server
	db           database.Db              // database
	inMsgQueue   chan wire.FtmInternalMsg //incoming message queue for factom application messages
	outMsgQueue  chan wire.FtmInternalMsg //outgoing message queue for factom application messages

	inCtlMsgQueue  chan wire.FtmInternalMsg //incoming message queue for factom control messages
	outCtlMsgQueue chan wire.FtmInternalMsg //outgoing message queue for factom control messages
)

// start up Factom queue(s) managers/processors
// this is to be called within the btcd's main code
func factomForkInit(s *server) {
	// tweak some config options
	cfg.DisableCheckpoints = true

	local_Server = s // local copy of our server pointer

	// Write outgoing factom messages into P2P network
	go func() {
		for msg := range outMsgQueue {
			switch msg.(type) {
			case *wire.MsgInt_DirBlock:
				dirBlock, _ := msg.(*wire.MsgInt_DirBlock)
				iv := wire.NewInvVect(wire.InvTypeFactomDirBlock, dirBlock.ShaHash)
				s.RelayInventory(iv, nil)

			case wire.Message:
				wireMsg, _ := msg.(wire.Message)
				//s.BroadcastMessage(wireMsg)
				if ClientOnly {
					//fmt.Println("broadcasting from client.")
					s.BroadcastMessage(wireMsg)
				} else {
					if _, ok := msg.(*wire.MsgAcknowledgement); ok {
						//fmt.Println("broadcasting from server.")
						s.BroadcastMessage(wireMsg)
					}
				}

			case *wire.MsgInt_ReSyncup:
				msg, _ := msg.(*wire.MsgInt_ReSyncup)
				//fmt.Println("MsgInt_ReSyncup: ", spew.Sdump(msg))
				p := local_Server.SyncPeer()
				if p != nil {
					//fmt.Println("start re-syncup: ")
					locator := DirBlockLocatorFromHash(msg.StartHash)
					p.PushGetDirBlocksMsg(locator, &zeroHash)
				}
				//fmt.Println("sync peer is nil, no syncup renewed.")

			default:
				panic(fmt.Sprintf("bad outMsgQueue message received: %v", msg))
			}
		}
	}()

	go func() {
		for msg := range outCtlMsgQueue {

			fmt.Printf("in range outCtlMsgQueue, msg:%+v\n", msg)

			msgEom, _ := msg.(*wire.MsgInt_EOM)

			//			switch msgEom.Command() {
			switch msg.Command() {

			case wire.CmdInt_EOM:

				switch msgEom.EOM_Type {

				case wire.END_MINUTE_10:
					panic(errors.New("unhandled END_MINUTE_10"))

				default:
					panic(errors.New("unhandled EOM type"))
				}

			default:
				panic(errors.New("unhandled CmdInt_EOM"))
			}

			/*
				switch msg.EOM_Type {

				case wire.END_MINUTE_10:
					//util.Trace("EOM10")
				default:
					//util.Trace("default")
				}
			*/

			/*
				switch msg.Command() {
				case factomwire.CmdTx:
					InMsgQueue <- msg //    for testing
					server.blockManager.QueueTx(msg.(*factomwire.MsgTx), nil)
				case factomwire.CmdConfirmation:
					server.blockManager.QueueConf(msg.(*factomwire.MsgConfirmation), nil)

				default:
					inMsgQueue <- msg
					outMsgQueue <- msg
				}
			*/
		}
	}()
}

func Start_btcd(
	ldb database.Db,
	inMsgQ chan wire.FtmInternalMsg,
	outMsgQ chan wire.FtmInternalMsg,
	inCtlMsgQ chan wire.FtmInternalMsg,
	outCtlMsgQ chan wire.FtmInternalMsg,
	user, pass string, clientMode bool) {

	factomdUser = user
	factomdPass = pass

	ClientOnly = clientMode

	if ClientOnly {
		cp.CP.AddUpdate(
			"FactomMode", // tag
			"system",     // Category
			"Factom Mode: Full Node (Client)", // Title
			"", // Message
			0)
		fmt.Println("\n\n>>>>>>>>>>>>>>>>>  CLIENT MODE <<<<<<<<<<<<<<<<<<<<<<<\n\n")
	} else {
		cp.CP.AddUpdate(
			"FactomMode",                    // tag
			"system",                        // Category
			"Factom Mode: Federated Server", // Title
			"", // Message
			0)
		fmt.Println("\n\n>>>>>>>>>>>>>>>>>  SERVER MODE <<<<<<<<<<<<<<<<<<<<<<<\n\n")
	}

	db = ldb

	inMsgQueue = inMsgQ
	outMsgQueue = outMsgQ

	inCtlMsgQueue = inCtlMsgQ
	outCtlMsgQueue = outCtlMsgQ

	// Use all processor cores.
	//runtime.GOMAXPROCS(runtime.NumCPU())

	// Up some limits.
	//if err := limits.SetLimits(); err != nil {
	//	os.Exit(1)
	//}

	// Call serviceMain on Windows to handle running as a service.  When
	// the return isService flag is true, exit now since we ran as a
	// service.  Otherwise, just fall through to normal operation.
	/*if runtime.GOOS == "windows" {
		isService, err := winServiceMain()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		if isService {
			os.Exit(0)
		}
	}
	*/

	// Work around defer not working after os.Exit()
	if err := btcdMain(nil); err != nil {
		os.Exit(1)
	}
}

// Handle factom app imcoming msg
func (p *peer) handleCommitChainMsg(msg *wire.MsgCommitChain) {
	// Add the msg to inbound msg queue
	if !ClientOnly {
		inMsgQueue <- msg
	}
}

// Handle factom app imcoming msg
func (p *peer) handleRevealChainMsg(msg *wire.MsgRevealChain) {
	// Add the msg to inbound msg queue
	//inMsgQueue <- msg
	if !ClientOnly {
		inMsgQueue <- msg
	}
}

// Handle factom app imcoming msg
func (p *peer) handleCommitEntryMsg(msg *wire.MsgCommitEntry) {
	// Add the msg to inbound msg queue
	if !ClientOnly {
		inMsgQueue <- msg
	}
}

// Handle factom app imcoming msg
func (p *peer) handleRevealEntryMsg(msg *wire.MsgRevealEntry) {
	// Add the msg to inbound msg queue
	if !ClientOnly {
		inMsgQueue <- msg
	}
}

// Handle factom app imcoming msg
func (p *peer) handleAcknoledgementMsg(msg *wire.MsgAcknowledgement) {
	// Add the msg to inbound msg queue
	if !ClientOnly {
		inMsgQueue <- msg
	}
}

// returns true if the message should be relayed, false otherwise
func (p *peer) shallRelay(msg interface{}) bool {
	hash, _ := wire.NewShaHashFromStruct(msg)
	iv := wire.NewInvVect(wire.InvTypeFactomRaw, hash)

	if !p.isKnownInventory(iv) {
		p.AddKnownInventory(iv)

		return true
	}

	fmt.Println("******************* SHALL NOT RELAY !!!!!!!!!!! ******************")

	return false
}

// Call FactomRelay to relay/broadcast a Factom message (to your peers).
// The intent is to call this function after certain 'processor' checks been done.
func (p *peer) FactomRelay(msg wire.Message) {

	// broadcast/relay only if hadn't been done for this peer
	if p.shallRelay(msg) {
		//		p.server.BroadcastMessage(msg, p)
		local_Server.BroadcastMessage(msg)
	}
}

/*
// func (pl *ProcessList) AddFtmTxToProcessList(msg wire.Message, msgHash *wire.ShaHash) error {
func fakehook1(msg wire.Message, msgHash *wire.ShaHash) error {
	return nil
}

func factom_PL_hook(tx *btcutil.Tx, label string) error {
	//util.Trace("label= " + label)

	_ = fakehook1(tx.MsgTx(), tx.Sha())

	return nil
}

// for Jack
func global_DeleteMemPoolEntry(hash *wire.ShaHash) {
	// TODO: ensure mutex-protection
}

// check a few btcd-related flags for sanity in our fork
func (b *blockManager) factomChecks() {
	//util.Trace()

	if cfg.AddrIndex {
		panic(errors.New("AddrIndex must be disabled and it is NOT !!!"))
	}

	// DisableCheckpoints should always be set
	if !cfg.DisableCheckpoints {
		panic(errors.New("checkpoints must be disabled and they are NOT !!!"))
	}

	if cfg.RegressionTest || cfg.SimNet || cfg.Generate {
		panic(100)
	}

	if cfg.TestNet3 {
		panic(errors.New("TestNet mode is NOT SUPPORTED (remove the option from the command line or from the .conf file)!"))
	}

	//util.Trace()
}

// feed all incoming Txs to the inner Factom code (for Jack)
// TODO: do this after proper mempool/orphanpool/validity triangulation & checks
func factomIngressTx_hook(tx *wire.MsgTx) error {
	//util.Trace()

	ecmap := make(map[wire.ShaHash]uint64)

	txid, _ := tx.TxSha()
	hash, _ := wire.NewShaHash(wire.Sha256(txid.Bytes()))

	ecmap[*hash] = 1

	// Use wallet's public key to add EC??
	sig := wallet.SignData(nil)
	hash2 := new(wire.ShaHash)
	hash2.SetBytes((*sig.Pub.Key)[:])

	ecmap[*hash2] = 100

	txHash, _ := tx.TxSha()
	fo := &wire.MsgInt_FactoidObj{tx, &txHash, ecmap}

	fmt.Println("ecmap len =", len(ecmap))

	inMsgQueue <- fo

	return nil
}

func factomIngressBlock_hook(hash *wire.ShaHash) error {
	//util.Trace(fmt.Sprintf("hash: %s", hash))

	fbo := &wire.MsgInt_FactoidBlock{
		ShaHash: *hash}

	doneFBlockQueue <- fbo

	return nil
}
*/

/*
func ExtractPkScriptAddrs(pkScript []byte, chainParams *chaincfg.Params) ([]btcutil.Address, int, error) {
	oldWay := false

	//util.Trace("bytes= " + spew.Sdump(pkScript))

	var addrs []btcutil.Address
	var requiredSigs int

	if oldWay {
		// A pay-to-pubkey script is of the form:
		//  <pubkey> OP_CHECKSIG
		// Therefore the pubkey is the first item on the stack.
		// Skip the pubkey if it's invalid for some reason.
		requiredSigs = 1
		addr, err := btcutil.NewAddressPubKey(pkScript, chainParams)
		if err == nil {
			addrs = append(addrs, addr)
		}

	} else {

		// A pay-to-pubkey-hash script is of the form:
		//  OP_DUP OP_HASH160 <hash> OP_EQUALVERIFY OP_CHECKSIG
		// Therefore the pubkey hash is the 3rd item on the stack.
		// Skip the pubkey hash if it's invalid for some reason.
		requiredSigs = 1
		//	addr, err := btcutil.NewAddressPubKeyHash(pops[2].data,
		addr, err := btcutil.NewAddressPubKeyHash(pkScript, chainParams)
		if err == nil {
			addrs = append(addrs, addr)
		}
	}

	//util.Trace("addrs= " + spew.Sdump(addrs))

	return addrs, requiredSigs, nil
}

// PayToAddrScript creates a new script to pay a transaction output to a the
// specified address.
func PayToAddrScript(addr btcutil.Address) ([]byte, error) {
	scrAddr := addr.ScriptAddress()

	//util.Trace("scrAddr= " + spew.Sdump(scrAddr))

	return scrAddr, nil

	//	panic(errors.New("PayToAddrScript -- NOT IMPLEMENTED !!!"))

	//	return payToPubKeyHashScript(addr.ScriptAddress())
}
*/
<|endoftext|>


<commit_before>package henchman

import (
	"fmt"
	"gopkg.in/yaml.v1"
	"strings"
)

type TaskVars map[string]interface{}

// A plan is a collection of tasks.
// All the tasks are executed serially, although the same plan
// is run concurrently on multiple machines
type Plan struct {
	Hosts []string
	Tasks []Task
	Vars  *TaskVars
	Name  string

	report map[string]string
	tasks  []map[string]string `yaml:"tasks"`
}

func mergeMap(source *TaskVars, destination *TaskVars) {
	src := *source
	dst := *destination
	for variable, value := range src {
		dst[variable] = value
	}
}

// Returns a new plan with a collection of tasks. The planBuf should be a valid
// 'yaml' representation. Additionally, this function also takes in any variable
// overrides that takes precedence over the variables present in the plan.
func NewPlanFromYAML(planBuf []byte, overrides *TaskVars) (*Plan, error) {
	plan := Plan{}
	err := yaml.Unmarshal(planBuf, &plan)
	if plan.Vars == nil {
		_vars := make(TaskVars)
		plan.Vars = &_vars
	}
	if err != nil {
		return nil, err
	}
	if overrides != nil {
		mergeMap(overrides, plan.Vars)
		if hosts, present := (*overrides)["hosts"]; present {
			plan.Hosts = strings.Split(hosts.(string), ",")
		}

	}
	plan.report = make(map[string]string)
	plan.parseTasks()
	return &plan, nil
}

func (plan *Plan) parseTasks() {
	for _, t := range plan.tasks {
		task := Task{}
		task.Name = t["name"]
		task.Action = t["action"]
		_, present := t["ignore_errors"]
		task.IgnoreErrors = bool(present)
		_, present = t["local"]
		task.LocalAction = bool(present)
		plan.Tasks = append(plan.Tasks, task)
	}
}

// Prints the summary of the Plan execution across all the hosts
func (plan *Plan) PrintReport() {
	var counts = make(map[string]int)

	total := len(plan.Tasks) * len(plan.Hosts)
	attempted := len(plan.report)
	skipped := total - attempted

	counts["skipped"] = skipped
	for _, status := range plan.report {
		_, present := counts[status]
		if !present {
			counts[status] = 1
		} else {
			counts[status]++
		}
	}
	fmt.Println()
	fmt.Println("---")
	fmt.Printf("Plan Report: %s\n", plan.Name)
	fmt.Println()
	for k, v := range counts {
		fmt.Printf("%s (all hosts):\t%d\n", k, v)
	}
	fmt.Println()
	fmt.Printf("Tasks total (all hosts):\t%d\n",
		len(plan.Tasks)*len(plan.Hosts))
	fmt.Printf("Tasks attempted (all hosts):\t%d\n", len(plan.report))
}

// Mark a given task's status.
// NOTE: Skipped tasks are not tracked here.
func (plan *Plan) SaveStatus(task *Task, status string) {
	plan.report[task.Id] = status
}

func (plan *Plan) String() string {
	status := "Plan '%s':"
	for k, v := range plan.report {
		status = status + fmt.Sprintf(" %s - %s;", k, v)
	}
	return status
}
<commit_msg>Fix String() for Plan<commit_after>package henchman

import (
	"fmt"
	"gopkg.in/yaml.v1"
	"strings"
)

type TaskVars map[string]interface{}

// A plan is a collection of tasks.
// All the tasks are executed serially, although the same plan
// is run concurrently on multiple machines
type Plan struct {
	Hosts []string
	Tasks []Task
	Vars  *TaskVars
	Name  string

	report map[string]string
	tasks  []map[string]string `yaml:"tasks"`
}

func mergeMap(source *TaskVars, destination *TaskVars) {
	src := *source
	dst := *destination
	for variable, value := range src {
		dst[variable] = value
	}
}

// Returns a new plan with a collection of tasks. The planBuf should be a valid
// 'yaml' representation. Additionally, this function also takes in any variable
// overrides that takes precedence over the variables present in the plan.
func NewPlanFromYAML(planBuf []byte, overrides *TaskVars) (*Plan, error) {
	plan := Plan{}
	err := yaml.Unmarshal(planBuf, &plan)
	if plan.Vars == nil {
		_vars := make(TaskVars)
		plan.Vars = &_vars
	}
	if err != nil {
		return nil, err
	}
	if overrides != nil {
		mergeMap(overrides, plan.Vars)
		if hosts, present := (*overrides)["hosts"]; present {
			plan.Hosts = strings.Split(hosts.(string), ",")
		}

	}
	plan.report = make(map[string]string)
	plan.parseTasks()
	return &plan, nil
}

func (plan *Plan) parseTasks() {
	for _, t := range plan.tasks {
		task := Task{}
		task.Name = t["name"]
		task.Action = t["action"]
		_, present := t["ignore_errors"]
		task.IgnoreErrors = bool(present)
		_, present = t["local"]
		task.LocalAction = bool(present)
		plan.Tasks = append(plan.Tasks, task)
	}
}

// Prints the summary of the Plan execution across all the hosts
func (plan *Plan) PrintReport() {
	var counts = make(map[string]int)

	total := len(plan.Tasks) * len(plan.Hosts)
	attempted := len(plan.report)
	skipped := total - attempted

	counts["skipped"] = skipped
	for _, status := range plan.report {
		_, present := counts[status]
		if !present {
			counts[status] = 1
		} else {
			counts[status]++
		}
	}
	fmt.Println()
	fmt.Println("---")
	fmt.Printf("Plan Report: %s\n", plan.Name)
	fmt.Println()
	for k, v := range counts {
		fmt.Printf("%s (all hosts):\t%d\n", k, v)
	}
	fmt.Println()
	fmt.Printf("Tasks total (all hosts):\t%d\n",
		len(plan.Tasks)*len(plan.Hosts))
	fmt.Printf("Tasks attempted (all hosts):\t%d\n", len(plan.report))
}

// Mark a given task's status.
// NOTE: Skipped tasks are not tracked here.
func (plan *Plan) SaveStatus(task *Task, status string) {
	plan.report[task.Id] = status
}

func (plan *Plan) String() string {
	status := fmt.Sprintf("Plan '%s' with %d tasks:", plan.Name, len(plan.Tasks))
	for k, v := range plan.report {
		status = status + fmt.Sprintf(" %s - %s;", k, v)
	}
	return status
}
<|endoftext|>