<commit_before>package ingester

import (
	"fmt"
	"net/http"
	"time"

	"golang.org/x/net/context"

	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/weaveworks/common/user"

	"github.com/cortexproject/cortex/pkg/chunk"
	"github.com/cortexproject/cortex/pkg/util"
	"github.com/grafana/loki/pkg/chunkenc"
	loki_util "github.com/grafana/loki/pkg/util"
)

var (
	chunkUtilization = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "loki",
		Name:      "ingester_chunk_utilization",
		Help:      "Distribution of stored chunk utilization (when stored).",
		Buckets:   prometheus.LinearBuckets(0, 0.2, 6),
	})
	memoryChunks = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "loki",
		Name:      "ingester_memory_chunks",
		Help:      "The total number of chunks in memory.",
	})
	chunkEntries = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "loki",
		Name:      "ingester_chunk_entries",
		Help:      "Distribution of stored lines per chunk (when stored).",
		Buckets:   prometheus.ExponentialBuckets(200, 2, 9), // biggest bucket is 200*2^(9-1) = 51200
	})
	chunkSize = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "loki",
		Name:      "ingester_chunk_size_bytes",
		Help:      "Distribution of stored chunk sizes (when stored).",
		Buckets:   prometheus.ExponentialBuckets(10000, 2, 7), // biggest bucket is 10000*2^(7-1) = 640000 (~640KB)
	})
	chunkCompressionRatio = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "loki",
		Name:      "ingester_chunk_compression_ratio",
		Help:      "Compression ratio of chunks (when stored).",
		Buckets:   prometheus.LinearBuckets(.75, 2, 10),
	})
	chunksPerTenant = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: "loki",
		Name:      "ingester_chunks_stored_total",
		Help:      "Total stored chunks per tenant.",
	}, []string{"tenant"})
	chunkSizePerTenant = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: "loki",
		Name:      "ingester_chunk_stored_bytes_total",
		Help:      "Total bytes stored in chunks per tenant.",
	}, []string{"tenant"})
	chunkAge = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "loki",
		Name:      "ingester_chunk_age_seconds",
		Help:      "Distribution of chunk ages (when stored).",
		// with default settings chunks should flush between 5 min and 12 hours
		// so buckets at 1min, 5min, 10min, 30min, 1hr, 2hr, 4hr, 10hr, 12hr, 16hr
		Buckets: []float64{60, 300, 600, 1800, 3600, 7200, 14400, 36000, 43200, 57600},
	})
	chunkEncodeTime = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "loki",
		Name:      "ingester_chunk_encode_time_seconds",
		Help:      "Distribution of chunk encode times.",
		// 10ms to 10s.
		Buckets: prometheus.ExponentialBuckets(0.01, 4, 6),
	})
)

const (
	// Backoff for retrying 'immediate' flushes. Only counts for queue
	// position, not wallclock time.
	flushBackoff = 1 * time.Second

	nameLabel = "__name__"
	logsValue = "logs"
)

// Flush triggers a flush of all the chunks and closes the flush queues.
// Called from the Lifecycler as part of the ingester shutdown.
func (i *Ingester) Flush() {
	i.sweepUsers(true)

	// Close the flush queues, to unblock waiting workers.
	for _, flushQueue := range i.flushQueues {
		flushQueue.Close()
	}

	i.flushQueuesDone.Wait()
}

// FlushHandler triggers a flush of all in memory chunks.  Mainly used for
// local testing.
func (i *Ingester) FlushHandler(w http.ResponseWriter, _ *http.Request) {
	i.sweepUsers(true)
	w.WriteHeader(http.StatusNoContent)
}

type flushOp struct {
	from      model.Time
	userID    string
	fp        model.Fingerprint
	immediate bool
}

func (o *flushOp) Key() string {
	return fmt.Sprintf("%s-%s-%v", o.userID, o.fp, o.immediate)
}

func (o *flushOp) Priority() int64 {
	return -int64(o.from)
}

// sweepUsers periodically schedules series for flushing and garbage collects users with no series
func (i *Ingester) sweepUsers(immediate bool) {
	instances := i.getInstances()

	for _, instance := range instances {
		i.sweepInstance(instance, immediate)
	}
}

func (i *Ingester) sweepInstance(instance *instance, immediate bool) {
	instance.streamsMtx.Lock()
	defer instance.streamsMtx.Unlock()

	for _, stream := range instance.streams {
		i.sweepStream(instance, stream, immediate)
		i.removeFlushedChunks(instance, stream)
	}
}

func (i *Ingester) sweepStream(instance *instance, stream *stream, immediate bool) {
	if len(stream.chunks) == 0 {
		return
	}

	lastChunk := stream.chunks[len(stream.chunks)-1]
	if len(stream.chunks) == 1 && time.Since(lastChunk.lastUpdated) < i.cfg.MaxChunkIdle && !immediate {
		return
	}

	flushQueueIndex := int(uint64(stream.fp) % uint64(i.cfg.ConcurrentFlushes))
	firstTime, _ := stream.chunks[0].chunk.Bounds()
	i.flushQueues[flushQueueIndex].Enqueue(&flushOp{
		model.TimeFromUnixNano(firstTime.UnixNano()), instance.instanceID,
		stream.fp, immediate,
	})
}

func (i *Ingester) flushLoop(j int) {
	defer func() {
		level.Debug(util.Logger).Log("msg", "Ingester.flushLoop() exited")
		i.flushQueuesDone.Done()
	}()

	for {
		o := i.flushQueues[j].Dequeue()
		if o == nil {
			return
		}
		op := o.(*flushOp)

		level.Debug(util.Logger).Log("msg", "flushing stream", "userid", op.userID, "fp", op.fp, "immediate", op.immediate)

		err := i.flushUserSeries(op.userID, op.fp, op.immediate)
		if err != nil {
			level.Error(util.WithUserID(op.userID, util.Logger)).Log("msg", "failed to flush user", "err", err)
		}

		// If we're exiting & we failed to flush, put the failed operation
		// back in the queue at a later point.
		if op.immediate && err != nil {
			op.from = op.from.Add(flushBackoff)
			i.flushQueues[j].Enqueue(op)
		}
	}
}

func (i *Ingester) flushUserSeries(userID string, fp model.Fingerprint, immediate bool) error {
	instance, ok := i.getInstanceByID(userID)
	if !ok {
		return nil
	}

	chunks, labels := i.collectChunksToFlush(instance, fp, immediate)
	if len(chunks) < 1 {
		return nil
	}

	ctx := user.InjectOrgID(context.Background(), userID)
	ctx, cancel := context.WithTimeout(ctx, i.cfg.FlushOpTimeout)
	defer cancel()
	err := i.flushChunks(ctx, fp, labels, chunks)
	if err != nil {
		return err
	}

	instance.streamsMtx.Lock()
	for _, chunk := range chunks {
		chunk.flushed = time.Now()
	}
	instance.streamsMtx.Unlock()
	return nil
}

func (i *Ingester) collectChunksToFlush(instance *instance, fp model.Fingerprint, immediate bool) ([]*chunkDesc, labels.Labels) {
	instance.streamsMtx.Lock()
	defer instance.streamsMtx.Unlock()

	stream, ok := instance.streams[fp]
	if !ok {
		return nil, nil
	}

	var result []*chunkDesc
	for j := range stream.chunks {
		if immediate || i.shouldFlushChunk(&stream.chunks[j]) {
			// Ensure no more writes happen to this chunk.
			if !stream.chunks[j].closed {
				stream.chunks[j].closed = true
			}
			// Flush this chunk if it hasn't already been successfully flushed.
			if stream.chunks[j].flushed.IsZero() {
				result = append(result, &stream.chunks[j])
			}
		}
	}
	return result, stream.labels
}

func (i *Ingester) shouldFlushChunk(chunk *chunkDesc) bool {
	// Append should close the chunk when the a new one is added.
	if chunk.closed {
		return true
	}

	if time.Since(chunk.lastUpdated) > i.cfg.MaxChunkIdle {
		chunk.closed = true
		return true
	}

	return false
}

func (i *Ingester) removeFlushedChunks(instance *instance, stream *stream) {
	now := time.Now()

	prevNumChunks := len(stream.chunks)
	for len(stream.chunks) > 0 {
		if stream.chunks[0].flushed.IsZero() || now.Sub(stream.chunks[0].flushed) < i.cfg.RetainPeriod {
			break
		}

		stream.chunks[0].chunk = nil // erase reference so the chunk can be garbage-collected
		stream.chunks = stream.chunks[1:]
	}
	memoryChunks.Sub(float64(prevNumChunks - len(stream.chunks)))

	if len(stream.chunks) == 0 {
		delete(instance.streams, stream.fp)
		instance.index.Delete(stream.labels, stream.fp)
		instance.streamsRemovedTotal.Inc()
		memoryStreams.Dec()
	}
}

func (i *Ingester) flushChunks(ctx context.Context, fp model.Fingerprint, labelPairs labels.Labels, cs []*chunkDesc) error {
	userID, err := user.ExtractOrgID(ctx)
	if err != nil {
		return err
	}

	labelsBuilder := labels.NewBuilder(labelPairs)
	labelsBuilder.Set(nameLabel, logsValue)
	metric := labelsBuilder.Labels()

	wireChunks := make([]chunk.Chunk, 0, len(cs))
	for _, c := range cs {
		firstTime, lastTime := loki_util.RoundToMilliseconds(c.chunk.Bounds())
		c := chunk.NewChunk(
			userID, fp, metric,
			chunkenc.NewFacade(c.chunk),
			firstTime,
			lastTime,
		)

		start := time.Now()
		if err := c.Encode(); err != nil {
			return err
		}
		chunkEncodeTime.Observe(time.Since(start).Seconds())
		wireChunks = append(wireChunks, c)
	}

	if err := i.store.Put(ctx, wireChunks); err != nil {
		return err
	}

	// Record statistsics only when actual put request did not return error.
	sizePerTenant := chunkSizePerTenant.WithLabelValues(userID)
	countPerTenant := chunksPerTenant.WithLabelValues(userID)
	for i, wc := range wireChunks {
		numEntries := cs[i].chunk.Size()
		byt, err := wc.Encoded()
		if err != nil {
			continue
		}

		compressedSize := float64(len(byt))
		uncompressedSize, ok := chunkenc.UncompressedSize(wc.Data)

		if ok && compressedSize > 0 {
			chunkCompressionRatio.Observe(float64(uncompressedSize) / compressedSize)
		}

		chunkUtilization.Observe(wc.Data.Utilization())
		chunkEntries.Observe(float64(numEntries))
		chunkSize.Observe(compressedSize)
		sizePerTenant.Add(compressedSize)
		countPerTenant.Inc()
		firstTime, _ := cs[i].chunk.Bounds()
		chunkAge.Observe(time.Since(firstTime).Seconds())
	}

	return nil
}
<commit_msg>change the chunk size histogram to allow for bigger buckets (#1409)<commit_after>package ingester

import (
	"fmt"
	"net/http"
	"time"

	"golang.org/x/net/context"

	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/weaveworks/common/user"

	"github.com/cortexproject/cortex/pkg/chunk"
	"github.com/cortexproject/cortex/pkg/util"
	"github.com/grafana/loki/pkg/chunkenc"
	loki_util "github.com/grafana/loki/pkg/util"
)

var (
	chunkUtilization = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "loki",
		Name:      "ingester_chunk_utilization",
		Help:      "Distribution of stored chunk utilization (when stored).",
		Buckets:   prometheus.LinearBuckets(0, 0.2, 6),
	})
	memoryChunks = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "loki",
		Name:      "ingester_memory_chunks",
		Help:      "The total number of chunks in memory.",
	})
	chunkEntries = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "loki",
		Name:      "ingester_chunk_entries",
		Help:      "Distribution of stored lines per chunk (when stored).",
		Buckets:   prometheus.ExponentialBuckets(200, 2, 9), // biggest bucket is 200*2^(9-1) = 51200
	})
	chunkSize = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "loki",
		Name:      "ingester_chunk_size_bytes",
		Help:      "Distribution of stored chunk sizes (when stored).",
		Buckets:   prometheus.ExponentialBuckets(20000, 2, 10), // biggest bucket is 20000*2^(10-1) = 10,240,000 (~10.2MB)
	})
	chunkCompressionRatio = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "loki",
		Name:      "ingester_chunk_compression_ratio",
		Help:      "Compression ratio of chunks (when stored).",
		Buckets:   prometheus.LinearBuckets(.75, 2, 10),
	})
	chunksPerTenant = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: "loki",
		Name:      "ingester_chunks_stored_total",
		Help:      "Total stored chunks per tenant.",
	}, []string{"tenant"})
	chunkSizePerTenant = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: "loki",
		Name:      "ingester_chunk_stored_bytes_total",
		Help:      "Total bytes stored in chunks per tenant.",
	}, []string{"tenant"})
	chunkAge = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "loki",
		Name:      "ingester_chunk_age_seconds",
		Help:      "Distribution of chunk ages (when stored).",
		// with default settings chunks should flush between 5 min and 12 hours
		// so buckets at 1min, 5min, 10min, 30min, 1hr, 2hr, 4hr, 10hr, 12hr, 16hr
		Buckets: []float64{60, 300, 600, 1800, 3600, 7200, 14400, 36000, 43200, 57600},
	})
	chunkEncodeTime = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "loki",
		Name:      "ingester_chunk_encode_time_seconds",
		Help:      "Distribution of chunk encode times.",
		// 10ms to 10s.
		Buckets: prometheus.ExponentialBuckets(0.01, 4, 6),
	})
)

const (
	// Backoff for retrying 'immediate' flushes. Only counts for queue
	// position, not wallclock time.
	flushBackoff = 1 * time.Second

	nameLabel = "__name__"
	logsValue = "logs"
)

// Flush triggers a flush of all the chunks and closes the flush queues.
// Called from the Lifecycler as part of the ingester shutdown.
func (i *Ingester) Flush() {
	i.sweepUsers(true)

	// Close the flush queues, to unblock waiting workers.
	for _, flushQueue := range i.flushQueues {
		flushQueue.Close()
	}

	i.flushQueuesDone.Wait()
}

// FlushHandler triggers a flush of all in memory chunks.  Mainly used for
// local testing.
func (i *Ingester) FlushHandler(w http.ResponseWriter, _ *http.Request) {
	i.sweepUsers(true)
	w.WriteHeader(http.StatusNoContent)
}

type flushOp struct {
	from      model.Time
	userID    string
	fp        model.Fingerprint
	immediate bool
}

func (o *flushOp) Key() string {
	return fmt.Sprintf("%s-%s-%v", o.userID, o.fp, o.immediate)
}

func (o *flushOp) Priority() int64 {
	return -int64(o.from)
}

// sweepUsers periodically schedules series for flushing and garbage collects users with no series
func (i *Ingester) sweepUsers(immediate bool) {
	instances := i.getInstances()

	for _, instance := range instances {
		i.sweepInstance(instance, immediate)
	}
}

func (i *Ingester) sweepInstance(instance *instance, immediate bool) {
	instance.streamsMtx.Lock()
	defer instance.streamsMtx.Unlock()

	for _, stream := range instance.streams {
		i.sweepStream(instance, stream, immediate)
		i.removeFlushedChunks(instance, stream)
	}
}

func (i *Ingester) sweepStream(instance *instance, stream *stream, immediate bool) {
	if len(stream.chunks) == 0 {
		return
	}

	lastChunk := stream.chunks[len(stream.chunks)-1]
	if len(stream.chunks) == 1 && time.Since(lastChunk.lastUpdated) < i.cfg.MaxChunkIdle && !immediate {
		return
	}

	flushQueueIndex := int(uint64(stream.fp) % uint64(i.cfg.ConcurrentFlushes))
	firstTime, _ := stream.chunks[0].chunk.Bounds()
	i.flushQueues[flushQueueIndex].Enqueue(&flushOp{
		model.TimeFromUnixNano(firstTime.UnixNano()), instance.instanceID,
		stream.fp, immediate,
	})
}

func (i *Ingester) flushLoop(j int) {
	defer func() {
		level.Debug(util.Logger).Log("msg", "Ingester.flushLoop() exited")
		i.flushQueuesDone.Done()
	}()

	for {
		o := i.flushQueues[j].Dequeue()
		if o == nil {
			return
		}
		op := o.(*flushOp)

		level.Debug(util.Logger).Log("msg", "flushing stream", "userid", op.userID, "fp", op.fp, "immediate", op.immediate)

		err := i.flushUserSeries(op.userID, op.fp, op.immediate)
		if err != nil {
			level.Error(util.WithUserID(op.userID, util.Logger)).Log("msg", "failed to flush user", "err", err)
		}

		// If we're exiting & we failed to flush, put the failed operation
		// back in the queue at a later point.
		if op.immediate && err != nil {
			op.from = op.from.Add(flushBackoff)
			i.flushQueues[j].Enqueue(op)
		}
	}
}

func (i *Ingester) flushUserSeries(userID string, fp model.Fingerprint, immediate bool) error {
	instance, ok := i.getInstanceByID(userID)
	if !ok {
		return nil
	}

	chunks, labels := i.collectChunksToFlush(instance, fp, immediate)
	if len(chunks) < 1 {
		return nil
	}

	ctx := user.InjectOrgID(context.Background(), userID)
	ctx, cancel := context.WithTimeout(ctx, i.cfg.FlushOpTimeout)
	defer cancel()
	err := i.flushChunks(ctx, fp, labels, chunks)
	if err != nil {
		return err
	}

	instance.streamsMtx.Lock()
	for _, chunk := range chunks {
		chunk.flushed = time.Now()
	}
	instance.streamsMtx.Unlock()
	return nil
}

func (i *Ingester) collectChunksToFlush(instance *instance, fp model.Fingerprint, immediate bool) ([]*chunkDesc, labels.Labels) {
	instance.streamsMtx.Lock()
	defer instance.streamsMtx.Unlock()

	stream, ok := instance.streams[fp]
	if !ok {
		return nil, nil
	}

	var result []*chunkDesc
	for j := range stream.chunks {
		if immediate || i.shouldFlushChunk(&stream.chunks[j]) {
			// Ensure no more writes happen to this chunk.
			if !stream.chunks[j].closed {
				stream.chunks[j].closed = true
			}
			// Flush this chunk if it hasn't already been successfully flushed.
			if stream.chunks[j].flushed.IsZero() {
				result = append(result, &stream.chunks[j])
			}
		}
	}
	return result, stream.labels
}

func (i *Ingester) shouldFlushChunk(chunk *chunkDesc) bool {
	// Append should close the chunk when the a new one is added.
	if chunk.closed {
		return true
	}

	if time.Since(chunk.lastUpdated) > i.cfg.MaxChunkIdle {
		chunk.closed = true
		return true
	}

	return false
}

func (i *Ingester) removeFlushedChunks(instance *instance, stream *stream) {
	now := time.Now()

	prevNumChunks := len(stream.chunks)
	for len(stream.chunks) > 0 {
		if stream.chunks[0].flushed.IsZero() || now.Sub(stream.chunks[0].flushed) < i.cfg.RetainPeriod {
			break
		}

		stream.chunks[0].chunk = nil // erase reference so the chunk can be garbage-collected
		stream.chunks = stream.chunks[1:]
	}
	memoryChunks.Sub(float64(prevNumChunks - len(stream.chunks)))

	if len(stream.chunks) == 0 {
		delete(instance.streams, stream.fp)
		instance.index.Delete(stream.labels, stream.fp)
		instance.streamsRemovedTotal.Inc()
		memoryStreams.Dec()
	}
}

func (i *Ingester) flushChunks(ctx context.Context, fp model.Fingerprint, labelPairs labels.Labels, cs []*chunkDesc) error {
	userID, err := user.ExtractOrgID(ctx)
	if err != nil {
		return err
	}

	labelsBuilder := labels.NewBuilder(labelPairs)
	labelsBuilder.Set(nameLabel, logsValue)
	metric := labelsBuilder.Labels()

	wireChunks := make([]chunk.Chunk, 0, len(cs))
	for _, c := range cs {
		firstTime, lastTime := loki_util.RoundToMilliseconds(c.chunk.Bounds())
		c := chunk.NewChunk(
			userID, fp, metric,
			chunkenc.NewFacade(c.chunk),
			firstTime,
			lastTime,
		)

		start := time.Now()
		if err := c.Encode(); err != nil {
			return err
		}
		chunkEncodeTime.Observe(time.Since(start).Seconds())
		wireChunks = append(wireChunks, c)
	}

	if err := i.store.Put(ctx, wireChunks); err != nil {
		return err
	}

	// Record statistsics only when actual put request did not return error.
	sizePerTenant := chunkSizePerTenant.WithLabelValues(userID)
	countPerTenant := chunksPerTenant.WithLabelValues(userID)
	for i, wc := range wireChunks {
		numEntries := cs[i].chunk.Size()
		byt, err := wc.Encoded()
		if err != nil {
			continue
		}

		compressedSize := float64(len(byt))
		uncompressedSize, ok := chunkenc.UncompressedSize(wc.Data)

		if ok && compressedSize > 0 {
			chunkCompressionRatio.Observe(float64(uncompressedSize) / compressedSize)
		}

		chunkUtilization.Observe(wc.Data.Utilization())
		chunkEntries.Observe(float64(numEntries))
		chunkSize.Observe(compressedSize)
		sizePerTenant.Add(compressedSize)
		countPerTenant.Inc()
		firstTime, _ := cs[i].chunk.Bounds()
		chunkAge.Observe(time.Since(firstTime).Seconds())
	}

	return nil
}
<|endoftext|>


<commit_before>package core

import (
	"bufio"
	"math/rand"
	"os"
	"path/filepath"
	"strings"

	"github.com/itsabot/abot/core/log"
	"github.com/itsabot/abot/shared/datatypes"
	"github.com/itsabot/abot/shared/nlp"
)

// Classifier is a set of common english word stems unique among their
// Structured Input Types. This enables extremely fast constant-time O(1)
// lookups of stems to their SITs with high accuracy and no training
// requirements. It consumes just a few MB in memory.
type Classifier map[string]struct{}

// ClassifyTokens builds a StructuredInput from a tokenized sentence.
func (c Classifier) ClassifyTokens(tokens []string) *nlp.StructuredInput {
	var s nlp.StructuredInput
	for _, t := range tokens {
		t = strings.ToLower(t)
		_, exists := c["C"+t]
		if exists {
			s.Commands = append(s.Commands, t)
		}
		_, exists = c["O"+t]
		if exists {
			s.Objects = append(s.Objects, t)
		}
	}
	return &s
}

// buildClassifier prepares the Named Entity Recognizer (NER) to find Commands
// and Objects using a simple dictionary lookup. This has the benefit of high
// speed--constant time, O(1)--with insignificant memory use and high accuracy
// given false positives (marking something as both a Command and an Object when
// it's really acting as an Object) are OK. Utlimately this should be a first
// pass, and any double-marked words should be passed through something like an
// n-gram Bayesian filter to determine the correct part of speech within its
// context in the sentence.
func buildClassifier() (Classifier, error) {
	ner := Classifier{}
	p := filepath.Join(os.Getenv("GOPATH"), "src", "github.com", "itsabot",
		"abot", "data", "ner")
	fi, err := os.Open(filepath.Join(p, "nouns.txt"))
	if err != nil {
		return ner, err
	}
	scanner := bufio.NewScanner(fi)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		ner["O"+scanner.Text()] = struct{}{}
	}
	if err = fi.Close(); err != nil {
		return ner, err
	}
	fi, err = os.Open(filepath.Join(p, "verbs.txt"))
	if err != nil {
		return ner, err
	}
	scanner = bufio.NewScanner(fi)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		ner["C"+scanner.Text()] = struct{}{}
	}
	if err = fi.Close(); err != nil {
		return ner, err
	}
	fi, err = os.Open(filepath.Join(p, "adjectives.txt"))
	if err != nil {
		return ner, err
	}
	scanner = bufio.NewScanner(fi)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		ner["O"+scanner.Text()] = struct{}{}
	}
	if err = fi.Close(); err != nil {
		return ner, err
	}
	fi, err = os.Open(filepath.Join(p, "adverbs.txt"))
	if err != nil {
		return ner, err
	}
	scanner = bufio.NewScanner(fi)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		ner["O"+scanner.Text()] = struct{}{}
	}
	if err = fi.Close(); err != nil {
		return ner, err
	}
	return ner, nil
}

// buildOffensiveMap creates a map of offensive terms for which Abot will refuse
// to respond. This helps ensure that users are somewhat respectful to Abot and
// her human trainers, since sentences caught by the OffensiveMap are rejected
// before any human ever sees them.
func buildOffensiveMap() (map[string]struct{}, error) {
	o := map[string]struct{}{}
	p := filepath.Join(os.Getenv("GOPATH"), "src", "github.com", "itsabot",
		"abot", "data", "offensive.txt")
	fi, err := os.Open(p)
	if err != nil {
		return o, err
	}
	scanner := bufio.NewScanner(fi)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		o[scanner.Text()] = struct{}{}
	}
	err = fi.Close()
	return o, err
}

// RespondWithNicety replies to niceties that humans use, but Abot can ignore.
// Words like "Thank you" are not necessary for a robot, but it's important Abot
// respond correctly nonetheless. The returned bool specifies whether a response
// is necessary, and the returned string is the response, if any.
func RespondWithNicety(in *dt.Msg) (responseNecessary bool, response string) {
	for _, w := range in.Stems {
		// Since these are stems, some of them look incorrectly spelled.
		// Needless to say, these are the correct Porter2 Snowball stems
		switch w {
		case "thank":
			return true, "You're welcome!"
		case "cool", "sweet", "awesom", "neat", "perfect":
			return false, ""
		case "sorri":
			return true, "That's OK. I forgive you."
		case "hi", "hello":
			return true, "Hi there. :)"
		}
	}
	return true, ""
}

// RespondWithOffense is a one-off function to respond to rude user language by
// refusing to process the command.
func RespondWithOffense(off map[string]struct{}, in *dt.Msg) string {
	for _, w := range in.Stems {
		_, offensive := off[w]
		if offensive {
			return "I'm sorry, but I don't respond to rude language."
		}
	}
	return ""
}

// ConfusedLang returns a randomized response signalling that Abot is confused
// or could not understand the user's request.
func ConfusedLang() string {
	n := rand.Intn(4)
	switch n {
	case 0:
		return "I'm not sure I understand you."
	case 1:
		return "I'm sorry, I don't understand that."
	case 2:
		return "Uh, what are you telling me to do?"
	case 3:
		return "What should I do?"
	}
	log.Debug("confused failed to return a response")
	return ""
}
<commit_msg>Fix Heroku deploy<commit_after>package core

import (
	"bufio"
	"math/rand"
	"os"
	"path/filepath"
	"strings"

	"github.com/itsabot/abot/core/log"
	"github.com/itsabot/abot/shared/datatypes"
	"github.com/itsabot/abot/shared/nlp"
)

// Classifier is a set of common english word stems unique among their
// Structured Input Types. This enables extremely fast constant-time O(1)
// lookups of stems to their SITs with high accuracy and no training
// requirements. It consumes just a few MB in memory.
type Classifier map[string]struct{}

// ClassifyTokens builds a StructuredInput from a tokenized sentence.
func (c Classifier) ClassifyTokens(tokens []string) *nlp.StructuredInput {
	var s nlp.StructuredInput
	for _, t := range tokens {
		t = strings.ToLower(t)
		_, exists := c["C"+t]
		if exists {
			s.Commands = append(s.Commands, t)
		}
		_, exists = c["O"+t]
		if exists {
			s.Objects = append(s.Objects, t)
		}
	}
	return &s
}

// buildClassifier prepares the Named Entity Recognizer (NER) to find Commands
// and Objects using a simple dictionary lookup. This has the benefit of high
// speed--constant time, O(1)--with insignificant memory use and high accuracy
// given false positives (marking something as both a Command and an Object when
// it's really acting as an Object) are OK. Utlimately this should be a first
// pass, and any double-marked words should be passed through something like an
// n-gram Bayesian filter to determine the correct part of speech within its
// context in the sentence.
func buildClassifier() (Classifier, error) {
	ner := Classifier{}
	p := filepath.Join("data", "ner")
	fi, err := os.Open(filepath.Join(p, "nouns.txt"))
	if err != nil {
		return ner, err
	}
	scanner := bufio.NewScanner(fi)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		ner["O"+scanner.Text()] = struct{}{}
	}
	if err = fi.Close(); err != nil {
		return ner, err
	}
	fi, err = os.Open(filepath.Join(p, "verbs.txt"))
	if err != nil {
		return ner, err
	}
	scanner = bufio.NewScanner(fi)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		ner["C"+scanner.Text()] = struct{}{}
	}
	if err = fi.Close(); err != nil {
		return ner, err
	}
	fi, err = os.Open(filepath.Join(p, "adjectives.txt"))
	if err != nil {
		return ner, err
	}
	scanner = bufio.NewScanner(fi)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		ner["O"+scanner.Text()] = struct{}{}
	}
	if err = fi.Close(); err != nil {
		return ner, err
	}
	fi, err = os.Open(filepath.Join(p, "adverbs.txt"))
	if err != nil {
		return ner, err
	}
	scanner = bufio.NewScanner(fi)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		ner["O"+scanner.Text()] = struct{}{}
	}
	if err = fi.Close(); err != nil {
		return ner, err
	}
	return ner, nil
}

// buildOffensiveMap creates a map of offensive terms for which Abot will refuse
// to respond. This helps ensure that users are somewhat respectful to Abot and
// her human trainers, since sentences caught by the OffensiveMap are rejected
// before any human ever sees them.
func buildOffensiveMap() (map[string]struct{}, error) {
	o := map[string]struct{}{}
	p := filepath.Join("data", "offensive.txt")
	fi, err := os.Open(p)
	if err != nil {
		return o, err
	}
	scanner := bufio.NewScanner(fi)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		o[scanner.Text()] = struct{}{}
	}
	err = fi.Close()
	return o, err
}

// RespondWithNicety replies to niceties that humans use, but Abot can ignore.
// Words like "Thank you" are not necessary for a robot, but it's important Abot
// respond correctly nonetheless. The returned bool specifies whether a response
// is necessary, and the returned string is the response, if any.
func RespondWithNicety(in *dt.Msg) (responseNecessary bool, response string) {
	for _, w := range in.Stems {
		// Since these are stems, some of them look incorrectly spelled.
		// Needless to say, these are the correct Porter2 Snowball stems
		switch w {
		case "thank":
			return true, "You're welcome!"
		case "cool", "sweet", "awesom", "neat", "perfect":
			return false, ""
		case "sorri":
			return true, "That's OK. I forgive you."
		case "hi", "hello":
			return true, "Hi there. :)"
		}
	}
	return true, ""
}

// RespondWithOffense is a one-off function to respond to rude user language by
// refusing to process the command.
func RespondWithOffense(off map[string]struct{}, in *dt.Msg) string {
	for _, w := range in.Stems {
		_, offensive := off[w]
		if offensive {
			return "I'm sorry, but I don't respond to rude language."
		}
	}
	return ""
}

// ConfusedLang returns a randomized response signalling that Abot is confused
// or could not understand the user's request.
func ConfusedLang() string {
	n := rand.Intn(4)
	switch n {
	case 0:
		return "I'm not sure I understand you."
	case 1:
		return "I'm sorry, I don't understand that."
	case 2:
		return "Uh, what are you telling me to do?"
	case 3:
		return "What should I do?"
	}
	log.Debug("confused failed to return a response")
	return ""
}
<|endoftext|>


<commit_before>package logger

import (
	"github.com/op/go-logging"
	"koding/tools/config"
)

// Stores current logging level.
var loggingLevel logging.Level

// Mappings of strings in config file to internal types.
var nameToLevelMapping = map[string]logging.Level{
	"debug":   logging.DEBUG,
	"warning": logging.WARNING,
	"error":   logging.ERROR,
}

var defaultLoggingLevel = logging.WARNING

// Get logging level from config file & find the appropriate logging.Level
func getLoggingLevelFromConfig(name string) logging.Level {
	var logLevelString, ok = config.Current.LogLevel[name]
	if !ok {
		return defaultLoggingLevel
	}

	loggingLevel, ok = nameToLevelMapping[logLevelString]
	if !ok {
		return defaultLoggingLevel
	}

	return loggingLevel
}
<commit_msg>logger: add more levels<commit_after>package logger

import (
	"github.com/op/go-logging"
	"koding/tools/config"
)

// Stores current logging level.
var loggingLevel logging.Level

// Mappings of strings in config file to internal types.
var nameToLevelMapping = map[string]logging.Level{
	"critical": logging.CRITICAL,
	"debug":    logging.DEBUG,
	"error":    logging.ERROR,
	"info":     logging.INFO,
	"notice":   logging.NOTICE,
	"warning":  logging.WARNING,
}

var defaultLoggingLevel = logging.WARNING

// Get logging level from config file & find the appropriate logging.Level
func getLoggingLevelFromConfig(name string) logging.Level {
	var logLevelString, ok = config.Current.LogLevel[name]
	if !ok {
		return defaultLoggingLevel
	}

	loggingLevel, ok = nameToLevelMapping[logLevelString]
	if !ok {
		return defaultLoggingLevel
	}

	return loggingLevel
}
<|endoftext|>


<commit_before>// +build linux darwin

package instance

// DirName returns the name of the subdirectory where instance data are stored.
// On Posix systems, it's the instance domain name.
func (i *Instance) DirName() string {
	return i.Domain
}
<commit_msg>Fix building cozy-stack on FreeBSD, again<commit_after>// +build !windows

package instance

// DirName returns the name of the subdirectory where instance data are stored.
// On Posix systems, it's the instance domain name.
func (i *Instance) DirName() string {
	return i.Domain
}
<|endoftext|>


<commit_before>package protocol

import "time"

// MaxPacketSizeIPv4 is the maximum packet size that we use for sending IPv4 packets.
const MaxPacketSizeIPv4 = 1252

// MaxPacketSizeIPv6 is the maximum packet size that we use for sending IPv6 packets.
const MaxPacketSizeIPv6 = 1232

// MaxCongestionWindowPackets is the maximum congestion window in packet.
const MaxCongestionWindowPackets = 10000

// MaxUndecryptablePackets limits the number of undecryptable packets that are queued in the session.
const MaxUndecryptablePackets = 33

// ConnectionFlowControlMultiplier determines how much larger the connection flow control windows needs to be relative to any stream's flow control window
// This is the value that Chromium is using
const ConnectionFlowControlMultiplier = 1.5

// InitialMaxStreamData is the stream-level flow control window for receiving data
const InitialMaxStreamData = (1 << 10) * 512 // 512 kb

// InitialMaxData is the connection-level flow control window for receiving data
const InitialMaxData = ConnectionFlowControlMultiplier * InitialMaxStreamData

// DefaultMaxReceiveStreamFlowControlWindow is the default maximum stream-level flow control window for receiving data, for the server
const DefaultMaxReceiveStreamFlowControlWindow = 6 * (1 << 20) // 6 MB

// DefaultMaxReceiveConnectionFlowControlWindow is the default connection-level flow control window for receiving data, for the server
const DefaultMaxReceiveConnectionFlowControlWindow = 15 * (1 << 20) // 12 MB

// WindowUpdateThreshold is the fraction of the receive window that has to be consumed before an higher offset is advertised to the client
const WindowUpdateThreshold = 0.25

// DefaultMaxIncomingStreams is the maximum number of streams that a peer may open
const DefaultMaxIncomingStreams = 100

// DefaultMaxIncomingUniStreams is the maximum number of unidirectional streams that a peer may open
const DefaultMaxIncomingUniStreams = 100

// MaxServerUnprocessedPackets is the max number of packets stored in the server that are not yet processed.
const MaxServerUnprocessedPackets = 1024

// MaxSessionUnprocessedPackets is the max number of packets stored in each session that are not yet processed.
const MaxSessionUnprocessedPackets = MaxCongestionWindowPackets

// SkipPacketAveragePeriodLength is the average period length in which one packet number is skipped to prevent an Optimistic ACK attack
const SkipPacketAveragePeriodLength PacketNumber = 500

// MaxTrackedSkippedPackets is the maximum number of skipped packet numbers the SentPacketHandler keep track of for Optimistic ACK attack mitigation
const MaxTrackedSkippedPackets = 10

// MaxAcceptQueueSize is the maximum number of sessions that the server queues for accepting.
// If the queue is full, new connection attempts will be rejected.
const MaxAcceptQueueSize = 32

// TokenValidity is the duration that a (non-retry) token is considered valid
const TokenValidity = 24 * time.Hour

// RetryTokenValidity is the duration that a retry token is considered valid
const RetryTokenValidity = 10 * time.Second

// MaxOutstandingSentPackets is maximum number of packets saved for retransmission.
// When reached, it imposes a soft limit on sending new packets:
// Sending ACKs and retransmission is still allowed, but now new regular packets can be sent.
const MaxOutstandingSentPackets = 2 * MaxCongestionWindowPackets

// MaxTrackedSentPackets is maximum number of sent packets saved for retransmission.
// When reached, no more packets will be sent.
// This value *must* be larger than MaxOutstandingSentPackets.
const MaxTrackedSentPackets = MaxOutstandingSentPackets * 5 / 4

// MaxNonAckElicitingAcks is the maximum number of packets containing an ACK,
// but no ack-eliciting frames, that we send in a row
const MaxNonAckElicitingAcks = 19

// MaxStreamFrameSorterGaps is the maximum number of gaps between received StreamFrames
// prevents DoS attacks against the streamFrameSorter
const MaxStreamFrameSorterGaps = 1000

// MinStreamFrameBufferSize is the minimum data length of a received STREAM frame
// that we use the buffer for. This protects against a DoS where an attacker would send us
// very small STREAM frames to consume a lot of memory.
const MinStreamFrameBufferSize = 128

// MinCoalescedPacketSize is the minimum size of a coalesced packet that we pack.
// If a packet has less than this number of bytes, we won't coalesce any more packets onto it.
const MinCoalescedPacketSize = 128

// MaxCryptoStreamOffset is the maximum offset allowed on any of the crypto streams.
// This limits the size of the ClientHello and Certificates that can be received.
const MaxCryptoStreamOffset = 16 * (1 << 10)

// MinRemoteIdleTimeout is the minimum value that we accept for the remote idle timeout
const MinRemoteIdleTimeout = 5 * time.Second

// DefaultIdleTimeout is the default idle timeout
const DefaultIdleTimeout = 30 * time.Second

// DefaultHandshakeTimeout is the default timeout for a connection until the crypto handshake succeeds.
const DefaultHandshakeTimeout = 10 * time.Second

// MaxKeepAliveInterval is the maximum time until we send a packet to keep a connection alive.
// It should be shorter than the time that NATs clear their mapping.
const MaxKeepAliveInterval = 20 * time.Second

// RetiredConnectionIDDeleteTimeout is the time we keep closed sessions around in order to retransmit the CONNECTION_CLOSE.
// after this time all information about the old connection will be deleted
const RetiredConnectionIDDeleteTimeout = 5 * time.Second

// MinStreamFrameSize is the minimum size that has to be left in a packet, so that we add another STREAM frame.
// This avoids splitting up STREAM frames into small pieces, which has 2 advantages:
// 1. it reduces the framing overhead
// 2. it reduces the head-of-line blocking, when a packet is lost
const MinStreamFrameSize ByteCount = 128

// MaxPostHandshakeCryptoFrameSize is the maximum size of CRYPTO frames
// we send after the handshake completes.
const MaxPostHandshakeCryptoFrameSize = 1000

// MaxAckFrameSize is the maximum size for an ACK frame that we write
// Due to the varint encoding, ACK frames can grow (almost) indefinitely large.
// The MaxAckFrameSize should be large enough to encode many ACK range,
// but must ensure that a maximum size ACK frame fits into one packet.
const MaxAckFrameSize ByteCount = 1000

// MaxNumAckRanges is the maximum number of ACK ranges that we send in an ACK frame.
// It also serves as a limit for the packet history.
// If at any point we keep track of more ranges, old ranges are discarded.
const MaxNumAckRanges = 500

// MinPacingDelay is the minimum duration that is used for packet pacing
// If the packet packing frequency is higher, multiple packets might be sent at once.
// Example: For a packet pacing delay of 20 microseconds, we would send 5 packets at once, wait for 100 microseconds, and so forth.
const MinPacingDelay time.Duration = 100 * time.Microsecond

// DefaultConnectionIDLength is the connection ID length that is used for multiplexed connections
// if no other value is configured.
const DefaultConnectionIDLength = 4

// MaxActiveConnectionIDs is the number of connection IDs that we're storing.
const MaxActiveConnectionIDs = 4

// MaxIssuedConnectionIDs is the maximum number of connection IDs that we're issuing at the same time.
const MaxIssuedConnectionIDs = 6

// PacketsPerConnectionID is the number of packets we send using one connection ID.
// If the peer provices us with enough new connection IDs, we switch to a new connection ID.
const PacketsPerConnectionID = 10000

// AckDelayExponent is the ack delay exponent used when sending ACKs.
const AckDelayExponent = 3

// Estimated timer granularity.
// The loss detection timer will not be set to a value smaller than granularity.
const TimerGranularity = time.Millisecond

// MaxAckDelay is the maximum time by which we delay sending ACKs.
const MaxAckDelay = 25 * time.Millisecond

// MaxAckDelayInclGranularity is the max_ack_delay including the timer granularity.
// This is the value that should be advertised to the peer.
const MaxAckDelayInclGranularity = MaxAckDelay + TimerGranularity

// KeyUpdateInterval is the maximum number of packets we send or receive before initiating a key udpate.
const KeyUpdateInterval = 100 * 1000

// Max0RTTQueueingDuration is the maximum time that we store 0-RTT packets in order to wait for the corresponding Initial to be received.
const Max0RTTQueueingDuration = 100 * time.Millisecond

// Max0RTTQueues is the maximum number of connections that we buffer 0-RTT packets for.
const Max0RTTQueues = 32

// Max0RTTQueueLen is the maximum number of 0-RTT packets that we buffer for each connection.
// When a new session is created, all buffered packets are passed to the session immediately.
// To avoid blocking, this value has to be smaller than MaxSessionUnprocessedPackets.
// To avoid packets being dropped as undecryptable by the session, this value has to be smaller than MaxUndecryptablePackets.
const Max0RTTQueueLen = 32
<commit_msg>reduce the length of the unprocessed packet chan in the session<commit_after>package protocol

import "time"

// MaxPacketSizeIPv4 is the maximum packet size that we use for sending IPv4 packets.
const MaxPacketSizeIPv4 = 1252

// MaxPacketSizeIPv6 is the maximum packet size that we use for sending IPv6 packets.
const MaxPacketSizeIPv6 = 1232

// MaxCongestionWindowPackets is the maximum congestion window in packet.
const MaxCongestionWindowPackets = 10000

// MaxUndecryptablePackets limits the number of undecryptable packets that are queued in the session.
const MaxUndecryptablePackets = 33

// ConnectionFlowControlMultiplier determines how much larger the connection flow control windows needs to be relative to any stream's flow control window
// This is the value that Chromium is using
const ConnectionFlowControlMultiplier = 1.5

// InitialMaxStreamData is the stream-level flow control window for receiving data
const InitialMaxStreamData = (1 << 10) * 512 // 512 kb

// InitialMaxData is the connection-level flow control window for receiving data
const InitialMaxData = ConnectionFlowControlMultiplier * InitialMaxStreamData

// DefaultMaxReceiveStreamFlowControlWindow is the default maximum stream-level flow control window for receiving data, for the server
const DefaultMaxReceiveStreamFlowControlWindow = 6 * (1 << 20) // 6 MB

// DefaultMaxReceiveConnectionFlowControlWindow is the default connection-level flow control window for receiving data, for the server
const DefaultMaxReceiveConnectionFlowControlWindow = 15 * (1 << 20) // 12 MB

// WindowUpdateThreshold is the fraction of the receive window that has to be consumed before an higher offset is advertised to the client
const WindowUpdateThreshold = 0.25

// DefaultMaxIncomingStreams is the maximum number of streams that a peer may open
const DefaultMaxIncomingStreams = 100

// DefaultMaxIncomingUniStreams is the maximum number of unidirectional streams that a peer may open
const DefaultMaxIncomingUniStreams = 100

// MaxServerUnprocessedPackets is the max number of packets stored in the server that are not yet processed.
const MaxServerUnprocessedPackets = 1024

// MaxSessionUnprocessedPackets is the max number of packets stored in each session that are not yet processed.
const MaxSessionUnprocessedPackets = 256

// SkipPacketAveragePeriodLength is the average period length in which one packet number is skipped to prevent an Optimistic ACK attack
const SkipPacketAveragePeriodLength PacketNumber = 500

// MaxTrackedSkippedPackets is the maximum number of skipped packet numbers the SentPacketHandler keep track of for Optimistic ACK attack mitigation
const MaxTrackedSkippedPackets = 10

// MaxAcceptQueueSize is the maximum number of sessions that the server queues for accepting.
// If the queue is full, new connection attempts will be rejected.
const MaxAcceptQueueSize = 32

// TokenValidity is the duration that a (non-retry) token is considered valid
const TokenValidity = 24 * time.Hour

// RetryTokenValidity is the duration that a retry token is considered valid
const RetryTokenValidity = 10 * time.Second

// MaxOutstandingSentPackets is maximum number of packets saved for retransmission.
// When reached, it imposes a soft limit on sending new packets:
// Sending ACKs and retransmission is still allowed, but now new regular packets can be sent.
const MaxOutstandingSentPackets = 2 * MaxCongestionWindowPackets

// MaxTrackedSentPackets is maximum number of sent packets saved for retransmission.
// When reached, no more packets will be sent.
// This value *must* be larger than MaxOutstandingSentPackets.
const MaxTrackedSentPackets = MaxOutstandingSentPackets * 5 / 4

// MaxNonAckElicitingAcks is the maximum number of packets containing an ACK,
// but no ack-eliciting frames, that we send in a row
const MaxNonAckElicitingAcks = 19

// MaxStreamFrameSorterGaps is the maximum number of gaps between received StreamFrames
// prevents DoS attacks against the streamFrameSorter
const MaxStreamFrameSorterGaps = 1000

// MinStreamFrameBufferSize is the minimum data length of a received STREAM frame
// that we use the buffer for. This protects against a DoS where an attacker would send us
// very small STREAM frames to consume a lot of memory.
const MinStreamFrameBufferSize = 128

// MinCoalescedPacketSize is the minimum size of a coalesced packet that we pack.
// If a packet has less than this number of bytes, we won't coalesce any more packets onto it.
const MinCoalescedPacketSize = 128

// MaxCryptoStreamOffset is the maximum offset allowed on any of the crypto streams.
// This limits the size of the ClientHello and Certificates that can be received.
const MaxCryptoStreamOffset = 16 * (1 << 10)

// MinRemoteIdleTimeout is the minimum value that we accept for the remote idle timeout
const MinRemoteIdleTimeout = 5 * time.Second

// DefaultIdleTimeout is the default idle timeout
const DefaultIdleTimeout = 30 * time.Second

// DefaultHandshakeTimeout is the default timeout for a connection until the crypto handshake succeeds.
const DefaultHandshakeTimeout = 10 * time.Second

// MaxKeepAliveInterval is the maximum time until we send a packet to keep a connection alive.
// It should be shorter than the time that NATs clear their mapping.
const MaxKeepAliveInterval = 20 * time.Second

// RetiredConnectionIDDeleteTimeout is the time we keep closed sessions around in order to retransmit the CONNECTION_CLOSE.
// after this time all information about the old connection will be deleted
const RetiredConnectionIDDeleteTimeout = 5 * time.Second

// MinStreamFrameSize is the minimum size that has to be left in a packet, so that we add another STREAM frame.
// This avoids splitting up STREAM frames into small pieces, which has 2 advantages:
// 1. it reduces the framing overhead
// 2. it reduces the head-of-line blocking, when a packet is lost
const MinStreamFrameSize ByteCount = 128

// MaxPostHandshakeCryptoFrameSize is the maximum size of CRYPTO frames
// we send after the handshake completes.
const MaxPostHandshakeCryptoFrameSize = 1000

// MaxAckFrameSize is the maximum size for an ACK frame that we write
// Due to the varint encoding, ACK frames can grow (almost) indefinitely large.
// The MaxAckFrameSize should be large enough to encode many ACK range,
// but must ensure that a maximum size ACK frame fits into one packet.
const MaxAckFrameSize ByteCount = 1000

// MaxNumAckRanges is the maximum number of ACK ranges that we send in an ACK frame.
// It also serves as a limit for the packet history.
// If at any point we keep track of more ranges, old ranges are discarded.
const MaxNumAckRanges = 500

// MinPacingDelay is the minimum duration that is used for packet pacing
// If the packet packing frequency is higher, multiple packets might be sent at once.
// Example: For a packet pacing delay of 20 microseconds, we would send 5 packets at once, wait for 100 microseconds, and so forth.
const MinPacingDelay time.Duration = 100 * time.Microsecond

// DefaultConnectionIDLength is the connection ID length that is used for multiplexed connections
// if no other value is configured.
const DefaultConnectionIDLength = 4

// MaxActiveConnectionIDs is the number of connection IDs that we're storing.
const MaxActiveConnectionIDs = 4

// MaxIssuedConnectionIDs is the maximum number of connection IDs that we're issuing at the same time.
const MaxIssuedConnectionIDs = 6

// PacketsPerConnectionID is the number of packets we send using one connection ID.
// If the peer provices us with enough new connection IDs, we switch to a new connection ID.
const PacketsPerConnectionID = 10000

// AckDelayExponent is the ack delay exponent used when sending ACKs.
const AckDelayExponent = 3

// Estimated timer granularity.
// The loss detection timer will not be set to a value smaller than granularity.
const TimerGranularity = time.Millisecond

// MaxAckDelay is the maximum time by which we delay sending ACKs.
const MaxAckDelay = 25 * time.Millisecond

// MaxAckDelayInclGranularity is the max_ack_delay including the timer granularity.
// This is the value that should be advertised to the peer.
const MaxAckDelayInclGranularity = MaxAckDelay + TimerGranularity

// KeyUpdateInterval is the maximum number of packets we send or receive before initiating a key udpate.
const KeyUpdateInterval = 100 * 1000

// Max0RTTQueueingDuration is the maximum time that we store 0-RTT packets in order to wait for the corresponding Initial to be received.
const Max0RTTQueueingDuration = 100 * time.Millisecond

// Max0RTTQueues is the maximum number of connections that we buffer 0-RTT packets for.
const Max0RTTQueues = 32

// Max0RTTQueueLen is the maximum number of 0-RTT packets that we buffer for each connection.
// When a new session is created, all buffered packets are passed to the session immediately.
// To avoid blocking, this value has to be smaller than MaxSessionUnprocessedPackets.
// To avoid packets being dropped as undecryptable by the session, this value has to be smaller than MaxUndecryptablePackets.
const Max0RTTQueueLen = 32
<|endoftext|>


<commit_before>package logger

import (
	"github.com/op/go-logging"
	"koding/tools/config"
	stdlog "log"
	"os"
)

var loggingLevel logging.Level

var nameToLevelMapping = map[string]logging.Level{
	"debug":   logging.DEBUG,
	"warning": logging.WARNING,
	"error":   logging.ERROR,
}

// Get logging level from config file and find the appropriate logging.Level
// from string.
func init() {
	var exists bool
	var logLevelString = config.Current.Neo4j.LogLevel

	loggingLevel, exists = nameToLevelMapping[logLevelString]
	if !exists {
		loggingLevel = logging.DEBUG
	}
}

func New(name string) *logging.Logger {
	logging.SetFormatter(logging.MustStringFormatter("[%{level:.8s}] - %{message}"))

	var logBackend = logging.NewLogBackend(os.Stderr, "", stdlog.LstdFlags|stdlog.Lshortfile)
	logBackend.Color = true

	var syslogBackend, err = logging.NewSyslogBackend("")
	if err != nil {
		panic(err)
	}

	logging.SetBackend(logBackend, syslogBackend)

	// Set logging level based on value in config.
	logging.SetLevel(loggingLevel, name)

	return logging.MustGetLogger(name)
}
<commit_msg>logger: copy gauges from old log/ pkg<commit_after>package logger

import (
	"fmt"
	"github.com/op/go-logging"
	"koding/tools/config"
	stdlog "log"
	"os"
	"os/exec"
	"strings"
	"time"
)

var loggingLevel logging.Level

var nameToLevelMapping = map[string]logging.Level{
	"debug":   logging.DEBUG,
	"warning": logging.WARNING,
	"error":   logging.ERROR,
}

// Get logging level from config file and find the appropriate logging.Level
// from string.
func init() {
	var exists bool
	var logLevelString = config.Current.Neo4j.LogLevel

	loggingLevel, exists = nameToLevelMapping[logLevelString]
	if !exists {
		loggingLevel = logging.DEBUG
	}
}

func New(name string) *logging.Logger {
	logging.SetFormatter(logging.MustStringFormatter("[%{level:.8s}] - %{message}"))

	var logBackend = logging.NewLogBackend(os.Stderr, "", stdlog.LstdFlags|stdlog.Lshortfile)
	logBackend.Color = true

	var syslogBackend, err = logging.NewSyslogBackend("")
	if err != nil {
		panic(err)
	}

	logging.SetBackend(logBackend, syslogBackend)

	// Set logging level based on value in config.
	logging.SetLevel(loggingLevel, name)

	return logging.MustGetLogger(name)
}

//----------------------------------------------------------
// Gauges. Copied over from koding/tool/log
//----------------------------------------------------------

type Unit string

type Gauge struct {
	Name   string  `json:"name"`
	Value  float64 `json:"value"`
	Time   int64   `json:"measure_time"`
	Source string  `json:"source"`
	unit   Unit
	input  func() float64
}

const ISO8601 = "2006-01-02T15:04:05.000"

var gauges = make([]*Gauge, 0)
var GaugeChanges = make(chan func())
var tags string
var hostname string

func CreateGauge(name string, unit Unit, input func() float64) {
	gauges = append(gauges, &Gauge{name, 0, 0, "", unit, input})
}

func CreateCounterGauge(name string, unit Unit, resetOnReport bool) func(int) {
	value := new(int)
	CreateGauge(name, unit, func() float64 {
		v := *value
		if resetOnReport {
			*value = 0
		}
		return float64(v)
	})
	return func(diff int) {
		GaugeChanges <- func() {
			*value += diff
		}
	}
}

var interval = 60000

func RunGaugesLoop() {
	reportTrigger := make(chan int64)
	go func() {
		reportInterval := int64(interval) / 1000
		nextReportTime := time.Now().Unix() / reportInterval * reportInterval
		for {
			nextReportTime += reportInterval
			time.Sleep(time.Duration(nextReportTime-time.Now().Unix()) * time.Second)
			reportTrigger <- nextReportTime
		}
	}()
	go func() {
		for {
			select {
			case change := <-GaugeChanges:
				change()
			}
		}
	}()
}

func LogGauges(reportTime int64) {
	if !config.Current.Opsview.Push {
		indent := strings.Repeat(" ", len(ISO8601)+1)
		stdlog.Printf("%s [gauges %s]\n", time.Now().Format(ISO8601), tags)

		for _, gauge := range gauges {
			stdlog.Printf("%s%s: %v\n", indent, gauge.Name, gauge.input())
		}
		return
	}

	for _, gauge := range gauges {
		gauge.Value = gauge.input()
		gauge.Time = reportTime
	}

	if config.Current.Opsview.Push {
		entries := make([]string, len(gauges))
		for i, gauge := range gauges {
			entries[i] = fmt.Sprintf("%s=%f%s", gauge.Name, gauge.Value, gauge.unit)
		}
		cmd := exec.Command("/usr/local/nagios/bin/send_nsca", "-H", config.Current.Opsview.Host, "-c", "/usr/local/nagios/etc/send_nsca.cfg")
		cmd.Stdin = strings.NewReader(hostname + "\tGauges\t0\tGauge Data Sent|" + strings.Join(entries, " ") + "\n")
		if out, err := cmd.CombinedOutput(); err != nil {
			stdlog.Printf("logger error: send_nsca failed.\n%v\n%v\n", err, string(out))
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package binlog

import (
	"fmt"
	"io"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/sync2"
	"github.com/youtube/vitess/go/vt/binlog/proto"
	"github.com/youtube/vitess/go/vt/mysqlctl"
	myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto"
)

var ClientEOF = fmt.Errorf("binlog stream consumer ended the reply stream")
var ServerEOF = fmt.Errorf("binlog stream connection was closed by mysqld")

// binlogConnStreamer streams binlog events from MySQL by connecting as a slave.
type binlogConnStreamer struct {
	// dbname and mysqld are set at creation.
	dbname string
	mysqld *mysqlctl.Mysqld

	svm  sync2.ServiceManager
	conn *mysqlctl.SlaveConnection

	// pos is the GTID of the most recent event we've seen.
	pos myproto.GTID
}

// newBinlogConnStreamer creates a BinlogStreamer.
//
// dbname specifes the db to stream events for.
func newBinlogConnStreamer(dbname string, mysqld *mysqlctl.Mysqld) BinlogStreamer {
	return &binlogConnStreamer{
		dbname: dbname,
		mysqld: mysqld,
	}
}

// Stream implements BinlogStreamer.Stream().
func (bls *binlogConnStreamer) Stream(startPos myproto.GTID, sendTransaction sendTransactionFunc) error {
	// Launch using service manager so we can stop this as needed.
	bls.svm.Go(func(svc *sync2.ServiceContext) error {
		var events <-chan proto.BinlogEvent
		var err error

		if bls.conn, err = mysqlctl.NewSlaveConnection(bls.mysqld); err != nil {
			return err
		}
		defer bls.conn.Close()

		events, err = bls.conn.StartBinlogDump(startPos)
		if err != nil {
			return err
		}
		// parseEvents will loop until the events channel is closed, the
		// service enters the SHUTTING_DOWN state, or an error occurs.
		err = bls.parseEvents(svc, events, sendTransaction)
		if err == ClientEOF {
			return nil
		}
		return err
	})

	// Wait for service to exit, and handle errors if any.
	err := bls.svm.Join()

	if err != nil {
		binlogStreamerErrors.Add("Stream", 1)
		err = fmt.Errorf("stream error @ %#v, error: %v", bls.pos, err)
		log.Error(err.Error())
		return err
	}

	log.Infof("stream ended @ %#v", bls.pos)
	return nil
}

// Stop implements BinlogStreamer.Stop().
func (bls *binlogConnStreamer) Stop() {
	bls.svm.Stop()
}

// parseEvents processes the raw binlog dump stream from the server, one event
// at a time, and groups them into transactions. It is called from within the
// service function launched by Stream().
//
// If the sendTransaction func returns io.EOF, parseEvents returns ClientEOF.
// If the events channel is closed, parseEvents returns ServerEOF.
func (bls *binlogConnStreamer) parseEvents(svc *sync2.ServiceContext, events <-chan proto.BinlogEvent, sendTransaction sendTransactionFunc) (err error) {
	var statements []proto.Statement
	var format proto.BinlogFormat

	// A commit can be triggered either by a COMMIT query, or by an XID_EVENT.
	commit := func(timestamp int64) error {
		trans := &proto.BinlogTransaction{
			Statements: statements,
			Timestamp:  timestamp,
			GTIDField:  myproto.GTIDField{Value: bls.pos},
		}
		if err = sendTransaction(trans); err != nil {
			if err == io.EOF {
				return ClientEOF
			}
			return fmt.Errorf("send reply error: %v", err)
		}
		statements = nil
		return nil
	}

	// Parse events.
	for svc.IsRunning() {
		var ev proto.BinlogEvent
		var ok bool

		select {
		case ev, ok = <-events:
			if !ok {
				// events channel has been closed, which means the connection died.
				log.Infof("reached end of binlog event stream")
				return ServerEOF
			}
		case <-svc.ShuttingDown:
			log.Infof("stopping early due to BinlogStreamer service shutdown")
			return nil
		}

		// Validate the buffer before reading fields from it.
		if !ev.IsValid() {
			return fmt.Errorf("can't parse binlog event, invalid data: %#v", ev)
		}

		// We can't parse anything until we get a FORMAT_DESCRIPTION_EVENT that
		// tells us the size of the event header.
		if format.IsZero() {
			if !ev.IsFormatDescription() {
				// The only thing that should come before the FORMAT_DESCRIPTION_EVENT
				// is a fake ROTATE_EVENT, which the master sends to tell us the name
				// of the current log file.
				if ev.IsRotate() {
					continue
				}
				return fmt.Errorf("got a real event before FORMAT_DESCRIPTION_EVENT: %#v", ev)
			}

			format, err = ev.Format()
			if err != nil {
				return fmt.Errorf("can't parse FORMAT_DESCRIPTION_EVENT: %v, event data: %#v", err, ev)
			}
			continue
		}

		// Update the GTID if the event has one. The actual event type could be
		// something special like GTID_EVENT (MariaDB, MySQL 5.6), or it could be
		// an arbitrary event with a GTID in the header (Google MySQL).
		if ev.HasGTID(format) {
			bls.pos, err = ev.GTID(format)
			if err != nil {
				return fmt.Errorf("can't get GTID from binlog event: %v, event data: %#v", err, ev)
			}
			// If it's a dedicated GTID_EVENT, there's nothing else to do.
			if ev.IsGTID() {
				continue
			}
		}

		switch true {
		case ev.IsXID(): // XID_EVENT (equivalent to COMMIT)
			if err = commit(int64(ev.Timestamp())); err != nil {
				return err
			}
		case ev.IsIntVar(): // INTVAR_EVENT
			name, value, err := ev.IntVar(format)
			if err != nil {
				return fmt.Errorf("can't parse INTVAR_EVENT: %v, event data: %#v", err, ev)
			}
			statements = append(statements, proto.Statement{
				Category: proto.BL_SET,
				Sql:      []byte(fmt.Sprintf("SET %s=%d", name, value)),
			})
		case ev.IsRand(): // RAND_EVENT
			seed1, seed2, err := ev.Rand(format)
			if err != nil {
				return fmt.Errorf("can't parse RAND_EVENT: %v, event data: %#v", err, ev)
			}
			statements = append(statements, proto.Statement{
				Category: proto.BL_SET,
				Sql:      []byte(fmt.Sprintf("SET @@RAND_SEED1=%d, @@RAND_SEED2=%d", seed1, seed2)),
			})
		case ev.IsQuery(): // QUERY_EVENT
			// Extract the query string and group into transactions.
			db, sql, err := ev.Query(format)
			if err != nil {
				return fmt.Errorf("can't get query from binlog event: %v, event data: %#v", err, ev)
			}
			switch cat := getStatementCategory(sql); cat {
			case proto.BL_BEGIN:
				if statements != nil {
					// If this happened, it would be a legitimate error.
					log.Errorf("BEGIN in binlog stream while still in another transaction; dropping %d statements: %v", len(statements), statements)
					binlogStreamerErrors.Add("ParseEvents", 1)
				}
				statements = make([]proto.Statement, 0, 10)
			case proto.BL_ROLLBACK:
				// Rollbacks are possible under some circumstances. So, let's honor them
				// by sending an empty transaction, which will contain the new binlog position.
				statements = nil
				fallthrough
			case proto.BL_COMMIT:
				if err = commit(int64(ev.Timestamp())); err != nil {
					return err
				}
			default: // BL_DDL, BL_DML, BL_SET, BL_UNRECOGNIZED
				if db != "" && db != bls.dbname {
					// Skip cross-db statements.
					continue
				}
				autocommit := false
				if statements == nil {
					// We didn't see a BEGIN. Assume autocommit.
					autocommit = true
				}
				statements = append(statements, proto.Statement{
					Category: proto.BL_SET,
					Sql:      []byte(fmt.Sprintf("SET TIMESTAMP=%d", ev.Timestamp())),
				})
				statements = append(statements, proto.Statement{Category: cat, Sql: sql})
				if autocommit {
					if err = commit(int64(ev.Timestamp())); err != nil {
						return err
					}
				}
			}
		}
	}

	return nil
}
<commit_msg>binlog_conn_streamer: improved autocommit state change<commit_after>// Copyright 2014, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package binlog

import (
	"fmt"
	"io"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/sync2"
	"github.com/youtube/vitess/go/vt/binlog/proto"
	"github.com/youtube/vitess/go/vt/mysqlctl"
	myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto"
)

var ClientEOF = fmt.Errorf("binlog stream consumer ended the reply stream")
var ServerEOF = fmt.Errorf("binlog stream connection was closed by mysqld")

// binlogConnStreamer streams binlog events from MySQL by connecting as a slave.
type binlogConnStreamer struct {
	// dbname and mysqld are set at creation.
	dbname string
	mysqld *mysqlctl.Mysqld

	svm  sync2.ServiceManager
	conn *mysqlctl.SlaveConnection

	// pos is the GTID of the most recent event we've seen.
	pos myproto.GTID
}

// newBinlogConnStreamer creates a BinlogStreamer.
//
// dbname specifes the db to stream events for.
func newBinlogConnStreamer(dbname string, mysqld *mysqlctl.Mysqld) BinlogStreamer {
	return &binlogConnStreamer{
		dbname: dbname,
		mysqld: mysqld,
	}
}

// Stream implements BinlogStreamer.Stream().
func (bls *binlogConnStreamer) Stream(startPos myproto.GTID, sendTransaction sendTransactionFunc) error {
	// Launch using service manager so we can stop this as needed.
	bls.svm.Go(func(svc *sync2.ServiceContext) error {
		var events <-chan proto.BinlogEvent
		var err error

		if bls.conn, err = mysqlctl.NewSlaveConnection(bls.mysqld); err != nil {
			return err
		}
		defer bls.conn.Close()

		events, err = bls.conn.StartBinlogDump(startPos)
		if err != nil {
			return err
		}
		// parseEvents will loop until the events channel is closed, the
		// service enters the SHUTTING_DOWN state, or an error occurs.
		err = bls.parseEvents(svc, events, sendTransaction)
		if err == ClientEOF {
			return nil
		}
		return err
	})

	// Wait for service to exit, and handle errors if any.
	err := bls.svm.Join()

	if err != nil {
		binlogStreamerErrors.Add("Stream", 1)
		err = fmt.Errorf("stream error @ %#v, error: %v", bls.pos, err)
		log.Error(err.Error())
		return err
	}

	log.Infof("stream ended @ %#v", bls.pos)
	return nil
}

// Stop implements BinlogStreamer.Stop().
func (bls *binlogConnStreamer) Stop() {
	bls.svm.Stop()
}

// parseEvents processes the raw binlog dump stream from the server, one event
// at a time, and groups them into transactions. It is called from within the
// service function launched by Stream().
//
// If the sendTransaction func returns io.EOF, parseEvents returns ClientEOF.
// If the events channel is closed, parseEvents returns ServerEOF.
func (bls *binlogConnStreamer) parseEvents(svc *sync2.ServiceContext, events <-chan proto.BinlogEvent, sendTransaction sendTransactionFunc) (err error) {
	var statements []proto.Statement
	var format proto.BinlogFormat
	var autocommit = true

	// A commit can be triggered either by a COMMIT query, or by an XID_EVENT.
	commit := func(timestamp int64) error {
		trans := &proto.BinlogTransaction{
			Statements: statements,
			Timestamp:  timestamp,
			GTIDField:  myproto.GTIDField{Value: bls.pos},
		}
		if err = sendTransaction(trans); err != nil {
			if err == io.EOF {
				return ClientEOF
			}
			return fmt.Errorf("send reply error: %v", err)
		}
		statements = nil
		autocommit = true
		return nil
	}

	// Parse events.
	for svc.IsRunning() {
		var ev proto.BinlogEvent
		var ok bool

		select {
		case ev, ok = <-events:
			if !ok {
				// events channel has been closed, which means the connection died.
				log.Infof("reached end of binlog event stream")
				return ServerEOF
			}
		case <-svc.ShuttingDown:
			log.Infof("stopping early due to BinlogStreamer service shutdown")
			return nil
		}

		// Validate the buffer before reading fields from it.
		if !ev.IsValid() {
			return fmt.Errorf("can't parse binlog event, invalid data: %#v", ev)
		}

		// We can't parse anything until we get a FORMAT_DESCRIPTION_EVENT that
		// tells us the size of the event header.
		if format.IsZero() {
			if !ev.IsFormatDescription() {
				// The only thing that should come before the FORMAT_DESCRIPTION_EVENT
				// is a fake ROTATE_EVENT, which the master sends to tell us the name
				// of the current log file.
				if ev.IsRotate() {
					continue
				}
				return fmt.Errorf("got a real event before FORMAT_DESCRIPTION_EVENT: %#v", ev)
			}

			format, err = ev.Format()
			if err != nil {
				return fmt.Errorf("can't parse FORMAT_DESCRIPTION_EVENT: %v, event data: %#v", err, ev)
			}
			continue
		}

		// Update the GTID if the event has one. The actual event type could be
		// something special like GTID_EVENT (MariaDB, MySQL 5.6), or it could be
		// an arbitrary event with a GTID in the header (Google MySQL).
		if ev.HasGTID(format) {
			bls.pos, err = ev.GTID(format)
			if err != nil {
				return fmt.Errorf("can't get GTID from binlog event: %v, event data: %#v", err, ev)
			}
			// If it's a dedicated GTID_EVENT, there's nothing else to do.
			if ev.IsGTID() {
				continue
			}
		}

		switch true {
		case ev.IsXID(): // XID_EVENT (equivalent to COMMIT)
			if err = commit(int64(ev.Timestamp())); err != nil {
				return err
			}
		case ev.IsIntVar(): // INTVAR_EVENT
			name, value, err := ev.IntVar(format)
			if err != nil {
				return fmt.Errorf("can't parse INTVAR_EVENT: %v, event data: %#v", err, ev)
			}
			statements = append(statements, proto.Statement{
				Category: proto.BL_SET,
				Sql:      []byte(fmt.Sprintf("SET %s=%d", name, value)),
			})
		case ev.IsRand(): // RAND_EVENT
			seed1, seed2, err := ev.Rand(format)
			if err != nil {
				return fmt.Errorf("can't parse RAND_EVENT: %v, event data: %#v", err, ev)
			}
			statements = append(statements, proto.Statement{
				Category: proto.BL_SET,
				Sql:      []byte(fmt.Sprintf("SET @@RAND_SEED1=%d, @@RAND_SEED2=%d", seed1, seed2)),
			})
		case ev.IsQuery(): // QUERY_EVENT
			// Extract the query string and group into transactions.
			db, sql, err := ev.Query(format)
			if err != nil {
				return fmt.Errorf("can't get query from binlog event: %v, event data: %#v", err, ev)
			}
			switch cat := getStatementCategory(sql); cat {
			case proto.BL_BEGIN:
				if statements != nil {
					// If this happened, it would be a legitimate error.
					log.Errorf("BEGIN in binlog stream while still in another transaction; dropping %d statements: %v", len(statements), statements)
					binlogStreamerErrors.Add("ParseEvents", 1)
				}
				statements = make([]proto.Statement, 0, 10)
				autocommit = false
			case proto.BL_ROLLBACK:
				// Rollbacks are possible under some circumstances. So, let's honor them
				// by sending an empty transaction, which will contain the new binlog position.
				statements = nil
				fallthrough
			case proto.BL_COMMIT:
				if err = commit(int64(ev.Timestamp())); err != nil {
					return err
				}
			default: // BL_DDL, BL_DML, BL_SET, BL_UNRECOGNIZED
				if db != "" && db != bls.dbname {
					// Skip cross-db statements.
					continue
				}
				statements = append(statements, proto.Statement{
					Category: proto.BL_SET,
					Sql:      []byte(fmt.Sprintf("SET TIMESTAMP=%d", ev.Timestamp())),
				})
				statements = append(statements, proto.Statement{Category: cat, Sql: sql})
				if autocommit {
					if err = commit(int64(ev.Timestamp())); err != nil {
						return err
					}
				}
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package net

import (
	"net"

	"nimona.io/pkg/crypto"
)

type Connection struct {
	Conn          net.Conn
	RemotePeerKey *crypto.Key
}
<commit_msg>Add net connection direction<commit_after>package net

import (
	"net"

	"nimona.io/pkg/crypto"
)

type Connection struct {
	Conn          net.Conn
	RemotePeerKey *crypto.Key
	IsIncoming    bool
	IsOutgoing    bool
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package save

import (
	"errors"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/blob"
	"github.com/jacobsa/comeback/internal/graph"
)

// For each incoming node, use the supplied blob store to ensure that the node
// has a non-nil list of scores. Incoming nodes must be in reverse
// topologically sorted order: children must appear before parents.
func fillInScores(
	ctx context.Context,
	blobStore blob.Store,
	nodeIn <-chan *fsNode,
	nodesOut chan<- *fsNode) (err error) {
	err = errors.New("TODO")
	return
}

// Create a graph.Visitor for *fsNode that saves to the supplied blob store,
// filling in the node's Scores field when it is nil. All visited nodes are
// then written to nodesOut.
func newVisitor(
	blobStore blob.Store,
	nodesOut chan<- *fsNode) (v graph.Visitor) {
	panic("TODO")
}

type visitor struct {
}

func (v *visitor) Visit(ctx context.Context, untyped graph.Node) (err error) {
	err = errors.New("TODO")
}
<commit_msg>Fixed a build error.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package save

import (
	"errors"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/blob"
	"github.com/jacobsa/comeback/internal/graph"
)

// For each incoming node, use the supplied blob store to ensure that the node
// has a non-nil list of scores. Incoming nodes must be in reverse
// topologically sorted order: children must appear before parents.
func fillInScores(
	ctx context.Context,
	blobStore blob.Store,
	nodeIn <-chan *fsNode,
	nodesOut chan<- *fsNode) (err error) {
	err = errors.New("TODO")
	return
}

// Create a graph.Visitor for *fsNode that saves to the supplied blob store,
// filling in the node's Scores field when it is nil. All visited nodes are
// then written to nodesOut.
func newVisitor(
	blobStore blob.Store,
	nodesOut chan<- *fsNode) (v graph.Visitor) {
	panic("TODO")
}

type visitor struct {
}

func (v *visitor) Visit(ctx context.Context, untyped graph.Node) (err error) {
	err = errors.New("TODO")
	return
}
<|endoftext|>


<commit_before>// Copyright Istio Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package publish

import (
	"fmt"
	"io/ioutil"
	"path"
	"path/filepath"
	"strings"

	"istio.io/pkg/log"
	"istio.io/release-builder/pkg/model"
	"istio.io/release-builder/pkg/util"
)

const DefaultVariant = "debug"

// Docker publishes all images to the given hub
func Docker(manifest model.Manifest, hub string, tags []string, cosignkey string) error {
	dockerArchives, err := ioutil.ReadDir(path.Join(manifest.Directory, "docker"))
	if err != nil {
		return fmt.Errorf("failed to read docker output of release: %v", err)
	}

	// Only attempt to sign images if a valid cosign key is provided and we are
	// able to run 'cosign public-key <key>'.
	cosignEnabled := false
	if cosignkey != "" {
		if err := util.VerboseCommand("cosign", "public-key", "-key", cosignkey).Run(); err != nil {
			log.Errorf("Argument '--cosignkey' nonempty but unable to access key %v, disabling signing.", err)
		} else {
			cosignEnabled = true
		}
	}

	for _, f := range dockerArchives {
		if !strings.HasSuffix(f.Name(), "tar.gz") {
			return fmt.Errorf("invalid image found in docker folder: %v", f.Name())
		}

		imageName, variant := getImageNameVariant(f.Name())
		if variant != "" {
			// Prepend - so it shows up as name-variant in the final tag
			variant = "-" + variant
		}
		if err := util.VerboseCommand("docker", "load", "-i", path.Join(manifest.Directory, "docker", f.Name())).Run(); err != nil {
			return fmt.Errorf("failed to load docker image %v: %v", f.Name(), err)
		}

		// Images are always built with the `istio` hub initially. We will retag these to the correct hub
		currentTag := fmt.Sprintf("%s/%s:%s%s", manifest.Docker, imageName, manifest.Version, variant)
		if len(tags) == 0 {
			tags = []string{manifest.Version}
		}
		variants := []string{variant}
		// Publish with no variant tag as well (ie "istio/pilot:tag")
		if variant == DefaultVariant {
			variants = append(variants, "")
		}
		for _, tag := range tags {
			for _, variant := range variants {
				newTag := fmt.Sprintf("%s/%s:%s%s", hub, imageName, tag, variant)
				if err := util.VerboseCommand("docker", "tag", currentTag, newTag).Run(); err != nil {
					return fmt.Errorf("failed to load docker image %v: %v", currentTag, err)
				}

				if err := util.VerboseCommand("docker", "push", newTag).Run(); err != nil {
					return fmt.Errorf("failed to push docker image %v: %v", newTag, err)
				}

				// Sign images *after* push -- cosign only works against real
				// repositories (not valid against tarballs)
				if cosignEnabled {
					if err := util.VerboseCommand("cosign", "sign", "-key", cosignkey, newTag).Run(); err != nil {
						return fmt.Errorf("failed to sign image %v with key %v: %v", newTag, cosignkey, err)
					}
				}
			}
		}
	}
	return nil
}

// getImageNameVariant determines the name of the image (eg, pilot) and variant (eg, distroless).
// This is derived from the file name.
func getImageNameVariant(fname string) (string, string) {
	imageName := strings.Split(fname, ".")[0]
	if match, _ := filepath.Match("*-distroless", imageName); match {
		return strings.TrimSuffix(imageName, "-distroless"), "distroless"
	}
	return imageName, ""
}
<commit_msg>Fix duplicating tagging in publish (#806)<commit_after>// Copyright Istio Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package publish

import (
	"fmt"
	"io/ioutil"
	"path"
	"path/filepath"
	"strings"

	"istio.io/pkg/log"
	"istio.io/release-builder/pkg/model"
	"istio.io/release-builder/pkg/util"
)

const DefaultVariant = "debug"

// Docker publishes all images to the given hub
func Docker(manifest model.Manifest, hub string, tags []string, cosignkey string) error {
	dockerArchives, err := ioutil.ReadDir(path.Join(manifest.Directory, "docker"))
	if err != nil {
		return fmt.Errorf("failed to read docker output of release: %v", err)
	}

	// Only attempt to sign images if a valid cosign key is provided and we are
	// able to run 'cosign public-key <key>'.
	cosignEnabled := false
	if cosignkey != "" {
		if err := util.VerboseCommand("cosign", "public-key", "-key", cosignkey).Run(); err != nil {
			log.Errorf("Argument '--cosignkey' nonempty but unable to access key %v, disabling signing.", err)
		} else {
			cosignEnabled = true
		}
	}

	for _, f := range dockerArchives {
		if !strings.HasSuffix(f.Name(), "tar.gz") {
			return fmt.Errorf("invalid image found in docker folder: %v", f.Name())
		}

		imageName, variant := getImageNameVariant(f.Name())
		if variant != "" {
			// Prepend - so it shows up as name-variant in the final tag
			variant = "-" + variant
		}
		if err := util.VerboseCommand("docker", "load", "-i", path.Join(manifest.Directory, "docker", f.Name())).Run(); err != nil {
			return fmt.Errorf("failed to load docker image %v: %v", f.Name(), err)
		}

		// Images are always built with the `istio` hub initially. We will retag these to the correct hub
		currentTag := fmt.Sprintf("%s/%s:%s%s", manifest.Docker, imageName, manifest.Version, variant)
		if len(tags) == 0 {
			tags = []string{manifest.Version}
		}
		variants := []string{variant}
		// Publish with no variant tag as well (ie "istio/pilot:tag")
		if variant == DefaultVariant {
			variants = append(variants, "")
		}
		for _, tag := range tags {
			for _, variant := range variants {
				newTag := fmt.Sprintf("%s/%s:%s%s", hub, imageName, tag, variant)
				if err := util.VerboseCommand("docker", "tag", currentTag, newTag).Run(); err != nil {
					return fmt.Errorf("failed to load docker image %v: %v", currentTag, err)
				}

				if err := util.VerboseCommand("docker", "push", newTag).Run(); err != nil {
					return fmt.Errorf("failed to push docker image %v: %v", newTag, err)
				}

				// Sign images *after* push -- cosign only works against real
				// repositories (not valid against tarballs)
				if cosignEnabled {
					if err := util.VerboseCommand("cosign", "sign", "-key", cosignkey, newTag).Run(); err != nil {
						return fmt.Errorf("failed to sign image %v with key %v: %v", newTag, cosignkey, err)
					}
				}
			}
		}
	}
	return nil
}

// getImageNameVariant determines the name of the image (eg, pilot) and variant (eg, distroless).
// This is derived from the file name.
func getImageNameVariant(fname string) (string, string) {
	imageName := strings.Split(fname, ".")[0]
	if match, _ := filepath.Match("*-distroless", imageName); match {
		return strings.TrimSuffix(imageName, "-distroless"), "distroless"
	}
	if match, _ := filepath.Match("*-debug", imageName); match {
		return strings.TrimSuffix(imageName, "-debug"), "debug"
	}
	return imageName, ""
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package testutil

import (
	"fmt"
	"os"
	"path/filepath"

	"v.io/x/devtools/internal/collect"
	"v.io/x/devtools/internal/tool"
	"v.io/x/devtools/internal/util"
	"v.io/x/devtools/internal/xunit"
)

var (
	mirrors = []Mirror{
		Mirror{
			name:         "environment",
			googlesource: "https://vanadium.googlesource.com/environment",
			github:       "git@github.com:vanadium/environment.git",
		},
		Mirror{
			name:         "browser",
			googlesource: "https://vanadium.googlesource.com/browser",
			github:       "git@github.com:vanadium/browser.git",
		},
		Mirror{
			name:         "go.v23",
			googlesource: "https://vanadium.googlesource.com/release.go.v23",
			github:       "git@github.com:vanadium/go.v23.git",
		},
		Mirror{
			name:         "go.devtools",
			googlesource: "https://vanadium.googlesource.com/release.go.x.devtools",
			github:       "git@github.com:vanadium/go.devtools.git",
		},
		Mirror{
			name:         "go.jni",
			googlesource: "https://vanadium.googlesource.com/release.go.x.jni",
			github:       "git@github.com:vanadium/go.jni.git",
		},
		Mirror{
			name:         "go.lib",
			googlesource: "https://vanadium.googlesource.com/release.go.x.lib",
			github:       "git@github.com:vanadium/go.lib.git",
		},
		Mirror{
			name:         "go.ref",
			googlesource: "https://vanadium.googlesource.com/release.go.x.ref",
			github:       "git@github.com:vanadium/go.ref.git",
		},
		Mirror{
			name:         "java",
			googlesource: "https://vanadium.googlesource.com/release.java",
			github:       "git@github.com:vanadium/java.git",
		},
		Mirror{
			name:         "js",
			googlesource: "https://vanadium.googlesource.com/release.js.core",
			github:       "git@github.com:vanadium/js.git",
		},
		Mirror{
			name:         "chat",
			googlesource: "https://vanadium.googlesource.com/release.projects.chat",
			github:       "git@github.com:vanadium/chat.git",
		},
		Mirror{
			name:         "pipe2browser",
			googlesource: "https://vanadium.googlesource.com/release.projects.pipe2browser",
			github:       "git@github.com:vanadium/pipe2browser.git",
		},
		Mirror{
			name:         "playground",
			googlesource: "https://vanadium.googlesource.com/release.projects.playground",
			github:       "git@github.com:vanadium/playground.git",
		},
		Mirror{
			name:         "scripts",
			googlesource: "https://vanadium.googlesource.com/scripts",
			github:       "git@github.com:vanadium/scripts.git",
		},
		Mirror{
			name:         "third_party",
			googlesource: "https://vanadium.googlesource.com/third_party",
			github:       "git@github.com:vanadium/third_party.git",
		},
		Mirror{
			name:         "www",
			googlesource: "https://vanadium.googlesource.com/www",
			github:       "git@github.com:vanadium/www.git",
		},
	}
)

type Mirror struct {
	name, googlesource, github string
}

// vanadiumGitHubMirror mirrors googlesource.com vanadium projects to
// github.com.
func vanadiumGitHubMirror(ctx *tool.Context, testName string, _ ...TestOpt) (_ *TestResult, e error) {
	// Initialize the test/task.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	root, err := util.VanadiumRoot()
	if err != nil {
		return nil, internalTestError{err, "VanadiumRoot"}
	}

	projects := filepath.Join(root, "projects")
	mode := os.FileMode(0755)
	if err := ctx.Run().MkdirAll(projects, mode); err != nil {
		return nil, internalTestError{err, "MkdirAll"}
	}

	allPassed := true
	suites := []xunit.TestSuite{}
	for _, mirror := range mirrors {
		suite, err := sync(ctx, mirror, projects)
		if err != nil {
			return nil, internalTestError{err, "sync"}
		}

		allPassed = allPassed && (suite.Failures == 0)
		suites = append(suites, *suite)
	}

	if err := xunit.CreateReport(ctx, testName, suites); err != nil {
		return nil, err
	}

	if !allPassed {
		return &TestResult{Status: TestFailed}, nil
	}

	return &TestResult{Status: TestPassed}, nil
}

func sync(ctx *tool.Context, mirror Mirror, projects string) (*xunit.TestSuite, error) {
	suite := xunit.TestSuite{Name: mirror.name}
	dirname := filepath.Join(projects, mirror.name)

	// If dirname does not exist `git clone` otherwise `git pull`.
	if _, err := os.Stat(dirname); err != nil {
		if !os.IsNotExist(err) {
			return nil, internalTestError{err, "Stat"}
		}

		err := clone(ctx, mirror, projects)
		testCase := makeTestCase("clone", err)
		if err != nil {
			suite.Failures++
		}
		suite.Cases = append(suite.Cases, *testCase)
	} else {
		err := pull(ctx, mirror, projects)
		testCase := makeTestCase("pull", err)
		if err != nil {
			suite.Failures++
		}
		suite.Cases = append(suite.Cases, *testCase)
	}

	err := push(ctx, mirror, projects)
	testCase := makeTestCase("push", err)
	if err != nil {
		suite.Failures++
	}
	suite.Cases = append(suite.Cases, *testCase)

	return &suite, nil
}

func makeTestCase(action string, err error) *xunit.TestCase {
	c := xunit.TestCase{
		Classname: "git",
		Name:      action,
	}

	if err != nil {
		f := xunit.Failure{
			Message: "git error",
			Data:    fmt.Sprintf("%v", err),
		}
		c.Failures = append(c.Failures, f)
	}

	return &c
}

func clone(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	return ctx.Git().Clone(mirror.googlesource, dirname)
}

func pull(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	opts := tool.RootDirOpt(dirname)
	return ctx.Git(opts).Pull("origin", "master")
}

func push(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	opts := tool.RootDirOpt(dirname)
	return ctx.Git(opts).Push(mirror.github, "master")
}
<commit_msg>devtools: fixing github-mirror target after making java and jni projects private<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package testutil

import (
	"fmt"
	"os"
	"path/filepath"

	"v.io/x/devtools/internal/collect"
	"v.io/x/devtools/internal/tool"
	"v.io/x/devtools/internal/util"
	"v.io/x/devtools/internal/xunit"
)

var (
	mirrors = []Mirror{
		Mirror{
			name:         "environment",
			googlesource: "https://vanadium.googlesource.com/environment",
			github:       "git@github.com:vanadium/environment.git",
		},
		Mirror{
			name:         "browser",
			googlesource: "https://vanadium.googlesource.com/browser",
			github:       "git@github.com:vanadium/browser.git",
		},
		Mirror{
			name:         "go.v23",
			googlesource: "https://vanadium.googlesource.com/release.go.v23",
			github:       "git@github.com:vanadium/go.v23.git",
		},
		Mirror{
			name:         "go.devtools",
			googlesource: "https://vanadium.googlesource.com/release.go.x.devtools",
			github:       "git@github.com:vanadium/go.devtools.git",
		},
		Mirror{
			name:         "go.lib",
			googlesource: "https://vanadium.googlesource.com/release.go.x.lib",
			github:       "git@github.com:vanadium/go.lib.git",
		},
		Mirror{
			name:         "go.ref",
			googlesource: "https://vanadium.googlesource.com/release.go.x.ref",
			github:       "git@github.com:vanadium/go.ref.git",
		},
		Mirror{
			name:         "js",
			googlesource: "https://vanadium.googlesource.com/release.js.core",
			github:       "git@github.com:vanadium/js.git",
		},
		Mirror{
			name:         "chat",
			googlesource: "https://vanadium.googlesource.com/release.projects.chat",
			github:       "git@github.com:vanadium/chat.git",
		},
		Mirror{
			name:         "pipe2browser",
			googlesource: "https://vanadium.googlesource.com/release.projects.pipe2browser",
			github:       "git@github.com:vanadium/pipe2browser.git",
		},
		Mirror{
			name:         "playground",
			googlesource: "https://vanadium.googlesource.com/release.projects.playground",
			github:       "git@github.com:vanadium/playground.git",
		},
		Mirror{
			name:         "scripts",
			googlesource: "https://vanadium.googlesource.com/scripts",
			github:       "git@github.com:vanadium/scripts.git",
		},
		Mirror{
			name:         "third_party",
			googlesource: "https://vanadium.googlesource.com/third_party",
			github:       "git@github.com:vanadium/third_party.git",
		},
		Mirror{
			name:         "www",
			googlesource: "https://vanadium.googlesource.com/www",
			github:       "git@github.com:vanadium/www.git",
		},
	}
)

type Mirror struct {
	name, googlesource, github string
}

// vanadiumGitHubMirror mirrors googlesource.com vanadium projects to
// github.com.
func vanadiumGitHubMirror(ctx *tool.Context, testName string, _ ...TestOpt) (_ *TestResult, e error) {
	// Initialize the test/task.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	root, err := util.VanadiumRoot()
	if err != nil {
		return nil, internalTestError{err, "VanadiumRoot"}
	}

	projects := filepath.Join(root, "projects")
	mode := os.FileMode(0755)
	if err := ctx.Run().MkdirAll(projects, mode); err != nil {
		return nil, internalTestError{err, "MkdirAll"}
	}

	allPassed := true
	suites := []xunit.TestSuite{}
	for _, mirror := range mirrors {
		suite, err := sync(ctx, mirror, projects)
		if err != nil {
			return nil, internalTestError{err, "sync"}
		}

		allPassed = allPassed && (suite.Failures == 0)
		suites = append(suites, *suite)
	}

	if err := xunit.CreateReport(ctx, testName, suites); err != nil {
		return nil, err
	}

	if !allPassed {
		return &TestResult{Status: TestFailed}, nil
	}

	return &TestResult{Status: TestPassed}, nil
}

func sync(ctx *tool.Context, mirror Mirror, projects string) (*xunit.TestSuite, error) {
	suite := xunit.TestSuite{Name: mirror.name}
	dirname := filepath.Join(projects, mirror.name)

	// If dirname does not exist `git clone` otherwise `git pull`.
	if _, err := os.Stat(dirname); err != nil {
		if !os.IsNotExist(err) {
			return nil, internalTestError{err, "Stat"}
		}

		err := clone(ctx, mirror, projects)
		testCase := makeTestCase("clone", err)
		if err != nil {
			suite.Failures++
		}
		suite.Cases = append(suite.Cases, *testCase)
	} else {
		err := pull(ctx, mirror, projects)
		testCase := makeTestCase("pull", err)
		if err != nil {
			suite.Failures++
		}
		suite.Cases = append(suite.Cases, *testCase)
	}

	err := push(ctx, mirror, projects)
	testCase := makeTestCase("push", err)
	if err != nil {
		suite.Failures++
	}
	suite.Cases = append(suite.Cases, *testCase)

	return &suite, nil
}

func makeTestCase(action string, err error) *xunit.TestCase {
	c := xunit.TestCase{
		Classname: "git",
		Name:      action,
	}

	if err != nil {
		f := xunit.Failure{
			Message: "git error",
			Data:    fmt.Sprintf("%v", err),
		}
		c.Failures = append(c.Failures, f)
	}

	return &c
}

func clone(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	return ctx.Git().Clone(mirror.googlesource, dirname)
}

func pull(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	opts := tool.RootDirOpt(dirname)
	return ctx.Git(opts).Pull("origin", "master")
}

func push(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	opts := tool.RootDirOpt(dirname)
	return ctx.Git(opts).Push(mirror.github, "master")
}
<|endoftext|>