<commit_before>package stream_test

import (
	"fmt"
	"testing"
	"time"

	"github.com/stretchr/testify/require"

	"nimona.io/pkg/config"
	"nimona.io/pkg/context"
	"nimona.io/pkg/daemon"
	"nimona.io/pkg/hyperspace/provider"
	"nimona.io/pkg/object"
	"nimona.io/pkg/peer"
	"nimona.io/pkg/sqlobjectstore"
	"nimona.io/pkg/stream"
	"nimona.io/pkg/tilde"
)

func TestSyncStrategy_Integration(t *testing.T) {
	_, c0 := provider.NewTestProvider(t, context.Background())

	d1, err := daemon.New(
		context.New(),
		daemon.WithConfigOptions(
			config.WithDefaultPath(t.TempDir()),
			config.WithDefaultListenOnLocalIPs(),
			config.WithDefaultListenOnPrivateIPs(),
			config.WithDefaultBootstraps([]peer.Shorthand{
				peer.Shorthand(fmt.Sprintf("%s@%s", c0.PublicKey, c0.Addresses[0])),
			}),
		),
	)
	require.NoError(t, err)

	m1, err := stream.NewManager(
		context.New(),
		d1.Network(),
		d1.Resolver(),
		d1.ObjectStore().(*sqlobjectstore.Store),
	)
	require.NoError(t, err)

	d2, err := daemon.New(
		context.New(),
		daemon.WithConfigOptions(
			config.WithDefaultPath(t.TempDir()),
			config.WithDefaultListenOnLocalIPs(),
			config.WithDefaultListenOnPrivateIPs(),
			config.WithDefaultBootstraps([]peer.Shorthand{
				peer.Shorthand(fmt.Sprintf("%s@%s", c0.PublicKey, c0.Addresses[0])),
			}),
		),
	)
	require.NoError(t, err)

	m2, err := stream.NewManager(
		context.New(),
		d2.Network(),
		d2.Resolver(),
		d2.ObjectStore().(*sqlobjectstore.Store),
	)
	require.NoError(t, err)

	o1 := &object.Object{
		Type:     "test",
		Metadata: object.Metadata{},
		Data: tilde.Map{
			"foo": tilde.String("bar"),
		},
	}
	h1 := o1.Hash()
	err = d1.ObjectStore().Put(o1)
	require.NoError(t, err)

	o1g, err := d1.ObjectStore().Get(h1)
	require.NoError(t, err)
	require.Equal(t, o1, o1g)

	o1r, err := d1.ObjectStore().GetByStream(h1)
	require.NoError(t, err)
	o1gs, err := object.ReadAll(o1r)
	require.Len(t, o1gs, 1)
	require.Equal(t, o1, o1gs[0])

	f1 := stream.NewTopographicalSyncStrategy(
		d1.Network(),
		d1.Resolver(),
		d1.ObjectStore(),
	)
	go f1.Serve(context.New(), m1)

	start := time.Now()
	s2 := stream.NewController(
		d2.Network(),
		d2.ObjectStore().(*sqlobjectstore.Store),
	)
	// f2 := stream.NewTopographicalSyncStrategy(
	// 	d2.Network(),
	// 	d2.Resolver(),
	// 	d2.ObjectStore(),
	// )
	n, err := m2.Fetch(context.New(), s2, h1)
	require.NoError(t, err)
	require.Equal(t, 1, n)
	fmt.Println("---", time.Since(start))
}
<commit_msg>chore(stream): improve test timings<commit_after>package stream_test

import (
	"fmt"
	"testing"
	"time"

	"github.com/stretchr/testify/require"

	"nimona.io/pkg/config"
	"nimona.io/pkg/context"
	"nimona.io/pkg/daemon"
	"nimona.io/pkg/hyperspace/provider"
	"nimona.io/pkg/object"
	"nimona.io/pkg/peer"
	"nimona.io/pkg/sqlobjectstore"
	"nimona.io/pkg/stream"
	"nimona.io/pkg/tilde"
)

func TestSyncStrategy_Integration(t *testing.T) {
	_, c0 := provider.NewTestProvider(t, context.Background())

	d1, err := daemon.New(
		context.New(),
		daemon.WithConfigOptions(
			config.WithDefaultPath(t.TempDir()),
			config.WithDefaultListenOnLocalIPs(),
			config.WithDefaultListenOnPrivateIPs(),
			config.WithDefaultBootstraps([]peer.Shorthand{
				peer.Shorthand(fmt.Sprintf("%s@%s", c0.PublicKey, c0.Addresses[0])),
			}),
		),
	)
	require.NoError(t, err)

	m1, err := stream.NewManager(
		context.New(),
		d1.Network(),
		d1.Resolver(),
		d1.ObjectStore().(*sqlobjectstore.Store),
	)
	require.NoError(t, err)

	time.Sleep(time.Second)

	d2, err := daemon.New(
		context.New(),
		daemon.WithConfigOptions(
			config.WithDefaultPath(t.TempDir()),
			config.WithDefaultListenOnLocalIPs(),
			config.WithDefaultListenOnPrivateIPs(),
			config.WithDefaultBootstraps([]peer.Shorthand{
				peer.Shorthand(fmt.Sprintf("%s@%s", c0.PublicKey, c0.Addresses[0])),
			}),
		),
	)
	require.NoError(t, err)

	m2, err := stream.NewManager(
		context.New(),
		d2.Network(),
		d2.Resolver(),
		d2.ObjectStore().(*sqlobjectstore.Store),
	)
	require.NoError(t, err)

	time.Sleep(time.Second)

	o1 := &object.Object{
		Type:     "test",
		Metadata: object.Metadata{},
		Data: tilde.Map{
			"foo": tilde.String("bar"),
		},
	}
	h1 := o1.Hash()
	err = d1.ObjectStore().Put(o1)
	require.NoError(t, err)

	o1g, err := d1.ObjectStore().Get(h1)
	require.NoError(t, err)
	require.Equal(t, o1, o1g)

	o1r, err := d1.ObjectStore().GetByStream(h1)
	require.NoError(t, err)
	o1gs, err := object.ReadAll(o1r)
	require.Len(t, o1gs, 1)
	require.Equal(t, o1, o1gs[0])

	f1 := stream.NewTopographicalSyncStrategy(
		d1.Network(),
		d1.Resolver(),
		d1.ObjectStore(),
	)
	go f1.Serve(context.New(), m1)

	start := time.Now()
	s2 := stream.NewController(
		d2.Network(),
		d2.ObjectStore().(*sqlobjectstore.Store),
	)
	// f2 := stream.NewTopographicalSyncStrategy(
	// 	d2.Network(),
	// 	d2.Resolver(),
	// 	d2.ObjectStore(),
	// )
	n, err := m2.Fetch(context.New(), s2, h1)
	require.NoError(t, err)
	require.Equal(t, 1, n)
	fmt.Println("---", time.Since(start))
}
<|endoftext|>


<commit_before>// Copyright 2016 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inhibit

import (
	"testing"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/model"

	"github.com/prometheus/alertmanager/config"
	"github.com/prometheus/alertmanager/provider"
	"github.com/prometheus/alertmanager/store"
	"github.com/prometheus/alertmanager/types"
)

var nopLogger = log.NewNopLogger()

func TestInhibitRuleHasEqual(t *testing.T) {
	t.Parallel()

	now := time.Now()
	cases := []struct {
		initial map[model.Fingerprint]*types.Alert
		equal   model.LabelNames
		input   model.LabelSet
		result  bool
	}{
		{
			// No source alerts at all.
			initial: map[model.Fingerprint]*types.Alert{},
			input:   model.LabelSet{"a": "b"},
			result:  false,
		},
		{
			// No equal labels, any source alerts satisfies the requirement.
			initial: map[model.Fingerprint]*types.Alert{1: &types.Alert{}},
			input:   model.LabelSet{"a": "b"},
			result:  true,
		},
		{
			// Matching but already resolved.
			initial: map[model.Fingerprint]*types.Alert{
				1: &types.Alert{
					Alert: model.Alert{
						Labels:   model.LabelSet{"a": "b", "b": "f"},
						StartsAt: now.Add(-time.Minute),
						EndsAt:   now.Add(-time.Second),
					},
				},
				2: &types.Alert{
					Alert: model.Alert{
						Labels:   model.LabelSet{"a": "b", "b": "c"},
						StartsAt: now.Add(-time.Minute),
						EndsAt:   now.Add(-time.Second),
					},
				},
			},
			equal:  model.LabelNames{"a", "b"},
			input:  model.LabelSet{"a": "b", "b": "c"},
			result: false,
		},
		{
			// Matching and unresolved.
			initial: map[model.Fingerprint]*types.Alert{
				1: &types.Alert{
					Alert: model.Alert{
						Labels:   model.LabelSet{"a": "b", "c": "d"},
						StartsAt: now.Add(-time.Minute),
						EndsAt:   now.Add(-time.Second),
					},
				},
				2: &types.Alert{
					Alert: model.Alert{
						Labels:   model.LabelSet{"a": "b", "c": "f"},
						StartsAt: now.Add(-time.Minute),
						EndsAt:   now.Add(time.Hour),
					},
				},
			},
			equal:  model.LabelNames{"a"},
			input:  model.LabelSet{"a": "b"},
			result: true,
		},
		{
			// Equal label does not match.
			initial: map[model.Fingerprint]*types.Alert{
				1: &types.Alert{
					Alert: model.Alert{
						Labels:   model.LabelSet{"a": "c", "c": "d"},
						StartsAt: now.Add(-time.Minute),
						EndsAt:   now.Add(-time.Second),
					},
				},
				2: &types.Alert{
					Alert: model.Alert{
						Labels:   model.LabelSet{"a": "c", "c": "f"},
						StartsAt: now.Add(-time.Minute),
						EndsAt:   now.Add(-time.Second),
					},
				},
			},
			equal:  model.LabelNames{"a"},
			input:  model.LabelSet{"a": "b"},
			result: false,
		},
	}

	for _, c := range cases {
		r := &InhibitRule{
			Equal:  map[model.LabelName]struct{}{},
			scache: store.NewAlerts(5 * time.Minute),
		}
		for _, ln := range c.equal {
			r.Equal[ln] = struct{}{}
		}
		for _, v := range c.initial {
			r.scache.Set(v)
		}

		if _, have := r.hasEqual(c.input, false); have != c.result {
			t.Errorf("Unexpected result %t, expected %t", have, c.result)
		}
	}
}

func TestInhibitRuleMatches(t *testing.T) {
	t.Parallel()

	rule1 := config.InhibitRule{
		SourceMatch: map[string]string{"s1": "1"},
		TargetMatch: map[string]string{"t1": "1"},
		Equal:       model.LabelNames{"e"},
	}
	rule2 := config.InhibitRule{
		SourceMatch: map[string]string{"s2": "1"},
		TargetMatch: map[string]string{"t2": "1"},
		Equal:       model.LabelNames{"e"},
	}
	m := types.NewMarker(prometheus.NewRegistry())
	ih := NewInhibitor(nil, []*config.InhibitRule{&rule1, &rule2}, m, nopLogger)
	now := time.Now()
	// Active alert that matches the source filter of rule1.
	sourceAlert1 := &types.Alert{
		Alert: model.Alert{
			Labels:   model.LabelSet{"s1": "1", "t1": "2", "e": "1"},
			StartsAt: now.Add(-time.Minute),
			EndsAt:   now.Add(time.Hour),
		},
	}
	// Active alert that matches the source filter _and_ the target filter of rule2.
	sourceAlert2 := &types.Alert{
		Alert: model.Alert{
			Labels:   model.LabelSet{"s2": "1", "t2": "1", "e": "1"},
			StartsAt: now.Add(-time.Minute),
			EndsAt:   now.Add(time.Hour),
		},
	}

	ih.rules[0].scache = store.NewAlerts(5 * time.Minute)
	ih.rules[0].scache.Set(sourceAlert1)
	ih.rules[1].scache = store.NewAlerts(5 * time.Minute)
	ih.rules[1].scache.Set(sourceAlert2)

	cases := []struct {
		target   model.LabelSet
		expected bool
	}{
		{
			// Matches target filter of rule1, inhibited.
			target:   model.LabelSet{"t1": "1", "e": "1"},
			expected: true,
		},
		{
			// Matches target filter of rule2, inhibited.
			target:   model.LabelSet{"t2": "1", "e": "1"},
			expected: true,
		},
		{
			// Matches target filter of rule1 (plus noise), inhibited.
			target:   model.LabelSet{"t1": "1", "t3": "1", "e": "1"},
			expected: true,
		},
		{
			// Matches target filter of rule1 plus rule2, inhibited.
			target:   model.LabelSet{"t1": "1", "t2": "1", "e": "1"},
			expected: true,
		},
		{
			// Doesn't match target filter, not inhibited.
			target:   model.LabelSet{"t1": "0", "e": "1"},
			expected: false,
		},
		{
			// Matches both source and target filters of rule1,
			// inhibited because sourceAlert1 matches only the
			// source filter of rule1.
			target:   model.LabelSet{"s1": "1", "t1": "1", "e": "1"},
			expected: true,
		},
		{
			// Matches both source and target filters of rule2,
			// inhibited because sourceAlert2 matches also both the
			// source and target filterof rule1.
			target:   model.LabelSet{"s2": "1", "t2": "1", "e": "1"},
			expected: false,
		},
		{
			// Matches target filter, equal label doesn't match, not inhibited
			target:   model.LabelSet{"t1": "1", "e": "0"},
			expected: false,
		},
	}

	for _, c := range cases {
		if actual := ih.Mutes(c.target); actual != c.expected {
			t.Errorf("Expected (*Inhibitor).Mutes(%v) to return %t but got %t", c.target, c.expected, actual)
		}
	}
}

type fakeAlerts struct {
	alerts   []*types.Alert
	finished chan struct{}
}

func newFakeAlerts(alerts []*types.Alert) *fakeAlerts {
	return &fakeAlerts{
		alerts:   alerts,
		finished: make(chan struct{}),
	}
}

func (f *fakeAlerts) GetPending() provider.AlertIterator          { return nil }
func (f *fakeAlerts) Get(model.Fingerprint) (*types.Alert, error) { return nil, nil }
func (f *fakeAlerts) Put(...*types.Alert) error                   { return nil }
func (f *fakeAlerts) Subscribe() provider.AlertIterator {
	ch := make(chan *types.Alert)
	done := make(chan struct{})
	go func() {
		for _, a := range f.alerts {
			ch <- a
		}
		// Send another (meaningless) alert to make sure that the inhibitor has
		// processed everything.
		ch <- &types.Alert{
			Alert: model.Alert{
				Labels:   model.LabelSet{},
				StartsAt: time.Now(),
			},
		}
		close(f.finished)
		<-done
	}()
	return provider.NewAlertIterator(ch, done, nil)
}

func TestInhibit(t *testing.T) {
	t.Parallel()

	now := time.Now()
	inhibitRule := func() *config.InhibitRule {
		return &config.InhibitRule{
			SourceMatch: map[string]string{"s": "1"},
			TargetMatch: map[string]string{"t": "1"},
			Equal:       model.LabelNames{"e"},
		}
	}
	// alertOne is muted by alertTwo when it is active.
	alertOne := func() *types.Alert {
		return &types.Alert{
			Alert: model.Alert{
				Labels:   model.LabelSet{"t": "1", "e": "f"},
				StartsAt: now.Add(-time.Minute),
				EndsAt:   now.Add(time.Hour),
			},
		}
	}
	alertTwo := func(resolved bool) *types.Alert {
		var end time.Time
		if resolved {
			end = now.Add(-time.Second)
		} else {
			end = now.Add(time.Hour)
		}
		return &types.Alert{
			Alert: model.Alert{
				Labels:   model.LabelSet{"s": "1", "e": "f"},
				StartsAt: now.Add(-time.Minute),
				EndsAt:   end,
			},
		}
	}

	type exp struct {
		lbls  model.LabelSet
		muted bool
	}
	for i, tc := range []struct {
		alerts   []*types.Alert
		expected []exp
	}{
		{
			// alertOne shouldn't be muted since alertTwo hasn't fired.
			alerts: []*types.Alert{alertOne()},
			expected: []exp{
				{
					lbls:  model.LabelSet{"t": "1", "e": "f"},
					muted: false,
				},
			},
		},
		{
			// alertOne should be muted by alertTwo which is active.
			alerts: []*types.Alert{alertOne(), alertTwo(false)},
			expected: []exp{
				{
					lbls:  model.LabelSet{"t": "1", "e": "f"},
					muted: true,
				},
				{
					lbls:  model.LabelSet{"s": "1", "e": "f"},
					muted: false,
				},
			},
		},
		{
			// alertOne shouldn't be muted since alertTwo is resolved.
			alerts: []*types.Alert{alertOne(), alertTwo(false), alertTwo(true)},
			expected: []exp{
				{
					lbls:  model.LabelSet{"t": "1", "e": "f"},
					muted: false,
				},
				{
					lbls:  model.LabelSet{"s": "1", "e": "f"},
					muted: false,
				},
			},
		},
	} {
		ap := newFakeAlerts(tc.alerts)
		mk := types.NewMarker(prometheus.NewRegistry())
		inhibitor := NewInhibitor(ap, []*config.InhibitRule{inhibitRule()}, mk, nopLogger)

		go func() {
			for ap.finished != nil {
				select {
				case <-ap.finished:
					ap.finished = nil
				default:
				}
			}
			inhibitor.Stop()
		}()
		inhibitor.Run()

		for _, expected := range tc.expected {
			if inhibitor.Mutes(expected.lbls) != expected.muted {
				mute := "unmuted"
				if expected.muted {
					mute = "muted"
				}
				t.Errorf("tc: %d, expected alert with labels %q to be %s", i, expected.lbls, mute)
			}
		}
	}
}
<commit_msg>Fix doc comment<commit_after>// Copyright 2016 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inhibit

import (
	"testing"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/model"

	"github.com/prometheus/alertmanager/config"
	"github.com/prometheus/alertmanager/provider"
	"github.com/prometheus/alertmanager/store"
	"github.com/prometheus/alertmanager/types"
)

var nopLogger = log.NewNopLogger()

func TestInhibitRuleHasEqual(t *testing.T) {
	t.Parallel()

	now := time.Now()
	cases := []struct {
		initial map[model.Fingerprint]*types.Alert
		equal   model.LabelNames
		input   model.LabelSet
		result  bool
	}{
		{
			// No source alerts at all.
			initial: map[model.Fingerprint]*types.Alert{},
			input:   model.LabelSet{"a": "b"},
			result:  false,
		},
		{
			// No equal labels, any source alerts satisfies the requirement.
			initial: map[model.Fingerprint]*types.Alert{1: &types.Alert{}},
			input:   model.LabelSet{"a": "b"},
			result:  true,
		},
		{
			// Matching but already resolved.
			initial: map[model.Fingerprint]*types.Alert{
				1: &types.Alert{
					Alert: model.Alert{
						Labels:   model.LabelSet{"a": "b", "b": "f"},
						StartsAt: now.Add(-time.Minute),
						EndsAt:   now.Add(-time.Second),
					},
				},
				2: &types.Alert{
					Alert: model.Alert{
						Labels:   model.LabelSet{"a": "b", "b": "c"},
						StartsAt: now.Add(-time.Minute),
						EndsAt:   now.Add(-time.Second),
					},
				},
			},
			equal:  model.LabelNames{"a", "b"},
			input:  model.LabelSet{"a": "b", "b": "c"},
			result: false,
		},
		{
			// Matching and unresolved.
			initial: map[model.Fingerprint]*types.Alert{
				1: &types.Alert{
					Alert: model.Alert{
						Labels:   model.LabelSet{"a": "b", "c": "d"},
						StartsAt: now.Add(-time.Minute),
						EndsAt:   now.Add(-time.Second),
					},
				},
				2: &types.Alert{
					Alert: model.Alert{
						Labels:   model.LabelSet{"a": "b", "c": "f"},
						StartsAt: now.Add(-time.Minute),
						EndsAt:   now.Add(time.Hour),
					},
				},
			},
			equal:  model.LabelNames{"a"},
			input:  model.LabelSet{"a": "b"},
			result: true,
		},
		{
			// Equal label does not match.
			initial: map[model.Fingerprint]*types.Alert{
				1: &types.Alert{
					Alert: model.Alert{
						Labels:   model.LabelSet{"a": "c", "c": "d"},
						StartsAt: now.Add(-time.Minute),
						EndsAt:   now.Add(-time.Second),
					},
				},
				2: &types.Alert{
					Alert: model.Alert{
						Labels:   model.LabelSet{"a": "c", "c": "f"},
						StartsAt: now.Add(-time.Minute),
						EndsAt:   now.Add(-time.Second),
					},
				},
			},
			equal:  model.LabelNames{"a"},
			input:  model.LabelSet{"a": "b"},
			result: false,
		},
	}

	for _, c := range cases {
		r := &InhibitRule{
			Equal:  map[model.LabelName]struct{}{},
			scache: store.NewAlerts(5 * time.Minute),
		}
		for _, ln := range c.equal {
			r.Equal[ln] = struct{}{}
		}
		for _, v := range c.initial {
			r.scache.Set(v)
		}

		if _, have := r.hasEqual(c.input, false); have != c.result {
			t.Errorf("Unexpected result %t, expected %t", have, c.result)
		}
	}
}

func TestInhibitRuleMatches(t *testing.T) {
	t.Parallel()

	rule1 := config.InhibitRule{
		SourceMatch: map[string]string{"s1": "1"},
		TargetMatch: map[string]string{"t1": "1"},
		Equal:       model.LabelNames{"e"},
	}
	rule2 := config.InhibitRule{
		SourceMatch: map[string]string{"s2": "1"},
		TargetMatch: map[string]string{"t2": "1"},
		Equal:       model.LabelNames{"e"},
	}
	m := types.NewMarker(prometheus.NewRegistry())
	ih := NewInhibitor(nil, []*config.InhibitRule{&rule1, &rule2}, m, nopLogger)
	now := time.Now()
	// Active alert that matches the source filter of rule1.
	sourceAlert1 := &types.Alert{
		Alert: model.Alert{
			Labels:   model.LabelSet{"s1": "1", "t1": "2", "e": "1"},
			StartsAt: now.Add(-time.Minute),
			EndsAt:   now.Add(time.Hour),
		},
	}
	// Active alert that matches the source filter _and_ the target filter of rule2.
	sourceAlert2 := &types.Alert{
		Alert: model.Alert{
			Labels:   model.LabelSet{"s2": "1", "t2": "1", "e": "1"},
			StartsAt: now.Add(-time.Minute),
			EndsAt:   now.Add(time.Hour),
		},
	}

	ih.rules[0].scache = store.NewAlerts(5 * time.Minute)
	ih.rules[0].scache.Set(sourceAlert1)
	ih.rules[1].scache = store.NewAlerts(5 * time.Minute)
	ih.rules[1].scache.Set(sourceAlert2)

	cases := []struct {
		target   model.LabelSet
		expected bool
	}{
		{
			// Matches target filter of rule1, inhibited.
			target:   model.LabelSet{"t1": "1", "e": "1"},
			expected: true,
		},
		{
			// Matches target filter of rule2, inhibited.
			target:   model.LabelSet{"t2": "1", "e": "1"},
			expected: true,
		},
		{
			// Matches target filter of rule1 (plus noise), inhibited.
			target:   model.LabelSet{"t1": "1", "t3": "1", "e": "1"},
			expected: true,
		},
		{
			// Matches target filter of rule1 plus rule2, inhibited.
			target:   model.LabelSet{"t1": "1", "t2": "1", "e": "1"},
			expected: true,
		},
		{
			// Doesn't match target filter, not inhibited.
			target:   model.LabelSet{"t1": "0", "e": "1"},
			expected: false,
		},
		{
			// Matches both source and target filters of rule1,
			// inhibited because sourceAlert1 matches only the
			// source filter of rule1.
			target:   model.LabelSet{"s1": "1", "t1": "1", "e": "1"},
			expected: true,
		},
		{
			// Matches both source and target filters of rule2,
			// not inhibited because sourceAlert2 matches also both the
			// source and target filter of rule1.
			target:   model.LabelSet{"s2": "1", "t2": "1", "e": "1"},
			expected: false,
		},
		{
			// Matches target filter, equal label doesn't match, not inhibited
			target:   model.LabelSet{"t1": "1", "e": "0"},
			expected: false,
		},
	}

	for _, c := range cases {
		if actual := ih.Mutes(c.target); actual != c.expected {
			t.Errorf("Expected (*Inhibitor).Mutes(%v) to return %t but got %t", c.target, c.expected, actual)
		}
	}
}

type fakeAlerts struct {
	alerts   []*types.Alert
	finished chan struct{}
}

func newFakeAlerts(alerts []*types.Alert) *fakeAlerts {
	return &fakeAlerts{
		alerts:   alerts,
		finished: make(chan struct{}),
	}
}

func (f *fakeAlerts) GetPending() provider.AlertIterator          { return nil }
func (f *fakeAlerts) Get(model.Fingerprint) (*types.Alert, error) { return nil, nil }
func (f *fakeAlerts) Put(...*types.Alert) error                   { return nil }
func (f *fakeAlerts) Subscribe() provider.AlertIterator {
	ch := make(chan *types.Alert)
	done := make(chan struct{})
	go func() {
		for _, a := range f.alerts {
			ch <- a
		}
		// Send another (meaningless) alert to make sure that the inhibitor has
		// processed everything.
		ch <- &types.Alert{
			Alert: model.Alert{
				Labels:   model.LabelSet{},
				StartsAt: time.Now(),
			},
		}
		close(f.finished)
		<-done
	}()
	return provider.NewAlertIterator(ch, done, nil)
}

func TestInhibit(t *testing.T) {
	t.Parallel()

	now := time.Now()
	inhibitRule := func() *config.InhibitRule {
		return &config.InhibitRule{
			SourceMatch: map[string]string{"s": "1"},
			TargetMatch: map[string]string{"t": "1"},
			Equal:       model.LabelNames{"e"},
		}
	}
	// alertOne is muted by alertTwo when it is active.
	alertOne := func() *types.Alert {
		return &types.Alert{
			Alert: model.Alert{
				Labels:   model.LabelSet{"t": "1", "e": "f"},
				StartsAt: now.Add(-time.Minute),
				EndsAt:   now.Add(time.Hour),
			},
		}
	}
	alertTwo := func(resolved bool) *types.Alert {
		var end time.Time
		if resolved {
			end = now.Add(-time.Second)
		} else {
			end = now.Add(time.Hour)
		}
		return &types.Alert{
			Alert: model.Alert{
				Labels:   model.LabelSet{"s": "1", "e": "f"},
				StartsAt: now.Add(-time.Minute),
				EndsAt:   end,
			},
		}
	}

	type exp struct {
		lbls  model.LabelSet
		muted bool
	}
	for i, tc := range []struct {
		alerts   []*types.Alert
		expected []exp
	}{
		{
			// alertOne shouldn't be muted since alertTwo hasn't fired.
			alerts: []*types.Alert{alertOne()},
			expected: []exp{
				{
					lbls:  model.LabelSet{"t": "1", "e": "f"},
					muted: false,
				},
			},
		},
		{
			// alertOne should be muted by alertTwo which is active.
			alerts: []*types.Alert{alertOne(), alertTwo(false)},
			expected: []exp{
				{
					lbls:  model.LabelSet{"t": "1", "e": "f"},
					muted: true,
				},
				{
					lbls:  model.LabelSet{"s": "1", "e": "f"},
					muted: false,
				},
			},
		},
		{
			// alertOne shouldn't be muted since alertTwo is resolved.
			alerts: []*types.Alert{alertOne(), alertTwo(false), alertTwo(true)},
			expected: []exp{
				{
					lbls:  model.LabelSet{"t": "1", "e": "f"},
					muted: false,
				},
				{
					lbls:  model.LabelSet{"s": "1", "e": "f"},
					muted: false,
				},
			},
		},
	} {
		ap := newFakeAlerts(tc.alerts)
		mk := types.NewMarker(prometheus.NewRegistry())
		inhibitor := NewInhibitor(ap, []*config.InhibitRule{inhibitRule()}, mk, nopLogger)

		go func() {
			for ap.finished != nil {
				select {
				case <-ap.finished:
					ap.finished = nil
				default:
				}
			}
			inhibitor.Stop()
		}()
		inhibitor.Run()

		for _, expected := range tc.expected {
			if inhibitor.Mutes(expected.lbls) != expected.muted {
				mute := "unmuted"
				if expected.muted {
					mute = "muted"
				}
				t.Errorf("tc: %d, expected alert with labels %q to be %s", i, expected.lbls, mute)
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/golang/protobuf/jsonpb" //nolint:golint,staticcheck
	"github.com/golang/protobuf/ptypes/empty"
	"google.golang.org/grpc"

	"github.com/GoogleContainerTools/skaffold/integration/skaffold"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/event"
	"github.com/GoogleContainerTools/skaffold/proto"
	"github.com/GoogleContainerTools/skaffold/testutil"
)

var (
	connectionRetries = 2
	readRetries       = 20
	numLogEntries     = 5
	waitTime          = 1 * time.Second
)

func TestEventsRPC(t *testing.T) {
	if testing.Short() || RunOnGCP() {
		t.Skip("skipping kind integration test")
	}

	rpcAddr := randomPort()
	setupSkaffoldWithArgs(t, "--rpc-port", rpcAddr, "--status-check=false")

	// start a grpc client and make sure we can connect properly
	var (
		conn   *grpc.ClientConn
		err    error
		client proto.SkaffoldServiceClient
	)

	// connect to the skaffold grpc server
	for i := 0; i < connectionRetries; i++ {
		conn, err = grpc.Dial(fmt.Sprintf(":%s", rpcAddr), grpc.WithInsecure())
		if err != nil {
			t.Logf("unable to establish skaffold grpc connection: retrying...")
			time.Sleep(waitTime)
			continue
		}
		defer conn.Close()

		client = proto.NewSkaffoldServiceClient(conn)
		break
	}

	if client == nil {
		t.Fatalf("error establishing skaffold grpc connection")
	}

	ctx, ctxCancel := context.WithCancel(context.Background())
	defer ctxCancel()

	// read the event log stream from the skaffold grpc server
	var stream proto.SkaffoldService_EventsClient
	for i := 0; i < readRetries; i++ {
		stream, err = client.Events(ctx, &empty.Empty{})
		if err != nil {
			t.Logf("waiting for connection...")
			time.Sleep(waitTime)
			continue
		}
	}
	if stream == nil {
		t.Fatalf("error retrieving event log: %v\n", err)
	}

	// read a preset number of entries from the event log
	var logEntries []*proto.LogEntry
	entriesReceived := 0
	for {
		entry, err := stream.Recv()
		if err != nil {
			t.Errorf("error receiving entry from stream: %s", err)
		}

		if entry != nil {
			logEntries = append(logEntries, entry)
			entriesReceived++
		}
		if entriesReceived == numLogEntries {
			break
		}
	}
	metaEntries, buildEntries, deployEntries := 0, 0, 0
	for _, entry := range logEntries {
		switch entry.Event.GetEventType().(type) {
		case *proto.Event_MetaEvent:
			metaEntries++
		case *proto.Event_BuildEvent:
			buildEntries++
		case *proto.Event_DeployEvent:
			deployEntries++
		default:
		}
	}
	// make sure we have exactly 1 meta entry, 2 deploy entries and 2 build entries
	testutil.CheckDeepEqual(t, 1, metaEntries)
	testutil.CheckDeepEqual(t, 2, deployEntries)
	testutil.CheckDeepEqual(t, 2, buildEntries)
}

func TestEventLogHTTP(t *testing.T) {
	if testing.Short() || RunOnGCP() {
		t.Skip("skipping kind integration test")
	}

	tests := []struct {
		description string
		endpoint    string
	}{
		{
			//TODO deprecate (https://github.com/GoogleContainerTools/skaffold/issues/3168)
			description: "/v1/event_log",
			endpoint:    "/v1/event_log",
		},
		{
			description: "/v1/events",
			endpoint:    "/v1/events",
		},
	}
	for _, test := range tests {
		t.Run(test.description, func(t *testing.T) {
			httpAddr := randomPort()
			setupSkaffoldWithArgs(t, "--rpc-http-port", httpAddr)
			time.Sleep(500 * time.Millisecond) // give skaffold time to process all events

			httpResponse, err := http.Get(fmt.Sprintf("http://localhost:%s%s", httpAddr, test.endpoint))
			if err != nil {
				t.Fatalf("error connecting to gRPC REST API: %s", err.Error())
			}
			defer httpResponse.Body.Close()

			numEntries := 0
			var logEntries []proto.LogEntry
			for {
				e := make([]byte, 1024)
				l, err := httpResponse.Body.Read(e)
				if err != nil {
					t.Errorf("error reading body from http response: %s", err.Error())
				}
				e = e[0:l] // remove empty bytes from slice

				// sometimes reads can encompass multiple log entries, since Read() doesn't count newlines as EOF.
				readEntries := strings.Split(string(e), "\n")
				for _, entryStr := range readEntries {
					if entryStr == "" {
						continue
					}
					var entry proto.LogEntry
					// the HTTP wrapper sticks the proto messages into a map of "result" -> message.
					// attempting to JSON unmarshal drops necessary proto information, so we just manually
					// strip the string off the response and unmarshal directly to the proto message
					entryStr = strings.Replace(entryStr, "{\"result\":", "", 1)
					entryStr = entryStr[:len(entryStr)-1]
					if err := jsonpb.UnmarshalString(entryStr, &entry); err != nil {
						t.Errorf("error converting http response to proto: %s", err.Error())
					}
					numEntries++
					logEntries = append(logEntries, entry)
				}
				if numEntries >= numLogEntries {
					break
				}
			}

			metaEntries, buildEntries, deployEntries := 0, 0, 0
			for _, entry := range logEntries {
				switch entry.Event.GetEventType().(type) {
				case *proto.Event_MetaEvent:
					metaEntries++
				case *proto.Event_BuildEvent:
					buildEntries++
				case *proto.Event_DeployEvent:
					deployEntries++
				default:
				}
			}
			// make sure we have exactly 1 meta entry, 2 deploy entries and 2 build entries
			testutil.CheckDeepEqual(t, 1, metaEntries)
			testutil.CheckDeepEqual(t, 2, deployEntries)
			testutil.CheckDeepEqual(t, 2, buildEntries)
		})
	}
}

func TestGetStateRPC(t *testing.T) {
	if testing.Short() || RunOnGCP() {
		t.Skip("skipping kind integration test")
	}

	rpcAddr := randomPort()
	// start a skaffold dev loop on an example
	setupSkaffoldWithArgs(t, "--rpc-port", rpcAddr)

	// start a grpc client and make sure we can connect properly
	var (
		conn   *grpc.ClientConn
		err    error
		client proto.SkaffoldServiceClient
	)

	for i := 0; i < connectionRetries; i++ {
		conn, err = grpc.Dial(fmt.Sprintf(":%s", rpcAddr), grpc.WithInsecure())
		if err != nil {
			t.Logf("unable to establish skaffold grpc connection: retrying...")
			time.Sleep(waitTime)
			continue
		}
		defer conn.Close()

		client = proto.NewSkaffoldServiceClient(conn)
		break
	}

	if client == nil {
		t.Fatalf("error establishing skaffold grpc connection")
	}

	ctx, ctxCancel := context.WithCancel(context.Background())
	defer ctxCancel()

	// try a few times and wait around until we see the build is complete, or fail.
	success := false
	var grpcState *proto.State
	for i := 0; i < readRetries; i++ {
		grpcState = retrieveRPCState(ctx, t, client)
		if grpcState != nil && checkBuildAndDeployComplete(*grpcState) {
			success = true
			break
		}
		time.Sleep(waitTime)
	}
	if !success {
		t.Errorf("skaffold build or deploy not complete. state: %+v\n", grpcState)
	}
}

func TestGetStateHTTP(t *testing.T) {
	if testing.Short() || RunOnGCP() {
		t.Skip("skipping kind integration test")
	}

	httpAddr := randomPort()
	setupSkaffoldWithArgs(t, "--rpc-http-port", httpAddr)
	time.Sleep(3 * time.Second) // give skaffold time to process all events

	success := false
	var httpState proto.State
	for i := 0; i < readRetries; i++ {
		httpState = retrieveHTTPState(t, httpAddr)
		if checkBuildAndDeployComplete(httpState) {
			success = true
			break
		}
		time.Sleep(waitTime)
	}
	if !success {
		t.Errorf("skaffold build or deploy not complete. state: %+v\n", httpState)
	}
}

func retrieveRPCState(ctx context.Context, t *testing.T, client proto.SkaffoldServiceClient) *proto.State {
	attempts := 0
	for {
		grpcState, err := client.GetState(ctx, &empty.Empty{})
		if err != nil {
			if attempts >= connectionRetries {
				t.Fatalf("error retrieving state: %v\n", err)
			}

			t.Logf("waiting for connection...")
			attempts++
			time.Sleep(waitTime)
			continue
		}

		return grpcState
	}
}

func retrieveHTTPState(t *testing.T, httpAddr string) proto.State {
	var httpState proto.State

	// retrieve the state via HTTP as well, and verify the result is the same
	httpResponse, err := http.Get(fmt.Sprintf("http://localhost:%s/v1/state", httpAddr))
	if err != nil {
		t.Fatalf("error connecting to gRPC REST API: %s", err.Error())
	}
	defer httpResponse.Body.Close()

	b, err := ioutil.ReadAll(httpResponse.Body)
	if err != nil {
		t.Errorf("error reading body from http response: %s", err.Error())
	}
	if err := jsonpb.UnmarshalString(string(b), &httpState); err != nil {
		t.Errorf("error converting http response to proto: %s", err.Error())
	}
	return httpState
}

func setupSkaffoldWithArgs(t *testing.T, args ...string) {
	Run(t, "testdata/dev", "sh", "-c", "echo foo > foo")

	// Run skaffold build first to fail quickly on a build failure
	skaffold.Build().InDir("testdata/dev").RunOrFail(t)

	// start a skaffold dev loop on an example
	ns, _ := SetupNamespace(t)

	skaffold.Dev(args...).InDir("testdata/dev").InNs(ns.Name).RunBackground(t)

	t.Cleanup(func() {
		Run(t, "testdata/dev", "rm", "foo")
	})
}

// randomPort chooses a port in range [1024, 65535]
func randomPort() string {
	return strconv.Itoa(1024 + rand.Intn(65536-1024))
}

func checkBuildAndDeployComplete(state proto.State) bool {
	if state.BuildState == nil || state.DeployState == nil {
		return false
	}

	for _, a := range state.BuildState.Artifacts {
		if a != event.Complete {
			return false
		}
	}

	return state.DeployState.Status == event.Complete
}

func apiEvents(t *testing.T, rpcAddr string) (proto.SkaffoldServiceClient, chan *proto.LogEntry) {
	client := setupRPCClient(t, rpcAddr)

	stream, err := readEventAPIStream(client, t, readRetries)
	if stream == nil {
		t.Fatalf("error retrieving event log: %v\n", err)
	}

	// read entries from the log
	entries := make(chan *proto.LogEntry)
	go func() {
		for {
			entry, _ := stream.Recv()
			if entry != nil {
				entries <- entry
			}
		}
	}()

	return client, entries
}

func readEventAPIStream(client proto.SkaffoldServiceClient, t *testing.T, retries int) (proto.SkaffoldService_EventLogClient, error) {
	t.Helper()
	// read the event log stream from the skaffold grpc server
	var stream proto.SkaffoldService_EventLogClient
	var err error
	for i := 0; i < retries; i++ {
		stream, err = client.EventLog(context.Background())
		if err != nil {
			t.Logf("waiting for connection...")
			time.Sleep(waitTime)
			continue
		}
	}
	return stream, err
}

func setupRPCClient(t *testing.T, port string) proto.SkaffoldServiceClient {
	// start a grpc client
	var (
		conn   *grpc.ClientConn
		err    error
		client proto.SkaffoldServiceClient
	)

	// connect to the skaffold grpc server
	for i := 0; i < connectionRetries; i++ {
		conn, err = grpc.Dial(fmt.Sprintf(":%s", port), grpc.WithInsecure())
		if err != nil {
			t.Logf("unable to establish skaffold grpc connection: retrying...")
			time.Sleep(waitTime)
			continue
		}

		client = proto.NewSkaffoldServiceClient(conn)
		break
	}

	if client == nil {
		t.Fatalf("error establishing skaffold grpc connection")
	}

	t.Cleanup(func() { conn.Close() })

	return client
}
<commit_msg>disable status check in TestEventLog<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/golang/protobuf/jsonpb" //nolint:golint,staticcheck
	"github.com/golang/protobuf/ptypes/empty"
	"google.golang.org/grpc"

	"github.com/GoogleContainerTools/skaffold/integration/skaffold"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/event"
	"github.com/GoogleContainerTools/skaffold/proto"
	"github.com/GoogleContainerTools/skaffold/testutil"
)

var (
	connectionRetries = 2
	readRetries       = 20
	numLogEntries     = 5
	waitTime          = 1 * time.Second
)

func TestEventsRPC(t *testing.T) {
	if testing.Short() || RunOnGCP() {
		t.Skip("skipping kind integration test")
	}

	rpcAddr := randomPort()
	setupSkaffoldWithArgs(t, "--rpc-port", rpcAddr, "--status-check=false")

	// start a grpc client and make sure we can connect properly
	var (
		conn   *grpc.ClientConn
		err    error
		client proto.SkaffoldServiceClient
	)

	// connect to the skaffold grpc server
	for i := 0; i < connectionRetries; i++ {
		conn, err = grpc.Dial(fmt.Sprintf(":%s", rpcAddr), grpc.WithInsecure())
		if err != nil {
			t.Logf("unable to establish skaffold grpc connection: retrying...")
			time.Sleep(waitTime)
			continue
		}
		defer conn.Close()

		client = proto.NewSkaffoldServiceClient(conn)
		break
	}

	if client == nil {
		t.Fatalf("error establishing skaffold grpc connection")
	}

	ctx, ctxCancel := context.WithCancel(context.Background())
	defer ctxCancel()

	// read the event log stream from the skaffold grpc server
	var stream proto.SkaffoldService_EventsClient
	for i := 0; i < readRetries; i++ {
		stream, err = client.Events(ctx, &empty.Empty{})
		if err != nil {
			t.Logf("waiting for connection...")
			time.Sleep(waitTime)
			continue
		}
	}
	if stream == nil {
		t.Fatalf("error retrieving event log: %v\n", err)
	}

	// read a preset number of entries from the event log
	var logEntries []*proto.LogEntry
	entriesReceived := 0
	for {
		entry, err := stream.Recv()
		if err != nil {
			t.Errorf("error receiving entry from stream: %s", err)
		}

		if entry != nil {
			logEntries = append(logEntries, entry)
			entriesReceived++
		}
		if entriesReceived == numLogEntries {
			break
		}
	}
	metaEntries, buildEntries, deployEntries := 0, 0, 0
	for _, entry := range logEntries {
		switch entry.Event.GetEventType().(type) {
		case *proto.Event_MetaEvent:
			metaEntries++
		case *proto.Event_BuildEvent:
			buildEntries++
		case *proto.Event_DeployEvent:
			deployEntries++
		default:
		}
	}
	// make sure we have exactly 1 meta entry, 2 deploy entries and 2 build entries
	testutil.CheckDeepEqual(t, 1, metaEntries)
	testutil.CheckDeepEqual(t, 2, deployEntries)
	testutil.CheckDeepEqual(t, 2, buildEntries)
}

func TestEventLogHTTP(t *testing.T) {
	if testing.Short() || RunOnGCP() {
		t.Skip("skipping kind integration test")
	}

	tests := []struct {
		description string
		endpoint    string
	}{
		{
			//TODO deprecate (https://github.com/GoogleContainerTools/skaffold/issues/3168)
			description: "/v1/event_log",
			endpoint:    "/v1/event_log",
		},
		{
			description: "/v1/events",
			endpoint:    "/v1/events",
		},
	}
	for _, test := range tests {
		t.Run(test.description, func(t *testing.T) {
			httpAddr := randomPort()
			setupSkaffoldWithArgs(t, "--rpc-http-port", httpAddr, "--status-check=false")
			time.Sleep(500 * time.Millisecond) // give skaffold time to process all events

			httpResponse, err := http.Get(fmt.Sprintf("http://localhost:%s%s", httpAddr, test.endpoint))
			if err != nil {
				t.Fatalf("error connecting to gRPC REST API: %s", err.Error())
			}
			defer httpResponse.Body.Close()

			numEntries := 0
			var logEntries []proto.LogEntry
			for {
				e := make([]byte, 1024)
				l, err := httpResponse.Body.Read(e)
				if err != nil {
					t.Errorf("error reading body from http response: %s", err.Error())
				}
				e = e[0:l] // remove empty bytes from slice

				// sometimes reads can encompass multiple log entries, since Read() doesn't count newlines as EOF.
				readEntries := strings.Split(string(e), "\n")
				for _, entryStr := range readEntries {
					if entryStr == "" {
						continue
					}
					var entry proto.LogEntry
					// the HTTP wrapper sticks the proto messages into a map of "result" -> message.
					// attempting to JSON unmarshal drops necessary proto information, so we just manually
					// strip the string off the response and unmarshal directly to the proto message
					entryStr = strings.Replace(entryStr, "{\"result\":", "", 1)
					entryStr = entryStr[:len(entryStr)-1]
					if err := jsonpb.UnmarshalString(entryStr, &entry); err != nil {
						t.Errorf("error converting http response to proto: %s", err.Error())
					}
					numEntries++
					logEntries = append(logEntries, entry)
				}
				if numEntries >= numLogEntries {
					break
				}
			}

			metaEntries, buildEntries, deployEntries := 0, 0, 0
			for _, entry := range logEntries {
				switch entry.Event.GetEventType().(type) {
				case *proto.Event_MetaEvent:
					metaEntries++
				case *proto.Event_BuildEvent:
					buildEntries++
				case *proto.Event_DeployEvent:
					deployEntries++
				default:
				}
			}
			// make sure we have exactly 1 meta entry, 2 deploy entries and 2 build entries
			testutil.CheckDeepEqual(t, 1, metaEntries)
			testutil.CheckDeepEqual(t, 2, deployEntries)
			testutil.CheckDeepEqual(t, 2, buildEntries)
		})
	}
}

func TestGetStateRPC(t *testing.T) {
	if testing.Short() || RunOnGCP() {
		t.Skip("skipping kind integration test")
	}

	rpcAddr := randomPort()
	// start a skaffold dev loop on an example
	setupSkaffoldWithArgs(t, "--rpc-port", rpcAddr)

	// start a grpc client and make sure we can connect properly
	var (
		conn   *grpc.ClientConn
		err    error
		client proto.SkaffoldServiceClient
	)

	for i := 0; i < connectionRetries; i++ {
		conn, err = grpc.Dial(fmt.Sprintf(":%s", rpcAddr), grpc.WithInsecure())
		if err != nil {
			t.Logf("unable to establish skaffold grpc connection: retrying...")
			time.Sleep(waitTime)
			continue
		}
		defer conn.Close()

		client = proto.NewSkaffoldServiceClient(conn)
		break
	}

	if client == nil {
		t.Fatalf("error establishing skaffold grpc connection")
	}

	ctx, ctxCancel := context.WithCancel(context.Background())
	defer ctxCancel()

	// try a few times and wait around until we see the build is complete, or fail.
	success := false
	var grpcState *proto.State
	for i := 0; i < readRetries; i++ {
		grpcState = retrieveRPCState(ctx, t, client)
		if grpcState != nil && checkBuildAndDeployComplete(*grpcState) {
			success = true
			break
		}
		time.Sleep(waitTime)
	}
	if !success {
		t.Errorf("skaffold build or deploy not complete. state: %+v\n", grpcState)
	}
}

func TestGetStateHTTP(t *testing.T) {
	if testing.Short() || RunOnGCP() {
		t.Skip("skipping kind integration test")
	}

	httpAddr := randomPort()
	setupSkaffoldWithArgs(t, "--rpc-http-port", httpAddr)
	time.Sleep(3 * time.Second) // give skaffold time to process all events

	success := false
	var httpState proto.State
	for i := 0; i < readRetries; i++ {
		httpState = retrieveHTTPState(t, httpAddr)
		if checkBuildAndDeployComplete(httpState) {
			success = true
			break
		}
		time.Sleep(waitTime)
	}
	if !success {
		t.Errorf("skaffold build or deploy not complete. state: %+v\n", httpState)
	}
}

func retrieveRPCState(ctx context.Context, t *testing.T, client proto.SkaffoldServiceClient) *proto.State {
	attempts := 0
	for {
		grpcState, err := client.GetState(ctx, &empty.Empty{})
		if err != nil {
			if attempts >= connectionRetries {
				t.Fatalf("error retrieving state: %v\n", err)
			}

			t.Logf("waiting for connection...")
			attempts++
			time.Sleep(waitTime)
			continue
		}

		return grpcState
	}
}

func retrieveHTTPState(t *testing.T, httpAddr string) proto.State {
	var httpState proto.State

	// retrieve the state via HTTP as well, and verify the result is the same
	httpResponse, err := http.Get(fmt.Sprintf("http://localhost:%s/v1/state", httpAddr))
	if err != nil {
		t.Fatalf("error connecting to gRPC REST API: %s", err.Error())
	}
	defer httpResponse.Body.Close()

	b, err := ioutil.ReadAll(httpResponse.Body)
	if err != nil {
		t.Errorf("error reading body from http response: %s", err.Error())
	}
	if err := jsonpb.UnmarshalString(string(b), &httpState); err != nil {
		t.Errorf("error converting http response to proto: %s", err.Error())
	}
	return httpState
}

func setupSkaffoldWithArgs(t *testing.T, args ...string) {
	Run(t, "testdata/dev", "sh", "-c", "echo foo > foo")

	// Run skaffold build first to fail quickly on a build failure
	skaffold.Build().InDir("testdata/dev").RunOrFail(t)

	// start a skaffold dev loop on an example
	ns, _ := SetupNamespace(t)

	skaffold.Dev(args...).InDir("testdata/dev").InNs(ns.Name).RunBackground(t)

	t.Cleanup(func() {
		Run(t, "testdata/dev", "rm", "foo")
	})
}

// randomPort chooses a port in range [1024, 65535]
func randomPort() string {
	return strconv.Itoa(1024 + rand.Intn(65536-1024))
}

func checkBuildAndDeployComplete(state proto.State) bool {
	if state.BuildState == nil || state.DeployState == nil {
		return false
	}

	for _, a := range state.BuildState.Artifacts {
		if a != event.Complete {
			return false
		}
	}

	return state.DeployState.Status == event.Complete
}

func apiEvents(t *testing.T, rpcAddr string) (proto.SkaffoldServiceClient, chan *proto.LogEntry) {
	client := setupRPCClient(t, rpcAddr)

	stream, err := readEventAPIStream(client, t, readRetries)
	if stream == nil {
		t.Fatalf("error retrieving event log: %v\n", err)
	}

	// read entries from the log
	entries := make(chan *proto.LogEntry)
	go func() {
		for {
			entry, _ := stream.Recv()
			if entry != nil {
				entries <- entry
			}
		}
	}()

	return client, entries
}

func readEventAPIStream(client proto.SkaffoldServiceClient, t *testing.T, retries int) (proto.SkaffoldService_EventLogClient, error) {
	t.Helper()
	// read the event log stream from the skaffold grpc server
	var stream proto.SkaffoldService_EventLogClient
	var err error
	for i := 0; i < retries; i++ {
		stream, err = client.EventLog(context.Background())
		if err != nil {
			t.Logf("waiting for connection...")
			time.Sleep(waitTime)
			continue
		}
	}
	return stream, err
}

func setupRPCClient(t *testing.T, port string) proto.SkaffoldServiceClient {
	// start a grpc client
	var (
		conn   *grpc.ClientConn
		err    error
		client proto.SkaffoldServiceClient
	)

	// connect to the skaffold grpc server
	for i := 0; i < connectionRetries; i++ {
		conn, err = grpc.Dial(fmt.Sprintf(":%s", port), grpc.WithInsecure())
		if err != nil {
			t.Logf("unable to establish skaffold grpc connection: retrying...")
			time.Sleep(waitTime)
			continue
		}

		client = proto.NewSkaffoldServiceClient(conn)
		break
	}

	if client == nil {
		t.Fatalf("error establishing skaffold grpc connection")
	}

	t.Cleanup(func() { conn.Close() })

	return client
}
<|endoftext|>


<commit_before>// +build integration

/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"testing"
	"time"

	kubernetesutil "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/clientcmd/api"
)

var gkeZone = flag.String("gke-zone", "us-central1-a", "gke zone")
var gkeClusterName = flag.String("gke-cluster-name", "integration-tests", "name of the integration test cluster")
var gcpProject = flag.String("gcp-project", "k8s-skaffold", "the gcp project where the integration test cluster lives")
var remote = flag.Bool("remote", false, "if true, run tests on a remote GKE cluster")

var client kubernetes.Interface

var context *api.Context

func TestMain(m *testing.M) {
	flag.Parse()
	if *remote {
		cmd := exec.Command("gcloud", "container", "clusters", "get-credentials", *gkeClusterName, "--zone", *gkeZone, "--project", *gcpProject)
		if err := util.RunCmd(cmd); err != nil {
			logrus.Fatalf("Error authenticating to GKE cluster stdout: %v", err)
		}
	}

	var err error
	client, err = kubernetesutil.GetClientset()
	if err != nil {
		logrus.Fatalf("Test setup error: getting kubernetes client: %s", err)
	}

	loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
	kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})

	cfg, err := kubeConfig.RawConfig()
	if err != nil {
		logrus.Fatalf("loading kubeconfig: %s", err)
	}

	context = cfg.Contexts[cfg.CurrentContext]

	exitCode := m.Run()

	// Reset default context and namespace
	if err := exec.Command("kubectl", "config", "set-context", context.Cluster, "--namespace", context.Namespace).Run(); err != nil {
		logrus.Warn(err)
	}

	os.Exit(exitCode)
}

func TestRun(t *testing.T) {
	type testObject struct {
		name string
	}

	type testRunCase struct {
		description string
		dir         string
		extraArgs   []string
		deployments []testObject
		pods        []testObject
		env         map[string]string

		remoteOnly bool
	}

	var testCases = []testRunCase{
		{
			description: "getting-started example",
			pods: []testObject{
				{
					name: "getting-started",
				},
			},
			dir: "../examples/getting-started",
		},
		{
			description: "no manifest example",
			deployments: []testObject{
				{
					name: "skaffold",
				},
			},
			dir: "../examples/no-manifest",
		},
		{
			description: "annotated getting-started example",
			pods: []testObject{
				{
					name: "getting-started",
				},
			},
			dir:       "../examples",
			extraArgs: []string{"-f", "annotated-skaffold.yaml"},
		},
		{
			description: "getting-started envTagger",
			pods: []testObject{
				{
					name: "getting-started",
				},
			},
			dir: "../examples/environment-variables",
			env: map[string]string{"FOO": "foo"},
		},
		// // Don't run this test for now. It takes awhile to download all the
		// // dependencies
		// {
		// 	description: "repository root skaffold.yaml",
		// 	pods: []testObject{
		// 		{
		// 			name:      "skaffold",
		// 			namespace: "default",
		// 		},
		// 	},
		// 	dir: "../",
		// },
		// // Add this test back if after looking at debug logs to see why is this
		// // failing. See #561 for more details.
		// {
		// 	description: "gcb builder example",
		// 	pods: []testObject{
		// 		{
		// 			name: "getting-started",
		// 		},
		// 	},
		// 	dir:        "../examples/getting-started",
		// 	extraArgs:  []string{"-p", "gcb"},
		// 	remoteOnly: true,
		// },
	}

	for _, testCase := range testCases {
		t.Run(testCase.description, func(t *testing.T) {
			if !*remote && testCase.remoteOnly {
				t.Skip("skipping remote only test")
			}

			ns, deleteNs := setupNamespace(t)
			defer deleteNs()

			args := []string{"run"}
			args = append(args, testCase.extraArgs...)
			cmd := exec.Command("skaffold", args...)
			env := os.Environ()
			for k, v := range testCase.env {
				env = append(env, fmt.Sprintf("%s=%s", k, v))
			}
			cmd.Env = env
			cmd.Dir = testCase.dir
			err := util.RunCmd(cmd)
			if err != nil {
				t.Fatalf("skaffold run: %v", err)
			}

			for _, p := range testCase.pods {
				if err := kubernetesutil.WaitForPodReady(client.CoreV1().Pods(ns.Name), p.name); err != nil {
					t.Fatalf("Timed out waiting for pod ready")
				}
			}

			for _, d := range testCase.deployments {
				if err := kubernetesutil.WaitForDeploymentToStabilize(client, ns.Name, d.name, 10*time.Minute); err != nil {
					t.Fatalf("Timed out waiting for deployment to stabilize")
				}
			}
		})
	}
}

func setupNamespace(t *testing.T) (*v1.Namespace, func()) {
	namespaceName := util.RandomID()
	ns, err := client.CoreV1().Namespaces().Create(&v1.Namespace{
		ObjectMeta: meta_v1.ObjectMeta{
			Name:      namespaceName,
			Namespace: namespaceName,
		},
	})
	if err != nil {
		t.Fatalf("creating namespace: %s", err)
	}

	kubectlCmd := exec.Command("kubectl", "config", "set-context", context.Cluster, "--namespace", ns.Name)
	if err := util.RunCmd(kubectlCmd); err != nil {
		t.Fatalf("kubectl config set-context --namespace: %v", err)
	}

	return ns, func() { client.CoreV1().Namespaces().Delete(ns.Name, &meta_v1.DeleteOptions{}); return }
}
func TestFix(t *testing.T) {
	_, deleteNs := setupNamespace(t)
	defer deleteNs()

	fixCmd := exec.Command("skaffold", "fix", "-f", "skaffold.yaml")
	fixCmd.Dir = "testdata/old-config"
	out, err := util.RunCmdOut(fixCmd)
	if err != nil {
		t.Fatalf("testing error: %v", err)
	}

	runCmd := exec.Command("skaffold", "run", "-f", "-")
	runCmd.Dir = "testdata/old-config"
	runCmd.Stdin = bytes.NewReader(out)
	err = util.RunCmd(runCmd)
	if err != nil {
		t.Fatalf("testing error: %v", err)
	}
}
<commit_msg>Restore GCB integration test<commit_after>// +build integration

/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"testing"
	"time"

	kubernetesutil "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/clientcmd/api"
)

var gkeZone = flag.String("gke-zone", "us-central1-a", "gke zone")
var gkeClusterName = flag.String("gke-cluster-name", "integration-tests", "name of the integration test cluster")
var gcpProject = flag.String("gcp-project", "k8s-skaffold", "the gcp project where the integration test cluster lives")
var remote = flag.Bool("remote", false, "if true, run tests on a remote GKE cluster")

var client kubernetes.Interface

var context *api.Context

func TestMain(m *testing.M) {
	flag.Parse()
	if *remote {
		cmd := exec.Command("gcloud", "container", "clusters", "get-credentials", *gkeClusterName, "--zone", *gkeZone, "--project", *gcpProject)
		if err := util.RunCmd(cmd); err != nil {
			logrus.Fatalf("Error authenticating to GKE cluster stdout: %v", err)
		}
	}

	var err error
	client, err = kubernetesutil.GetClientset()
	if err != nil {
		logrus.Fatalf("Test setup error: getting kubernetes client: %s", err)
	}

	loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
	kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})

	cfg, err := kubeConfig.RawConfig()
	if err != nil {
		logrus.Fatalf("loading kubeconfig: %s", err)
	}

	context = cfg.Contexts[cfg.CurrentContext]

	exitCode := m.Run()

	// Reset default context and namespace
	if err := exec.Command("kubectl", "config", "set-context", context.Cluster, "--namespace", context.Namespace).Run(); err != nil {
		logrus.Warn(err)
	}

	os.Exit(exitCode)
}

func TestRun(t *testing.T) {
	type testObject struct {
		name string
	}

	type testRunCase struct {
		description string
		dir         string
		extraArgs   []string
		deployments []testObject
		pods        []testObject
		env         map[string]string

		remoteOnly bool
	}

	var testCases = []testRunCase{
		{
			description: "getting-started example",
			pods: []testObject{
				{
					name: "getting-started",
				},
			},
			dir: "../examples/getting-started",
		},
		{
			description: "no manifest example",
			deployments: []testObject{
				{
					name: "skaffold",
				},
			},
			dir: "../examples/no-manifest",
		},
		{
			description: "annotated getting-started example",
			pods: []testObject{
				{
					name: "getting-started",
				},
			},
			dir:       "../examples",
			extraArgs: []string{"-f", "annotated-skaffold.yaml"},
		},
		{
			description: "getting-started envTagger",
			pods: []testObject{
				{
					name: "getting-started",
				},
			},
			dir: "../examples/environment-variables",
			env: map[string]string{"FOO": "foo"},
		},
		// // Don't run this test for now. It takes awhile to download all the
		// // dependencies
		// {
		// 	description: "repository root skaffold.yaml",
		// 	pods: []testObject{
		// 		{
		// 			name:      "skaffold",
		// 			namespace: "default",
		// 		},
		// 	},
		// 	dir: "../",
		// },
		{
			description: "gcb builder example",
			pods: []testObject{
				{
					name: "getting-started",
				},
			},
			dir:        "../examples/getting-started",
			extraArgs:  []string{"-p", "gcb"},
			remoteOnly: true,
		},
	}

	for _, testCase := range testCases {
		t.Run(testCase.description, func(t *testing.T) {
			if !*remote && testCase.remoteOnly {
				t.Skip("skipping remote only test")
			}

			ns, deleteNs := setupNamespace(t)
			defer deleteNs()

			args := []string{"run"}
			args = append(args, testCase.extraArgs...)
			cmd := exec.Command("skaffold", args...)
			env := os.Environ()
			for k, v := range testCase.env {
				env = append(env, fmt.Sprintf("%s=%s", k, v))
			}
			cmd.Env = env
			cmd.Dir = testCase.dir
			err := util.RunCmd(cmd)
			if err != nil {
				t.Fatalf("skaffold run: %v", err)
			}

			for _, p := range testCase.pods {
				if err := kubernetesutil.WaitForPodReady(client.CoreV1().Pods(ns.Name), p.name); err != nil {
					t.Fatalf("Timed out waiting for pod ready")
				}
			}

			for _, d := range testCase.deployments {
				if err := kubernetesutil.WaitForDeploymentToStabilize(client, ns.Name, d.name, 10*time.Minute); err != nil {
					t.Fatalf("Timed out waiting for deployment to stabilize")
				}
			}
		})
	}
}

func setupNamespace(t *testing.T) (*v1.Namespace, func()) {
	namespaceName := util.RandomID()
	ns, err := client.CoreV1().Namespaces().Create(&v1.Namespace{
		ObjectMeta: meta_v1.ObjectMeta{
			Name:      namespaceName,
			Namespace: namespaceName,
		},
	})
	if err != nil {
		t.Fatalf("creating namespace: %s", err)
	}

	kubectlCmd := exec.Command("kubectl", "config", "set-context", context.Cluster, "--namespace", ns.Name)
	if err := util.RunCmd(kubectlCmd); err != nil {
		t.Fatalf("kubectl config set-context --namespace: %v", err)
	}

	return ns, func() { client.CoreV1().Namespaces().Delete(ns.Name, &meta_v1.DeleteOptions{}); return }
}
func TestFix(t *testing.T) {
	_, deleteNs := setupNamespace(t)
	defer deleteNs()

	fixCmd := exec.Command("skaffold", "fix", "-f", "skaffold.yaml")
	fixCmd.Dir = "testdata/old-config"
	out, err := util.RunCmdOut(fixCmd)
	if err != nil {
		t.Fatalf("testing error: %v", err)
	}

	runCmd := exec.Command("skaffold", "run", "-f", "-")
	runCmd.Dir = "testdata/old-config"
	runCmd.Stdin = bytes.NewReader(out)
	err = util.RunCmd(runCmd)
	if err != nil {
		t.Fatalf("testing error: %v", err)
	}
}
<|endoftext|>


<commit_before>package command

const manual = `NAME
     noti - trigger notifications when a process completes

SYNOPSIS
     noti [options] [utility [args...]]

OPTIONS
    -t <string>, --title <string>
        Notification title. Default is utility name.
    -m <string>, --message <string>
        Notification message. Default is "Done!"
    -w <pid>, --pwatch <pid>
        Trigger notification after PID disappears.

    -b, --banner
        Trigger a banner notification. Default is true. To disable this
        notification set this flag to false.
    -s, --speech
        Trigger a speech notification. Optionally, customize the voice with
        NOTI_VOICE.

    -c, --bearychat
        Trigger a BearyChat notification. Requries NOTI_BC_INCOMING_URI
        to be set.
    -i, --hipchat
        Trigger a HipChat notification. Requires NOTI_HIPCHAT_TOK and
        NOTI_HIPCHAT_DEST to be set.
    -p, --pushbullet
        Trigger a Pushbullet notification. Requires NOTI_PUSHBULLET_TOK to
        be set.
    -o, --pushover
        Trigger a Pushover notification. Requires NOTI_PUSHOVER_TOK and
        NOTI_PUSHOVER_DEST to be set.
    -u, --pushsafer
        Trigger a Pushsafer notification. Requires NOTI_PUSHSAFER_KEY
        to be set.
    -l, --simplepush
        Trigger a Simplepush notification. Requires NOTI_SIMPLEPUSH_KEY
        to be set. Optionally, customize ringtone and vibration with
        NOTI_SIMPLEPUSH_EVENT.
    -k, --slack
        Trigger a Slack notification. Requires NOTI_SLACK_TOK and
        NOTI_SLACK_DEST to be set.

    -v, --version
        Print noti version and exit.
    -h, --help
        Display help information and exit.

ENVIRONMENT
    NOTI_DEFAULT
        Notification types noti should trigger in a space-delimited list. For
        example, set NOTI_DEFAULT="banner speech pushbullet slack" to enable
        all available notifications to fire sequentially.
    NOTI_BC_INCOMING_URI
        BearyChat incoming URI.
    NOTI_HIPCHAT_TOK
        HipChat access token. Log into your HipChat account and retrieve a token
        from the Room Notification Tokens page.
    NOTI_HIPCHAT_DEST
        HipChat message destination. Can be either a Room name or ID.
    NOTI_PUSHBULLET_TOK
        Pushbullet access token. Log into your Pushbullet account and retrieve a
        token from the Account Settings page.
    NOTI_PUSHOVER_TOK
        Pushover access token. Log into your Pushover account and create a
        token from the Create New Application/Plugin page.
    NOTI_PUSHOVER_DEST
        Pushover message destination. Should be your User Key.
    NOTI_PUSHSAFER_KEY
        Pushsafer private or alias key. Log into your Pushsafer account and note
        your private or alias key.
    NOTI_SIMPLEPUSH_KEY
        Simplepush key. Install the Simplepush app and retrieve your key there.
    NOTI_SLACK_TOK
        Slack access token. Log into your Slack account and retrieve a token
        from the Slack Web API page.
    NOTI_SLACK_DEST
        Slack message destination. Can be either a #channel or a @username.%s

EXAMPLES
    Display a notification when tar finishes compressing files.
        noti tar -cjf music.tar.bz2 Music/
    You can also add noti after a command, in case you forgot at the beginning.
        clang foo.c -Wall -lm -L/usr/X11R6/lib -lX11 -o bizz; noti
`

const osxManual = `
    NOTI_SOUND
        Banner success sound. Default is Ping. Possible options are Basso, Blow,
        Bottle, Frog, Funk, Glass, Hero, Morse, Ping, Pop, Purr, Sosumi,
        Submarine, Tink. See /System/Library/Sounds for available sounds.
    NOTI_SOUND_FAIL
        Banner failure sound. Default is Basso. Possible options are Basso,
        Blow, Bottle, Frog, Funk, Glass, Hero, Morse, Ping, Pop, Purr, Sosumi,
        Submarine, Tink. See /System/Library/Sounds for available sounds.
    NOTI_VOICE
        Name of voice used for speech notifications. See "say -v ?" for
        available voices.
BUGS
    Banner notifications don't fire in tmux.
    Clicking on banner notifications causes unexpected behavior.`

const linuxFreeBSDManual = `
    NOTI_VOICE
        Name of voice used for speech notifications. See "espeak --voices" for
        available voices.`
<commit_msg>Add link to bugs on GitHub<commit_after>package command

const manual = `NAME
     noti - trigger notifications when a process completes

SYNOPSIS
     noti [options] [utility [args...]]

OPTIONS
    -t <string>, --title <string>
        Notification title. Default is utility name.
    -m <string>, --message <string>
        Notification message. Default is "Done!"
    -w <pid>, --pwatch <pid>
        Trigger notification after PID disappears.

    -b, --banner
        Trigger a banner notification. Default is true. To disable this
        notification set this flag to false.
    -s, --speech
        Trigger a speech notification. Optionally, customize the voice with
        NOTI_VOICE.

    -c, --bearychat
        Trigger a BearyChat notification. Requries NOTI_BC_INCOMING_URI
        to be set.
    -i, --hipchat
        Trigger a HipChat notification. Requires NOTI_HIPCHAT_TOK and
        NOTI_HIPCHAT_DEST to be set.
    -p, --pushbullet
        Trigger a Pushbullet notification. Requires NOTI_PUSHBULLET_TOK to
        be set.
    -o, --pushover
        Trigger a Pushover notification. Requires NOTI_PUSHOVER_TOK and
        NOTI_PUSHOVER_DEST to be set.
    -u, --pushsafer
        Trigger a Pushsafer notification. Requires NOTI_PUSHSAFER_KEY
        to be set.
    -l, --simplepush
        Trigger a Simplepush notification. Requires NOTI_SIMPLEPUSH_KEY
        to be set. Optionally, customize ringtone and vibration with
        NOTI_SIMPLEPUSH_EVENT.
    -k, --slack
        Trigger a Slack notification. Requires NOTI_SLACK_TOK and
        NOTI_SLACK_DEST to be set.

    -v, --version
        Print noti version and exit.
    -h, --help
        Display help information and exit.

ENVIRONMENT
    NOTI_DEFAULT
        Notification types noti should trigger in a space-delimited list. For
        example, set NOTI_DEFAULT="banner speech pushbullet slack" to enable
        all available notifications to fire sequentially.
    NOTI_BC_INCOMING_URI
        BearyChat incoming URI.
    NOTI_HIPCHAT_TOK
        HipChat access token. Log into your HipChat account and retrieve a token
        from the Room Notification Tokens page.
    NOTI_HIPCHAT_DEST
        HipChat message destination. Can be either a Room name or ID.
    NOTI_PUSHBULLET_TOK
        Pushbullet access token. Log into your Pushbullet account and retrieve a
        token from the Account Settings page.
    NOTI_PUSHOVER_TOK
        Pushover access token. Log into your Pushover account and create a
        token from the Create New Application/Plugin page.
    NOTI_PUSHOVER_DEST
        Pushover message destination. Should be your User Key.
    NOTI_PUSHSAFER_KEY
        Pushsafer private or alias key. Log into your Pushsafer account and note
        your private or alias key.
    NOTI_SIMPLEPUSH_KEY
        Simplepush key. Install the Simplepush app and retrieve your key there.
    NOTI_SLACK_TOK
        Slack access token. Log into your Slack account and retrieve a token
        from the Slack Web API page.
    NOTI_SLACK_DEST
        Slack message destination. Can be either a #channel or a @username.%s

EXAMPLES
    Display a notification when tar finishes compressing files.
        noti tar -cjf music.tar.bz2 Music/
    You can also add noti after a command, in case you forgot at the beginning.
        clang foo.c -Wall -lm -L/usr/X11R6/lib -lX11 -o bizz; noti
`

const osxManual = `
    NOTI_SOUND
        Banner success sound. Default is Ping. Possible options are Basso, Blow,
        Bottle, Frog, Funk, Glass, Hero, Morse, Ping, Pop, Purr, Sosumi,
        Submarine, Tink. See /System/Library/Sounds for available sounds.
    NOTI_SOUND_FAIL
        Banner failure sound. Default is Basso. Possible options are Basso,
        Blow, Bottle, Frog, Funk, Glass, Hero, Morse, Ping, Pop, Purr, Sosumi,
        Submarine, Tink. See /System/Library/Sounds for available sounds.
    NOTI_VOICE
        Name of voice used for speech notifications. See "say -v ?" for
        available voices.
BUGS
    See GitHub for latest bugs.
    https://github.com/variadico/noti/issues?q=is%3Aissue+is%3Aopen+label%3Abug`

const linuxFreeBSDManual = `
    NOTI_VOICE
        Name of voice used for speech notifications. See "espeak --voices" for
        available voices.`
<|endoftext|>


<commit_before>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package entry

import (
	"fmt"
	"net/http"
	"regexp"
	"sort"
	"strings"
)

// 表示正则路由中，表达式的起止字符
const (
	RegexpStart = '{'
	RegexpEnd   = '}'
)

// 表示 Entry 接口的类型
const (
	TypeBasic = iota + 1
	TypeStatic
	TypeRegexp
)

// Entry 表示一类资源的进入点，拥有统一的路由匹配模式。
type Entry interface {
	// 返回路由的匹配字符串
	Pattern() string

	// url 与当前的匹配程度：
	//  -1 表示完全不匹配；
	//  0  表示完全匹配；
	//  >0 表示部分匹配，值越小表示匹配程度越高。
	Match(url string) int

	// 获取路由中的参数，非正则匹配返回 nil。
	Params(url string) map[string]string

	// 接口的实现类型
	Type() int

	// 获取指定请求方法对应的 http.Handler 实例，若不存在，则返回 nil。
	Handler(method string) http.Handler

	// 添加请求方法及其对应的处理函数。
	//
	// 若已经存在，则返回错误。
	// 若 method == http.MethodOptions，则可以去覆盖默认的处理方式。
	Add(handler http.Handler, methods ...string) error

	// 移除指定方法的处理池数。若 Entry 中已经没有任何 http.Handler，则返回 true
	//
	// 可以通过指定 http.MethodOptions 的方式，来强制删除 OPTIONS 请求方法的处理。
	Remove(method ...string) (empty bool)

	// 手动设置 OPTIONS 的 Allow 报头。不调用此函数，会自动根据
	// 当前的 Add 和 Remove 调整 Allow 报头，调用 SetAll() 之后，
	// 这些自动设置不再启作用。
	SetAllow(string)
}

// 最基本的字符串匹配，只能全字符串匹配。
type basic struct {
	*items
	pattern string
}

// 静态文件匹配路由项，只要路径中的开头字符串与 pattern 相同，
// 且 pattern 以 / 结尾，即表示匹配成功。根据 match() 的返回值来确定哪个最匹配。
type static struct {
	*items
	pattern string
}

// 正则表达式匹配。
type regexpr struct {
	*items
	pattern   string
	expr      *regexp.Regexp
	hasParams bool
}

func (b *basic) Pattern() string {
	return b.pattern
}

func (b *basic) Type() int {
	return TypeBasic
}

func (b *basic) Params(url string) map[string]string {
	return nil
}

func (b *basic) Match(url string) int {
	if url == b.pattern {
		return 0
	}
	return -1
}

func (s *static) Pattern() string {
	return s.pattern
}

func (s *static) Type() int {
	return TypeStatic
}

func (s *static) Params(url string) map[string]string {
	return nil
}

func (s *static) Match(url string) int {
	l := len(url) - len(s.pattern)
	if l < 0 {
		return -1
	}

	// 由 New 函数确保 s.pattern 都是以 '/' 结尾的
	if s.pattern == url[:len(s.pattern)] {
		return l
	}
	return -1
}

func (re *regexpr) Pattern() string {
	return re.pattern
}

func (re *regexpr) Type() int {
	return TypeRegexp
}

func (re *regexpr) Match(url string) int {
	loc := re.expr.FindStringIndex(url)

	if loc != nil &&
		loc[0] == 0 &&
		loc[1] == len(url) {
		return 0
	}
	return -1
}

// 将 url 与当前的表达式进行匹配，返回其命名路由参数的值。若不匹配，则返回 nil
func (re *regexpr) Params(url string) map[string]string {
	if !re.hasParams {
		return nil
	}

	// 正确匹配正则表达式，则获相关的正则表达式命名变量。
	mapped := make(map[string]string, 3)
	subexps := re.expr.SubexpNames()
	args := re.expr.FindStringSubmatch(url)
	for index, name := range subexps {
		if len(name) > 0 && index < len(args) {
			mapped[name] = args[index]
		}
	}
	return mapped
}

// New 根据内容，生成相应的 Entry 接口实例。
//
// pattern 匹配内容。
// h 对应的 http.Handler，外层调用者确保该值不能为 nil.
func New(pattern string, h http.Handler) (Entry, error) {
	strs := split(pattern)

	if len(strs) > 1 { // 正则路由
		p, hasParams, err := toPattern(strs)
		if err != nil {
			return nil, err
		}

		return &regexpr{
			items:     newItems(),
			pattern:   pattern,
			hasParams: hasParams,
			expr:      regexp.MustCompile(p),
		}, nil
	}

	if pattern[len(pattern)-1] == '/' {
		return &static{
			items:   newItems(),
			pattern: pattern,
		}, nil
	}

	return &basic{
		items:   newItems(),
		pattern: pattern,
	}, nil
}

// 将 strs 按照顺序合并成一个正则表达式
// 返回参数正则表达式的字符串，和一个 bool 值用以表式正则中是否包含了命名匹配。
func toPattern(strs []string) (string, bool, error) {
	pattern := ""
	hasParams := false
	names := []string{}

	for _, v := range strs {
		lastIndex := len(v) - 1
		if v[0] != RegexpStart || v[lastIndex] != RegexpEnd { // 普通字符串
			pattern += v
			continue
		}

		v = v[1:lastIndex] // 去掉首尾的{}符号

		index := strings.IndexByte(v, ':')
		if index < 0 { // 只存在命名，而不存在正则表达式，默认匹配[^/]
			pattern += "(?P<" + v + ">[^/]+)"
			hasParams = true
			names = append(names, v)
			continue
		}

		if index == 0 { // 不存在命名，但有正则表达式
			pattern += v[1:]
			continue
		}

		pattern += "(?P<" + v[:index] + ">" + v[index+1:] + ")"
		names = append(names, v[:index])
		hasParams = true
	}

	// 检测是否存在同名参数：
	// 先按名称排序，之后只要检测相邻两个名称是否相同即可。
	if len(names) > 1 {
		sort.Strings(names)
		for i := 1; i < len(names); i++ {
			if names[i] == names[i-1] {
				return "", false, fmt.Errorf("相同的路由参数名：%v", names[i])
			}
		}
	}
	return pattern, hasParams, nil
}

// 将 str 以 { 和 } 为分隔符进行分隔。
// 符号 { 和 } 必须成对出现，且不能嵌套，否则结果是未知的。
//  /api/{id:\\d+}/users/ ==> {"/api/", "{id:\\d+}", "/users/"}
func split(str string) []string {
	ret := []string{}

	var start, end int
	for {
		if len(str) == 0 {
			return ret
		}

		start = strings.IndexByte(str, RegexpStart)
		if start < 0 { // 不存在 start
			return append(ret, str)
		}

		end = strings.IndexByte(str[start:], RegexpEnd)
		if end < 0 { // 不存在 end
			return append(ret, str)
		}
		end++
		end += start

		if start > 0 {
			ret = append(ret, str[:start])
		}

		ret = append(ret, str[start:end])
		str = str[end:]
	}
}
<commit_msg>[internal/entry] 编译正则表达式出错是直接返回错误，而不是 panic<commit_after>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package entry

import (
	"fmt"
	"net/http"
	"regexp"
	"sort"
	"strings"
)

// 表示正则路由中，表达式的起止字符
const (
	RegexpStart = '{'
	RegexpEnd   = '}'
)

// 表示 Entry 接口的类型
const (
	TypeBasic = iota + 1
	TypeStatic
	TypeRegexp
)

// Entry 表示一类资源的进入点，拥有统一的路由匹配模式。
type Entry interface {
	// 返回路由的匹配字符串
	Pattern() string

	// url 与当前的匹配程度：
	//  -1 表示完全不匹配；
	//  0  表示完全匹配；
	//  >0 表示部分匹配，值越小表示匹配程度越高。
	Match(url string) int

	// 获取路由中的参数，非正则匹配返回 nil。
	Params(url string) map[string]string

	// 接口的实现类型
	Type() int

	// 获取指定请求方法对应的 http.Handler 实例，若不存在，则返回 nil。
	Handler(method string) http.Handler

	// 添加请求方法及其对应的处理函数。
	//
	// 若已经存在，则返回错误。
	// 若 method == http.MethodOptions，则可以去覆盖默认的处理方式。
	Add(handler http.Handler, methods ...string) error

	// 移除指定方法的处理池数。若 Entry 中已经没有任何 http.Handler，则返回 true
	//
	// 可以通过指定 http.MethodOptions 的方式，来强制删除 OPTIONS 请求方法的处理。
	Remove(method ...string) (empty bool)

	// 手动设置 OPTIONS 的 Allow 报头。不调用此函数，会自动根据
	// 当前的 Add 和 Remove 调整 Allow 报头，调用 SetAll() 之后，
	// 这些自动设置不再启作用。
	SetAllow(string)
}

// 最基本的字符串匹配，只能全字符串匹配。
type basic struct {
	*items
	pattern string
}

// 静态文件匹配路由项，只要路径中的开头字符串与 pattern 相同，
// 且 pattern 以 / 结尾，即表示匹配成功。根据 match() 的返回值来确定哪个最匹配。
type static struct {
	*items
	pattern string
}

// 正则表达式匹配。
type regexpr struct {
	*items
	pattern   string
	expr      *regexp.Regexp
	hasParams bool
}

func (b *basic) Pattern() string {
	return b.pattern
}

func (b *basic) Type() int {
	return TypeBasic
}

func (b *basic) Params(url string) map[string]string {
	return nil
}

func (b *basic) Match(url string) int {
	if url == b.pattern {
		return 0
	}
	return -1
}

func (s *static) Pattern() string {
	return s.pattern
}

func (s *static) Type() int {
	return TypeStatic
}

func (s *static) Params(url string) map[string]string {
	return nil
}

func (s *static) Match(url string) int {
	l := len(url) - len(s.pattern)
	if l < 0 {
		return -1
	}

	// 由 New 函数确保 s.pattern 都是以 '/' 结尾的
	if s.pattern == url[:len(s.pattern)] {
		return l
	}
	return -1
}

func (re *regexpr) Pattern() string {
	return re.pattern
}

func (re *regexpr) Type() int {
	return TypeRegexp
}

func (re *regexpr) Match(url string) int {
	loc := re.expr.FindStringIndex(url)

	if loc != nil &&
		loc[0] == 0 &&
		loc[1] == len(url) {
		return 0
	}
	return -1
}

// 将 url 与当前的表达式进行匹配，返回其命名路由参数的值。若不匹配，则返回 nil
func (re *regexpr) Params(url string) map[string]string {
	if !re.hasParams {
		return nil
	}

	// 正确匹配正则表达式，则获相关的正则表达式命名变量。
	mapped := make(map[string]string, 3)
	subexps := re.expr.SubexpNames()
	args := re.expr.FindStringSubmatch(url)
	for index, name := range subexps {
		if len(name) > 0 && index < len(args) {
			mapped[name] = args[index]
		}
	}
	return mapped
}

// New 根据内容，生成相应的 Entry 接口实例。
//
// pattern 匹配内容。
// h 对应的 http.Handler，外层调用者确保该值不能为 nil.
func New(pattern string, h http.Handler) (Entry, error) {
	strs := split(pattern)

	if len(strs) > 1 { // 正则路由
		p, hasParams, err := toPattern(strs)
		if err != nil {
			return nil, err
		}

		expr, err := regexp.Compile(p)
		if err != nil {
			return nil, err
		}

		return &regexpr{
			items:     newItems(),
			pattern:   pattern,
			hasParams: hasParams,
			expr:      expr,
		}, nil
	}

	if pattern[len(pattern)-1] == '/' {
		return &static{
			items:   newItems(),
			pattern: pattern,
		}, nil
	}

	return &basic{
		items:   newItems(),
		pattern: pattern,
	}, nil
}

// 将 strs 按照顺序合并成一个正则表达式
// 返回参数正则表达式的字符串，和一个 bool 值用以表式正则中是否包含了命名匹配。
func toPattern(strs []string) (string, bool, error) {
	pattern := ""
	hasParams := false
	names := []string{}

	for _, v := range strs {
		lastIndex := len(v) - 1
		if v[0] != RegexpStart || v[lastIndex] != RegexpEnd { // 普通字符串
			pattern += v
			continue
		}

		v = v[1:lastIndex] // 去掉首尾的{}符号

		index := strings.IndexByte(v, ':')
		if index < 0 { // 只存在命名，而不存在正则表达式，默认匹配[^/]
			pattern += "(?P<" + v + ">[^/]+)"
			hasParams = true
			names = append(names, v)
			continue
		}

		if index == 0 { // 不存在命名，但有正则表达式
			pattern += v[1:]
			continue
		}

		pattern += "(?P<" + v[:index] + ">" + v[index+1:] + ")"
		names = append(names, v[:index])
		hasParams = true
	}

	// 检测是否存在同名参数：
	// 先按名称排序，之后只要检测相邻两个名称是否相同即可。
	if len(names) > 1 {
		sort.Strings(names)
		for i := 1; i < len(names); i++ {
			if names[i] == names[i-1] {
				return "", false, fmt.Errorf("相同的路由参数名：%v", names[i])
			}
		}
	}
	return pattern, hasParams, nil
}

// 将 str 以 { 和 } 为分隔符进行分隔。
// 符号 { 和 } 必须成对出现，且不能嵌套，否则结果是未知的。
//  /api/{id:\\d+}/users/ ==> {"/api/", "{id:\\d+}", "/users/"}
func split(str string) []string {
	ret := []string{}

	var start, end int
	for {
		if len(str) == 0 {
			return ret
		}

		start = strings.IndexByte(str, RegexpStart)
		if start < 0 { // 不存在 start
			return append(ret, str)
		}

		end = strings.IndexByte(str[start:], RegexpEnd)
		if end < 0 { // 不存在 end
			return append(ret, str)
		}
		end++
		end += start

		if start > 0 {
			ret = append(ret, str[:start])
		}

		ret = append(ret, str[start:end])
		str = str[end:]
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

import (
	"context"
	"errors"
	"expvar"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sync"
	"time"

	"github.com/golang/glog"

	"github.com/google/mtail/internal/logline"
	"github.com/google/mtail/internal/tailer/logstream"
	"github.com/google/mtail/internal/waker"
)

var (
	// logCount records the number of logs that are being tailed
	logCount = expvar.NewInt("log_count")
)

// Tailer polls the filesystem for log sources that match given
// `LogPathPatterns` and creates `LogStream`s to tail them.
type Tailer struct {
	ctx   context.Context
	wg    sync.WaitGroup // Wait for our subroutines to finish
	lines chan<- *logline.LogLine

	globPatternsMu     sync.RWMutex        // protects `globPatterns'
	globPatterns       map[string]struct{} // glob patterns to match newly created logs in dir paths against
	ignoreRegexPattern *regexp.Regexp

	oneShot bool

	pollMu sync.Mutex // protects Poll()

	logstreamPollWaker waker.Waker                    // Used for waking idle logstreams
	logstreamsMu       sync.RWMutex                   // protects `logstreams`.
	logstreams         map[string]logstream.LogStream // Map absolte pathname to logstream reading that pathname.

	initDone chan struct{}
}

// Option configures a new Tailer.
type Option interface {
	apply(*Tailer) error
}

type niladicOption struct {
	applyfunc func(*Tailer) error
}

func (n *niladicOption) apply(t *Tailer) error {
	return n.applyfunc(t)
}

// OneShot puts the tailer in one-shot mode, where sources are read once from the start and then closed.
var OneShot = &niladicOption{func(t *Tailer) error { t.oneShot = true; return nil }}

// LogPatterns sets the glob patterns to use to match pathnames.
type LogPatterns []string

func (opt LogPatterns) apply(t *Tailer) error {
	for _, p := range opt {
		if err := t.AddPattern(p); err != nil {
			return err
		}
	}
	return nil
}

// IgnoreRegex sets the regular expression to use to filter away pathnames that match the LogPatterns glob
type IgnoreRegex string

func (opt IgnoreRegex) apply(t *Tailer) error {
	t.SetIgnorePattern(string(opt))
	return nil
}

// StaleLogGcWaker triggers garbage collection runs for stale logs in the tailer.
func StaleLogGcWaker(w waker.Waker) Option {
	return &staleLogGcWaker{w}
}

type staleLogGcWaker struct {
	waker.Waker
}

func (opt staleLogGcWaker) apply(t *Tailer) error {
	t.StartGcLoop(opt.Waker)
	return nil
}

// LogPatternPollWaker triggers polls on the filesystem for new logs that match the log glob patterns.
func LogPatternPollWaker(w waker.Waker) Option {
	return &logPatternPollWaker{w}
}

type logPatternPollWaker struct {
	waker.Waker
}

func (opt logPatternPollWaker) apply(t *Tailer) error {
	t.StartLogPatternPollLoop(opt.Waker)
	return nil
}

// LogstreamPollWaker wakes idle logstreams.
func LogstreamPollWaker(w waker.Waker) Option {
	return &logstreamPollWaker{w}
}

type logstreamPollWaker struct {
	waker.Waker
}

func (opt logstreamPollWaker) apply(t *Tailer) error {
	t.logstreamPollWaker = opt.Waker
	return nil
}

// New creates a new Tailer.
func New(ctx context.Context, wg *sync.WaitGroup, lines chan<- *logline.LogLine, options ...Option) (*Tailer, error) {
	if lines == nil {
		return nil, errors.New("Tailer needs a lines channel")
	}
	t := &Tailer{
		ctx:          ctx,
		lines:        lines,
		initDone:     make(chan struct{}),
		globPatterns: make(map[string]struct{}),
		logstreams:   make(map[string]logstream.LogStream),
	}
	defer close(t.initDone)
	if err := t.SetOption(options...); err != nil {
		return nil, err
	}
	if len(t.globPatterns) == 0 {
		glog.Info("No patterns to tail, tailer done.")
		close(t.lines)
		return t, nil
	}
	// Guarantee all existing logs get tailed before we leave.  Also necessary
	// in case oneshot mode is active, the logs get read!
	if err := t.PollLogPatterns(); err != nil {
		return nil, err
	}
	// Setup for shutdown, once all routines are finished.
	wg.Add(1)
	go func() {
		defer wg.Done()
		<-t.initDone
		// We need to wait for context.Done() before we wait for the subbies
		// because we don't know how many are running at any point -- as soon
		// as t.wg.Wait begins the number of waited-on goroutines is fixed, and
		// we may end up leaking a LogStream goroutine and it'll try to send on
		// a closed channel as a result.  But in tests and oneshot, we want to
		// make sure the whole log gets read so we can't wait on context.Done
		// here.
		if !t.oneShot {
			<-t.ctx.Done()
		}
		t.wg.Wait()
		close(t.lines)
	}()
	return t, nil
}

var ErrNilOption = errors.New("nil option supplied")

// SetOption takes one or more option functions and applies them in order to Tailer.
func (t *Tailer) SetOption(options ...Option) error {
	for _, option := range options {
		if option == nil {
			return ErrNilOption
		}
		if err := option.apply(t); err != nil {
			return err
		}
	}
	return nil
}

// AddPattern adds a pattern to the list of patterns to filter filenames against.
func (t *Tailer) AddPattern(pattern string) error {
	absPath, err := filepath.Abs(pattern)
	if err != nil {
		glog.V(2).Infof("Couldn't canonicalize path %q: %s", pattern, err)
		return err
	}
	glog.V(2).Infof("AddPattern: %s", absPath)
	t.globPatternsMu.Lock()
	t.globPatterns[absPath] = struct{}{}
	t.globPatternsMu.Unlock()
	return nil
}

func (t *Tailer) Ignore(pathname string) (bool, error) {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return false, err
	}
	fi, err := os.Stat(absPath)
	if err != nil {
		return false, err
	}
	if fi.Mode().IsDir() {
		glog.V(2).Infof("ignore path %q because it is a folder", pathname)
		return true, nil
	}
	return t.ignoreRegexPattern != nil && t.ignoreRegexPattern.MatchString(fi.Name()), nil
}

func (t *Tailer) SetIgnorePattern(pattern string) error {
	if len(pattern) == 0 {
		return nil
	}
	glog.V(2).Infof("Set filename ignore regex pattern %q", pattern)
	ignoreRegexPattern, err := regexp.Compile(pattern)
	if err != nil {
		glog.V(2).Infof("Couldn't compile regex %q: %s", pattern, err)
		fmt.Println(fmt.Sprintf("error: %v", err))
		return err
	}
	t.ignoreRegexPattern = ignoreRegexPattern
	return nil
}

// TailPath registers a filesystem pathname to be tailed.
func (t *Tailer) TailPath(pathname string) error {
	t.logstreamsMu.Lock()
	defer t.logstreamsMu.Unlock()
	if l, ok := t.logstreams[pathname]; ok {
		if !l.IsComplete() {
			glog.V(2).Infof("already got a logstream on %q", pathname)
			return nil
		}
		logCount.Add(-1) // Removing the current entry before re-adding.
		glog.V(2).Infof("Existing logstream is finished, creating a new one.")
	}
	l, err := logstream.New(t.ctx, &t.wg, t.logstreamPollWaker, pathname, t.lines, t.oneShot)
	if err != nil {
		return err
	}
	if t.oneShot {
		glog.V(2).Infof("Starting oneshot read at startup of %q", pathname)
		l.Stop()
	}
	t.logstreams[pathname] = l
	glog.Infof("Tailing %s", pathname)
	logCount.Add(1)
	return nil
}

// Gc removes logstreams that have had no reads for 24h or more.
func (t *Tailer) Gc() error {
	t.logstreamsMu.Lock()
	defer t.logstreamsMu.Unlock()
	for _, v := range t.logstreams {
		if time.Since(v.LastReadTime()) > (time.Hour * 24) {
			v.Stop()
		}
	}
	return nil
}

// StartGcLoop runs a permanent goroutine to expire metrics every duration.
func (t *Tailer) StartGcLoop(waker waker.Waker) {
	if waker == nil {
		glog.Info("Log handle expiration disabled")
		return
	}
	t.wg.Add(1)
	go func() {
		defer t.wg.Done()
		<-t.initDone
		if t.oneShot {
			glog.Info("No gc loop in oneshot mode.")
			return
		}
		//glog.Infof("Starting log handle expiry loop every %s", duration.String())
		for {
			select {
			case <-t.ctx.Done():
				return
			case <-waker.Wake():
				if err := t.Gc(); err != nil {
					glog.Info(err)
				}
			}
		}
	}()
}

// StartLogPatternPollLoop runs a permanent goroutine to poll for new log files.
func (t *Tailer) StartLogPatternPollLoop(waker waker.Waker) {
	if waker == nil {
		glog.Info("Log pattern polling disabled")
		return
	}
	t.wg.Add(1)
	go func() {
		defer t.wg.Done()
		<-t.initDone
		if t.oneShot {
			glog.Info("No polling loop in oneshot mode.")
			return
		}
		//glog.Infof("Starting log pattern poll loop every %s", duration.String())
		for {
			select {
			case <-t.ctx.Done():
				return
			case <-waker.Wake():
				if err := t.Poll(); err != nil {
					glog.Info(err)
				}
			}
		}
	}()
}

func (t *Tailer) PollLogPatterns() error {
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()
	for pattern := range t.globPatterns {
		matches, err := filepath.Glob(pattern)
		if err != nil {
			return err
		}
		glog.V(1).Infof("glob matches: %v", matches)
		for _, pathname := range matches {
			ignore, err := t.Ignore(pathname)
			if err != nil {
				return err
			}
			if ignore {
				continue
			}
			absPath, err := filepath.Abs(pathname)
			if err != nil {
				return err
			}
			glog.Infof("watched path is %q", absPath)
			if err := t.TailPath(absPath); err != nil {
				glog.Info(err)
			}
		}
	}
	return nil
}

// PollLogStreams looks at the existing paths and checks if they're already
// complete, removing it from the map if so.
func (t *Tailer) PollLogStreams() error {
	t.logstreamsMu.Lock()
	defer t.logstreamsMu.Unlock()
	for name, l := range t.logstreams {
		if l.IsComplete() {
			glog.Infof("%s is complete", name)
			delete(t.logstreams, name)
			logCount.Add(-1)
			continue
		}
	}
	return nil
}

func (t *Tailer) Poll() error {
	t.pollMu.Lock()
	defer t.pollMu.Unlock()
	if err := t.PollLogPatterns(); err != nil {
		return err
	}
	if err := t.PollLogStreams(); err != nil {
		return err
	}
	return nil
}
<commit_msg>Move a debug log message to level 2 verbosity.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

import (
	"context"
	"errors"
	"expvar"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sync"
	"time"

	"github.com/golang/glog"

	"github.com/google/mtail/internal/logline"
	"github.com/google/mtail/internal/tailer/logstream"
	"github.com/google/mtail/internal/waker"
)

var (
	// logCount records the number of logs that are being tailed
	logCount = expvar.NewInt("log_count")
)

// Tailer polls the filesystem for log sources that match given
// `LogPathPatterns` and creates `LogStream`s to tail them.
type Tailer struct {
	ctx   context.Context
	wg    sync.WaitGroup // Wait for our subroutines to finish
	lines chan<- *logline.LogLine

	globPatternsMu     sync.RWMutex        // protects `globPatterns'
	globPatterns       map[string]struct{} // glob patterns to match newly created logs in dir paths against
	ignoreRegexPattern *regexp.Regexp

	oneShot bool

	pollMu sync.Mutex // protects Poll()

	logstreamPollWaker waker.Waker                    // Used for waking idle logstreams
	logstreamsMu       sync.RWMutex                   // protects `logstreams`.
	logstreams         map[string]logstream.LogStream // Map absolte pathname to logstream reading that pathname.

	initDone chan struct{}
}

// Option configures a new Tailer.
type Option interface {
	apply(*Tailer) error
}

type niladicOption struct {
	applyfunc func(*Tailer) error
}

func (n *niladicOption) apply(t *Tailer) error {
	return n.applyfunc(t)
}

// OneShot puts the tailer in one-shot mode, where sources are read once from the start and then closed.
var OneShot = &niladicOption{func(t *Tailer) error { t.oneShot = true; return nil }}

// LogPatterns sets the glob patterns to use to match pathnames.
type LogPatterns []string

func (opt LogPatterns) apply(t *Tailer) error {
	for _, p := range opt {
		if err := t.AddPattern(p); err != nil {
			return err
		}
	}
	return nil
}

// IgnoreRegex sets the regular expression to use to filter away pathnames that match the LogPatterns glob
type IgnoreRegex string

func (opt IgnoreRegex) apply(t *Tailer) error {
	t.SetIgnorePattern(string(opt))
	return nil
}

// StaleLogGcWaker triggers garbage collection runs for stale logs in the tailer.
func StaleLogGcWaker(w waker.Waker) Option {
	return &staleLogGcWaker{w}
}

type staleLogGcWaker struct {
	waker.Waker
}

func (opt staleLogGcWaker) apply(t *Tailer) error {
	t.StartGcLoop(opt.Waker)
	return nil
}

// LogPatternPollWaker triggers polls on the filesystem for new logs that match the log glob patterns.
func LogPatternPollWaker(w waker.Waker) Option {
	return &logPatternPollWaker{w}
}

type logPatternPollWaker struct {
	waker.Waker
}

func (opt logPatternPollWaker) apply(t *Tailer) error {
	t.StartLogPatternPollLoop(opt.Waker)
	return nil
}

// LogstreamPollWaker wakes idle logstreams.
func LogstreamPollWaker(w waker.Waker) Option {
	return &logstreamPollWaker{w}
}

type logstreamPollWaker struct {
	waker.Waker
}

func (opt logstreamPollWaker) apply(t *Tailer) error {
	t.logstreamPollWaker = opt.Waker
	return nil
}

// New creates a new Tailer.
func New(ctx context.Context, wg *sync.WaitGroup, lines chan<- *logline.LogLine, options ...Option) (*Tailer, error) {
	if lines == nil {
		return nil, errors.New("Tailer needs a lines channel")
	}
	t := &Tailer{
		ctx:          ctx,
		lines:        lines,
		initDone:     make(chan struct{}),
		globPatterns: make(map[string]struct{}),
		logstreams:   make(map[string]logstream.LogStream),
	}
	defer close(t.initDone)
	if err := t.SetOption(options...); err != nil {
		return nil, err
	}
	if len(t.globPatterns) == 0 {
		glog.Info("No patterns to tail, tailer done.")
		close(t.lines)
		return t, nil
	}
	// Guarantee all existing logs get tailed before we leave.  Also necessary
	// in case oneshot mode is active, the logs get read!
	if err := t.PollLogPatterns(); err != nil {
		return nil, err
	}
	// Setup for shutdown, once all routines are finished.
	wg.Add(1)
	go func() {
		defer wg.Done()
		<-t.initDone
		// We need to wait for context.Done() before we wait for the subbies
		// because we don't know how many are running at any point -- as soon
		// as t.wg.Wait begins the number of waited-on goroutines is fixed, and
		// we may end up leaking a LogStream goroutine and it'll try to send on
		// a closed channel as a result.  But in tests and oneshot, we want to
		// make sure the whole log gets read so we can't wait on context.Done
		// here.
		if !t.oneShot {
			<-t.ctx.Done()
		}
		t.wg.Wait()
		close(t.lines)
	}()
	return t, nil
}

var ErrNilOption = errors.New("nil option supplied")

// SetOption takes one or more option functions and applies them in order to Tailer.
func (t *Tailer) SetOption(options ...Option) error {
	for _, option := range options {
		if option == nil {
			return ErrNilOption
		}
		if err := option.apply(t); err != nil {
			return err
		}
	}
	return nil
}

// AddPattern adds a pattern to the list of patterns to filter filenames against.
func (t *Tailer) AddPattern(pattern string) error {
	absPath, err := filepath.Abs(pattern)
	if err != nil {
		glog.V(2).Infof("Couldn't canonicalize path %q: %s", pattern, err)
		return err
	}
	glog.V(2).Infof("AddPattern: %s", absPath)
	t.globPatternsMu.Lock()
	t.globPatterns[absPath] = struct{}{}
	t.globPatternsMu.Unlock()
	return nil
}

func (t *Tailer) Ignore(pathname string) (bool, error) {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return false, err
	}
	fi, err := os.Stat(absPath)
	if err != nil {
		return false, err
	}
	if fi.Mode().IsDir() {
		glog.V(2).Infof("ignore path %q because it is a folder", pathname)
		return true, nil
	}
	return t.ignoreRegexPattern != nil && t.ignoreRegexPattern.MatchString(fi.Name()), nil
}

func (t *Tailer) SetIgnorePattern(pattern string) error {
	if len(pattern) == 0 {
		return nil
	}
	glog.V(2).Infof("Set filename ignore regex pattern %q", pattern)
	ignoreRegexPattern, err := regexp.Compile(pattern)
	if err != nil {
		glog.V(2).Infof("Couldn't compile regex %q: %s", pattern, err)
		fmt.Println(fmt.Sprintf("error: %v", err))
		return err
	}
	t.ignoreRegexPattern = ignoreRegexPattern
	return nil
}

// TailPath registers a filesystem pathname to be tailed.
func (t *Tailer) TailPath(pathname string) error {
	t.logstreamsMu.Lock()
	defer t.logstreamsMu.Unlock()
	if l, ok := t.logstreams[pathname]; ok {
		if !l.IsComplete() {
			glog.V(2).Infof("already got a logstream on %q", pathname)
			return nil
		}
		logCount.Add(-1) // Removing the current entry before re-adding.
		glog.V(2).Infof("Existing logstream is finished, creating a new one.")
	}
	l, err := logstream.New(t.ctx, &t.wg, t.logstreamPollWaker, pathname, t.lines, t.oneShot)
	if err != nil {
		return err
	}
	if t.oneShot {
		glog.V(2).Infof("Starting oneshot read at startup of %q", pathname)
		l.Stop()
	}
	t.logstreams[pathname] = l
	glog.Infof("Tailing %s", pathname)
	logCount.Add(1)
	return nil
}

// Gc removes logstreams that have had no reads for 24h or more.
func (t *Tailer) Gc() error {
	t.logstreamsMu.Lock()
	defer t.logstreamsMu.Unlock()
	for _, v := range t.logstreams {
		if time.Since(v.LastReadTime()) > (time.Hour * 24) {
			v.Stop()
		}
	}
	return nil
}

// StartGcLoop runs a permanent goroutine to expire metrics every duration.
func (t *Tailer) StartGcLoop(waker waker.Waker) {
	if waker == nil {
		glog.Info("Log handle expiration disabled")
		return
	}
	t.wg.Add(1)
	go func() {
		defer t.wg.Done()
		<-t.initDone
		if t.oneShot {
			glog.Info("No gc loop in oneshot mode.")
			return
		}
		//glog.Infof("Starting log handle expiry loop every %s", duration.String())
		for {
			select {
			case <-t.ctx.Done():
				return
			case <-waker.Wake():
				if err := t.Gc(); err != nil {
					glog.Info(err)
				}
			}
		}
	}()
}

// StartLogPatternPollLoop runs a permanent goroutine to poll for new log files.
func (t *Tailer) StartLogPatternPollLoop(waker waker.Waker) {
	if waker == nil {
		glog.Info("Log pattern polling disabled")
		return
	}
	t.wg.Add(1)
	go func() {
		defer t.wg.Done()
		<-t.initDone
		if t.oneShot {
			glog.Info("No polling loop in oneshot mode.")
			return
		}
		//glog.Infof("Starting log pattern poll loop every %s", duration.String())
		for {
			select {
			case <-t.ctx.Done():
				return
			case <-waker.Wake():
				if err := t.Poll(); err != nil {
					glog.Info(err)
				}
			}
		}
	}()
}

func (t *Tailer) PollLogPatterns() error {
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()
	for pattern := range t.globPatterns {
		matches, err := filepath.Glob(pattern)
		if err != nil {
			return err
		}
		glog.V(1).Infof("glob matches: %v", matches)
		for _, pathname := range matches {
			ignore, err := t.Ignore(pathname)
			if err != nil {
				return err
			}
			if ignore {
				continue
			}
			absPath, err := filepath.Abs(pathname)
			if err != nil {
				return err
			}
			glog.V(2).Infof("watched path is %q", absPath)
			if err := t.TailPath(absPath); err != nil {
				glog.Info(err)
			}
		}
	}
	return nil
}

// PollLogStreams looks at the existing paths and checks if they're already
// complete, removing it from the map if so.
func (t *Tailer) PollLogStreams() error {
	t.logstreamsMu.Lock()
	defer t.logstreamsMu.Unlock()
	for name, l := range t.logstreams {
		if l.IsComplete() {
			glog.Infof("%s is complete", name)
			delete(t.logstreams, name)
			logCount.Add(-1)
			continue
		}
	}
	return nil
}

func (t *Tailer) Poll() error {
	t.pollMu.Lock()
	defer t.pollMu.Unlock()
	if err := t.PollLogPatterns(); err != nil {
		return err
	}
	if err := t.PollLogStreams(); err != nil {
		return err
	}
	return nil
}
<|endoftext|>