<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"io"
	"io/ioutil"
	"runtime"
	"testing"
	"time"

	"github.com/GoogleContainerTools/skaffold/testutil"
)

func TestCreateTar(t *testing.T) {
	testutil.Run(t, "", func(t *testutil.T) {
		files := map[string]string{
			"foo":     "baz1",
			"bar/bat": "baz2",
			"bar/baz": "baz3",
		}
		_, paths := prepareFiles(t, files)

		var b bytes.Buffer
		err := CreateTar(&b, ".", paths)
		t.CheckNoError(err)

		// Make sure the contents match.
		tarFiles := make(map[string]string)
		tr := tar.NewReader(&b)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break
			}
			t.CheckNoError(err)

			content, err := ioutil.ReadAll(tr)
			t.CheckNoError(err)

			tarFiles[hdr.Name] = string(content)
		}

		t.CheckDeepEqual(files, tarFiles)
	})
}

func TestCreateTarWithParents(t *testing.T) {
	testutil.Run(t, "", func(t *testutil.T) {
		files := map[string]string{
			"foo":     "baz1",
			"bar/bat": "baz2",
			"bar/baz": "baz3",
		}
		_, paths := prepareFiles(t, files)

		var b bytes.Buffer
		err := CreateTarWithParents(&b, ".", paths, 1000, 1000, time.Now())
		t.CheckNoError(err)

		// Make sure the contents match.
		tarFiles := make(map[string]string)
		tr := tar.NewReader(&b)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break
			}
			t.CheckNoError(err)

			content, err := ioutil.ReadAll(tr)
			t.CheckNoError(err)

			tarFiles[hdr.Name] = string(content)
		}

		t.CheckDeepEqual(map[string]string{
			"foo":     "baz1",
			"bar":     "",
			"bar/bat": "baz2",
			"bar/baz": "baz3",
		}, tarFiles)
	})
}

func TestCreateTarGz(t *testing.T) {
	testutil.Run(t, "", func(t *testutil.T) {
		files := map[string]string{
			"foo":     "baz1",
			"bar/bat": "baz2",
			"bar/baz": "baz3",
		}
		_, paths := prepareFiles(t, files)

		var b bytes.Buffer
		err := CreateTarGz(&b, ".", paths)
		t.CheckNoError(err)

		// Make sure the contents match.
		tarFiles := make(map[string]string)
		gzr, err := gzip.NewReader(&b)
		t.CheckNoError(err)
		tr := tar.NewReader(gzr)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break
			}
			t.CheckNoError(err)

			content, err := ioutil.ReadAll(tr)
			t.CheckNoError(err)

			tarFiles[hdr.Name] = string(content)
		}

		t.CheckDeepEqual(files, tarFiles)
	})
}

func TestCreateTarSubDirectory(t *testing.T) {
	testutil.Run(t, "", func(t *testutil.T) {
		files := map[string]string{
			"sub/foo":     "baz1",
			"sub/bar/bat": "baz2",
			"sub/bar/baz": "baz3",
		}
		_, paths := prepareFiles(t, files)

		var b bytes.Buffer
		err := CreateTar(&b, "sub", paths)
		t.CheckNoError(err)

		// Make sure the contents match.
		tarFiles := make(map[string]string)
		tr := tar.NewReader(&b)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break
			}
			t.CheckNoError(err)

			content, err := ioutil.ReadAll(tr)
			t.CheckNoError(err)

			tarFiles["sub/"+hdr.Name] = string(content)
		}

		t.CheckDeepEqual(files, tarFiles)
	})
}

func TestCreateTarEmptyFolder(t *testing.T) {
	testutil.Run(t, "", func(t *testutil.T) {
		t.NewTempDir().
			Mkdir("empty").
			Chdir()

		var b bytes.Buffer
		err := CreateTar(&b, ".", []string{"empty"})
		t.CheckNoError(err)

		// Make sure the contents match.
		var tarFolders []string
		tr := tar.NewReader(&b)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break
			}
			t.CheckNoError(err)

			if hdr.FileInfo().IsDir() {
				tarFolders = append(tarFolders, hdr.Name)
			}
		}

		t.CheckNoError(err)
		t.CheckDeepEqual([]string{"empty"}, tarFolders)
	})
}

func TestCreateTarWithAbsolutePaths(t *testing.T) {
	testutil.Run(t, "", func(t *testutil.T) {
		files := map[string]string{
			"foo":     "baz1",
			"bar/bat": "baz2",
			"bar/baz": "baz3",
		}
		tmpDir, paths := prepareFiles(t, files)

		var b bytes.Buffer
		err := CreateTar(&b, tmpDir.Root(), tmpDir.Paths(paths...))
		t.CheckNoError(err)

		// Make sure the contents match.
		tarFiles := make(map[string]string)
		tr := tar.NewReader(&b)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break
			}
			t.CheckNoError(err)

			content, err := ioutil.ReadAll(tr)
			t.CheckNoError(err)

			tarFiles[hdr.Name] = string(content)
		}

		t.CheckDeepEqual(files, tarFiles)
	})
}

func TestAddFileToTarSymlinks(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("creating symlinks requires extra privileges on Windows")
	}

	testutil.Run(t, "", func(t *testutil.T) {
		files := map[string]string{
			"foo":     "baz1",
			"bar/bat": "baz2",
			"bar/baz": "baz3",
		}
		tmpDir, paths := prepareFiles(t, files)

		links := map[string]string{
			"foo.link":     "foo",
			"bat.link":     "bar/bat",
			"bat/baz.link": "bar/baz",
		}
		for link, file := range links {
			tmpDir.Symlink(file, link)
			paths = append(paths, link)
		}

		var b bytes.Buffer
		err := CreateTar(&b, tmpDir.Root(), tmpDir.Paths(paths...))
		t.CheckNoError(err)

		// Make sure the links match.
		tr := tar.NewReader(&b)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break
			}
			t.CheckNoError(err)

			if _, isFile := files[hdr.Name]; isFile {
				continue
			}

			link, isLink := links[hdr.Name]
			if !isLink {
				t.Errorf("Unexpected file/link in tar: %s", hdr.Name)
			}
			if hdr.Linkname != link {
				t.Errorf("Link destination doesn't match. %s != %s.", link, hdr.Linkname)
			}
		}
	})
}

func prepareFiles(t *testutil.T, files map[string]string) (*testutil.TempDir, []string) {
	tmpDir := t.NewTempDir().Chdir()

	var paths []string
	for path, content := range files {
		tmpDir.Write(path, content)
		paths = append(paths, path)
	}

	return tmpDir, paths
}
<commit_msg>More assertions<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"io"
	"io/ioutil"
	"runtime"
	"testing"
	"time"

	"github.com/GoogleContainerTools/skaffold/testutil"
)

func TestCreateTar(t *testing.T) {
	testutil.Run(t, "", func(t *testutil.T) {
		files := map[string]string{
			"foo":     "baz1",
			"bar/bat": "baz2",
			"bar/baz": "baz3",
		}
		_, paths := prepareFiles(t, files)

		var b bytes.Buffer
		err := CreateTar(&b, ".", paths)
		t.CheckNoError(err)

		// Make sure the contents match.
		tarFiles := make(map[string]string)
		tr := tar.NewReader(&b)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break
			}
			t.CheckNoError(err)

			content, err := ioutil.ReadAll(tr)
			t.CheckNoError(err)

			tarFiles[hdr.Name] = string(content)
		}

		t.CheckDeepEqual(files, tarFiles)
	})
}

func TestCreateTarWithParents(t *testing.T) {
	testutil.Run(t, "", func(t *testutil.T) {
		files := map[string]string{
			"foo":     "baz1",
			"bar/bat": "baz2",
			"bar/baz": "baz3",
		}
		_, paths := prepareFiles(t, files)

		var b bytes.Buffer
		err := CreateTarWithParents(&b, ".", paths, 10, 100, time.Now())
		t.CheckNoError(err)

		// Make sure the contents match.
		tarFiles := make(map[string]string)
		tr := tar.NewReader(&b)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break
			}
			t.CheckNoError(err)
			t.CheckDeepEqual(10, hdr.Uid)
			t.CheckDeepEqual(100, hdr.Gid)

			content, err := ioutil.ReadAll(tr)
			t.CheckNoError(err)

			tarFiles[hdr.Name] = string(content)
		}

		t.CheckDeepEqual(map[string]string{
			"foo":     "baz1",
			"bar":     "",
			"bar/bat": "baz2",
			"bar/baz": "baz3",
		}, tarFiles)
	})
}

func TestCreateTarGz(t *testing.T) {
	testutil.Run(t, "", func(t *testutil.T) {
		files := map[string]string{
			"foo":     "baz1",
			"bar/bat": "baz2",
			"bar/baz": "baz3",
		}
		_, paths := prepareFiles(t, files)

		var b bytes.Buffer
		err := CreateTarGz(&b, ".", paths)
		t.CheckNoError(err)

		// Make sure the contents match.
		tarFiles := make(map[string]string)
		gzr, err := gzip.NewReader(&b)
		t.CheckNoError(err)
		tr := tar.NewReader(gzr)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break
			}
			t.CheckNoError(err)

			content, err := ioutil.ReadAll(tr)
			t.CheckNoError(err)

			tarFiles[hdr.Name] = string(content)
		}

		t.CheckDeepEqual(files, tarFiles)
	})
}

func TestCreateTarSubDirectory(t *testing.T) {
	testutil.Run(t, "", func(t *testutil.T) {
		files := map[string]string{
			"sub/foo":     "baz1",
			"sub/bar/bat": "baz2",
			"sub/bar/baz": "baz3",
		}
		_, paths := prepareFiles(t, files)

		var b bytes.Buffer
		err := CreateTar(&b, "sub", paths)
		t.CheckNoError(err)

		// Make sure the contents match.
		tarFiles := make(map[string]string)
		tr := tar.NewReader(&b)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break
			}
			t.CheckNoError(err)

			content, err := ioutil.ReadAll(tr)
			t.CheckNoError(err)

			tarFiles["sub/"+hdr.Name] = string(content)
		}

		t.CheckDeepEqual(files, tarFiles)
	})
}

func TestCreateTarEmptyFolder(t *testing.T) {
	testutil.Run(t, "", func(t *testutil.T) {
		t.NewTempDir().
			Mkdir("empty").
			Chdir()

		var b bytes.Buffer
		err := CreateTar(&b, ".", []string{"empty"})
		t.CheckNoError(err)

		// Make sure the contents match.
		var tarFolders []string
		tr := tar.NewReader(&b)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break
			}
			t.CheckNoError(err)

			if hdr.FileInfo().IsDir() {
				tarFolders = append(tarFolders, hdr.Name)
			}
		}

		t.CheckNoError(err)
		t.CheckDeepEqual([]string{"empty"}, tarFolders)
	})
}

func TestCreateTarWithAbsolutePaths(t *testing.T) {
	testutil.Run(t, "", func(t *testutil.T) {
		files := map[string]string{
			"foo":     "baz1",
			"bar/bat": "baz2",
			"bar/baz": "baz3",
		}
		tmpDir, paths := prepareFiles(t, files)

		var b bytes.Buffer
		err := CreateTar(&b, tmpDir.Root(), tmpDir.Paths(paths...))
		t.CheckNoError(err)

		// Make sure the contents match.
		tarFiles := make(map[string]string)
		tr := tar.NewReader(&b)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break
			}
			t.CheckNoError(err)

			content, err := ioutil.ReadAll(tr)
			t.CheckNoError(err)

			tarFiles[hdr.Name] = string(content)
		}

		t.CheckDeepEqual(files, tarFiles)
	})
}

func TestAddFileToTarSymlinks(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("creating symlinks requires extra privileges on Windows")
	}

	testutil.Run(t, "", func(t *testutil.T) {
		files := map[string]string{
			"foo":     "baz1",
			"bar/bat": "baz2",
			"bar/baz": "baz3",
		}
		tmpDir, paths := prepareFiles(t, files)

		links := map[string]string{
			"foo.link":     "foo",
			"bat.link":     "bar/bat",
			"bat/baz.link": "bar/baz",
		}
		for link, file := range links {
			tmpDir.Symlink(file, link)
			paths = append(paths, link)
		}

		var b bytes.Buffer
		err := CreateTar(&b, tmpDir.Root(), tmpDir.Paths(paths...))
		t.CheckNoError(err)

		// Make sure the links match.
		tr := tar.NewReader(&b)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break
			}
			t.CheckNoError(err)

			if _, isFile := files[hdr.Name]; isFile {
				continue
			}

			link, isLink := links[hdr.Name]
			if !isLink {
				t.Errorf("Unexpected file/link in tar: %s", hdr.Name)
			}
			if hdr.Linkname != link {
				t.Errorf("Link destination doesn't match. %s != %s.", link, hdr.Linkname)
			}
		}
	})
}

func prepareFiles(t *testutil.T, files map[string]string) (*testutil.TempDir, []string) {
	tmpDir := t.NewTempDir().Chdir()

	var paths []string
	for path, content := range files {
		tmpDir.Write(path, content)
		paths = append(paths, path)
	}

	return tmpDir, paths
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kafka

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"golang.org/x/time/rate"

	ctrl "knative.dev/control-protocol/pkg"
	ctrlnetwork "knative.dev/control-protocol/pkg/network"

	"github.com/Shopify/sarama"
	"go.opencensus.io/trace"
	"go.uber.org/zap"

	"knative.dev/pkg/logging"
	pkgsource "knative.dev/pkg/source"

	"knative.dev/eventing/pkg/adapter/v2"
	"knative.dev/eventing/pkg/kncloudevents"

	"knative.dev/eventing-kafka/pkg/common/consumer"
	"knative.dev/eventing-kafka/pkg/source/client"
	kafkasourcecontrol "knative.dev/eventing-kafka/pkg/source/control"
)

const (
	resourceGroup = "kafkasources.sources.knative.dev"
)

type AdapterConfig struct {
	adapter.EnvConfig
	client.KafkaEnvConfig

	Topics        []string `envconfig:"KAFKA_TOPICS" required:"true"`
	ConsumerGroup string   `envconfig:"KAFKA_CONSUMER_GROUP" required:"true"`
	Name          string   `envconfig:"NAME" required:"true"`
	KeyType       string   `envconfig:"KEY_TYPE" required:"false"`

	// Turn off the control server.
	DisableControlServer bool
}

func NewEnvConfig() adapter.EnvConfigAccessor {
	return &AdapterConfig{}
}

type Adapter struct {
	config        *AdapterConfig
	controlServer *ctrlnetwork.ControlServer
	saramaConfig  *sarama.Config

	httpMessageSender *kncloudevents.HTTPMessageSender
	reporter          pkgsource.StatsReporter
	logger            *zap.SugaredLogger
	keyTypeMapper     func([]byte) interface{}
	rateLimiter       *rate.Limiter
}

var _ adapter.MessageAdapter = (*Adapter)(nil)
var _ consumer.KafkaConsumerHandler = (*Adapter)(nil)
var _ consumer.SaramaConsumerLifecycleListener = (*Adapter)(nil)
var _ adapter.MessageAdapterConstructor = NewAdapter

func NewAdapter(ctx context.Context, processed adapter.EnvConfigAccessor, httpMessageSender *kncloudevents.HTTPMessageSender, reporter pkgsource.StatsReporter) adapter.MessageAdapter {
	logger := logging.FromContext(ctx)
	config := processed.(*AdapterConfig)

	return &Adapter{
		config:            config,
		httpMessageSender: httpMessageSender,
		reporter:          reporter,
		logger:            logger,
		keyTypeMapper:     getKeyTypeMapper(config.KeyType),
	}
}
func (a *Adapter) GetConsumerGroup() string {
	return a.config.ConsumerGroup
}

func (a *Adapter) Start(ctx context.Context) (err error) {
	a.logger.Infow("Starting with config: ",
		zap.String("Topics", strings.Join(a.config.Topics, ",")),
		zap.String("ConsumerGroup", a.config.ConsumerGroup),
		zap.String("SinkURI", a.config.Sink),
		zap.String("Name", a.config.Name),
		zap.String("Namespace", a.config.Namespace),
	)

	// Init control service
	if !a.config.DisableControlServer {
		a.controlServer, err = ctrlnetwork.StartInsecureControlServer(ctx)
		if err != nil {
			return err
		}
		a.controlServer.MessageHandler(a)
	}

	// init consumer group
	addrs, config, err := client.NewConfigWithEnv(context.Background(), &a.config.KafkaEnvConfig)
	if err != nil {
		return fmt.Errorf("failed to create the config: %w", err)
	}
	a.saramaConfig = config

	options := []consumer.SaramaConsumerHandlerOption{consumer.WithSaramaConsumerLifecycleListener(a)}
	consumerGroupFactory := consumer.NewConsumerGroupFactory(addrs, config)
	group, err := consumerGroupFactory.StartConsumerGroup(
		a.config.ConsumerGroup,
		a.config.Topics,
		a.logger,
		a,
		options...,
	)
	if err != nil {
		return fmt.Errorf("failed to start consumer group: %w", err)
	}
	defer func() {
		err := group.Close()
		if err != nil {
			a.logger.Errorw("Failed to close consumer group", zap.Error(err))
		}
	}()

	// Track errors
	go func() {
		for err := range group.Errors() {
			a.logger.Errorw("Error while consuming messages", zap.Error(err))
		}
	}()

	<-ctx.Done()
	a.logger.Info("Shutting down...")
	return nil
}

func (a *Adapter) SetReady(int32, bool) {}

func (a *Adapter) Handle(ctx context.Context, msg *sarama.ConsumerMessage) (bool, error) {
	if a.rateLimiter != nil {
		a.rateLimiter.Wait(ctx)
	}

	ctx, span := trace.StartSpan(ctx, "kafka-source")
	defer span.End()

	req, err := a.httpMessageSender.NewCloudEventRequest(ctx)
	if err != nil {
		return false, err
	}

	err = a.ConsumerMessageToHttpRequest(ctx, msg, req)
	if err != nil {
		a.logger.Debug("failed to create request", zap.Error(err))
		return true, err
	}

	res, err := a.httpMessageSender.Send(req)

	if err != nil {
		a.logger.Debug("Error while sending the message", zap.Error(err))
		return false, err // Error while sending, don't commit offset
	}
	// Always try to read and close body so the connection can be reused afterwards
	if res.Body != nil {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}

	if res.StatusCode/100 != 2 {
		a.logger.Debug("Unexpected status code", zap.Int("status code", res.StatusCode))
		return false, fmt.Errorf("%d %s", res.StatusCode, http.StatusText(res.StatusCode))
	}

	reportArgs := &pkgsource.ReportArgs{
		Namespace:     a.config.Namespace,
		Name:          a.config.Name,
		ResourceGroup: resourceGroup,
	}

	_ = a.reporter.ReportEventCount(reportArgs, res.StatusCode)
	return true, nil
}

// SetRateLimiter sets the global consumer rate limiter
func (a *Adapter) SetRateLimits(r rate.Limit, b int) {
	a.rateLimiter = rate.NewLimiter(r, b)
}

func (a *Adapter) HandleServiceMessage(ctx context.Context, message ctrl.ServiceMessage) {
	// In this first PR, there is only the RA sending messages to control plane,
	// there is no message the control plane should send to the RA
	a.logger.Info("Received unexpected control message")
	message.Ack()
}

func (a *Adapter) Setup(sess sarama.ConsumerGroupSession) {
	if a.controlServer != nil {
		if err := a.controlServer.SendAndWaitForAck(kafkasourcecontrol.NotifySetupClaimsOpCode, kafkasourcecontrol.Claims(sess.Claims())); err != nil {
			a.logger.Warnf("Cannot send the claims update: %v", err)
		}
	}

	// Preemptively initialize consumer group offsets to be able to mark the source as ready
	// as soon as possible.
	if err := a.InitOffsets(sess); err != nil {
		a.logger.Warnf("Cannot initialized consumer group offsets: %v", err)
	}
}

func (a *Adapter) Cleanup(sess sarama.ConsumerGroupSession) {
	if a.controlServer != nil {
		if err := a.controlServer.SendAndWaitForAck(kafkasourcecontrol.NotifyCleanupClaimsOpCode, kafkasourcecontrol.Claims(sess.Claims())); err != nil {
			a.logger.Warnf("Cannot send the claims update: %v", err)
		}
	}
}

// InitOffsets makes sure all consumer group offsets are set.
func (a *Adapter) InitOffsets(session sarama.ConsumerGroupSession) error {
	if a.saramaConfig.Consumer.Offsets.Initial == sarama.OffsetNewest {
		// We want to make sure that ALL consumer group offsets are set to avoid
		// losing events in case the consumer group session is closed before at least one message is
		// consumed from ALL partitions.
		// If not, an event sent to a partition with an uninitialized offset
		// will not be forwarded when the session is closed (or a rebalancing is in progress).
		kafkaClient, err := sarama.NewClient(a.config.BootstrapServers, a.saramaConfig)
		if err != nil {
			return fmt.Errorf("failed to create a Kafka client: %w", err)
		}
		defer kafkaClient.Close()

		kafkaAdminClient, err := sarama.NewClusterAdminFromClient(kafkaClient)
		if err != nil {
			return fmt.Errorf("failed to create a Kafka admin client: %w", err)
		}
		defer kafkaAdminClient.Close()

		// Retrieve all partitions
		topicPartitions := make(map[string][]int32)
		for _, topic := range a.config.Topics {
			partitions, err := kafkaClient.Partitions(topic)

			if err != nil {
				return fmt.Errorf("failed to get partitions for topic %s: %w", topic, err)
			}

			topicPartitions[topic] = partitions
		}

		// Look for uninitialized offset (-1)
		offsets, err := kafkaAdminClient.ListConsumerGroupOffsets(a.config.ConsumerGroup, topicPartitions)
		if err != nil {
			return err
		}

		dirty := false
		for topic, partitions := range offsets.Blocks {
			for partition, block := range partitions {
				if block.Offset == -1 { // not initialized?

					// Fetch the newest offset in the topic/partition and set it in the consumer group
					offset, err := kafkaClient.GetOffset(topic, partition, sarama.OffsetNewest)
					if err != nil {
						return fmt.Errorf("failed to get the offset for topic %s and partition %d: %w", topic, partition, err)
					}

					a.logger.Infow("initializing offset", zap.String("topic", topic), zap.Int32("partition", partition), zap.Int64("offset", offset))

					session.MarkOffset(topic, partition, offset, "")
					dirty = true
				}
			}
		}

		if dirty {
			session.Commit()

			a.logger.Infow("consumer group offsets committed", zap.String("consumergroup", a.config.ConsumerGroup))
		}
	}

	// At this stage the KafkaSource instance is considered Ready (TODO: update KafkaSource status)
	return nil
}
<commit_msg>KafkaSource retries on error (#666)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kafka

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"strings"
	"time"

	"golang.org/x/time/rate"
	ctrl "knative.dev/control-protocol/pkg"
	ctrlnetwork "knative.dev/control-protocol/pkg/network"

	"github.com/Shopify/sarama"
	"go.opencensus.io/trace"
	"go.uber.org/zap"

	"knative.dev/pkg/logging"
	pkgsource "knative.dev/pkg/source"

	"knative.dev/eventing/pkg/adapter/v2"
	"knative.dev/eventing/pkg/kncloudevents"

	"knative.dev/eventing-kafka/pkg/common/consumer"
	"knative.dev/eventing-kafka/pkg/source/client"
	kafkasourcecontrol "knative.dev/eventing-kafka/pkg/source/control"
)

const (
	resourceGroup = "kafkasources.sources.knative.dev"
)

type AdapterConfig struct {
	adapter.EnvConfig
	client.KafkaEnvConfig

	Topics        []string `envconfig:"KAFKA_TOPICS" required:"true"`
	ConsumerGroup string   `envconfig:"KAFKA_CONSUMER_GROUP" required:"true"`
	Name          string   `envconfig:"NAME" required:"true"`
	KeyType       string   `envconfig:"KEY_TYPE" required:"false"`

	// Turn off the control server.
	DisableControlServer bool
}

func NewEnvConfig() adapter.EnvConfigAccessor {
	return &AdapterConfig{}
}

type Adapter struct {
	config        *AdapterConfig
	controlServer *ctrlnetwork.ControlServer
	saramaConfig  *sarama.Config

	httpMessageSender *kncloudevents.HTTPMessageSender
	reporter          pkgsource.StatsReporter
	logger            *zap.SugaredLogger
	keyTypeMapper     func([]byte) interface{}
	rateLimiter       *rate.Limiter
}

var (
	_           adapter.MessageAdapter                   = (*Adapter)(nil)
	_           consumer.KafkaConsumerHandler            = (*Adapter)(nil)
	_           consumer.SaramaConsumerLifecycleListener = (*Adapter)(nil)
	_           adapter.MessageAdapterConstructor        = NewAdapter
	retryConfig                                          = defaultRetryConfig()
)

func NewAdapter(ctx context.Context, processed adapter.EnvConfigAccessor, httpMessageSender *kncloudevents.HTTPMessageSender, reporter pkgsource.StatsReporter) adapter.MessageAdapter {
	logger := logging.FromContext(ctx)
	config := processed.(*AdapterConfig)

	return &Adapter{
		config:            config,
		httpMessageSender: httpMessageSender,
		reporter:          reporter,
		logger:            logger,
		keyTypeMapper:     getKeyTypeMapper(config.KeyType),
	}
}
func (a *Adapter) GetConsumerGroup() string {
	return a.config.ConsumerGroup
}

func (a *Adapter) Start(ctx context.Context) (err error) {
	a.logger.Infow("Starting with config: ",
		zap.String("Topics", strings.Join(a.config.Topics, ",")),
		zap.String("ConsumerGroup", a.config.ConsumerGroup),
		zap.String("SinkURI", a.config.Sink),
		zap.String("Name", a.config.Name),
		zap.String("Namespace", a.config.Namespace),
	)

	// Init control service
	if !a.config.DisableControlServer {
		a.controlServer, err = ctrlnetwork.StartInsecureControlServer(ctx)
		if err != nil {
			return err
		}
		a.controlServer.MessageHandler(a)
	}

	// init consumer group
	addrs, config, err := client.NewConfigWithEnv(context.Background(), &a.config.KafkaEnvConfig)
	if err != nil {
		return fmt.Errorf("failed to create the config: %w", err)
	}
	a.saramaConfig = config

	options := []consumer.SaramaConsumerHandlerOption{consumer.WithSaramaConsumerLifecycleListener(a)}
	consumerGroupFactory := consumer.NewConsumerGroupFactory(addrs, config)
	group, err := consumerGroupFactory.StartConsumerGroup(
		a.config.ConsumerGroup,
		a.config.Topics,
		a.logger,
		a,
		options...,
	)
	if err != nil {
		return fmt.Errorf("failed to start consumer group: %w", err)
	}
	defer func() {
		err := group.Close()
		if err != nil {
			a.logger.Errorw("Failed to close consumer group", zap.Error(err))
		}
	}()

	// Track errors
	go func() {
		for err := range group.Errors() {
			a.logger.Errorw("Error while consuming messages", zap.Error(err))
		}
	}()

	<-ctx.Done()
	a.logger.Info("Shutting down...")
	return nil
}

func (a *Adapter) SetReady(int32, bool) {}

func (a *Adapter) Handle(ctx context.Context, msg *sarama.ConsumerMessage) (bool, error) {
	if a.rateLimiter != nil {
		a.rateLimiter.Wait(ctx)
	}

	ctx, span := trace.StartSpan(ctx, "kafka-source")
	defer span.End()

	req, err := a.httpMessageSender.NewCloudEventRequest(ctx)
	if err != nil {
		return false, err
	}

	err = a.ConsumerMessageToHttpRequest(ctx, msg, req)
	if err != nil {
		a.logger.Debug("failed to create request", zap.Error(err))
		return true, err
	}

	res, err := a.httpMessageSender.SendWithRetries(req, retryConfig)

	if err != nil {
		a.logger.Debug("Error while sending the message", zap.Error(err))
		return false, err // Error while sending, don't commit offset
	}
	// Always try to read and close body so the connection can be reused afterwards
	if res.Body != nil {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}

	if res.StatusCode/100 != 2 {
		a.logger.Debug("Unexpected status code", zap.Int("status code", res.StatusCode))
		return false, fmt.Errorf("%d %s", res.StatusCode, http.StatusText(res.StatusCode))
	}

	reportArgs := &pkgsource.ReportArgs{
		Namespace:     a.config.Namespace,
		Name:          a.config.Name,
		ResourceGroup: resourceGroup,
	}

	_ = a.reporter.ReportEventCount(reportArgs, res.StatusCode)
	return true, nil
}

// SetRateLimiter sets the global consumer rate limiter
func (a *Adapter) SetRateLimits(r rate.Limit, b int) {
	a.rateLimiter = rate.NewLimiter(r, b)
}

func (a *Adapter) HandleServiceMessage(ctx context.Context, message ctrl.ServiceMessage) {
	// In this first PR, there is only the RA sending messages to control plane,
	// there is no message the control plane should send to the RA
	a.logger.Info("Received unexpected control message")
	message.Ack()
}

func (a *Adapter) Setup(sess sarama.ConsumerGroupSession) {
	if a.controlServer != nil {
		if err := a.controlServer.SendAndWaitForAck(kafkasourcecontrol.NotifySetupClaimsOpCode, kafkasourcecontrol.Claims(sess.Claims())); err != nil {
			a.logger.Warnf("Cannot send the claims update: %v", err)
		}
	}

	// Preemptively initialize consumer group offsets to be able to mark the source as ready
	// as soon as possible.
	if err := a.InitOffsets(sess); err != nil {
		a.logger.Warnf("Cannot initialized consumer group offsets: %v", err)
	}
}

func (a *Adapter) Cleanup(sess sarama.ConsumerGroupSession) {
	if a.controlServer != nil {
		if err := a.controlServer.SendAndWaitForAck(kafkasourcecontrol.NotifyCleanupClaimsOpCode, kafkasourcecontrol.Claims(sess.Claims())); err != nil {
			a.logger.Warnf("Cannot send the claims update: %v", err)
		}
	}
}

// InitOffsets makes sure all consumer group offsets are set.
func (a *Adapter) InitOffsets(session sarama.ConsumerGroupSession) error {
	if a.saramaConfig.Consumer.Offsets.Initial == sarama.OffsetNewest {
		// We want to make sure that ALL consumer group offsets are set to avoid
		// losing events in case the consumer group session is closed before at least one message is
		// consumed from ALL partitions.
		// If not, an event sent to a partition with an uninitialized offset
		// will not be forwarded when the session is closed (or a rebalancing is in progress).
		kafkaClient, err := sarama.NewClient(a.config.BootstrapServers, a.saramaConfig)
		if err != nil {
			return fmt.Errorf("failed to create a Kafka client: %w", err)
		}
		defer kafkaClient.Close()

		kafkaAdminClient, err := sarama.NewClusterAdminFromClient(kafkaClient)
		if err != nil {
			return fmt.Errorf("failed to create a Kafka admin client: %w", err)
		}
		defer kafkaAdminClient.Close()

		// Retrieve all partitions
		topicPartitions := make(map[string][]int32)
		for _, topic := range a.config.Topics {
			partitions, err := kafkaClient.Partitions(topic)

			if err != nil {
				return fmt.Errorf("failed to get partitions for topic %s: %w", topic, err)
			}

			topicPartitions[topic] = partitions
		}

		// Look for uninitialized offset (-1)
		offsets, err := kafkaAdminClient.ListConsumerGroupOffsets(a.config.ConsumerGroup, topicPartitions)
		if err != nil {
			return err
		}

		dirty := false
		for topic, partitions := range offsets.Blocks {
			for partition, block := range partitions {
				if block.Offset == -1 { // not initialized?

					// Fetch the newest offset in the topic/partition and set it in the consumer group
					offset, err := kafkaClient.GetOffset(topic, partition, sarama.OffsetNewest)
					if err != nil {
						return fmt.Errorf("failed to get the offset for topic %s and partition %d: %w", topic, partition, err)
					}

					a.logger.Infow("initializing offset", zap.String("topic", topic), zap.Int32("partition", partition), zap.Int64("offset", offset))

					session.MarkOffset(topic, partition, offset, "")
					dirty = true
				}
			}
		}

		if dirty {
			session.Commit()

			a.logger.Infow("consumer group offsets committed", zap.String("consumergroup", a.config.ConsumerGroup))
		}
	}

	// At this stage the KafkaSource instance is considered Ready (TODO: update KafkaSource status)
	return nil
}

// Default retry configuration, 5 retries, exponential backoff with 50ms delay
func defaultRetryConfig() *kncloudevents.RetryConfig {
	return &kncloudevents.RetryConfig{
		CheckRetry: kncloudevents.SelectiveRetry,
		RetryMax:   5,
		Backoff: func(attemptNum int, resp *http.Response) time.Duration {
			return 50 * time.Millisecond * time.Duration(math.Exp2(float64(attemptNum)))
		},
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package timeutil

import (
	"fmt"
	"testing"
	"time"
)

func mustParseTime(t string) time.Time {
	ref, err := time.Parse("Mon Jan 2 15:04:05 MST 2006", t)
	if err != nil {
		panic(fmt.Sprintf("parsing time %s failed: %s", t, err))
	}
	return ref
}

func TestPeriodicParse(t *testing.T) {
	tests := []struct {
		start    string
		duration string
		err      bool
	}{
		{ // Valid start time
			start:    "14:00",
			duration: "1h",
			err:      false,
		},
		{ // Start hour out of range
			start:    "25:00",
			duration: "1h",
			err:      true,
		},
		{ // Start minute out of range
			start:    "23:61",
			duration: "1h",
			err:      true,
		},
		{ // Bad time of day
			start:    "14",
			duration: "1h",
			err:      true,
		},
		{ // Bad day of week
			start:    "foo 14:00",
			duration: "1h",
			err:      true,
		},
		{ // Bad duration
			start:    "sat 14:00",
			duration: "1j",
			err:      true,
		},
		{ // Negative duration
			start:    "sat 14:00",
			duration: "-1h",
			err:      true,
		},
		{ // Too many start fields
			start:    "sat 14:00 1234",
			duration: "1h",
			err:      true,
		},
		{ // Specified just a day
			start:    "sat",
			duration: "1h",
			err:      true,
		},
	}

	for i, tt := range tests {
		_, err := ParsePeriodic(tt.start, tt.duration)
		if err != nil && tt.err == false {
			t.Errorf("#%d: unexpected error: %q", i, err)
			continue
		}
		if err == nil && tt.err == true {
			t.Errorf("#%d: expected error and did not get it", i)
			continue
		}
	}
}

func TestToStart(t *testing.T) {
	tests := []struct {
		start    string
		duration string
		time     string
		toStart  time.Duration
	}{
		{ // Daily window in 15 minutes.
			start:    "00:15",
			duration: "10s",
			time:     "Thu May 21 00:00:00 PDT 2015",
			toStart:  15 * time.Minute,
		},
		{ // Daily window now.
			start:    "00:00",
			duration: "1h",
			time:     "Thu May 21 00:00:00 PDT 2015",
			toStart:  0,
		},
		{ // Daily window started 1 second ago.
			start:    "00:00",
			duration: "1h",
			time:     "Thu May 21 00:00:01 PDT 2015",
			toStart:  -1 * time.Second,
		},
		{ // Daily window started 10 hours ago but closing edge.
			start:    "02:33",
			duration: "10h",
			time:     "Thu May 21 12:33:00 PDT 2015",
			toStart:  -10 * time.Hour,
		},
		{ // Daily window started 10 hours ago but _just_ closed, expect next.
			start:    "02:33",
			duration: "10h",
			time:     "Thu May 21 12:33:01 PDT 2015",
			toStart:  13*time.Hour + 59*time.Minute + 59*time.Second,
		},
		{ // Daily window started last night but extends into now on the next day.
			start:    "23:05",
			duration: "11h",
			time:     "Thu May 21 09:33:01 PDT 2015",
			toStart:  -(10*time.Hour + 28*time.Minute + 1*time.Second),
		},
		{ // Weekly window started in 15 minutes.
			start:    "Thu 00:15",
			duration: "10s",
			time:     "Thu May 21 00:00:00 PDT 2015",
			toStart:  15 * time.Minute,
		},
		{ // Weekly window now.
			start:    "Thu 00:00",
			duration: "1h",
			time:     "Thu May 21 00:00:00 PDT 2015",
			toStart:  0,
		},
		{ // Weekly window started 1 second ago.
			start:    "Sun 00:00",
			duration: "1h",
			time:     "Sun May 17 00:00:01 PDT 2015",
			toStart:  -1 * time.Second,
		},
		{ // Weekly window started 10 hours ago but closing edge.
			start:    "Sun 02:33",
			duration: "10h",
			time:     "Sun May 17 12:33:00 PDT 2015",
			toStart:  -10 * time.Hour,
		},
		{ // Weekly window started last night in previous week but extends into now on the next day.
			start:    "Sat 23:05",
			duration: "11h",
			time:     "Sun May 17 09:33:01 PDT 2015",
			toStart:  -(10*time.Hour + 28*time.Minute + 1*time.Second),
		},
		{ // Weekly window where next period begins next month
			start:    "Mon 9:00",
			duration: "1h",
			time:     "Sat May 30 23:00:00 PDT 2015",
			toStart:  34 * time.Hour,
		},
	}

	for i, tt := range tests {
		p, err := ParsePeriodic(tt.start, tt.duration)
		if err != nil {
			t.Errorf("#%d: periodic parse failed: %v", i, err)
			continue
		}
		ref := mustParseTime(tt.time)
		if dts := p.DurationToStart(ref); dts != tt.toStart {
			t.Errorf("#%d: got %v, want %v", i, dts, tt.toStart)
		}
	}
}
<commit_msg>pkg/timeutil: add test for a period that began on the last day of the previous month<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package timeutil

import (
	"fmt"
	"testing"
	"time"
)

func mustParseTime(t string) time.Time {
	ref, err := time.Parse("Mon Jan 2 15:04:05 MST 2006", t)
	if err != nil {
		panic(fmt.Sprintf("parsing time %s failed: %s", t, err))
	}
	return ref
}

func TestPeriodicParse(t *testing.T) {
	tests := []struct {
		start    string
		duration string
		err      bool
	}{
		{ // Valid start time
			start:    "14:00",
			duration: "1h",
			err:      false,
		},
		{ // Start hour out of range
			start:    "25:00",
			duration: "1h",
			err:      true,
		},
		{ // Start minute out of range
			start:    "23:61",
			duration: "1h",
			err:      true,
		},
		{ // Bad time of day
			start:    "14",
			duration: "1h",
			err:      true,
		},
		{ // Bad day of week
			start:    "foo 14:00",
			duration: "1h",
			err:      true,
		},
		{ // Bad duration
			start:    "sat 14:00",
			duration: "1j",
			err:      true,
		},
		{ // Negative duration
			start:    "sat 14:00",
			duration: "-1h",
			err:      true,
		},
		{ // Too many start fields
			start:    "sat 14:00 1234",
			duration: "1h",
			err:      true,
		},
		{ // Specified just a day
			start:    "sat",
			duration: "1h",
			err:      true,
		},
	}

	for i, tt := range tests {
		_, err := ParsePeriodic(tt.start, tt.duration)
		if err != nil && tt.err == false {
			t.Errorf("#%d: unexpected error: %q", i, err)
			continue
		}
		if err == nil && tt.err == true {
			t.Errorf("#%d: expected error and did not get it", i)
			continue
		}
	}
}

func TestToStart(t *testing.T) {
	tests := []struct {
		start    string
		duration string
		time     string
		toStart  time.Duration
	}{
		{ // Daily window in 15 minutes.
			start:    "00:15",
			duration: "10s",
			time:     "Thu May 21 00:00:00 PDT 2015",
			toStart:  15 * time.Minute,
		},
		{ // Daily window now.
			start:    "00:00",
			duration: "1h",
			time:     "Thu May 21 00:00:00 PDT 2015",
			toStart:  0,
		},
		{ // Daily window started 1 second ago.
			start:    "00:00",
			duration: "1h",
			time:     "Thu May 21 00:00:01 PDT 2015",
			toStart:  -1 * time.Second,
		},
		{ // Daily window started 10 hours ago but closing edge.
			start:    "02:33",
			duration: "10h",
			time:     "Thu May 21 12:33:00 PDT 2015",
			toStart:  -10 * time.Hour,
		},
		{ // Daily window started 10 hours ago but _just_ closed, expect next.
			start:    "02:33",
			duration: "10h",
			time:     "Thu May 21 12:33:01 PDT 2015",
			toStart:  13*time.Hour + 59*time.Minute + 59*time.Second,
		},
		{ // Daily window started last night but extends into now on the next day.
			start:    "23:05",
			duration: "11h",
			time:     "Thu May 21 09:33:01 PDT 2015",
			toStart:  -(10*time.Hour + 28*time.Minute + 1*time.Second),
		},
		{ // Weekly window started in 15 minutes.
			start:    "Thu 00:15",
			duration: "10s",
			time:     "Thu May 21 00:00:00 PDT 2015",
			toStart:  15 * time.Minute,
		},
		{ // Weekly window now.
			start:    "Thu 00:00",
			duration: "1h",
			time:     "Thu May 21 00:00:00 PDT 2015",
			toStart:  0,
		},
		{ // Weekly window started 1 second ago.
			start:    "Sun 00:00",
			duration: "1h",
			time:     "Sun May 17 00:00:01 PDT 2015",
			toStart:  -1 * time.Second,
		},
		{ // Weekly window started 10 hours ago but closing edge.
			start:    "Sun 02:33",
			duration: "10h",
			time:     "Sun May 17 12:33:00 PDT 2015",
			toStart:  -10 * time.Hour,
		},
		{ // Weekly window started last night in previous week but extends into now on the next day.
			start:    "Sat 23:05",
			duration: "11h",
			time:     "Sun May 17 09:33:01 PDT 2015",
			toStart:  -(10*time.Hour + 28*time.Minute + 1*time.Second),
		},
		{ // Weekly window where next period begins next month
			start:    "Mon 9:00",
			duration: "1h",
			time:     "Sat May 30 23:00:00 PDT 2015",
			toStart:  34 * time.Hour,
		},
		{ // Weekly window where the period started on the last day of the previous month
			start:    "Sun 23:00",
			duration: "4h",
			time:     "Mon Jun 1 01:00:00 PDT 2015",
			toStart:  -(2 * time.Hour),
		},
	}

	for i, tt := range tests {
		p, err := ParsePeriodic(tt.start, tt.duration)
		if err != nil {
			t.Errorf("#%d: periodic parse failed: %v", i, err)
			continue
		}
		ref := mustParseTime(tt.time)
		if dts := p.DurationToStart(ref); dts != tt.toStart {
			t.Errorf("#%d: got %v, want %v", i, dts, tt.toStart)
		}
	}
}
<|endoftext|>


<commit_before>package exec

import (
	"archive/tar"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"time"

	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/realtime"

	"github.com/spf13/afero"
)

// KonnectorOptions contains the options to execute a konnector.
type KonnectorOptions struct {
	Konnector    string `json:"konnector"`
	Account      string `json:"account"`
	FolderToSave string `json:"folder_to_save"`
}

type konnectorWorker struct {
	opts     *KonnectorOptions
	man      *apps.KonnManifest
	messages []konnectorMsg
}

// result stores the result of a konnector execution.
type result struct {
	DocID       string    `json:"_id,omitempty"`
	DocRev      string    `json:"_rev,omitempty"`
	CreatedAt   time.Time `json:"last_execution"`
	LastSuccess time.Time `json:"last_success"`
	Account     string    `json:"account"`
	State       string    `json:"state"`
	Error       string    `json:"error"`
}

func (r *result) ID() string         { return r.DocID }
func (r *result) Rev() string        { return r.DocRev }
func (r *result) DocType() string    { return consts.KonnectorResults }
func (r *result) Clone() couchdb.Doc { c := *r; return &c }
func (r *result) SetID(id string)    { r.DocID = id }
func (r *result) SetRev(rev string)  { r.DocRev = rev }

const konnectorMsgTypeError string = "error"

// const konnectorMsgTypeDebug string = "debug"
// const konnectorMsgTypeWarning string = "warning"
// const konnectorMsgTypeProgress string = "progress"

type konnectorMsg struct {
	Type    string `json:"type"`
	Message string `json:"message"`
}

type konnectorLogs struct {
	Slug     string         `json:"_id,omitempty"`
	DocRev   string         `json:"_rev,omitempty"`
	Messages []konnectorMsg `json:"logs"`
}

func (kl *konnectorLogs) ID() string      { return kl.Slug }
func (kl *konnectorLogs) Rev() string     { return kl.DocRev }
func (kl *konnectorLogs) DocType() string { return consts.KonnectorLogs }
func (kl *konnectorLogs) Clone() couchdb.Doc {
	cloned := *kl
	cloned.Messages = make([]konnectorMsg, len(kl.Messages))
	copy(cloned.Messages, kl.Messages)
	return &cloned
}
func (kl *konnectorLogs) SetID(id string)   {}
func (kl *konnectorLogs) SetRev(rev string) { kl.DocRev = rev }

func (w *konnectorWorker) PrepareWorkDir(i *instance.Instance, m *jobs.Message) (workDir string, err error) {
	opts := &KonnectorOptions{}
	if err = m.Unmarshal(&opts); err != nil {
		return
	}

	slug := opts.Konnector

	man, err := apps.GetKonnectorBySlug(i, slug)
	if err != nil {
		return
	}
	if man.State() != apps.Ready {
		err = errors.New("Konnector is not ready")
		return
	}

	w.opts = opts
	w.man = man

	osFS := afero.NewOsFs()
	workDir, err = afero.TempDir(osFS, "", "konnector-"+slug)
	if err != nil {
		return
	}
	workFS := afero.NewBasePathFs(osFS, workDir)

	fileServer := i.KonnectorsFileServer()
	tarFile, err := fileServer.Open(slug, man.Version(), apps.KonnectorArchiveName)
	if err != nil {
		return
	}

	tr := tar.NewReader(tarFile)
	for {
		var hdr *tar.Header
		hdr, err = tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return
		}
		dirname := path.Dir(hdr.Name)
		if dirname != "." {
			if err = workFS.MkdirAll(dirname, 0755); err != nil {
				return
			}
		}
		var f afero.File
		f, err = workFS.OpenFile(hdr.Name, os.O_CREATE|os.O_WRONLY, 0640)
		if err != nil {
			return
		}
		_, err = io.Copy(f, tr)
		errc := f.Close()
		if err != nil {
			return
		}
		if errc != nil {
			err = errc
			return
		}
	}

	return workDir, nil
}

func (w *konnectorWorker) PrepareCmdEnv(i *instance.Instance, m *jobs.Message) (cmd string, env []string, jobID string, err error) {
	jobID = fmt.Sprintf("konnector/%s/%s", w.opts.Konnector, i.Domain)

	fields := struct {
		Account      string `json:"account"`
		FolderToSave string `json:"folder_to_save,omitempty"`
	}{
		Account:      w.opts.Account,
		FolderToSave: w.opts.FolderToSave,
	}

	fieldsJSON, err := json.Marshal(fields)
	if err != nil {
		return
	}

	paramsJSON, err := json.Marshal(w.man.Parameters)
	if err != nil {
		return
	}

	token := i.BuildKonnectorToken(w.man)

	cmd = config.GetConfig().Konnectors.Cmd
	env = []string{
		"COZY_URL=" + i.PageURL("/", nil),
		"COZY_CREDENTIALS=" + token,
		"COZY_FIELDS=" + string(fieldsJSON),
		"COZY_PARAMETERS=" + string(paramsJSON),
		"COZY_TYPE=" + w.man.Type,
		"COZY_LOCALE=" + i.Locale,
		"COZY_JOB_ID=" + jobID,
	}
	return
}

func (w *konnectorWorker) ScanOuput(i *instance.Instance, line []byte) error {
	var msg konnectorMsg
	if err := json.Unmarshal(line, &msg); err != nil {
		return fmt.Errorf("Could not parse stdout as JSON: %q", string(line))
	}
	// TODO: filter some of the messages
	w.messages = append(w.messages, msg)
	realtime.GetHub().Publish(&realtime.Event{
		Verb: realtime.EventCreate,
		Doc: couchdb.JSONDoc{Type: consts.JobEvents, M: map[string]interface{}{
			"type":    msg.Type,
			"message": msg.Message,
		}},
		Domain: i.Domain,
	})
	return nil
}

func (w *konnectorWorker) Error(i *instance.Instance, err error) error {
	errLogs := couchdb.Upsert(i, &konnectorLogs{
		Slug:     w.opts.Konnector,
		Messages: w.messages,
	})
	if errLogs != nil {
		fmt.Println("Failed to save konnector logs", errLogs)
	}

	for _, msg := range w.messages {
		if msg.Type == konnectorMsgTypeError {
			// konnector err is more explicit
			return errors.New(msg.Message)
		}
	}

	return err
}

func (w *konnectorWorker) Commit(ctx context.Context, msg *jobs.Message, errjob error) error {
	if w.opts == nil {
		return nil
	}

	slug := w.opts.Konnector
	domain := ctx.Value(jobs.ContextDomainKey).(string)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	lastResult := &result{}
	err = couchdb.GetDoc(inst, consts.KonnectorResults, slug, lastResult)
	if err != nil {
		if !couchdb.IsNotFoundError(err) {
			return err
		}
		lastResult = nil
	}

	var state, errstr string
	var lastSuccess time.Time
	if errjob != nil {
		if lastResult != nil {
			lastSuccess = lastResult.LastSuccess
		}
		errstr = errjob.Error()
		state = jobs.Errored
	} else {
		lastSuccess = time.Now()
		state = jobs.Done
	}
	result := &result{
		DocID:       slug,
		Account:     w.opts.Account,
		CreatedAt:   time.Now(),
		LastSuccess: lastSuccess,
		State:       state,
		Error:       errstr,
	}
	if lastResult == nil {
		err = couchdb.CreateNamedDocWithDB(inst, result)
	} else {
		result.SetRev(lastResult.Rev())
		err = couchdb.UpdateDoc(inst, result)
	}
	// if err != nil {
	// 	return err
	// }

	return err
	// // if it is the first try we do not take into account an error, we bail.
	// if lastResult == nil {
	// 	return nil
	// }
	// // if the job has not errored, or the last one was already errored, we bail.
	// if state != jobs.Errored || lastResult.State == jobs.Errored {
	// 	return nil
	// }

	// konnectorURL := inst.SubDomain(consts.CollectSlug)
	// konnectorURL.Fragment = "/category/all/" + slug
	// mail := mails.Options{
	// 	Mode:         mails.ModeNoReply,
	// 	Subject:      inst.Translate("Error Konnector execution", domain),
	// 	TemplateName: "konnector_error_" + inst.Locale,
	// 	TemplateValues: map[string]string{
	// 		"KonnectorName": slug,
	// 		"KonnectorPage": konnectorURL.String(),
	// 	},
	// }
	// msg, err := jobs.NewMessage(jobs.JSONEncoding, &mail)
	// if err != nil {
	// 	return err
	// }
	// log := logger.WithDomain(domain)
	// log.Info("Konnector has failed definitively, should send mail.", mail)
	// _, err = globals.GetBroker().PushJob(&jobs.JobRequest{
	// 	Domain:     domain,
	// 	WorkerType: "sendmail",
	// 	Message:    msg,
	// })
	// return err
}
<commit_msg>Add a created_at for konnector logs<commit_after>package exec

import (
	"archive/tar"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"time"

	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/realtime"

	"github.com/spf13/afero"
)

// KonnectorOptions contains the options to execute a konnector.
type KonnectorOptions struct {
	Konnector    string `json:"konnector"`
	Account      string `json:"account"`
	FolderToSave string `json:"folder_to_save"`
}

type konnectorWorker struct {
	opts     *KonnectorOptions
	man      *apps.KonnManifest
	messages []konnectorMsg
}

// result stores the result of a konnector execution.
type result struct {
	DocID       string    `json:"_id,omitempty"`
	DocRev      string    `json:"_rev,omitempty"`
	CreatedAt   time.Time `json:"last_execution"`
	LastSuccess time.Time `json:"last_success"`
	Account     string    `json:"account"`
	State       string    `json:"state"`
	Error       string    `json:"error"`
}

func (r *result) ID() string         { return r.DocID }
func (r *result) Rev() string        { return r.DocRev }
func (r *result) DocType() string    { return consts.KonnectorResults }
func (r *result) Clone() couchdb.Doc { c := *r; return &c }
func (r *result) SetID(id string)    { r.DocID = id }
func (r *result) SetRev(rev string)  { r.DocRev = rev }

const konnectorMsgTypeError string = "error"

// const konnectorMsgTypeDebug string = "debug"
// const konnectorMsgTypeWarning string = "warning"
// const konnectorMsgTypeProgress string = "progress"

type konnectorMsg struct {
	Type    string `json:"type"`
	Message string `json:"message"`
}

type konnectorLogs struct {
	Slug      string         `json:"_id,omitempty"`
	DocRev    string         `json:"_rev,omitempty"`
	Messages  []konnectorMsg `json:"logs"`
	CreatedAt time.Time      `json:"created_at"`
}

func (kl *konnectorLogs) ID() string      { return kl.Slug }
func (kl *konnectorLogs) Rev() string     { return kl.DocRev }
func (kl *konnectorLogs) DocType() string { return consts.KonnectorLogs }
func (kl *konnectorLogs) Clone() couchdb.Doc {
	cloned := *kl
	cloned.Messages = make([]konnectorMsg, len(kl.Messages))
	copy(cloned.Messages, kl.Messages)
	return &cloned
}
func (kl *konnectorLogs) SetID(id string)   {}
func (kl *konnectorLogs) SetRev(rev string) { kl.DocRev = rev }

func (w *konnectorWorker) PrepareWorkDir(i *instance.Instance, m *jobs.Message) (workDir string, err error) {
	opts := &KonnectorOptions{}
	if err = m.Unmarshal(&opts); err != nil {
		return
	}

	slug := opts.Konnector

	man, err := apps.GetKonnectorBySlug(i, slug)
	if err != nil {
		return
	}
	if man.State() != apps.Ready {
		err = errors.New("Konnector is not ready")
		return
	}

	w.opts = opts
	w.man = man

	osFS := afero.NewOsFs()
	workDir, err = afero.TempDir(osFS, "", "konnector-"+slug)
	if err != nil {
		return
	}
	workFS := afero.NewBasePathFs(osFS, workDir)

	fileServer := i.KonnectorsFileServer()
	tarFile, err := fileServer.Open(slug, man.Version(), apps.KonnectorArchiveName)
	if err != nil {
		return
	}

	tr := tar.NewReader(tarFile)
	for {
		var hdr *tar.Header
		hdr, err = tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return
		}
		dirname := path.Dir(hdr.Name)
		if dirname != "." {
			if err = workFS.MkdirAll(dirname, 0755); err != nil {
				return
			}
		}
		var f afero.File
		f, err = workFS.OpenFile(hdr.Name, os.O_CREATE|os.O_WRONLY, 0640)
		if err != nil {
			return
		}
		_, err = io.Copy(f, tr)
		errc := f.Close()
		if err != nil {
			return
		}
		if errc != nil {
			err = errc
			return
		}
	}

	return workDir, nil
}

func (w *konnectorWorker) PrepareCmdEnv(i *instance.Instance, m *jobs.Message) (cmd string, env []string, jobID string, err error) {
	jobID = fmt.Sprintf("konnector/%s/%s", w.opts.Konnector, i.Domain)

	fields := struct {
		Account      string `json:"account"`
		FolderToSave string `json:"folder_to_save,omitempty"`
	}{
		Account:      w.opts.Account,
		FolderToSave: w.opts.FolderToSave,
	}

	fieldsJSON, err := json.Marshal(fields)
	if err != nil {
		return
	}

	paramsJSON, err := json.Marshal(w.man.Parameters)
	if err != nil {
		return
	}

	token := i.BuildKonnectorToken(w.man)

	cmd = config.GetConfig().Konnectors.Cmd
	env = []string{
		"COZY_URL=" + i.PageURL("/", nil),
		"COZY_CREDENTIALS=" + token,
		"COZY_FIELDS=" + string(fieldsJSON),
		"COZY_PARAMETERS=" + string(paramsJSON),
		"COZY_TYPE=" + w.man.Type,
		"COZY_LOCALE=" + i.Locale,
		"COZY_JOB_ID=" + jobID,
	}
	return
}

func (w *konnectorWorker) ScanOuput(i *instance.Instance, line []byte) error {
	var msg konnectorMsg
	if err := json.Unmarshal(line, &msg); err != nil {
		return fmt.Errorf("Could not parse stdout as JSON: %q", string(line))
	}
	// TODO: filter some of the messages
	w.messages = append(w.messages, msg)
	realtime.GetHub().Publish(&realtime.Event{
		Verb: realtime.EventCreate,
		Doc: couchdb.JSONDoc{Type: consts.JobEvents, M: map[string]interface{}{
			"type":    msg.Type,
			"message": msg.Message,
		}},
		Domain: i.Domain,
	})
	return nil
}

func (w *konnectorWorker) Error(i *instance.Instance, err error) error {
	errLogs := couchdb.Upsert(i, &konnectorLogs{
		Slug:      w.opts.Konnector,
		Messages:  w.messages,
		CreatedAt: time.Now(),
	})
	if errLogs != nil {
		fmt.Println("Failed to save konnector logs", errLogs)
	}

	for _, msg := range w.messages {
		if msg.Type == konnectorMsgTypeError {
			// konnector err is more explicit
			return errors.New(msg.Message)
		}
	}

	return err
}

func (w *konnectorWorker) Commit(ctx context.Context, msg *jobs.Message, errjob error) error {
	if w.opts == nil {
		return nil
	}

	slug := w.opts.Konnector
	domain := ctx.Value(jobs.ContextDomainKey).(string)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	lastResult := &result{}
	err = couchdb.GetDoc(inst, consts.KonnectorResults, slug, lastResult)
	if err != nil {
		if !couchdb.IsNotFoundError(err) {
			return err
		}
		lastResult = nil
	}

	var state, errstr string
	var lastSuccess time.Time
	if errjob != nil {
		if lastResult != nil {
			lastSuccess = lastResult.LastSuccess
		}
		errstr = errjob.Error()
		state = jobs.Errored
	} else {
		lastSuccess = time.Now()
		state = jobs.Done
	}
	result := &result{
		DocID:       slug,
		Account:     w.opts.Account,
		CreatedAt:   time.Now(),
		LastSuccess: lastSuccess,
		State:       state,
		Error:       errstr,
	}
	if lastResult == nil {
		err = couchdb.CreateNamedDocWithDB(inst, result)
	} else {
		result.SetRev(lastResult.Rev())
		err = couchdb.UpdateDoc(inst, result)
	}
	// if err != nil {
	// 	return err
	// }

	return err
	// // if it is the first try we do not take into account an error, we bail.
	// if lastResult == nil {
	// 	return nil
	// }
	// // if the job has not errored, or the last one was already errored, we bail.
	// if state != jobs.Errored || lastResult.State == jobs.Errored {
	// 	return nil
	// }

	// konnectorURL := inst.SubDomain(consts.CollectSlug)
	// konnectorURL.Fragment = "/category/all/" + slug
	// mail := mails.Options{
	// 	Mode:         mails.ModeNoReply,
	// 	Subject:      inst.Translate("Error Konnector execution", domain),
	// 	TemplateName: "konnector_error_" + inst.Locale,
	// 	TemplateValues: map[string]string{
	// 		"KonnectorName": slug,
	// 		"KonnectorPage": konnectorURL.String(),
	// 	},
	// }
	// msg, err := jobs.NewMessage(jobs.JSONEncoding, &mail)
	// if err != nil {
	// 	return err
	// }
	// log := logger.WithDomain(domain)
	// log.Info("Konnector has failed definitively, should send mail.", mail)
	// _, err = globals.GetBroker().PushJob(&jobs.JobRequest{
	// 	Domain:     domain,
	// 	WorkerType: "sendmail",
	// 	Message:    msg,
	// })
	// return err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/starkandwayne/shield/plugin"
	"time"
)

func main() {
	p := RedisBrokerPlugin{
		Name:    "Pivotal Redis Broker Backup Plugin",
		Author:  "Stark & Wayne",
		Version: "0.0.1",
		Features: plugin.PluginFeatures{
			Target: "yes",
			Store:  "no",
		},
	}

	plugin.Run(p)
}

type RedisBrokerPlugin plugin.PluginInfo

type RedisEndpoint struct {
	Mode string
}

func (p RedisBrokerPlugin) Meta() plugin.PluginInfo {
	return plugin.PluginInfo(p)
}

func (p RedisBrokerPlugin) Backup(endpoint plugin.ShieldEndpoint) error {
	err := plugin.Exec("tar -c -C /var/vcap/store .", plugin.STDOUT)
	if err != nil {
		return err
	}

	return nil
}

func (p RedisBrokerPlugin) Restore(endpoint plugin.ShieldEndpoint) error {
	redis, err := getRedisEndpoint(endpoint)
	if err != nil {
		return err
	}

	var services = []string{"cf-redis-broker"}
	if redis.Mode == "dedicated" {
		services = []string{"redis", "redis-agent"}
	}

	for _, svc := range services {
		err = plugin.Exec(fmt.Sprintf("/var/vcap/bosh/bin/monit stop %s", svc), plugin.STDOUT)
		if err != nil {
			return err
		}
	}

	for _, svc := range services {
		err = plugin.Exec(fmt.Sprintf("bash -c \"while [[ $(/var/vcap/bosh/bin/monit summary %s | grep running) ]]; do sleep 1; done\"", svc), plugin.STDOUT)
		if err != nil {
			return err
		}
	}

	// Don't look for errors here, because pkill will return non-zero if there
	// were no processes to kill in the first place.
	// FIXME: handle this better, so we know we're pkilling properly
	plugin.Exec("pkill redis-server", plugin.STDOUT)
	time.Sleep(2 * time.Second)
	plugin.Exec("pkill -9 redis-server", plugin.STDOUT)
	time.Sleep(1 * time.Second)

	err = plugin.Exec("tar -x -C /var/vcap/store . ", plugin.STDIN)
	if err != nil {
		return err
	}

	err = plugin.Exec("bash -c 'yes | find /var/vcap/store -name appendonly.aof -exec /var/vcap/packages/redis/bin/redis-check-aof --fix {} \\;'", plugin.STDOUT)
	if err != nil {
		return err
	}

	for _, svc := range services {
		err = plugin.Exec(fmt.Sprintf("/var/vcap/bosh/bin/monit start %s", svc), plugin.STDOUT)
		if err != nil {
			return err
		}
	}
	return nil
}

func (p RedisBrokerPlugin) Store(endpoint plugin.ShieldEndpoint) (string, error) {
	return "", plugin.UNIMPLEMENTED
}

func (p RedisBrokerPlugin) Retrieve(endpoint plugin.ShieldEndpoint, file string) error {
	return plugin.UNIMPLEMENTED
}

func (p RedisBrokerPlugin) Purge(endpoint plugin.ShieldEndpoint, file string) error {
	return plugin.UNIMPLEMENTED
}

func getRedisEndpoint(endpoint plugin.ShieldEndpoint) (RedisEndpoint, error) {
	mode, err := endpoint.StringValue("redis_type")
	if err != nil {
		return RedisEndpoint{}, err
	}
	return RedisEndpoint{Mode: mode}, nil
}
<commit_msg>redis-broker plugin monit fix<commit_after>package main

import (
	"fmt"
	"github.com/starkandwayne/shield/plugin"
	"time"
)

func main() {
	p := RedisBrokerPlugin{
		Name:    "Pivotal Redis Broker Backup Plugin",
		Author:  "Stark & Wayne",
		Version: "0.0.1",
		Features: plugin.PluginFeatures{
			Target: "yes",
			Store:  "no",
		},
	}

	plugin.Run(p)
}

type RedisBrokerPlugin plugin.PluginInfo

type RedisEndpoint struct {
	Mode string
}

func (p RedisBrokerPlugin) Meta() plugin.PluginInfo {
	return plugin.PluginInfo(p)
}

func (p RedisBrokerPlugin) Backup(endpoint plugin.ShieldEndpoint) error {
	err := plugin.Exec("tar -c -C /var/vcap/store .", plugin.STDOUT)
	if err != nil {
		return err
	}

	return nil
}

func (p RedisBrokerPlugin) Restore(endpoint plugin.ShieldEndpoint) error {
	redis, err := getRedisEndpoint(endpoint)
	if err != nil {
		return err
	}

	var services = []string{"cf-redis-broker"}
	if redis.Mode == "dedicated" {
		services = []string{"redis", "redis-agent"}
	}

	for _, svc := range services {
		err = plugin.Exec(fmt.Sprintf("/var/vcap/bosh/bin/monit stop %s", svc), plugin.STDOUT)
		if err != nil {
			return err
		}
	}

	err = plugin.Exec(fmt.Sprintf("bash -c \"while [[ $(/var/vcap/bosh/bin/monit summary | grep redis | grep running) ]]; do sleep 1; done\"", svc), plugin.STDOUT)
	if err != nil {
		return err
	}

	// Don't look for errors here, because pkill will return non-zero if there
	// were no processes to kill in the first place.
	// FIXME: handle this better, so we know we're pkilling properly
	plugin.Exec("pkill redis-server", plugin.STDOUT)
	time.Sleep(2 * time.Second)
	plugin.Exec("pkill -9 redis-server", plugin.STDOUT)
	time.Sleep(1 * time.Second)

	err = plugin.Exec("tar -x -C /var/vcap/store . ", plugin.STDIN)
	if err != nil {
		return err
	}

	err = plugin.Exec("bash -c 'yes | find /var/vcap/store -name appendonly.aof -exec /var/vcap/packages/redis/bin/redis-check-aof --fix {} \\;'", plugin.STDOUT)
	if err != nil {
		return err
	}

	for _, svc := range services {
		err = plugin.Exec(fmt.Sprintf("/var/vcap/bosh/bin/monit start %s", svc), plugin.STDOUT)
		if err != nil {
			return err
		}
	}
	return nil
}

func (p RedisBrokerPlugin) Store(endpoint plugin.ShieldEndpoint) (string, error) {
	return "", plugin.UNIMPLEMENTED
}

func (p RedisBrokerPlugin) Retrieve(endpoint plugin.ShieldEndpoint, file string) error {
	return plugin.UNIMPLEMENTED
}

func (p RedisBrokerPlugin) Purge(endpoint plugin.ShieldEndpoint, file string) error {
	return plugin.UNIMPLEMENTED
}

func getRedisEndpoint(endpoint plugin.ShieldEndpoint) (RedisEndpoint, error) {
	mode, err := endpoint.StringValue("redis_type")
	if err != nil {
		return RedisEndpoint{}, err
	}
	return RedisEndpoint{Mode: mode}, nil
}
<|endoftext|>


<commit_before>package comb

func (s *State) Char(r rune) Parser {
	return func() (interface{}, error) {
		if s.currentRune() != r {
			return nil, NewError("Invalid character. '%c'", s.currentRune())
		}

		s.increment()

		return r, nil
	}
}

func (s *State) NotChar(r rune) Parser {
	return func() (interface{}, error) {
		if s.currentRune() == r {
			return nil, NewError("Should not be '%c'.", r)
		}

		defer s.increment()

		return s.currentRune(), nil
	}
}

func (s *State) String(str string) Parser {
	rs := ([]rune)(str)
	ps := make([]Parser, len(rs))

	for i, r := range rs {
		ps[i] = s.Char(r)
	}

	return s.And(ps...)
}

func (s *State) InString(str string) Parser {
	rs := stringToRuneSet(str)

	return func() (interface{}, error) {
		if _, ok := rs[s.currentRune()]; ok {
			defer s.increment()
			return s.currentRune(), nil
		}

		return nil, NewError("Invalid character. '%c'", s.currentRune())
	}
}

func (s *State) NotInString(str string) Parser {
	rs := stringToRuneSet(str)

	return func() (interface{}, error) {
		if _, ok := rs[s.currentRune()]; !ok {
			defer s.increment()
			return s.currentRune(), nil
		}

		return nil, NewError("Invalid character. '%c'", s.currentRune())
	}
}

func (s *State) Wrap(l, m, r Parser) Parser {
	p := s.And(l, m, r)

	return func() (interface{}, error) {
		results, err := p()

		if results, ok := results.([]interface{}); ok {
			return results[1], err
		}

		return results, err
	}
}

func (s *State) Many(p Parser) Parser {
	return func() (interface{}, error) {
		results, err := s.Many1(p)()

		if err != nil {
			return []interface{}{}, nil
		}

		return results, nil
	}
}

func (s *State) Many1(p Parser) Parser {
	return func() (interface{}, error) {
		var results []interface{}

		for i := 0; ; i++ {
			result, err := p()

			if err != nil && i == 0 {
				return nil, err
			} else if err != nil {
				break
			}

			results = append(results, result)
		}

		return results, nil
	}
}

func (s *State) Or(ps ...Parser) Parser {
	return func() (interface{}, error) {
		var err error
		old := *s

		for _, p := range ps {
			var result interface{}

			result, err = p()

			if err == nil {
				return result, nil
			}

			*s = old
		}

		return nil, err
	}
}

func (State) And(ps ...Parser) Parser {
	return func() (interface{}, error) {
		results := make([]interface{}, len(ps))

		for i, p := range ps {
			result, err := p()

			if err != nil {
				return nil, err
			}

			results[i] = result
		}

		return results, nil
	}
}

func (State) Void(p Parser) Parser {
	return func() (interface{}, error) {
		_, err := p()
		return nil, err
	}
}

func stringToRuneSet(s string) map[rune]bool {
	rs := make(map[rune]bool)

	for _, r := range s {
		rs[r] = true
	}

	return rs
}
<commit_msg>Make And backtrack<commit_after>package comb

func (s *State) Char(r rune) Parser {
	return func() (interface{}, error) {
		if s.currentRune() != r {
			return nil, NewError("Invalid character. '%c'", s.currentRune())
		}

		s.increment()

		return r, nil
	}
}

func (s *State) NotChar(r rune) Parser {
	return func() (interface{}, error) {
		if s.currentRune() == r {
			return nil, NewError("Should not be '%c'.", r)
		}

		defer s.increment()

		return s.currentRune(), nil
	}
}

func (s *State) String(str string) Parser {
	rs := ([]rune)(str)
	ps := make([]Parser, len(rs))

	for i, r := range rs {
		ps[i] = s.Char(r)
	}

	return s.And(ps...)
}

func (s *State) InString(str string) Parser {
	rs := stringToRuneSet(str)

	return func() (interface{}, error) {
		if _, ok := rs[s.currentRune()]; ok {
			defer s.increment()
			return s.currentRune(), nil
		}

		return nil, NewError("Invalid character. '%c'", s.currentRune())
	}
}

func (s *State) NotInString(str string) Parser {
	rs := stringToRuneSet(str)

	return func() (interface{}, error) {
		if _, ok := rs[s.currentRune()]; !ok {
			defer s.increment()
			return s.currentRune(), nil
		}

		return nil, NewError("Invalid character. '%c'", s.currentRune())
	}
}

func (s *State) Wrap(l, m, r Parser) Parser {
	p := s.And(l, m, r)

	return func() (interface{}, error) {
		results, err := p()

		if results, ok := results.([]interface{}); ok {
			return results[1], err
		}

		return results, err
	}
}

func (s *State) Many(p Parser) Parser {
	return func() (interface{}, error) {
		results, err := s.Many1(p)()

		if err != nil {
			return []interface{}{}, nil
		}

		return results, nil
	}
}

func (s *State) Many1(p Parser) Parser {
	return func() (interface{}, error) {
		var results []interface{}

		for i := 0; ; i++ {
			result, err := p()

			if err != nil && i == 0 {
				return nil, err
			} else if err != nil {
				break
			}

			results = append(results, result)
		}

		return results, nil
	}
}

func (s *State) Or(ps ...Parser) Parser {
	return func() (interface{}, error) {
		var err error
		old := *s

		for _, p := range ps {
			var result interface{}

			result, err = p()

			if err == nil {
				return result, nil
			}

			*s = old
		}

		return nil, err
	}
}

func (s *State) And(ps ...Parser) Parser {
	return func() (interface{}, error) {
		results := make([]interface{}, len(ps))
		old := *s

		for i, p := range ps {
			result, err := p()

			if err != nil {
				*s = old
				return nil, err
			}

			results[i] = result
		}

		return results, nil
	}
}

func (State) Void(p Parser) Parser {
	return func() (interface{}, error) {
		_, err := p()
		return nil, err
	}
}

func stringToRuneSet(s string) map[rune]bool {
	rs := make(map[rune]bool)

	for _, r := range s {
		rs[r] = true
	}

	return rs
}
<|endoftext|>


<commit_before>package pipeline

import (
	"context"

	"github.com/rancher/rancher/pkg/controllers/managementuserlegacy/pipeline/controller/pipeline"
	"github.com/rancher/rancher/pkg/controllers/managementuserlegacy/pipeline/controller/pipelineexecution"
	"github.com/rancher/rancher/pkg/controllers/managementuserlegacy/pipeline/controller/project"
	"github.com/rancher/rancher/pkg/types/config"
)

func Register(ctx context.Context, cluster *config.UserContext) {
	pipeline.Register(ctx, cluster)
	pipelineexecution.Register(ctx, cluster)
	project.Register(ctx, cluster)
}
<commit_msg>Only start pipeline user controller if v3.Pipeline exists<commit_after>package pipeline

import (
	"context"

	v3 "github.com/rancher/rancher/pkg/apis/project.cattle.io/v3"
	"github.com/rancher/rancher/pkg/controllers/managementuserlegacy/pipeline/controller/pipeline"
	"github.com/rancher/rancher/pkg/controllers/managementuserlegacy/pipeline/controller/pipelineexecution"
	"github.com/rancher/rancher/pkg/controllers/managementuserlegacy/pipeline/controller/project"
	"github.com/rancher/rancher/pkg/types/config"
	"k8s.io/apimachinery/pkg/runtime"
)

func Register(ctx context.Context, cluster *config.UserContext) {
	starter := cluster.DeferredStart(ctx, func(ctx context.Context) error {
		registerDeferred(ctx, cluster)
		return nil
	})
	AddStarter(ctx, cluster, starter)
}

func AddStarter(ctx context.Context, cluster *config.UserContext, starter func() error) {
	pipelines := cluster.Management.Project.Pipelines("")
	pipelines.AddClusterScopedHandler(ctx, "pipeline-deferred", cluster.ClusterName, func(key string, obj *v3.Pipeline) (runtime.Object, error) {
		return obj, starter()
	})
}

func registerDeferred(ctx context.Context, cluster *config.UserContext) {
	pipeline.Register(ctx, cluster)
	pipelineexecution.Register(ctx, cluster)
	project.Register(ctx, cluster)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package workflow

import (
	"errors"
	"flag"
	"fmt"
	"strconv"
	"sync"

	"golang.org/x/net/context"

	"github.com/golang/protobuf/proto"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/logutil"
	"vitess.io/vitess/go/vt/topo"

	workflowpb "vitess.io/vitess/go/vt/proto/workflow"
)

const (
	testWorkflowFactoryName = "test_workflow"

	phaseSimple PhaseType = "simple"

	errMessage = "fake error for testing retry"
)

func createTestTaskID(phase PhaseType, count int) string {
	return fmt.Sprintf("%s/%v", phase, count)
}

func init() {
	Register(testWorkflowFactoryName, &TestWorkflowFactory{})
}

// TestWorkflowFactory is the factory to create a test workflow.
type TestWorkflowFactory struct{}

// Init is part of the workflow.Factory interface.
func (*TestWorkflowFactory) Init(_ *Manager, w *workflowpb.Workflow, args []string) error {
	subFlags := flag.NewFlagSet(testWorkflowFactoryName, flag.ContinueOnError)
	retryFlag := subFlags.Bool("retry", false, "The retry flag should be true if the retry action should be tested")
	count := subFlags.Int("count", 0, "The number of simple tasks")
	enableApprovals := subFlags.Bool("enable_approvals", false, "If true, executions of tasks require user's approvals on the UI.")
	sequential := subFlags.Bool("sequential", false, "If true, executions of tasks are sequential")
	if err := subFlags.Parse(args); err != nil {
		return err
	}

	// Initialize the checkpoint.
	taskMap := make(map[string]*workflowpb.Task)
	for i := 0; i < *count; i++ {
		taskID := createTestTaskID(phaseSimple, i)
		taskMap[taskID] = &workflowpb.Task{
			Id:         taskID,
			State:      workflowpb.TaskState_TaskNotStarted,
			Attributes: map[string]string{"number": fmt.Sprintf("%v", i)},
		}
	}
	checkpoint := &workflowpb.WorkflowCheckpoint{
		CodeVersion: 0,
		Tasks:       taskMap,
		Settings:    map[string]string{"count": fmt.Sprintf("%v", *count), "retry": fmt.Sprintf("%v", *retryFlag), "enable_approvals": fmt.Sprintf("%v", *enableApprovals), "sequential": fmt.Sprintf("%v", *sequential)},
	}
	var err error
	w.Data, err = proto.Marshal(checkpoint)
	if err != nil {
		return err
	}
	return nil
}

// Instantiate is part the workflow.Factory interface.
func (*TestWorkflowFactory) Instantiate(m *Manager, w *workflowpb.Workflow, rootNode *Node) (Workflow, error) {
	checkpoint := &workflowpb.WorkflowCheckpoint{}
	if err := proto.Unmarshal(w.Data, checkpoint); err != nil {
		return nil, err
	}
	// Get the retry flags for all tasks from the checkpoint.
	retry, err := strconv.ParseBool(checkpoint.Settings["retry"])
	if err != nil {
		log.Errorf("converting retry in checkpoint.Settings to bool fails: %v", checkpoint.Settings["retry"])
		return nil, err
	}
	retryFlags := make(map[string]bool)
	for _, task := range checkpoint.Tasks {
		retryFlags[task.Id] = retry
	}

	// Get the user control flags from the checkpoint.
	enableApprovals, err := strconv.ParseBool(checkpoint.Settings["enable_approvals"])
	if err != nil {
		log.Errorf("converting enable_approvals in checkpoint.Settings to bool fails: %v", checkpoint.Settings["user_control"])
		return nil, err
	}

	sequential, err := strconv.ParseBool(checkpoint.Settings["sequential"])
	if err != nil {
		log.Errorf("converting sequential in checkpoint.Settings to bool fails: %v", checkpoint.Settings["user_control"])
		return nil, err
	}

	tw := &TestWorkflow{
		topoServer:      m.TopoServer(),
		manager:         m,
		checkpoint:      checkpoint,
		rootUINode:      rootNode,
		logger:          logutil.NewMemoryLogger(),
		retryFlags:      retryFlags,
		enableApprovals: enableApprovals,
		sequential:      sequential,
	}

	count, err := strconv.Atoi(checkpoint.Settings["count"])
	if err != nil {
		log.Errorf("converting count in checkpoint.Settings to int fails: %v", checkpoint.Settings["count"])
		return nil, err
	}

	phaseNode := &Node{
		Name:     string(phaseSimple),
		PathName: string(phaseSimple),
	}
	tw.rootUINode.Children = append(tw.rootUINode.Children, phaseNode)

	for i := 0; i < count; i++ {
		taskName := fmt.Sprintf("%v", i)
		taskUINode := &Node{
			Name:     taskName,
			PathName: taskName,
		}
		phaseNode.Children = append(phaseNode.Children, taskUINode)
	}
	return tw, nil
}

// TestWorkflow is used to unit test the ParallelRunner object. It is a
// simplified workflow of one phase. To test the ParallelRunner's retry
// behavior, we can let the tasks explicitly fail initially and succeed
// after a retry.
type TestWorkflow struct {
	ctx        context.Context
	manager    *Manager
	topoServer *topo.Server
	wi         *topo.WorkflowInfo
	logger     *logutil.MemoryLogger

	retryMu sync.Mutex
	// retryFlags stores the retry flag for all tasks.
	retryFlags map[string]bool

	rootUINode *Node

	checkpoint       *workflowpb.WorkflowCheckpoint
	checkpointWriter *CheckpointWriter

	enableApprovals bool
	sequential      bool
}

// Run implements the workflow.Workflow interface.
func (tw *TestWorkflow) Run(ctx context.Context, manager *Manager, wi *topo.WorkflowInfo) error {
	tw.ctx = ctx
	tw.wi = wi
	tw.checkpointWriter = NewCheckpointWriter(tw.topoServer, tw.checkpoint, tw.wi)

	tw.rootUINode.Display = NodeDisplayDeterminate
	tw.rootUINode.BroadcastChanges(true /* updateChildren */)

	simpleTasks := tw.getTasks(phaseSimple)
	concurrencyLevel := Parallel
	if tw.sequential {
		concurrencyLevel = Sequential
	}
	simpleRunner := NewParallelRunner(tw.ctx, tw.rootUINode, tw.checkpointWriter, simpleTasks, tw.runSimple, concurrencyLevel, tw.enableApprovals)
	simpleRunner.Run()
	return nil
}

func (tw *TestWorkflow) getTasks(phaseName PhaseType) []*workflowpb.Task {
	count, err := strconv.Atoi(tw.checkpoint.Settings["count"])
	if err != nil {
		log.Info("converting count in checkpoint.Settings to int failed: %v", tw.checkpoint.Settings["count"])
		return nil
	}
	var tasks []*workflowpb.Task
	for i := 0; i < count; i++ {
		taskID := createTestTaskID(phaseName, i)
		tasks = append(tasks, tw.checkpoint.Tasks[taskID])
	}
	return tasks
}

func (tw *TestWorkflow) runSimple(ctx context.Context, t *workflowpb.Task) error {
	log.Info("The number passed to me is %v", t.Attributes["number"])

	tw.retryMu.Lock()
	defer tw.retryMu.Unlock()
	if tw.retryFlags[t.Id] {
		log.Info("I will fail at this time since retry flag is true.")
		tw.retryFlags[t.Id] = false
		return errors.New(errMessage)
	}
	return nil
}
<commit_msg>Fix potential issue with return<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package workflow

import (
	"errors"
	"flag"
	"fmt"
	"strconv"
	"sync"

	"golang.org/x/net/context"

	"github.com/golang/protobuf/proto"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/logutil"
	"vitess.io/vitess/go/vt/topo"

	workflowpb "vitess.io/vitess/go/vt/proto/workflow"
)

const (
	testWorkflowFactoryName = "test_workflow"

	phaseSimple PhaseType = "simple"

	errMessage = "fake error for testing retry"
)

func createTestTaskID(phase PhaseType, count int) string {
	return fmt.Sprintf("%s/%v", phase, count)
}

func init() {
	Register(testWorkflowFactoryName, &TestWorkflowFactory{})
}

// TestWorkflowFactory is the factory to create a test workflow.
type TestWorkflowFactory struct{}

// Init is part of the workflow.Factory interface.
func (*TestWorkflowFactory) Init(_ *Manager, w *workflowpb.Workflow, args []string) error {
	subFlags := flag.NewFlagSet(testWorkflowFactoryName, flag.ContinueOnError)
	retryFlag := subFlags.Bool("retry", false, "The retry flag should be true if the retry action should be tested")
	count := subFlags.Int("count", 0, "The number of simple tasks")
	enableApprovals := subFlags.Bool("enable_approvals", false, "If true, executions of tasks require user's approvals on the UI.")
	sequential := subFlags.Bool("sequential", false, "If true, executions of tasks are sequential")
	if err := subFlags.Parse(args); err != nil {
		return err
	}

	// Initialize the checkpoint.
	taskMap := make(map[string]*workflowpb.Task)
	for i := 0; i < *count; i++ {
		taskID := createTestTaskID(phaseSimple, i)
		taskMap[taskID] = &workflowpb.Task{
			Id:         taskID,
			State:      workflowpb.TaskState_TaskNotStarted,
			Attributes: map[string]string{"number": fmt.Sprintf("%v", i)},
		}
	}
	checkpoint := &workflowpb.WorkflowCheckpoint{
		CodeVersion: 0,
		Tasks:       taskMap,
		Settings:    map[string]string{"count": fmt.Sprintf("%v", *count), "retry": fmt.Sprintf("%v", *retryFlag), "enable_approvals": fmt.Sprintf("%v", *enableApprovals), "sequential": fmt.Sprintf("%v", *sequential)},
	}
	var err error
	w.Data, err = proto.Marshal(checkpoint)
	if err != nil {
		return err
	}
	return nil
}

// Instantiate is part the workflow.Factory interface.
func (*TestWorkflowFactory) Instantiate(m *Manager, w *workflowpb.Workflow, rootNode *Node) (Workflow, error) {
	checkpoint := &workflowpb.WorkflowCheckpoint{}
	if err := proto.Unmarshal(w.Data, checkpoint); err != nil {
		return nil, err
	}
	// Get the retry flags for all tasks from the checkpoint.
	retry, err := strconv.ParseBool(checkpoint.Settings["retry"])
	if err != nil {
		log.Errorf("converting retry in checkpoint.Settings to bool fails: %v", checkpoint.Settings["retry"])
		return nil, err
	}
	retryFlags := make(map[string]bool)
	for _, task := range checkpoint.Tasks {
		retryFlags[task.Id] = retry
	}

	// Get the user control flags from the checkpoint.
	enableApprovals, err := strconv.ParseBool(checkpoint.Settings["enable_approvals"])
	if err != nil {
		log.Errorf("converting enable_approvals in checkpoint.Settings to bool fails: %v", checkpoint.Settings["user_control"])
		return nil, err
	}

	sequential, err := strconv.ParseBool(checkpoint.Settings["sequential"])
	if err != nil {
		log.Errorf("converting sequential in checkpoint.Settings to bool fails: %v", checkpoint.Settings["user_control"])
		return nil, err
	}

	tw := &TestWorkflow{
		topoServer:      m.TopoServer(),
		manager:         m,
		checkpoint:      checkpoint,
		rootUINode:      rootNode,
		logger:          logutil.NewMemoryLogger(),
		retryFlags:      retryFlags,
		enableApprovals: enableApprovals,
		sequential:      sequential,
	}

	count, err := strconv.Atoi(checkpoint.Settings["count"])
	if err != nil {
		log.Errorf("converting count in checkpoint.Settings to int fails: %v", checkpoint.Settings["count"])
		return nil, err
	}

	phaseNode := &Node{
		Name:     string(phaseSimple),
		PathName: string(phaseSimple),
	}
	tw.rootUINode.Children = append(tw.rootUINode.Children, phaseNode)

	for i := 0; i < count; i++ {
		taskName := fmt.Sprintf("%v", i)
		taskUINode := &Node{
			Name:     taskName,
			PathName: taskName,
		}
		phaseNode.Children = append(phaseNode.Children, taskUINode)
	}
	return tw, nil
}

// TestWorkflow is used to unit test the ParallelRunner object. It is a
// simplified workflow of one phase. To test the ParallelRunner's retry
// behavior, we can let the tasks explicitly fail initially and succeed
// after a retry.
type TestWorkflow struct {
	ctx        context.Context
	manager    *Manager
	topoServer *topo.Server
	wi         *topo.WorkflowInfo
	logger     *logutil.MemoryLogger

	retryMu sync.Mutex
	// retryFlags stores the retry flag for all tasks.
	retryFlags map[string]bool

	rootUINode *Node

	checkpoint       *workflowpb.WorkflowCheckpoint
	checkpointWriter *CheckpointWriter

	enableApprovals bool
	sequential      bool
}

// Run implements the workflow.Workflow interface.
func (tw *TestWorkflow) Run(ctx context.Context, manager *Manager, wi *topo.WorkflowInfo) error {
	tw.ctx = ctx
	tw.wi = wi
	tw.checkpointWriter = NewCheckpointWriter(tw.topoServer, tw.checkpoint, tw.wi)

	tw.rootUINode.Display = NodeDisplayDeterminate
	tw.rootUINode.BroadcastChanges(true /* updateChildren */)

	simpleTasks := tw.getTasks(phaseSimple)
	concurrencyLevel := Parallel
	if tw.sequential {
		concurrencyLevel = Sequential
	}
	simpleRunner := NewParallelRunner(tw.ctx, tw.rootUINode, tw.checkpointWriter, simpleTasks, tw.runSimple, concurrencyLevel, tw.enableApprovals)
	return simpleRunner.Run()
}

func (tw *TestWorkflow) getTasks(phaseName PhaseType) []*workflowpb.Task {
	count, err := strconv.Atoi(tw.checkpoint.Settings["count"])
	if err != nil {
		log.Info("converting count in checkpoint.Settings to int failed: %v", tw.checkpoint.Settings["count"])
		return nil
	}
	var tasks []*workflowpb.Task
	for i := 0; i < count; i++ {
		taskID := createTestTaskID(phaseName, i)
		tasks = append(tasks, tw.checkpoint.Tasks[taskID])
	}
	return tasks
}

func (tw *TestWorkflow) runSimple(ctx context.Context, t *workflowpb.Task) error {
	log.Info("The number passed to me is %v", t.Attributes["number"])

	tw.retryMu.Lock()
	defer tw.retryMu.Unlock()
	if tw.retryFlags[t.Id] {
		log.Info("I will fail at this time since retry flag is true.")
		tw.retryFlags[t.Id] = false
		return errors.New(errMessage)
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var (
	skippedPaths   = regexp.MustCompile(`Godeps|third_party|_gopath|_output|\.git|cluster/env.sh|vendor|test/e2e/generated/bindata.go|site/themes/docsy`)
	boilerplatedir = flag.String("boilerplate-dir", ".", "Boilerplate directory for boilerplate files")
	rootdir = flag.String("rootdir", "../../", "Root directory to examine")
)

func main() {
	flag.Parse()
	refs := boilerplateRefs(*boilerplatedir)
	if len(refs) == 0 {
		log.Fatal("no references in ", *boilerplatedir)
	}
	files := filesToCheck(*rootdir, refs)
	for _, file := range files {
		if !filePasses(file, refs[fileExtension(file)]) {
			fmt.Println(file)
		}
	}

}

/*
This function is to populate the refs variable with the
different boilerplate/template for different extension.
*/
func boilerplateRefs(dir string) map[string][]byte {
	refs := make(map[string][]byte)
	files, _ := filepath.Glob(dir + "/*.txt")
	for _, filename := range files {
		re := regexp.MustCompile(`\.txt`)
		extension := strings.ToLower(fileExtension(re.ReplaceAllString(filename, "")))
		data, err := ioutil.ReadFile(filename)
		if err != nil {
			log.Fatal(err)
		}
		re = regexp.MustCompile(`\r`)
		refs[extension] = re.ReplaceAll(data, nil)
	}
	return refs
}

/*
Function to check whether the processed file
is valid.
Returning false means that the file does not the
proper boilerplate template
*/
func filePasses(filename string, ref []byte) bool {
	var re *regexp.Regexp
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatal(err)
	}
	re = regexp.MustCompile(`\r`)
	data = re.ReplaceAll(data, nil)

	extension := fileExtension(filename)

	// remove build tags from the top of Go files
	if extension == "go" {
		re = regexp.MustCompile(`(?m)^(// \+build.*\n)+\n`)
		data = re.ReplaceAll(data, nil)
	}

	// remove shebang from the top of shell files
	if extension == "sh" {
		re = regexp.MustCompile(`(?m)^(#!.*\n)\n*`)
		data = re.ReplaceAll(data, nil)
	}

	// if our test file is smaller than the reference it surely fails!
	if len(data) < len(ref) {
		return false
	}

	data = data[:len(ref)]

	// Search for "Copyright YEAR" which exists in the boilerplate, but shouldn't in the real thing
	re = regexp.MustCompile(`Copyright YEAR`)
	if re.Match(data) {
		return false
	}

	// Replace all occurrences of the regex "Copyright \d{4}" with "Copyright YEAR"
	re = regexp.MustCompile(`Copyright \d{4}`)
	data = re.ReplaceAll(data, []byte(`Copyright YEAR`))

	return bytes.Equal(data, ref)
}

/**
Function to get the file extensin or the filename if the file has no extension
*/
func fileExtension(filename string) string {
	splitted := strings.Split(filepath.Base(filename), ".")
	return strings.ToLower(splitted[len(splitted)-1])
}

/**
Function to get all the files from the directory that heeds to be checked.
*/
func filesToCheck(rootDir string, extensions map[string][]byte) []string {
	var outFiles []string
	err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
		// remove current workdir from the beginig of the path in case it matches the skipped path
		cwd, _ := os.Getwd()
		// replace "\" with "\\" for windows style path
		re := regexp.MustCompile(`\\`)
		re = regexp.MustCompile(`^` + re.ReplaceAllString(cwd, `\\`))
		if !info.IsDir() && !skippedPaths.MatchString(re.ReplaceAllString(filepath.Dir(path), "")) {
			if extensions[strings.ToLower(fileExtension(path))] != nil {
				outFiles = append(outFiles, path)
			}
		}
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}
	return outFiles
}
<commit_msg>Comments refactored in boilerplate.go<commit_after>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var (
	skippedPaths   = regexp.MustCompile(`Godeps|third_party|_gopath|_output|\.git|cluster/env.sh|vendor|test/e2e/generated/bindata.go|site/themes/docsy`)
	boilerplatedir = flag.String("boilerplate-dir", ".", "Boilerplate directory for boilerplate files")
	rootdir        = flag.String("rootdir", "../../", "Root directory to examine")
)

func main() {
	flag.Parse()
	refs := extensionToBoilerplate(*boilerplatedir)
	if len(refs) == 0 {
		log.Fatal("no references in ", *boilerplatedir)
	}
	files := filesToCheck(*rootdir, refs)
	for _, file := range files {
		if !filePasses(file, refs[fileExtension(file)]) {
			fmt.Println(file)
		}
	}

}

// extensionToBoilerplate returns a map of file extension to required boilerplate text
func extensionToBoilerplate(dir string) map[string][]byte {
	refs := make(map[string][]byte)
	files, _ := filepath.Glob(dir + "/*.txt")
	for _, filename := range files {
		re := regexp.MustCompile(`\.txt`)
		extension := strings.ToLower(fileExtension(re.ReplaceAllString(filename, "")))
		data, err := ioutil.ReadFile(filename)
		if err != nil {
			log.Fatal(err)
		}
		re = regexp.MustCompile(`\r`)
		refs[extension] = re.ReplaceAll(data, nil)
	}
	return refs
}

// filePasses checks whether the processed file is valid. Returning false means that the file does not the proper boilerplate template.
func filePasses(filename string, expectedBoilerplate []byte) bool {
	var re *regexp.Regexp
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatal(err)
	}
	re = regexp.MustCompile(`\r`)
	data = re.ReplaceAll(data, nil)

	extension := fileExtension(filename)

	// remove build tags from the top of Go files
	if extension == "go" {
		re = regexp.MustCompile(`(?m)^(// \+build.*\n)+\n`)
		data = re.ReplaceAll(data, nil)
	}

	// remove shebang from the top of shell files
	if extension == "sh" {
		re = regexp.MustCompile(`(?m)^(#!.*\n)\n*`)
		data = re.ReplaceAll(data, nil)
	}

	// if our test file is smaller than the reference it surely fails!
	if len(data) < len(expectedBoilerplate) {
		return false
	}

	data = data[:len(expectedBoilerplate)]

	// Search for "Copyright YEAR" which exists in the boilerplate, but shouldn't in the real thing
	re = regexp.MustCompile(`Copyright YEAR`)
	if re.Match(data) {
		return false
	}

	// Replace all occurrences of the regex "Copyright \d{4}" with "Copyright YEAR"
	re = regexp.MustCompile(`Copyright \d{4}`)
	data = re.ReplaceAll(data, []byte(`Copyright YEAR`))

	return bytes.Equal(data, expectedBoilerplate)
}

/**
Function to get the file extensin or the filename if the file has no extension
*/
func fileExtension(filename string) string {
	splitted := strings.Split(filepath.Base(filename), ".")
	return strings.ToLower(splitted[len(splitted)-1])
}

// filesToCheck returns the list of the filers that will be checked for the boilerplate.
func filesToCheck(rootDir string, extensions map[string][]byte) []string {
	var outFiles []string
	err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
		// remove current workdir from the beginig of the path in case it matches the skipped path
		cwd, _ := os.Getwd()
		// replace "\" with "\\" for windows style path
		re := regexp.MustCompile(`\\`)
		re = regexp.MustCompile(`^` + re.ReplaceAllString(cwd, `\\`))
		if !info.IsDir() && !skippedPaths.MatchString(re.ReplaceAllString(filepath.Dir(path), "")) {
			if extensions[strings.ToLower(fileExtension(path))] != nil {
				outFiles = append(outFiles, path)
			}
		}
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}
	return outFiles
}
<|endoftext|>