<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kafka

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"strings"
	"time"

	"golang.org/x/time/rate"
	ctrl "knative.dev/control-protocol/pkg"
	ctrlnetwork "knative.dev/control-protocol/pkg/network"

	"github.com/Shopify/sarama"
	"go.opencensus.io/trace"
	"go.uber.org/zap"

	"knative.dev/pkg/logging"
	pkgsource "knative.dev/pkg/source"

	"knative.dev/eventing/pkg/adapter/v2"
	"knative.dev/eventing/pkg/kncloudevents"

	"knative.dev/eventing-kafka/pkg/common/consumer"
	"knative.dev/eventing-kafka/pkg/source/client"
	kafkasourcecontrol "knative.dev/eventing-kafka/pkg/source/control"
)

const (
	resourceGroup = "kafkasources.sources.knative.dev"
)

type AdapterConfig struct {
	adapter.EnvConfig
	client.KafkaEnvConfig

	Topics        []string `envconfig:"KAFKA_TOPICS" required:"true"`
	ConsumerGroup string   `envconfig:"KAFKA_CONSUMER_GROUP" required:"true"`
	Name          string   `envconfig:"NAME" required:"true"`
	KeyType       string   `envconfig:"KEY_TYPE" required:"false"`

	// Turn off the control server.
	DisableControlServer bool
}

func NewEnvConfig() adapter.EnvConfigAccessor {
	return &AdapterConfig{}
}

type Adapter struct {
	config        *AdapterConfig
	controlServer *ctrlnetwork.ControlServer
	saramaConfig  *sarama.Config

	httpMessageSender *kncloudevents.HTTPMessageSender
	reporter          pkgsource.StatsReporter
	logger            *zap.SugaredLogger
	keyTypeMapper     func([]byte) interface{}
	rateLimiter       *rate.Limiter
}

var (
	_           adapter.MessageAdapter                   = (*Adapter)(nil)
	_           consumer.KafkaConsumerHandler            = (*Adapter)(nil)
	_           consumer.SaramaConsumerLifecycleListener = (*Adapter)(nil)
	_           adapter.MessageAdapterConstructor        = NewAdapter
	retryConfig                                          = defaultRetryConfig()
)

func NewAdapter(ctx context.Context, processed adapter.EnvConfigAccessor, httpMessageSender *kncloudevents.HTTPMessageSender, reporter pkgsource.StatsReporter) adapter.MessageAdapter {
	logger := logging.FromContext(ctx)
	config := processed.(*AdapterConfig)

	return &Adapter{
		config:            config,
		httpMessageSender: httpMessageSender,
		reporter:          reporter,
		logger:            logger,
		keyTypeMapper:     getKeyTypeMapper(config.KeyType),
	}
}
func (a *Adapter) GetConsumerGroup() string {
	return a.config.ConsumerGroup
}

func (a *Adapter) Start(ctx context.Context) (err error) {
	a.logger.Infow("Starting with config: ",
		zap.String("Topics", strings.Join(a.config.Topics, ",")),
		zap.String("ConsumerGroup", a.config.ConsumerGroup),
		zap.String("SinkURI", a.config.Sink),
		zap.String("Name", a.config.Name),
		zap.String("Namespace", a.config.Namespace),
	)

	// Init control service
	if !a.config.DisableControlServer {
		a.controlServer, err = ctrlnetwork.StartInsecureControlServer(ctx)
		if err != nil {
			return err
		}
		a.controlServer.MessageHandler(a)
	}

	// init consumer group
	addrs, config, err := client.NewConfigWithEnv(context.Background(), &a.config.KafkaEnvConfig)
	if err != nil {
		return fmt.Errorf("failed to create the config: %w", err)
	}
	a.saramaConfig = config

	options := []consumer.SaramaConsumerHandlerOption{consumer.WithSaramaConsumerLifecycleListener(a)}
	consumerGroupFactory := consumer.NewConsumerGroupFactory(addrs, config)
	group, err := consumerGroupFactory.StartConsumerGroup(
		a.config.ConsumerGroup,
		a.config.Topics,
		a.logger,
		a,
		options...,
	)
	if err != nil {
		return fmt.Errorf("failed to start consumer group: %w", err)
	}
	defer func() {
		err := group.Close()
		if err != nil {
			a.logger.Errorw("Failed to close consumer group", zap.Error(err))
		}
	}()

	// Track errors
	go func() {
		for err := range group.Errors() {
			a.logger.Errorw("Error while consuming messages", zap.Error(err))
		}
	}()

	<-ctx.Done()
	a.logger.Info("Shutting down...")
	return nil
}

func (a *Adapter) SetReady(int32, bool) {}

func (a *Adapter) Handle(ctx context.Context, msg *sarama.ConsumerMessage) (bool, error) {
	if a.rateLimiter != nil {
		a.rateLimiter.Wait(ctx)
	}

	ctx, span := trace.StartSpan(ctx, "kafka-source")
	defer span.End()

	req, err := a.httpMessageSender.NewCloudEventRequest(ctx)
	if err != nil {
		return false, err
	}

	err = a.ConsumerMessageToHttpRequest(ctx, msg, req)
	if err != nil {
		a.logger.Debug("failed to create request", zap.Error(err))
		return true, err
	}

	res, err := a.httpMessageSender.SendWithRetries(req, retryConfig)

	if err != nil {
		a.logger.Debug("Error while sending the message", zap.Error(err))
		return false, err // Error while sending, don't commit offset
	}
	// Always try to read and close body so the connection can be reused afterwards
	if res.Body != nil {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}

	if res.StatusCode/100 != 2 {
		a.logger.Debug("Unexpected status code", zap.Int("status code", res.StatusCode))
		return false, fmt.Errorf("%d %s", res.StatusCode, http.StatusText(res.StatusCode))
	}

	reportArgs := &pkgsource.ReportArgs{
		Namespace:     a.config.Namespace,
		Name:          a.config.Name,
		ResourceGroup: resourceGroup,
	}

	_ = a.reporter.ReportEventCount(reportArgs, res.StatusCode)
	return true, nil
}

// SetRateLimiter sets the global consumer rate limiter
func (a *Adapter) SetRateLimits(r rate.Limit, b int) {
	a.rateLimiter = rate.NewLimiter(r, b)
}

func (a *Adapter) HandleServiceMessage(ctx context.Context, message ctrl.ServiceMessage) {
	// In this first PR, there is only the RA sending messages to control plane,
	// there is no message the control plane should send to the RA
	a.logger.Info("Received unexpected control message")
	message.Ack()
}

func (a *Adapter) Setup(sess sarama.ConsumerGroupSession) {
	if a.controlServer != nil {
		if err := a.controlServer.SendAndWaitForAck(kafkasourcecontrol.NotifySetupClaimsOpCode, kafkasourcecontrol.Claims(sess.Claims())); err != nil {
			a.logger.Warnf("Cannot send the claims update: %v", err)
		}
	}

	// Preemptively initialize consumer group offsets to be able to mark the source as ready
	// as soon as possible.
	if err := a.InitOffsets(sess); err != nil {
		a.logger.Warnf("Cannot initialized consumer group offsets: %v", err)
	}
}

func (a *Adapter) Cleanup(sess sarama.ConsumerGroupSession) {
	if a.controlServer != nil {
		if err := a.controlServer.SendAndWaitForAck(kafkasourcecontrol.NotifyCleanupClaimsOpCode, kafkasourcecontrol.Claims(sess.Claims())); err != nil {
			a.logger.Warnf("Cannot send the claims update: %v", err)
		}
	}
}

// InitOffsets makes sure all consumer group offsets are set.
func (a *Adapter) InitOffsets(session sarama.ConsumerGroupSession) error {
	if a.saramaConfig.Consumer.Offsets.Initial == sarama.OffsetNewest {
		// We want to make sure that ALL consumer group offsets are set to avoid
		// losing events in case the consumer group session is closed before at least one message is
		// consumed from ALL partitions.
		// If not, an event sent to a partition with an uninitialized offset
		// will not be forwarded when the session is closed (or a rebalancing is in progress).
		kafkaClient, err := sarama.NewClient(a.config.BootstrapServers, a.saramaConfig)
		if err != nil {
			return fmt.Errorf("failed to create a Kafka client: %w", err)
		}
		defer kafkaClient.Close()

		kafkaAdminClient, err := sarama.NewClusterAdminFromClient(kafkaClient)
		if err != nil {
			return fmt.Errorf("failed to create a Kafka admin client: %w", err)
		}
		defer kafkaAdminClient.Close()

		// Retrieve all partitions
		topicPartitions := make(map[string][]int32)
		for _, topic := range a.config.Topics {
			partitions, err := kafkaClient.Partitions(topic)

			if err != nil {
				return fmt.Errorf("failed to get partitions for topic %s: %w", topic, err)
			}

			topicPartitions[topic] = partitions
		}

		// Look for uninitialized offset (-1)
		offsets, err := kafkaAdminClient.ListConsumerGroupOffsets(a.config.ConsumerGroup, topicPartitions)
		if err != nil {
			return err
		}

		dirty := false
		for topic, partitions := range offsets.Blocks {
			for partition, block := range partitions {
				if block.Offset == -1 { // not initialized?

					// Fetch the newest offset in the topic/partition and set it in the consumer group
					offset, err := kafkaClient.GetOffset(topic, partition, sarama.OffsetNewest)
					if err != nil {
						return fmt.Errorf("failed to get the offset for topic %s and partition %d: %w", topic, partition, err)
					}

					a.logger.Infow("initializing offset", zap.String("topic", topic), zap.Int32("partition", partition), zap.Int64("offset", offset))

					session.MarkOffset(topic, partition, offset, "")
					dirty = true
				}
			}
		}

		if dirty {
			session.Commit()

			a.logger.Infow("consumer group offsets committed", zap.String("consumergroup", a.config.ConsumerGroup))
		}
	}

	// At this stage the KafkaSource instance is considered Ready (TODO: update KafkaSource status)
	return nil
}

// Default retry configuration, 5 retries, exponential backoff with 50ms delay
func defaultRetryConfig() *kncloudevents.RetryConfig {
	return &kncloudevents.RetryConfig{
		CheckRetry: kncloudevents.SelectiveRetry,
		RetryMax:   5,
		Backoff: func(attemptNum int, resp *http.Response) time.Duration {
			return 50 * time.Millisecond * time.Duration(math.Exp2(float64(attemptNum)))
		},
	}
}
<commit_msg>Only run InitOffset once after the adapter has started (#695)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kafka

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"strings"
	"time"

	"golang.org/x/time/rate"
	ctrl "knative.dev/control-protocol/pkg"
	ctrlnetwork "knative.dev/control-protocol/pkg/network"

	"github.com/Shopify/sarama"
	"go.opencensus.io/trace"
	"go.uber.org/zap"

	"knative.dev/pkg/logging"
	pkgsource "knative.dev/pkg/source"

	"knative.dev/eventing/pkg/adapter/v2"
	"knative.dev/eventing/pkg/kncloudevents"

	"knative.dev/eventing-kafka/pkg/common/consumer"
	"knative.dev/eventing-kafka/pkg/source/client"
	kafkasourcecontrol "knative.dev/eventing-kafka/pkg/source/control"
)

const (
	resourceGroup = "kafkasources.sources.knative.dev"
)

type AdapterConfig struct {
	adapter.EnvConfig
	client.KafkaEnvConfig

	Topics        []string `envconfig:"KAFKA_TOPICS" required:"true"`
	ConsumerGroup string   `envconfig:"KAFKA_CONSUMER_GROUP" required:"true"`
	Name          string   `envconfig:"NAME" required:"true"`
	KeyType       string   `envconfig:"KEY_TYPE" required:"false"`

	// Turn off the control server.
	DisableControlServer bool
}

func NewEnvConfig() adapter.EnvConfigAccessor {
	return &AdapterConfig{}
}

type Adapter struct {
	config            *AdapterConfig
	controlServer     *ctrlnetwork.ControlServer
	saramaConfig      *sarama.Config
	offsetInitialized bool

	httpMessageSender *kncloudevents.HTTPMessageSender
	reporter          pkgsource.StatsReporter
	logger            *zap.SugaredLogger
	keyTypeMapper     func([]byte) interface{}
	rateLimiter       *rate.Limiter
}

var (
	_           adapter.MessageAdapter                   = (*Adapter)(nil)
	_           consumer.KafkaConsumerHandler            = (*Adapter)(nil)
	_           consumer.SaramaConsumerLifecycleListener = (*Adapter)(nil)
	_           adapter.MessageAdapterConstructor        = NewAdapter
	retryConfig                                          = defaultRetryConfig()
)

func NewAdapter(ctx context.Context, processed adapter.EnvConfigAccessor, httpMessageSender *kncloudevents.HTTPMessageSender, reporter pkgsource.StatsReporter) adapter.MessageAdapter {
	logger := logging.FromContext(ctx)
	config := processed.(*AdapterConfig)

	return &Adapter{
		config:            config,
		httpMessageSender: httpMessageSender,
		reporter:          reporter,
		logger:            logger,
		keyTypeMapper:     getKeyTypeMapper(config.KeyType),
	}
}
func (a *Adapter) GetConsumerGroup() string {
	return a.config.ConsumerGroup
}

func (a *Adapter) Start(ctx context.Context) (err error) {
	a.logger.Infow("Starting with config: ",
		zap.String("Topics", strings.Join(a.config.Topics, ",")),
		zap.String("ConsumerGroup", a.config.ConsumerGroup),
		zap.String("SinkURI", a.config.Sink),
		zap.String("Name", a.config.Name),
		zap.String("Namespace", a.config.Namespace),
	)

	// Init control service
	if !a.config.DisableControlServer {
		a.controlServer, err = ctrlnetwork.StartInsecureControlServer(ctx)
		if err != nil {
			return err
		}
		a.controlServer.MessageHandler(a)
	}

	// init consumer group
	addrs, config, err := client.NewConfigWithEnv(context.Background(), &a.config.KafkaEnvConfig)
	if err != nil {
		return fmt.Errorf("failed to create the config: %w", err)
	}
	a.saramaConfig = config

	options := []consumer.SaramaConsumerHandlerOption{consumer.WithSaramaConsumerLifecycleListener(a)}
	consumerGroupFactory := consumer.NewConsumerGroupFactory(addrs, config)
	group, err := consumerGroupFactory.StartConsumerGroup(
		a.config.ConsumerGroup,
		a.config.Topics,
		a.logger,
		a,
		options...,
	)
	if err != nil {
		return fmt.Errorf("failed to start consumer group: %w", err)
	}
	defer func() {
		err := group.Close()
		if err != nil {
			a.logger.Errorw("Failed to close consumer group", zap.Error(err))
		}
	}()

	// Track errors
	go func() {
		for err := range group.Errors() {
			a.logger.Errorw("Error while consuming messages", zap.Error(err))
		}
	}()

	<-ctx.Done()
	a.logger.Info("Shutting down...")
	return nil
}

func (a *Adapter) SetReady(int32, bool) {}

func (a *Adapter) Handle(ctx context.Context, msg *sarama.ConsumerMessage) (bool, error) {
	if a.rateLimiter != nil {
		a.rateLimiter.Wait(ctx)
	}

	ctx, span := trace.StartSpan(ctx, "kafka-source")
	defer span.End()

	req, err := a.httpMessageSender.NewCloudEventRequest(ctx)
	if err != nil {
		return false, err
	}

	err = a.ConsumerMessageToHttpRequest(ctx, msg, req)
	if err != nil {
		a.logger.Debug("failed to create request", zap.Error(err))
		return true, err
	}

	res, err := a.httpMessageSender.SendWithRetries(req, retryConfig)

	if err != nil {
		a.logger.Debug("Error while sending the message", zap.Error(err))
		return false, err // Error while sending, don't commit offset
	}
	// Always try to read and close body so the connection can be reused afterwards
	if res.Body != nil {
		io.Copy(ioutil.Discard, res.Body)
		res.Body.Close()
	}

	if res.StatusCode/100 != 2 {
		a.logger.Debug("Unexpected status code", zap.Int("status code", res.StatusCode))
		return false, fmt.Errorf("%d %s", res.StatusCode, http.StatusText(res.StatusCode))
	}

	reportArgs := &pkgsource.ReportArgs{
		Namespace:     a.config.Namespace,
		Name:          a.config.Name,
		ResourceGroup: resourceGroup,
	}

	_ = a.reporter.ReportEventCount(reportArgs, res.StatusCode)
	return true, nil
}

// SetRateLimiter sets the global consumer rate limiter
func (a *Adapter) SetRateLimits(r rate.Limit, b int) {
	a.rateLimiter = rate.NewLimiter(r, b)
}

func (a *Adapter) HandleServiceMessage(ctx context.Context, message ctrl.ServiceMessage) {
	// In this first PR, there is only the RA sending messages to control plane,
	// there is no message the control plane should send to the RA
	a.logger.Info("Received unexpected control message")
	message.Ack()
}

func (a *Adapter) Setup(sess sarama.ConsumerGroupSession) {
	if a.controlServer != nil {
		if err := a.controlServer.SendAndWaitForAck(kafkasourcecontrol.NotifySetupClaimsOpCode, kafkasourcecontrol.Claims(sess.Claims())); err != nil {
			a.logger.Warnf("Cannot send the claims update: %v", err)
		}
	}

	// Preemptively initialize consumer group offsets to be able to mark the source as ready
	// as soon as possible.
	if err := a.InitOffsets(sess); err != nil {
		a.logger.Warnf("Cannot initialized consumer group offsets: %v", err)
	}
}

func (a *Adapter) Cleanup(sess sarama.ConsumerGroupSession) {
	if a.controlServer != nil {
		if err := a.controlServer.SendAndWaitForAck(kafkasourcecontrol.NotifyCleanupClaimsOpCode, kafkasourcecontrol.Claims(sess.Claims())); err != nil {
			a.logger.Warnf("Cannot send the claims update: %v", err)
		}
	}
}

// InitOffsets makes sure all consumer group offsets are set.
func (a *Adapter) InitOffsets(session sarama.ConsumerGroupSession) error {
	if !a.offsetInitialized && a.saramaConfig.Consumer.Offsets.Initial == sarama.OffsetNewest {
		// We want to make sure that ALL consumer group offsets are set to avoid
		// losing events in case the consumer group session is closed before at least one message is
		// consumed from ALL partitions.
		// If not, an event sent to a partition with an uninitialized offset
		// will not be forwarded when the session is closed (or a rebalancing is in progress).
		kafkaClient, err := sarama.NewClient(a.config.BootstrapServers, a.saramaConfig)
		if err != nil {
			return fmt.Errorf("failed to create a Kafka client: %w", err)
		}
		defer kafkaClient.Close()

		kafkaAdminClient, err := sarama.NewClusterAdminFromClient(kafkaClient)
		if err != nil {
			return fmt.Errorf("failed to create a Kafka admin client: %w", err)
		}
		defer kafkaAdminClient.Close()

		// Retrieve all partitions
		topicPartitions := make(map[string][]int32)
		for _, topic := range a.config.Topics {
			partitions, err := kafkaClient.Partitions(topic)

			if err != nil {
				return fmt.Errorf("failed to get partitions for topic %s: %w", topic, err)
			}

			topicPartitions[topic] = partitions
		}

		// Look for uninitialized offset (-1)
		offsets, err := kafkaAdminClient.ListConsumerGroupOffsets(a.config.ConsumerGroup, topicPartitions)
		if err != nil {
			return err
		}

		dirty := false
		for topic, partitions := range offsets.Blocks {
			for partition, block := range partitions {
				if block.Offset == -1 { // not initialized?

					// Fetch the newest offset in the topic/partition and set it in the consumer group
					offset, err := kafkaClient.GetOffset(topic, partition, sarama.OffsetNewest)
					if err != nil {
						return fmt.Errorf("failed to get the offset for topic %s and partition %d: %w", topic, partition, err)
					}

					a.logger.Infow("initializing offset", zap.String("topic", topic), zap.Int32("partition", partition), zap.Int64("offset", offset))

					session.MarkOffset(topic, partition, offset, "")
					dirty = true
				}
			}
		}

		if dirty {
			session.Commit()

			a.logger.Infow("consumer group offsets committed", zap.String("consumergroup", a.config.ConsumerGroup))
		}

		a.offsetInitialized = true
	}

	// At this stage the KafkaSource instance is considered Ready (TODO: update KafkaSource status)
	return nil
}

// Default retry configuration, 5 retries, exponential backoff with 50ms delay
func defaultRetryConfig() *kncloudevents.RetryConfig {
	return &kncloudevents.RetryConfig{
		CheckRetry: kncloudevents.SelectiveRetry,
		RetryMax:   5,
		Backoff: func(attemptNum int, resp *http.Response) time.Duration {
			return 50 * time.Millisecond * time.Duration(math.Exp2(float64(attemptNum)))
		},
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cacher

import (
	"k8s.io/component-base/metrics"
	"k8s.io/component-base/metrics/legacyregistry"
)

/*
 * By default, all the following metrics are defined as falling under
 * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes)
 *
 * Promoting the stability level of the metric is a responsibility of the component owner, since it
 * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
 * the metric stability policy.
 */
var (
	initCounter = metrics.NewCounterVec(
		&metrics.CounterOpts{
			Name:           "apiserver_init_events_total",
			Help:           "Counter of init events processed in watchcache broken by resource type.",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"resource"},
	)

	terminatedWatchersCounter = metrics.NewCounterVec(
		&metrics.CounterOpts{
			Name:           "apiserver_terminated_watchers_total",
			Help:           "Counter of watchers closed due to unresponsiveness broken by resource type.",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"resource"},
	)

	watchCacheCapacityIncreaseTotal = metrics.NewCounterVec(
		&metrics.CounterOpts{
			Name:           "watch_cache_capacity_increase_total",
			Help:           "Total number of watch cache capacity increase events broken by resource type.",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"resource"},
	)

	watchCacheCapacityDecreaseTotal = metrics.NewCounterVec(
		&metrics.CounterOpts{
			Name:           "watch_cache_capacity_decrease_total",
			Help:           "Total number of watch cache capacity decrease events broken by resource type.",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"resource"},
	)

	watchCacheCapacity = metrics.NewGaugeVec(
		&metrics.GaugeOpts{
			Name:           "watch_cache_capacity",
			Help:           "Total capacity of watch cache broken by resource type.",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"resource"},
	)
)

func init() {
	legacyregistry.MustRegister(initCounter)
	legacyregistry.MustRegister(terminatedWatchersCounter)
	legacyregistry.MustRegister(watchCacheCapacityIncreaseTotal)
	legacyregistry.MustRegister(watchCacheCapacityDecreaseTotal)
	legacyregistry.MustRegister(watchCacheCapacity)
}

// recordsWatchCacheCapacityChange record watchCache capacity resize(increase or decrease) operations.
func recordsWatchCacheCapacityChange(objType string, old, new int) {
	if old < new {
		watchCacheCapacityIncreaseTotal.WithLabelValues(objType).Inc()
		return
	}
	watchCacheCapacityDecreaseTotal.WithLabelValues(objType).Inc()
	watchCacheCapacity.WithLabelValues(objType).Set(float64(new))
}
<commit_msg>fix watch_cache_capacity metrics<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cacher

import (
	"k8s.io/component-base/metrics"
	"k8s.io/component-base/metrics/legacyregistry"
)

/*
 * By default, all the following metrics are defined as falling under
 * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes)
 *
 * Promoting the stability level of the metric is a responsibility of the component owner, since it
 * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
 * the metric stability policy.
 */
var (
	initCounter = metrics.NewCounterVec(
		&metrics.CounterOpts{
			Name:           "apiserver_init_events_total",
			Help:           "Counter of init events processed in watchcache broken by resource type.",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"resource"},
	)

	terminatedWatchersCounter = metrics.NewCounterVec(
		&metrics.CounterOpts{
			Name:           "apiserver_terminated_watchers_total",
			Help:           "Counter of watchers closed due to unresponsiveness broken by resource type.",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"resource"},
	)

	watchCacheCapacityIncreaseTotal = metrics.NewCounterVec(
		&metrics.CounterOpts{
			Name:           "watch_cache_capacity_increase_total",
			Help:           "Total number of watch cache capacity increase events broken by resource type.",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"resource"},
	)

	watchCacheCapacityDecreaseTotal = metrics.NewCounterVec(
		&metrics.CounterOpts{
			Name:           "watch_cache_capacity_decrease_total",
			Help:           "Total number of watch cache capacity decrease events broken by resource type.",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"resource"},
	)

	watchCacheCapacity = metrics.NewGaugeVec(
		&metrics.GaugeOpts{
			Name:           "watch_cache_capacity",
			Help:           "Total capacity of watch cache broken by resource type.",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"resource"},
	)
)

func init() {
	legacyregistry.MustRegister(initCounter)
	legacyregistry.MustRegister(terminatedWatchersCounter)
	legacyregistry.MustRegister(watchCacheCapacityIncreaseTotal)
	legacyregistry.MustRegister(watchCacheCapacityDecreaseTotal)
	legacyregistry.MustRegister(watchCacheCapacity)
}

// recordsWatchCacheCapacityChange record watchCache capacity resize(increase or decrease) operations.
func recordsWatchCacheCapacityChange(objType string, old, new int) {
	watchCacheCapacity.WithLabelValues(objType).Set(float64(new))
	if old < new {
		watchCacheCapacityIncreaseTotal.WithLabelValues(objType).Inc()
		return
	}
	watchCacheCapacityDecreaseTotal.WithLabelValues(objType).Inc()
}
<|endoftext|>


<commit_before>package objchange

import (
	"fmt"
	"strconv"

	"github.com/zclconf/go-cty/cty"
	"github.com/zclconf/go-cty/cty/convert"

	"github.com/hashicorp/terraform/configs/configschema"
)

// AssertObjectCompatible checks whether the given "actual" value is a valid
// completion of the possibly-partially-unknown "planned" value.
//
// This means that any known leaf value in "planned" must be equal to the
// corresponding value in "actual", and various other similar constraints.
//
// Any inconsistencies are reported by returning a non-zero number of errors.
// These errors are usually (but not necessarily) cty.PathError values
// referring to a particular nested value within the "actual" value.
//
// The two values must have types that conform to the given schema's implied
// type, or this function will panic.
func AssertObjectCompatible(schema *configschema.Block, planned, actual cty.Value) []error {
	return assertObjectCompatible(schema, planned, actual, nil)
}

func assertObjectCompatible(schema *configschema.Block, planned, actual cty.Value, path cty.Path) []error {
	var errs []error
	if planned.IsNull() && !actual.IsNull() {
		errs = append(errs, path.NewErrorf("was absent, but now present"))
		return errs
	}
	if actual.IsNull() && !planned.IsNull() {
		errs = append(errs, path.NewErrorf("was present, but now absent"))
		return errs
	}
	if planned.IsNull() {
		// No further checks possible if both values are null
		return errs
	}

	for name := range schema.Attributes {
		plannedV := planned.GetAttr(name)
		actualV := actual.GetAttr(name)

		path := append(path, cty.GetAttrStep{Name: name})
		moreErrs := assertValueCompatible(plannedV, actualV, path)
		errs = append(errs, moreErrs...)
	}
	for name, blockS := range schema.BlockTypes {
		plannedV := planned.GetAttr(name)
		actualV := actual.GetAttr(name)

		// As a special case, we permit a "planned" block with exactly one
		// element where all of the "leaf" values are unknown, since that's
		// what HCL's dynamic block extension generates if the for_each
		// expression is itself unknown and thus it cannot predict how many
		// child blocks will get created.
		switch blockS.Nesting {
		case configschema.NestingSingle:
			if allLeafValuesUnknown(plannedV) && !plannedV.IsNull() {
				return errs
			}
		case configschema.NestingList, configschema.NestingMap, configschema.NestingSet:
			if plannedV.LengthInt() == 1 {
				elemVs := plannedV.AsValueSlice()
				if allLeafValuesUnknown(elemVs[0]) {
					return errs
				}
			}
		default:
			panic(fmt.Sprintf("unsupported nesting mode %s", blockS.Nesting))
		}

		path := append(path, cty.GetAttrStep{Name: name})
		switch blockS.Nesting {
		case configschema.NestingSingle:
			moreErrs := assertObjectCompatible(&blockS.Block, plannedV, actualV, path)
			errs = append(errs, moreErrs...)
		case configschema.NestingList:
			// A NestingList might either be a list or a tuple, depending on
			// whether there are dynamically-typed attributes inside. However,
			// both support a similar-enough API that we can treat them the
			// same for our purposes here.
			plannedL := plannedV.LengthInt()
			actualL := actualV.LengthInt()
			if plannedL != actualL {
				errs = append(errs, path.NewErrorf("block count changed from %d to %d", plannedL, actualL))
				continue
			}
			for it := plannedV.ElementIterator(); it.Next(); {
				idx, plannedEV := it.Element()
				if !actualV.HasIndex(idx).True() {
					continue
				}
				actualEV := actualV.Index(idx)
				moreErrs := assertObjectCompatible(&blockS.Block, plannedEV, actualEV, append(path, cty.IndexStep{Key: idx}))
				errs = append(errs, moreErrs...)
			}
		case configschema.NestingMap:
			// A NestingMap might either be a map or an object, depending on
			// whether there are dynamically-typed attributes inside, but
			// that's decided statically and so both values will have the same
			// kind.
			if plannedV.Type().IsObjectType() {
				plannedAtys := plannedV.Type().AttributeTypes()
				actualAtys := actualV.Type().AttributeTypes()
				for k := range plannedAtys {
					if _, ok := actualAtys[k]; !ok {
						errs = append(errs, path.NewErrorf("block key %q has vanished", k))
						continue
					}

					plannedEV := plannedV.GetAttr(k)
					actualEV := actualV.GetAttr(k)
					moreErrs := assertObjectCompatible(&blockS.Block, plannedEV, actualEV, append(path, cty.GetAttrStep{Name: k}))
					errs = append(errs, moreErrs...)
				}
				for k := range actualAtys {
					if _, ok := plannedAtys[k]; !ok {
						errs = append(errs, path.NewErrorf("new block key %q has appeared", k))
						continue
					}
				}
			} else {
				plannedL := plannedV.LengthInt()
				actualL := actualV.LengthInt()
				if plannedL != actualL {
					errs = append(errs, path.NewErrorf("block count changed from %d to %d", plannedL, actualL))
					continue
				}
				for it := plannedV.ElementIterator(); it.Next(); {
					idx, plannedEV := it.Element()
					if !actualV.HasIndex(idx).True() {
						continue
					}
					actualEV := actualV.Index(idx)
					moreErrs := assertObjectCompatible(&blockS.Block, plannedEV, actualEV, append(path, cty.IndexStep{Key: idx}))
					errs = append(errs, moreErrs...)
				}
			}
		case configschema.NestingSet:
			// We can't do any reasonable matching of set elements since their
			// content is also their key, and so we have no way to correlate
			// them. Because of this, we simply verify that we still have the
			// same number of elements.
			plannedL := plannedV.LengthInt()
			actualL := actualV.LengthInt()
			if plannedL < actualL {
				errs = append(errs, path.NewErrorf("block set length changed from %d to %d", plannedL, actualL))
			}
		default:
			panic(fmt.Sprintf("unsupported nesting mode %s", blockS.Nesting))
		}
	}
	return errs
}

func assertValueCompatible(planned, actual cty.Value, path cty.Path) []error {
	// NOTE: We don't normally use the GoString rendering of cty.Value in
	// user-facing error messages as a rule, but we make an exception
	// for this function because we expect the user to pass this message on
	// verbatim to the provider development team and so more detail is better.

	var errs []error
	if planned.Type() == cty.DynamicPseudoType {
		// Anything goes, then
		return errs
	}
	if problems := planned.Type().TestConformance(actual.Type()); len(problems) > 0 {
		errs = append(errs, path.NewErrorf("wrong final value type: %s", convert.MismatchMessage(actual.Type(), planned.Type())))
		// If the types don't match then we can't do any other comparisons,
		// so we bail early.
		return errs
	}

	ty := planned.Type()
	switch {

	case ty == cty.DynamicPseudoType || !planned.IsKnown():
		// We didn't know what were going to end up with during plan, so
		// anything goes during apply.
		return errs

	case !actual.IsKnown():
		errs = append(errs, path.NewErrorf("was known, but now unknown"))

	case ty.IsPrimitiveType():
		if !actual.Equals(planned).True() {
			errs = append(errs, path.NewErrorf("was %#v, but now %#v", planned, actual))
		}

	case ty.IsListType() || ty.IsMapType() || ty.IsTupleType():
		for it := planned.ElementIterator(); it.Next(); {
			k, plannedV := it.Element()
			if !actual.HasIndex(k).True() {
				errs = append(errs, path.NewErrorf("element %s has vanished", indexStrForErrors(k)))
				continue
			}

			actualV := actual.Index(k)
			moreErrs := assertValueCompatible(plannedV, actualV, append(path, cty.IndexStep{Key: k}))
			errs = append(errs, moreErrs...)
		}

		for it := actual.ElementIterator(); it.Next(); {
			k, _ := it.Element()
			if !planned.HasIndex(k).True() {
				errs = append(errs, path.NewErrorf("new element %s has appeared", indexStrForErrors(k)))
			}
		}

	case ty.IsObjectType():
		atys := ty.AttributeTypes()
		for name := range atys {
			// Because we already tested that the two values have the same type,
			// we can assume that the same attributes are present in both and
			// focus just on testing their values.
			plannedV := planned.GetAttr(name)
			actualV := actual.GetAttr(name)
			moreErrs := assertValueCompatible(plannedV, actualV, append(path, cty.GetAttrStep{Name: name}))
			errs = append(errs, moreErrs...)
		}

	case ty.IsSetType():
		// We can't really do anything useful for sets here because changing
		// an unknown element to known changes the identity of the element, and
		// so we can't correlate them properly. However, we will at least check
		// to ensure that the number of elements is consistent, along with
		// the general type-match checks we ran earlier in this function.
		plannedL := planned.LengthInt()
		actualL := actual.LengthInt()
		if plannedL < actualL {
			errs = append(errs, path.NewErrorf("length changed from %d to %d", plannedL, actualL))
		}
	}

	return errs
}

func indexStrForErrors(v cty.Value) string {
	switch v.Type() {
	case cty.Number:
		return v.AsBigFloat().Text('f', -1)
	case cty.String:
		return strconv.Quote(v.AsString())
	default:
		// Should be impossible, since no other index types are allowed!
		return fmt.Sprintf("%#v", v)
	}
}

func allLeafValuesUnknown(v cty.Value) bool {
	seenKnownValue := false
	cty.Walk(v, func(path cty.Path, cv cty.Value) (bool, error) {
		if cv.IsNull() {
			seenKnownValue = true
		}
		if cv.Type().IsPrimitiveType() && cv.IsKnown() {
			seenKnownValue = true
		}
		return true, nil
	})
	return !seenKnownValue
}
<commit_msg>plans/objchange: Don't panic if old or new values are null<commit_after>package objchange

import (
	"fmt"
	"strconv"

	"github.com/zclconf/go-cty/cty"
	"github.com/zclconf/go-cty/cty/convert"

	"github.com/hashicorp/terraform/configs/configschema"
)

// AssertObjectCompatible checks whether the given "actual" value is a valid
// completion of the possibly-partially-unknown "planned" value.
//
// This means that any known leaf value in "planned" must be equal to the
// corresponding value in "actual", and various other similar constraints.
//
// Any inconsistencies are reported by returning a non-zero number of errors.
// These errors are usually (but not necessarily) cty.PathError values
// referring to a particular nested value within the "actual" value.
//
// The two values must have types that conform to the given schema's implied
// type, or this function will panic.
func AssertObjectCompatible(schema *configschema.Block, planned, actual cty.Value) []error {
	return assertObjectCompatible(schema, planned, actual, nil)
}

func assertObjectCompatible(schema *configschema.Block, planned, actual cty.Value, path cty.Path) []error {
	var errs []error
	if planned.IsNull() && !actual.IsNull() {
		errs = append(errs, path.NewErrorf("was absent, but now present"))
		return errs
	}
	if actual.IsNull() && !planned.IsNull() {
		errs = append(errs, path.NewErrorf("was present, but now absent"))
		return errs
	}
	if planned.IsNull() {
		// No further checks possible if both values are null
		return errs
	}

	for name := range schema.Attributes {
		plannedV := planned.GetAttr(name)
		actualV := actual.GetAttr(name)

		path := append(path, cty.GetAttrStep{Name: name})
		moreErrs := assertValueCompatible(plannedV, actualV, path)
		errs = append(errs, moreErrs...)
	}
	for name, blockS := range schema.BlockTypes {
		plannedV := planned.GetAttr(name)
		actualV := actual.GetAttr(name)

		// As a special case, we permit a "planned" block with exactly one
		// element where all of the "leaf" values are unknown, since that's
		// what HCL's dynamic block extension generates if the for_each
		// expression is itself unknown and thus it cannot predict how many
		// child blocks will get created.
		switch blockS.Nesting {
		case configschema.NestingSingle:
			if allLeafValuesUnknown(plannedV) && !plannedV.IsNull() {
				return errs
			}
		case configschema.NestingList, configschema.NestingMap, configschema.NestingSet:
			if plannedV.LengthInt() == 1 {
				elemVs := plannedV.AsValueSlice()
				if allLeafValuesUnknown(elemVs[0]) {
					return errs
				}
			}
		default:
			panic(fmt.Sprintf("unsupported nesting mode %s", blockS.Nesting))
		}

		path := append(path, cty.GetAttrStep{Name: name})
		switch blockS.Nesting {
		case configschema.NestingSingle:
			moreErrs := assertObjectCompatible(&blockS.Block, plannedV, actualV, path)
			errs = append(errs, moreErrs...)
		case configschema.NestingList:
			// A NestingList might either be a list or a tuple, depending on
			// whether there are dynamically-typed attributes inside. However,
			// both support a similar-enough API that we can treat them the
			// same for our purposes here.
			plannedL := plannedV.LengthInt()
			actualL := actualV.LengthInt()
			if plannedL != actualL {
				errs = append(errs, path.NewErrorf("block count changed from %d to %d", plannedL, actualL))
				continue
			}
			for it := plannedV.ElementIterator(); it.Next(); {
				idx, plannedEV := it.Element()
				if !actualV.HasIndex(idx).True() {
					continue
				}
				actualEV := actualV.Index(idx)
				moreErrs := assertObjectCompatible(&blockS.Block, plannedEV, actualEV, append(path, cty.IndexStep{Key: idx}))
				errs = append(errs, moreErrs...)
			}
		case configschema.NestingMap:
			// A NestingMap might either be a map or an object, depending on
			// whether there are dynamically-typed attributes inside, but
			// that's decided statically and so both values will have the same
			// kind.
			if plannedV.Type().IsObjectType() {
				plannedAtys := plannedV.Type().AttributeTypes()
				actualAtys := actualV.Type().AttributeTypes()
				for k := range plannedAtys {
					if _, ok := actualAtys[k]; !ok {
						errs = append(errs, path.NewErrorf("block key %q has vanished", k))
						continue
					}

					plannedEV := plannedV.GetAttr(k)
					actualEV := actualV.GetAttr(k)
					moreErrs := assertObjectCompatible(&blockS.Block, plannedEV, actualEV, append(path, cty.GetAttrStep{Name: k}))
					errs = append(errs, moreErrs...)
				}
				for k := range actualAtys {
					if _, ok := plannedAtys[k]; !ok {
						errs = append(errs, path.NewErrorf("new block key %q has appeared", k))
						continue
					}
				}
			} else {
				plannedL := plannedV.LengthInt()
				actualL := actualV.LengthInt()
				if plannedL != actualL {
					errs = append(errs, path.NewErrorf("block count changed from %d to %d", plannedL, actualL))
					continue
				}
				for it := plannedV.ElementIterator(); it.Next(); {
					idx, plannedEV := it.Element()
					if !actualV.HasIndex(idx).True() {
						continue
					}
					actualEV := actualV.Index(idx)
					moreErrs := assertObjectCompatible(&blockS.Block, plannedEV, actualEV, append(path, cty.IndexStep{Key: idx}))
					errs = append(errs, moreErrs...)
				}
			}
		case configschema.NestingSet:
			// We can't do any reasonable matching of set elements since their
			// content is also their key, and so we have no way to correlate
			// them. Because of this, we simply verify that we still have the
			// same number of elements.
			plannedL := plannedV.LengthInt()
			actualL := actualV.LengthInt()
			if plannedL < actualL {
				errs = append(errs, path.NewErrorf("block set length changed from %d to %d", plannedL, actualL))
			}
		default:
			panic(fmt.Sprintf("unsupported nesting mode %s", blockS.Nesting))
		}
	}
	return errs
}

func assertValueCompatible(planned, actual cty.Value, path cty.Path) []error {
	// NOTE: We don't normally use the GoString rendering of cty.Value in
	// user-facing error messages as a rule, but we make an exception
	// for this function because we expect the user to pass this message on
	// verbatim to the provider development team and so more detail is better.

	var errs []error
	if planned.Type() == cty.DynamicPseudoType {
		// Anything goes, then
		return errs
	}
	if problems := planned.Type().TestConformance(actual.Type()); len(problems) > 0 {
		errs = append(errs, path.NewErrorf("wrong final value type: %s", convert.MismatchMessage(actual.Type(), planned.Type())))
		// If the types don't match then we can't do any other comparisons,
		// so we bail early.
		return errs
	}

	if actual.IsNull() {
		if planned.IsNull() {
			return nil
		}
		errs = append(errs, path.NewErrorf("was %#v, but now null", planned))
	}
	if planned.IsNull() {
		errs = append(errs, path.NewErrorf("was null, but now %#v", actual))
	}

	ty := planned.Type()
	switch {

	case ty == cty.DynamicPseudoType || !planned.IsKnown():
		// We didn't know what were going to end up with during plan, so
		// anything goes during apply.
		return errs

	case !actual.IsKnown():
		errs = append(errs, path.NewErrorf("was known, but now unknown"))

	case ty.IsPrimitiveType():
		if !actual.Equals(planned).True() {
			errs = append(errs, path.NewErrorf("was %#v, but now %#v", planned, actual))
		}

	case ty.IsListType() || ty.IsMapType() || ty.IsTupleType():
		for it := planned.ElementIterator(); it.Next(); {
			k, plannedV := it.Element()
			if !actual.HasIndex(k).True() {
				errs = append(errs, path.NewErrorf("element %s has vanished", indexStrForErrors(k)))
				continue
			}

			actualV := actual.Index(k)
			moreErrs := assertValueCompatible(plannedV, actualV, append(path, cty.IndexStep{Key: k}))
			errs = append(errs, moreErrs...)
		}

		for it := actual.ElementIterator(); it.Next(); {
			k, _ := it.Element()
			if !planned.HasIndex(k).True() {
				errs = append(errs, path.NewErrorf("new element %s has appeared", indexStrForErrors(k)))
			}
		}

	case ty.IsObjectType():
		atys := ty.AttributeTypes()
		for name := range atys {
			// Because we already tested that the two values have the same type,
			// we can assume that the same attributes are present in both and
			// focus just on testing their values.
			plannedV := planned.GetAttr(name)
			actualV := actual.GetAttr(name)
			moreErrs := assertValueCompatible(plannedV, actualV, append(path, cty.GetAttrStep{Name: name}))
			errs = append(errs, moreErrs...)
		}

	case ty.IsSetType():
		// We can't really do anything useful for sets here because changing
		// an unknown element to known changes the identity of the element, and
		// so we can't correlate them properly. However, we will at least check
		// to ensure that the number of elements is consistent, along with
		// the general type-match checks we ran earlier in this function.
		plannedL := planned.LengthInt()
		actualL := actual.LengthInt()
		if plannedL < actualL {
			errs = append(errs, path.NewErrorf("length changed from %d to %d", plannedL, actualL))
		}
	}

	return errs
}

func indexStrForErrors(v cty.Value) string {
	switch v.Type() {
	case cty.Number:
		return v.AsBigFloat().Text('f', -1)
	case cty.String:
		return strconv.Quote(v.AsString())
	default:
		// Should be impossible, since no other index types are allowed!
		return fmt.Sprintf("%#v", v)
	}
}

func allLeafValuesUnknown(v cty.Value) bool {
	seenKnownValue := false
	cty.Walk(v, func(path cty.Path, cv cty.Value) (bool, error) {
		if cv.IsNull() {
			seenKnownValue = true
		}
		if cv.Type().IsPrimitiveType() && cv.IsKnown() {
			seenKnownValue = true
		}
		return true, nil
	})
	return !seenKnownValue
}
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package azure

import (
	"encoding/xml"
	"fmt"

	"github.com/Azure/azure-sdk-for-go/management/storageservice"
)

var (
	azureImageURL = "services/images"
)

func (a *API) GetStorageServiceKeys(account string) (storageservice.GetStorageServiceKeysResponse, error) {
	return storageservice.NewClient(a.client).GetStorageServiceKeys(account)
}

// https://msdn.microsoft.com/en-us/library/azure/jj157192.aspx
func (a *API) AddOSImage(md *OSImage) error {
	data, err := xml.Marshal(md)
	if err != nil {
		return err
	}

	op, err := a.client.SendAzurePostRequest(azureImageURL, data)
	if err != nil {
		return err
	}

	return a.client.WaitForOperation(op, nil)
}

func (a *API) OSImageExists(name string) (bool, error) {
	url := fmt.Sprintf("%s/%s", azureImageURL, name)
	response, err := a.client.SendAzureGetRequest(url)
	if err != nil {
		return false, err
	}

	var image OSImage

	if err := xml.Unmarshal(response, &image); err != nil {
		return false, err
	}

	if image.Name == name {
		return true, nil
	}

	return false, nil
}
<commit_msg>platform/api/azure: convert resource not found error into bool correctly for OSImageExists<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package azure

import (
	"encoding/xml"
	"fmt"

	"github.com/Azure/azure-sdk-for-go/management"
	"github.com/Azure/azure-sdk-for-go/management/storageservice"
)

var (
	azureImageURL = "services/images"
)

func (a *API) GetStorageServiceKeys(account string) (storageservice.GetStorageServiceKeysResponse, error) {
	return storageservice.NewClient(a.client).GetStorageServiceKeys(account)
}

// https://msdn.microsoft.com/en-us/library/azure/jj157192.aspx
func (a *API) AddOSImage(md *OSImage) error {
	data, err := xml.Marshal(md)
	if err != nil {
		return err
	}

	op, err := a.client.SendAzurePostRequest(azureImageURL, data)
	if err != nil {
		return err
	}

	return a.client.WaitForOperation(op, nil)
}

func (a *API) OSImageExists(name string) (bool, error) {
	url := fmt.Sprintf("%s/%s", azureImageURL, name)
	response, err := a.client.SendAzureGetRequest(url)
	if err != nil {
		if management.IsResourceNotFoundError(err) {
			return false, nil
		}

		return false, err
	}

	var image OSImage

	if err := xml.Unmarshal(response, &image); err != nil {
		return false, err
	}

	if image.Name == name {
		return true, nil
	}

	return false, nil
}
<|endoftext|>


<commit_before>package freetree

import (
	"log"
	"testing"
)

// -----------------------------------------------------------------------------

type intTest int

func (i1 intTest) Less(i2 Comparable) bool { return i1 < i2.(intTest) }

func TestSimpleTree_pair_sorted_input(t *testing.T) {
	expected := ComparableArray{intTest(1), intTest(3), intTest(2), intTest(5), intTest(6), intTest(4)}

	st := NewSimpleTree()
	cs := ComparableArray{intTest(1), intTest(2), intTest(3), intTest(4), intTest(5), intTest(6)}

	st.InsertArray(cs)
	flat := st.Flatten()

	for i := range flat {
		if flat[i] != expected[i] {
			t.Error("expected != flat")
			t.FailNow()
		}
	}

	if st.Ascend(intTest(1)).(intTest) != intTest(1) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(2)).(intTest) != intTest(2) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(5)).(intTest) != intTest(5) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(6)).(intTest) != intTest(6) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(7)) != nil {
		log.Println(st.Ascend(intTest(7)).(intTest))
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(8)) != nil {
		log.Println(st.Ascend(intTest(8)).(intTest))
		t.Error("unexpected retval")
	}
}

func TestSimpleTree_pair_sorted_input_rebalanced(t *testing.T) {
	expected := ComparableArray{intTest(1), intTest(3), intTest(2), intTest(5), intTest(6), intTest(4)}

	st := NewSimpleTree()
	cs := ComparableArray{intTest(1), intTest(2), intTest(3), intTest(4), intTest(5), intTest(6)}

	st.InsertArray(cs)
	flat := st.Flatten()

	for i := range flat {
		if flat[i] != expected[i] {
			t.Error("expected != flat")
			t.FailNow()
		}
	}

	if st.Ascend(intTest(1)).(intTest) != intTest(1) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(2)).(intTest) != intTest(2) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(5)).(intTest) != intTest(5) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(6)).(intTest) != intTest(6) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(7)) != nil {
		log.Println(st.Ascend(intTest(7)).(intTest))
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(8)) != nil {
		log.Println(st.Ascend(intTest(8)).(intTest))
		t.Error("unexpected retval")
	}
}
<commit_msg>added TestSimpleTree_pair_unsorted_input<commit_after>package freetree

import (
	"log"
	"testing"
)

// -----------------------------------------------------------------------------

type intTest int

func (i1 intTest) Less(i2 Comparable) bool { return i1 < i2.(intTest) }

func TestSimpleTree_pair_sorted_input(t *testing.T) {
	expected := ComparableArray{intTest(1), intTest(3), intTest(2), intTest(5), intTest(6), intTest(4)}

	st := NewSimpleTree()
	cs := ComparableArray{intTest(1), intTest(2), intTest(3), intTest(4), intTest(5), intTest(6)}

	st.InsertArray(cs)
	flat := st.Flatten()

	for i := range flat {
		if flat[i] != expected[i] {
			t.Error("expected != flat")
			t.FailNow()
		}
	}

	if st.Ascend(intTest(1)).(intTest) != intTest(1) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(2)).(intTest) != intTest(2) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(5)).(intTest) != intTest(5) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(6)).(intTest) != intTest(6) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(7)) != nil {
		log.Println(st.Ascend(intTest(7)).(intTest))
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(8)) != nil {
		log.Println(st.Ascend(intTest(8)).(intTest))
		t.Error("unexpected retval")
	}
}

func TestSimpleTree_pair_sorted_input_rebalanced(t *testing.T) {
	expected := ComparableArray{intTest(1), intTest(3), intTest(2), intTest(5), intTest(6), intTest(4)}

	st := NewSimpleTree()
	cs := ComparableArray{intTest(1), intTest(2), intTest(3), intTest(4), intTest(5), intTest(6)}

	st.InsertArray(cs)
	flat := st.Flatten()

	for i := range flat {
		if flat[i] != expected[i] {
			t.Error("expected != flat")
			t.FailNow()
		}
	}

	if st.Ascend(intTest(1)).(intTest) != intTest(1) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(2)).(intTest) != intTest(2) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(5)).(intTest) != intTest(5) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(6)).(intTest) != intTest(6) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(7)) != nil {
		log.Println(st.Ascend(intTest(7)).(intTest))
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(8)) != nil {
		log.Println(st.Ascend(intTest(8)).(intTest))
		t.Error("unexpected retval")
	}
}

func TestSimpleTree_pair_unsorted_input(t *testing.T) {
	expected := ComparableArray{intTest(3), intTest(2), intTest(6), intTest(5), intTest(4), intTest(1)}

	st := NewSimpleTree()
	cs := ComparableArray{intTest(5), intTest(4), intTest(6), intTest(1), intTest(3), intTest(2)}

	st.InsertArray(cs)
	flat := st.Flatten()

	for i := range flat {
		if flat[i] != expected[i] {
			t.Error("expected != flat")
			t.FailNow()
		}
	}

	if st.Ascend(intTest(1)).(intTest) != intTest(1) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(2)).(intTest) != intTest(2) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(5)).(intTest) != intTest(5) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(6)).(intTest) != intTest(6) {
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(7)) != nil {
		log.Println(st.Ascend(intTest(7)).(intTest))
		t.Error("unexpected retval")
	}
	if st.Ascend(intTest(8)) != nil {
		log.Println(st.Ascend(intTest(8)).(intTest))
		t.Error("unexpected retval")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package generators

import (
	"fmt"
	"path"
	"path/filepath"
	"strings"

	"k8s.io/gengo/args"
	"k8s.io/gengo/generator"
	"k8s.io/gengo/namer"
	"k8s.io/gengo/types"
	"k8s.io/klog"

	"k8s.io/code-generator/cmd/client-gen/generators/util"
	clientgentypes "k8s.io/code-generator/cmd/client-gen/types"
	informergenargs "k8s.io/code-generator/cmd/informer-gen/args"
)

// NameSystems returns the name system used by the generators in this package.
func NameSystems() namer.NameSystems {
	pluralExceptions := map[string]string{
		"Endpoints": "Endpoints",
	}
	return namer.NameSystems{
		"public":             namer.NewPublicNamer(0),
		"private":            namer.NewPrivateNamer(0),
		"raw":                namer.NewRawNamer("", nil),
		"publicPlural":       namer.NewPublicPluralNamer(pluralExceptions),
		"allLowercasePlural": namer.NewAllLowercasePluralNamer(pluralExceptions),
		"lowercaseSingular":  &lowercaseSingularNamer{},
	}
}

// lowercaseSingularNamer implements Namer
type lowercaseSingularNamer struct{}

// Name returns t's name in all lowercase.
func (n *lowercaseSingularNamer) Name(t *types.Type) string {
	return strings.ToLower(t.Name.Name)
}

// DefaultNameSystem returns the default name system for ordering the types to be
// processed by the generators in this package.
func DefaultNameSystem() string {
	return "public"
}

// objectMetaForPackage returns the type of ObjectMeta used by package p.
func objectMetaForPackage(p *types.Package) (*types.Type, bool, error) {
	generatingForPackage := false
	for _, t := range p.Types {
		if !util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...)).GenerateClient {
			continue
		}
		generatingForPackage = true
		for _, member := range t.Members {
			if member.Name == "ObjectMeta" {
				return member.Type, isInternal(member), nil
			}
		}
	}
	if generatingForPackage {
		return nil, false, fmt.Errorf("unable to find ObjectMeta for any types in package %s", p.Path)
	}
	return nil, false, nil
}

// isInternal returns true if the tags for a member do not contain a json tag
func isInternal(m types.Member) bool {
	return !strings.Contains(m.Tags, "json")
}

func packageForInternalInterfaces(base string) string {
	return filepath.Join(base, "internalinterfaces")
}

func vendorless(p string) string {
	if pos := strings.LastIndex(p, "/vendor/"); pos != -1 {
		return p[pos+len("/vendor/"):]
	}
	return p
}

// Packages makes the client package definition.
func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages {
	boilerplate, err := arguments.LoadGoBoilerplate()
	if err != nil {
		klog.Fatalf("Failed loading boilerplate: %v", err)
	}

	customArgs, ok := arguments.CustomArgs.(*informergenargs.CustomArgs)
	if !ok {
		klog.Fatalf("Wrong CustomArgs type: %T", arguments.CustomArgs)
	}

	internalVersionPackagePath := filepath.Join(arguments.OutputPackagePath)
	externalVersionPackagePath := filepath.Join(arguments.OutputPackagePath)
	if !customArgs.SingleDirectory {
		internalVersionPackagePath = filepath.Join(arguments.OutputPackagePath, "internalversion")
		externalVersionPackagePath = filepath.Join(arguments.OutputPackagePath, "externalversions")
	}

	var packageList generator.Packages
	typesForGroupVersion := make(map[clientgentypes.GroupVersion][]*types.Type)

	externalGroupVersions := make(map[string]clientgentypes.GroupVersions)
	internalGroupVersions := make(map[string]clientgentypes.GroupVersions)
	groupGoNames := make(map[string]string)
	for _, inputDir := range arguments.InputDirs {
		p := context.Universe.Package(vendorless(inputDir))

		objectMeta, internal, err := objectMetaForPackage(p)
		if err != nil {
			klog.Fatal(err)
		}
		if objectMeta == nil {
			// no types in this package had genclient
			continue
		}

		var gv clientgentypes.GroupVersion
		var targetGroupVersions map[string]clientgentypes.GroupVersions

		if internal {
			lastSlash := strings.LastIndex(p.Path, "/")
			if lastSlash == -1 {
				klog.Fatalf("error constructing internal group version for package %q", p.Path)
			}
			gv.Group = clientgentypes.Group(p.Path[lastSlash+1:])
			targetGroupVersions = internalGroupVersions
		} else {
			parts := strings.Split(p.Path, "/")
			gv.Group = clientgentypes.Group(parts[len(parts)-2])
			gv.Version = clientgentypes.Version(parts[len(parts)-1])
			targetGroupVersions = externalGroupVersions
		}
		groupPackageName := gv.Group.NonEmpty()
		gvPackage := path.Clean(p.Path)

		// If there's a comment of the form "// +groupName=somegroup" or
		// "// +groupName=somegroup.foo.bar.io", use the first field (somegroup) as the name of the
		// group when generating.
		if override := types.ExtractCommentTags("+", p.Comments)["groupName"]; override != nil {
			gv.Group = clientgentypes.Group(override[0])
		}

		// If there's a comment of the form "// +groupGoName=SomeUniqueShortName", use that as
		// the Go group identifier in CamelCase. It defaults
		groupGoNames[groupPackageName] = namer.IC(strings.Split(gv.Group.NonEmpty(), ".")[0])
		if override := types.ExtractCommentTags("+", p.Comments)["groupGoName"]; override != nil {
			groupGoNames[groupPackageName] = namer.IC(override[0])
		}

		var typesToGenerate []*types.Type
		for _, t := range p.Types {
			tags := util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...))
			if !tags.GenerateClient || tags.NoVerbs || !tags.HasVerb("list") || !tags.HasVerb("watch") {
				continue
			}

			typesToGenerate = append(typesToGenerate, t)

			if _, ok := typesForGroupVersion[gv]; !ok {
				typesForGroupVersion[gv] = []*types.Type{}
			}
			typesForGroupVersion[gv] = append(typesForGroupVersion[gv], t)
		}
		if len(typesToGenerate) == 0 {
			continue
		}

		groupVersionsEntry, ok := targetGroupVersions[groupPackageName]
		if !ok {
			groupVersionsEntry = clientgentypes.GroupVersions{
				PackageName: groupPackageName,
				Group:       gv.Group,
			}
		}
		groupVersionsEntry.Versions = append(groupVersionsEntry.Versions, clientgentypes.PackageVersion{Version: gv.Version, Package: gvPackage})
		targetGroupVersions[groupPackageName] = groupVersionsEntry

		orderer := namer.Orderer{Namer: namer.NewPrivateNamer(0)}
		typesToGenerate = orderer.OrderTypes(typesToGenerate)

		if internal {
			packageList = append(packageList, versionPackage(internalVersionPackagePath, groupPackageName, gv, groupGoNames[groupPackageName], boilerplate, typesToGenerate, customArgs.InternalClientSetPackage, customArgs.ListersPackage))
		} else {
			packageList = append(packageList, versionPackage(externalVersionPackagePath, groupPackageName, gv, groupGoNames[groupPackageName], boilerplate, typesToGenerate, customArgs.VersionedClientSetPackage, customArgs.ListersPackage))
		}
	}

	if len(externalGroupVersions) != 0 {
		packageList = append(packageList, factoryInterfacePackage(externalVersionPackagePath, boilerplate, customArgs.VersionedClientSetPackage))
		packageList = append(packageList, factoryPackage(externalVersionPackagePath, boilerplate, groupGoNames, externalGroupVersions, customArgs.VersionedClientSetPackage, typesForGroupVersion))
		for _, gvs := range externalGroupVersions {
			packageList = append(packageList, groupPackage(externalVersionPackagePath, gvs, boilerplate))
		}
	}

	if len(internalGroupVersions) != 0 {
		packageList = append(packageList, factoryInterfacePackage(internalVersionPackagePath, boilerplate, customArgs.InternalClientSetPackage))
		packageList = append(packageList, factoryPackage(internalVersionPackagePath, boilerplate, groupGoNames, internalGroupVersions, customArgs.InternalClientSetPackage, typesForGroupVersion))
		for _, gvs := range internalGroupVersions {
			packageList = append(packageList, groupPackage(internalVersionPackagePath, gvs, boilerplate))
		}
	}

	return packageList
}

func factoryPackage(basePackage string, boilerplate []byte, groupGoNames map[string]string, groupVersions map[string]clientgentypes.GroupVersions, clientSetPackage string, typesForGroupVersion map[clientgentypes.GroupVersion][]*types.Type) generator.Package {
	return &generator.DefaultPackage{
		PackageName: filepath.Base(basePackage),
		PackagePath: basePackage,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &factoryGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "factory",
				},
				outputPackage:             basePackage,
				imports:                   generator.NewImportTracker(),
				groupVersions:             groupVersions,
				clientSetPackage:          clientSetPackage,
				internalInterfacesPackage: packageForInternalInterfaces(basePackage),
				gvGoNames:                 groupGoNames,
			})

			generators = append(generators, &genericGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "generic",
				},
				outputPackage:        basePackage,
				imports:              generator.NewImportTracker(),
				groupVersions:        groupVersions,
				typesForGroupVersion: typesForGroupVersion,
				groupGoNames:         groupGoNames,
			})

			return generators
		},
	}
}

func factoryInterfacePackage(basePackage string, boilerplate []byte, clientSetPackage string) generator.Package {
	packagePath := packageForInternalInterfaces(basePackage)

	return &generator.DefaultPackage{
		PackageName: filepath.Base(packagePath),
		PackagePath: packagePath,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &factoryInterfaceGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "factory_interfaces",
				},
				outputPackage:    packagePath,
				imports:          generator.NewImportTracker(),
				clientSetPackage: clientSetPackage,
			})

			return generators
		},
	}
}

func groupPackage(basePackage string, groupVersions clientgentypes.GroupVersions, boilerplate []byte) generator.Package {
	packagePath := filepath.Join(basePackage, groupVersions.PackageName)

	return &generator.DefaultPackage{
		PackageName: groupVersions.PackageName,
		PackagePath: packagePath,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &groupInterfaceGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "interface",
				},
				outputPackage:             packagePath,
				groupVersions:             groupVersions,
				imports:                   generator.NewImportTracker(),
				internalInterfacesPackage: packageForInternalInterfaces(basePackage),
			})
			return generators
		},
		FilterFunc: func(c *generator.Context, t *types.Type) bool {
			tags := util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...))
			return tags.GenerateClient && tags.HasVerb("list") && tags.HasVerb("watch")
		},
	}
}

func versionPackage(basePackage string, groupPkgName string, gv clientgentypes.GroupVersion, groupGoName string, boilerplate []byte, typesToGenerate []*types.Type, clientSetPackage, listersPackage string) generator.Package {
	packagePath := filepath.Join(basePackage, groupPkgName, strings.ToLower(gv.Version.NonEmpty()))

	return &generator.DefaultPackage{
		PackageName: strings.ToLower(gv.Version.NonEmpty()),
		PackagePath: packagePath,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &versionInterfaceGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "interface",
				},
				outputPackage:             packagePath,
				imports:                   generator.NewImportTracker(),
				types:                     typesToGenerate,
				internalInterfacesPackage: packageForInternalInterfaces(basePackage),
			})

			for _, t := range typesToGenerate {
				generators = append(generators, &informerGenerator{
					DefaultGen: generator.DefaultGen{
						OptionalName: strings.ToLower(t.Name.Name),
					},
					outputPackage:             packagePath,
					groupPkgName:              groupPkgName,
					groupVersion:              gv,
					groupGoName:               groupGoName,
					typeToGenerate:            t,
					imports:                   generator.NewImportTracker(),
					clientSetPackage:          clientSetPackage,
					listersPackage:            listersPackage,
					internalInterfacesPackage: packageForInternalInterfaces(basePackage),
				})
			}
			return generators
		},
		FilterFunc: func(c *generator.Context, t *types.Type) bool {
			tags := util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...))
			return tags.GenerateClient && tags.HasVerb("list") && tags.HasVerb("watch")
		},
	}
}
<commit_msg>informer-gen: allow package names containing dots<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package generators

import (
	"fmt"
	"path"
	"path/filepath"
	"strings"

	"k8s.io/gengo/args"
	"k8s.io/gengo/generator"
	"k8s.io/gengo/namer"
	"k8s.io/gengo/types"
	"k8s.io/klog"

	"k8s.io/code-generator/cmd/client-gen/generators/util"
	clientgentypes "k8s.io/code-generator/cmd/client-gen/types"
	informergenargs "k8s.io/code-generator/cmd/informer-gen/args"
)

// NameSystems returns the name system used by the generators in this package.
func NameSystems() namer.NameSystems {
	pluralExceptions := map[string]string{
		"Endpoints": "Endpoints",
	}
	return namer.NameSystems{
		"public":             namer.NewPublicNamer(0),
		"private":            namer.NewPrivateNamer(0),
		"raw":                namer.NewRawNamer("", nil),
		"publicPlural":       namer.NewPublicPluralNamer(pluralExceptions),
		"allLowercasePlural": namer.NewAllLowercasePluralNamer(pluralExceptions),
		"lowercaseSingular":  &lowercaseSingularNamer{},
	}
}

// lowercaseSingularNamer implements Namer
type lowercaseSingularNamer struct{}

// Name returns t's name in all lowercase.
func (n *lowercaseSingularNamer) Name(t *types.Type) string {
	return strings.ToLower(t.Name.Name)
}

// DefaultNameSystem returns the default name system for ordering the types to be
// processed by the generators in this package.
func DefaultNameSystem() string {
	return "public"
}

// objectMetaForPackage returns the type of ObjectMeta used by package p.
func objectMetaForPackage(p *types.Package) (*types.Type, bool, error) {
	generatingForPackage := false
	for _, t := range p.Types {
		if !util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...)).GenerateClient {
			continue
		}
		generatingForPackage = true
		for _, member := range t.Members {
			if member.Name == "ObjectMeta" {
				return member.Type, isInternal(member), nil
			}
		}
	}
	if generatingForPackage {
		return nil, false, fmt.Errorf("unable to find ObjectMeta for any types in package %s", p.Path)
	}
	return nil, false, nil
}

// isInternal returns true if the tags for a member do not contain a json tag
func isInternal(m types.Member) bool {
	return !strings.Contains(m.Tags, "json")
}

func packageForInternalInterfaces(base string) string {
	return filepath.Join(base, "internalinterfaces")
}

func vendorless(p string) string {
	if pos := strings.LastIndex(p, "/vendor/"); pos != -1 {
		return p[pos+len("/vendor/"):]
	}
	return p
}

// Packages makes the client package definition.
func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages {
	boilerplate, err := arguments.LoadGoBoilerplate()
	if err != nil {
		klog.Fatalf("Failed loading boilerplate: %v", err)
	}

	customArgs, ok := arguments.CustomArgs.(*informergenargs.CustomArgs)
	if !ok {
		klog.Fatalf("Wrong CustomArgs type: %T", arguments.CustomArgs)
	}

	internalVersionPackagePath := filepath.Join(arguments.OutputPackagePath)
	externalVersionPackagePath := filepath.Join(arguments.OutputPackagePath)
	if !customArgs.SingleDirectory {
		internalVersionPackagePath = filepath.Join(arguments.OutputPackagePath, "internalversion")
		externalVersionPackagePath = filepath.Join(arguments.OutputPackagePath, "externalversions")
	}

	var packageList generator.Packages
	typesForGroupVersion := make(map[clientgentypes.GroupVersion][]*types.Type)

	externalGroupVersions := make(map[string]clientgentypes.GroupVersions)
	internalGroupVersions := make(map[string]clientgentypes.GroupVersions)
	groupGoNames := make(map[string]string)
	for _, inputDir := range arguments.InputDirs {
		p := context.Universe.Package(vendorless(inputDir))

		objectMeta, internal, err := objectMetaForPackage(p)
		if err != nil {
			klog.Fatal(err)
		}
		if objectMeta == nil {
			// no types in this package had genclient
			continue
		}

		var gv clientgentypes.GroupVersion
		var targetGroupVersions map[string]clientgentypes.GroupVersions

		if internal {
			lastSlash := strings.LastIndex(p.Path, "/")
			if lastSlash == -1 {
				klog.Fatalf("error constructing internal group version for package %q", p.Path)
			}
			gv.Group = clientgentypes.Group(p.Path[lastSlash+1:])
			targetGroupVersions = internalGroupVersions
		} else {
			parts := strings.Split(p.Path, "/")
			gv.Group = clientgentypes.Group(parts[len(parts)-2])
			gv.Version = clientgentypes.Version(parts[len(parts)-1])
			targetGroupVersions = externalGroupVersions
		}
		groupPackageName := gv.Group.NonEmpty()
		gvPackage := path.Clean(p.Path)

		// If there's a comment of the form "// +groupName=somegroup" or
		// "// +groupName=somegroup.foo.bar.io", use the first field (somegroup) as the name of the
		// group when generating.
		if override := types.ExtractCommentTags("+", p.Comments)["groupName"]; override != nil {
			gv.Group = clientgentypes.Group(override[0])
		}

		// If there's a comment of the form "// +groupGoName=SomeUniqueShortName", use that as
		// the Go group identifier in CamelCase. It defaults
		groupGoNames[groupPackageName] = namer.IC(strings.Split(gv.Group.NonEmpty(), ".")[0])
		if override := types.ExtractCommentTags("+", p.Comments)["groupGoName"]; override != nil {
			groupGoNames[groupPackageName] = namer.IC(override[0])
		}

		var typesToGenerate []*types.Type
		for _, t := range p.Types {
			tags := util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...))
			if !tags.GenerateClient || tags.NoVerbs || !tags.HasVerb("list") || !tags.HasVerb("watch") {
				continue
			}

			typesToGenerate = append(typesToGenerate, t)

			if _, ok := typesForGroupVersion[gv]; !ok {
				typesForGroupVersion[gv] = []*types.Type{}
			}
			typesForGroupVersion[gv] = append(typesForGroupVersion[gv], t)
		}
		if len(typesToGenerate) == 0 {
			continue
		}

		groupVersionsEntry, ok := targetGroupVersions[groupPackageName]
		if !ok {
			groupVersionsEntry = clientgentypes.GroupVersions{
				PackageName: groupPackageName,
				Group:       gv.Group,
			}
		}
		groupVersionsEntry.Versions = append(groupVersionsEntry.Versions, clientgentypes.PackageVersion{Version: gv.Version, Package: gvPackage})
		targetGroupVersions[groupPackageName] = groupVersionsEntry

		orderer := namer.Orderer{Namer: namer.NewPrivateNamer(0)}
		typesToGenerate = orderer.OrderTypes(typesToGenerate)

		if internal {
			packageList = append(packageList, versionPackage(internalVersionPackagePath, groupPackageName, gv, groupGoNames[groupPackageName], boilerplate, typesToGenerate, customArgs.InternalClientSetPackage, customArgs.ListersPackage))
		} else {
			packageList = append(packageList, versionPackage(externalVersionPackagePath, groupPackageName, gv, groupGoNames[groupPackageName], boilerplate, typesToGenerate, customArgs.VersionedClientSetPackage, customArgs.ListersPackage))
		}
	}

	if len(externalGroupVersions) != 0 {
		packageList = append(packageList, factoryInterfacePackage(externalVersionPackagePath, boilerplate, customArgs.VersionedClientSetPackage))
		packageList = append(packageList, factoryPackage(externalVersionPackagePath, boilerplate, groupGoNames, externalGroupVersions, customArgs.VersionedClientSetPackage, typesForGroupVersion))
		for _, gvs := range externalGroupVersions {
			packageList = append(packageList, groupPackage(externalVersionPackagePath, gvs, boilerplate))
		}
	}

	if len(internalGroupVersions) != 0 {
		packageList = append(packageList, factoryInterfacePackage(internalVersionPackagePath, boilerplate, customArgs.InternalClientSetPackage))
		packageList = append(packageList, factoryPackage(internalVersionPackagePath, boilerplate, groupGoNames, internalGroupVersions, customArgs.InternalClientSetPackage, typesForGroupVersion))
		for _, gvs := range internalGroupVersions {
			packageList = append(packageList, groupPackage(internalVersionPackagePath, gvs, boilerplate))
		}
	}

	return packageList
}

func factoryPackage(basePackage string, boilerplate []byte, groupGoNames map[string]string, groupVersions map[string]clientgentypes.GroupVersions, clientSetPackage string, typesForGroupVersion map[clientgentypes.GroupVersion][]*types.Type) generator.Package {
	return &generator.DefaultPackage{
		PackageName: filepath.Base(basePackage),
		PackagePath: basePackage,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &factoryGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "factory",
				},
				outputPackage:             basePackage,
				imports:                   generator.NewImportTracker(),
				groupVersions:             groupVersions,
				clientSetPackage:          clientSetPackage,
				internalInterfacesPackage: packageForInternalInterfaces(basePackage),
				gvGoNames:                 groupGoNames,
			})

			generators = append(generators, &genericGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "generic",
				},
				outputPackage:        basePackage,
				imports:              generator.NewImportTracker(),
				groupVersions:        groupVersions,
				typesForGroupVersion: typesForGroupVersion,
				groupGoNames:         groupGoNames,
			})

			return generators
		},
	}
}

func factoryInterfacePackage(basePackage string, boilerplate []byte, clientSetPackage string) generator.Package {
	packagePath := packageForInternalInterfaces(basePackage)

	return &generator.DefaultPackage{
		PackageName: filepath.Base(packagePath),
		PackagePath: packagePath,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &factoryInterfaceGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "factory_interfaces",
				},
				outputPackage:    packagePath,
				imports:          generator.NewImportTracker(),
				clientSetPackage: clientSetPackage,
			})

			return generators
		},
	}
}

func groupPackage(basePackage string, groupVersions clientgentypes.GroupVersions, boilerplate []byte) generator.Package {
	packagePath := filepath.Join(basePackage, groupVersions.PackageName)
	groupPkgName := strings.Split(string(groupVersions.PackageName), ".")[0]

	return &generator.DefaultPackage{
		PackageName: groupPkgName,
		PackagePath: packagePath,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &groupInterfaceGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "interface",
				},
				outputPackage:             packagePath,
				groupVersions:             groupVersions,
				imports:                   generator.NewImportTracker(),
				internalInterfacesPackage: packageForInternalInterfaces(basePackage),
			})
			return generators
		},
		FilterFunc: func(c *generator.Context, t *types.Type) bool {
			tags := util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...))
			return tags.GenerateClient && tags.HasVerb("list") && tags.HasVerb("watch")
		},
	}
}

func versionPackage(basePackage string, groupPkgName string, gv clientgentypes.GroupVersion, groupGoName string, boilerplate []byte, typesToGenerate []*types.Type, clientSetPackage, listersPackage string) generator.Package {
	packagePath := filepath.Join(basePackage, groupPkgName, strings.ToLower(gv.Version.NonEmpty()))

	return &generator.DefaultPackage{
		PackageName: strings.ToLower(gv.Version.NonEmpty()),
		PackagePath: packagePath,
		HeaderText:  boilerplate,
		GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) {
			generators = append(generators, &versionInterfaceGenerator{
				DefaultGen: generator.DefaultGen{
					OptionalName: "interface",
				},
				outputPackage:             packagePath,
				imports:                   generator.NewImportTracker(),
				types:                     typesToGenerate,
				internalInterfacesPackage: packageForInternalInterfaces(basePackage),
			})

			for _, t := range typesToGenerate {
				generators = append(generators, &informerGenerator{
					DefaultGen: generator.DefaultGen{
						OptionalName: strings.ToLower(t.Name.Name),
					},
					outputPackage:             packagePath,
					groupPkgName:              groupPkgName,
					groupVersion:              gv,
					groupGoName:               groupGoName,
					typeToGenerate:            t,
					imports:                   generator.NewImportTracker(),
					clientSetPackage:          clientSetPackage,
					listersPackage:            listersPackage,
					internalInterfacesPackage: packageForInternalInterfaces(basePackage),
				})
			}
			return generators
		},
		FilterFunc: func(c *generator.Context, t *types.Type) bool {
			tags := util.MustParseClientGenTags(append(t.SecondClosestCommentLines, t.CommentLines...))
			return tags.GenerateClient && tags.HasVerb("list") && tags.HasVerb("watch")
		},
	}
}
<|endoftext|>


<commit_before><commit_msg>"Jeff: Give yourself to the Dark Side. It is the only way you can save your friends."<commit_after><|endoftext|>


<commit_before><commit_msg>"Jeff: Come to the dark side... together we will rule the galaxy."<commit_after><|endoftext|>


<commit_before><commit_msg>S01E01<commit_after><|endoftext|>


<commit_before><commit_msg>Add proxy flag<commit_after><|endoftext|>


<commit_before>package fstree

import (
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sync"

	"github.com/caelifer/dups/balancer"
)

// Distributed file system tree walker

// Heler type - matches parameter signature of filepath.Walk()
type nodeFn func(path string, info os.FileInfo, err error) error

// Primary interface - matches signature of filepath.Walk()
func Walk(workQueue balancer.WorkQueue, path string, fn nodeFn) error {
	// Create walker object
	w := newWalker(workQueue, path)

	// Construct node from provided path
	info, err := os.Lstat(path)

	// On success ...
	if err == nil {
		// Process node
		err = w.walkNode(newNode(path, info), nil, fn)
	}

	// Wait util all nodes are processed
	w.wg.Wait()

	return err
}

type node struct {
	path string
	info os.FileInfo
}

func newNode(path string, info os.FileInfo) *node {
	return &node{path: filepath.Clean(path), info: info}
}

type walker struct {
	root      string
	workQueue balancer.WorkQueue
	wg        sync.WaitGroup
}

func newWalker(workQueue balancer.WorkQueue, root string) *walker {
	return &walker{
		root:      root,
		workQueue: workQueue,
	}
}

func (w *walker) walkNode(node *node, err error, fn nodeFn) error {
	// Process node by calling client function
	err = fn(node.path, node.info, err)

	// ... then, recursively process directories
	if node.info.IsDir() {
		// Traverse directrory asnyncronously using balancer
		w.walkDir(node, err, fn)
	}

	return err
}

func (w *walker) walkDir(node *node, err error, fn nodeFn) {
	if err != nil {
		log.Println("WARN", err)
		return
	}

	// Make sure we are not finished until all recursive calls are done
	w.wg.Add(1)

	// Send to be processed in the workpool
	go func() {
		w.workQueue <- func() {
			defer w.wg.Done() // Signal done at the end of the function

			// Read directory entries
			dirents, err := ioutil.ReadDir(node.path)
			if err != nil {
				log.Println("WARN", err)

				// erly termination if we cannot read directory
				return
			}

			// Read all entries in current directory
			for _, entry := range dirents {
				// path := node.path + string(os.PathSeparator) + entry.Name()

				// Use custom fast string concatenation rutine
				path := fastStringConcat(node.path, os.PathSeparator, entry.Name())

				// Process node, ignore errors
				w.walkNode(newNode(path, entry), nil, fn)
			}
		}
	}()
}

// Little helper for specialized fast string / rune concatenation
// Inspired by http://golang-examples.tumblr.com/post/86169510884/fastest-string-contatenation
func fastStringConcat(first string, second byte, third string) string {
	res := make([]byte, 0, len(first)+1+len(third))
	res = append(res, []byte(first)...)
	res = append(res, second)
	res = append(res, []byte(third)...)
	return string(res)
}
<commit_msg>Improve comment for the Walk() function<commit_after>package fstree

import (
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sync"

	"github.com/caelifer/dups/balancer"
)

// Distributed file system tree walker

// Heler type - matches parameter signature of filepath.Walk()
type nodeFn func(path string, info os.FileInfo, err error) error

// Walk is a primary interface to this package. It matches signature of filepath.Walk().
func Walk(workQueue balancer.WorkQueue, path string, fn nodeFn) error {
	// Create walker object
	w := newWalker(workQueue, path)

	// Construct node from provided path
	info, err := os.Lstat(path)

	// On success ...
	if err == nil {
		// Process node
		err = w.walkNode(newNode(path, info), nil, fn)
	}

	// Wait util all nodes are processed
	w.wg.Wait()

	return err
}

type node struct {
	path string
	info os.FileInfo
}

func newNode(path string, info os.FileInfo) *node {
	return &node{path: filepath.Clean(path), info: info}
}

type walker struct {
	root      string
	workQueue balancer.WorkQueue
	wg        sync.WaitGroup
}

func newWalker(workQueue balancer.WorkQueue, root string) *walker {
	return &walker{
		root:      root,
		workQueue: workQueue,
	}
}

func (w *walker) walkNode(node *node, err error, fn nodeFn) error {
	// Process node by calling client function
	err = fn(node.path, node.info, err)

	// ... then, recursively process directories
	if node.info.IsDir() {
		// Traverse directrory asnyncronously using balancer
		w.walkDir(node, err, fn)
	}

	return err
}

func (w *walker) walkDir(node *node, err error, fn nodeFn) {
	if err != nil {
		log.Println("WARN", err)
		return
	}

	// Make sure we are not finished until all recursive calls are done
	w.wg.Add(1)

	// Send to be processed in the workpool
	go func() {
		w.workQueue <- func() {
			defer w.wg.Done() // Signal done at the end of the function

			// Read directory entries
			dirents, err := ioutil.ReadDir(node.path)
			if err != nil {
				log.Println("WARN", err)

				// erly termination if we cannot read directory
				return
			}

			// Read all entries in current directory
			for _, entry := range dirents {
				// path := node.path + string(os.PathSeparator) + entry.Name()

				// Use custom fast string concatenation rutine
				path := fastStringConcat(node.path, os.PathSeparator, entry.Name())

				// Process node, ignore errors
				w.walkNode(newNode(path, entry), nil, fn)
			}
		}
	}()
}

// Little helper for specialized fast string / rune concatenation
// Inspired by http://golang-examples.tumblr.com/post/86169510884/fastest-string-contatenation
func fastStringConcat(first string, second byte, third string) string {
	res := make([]byte, 0, len(first)+1+len(third))
	res = append(res, []byte(first)...)
	res = append(res, second)
	res = append(res, []byte(third)...)
	return string(res)
}
<|endoftext|>


<commit_before><commit_msg>adding a proper times.Now()<commit_after><|endoftext|>


<commit_before><commit_msg>fixing the wrapper Times struct to work with time.Time<commit_after><|endoftext|>


<commit_before>// A FUSE filesystem that shunts all request to an underlying file
// system.  Its main purpose is to provide test coverage without
// having to build an actual synthetic filesystem.

package fuse

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"syscall"
)

var _ = fmt.Println
var _ = log.Println

type LoopbackFileSystem struct {
	root string

	DefaultFileSystem
}

func NewLoopbackFileSystem(root string) (out *LoopbackFileSystem) {
	out = new(LoopbackFileSystem)
	out.root = root

	return out
}

func (me *LoopbackFileSystem) GetPath(relPath string) string {
	return filepath.Join(me.root, relPath)
}

func (me *LoopbackFileSystem) GetAttr(name string) (*Attr, Status) {
	fullPath := me.GetPath(name)
	fi, err := os.Lstat(fullPath)
	if err != nil {
		return nil, ENOENT
	}
	out := new(Attr)
	CopyFileInfo(fi, out)

	return out, OK
}

func (me *LoopbackFileSystem) OpenDir(name string) (stream chan DirEntry, status Status) {
	// What other ways beyond O_RDONLY are there to open
	// directories?
	f, err := os.Open(me.GetPath(name))
	if err != nil {
		return nil, OsErrorToErrno(err)
	}
	want := 500
	output := make(chan DirEntry, want)
	go func() {
		for {
			infos, err := f.Readdir(want)
			for i, _ := range infos {
				output <- DirEntry{
					Name: infos[i].Name,
					Mode: infos[i].Mode,
				}
			}
			if len(infos) < want {
				break
			}
			if err != nil {
				// TODO - how to signal error
				break
			}
		}
		close(output)
		f.Close()
	}()

	return output, OK
}

func (me *LoopbackFileSystem) Open(name string, flags uint32) (fuseFile File, status Status) {
	f, err := os.OpenFile(me.GetPath(name), int(flags), 0)
	if err != nil {
		return nil, OsErrorToErrno(err)
	}
	return &LoopbackFile{file: f}, OK
}

func (me *LoopbackFileSystem) Chmod(path string, mode uint32) (code Status) {
	err := os.Chmod(me.GetPath(path), mode)
	return OsErrorToErrno(err)
}

func (me *LoopbackFileSystem) Chown(path string, uid uint32, gid uint32) (code Status) {
	return OsErrorToErrno(os.Chown(me.GetPath(path), int(uid), int(gid)))
}

func (me *LoopbackFileSystem) Truncate(path string, offset uint64) (code Status) {
	return OsErrorToErrno(os.Truncate(me.GetPath(path), int64(offset)))
}

func (me *LoopbackFileSystem) Utimens(path string, AtimeNs uint64, MtimeNs uint64) (code Status) {
	return OsErrorToErrno(os.Chtimes(me.GetPath(path), int64(AtimeNs), int64(MtimeNs)))
}

func (me *LoopbackFileSystem) Readlink(name string) (out string, code Status) {
	f, err := os.Readlink(me.GetPath(name))
	return f, OsErrorToErrno(err)
}

func (me *LoopbackFileSystem) Mknod(name string, mode uint32, dev uint32) (code Status) {
	return Status(syscall.Mknod(me.GetPath(name), mode, int(dev)))
}

func (me *LoopbackFileSystem) Mkdir(path string, mode uint32) (code Status) {
	return OsErrorToErrno(os.Mkdir(me.GetPath(path), mode))
}

// Don't use os.Remove, it removes twice (unlink followed by rmdir).
func (me *LoopbackFileSystem) Unlink(name string) (code Status) {
	return Status(syscall.Unlink(me.GetPath(name)))
}

func (me *LoopbackFileSystem) Rmdir(name string) (code Status) {
	return Status(syscall.Rmdir(me.GetPath(name)))
}

func (me *LoopbackFileSystem) Symlink(pointedTo string, linkName string) (code Status) {
	return OsErrorToErrno(os.Symlink(pointedTo, me.GetPath(linkName)))
}

func (me *LoopbackFileSystem) Rename(oldPath string, newPath string) (code Status) {
	err := os.Rename(me.GetPath(oldPath), me.GetPath(newPath))
	return OsErrorToErrno(err)
}

func (me *LoopbackFileSystem) Link(orig string, newName string) (code Status) {
	return OsErrorToErrno(os.Link(me.GetPath(orig), me.GetPath(newName)))
}

func (me *LoopbackFileSystem) Access(name string, mode uint32) (code Status) {
	return Status(syscall.Access(me.GetPath(name), mode))
}

func (me *LoopbackFileSystem) Create(path string, flags uint32, mode uint32) (fuseFile File, code Status) {
	f, err := os.OpenFile(me.GetPath(path), int(flags)|os.O_CREATE, mode)
	return &LoopbackFile{file: f}, OsErrorToErrno(err)
}

func (me *LoopbackFileSystem) GetXAttr(name string, attr string) ([]byte, Status) {
	data, errNo := GetXAttr(me.GetPath(name), attr)

	return data, Status(errNo)
}

func (me *LoopbackFileSystem) ListXAttr(name string) ([]string, Status) {
	data, errNo := ListXAttr(me.GetPath(name))

	return data, Status(errNo)
}

func (me *LoopbackFileSystem) RemoveXAttr(name string, attr string) Status {
	return Status(Removexattr(me.GetPath(name), attr))
}

////////////////////////////////////////////////////////////////

type LoopbackFile struct {
	file *os.File

	DefaultFile
}

func (me *LoopbackFile) Read(input *ReadIn, buffers *BufferPool) ([]byte, Status) {
	slice := buffers.AllocBuffer(input.Size)

	n, err := me.file.ReadAt(slice, int64(input.Offset))
	if err == os.EOF {
		return slice[:n], OK
	}
	return slice[:n], OsErrorToErrno(err)
}

func (me *LoopbackFile) Write(input *WriteIn, data []byte) (uint32, Status) {
	n, err := me.file.WriteAt(data, int64(input.Offset))
	return uint32(n), OsErrorToErrno(err)
}

func (me *LoopbackFile) Release() {
	me.file.Close()
}

func (me *LoopbackFile) Fsync(*FsyncIn) (code Status) {
	return Status(syscall.Fsync(me.file.Fd()))
}
<commit_msg>Implement LoopbackFile.Truncate.<commit_after>// A FUSE filesystem that shunts all request to an underlying file
// system.  Its main purpose is to provide test coverage without
// having to build an actual synthetic filesystem.

package fuse

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"syscall"
)

var _ = fmt.Println
var _ = log.Println

type LoopbackFileSystem struct {
	root string

	DefaultFileSystem
}

func NewLoopbackFileSystem(root string) (out *LoopbackFileSystem) {
	out = new(LoopbackFileSystem)
	out.root = root

	return out
}

func (me *LoopbackFileSystem) GetPath(relPath string) string {
	return filepath.Join(me.root, relPath)
}

func (me *LoopbackFileSystem) GetAttr(name string) (*Attr, Status) {
	fullPath := me.GetPath(name)
	fi, err := os.Lstat(fullPath)
	if err != nil {
		return nil, ENOENT
	}
	out := new(Attr)
	CopyFileInfo(fi, out)

	return out, OK
}

func (me *LoopbackFileSystem) OpenDir(name string) (stream chan DirEntry, status Status) {
	// What other ways beyond O_RDONLY are there to open
	// directories?
	f, err := os.Open(me.GetPath(name))
	if err != nil {
		return nil, OsErrorToErrno(err)
	}
	want := 500
	output := make(chan DirEntry, want)
	go func() {
		for {
			infos, err := f.Readdir(want)
			for i, _ := range infos {
				output <- DirEntry{
					Name: infos[i].Name,
					Mode: infos[i].Mode,
				}
			}
			if len(infos) < want {
				break
			}
			if err != nil {
				// TODO - how to signal error
				break
			}
		}
		close(output)
		f.Close()
	}()

	return output, OK
}

func (me *LoopbackFileSystem) Open(name string, flags uint32) (fuseFile File, status Status) {
	f, err := os.OpenFile(me.GetPath(name), int(flags), 0)
	if err != nil {
		return nil, OsErrorToErrno(err)
	}
	return &LoopbackFile{file: f}, OK
}

func (me *LoopbackFileSystem) Chmod(path string, mode uint32) (code Status) {
	err := os.Chmod(me.GetPath(path), mode)
	return OsErrorToErrno(err)
}

func (me *LoopbackFileSystem) Chown(path string, uid uint32, gid uint32) (code Status) {
	return OsErrorToErrno(os.Chown(me.GetPath(path), int(uid), int(gid)))
}

func (me *LoopbackFileSystem) Truncate(path string, offset uint64) (code Status) {
	return OsErrorToErrno(os.Truncate(me.GetPath(path), int64(offset)))
}

func (me *LoopbackFileSystem) Utimens(path string, AtimeNs uint64, MtimeNs uint64) (code Status) {
	return OsErrorToErrno(os.Chtimes(me.GetPath(path), int64(AtimeNs), int64(MtimeNs)))
}

func (me *LoopbackFileSystem) Readlink(name string) (out string, code Status) {
	f, err := os.Readlink(me.GetPath(name))
	return f, OsErrorToErrno(err)
}

func (me *LoopbackFileSystem) Mknod(name string, mode uint32, dev uint32) (code Status) {
	return Status(syscall.Mknod(me.GetPath(name), mode, int(dev)))
}

func (me *LoopbackFileSystem) Mkdir(path string, mode uint32) (code Status) {
	return OsErrorToErrno(os.Mkdir(me.GetPath(path), mode))
}

// Don't use os.Remove, it removes twice (unlink followed by rmdir).
func (me *LoopbackFileSystem) Unlink(name string) (code Status) {
	return Status(syscall.Unlink(me.GetPath(name)))
}

func (me *LoopbackFileSystem) Rmdir(name string) (code Status) {
	return Status(syscall.Rmdir(me.GetPath(name)))
}

func (me *LoopbackFileSystem) Symlink(pointedTo string, linkName string) (code Status) {
	return OsErrorToErrno(os.Symlink(pointedTo, me.GetPath(linkName)))
}

func (me *LoopbackFileSystem) Rename(oldPath string, newPath string) (code Status) {
	err := os.Rename(me.GetPath(oldPath), me.GetPath(newPath))
	return OsErrorToErrno(err)
}

func (me *LoopbackFileSystem) Link(orig string, newName string) (code Status) {
	return OsErrorToErrno(os.Link(me.GetPath(orig), me.GetPath(newName)))
}

func (me *LoopbackFileSystem) Access(name string, mode uint32) (code Status) {
	return Status(syscall.Access(me.GetPath(name), mode))
}

func (me *LoopbackFileSystem) Create(path string, flags uint32, mode uint32) (fuseFile File, code Status) {
	f, err := os.OpenFile(me.GetPath(path), int(flags)|os.O_CREATE, mode)
	return &LoopbackFile{file: f}, OsErrorToErrno(err)
}

func (me *LoopbackFileSystem) GetXAttr(name string, attr string) ([]byte, Status) {
	data, errNo := GetXAttr(me.GetPath(name), attr)

	return data, Status(errNo)
}

func (me *LoopbackFileSystem) ListXAttr(name string) ([]string, Status) {
	data, errNo := ListXAttr(me.GetPath(name))

	return data, Status(errNo)
}

func (me *LoopbackFileSystem) RemoveXAttr(name string, attr string) Status {
	return Status(Removexattr(me.GetPath(name), attr))
}

////////////////////////////////////////////////////////////////

type LoopbackFile struct {
	file *os.File

	DefaultFile
}

func (me *LoopbackFile) Read(input *ReadIn, buffers *BufferPool) ([]byte, Status) {
	slice := buffers.AllocBuffer(input.Size)

	n, err := me.file.ReadAt(slice, int64(input.Offset))
	if err == os.EOF {
		return slice[:n], OK
	}
	return slice[:n], OsErrorToErrno(err)
}

func (me *LoopbackFile) Write(input *WriteIn, data []byte) (uint32, Status) {
	n, err := me.file.WriteAt(data, int64(input.Offset))
	return uint32(n), OsErrorToErrno(err)
}

func (me *LoopbackFile) Release() {
	me.file.Close()
}

func (me *LoopbackFile) Fsync(*FsyncIn) (code Status) {
	return Status(syscall.Fsync(me.file.Fd()))
}

func (me *LoopbackFile) Truncate(size uint64) Status {
	return Status(syscall.Ftruncate(me.file.Fd(), int64(size)))
}
<|endoftext|>