<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Link, typically invoked as ``go tool link,'' reads the Go archive or object
for a package main, along with its dependencies, and combines them
into an executable binary.

Command Line

Usage:

	go tool link [flags] main.a

Flags:

	-B note
		Add an ELF_NT_GNU_BUILD_ID note when using ELF.
		The value should start with 0x and be an even number of hex digits.
	-D address
		Set data segment address.
	-E entry
		Set entry symbol name.
	-H type
		Set executable format type.
		The default format is inferred from GOOS and GOARCH.
		On Windows, -H windowsgui writes a "GUI binary" instead of a "console binary."
	-I interpreter
		Set the ELF dynamic linker to use.
	-L dir1 -L dir2
		Search for imported packages in dir1, dir2, etc,
		after consulting $GOROOT/pkg/$GOOS_$GOARCH.
	-R quantum
		Set address rounding quantum.
	-T address
		Set text segment address.
	-V
		Print linker version and exit.
	-X importpath.name=value
		Set the value of the string variable in importpath named name to value.
		This is only effective if the variable is declared in the source code either uninitialized
		or initialized to a constant string expression. -X will not work if the initializer makes
		a function call or refers to other variables.
		Note that before Go 1.5 this option took two separate arguments.
	-a
		Disassemble output.
	-buildid id
		Record id as Go toolchain build id.
	-buildmode mode
		Set build mode (default exe).
	-c
		Dump call graphs.
	-compressdwarf
		Compress DWARF if possible (default true).
	-cpuprofile file
		Write CPU profile to file.
	-d
		Disable generation of dynamic executables.
		The emitted code is the same in either case; the option
		controls only whether a dynamic header is included.
		The dynamic header is on by default, even without any
		references to dynamic libraries, because many common
		system tools now assume the presence of the header.
	-debugtramp int
		Debug trampolines.
	-dumpdep
		Dump symbol dependency graph.
	-extar ar
		Set the external archive program (default "ar").
		Used only for -buildmode=c-archive.
	-extld linker
		Set the external linker (default "clang" or "gcc").
	-extldflags flags
		Set space-separated flags to pass to the external linker.
	-f
		Ignore version mismatch in the linked archives.
	-g
		Disable Go package data checks.
	-importcfg file
		Read import configuration from file.
		In the file, set packagefile, packageshlib to specify import resolution.
	-installsuffix suffix
		Look for packages in $GOROOT/pkg/$GOOS_$GOARCH_suffix
		instead of $GOROOT/pkg/$GOOS_$GOARCH.
	-k symbol
		Set field tracking symbol. Use this flag when GOEXPERIMENT=fieldtrack is set.
	-libgcc file
		Set name of compiler support library.
		This is only used in internal link mode.
		If not set, default value comes from running the compiler,
		which may be set by the -extld option.
		Set to "none" to use no support library.
	-linkmode mode
		Set link mode (internal, external, auto).
		This sets the linking mode as described in cmd/cgo/doc.go.
	-linkshared
		Link against installed Go shared libraries (experimental).
	-memprofile file
		Write memory profile to file.
	-memprofilerate rate
		Set runtime.MemProfileRate to rate.
	-msan
		Link with C/C++ memory sanitizer support.
	-n
		Dump symbol table.
	-o file
		Write output to file (default a.out, or a.out.exe on Windows).
	-pluginpath path
		The path name used to prefix exported plugin symbols.
	-r dir1:dir2:...
		Set the ELF dynamic linker search path.
	-race
		Link with race detection libraries.
	-s
		Omit the symbol table and debug information.
	-shared
		Generated shared object (implies -linkmode external; experimental).
	-tmpdir dir
		Write temporary files to dir.
		Temporary files are only used in external linking mode.
	-u
		Reject unsafe packages.
	-v
		Print trace of linker operations.
	-w
		Omit the DWARF symbol table.
*/
package main
<commit_msg>cmd/link: move comma outside quotes<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Link, typically invoked as ``go tool link'', reads the Go archive or object
for a package main, along with its dependencies, and combines them
into an executable binary.

Command Line

Usage:

	go tool link [flags] main.a

Flags:

	-B note
		Add an ELF_NT_GNU_BUILD_ID note when using ELF.
		The value should start with 0x and be an even number of hex digits.
	-D address
		Set data segment address.
	-E entry
		Set entry symbol name.
	-H type
		Set executable format type.
		The default format is inferred from GOOS and GOARCH.
		On Windows, -H windowsgui writes a "GUI binary" instead of a "console binary."
	-I interpreter
		Set the ELF dynamic linker to use.
	-L dir1 -L dir2
		Search for imported packages in dir1, dir2, etc,
		after consulting $GOROOT/pkg/$GOOS_$GOARCH.
	-R quantum
		Set address rounding quantum.
	-T address
		Set text segment address.
	-V
		Print linker version and exit.
	-X importpath.name=value
		Set the value of the string variable in importpath named name to value.
		This is only effective if the variable is declared in the source code either uninitialized
		or initialized to a constant string expression. -X will not work if the initializer makes
		a function call or refers to other variables.
		Note that before Go 1.5 this option took two separate arguments.
	-a
		Disassemble output.
	-buildid id
		Record id as Go toolchain build id.
	-buildmode mode
		Set build mode (default exe).
	-c
		Dump call graphs.
	-compressdwarf
		Compress DWARF if possible (default true).
	-cpuprofile file
		Write CPU profile to file.
	-d
		Disable generation of dynamic executables.
		The emitted code is the same in either case; the option
		controls only whether a dynamic header is included.
		The dynamic header is on by default, even without any
		references to dynamic libraries, because many common
		system tools now assume the presence of the header.
	-debugtramp int
		Debug trampolines.
	-dumpdep
		Dump symbol dependency graph.
	-extar ar
		Set the external archive program (default "ar").
		Used only for -buildmode=c-archive.
	-extld linker
		Set the external linker (default "clang" or "gcc").
	-extldflags flags
		Set space-separated flags to pass to the external linker.
	-f
		Ignore version mismatch in the linked archives.
	-g
		Disable Go package data checks.
	-importcfg file
		Read import configuration from file.
		In the file, set packagefile, packageshlib to specify import resolution.
	-installsuffix suffix
		Look for packages in $GOROOT/pkg/$GOOS_$GOARCH_suffix
		instead of $GOROOT/pkg/$GOOS_$GOARCH.
	-k symbol
		Set field tracking symbol. Use this flag when GOEXPERIMENT=fieldtrack is set.
	-libgcc file
		Set name of compiler support library.
		This is only used in internal link mode.
		If not set, default value comes from running the compiler,
		which may be set by the -extld option.
		Set to "none" to use no support library.
	-linkmode mode
		Set link mode (internal, external, auto).
		This sets the linking mode as described in cmd/cgo/doc.go.
	-linkshared
		Link against installed Go shared libraries (experimental).
	-memprofile file
		Write memory profile to file.
	-memprofilerate rate
		Set runtime.MemProfileRate to rate.
	-msan
		Link with C/C++ memory sanitizer support.
	-n
		Dump symbol table.
	-o file
		Write output to file (default a.out, or a.out.exe on Windows).
	-pluginpath path
		The path name used to prefix exported plugin symbols.
	-r dir1:dir2:...
		Set the ELF dynamic linker search path.
	-race
		Link with race detection libraries.
	-s
		Omit the symbol table and debug information.
	-shared
		Generated shared object (implies -linkmode external; experimental).
	-tmpdir dir
		Write temporary files to dir.
		Temporary files are only used in external linking mode.
	-u
		Reject unsafe packages.
	-v
		Print trace of linker operations.
	-w
		Omit the DWARF symbol table.
*/
package main
<|endoftext|>


<commit_before>package kafkamdm

import (
	"flag"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"

	"github.com/raintank/metrictank/cluster"
	"github.com/raintank/metrictank/input"
	"github.com/raintank/metrictank/kafka"
	"github.com/raintank/metrictank/stats"
	"gopkg.in/raintank/schema.v1"
)

// metric input.kafka-mdm.metrics_per_message is how many metrics per message were seen.
var metricsPerMessage = stats.NewMeter32("input.kafka-mdm.metrics_per_message", false)

// metric input.kafka-mdm.metrics_decode_err is a count of times an input message failed to parse
var metricsDecodeErr = stats.NewCounter32("input.kafka-mdm.metrics_decode_err")

type KafkaMdm struct {
	input.Handler
	consumer   sarama.Consumer
	client     sarama.Client
	lagMonitor *LagMonitor
	wg         sync.WaitGroup

	// signal to PartitionConsumers to shutdown
	stopConsuming chan struct{}
}

func (k *KafkaMdm) Name() string {
	return "kafka-mdm"
}

var LogLevel int
var Enabled bool
var brokerStr string
var brokers []string
var topicStr string
var topics []string
var partitionStr string
var partitions []int32
var offsetStr string
var DataDir string
var config *sarama.Config
var channelBufferSize int
var consumerFetchMin int
var consumerFetchDefault int
var consumerMaxWaitTime time.Duration
var consumerMaxProcessingTime time.Duration
var netMaxOpenRequests int
var offsetMgr *kafka.OffsetMgr
var offsetDuration time.Duration
var offsetCommitInterval time.Duration
var partitionOffset map[int32]*stats.Gauge64
var partitionLogSize map[int32]*stats.Gauge64
var partitionLag map[int32]*stats.Gauge64

func ConfigSetup() {
	inKafkaMdm := flag.NewFlagSet("kafka-mdm-in", flag.ExitOnError)
	inKafkaMdm.BoolVar(&Enabled, "enabled", false, "")
	inKafkaMdm.StringVar(&brokerStr, "brokers", "kafka:9092", "tcp address for kafka (may be be given multiple times as a comma-separated list)")
	inKafkaMdm.StringVar(&topicStr, "topics", "mdm", "kafka topic (may be given multiple times as a comma-separated list)")
	inKafkaMdm.StringVar(&offsetStr, "offset", "last", "Set the offset to start consuming from. Can be one of newest, oldest,last or a time duration")
	inKafkaMdm.StringVar(&partitionStr, "partitions", "*", "kafka partitions to consume. use '*' or a comma separated list of id's")
	inKafkaMdm.DurationVar(&offsetCommitInterval, "offset-commit-interval", time.Second*5, "Interval at which offsets should be saved.")
	inKafkaMdm.StringVar(&DataDir, "data-dir", "", "Directory to store partition offsets index")
	inKafkaMdm.IntVar(&channelBufferSize, "channel-buffer-size", 1000000, "The number of metrics to buffer in internal and external channels")
	inKafkaMdm.IntVar(&consumerFetchMin, "consumer-fetch-min", 1, "The minimum number of message bytes to fetch in a request")
	inKafkaMdm.IntVar(&consumerFetchDefault, "consumer-fetch-default", 32768, "The default number of message bytes to fetch in a request")
	inKafkaMdm.DurationVar(&consumerMaxWaitTime, "consumer-max-wait-time", time.Second, "The maximum amount of time the broker will wait for Consumer.Fetch.Min bytes to become available before it returns fewer than that anyway")
	inKafkaMdm.DurationVar(&consumerMaxProcessingTime, "consumer-max-processing-time", time.Second, "The maximum amount of time the consumer expects a message takes to process")
	inKafkaMdm.IntVar(&netMaxOpenRequests, "net-max-open-requests", 100, "How many outstanding requests a connection is allowed to have before sending on it blocks")
	globalconf.Register("kafka-mdm-in", inKafkaMdm)
}

func ConfigProcess(instance string) {
	if !Enabled {
		return
	}

	if offsetCommitInterval == 0 {
		log.Fatal(4, "kafkamdm: offset-commit-interval must be greater then 0")
	}
	if consumerMaxWaitTime == 0 {
		log.Fatal(4, "kafkamdm: consumer-max-wait-time must be greater then 0")
	}
	if consumerMaxProcessingTime == 0 {
		log.Fatal(4, "kafkamdm: consumer-max-processing-time must be greater then 0")
	}
	var err error
	switch offsetStr {
	case "last":
	case "oldest":
	case "newest":
	default:
		offsetDuration, err = time.ParseDuration(offsetStr)
		if err != nil {
			log.Fatal(4, "kafkamdm: invalid offest format. %s", err)
		}
	}

	offsetMgr, err = kafka.NewOffsetMgr(DataDir)
	if err != nil {
		log.Fatal(4, "kafka-mdm couldnt create offsetMgr. %s", err)
	}
	brokers = strings.Split(brokerStr, ",")
	topics = strings.Split(topicStr, ",")

	config = sarama.NewConfig()

	config.ClientID = instance + "-mdm"
	config.ChannelBufferSize = channelBufferSize
	config.Consumer.Fetch.Min = int32(consumerFetchMin)
	config.Consumer.Fetch.Default = int32(consumerFetchDefault)
	config.Consumer.MaxWaitTime = consumerMaxWaitTime
	config.Consumer.MaxProcessingTime = consumerMaxProcessingTime
	config.Net.MaxOpenRequests = netMaxOpenRequests
	config.Version = sarama.V0_10_0_0
	err = config.Validate()
	if err != nil {
		log.Fatal(2, "kafka-mdm invalid config: %s", err)
	}
	// validate our partitions
	client, err := sarama.NewClient(brokers, config)
	if err != nil {
		log.Fatal(4, "kafka-mdm failed to create client. %s", err)
	}
	defer client.Close()

	availParts, err := kafka.GetPartitions(client, topics)
	if err != nil {
		log.Fatal(4, "kafka-mdm: %s", err.Error())
	}
	log.Info("kafka-mdm: available partitions %v", availParts)
	if partitionStr == "*" {
		partitions = availParts
	} else {
		parts := strings.Split(partitionStr, ",")
		for _, part := range parts {
			i, err := strconv.Atoi(part)
			if err != nil {
				log.Fatal(4, "could not parse partition %q. partitions must be '*' or a comma separated list of id's", part)
			}
			partitions = append(partitions, int32(i))
		}
		missing := kafka.DiffPartitions(partitions, availParts)
		if len(missing) > 0 {
			log.Fatal(4, "kafka-mdm: configured partitions not in list of available partitions. missing %v", missing)
		}
	}
	// record our partitions so others (MetricIdx) can use the partitioning information.
	// but only if the manager has been created (e.g. in metrictank), not when this input plugin is used in other contexts
	if cluster.Manager != nil {
		cluster.Manager.SetPartitions(partitions)
	}

	// initialize our offset metrics
	partitionOffset = make(map[int32]*stats.Gauge64)
	partitionLogSize = make(map[int32]*stats.Gauge64)
	partitionLag = make(map[int32]*stats.Gauge64)
	for _, part := range partitions {
		partitionOffset[part] = stats.NewGauge64(fmt.Sprintf("input.kafka-mdm.partition.%d.offset", part))
		partitionLogSize[part] = stats.NewGauge64(fmt.Sprintf("input.kafka-mdm.partition.%d.log_size", part))
		partitionLag[part] = stats.NewGauge64(fmt.Sprintf("input.kafka-mdm.partition.%d.lag", part))
	}
}

func New() *KafkaMdm {
	client, err := sarama.NewClient(brokers, config)
	if err != nil {
		log.Fatal(4, "kafka-mdm failed to create client. %s", err)
	}
	consumer, err := sarama.NewConsumerFromClient(client)
	if err != nil {
		log.Fatal(2, "kafka-mdm failed to create consumer: %s", err)
	}
	log.Info("kafka-mdm consumer created without error")
	k := KafkaMdm{
		consumer:      consumer,
		client:        client,
		lagMonitor:    NewLagMonitor(10, partitions),
		stopConsuming: make(chan struct{}),
	}

	return &k
}

func (k *KafkaMdm) Start(handler input.Handler) {
	k.Handler = handler
	var err error
	for _, topic := range topics {
		for _, partition := range partitions {
			var offset int64
			switch offsetStr {
			case "oldest":
				offset = sarama.OffsetOldest
			case "newest":
				offset = sarama.OffsetNewest
			case "last":
				offset, err = offsetMgr.Last(topic, partition)
			default:
				offset, err = k.client.GetOffset(topic, partition, time.Now().Add(-1*offsetDuration).UnixNano()/int64(time.Millisecond))
			}
			if err != nil {
				log.Fatal(4, "kafka-mdm: Failed to get %q duration offset for %s:%d. %q", offsetStr, topic, partition, err)
			}
			go k.consumePartition(topic, partition, offset)
		}
	}
}

// this will continually consume from the topic until k.stopConsuming is triggered.
func (k *KafkaMdm) consumePartition(topic string, partition int32, currentOffset int64) {
	k.wg.Add(1)
	defer k.wg.Done()

	pc, err := k.consumer.ConsumePartition(topic, partition, currentOffset)
	if err != nil {
		log.Fatal(4, "kafka-mdm: failed to start partitionConsumer for %s:%d. %s", topic, partition, err)
	}

	partitionOffsetMetric := partitionOffset[partition]
	partitionLogSizeMetric := partitionLogSize[partition]
	partitionLagMetric := partitionLag[partition]
	offset, err := k.client.GetOffset(topic, partition, sarama.OffsetNewest)
	if err != nil {
		log.Error(3, "kafka-mdm failed to get log-size of partition %s:%d. %s", topic, partition, err)
	} else {
		partitionLogSizeMetric.Set(int(offset))
	}
	if currentOffset >= 0 {
		// we cant set the offsetMetrics until we know what offset we are at.
		partitionOffsetMetric.Set(int(currentOffset))
		// we need the currentLogSize to be able to record our initial Lag.
		if err == nil {
			partitionLagMetric.Set(int(offset - currentOffset))
		}
	}

	log.Info("kafka-mdm: consuming from %s:%d from offset %d", topic, partition, currentOffset)
	messages := pc.Messages()
	ticker := time.NewTicker(offsetCommitInterval)
	for {
		select {
		case msg := <-messages:
			if LogLevel < 2 {
				log.Debug("kafka-mdm received message: Topic %s, Partition: %d, Offset: %d, Key: %x", msg.Topic, msg.Partition, msg.Offset, msg.Key)
			}
			k.handleMsg(msg.Value, partition)
			currentOffset = msg.Offset
		case ts := <-ticker.C:
			if err := offsetMgr.Commit(topic, partition, currentOffset); err != nil {
				log.Error(3, "kafka-mdm failed to commit offset for %s:%d, %s", topic, partition, err)
			}
			k.lagMonitor.StoreOffset(partition, currentOffset, ts)
			offset, err := k.client.GetOffset(topic, partition, sarama.OffsetNewest)
			if err != nil {
				log.Error(3, "kafka-mdm failed to get log-size of partition %s:%d. %s", topic, partition, err)
			} else {
				partitionLogSizeMetric.Set(int(offset))
			}
			if currentOffset < 0 {
				// we're set to consume oldest/newest and
				// we have not yet consumed any messages.
				continue
			}
			partitionOffsetMetric.Set(int(currentOffset))
			if err == nil {
				lag := int(offset - currentOffset)
				partitionLagMetric.Set(lag)
				k.lagMonitor.StoreLag(partition, lag)
			}
		case <-k.stopConsuming:
			pc.Close()
			if err := offsetMgr.Commit(topic, partition, currentOffset); err != nil {
				log.Error(3, "kafka-mdm failed to commit offset for %s:%d, %s", topic, partition, err)
			}
			log.Info("kafka-mdm consumer for %s:%d ended.", topic, partition)
			return
		}
	}
}

func (k *KafkaMdm) handleMsg(data []byte, partition int32) {
	md := schema.MetricData{}
	_, err := md.UnmarshalMsg(data)
	if err != nil {
		metricsDecodeErr.Inc()
		log.Error(3, "kafka-mdm decode error, skipping message. %s", err)
		return
	}
	metricsPerMessage.ValueUint32(1)
	k.Handler.Process(&md, partition)
}

// Stop will initiate a graceful stop of the Consumer (permanent)
// and block until it stopped.
func (k *KafkaMdm) Stop() {
	// closes notifications and messages channels, amongst others
	close(k.stopConsuming)
	k.wg.Wait()
	k.client.Close()
	offsetMgr.Close()
}

func (k *KafkaMdm) MaintainPriority() {
	go func() {
		ticker := time.NewTicker(time.Second * 10)
		for {
			select {
			case <-k.stopConsuming:
				return
			case <-ticker.C:
				cluster.Manager.SetPriority(k.lagMonitor.Metric())
			}
		}
	}()
}
<commit_msg>more offset/lag reporting, fix priority reporting when no consumption<commit_after>package kafkamdm

import (
	"flag"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"

	"github.com/raintank/metrictank/cluster"
	"github.com/raintank/metrictank/input"
	"github.com/raintank/metrictank/kafka"
	"github.com/raintank/metrictank/stats"
	"gopkg.in/raintank/schema.v1"
)

// metric input.kafka-mdm.metrics_per_message is how many metrics per message were seen.
var metricsPerMessage = stats.NewMeter32("input.kafka-mdm.metrics_per_message", false)

// metric input.kafka-mdm.metrics_decode_err is a count of times an input message failed to parse
var metricsDecodeErr = stats.NewCounter32("input.kafka-mdm.metrics_decode_err")

type KafkaMdm struct {
	input.Handler
	consumer   sarama.Consumer
	client     sarama.Client
	lagMonitor *LagMonitor
	wg         sync.WaitGroup

	// signal to PartitionConsumers to shutdown
	stopConsuming chan struct{}
}

func (k *KafkaMdm) Name() string {
	return "kafka-mdm"
}

var LogLevel int
var Enabled bool
var brokerStr string
var brokers []string
var topicStr string
var topics []string
var partitionStr string
var partitions []int32
var offsetStr string
var DataDir string
var config *sarama.Config
var channelBufferSize int
var consumerFetchMin int
var consumerFetchDefault int
var consumerMaxWaitTime time.Duration
var consumerMaxProcessingTime time.Duration
var netMaxOpenRequests int
var offsetMgr *kafka.OffsetMgr
var offsetDuration time.Duration
var offsetCommitInterval time.Duration
var partitionOffset map[int32]*stats.Gauge64
var partitionLogSize map[int32]*stats.Gauge64
var partitionLag map[int32]*stats.Gauge64

func ConfigSetup() {
	inKafkaMdm := flag.NewFlagSet("kafka-mdm-in", flag.ExitOnError)
	inKafkaMdm.BoolVar(&Enabled, "enabled", false, "")
	inKafkaMdm.StringVar(&brokerStr, "brokers", "kafka:9092", "tcp address for kafka (may be be given multiple times as a comma-separated list)")
	inKafkaMdm.StringVar(&topicStr, "topics", "mdm", "kafka topic (may be given multiple times as a comma-separated list)")
	inKafkaMdm.StringVar(&offsetStr, "offset", "last", "Set the offset to start consuming from. Can be one of newest, oldest,last or a time duration")
	inKafkaMdm.StringVar(&partitionStr, "partitions", "*", "kafka partitions to consume. use '*' or a comma separated list of id's")
	inKafkaMdm.DurationVar(&offsetCommitInterval, "offset-commit-interval", time.Second*5, "Interval at which offsets should be saved.")
	inKafkaMdm.StringVar(&DataDir, "data-dir", "", "Directory to store partition offsets index")
	inKafkaMdm.IntVar(&channelBufferSize, "channel-buffer-size", 1000000, "The number of metrics to buffer in internal and external channels")
	inKafkaMdm.IntVar(&consumerFetchMin, "consumer-fetch-min", 1, "The minimum number of message bytes to fetch in a request")
	inKafkaMdm.IntVar(&consumerFetchDefault, "consumer-fetch-default", 32768, "The default number of message bytes to fetch in a request")
	inKafkaMdm.DurationVar(&consumerMaxWaitTime, "consumer-max-wait-time", time.Second, "The maximum amount of time the broker will wait for Consumer.Fetch.Min bytes to become available before it returns fewer than that anyway")
	inKafkaMdm.DurationVar(&consumerMaxProcessingTime, "consumer-max-processing-time", time.Second, "The maximum amount of time the consumer expects a message takes to process")
	inKafkaMdm.IntVar(&netMaxOpenRequests, "net-max-open-requests", 100, "How many outstanding requests a connection is allowed to have before sending on it blocks")
	globalconf.Register("kafka-mdm-in", inKafkaMdm)
}

func ConfigProcess(instance string) {
	if !Enabled {
		return
	}

	if offsetCommitInterval == 0 {
		log.Fatal(4, "kafkamdm: offset-commit-interval must be greater then 0")
	}
	if consumerMaxWaitTime == 0 {
		log.Fatal(4, "kafkamdm: consumer-max-wait-time must be greater then 0")
	}
	if consumerMaxProcessingTime == 0 {
		log.Fatal(4, "kafkamdm: consumer-max-processing-time must be greater then 0")
	}
	var err error
	switch offsetStr {
	case "last":
	case "oldest":
	case "newest":
	default:
		offsetDuration, err = time.ParseDuration(offsetStr)
		if err != nil {
			log.Fatal(4, "kafkamdm: invalid offest format. %s", err)
		}
	}

	offsetMgr, err = kafka.NewOffsetMgr(DataDir)
	if err != nil {
		log.Fatal(4, "kafka-mdm couldnt create offsetMgr. %s", err)
	}
	brokers = strings.Split(brokerStr, ",")
	topics = strings.Split(topicStr, ",")

	config = sarama.NewConfig()

	config.ClientID = instance + "-mdm"
	config.ChannelBufferSize = channelBufferSize
	config.Consumer.Fetch.Min = int32(consumerFetchMin)
	config.Consumer.Fetch.Default = int32(consumerFetchDefault)
	config.Consumer.MaxWaitTime = consumerMaxWaitTime
	config.Consumer.MaxProcessingTime = consumerMaxProcessingTime
	config.Net.MaxOpenRequests = netMaxOpenRequests
	config.Version = sarama.V0_10_0_0
	err = config.Validate()
	if err != nil {
		log.Fatal(2, "kafka-mdm invalid config: %s", err)
	}
	// validate our partitions
	client, err := sarama.NewClient(brokers, config)
	if err != nil {
		log.Fatal(4, "kafka-mdm failed to create client. %s", err)
	}
	defer client.Close()

	availParts, err := kafka.GetPartitions(client, topics)
	if err != nil {
		log.Fatal(4, "kafka-mdm: %s", err.Error())
	}
	log.Info("kafka-mdm: available partitions %v", availParts)
	if partitionStr == "*" {
		partitions = availParts
	} else {
		parts := strings.Split(partitionStr, ",")
		for _, part := range parts {
			i, err := strconv.Atoi(part)
			if err != nil {
				log.Fatal(4, "could not parse partition %q. partitions must be '*' or a comma separated list of id's", part)
			}
			partitions = append(partitions, int32(i))
		}
		missing := kafka.DiffPartitions(partitions, availParts)
		if len(missing) > 0 {
			log.Fatal(4, "kafka-mdm: configured partitions not in list of available partitions. missing %v", missing)
		}
	}
	// record our partitions so others (MetricIdx) can use the partitioning information.
	// but only if the manager has been created (e.g. in metrictank), not when this input plugin is used in other contexts
	if cluster.Manager != nil {
		cluster.Manager.SetPartitions(partitions)
	}

	// initialize our offset metrics
	partitionOffset = make(map[int32]*stats.Gauge64)
	partitionLogSize = make(map[int32]*stats.Gauge64)
	partitionLag = make(map[int32]*stats.Gauge64)
	for _, part := range partitions {
		partitionOffset[part] = stats.NewGauge64(fmt.Sprintf("input.kafka-mdm.partition.%d.offset", part))
		partitionLogSize[part] = stats.NewGauge64(fmt.Sprintf("input.kafka-mdm.partition.%d.log_size", part))
		partitionLag[part] = stats.NewGauge64(fmt.Sprintf("input.kafka-mdm.partition.%d.lag", part))
	}
}

func New() *KafkaMdm {
	client, err := sarama.NewClient(brokers, config)
	if err != nil {
		log.Fatal(4, "kafka-mdm failed to create client. %s", err)
	}
	consumer, err := sarama.NewConsumerFromClient(client)
	if err != nil {
		log.Fatal(2, "kafka-mdm failed to create consumer: %s", err)
	}
	log.Info("kafka-mdm consumer created without error")
	k := KafkaMdm{
		consumer:      consumer,
		client:        client,
		lagMonitor:    NewLagMonitor(10, partitions),
		stopConsuming: make(chan struct{}),
	}

	return &k
}

func (k *KafkaMdm) Start(handler input.Handler) {
	k.Handler = handler
	var err error
	for _, topic := range topics {
		for _, partition := range partitions {
			var offset int64
			switch offsetStr {
			case "oldest":
				offset = sarama.OffsetOldest
			case "newest":
				offset = sarama.OffsetNewest
			case "last":
				offset, err = offsetMgr.Last(topic, partition)
			default:
				offset, err = k.client.GetOffset(topic, partition, time.Now().Add(-1*offsetDuration).UnixNano()/int64(time.Millisecond))
			}
			if err != nil {
				log.Fatal(4, "kafka-mdm: Failed to get %q duration offset for %s:%d. %q", offsetStr, topic, partition, err)
			}
			go k.consumePartition(topic, partition, offset)
		}
	}
}

// this will continually consume from the topic until k.stopConsuming is triggered.
func (k *KafkaMdm) consumePartition(topic string, partition int32, currentOffset int64) {
	k.wg.Add(1)
	defer k.wg.Done()

	pc, err := k.consumer.ConsumePartition(topic, partition, currentOffset)
	if err != nil {
		log.Fatal(4, "kafka-mdm: failed to start partitionConsumer for %s:%d. %s", topic, partition, err)
	}

	partitionOffsetMetric := partitionOffset[partition]
	partitionLogSizeMetric := partitionLogSize[partition]
	partitionLagMetric := partitionLag[partition]
	var initialOffset int64
	offset, err := k.client.GetOffset(topic, partition, sarama.OffsetNewest)
	if err != nil {
		log.Error(3, "kafka-mdm failed to get log-size of partition %s:%d. %s", topic, partition, err)
	} else {
		partitionLogSizeMetric.Set(int(offset))
	}
	var err2 error
	if currentOffset == sarama.OffsetNewest {
		initialOffset = offset
	} else if currentOffset == sarama.OffsetOldest {
		initialOffset, err2 = k.client.GetOffset(topic, partition, currentOffset)
		if err != nil {
			log.Error(3, "kafka-mdm failed to get oldest offset of partition %s:%d. %s", topic, partition, err)
		}
	}

	if err == nil && err2 == nil {
		partitionOffsetMetric.Set(int(initialOffset))
		partitionLagMetric.Set(int(offset - initialOffset))
	}

	log.Info("kafka-mdm: consuming from %s:%d from offset %d", topic, partition, currentOffset)
	messages := pc.Messages()
	ticker := time.NewTicker(offsetCommitInterval)
	for {
		select {
		case msg := <-messages:
			if LogLevel < 2 {
				log.Debug("kafka-mdm received message: Topic %s, Partition: %d, Offset: %d, Key: %x", msg.Topic, msg.Partition, msg.Offset, msg.Key)
			}
			k.handleMsg(msg.Value, partition)
			currentOffset = msg.Offset
		case ts := <-ticker.C:
			if err := offsetMgr.Commit(topic, partition, currentOffset); err != nil {
				log.Error(3, "kafka-mdm failed to commit offset for %s:%d, %s", topic, partition, err)
			}
			k.lagMonitor.StoreOffset(partition, currentOffset, ts)
			offset, err := k.client.GetOffset(topic, partition, sarama.OffsetNewest)
			if err != nil {
				log.Error(3, "kafka-mdm failed to get log-size of partition %s:%d. %s", topic, partition, err)
			} else {
				partitionLogSizeMetric.Set(int(offset))
			}

			effectiveOffset := currentOffset
			if currentOffset < 0 {
				// we're set to consume oldest/newest and we have not yet consumed any messages.
				// so we're still stuck at the offset we got initially.
				effectiveOffset = initialOffset
			}
			partitionOffsetMetric.Set(int(effectiveOffset))
			if err == nil {
				lag := int(offset - effectiveOffset)
				partitionLagMetric.Set(lag)
				k.lagMonitor.StoreLag(partition, lag)
			}
		case <-k.stopConsuming:
			pc.Close()
			if err := offsetMgr.Commit(topic, partition, currentOffset); err != nil {
				log.Error(3, "kafka-mdm failed to commit offset for %s:%d, %s", topic, partition, err)
			}
			log.Info("kafka-mdm consumer for %s:%d ended.", topic, partition)
			return
		}
	}
}

func (k *KafkaMdm) handleMsg(data []byte, partition int32) {
	md := schema.MetricData{}
	_, err := md.UnmarshalMsg(data)
	if err != nil {
		metricsDecodeErr.Inc()
		log.Error(3, "kafka-mdm decode error, skipping message. %s", err)
		return
	}
	metricsPerMessage.ValueUint32(1)
	k.Handler.Process(&md, partition)
}

// Stop will initiate a graceful stop of the Consumer (permanent)
// and block until it stopped.
func (k *KafkaMdm) Stop() {
	// closes notifications and messages channels, amongst others
	close(k.stopConsuming)
	k.wg.Wait()
	k.client.Close()
	offsetMgr.Close()
}

func (k *KafkaMdm) MaintainPriority() {
	go func() {
		ticker := time.NewTicker(time.Second * 10)
		for {
			select {
			case <-k.stopConsuming:
				return
			case <-ticker.C:
				cluster.Manager.SetPriority(k.lagMonitor.Metric())
			}
		}
	}()
}
<|endoftext|>


<commit_before>package install

import (
	"fmt"
	"testing"

	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/assert"
	"k8s.io/api/core/v1"
	v1beta1extensions "k8s.io/api/extensions/v1beta1"
	v1beta1rbac "k8s.io/api/rbac/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"reflect"

	"github.com/coreos-inc/alm/apis"
	"github.com/coreos-inc/alm/apis/clusterserviceversion/v1alpha1"
	"github.com/coreos-inc/alm/client"
	"github.com/stretchr/testify/require"
)

func testDepoyment(name, namespace string, mockOwnerMeta metav1.ObjectMeta) v1beta1extensions.Deployment {
	testDeploymentLabels := map[string]string{"alm-owner-name": mockOwnerMeta.Name, "alm-owner-namespace": mockOwnerMeta.Namespace}

	deployment := v1beta1extensions.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Namespace:    namespace,
			GenerateName: fmt.Sprintf("%s-", mockOwnerMeta.Name),
			OwnerReferences: []metav1.OwnerReference{
				{
					APIVersion:         apis.GroupName,
					Kind:               v1alpha1.ClusterServiceVersionKind,
					Name:               mockOwnerMeta.GetName(),
					UID:                mockOwnerMeta.UID,
					Controller:         &Controller,
					BlockOwnerDeletion: &BlockOwnerDeletion,
				},
			},
			Labels: testDeploymentLabels,
		},
	}
	return deployment
}

func testServiceAccount(name string, mockOwnerMeta metav1.ObjectMeta) *v1.ServiceAccount {
	serviceAccount := &v1.ServiceAccount{}
	serviceAccount.SetName(name)
	serviceAccount.SetOwnerReferences([]metav1.OwnerReference{
		{
			APIVersion:         apis.GroupName,
			Kind:               v1alpha1.ClusterServiceVersionKind,
			Name:               mockOwnerMeta.GetName(),
			UID:                mockOwnerMeta.UID,
			Controller:         &Controller,
			BlockOwnerDeletion: &BlockOwnerDeletion,
		},
	})
	return serviceAccount
}

type RoleMatcher struct{ rules []v1beta1rbac.PolicyRule }

func MatchesRoleRules(rules []v1beta1rbac.PolicyRule) gomock.Matcher {
	return &RoleMatcher{rules}
}

func (e *RoleMatcher) Matches(x interface{}) bool {
	role, ok := x.(*v1beta1rbac.Role)
	if !ok {
		return false
	}
	return reflect.DeepEqual(role.Rules, e.rules)
}

func (e *RoleMatcher) String() string {
	return "matches expected rules"
}

func strategy(n int, namespace string, mockOwnerMeta metav1.ObjectMeta) *StrategyDetailsDeployment {
	var deploymentSpecs = []v1beta1extensions.DeploymentSpec{}
	var permissions = []StrategyDeploymentPermissions{}
	for i := 1; i <= n; i++ {
		deploymentSpecs = append(deploymentSpecs, testDepoyment(fmt.Sprintf("alm-dep-%d", i), namespace, mockOwnerMeta).Spec)
		serviceAccount := testServiceAccount(fmt.Sprintf("alm-sa-%d", i), mockOwnerMeta)
		permissions = append(permissions, StrategyDeploymentPermissions{
			ServiceAccountName: serviceAccount.Name,
			Rules: []v1beta1rbac.PolicyRule{
				{
					Verbs:     []string{"list", "delete"},
					APIGroups: []string{""},
					Resources: []string{"pods"},
				},
			},
		})
	}
	return &StrategyDetailsDeployment{
		DeploymentSpecs: deploymentSpecs,
		Permissions:     permissions,
	}
}

func TestInstallStrategyDeployment(t *testing.T) {
	namespace := "alm-test-deployment"
	mockOwnerMeta := metav1.ObjectMeta{
		Name:         "clusterserviceversion-owner",
		Namespace:    namespace,
		GenerateName: fmt.Sprintf("%s-", namespace),
	}
	mockOwnerType := metav1.TypeMeta{
		Kind:       "kind",
		APIVersion: "APIString",
	}

	tests := []struct {
		numMockServiceAccounts int
		numMockDeployments     int
		numExpected            int
		description            string
	}{
		{
			numMockServiceAccounts: 0,
			numMockDeployments:     0,
			numExpected:            1,
			description:            "NoServiceAccount/NoDeployment/Require1,1",
		},
		{
			numMockServiceAccounts: 1,
			numMockDeployments:     1,
			numExpected:            1,
			description:            "1ServiceAccount/1Deployment/Require1,1",
		},
		{
			numMockServiceAccounts: 0,
			numMockDeployments:     1,
			numExpected:            1,
			description:            "0ServiceAccount/1Deployment/Require1,1",
		},
		{
			numMockServiceAccounts: 1,
			numMockDeployments:     0,
			numExpected:            1,
			description:            "1ServiceAccount/0Deployment/Require1,1",
		},
		{
			numMockServiceAccounts: 3,
			numMockDeployments:     3,
			numExpected:            3,
			description:            "3ServiceAccount/3Deployment/Require3,3",
		},
	}

	for _, tt := range tests {
		t.Run(tt.description, func(t *testing.T) {
			ctrl := gomock.NewController(t)
			mockClient := client.NewMockInstallStrategyDeploymentInterface(ctrl)
			strategy := strategy(tt.numExpected, namespace, mockOwnerMeta)
			for i, p := range strategy.Permissions {
				if i < tt.numMockServiceAccounts {
					t.Logf("mocking %s true", p.ServiceAccountName)
					mockClient.EXPECT().
						CheckServiceAccount(p.ServiceAccountName).
						Return(true, nil)
				}
				if i == tt.numMockServiceAccounts {
					t.Logf("mocking %s false", p.ServiceAccountName)
					mockClient.EXPECT().
						CheckServiceAccount(p.ServiceAccountName).
						Return(false, nil)
				}

				serviceAccount := testServiceAccount(p.ServiceAccountName, mockOwnerMeta)
				mockClient.EXPECT().GetOrCreateServiceAccount(serviceAccount).Return(serviceAccount, nil)
				mockClient.EXPECT().
					CreateRole(MatchesRoleRules(p.Rules)).
					Return(&v1beta1rbac.Role{Rules: p.Rules}, nil)
				mockClient.EXPECT().CreateRoleBinding(gomock.Any()).Return(&v1beta1rbac.RoleBinding{}, nil)
			}
			if tt.numMockServiceAccounts == tt.numExpected {
				t.Log("mocking dep check")
				// if all serviceaccounts exist then we check if deployments exist
				mockClient.EXPECT().CheckOwnedDeployments(mockOwnerMeta, strategy.DeploymentSpecs).Return(tt.numMockDeployments == len(strategy.DeploymentSpecs), nil)
			}

			for i := range make([]int, len(strategy.DeploymentSpecs)) {
				deployment := testDepoyment(fmt.Sprintf("alm-dep-%d", i), namespace, mockOwnerMeta)
				mockClient.EXPECT().
					CreateDeployment(&deployment).
					Return(&deployment, nil)
			}

			installer := &StrategyDeploymentInstaller{
				strategyClient: mockClient,
				ownerMeta:      mockOwnerMeta,
				ownerType:      mockOwnerType,
			}
			installed, err := installer.CheckInstalled(strategy)
			if tt.numMockServiceAccounts == tt.numExpected && tt.numMockDeployments == tt.numExpected {
				require.True(t, installed)
			} else {
				require.False(t, installed)
			}
			assert.NoError(t, err)
			assert.NoError(t, installer.Install(strategy))

			ctrl.Finish()
		})
	}
}
<commit_msg>Deploy install strategy: Add tests for error cases<commit_after>package install

import (
	"fmt"
	"testing"

	opClient "github.com/coreos-inc/operator-client/pkg/client"
	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/assert"
	"k8s.io/api/core/v1"
	v1beta1extensions "k8s.io/api/extensions/v1beta1"
	v1beta1rbac "k8s.io/api/rbac/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"reflect"

	"github.com/coreos-inc/alm/apis"
	"github.com/coreos-inc/alm/apis/clusterserviceversion/v1alpha1"
	"github.com/coreos-inc/alm/client"
	"github.com/stretchr/testify/require"
)

func testDepoyment(name, namespace string, mockOwnerMeta metav1.ObjectMeta) v1beta1extensions.Deployment {
	testDeploymentLabels := map[string]string{"alm-owner-name": mockOwnerMeta.Name, "alm-owner-namespace": mockOwnerMeta.Namespace}

	deployment := v1beta1extensions.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Namespace:    namespace,
			GenerateName: fmt.Sprintf("%s-", mockOwnerMeta.Name),
			OwnerReferences: []metav1.OwnerReference{
				{
					APIVersion:         apis.GroupName,
					Kind:               v1alpha1.ClusterServiceVersionKind,
					Name:               mockOwnerMeta.GetName(),
					UID:                mockOwnerMeta.UID,
					Controller:         &Controller,
					BlockOwnerDeletion: &BlockOwnerDeletion,
				},
			},
			Labels: testDeploymentLabels,
		},
	}
	return deployment
}

func testServiceAccount(name string, mockOwnerMeta metav1.ObjectMeta) *v1.ServiceAccount {
	serviceAccount := &v1.ServiceAccount{}
	serviceAccount.SetName(name)
	serviceAccount.SetOwnerReferences([]metav1.OwnerReference{
		{
			APIVersion:         apis.GroupName,
			Kind:               v1alpha1.ClusterServiceVersionKind,
			Name:               mockOwnerMeta.GetName(),
			UID:                mockOwnerMeta.UID,
			Controller:         &Controller,
			BlockOwnerDeletion: &BlockOwnerDeletion,
		},
	})
	return serviceAccount
}

type RoleMatcher struct{ rules []v1beta1rbac.PolicyRule }

func MatchesRoleRules(rules []v1beta1rbac.PolicyRule) gomock.Matcher {
	return &RoleMatcher{rules}
}

func (e *RoleMatcher) Matches(x interface{}) bool {
	role, ok := x.(*v1beta1rbac.Role)
	if !ok {
		return false
	}
	return reflect.DeepEqual(role.Rules, e.rules)
}

func (e *RoleMatcher) String() string {
	return "matches expected rules"
}

func strategy(n int, namespace string, mockOwnerMeta metav1.ObjectMeta) *StrategyDetailsDeployment {
	var deploymentSpecs = []v1beta1extensions.DeploymentSpec{}
	var permissions = []StrategyDeploymentPermissions{}
	for i := 1; i <= n; i++ {
		deploymentSpecs = append(deploymentSpecs, testDepoyment(fmt.Sprintf("alm-dep-%d", i), namespace, mockOwnerMeta).Spec)
		serviceAccount := testServiceAccount(fmt.Sprintf("alm-sa-%d", i), mockOwnerMeta)
		permissions = append(permissions, StrategyDeploymentPermissions{
			ServiceAccountName: serviceAccount.Name,
			Rules: []v1beta1rbac.PolicyRule{
				{
					Verbs:     []string{"list", "delete"},
					APIGroups: []string{""},
					Resources: []string{"pods"},
				},
			},
		})
	}
	return &StrategyDetailsDeployment{
		DeploymentSpecs: deploymentSpecs,
		Permissions:     permissions,
	}
}

func TestInstallStrategyDeployment(t *testing.T) {
	namespace := "alm-test-deployment"
	mockOwnerMeta := metav1.ObjectMeta{
		Name:         "clusterserviceversion-owner",
		Namespace:    namespace,
		GenerateName: fmt.Sprintf("%s-", namespace),
	}
	mockOwnerType := metav1.TypeMeta{
		Kind:       "kind",
		APIVersion: "APIString",
	}

	tests := []struct {
		numMockServiceAccounts int
		numMockDeployments     int
		numExpected            int
		description            string
	}{
		{
			numMockServiceAccounts: 0,
			numMockDeployments:     0,
			numExpected:            1,
			description:            "NoServiceAccount/NoDeployment/Require1,1",
		},
		{
			numMockServiceAccounts: 1,
			numMockDeployments:     1,
			numExpected:            1,
			description:            "1ServiceAccount/1Deployment/Require1,1",
		},
		{
			numMockServiceAccounts: 0,
			numMockDeployments:     1,
			numExpected:            1,
			description:            "0ServiceAccount/1Deployment/Require1,1",
		},
		{
			numMockServiceAccounts: 1,
			numMockDeployments:     0,
			numExpected:            1,
			description:            "1ServiceAccount/0Deployment/Require1,1",
		},
		{
			numMockServiceAccounts: 3,
			numMockDeployments:     3,
			numExpected:            3,
			description:            "3ServiceAccount/3Deployment/Require3,3",
		},
	}

	for _, tt := range tests {
		t.Run(tt.description, func(t *testing.T) {
			ctrl := gomock.NewController(t)
			mockClient := client.NewMockInstallStrategyDeploymentInterface(ctrl)
			strategy := strategy(tt.numExpected, namespace, mockOwnerMeta)
			for i, p := range strategy.Permissions {
				if i < tt.numMockServiceAccounts {
					t.Logf("mocking %s true", p.ServiceAccountName)
					mockClient.EXPECT().
						CheckServiceAccount(p.ServiceAccountName).
						Return(true, nil)
				}
				if i == tt.numMockServiceAccounts {
					t.Logf("mocking %s false", p.ServiceAccountName)
					mockClient.EXPECT().
						CheckServiceAccount(p.ServiceAccountName).
						Return(false, nil)
				}

				serviceAccount := testServiceAccount(p.ServiceAccountName, mockOwnerMeta)
				mockClient.EXPECT().GetOrCreateServiceAccount(serviceAccount).Return(serviceAccount, nil)
				mockClient.EXPECT().
					CreateRole(MatchesRoleRules(p.Rules)).
					Return(&v1beta1rbac.Role{Rules: p.Rules}, nil)
				mockClient.EXPECT().CreateRoleBinding(gomock.Any()).Return(&v1beta1rbac.RoleBinding{}, nil)
			}
			if tt.numMockServiceAccounts == tt.numExpected {
				t.Log("mocking dep check")
				// if all serviceaccounts exist then we check if deployments exist
				mockClient.EXPECT().CheckOwnedDeployments(mockOwnerMeta, strategy.DeploymentSpecs).Return(tt.numMockDeployments == len(strategy.DeploymentSpecs), nil)
			}

			for i := range make([]int, len(strategy.DeploymentSpecs)) {
				deployment := testDepoyment(fmt.Sprintf("alm-dep-%d", i), namespace, mockOwnerMeta)
				mockClient.EXPECT().
					CreateDeployment(&deployment).
					Return(&deployment, nil)
			}

			installer := &StrategyDeploymentInstaller{
				strategyClient: mockClient,
				ownerMeta:      mockOwnerMeta,
				ownerType:      mockOwnerType,
			}
			installed, err := installer.CheckInstalled(strategy)
			if tt.numMockServiceAccounts == tt.numExpected && tt.numMockDeployments == tt.numExpected {
				require.True(t, installed)
			} else {
				require.False(t, installed)
			}
			assert.NoError(t, err)
			assert.NoError(t, installer.Install(strategy))

			ctrl.Finish()
		})
	}
}

type BadStrategy struct{}

func (b *BadStrategy) GetStrategyName() string {
	return "bad"
}

func TestNewStrategyDeploymentInstaller(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	mockClient := opClient.NewMockInterface(ctrl)
	mockOwnerMeta := metav1.ObjectMeta{
		Name:         "clusterserviceversion-owner",
		Namespace:    "ns",
		GenerateName: fmt.Sprintf("%s-", "ns"),
	}
	mockOwnerType := metav1.TypeMeta{
		Kind:       "kind",
		APIVersion: "APIString",
	}
	strategy := NewStrategyDeploymentInstaller(mockClient, mockOwnerMeta, mockOwnerType)
	require.Implements(t, (*StrategyInstaller)(nil), strategy)
	require.Error(t, strategy.Install(&BadStrategy{}))
	_, err := strategy.CheckInstalled(&BadStrategy{})
	require.Error(t, err)
}

func TestInstallStrategyDeploymentCheckInstallErrors(t *testing.T) {
	namespace := "alm-test-deployment"
	mockOwnerMeta := metav1.ObjectMeta{
		Name:         "clusterserviceversion-owner",
		Namespace:    namespace,
		GenerateName: fmt.Sprintf("%s-", namespace),
	}
	mockOwnerType := metav1.TypeMeta{
		Kind:       "kind",
		APIVersion: "APIString",
	}

	tests := []struct {
		createRoleErr           error
		createRoleBindingErr    error
		createServiceAccountErr error
		createDeploymentErr     error
		checkServiceAccountErr  error
		checkDeploymentErr      error
		description             string
	}{
		{
			checkServiceAccountErr: fmt.Errorf("couldn't query serviceaccount"),
			description:            "ErrorCheckingForServiceAccount",
		},
		{
			checkDeploymentErr: fmt.Errorf("couldn't query deployments"),
			description:        "ErrorCheckingForDeployments",
		},
		{
			createRoleErr: fmt.Errorf("error creating role"),
			description:   "ErrorCreatingRole",
		},
		{
			createServiceAccountErr: fmt.Errorf("error creating serviceaccount"),
			description:             "ErrorCreateingServiceAccount",
		},
		{
			createRoleBindingErr: fmt.Errorf("error creating rolebinding"),
			description:          "ErrorCreatingRoleBinding",
		},
		{
			createDeploymentErr: fmt.Errorf("error creating deployment"),
			description:         "ErrorCreatingDeployment",
		},
	}

	for _, tt := range tests {
		t.Run(tt.description, func(t *testing.T) {
			ctrl := gomock.NewController(t)
			defer ctrl.Finish()

			mockClient := client.NewMockInstallStrategyDeploymentInterface(ctrl)
			strategy := strategy(1, namespace, mockOwnerMeta)
			installer := &StrategyDeploymentInstaller{
				strategyClient: mockClient,
				ownerMeta:      mockOwnerMeta,
				ownerType:      mockOwnerType,
			}

			skipInstall := tt.checkDeploymentErr != nil || tt.checkServiceAccountErr != nil

			mockClient.EXPECT().
				CheckServiceAccount(strategy.Permissions[0].ServiceAccountName).
				Return(tt.checkServiceAccountErr == nil, tt.checkServiceAccountErr)
			if tt.checkServiceAccountErr == nil {
				mockClient.EXPECT().CheckOwnedDeployments(mockOwnerMeta, strategy.DeploymentSpecs).Return(tt.checkDeploymentErr == nil, tt.checkDeploymentErr)
			}

			installed, err := installer.CheckInstalled(strategy)

			if skipInstall {
				require.False(t, installed)
				require.Error(t, err)
				return
			} else {
				require.True(t, installed)
				require.NoError(t, err)
			}

			mockClient.EXPECT().
				CreateRole(MatchesRoleRules(strategy.Permissions[0].Rules)).
				Return(&v1beta1rbac.Role{Rules: strategy.Permissions[0].Rules}, tt.createRoleErr)

			if tt.createRoleErr != nil {
				err := installer.Install(strategy)
				require.Error(t, err)
				return
			}

			serviceAccount := testServiceAccount(strategy.Permissions[0].ServiceAccountName, mockOwnerMeta)
			mockClient.EXPECT().GetOrCreateServiceAccount(serviceAccount).Return(serviceAccount, tt.createServiceAccountErr)

			if tt.createServiceAccountErr != nil {
				err := installer.Install(strategy)
				require.Error(t, err)
				return
			}

			mockClient.EXPECT().CreateRoleBinding(gomock.Any()).Return(&v1beta1rbac.RoleBinding{}, tt.createRoleBindingErr)

			if tt.createRoleBindingErr != nil {
				err := installer.Install(strategy)
				require.Error(t, err)
				return
			}

			deployment := testDepoyment("alm-dep", namespace, mockOwnerMeta)
			mockClient.EXPECT().
				CreateDeployment(&deployment).
				Return(&deployment, tt.createDeploymentErr)

			if tt.createDeploymentErr != nil {
				err := installer.Install(strategy)
				require.Error(t, err)
				return
			}
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"
)

const ucdFileName = "UnicodeData.txt"
const ucdBaseUrl = "http://www.unicode.org/Public/UCD/latest/ucd/"

func progressDisplay(running <-chan bool) {
	for {
		select {
		case <-running:
			fmt.Println()
		case <-time.After(200 * time.Millisecond):
			fmt.Print(".")
		}
	}
}

func getUcdFile(fileName string) {

	url := ucdBaseUrl + ucdFileName
	fmt.Printf("%s not found\nretrieving from %s\n", ucdFileName, url)
	running := make(chan bool)
	go progressDisplay(running)
	defer func() {
		running <- false
	}()
	response, err := http.Get(url)
	if err != nil {
		panic(err)
	}
	defer response.Body.Close()
	file, err := os.Create(fileName)
	if err != nil {
		panic(err)
	}
	_, err = io.Copy(file, response.Body)
	if err != nil {
		panic(err)
	}
	file.Close()
}

type runesSlice []rune

func (rs runesSlice) Len() int           { return len(rs) }
func (rs runesSlice) Less(i, j int) bool { return rs[i] < rs[j] }
func (rs runesSlice) Swap(i, j int)      { rs[i], rs[j] = rs[j], rs[i] }

type runeSet map[rune]struct{}

func (rs runeSet) Intersection(other runeSet) runeSet {
	result := runeSet{}
	for k := range rs {
		if _, occurs := other[k]; occurs {
			result[k] = struct{}{}
		}
	}
	return result
}

func buildIndex(fileName string) (map[string]runesSlice, map[rune]string) {
	if _, err := os.Stat(fileName); os.IsNotExist(err) {
		getUcdFile(fileName)
	}
	content, err := ioutil.ReadFile(fileName)
	if err != nil {
		panic(err)
	}

	lines := strings.Split(string(content), "\n")

	index := map[string]runesSlice{}
	names := map[rune]string{}

	for _, line := range lines {
		var uchar rune
		fields := strings.Split(line, ";")
		if len(fields) >= 2 {
			code64, _ := strconv.ParseInt(fields[0], 16, 0)
			uchar = rune(code64)
			names[uchar] = fields[1]
			// fmt.Printf("%#v", index)
			for _, word := range strings.Split(fields[1], " ") {
				var entries runesSlice
				if len(index[word]) < 1 {
					entries = runesSlice{}
				} else {
					entries = index[word]
				}
				index[word] = append(entries, uchar)
			}
		}

	}
	return index, names
}

func findRunes(query []string, index map[string]runesSlice) runesSlice {
	foundList := []runeSet{}
	for _, word := range query {
		word = strings.ToUpper(word)
		found := runeSet{}
		for _, uchar := range index[word] {
			found[uchar] = struct{}{}
		}
		foundList = append(foundList, found)
	}
	commonRunes := foundList[0]
	for _, found := range foundList[1:] {
		commonRunes = commonRunes.Intersection(found)
	}
	result := runesSlice{}
	for uchar := range commonRunes {
		result = append(result, uchar)
	}
	sort.Sort(result)
	return result
}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("Usage:  runefinder <word>\texample: runefinder cat")
		os.Exit(1)
	}
	words := os.Args[1:]

	dir, _ := os.Getwd()
	path := path.Join(dir, ucdFileName)
	index, names := buildIndex(path)

	count := 0
	format := "U+%04X  %c \t%s\n"
	for _, uchar := range findRunes(words, index) {
		if uchar > 0xFFFF {
			format = "U+%5X %c \t%s\n"
		}
		fmt.Printf(format, uchar, uchar, names[uchar])
		count++
	}
	fmt.Printf("%d characters found\n", count)

}
<commit_msg>converted index to map from word to runeSet<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"
)

const ucdFileName = "UnicodeData.txt"
const ucdBaseUrl = "http://www.unicode.org/Public/UCD/latest/ucd/"

func progressDisplay(running <-chan bool) {
	for {
		select {
		case <-running:
			fmt.Println()
		case <-time.After(200 * time.Millisecond):
			fmt.Print(".")
		}
	}
}

func getUcdFile(fileName string) {

	url := ucdBaseUrl + ucdFileName
	fmt.Printf("%s not found\nretrieving from %s\n", ucdFileName, url)
	running := make(chan bool)
	go progressDisplay(running)
	defer func() {
		running <- false
	}()
	response, err := http.Get(url)
	if err != nil {
		panic(err)
	}
	defer response.Body.Close()
	file, err := os.Create(fileName)
	if err != nil {
		panic(err)
	}
	_, err = io.Copy(file, response.Body)
	if err != nil {
		panic(err)
	}
	file.Close()
}

type runeSlice []rune

func (rs runeSlice) Len() int           { return len(rs) }
func (rs runeSlice) Less(i, j int) bool { return rs[i] < rs[j] }
func (rs runeSlice) Swap(i, j int)      { rs[i], rs[j] = rs[j], rs[i] }

type runeSet map[rune]struct{}

func (rs runeSet) Intersection(other runeSet) runeSet {
	result := runeSet{}
	for k := range rs {
		if _, occurs := other[k]; occurs {
			result[k] = struct{}{}
		}
	}
	return result
}

func buildIndex(fileName string) (map[string]runeSet, map[rune]string) {
	if _, err := os.Stat(fileName); os.IsNotExist(err) {
		getUcdFile(fileName)
	}
	content, err := ioutil.ReadFile(fileName)
	if err != nil {
		panic(err)
	}
	lines := strings.Split(string(content), "\n")

	index := map[string]runeSet{}
	names := map[rune]string{}

	for _, line := range lines {
		var uchar rune
		fields := strings.Split(line, ";")
		if len(fields) >= 2 {
			code64, _ := strconv.ParseInt(fields[0], 16, 0)
			uchar = rune(code64)
			names[uchar] = fields[1]
			for _, word := range strings.Split(fields[1], " ") {
				var existing runeSet
				if len(index[word]) < 1 {
					existing = runeSet{}
				} else {
					existing = index[word]
				}
				existing[uchar] = struct{}{}
				index[word] = existing
			}
		}

	}
	return index, names
}

func getIndex() (map[string]runeSet, map[rune]string) {
	dir, _ := os.Getwd()
	path := path.Join(dir, ucdFileName)
	return buildIndex(path)
}

func findRunes(query []string, index map[string]runeSet) runeSlice {
	commonRunes := runeSet{}
	for i, word := range query {
		word = strings.ToUpper(word)
		found := index[word]
		if i == 0 {
			commonRunes = found
		} else {
			commonRunes = commonRunes.Intersection(found)
		}
		if len(commonRunes) == 0 {
			break
		}
	}
	result := runeSlice{}
	for uchar := range commonRunes {
		result = append(result, uchar)
	}
	sort.Sort(result)
	return result
}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("Usage:  runefinder <word>\texample: runefinder cat")
		os.Exit(1)
	}
	words := os.Args[1:]

	index, names := getIndex()

	count := 0
	format := "U+%04X  %c \t%s\n"
	for _, uchar := range findRunes(words, index) {
		if uchar > 0xFFFF {
			format = "U+%5X %c \t%s\n"
		}
		fmt.Printf(format, uchar, uchar, names[uchar])
		count++
	}
	fmt.Printf("%d characters found\n", count)

}
<|endoftext|>


<commit_before>package selenium

import (
	"flag"
	"fmt"
	"strings"
	"testing"

	"github.com/blang/semver"
	"github.com/tebeka/selenium/sauce"
)

var (
	enableSauce      = flag.Bool("experimental_enable_sauce", false, "If true, perform integration tests on SauceLabs remote infrastructure.")
	sauceUserName    = flag.String("sauce_user_name", "", "The username to use for SauceLabs.")
	sauceAccessKey   = flag.String("sauce_access_key", "", "The access key to use for SauceLabs.")
	sauceConnectPath = flag.String("sauce_connect_path", "vendor/sauce-connect/bin/sc", "The path to the Sauce Connect binary.")
)

func TestSauce(t *testing.T) {
	if !*enableSauce {
		t.Skip("Skipping Sauce tests. Enable via --experimental_sauce_tests")
	}
	if testing.Verbose() {
		SetDebug(true)
	}
	if *sauceUserName == "" {
		t.Fatalf("--sauce_user_name is required.")
	}
	if *sauceAccessKey == "" {
		t.Fatalf("--sauce_access_key is required.")
	}

	addr := sauce.Addr(*sauceUserName, *sauceAccessKey)
	if *sauceConnectPath != "" {
		port, err := pickUnusedPort()
		if err != nil {
			t.Fatalf("pickUnusedPort() returned error: %s", err)
		}
		sc := sauce.Connect{
			Path:         *sauceConnectPath,
			UserName:     *sauceUserName,
			AccessKey:    *sauceAccessKey,
			SeleniumPort: port,
		}
		if testing.Verbose() {
			sc.ExtraVerbose = true
		}
		if err := sc.Start(); err != nil {
			t.Fatalf("sc.Start() returned error: %s", err)
		}
		defer func() {
			if err := sc.Stop(); err != nil {
				t.Fatalf("sc.Stop() returned error: %s", err)
			}
		}()
		addr = sc.Addr()
	}

	const browser = "Firefox"
	for _, tc := range []struct {
		version, platform, selenium string
	}{
		{"Latest", "Windows 10", "3.4.0"},
		{"45.0", "Linux", "2.53.1"},
	} {
		name := fmt.Sprintf("%s/%s/%s", browser, tc.version, tc.platform)
		browser, version := strings.ToLower(browser), strings.ToLower(tc.version)

		t.Run(name, func(t *testing.T) {
			runFirefoxSubTests(t, config{
				browser:         browser,
				seleniumVersion: semver.MustParse(tc.selenium),
				sauce: &sauce.Capabilities{
					Browser:         browser,
					Version:         version,
					Platform:        tc.platform,
					SeleniumVersion: tc.selenium,
				},
				addr: addr,
			})
		})
	}
}
<commit_msg>Add the Selenium version number to the Sauce-based test names<commit_after>package selenium

import (
	"flag"
	"fmt"
	"strings"
	"testing"

	"github.com/blang/semver"
	"github.com/tebeka/selenium/sauce"
)

var (
	enableSauce      = flag.Bool("experimental_enable_sauce", false, "If true, perform integration tests on SauceLabs remote infrastructure.")
	sauceUserName    = flag.String("sauce_user_name", "", "The username to use for SauceLabs.")
	sauceAccessKey   = flag.String("sauce_access_key", "", "The access key to use for SauceLabs.")
	sauceConnectPath = flag.String("sauce_connect_path", "vendor/sauce-connect/bin/sc", "The path to the Sauce Connect binary.")
)

func TestSauce(t *testing.T) {
	if !*enableSauce {
		t.Skip("Skipping Sauce tests. Enable via --experimental_sauce_tests")
	}
	if testing.Verbose() {
		SetDebug(true)
	}
	if *sauceUserName == "" {
		t.Fatalf("--sauce_user_name is required.")
	}
	if *sauceAccessKey == "" {
		t.Fatalf("--sauce_access_key is required.")
	}

	addr := sauce.Addr(*sauceUserName, *sauceAccessKey)
	if *sauceConnectPath != "" {
		port, err := pickUnusedPort()
		if err != nil {
			t.Fatalf("pickUnusedPort() returned error: %s", err)
		}
		sc := sauce.Connect{
			Path:         *sauceConnectPath,
			UserName:     *sauceUserName,
			AccessKey:    *sauceAccessKey,
			SeleniumPort: port,
		}
		if testing.Verbose() {
			sc.ExtraVerbose = true
		}
		if err := sc.Start(); err != nil {
			t.Fatalf("sc.Start() returned error: %s", err)
		}
		defer func() {
			if err := sc.Stop(); err != nil {
				t.Fatalf("sc.Stop() returned error: %s", err)
			}
		}()
		addr = sc.Addr()
	}

	const browser = "Firefox"
	for _, tc := range []struct {
		version, platform, selenium string
	}{
		{"Latest", "Windows 10", "3.4.0"},
		{"45.0", "Linux", "2.53.1"},
	} {
		name := fmt.Sprintf("%s/%s/%s/%s", browser, tc.version, tc.platform, tc.selenium)
		browser, version := strings.ToLower(browser), strings.ToLower(tc.version)

		t.Run(name, func(t *testing.T) {
			runFirefoxSubTests(t, config{
				browser:         browser,
				seleniumVersion: semver.MustParse(tc.selenium),
				sauce: &sauce.Capabilities{
					Browser:         browser,
					Version:         version,
					Platform:        tc.platform,
					SeleniumVersion: tc.selenium,
				},
				addr: addr,
			})
		})
	}
}
<|endoftext|>


<commit_before>package rtime

import (
	"strings"
	"time"
)

// Returns the minute offset within the hour specified by t, in the range [0, 59].
func (t Time) Minute() int {
	return time.Time(t).Minute()
}

// Returns the second offset within the minute specified by t, in the range [0, 59].
func (t Time) Second() int {
	return time.Time(t).Second()
}

// Returns the hour within the day specified by t, in the range [0, 23].
func (t Time) Hour() int {
	return time.Time(t).Hour()
}

// Returns string representation of Hour
func (t Time) HourString() (result string) {
	result = numberInString(t.Hour(), false)
	switch t.Hour() {
	case 1:
		result += " час"
	case 2, 3, 4:
		result += " часа"
	default:
		result += " часов"
	}
	return
}

// Returns string representation of Minute
func (t Time) MinuteString() (result string) {
	result = numberInString(t.Minute(), true)
	switch t.Minute() % 10 {
	case 1:
		result += " минута"
	case 2, 3, 4:
		result += " минуты"
	default:
		result += " минут"
	}
	return
}

// Returns string representation of Second
func (t Time) SecondString() (result string) {
	result = numberInString(t.Second(), true)
	switch t.Second() % 10 {
	case 1:
		result += " секунда"
	case 2, 3, 4:
		result += " секунды"
	default:
		result += " секунд"
	}
	return
}

func numberInString(num int, female bool) (result string) {
	switch num {
	case 0:
		result = "Ноль"
	case 1:
		if female {
			result = "Одна"
		} else {
			result = "Один"
		}
	case 2:
		if female {
			result = "Две"
		} else {
			result = "Две"
		}
	case 3:
		result = "Три"
	case 4:
		result = "Четыре"
	case 5:
		result = "Пять"
	case 6:
		result = "Шесть"
	case 7:
		result = "Семь"
	case 8:
		result = "Восемь"
	case 9:
		result = "Девять"
	case 10:
		result = "Десять"
	case 11:
		result = "Одиннадцать"
	case 12:
		result = "Двенадцать"
	case 13:
		result = "Тринадцать"
	case 14:
		result = "Четырнадцать"
	case 15:
		result = "Пятнадцать"
	case 16:
		result = "Шестнадцать"
	case 17:
		result = "Семнадцать"
	case 18:
		result = "Восемнадцать"
	case 19:
		result = "Девятнадцать"
	case 20:
		result = "Двадцать"
	case 30:
		result = "Тридцать"
	case 40:
		result = "Сорок"
	case 50:
		result = "Пятьдесят"
	case 60:
		result = "Шестьдесят"
	case 70:
		result = "Семьдесят"
	case 80:
		result = "Восемьдесят"
	case 90:
		result = "Девяносто"
	default:
		lastDigit := num % 10
		result = numberInString(num-lastDigit, female)
		result += " " + strings.ToLower(numberInString(lastDigit, female))
	}
	return
}
<commit_msg>Fix when minutes or seconds less 10<commit_after>package rtime

import (
	"strings"
	"time"
)

// Returns the minute offset within the hour specified by t, in the range [0, 59].
func (t Time) Minute() int {
	return time.Time(t).Minute()
}

// Returns the second offset within the minute specified by t, in the range [0, 59].
func (t Time) Second() int {
	return time.Time(t).Second()
}

// Returns the hour within the day specified by t, in the range [0, 23].
func (t Time) Hour() int {
	return time.Time(t).Hour()
}

// Returns string representation of Hour
func (t Time) HourString() (result string) {
	result = numberInString(t.Hour(), false)
	switch t.Hour() {
	case 1:
		result += " час"
	case 2, 3, 4:
		result += " часа"
	default:
		result += " часов"
	}
	return
}

// Returns string representation of Minute
func (t Time) MinuteString() (result string) {
	result = numberInString(t.Minute(), true)
	minutes := t.Minute()
	if minutes > 20 {
		minutes = minutes % 10
	}
	switch minutes {
	case 1:
		result += " минута"
	case 2, 3, 4:
		result += " минуты"
	default:
		result += " минут"
	}
	return
}

// Returns string representation of Second
func (t Time) SecondString() (result string) {
	result = numberInString(t.Second(), true)
	seconds := t.Second()
	if seconds > 20 {
		seconds = seconds % 10
	}
	switch seconds {
	case 1:
		result += " секунда"
	case 2, 3, 4:
		result += " секунды"
	default:
		result += " секунд"
	}
	return
}

func numberInString(num int, female bool) (result string) {
	switch num {
	case 0:
		result = "Ноль"
	case 1:
		if female {
			result = "Одна"
		} else {
			result = "Один"
		}
	case 2:
		if female {
			result = "Две"
		} else {
			result = "Две"
		}
	case 3:
		result = "Три"
	case 4:
		result = "Четыре"
	case 5:
		result = "Пять"
	case 6:
		result = "Шесть"
	case 7:
		result = "Семь"
	case 8:
		result = "Восемь"
	case 9:
		result = "Девять"
	case 10:
		result = "Десять"
	case 11:
		result = "Одиннадцать"
	case 12:
		result = "Двенадцать"
	case 13:
		result = "Тринадцать"
	case 14:
		result = "Четырнадцать"
	case 15:
		result = "Пятнадцать"
	case 16:
		result = "Шестнадцать"
	case 17:
		result = "Семнадцать"
	case 18:
		result = "Восемнадцать"
	case 19:
		result = "Девятнадцать"
	case 20:
		result = "Двадцать"
	case 30:
		result = "Тридцать"
	case 40:
		result = "Сорок"
	case 50:
		result = "Пятьдесят"
	case 60:
		result = "Шестьдесят"
	case 70:
		result = "Семьдесят"
	case 80:
		result = "Восемьдесят"
	case 90:
		result = "Девяносто"
	default:
		lastDigit := num % 10
		result = numberInString(num-lastDigit, female)
		result += " " + strings.ToLower(numberInString(lastDigit, female))
	}
	return
}
<|endoftext|>


<commit_before>package tlsh

import (
	"fmt"
	"io/ioutil"
)

var windowLength = 5
var effBuckets = 256
var codeSize = 32
var numBuckets = 256

// LSH holds the hash components
type LSH struct {
	Checksum byte
	Length   byte
}

func getTriplets(slice []byte) (triplets [][]byte) {
	triplets = [][]byte{
		{slice[0], slice[1], slice[2]},
		{slice[0], slice[1], slice[3]},
		{slice[0], slice[2], slice[3]},
		{slice[0], slice[2], slice[4]},
		{slice[0], slice[1], slice[4]},
		{slice[0], slice[3], slice[4]},
	}
	return triplets
}

func quartilePoints(buckets []byte) (q1, q2, q3 byte) {
	buckets2 := make([]byte, effBuckets)
	copy(buckets2, buckets[:effBuckets])
	sortedBuckets := SortByteArray(buckets2)
	// 25%, 50% and 75%
	return sortedBuckets[(effBuckets/4)-1], sortedBuckets[(effBuckets/2)-1], sortedBuckets[effBuckets-(effBuckets/4)-1]
}

func makeHash(buckets []byte, q1, q2, q3 byte) []uint {
	var biHash []uint

	for i := 0; i < codeSize; i++ {
		var h uint
		for j := 0; j < 4; j++ {
			k := buckets[4*i+j]
			if q3 < k {
				h += 3 << (uint(j) * 2)
			} else if q2 < k {
				h += 2 << (uint(j) * 2)
			} else if q1 < k {
				h += 1 << (uint(j) * 2)
			}
		}
		biHash = append([]uint{h}, biHash...)
	}

	return biHash
}

//Hash calculates the TLSH for the input file
func Hash(filename string) (hash string, err error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}
	buckets := make([]byte, numBuckets)
	chunk := make([]byte, windowLength)
	sw := 0
	for sw <= len(data)-windowLength {

		for j, x := sw+windowLength-1, 0; j >= sw; j, x = j-1, x+1 {
			chunk[x] = data[j]
		}

		sw++
		triplets := getTriplets(chunk)
		salt := []byte{2, 3, 5, 7, 11, 13}
		for i, triplet := range triplets {
			buckets[pearsonHash(salt[i], triplet)]++
		}
	}
	q1, q2, q3 := quartilePoints(buckets)
	fmt.Println(q1, q2, q3)
	q1Ratio := (q1 * 100 / q3) % 16
	q2Ratio := (q2 * 100 / q3) % 16
	fmt.Println(q1Ratio, q2Ratio)
	//checksum := pearsonHash(0, triplet)
	//fmt.Println(checksum)

	biHash := makeHash(buckets, q1, q2, q3)

	for i := 0; i < len(biHash); i++ {
		hash += fmt.Sprintf("%02X", biHash[i])
	}

	return hash, nil
}
<commit_msg>Add lValue generation for header construction<commit_after>package tlsh

import (
	"fmt"
	"math"
	"io/ioutil"
)

const (
	LOG_1_5 = 0.4054651
	LOG_1_3 = 0.26236426
	LOG_1_1 = 0.095310180
)

var windowLength = 5
var effBuckets = 256
var codeSize = 32
var numBuckets = 256

// LSH holds the hash components
type LSH struct {
	Checksum byte
	Length   byte
}

func getTriplets(slice []byte) (triplets [][]byte) {
	triplets = [][]byte{
		{slice[0], slice[1], slice[2]},
		{slice[0], slice[1], slice[3]},
		{slice[0], slice[2], slice[3]},
		{slice[0], slice[2], slice[4]},
		{slice[0], slice[1], slice[4]},
		{slice[0], slice[3], slice[4]},
	}
	return triplets
}

func quartilePoints(buckets []byte) (q1, q2, q3 byte) {
	buckets2 := make([]byte, effBuckets)
	copy(buckets2, buckets[:effBuckets])
	sortedBuckets := SortByteArray(buckets2)
	// 25%, 50% and 75%
	return sortedBuckets[(effBuckets/4)-1], sortedBuckets[(effBuckets/2)-1], sortedBuckets[effBuckets-(effBuckets/4)-1]
}

func lValue(length int) uint {
	var l uint

	if length <= 656 {
		l = uint(math.Floor(math.Log(float64(length)) / LOG_1_5))
	} else if length <= 3199 {
		l = uint(math.Floor(math.Log(float64(length)) / LOG_1_3 - 8.72777))
	} else {
		l = uint(math.Floor(math.Log(float64(length)) / LOG_1_1 - 62.5472))
	}

	return l % 256
}

func swapByte(in uint) uint {
	var out uint
	out = ((in & 0xF0) >> 4 ) & 0x0F
	out |= ((in & 0x0F) << 4 ) & 0xF0

	return out
}

func makeHash(buckets []byte, q1, q2, q3 byte) []uint {
	var biHash []uint

	for i := 0; i < codeSize; i++ {
		var h uint
		for j := 0; j < 4; j++ {
			k := buckets[4*i+j]
			if q3 < k {
				h += 3 << (uint(j) * 2)
			} else if q2 < k {
				h += 2 << (uint(j) * 2)
			} else if q1 < k {
				h += 1 << (uint(j) * 2)
			}
		}
		biHash = append([]uint{h}, biHash...)
	}

	return biHash
}

//Hash calculates the TLSH for the input file
func Hash(filename string) (hash string, err error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}
	buckets := make([]byte, numBuckets)
	chunk := make([]byte, windowLength)
	sw := 0
	for sw <= len(data)-windowLength {

		for j, x := sw+windowLength-1, 0; j >= sw; j, x = j-1, x+1 {
			chunk[x] = data[j]
		}

		sw++
		triplets := getTriplets(chunk)
		salt := []byte{2, 3, 5, 7, 11, 13}
		for i, triplet := range triplets {
			buckets[pearsonHash(salt[i], triplet)]++
		}
	}
	q1, q2, q3 := quartilePoints(buckets)
	fmt.Println(q1, q2, q3)
	q1Ratio := (q1 * 100 / q3) % 16
	q2Ratio := (q2 * 100 / q3) % 16
	lValue := lValue(len(data))
	fmt.Println(q1Ratio, q2Ratio)
	//checksum := pearsonHash(0, triplet)
	//fmt.Println(checksum)

	biHash := makeHash(buckets, q1, q2, q3)

	fmt.Printf("\nL=%02X\n", swapByte(lValue))
	for i := 0; i < len(biHash); i++ {
		hash += fmt.Sprintf("%02X", biHash[i])
	}

	return hash, nil
}
<|endoftext|>


<commit_before>/* Copyright (c) 2014-2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import "html/template"

var tmpl *template.Template

func loadTemplates() {
	for name, s := range templates {
		var t *template.Template
		if tmpl == nil {
			tmpl = template.New(name)
		}
		if name == tmpl.Name() {
			t = tmpl
		} else {
			t = tmpl.New(name)
		}
		if _, err := t.Parse(s); err != nil {
			panic("could not load templates")
		}
	}
}

var templates = map[string]string{
	"/": `

Set up an alias:

    $ alias pcat='curl -F "{{.FieldName}}=<-" {{.SiteURL}}'

Upload a new paste:

    $ echo foo | pcat
    {{.SiteURL}}/a63d03b9

Fetch it:

    $ curl {{.SiteURL}}/a63d03b9
    foo

You can also use the web form.
{{if gt .MaxSize 0.0}}
Maximum size per paste is {{.MaxSize}}.
{{end}}{{if gt .LifeTime 0}}
Pastes will be deleted after {{.LifeTime}}.
{{end}}
github.com/mvdan/pastecat

`,
	"/form": `

	<form action="{{.SiteURL}}" method="post" enctype="multipart/form-data">
		<textarea cols=80 rows=24 name="{{.FieldName}}"></textarea>

		<button type="submit">Paste text</button>
	</form>

	<form action="{{.SiteURL}}" method="post" enctype="multipart/form-data">
		<input type="file" name="{{.FieldName}}"></input>

		<button type="submit">Paste file</button>
	</form>

{{.LimitDesc}}

`,
}
<commit_msg>Remove limits from the form page<commit_after>/* Copyright (c) 2014-2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import "html/template"

var tmpl *template.Template

func loadTemplates() {
	for name, s := range templates {
		var t *template.Template
		if tmpl == nil {
			tmpl = template.New(name)
		}
		if name == tmpl.Name() {
			t = tmpl
		} else {
			t = tmpl.New(name)
		}
		if _, err := t.Parse(s); err != nil {
			panic("could not load templates")
		}
	}
}

var templates = map[string]string{
	"/": `

Set up an alias:

    $ alias pcat='curl -F "{{.FieldName}}=<-" {{.SiteURL}}'

Upload a new paste:

    $ echo foo | pcat
    {{.SiteURL}}/a63d03b9

Fetch it:

    $ curl {{.SiteURL}}/a63d03b9
    foo

You can also use the web form.
{{if gt .MaxSize 0.0}}
Maximum size per paste is {{.MaxSize}}
{{end}}{{if gt .LifeTime 0}}
Pastes will be deleted after {{.LifeTime}}
{{end}}
github.com/mvdan/pastecat

`,
	"/form": `

	<form action="{{.SiteURL}}" method="post" enctype="multipart/form-data">
		<textarea cols=80 rows=24 name="{{.FieldName}}"></textarea>

		<button type="submit">Paste text</button>
	</form>

	<form action="{{.SiteURL}}" method="post" enctype="multipart/form-data">
		<input type="file" name="{{.FieldName}}"></input>

		<button type="submit">Paste file</button>
	</form>

`,
}
<|endoftext|>


<commit_before>package restic

import (
	"errors"
	"fmt"
	"path/filepath"
	"sort"

	"github.com/restic/restic/backend"
	"github.com/restic/restic/debug"
)

type Tree struct {
	Nodes []*Node `json:"nodes"`
	Map   *Map    `json:"map"`
}

var (
	ErrNodeNotFound      = errors.New("named node not found")
	ErrNodeAlreadyInTree = errors.New("node already present")
)

func NewTree() *Tree {
	return &Tree{
		Nodes: []*Node{},
		Map:   NewMap(),
	}
}

func (t Tree) String() string {
	return fmt.Sprintf("Tree<%d nodes, %d blobs>", len(t.Nodes), len(t.Map.list))
}

func LoadTree(s Server, blob Blob) (*Tree, error) {
	tree := &Tree{}
	err := s.LoadJSON(backend.Tree, blob, tree)
	if err != nil {
		return nil, err
	}

	return tree, nil
}

// LoadTreeRecursive loads the tree and all subtrees via s.
func LoadTreeRecursive(path string, s Server, blob Blob) (*Tree, error) {
	// TODO: load subtrees in parallel
	tree, err := LoadTree(s, blob)
	if err != nil {
		return nil, err
	}

	for _, n := range tree.Nodes {
		n.path = filepath.Join(path, n.Name)
		if n.Type == "dir" && n.Subtree != nil {
			subtreeBlob, err := tree.Map.FindID(n.Subtree)
			if err != nil {
				return nil, err
			}

			t, err := LoadTreeRecursive(n.path, s, subtreeBlob)
			if err != nil {
				return nil, err
			}

			n.tree = t
		}
	}

	return tree, nil
}

// CopyFrom recursively copies all content from other to t.
func (t Tree) CopyFrom(other *Tree, s *Server) error {
	debug.Log("Tree.CopyFrom", "CopyFrom(%v)\n", other)
	for _, node := range t.Nodes {
		// only process files and dirs
		if node.Type != "file" && node.Type != "dir" {
			continue
		}

		// find entry in other tree
		oldNode, err := other.Find(node.Name)

		// if the node could not be found or the type has changed, proceed to the next
		if err == ErrNodeNotFound || node.Type != oldNode.Type {
			debug.Log("Tree.CopyFrom", "  node %v is new\n", node)
			continue
		}

		if node.Type == "file" {
			// compare content
			if node.SameContent(oldNode) {
				debug.Log("Tree.CopyFrom", "  file node %v has same content\n", node)

				// check if all content is still available in the repository
				for _, id := range oldNode.Content {
					blob, err := other.Map.FindID(id)
					if err != nil {
						continue
					}

					if ok, err := s.Test(backend.Data, blob.Storage); !ok || err != nil {
						continue
					}
				}

				// copy Content
				node.Content = oldNode.Content

				// copy storage IDs
				for _, id := range node.Content {
					blob, err := other.Map.FindID(id)
					if err != nil {
						return err
					}

					debug.Log("Tree.CopyFrom", "    insert blob %v\n", blob)
					t.Map.Insert(blob)
				}
			}
		} else if node.Type == "dir" {
			// fill in all subtrees from old subtree
			err := node.tree.CopyFrom(oldNode.tree, s)
			if err != nil {
				return err
			}

			// check if tree has changed
			if node.tree.Equals(*oldNode.tree) {
				debug.Log("Tree.CopyFrom", "  tree node %v has same content\n", node)

				// if nothing has changed, copy subtree ID
				node.Subtree = oldNode.Subtree

				// and store blob in bloblist
				blob, err := other.Map.FindID(oldNode.Subtree)
				if err != nil {
					return err
				}

				debug.Log("Tree.CopyFrom", "    insert blob %v\n", blob)
				t.Map.Insert(blob)
			} else {
				debug.Log("Tree.CopyFrom", "  trees are not equal: %v\n", node)
				debug.Log("Tree.CopyFrom", "    %#v\n", node.tree)
				debug.Log("Tree.CopyFrom", "    %#v\n", oldNode.tree)
			}
		}
	}

	return nil
}

// Equals returns true if t and other have exactly the same nodes and map.
func (t Tree) Equals(other Tree) bool {
	if len(t.Nodes) != len(other.Nodes) {
		debug.Log("Tree.Equals", "tree.Equals(): trees have different number of nodes")
		return false
	}

	if !t.Map.Equals(other.Map) {
		debug.Log("Tree.Equals", "tree.Equals(): maps aren't equal")
		return false
	}

	for i := 0; i < len(t.Nodes); i++ {
		if !t.Nodes[i].Equals(*other.Nodes[i]) {
			debug.Log("Tree.Equals", "tree.Equals(): node %d is different:", i)
			debug.Log("Tree.Equals", "  %#v", t.Nodes[i])
			debug.Log("Tree.Equals", "  %#v", other.Nodes[i])
			return false
		}
	}

	return true
}

func (t *Tree) Insert(node *Node) error {
	pos, _, err := t.find(node.Name)
	if err == nil {
		// already present
		return ErrNodeAlreadyInTree
	}

	// insert blob
	// https://code.google.com/p/go-wiki/wiki/bliceTricks
	t.Nodes = append(t.Nodes, &Node{})
	copy(t.Nodes[pos+1:], t.Nodes[pos:])
	t.Nodes[pos] = node

	return nil
}

func (t Tree) find(name string) (int, *Node, error) {
	pos := sort.Search(len(t.Nodes), func(i int) bool {
		return t.Nodes[i].Name >= name
	})

	if pos < len(t.Nodes) && t.Nodes[pos].Name == name {
		return pos, t.Nodes[pos], nil
	}

	return pos, nil, ErrNodeNotFound
}

func (t Tree) Find(name string) (*Node, error) {
	_, node, err := t.find(name)
	return node, err
}

func (t Tree) Stat() Stat {
	s := Stat{}
	for _, n := range t.Nodes {
		switch n.Type {
		case "file":
			s.Files++
			s.Bytes += n.Size
		case "dir":
			s.Dirs++
			if n.tree != nil {
				s.Add(n.tree.Stat())
			}
		}
	}

	return s
}

func (t Tree) StatTodo() Stat {
	s := Stat{}
	for _, n := range t.Nodes {
		switch n.Type {
		case "file":
			if n.Content == nil {
				s.Files++
				s.Bytes += n.Size
			}
		case "dir":
			if n.Subtree == nil {
				s.Dirs++
				if n.tree != nil {
					s.Add(n.tree.StatTodo())
				}
			}
		}
	}

	return s
}
<commit_msg>tree: Add concurrent traversing functions<commit_after>package restic

import (
	"errors"
	"fmt"
	"path/filepath"
	"sort"
	"sync"

	"github.com/restic/restic/backend"
	"github.com/restic/restic/debug"
)

type Tree struct {
	Nodes []*Node `json:"nodes"`
	Map   *Map    `json:"map"`
}

var (
	ErrNodeNotFound      = errors.New("named node not found")
	ErrNodeAlreadyInTree = errors.New("node already present")
)

func NewTree() *Tree {
	return &Tree{
		Nodes: []*Node{},
		Map:   NewMap(),
	}
}

func (t Tree) String() string {
	return fmt.Sprintf("Tree<%d nodes, %d blobs>", len(t.Nodes), len(t.Map.list))
}

func LoadTree(s Server, blob Blob) (*Tree, error) {
	tree := &Tree{}
	err := s.LoadJSON(backend.Tree, blob, tree)
	if err != nil {
		return nil, err
	}

	return tree, nil
}

// LoadTreeRecursive loads the tree and all subtrees via s.
func LoadTreeRecursive(path string, s Server, blob Blob) (*Tree, error) {
	// TODO: load subtrees in parallel
	tree, err := LoadTree(s, blob)
	if err != nil {
		return nil, err
	}

	for _, n := range tree.Nodes {
		n.path = filepath.Join(path, n.Name)
		if n.Type == "dir" && n.Subtree != nil {
			subtreeBlob, err := tree.Map.FindID(n.Subtree)
			if err != nil {
				return nil, err
			}

			t, err := LoadTreeRecursive(n.path, s, subtreeBlob)
			if err != nil {
				return nil, err
			}

			n.tree = t
		}
	}

	return tree, nil
}

// CopyFrom recursively copies all content from other to t.
func (t Tree) CopyFrom(other *Tree, s *Server) error {
	debug.Log("Tree.CopyFrom", "CopyFrom(%v)\n", other)
	for _, node := range t.Nodes {
		// only process files and dirs
		if node.Type != "file" && node.Type != "dir" {
			continue
		}

		// find entry in other tree
		oldNode, err := other.Find(node.Name)

		// if the node could not be found or the type has changed, proceed to the next
		if err == ErrNodeNotFound || node.Type != oldNode.Type {
			debug.Log("Tree.CopyFrom", "  node %v is new\n", node)
			continue
		}

		if node.Type == "file" {
			// compare content
			if node.SameContent(oldNode) {
				debug.Log("Tree.CopyFrom", "  file node %v has same content\n", node)

				// check if all content is still available in the repository
				for _, id := range oldNode.Content {
					blob, err := other.Map.FindID(id)
					if err != nil {
						continue
					}

					if ok, err := s.Test(backend.Data, blob.Storage); !ok || err != nil {
						continue
					}
				}

				// copy Content
				node.Content = oldNode.Content

				// copy storage IDs
				for _, id := range node.Content {
					blob, err := other.Map.FindID(id)
					if err != nil {
						return err
					}

					debug.Log("Tree.CopyFrom", "    insert blob %v\n", blob)
					t.Map.Insert(blob)
				}
			}
		} else if node.Type == "dir" {
			// fill in all subtrees from old subtree
			err := node.tree.CopyFrom(oldNode.tree, s)
			if err != nil {
				return err
			}

			// check if tree has changed
			if node.tree.Equals(*oldNode.tree) {
				debug.Log("Tree.CopyFrom", "  tree node %v has same content\n", node)

				// if nothing has changed, copy subtree ID
				node.Subtree = oldNode.Subtree

				// and store blob in bloblist
				blob, err := other.Map.FindID(oldNode.Subtree)
				if err != nil {
					return err
				}

				debug.Log("Tree.CopyFrom", "    insert blob %v\n", blob)
				t.Map.Insert(blob)
			} else {
				debug.Log("Tree.CopyFrom", "  trees are not equal: %v\n", node)
				debug.Log("Tree.CopyFrom", "    %#v\n", node.tree)
				debug.Log("Tree.CopyFrom", "    %#v\n", oldNode.tree)
			}
		}
	}

	return nil
}

// Equals returns true if t and other have exactly the same nodes and map.
func (t Tree) Equals(other Tree) bool {
	if len(t.Nodes) != len(other.Nodes) {
		debug.Log("Tree.Equals", "tree.Equals(): trees have different number of nodes")
		return false
	}

	if !t.Map.Equals(other.Map) {
		debug.Log("Tree.Equals", "tree.Equals(): maps aren't equal")
		return false
	}

	for i := 0; i < len(t.Nodes); i++ {
		if !t.Nodes[i].Equals(*other.Nodes[i]) {
			debug.Log("Tree.Equals", "tree.Equals(): node %d is different:", i)
			debug.Log("Tree.Equals", "  %#v", t.Nodes[i])
			debug.Log("Tree.Equals", "  %#v", other.Nodes[i])
			return false
		}
	}

	return true
}

func (t *Tree) Insert(node *Node) error {
	pos, _, err := t.find(node.Name)
	if err == nil {
		// already present
		return ErrNodeAlreadyInTree
	}

	// insert blob
	// https://code.google.com/p/go-wiki/wiki/bliceTricks
	t.Nodes = append(t.Nodes, &Node{})
	copy(t.Nodes[pos+1:], t.Nodes[pos:])
	t.Nodes[pos] = node

	return nil
}

func (t Tree) find(name string) (int, *Node, error) {
	pos := sort.Search(len(t.Nodes), func(i int) bool {
		return t.Nodes[i].Name >= name
	})

	if pos < len(t.Nodes) && t.Nodes[pos].Name == name {
		return pos, t.Nodes[pos], nil
	}

	return pos, nil, ErrNodeNotFound
}

func (t Tree) Find(name string) (*Node, error) {
	_, node, err := t.find(name)
	return node, err
}

func (t Tree) Stat() Stat {
	s := Stat{}
	for _, n := range t.Nodes {
		switch n.Type {
		case "file":
			s.Files++
			s.Bytes += n.Size
		case "dir":
			s.Dirs++
			if n.tree != nil {
				s.Add(n.tree.Stat())
			}
		}
	}

	return s
}

func (t Tree) StatTodo() Stat {
	s := Stat{}
	for _, n := range t.Nodes {
		switch n.Type {
		case "file":
			if n.Content == nil {
				s.Files++
				s.Bytes += n.Size
			}
		case "dir":
			if n.Subtree == nil {
				s.Dirs++
				if n.tree != nil {
					s.Add(n.tree.StatTodo())
				}
			}
		}
	}

	return s
}

// EachNode calls fn recursively for each node in t and all subtrees
// (depth-first). This in done concurrently in p goroutines.
func (t *Tree) EachNode(p int, fn func(*Node)) {
	processNodeWorker := func(wg *sync.WaitGroup, ch <-chan *Node, f func(*Node)) {
		for node := range ch {
			f(node)
		}
		wg.Done()
	}

	// start workers
	var wg sync.WaitGroup
	ch := make(chan *Node)

	for i := 0; i < p; i++ {
		wg.Add(1)
		go processNodeWorker(&wg, ch, fn)
	}

	var processTree func(t *Tree, ch chan<- *Node)
	processTree = func(t *Tree, ch chan<- *Node) {
		for _, n := range t.Nodes {
			if n.Type == "dir" && n.Tree() != nil {
				processTree(n.Tree(), ch)
			}

			ch <- n
		}
	}

	// run on root
	processTree(t, ch)
	close(ch)

	// wait for all goroutines to terminate
	wg.Wait()
}

// EachSubtree calls fn recursively for t and all subtrees (depth-first). This
// is done concurrently in p goroutines.
func (t *Tree) EachSubtree(p int, fn func(*Tree)) {
	processTreeWorker := func(wg *sync.WaitGroup, ch <-chan *Tree, f func(*Tree)) {
		for tree := range ch {
			f(tree)
		}
		wg.Done()
	}

	// start workers
	var wg sync.WaitGroup
	ch := make(chan *Tree)

	for i := 0; i < p; i++ {
		wg.Add(1)
		go processTreeWorker(&wg, ch, fn)
	}

	// extra variable needed for recursion
	var processTree func(t *Tree, ch chan<- *Tree)
	processTree = func(t *Tree, ch chan<- *Tree) {
		for _, n := range t.Nodes {
			if n.Type == "dir" && n.Tree() != nil {
				processTree(n.Tree(), ch)
			}
		}

		ch <- t
	}

	// run on root
	processTree(t, ch)
	close(ch)

	// wait for all goroutines to terminate
	wg.Wait()
}
<|endoftext|>