<commit_before>// test_consistentHash
package consistentHash

import (
	"crypto/rand"
	"fmt"
	"github.com/GaryBoone/GoStats/stats"
	"github.com/stretchr/testify/assert"
	"strconv"
	"testing"
)

var (
	keys [][]byte
)

func init() {
	keys = make([][]byte, 10000)
	for i := 0; i < len(keys); i++ {
		keys[i] = rand_bytes(10)
	}

}

func rand_bytes(size int) []byte {
	var bytes = make([]byte, size)
	rand.Read(bytes)
	return bytes
}

// Test_vnodeAdd verifies that the correct number of vnodes are added after an Add() call
func Test_vnodeAdd(t *testing.T) {
	c := New()
	c.Add("localhost")
	assert.Equal(t, c.vnodeCount, len(c.vnodes))

}

// Test_Distribution tests how well keys are distributed across servers
// and how many keys are remapped after a node is removed
// This is informational, not a pass/fail test
func Test_Distribution(t *testing.T) {
	c := New()
	serverCount := 10
	for i := 0; i < serverCount; i++ {
		c.Add("server" + strconv.Itoa(i))
	}
	distribution := make(map[string]int)
	keymapping := make(map[string]string)
	for i := 0; i < len(keys); i++ {
		key := keys[i]
		server, _ := c.Get(key)
		keymapping[string(key)] = server
		distribution[server]++
	}
	stat := stats.Stats{}
	for key, count := range distribution {
		stat.Update(float64(count))
		delete(distribution, key)
	}
	fmt.Printf("Stddev for %d keys mapped across %d servers = %.2f\n", len(keys), serverCount, stat.PopulationStandardDeviation())

	c.Remove("server" + strconv.Itoa(serverCount/2))
	stat = stats.Stats{}
	for i := 0; i < len(keys); i++ {
		key := keys[i]
		server, _ := c.Get(key)
		if keymapping[string(key)] == server {
			delete(keymapping, string(key))
		}
		distribution[server]++
	}
	for key, count := range distribution {
		stat.Update(float64(count))
		delete(distribution, key)
	}
	fmt.Printf("Stddev for %d keys mapped across %d servers after one server removed = %.2f\n", len(keys), serverCount, stat.PopulationStandardDeviation())
	fmt.Printf("Number of keys out of %d remapped after removing 1 out of %d servers = %d\n", len(keys), serverCount, len(keymapping))
}

// Benchmark_DefaultLookup tests how fast lookups are if each node has the default number of vnodes
func Benchmark_DefaultLookup(b *testing.B) {
	c := New()
	serverCount := 10
	for i := 0; i < serverCount; i++ {
		c.Add("server" + strconv.Itoa(i))
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		c.Get(keys[i%len(keys)])
	}
}

// Benchmark_SingleVnodeLookup tests how fast lookups are if each node has 1 vnode
// Note that this would have very poor distribution
func Benchmark_SingleVnodeLookup(b *testing.B) {
	c := New()
	c.SetVnodeCount(1)
	serverCount := 10
	for i := 0; i < serverCount; i++ {
		c.Add("server" + strconv.Itoa(i))
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		c.Get(keys[i%len(keys)])
	}
}

// Benchmark_1000VnodeLookup tests how fast lookups are if each node has 1000 vnodes
func Benchmark_1000VnodeLookup(b *testing.B) {
	c := New()
	c.SetVnodeCount(1000)
	serverCount := 10
	for i := 0; i < serverCount; i++ {
		c.Add("server" + strconv.Itoa(i))
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		c.Get(keys[i%len(keys)])
	}
}

// Test_insertVnode verifies that vnodes are correctly inserted in the proper order
func Test_insertVnode(t *testing.T) {
	ch := New()
	v1 := vnode{100, "a"}
	v2 := vnode{50, "b"}
	v3 := vnode{1001, "c"}
	v4 := vnode{1000, "d"}
	ch.insertVnode(v1)
	ch.insertVnode(v2)
	ch.insertVnode(v3)
	ch.insertVnode(v4)
	assert.Equal(t, 4, len(ch.vnodes))
	assert.Equal(t, v2, ch.vnodes[0])
	assert.Equal(t, v1, ch.vnodes[1])
	assert.Equal(t, v3, ch.vnodes[3])
	assert.Equal(t, v4, ch.vnodes[2])

}

func Test_Get2(t *testing.T) {
	ch := New()
	ch.Add("server1")
	ch.Add("server2")
	server1, server2, err := ch.Get2([]byte("testKey"))
	assert.Nil(t, err)
	assert.True(t, (server1 == "server1" && server2 == "server2") || (server1 == "server2" && server2 == "server1"))
}

func Test_GetN(t *testing.T) {
	ch := New()
	ch.Add("server1")
	ch.Add("server2")
	_, err := ch.GetN([]byte("testKey"), 3)
	assert.Equal(t, err, notEnoughMembersError)
	ch.Add("server3")
	servers, err := ch.GetN([]byte("testKey"), 3)
	assert.Nil(t, err)
	assert.Equal(t, 3, len(servers))
}

// Test_removeVnode verifies that vnodes are correctly removed
func Test_removeVnode(t *testing.T) {
	ch := New()
	v1 := vnode{100, "a"}
	v2 := vnode{50, "b"}
	v3 := vnode{1001, "c"}
	v4 := vnode{1000, "d"}
	ch.insertVnode(v1)
	ch.insertVnode(v2)
	ch.insertVnode(v3)
	ch.insertVnode(v4)
	ch.removeVnode(50)
	assert.Equal(t, 3, len(ch.vnodes))
	ch.removeVnode(1001)
	ch.removeVnode(100)
	ch.removeVnode(1000)
	assert.Empty(t, ch.vnodes)

}

func Example_basic() {
	// Output: key=A server=server3
	//key=B server=server3
	//key=C server=server1
	//key=D server=server3
	//key=E server=server2
	//key=F server=server2
	//key=G server=server1
	ch := New()
	ch.Add("server1")
	ch.Add("server2")
	ch.Add("server3")
	keys := []string{"A", "B", "C", "D", "E", "F", "G"}
	for _, key := range keys {
		server, err := ch.Get([]byte(key))
		if err != nil {
			panic(err)
		}
		fmt.Printf("key=%s server=%s\n", key, server)
	}
}

func Example_remove() {
	//Output: 3 servers
	//key=A server=server3
	//key=B server=server3
	//key=C server=server1
	//key=D server=server3
	//key=E server=server2
	//key=F server=server2
	//key=G server=server1
	//Removing server3
	//key=A server=server1
	//key=B server=server2
	//key=C server=server1
	//key=D server=server1
	//key=E server=server2
	//key=F server=server2
	//key=G server=server1
	ch := New()
	ch.Add("server1")
	ch.Add("server2")
	ch.Add("server3")
	keys := []string{"A", "B", "C", "D", "E", "F", "G"}
	fmt.Println("3 servers")
	for _, key := range keys {
		server, _ := ch.Get([]byte(key))
		fmt.Printf("key=%s server=%s\n", key, server)
	}
	fmt.Println("Removing server3")
	ch.Remove("server3")
	for _, key := range keys {
		server, _ := ch.Get([]byte(key))
		fmt.Printf("key=%s server=%s\n", key, server)
	}
}
<commit_msg>test output should only print if test -v is used<commit_after>// test_consistentHash
package consistentHash

import (
	"crypto/rand"
	"fmt"
	"github.com/GaryBoone/GoStats/stats"
	"github.com/stretchr/testify/assert"
	"strconv"
	"testing"
)

var (
	keys [][]byte
)

func init() {
	keys = make([][]byte, 10000)
	for i := 0; i < len(keys); i++ {
		keys[i] = rand_bytes(10)
	}

}

func rand_bytes(size int) []byte {
	var bytes = make([]byte, size)
	rand.Read(bytes)
	return bytes
}

// Test_vnodeAdd verifies that the correct number of vnodes are added after an Add() call
func Test_vnodeAdd(t *testing.T) {
	c := New()
	c.Add("localhost")
	assert.Equal(t, c.vnodeCount, len(c.vnodes))

}

// Test_Distribution tests how well keys are distributed across servers
// and how many keys are remapped after a node is removed
// This is informational, not a pass/fail test
func Test_Distribution(t *testing.T) {
	c := New()
	serverCount := 10
	for i := 0; i < serverCount; i++ {
		c.Add("server" + strconv.Itoa(i))
	}
	distribution := make(map[string]int)
	keymapping := make(map[string]string)
	for i := 0; i < len(keys); i++ {
		key := keys[i]
		server, _ := c.Get(key)
		keymapping[string(key)] = server
		distribution[server]++
	}
	stat := stats.Stats{}
	for key, count := range distribution {
		stat.Update(float64(count))
		delete(distribution, key)
	}
	t.Logf("Stddev for %d keys mapped across %d servers = %.2f\n", len(keys), serverCount, stat.PopulationStandardDeviation())

	c.Remove("server" + strconv.Itoa(serverCount/2))
	stat = stats.Stats{}
	for i := 0; i < len(keys); i++ {
		key := keys[i]
		server, _ := c.Get(key)
		if keymapping[string(key)] == server {
			delete(keymapping, string(key))
		}
		distribution[server]++
	}
	for key, count := range distribution {
		stat.Update(float64(count))
		delete(distribution, key)
	}
	t.Logf("Stddev for %d keys mapped across %d servers after one server removed = %.2f\n", len(keys), serverCount, stat.PopulationStandardDeviation())
	t.Logf("Number of keys out of %d remapped after removing 1 out of %d servers = %d\n", len(keys), serverCount, len(keymapping))
}

// Benchmark_DefaultLookup tests how fast lookups are if each node has the default number of vnodes
func Benchmark_DefaultLookup(b *testing.B) {
	c := New()
	serverCount := 10
	for i := 0; i < serverCount; i++ {
		c.Add("server" + strconv.Itoa(i))
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		c.Get(keys[i%len(keys)])
	}
}

// Benchmark_SingleVnodeLookup tests how fast lookups are if each node has 1 vnode
// Note that this would have very poor distribution
func Benchmark_SingleVnodeLookup(b *testing.B) {
	c := New()
	c.SetVnodeCount(1)
	serverCount := 10
	for i := 0; i < serverCount; i++ {
		c.Add("server" + strconv.Itoa(i))
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		c.Get(keys[i%len(keys)])
	}
}

// Benchmark_1000VnodeLookup tests how fast lookups are if each node has 1000 vnodes
func Benchmark_1000VnodeLookup(b *testing.B) {
	c := New()
	c.SetVnodeCount(1000)
	serverCount := 10
	for i := 0; i < serverCount; i++ {
		c.Add("server" + strconv.Itoa(i))
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		c.Get(keys[i%len(keys)])
	}
}

// Test_insertVnode verifies that vnodes are correctly inserted in the proper order
func Test_insertVnode(t *testing.T) {
	ch := New()
	v1 := vnode{100, "a"}
	v2 := vnode{50, "b"}
	v3 := vnode{1001, "c"}
	v4 := vnode{1000, "d"}
	ch.insertVnode(v1)
	ch.insertVnode(v2)
	ch.insertVnode(v3)
	ch.insertVnode(v4)
	assert.Equal(t, 4, len(ch.vnodes))
	assert.Equal(t, v2, ch.vnodes[0])
	assert.Equal(t, v1, ch.vnodes[1])
	assert.Equal(t, v3, ch.vnodes[3])
	assert.Equal(t, v4, ch.vnodes[2])

}

func Test_Get2(t *testing.T) {
	ch := New()
	ch.Add("server1")
	ch.Add("server2")
	server1, server2, err := ch.Get2([]byte("testKey"))
	assert.Nil(t, err)
	assert.True(t, (server1 == "server1" && server2 == "server2") || (server1 == "server2" && server2 == "server1"))
}

func Test_GetN(t *testing.T) {
	ch := New()
	ch.Add("server1")
	ch.Add("server2")
	_, err := ch.GetN([]byte("testKey"), 3)
	assert.Equal(t, err, notEnoughMembersError)
	ch.Add("server3")
	servers, err := ch.GetN([]byte("testKey"), 3)
	assert.Nil(t, err)
	assert.Equal(t, 3, len(servers))
}

// Test_removeVnode verifies that vnodes are correctly removed
func Test_removeVnode(t *testing.T) {
	ch := New()
	v1 := vnode{100, "a"}
	v2 := vnode{50, "b"}
	v3 := vnode{1001, "c"}
	v4 := vnode{1000, "d"}
	ch.insertVnode(v1)
	ch.insertVnode(v2)
	ch.insertVnode(v3)
	ch.insertVnode(v4)
	ch.removeVnode(50)
	assert.Equal(t, 3, len(ch.vnodes))
	ch.removeVnode(1001)
	ch.removeVnode(100)
	ch.removeVnode(1000)
	assert.Empty(t, ch.vnodes)

}

func Example_basic() {
	// Output: key=A server=server3
	//key=B server=server3
	//key=C server=server1
	//key=D server=server3
	//key=E server=server2
	//key=F server=server2
	//key=G server=server1
	ch := New()
	ch.Add("server1")
	ch.Add("server2")
	ch.Add("server3")
	keys := []string{"A", "B", "C", "D", "E", "F", "G"}
	for _, key := range keys {
		server, err := ch.Get([]byte(key))
		if err != nil {
			panic(err)
		}
		fmt.Printf("key=%s server=%s\n", key, server)
	}
}

func Example_remove() {
	//Output: 3 servers
	//key=A server=server3
	//key=B server=server3
	//key=C server=server1
	//key=D server=server3
	//key=E server=server2
	//key=F server=server2
	//key=G server=server1
	//Removing server3
	//key=A server=server1
	//key=B server=server2
	//key=C server=server1
	//key=D server=server1
	//key=E server=server2
	//key=F server=server2
	//key=G server=server1
	ch := New()
	ch.Add("server1")
	ch.Add("server2")
	ch.Add("server3")
	keys := []string{"A", "B", "C", "D", "E", "F", "G"}
	fmt.Println("3 servers")
	for _, key := range keys {
		server, _ := ch.Get([]byte(key))
		fmt.Printf("key=%s server=%s\n", key, server)
	}
	fmt.Println("Removing server3")
	ch.Remove("server3")
	for _, key := range keys {
		server, _ := ch.Get([]byte(key))
		fmt.Printf("key=%s server=%s\n", key, server)
	}
}
<|endoftext|>


<commit_before>package constants

//
// Version the version tgbot
//
const Version = "dev-0.1"
<commit_msg>Added command processor<commit_after>package constants

//
// Version the version tgbot
//
const Version = "dev-0.2"
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"fmt"
	metrics "github.com/rcrowley/go-metrics"
	kafkaClient "github.com/stealthly/go_kafka_client"
	"net"
	"os"
	"os/signal"
	"strconv"
	"time"
)

func resolveConfig() (*kafkaClient.ConsumerConfig, string, int, string, time.Duration) {
	rawConfig, err := kafkaClient.LoadConfiguration("consumers.properties")
	if err != nil {
		panic("Failed to load configuration file")
	}
	numConsumers, _ := strconv.Atoi(rawConfig["num_consumers"])
	zkTimeout, _ := time.ParseDuration(rawConfig["zookeeper_timeout"])

	numWorkers, _ := strconv.Atoi(rawConfig["num_workers"])
	maxWorkerRetries, _ := strconv.Atoi(rawConfig["max_worker_retries"])
	workerBackoff, _ := time.ParseDuration(rawConfig["worker_backoff"])
	workerRetryThreshold, _ := strconv.Atoi(rawConfig["worker_retry_threshold"])
	workerConsideredFailedTimeWindow, _ := time.ParseDuration(rawConfig["worker_considered_failed_time_window"])
	workerTaskTimeout, _ := time.ParseDuration(rawConfig["worker_task_timeout"])
	workerManagersStopTimeout, _ := time.ParseDuration(rawConfig["worker_managers_stop_timeout"])

	rebalanceBarrierTimeout, _ := time.ParseDuration(rawConfig["rebalance_barrier_timeout"])
	rebalanceMaxRetries, _ := strconv.Atoi(rawConfig["rebalance_max_retries"])
	rebalanceBackoff, _ := time.ParseDuration(rawConfig["rebalance_backoff"])
	partitionAssignmentStrategy, _ := rawConfig["partition_assignment_strategy"]
	excludeInternalTopics, _ := strconv.ParseBool(rawConfig["exclude_internal_topics"])

	numConsumerFetchers, _ := strconv.Atoi(rawConfig["num_consumer_fetchers"])
	fetchBatchSize, _ := strconv.Atoi(rawConfig["fetch_batch_size"])
	fetchMessageMaxBytes, _ := strconv.Atoi(rawConfig["fetch_message_max_bytes"])
	fetchMinBytes, _ := strconv.Atoi(rawConfig["fetch_min_bytes"])
	fetchBatchTimeout, _ := time.ParseDuration(rawConfig["fetch_batch_timeout"])
	requeueAskNextBackoff, _ := time.ParseDuration(rawConfig["requeue_ask_next_backoff"])
	fetchWaitMaxMs, _ := strconv.Atoi(rawConfig["fetch_wait_max_ms"])
	socketTimeout, _ := time.ParseDuration(rawConfig["socket_timeout"])
	queuedMaxMessages, _ := strconv.Atoi(rawConfig["queued_max_messages"])
	refreshLeaderBackoff, _ := time.ParseDuration(rawConfig["refresh_leader_backoff"])
	fetchMetadataRetries, _ := strconv.Atoi(rawConfig["fetch_metadata_retries"])
	fetchMetadataBackoff, _ := time.ParseDuration(rawConfig["fetch_metadata_backoff"])

	time.ParseDuration(rawConfig["fetch_metadata_backoff"])

	offsetsCommitMaxRetries, _ := strconv.Atoi(rawConfig["offsets_commit_max_retries"])

	flushInterval, _ := time.ParseDuration(rawConfig["flush_interval"])
	deploymentTimeout, _ := time.ParseDuration(rawConfig["deployment_timeout"])

	zkConfig := kafkaClient.NewZookeeperConfig()
	zkConfig.ZookeeperConnect = []string{rawConfig["zookeeper_connect"]}
	zkConfig.ZookeeperTimeout = zkTimeout

	config := kafkaClient.DefaultConsumerConfig()
	config.Groupid = rawConfig["group_id"]
	config.NumWorkers = numWorkers
	config.MaxWorkerRetries = maxWorkerRetries
	config.WorkerBackoff = workerBackoff
	config.WorkerRetryThreshold = int32(workerRetryThreshold)
	config.WorkerThresholdTimeWindow = workerConsideredFailedTimeWindow
	config.WorkerTaskTimeout = workerTaskTimeout
	config.WorkerManagersStopTimeout = workerManagersStopTimeout
	config.BarrierTimeout = rebalanceBarrierTimeout
	config.RebalanceMaxRetries = int32(rebalanceMaxRetries)
	config.RebalanceBackoff = rebalanceBackoff
	config.PartitionAssignmentStrategy = partitionAssignmentStrategy
	config.ExcludeInternalTopics = excludeInternalTopics
	config.NumConsumerFetchers = numConsumerFetchers
	config.FetchBatchSize = fetchBatchSize
	config.FetchMessageMaxBytes = int32(fetchMessageMaxBytes)
	config.FetchMinBytes = int32(fetchMinBytes)
	config.FetchBatchTimeout = fetchBatchTimeout
	config.FetchTopicMetadataRetries = fetchMetadataRetries
	config.FetchTopicMetadataBackoff = fetchMetadataBackoff
	config.RequeueAskNextBackoff = requeueAskNextBackoff
	config.FetchWaitMaxMs = int32(fetchWaitMaxMs)
	config.SocketTimeout = socketTimeout
	config.QueuedMaxMessages = int32(queuedMaxMessages)
	config.RefreshLeaderBackoff = refreshLeaderBackoff
	config.Coordinator = kafkaClient.NewZookeeperCoordinator(zkConfig)
	config.OffsetsStorage = rawConfig["offsets_storage"]
	config.AutoOffsetReset = rawConfig["auto_offset_reset"]
	config.OffsetsCommitMaxRetries = offsetsCommitMaxRetries
	config.DeploymentTimeout = deploymentTimeout

	return config, rawConfig["topic"], numConsumers, rawConfig["graphite_connect"], flushInterval
}

func main() {
	config, topic, numConsumers, graphiteConnect, graphiteFlushInterval := resolveConfig()
	if graphiteConnect != "" {
		startMetrics(graphiteConnect, graphiteFlushInterval)
	}

	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)

	consumers := make([]*kafkaClient.Consumer, numConsumers)
	for i := 0; i < numConsumers; i++ {
		consumers[i] = startNewConsumer(*config, topic)
		time.Sleep(10 * time.Second)
	}

	<-ctrlc
	fmt.Println("Shutdown triggered, closing all alive consumers")
	for _, consumer := range consumers {
		<-consumer.Close()
	}
	fmt.Println("Successfully shut down all consumers")
}

func startMetrics(graphiteConnect string, graphiteFlushInterval time.Duration) {
	addr, err := net.ResolveTCPAddr("tcp", graphiteConnect)
	if err != nil {
		panic(err)
	}
	go metrics.GraphiteWithConfig(metrics.GraphiteConfig{
		Addr:          addr,
		Registry:      metrics.DefaultRegistry,
		FlushInterval: graphiteFlushInterval,
		DurationUnit:  time.Second,
		Prefix:        "metrics",
		Percentiles:   []float64{0.5, 0.75, 0.95, 0.99, 0.999},
	})
}

func startNewConsumer(config kafkaClient.ConsumerConfig, topic string) *kafkaClient.Consumer {
	config.Strategy = GetStrategy(config.Consumerid)
	config.WorkerFailureCallback = FailedCallback
	config.WorkerFailedAttemptCallback = FailedAttemptCallback
	consumer := kafkaClient.NewConsumer(&config)
	topics := map[string]int{topic: config.NumConsumerFetchers}
	go func() {
		consumer.StartStatic(topics)
	}()
	return consumer
}

func GetStrategy(consumerId string) func(*kafkaClient.Worker, *kafkaClient.Message, kafkaClient.TaskId) kafkaClient.WorkerResult {
	consumeRate := metrics.NewRegisteredMeter(fmt.Sprintf("%s-ConsumeRate", consumerId), metrics.DefaultRegistry)
	return func(_ *kafkaClient.Worker, msg *kafkaClient.Message, id kafkaClient.TaskId) kafkaClient.WorkerResult {
		kafkaClient.Infof("main", "Got a message: %s", string(msg.Value))
		consumeRate.Mark(1)

		return kafkaClient.NewSuccessfulResult(id)
	}
}

func FailedCallback(wm *kafkaClient.WorkerManager) kafkaClient.FailedDecision {
	kafkaClient.Info("main", "Failed callback")

	return kafkaClient.DoNotCommitOffsetAndStop
}

func FailedAttemptCallback(task *kafkaClient.Task, result kafkaClient.WorkerResult) kafkaClient.FailedDecision {
	kafkaClient.Info("main", "Failed attempt")

	return kafkaClient.CommitOffsetAndContinue
}
<commit_msg>fixed broken consumers.go. re #22<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"fmt"
	metrics "github.com/rcrowley/go-metrics"
	kafkaClient "github.com/stealthly/go_kafka_client"
	"net"
	"os"
	"os/signal"
	"strconv"
	"time"
)

func resolveConfig() (*kafkaClient.ConsumerConfig, string, int, string, time.Duration) {
	rawConfig, err := kafkaClient.LoadConfiguration("consumers.properties")
	if err != nil {
		panic("Failed to load configuration file")
	}
	numConsumers, _ := strconv.Atoi(rawConfig["num_consumers"])
	zkTimeout, _ := time.ParseDuration(rawConfig["zookeeper_timeout"])

	numWorkers, _ := strconv.Atoi(rawConfig["num_workers"])
	maxWorkerRetries, _ := strconv.Atoi(rawConfig["max_worker_retries"])
	workerBackoff, _ := time.ParseDuration(rawConfig["worker_backoff"])
	workerRetryThreshold, _ := strconv.Atoi(rawConfig["worker_retry_threshold"])
	workerConsideredFailedTimeWindow, _ := time.ParseDuration(rawConfig["worker_considered_failed_time_window"])
	workerTaskTimeout, _ := time.ParseDuration(rawConfig["worker_task_timeout"])
	workerManagersStopTimeout, _ := time.ParseDuration(rawConfig["worker_managers_stop_timeout"])

	rebalanceBarrierTimeout, _ := time.ParseDuration(rawConfig["rebalance_barrier_timeout"])
	rebalanceMaxRetries, _ := strconv.Atoi(rawConfig["rebalance_max_retries"])
	rebalanceBackoff, _ := time.ParseDuration(rawConfig["rebalance_backoff"])
	partitionAssignmentStrategy, _ := rawConfig["partition_assignment_strategy"]
	excludeInternalTopics, _ := strconv.ParseBool(rawConfig["exclude_internal_topics"])

	numConsumerFetchers, _ := strconv.Atoi(rawConfig["num_consumer_fetchers"])
	fetchBatchSize, _ := strconv.Atoi(rawConfig["fetch_batch_size"])
	fetchMessageMaxBytes, _ := strconv.Atoi(rawConfig["fetch_message_max_bytes"])
	fetchMinBytes, _ := strconv.Atoi(rawConfig["fetch_min_bytes"])
	fetchBatchTimeout, _ := time.ParseDuration(rawConfig["fetch_batch_timeout"])
	requeueAskNextBackoff, _ := time.ParseDuration(rawConfig["requeue_ask_next_backoff"])
	fetchWaitMaxMs, _ := strconv.Atoi(rawConfig["fetch_wait_max_ms"])
	socketTimeout, _ := time.ParseDuration(rawConfig["socket_timeout"])
	queuedMaxMessages, _ := strconv.Atoi(rawConfig["queued_max_messages"])
	refreshLeaderBackoff, _ := time.ParseDuration(rawConfig["refresh_leader_backoff"])
	fetchMetadataRetries, _ := strconv.Atoi(rawConfig["fetch_metadata_retries"])
	fetchMetadataBackoff, _ := time.ParseDuration(rawConfig["fetch_metadata_backoff"])

	time.ParseDuration(rawConfig["fetch_metadata_backoff"])

	offsetsCommitMaxRetries, _ := strconv.Atoi(rawConfig["offsets_commit_max_retries"])

	flushInterval, _ := time.ParseDuration(rawConfig["flush_interval"])
	deploymentTimeout, _ := time.ParseDuration(rawConfig["deployment_timeout"])

	zkConfig := kafkaClient.NewZookeeperConfig()
	zkConfig.ZookeeperConnect = []string{rawConfig["zookeeper_connect"]}
	zkConfig.ZookeeperTimeout = zkTimeout

	config := kafkaClient.DefaultConsumerConfig()
	config.Groupid = rawConfig["group_id"]
	config.NumWorkers = numWorkers
	config.MaxWorkerRetries = maxWorkerRetries
	config.WorkerBackoff = workerBackoff
	config.WorkerRetryThreshold = int32(workerRetryThreshold)
	config.WorkerThresholdTimeWindow = workerConsideredFailedTimeWindow
	config.WorkerTaskTimeout = workerTaskTimeout
	config.WorkerManagersStopTimeout = workerManagersStopTimeout
	config.BarrierTimeout = rebalanceBarrierTimeout
	config.RebalanceMaxRetries = int32(rebalanceMaxRetries)
	config.RebalanceBackoff = rebalanceBackoff
	config.PartitionAssignmentStrategy = partitionAssignmentStrategy
	config.ExcludeInternalTopics = excludeInternalTopics
	config.NumConsumerFetchers = numConsumerFetchers
	config.FetchBatchSize = fetchBatchSize
	config.FetchMessageMaxBytes = int32(fetchMessageMaxBytes)
	config.FetchMinBytes = int32(fetchMinBytes)
	config.FetchBatchTimeout = fetchBatchTimeout
	config.FetchTopicMetadataRetries = fetchMetadataRetries
	config.FetchTopicMetadataBackoff = fetchMetadataBackoff
	config.RequeueAskNextBackoff = requeueAskNextBackoff
	config.FetchWaitMaxMs = int32(fetchWaitMaxMs)
	config.SocketTimeout = socketTimeout
	config.QueuedMaxMessages = int32(queuedMaxMessages)
	config.RefreshLeaderBackoff = refreshLeaderBackoff
	config.Coordinator = kafkaClient.NewZookeeperCoordinator(zkConfig)
	config.AutoOffsetReset = rawConfig["auto_offset_reset"]
	config.OffsetsCommitMaxRetries = offsetsCommitMaxRetries
	config.DeploymentTimeout = deploymentTimeout

	return config, rawConfig["topic"], numConsumers, rawConfig["graphite_connect"], flushInterval
}

func main() {
	config, topic, numConsumers, graphiteConnect, graphiteFlushInterval := resolveConfig()
	if graphiteConnect != "" {
		startMetrics(graphiteConnect, graphiteFlushInterval)
	}

	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)

	consumers := make([]*kafkaClient.Consumer, numConsumers)
	for i := 0; i < numConsumers; i++ {
		consumers[i] = startNewConsumer(*config, topic)
		time.Sleep(10 * time.Second)
	}

	<-ctrlc
	fmt.Println("Shutdown triggered, closing all alive consumers")
	for _, consumer := range consumers {
		<-consumer.Close()
	}
	fmt.Println("Successfully shut down all consumers")
}

func startMetrics(graphiteConnect string, graphiteFlushInterval time.Duration) {
	addr, err := net.ResolveTCPAddr("tcp", graphiteConnect)
	if err != nil {
		panic(err)
	}
	go metrics.GraphiteWithConfig(metrics.GraphiteConfig{
		Addr:          addr,
		Registry:      metrics.DefaultRegistry,
		FlushInterval: graphiteFlushInterval,
		DurationUnit:  time.Second,
		Prefix:        "metrics",
		Percentiles:   []float64{0.5, 0.75, 0.95, 0.99, 0.999},
	})
}

func startNewConsumer(config kafkaClient.ConsumerConfig, topic string) *kafkaClient.Consumer {
	config.Strategy = GetStrategy(config.Consumerid)
	config.WorkerFailureCallback = FailedCallback
	config.WorkerFailedAttemptCallback = FailedAttemptCallback
	consumer := kafkaClient.NewConsumer(&config)
	topics := map[string]int{topic: config.NumConsumerFetchers}
	go func() {
		consumer.StartStatic(topics)
	}()
	return consumer
}

func GetStrategy(consumerId string) func(*kafkaClient.Worker, *kafkaClient.Message, kafkaClient.TaskId) kafkaClient.WorkerResult {
	consumeRate := metrics.NewRegisteredMeter(fmt.Sprintf("%s-ConsumeRate", consumerId), metrics.DefaultRegistry)
	return func(_ *kafkaClient.Worker, msg *kafkaClient.Message, id kafkaClient.TaskId) kafkaClient.WorkerResult {
		kafkaClient.Infof("main", "Got a message: %s", string(msg.Value))
		consumeRate.Mark(1)

		return kafkaClient.NewSuccessfulResult(id)
	}
}

func FailedCallback(wm *kafkaClient.WorkerManager) kafkaClient.FailedDecision {
	kafkaClient.Info("main", "Failed callback")

	return kafkaClient.DoNotCommitOffsetAndStop
}

func FailedAttemptCallback(task *kafkaClient.Task, result kafkaClient.WorkerResult) kafkaClient.FailedDecision {
	kafkaClient.Info("main", "Failed attempt")

	return kafkaClient.CommitOffsetAndContinue
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"

	"github.com/hashicorp/terraform/helper/schema"
)

// See https://github.com/fog/fog-aws/pull/332/files
// This list isn't exposed by AWS - it's been found through
// trouble solving
var elbHostedZoneIdPerRegionMap = map[string]string{
	"ap-northeast-1": "Z14GRHDCWA56QT",
	"ap-northeast-2": "ZWKZPGTI48KDX",
	"ap-south-1":     "ZP97RAFLXTNZK",
	"ap-southeast-1": "Z1LMS91P8CMLE5",
	"ap-southeast-2": "Z1GM3OXH4ZPM65",
	"ca-central-1":   "ZQSVJUPU6J1EY",
	"eu-central-1":   "Z215JYRZR1TBD5",
	"eu-west-1":      "Z32O12XQLNTSW2",
	"eu-west-2":      "ZHURV8PSTC4K8",
	"us-east-1":      "Z35SXDOTRQ7X7K",
	"us-east-2":      "Z3AADJGX6KTTL2",
	"us-west-1":      "Z368ELLRRE2KJ0",
	"us-west-2":      "Z1H1FL5HABSF5",
	"sa-east-1":      "Z2P70J7HTTTPLU",
}

func dataSourceAwsElbHostedZoneId() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsElbHostedZoneIdRead,

		Schema: map[string]*schema.Schema{
			"region": {
				Type:     schema.TypeString,
				Optional: true,
			},
		},
	}
}

func dataSourceAwsElbHostedZoneIdRead(d *schema.ResourceData, meta interface{}) error {
	region := meta.(*AWSClient).region
	if v, ok := d.GetOk("region"); ok {
		region = v.(string)
	}

	if zoneId, ok := elbHostedZoneIdPerRegionMap[region]; ok {
		d.SetId(zoneId)
		return nil
	}

	return fmt.Errorf("Unknown region (%q)", region)
}
<commit_msg>provider/aws: Add gov and cn hosted zone Ids to aws_elb_hosted_zone data source (#15149)<commit_after>package aws

import (
	"fmt"

	"github.com/hashicorp/terraform/helper/schema"
)

// See https://github.com/fog/fog-aws/pull/332/files
// This list isn't exposed by AWS - it's been found through
// trouble solving
var elbHostedZoneIdPerRegionMap = map[string]string{
	"ap-northeast-1": "Z14GRHDCWA56QT",
	"ap-northeast-2": "ZWKZPGTI48KDX",
	"ap-south-1":     "ZP97RAFLXTNZK",
	"ap-southeast-1": "Z1LMS91P8CMLE5",
	"ap-southeast-2": "Z1GM3OXH4ZPM65",
	"ca-central-1":   "ZQSVJUPU6J1EY",
	"eu-central-1":   "Z215JYRZR1TBD5",
	"eu-west-1":      "Z32O12XQLNTSW2",
	"eu-west-2":      "ZHURV8PSTC4K8",
	"us-east-1":      "Z35SXDOTRQ7X7K",
	"us-east-2":      "Z3AADJGX6KTTL2",
	"us-west-1":      "Z368ELLRRE2KJ0",
	"us-west-2":      "Z1H1FL5HABSF5",
	"sa-east-1":      "Z2P70J7HTTTPLU",
	"us-gov-west-1":  "048591011584",
	"cn-north-1":     "638102146993",
}

func dataSourceAwsElbHostedZoneId() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsElbHostedZoneIdRead,

		Schema: map[string]*schema.Schema{
			"region": {
				Type:     schema.TypeString,
				Optional: true,
			},
		},
	}
}

func dataSourceAwsElbHostedZoneIdRead(d *schema.ResourceData, meta interface{}) error {
	region := meta.(*AWSClient).region
	if v, ok := d.GetOk("region"); ok {
		region = v.(string)
	}

	if zoneId, ok := elbHostedZoneIdPerRegionMap[region]; ok {
		d.SetId(zoneId)
		return nil
	}

	return fmt.Errorf("Unknown region (%q)", region)
}
<|endoftext|>


<commit_before>package azurerm

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
)

func TestAccAzureRMResourceGroup_importBasic(t *testing.T) {
	resourceName := "azurerm_resource_group.test"

	ri := acctest.RandInt()
	config := fmt.Sprintf(testAccAzureRMResourceGroup_basic, ri)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testCheckAzureRMResourceGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: config,
			},

			resource.TestStep{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
				//ImportStateVerifyIgnore: []string{"resource_group_name"},
				//this isn't returned from the API!
			},
		},
	})
}
<commit_msg>Update import_arm_resource_group_test.go<commit_after>package azurerm

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
)

func TestAccAzureRMResourceGroup_importBasic(t *testing.T) {
	resourceName := "azurerm_resource_group.test"

	ri := acctest.RandInt()
	config := fmt.Sprintf(testAccAzureRMResourceGroup_basic, ri)

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testCheckAzureRMResourceGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: config,
			},

			resource.TestStep{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}
<|endoftext|>


<commit_before>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package store

import (
	"container/heap"
)

// An TTLKeyHeap is a min-heap of TTLKeys order by expiration time
type ttlKeyHeap struct {
	array  []*node
	keyMap map[*node]int
}

func newTtlKeyHeap() *ttlKeyHeap {
	h := &ttlKeyHeap{keyMap: make(map[*node]int)}
	heap.Init(h)
	return h
}

func (h ttlKeyHeap) Len() int {
	return len(h.array)
}

func (h ttlKeyHeap) Less(i, j int) bool {
	return h.array[i].ExpireTime.Before(h.array[j].ExpireTime)
}

func (h ttlKeyHeap) Swap(i, j int) {
	// swap node
	h.array[i], h.array[j] = h.array[j], h.array[i]

	// update map
	h.keyMap[h.array[i]] = i
	h.keyMap[h.array[j]] = j
}

func (h *ttlKeyHeap) Push(x interface{}) {
	n, _ := x.(*node)
	h.keyMap[n] = len(h.array)
	h.array = append(h.array, n)
}

func (h *ttlKeyHeap) Pop() interface{} {
	old := h.array
	n := len(old)
	x := old[n-1]
	h.array = old[0 : n-1]
	delete(h.keyMap, x)
	return x
}

func (h *ttlKeyHeap) top() *node {
	if h.Len() != 0 {
		return h.array[0]
	}
	return nil
}

func (h *ttlKeyHeap) pop() *node {
	x := heap.Pop(h)
	n, _ := x.(*node)
	return n
}

func (h *ttlKeyHeap) push(x interface{}) {
	heap.Push(h, x)
}

func (h *ttlKeyHeap) update(n *node) {
	index, ok := h.keyMap[n]
	if ok {
		heap.Remove(h, index)
		heap.Push(h, n)
	}
}

func (h *ttlKeyHeap) remove(n *node) {
	index, ok := h.keyMap[n]
	if ok {
		heap.Remove(h, index)
	}
}
<commit_msg>store: optimize ttlKeyHeap GC<commit_after>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package store

import (
	"container/heap"
)

// An TTLKeyHeap is a min-heap of TTLKeys order by expiration time
type ttlKeyHeap struct {
	array  []*node
	keyMap map[*node]int
}

func newTtlKeyHeap() *ttlKeyHeap {
	h := &ttlKeyHeap{keyMap: make(map[*node]int)}
	heap.Init(h)
	return h
}

func (h ttlKeyHeap) Len() int {
	return len(h.array)
}

func (h ttlKeyHeap) Less(i, j int) bool {
	return h.array[i].ExpireTime.Before(h.array[j].ExpireTime)
}

func (h ttlKeyHeap) Swap(i, j int) {
	// swap node
	h.array[i], h.array[j] = h.array[j], h.array[i]

	// update map
	h.keyMap[h.array[i]] = i
	h.keyMap[h.array[j]] = j
}

func (h *ttlKeyHeap) Push(x interface{}) {
	n, _ := x.(*node)
	h.keyMap[n] = len(h.array)
	h.array = append(h.array, n)
}

func (h *ttlKeyHeap) Pop() interface{} {
	old := h.array
	n := len(old)
	x := old[n-1]
	// Set slice element to nil, so GC can recycle the node.
	// This is due to golang GC doesn't support partial recycling:
	// https://github.com/golang/go/issues/9618
	old[n-1] = nil
	h.array = old[0 : n-1]
	delete(h.keyMap, x)
	return x
}

func (h *ttlKeyHeap) top() *node {
	if h.Len() != 0 {
		return h.array[0]
	}
	return nil
}

func (h *ttlKeyHeap) pop() *node {
	x := heap.Pop(h)
	n, _ := x.(*node)
	return n
}

func (h *ttlKeyHeap) push(x interface{}) {
	heap.Push(h, x)
}

func (h *ttlKeyHeap) update(n *node) {
	index, ok := h.keyMap[n]
	if ok {
		heap.Remove(h, index)
		heap.Push(h, n)
	}
}

func (h *ttlKeyHeap) remove(n *node) {
	index, ok := h.keyMap[n]
	if ok {
		heap.Remove(h, index)
	}
}
<|endoftext|>


<commit_before>package structs

import (
	"bytes"
	"encoding/xml"
	"io/ioutil"

	"github.com/prometheus/common/log"
)

// VolumeInfoXML struct repesents cliOutput element of "gluster volume info" command
type VolumeInfoXML struct {
	XMLName  xml.Name `xml:"cliOutput"`
	OpRet    int      `xml:"opRet"`
	OpErrno  int      `xml:"opErrno"`
	OpErrstr string   `xml:"opErrstr"`
	VolInfo  VolInfo  `xml:"volInfo"`
}

// VolInfo element of "gluster volume info" command
type VolInfo struct {
	XMLName xml.Name `xml:"volInfo"`
	Volumes Volumes  `xml:"volumes"`
}

// Volumes element of "gluster volume info" command
type Volumes struct {
	XMLName xml.Name `xml:"volumes"`
	Volume  []Volume `xml:"volume"`
	Count   int      `xml:"count"`
}

// Volume element of "gluster volume info" command
type Volume struct {
	XMLName    xml.Name `xml:"volume"`
	Name       string   `xml:"name"`
	ID         string   `xml:"id"`
	Status     int      `xml:"status"`
	StatusStr  string   `xml:"statusStr"`
	BrickCount int      `xml:"brickCount"`
	Bricks     []Brick  `xml:"bricks"`
	DistCount  int      `xml:"distCount"`
}

// Brick element of "gluster volume info" command
type Brick struct {
	UUID      string `xml:"brick>uuid"`
	Name      string `xml:"brick>name"`
	HostUUID  string `xml:"brick>hostUuid"`
	IsArbiter int    `xml:"brick>isArbiter"`
}

// VolumeListXML struct repesents cliOutput element of "gluster volume list" command
type VolumeListXML struct {
	XMLName  xml.Name `xml:"cliOutput"`
	OpRet    int      `xml:"opRet"`
	OpErrno  int      `xml:"opErrno"`
	OpErrstr string   `xml:"opErrstr"`
	VolList  VolList  `xml:"volList"`
}

// VolList element of "gluster volume list" command
type VolList struct {
	Count  int      `xml:"count"`
	Volume []string `xml:"volume"`
}

// PeerStatusXML struct represents cliOutput element of "gluster peer status" command
type PeerStatusXML struct {
	XMLName    xml.Name   `xml:"cliOutput"`
	OpRet      int        `xml:"opRet"`
	OpErrno    int        `xml:"opErrno"`
	OpErrstr   string     `xml:"opErrstr"`
	PeerStatus PeerStatus `xml:"peerStatus"`
}

// PeerStatus element of "gluster peer status" command
type PeerStatus struct {
	XMLName xml.Name `xml:"peerStatus"`
	Peer    []Peer   `xml:"peer"`
}

// Peer element of "gluster peer status" command
type Peer struct {
	XMLName   xml.Name  `xml:"peer"`
	UUID      string    `xml:"uuid"`
	Hostname  string    `xml:"hostname"`
	Hostnames Hostnames `xml:"hostnames"`
	Connected int       `xml:"connected"`
	State     int       `xml:"state"`
	StateStr  string    `xml:"stateStr"`
}

// Hostnames element of "gluster peer status" command
type Hostnames struct {
	Hostname string `xml:"hostname"`
}

// VolumeProfileXML struct repesents cliOutput element of "gluster volume {volume} profile" command
type VolumeProfileXML struct {
	XMLName    xml.Name   `xml:"cliOutput"`
	OpRet      int        `xml:"opRet"`
	OpErrno    int        `xml:"opErrno"`
	OpErrstr   string     `xml:"opErrstr"`
	VolProfile VolProfile `xml:"volProfile"`
}

// VolProfile element of "gluster volume {volume} profile" command
type VolProfile struct {
	Volname    string         `xml:"volname"`
	BrickCount int            `xml:"brickCount"`
	Brick      []BrickProfile `xml:"brick"`
}

// BrickProfile struct for element brick of "gluster volume {volume} profile" command
type BrickProfile struct {
	//XMLName xml.Name `xml:"brick"`
	BrickName       string          `xml:"brickName"`
	CumulativeStats CumulativeStats `xml:"cumulativeStats"`
}

// CumulativeStats element of "gluster volume {volume} profile" command
type CumulativeStats struct {
	FopStats   FopStats `xml:"fopStats"`
	Duration   int      `xml:"duration"`
	TotalRead  int      `xml:"totalRead"`
	TotalWrite int      `xml:"totalWrite"`
}

// FopStats element of "gluster volume {volume} profile" command
type FopStats struct {
	Fop []Fop `xml:"fop"`
}

// Fop is struct for FopStats
type Fop struct {
	Name       string  `xml:"name"`
	Hits       int     `xml:"hits"`
	AvgLatency float64 `xml:"avgLatency"`
	MinLatency float64 `xml:"minLatency"`
	MaxLatency float64 `xml:"maxLatency"`
}

// HealInfoBrick is a struct of HealInfoBricks
type HealInfoBrick struct {
	XMLName         xml.Name `xml:"brick"`
	Name            string   `xml:"name"`
	Status          string   `xml:"status"`
	NumberOfEntries string   `xml:"numberOfEntries"`
}

// HealInfoBricks is a struct of HealInfo
type HealInfoBricks struct {
	XMLName xml.Name        `xml:"bricks"`
	Brick   []HealInfoBrick `xml:"brick"`
}

// HealInfo is a struct of VolumenHealInfoXML
type HealInfo struct {
	XMLName xml.Name       `xml:"healInfo"`
	Bricks  HealInfoBricks `xml:"bricks"`
}

// VolumeHealInfoXML struct repesents cliOutput element of "gluster volume {volume} heal info" command
type VolumeHealInfoXML struct {
	XMLName  xml.Name `xml:"cliOutput"`
	OpRet    int      `xml:"opRet"`
	OpErrno  int      `xml:"opErrno"`
	OpErrstr string   `xml:"opErrstr"`
	HealInfo HealInfo `xml:"healInfo"`
}

// VolumeHealInfoXMLUnmarshall unmarshalls heal info of gluster cluster
func VolumeHealInfoXMLUnmarshall(cmdOutBuff *bytes.Buffer) (VolumeHealInfoXML, error) {
	var vol VolumeHealInfoXML
	b, err := ioutil.ReadAll(cmdOutBuff)
	if err != nil {
		log.Error(err)
		return vol, err
	}
	err = xml.Unmarshal(b, &vol)
	if err != nil {
		log.Error(err)
	}
	return vol, nil
}

// VolumeListXMLUnmarshall unmarshalls bytes to VolumeListXML struct
func VolumeListXMLUnmarshall(cmdOutBuff *bytes.Buffer) (VolumeListXML, error) {
	var vol VolumeListXML
	b, err := ioutil.ReadAll(cmdOutBuff)
	if err != nil {
		log.Error(err)
		return vol, err
	}
	xml.Unmarshal(b, &vol)
	return vol, nil
}

// VolumeInfoXMLUnmarshall unmarshalls bytes to VolumeInfoXML struct
func VolumeInfoXMLUnmarshall(cmdOutBuff *bytes.Buffer) (VolumeInfoXML, error) {
	var vol VolumeInfoXML
	b, err := ioutil.ReadAll(cmdOutBuff)
	if err != nil {
		log.Error(err)
		return vol, err
	}
	xml.Unmarshal(b, &vol)
	return vol, nil
}

// PeerStatusXMLUnmarshall unmarshalls bytes to PeerStatusXML struct
func PeerStatusXMLUnmarshall(cmdOutBuff *bytes.Buffer) (PeerStatusXML, error) {
	var vol PeerStatusXML
	b, err := ioutil.ReadAll(cmdOutBuff)
	if err != nil {
		log.Error(err)
		return vol, err
	}
	xml.Unmarshal(b, &vol)
	return vol, nil
}

// VolumeProfileGvInfoCumulativeXMLUnmarshall unmarshalls cumulative profile of gluster volume profile
func VolumeProfileGvInfoCumulativeXMLUnmarshall(cmdOutBuff *bytes.Buffer) (VolumeProfileXML, error) {
	var vol VolumeProfileXML
	b, err := ioutil.ReadAll(cmdOutBuff)
	if err != nil {
		log.Error(err)
		return vol, err
	}
	xml.Unmarshal(b, &vol)
	return vol, nil
}

// VolumeStatusXML XML type of "gluster volume status"
type VolumeStatusXML struct {
	XMLName   xml.Name `xml:"cliOutput"`
	OpRet     int      `xml:"opRet"`
	OpErrno   int      `xml:"opErrno"`
	OpErrstr  string   `xml:"opErrstr"`
	VolStatus struct {
		Volumes struct {
			Volume []struct {
				VolName   string `xml:"volName"`
				NodeCount int    `xml:"nodeCount"`
				Node      []struct {
					Hostname string `xml:"hostname"`
					Path     string `xml:"path"`
					PeerID   string `xml:"peerid"`
					Status   int    `xml:"status"`
					Port     int    `xml:"port"`
					Ports    struct {
						TCP  int    `xml:"tcp"`
						RDMA string `xml:"rdma"`
					} `xml:"ports"`
					Pid        int    `xml:"pid"`
					SizeTotal  uint64 `xml:"sizeTotal"`
					SizeFree   uint64 `xml:"sizeFree"`
					Device     string `xml:"device"`
					BlockSize  int    `xml:"blockSize"`
					MntOptions string `xml:"mntOptions"`
					FsName     string `xml:"fsName"`
				} `xml:"node"`
			} `xml:"volume"`
		} `xml:"volumes"`
	} `xml:"volStatus"`
}

// VolumeStatusAllDetailXMLUnmarshall reads bytes.buffer and returns unmarshalled xml
func VolumeStatusAllDetailXMLUnmarshall(cmdOutBuff *bytes.Buffer) (VolumeStatusXML, error) {
	var vol VolumeStatusXML
	b, err := ioutil.ReadAll(cmdOutBuff)
	if err != nil {
		log.Error(err)
		return vol, err
	}
	xml.Unmarshal(b, &vol)
	return vol, nil
}

// QuotaLimit is a struct of VolQuota
type QuotaLimit struct {
	XMLName        xml.Name `xml:"limit"`
	Path           string   `xml:"path"`
	HardLimit      uint64   `xml:"hard_limit"`
	SoftLimitValue uint64   `xml:"soft_limit_value"`
	UsedSpace      uint64   `xml:"used_space"`
	AvailSpace     uint64   `xml:"avail_space"`
	SlExceeded     string   `xml:"sl_exceeded"`
	HlExceeded     string   `xml:"hl_exceeded"`
}

// VolQuota is a struct of VolumeQuotaXML
type VolQuota struct {
	XMLName     xml.Name     `xml:"volQuota"`
	QuotaLimits []QuotaLimit `xml:"limit"`
}

// VolumeQuotaXML XML type of "gluster volume quota list"
type VolumeQuotaXML struct {
	XMLName  xml.Name `xml:"cliOutput"`
	OpRet    int      `xml:"opRet"`
	OpErrno  int      `xml:"opErrno"`
	OpErrstr string   `xml:"opErrstr"`
	VolQuota VolQuota `xml:"volQuota"`
}

// VolumeQuotaListXMLUnmarshall fuction parse "gluster volume quota list" XML output
func VolumeQuotaListXMLUnmarshall(cmdOutBuff *bytes.Buffer) (VolumeQuotaXML, error) {
	var volQuotaXML VolumeQuotaXML
	b, err := ioutil.ReadAll(cmdOutBuff)
	if err != nil {
		log.Error(err)
		return volQuotaXML, err
	}
	xml.Unmarshal(b, &volQuotaXML)
	return volQuotaXML, nil
}
<commit_msg>Fix interfacer suggestions<commit_after>package structs

import (
	"encoding/xml"
	"io"
	"io/ioutil"

	"github.com/prometheus/common/log"
)

// VolumeInfoXML struct repesents cliOutput element of "gluster volume info" command
type VolumeInfoXML struct {
	XMLName  xml.Name `xml:"cliOutput"`
	OpRet    int      `xml:"opRet"`
	OpErrno  int      `xml:"opErrno"`
	OpErrstr string   `xml:"opErrstr"`
	VolInfo  VolInfo  `xml:"volInfo"`
}

// VolInfo element of "gluster volume info" command
type VolInfo struct {
	XMLName xml.Name `xml:"volInfo"`
	Volumes Volumes  `xml:"volumes"`
}

// Volumes element of "gluster volume info" command
type Volumes struct {
	XMLName xml.Name `xml:"volumes"`
	Volume  []Volume `xml:"volume"`
	Count   int      `xml:"count"`
}

// Volume element of "gluster volume info" command
type Volume struct {
	XMLName    xml.Name `xml:"volume"`
	Name       string   `xml:"name"`
	ID         string   `xml:"id"`
	Status     int      `xml:"status"`
	StatusStr  string   `xml:"statusStr"`
	BrickCount int      `xml:"brickCount"`
	Bricks     []Brick  `xml:"bricks"`
	DistCount  int      `xml:"distCount"`
}

// Brick element of "gluster volume info" command
type Brick struct {
	UUID      string `xml:"brick>uuid"`
	Name      string `xml:"brick>name"`
	HostUUID  string `xml:"brick>hostUuid"`
	IsArbiter int    `xml:"brick>isArbiter"`
}

// VolumeListXML struct repesents cliOutput element of "gluster volume list" command
type VolumeListXML struct {
	XMLName  xml.Name `xml:"cliOutput"`
	OpRet    int      `xml:"opRet"`
	OpErrno  int      `xml:"opErrno"`
	OpErrstr string   `xml:"opErrstr"`
	VolList  VolList  `xml:"volList"`
}

// VolList element of "gluster volume list" command
type VolList struct {
	Count  int      `xml:"count"`
	Volume []string `xml:"volume"`
}

// PeerStatusXML struct represents cliOutput element of "gluster peer status" command
type PeerStatusXML struct {
	XMLName    xml.Name   `xml:"cliOutput"`
	OpRet      int        `xml:"opRet"`
	OpErrno    int        `xml:"opErrno"`
	OpErrstr   string     `xml:"opErrstr"`
	PeerStatus PeerStatus `xml:"peerStatus"`
}

// PeerStatus element of "gluster peer status" command
type PeerStatus struct {
	XMLName xml.Name `xml:"peerStatus"`
	Peer    []Peer   `xml:"peer"`
}

// Peer element of "gluster peer status" command
type Peer struct {
	XMLName   xml.Name  `xml:"peer"`
	UUID      string    `xml:"uuid"`
	Hostname  string    `xml:"hostname"`
	Hostnames Hostnames `xml:"hostnames"`
	Connected int       `xml:"connected"`
	State     int       `xml:"state"`
	StateStr  string    `xml:"stateStr"`
}

// Hostnames element of "gluster peer status" command
type Hostnames struct {
	Hostname string `xml:"hostname"`
}

// VolumeProfileXML struct repesents cliOutput element of "gluster volume {volume} profile" command
type VolumeProfileXML struct {
	XMLName    xml.Name   `xml:"cliOutput"`
	OpRet      int        `xml:"opRet"`
	OpErrno    int        `xml:"opErrno"`
	OpErrstr   string     `xml:"opErrstr"`
	VolProfile VolProfile `xml:"volProfile"`
}

// VolProfile element of "gluster volume {volume} profile" command
type VolProfile struct {
	Volname    string         `xml:"volname"`
	BrickCount int            `xml:"brickCount"`
	Brick      []BrickProfile `xml:"brick"`
}

// BrickProfile struct for element brick of "gluster volume {volume} profile" command
type BrickProfile struct {
	//XMLName xml.Name `xml:"brick"`
	BrickName       string          `xml:"brickName"`
	CumulativeStats CumulativeStats `xml:"cumulativeStats"`
}

// CumulativeStats element of "gluster volume {volume} profile" command
type CumulativeStats struct {
	FopStats   FopStats `xml:"fopStats"`
	Duration   int      `xml:"duration"`
	TotalRead  int      `xml:"totalRead"`
	TotalWrite int      `xml:"totalWrite"`
}

// FopStats element of "gluster volume {volume} profile" command
type FopStats struct {
	Fop []Fop `xml:"fop"`
}

// Fop is struct for FopStats
type Fop struct {
	Name       string  `xml:"name"`
	Hits       int     `xml:"hits"`
	AvgLatency float64 `xml:"avgLatency"`
	MinLatency float64 `xml:"minLatency"`
	MaxLatency float64 `xml:"maxLatency"`
}

// HealInfoBrick is a struct of HealInfoBricks
type HealInfoBrick struct {
	XMLName         xml.Name `xml:"brick"`
	Name            string   `xml:"name"`
	Status          string   `xml:"status"`
	NumberOfEntries string   `xml:"numberOfEntries"`
}

// HealInfoBricks is a struct of HealInfo
type HealInfoBricks struct {
	XMLName xml.Name        `xml:"bricks"`
	Brick   []HealInfoBrick `xml:"brick"`
}

// HealInfo is a struct of VolumenHealInfoXML
type HealInfo struct {
	XMLName xml.Name       `xml:"healInfo"`
	Bricks  HealInfoBricks `xml:"bricks"`
}

// VolumeHealInfoXML struct repesents cliOutput element of "gluster volume {volume} heal info" command
type VolumeHealInfoXML struct {
	XMLName  xml.Name `xml:"cliOutput"`
	OpRet    int      `xml:"opRet"`
	OpErrno  int      `xml:"opErrno"`
	OpErrstr string   `xml:"opErrstr"`
	HealInfo HealInfo `xml:"healInfo"`
}

// VolumeHealInfoXMLUnmarshall unmarshalls heal info of gluster cluster
func VolumeHealInfoXMLUnmarshall(cmdOutBuff io.Reader) (VolumeHealInfoXML, error) {
	var vol VolumeHealInfoXML
	b, err := ioutil.ReadAll(cmdOutBuff)
	if err != nil {
		log.Error(err)
		return vol, err
	}
	err = xml.Unmarshal(b, &vol)
	if err != nil {
		log.Error(err)
	}
	return vol, nil
}

// VolumeListXMLUnmarshall unmarshalls bytes to VolumeListXML struct
func VolumeListXMLUnmarshall(cmdOutBuff io.Reader) (VolumeListXML, error) {
	var vol VolumeListXML
	b, err := ioutil.ReadAll(cmdOutBuff)
	if err != nil {
		log.Error(err)
		return vol, err
	}
	xml.Unmarshal(b, &vol)
	return vol, nil
}

// VolumeInfoXMLUnmarshall unmarshalls bytes to VolumeInfoXML struct
func VolumeInfoXMLUnmarshall(cmdOutBuff io.Reader) (VolumeInfoXML, error) {
	var vol VolumeInfoXML
	b, err := ioutil.ReadAll(cmdOutBuff)
	if err != nil {
		log.Error(err)
		return vol, err
	}
	xml.Unmarshal(b, &vol)
	return vol, nil
}

// PeerStatusXMLUnmarshall unmarshalls bytes to PeerStatusXML struct
func PeerStatusXMLUnmarshall(cmdOutBuff io.Reader) (PeerStatusXML, error) {
	var vol PeerStatusXML
	b, err := ioutil.ReadAll(cmdOutBuff)
	if err != nil {
		log.Error(err)
		return vol, err
	}
	xml.Unmarshal(b, &vol)
	return vol, nil
}

// VolumeProfileGvInfoCumulativeXMLUnmarshall unmarshalls cumulative profile of gluster volume profile
func VolumeProfileGvInfoCumulativeXMLUnmarshall(cmdOutBuff io.Reader) (VolumeProfileXML, error) {
	var vol VolumeProfileXML
	b, err := ioutil.ReadAll(cmdOutBuff)
	if err != nil {
		log.Error(err)
		return vol, err
	}
	xml.Unmarshal(b, &vol)
	return vol, nil
}

// VolumeStatusXML XML type of "gluster volume status"
type VolumeStatusXML struct {
	XMLName   xml.Name `xml:"cliOutput"`
	OpRet     int      `xml:"opRet"`
	OpErrno   int      `xml:"opErrno"`
	OpErrstr  string   `xml:"opErrstr"`
	VolStatus struct {
		Volumes struct {
			Volume []struct {
				VolName   string `xml:"volName"`
				NodeCount int    `xml:"nodeCount"`
				Node      []struct {
					Hostname string `xml:"hostname"`
					Path     string `xml:"path"`
					PeerID   string `xml:"peerid"`
					Status   int    `xml:"status"`
					Port     int    `xml:"port"`
					Ports    struct {
						TCP  int    `xml:"tcp"`
						RDMA string `xml:"rdma"`
					} `xml:"ports"`
					Pid        int    `xml:"pid"`
					SizeTotal  uint64 `xml:"sizeTotal"`
					SizeFree   uint64 `xml:"sizeFree"`
					Device     string `xml:"device"`
					BlockSize  int    `xml:"blockSize"`
					MntOptions string `xml:"mntOptions"`
					FsName     string `xml:"fsName"`
				} `xml:"node"`
			} `xml:"volume"`
		} `xml:"volumes"`
	} `xml:"volStatus"`
}

// VolumeStatusAllDetailXMLUnmarshall reads bytes.buffer and returns unmarshalled xml
func VolumeStatusAllDetailXMLUnmarshall(cmdOutBuff io.Reader) (VolumeStatusXML, error) {
	var vol VolumeStatusXML
	b, err := ioutil.ReadAll(cmdOutBuff)
	if err != nil {
		log.Error(err)
		return vol, err
	}
	xml.Unmarshal(b, &vol)
	return vol, nil
}

// QuotaLimit is a struct of VolQuota
type QuotaLimit struct {
	XMLName        xml.Name `xml:"limit"`
	Path           string   `xml:"path"`
	HardLimit      uint64   `xml:"hard_limit"`
	SoftLimitValue uint64   `xml:"soft_limit_value"`
	UsedSpace      uint64   `xml:"used_space"`
	AvailSpace     uint64   `xml:"avail_space"`
	SlExceeded     string   `xml:"sl_exceeded"`
	HlExceeded     string   `xml:"hl_exceeded"`
}

// VolQuota is a struct of VolumeQuotaXML
type VolQuota struct {
	XMLName     xml.Name     `xml:"volQuota"`
	QuotaLimits []QuotaLimit `xml:"limit"`
}

// VolumeQuotaXML XML type of "gluster volume quota list"
type VolumeQuotaXML struct {
	XMLName  xml.Name `xml:"cliOutput"`
	OpRet    int      `xml:"opRet"`
	OpErrno  int      `xml:"opErrno"`
	OpErrstr string   `xml:"opErrstr"`
	VolQuota VolQuota `xml:"volQuota"`
}

// VolumeQuotaListXMLUnmarshall fuction parse "gluster volume quota list" XML output
func VolumeQuotaListXMLUnmarshall(cmdOutBuff io.Reader) (VolumeQuotaXML, error) {
	var volQuotaXML VolumeQuotaXML
	b, err := ioutil.ReadAll(cmdOutBuff)
	if err != nil {
		log.Error(err)
		return volQuotaXML, err
	}
	xml.Unmarshal(b, &volQuotaXML)
	return volQuotaXML, nil
}
<|endoftext|>


<commit_before>package azurerm

import (
	"fmt"
	"log"
	"net/http"

	"github.com/Azure/azure-sdk-for-go/arm/network"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceArmVirtualNetwork() *schema.Resource {
	return &schema.Resource{
		Create: resourceArmVirtualNetworkCreate,
		Read:   resourceArmVirtualNetworkRead,
		Update: resourceArmVirtualNetworkCreate,
		Delete: resourceArmVirtualNetworkDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"address_space": {
				Type:     schema.TypeList,
				Required: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"dns_servers": {
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"subnet": {
				Type:     schema.TypeSet,
				Optional: true,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"name": {
							Type:     schema.TypeString,
							Required: true,
						},
						"address_prefix": {
							Type:     schema.TypeString,
							Required: true,
						},
						"security_group": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
				Set: resourceAzureSubnetHash,
			},

			"location": locationSchema(),

			"resource_group_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"tags": tagsSchema(),
		},
	}
}

func resourceArmVirtualNetworkCreate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*ArmClient)
	vnetClient := client.vnetClient

	log.Printf("[INFO] preparing arguments for Azure ARM virtual network creation.")

	name := d.Get("name").(string)
	location := d.Get("location").(string)
	resGroup := d.Get("resource_group_name").(string)
	tags := d.Get("tags").(map[string]interface{})

	vnet := network.VirtualNetwork{
		Name:                           &name,
		Location:                       &location,
		VirtualNetworkPropertiesFormat: getVirtualNetworkProperties(d),
		Tags: expandTags(tags),
	}

	networkSecurityGroupNames := make([]string, 0)
	for _, subnet := range *vnet.VirtualNetworkPropertiesFormat.Subnets {
		if subnet.NetworkSecurityGroup != nil {
			nsgName, err := parseNetworkSecurityGroupName(*subnet.NetworkSecurityGroup.ID)
			if err != nil {
				return err
			}

			networkSecurityGroupNames = append(networkSecurityGroupNames, nsgName)
		}
	}

	azureRMVirtualNetworkLockNetworkSecurityGroups(&networkSecurityGroupNames)
	defer azureRMVirtualNetworkUnlockNetworkSecurityGroups(&networkSecurityGroupNames)

	_, err := vnetClient.CreateOrUpdate(resGroup, name, vnet, make(chan struct{}))
	if err != nil {
		return err
	}

	read, err := vnetClient.Get(resGroup, name, "")
	if err != nil {
		return err
	}
	if read.ID == nil {
		return fmt.Errorf("Cannot read Virtual Network %s (resource group %s) ID", name, resGroup)
	}

	d.SetId(*read.ID)

	return resourceArmVirtualNetworkRead(d, meta)
}

func resourceArmVirtualNetworkRead(d *schema.ResourceData, meta interface{}) error {
	vnetClient := meta.(*ArmClient).vnetClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	resGroup := id.ResourceGroup
	name := id.Path["virtualNetworks"]

	resp, err := vnetClient.Get(resGroup, name, "")
	if err != nil {
		if resp.StatusCode == http.StatusNotFound {
			d.SetId("")
			return nil
		}
		return fmt.Errorf("Error making Read request on Azure virtual network %s: %s", name, err)
	}

	vnet := *resp.VirtualNetworkPropertiesFormat

	// update appropriate values
	d.Set("resource_group_name", resGroup)
	d.Set("name", resp.Name)
	d.Set("location", resp.Location)
	d.Set("address_space", vnet.AddressSpace.AddressPrefixes)

	subnets := &schema.Set{
		F: resourceAzureSubnetHash,
	}

	for _, subnet := range *vnet.Subnets {
		s := map[string]interface{}{}

		s["name"] = *subnet.Name
		s["address_prefix"] = *subnet.SubnetPropertiesFormat.AddressPrefix
		if subnet.SubnetPropertiesFormat.NetworkSecurityGroup != nil {
			s["security_group"] = *subnet.SubnetPropertiesFormat.NetworkSecurityGroup.ID
		}

		subnets.Add(s)
	}
	d.Set("subnet", subnets)

	if vnet.DhcpOptions != nil && vnet.DhcpOptions.DNSServers != nil {
		dnses := []string{}
		for _, dns := range *vnet.DhcpOptions.DNSServers {
			dnses = append(dnses, dns)
		}
		d.Set("dns_servers", dnses)
	}

	flattenAndSetTags(d, resp.Tags)

	return nil
}

func resourceArmVirtualNetworkDelete(d *schema.ResourceData, meta interface{}) error {
	vnetClient := meta.(*ArmClient).vnetClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	resGroup := id.ResourceGroup
	name := id.Path["virtualNetworks"]

	nsgNames, err := expandAzureRmVirtualNetworkVirtualNetworkSecurityGroupNames(d)
	if err != nil {
		return fmt.Errorf("[ERROR] Error parsing Network Security Group ID's: %+v", err)
	}

	azureRMVirtualNetworkLockNetworkSecurityGroups(&nsgNames)
	defer azureRMVirtualNetworkUnlockNetworkSecurityGroups(&nsgNames)

	_, err = vnetClient.Delete(resGroup, name, make(chan struct{}))

	return err
}

func getVirtualNetworkProperties(d *schema.ResourceData) *network.VirtualNetworkPropertiesFormat {
	// first; get address space prefixes:
	prefixes := []string{}
	for _, prefix := range d.Get("address_space").([]interface{}) {
		prefixes = append(prefixes, prefix.(string))
	}

	// then; the dns servers:
	dnses := []string{}
	for _, dns := range d.Get("dns_servers").([]interface{}) {
		dnses = append(dnses, dns.(string))
	}

	// then; the subnets:
	subnets := []network.Subnet{}
	if subs := d.Get("subnet").(*schema.Set); subs.Len() > 0 {
		for _, subnet := range subs.List() {
			subnet := subnet.(map[string]interface{})

			name := subnet["name"].(string)
			prefix := subnet["address_prefix"].(string)
			secGroup := subnet["security_group"].(string)

			var subnetObj network.Subnet
			subnetObj.Name = &name
			subnetObj.SubnetPropertiesFormat = &network.SubnetPropertiesFormat{}
			subnetObj.SubnetPropertiesFormat.AddressPrefix = &prefix

			if secGroup != "" {
				subnetObj.SubnetPropertiesFormat.NetworkSecurityGroup = &network.SecurityGroup{
					ID: &secGroup,
				}
			}

			subnets = append(subnets, subnetObj)
		}
	}

	// finally; return the struct:
	return &network.VirtualNetworkPropertiesFormat{
		AddressSpace: &network.AddressSpace{
			AddressPrefixes: &prefixes,
		},
		DhcpOptions: &network.DhcpOptions{
			DNSServers: &dnses,
		},
		Subnets: &subnets,
	}
}

func resourceAzureSubnetHash(v interface{}) int {
	m := v.(map[string]interface{})
	subnet := m["name"].(string) + m["address_prefix"].(string)
	if securityGroup, present := m["security_group"]; present {
		subnet = subnet + securityGroup.(string)
	}
	return hashcode.String(subnet)
}

func expandAzureRmVirtualNetworkVirtualNetworkSecurityGroupNames(d *schema.ResourceData) ([]string, error) {
	nsgNames := make([]string, 0)

	if v, ok := d.GetOk("subnet"); ok {
		subnets := v.(*schema.Set).List()
		for _, subnet := range subnets {
			subnet := subnet.(map[string]interface{})

			networkSecurityGroupId := subnet["security_group"].(string)
			if networkSecurityGroupId != "" {
				nsgName, err := parseNetworkSecurityGroupName(networkSecurityGroupId)
				if err != nil {
					return nil, err
				}

				nsgNames = append(nsgNames, nsgName)
			}
		}
	}

	return nsgNames, nil
}

func azureRMVirtualNetworkUnlockNetworkSecurityGroups(networkSecurityGroupNames *[]string) {
	for _, networkSecurityGroupName := range *networkSecurityGroupNames {
		armMutexKV.Unlock(networkSecurityGroupName)
	}
}
func azureRMVirtualNetworkLockNetworkSecurityGroups(networkSecurityGroupNames *[]string) {
	for _, networkSecurityGroupName := range *networkSecurityGroupNames {
		armMutexKV.Lock(networkSecurityGroupName)
	}
}
<commit_msg>Checking the type of the subnet before using it<commit_after>package azurerm

import (
	"fmt"
	"log"
	"net/http"

	"github.com/Azure/azure-sdk-for-go/arm/network"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceArmVirtualNetwork() *schema.Resource {
	return &schema.Resource{
		Create: resourceArmVirtualNetworkCreate,
		Read:   resourceArmVirtualNetworkRead,
		Update: resourceArmVirtualNetworkCreate,
		Delete: resourceArmVirtualNetworkDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"address_space": {
				Type:     schema.TypeList,
				Required: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"dns_servers": {
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"subnet": {
				Type:     schema.TypeSet,
				Optional: true,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"name": {
							Type:     schema.TypeString,
							Required: true,
						},
						"address_prefix": {
							Type:     schema.TypeString,
							Required: true,
						},
						"security_group": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
				Set: resourceAzureSubnetHash,
			},

			"location": locationSchema(),

			"resource_group_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"tags": tagsSchema(),
		},
	}
}

func resourceArmVirtualNetworkCreate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*ArmClient)
	vnetClient := client.vnetClient

	log.Printf("[INFO] preparing arguments for Azure ARM virtual network creation.")

	name := d.Get("name").(string)
	location := d.Get("location").(string)
	resGroup := d.Get("resource_group_name").(string)
	tags := d.Get("tags").(map[string]interface{})

	vnet := network.VirtualNetwork{
		Name:                           &name,
		Location:                       &location,
		VirtualNetworkPropertiesFormat: getVirtualNetworkProperties(d),
		Tags: expandTags(tags),
	}

	networkSecurityGroupNames := make([]string, 0)
	for _, subnet := range *vnet.VirtualNetworkPropertiesFormat.Subnets {
		if subnet.NetworkSecurityGroup != nil {
			nsgName, err := parseNetworkSecurityGroupName(*subnet.NetworkSecurityGroup.ID)
			if err != nil {
				return err
			}

			networkSecurityGroupNames = append(networkSecurityGroupNames, nsgName)
		}
	}

	azureRMVirtualNetworkLockNetworkSecurityGroups(&networkSecurityGroupNames)
	defer azureRMVirtualNetworkUnlockNetworkSecurityGroups(&networkSecurityGroupNames)

	_, err := vnetClient.CreateOrUpdate(resGroup, name, vnet, make(chan struct{}))
	if err != nil {
		return err
	}

	read, err := vnetClient.Get(resGroup, name, "")
	if err != nil {
		return err
	}
	if read.ID == nil {
		return fmt.Errorf("Cannot read Virtual Network %s (resource group %s) ID", name, resGroup)
	}

	d.SetId(*read.ID)

	return resourceArmVirtualNetworkRead(d, meta)
}

func resourceArmVirtualNetworkRead(d *schema.ResourceData, meta interface{}) error {
	vnetClient := meta.(*ArmClient).vnetClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	resGroup := id.ResourceGroup
	name := id.Path["virtualNetworks"]

	resp, err := vnetClient.Get(resGroup, name, "")
	if err != nil {
		if resp.StatusCode == http.StatusNotFound {
			d.SetId("")
			return nil
		}
		return fmt.Errorf("Error making Read request on Azure virtual network %s: %s", name, err)
	}

	vnet := *resp.VirtualNetworkPropertiesFormat

	// update appropriate values
	d.Set("resource_group_name", resGroup)
	d.Set("name", resp.Name)
	d.Set("location", resp.Location)
	d.Set("address_space", vnet.AddressSpace.AddressPrefixes)

	subnets := &schema.Set{
		F: resourceAzureSubnetHash,
	}

	for _, subnet := range *vnet.Subnets {
		s := map[string]interface{}{}

		s["name"] = *subnet.Name
		s["address_prefix"] = *subnet.SubnetPropertiesFormat.AddressPrefix
		if subnet.SubnetPropertiesFormat.NetworkSecurityGroup != nil {
			s["security_group"] = *subnet.SubnetPropertiesFormat.NetworkSecurityGroup.ID
		}

		subnets.Add(s)
	}
	d.Set("subnet", subnets)

	if vnet.DhcpOptions != nil && vnet.DhcpOptions.DNSServers != nil {
		dnses := []string{}
		for _, dns := range *vnet.DhcpOptions.DNSServers {
			dnses = append(dnses, dns)
		}
		d.Set("dns_servers", dnses)
	}

	flattenAndSetTags(d, resp.Tags)

	return nil
}

func resourceArmVirtualNetworkDelete(d *schema.ResourceData, meta interface{}) error {
	vnetClient := meta.(*ArmClient).vnetClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	resGroup := id.ResourceGroup
	name := id.Path["virtualNetworks"]

	nsgNames, err := expandAzureRmVirtualNetworkVirtualNetworkSecurityGroupNames(d)
	if err != nil {
		return fmt.Errorf("[ERROR] Error parsing Network Security Group ID's: %+v", err)
	}

	azureRMVirtualNetworkLockNetworkSecurityGroups(&nsgNames)
	defer azureRMVirtualNetworkUnlockNetworkSecurityGroups(&nsgNames)

	_, err = vnetClient.Delete(resGroup, name, make(chan struct{}))

	return err
}

func getVirtualNetworkProperties(d *schema.ResourceData) *network.VirtualNetworkPropertiesFormat {
	// first; get address space prefixes:
	prefixes := []string{}
	for _, prefix := range d.Get("address_space").([]interface{}) {
		prefixes = append(prefixes, prefix.(string))
	}

	// then; the dns servers:
	dnses := []string{}
	for _, dns := range d.Get("dns_servers").([]interface{}) {
		dnses = append(dnses, dns.(string))
	}

	// then; the subnets:
	subnets := []network.Subnet{}
	if subs := d.Get("subnet").(*schema.Set); subs.Len() > 0 {
		for _, subnet := range subs.List() {
			subnet := subnet.(map[string]interface{})

			name := subnet["name"].(string)
			prefix := subnet["address_prefix"].(string)
			secGroup := subnet["security_group"].(string)

			var subnetObj network.Subnet
			subnetObj.Name = &name
			subnetObj.SubnetPropertiesFormat = &network.SubnetPropertiesFormat{}
			subnetObj.SubnetPropertiesFormat.AddressPrefix = &prefix

			if secGroup != "" {
				subnetObj.SubnetPropertiesFormat.NetworkSecurityGroup = &network.SecurityGroup{
					ID: &secGroup,
				}
			}

			subnets = append(subnets, subnetObj)
		}
	}

	// finally; return the struct:
	return &network.VirtualNetworkPropertiesFormat{
		AddressSpace: &network.AddressSpace{
			AddressPrefixes: &prefixes,
		},
		DhcpOptions: &network.DhcpOptions{
			DNSServers: &dnses,
		},
		Subnets: &subnets,
	}
}

func resourceAzureSubnetHash(v interface{}) int {
	m := v.(map[string]interface{})
	subnet := m["name"].(string) + m["address_prefix"].(string)
	if securityGroup, present := m["security_group"]; present {
		subnet = subnet + securityGroup.(string)
	}
	return hashcode.String(subnet)
}

func expandAzureRmVirtualNetworkVirtualNetworkSecurityGroupNames(d *schema.ResourceData) ([]string, error) {
	nsgNames := make([]string, 0)

	if v, ok := d.GetOk("subnet"); ok {
		subnets := v.(*schema.Set).List()
		for _, subnet := range subnets {
			subnet, ok := subnet.(map[string]interface{})
			if !ok {
				return nil, fmt.Errorf("[ERROR] Subnet should be a Hash - was '%+v'", subnet)
			}

			networkSecurityGroupId := subnet["security_group"].(string)
			if networkSecurityGroupId != "" {
				nsgName, err := parseNetworkSecurityGroupName(networkSecurityGroupId)
				if err != nil {
					return nil, err
				}

				nsgNames = append(nsgNames, nsgName)
			}
		}
	}

	return nsgNames, nil
}

func azureRMVirtualNetworkUnlockNetworkSecurityGroups(networkSecurityGroupNames *[]string) {
	for _, networkSecurityGroupName := range *networkSecurityGroupNames {
		armMutexKV.Unlock(networkSecurityGroupName)
	}
}
func azureRMVirtualNetworkLockNetworkSecurityGroups(networkSecurityGroupNames *[]string) {
	for _, networkSecurityGroupName := range *networkSecurityGroupNames {
		armMutexKV.Lock(networkSecurityGroupName)
	}
}
<|endoftext|>


<commit_before>package update

import (
	"fmt"
	"strings"

	"github.com/pkg/errors"

	"github.com/go-kit/kit/log"
	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/cluster"
	"github.com/weaveworks/flux/image"
	"github.com/weaveworks/flux/policy"
	"github.com/weaveworks/flux/registry"
)

const (
	ResourceSpecAll = ResourceSpec("<all>")
	ImageSpecLatest = ImageSpec("<all latest>")
)

var (
	ErrInvalidReleaseKind = errors.New("invalid release kind")
)

// ReleaseKind says whether a release is to be planned only, or planned then executed
type ReleaseKind string
type ReleaseType string

const (
	ReleaseKindPlan    ReleaseKind = "plan"
	ReleaseKindExecute ReleaseKind = "execute"
)

func ParseReleaseKind(s string) (ReleaseKind, error) {
	switch s {
	case string(ReleaseKindPlan):
		return ReleaseKindPlan, nil
	case string(ReleaseKindExecute):
		return ReleaseKindExecute, nil
	default:
		return "", ErrInvalidReleaseKind
	}
}

const UserAutomated = "<automated>"

type ReleaseContext interface {
	SelectServices(Result, []ControllerFilter, []ControllerFilter) ([]*ControllerUpdate, error)
	ServicesWithPolicies() (policy.ResourceMap, error)
	Registry() registry.Registry
	Manifests() cluster.Manifests
}

// NB: these get sent from fluxctl, so we have to maintain the json format of
// this. Eugh.
type ReleaseSpec struct {
	ServiceSpecs []ResourceSpec
	ImageSpec    ImageSpec
	Kind         ReleaseKind
	Excludes     []flux.ResourceID
}

// ReleaseType gives a one-word description of the release, mainly
// useful for labelling metrics or log messages.
func (s ReleaseSpec) ReleaseType() ReleaseType {
	switch {
	case s.ImageSpec == ImageSpecLatest:
		return "latest_images"
	default:
		return "specific_image"
	}
}

func (s ReleaseSpec) CalculateRelease(rc ReleaseContext, logger log.Logger) ([]*ControllerUpdate, Result, error) {
	results := Result{}
	timer := NewStageTimer("select_services")
	updates, err := s.selectServices(rc, results)
	timer.ObserveDuration()
	if err != nil {
		return nil, nil, err
	}
	s.markSkipped(results)

	timer = NewStageTimer("lookup_images")
	updates, err = s.calculateImageUpdates(rc, updates, results, logger)
	timer.ObserveDuration()
	if err != nil {
		return nil, nil, err
	}
	return updates, results, nil
}

func (s ReleaseSpec) ReleaseKind() ReleaseKind {
	return s.Kind
}

func (s ReleaseSpec) CommitMessage() string {
	image := strings.Trim(s.ImageSpec.String(), "<>")
	var services []string
	for _, spec := range s.ServiceSpecs {
		services = append(services, strings.Trim(spec.String(), "<>"))
	}
	return fmt.Sprintf("Release %s to %s", image, strings.Join(services, ", "))
}

// Take the spec given in the job, and figure out which services are
// in question based on the running services and those defined in the
// repo. Fill in the release results along the way.
func (s ReleaseSpec) selectServices(rc ReleaseContext, results Result) ([]*ControllerUpdate, error) {
	// Build list of filters
	prefilters, postfilters, err := s.filters(rc)
	if err != nil {
		return nil, err
	}
	// Find and filter services
	return rc.SelectServices(results, prefilters, postfilters)
}

func (s ReleaseSpec) filters(rc ReleaseContext) ([]ControllerFilter, []ControllerFilter, error) {
	var prefilters, postfilters []ControllerFilter

	ids := []flux.ResourceID{}
	for _, s := range s.ServiceSpecs {
		if s == ResourceSpecAll {
			// "<all>" Overrides any other filters
			ids = []flux.ResourceID{}
			break
		}
		id, err := flux.ParseResourceID(string(s))
		if err != nil {
			return nil, nil, err
		}
		ids = append(ids, id)
	}
	if len(ids) > 0 {
		prefilters = append(prefilters, &IncludeFilter{ids})
	}

	// Exclude filter
	if len(s.Excludes) > 0 {
		prefilters = append(prefilters, &ExcludeFilter{s.Excludes})
	}

	// Image filter
	if s.ImageSpec != ImageSpecLatest {
		id, err := image.ParseRef(s.ImageSpec.String())
		if err != nil {
			return nil, nil, err
		}
		postfilters = append(postfilters, &SpecificImageFilter{id})
	}

	// Locked filter
	services, err := rc.ServicesWithPolicies()
	if err != nil {
		return nil, nil, err
	}
	lockedSet := services.OnlyWithPolicy(policy.Locked)
	postfilters = append(postfilters, &LockedFilter{lockedSet.ToSlice()})

	return prefilters, postfilters, nil
}

func (s ReleaseSpec) markSkipped(results Result) {
	for _, v := range s.ServiceSpecs {
		if v == ResourceSpecAll {
			continue
		}
		id, err := v.AsID()
		if err != nil {
			continue
		}
		if _, ok := results[id]; !ok {
			results[id] = ControllerResult{
				Status: ReleaseStatusSkipped,
				Error:  NotInRepo,
			}
		}
	}
}

// Find all the image updates that should be performed, and do
// replacements. (For a dry-run, we don't strictly need to do the
// replacements, since we won't be committing any changes back;
// however we do want to see if we *can* do the replacements, because
// if not, it indicates there's likely some problem with the running
// system vs the definitions given in the repo.)
func (s ReleaseSpec) calculateImageUpdates(rc ReleaseContext, candidates []*ControllerUpdate, results Result, logger log.Logger) ([]*ControllerUpdate, error) {
	// Compile an `ImageMap` of all relevant images
	var images ImageMap
	var singleRepo image.CanonicalName
	var err error

	switch s.ImageSpec {
	case ImageSpecLatest:
		images, err = collectUpdateImages(rc.Registry(), candidates, logger)
	default:
		var ref image.Ref
		ref, err = s.ImageSpec.AsRef()
		if err == nil {
			singleRepo = ref.CanonicalName()
			images, err = exactImages(rc.Registry(), []image.Ref{ref})
		}
	}

	if err != nil {
		return nil, err
	}

	// Look through all the services' containers to see which have an
	// image that could be updated.
	var updates []*ControllerUpdate
	for _, u := range candidates {
		containers, err := u.Controller.ContainersOrError()
		if err != nil {
			results[u.ResourceID] = ControllerResult{
				Status: ReleaseStatusFailed,
				Error:  err.Error(),
			}
			continue
		}

		// If at least one container used an image in question, we say
		// we're skipping it rather than ignoring it. This is mainly
		// for the purpose of filtering the output.
		ignoredOrSkipped := ReleaseStatusIgnored
		var containerUpdates []ContainerUpdate

		for _, container := range containers {
			currentImageID, err := image.ParseRef(container.Image)
			if err != nil {
				// We may hope never to find a malformed image ID, but
				// anything is possible.
				return nil, err
			}

			latestImage, ok := images.LatestImage(currentImageID.Name, "*")
			if !ok {
				if currentImageID.CanonicalName() != singleRepo {
					ignoredOrSkipped = ReleaseStatusIgnored
				} else {
					ignoredOrSkipped = ReleaseStatusUnknown
				}
				continue
			}

			if currentImageID == latestImage.ID {
				ignoredOrSkipped = ReleaseStatusSkipped
				continue
			}

			// We want to update the image with respect to the form it
			// appears in the manifest, whereas what we have is the
			// canonical form.
			newImageID := currentImageID.WithNewTag(latestImage.ID.Tag)

			u.ManifestBytes, err = rc.Manifests().UpdateDefinition(u.ManifestBytes, container.Name, newImageID)
			if err != nil {
				return nil, err
			}

			containerUpdates = append(containerUpdates, ContainerUpdate{
				Container: container.Name,
				Current:   currentImageID,
				Target:    newImageID,
			})
		}

		switch {
		case len(containerUpdates) > 0:
			u.Updates = containerUpdates
			updates = append(updates, u)
			results[u.ResourceID] = ControllerResult{
				Status:       ReleaseStatusSuccess,
				PerContainer: containerUpdates,
			}
		case ignoredOrSkipped == ReleaseStatusSkipped:
			results[u.ResourceID] = ControllerResult{
				Status: ReleaseStatusSkipped,
				Error:  ImageUpToDate,
			}
		case ignoredOrSkipped == ReleaseStatusIgnored:
			results[u.ResourceID] = ControllerResult{
				Status: ReleaseStatusIgnored,
				Error:  DoesNotUseImage,
			}
		case ignoredOrSkipped == ReleaseStatusUnknown:
			results[u.ResourceID] = ControllerResult{
				Status: ReleaseStatusSkipped,
				Error:  ImageNotFound,
			}
		}
	}

	return updates, nil
}

type ResourceSpec string // ResourceID or "<all>"

func ParseResourceSpec(s string) (ResourceSpec, error) {
	if s == string(ResourceSpecAll) {
		return ResourceSpecAll, nil
	}
	id, err := flux.ParseResourceID(s)
	if err != nil {
		return "", errors.Wrap(err, "invalid service spec")
	}
	return ResourceSpec(id.String()), nil
}

func MakeResourceSpec(id flux.ResourceID) ResourceSpec {
	return ResourceSpec(id.String())
}

func (s ResourceSpec) AsID() (flux.ResourceID, error) {
	return flux.ParseResourceID(string(s))
}

func (s ResourceSpec) String() string {
	return string(s)
}

// ImageSpec is an ImageID, or "<all latest>" (update all containers
// to the latest available).
type ImageSpec string

func ParseImageSpec(s string) (ImageSpec, error) {
	if s == string(ImageSpecLatest) {
		return ImageSpecLatest, nil
	}

	id, err := image.ParseRef(s)
	if err != nil {
		return "", err
	}
	if id.Tag == "" {
		return "", errors.Wrap(image.ErrInvalidImageID, "blank tag (if you want latest, explicitly state the tag :latest)")
	}
	return ImageSpec(id.String()), err
}

func (s ImageSpec) String() string {
	return string(s)
}

func (s ImageSpec) AsRef() (image.Ref, error) {
	return image.ParseRef(s.String())
}

func ImageSpecFromRef(id image.Ref) ImageSpec {
	return ImageSpec(id.String())
}
<commit_msg>added force flag for later work on forced deployments<commit_after>package update

import (
	"fmt"
	"strings"

	"github.com/pkg/errors"

	"github.com/go-kit/kit/log"
	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/cluster"
	"github.com/weaveworks/flux/image"
	"github.com/weaveworks/flux/policy"
	"github.com/weaveworks/flux/registry"
)

const (
	ResourceSpecAll = ResourceSpec("<all>")
	ImageSpecLatest = ImageSpec("<all latest>")
)

var (
	ErrInvalidReleaseKind = errors.New("invalid release kind")
)

// ReleaseKind says whether a release is to be planned only, or planned then executed
type ReleaseKind string
type ReleaseType string

const (
	ReleaseKindPlan    ReleaseKind = "plan"
	ReleaseKindExecute ReleaseKind = "execute"
)

func ParseReleaseKind(s string) (ReleaseKind, error) {
	switch s {
	case string(ReleaseKindPlan):
		return ReleaseKindPlan, nil
	case string(ReleaseKindExecute):
		return ReleaseKindExecute, nil
	default:
		return "", ErrInvalidReleaseKind
	}
}

const UserAutomated = "<automated>"

type ReleaseContext interface {
	SelectServices(Result, []ControllerFilter, []ControllerFilter) ([]*ControllerUpdate, error)
	ServicesWithPolicies() (policy.ResourceMap, error)
	Registry() registry.Registry
	Manifests() cluster.Manifests
}

// NB: these get sent from fluxctl, so we have to maintain the json format of
// this. Eugh.
type ReleaseSpec struct {
	ServiceSpecs []ResourceSpec
	ImageSpec    ImageSpec
	Kind         ReleaseKind
	Excludes     []flux.ResourceID
	Force        bool
}

// ReleaseType gives a one-word description of the release, mainly
// useful for labelling metrics or log messages.
func (s ReleaseSpec) ReleaseType() ReleaseType {
	switch {
	case s.ImageSpec == ImageSpecLatest:
		return "latest_images"
	default:
		return "specific_image"
	}
}

func (s ReleaseSpec) CalculateRelease(rc ReleaseContext, logger log.Logger) ([]*ControllerUpdate, Result, error) {
	results := Result{}
	timer := NewStageTimer("select_services")
	updates, err := s.selectServices(rc, results)
	timer.ObserveDuration()
	if err != nil {
		return nil, nil, err
	}
	s.markSkipped(results)

	timer = NewStageTimer("lookup_images")
	updates, err = s.calculateImageUpdates(rc, updates, results, logger)
	timer.ObserveDuration()
	if err != nil {
		return nil, nil, err
	}
	return updates, results, nil
}

func (s ReleaseSpec) ReleaseKind() ReleaseKind {
	return s.Kind
}

func (s ReleaseSpec) CommitMessage() string {
	image := strings.Trim(s.ImageSpec.String(), "<>")
	var services []string
	for _, spec := range s.ServiceSpecs {
		services = append(services, strings.Trim(spec.String(), "<>"))
	}
	return fmt.Sprintf("Release %s to %s", image, strings.Join(services, ", "))
}

// Take the spec given in the job, and figure out which services are
// in question based on the running services and those defined in the
// repo. Fill in the release results along the way.
func (s ReleaseSpec) selectServices(rc ReleaseContext, results Result) ([]*ControllerUpdate, error) {
	// Build list of filters
	prefilters, postfilters, err := s.filters(rc)
	if err != nil {
		return nil, err
	}
	// Find and filter services
	return rc.SelectServices(results, prefilters, postfilters)
}

func (s ReleaseSpec) filters(rc ReleaseContext) ([]ControllerFilter, []ControllerFilter, error) {
	var prefilters, postfilters []ControllerFilter

	ids := []flux.ResourceID{}
	for _, s := range s.ServiceSpecs {
		if s == ResourceSpecAll {
			// "<all>" Overrides any other filters
			ids = []flux.ResourceID{}
			break
		}
		id, err := flux.ParseResourceID(string(s))
		if err != nil {
			return nil, nil, err
		}
		ids = append(ids, id)
	}
	if len(ids) > 0 {
		prefilters = append(prefilters, &IncludeFilter{ids})
	}

	// Exclude filter
	if len(s.Excludes) > 0 {
		prefilters = append(prefilters, &ExcludeFilter{s.Excludes})
	}

	// Image filter
	if s.ImageSpec != ImageSpecLatest {
		id, err := image.ParseRef(s.ImageSpec.String())
		if err != nil {
			return nil, nil, err
		}
		postfilters = append(postfilters, &SpecificImageFilter{id})
	}

	// Locked filter
	services, err := rc.ServicesWithPolicies()
	if err != nil {
		return nil, nil, err
	}
	lockedSet := services.OnlyWithPolicy(policy.Locked)
	postfilters = append(postfilters, &LockedFilter{lockedSet.ToSlice()})

	return prefilters, postfilters, nil
}

func (s ReleaseSpec) markSkipped(results Result) {
	for _, v := range s.ServiceSpecs {
		if v == ResourceSpecAll {
			continue
		}
		id, err := v.AsID()
		if err != nil {
			continue
		}
		if _, ok := results[id]; !ok {
			results[id] = ControllerResult{
				Status: ReleaseStatusSkipped,
				Error:  NotInRepo,
			}
		}
	}
}

// Find all the image updates that should be performed, and do
// replacements. (For a dry-run, we don't strictly need to do the
// replacements, since we won't be committing any changes back;
// however we do want to see if we *can* do the replacements, because
// if not, it indicates there's likely some problem with the running
// system vs the definitions given in the repo.)
func (s ReleaseSpec) calculateImageUpdates(rc ReleaseContext, candidates []*ControllerUpdate, results Result, logger log.Logger) ([]*ControllerUpdate, error) {
	// Compile an `ImageMap` of all relevant images
	var images ImageMap
	var singleRepo image.CanonicalName
	var err error

	switch s.ImageSpec {
	case ImageSpecLatest:
		images, err = collectUpdateImages(rc.Registry(), candidates, logger)
	default:
		var ref image.Ref
		ref, err = s.ImageSpec.AsRef()
		if err == nil {
			singleRepo = ref.CanonicalName()
			images, err = exactImages(rc.Registry(), []image.Ref{ref})
		}
	}

	if err != nil {
		return nil, err
	}

	// Look through all the services' containers to see which have an
	// image that could be updated.
	var updates []*ControllerUpdate
	for _, u := range candidates {
		containers, err := u.Controller.ContainersOrError()
		if err != nil {
			results[u.ResourceID] = ControllerResult{
				Status: ReleaseStatusFailed,
				Error:  err.Error(),
			}
			continue
		}

		// If at least one container used an image in question, we say
		// we're skipping it rather than ignoring it. This is mainly
		// for the purpose of filtering the output.
		ignoredOrSkipped := ReleaseStatusIgnored
		var containerUpdates []ContainerUpdate

		for _, container := range containers {
			currentImageID, err := image.ParseRef(container.Image)
			if err != nil {
				// We may hope never to find a malformed image ID, but
				// anything is possible.
				return nil, err
			}

			latestImage, ok := images.LatestImage(currentImageID.Name, "*")
			if !ok {
				if currentImageID.CanonicalName() != singleRepo {
					ignoredOrSkipped = ReleaseStatusIgnored
				} else {
					ignoredOrSkipped = ReleaseStatusUnknown
				}
				continue
			}

			if currentImageID == latestImage.ID {
				ignoredOrSkipped = ReleaseStatusSkipped
				continue
			}

			// We want to update the image with respect to the form it
			// appears in the manifest, whereas what we have is the
			// canonical form.
			newImageID := currentImageID.WithNewTag(latestImage.ID.Tag)

			u.ManifestBytes, err = rc.Manifests().UpdateDefinition(u.ManifestBytes, container.Name, newImageID)
			if err != nil {
				return nil, err
			}

			containerUpdates = append(containerUpdates, ContainerUpdate{
				Container: container.Name,
				Current:   currentImageID,
				Target:    newImageID,
			})
		}

		switch {
		case len(containerUpdates) > 0:
			u.Updates = containerUpdates
			updates = append(updates, u)
			results[u.ResourceID] = ControllerResult{
				Status:       ReleaseStatusSuccess,
				PerContainer: containerUpdates,
			}
		case ignoredOrSkipped == ReleaseStatusSkipped:
			results[u.ResourceID] = ControllerResult{
				Status: ReleaseStatusSkipped,
				Error:  ImageUpToDate,
			}
		case ignoredOrSkipped == ReleaseStatusIgnored:
			results[u.ResourceID] = ControllerResult{
				Status: ReleaseStatusIgnored,
				Error:  DoesNotUseImage,
			}
		case ignoredOrSkipped == ReleaseStatusUnknown:
			results[u.ResourceID] = ControllerResult{
				Status: ReleaseStatusSkipped,
				Error:  ImageNotFound,
			}
		}
	}

	return updates, nil
}

type ResourceSpec string // ResourceID or "<all>"

func ParseResourceSpec(s string) (ResourceSpec, error) {
	if s == string(ResourceSpecAll) {
		return ResourceSpecAll, nil
	}
	id, err := flux.ParseResourceID(s)
	if err != nil {
		return "", errors.Wrap(err, "invalid service spec")
	}
	return ResourceSpec(id.String()), nil
}

func MakeResourceSpec(id flux.ResourceID) ResourceSpec {
	return ResourceSpec(id.String())
}

func (s ResourceSpec) AsID() (flux.ResourceID, error) {
	return flux.ParseResourceID(string(s))
}

func (s ResourceSpec) String() string {
	return string(s)
}

// ImageSpec is an ImageID, or "<all latest>" (update all containers
// to the latest available).
type ImageSpec string

func ParseImageSpec(s string) (ImageSpec, error) {
	if s == string(ImageSpecLatest) {
		return ImageSpecLatest, nil
	}

	id, err := image.ParseRef(s)
	if err != nil {
		return "", err
	}
	if id.Tag == "" {
		return "", errors.Wrap(image.ErrInvalidImageID, "blank tag (if you want latest, explicitly state the tag :latest)")
	}
	return ImageSpec(id.String()), err
}

func (s ImageSpec) String() string {
	return string(s)
}

func (s ImageSpec) AsRef() (image.Ref, error) {
	return image.ParseRef(s.String())
}

func ImageSpecFromRef(id image.Ref) ImageSpec {
	return ImageSpec(id.String())
}
<|endoftext|>