<commit_before>package mpelasticsearch

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"strings"

	mp "github.com/mackerelio/go-mackerel-plugin"
	"github.com/mackerelio/golib/logging"
)

var logger = logging.GetLogger("metrics.plugin.elasticsearch")

var metricPlace = map[string][]string{
	"http_opened":                 {"http", "total_opened"},
	"total_indexing_index":        {"indices", "indexing", "index_total"},
	"total_indexing_delete":       {"indices", "indexing", "delete_total"},
	"total_get":                   {"indices", "get", "total"},
	"total_search_query":          {"indices", "search", "query_total"},
	"total_search_fetch":          {"indices", "search", "fetch_total"},
	"total_merges":                {"indices", "merges", "total"},
	"total_refresh":               {"indices", "refresh", "total"},
	"total_flush":                 {"indices", "flush", "total"},
	"total_warmer":                {"indices", "warmer", "total"},
	"total_percolate":             {"indices", "percolate", "total"},
	"total_suggest":               {"indices", "suggest", "total"},
	"docs_count":                  {"indices", "docs", "count"},
	"docs_deleted":                {"indices", "docs", "deleted"},
	"fielddata_size":              {"indices", "fielddata", "memory_size_in_bytes"},
	"filter_cache_size":           {"indices", "filter_cache", "memory_size_in_bytes"},
	"segments_size":               {"indices", "segments", "memory_in_bytes"},
	"segments_index_writer_size":  {"indices", "segments", "index_writer_memory_in_bytes"},
	"segments_version_map_size":   {"indices", "segments", "version_map_memory_in_bytes"},
	"segments_fixed_bit_set_size": {"indices", "segments", "fixed_bit_set_memory_in_bytes"},
	"evictions_fielddata":         {"indices", "fielddata", "evictions"},
	"evictions_filter_cache":      {"indices", "filter_cache", "evictions"},
	"heap_used":                   {"jvm", "mem", "heap_used_in_bytes"},
	"heap_max":                    {"jvm", "mem", "heap_max_in_bytes"},
	"threads_generic":             {"thread_pool", "generic", "threads"},
	"threads_index":               {"thread_pool", "index", "threads"},
	"threads_snapshot_data":       {"thread_pool", "snapshot_data", "threads"},
	"threads_get":                 {"thread_pool", "get", "threads"},
	"threads_bench":               {"thread_pool", "bench", "threads"},
	"threads_snapshot":            {"thread_pool", "snapshot", "threads"},
	"threads_merge":               {"thread_pool", "merge", "threads"},
	"threads_suggest":             {"thread_pool", "suggest", "threads"},
	"threads_bulk":                {"thread_pool", "bulk", "threads"},
	"threads_optimize":            {"thread_pool", "optimize", "threads"},
	"threads_warmer":              {"thread_pool", "warmer", "threads"},
	"threads_flush":               {"thread_pool", "flush", "threads"},
	"threads_search":              {"thread_pool", "search", "threads"},
	"threads_percolate":           {"thread_pool", "percolate", "threads"},
	"threads_refresh":             {"thread_pool", "refresh", "threads"},
	"threads_management":          {"thread_pool", "management", "threads"},
	"threads_fetch_shard_started": {"thread_pool", "fetch_shard_started", "threads"},
	"threads_fetch_shard_store":   {"thread_pool", "fetch_shard_store", "threads"},
	"threads_listener":            {"thread_pool", "listener", "threads"},
	"count_rx":                    {"transport", "rx_count"},
	"count_tx":                    {"transport", "tx_count"},
	"open_file_descriptors":       {"process", "open_file_descriptors"},
}

func getFloatValue(s map[string]interface{}, keys []string) (float64, error) {
	var val float64
	sm := s
	for i, k := range keys {
		if i+1 < len(keys) {
			switch sm[k].(type) {
			case map[string]interface{}:
				sm = sm[k].(map[string]interface{})
			default:
				return 0, errors.New("Cannot handle as a hash")
			}
		} else {
			switch sm[k].(type) {
			case float64:
				val = sm[k].(float64)
			default:
				return 0, errors.New("Not float64")
			}
		}
	}

	return val, nil
}

// ElasticsearchPlugin mackerel plugin for Elasticsearch
type ElasticsearchPlugin struct {
	URI         string
	Prefix      string
	LabelPrefix string
}

// FetchMetrics interface for mackerelplugin
func (p ElasticsearchPlugin) FetchMetrics() (map[string]float64, error) {
	req, err := http.NewRequest(http.MethodGet, p.URI+"/_nodes/_local/stats", nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", "mackerel-plugin-elasticsearch")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	stat := make(map[string]float64)
	decoder := json.NewDecoder(resp.Body)

	var s map[string]interface{}
	err = decoder.Decode(&s)
	if err != nil {
		return nil, err
	}

	nodes := s["nodes"].(map[string]interface{})
	n := ""
	for k := range nodes {
		if n != "" {
			return nil, errors.New("Multiple node found")
		}
		n = k
	}
	node := nodes[n].(map[string]interface{})

	for k, v := range metricPlace {
		val, err := getFloatValue(node, v)
		if err != nil {
			logger.Errorf("Failed to find '%s': %s", k, err)
			continue
		}

		stat[k] = val
	}

	return stat, nil
}

// GraphDefinition interface for mackerelplugin
func (p ElasticsearchPlugin) GraphDefinition() map[string]mp.Graphs {
	var graphdef = map[string]mp.Graphs{
		p.Prefix + ".http": {
			Label: (p.LabelPrefix + " HTTP"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "http_opened", Label: "Opened", Diff: true},
			},
		},
		p.Prefix + ".indices": {
			Label: (p.LabelPrefix + " Indices"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "total_indexing_index", Label: "Indexing-Index", Diff: true, Stacked: true},
				{Name: "total_indexing_delete", Label: "Indexing-Delete", Diff: true, Stacked: true},
				{Name: "total_get", Label: "Get", Diff: true, Stacked: true},
				{Name: "total_search_query", Label: "Search-Query", Diff: true, Stacked: true},
				{Name: "total_search_fetch", Label: "Search-fetch", Diff: true, Stacked: true},
				{Name: "total_merges", Label: "Merges", Diff: true, Stacked: true},
				{Name: "total_refresh", Label: "Refresh", Diff: true, Stacked: true},
				{Name: "total_flush", Label: "Flush", Diff: true, Stacked: true},
				{Name: "total_warmer", Label: "Warmer", Diff: true, Stacked: true},
				{Name: "total_percolate", Label: "Percolate", Diff: true, Stacked: true},
				{Name: "total_suggest", Label: "Suggest", Diff: true, Stacked: true},
			},
		},
		p.Prefix + ".indices.docs": {
			Label: (p.LabelPrefix + " Indices Docs"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "docs_count", Label: "Count", Stacked: true},
				{Name: "docs_deleted", Label: "Deleted", Stacked: true},
			},
		},
		p.Prefix + ".indices.memory_size": {
			Label: (p.LabelPrefix + " Indices Memory Size"),
			Unit:  "bytes",
			Metrics: []mp.Metrics{
				{Name: "fielddata_size", Label: "Fielddata", Stacked: true},
				{Name: "filter_cache_size", Label: "Filter Cache", Stacked: true},
				{Name: "segments_size", Label: "Lucene Segments", Stacked: true},
				{Name: "segments_index_writer_size", Label: "Lucene Segments Index Writer", Stacked: true},
				{Name: "segments_version_map_size", Label: "Lucene Segments Version Map", Stacked: true},
				{Name: "segments_fixed_bit_set_size", Label: "Lucene Segments Fixed Bit Set", Stacked: true},
			},
		},
		p.Prefix + ".indices.evictions": {
			Label: (p.LabelPrefix + " Indices Evictions"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "evictions_fielddata", Label: "Fielddata", Diff: true},
				{Name: "evictions_filter_cache", Label: "Filter Cache", Diff: true},
			},
		},
		p.Prefix + ".jvm.heap": {
			Label: (p.LabelPrefix + " JVM Heap Mem"),
			Unit:  "bytes",
			Metrics: []mp.Metrics{
				{Name: "heap_used", Label: "Used"},
				{Name: "heap_max", Label: "Max"},
			},
		},
		p.Prefix + ".thread_pool.threads": {
			Label: (p.LabelPrefix + " Thread-Pool Threads"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "threads_generic", Label: "Generic", Stacked: true},
				{Name: "threads_index", Label: "Index", Stacked: true},
				{Name: "threads_snapshot_data", Label: "Snapshot Data", Stacked: true},
				{Name: "threads_get", Label: "Get", Stacked: true},
				{Name: "threads_bench", Label: "Bench", Stacked: true},
				{Name: "threads_snapshot", Label: "Snapshot", Stacked: true},
				{Name: "threads_merge", Label: "Merge", Stacked: true},
				{Name: "threads_suggest", Label: "Suggest", Stacked: true},
				{Name: "threads_bulk", Label: "Bulk", Stacked: true},
				{Name: "threads_optimize", Label: "Optimize", Stacked: true},
				{Name: "threads_warmer", Label: "Warmer", Stacked: true},
				{Name: "threads_flush", Label: "Flush", Stacked: true},
				{Name: "threads_search", Label: "Search", Stacked: true},
				{Name: "threads_percolate", Label: "Percolate", Stacked: true},
				{Name: "threads_refresh", Label: "Refresh", Stacked: true},
				{Name: "threads_management", Label: "Management", Stacked: true},
				{Name: "threads_fetch_shard_started", Label: "Fetch Shard Started", Stacked: true},
				{Name: "threads_fetch_shard_store", Label: "Fetch Shard Store", Stacked: true},
				{Name: "threads_listener", Label: "Listener", Stacked: true},
			},
		},
		p.Prefix + ".transport.count": {
			Label: (p.LabelPrefix + " Transport Count"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "count_rx", Label: "TX", Diff: true},
				{Name: "count_tx", Label: "RX", Diff: true},
			},
		},
		p.Prefix + ".process": {
			Label: (p.LabelPrefix + " Process"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "open_file_descriptors", Label: "Open File Descriptors"},
			},
		},
	}

	return graphdef
}

// Do the plugin
func Do() {
	optScheme := flag.String("scheme", "http", "Scheme")
	optHost := flag.String("host", "localhost", "Host")
	optPort := flag.String("port", "9200", "Port")
	optPrefix := flag.String("metric-key-prefix", "elasticsearch", "Metric key prefix")
	optLabelPrefix := flag.String("metric-label-prefix", "", "Metric Label prefix")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var elasticsearch ElasticsearchPlugin
	elasticsearch.URI = fmt.Sprintf("%s://%s:%s", *optScheme, *optHost, *optPort)
	elasticsearch.Prefix = *optPrefix
	if *optLabelPrefix == "" {
		elasticsearch.LabelPrefix = strings.Title(*optPrefix)
	} else {
		elasticsearch.LabelPrefix = *optLabelPrefix
	}

	helper := mp.NewMackerelPlugin(elasticsearch)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.SetTempfileByBasename(fmt.Sprintf("mackerel-plugin-elasticsearch-%s-%s", *optHost, *optPort))
	}

	helper.Run()
}
<commit_msg>Add flags -insecure, -user, -password, -suppress-missing-error<commit_after>package mpelasticsearch

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"strings"

	mp "github.com/mackerelio/go-mackerel-plugin"
	"github.com/mackerelio/golib/logging"
)

var logger = logging.GetLogger("metrics.plugin.elasticsearch")

var metricPlace = map[string][]string{
	"http_opened":                 {"http", "total_opened"},
	"total_indexing_index":        {"indices", "indexing", "index_total"},
	"total_indexing_delete":       {"indices", "indexing", "delete_total"},
	"total_get":                   {"indices", "get", "total"},
	"total_search_query":          {"indices", "search", "query_total"},
	"total_search_fetch":          {"indices", "search", "fetch_total"},
	"total_merges":                {"indices", "merges", "total"},
	"total_refresh":               {"indices", "refresh", "total"},
	"total_flush":                 {"indices", "flush", "total"},
	"total_warmer":                {"indices", "warmer", "total"},
	"total_percolate":             {"indices", "percolate", "total"},
	"total_suggest":               {"indices", "suggest", "total"},
	"docs_count":                  {"indices", "docs", "count"},
	"docs_deleted":                {"indices", "docs", "deleted"},
	"fielddata_size":              {"indices", "fielddata", "memory_size_in_bytes"},
	"filter_cache_size":           {"indices", "filter_cache", "memory_size_in_bytes"},
	"segments_size":               {"indices", "segments", "memory_in_bytes"},
	"segments_index_writer_size":  {"indices", "segments", "index_writer_memory_in_bytes"},
	"segments_version_map_size":   {"indices", "segments", "version_map_memory_in_bytes"},
	"segments_fixed_bit_set_size": {"indices", "segments", "fixed_bit_set_memory_in_bytes"},
	"evictions_fielddata":         {"indices", "fielddata", "evictions"},
	"evictions_filter_cache":      {"indices", "filter_cache", "evictions"},
	"heap_used":                   {"jvm", "mem", "heap_used_in_bytes"},
	"heap_max":                    {"jvm", "mem", "heap_max_in_bytes"},
	"threads_generic":             {"thread_pool", "generic", "threads"},
	"threads_index":               {"thread_pool", "index", "threads"},
	"threads_snapshot_data":       {"thread_pool", "snapshot_data", "threads"},
	"threads_get":                 {"thread_pool", "get", "threads"},
	"threads_bench":               {"thread_pool", "bench", "threads"},
	"threads_snapshot":            {"thread_pool", "snapshot", "threads"},
	"threads_merge":               {"thread_pool", "merge", "threads"},
	"threads_suggest":             {"thread_pool", "suggest", "threads"},
	"threads_bulk":                {"thread_pool", "bulk", "threads"},
	"threads_optimize":            {"thread_pool", "optimize", "threads"},
	"threads_warmer":              {"thread_pool", "warmer", "threads"},
	"threads_flush":               {"thread_pool", "flush", "threads"},
	"threads_search":              {"thread_pool", "search", "threads"},
	"threads_percolate":           {"thread_pool", "percolate", "threads"},
	"threads_refresh":             {"thread_pool", "refresh", "threads"},
	"threads_management":          {"thread_pool", "management", "threads"},
	"threads_fetch_shard_started": {"thread_pool", "fetch_shard_started", "threads"},
	"threads_fetch_shard_store":   {"thread_pool", "fetch_shard_store", "threads"},
	"threads_listener":            {"thread_pool", "listener", "threads"},
	"count_rx":                    {"transport", "rx_count"},
	"count_tx":                    {"transport", "tx_count"},
	"open_file_descriptors":       {"process", "open_file_descriptors"},
}

func getFloatValue(s map[string]interface{}, keys []string) (float64, error) {
	var val float64
	sm := s
	for i, k := range keys {
		if i+1 < len(keys) {
			switch sm[k].(type) {
			case map[string]interface{}:
				sm = sm[k].(map[string]interface{})
			default:
				return 0, errors.New("Cannot handle as a hash")
			}
		} else {
			switch sm[k].(type) {
			case float64:
				val = sm[k].(float64)
			default:
				return 0, errors.New("Not float64")
			}
		}
	}

	return val, nil
}

// ElasticsearchPlugin mackerel plugin for Elasticsearch
type ElasticsearchPlugin struct {
	URI                  string
	Prefix               string
	LabelPrefix          string
	Insecure             bool
	User                 string
	Password             string
	SuppressMissingError bool
}

// FetchMetrics interface for mackerelplugin
func (p ElasticsearchPlugin) FetchMetrics() (map[string]float64, error) {
	req, err := http.NewRequest(http.MethodGet, p.URI+"/_nodes/_local/stats", nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", "mackerel-plugin-elasticsearch")
	if p.User != "" && p.Password != "" {
		req.SetBasicAuth(p.User, p.Password)
	}
	client := http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: p.Insecure},
		},
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	stat := make(map[string]float64)
	decoder := json.NewDecoder(resp.Body)

	var s map[string]interface{}
	err = decoder.Decode(&s)
	if err != nil {
		return nil, err
	}

	nodes := s["nodes"].(map[string]interface{})
	n := ""
	for k := range nodes {
		if n != "" {
			return nil, errors.New("Multiple node found")
		}
		n = k
	}
	node := nodes[n].(map[string]interface{})

	for k, v := range metricPlace {
		val, err := getFloatValue(node, v)
		if err != nil {
			if !p.SuppressMissingError {
				logger.Errorf("Failed to find '%s': %s", k, err)
			}
			continue
		}

		stat[k] = val
	}

	return stat, nil
}

// GraphDefinition interface for mackerelplugin
func (p ElasticsearchPlugin) GraphDefinition() map[string]mp.Graphs {
	var graphdef = map[string]mp.Graphs{
		p.Prefix + ".http": {
			Label: (p.LabelPrefix + " HTTP"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "http_opened", Label: "Opened", Diff: true},
			},
		},
		p.Prefix + ".indices": {
			Label: (p.LabelPrefix + " Indices"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "total_indexing_index", Label: "Indexing-Index", Diff: true, Stacked: true},
				{Name: "total_indexing_delete", Label: "Indexing-Delete", Diff: true, Stacked: true},
				{Name: "total_get", Label: "Get", Diff: true, Stacked: true},
				{Name: "total_search_query", Label: "Search-Query", Diff: true, Stacked: true},
				{Name: "total_search_fetch", Label: "Search-fetch", Diff: true, Stacked: true},
				{Name: "total_merges", Label: "Merges", Diff: true, Stacked: true},
				{Name: "total_refresh", Label: "Refresh", Diff: true, Stacked: true},
				{Name: "total_flush", Label: "Flush", Diff: true, Stacked: true},
				{Name: "total_warmer", Label: "Warmer", Diff: true, Stacked: true},
				{Name: "total_percolate", Label: "Percolate", Diff: true, Stacked: true},
				{Name: "total_suggest", Label: "Suggest", Diff: true, Stacked: true},
			},
		},
		p.Prefix + ".indices.docs": {
			Label: (p.LabelPrefix + " Indices Docs"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "docs_count", Label: "Count", Stacked: true},
				{Name: "docs_deleted", Label: "Deleted", Stacked: true},
			},
		},
		p.Prefix + ".indices.memory_size": {
			Label: (p.LabelPrefix + " Indices Memory Size"),
			Unit:  "bytes",
			Metrics: []mp.Metrics{
				{Name: "fielddata_size", Label: "Fielddata", Stacked: true},
				{Name: "filter_cache_size", Label: "Filter Cache", Stacked: true},
				{Name: "segments_size", Label: "Lucene Segments", Stacked: true},
				{Name: "segments_index_writer_size", Label: "Lucene Segments Index Writer", Stacked: true},
				{Name: "segments_version_map_size", Label: "Lucene Segments Version Map", Stacked: true},
				{Name: "segments_fixed_bit_set_size", Label: "Lucene Segments Fixed Bit Set", Stacked: true},
			},
		},
		p.Prefix + ".indices.evictions": {
			Label: (p.LabelPrefix + " Indices Evictions"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "evictions_fielddata", Label: "Fielddata", Diff: true},
				{Name: "evictions_filter_cache", Label: "Filter Cache", Diff: true},
			},
		},
		p.Prefix + ".jvm.heap": {
			Label: (p.LabelPrefix + " JVM Heap Mem"),
			Unit:  "bytes",
			Metrics: []mp.Metrics{
				{Name: "heap_used", Label: "Used"},
				{Name: "heap_max", Label: "Max"},
			},
		},
		p.Prefix + ".thread_pool.threads": {
			Label: (p.LabelPrefix + " Thread-Pool Threads"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "threads_generic", Label: "Generic", Stacked: true},
				{Name: "threads_index", Label: "Index", Stacked: true},
				{Name: "threads_snapshot_data", Label: "Snapshot Data", Stacked: true},
				{Name: "threads_get", Label: "Get", Stacked: true},
				{Name: "threads_bench", Label: "Bench", Stacked: true},
				{Name: "threads_snapshot", Label: "Snapshot", Stacked: true},
				{Name: "threads_merge", Label: "Merge", Stacked: true},
				{Name: "threads_suggest", Label: "Suggest", Stacked: true},
				{Name: "threads_bulk", Label: "Bulk", Stacked: true},
				{Name: "threads_optimize", Label: "Optimize", Stacked: true},
				{Name: "threads_warmer", Label: "Warmer", Stacked: true},
				{Name: "threads_flush", Label: "Flush", Stacked: true},
				{Name: "threads_search", Label: "Search", Stacked: true},
				{Name: "threads_percolate", Label: "Percolate", Stacked: true},
				{Name: "threads_refresh", Label: "Refresh", Stacked: true},
				{Name: "threads_management", Label: "Management", Stacked: true},
				{Name: "threads_fetch_shard_started", Label: "Fetch Shard Started", Stacked: true},
				{Name: "threads_fetch_shard_store", Label: "Fetch Shard Store", Stacked: true},
				{Name: "threads_listener", Label: "Listener", Stacked: true},
			},
		},
		p.Prefix + ".transport.count": {
			Label: (p.LabelPrefix + " Transport Count"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "count_rx", Label: "TX", Diff: true},
				{Name: "count_tx", Label: "RX", Diff: true},
			},
		},
		p.Prefix + ".process": {
			Label: (p.LabelPrefix + " Process"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "open_file_descriptors", Label: "Open File Descriptors"},
			},
		},
	}

	return graphdef
}

// Do the plugin
func Do() {
	optScheme := flag.String("scheme", "http", "Scheme")
	optHost := flag.String("host", "localhost", "Host")
	optPort := flag.String("port", "9200", "Port")
	optPrefix := flag.String("metric-key-prefix", "elasticsearch", "Metric key prefix")
	optLabelPrefix := flag.String("metric-label-prefix", "", "Metric Label prefix")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	optInsecure := flag.Bool("insecure", false, "Skip TLS certificate verification")
	optUser := flag.String("user", "", "Basic auth user")
	optPassword := flag.String("password", "", "Basic auth password")
	optSuppressMissingError := flag.Bool("suppress-missing-error", false, "Suppress ERROR for missing values")
	flag.Parse()

	var elasticsearch ElasticsearchPlugin
	elasticsearch.URI = fmt.Sprintf("%s://%s:%s", *optScheme, *optHost, *optPort)
	elasticsearch.Prefix = *optPrefix
	if *optLabelPrefix == "" {
		elasticsearch.LabelPrefix = strings.Title(*optPrefix)
	} else {
		elasticsearch.LabelPrefix = *optLabelPrefix
	}
	elasticsearch.Insecure = *optInsecure
	elasticsearch.User = *optUser
	elasticsearch.Password = *optPassword
	elasticsearch.SuppressMissingError = *optSuppressMissingError

	helper := mp.NewMackerelPlugin(elasticsearch)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.SetTempfileByBasename(fmt.Sprintf("mackerel-plugin-elasticsearch-%s-%s", *optHost, *optPort))
	}

	helper.Run()
}
<|endoftext|>


<commit_before>// Copyright 2016 Red Hat, Inc. and/or its affiliates
// and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package backend
package sqlite

import (
	"database/sql"
	"fmt"
	"log"
	"net/url"
	"regexp"

	_ "github.com/mattn/go-sqlite3"
	"github.com/yaacov/mohawk/backend"
)

type Backend struct {
	db *sql.DB
}

// Backend functions
// Required by backend interface

func (r Backend) Name() string {
	return "Backend-Sqlite3"
}

func (r *Backend) Open(options url.Values) {
	var err error
	var filename string

	// get backend options
	filename = options.Get("db-filename")
	if filename == "" {
		filename = "./server.db"
	}

	r.db, err = sql.Open("sqlite3", filename)
	if err != nil {
		log.Printf("%q\n", err)
	}

	sqlStmt := `
	create table if not exists ids (
		id    text,
		primary key (id));
	create table if not exists tags (
		id    text,
		tag   text,
		value text,
		primary key (id, tag));
	`
	_, err = r.db.Exec(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
		return
	}
}

func (r Backend) GetItemList(tags map[string]string) []backend.Item {
	res := make([]backend.Item, 0)

	// create one item per id
	sqlStmt := "select id from ids"
	rows, err := r.db.Query(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
	}
	defer rows.Close()
	for rows.Next() {
		var id string

		err = rows.Scan(&id)
		if err != nil {
			log.Printf("%q\n", err)
		}
		res = append(res, backend.Item{
			Id:   id,
			Type: "gauge",
			Tags: map[string]string{},
		})
	}
	err = rows.Err()
	if err != nil {
		log.Printf("%q\n", err)
	}

	// update item tags
	rows, err = r.db.Query("select id, tag, value from tags")
	if err != nil {
		log.Printf("%q\n", err)
	}
	defer rows.Close()
	for rows.Next() {
		var id string
		var tag string
		var value string

		err = rows.Scan(&id, &tag, &value)
		if err != nil {
			log.Printf("%q\n", err)
		}
		res = r.UpdateTag(res, id, tag, value)
	}
	err = rows.Err()
	if err != nil {
		log.Printf("%q\n", err)
	}

	// filter using tags
	// FIXME: we should do this in the sql
	if len(tags) > 0 {
		for key, value := range tags {
			res = backend.FilterItems(res, func(i backend.Item) bool {
				r, _ := regexp.Compile("^" + value + "$")
				return r.MatchString(i.Tags[key])
			})
		}
	}

	return res
}

func (r Backend) GetRawData(id string, end int64, start int64, limit int64, order string) []backend.DataItem {
	res := make([]backend.DataItem, 0)

	// check if id exist
	if !r.IdExist(id) {
		return res
	}

	// id exist, get timestamp, value pairs
	sqlStmt := fmt.Sprintf(`select timestamp, value
		from '%s'
		where timestamp > %d and timestamp <= %d
		order by timestamp %s limit %d`,
		id, start, end, order, limit)
	rows, err := r.db.Query(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
	}
	defer rows.Close()
	for rows.Next() {
		var timestamp int64
		var value float64

		err = rows.Scan(&timestamp, &value)
		if err != nil {
			log.Printf("%q\n", err)
		}
		res = append(res, backend.DataItem{
			Timestamp: timestamp,
			Value:     value,
		})
	}
	err = rows.Err()
	if err != nil {
		log.Printf("%q\n", err)
	}

	return res
}

func (r Backend) GetStatData(id string, end int64, start int64, limit int64, order string, bucketDuration int64) []backend.StatItem {
	var t int64
	res := make([]backend.StatItem, 0)

	// check if id exist
	if !r.IdExist(id) {
		return res
	}

	// id exist, get timestamp, value pairs
	sqlStmt := fmt.Sprintf(`select
		count(timestamp) as samples, cast((timestamp / %d) as integer) * %d as start, max(timestamp) as end,
		min(value) as min, max(value) as max, avg(value) as avg, sum(value) as sum
		from '%s'
		where timestamp > %d and timestamp <= %d
		group by start
		order by start %s limit %d`,
		bucketDuration*1000, bucketDuration*1000, id, start, end, order, limit)
	rows, err := r.db.Query(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
	}
	defer rows.Close()
	t = int64(start/(bucketDuration*1000)) * (bucketDuration * 1000)
	for rows.Next() {
		var samples int64
		var startT int64
		var endT int64
		var min float64
		var max float64
		var avg float64
		var sum float64

		err = rows.Scan(&samples, &startT, &endT, &min, &max, &avg, &sum)
		if err != nil {
			log.Printf("%q\n", err)
		}

		// append missing
		for t < (startT - bucketDuration*1000) {
			res = append(res, backend.StatItem{
				Start:   t,
				End:     t + bucketDuration*1000,
				Empty:   true,
				Samples: 0,
				Min:     0,
				Max:     0,
				Avg:     0,
				Median:  0,
				Sum:     0,
			})
			t += bucketDuration * 1000
		}

		// add data
		res = append(res, backend.StatItem{
			Start:   startT,
			End:     startT + bucketDuration*1000,
			Empty:   false,
			Samples: samples,
			Min:     min,
			Max:     max,
			Avg:     avg,
			Median:  0,
			Sum:     sum,
		})
		t += bucketDuration * 1000
	}
	err = rows.Err()
	if err != nil {
		log.Printf("%q\n", err)
	}

	// append missing
	for t < (end - bucketDuration*1000) {
		res = append(res, backend.StatItem{
			Start:   t,
			End:     t + bucketDuration*1000,
			Empty:   true,
			Samples: 0,
			Min:     0,
			Max:     0,
			Avg:     0,
			Median:  0,
			Sum:     0,
		})
		t += bucketDuration * 1000
	}

	return res
}

func (r Backend) PostRawData(id string, t int64, v float64) bool {
	// check if id exist
	if !r.IdExist(id) {
		r.createId(id)
	}

	r.insertData(id, t, v)
	return true
}

func (r Backend) PutTags(id string, tags map[string]string) bool {
	// check if id exist
	if !r.IdExist(id) {
		r.createId(id)
	}

	for k, v := range tags {
		r.insertTag(id, k, v)
	}
	return true
}

// Helper functions
// Not required by backend interface

func (r Backend) IdExist(id string) bool {
	var _id string
	sqlStmt := fmt.Sprintf("select id from ids where id='%s'", id)
	err := r.db.QueryRow(sqlStmt).Scan(&_id)
	return err != sql.ErrNoRows
}

func (r Backend) insertData(id string, t int64, v float64) {
	sqlStmt := fmt.Sprintf("insert into '%s' values (%d, %f)", id, t, v)
	_, err := r.db.Exec(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
	}
}

func (r Backend) insertTag(id string, k string, v string) {
	sqlStmt := fmt.Sprintf("insert or replace into tags values ('%s', '%s', '%s')", id, k, v)
	_, err := r.db.Exec(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
	}
}

func (r Backend) createId(id string) bool {
	sqlStmt := fmt.Sprintf("insert into ids values ('%s')", id)
	_, err := r.db.Exec(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
		return false
	}

	sqlStmt = fmt.Sprintf(`
	create table if not exists '%s' (
		timestamp integer,
		value     numeric,
		primary key (timestamp));
	`, id)

	_, err = r.db.Exec(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
		return false
	}

	return true
}

func (r Backend) UpdateTag(items []backend.Item, id string, tag string, value string) []backend.Item {
	// try to update tag if item exist
	for i, item := range items {
		if item.Id == id {
			items[i].Tags[tag] = value
			return items
		}
	}

	// if here we did not find a matching item
	items = append(items, backend.Item{
		Id:   id,
		Tags: map[string]string{tag: value},
	})

	return items
}
<commit_msg>we do not know to desc or limit stats<commit_after>// Copyright 2016 Red Hat, Inc. and/or its affiliates
// and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package backend
package sqlite

import (
	"database/sql"
	"fmt"
	"log"
	"net/url"
	"regexp"

	_ "github.com/mattn/go-sqlite3"
	"github.com/yaacov/mohawk/backend"
)

type Backend struct {
	db *sql.DB
}

// Backend functions
// Required by backend interface

func (r Backend) Name() string {
	return "Backend-Sqlite3"
}

func (r *Backend) Open(options url.Values) {
	var err error
	var filename string

	// get backend options
	filename = options.Get("db-filename")
	if filename == "" {
		filename = "./server.db"
	}

	r.db, err = sql.Open("sqlite3", filename)
	if err != nil {
		log.Printf("%q\n", err)
	}

	sqlStmt := `
	create table if not exists ids (
		id    text,
		primary key (id));
	create table if not exists tags (
		id    text,
		tag   text,
		value text,
		primary key (id, tag));
	`
	_, err = r.db.Exec(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
		return
	}
}

func (r Backend) GetItemList(tags map[string]string) []backend.Item {
	res := make([]backend.Item, 0)

	// create one item per id
	sqlStmt := "select id from ids"
	rows, err := r.db.Query(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
	}
	defer rows.Close()
	for rows.Next() {
		var id string

		err = rows.Scan(&id)
		if err != nil {
			log.Printf("%q\n", err)
		}
		res = append(res, backend.Item{
			Id:   id,
			Type: "gauge",
			Tags: map[string]string{},
		})
	}
	err = rows.Err()
	if err != nil {
		log.Printf("%q\n", err)
	}

	// update item tags
	rows, err = r.db.Query("select id, tag, value from tags")
	if err != nil {
		log.Printf("%q\n", err)
	}
	defer rows.Close()
	for rows.Next() {
		var id string
		var tag string
		var value string

		err = rows.Scan(&id, &tag, &value)
		if err != nil {
			log.Printf("%q\n", err)
		}
		res = r.UpdateTag(res, id, tag, value)
	}
	err = rows.Err()
	if err != nil {
		log.Printf("%q\n", err)
	}

	// filter using tags
	if len(tags) > 0 {
		for key, value := range tags {
			res = backend.FilterItems(res, func(i backend.Item) bool {
				r, _ := regexp.Compile("^" + value + "$")
				return r.MatchString(i.Tags[key])
			})
		}
	}

	return res
}

func (r Backend) GetRawData(id string, end int64, start int64, limit int64, order string) []backend.DataItem {
	res := make([]backend.DataItem, 0)

	// check if id exist
	if !r.IdExist(id) {
		return res
	}

	// id exist, get timestamp, value pairs
	sqlStmt := fmt.Sprintf(`select timestamp, value
		from '%s'
		where timestamp > %d and timestamp <= %d
		order by timestamp %s limit %d`,
		id, start, end, order, limit)
	rows, err := r.db.Query(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
	}
	defer rows.Close()
	for rows.Next() {
		var timestamp int64
		var value float64

		err = rows.Scan(&timestamp, &value)
		if err != nil {
			log.Printf("%q\n", err)
		}
		res = append(res, backend.DataItem{
			Timestamp: timestamp,
			Value:     value,
		})
	}
	err = rows.Err()
	if err != nil {
		log.Printf("%q\n", err)
	}

	return res
}

func (r Backend) GetStatData(id string, end int64, start int64, limit int64, order string, bucketDuration int64) []backend.StatItem {
	var t int64
	res := make([]backend.StatItem, 0)

	// check if id exist
	if !r.IdExist(id) {
		return res
	}

	// id exist, get timestamp, value pairs
	sqlStmt := fmt.Sprintf(`select
		count(timestamp) as samples, cast((timestamp / %d) as integer) * %d as start, max(timestamp) as end,
		min(value) as min, max(value) as max, avg(value) as avg, sum(value) as sum
		from '%s'
		where timestamp > %d and timestamp <= %d
		group by start
		order by start ASC`,
		bucketDuration*1000, bucketDuration*1000, id, start, end)
	rows, err := r.db.Query(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
	}
	defer rows.Close()
	t = int64(start/(bucketDuration*1000)) * (bucketDuration * 1000)
	for rows.Next() {
		var samples int64
		var startT int64
		var endT int64
		var min float64
		var max float64
		var avg float64
		var sum float64

		err = rows.Scan(&samples, &startT, &endT, &min, &max, &avg, &sum)
		if err != nil {
			log.Printf("%q\n", err)
		}

		// append missing
		for t < startT {
			res = append(res, backend.StatItem{
				Start:   t,
				End:     t + bucketDuration*1000,
				Empty:   true,
				Samples: 0,
				Min:     0,
				Max:     0,
				Avg:     0,
				Median:  0,
				Sum:     0,
			})
			t += bucketDuration * 1000
		}

		// add data
		res = append(res, backend.StatItem{
			Start:   startT,
			End:     startT + bucketDuration*1000,
			Empty:   false,
			Samples: samples,
			Min:     min,
			Max:     max,
			Avg:     avg,
			Median:  0,
			Sum:     sum,
		})
		t += bucketDuration * 1000
	}
	err = rows.Err()
	if err != nil {
		log.Printf("%q\n", err)
	}

	// append missing
	for t < end {
		res = append(res, backend.StatItem{
			Start:   t,
			End:     t + bucketDuration*1000,
			Empty:   true,
			Samples: 0,
			Min:     0,
			Max:     0,
			Avg:     0,
			Median:  0,
			Sum:     0,
		})
		t += bucketDuration * 1000
	}

	return res
}

func (r Backend) PostRawData(id string, t int64, v float64) bool {
	// check if id exist
	if !r.IdExist(id) {
		r.createId(id)
	}

	r.insertData(id, t, v)
	return true
}

func (r Backend) PutTags(id string, tags map[string]string) bool {
	// check if id exist
	if !r.IdExist(id) {
		r.createId(id)
	}

	for k, v := range tags {
		r.insertTag(id, k, v)
	}
	return true
}

// Helper functions
// Not required by backend interface

func (r Backend) IdExist(id string) bool {
	var _id string
	sqlStmt := fmt.Sprintf("select id from ids where id='%s'", id)
	err := r.db.QueryRow(sqlStmt).Scan(&_id)
	return err != sql.ErrNoRows
}

func (r Backend) insertData(id string, t int64, v float64) {
	sqlStmt := fmt.Sprintf("insert into '%s' values (%d, %f)", id, t, v)
	_, err := r.db.Exec(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
	}
}

func (r Backend) insertTag(id string, k string, v string) {
	sqlStmt := fmt.Sprintf("insert or replace into tags values ('%s', '%s', '%s')", id, k, v)
	_, err := r.db.Exec(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
	}
}

func (r Backend) createId(id string) bool {
	sqlStmt := fmt.Sprintf("insert into ids values ('%s')", id)
	_, err := r.db.Exec(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
		return false
	}

	sqlStmt = fmt.Sprintf(`
	create table if not exists '%s' (
		timestamp integer,
		value     numeric,
		primary key (timestamp));
	`, id)

	_, err = r.db.Exec(sqlStmt)
	if err != nil {
		log.Printf("%q: %s\n", err, sqlStmt)
		return false
	}

	return true
}

func (r Backend) UpdateTag(items []backend.Item, id string, tag string, value string) []backend.Item {
	// try to update tag if item exist
	for i, item := range items {
		if item.Id == id {
			items[i].Tags[tag] = value
			return items
		}
	}

	// if here we did not find a matching item
	items = append(items, backend.Item{
		Id:   id,
		Tags: map[string]string{tag: value},
	})

	return items
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2020 Red Hat, Inc.
 *
 */

package network

import (
	"fmt"
	"strconv"
	"time"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"

	batchv1 "k8s.io/api/batch/v1"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/client-go/kubecli"
	"kubevirt.io/kubevirt/tests"
	"kubevirt.io/kubevirt/tests/libnet"
	"kubevirt.io/kubevirt/tests/libvmi"
)

var _ = SIGDescribe("[Serial]Services", func() {
	var virtClient kubecli.KubevirtClient

	runTCPClientExpectingHelloWorldFromServer := func(host, port, namespace string) *batchv1.Job {
		job := tests.NewHelloWorldJob(host, port)
		job, err := virtClient.BatchV1().Jobs(namespace).Create(job)
		ExpectWithOffset(1, err).ToNot(HaveOccurred())
		return job
	}

	exposeExistingVMISpec := func(vmi *v1.VirtualMachineInstance, subdomain string, hostname string, selectorLabelKey string, selectorLabelValue string) *v1.VirtualMachineInstance {
		vmi.Labels = map[string]string{selectorLabelKey: selectorLabelValue}
		vmi.Spec.Subdomain = subdomain
		vmi.Spec.Hostname = hostname

		return vmi
	}

	readyVMI := func(vmi *v1.VirtualMachineInstance) *v1.VirtualMachineInstance {
		createdVMI, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
		Expect(err).ToNot(HaveOccurred())

		return tests.WaitUntilVMIReady(createdVMI, tests.LoggedInCirrosExpecter)
	}

	cleanupVMI := func(virtClient kubecli.KubevirtClient, vmi *v1.VirtualMachineInstance) {
		By("Deleting the VMI")
		Expect(virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Delete(vmi.GetName(), &k8smetav1.DeleteOptions{})).To(Succeed())

		By("Waiting for the VMI to be gone")
		Eventually(func() error {
			_, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.GetName(), &k8smetav1.GetOptions{})
			return err
		}, 2*time.Minute, time.Second).Should(SatisfyAll(HaveOccurred(), WithTransform(errors.IsNotFound, BeTrue())), "The VMI should be gone within the given timeout")
	}

	cleanupService := func(namespace string, serviceName string) error {
		return virtClient.CoreV1().Services(namespace).Delete(serviceName, &k8smetav1.DeleteOptions{})
	}

	assertConnectivityToService := func(serviceName, namespace string, servicePort int) (func() error, error) {
		serviceFQDN := fmt.Sprintf("%s.%s", serviceName, namespace)

		By(fmt.Sprintf("starting a job which tries to reach the vmi via service %s", serviceFQDN))
		job := runTCPClientExpectingHelloWorldFromServer(serviceFQDN, strconv.Itoa(servicePort), namespace)

		By(fmt.Sprintf("waiting for the job to report a SUCCESSFUL connection attempt to service %s on port %d", serviceFQDN, servicePort))
		err := tests.WaitForJobToSucceed(job, 90*time.Second)
		return func() error {
			return virtClient.BatchV1().Jobs(tests.NamespaceTestDefault).Delete(job.Name, &k8smetav1.DeleteOptions{})
		}, err
	}

	assertNoConnectivityToService := func(serviceName, namespace string, servicePort int) (func() error, error) {
		serviceFQDN := fmt.Sprintf("%s.%s", serviceName, namespace)

		By(fmt.Sprintf("starting a job which tries to reach the vmi via service %s", serviceFQDN))
		job := runTCPClientExpectingHelloWorldFromServer(serviceFQDN, strconv.Itoa(servicePort), namespace)

		By(fmt.Sprintf("waiting for the job to report a FAILED connection attempt to service %s on port %d", serviceFQDN, servicePort))
		err := tests.WaitForJobToFail(job, 90*time.Second)
		return func() error {
			return virtClient.BatchV1().Jobs(tests.NamespaceTestDefault).Delete(job.Name, &k8smetav1.DeleteOptions{})
		}, err
	}

	BeforeEach(func() {
		var err error
		virtClient, err = kubecli.GetKubevirtClient()
		Expect(err).NotTo(HaveOccurred(), "Should successfully initialize an API client")
	})

	Context("bridge interface binding", func() {
		var inboundVMI *v1.VirtualMachineInstance
		var serviceName string

		const (
			selectorLabelKey   = "expose"
			selectorLabelValue = "me"
			servicePort        = 1500
		)

		createVMISpecWithBridgeInterface := func() *v1.VirtualMachineInstance {
			return libvmi.NewCirros(
				libvmi.WithInterface(libvmi.InterfaceDeviceWithBridgeBinding()),
				libvmi.WithNetwork(v1.DefaultPodNetwork()))
		}

		createReadyVMIWithBridgeBindingAndExposedService := func(hostname string, subdomain string) *v1.VirtualMachineInstance {
			return readyVMI(
				exposeExistingVMISpec(
					createVMISpecWithBridgeInterface(), subdomain, hostname, selectorLabelKey, selectorLabelValue))
		}

		BeforeEach(func() {
			subdomain := "vmi"
			hostname := "inbound"

			inboundVMI = createReadyVMIWithBridgeBindingAndExposedService(hostname, subdomain)
			tests.StartTCPServer(inboundVMI, servicePort)
		})

		AfterEach(func() {
			Expect(inboundVMI).NotTo(BeNil(), "the VMI object must exist in order to be deleted.")
			cleanupVMI(virtClient, inboundVMI)
		})

		Context("with a service matching the vmi exposed", func() {
			var jobCleanup func() error

			BeforeEach(func() {
				serviceName = "myservice"

				service := buildServiceSpec(serviceName, servicePort, servicePort, selectorLabelKey, selectorLabelValue)
				_, err := virtClient.CoreV1().Services(inboundVMI.Namespace).Create(service)
				Expect(err).ToNot(HaveOccurred())
			})

			AfterEach(func() {
				Expect(cleanupService(inboundVMI.GetNamespace(), serviceName)).To(Succeed(), "cleaning up the k8sv1.Service entity should have succeeded.")
			})

			AfterEach(func() {
				Expect(jobCleanup).NotTo(BeNil(), "a k8sv1.Job cleaning up function should exist")
				Expect(jobCleanup()).To(Succeed(), "cleaning up the k8sv1.Job entity should have succeeded.")
				jobCleanup = nil
			})

			It("[test_id:1547] should be able to reach the vmi based on labels specified on the vmi", func() {
				var err error

				jobCleanup, err = assertConnectivityToService(serviceName, inboundVMI.Namespace, servicePort)
				Expect(err).NotTo(HaveOccurred(), "connectivity is expected to the exposed service")
			})

			It("[test_id:1548] should fail to reach the vmi if an invalid servicename is used", func() {
				var err error

				jobCleanup, err = assertNoConnectivityToService("wrongservice", inboundVMI.Namespace, servicePort)
				Expect(err).NotTo(HaveOccurred(), "connectivity is *not* expected, since there isn't an exposed service")
			})
		})

		Context("with a subdomain and a headless service given", func() {
			var jobCleanup func() error

			BeforeEach(func() {
				serviceName = inboundVMI.Spec.Subdomain

				service := buildHeadlessServiceSpec(serviceName, servicePort, servicePort, selectorLabelKey, selectorLabelValue)
				_, err := virtClient.CoreV1().Services(inboundVMI.Namespace).Create(service)
				Expect(err).ToNot(HaveOccurred())
			})

			AfterEach(func() {
				Expect(virtClient.CoreV1().Services(inboundVMI.Namespace).Delete(serviceName, &k8smetav1.DeleteOptions{})).To(Succeed())
			})

			AfterEach(func() {
				Expect(jobCleanup()).To(Succeed(), "cleaning up the k8sv1.Service entity should have succeeded.")
			})

			It("[test_id:1549]should be able to reach the vmi via its unique fully qualified domain name", func() {
				var err error
				serviceHostnameWithSubdomain := fmt.Sprintf("%s.%s", inboundVMI.Spec.Hostname, inboundVMI.Spec.Subdomain)

				jobCleanup, err = assertConnectivityToService(serviceHostnameWithSubdomain, inboundVMI.Namespace, servicePort)
				Expect(err).NotTo(HaveOccurred(), "connectivity is expected to the exposed service")
			})
		})
	})

	Context("Masquerade interface binding", func() {
		var inboundVMI *v1.VirtualMachineInstance

		const (
			selectorLabelKey   = "expose"
			selectorLabelValue = "me"
			servicePort        = 1500
		)

		createReadyVMIWithMasqueradeBindingAndExposedService := func(hostname string, subdomain string) *v1.VirtualMachineInstance {
			vmi := libvmi.NewCirros(
				libvmi.WithInterface(libvmi.InterfaceDeviceWithMasqueradeBinding()),
				libvmi.WithNetwork(v1.DefaultPodNetwork()))
			return readyVMI(
				exposeExistingVMISpec(vmi, subdomain, hostname, selectorLabelKey, selectorLabelValue))
		}

		BeforeEach(func() {
			subdomain := "vmi"
			hostname := "inbound"

			inboundVMI = createReadyVMIWithMasqueradeBindingAndExposedService(hostname, subdomain)
			tests.StartTCPServer(inboundVMI, servicePort)
		})

		AfterEach(func() {
			Expect(inboundVMI).NotTo(BeNil(), "the VMI object must exist in order to be deleted.")
			cleanupVMI(virtClient, inboundVMI)
		})

		Context("with a service matching the vmi exposed", func() {
			var jobCleanup func() error
			var service *k8sv1.Service

			AfterEach(func() {
				Expect(jobCleanup).NotTo(BeNil(), "a k8sv1.Job cleaning up function should exist")
				Expect(jobCleanup()).To(Succeed(), "cleaning up the k8sv1.Job entity should have succeeded.")
				jobCleanup = nil
			})

			AfterEach(func() {
				Expect(cleanupService(inboundVMI.GetNamespace(), service.Name)).To(Succeed(), "cleaning up the k8sv1.Service entity should have succeeded.")
			})

			table.DescribeTable("[Conformance] should be able to reach the vmi based on labels specified on the vmi", func(ipFamily k8sv1.IPFamily) {
				serviceName := "myservice"
				if ipFamily == k8sv1.IPv6Protocol {
					libnet.SkipWhenNotDualStackCluster(virtClient)

					serviceName = serviceName + "v6"
					service = buildIPv6ServiceSpec(serviceName, servicePort, servicePort, selectorLabelKey, selectorLabelValue)
				} else {
					service = buildServiceSpec(serviceName, servicePort, servicePort, selectorLabelKey, selectorLabelValue)
				}

				_, err := virtClient.CoreV1().Services(inboundVMI.Namespace).Create(service)
				Expect(err).NotTo(HaveOccurred(), "the k8sv1.Service entity should have been created.")

				jobCleanup, err = assertConnectivityToService(serviceName, inboundVMI.Namespace, servicePort)
				Expect(err).NotTo(HaveOccurred(), "connectivity is expected to the exposed service")
			},
				table.Entry("when the service is exposed by an IPv4 address.", k8sv1.IPv4Protocol),
				table.Entry("when the service is exposed by an IPv6 address.", k8sv1.IPv6Protocol),
			)
		})

		Context("*without* a service matching the vmi exposed", func() {
			var jobCleanup func() error
			var serviceName string

			AfterEach(func() {
				Expect(jobCleanup).NotTo(BeNil(), "a k8sv1.Job cleaning up function should exist")
				Expect(jobCleanup()).To(Succeed(), "cleaning up the k8sv1.Job entity should have succeeded.")
				jobCleanup = nil
			})

			It("should fail to reach the vmi", func() {
				var err error
				serviceName = "missingservice"

				jobCleanup, err = assertNoConnectivityToService(serviceName, inboundVMI.Namespace, servicePort)
				Expect(err).NotTo(HaveOccurred(), "connectivity is *not* expected, since there isn't an exposed service")
			})
		})
	})
})

func buildHeadlessServiceSpec(serviceName string, exposedPort int, portToExpose int, selectorKey string, selectorValue string) *k8sv1.Service {
	service := buildServiceSpec(serviceName, exposedPort, portToExpose, selectorKey, selectorValue)
	service.Spec.ClusterIP = k8sv1.ClusterIPNone
	return service
}

func buildIPv6ServiceSpec(serviceName string, exposedPort int, portToExpose int, selectorKey string, selectorValue string) *k8sv1.Service {
	service := buildServiceSpec(serviceName, exposedPort, portToExpose, selectorKey, selectorValue)
	ipv6Family := k8sv1.IPv6Protocol
	service.Spec.IPFamily = &ipv6Family

	return service
}

func buildServiceSpec(serviceName string, exposedPort int, portToExpose int, selectorKey string, selectorValue string) *k8sv1.Service {
	return &k8sv1.Service{
		ObjectMeta: k8smetav1.ObjectMeta{
			Name: serviceName,
		},
		Spec: k8sv1.ServiceSpec{
			Selector: map[string]string{
				selectorKey: selectorValue,
			},
			Ports: []k8sv1.ServicePort{
				{Protocol: k8sv1.ProtocolTCP, Port: int32(portToExpose), TargetPort: intstr.FromInt(exposedPort)},
			},
		},
	}
}
<commit_msg>dual stack, services, tests: enclose test setup in a `By` clause<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2020 Red Hat, Inc.
 *
 */

package network

import (
	"fmt"
	"strconv"
	"time"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"

	batchv1 "k8s.io/api/batch/v1"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/client-go/kubecli"
	"kubevirt.io/kubevirt/tests"
	"kubevirt.io/kubevirt/tests/libnet"
	"kubevirt.io/kubevirt/tests/libvmi"
)

var _ = SIGDescribe("[Serial]Services", func() {
	var virtClient kubecli.KubevirtClient

	runTCPClientExpectingHelloWorldFromServer := func(host, port, namespace string) *batchv1.Job {
		job := tests.NewHelloWorldJob(host, port)
		job, err := virtClient.BatchV1().Jobs(namespace).Create(job)
		ExpectWithOffset(1, err).ToNot(HaveOccurred())
		return job
	}

	exposeExistingVMISpec := func(vmi *v1.VirtualMachineInstance, subdomain string, hostname string, selectorLabelKey string, selectorLabelValue string) *v1.VirtualMachineInstance {
		vmi.Labels = map[string]string{selectorLabelKey: selectorLabelValue}
		vmi.Spec.Subdomain = subdomain
		vmi.Spec.Hostname = hostname

		return vmi
	}

	readyVMI := func(vmi *v1.VirtualMachineInstance) *v1.VirtualMachineInstance {
		createdVMI, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
		Expect(err).ToNot(HaveOccurred())

		return tests.WaitUntilVMIReady(createdVMI, tests.LoggedInCirrosExpecter)
	}

	cleanupVMI := func(virtClient kubecli.KubevirtClient, vmi *v1.VirtualMachineInstance) {
		By("Deleting the VMI")
		Expect(virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Delete(vmi.GetName(), &k8smetav1.DeleteOptions{})).To(Succeed())

		By("Waiting for the VMI to be gone")
		Eventually(func() error {
			_, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.GetName(), &k8smetav1.GetOptions{})
			return err
		}, 2*time.Minute, time.Second).Should(SatisfyAll(HaveOccurred(), WithTransform(errors.IsNotFound, BeTrue())), "The VMI should be gone within the given timeout")
	}

	cleanupService := func(namespace string, serviceName string) error {
		return virtClient.CoreV1().Services(namespace).Delete(serviceName, &k8smetav1.DeleteOptions{})
	}

	assertConnectivityToService := func(serviceName, namespace string, servicePort int) (func() error, error) {
		serviceFQDN := fmt.Sprintf("%s.%s", serviceName, namespace)

		By(fmt.Sprintf("starting a job which tries to reach the vmi via service %s", serviceFQDN))
		job := runTCPClientExpectingHelloWorldFromServer(serviceFQDN, strconv.Itoa(servicePort), namespace)

		By(fmt.Sprintf("waiting for the job to report a SUCCESSFUL connection attempt to service %s on port %d", serviceFQDN, servicePort))
		err := tests.WaitForJobToSucceed(job, 90*time.Second)
		return func() error {
			return virtClient.BatchV1().Jobs(tests.NamespaceTestDefault).Delete(job.Name, &k8smetav1.DeleteOptions{})
		}, err
	}

	assertNoConnectivityToService := func(serviceName, namespace string, servicePort int) (func() error, error) {
		serviceFQDN := fmt.Sprintf("%s.%s", serviceName, namespace)

		By(fmt.Sprintf("starting a job which tries to reach the vmi via service %s", serviceFQDN))
		job := runTCPClientExpectingHelloWorldFromServer(serviceFQDN, strconv.Itoa(servicePort), namespace)

		By(fmt.Sprintf("waiting for the job to report a FAILED connection attempt to service %s on port %d", serviceFQDN, servicePort))
		err := tests.WaitForJobToFail(job, 90*time.Second)
		return func() error {
			return virtClient.BatchV1().Jobs(tests.NamespaceTestDefault).Delete(job.Name, &k8smetav1.DeleteOptions{})
		}, err
	}

	BeforeEach(func() {
		var err error
		virtClient, err = kubecli.GetKubevirtClient()
		Expect(err).NotTo(HaveOccurred(), "Should successfully initialize an API client")
	})

	Context("bridge interface binding", func() {
		var inboundVMI *v1.VirtualMachineInstance
		var serviceName string

		const (
			selectorLabelKey   = "expose"
			selectorLabelValue = "me"
			servicePort        = 1500
		)

		createVMISpecWithBridgeInterface := func() *v1.VirtualMachineInstance {
			return libvmi.NewCirros(
				libvmi.WithInterface(libvmi.InterfaceDeviceWithBridgeBinding()),
				libvmi.WithNetwork(v1.DefaultPodNetwork()))
		}

		createReadyVMIWithBridgeBindingAndExposedService := func(hostname string, subdomain string) *v1.VirtualMachineInstance {
			return readyVMI(
				exposeExistingVMISpec(
					createVMISpecWithBridgeInterface(), subdomain, hostname, selectorLabelKey, selectorLabelValue))
		}

		BeforeEach(func() {
			subdomain := "vmi"
			hostname := "inbound"

			inboundVMI = createReadyVMIWithBridgeBindingAndExposedService(hostname, subdomain)
			tests.StartTCPServer(inboundVMI, servicePort)
		})

		AfterEach(func() {
			Expect(inboundVMI).NotTo(BeNil(), "the VMI object must exist in order to be deleted.")
			cleanupVMI(virtClient, inboundVMI)
		})

		Context("with a service matching the vmi exposed", func() {
			var jobCleanup func() error

			BeforeEach(func() {
				serviceName = "myservice"

				service := buildServiceSpec(serviceName, servicePort, servicePort, selectorLabelKey, selectorLabelValue)
				_, err := virtClient.CoreV1().Services(inboundVMI.Namespace).Create(service)
				Expect(err).ToNot(HaveOccurred())
			})

			AfterEach(func() {
				Expect(cleanupService(inboundVMI.GetNamespace(), serviceName)).To(Succeed(), "cleaning up the k8sv1.Service entity should have succeeded.")
			})

			AfterEach(func() {
				Expect(jobCleanup).NotTo(BeNil(), "a k8sv1.Job cleaning up function should exist")
				Expect(jobCleanup()).To(Succeed(), "cleaning up the k8sv1.Job entity should have succeeded.")
				jobCleanup = nil
			})

			It("[test_id:1547] should be able to reach the vmi based on labels specified on the vmi", func() {
				var err error

				jobCleanup, err = assertConnectivityToService(serviceName, inboundVMI.Namespace, servicePort)
				Expect(err).NotTo(HaveOccurred(), "connectivity is expected to the exposed service")
			})

			It("[test_id:1548] should fail to reach the vmi if an invalid servicename is used", func() {
				var err error

				jobCleanup, err = assertNoConnectivityToService("wrongservice", inboundVMI.Namespace, servicePort)
				Expect(err).NotTo(HaveOccurred(), "connectivity is *not* expected, since there isn't an exposed service")
			})
		})

		Context("with a subdomain and a headless service given", func() {
			var jobCleanup func() error

			BeforeEach(func() {
				serviceName = inboundVMI.Spec.Subdomain

				service := buildHeadlessServiceSpec(serviceName, servicePort, servicePort, selectorLabelKey, selectorLabelValue)
				_, err := virtClient.CoreV1().Services(inboundVMI.Namespace).Create(service)
				Expect(err).ToNot(HaveOccurred())
			})

			AfterEach(func() {
				Expect(virtClient.CoreV1().Services(inboundVMI.Namespace).Delete(serviceName, &k8smetav1.DeleteOptions{})).To(Succeed())
			})

			AfterEach(func() {
				Expect(jobCleanup()).To(Succeed(), "cleaning up the k8sv1.Service entity should have succeeded.")
			})

			It("[test_id:1549]should be able to reach the vmi via its unique fully qualified domain name", func() {
				var err error
				serviceHostnameWithSubdomain := fmt.Sprintf("%s.%s", inboundVMI.Spec.Hostname, inboundVMI.Spec.Subdomain)

				jobCleanup, err = assertConnectivityToService(serviceHostnameWithSubdomain, inboundVMI.Namespace, servicePort)
				Expect(err).NotTo(HaveOccurred(), "connectivity is expected to the exposed service")
			})
		})
	})

	Context("Masquerade interface binding", func() {
		var inboundVMI *v1.VirtualMachineInstance

		const (
			selectorLabelKey   = "expose"
			selectorLabelValue = "me"
			servicePort        = 1500
		)

		createReadyVMIWithMasqueradeBindingAndExposedService := func(hostname string, subdomain string) *v1.VirtualMachineInstance {
			vmi := libvmi.NewCirros(
				libvmi.WithInterface(libvmi.InterfaceDeviceWithMasqueradeBinding()),
				libvmi.WithNetwork(v1.DefaultPodNetwork()))
			return readyVMI(
				exposeExistingVMISpec(vmi, subdomain, hostname, selectorLabelKey, selectorLabelValue))
		}

		BeforeEach(func() {
			subdomain := "vmi"
			hostname := "inbound"

			inboundVMI = createReadyVMIWithMasqueradeBindingAndExposedService(hostname, subdomain)
			tests.StartTCPServer(inboundVMI, servicePort)
		})

		AfterEach(func() {
			Expect(inboundVMI).NotTo(BeNil(), "the VMI object must exist in order to be deleted.")
			cleanupVMI(virtClient, inboundVMI)
		})

		Context("with a service matching the vmi exposed", func() {
			var jobCleanup func() error
			var service *k8sv1.Service

			AfterEach(func() {
				Expect(jobCleanup).NotTo(BeNil(), "a k8sv1.Job cleaning up function should exist")
				Expect(jobCleanup()).To(Succeed(), "cleaning up the k8sv1.Job entity should have succeeded.")
				jobCleanup = nil
			})

			AfterEach(func() {
				Expect(cleanupService(inboundVMI.GetNamespace(), service.Name)).To(Succeed(), "cleaning up the k8sv1.Service entity should have succeeded.")
			})

			table.DescribeTable("[Conformance] should be able to reach the vmi based on labels specified on the vmi", func(ipFamily k8sv1.IPFamily) {
				serviceName := "myservice"
				By("setting up resources to expose the VMI via a service", func() {
					if ipFamily == k8sv1.IPv6Protocol {
						libnet.SkipWhenNotDualStackCluster(virtClient)

						serviceName = serviceName + "v6"
						service = buildIPv6ServiceSpec(serviceName, servicePort, servicePort, selectorLabelKey, selectorLabelValue)
					} else {
						service = buildServiceSpec(serviceName, servicePort, servicePort, selectorLabelKey, selectorLabelValue)
					}

					_, err := virtClient.CoreV1().Services(inboundVMI.Namespace).Create(service)
					Expect(err).NotTo(HaveOccurred(), "the k8sv1.Service entity should have been created.")
				})

				By("checking connectivity the the exposed service")
				var err error

				jobCleanup, err = assertConnectivityToService(serviceName, inboundVMI.Namespace, servicePort)
				Expect(err).NotTo(HaveOccurred(), "connectivity is expected to the exposed service")
			},
				table.Entry("when the service is exposed by an IPv4 address.", k8sv1.IPv4Protocol),
				table.Entry("when the service is exposed by an IPv6 address.", k8sv1.IPv6Protocol),
			)
		})

		Context("*without* a service matching the vmi exposed", func() {
			var jobCleanup func() error
			var serviceName string

			AfterEach(func() {
				Expect(jobCleanup).NotTo(BeNil(), "a k8sv1.Job cleaning up function should exist")
				Expect(jobCleanup()).To(Succeed(), "cleaning up the k8sv1.Job entity should have succeeded.")
				jobCleanup = nil
			})

			It("should fail to reach the vmi", func() {
				var err error
				serviceName = "missingservice"

				jobCleanup, err = assertNoConnectivityToService(serviceName, inboundVMI.Namespace, servicePort)
				Expect(err).NotTo(HaveOccurred(), "connectivity is *not* expected, since there isn't an exposed service")
			})
		})
	})
})

func buildHeadlessServiceSpec(serviceName string, exposedPort int, portToExpose int, selectorKey string, selectorValue string) *k8sv1.Service {
	service := buildServiceSpec(serviceName, exposedPort, portToExpose, selectorKey, selectorValue)
	service.Spec.ClusterIP = k8sv1.ClusterIPNone
	return service
}

func buildIPv6ServiceSpec(serviceName string, exposedPort int, portToExpose int, selectorKey string, selectorValue string) *k8sv1.Service {
	service := buildServiceSpec(serviceName, exposedPort, portToExpose, selectorKey, selectorValue)
	ipv6Family := k8sv1.IPv6Protocol
	service.Spec.IPFamily = &ipv6Family

	return service
}

func buildServiceSpec(serviceName string, exposedPort int, portToExpose int, selectorKey string, selectorValue string) *k8sv1.Service {
	return &k8sv1.Service{
		ObjectMeta: k8smetav1.ObjectMeta{
			Name: serviceName,
		},
		Spec: k8sv1.ServiceSpec{
			Selector: map[string]string{
				selectorKey: selectorValue,
			},
			Ports: []k8sv1.ServicePort{
				{Protocol: k8sv1.ProtocolTCP, Port: int32(portToExpose), TargetPort: intstr.FromInt(exposedPort)},
			},
		},
	}
}
<|endoftext|>


<commit_before>package quotasets

import (
	"github.com/mitchellh/mapstructure"
	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/pagination"
)

// QuotaSet is a set of operational limits that allow for control of compute usage.
type QuotaSet struct {
	//ID is tenant associated with this quota_set
	ID string `mapstructure:"id"`
	//FixedIps is number of fixed ips alloted this quota_set
	FixedIps int `mapstructure:"fixed_ips"`
	// FloatingIps is number of floatinh ips alloted this quota_set
	FloatingIps int `mapstructure:"floating_ips"`
	// InjectedFileContentBytes is content bytes allowed for each injected file
	InjectedFileContentBytes int `mapstructure:"injected_file_content_bytes"`
	// InjectedFilePathBytes is allowed bytes for each injected file path
	InjectedFilePathBytes int `mapstructure:"injected_file_path_bytes"`
	// InjectedFiles is injected files allowed for each project
	InjectedFiles int `mapstructure:"injected_files"`
	// KeyPairs is number of ssh keypairs
	KeyPairs int `mapstructure:"keypairs"`
	// MetadataItems is number of metadata items allowed for each instance
	MetadataItems int `mapstructure:"metadata_items"`
	// Ram is megabytes allowed for each instance
	Ram int `mapstructure:"ram"`
	// SecurityGroupRules is rules allowed for each security group
	SecurityGroupRules int `mapstructure:"security_group_rules"`
	// SecurityGroups security groups allowed for each project
	SecurityGroups int `mapstructure:"security_groups"`
	// Cores is number of instance cores allowed for each project
	Cores int `mapstructure:"cores"`
	// Instances is number of instances allowed for each project
	Instances int `mapstructure:"instances"`
}

// QuotaSetPage stores a single, only page of QuotaSet results from a List call.
type QuotaSetPage struct {
	pagination.SinglePageBase
}

// IsEmpty determines whether or not a QuotaSetsetPage is empty.
func (page QuotaSetPage) IsEmpty() (bool, error) {
	ks, err := ExtractQuotaSets(page)
	return len(ks) == 0, err
}

// ExtractQuotaSets interprets a page of results as a slice of QuotaSets.
func ExtractQuotaSets(page pagination.Page) ([]QuotaSet, error) {
	var resp struct {
		QuotaSets []QuotaSet `mapstructure:"quotas"`
	}

	err := mapstructure.Decode(page.(QuotaSetPage).Body, &resp)
	results := make([]QuotaSet, len(resp.QuotaSets))
	for i, q := range resp.QuotaSets {
		results[i] = q
	}
	return results, err
}

type quotaResult struct {
	gophercloud.Result
}

// Extract is a method that attempts to interpret any QuotaSet resource response as a QuotaSet struct.
func (r quotaResult) Extract() (*QuotaSet, error) {
	if r.Err != nil {
		return nil, r.Err
	}

	var res struct {
		QuotaSet *QuotaSet `json:"quota_set" mapstructure:"quota_set"`
	}

	err := mapstructure.Decode(r.Body, &res)
	return res.QuotaSet, err
}

// GetResult is the response from a Get operation. Call its Extract method to interpret it
// as a QuotaSet.
type GetResult struct {
	quotaResult
}
<commit_msg>spelling<commit_after>package quotasets

import (
	"github.com/mitchellh/mapstructure"
	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/pagination"
)

// QuotaSet is a set of operational limits that allow for control of compute usage.
type QuotaSet struct {
	//ID is tenant associated with this quota_set
	ID string `mapstructure:"id"`
	//FixedIps is number of fixed ips alloted this quota_set
	FixedIps int `mapstructure:"fixed_ips"`
	// FloatingIps is number of floating ips alloted this quota_set
	FloatingIps int `mapstructure:"floating_ips"`
	// InjectedFileContentBytes is content bytes allowed for each injected file
	InjectedFileContentBytes int `mapstructure:"injected_file_content_bytes"`
	// InjectedFilePathBytes is allowed bytes for each injected file path
	InjectedFilePathBytes int `mapstructure:"injected_file_path_bytes"`
	// InjectedFiles is injected files allowed for each project
	InjectedFiles int `mapstructure:"injected_files"`
	// KeyPairs is number of ssh keypairs
	KeyPairs int `mapstructure:"keypairs"`
	// MetadataItems is number of metadata items allowed for each instance
	MetadataItems int `mapstructure:"metadata_items"`
	// Ram is megabytes allowed for each instance
	Ram int `mapstructure:"ram"`
	// SecurityGroupRules is rules allowed for each security group
	SecurityGroupRules int `mapstructure:"security_group_rules"`
	// SecurityGroups security groups allowed for each project
	SecurityGroups int `mapstructure:"security_groups"`
	// Cores is number of instance cores allowed for each project
	Cores int `mapstructure:"cores"`
	// Instances is number of instances allowed for each project
	Instances int `mapstructure:"instances"`
}

// QuotaSetPage stores a single, only page of QuotaSet results from a List call.
type QuotaSetPage struct {
	pagination.SinglePageBase
}

// IsEmpty determines whether or not a QuotaSetsetPage is empty.
func (page QuotaSetPage) IsEmpty() (bool, error) {
	ks, err := ExtractQuotaSets(page)
	return len(ks) == 0, err
}

// ExtractQuotaSets interprets a page of results as a slice of QuotaSets.
func ExtractQuotaSets(page pagination.Page) ([]QuotaSet, error) {
	var resp struct {
		QuotaSets []QuotaSet `mapstructure:"quotas"`
	}

	err := mapstructure.Decode(page.(QuotaSetPage).Body, &resp)
	results := make([]QuotaSet, len(resp.QuotaSets))
	for i, q := range resp.QuotaSets {
		results[i] = q
	}
	return results, err
}

type quotaResult struct {
	gophercloud.Result
}

// Extract is a method that attempts to interpret any QuotaSet resource response as a QuotaSet struct.
func (r quotaResult) Extract() (*QuotaSet, error) {
	if r.Err != nil {
		return nil, r.Err
	}

	var res struct {
		QuotaSet *QuotaSet `json:"quota_set" mapstructure:"quota_set"`
	}

	err := mapstructure.Decode(r.Body, &res)
	return res.QuotaSet, err
}

// GetResult is the response from a Get operation. Call its Extract method to interpret it
// as a QuotaSet.
type GetResult struct {
	quotaResult
}
<|endoftext|>


<commit_before>package ftps

import (
	"bufio"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"net/textproto"
	"os"
	"strconv"
	"strings"
	"time"
)

type FTPS struct {
	host string

	conn net.Conn
	text *textproto.Conn

	Debug     bool
	TLSConfig tls.Config
}

func (ftps *FTPS) Connect(host string, port int) (err error) {

	ftps.host = host

	ftps.conn, err = net.Dial("tcp", fmt.Sprintf("%s:%d", host, port))
	if err != nil {
		return err
	}

	ftps.text = textproto.NewConn(ftps.conn)

	_, err = ftps.response(220)
	if err != nil {
		return err
	}

	_, err = ftps.request("AUTH TLS", 234)
	if err != nil {
		return err
	}

	ftps.conn = ftps.upgradeConnToTLS(ftps.conn)
	ftps.text = textproto.NewConn(ftps.conn) // TODO use sync or something similar?

	return
}

func (ftps *FTPS) isConnEstablished() {

	if ftps.conn == nil {
		panic("Connection is not established yet")
	}
}

func (ftps *FTPS) Login(username, password string) (err error) {

	ftps.isConnEstablished()

	_, err = ftps.request(fmt.Sprintf("USER %s", username), 331)
	if err != nil {
		return err
	}

	_, err = ftps.request(fmt.Sprintf("PASS %s", password), 230)
	if err != nil {
		return err
	}

	_, err = ftps.request("TYPE I", 200) // use binary mode
	if err != nil {
		return err
	}

	_, err = ftps.request("PBSZ 0", 200)
	if err != nil {
		return err
	}

	_, err = ftps.request("PROT P", 200) // encrypt data connection
	if err != nil {
		return err
	}

	return
}

func (ftps *FTPS) request(cmd string, expected int) (message string, err error) {

	ftps.isConnEstablished()

	ftps.debugInfo("<*cmd*> " + cmd)

	_, err = ftps.text.Cmd(cmd)
	if err != nil {
		return
	}

	message, err = ftps.response(expected)

	return
}

func (ftps *FTPS) requestDataConn(cmd string, expected int) (dataConn net.Conn, err error) {

	port, err := ftps.pasv()
	if err != nil {
		return
	}

	dataConn, err = ftps.openDataConn(port)
	if err != nil {
		return nil, err
	}

	_, err = ftps.request(cmd, expected)
	if err != nil {
		dataConn.Close()
		return nil, err
	}

	dataConn = ftps.upgradeConnToTLS(dataConn)

	return
}

func (ftps *FTPS) response(expected int) (message string, err error) {

	ftps.isConnEstablished()

	code, message, err := ftps.text.ReadResponse(expected)

	ftps.debugInfo(fmt.Sprintf("<*code*> %d", code))
	ftps.debugInfo("<*message*> " + message)

	return
}

func (ftps *FTPS) upgradeConnToTLS(conn net.Conn) (upgradedConn net.Conn) {

	var tlsConn *tls.Conn
	tlsConn = tls.Client(conn, &ftps.TLSConfig)

	tlsConn.Handshake()
	upgradedConn = net.Conn(tlsConn)

	// TODO verify that TLS connection is established

	return
}

func (ftps *FTPS) pasv() (port int, err error) {

	message, err := ftps.request("PASV", 227)
	if err != nil {
		return 0, err
	}

	start := strings.Index(message, "(")
	end := strings.LastIndex(message, ")")

	if start == -1 || end == -1 {
		err = errors.New("Invalid PASV response format")
		return 0, err
	}

	pasvData := strings.Split(message[start+1:end], ",")

	portPart1, err := strconv.Atoi(pasvData[4])
	if err != nil {
		return 0, err
	}

	portPart2, err := strconv.Atoi(pasvData[5])
	if err != nil {
		return 0, err
	}

	// Recompose port
	port = int(portPart1)*256 + int(portPart2)

	return
}

func (ftps *FTPS) PrintWorkingDirectory() (directory string, err error) {

	directory, err = ftps.request("PWD", 257)
	return
}

func (ftps *FTPS) ChangeWorkingDirectory(path string) (err error) {

	_, err = ftps.request(fmt.Sprintf("CWD %s", path), 250)
	return
}

func (ftps *FTPS) MakeDirectory(path string) (err error) {

	_, err = ftps.request(fmt.Sprintf("MKD %s", path), 257)
	return
}

func (ftps *FTPS) DeleteFile(path string) (err error) {

	_, err = ftps.request(fmt.Sprintf("DELE %s", path), 250)
	return
}

func (ftps *FTPS) RemoveDirectory(path string) (err error) {

	_, err = ftps.request(fmt.Sprintf("RMD %s", path), 250)
	return
}

func (ftps *FTPS) List() (entries []Entry, err error) {

	// TODO add support for MLSD

	dataConn, err := ftps.requestDataConn("LIST -a", 150) // TODO use also -L to resolve links?
	if err != nil {
		return
	}
	defer dataConn.Close()

	reader := bufio.NewReader(dataConn)
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF {
			break
		}

		entry, err := ftps.parseEntryLine(line)
		entries = append(entries, *entry)
	}
	dataConn.Close()

	_, err = ftps.response(226)
	if err != nil {
		return
	}

	return
}

func (ftps *FTPS) parseEntryLine(line string) (entry *Entry, err error) {

	// TODO Function mostly copied from https://github.com/jlaffaye/ftp

	fields := strings.Fields(line)
	if len(fields) < 9 {
		return nil, errors.New("Unsupported line format.")
	}

	entry = &Entry{}

	// parse type
	switch fields[0][0] {
	case '-':
		entry.Type = EntryTypeFile
	case 'd':
		entry.Type = EntryTypeFolder
	case 'l':
		entry.Type = EntryTypeLink
	default:
		return nil, errors.New("Unknown entry type.")
	}

	// parse size
	size, err := strconv.ParseUint(fields[4], 10, 0)
	if err != nil {
		return nil, err
	}
	entry.Size = size

	// parse time
	var timeStr string
	if strings.Contains(fields[7], ":") { // this year
		thisYear, _, _ := time.Now().Date()
		timeStr = fmt.Sprintf("%s %s %s %s GMT", fields[6], fields[5], strconv.Itoa(thisYear)[2:4], fields[7])
	} else { // not this year
		timeStr = fmt.Sprintf("%s %s %s 00:00 GMT", fields[6], fields[5], fields[7][2:4])
	}
	t, err := time.Parse("_2 Jan 06 15:04 MST", timeStr)
	if err != nil {
		return nil, err
	}
	entry.Time = t // TODO set timezone

	// parse name
	entry.Name = strings.Join(fields[8:], " ")

	return
}

func (ftps *FTPS) StoreFile(remoteFilepath string, data []byte) (err error) {

	dataConn, err := ftps.requestDataConn(fmt.Sprintf("STOR %s", remoteFilepath), 150)
	if err != nil {
		return
	}
	defer dataConn.Close()

	count, err := dataConn.Write(data)
	if err != nil {
		return
	}
	dataConn.Close()

	if len(data) != count {
		return errors.New("File transfer not complete.")
	}

	_, err = ftps.response(226)
	if err != nil {
		return
	}

	return
}

func (ftps *FTPS) RetrieveFile(remoteFilepath, localFilepath string) (err error) {

	dataConn, err := ftps.requestDataConn(fmt.Sprintf("RETR %s", remoteFilepath), 150)
	if err != nil {
		return
	}
	defer dataConn.Close()

	file, err := os.Create(localFilepath)
	if err != nil {
		return
	}
	defer file.Close()

	_, err = io.Copy(file, dataConn)
	if err != nil {
		return
	}
	dataConn.Close()

	_, err = ftps.response(226)
	if err != nil {
		return
	}

	return
}

func (ftps *FTPS) Quit() (err error) {

	_, err = ftps.request("QUIT", 221)
	if err != nil {
		return
	}
	ftps.conn.Close()

	return
}

func (ftps *FTPS) openDataConn(port int) (dataConn net.Conn, err error) {

	dataConn, err = net.Dial("tcp", fmt.Sprintf("%s:%d", ftps.host, port))
	if err != nil {
		return
	}

	return
}

func (ftps *FTPS) debugInfo(message string) {

	if ftps.Debug {
		log.Println(message)
	}
}
<commit_msg>changed error message<commit_after>package ftps

import (
	"bufio"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"net/textproto"
	"os"
	"strconv"
	"strings"
	"time"
)

type FTPS struct {
	host string

	conn net.Conn
	text *textproto.Conn

	Debug     bool
	TLSConfig tls.Config
}

func (ftps *FTPS) Connect(host string, port int) (err error) {

	ftps.host = host

	ftps.conn, err = net.Dial("tcp", fmt.Sprintf("%s:%d", host, port))
	if err != nil {
		return err
	}

	ftps.text = textproto.NewConn(ftps.conn)

	_, err = ftps.response(220)
	if err != nil {
		return err
	}

	_, err = ftps.request("AUTH TLS", 234)
	if err != nil {
		return err
	}

	ftps.conn = ftps.upgradeConnToTLS(ftps.conn)
	ftps.text = textproto.NewConn(ftps.conn) // TODO use sync or something similar?

	return
}

func (ftps *FTPS) isConnEstablished() {

	if ftps.conn == nil {
		panic("Connection is not established yet")
	}
}

func (ftps *FTPS) Login(username, password string) (err error) {

	ftps.isConnEstablished()

	_, err = ftps.request(fmt.Sprintf("USER %s", username), 331)
	if err != nil {
		return err
	}

	_, err = ftps.request(fmt.Sprintf("PASS %s", password), 230)
	if err != nil {
		return err
	}

	_, err = ftps.request("TYPE I", 200) // use binary mode
	if err != nil {
		return err
	}

	_, err = ftps.request("PBSZ 0", 200)
	if err != nil {
		return err
	}

	_, err = ftps.request("PROT P", 200) // encrypt data connection
	if err != nil {
		return err
	}

	return
}

func (ftps *FTPS) request(cmd string, expected int) (message string, err error) {

	ftps.isConnEstablished()

	ftps.debugInfo("<*cmd*> " + cmd)

	_, err = ftps.text.Cmd(cmd)
	if err != nil {
		return
	}

	message, err = ftps.response(expected)

	return
}

func (ftps *FTPS) requestDataConn(cmd string, expected int) (dataConn net.Conn, err error) {

	port, err := ftps.pasv()
	if err != nil {
		return
	}

	dataConn, err = ftps.openDataConn(port)
	if err != nil {
		return nil, err
	}

	_, err = ftps.request(cmd, expected)
	if err != nil {
		dataConn.Close()
		return nil, err
	}

	dataConn = ftps.upgradeConnToTLS(dataConn)

	return
}

func (ftps *FTPS) response(expected int) (message string, err error) {

	ftps.isConnEstablished()

	code, message, err := ftps.text.ReadResponse(expected)

	ftps.debugInfo(fmt.Sprintf("<*code*> %d", code))
	ftps.debugInfo("<*message*> " + message)

	return
}

func (ftps *FTPS) upgradeConnToTLS(conn net.Conn) (upgradedConn net.Conn) {

	var tlsConn *tls.Conn
	tlsConn = tls.Client(conn, &ftps.TLSConfig)

	tlsConn.Handshake()
	upgradedConn = net.Conn(tlsConn)

	// TODO verify that TLS connection is established

	return
}

func (ftps *FTPS) pasv() (port int, err error) {

	message, err := ftps.request("PASV", 227)
	if err != nil {
		return 0, err
	}

	start := strings.Index(message, "(")
	end := strings.LastIndex(message, ")")

	if start == -1 || end == -1 {
		err = errors.New("Invalid PASV response format")
		return 0, err
	}

	pasvData := strings.Split(message[start+1:end], ",")

	portPart1, err := strconv.Atoi(pasvData[4])
	if err != nil {
		return 0, err
	}

	portPart2, err := strconv.Atoi(pasvData[5])
	if err != nil {
		return 0, err
	}

	// Recompose port
	port = int(portPart1)*256 + int(portPart2)

	return
}

func (ftps *FTPS) PrintWorkingDirectory() (directory string, err error) {

	directory, err = ftps.request("PWD", 257)
	return
}

func (ftps *FTPS) ChangeWorkingDirectory(path string) (err error) {

	_, err = ftps.request(fmt.Sprintf("CWD %s", path), 250)
	return
}

func (ftps *FTPS) MakeDirectory(path string) (err error) {

	_, err = ftps.request(fmt.Sprintf("MKD %s", path), 257)
	return
}

func (ftps *FTPS) DeleteFile(path string) (err error) {

	_, err = ftps.request(fmt.Sprintf("DELE %s", path), 250)
	return
}

func (ftps *FTPS) RemoveDirectory(path string) (err error) {

	_, err = ftps.request(fmt.Sprintf("RMD %s", path), 250)
	return
}

func (ftps *FTPS) List() (entries []Entry, err error) {

	// TODO add support for MLSD

	dataConn, err := ftps.requestDataConn("LIST -a", 150) // TODO use also -L to resolve links?
	if err != nil {
		return
	}
	defer dataConn.Close()

	reader := bufio.NewReader(dataConn)
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF {
			break
		}

		entry, err := ftps.parseEntryLine(line)
		entries = append(entries, *entry)
	}
	dataConn.Close()

	_, err = ftps.response(226)
	if err != nil {
		return
	}

	return
}

func (ftps *FTPS) parseEntryLine(line string) (entry *Entry, err error) {

	// TODO Function mostly copied from https://github.com/jlaffaye/ftp

	fields := strings.Fields(line)
	if len(fields) < 9 {
		return nil, errors.New("Unsupported line format.")
	}

	entry = &Entry{}

	// parse type
	switch fields[0][0] {
	case '-':
		entry.Type = EntryTypeFile
	case 'd':
		entry.Type = EntryTypeFolder
	case 'l':
		entry.Type = EntryTypeLink
	default:
		return nil, errors.New("Unknown entry type.")
	}

	// parse size
	size, err := strconv.ParseUint(fields[4], 10, 0)
	if err != nil {
		return nil, err
	}
	entry.Size = size

	// parse time
	var timeStr string
	if strings.Contains(fields[7], ":") { // this year
		thisYear, _, _ := time.Now().Date()
		timeStr = fmt.Sprintf("%s %s %s %s GMT", fields[6], fields[5], strconv.Itoa(thisYear)[2:4], fields[7])
	} else { // not this year
		timeStr = fmt.Sprintf("%s %s %s 00:00 GMT", fields[6], fields[5], fields[7][2:4])
	}
	t, err := time.Parse("_2 Jan 06 15:04 MST", timeStr)
	if err != nil {
		return nil, err
	}
	entry.Time = t // TODO set timezone

	// parse name
	entry.Name = strings.Join(fields[8:], " ")

	return
}

func (ftps *FTPS) StoreFile(remoteFilepath string, data []byte) (err error) {

	dataConn, err := ftps.requestDataConn(fmt.Sprintf("STOR %s", remoteFilepath), 150)
	if err != nil {
		return
	}
	defer dataConn.Close()

	count, err := dataConn.Write(data)
	if err != nil {
		return
	}
	dataConn.Close()

	if len(data) != count {
		return errors.New("file transfer not complete")
	}

	_, err = ftps.response(226)
	if err != nil {
		return
	}

	return
}

func (ftps *FTPS) RetrieveFile(remoteFilepath, localFilepath string) (err error) {

	dataConn, err := ftps.requestDataConn(fmt.Sprintf("RETR %s", remoteFilepath), 150)
	if err != nil {
		return
	}
	defer dataConn.Close()

	file, err := os.Create(localFilepath)
	if err != nil {
		return
	}
	defer file.Close()

	_, err = io.Copy(file, dataConn)
	if err != nil {
		return
	}
	dataConn.Close()

	_, err = ftps.response(226)
	if err != nil {
		return
	}

	return
}

func (ftps *FTPS) Quit() (err error) {

	_, err = ftps.request("QUIT", 221)
	if err != nil {
		return
	}
	ftps.conn.Close()

	return
}

func (ftps *FTPS) openDataConn(port int) (dataConn net.Conn, err error) {

	dataConn, err = net.Dial("tcp", fmt.Sprintf("%s:%d", ftps.host, port))
	if err != nil {
		return
	}

	return
}

func (ftps *FTPS) debugInfo(message string) {

	if ftps.Debug {
		log.Println(message)
	}
}
<|endoftext|>