<commit_before>/*
 *    Copyright (C) 2014 Daniel 'grindhold' Brendle
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Lesser General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Daniel 'grindhold' Brendle <grindhold@skarphed.org>
 */

package gominatim

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
)

type searchResultError struct {
	error string `json:"error"`
}

type SearchResult struct {
	PlaceId       string     `json:"place_id"`
	License       string     `json:"license"`
	OsmType       string     `json:"osm_type"`
	OsmId         string     `json:"osm_id"`
	Boundingbox   []string   `json:"boundingbox"`
	Polygonpoints [][]string `json:"polygonpoints"`
	Lat           string     `json:"lat"`
	Lon           string     `json:"lon"`
	DisplayName   string     `json:"display_name"`
	Class         string     `json:"class"`
	Type          string     `json:"type"`
	Address       Address    `json:"address"`
	Importance    float32    `json:"importance""`
}

type SearchQuery struct {
	JsonCallback    interface{}
	AcceptLanguage  string
	Q               string
	Street          string
	City            string
	County          string
	State           string
	Postalcode      string
	Countrycodes    []string
	Viewbox         string
	Bounded         bool
	Polygon         bool
	Addressdetails  bool
	Email           string
	ExcludePlaceIds []string
	Limit           int
	PolygonGeojson  bool
	PolygonKml      bool
	PolygonText     bool
	PolygonSvg      bool
}

func (q *SearchQuery) specificFieldsUsed() bool {
	return q.Street != "" || q.City != "" || q.County != "" || q.State != "" || q.Postalcode != ""
}

func (q *SearchQuery) buildQuery() (string, error) {
	if server == "" {
		return "", errors.New("Server is not set. Set via gominatim.SetServer(srv string)")
	}
	s := server
	s = s + "/search?format=json"
	if q.JsonCallback != nil {
		cb, err := json.Marshal(q.JsonCallback)
		if err != nil {
			return "", err
		}
		s += "&json_callback=" + string(cb)
	}
	if q.AcceptLanguage != "" {
		s += "&accept-language=" + url.QueryEscape(q.AcceptLanguage)
		print(s)
	}
	if q.Q != "" {
		s += "&q=" + url.QueryEscape(q.Q)
	} else {
		if q.specificFieldsUsed() {
			if q.Street != "" {
				s += "&street=" + url.QueryEscape(q.Street)
			}
			if q.City != "" {
				s += "&city=" + url.QueryEscape(q.City)
			}
			if q.County != "" {
				s += "&county=" + url.QueryEscape(q.County)
			}
			if q.State != "" {
				s += "&state=" + url.QueryEscape(q.State)
			}
			if q.Postalcode != "" {
				s += "&postalcode=" + url.QueryEscape(q.Postalcode)
			}
		} else {
			return "", errors.New("You must use either Q or one or more of Street, City, County, State, Postalcode. The latter will be ignored if the further is used.")
		}
	}
	if q.Countrycodes != nil && len(q.Countrycodes) > 0 {
		als := ""
		first := true
		for i := range q.Countrycodes {
			if !first {
				als = als + ","
			}
			als = als + q.Countrycodes[i]
			if first {
				first = false
			}
		}
		s += "&countrycodes=" + url.QueryEscape(als)
	}
	if q.Viewbox != "" {
		s += "&viewbox=" + url.QueryEscape(q.Viewbox)
	}
	if q.Bounded {
		s += "&bounded=1"
	} else {
		s += "&bounded=0"
	}
	if q.Polygon {
		s += "&polygon=1"
	} else {
		s += "&polygon=0"
	}
	if q.Addressdetails {
		s += "&addressdetails=1"
	} else {
		s += "&addressdetails=0"
	}
	if q.Email != "" {
		s += "&email=" + url.QueryEscape(q.Email)
	}
	if q.ExcludePlaceIds != nil && len(q.ExcludePlaceIds) > 0 {
		als := ""
		first := true
		for i := range q.ExcludePlaceIds {
			if !first {
				als = als + ","
			}
			als = als + q.ExcludePlaceIds[i]
			if first {
				first = false
			}
		}
		s += "&exclude_place_ids=" + url.QueryEscape(als)
	}
	if q.Limit > 0 {
		s += "&limit=" + strconv.Itoa(q.Limit)
	}
	if q.PolygonGeojson {
		s += "&polygon_geojson=1"
	} else {
		s += "&polygon_geojson=0"
	}
	if q.PolygonKml {
		s += "&polygon_kml=1"
	} else {
		s += "&polygon_kml=0"
	}
	if q.PolygonSvg {
		s += "&polygon_svg=1"
	} else {
		s += "&polygon_svg=0"
	}
	if q.PolygonText {
		s += "&polygon_text=1"
	} else {
		s += "&polygon_text=0"
	}
	println(s)
	return s, nil
}

func (q *SearchQuery) Get() ([]SearchResult, error) {
	querystring, err := q.buildQuery()
	if err != nil {
		return nil, err
	}
	resp, err := http.Get(querystring)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	result := make([]SearchResult, 0)
	err = json.Unmarshal(body, &result)
	if err != nil {
		err_obj := new(searchResultError)
		err := json.Unmarshal(body, err_obj)
		if err != nil {
			return nil, err
		} else {
			return nil, errors.New(err_obj.error)
		}
	}
	if len(result) == 0 {
		return nil, errors.New("Nothing found; sorry :/")
	}
	return result, nil
}
<commit_msg>Remove debugging<commit_after>/*
 *    Copyright (C) 2014 Daniel 'grindhold' Brendle
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Lesser General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Daniel 'grindhold' Brendle <grindhold@skarphed.org>
 */

package gominatim

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
)

type searchResultError struct {
	error string `json:"error"`
}

type SearchResult struct {
	PlaceId       string     `json:"place_id"`
	License       string     `json:"license"`
	OsmType       string     `json:"osm_type"`
	OsmId         string     `json:"osm_id"`
	Boundingbox   []string   `json:"boundingbox"`
	Polygonpoints [][]string `json:"polygonpoints"`
	Lat           string     `json:"lat"`
	Lon           string     `json:"lon"`
	DisplayName   string     `json:"display_name"`
	Class         string     `json:"class"`
	Type          string     `json:"type"`
	Address       Address    `json:"address"`
	Importance    float32    `json:"importance""`
}

type SearchQuery struct {
	JsonCallback    interface{}
	AcceptLanguage  string
	Q               string
	Street          string
	City            string
	County          string
	State           string
	Postalcode      string
	Countrycodes    []string
	Viewbox         string
	Bounded         bool
	Polygon         bool
	Addressdetails  bool
	Email           string
	ExcludePlaceIds []string
	Limit           int
	PolygonGeojson  bool
	PolygonKml      bool
	PolygonText     bool
	PolygonSvg      bool
}

func (q *SearchQuery) specificFieldsUsed() bool {
	return q.Street != "" || q.City != "" || q.County != "" || q.State != "" || q.Postalcode != ""
}

func (q *SearchQuery) buildQuery() (string, error) {
	if server == "" {
		return "", errors.New("Server is not set. Set via gominatim.SetServer(srv string)")
	}
	s := server
	s = s + "/search?format=json"
	if q.JsonCallback != nil {
		cb, err := json.Marshal(q.JsonCallback)
		if err != nil {
			return "", err
		}
		s += "&json_callback=" + string(cb)
	}
	if q.AcceptLanguage != "" {
		s += "&accept-language=" + url.QueryEscape(q.AcceptLanguage)
	}
	if q.Q != "" {
		s += "&q=" + url.QueryEscape(q.Q)
	} else {
		if q.specificFieldsUsed() {
			if q.Street != "" {
				s += "&street=" + url.QueryEscape(q.Street)
			}
			if q.City != "" {
				s += "&city=" + url.QueryEscape(q.City)
			}
			if q.County != "" {
				s += "&county=" + url.QueryEscape(q.County)
			}
			if q.State != "" {
				s += "&state=" + url.QueryEscape(q.State)
			}
			if q.Postalcode != "" {
				s += "&postalcode=" + url.QueryEscape(q.Postalcode)
			}
		} else {
			return "", errors.New("You must use either Q or one or more of Street, City, County, State, Postalcode. The latter will be ignored if the further is used.")
		}
	}
	if q.Countrycodes != nil && len(q.Countrycodes) > 0 {
		als := ""
		first := true
		for i := range q.Countrycodes {
			if !first {
				als = als + ","
			}
			als = als + q.Countrycodes[i]
			if first {
				first = false
			}
		}
		s += "&countrycodes=" + url.QueryEscape(als)
	}
	if q.Viewbox != "" {
		s += "&viewbox=" + url.QueryEscape(q.Viewbox)
	}
	if q.Bounded {
		s += "&bounded=1"
	} else {
		s += "&bounded=0"
	}
	if q.Polygon {
		s += "&polygon=1"
	} else {
		s += "&polygon=0"
	}
	if q.Addressdetails {
		s += "&addressdetails=1"
	} else {
		s += "&addressdetails=0"
	}
	if q.Email != "" {
		s += "&email=" + url.QueryEscape(q.Email)
	}
	if q.ExcludePlaceIds != nil && len(q.ExcludePlaceIds) > 0 {
		als := ""
		first := true
		for i := range q.ExcludePlaceIds {
			if !first {
				als = als + ","
			}
			als = als + q.ExcludePlaceIds[i]
			if first {
				first = false
			}
		}
		s += "&exclude_place_ids=" + url.QueryEscape(als)
	}
	if q.Limit > 0 {
		s += "&limit=" + strconv.Itoa(q.Limit)
	}
	if q.PolygonGeojson {
		s += "&polygon_geojson=1"
	} else {
		s += "&polygon_geojson=0"
	}
	if q.PolygonKml {
		s += "&polygon_kml=1"
	} else {
		s += "&polygon_kml=0"
	}
	if q.PolygonSvg {
		s += "&polygon_svg=1"
	} else {
		s += "&polygon_svg=0"
	}
	if q.PolygonText {
		s += "&polygon_text=1"
	} else {
		s += "&polygon_text=0"
	}
	return s, nil
}

func (q *SearchQuery) Get() ([]SearchResult, error) {
	querystring, err := q.buildQuery()
	if err != nil {
		return nil, err
	}
	resp, err := http.Get(querystring)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	result := make([]SearchResult, 0)
	err = json.Unmarshal(body, &result)
	if err != nil {
		err_obj := new(searchResultError)
		err := json.Unmarshal(body, err_obj)
		if err != nil {
			return nil, err
		} else {
			return nil, errors.New(err_obj.error)
		}
	}
	if len(result) == 0 {
		return nil, errors.New("Nothing found; sorry :/")
	}
	return result, nil
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package productsearch

import (
	"bytes"
	"strings"
	"testing"

	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
)

func TestRemoveProductFromProductSet(t *testing.T) {
	tc := testutil.SystemTest(t)

	const location = "us-west1"
	const productSetDisplayName = "fake_product_set_display_name_for_testing"
	const productSetID = "fake_product_set_id_for_testing"
	const productDisplayName = "fake_product_display_name_for_testing"
	const productCategory = "homegoods"
	const productID = "fake_product_id_for_testing"

	var buf bytes.Buffer

	// Ensure re-used resource names don't exist prior to test start.
	if err := getProductSet(&buf, tc.ProjectID, location, productSetID); err == nil {
		deleteProductSet(&buf, tc.ProjectID, location, productSetID)
	}
	if err := getProduct(&buf, tc.ProjectID, location, productID); err == nil {
		deleteProduct(&buf, tc.ProjectID, location, productID)
	}
	buf.Reset()

	// Create fake product set and product.
	if err := createProductSet(&buf, tc.ProjectID, location, productSetID, productSetDisplayName); err != nil {
		t.Fatalf("createProductSet: %v", err)
	}
	if err := createProduct(&buf, tc.ProjectID, location, productID, productDisplayName, productCategory); err != nil {
		t.Fatalf("createProduct: %v", err)
	}

	// Add product to product set
	if err := addProductToProductSet(&buf, tc.ProjectID, location, productID, productSetID); err != nil {
		t.Fatalf("addProductToProductSet: %v", err)
	}

	// Confirm that the product is in the product set now.
	buf.Reset()
	if err := listProductsInProductSet(&buf, tc.ProjectID, location, productSetID); err != nil {
		t.Fatalf("listProductsInProductSet: %v", err)
	}
	if got := buf.String(); !strings.Contains(got, productID) {
		t.Errorf("Product ID %s is not in product set", productID)
	}

	// Remove product from product set.
	if err := removeProductFromProductSet(&buf, tc.ProjectID, location, productID, productSetID); err != nil {
		t.Fatalf("removeProductFromProductSet: %v", err)
	}

	// Check that the product is not in the product set now.
	buf.Reset()
	if err := listProductsInProductSet(&buf, tc.ProjectID, location, productSetID); err != nil {
		t.Fatalf("listProductsInProductSet: %v", err)
	}
	if got := buf.String(); strings.Contains(got, productID) {
		t.Errorf("Product ID %s still in product set", productID)
	}

	// Clean up.
	deleteProduct(&buf, tc.ProjectID, location, productID)
	deleteProductSet(&buf, tc.ProjectID, location, productSetID)
}
<commit_msg>vision/product_set: add retries to remove test (#1530)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package productsearch

import (
	"bytes"
	"strings"
	"testing"
	"time"

	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
)

func TestRemoveProductFromProductSet(t *testing.T) {
	tc := testutil.SystemTest(t)

	const location = "us-west1"
	const productSetDisplayName = "fake_product_set_display_name_for_testing"
	const productSetID = "fake_product_set_id_for_testing"
	const productDisplayName = "fake_product_display_name_for_testing"
	const productCategory = "homegoods"
	const productID = "fake_product_id_for_testing"

	testutil.Retry(t, 5, 10*time.Second, func(r *testutil.R) {
		var buf bytes.Buffer

		// Ensure re-used resource names don't exist prior to test start.
		if err := getProductSet(&buf, tc.ProjectID, location, productSetID); err == nil {
			deleteProductSet(&buf, tc.ProjectID, location, productSetID)
		}
		if err := getProduct(&buf, tc.ProjectID, location, productID); err == nil {
			deleteProduct(&buf, tc.ProjectID, location, productID)
		}
		buf.Reset()

		// Create fake product set and product.
		if err := createProductSet(&buf, tc.ProjectID, location, productSetID, productSetDisplayName); err != nil {
			r.Errorf("createProductSet: %v", err)
			return
		}
		if err := createProduct(&buf, tc.ProjectID, location, productID, productDisplayName, productCategory); err != nil {
			r.Errorf("createProduct: %v", err)
			return
		}

		// Add product to product set
		if err := addProductToProductSet(&buf, tc.ProjectID, location, productID, productSetID); err != nil {
			r.Errorf("addProductToProductSet: %v", err)
			return
		}

		// Confirm that the product is in the product set now.
		buf.Reset()
		if err := listProductsInProductSet(&buf, tc.ProjectID, location, productSetID); err != nil {
			r.Errorf("listProductsInProductSet: %v", err)
			return
		}
		if got := buf.String(); !strings.Contains(got, productID) {
			r.Errorf("Product ID %s is not in product set", productID)
		}

		// Remove product from product set.
		if err := removeProductFromProductSet(&buf, tc.ProjectID, location, productID, productSetID); err != nil {
			r.Errorf("removeProductFromProductSet: %v", err)
			return
		}

		// Check that the product is not in the product set now.
		buf.Reset()
		if err := listProductsInProductSet(&buf, tc.ProjectID, location, productSetID); err != nil {
			r.Errorf("listProductsInProductSet: %v", err)
			return
		}
		if got := buf.String(); strings.Contains(got, productID) {
			r.Errorf("Product ID %s still in product set", productID)
		}

		// Clean up.
		deleteProduct(&buf, tc.ProjectID, location, productID)
		deleteProductSet(&buf, tc.ProjectID, location, productSetID)
	})
}
<|endoftext|>


<commit_before>package repeater

import (
	lg "github.com/advantageous/go-logback/logging"
	m "github.com/advantageous/metricsd/metric"
	"github.com/advantageous/metricsd/util"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"strings"
	"time"
)

type AwsCloudMetricRepeater struct {
	logger lg.Logger
	conn   *cloudwatch.CloudWatch
	config *m.Config
}

func (cw AwsCloudMetricRepeater) ProcessMetrics(metrics []m.Metric) error {

	timestamp := aws.Time(time.Now())

	createDatum := func(name string, provider string) *cloudwatch.MetricDatum {

		dimensions := make([]*cloudwatch.Dimension, 0, 3)

		instanceIdDim := &cloudwatch.Dimension{
			Name:  aws.String("instanceId"),
			Value: aws.String(cw.config.EC2InstanceId),
		}
		dimensions = append(dimensions, instanceIdDim)

		if cw.config.IpAddress != "" {
			ipDim := &cloudwatch.Dimension{
				Name:  aws.String("ip"),
				Value: aws.String(cw.config.IpAddress),
			}
			dimensions = append(dimensions, ipDim)
		}

		if cw.config.Env != "" {
			dim := &cloudwatch.Dimension{
				Name:  aws.String("Environment"),
				Value: aws.String(cw.config.Env),
			}
			dimensions = append(dimensions, dim)
		}

		if cw.config.EC2InstanceNameTag != "" {
			dim := &cloudwatch.Dimension{
				Name:  aws.String("instanceName"),
				Value: aws.String(cw.config.EC2InstanceNameTag),
			}
			dimensions = append(dimensions, dim)
		}

		if cw.config.ServerRole != "" {
			dim := &cloudwatch.Dimension{
				Name:  aws.String("serverRole"),
				Value: aws.String(cw.config.ServerRole),
			}
			dimensions = append(dimensions, dim)
		}

		if provider != "" {
			dim := &cloudwatch.Dimension{
				Name:  aws.String("Provider"),
				Value: aws.String(provider),
			}
			dimensions = append(dimensions, dim)
		}
		return &cloudwatch.MetricDatum{
			MetricName: aws.String(name),
			Timestamp:  timestamp,
			Dimensions: dimensions,
		}
	}

	data := []*cloudwatch.MetricDatum{}

	var err error

	for index, d := range metrics {

		if cw.config.Debug {
			cw.logger.Printf("%s %d %d %s", d.GetName(), d.GetType(), d.GetValue(), d.GetProvider())
		}
		value := float64(d.GetValue())
		datum := createDatum(d.GetName(), d.GetProvider())
		datum.Value = aws.Float64(float64(value))

		switch d.GetType() {
		case m.COUNT:
			datum.Unit = aws.String(cloudwatch.StandardUnitCount)
		case m.LEVEL:
			datum.Unit = aws.String(cloudwatch.StandardUnitKilobytes)
		case m.LEVEL_PERCENT:
			datum.Unit = aws.String(cloudwatch.StandardUnitPercent)
		case m.TIMING:
			datum.Unit = aws.String(cloudwatch.StandardUnitMilliseconds)
		default:
			datum.Unit = aws.String(cloudwatch.StandardUnitCount)
		}

		data = append(data, datum)

		if index%20 == 0 && index != 0 {
			data = []*cloudwatch.MetricDatum{}

			if len(data) > 0 {
				request := &cloudwatch.PutMetricDataInput{
					Namespace:  aws.String(cw.config.NameSpace),
					MetricData: data,
				}
				_, err = cw.conn.PutMetricData(request)
				if err != nil {
					cw.logger.PrintError("Error writing metrics", err)
					cw.logger.Error("Error writing metrics", err, index)
				} else {
					if cw.config.Debug {
						cw.logger.Info("SENT..........................")
					}
				}
			}

		}

	}

	if len(data) > 0 {
		request := &cloudwatch.PutMetricDataInput{
			Namespace:  aws.String(cw.config.NameSpace),
			MetricData: data,
		}
		_, err = cw.conn.PutMetricData(request)

	}
	return err
}

func NewAwsCloudMetricRepeater(config *m.Config) AwsCloudMetricRepeater {
	session := util.NewAWSSession(config)
	logger := lg.NewSimpleLogger("log-repeater")
	return AwsCloudMetricRepeater{logger, cloudwatch.New(session), config}
}
<commit_msg>cleanup<commit_after>package repeater

import (
	lg "github.com/advantageous/go-logback/logging"
	m "github.com/advantageous/metricsd/metric"
	"github.com/advantageous/metricsd/util"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"time"
)

type AwsCloudMetricRepeater struct {
	logger lg.Logger
	conn   *cloudwatch.CloudWatch
	config *m.Config
}

func (cw AwsCloudMetricRepeater) ProcessMetrics(metrics []m.Metric) error {

	timestamp := aws.Time(time.Now())

	createDatum := func(name string, provider string) *cloudwatch.MetricDatum {

		dimensions := make([]*cloudwatch.Dimension, 0, 3)

		instanceIdDim := &cloudwatch.Dimension{
			Name:  aws.String("instanceId"),
			Value: aws.String(cw.config.EC2InstanceId),
		}
		dimensions = append(dimensions, instanceIdDim)

		if cw.config.IpAddress != "" {
			ipDim := &cloudwatch.Dimension{
				Name:  aws.String("ip"),
				Value: aws.String(cw.config.IpAddress),
			}
			dimensions = append(dimensions, ipDim)
		}

		if cw.config.Env != "" {
			dim := &cloudwatch.Dimension{
				Name:  aws.String("Environment"),
				Value: aws.String(cw.config.Env),
			}
			dimensions = append(dimensions, dim)
		}

		if cw.config.EC2InstanceNameTag != "" {
			dim := &cloudwatch.Dimension{
				Name:  aws.String("instanceName"),
				Value: aws.String(cw.config.EC2InstanceNameTag),
			}
			dimensions = append(dimensions, dim)
		}

		if cw.config.ServerRole != "" {
			dim := &cloudwatch.Dimension{
				Name:  aws.String("serverRole"),
				Value: aws.String(cw.config.ServerRole),
			}
			dimensions = append(dimensions, dim)
		}

		if provider != "" {
			dim := &cloudwatch.Dimension{
				Name:  aws.String("Provider"),
				Value: aws.String(provider),
			}
			dimensions = append(dimensions, dim)
		}
		return &cloudwatch.MetricDatum{
			MetricName: aws.String(name),
			Timestamp:  timestamp,
			Dimensions: dimensions,
		}
	}

	data := []*cloudwatch.MetricDatum{}

	var err error

	for index, d := range metrics {

		if cw.config.Debug {
			cw.logger.Printf("%s %d %d %s", d.GetName(), d.GetType(), d.GetValue(), d.GetProvider())
		}
		value := float64(d.GetValue())
		datum := createDatum(d.GetName(), d.GetProvider())
		datum.Value = aws.Float64(float64(value))

		switch d.GetType() {
		case m.COUNT:
			datum.Unit = aws.String(cloudwatch.StandardUnitCount)
		case m.LEVEL:
			datum.Unit = aws.String(cloudwatch.StandardUnitKilobytes)
		case m.LEVEL_PERCENT:
			datum.Unit = aws.String(cloudwatch.StandardUnitPercent)
		case m.TIMING:
			datum.Unit = aws.String(cloudwatch.StandardUnitMilliseconds)
		default:
			datum.Unit = aws.String(cloudwatch.StandardUnitCount)
		}

		data = append(data, datum)

		if index%20 == 0 && index != 0 {
			data = []*cloudwatch.MetricDatum{}

			if len(data) > 0 {
				request := &cloudwatch.PutMetricDataInput{
					Namespace:  aws.String(cw.config.NameSpace),
					MetricData: data,
				}
				_, err = cw.conn.PutMetricData(request)
				if err != nil {
					cw.logger.PrintError("Error writing metrics", err)
					cw.logger.Error("Error writing metrics", err, index)
				} else {
					if cw.config.Debug {
						cw.logger.Info("SENT..........................")
					}
				}
			}

		}

	}

	if len(data) > 0 {
		request := &cloudwatch.PutMetricDataInput{
			Namespace:  aws.String(cw.config.NameSpace),
			MetricData: data,
		}
		_, err = cw.conn.PutMetricData(request)

	}
	return err
}

func NewAwsCloudMetricRepeater(config *m.Config) AwsCloudMetricRepeater {
	session := util.NewAWSSession(config)
	logger := lg.NewSimpleLogger("log-repeater")
	return AwsCloudMetricRepeater{logger, cloudwatch.New(session), config}
}
<|endoftext|>


<commit_before>package stack

type Stack struct  {
	array []interface{}
	len int
}

func New() *Stack {
	//todo 初始化一个有初始容量的slice
	s := new(Stack)
	s.len = 0
	return s
}

//放入元素
func (this *Stack) Push(value ...interface{}) {
	this.array = append(this.array, value...)
	this.len ++
}

//返回下一个元素,并从Stack移除元素
func (this *Stack) Pop() interface{} {
	if this.len == 0 {
		return nil
	}
	this.len --
	value := this.array[this.len]
	this.array = append(this.array[:this.len])
	return value
}

//返回指定索引的元素
func (this *Stack) Get(i int) interface{} {
	return this.array[i]
}

//是否还有下一个元素
func (this *Stack) HasNext() bool {
	return this.len > 0
}

//func (this *Stack) String() string {
//	return strings.Join(this.array, ",")
//}<commit_msg>add stack.Top(); change stack.HasNext() to stack.IsEmpty()<commit_after>package stack

type Stack struct  {
	array []interface{}
	len int
}

func New() *Stack {
	//todo 初始化一个有初始容量的slice
	s := new(Stack)
	s.len = 0
	return s
}

//放入元素
func (this *Stack) Push(value ...interface{}) {
	this.array = append(this.array, value...)
	this.len ++
}

//返回下一个元素,并从Stack移除元素
func (this *Stack) Pop() interface{} {
	if this.len == 0 {
		return nil
	}
	this.len --
	value := this.array[this.len]
	this.array = append(this.array[:this.len])
	return value
}

//查看栈顶元素
func (this *Stack) Top() interface{} {
	return this.array[this.len-1]
}

//返回指定索引的元素
func (this *Stack) Get(i int) interface{} {
	return this.array[i]
}

//是否是空
func (this *Stack) IsEmpty() bool {
	return this.len == 0
}

//func (this *Stack) String() string {
//	return strings.Join(this.array, ",")
//}<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"

	"github.com/Sirupsen/logrus"
	"github.com/kubernetes-incubator/external-storage/iscsi/targetd/provisioner"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

var log = logrus.New()

// start-controllerCmd represents the start-controller command
var startcontrollerCmd = &cobra.Command{
	Use:   "start",
	Short: "Start an iscsi dynamic provisioner",
	Long:  `Start an iscsi dynamic provisioner`,
	Run: func(cmd *cobra.Command, args []string) {
		initLog()
		log.Debugln("start called")
		var config *rest.Config
		var err error
		master := viper.GetString("master")
		kubeconfig := viper.GetString("kubeconfig")
		// creates the in-cluster config
		log.Debugln("creating in cluster default kube client config")
		if master != "" || kubeconfig != "" {
			config, err = clientcmd.BuildConfigFromFlags(master, kubeconfig)
		} else {
			config, err = rest.InClusterConfig()
		}
		if err != nil {
			log.Fatalln(err)
		}
		log.WithFields(logrus.Fields{
			"config-host": config.Host,
		}).Debugln("kube client config created")

		// creates the clientset
		log.Debugln("creating kube client set")
		kubernetesClientSet, err := kubernetes.NewForConfig(config)
		if err != nil {
			log.Fatalln(err)
		}
		log.Debugln("kube client set created")

		// The controller needs to know what the server version is because out-of-tree
		// provisioners aren't officially supported until 1.5
		serverVersion, err := kubernetesClientSet.Discovery().ServerVersion()
		if err != nil {
			log.Fatalf("Error getting server version: %v", err)
		}

		url := fmt.Sprintf("%s://%s:%s@%s:%d/targetrpc", viper.GetString("targetd-scheme"), viper.GetString("targetd-username"), viper.GetString("targetd-password"), viper.GetString("targetd-address"), viper.GetInt("targetd-port"))

		log.Debugln("targed URL", url)

		iscsiProvisioner := provisioner.NewiscsiProvisioner(url)
		log.Debugln("iscsi provisioner created")

		pc := controller.NewProvisionController(kubernetesClientSet, viper.GetString("provisioner-name"), iscsiProvisioner, serverVersion.GitVersion)
		controller.ResyncPeriod(viper.GetDuration("resync-period"))
		controller.ExponentialBackOffOnError(viper.GetBool("exponential-backoff-on-error"))
		controller.FailedProvisionThreshold(viper.GetInt("fail-retry-threshold"))
		controller.FailedDeleteThreshold(viper.GetInt("fail-retry-threshold"))
		controller.LeaseDuration(viper.GetDuration("lease-period"))
		controller.RenewDeadline(viper.GetDuration("renew-deadline"))
		controller.TermLimit(viper.GetDuration("term-limit"))
		controller.RetryPeriod(viper.GetDuration("retry-period"))
		log.Debugln("iscsi controller created, running forever...")
		pc.Run(wait.NeverStop)
	},
}

func init() {
	RootCmd.AddCommand(startcontrollerCmd)
	startcontrollerCmd.Flags().String("provisioner-name", "iscsi-targetd", "name of this provisioner, must match what is passed in the storage class annotation")
	viper.BindPFlag("provisioner-name", startcontrollerCmd.Flags().Lookup("provisioner-name"))
	startcontrollerCmd.Flags().Duration("resync-period", controller.DefaultResyncPeriod, "how often to poll the master API for updates")
	viper.BindPFlag("resync-period", startcontrollerCmd.Flags().Lookup("resync-period"))
	startcontrollerCmd.Flags().Bool("exponential-backoff-on-error", controller.DefaultExponentialBackOffOnError, "exponential-backoff-on-error doubles the retry-period everytime there is an error")
	viper.BindPFlag("exponential-backoff-on-error", startcontrollerCmd.Flags().Lookup("exponential-backoff-on-error"))
	startcontrollerCmd.Flags().Int("fail-retry-threshold", controller.DefaultFailedProvisionThreshold, "Threshold for max number of retries on failure of provisioner")
	viper.BindPFlag("fail-retry-threshold", startcontrollerCmd.Flags().Lookup("fail-retry-threshold"))
	startcontrollerCmd.Flags().Duration("lease-period", controller.DefaultLeaseDuration, "LeaseDuration is the duration that non-leader candidates will wait to force acquire leadership. This is measured against time of last observed ack")
	viper.BindPFlag("lease-period", startcontrollerCmd.Flags().Lookup("lease-period"))
	startcontrollerCmd.Flags().Duration("renew-deadline", controller.DefaultRenewDeadline, "RenewDeadline is the duration that the acting master will retry refreshing leadership before giving up")
	viper.BindPFlag("renew-deadline", startcontrollerCmd.Flags().Lookup("renew-deadline"))
	startcontrollerCmd.Flags().Duration("retry-period", controller.DefaultRetryPeriod, "RetryPeriod is the duration the LeaderElector clients should wait between tries of actions")
	viper.BindPFlag("retry-period", startcontrollerCmd.Flags().Lookup("retry-period"))
	startcontrollerCmd.Flags().Duration("term-limit", controller.DefaultTermLimit, "TermLimit is the maximum duration that a leader may remain the leader to complete the task before it must give up its leadership. 0 for forever or indefinite.")
	viper.BindPFlag("term-limit", startcontrollerCmd.Flags().Lookup("term-limit"))
	startcontrollerCmd.Flags().String("targetd-scheme", "http", "scheme of the targetd connection, can be http or https")
	viper.BindPFlag("targetd-scheme", startcontrollerCmd.Flags().Lookup("targetd-scheme"))
	startcontrollerCmd.Flags().String("targetd-username", "admin", "username for the targetd connection")
	viper.BindPFlag("targetd-username", startcontrollerCmd.Flags().Lookup("targetd-username"))
	startcontrollerCmd.Flags().String("targetd-password", "", "password for the targetd connection")
	viper.BindPFlag("targetd-password", startcontrollerCmd.Flags().Lookup("targetd-password"))
	startcontrollerCmd.Flags().String("targetd-address", "localhost", "ip or dns of the targetd server")
	viper.BindPFlag("targetd-address", startcontrollerCmd.Flags().Lookup("targetd-address"))
	startcontrollerCmd.Flags().Int("targetd-port", 18700, "port on which targetd is listening")
	viper.BindPFlag("targetd-port", startcontrollerCmd.Flags().Lookup("targetd-port"))
	startcontrollerCmd.Flags().String("default-fs", "xfs", "filesystem to use when not specified")
	viper.BindPFlag("default-fs", startcontrollerCmd.Flags().Lookup("default-fs"))
	startcontrollerCmd.Flags().String("master", "", "Master URL")
	viper.BindPFlag("master", startcontrollerCmd.Flags().Lookup("master"))
	startcontrollerCmd.Flags().String("kubeconfig", "", "Absolute path to the kubeconfig")
	viper.BindPFlag("kubeconfig", startcontrollerCmd.Flags().Lookup("kubeconfig"))
	startcontrollerCmd.Flags().String("session-chap-credential-file-path", "/var/run/secrets/iscsi-provisioner/session-chap-credential.properties", "path where the credential for session chap authentication can be found")
	viper.BindPFlag("session-chap-credential-file-path", startcontrollerCmd.Flags().Lookup("session-chap-credential-file-path"))

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// start-controllerCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// start-controllerCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")

}

func initLog() {
	var err error
	log.Level, err = logrus.ParseLevel(viper.GetString("log-level"))
	if err != nil {
		log.Fatalln(err)
	}
}
<commit_msg>Set iscsi-targetd threadiness to 1 to avoid LUN number race<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"

	"github.com/Sirupsen/logrus"
	"github.com/kubernetes-incubator/external-storage/iscsi/targetd/provisioner"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

var log = logrus.New()

// start-controllerCmd represents the start-controller command
var startcontrollerCmd = &cobra.Command{
	Use:   "start",
	Short: "Start an iscsi dynamic provisioner",
	Long:  `Start an iscsi dynamic provisioner`,
	Run: func(cmd *cobra.Command, args []string) {
		initLog()
		log.Debugln("start called")
		var config *rest.Config
		var err error
		master := viper.GetString("master")
		kubeconfig := viper.GetString("kubeconfig")
		// creates the in-cluster config
		log.Debugln("creating in cluster default kube client config")
		if master != "" || kubeconfig != "" {
			config, err = clientcmd.BuildConfigFromFlags(master, kubeconfig)
		} else {
			config, err = rest.InClusterConfig()
		}
		if err != nil {
			log.Fatalln(err)
		}
		log.WithFields(logrus.Fields{
			"config-host": config.Host,
		}).Debugln("kube client config created")

		// creates the clientset
		log.Debugln("creating kube client set")
		kubernetesClientSet, err := kubernetes.NewForConfig(config)
		if err != nil {
			log.Fatalln(err)
		}
		log.Debugln("kube client set created")

		// The controller needs to know what the server version is because out-of-tree
		// provisioners aren't officially supported until 1.5
		serverVersion, err := kubernetesClientSet.Discovery().ServerVersion()
		if err != nil {
			log.Fatalf("Error getting server version: %v", err)
		}

		url := fmt.Sprintf("%s://%s:%s@%s:%d/targetrpc", viper.GetString("targetd-scheme"), viper.GetString("targetd-username"), viper.GetString("targetd-password"), viper.GetString("targetd-address"), viper.GetInt("targetd-port"))

		log.Debugln("targed URL", url)

		iscsiProvisioner := provisioner.NewiscsiProvisioner(url)
		log.Debugln("iscsi provisioner created")

		pc := controller.NewProvisionController(kubernetesClientSet, viper.GetString("provisioner-name"), iscsiProvisioner, serverVersion.GitVersion, controller.Threadiness(1))
		controller.ResyncPeriod(viper.GetDuration("resync-period"))
		controller.ExponentialBackOffOnError(viper.GetBool("exponential-backoff-on-error"))
		controller.FailedProvisionThreshold(viper.GetInt("fail-retry-threshold"))
		controller.FailedDeleteThreshold(viper.GetInt("fail-retry-threshold"))
		controller.LeaseDuration(viper.GetDuration("lease-period"))
		controller.RenewDeadline(viper.GetDuration("renew-deadline"))
		controller.TermLimit(viper.GetDuration("term-limit"))
		controller.RetryPeriod(viper.GetDuration("retry-period"))
		log.Debugln("iscsi controller created, running forever...")
		pc.Run(wait.NeverStop)
	},
}

func init() {
	RootCmd.AddCommand(startcontrollerCmd)
	startcontrollerCmd.Flags().String("provisioner-name", "iscsi-targetd", "name of this provisioner, must match what is passed in the storage class annotation")
	viper.BindPFlag("provisioner-name", startcontrollerCmd.Flags().Lookup("provisioner-name"))
	startcontrollerCmd.Flags().Duration("resync-period", controller.DefaultResyncPeriod, "how often to poll the master API for updates")
	viper.BindPFlag("resync-period", startcontrollerCmd.Flags().Lookup("resync-period"))
	startcontrollerCmd.Flags().Bool("exponential-backoff-on-error", controller.DefaultExponentialBackOffOnError, "exponential-backoff-on-error doubles the retry-period everytime there is an error")
	viper.BindPFlag("exponential-backoff-on-error", startcontrollerCmd.Flags().Lookup("exponential-backoff-on-error"))
	startcontrollerCmd.Flags().Int("fail-retry-threshold", controller.DefaultFailedProvisionThreshold, "Threshold for max number of retries on failure of provisioner")
	viper.BindPFlag("fail-retry-threshold", startcontrollerCmd.Flags().Lookup("fail-retry-threshold"))
	startcontrollerCmd.Flags().Duration("lease-period", controller.DefaultLeaseDuration, "LeaseDuration is the duration that non-leader candidates will wait to force acquire leadership. This is measured against time of last observed ack")
	viper.BindPFlag("lease-period", startcontrollerCmd.Flags().Lookup("lease-period"))
	startcontrollerCmd.Flags().Duration("renew-deadline", controller.DefaultRenewDeadline, "RenewDeadline is the duration that the acting master will retry refreshing leadership before giving up")
	viper.BindPFlag("renew-deadline", startcontrollerCmd.Flags().Lookup("renew-deadline"))
	startcontrollerCmd.Flags().Duration("retry-period", controller.DefaultRetryPeriod, "RetryPeriod is the duration the LeaderElector clients should wait between tries of actions")
	viper.BindPFlag("retry-period", startcontrollerCmd.Flags().Lookup("retry-period"))
	startcontrollerCmd.Flags().Duration("term-limit", controller.DefaultTermLimit, "TermLimit is the maximum duration that a leader may remain the leader to complete the task before it must give up its leadership. 0 for forever or indefinite.")
	viper.BindPFlag("term-limit", startcontrollerCmd.Flags().Lookup("term-limit"))
	startcontrollerCmd.Flags().String("targetd-scheme", "http", "scheme of the targetd connection, can be http or https")
	viper.BindPFlag("targetd-scheme", startcontrollerCmd.Flags().Lookup("targetd-scheme"))
	startcontrollerCmd.Flags().String("targetd-username", "admin", "username for the targetd connection")
	viper.BindPFlag("targetd-username", startcontrollerCmd.Flags().Lookup("targetd-username"))
	startcontrollerCmd.Flags().String("targetd-password", "", "password for the targetd connection")
	viper.BindPFlag("targetd-password", startcontrollerCmd.Flags().Lookup("targetd-password"))
	startcontrollerCmd.Flags().String("targetd-address", "localhost", "ip or dns of the targetd server")
	viper.BindPFlag("targetd-address", startcontrollerCmd.Flags().Lookup("targetd-address"))
	startcontrollerCmd.Flags().Int("targetd-port", 18700, "port on which targetd is listening")
	viper.BindPFlag("targetd-port", startcontrollerCmd.Flags().Lookup("targetd-port"))
	startcontrollerCmd.Flags().String("default-fs", "xfs", "filesystem to use when not specified")
	viper.BindPFlag("default-fs", startcontrollerCmd.Flags().Lookup("default-fs"))
	startcontrollerCmd.Flags().String("master", "", "Master URL")
	viper.BindPFlag("master", startcontrollerCmd.Flags().Lookup("master"))
	startcontrollerCmd.Flags().String("kubeconfig", "", "Absolute path to the kubeconfig")
	viper.BindPFlag("kubeconfig", startcontrollerCmd.Flags().Lookup("kubeconfig"))
	startcontrollerCmd.Flags().String("session-chap-credential-file-path", "/var/run/secrets/iscsi-provisioner/session-chap-credential.properties", "path where the credential for session chap authentication can be found")
	viper.BindPFlag("session-chap-credential-file-path", startcontrollerCmd.Flags().Lookup("session-chap-credential-file-path"))

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// start-controllerCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// start-controllerCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")

}

func initLog() {
	var err error
	log.Level, err = logrus.ParseLevel(viper.GetString("log-level"))
	if err != nil {
		log.Fatalln(err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logic

import (
	"context"
	"fmt"
	"time"

	"golang.org/x/time/rate"

	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"
	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
	vpa_lister "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/eviction"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/priority"
	metrics_updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/updater"
	vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa"
	kube_client "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/fake"
	"k8s.io/client-go/kubernetes/scheme"
	clientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
	v1lister "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/klog"
)

// Updater performs updates on pods if recommended by Vertical Pod Autoscaler
type Updater interface {
	// RunOnce represents single iteration in the main-loop of Updater
	RunOnce(context.Context)
}

type updater struct {
	vpaLister               vpa_lister.VerticalPodAutoscalerLister
	podLister               v1lister.PodLister
	eventRecorder           record.EventRecorder
	evictionFactory         eviction.PodsEvictionRestrictionFactory
	recommendationProcessor vpa_api_util.RecommendationProcessor
	evictionAdmission       priority.PodEvictionAdmission
	evictionRateLimiter     *rate.Limiter
	selectorFetcher         target.VpaTargetSelectorFetcher
}

// NewUpdater creates Updater with given configuration
func NewUpdater(kubeClient kube_client.Interface, vpaClient *vpa_clientset.Clientset, minReplicasForEvicition int, evictionRateLimit float64, evictionRateBurst int, evictionToleranceFraction float64, recommendationProcessor vpa_api_util.RecommendationProcessor, evictionAdmission priority.PodEvictionAdmission, selectorFetcher target.VpaTargetSelectorFetcher) (Updater, error) {
	evictionRateLimiter := getRateLimiter(evictionRateLimit, evictionRateBurst)
	factory, err := eviction.NewPodsEvictionRestrictionFactory(kubeClient, minReplicasForEvicition, evictionToleranceFraction)
	if err != nil {
		return nil, fmt.Errorf("Failed to create eviction restriction factory: %v", err)
	}
	return &updater{
		vpaLister:               vpa_api_util.NewAllVpasLister(vpaClient, make(chan struct{})),
		podLister:               newPodLister(kubeClient),
		eventRecorder:           newEventRecorder(kubeClient),
		evictionFactory:         factory,
		recommendationProcessor: recommendationProcessor,
		evictionRateLimiter:     evictionRateLimiter,
		evictionAdmission:       evictionAdmission,
		selectorFetcher:         selectorFetcher,
	}, nil
}

// RunOnce represents single iteration in the main-loop of Updater
func (u *updater) RunOnce(ctx context.Context) {
	timer := metrics_updater.NewExecutionTimer()

	vpaList, err := u.vpaLister.List(labels.Everything())
	if err != nil {
		klog.Fatalf("failed get VPA list: %v", err)
	}
	timer.ObserveStep("ListVPAs")

	vpas := make([]*vpa_api_util.VpaWithSelector, 0)

	for _, vpa := range vpaList {
		if vpa_api_util.GetUpdateMode(vpa) != vpa_types.UpdateModeRecreate &&
			vpa_api_util.GetUpdateMode(vpa) != vpa_types.UpdateModeAuto {
			klog.V(3).Infof("skipping VPA object %v because its mode is not \"Recreate\" or \"Auto\"", vpa.Name)
			continue
		}
		selector, err := u.selectorFetcher.Fetch(vpa)
		if err != nil {
			klog.V(3).Infof("skipping VPA object %v because we cannot fetch selector", vpa.Name)
			continue
		}

		vpas = append(vpas, &vpa_api_util.VpaWithSelector{
			Vpa:      vpa,
			Selector: selector,
		})
	}

	if len(vpas) == 0 {
		klog.Warningf("no VPA objects to process")
		if u.evictionAdmission != nil {
			u.evictionAdmission.CleanUp()
		}
		timer.ObserveTotal()
		return
	}

	podsList, err := u.podLister.List(labels.Everything())
	if err != nil {
		klog.Errorf("failed to get pods list: %v", err)
		timer.ObserveTotal()
		return
	}
	timer.ObserveStep("ListPods")
	allLivePods := filterDeletedPods(podsList)

	controlledPods := make(map[*vpa_types.VerticalPodAutoscaler][]*apiv1.Pod)
	for _, pod := range allLivePods {
		controllingVPA := vpa_api_util.GetControllingVPAForPod(pod, vpas)
		if controllingVPA != nil {
			controlledPods[controllingVPA.Vpa] = append(controlledPods[controllingVPA.Vpa], pod)
		}
	}
	timer.ObserveStep("FilterPods")

	if u.evictionAdmission != nil {
		u.evictionAdmission.LoopInit(allLivePods, controlledPods)
	}
	timer.ObserveStep("AdmissionInit")

	for vpa, livePods := range controlledPods {
		evictionLimiter := u.evictionFactory.NewPodsEvictionRestriction(livePods)
		podsForUpdate := u.getPodsUpdateOrder(filterNonEvictablePods(livePods, evictionLimiter), vpa)

		for _, pod := range podsForUpdate {
			if !evictionLimiter.CanEvict(pod) {
				continue
			}
			err := u.evictionRateLimiter.Wait(ctx)
			if err != nil {
				klog.Warningf("evicting pod %v failed: %v", pod.Name, err)
				return
			}
			klog.V(2).Infof("evicting pod %v", pod.Name)
			evictErr := evictionLimiter.Evict(pod, u.eventRecorder)
			if evictErr != nil {
				klog.Warningf("evicting pod %v failed: %v", pod.Name, evictErr)
			}
		}
	}
	timer.ObserveStep("EvictPods")
	timer.ObserveTotal()
}

func getRateLimiter(evictionRateLimit float64, evictionRateLimitBurst int) *rate.Limiter {
	var evictionRateLimiter *rate.Limiter
	if evictionRateLimit <= 0 {
		// As a special case if the rate is set to rate.Inf, the burst rate is ignored
		// see https://github.com/golang/time/blob/master/rate/rate.go#L37
		evictionRateLimiter = rate.NewLimiter(rate.Inf, 0)
		klog.V(1).Info("Rate limit disabled")
	} else {
		evictionRateLimiter = rate.NewLimiter(rate.Limit(evictionRateLimit), evictionRateLimitBurst)
	}
	return evictionRateLimiter
}

// getPodsUpdateOrder returns list of pods that should be updated ordered by update priority
func (u *updater) getPodsUpdateOrder(pods []*apiv1.Pod, vpa *vpa_types.VerticalPodAutoscaler) []*apiv1.Pod {
	priorityCalculator := priority.NewUpdatePriorityCalculator(vpa.Spec.ResourcePolicy, vpa.Status.Conditions, nil, u.recommendationProcessor)
	recommendation := vpa.Status.Recommendation

	for _, pod := range pods {
		priorityCalculator.AddPod(pod, recommendation, time.Now())
	}

	return priorityCalculator.GetSortedPods(u.evictionAdmission)
}

func filterNonEvictablePods(pods []*apiv1.Pod, evictionRestriciton eviction.PodsEvictionRestriction) []*apiv1.Pod {
	result := make([]*apiv1.Pod, 0)
	for _, pod := range pods {
		if evictionRestriciton.CanEvict(pod) {
			result = append(result, pod)
		}
	}
	return result
}

func filterDeletedPods(pods []*apiv1.Pod) []*apiv1.Pod {
	result := make([]*apiv1.Pod, 0)
	for _, pod := range pods {
		if pod.DeletionTimestamp == nil {
			result = append(result, pod)
		}
	}
	return result
}

func newPodLister(kubeClient kube_client.Interface) v1lister.PodLister {
	selector := fields.ParseSelectorOrDie("spec.nodeName!=" + "" + ",status.phase!=" +
		string(apiv1.PodSucceeded) + ",status.phase!=" + string(apiv1.PodFailed))
	podListWatch := cache.NewListWatchFromClient(kubeClient.CoreV1().RESTClient(), "pods", apiv1.NamespaceAll, selector)
	store := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
	podLister := v1lister.NewPodLister(store)
	podReflector := cache.NewReflector(podListWatch, &apiv1.Pod{}, store, time.Hour)
	stopCh := make(chan struct{})
	go podReflector.Run(stopCh)

	return podLister
}

func newEventRecorder(kubeClient kube_client.Interface) record.EventRecorder {
	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(klog.V(4).Infof)
	if _, isFake := kubeClient.(*fake.Clientset); !isFake {
		eventBroadcaster.StartRecordingToSink(&clientv1.EventSinkImpl{Interface: clientv1.New(kubeClient.CoreV1().RESTClient()).Events("")})
	}
	return eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{Component: "vpa-updater"})
}
<commit_msg>updater timer ObserveTotal cleanup<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logic

import (
	"context"
	"fmt"
	"time"

	"golang.org/x/time/rate"

	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"
	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
	vpa_lister "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/eviction"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/priority"
	metrics_updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/updater"
	vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa"
	kube_client "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/fake"
	"k8s.io/client-go/kubernetes/scheme"
	clientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
	v1lister "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/klog"
)

// Updater performs updates on pods if recommended by Vertical Pod Autoscaler
type Updater interface {
	// RunOnce represents single iteration in the main-loop of Updater
	RunOnce(context.Context)
}

type updater struct {
	vpaLister               vpa_lister.VerticalPodAutoscalerLister
	podLister               v1lister.PodLister
	eventRecorder           record.EventRecorder
	evictionFactory         eviction.PodsEvictionRestrictionFactory
	recommendationProcessor vpa_api_util.RecommendationProcessor
	evictionAdmission       priority.PodEvictionAdmission
	evictionRateLimiter     *rate.Limiter
	selectorFetcher         target.VpaTargetSelectorFetcher
}

// NewUpdater creates Updater with given configuration
func NewUpdater(kubeClient kube_client.Interface, vpaClient *vpa_clientset.Clientset, minReplicasForEvicition int, evictionRateLimit float64, evictionRateBurst int, evictionToleranceFraction float64, recommendationProcessor vpa_api_util.RecommendationProcessor, evictionAdmission priority.PodEvictionAdmission, selectorFetcher target.VpaTargetSelectorFetcher) (Updater, error) {
	evictionRateLimiter := getRateLimiter(evictionRateLimit, evictionRateBurst)
	factory, err := eviction.NewPodsEvictionRestrictionFactory(kubeClient, minReplicasForEvicition, evictionToleranceFraction)
	if err != nil {
		return nil, fmt.Errorf("Failed to create eviction restriction factory: %v", err)
	}
	return &updater{
		vpaLister:               vpa_api_util.NewAllVpasLister(vpaClient, make(chan struct{})),
		podLister:               newPodLister(kubeClient),
		eventRecorder:           newEventRecorder(kubeClient),
		evictionFactory:         factory,
		recommendationProcessor: recommendationProcessor,
		evictionRateLimiter:     evictionRateLimiter,
		evictionAdmission:       evictionAdmission,
		selectorFetcher:         selectorFetcher,
	}, nil
}

// RunOnce represents single iteration in the main-loop of Updater
func (u *updater) RunOnce(ctx context.Context) {
	timer := metrics_updater.NewExecutionTimer()
	defer timer.ObserveTotal()

	vpaList, err := u.vpaLister.List(labels.Everything())
	if err != nil {
		klog.Fatalf("failed get VPA list: %v", err)
	}
	timer.ObserveStep("ListVPAs")

	vpas := make([]*vpa_api_util.VpaWithSelector, 0)

	for _, vpa := range vpaList {
		if vpa_api_util.GetUpdateMode(vpa) != vpa_types.UpdateModeRecreate &&
			vpa_api_util.GetUpdateMode(vpa) != vpa_types.UpdateModeAuto {
			klog.V(3).Infof("skipping VPA object %v because its mode is not \"Recreate\" or \"Auto\"", vpa.Name)
			continue
		}
		selector, err := u.selectorFetcher.Fetch(vpa)
		if err != nil {
			klog.V(3).Infof("skipping VPA object %v because we cannot fetch selector", vpa.Name)
			continue
		}

		vpas = append(vpas, &vpa_api_util.VpaWithSelector{
			Vpa:      vpa,
			Selector: selector,
		})
	}

	if len(vpas) == 0 {
		klog.Warningf("no VPA objects to process")
		if u.evictionAdmission != nil {
			u.evictionAdmission.CleanUp()
		}
		return
	}

	podsList, err := u.podLister.List(labels.Everything())
	if err != nil {
		klog.Errorf("failed to get pods list: %v", err)
		return
	}
	timer.ObserveStep("ListPods")
	allLivePods := filterDeletedPods(podsList)

	controlledPods := make(map[*vpa_types.VerticalPodAutoscaler][]*apiv1.Pod)
	for _, pod := range allLivePods {
		controllingVPA := vpa_api_util.GetControllingVPAForPod(pod, vpas)
		if controllingVPA != nil {
			controlledPods[controllingVPA.Vpa] = append(controlledPods[controllingVPA.Vpa], pod)
		}
	}
	timer.ObserveStep("FilterPods")

	if u.evictionAdmission != nil {
		u.evictionAdmission.LoopInit(allLivePods, controlledPods)
	}
	timer.ObserveStep("AdmissionInit")

	for vpa, livePods := range controlledPods {
		evictionLimiter := u.evictionFactory.NewPodsEvictionRestriction(livePods)
		podsForUpdate := u.getPodsUpdateOrder(filterNonEvictablePods(livePods, evictionLimiter), vpa)

		for _, pod := range podsForUpdate {
			if !evictionLimiter.CanEvict(pod) {
				continue
			}
			err := u.evictionRateLimiter.Wait(ctx)
			if err != nil {
				klog.Warningf("evicting pod %v failed: %v", pod.Name, err)
				return
			}
			klog.V(2).Infof("evicting pod %v", pod.Name)
			evictErr := evictionLimiter.Evict(pod, u.eventRecorder)
			if evictErr != nil {
				klog.Warningf("evicting pod %v failed: %v", pod.Name, evictErr)
			}
		}
	}
	timer.ObserveStep("EvictPods")
}

func getRateLimiter(evictionRateLimit float64, evictionRateLimitBurst int) *rate.Limiter {
	var evictionRateLimiter *rate.Limiter
	if evictionRateLimit <= 0 {
		// As a special case if the rate is set to rate.Inf, the burst rate is ignored
		// see https://github.com/golang/time/blob/master/rate/rate.go#L37
		evictionRateLimiter = rate.NewLimiter(rate.Inf, 0)
		klog.V(1).Info("Rate limit disabled")
	} else {
		evictionRateLimiter = rate.NewLimiter(rate.Limit(evictionRateLimit), evictionRateLimitBurst)
	}
	return evictionRateLimiter
}

// getPodsUpdateOrder returns list of pods that should be updated ordered by update priority
func (u *updater) getPodsUpdateOrder(pods []*apiv1.Pod, vpa *vpa_types.VerticalPodAutoscaler) []*apiv1.Pod {
	priorityCalculator := priority.NewUpdatePriorityCalculator(vpa.Spec.ResourcePolicy, vpa.Status.Conditions, nil, u.recommendationProcessor)
	recommendation := vpa.Status.Recommendation

	for _, pod := range pods {
		priorityCalculator.AddPod(pod, recommendation, time.Now())
	}

	return priorityCalculator.GetSortedPods(u.evictionAdmission)
}

func filterNonEvictablePods(pods []*apiv1.Pod, evictionRestriciton eviction.PodsEvictionRestriction) []*apiv1.Pod {
	result := make([]*apiv1.Pod, 0)
	for _, pod := range pods {
		if evictionRestriciton.CanEvict(pod) {
			result = append(result, pod)
		}
	}
	return result
}

func filterDeletedPods(pods []*apiv1.Pod) []*apiv1.Pod {
	result := make([]*apiv1.Pod, 0)
	for _, pod := range pods {
		if pod.DeletionTimestamp == nil {
			result = append(result, pod)
		}
	}
	return result
}

func newPodLister(kubeClient kube_client.Interface) v1lister.PodLister {
	selector := fields.ParseSelectorOrDie("spec.nodeName!=" + "" + ",status.phase!=" +
		string(apiv1.PodSucceeded) + ",status.phase!=" + string(apiv1.PodFailed))
	podListWatch := cache.NewListWatchFromClient(kubeClient.CoreV1().RESTClient(), "pods", apiv1.NamespaceAll, selector)
	store := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
	podLister := v1lister.NewPodLister(store)
	podReflector := cache.NewReflector(podListWatch, &apiv1.Pod{}, store, time.Hour)
	stopCh := make(chan struct{})
	go podReflector.Run(stopCh)

	return podLister
}

func newEventRecorder(kubeClient kube_client.Interface) record.EventRecorder {
	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(klog.V(4).Infof)
	if _, isFake := kubeClient.(*fake.Clientset); !isFake {
		eventBroadcaster.StartRecordingToSink(&clientv1.EventSinkImpl{Interface: clientv1.New(kubeClient.CoreV1().RESTClient()).Events("")})
	}
	return eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{Component: "vpa-updater"})
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	labels "k8s.io/apimachinery/pkg/labels"
)

// ClusterState holds all runtime information about the cluster required for the
// VPA operations, i.e. configuration of resources (pods, containers,
// VPA objects), aggregated utilization of compute resources (CPU, memory) and
// events (container OOMs).
// All input to the VPA Recommender algorithm lives in this structure.
// TODO(kgrygiel): Limit the ClusterState object to a single namespace.
type ClusterState struct {
	// Pods in the cluster.
	Pods map[PodID]*PodState
	// VPA objects in the cluster.
	Vpas map[VpaID]*Vpa
}

// PodState holds runtime information about a single Pod.
type PodState struct {
	// Unique id of the Pod.
	ID PodID
	// Set of labels attached to the Pod.
	Labels labels.Set
	// Containers that belong to the Pod, keyed by the container name.
	Containers map[string]*ContainerState
	// VPA managing this pod (can be nil).
	Vpa *Vpa
	// All VPA objects that match this Pod. While it is incorrect to let
	// multiple VPA objects match the same pod, the model has no means to
	// prevent such situation. In such case the pod is controlled by one of the
	// matching VPAs.
	MatchingVpas map[VpaID]*Vpa
}

// NewClusterState returns a new ClusterState with no pods.
func NewClusterState() *ClusterState {
	return &ClusterState{
		make(map[PodID]*PodState), // empty pods map.
		make(map[VpaID]*Vpa),      // empty vpas map.
	}
}

// ContainerUsageSampleWithKey holds a ContainerUsageSample together with the
// ID of the container it belongs to.
type ContainerUsageSampleWithKey struct {
	ContainerUsageSample
	Container ContainerID
}

// AddOrUpdatePod udpates the state of the pod with a given PodID, if it is
// present in the cluster object. Otherwise a new pod is created and added to
// the Cluster object.
// If the labels of the pod have changed, it updates the links between the pod
// and the matching Vpa.
func (cluster *ClusterState) AddOrUpdatePod(podID PodID, newLabels labels.Set) {
	pod, podExists := cluster.Pods[podID]
	if !podExists {
		pod = newPod(podID)
		cluster.Pods[podID] = pod
	}
	if !podExists || !labels.Equals(pod.Labels, newLabels) {
		// Update the labels and the links between the pod and Vpas.
		pod.Labels = newLabels
		for _, vpa := range cluster.Vpas {
			vpa.UpdatePodLink(pod)
		}
	}
}

// DeletePod removes an existing pod from the cluster.
func (cluster *ClusterState) DeletePod(podID PodID) error {
	pod, podExists := cluster.Pods[podID]
	if !podExists {
		return NewKeyError(podID)
	}
	// Set labels to nil so that no VPA matches the pod.
	pod.Labels = nil
	for _, vpa := range pod.MatchingVpas {
		vpa.UpdatePodLink(pod)
	}
	delete(cluster.Pods, podID)
	return nil
}

// AddOrUpdateContainer creates a new container with the given ContainerID and
// adds it to the parent pod in the ClusterState object, if not yet present.
// Requires the pod to be added to the ClusterState first. Otherwise an error is
// returned.
func (cluster *ClusterState) AddOrUpdateContainer(containerID ContainerID) error {
	pod, podExists := cluster.Pods[containerID.PodID]
	if !podExists {
		return NewKeyError(containerID.PodID)
	}
	if _, containerExists := pod.Containers[containerID.ContainerName]; !containerExists {
		pod.Containers[containerID.ContainerName] = NewContainerState()
	}
	return nil
}

// AddSample adds a new usage sample to the proper container in the ClusterState
// object. Requires the container as well as the parent pod to be added to the
// ClusterState first. Otherwise an error is returned.
func (cluster *ClusterState) AddSample(sample *ContainerUsageSampleWithKey) error {
	pod, podExists := cluster.Pods[sample.Container.PodID]
	if !podExists {
		return NewKeyError(sample.Container.PodID)
	}
	containerState, containerExists := pod.Containers[sample.Container.ContainerName]
	if !containerExists {
		return NewKeyError(sample.Container)
	}
	containerState.AddSample(&sample.ContainerUsageSample)
	return nil
}

// AddOrUpdateVpa adds a new VPA with a given ID to the ClusterState if it
// didn't yet exist. If the VPA already existed but had a different pod
// selector, the pod selector is updated. Updates the links between the VPA and
// all pods it matches.
func (cluster *ClusterState) AddOrUpdateVpa(vpaID VpaID, podSelectorStr string) error {
	vpa, vpaExists := cluster.Vpas[vpaID]
	if vpaExists && vpa.PodSelectorStr != podSelectorStr {
		// Pod selector was changed. Delete the VPA object and recreate
		// it with the new selector.
		if err := cluster.DeleteVpa(vpaID); err != nil {
			return err
		}
		vpaExists = false
	}
	if !vpaExists {
		vpa, err := NewVpa(vpaID, podSelectorStr)
		if err != nil {
			return err
		}
		cluster.Vpas[vpaID] = vpa
		for _, pod := range cluster.Pods {
			vpa.UpdatePodLink(pod)
		}
	}
	return nil
}

// DeleteVpa removes a VPA with the given ID from the ClusterState.
func (cluster *ClusterState) DeleteVpa(vpaID VpaID) error {
	vpa, vpaExists := cluster.Vpas[vpaID]
	if !vpaExists {
		return NewKeyError(vpaID)
	}
	// Change the selector to not match any pod and detach all pods.
	vpa.SetPodSelectorStr("0=1")
	for _, pod := range vpa.Pods {
		vpa.UpdatePodLink(pod)
	}
	return nil
}

func newPod(id PodID) *PodState {
	return &PodState{
		id,
		make(map[string]string),          // empty labels.
		make(map[string]*ContainerState), // empty containers.
		nil,                  // Vpa.
		make(map[VpaID]*Vpa), // empty MatchingVpas.
	}
}
<commit_msg>Add method GetContainer to Cluster.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	labels "k8s.io/apimachinery/pkg/labels"
)

// ClusterState holds all runtime information about the cluster required for the
// VPA operations, i.e. configuration of resources (pods, containers,
// VPA objects), aggregated utilization of compute resources (CPU, memory) and
// events (container OOMs).
// All input to the VPA Recommender algorithm lives in this structure.
// TODO(kgrygiel): Limit the ClusterState object to a single namespace.
type ClusterState struct {
	// Pods in the cluster.
	Pods map[PodID]*PodState
	// VPA objects in the cluster.
	Vpas map[VpaID]*Vpa
}

// PodState holds runtime information about a single Pod.
type PodState struct {
	// Unique id of the Pod.
	ID PodID
	// Set of labels attached to the Pod.
	Labels labels.Set
	// Containers that belong to the Pod, keyed by the container name.
	Containers map[string]*ContainerState
	// VPA managing this pod (can be nil).
	Vpa *Vpa
	// All VPA objects that match this Pod. While it is incorrect to let
	// multiple VPA objects match the same pod, the model has no means to
	// prevent such situation. In such case the pod is controlled by one of the
	// matching VPAs.
	MatchingVpas map[VpaID]*Vpa
}

// NewClusterState returns a new ClusterState with no pods.
func NewClusterState() *ClusterState {
	return &ClusterState{
		make(map[PodID]*PodState), // empty pods map.
		make(map[VpaID]*Vpa),      // empty vpas map.
	}
}

// ContainerUsageSampleWithKey holds a ContainerUsageSample together with the
// ID of the container it belongs to.
type ContainerUsageSampleWithKey struct {
	ContainerUsageSample
	Container ContainerID
}

// AddOrUpdatePod udpates the state of the pod with a given PodID, if it is
// present in the cluster object. Otherwise a new pod is created and added to
// the Cluster object.
// If the labels of the pod have changed, it updates the links between the pod
// and the matching Vpa.
func (cluster *ClusterState) AddOrUpdatePod(podID PodID, newLabels labels.Set) {
	pod, podExists := cluster.Pods[podID]
	if !podExists {
		pod = newPod(podID)
		cluster.Pods[podID] = pod
	}
	if !podExists || !labels.Equals(pod.Labels, newLabels) {
		// Update the labels and the links between the pod and Vpas.
		pod.Labels = newLabels
		for _, vpa := range cluster.Vpas {
			vpa.UpdatePodLink(pod)
		}
	}
}

// GetContainer returns the ContainerState object for a given ContainerID or
// null if it's not present in the model.
func (cluster *ClusterState) GetContainer(containerID ContainerID) *ContainerState {
	pod, podExists := cluster.Pods[containerID.PodID]
	if podExists {
		container, containerExists := pod.Containers[containerID.ContainerName]
		if containerExists {
			return container
		}
	}
	return nil
}

// DeletePod removes an existing pod from the cluster.
func (cluster *ClusterState) DeletePod(podID PodID) error {
	pod, podExists := cluster.Pods[podID]
	if !podExists {
		return NewKeyError(podID)
	}
	// Set labels to nil so that no VPA matches the pod.
	pod.Labels = nil
	for _, vpa := range pod.MatchingVpas {
		vpa.UpdatePodLink(pod)
	}
	delete(cluster.Pods, podID)
	return nil
}

// AddOrUpdateContainer creates a new container with the given ContainerID and
// adds it to the parent pod in the ClusterState object, if not yet present.
// Requires the pod to be added to the ClusterState first. Otherwise an error is
// returned.
func (cluster *ClusterState) AddOrUpdateContainer(containerID ContainerID) error {
	pod, podExists := cluster.Pods[containerID.PodID]
	if !podExists {
		return NewKeyError(containerID.PodID)
	}
	if _, containerExists := pod.Containers[containerID.ContainerName]; !containerExists {
		pod.Containers[containerID.ContainerName] = NewContainerState()
	}
	return nil
}

// AddSample adds a new usage sample to the proper container in the ClusterState
// object. Requires the container as well as the parent pod to be added to the
// ClusterState first. Otherwise an error is returned.
func (cluster *ClusterState) AddSample(sample *ContainerUsageSampleWithKey) error {
	pod, podExists := cluster.Pods[sample.Container.PodID]
	if !podExists {
		return NewKeyError(sample.Container.PodID)
	}
	containerState, containerExists := pod.Containers[sample.Container.ContainerName]
	if !containerExists {
		return NewKeyError(sample.Container)
	}
	containerState.AddSample(&sample.ContainerUsageSample)
	return nil
}

// AddOrUpdateVpa adds a new VPA with a given ID to the ClusterState if it
// didn't yet exist. If the VPA already existed but had a different pod
// selector, the pod selector is updated. Updates the links between the VPA and
// all pods it matches.
func (cluster *ClusterState) AddOrUpdateVpa(vpaID VpaID, podSelectorStr string) error {
	vpa, vpaExists := cluster.Vpas[vpaID]
	if vpaExists && vpa.PodSelectorStr != podSelectorStr {
		// Pod selector was changed. Delete the VPA object and recreate
		// it with the new selector.
		if err := cluster.DeleteVpa(vpaID); err != nil {
			return err
		}
		vpaExists = false
	}
	if !vpaExists {
		vpa, err := NewVpa(vpaID, podSelectorStr)
		if err != nil {
			return err
		}
		cluster.Vpas[vpaID] = vpa
		for _, pod := range cluster.Pods {
			vpa.UpdatePodLink(pod)
		}
	}
	return nil
}

// DeleteVpa removes a VPA with the given ID from the ClusterState.
func (cluster *ClusterState) DeleteVpa(vpaID VpaID) error {
	vpa, vpaExists := cluster.Vpas[vpaID]
	if !vpaExists {
		return NewKeyError(vpaID)
	}
	// Change the selector to not match any pod and detach all pods.
	vpa.SetPodSelectorStr("0=1")
	for _, pod := range vpa.Pods {
		vpa.UpdatePodLink(pod)
	}
	return nil
}

func newPod(id PodID) *PodState {
	return &PodState{
		id,
		make(map[string]string),          // empty labels.
		make(map[string]*ContainerState), // empty containers.
		nil,                  // Vpa.
		make(map[VpaID]*Vpa), // empty MatchingVpas.
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// thrift-gen generates code for Thrift services that can be used with the
// uber/tchannel/thrift package. thrift-gen generated code relies on the
// Apache Thrift generated code for serialization/deserialization, and should
// be a part of the generated code's package.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"text/template"

	"github.com/samuel/go-thrift/parser"
)

const tchannelThriftImport = "github.com/uber/tchannel/golang/thrift"

var (
	generateThrift     = flag.Bool("generateThrift", false, "Whether to generate all Thrift go code")
	apacheThriftImport = flag.String("thriftImport", "github.com/apache/thrift/lib/go/thrift", "Go package to use for the Thrift import")
	inputFile          = flag.String("inputFile", "", "The .thrift file to generate a client for")
	outputFile         = flag.String("outputFile", "", "The output file to generate go code to")
	nlSpaceNL          = regexp.MustCompile(`\n[ \t]+\n`)
)

// TemplateData is the data passed to the template that generates code.
type TemplateData struct {
	Package        string
	Services       []*Service
	ThriftImport   string
	TChannelImport string
}

func main() {
	flag.Parse()
	if *inputFile == "" {
		log.Fatalf("Please specify an inputFile")
	}

	if err := processFile(*generateThrift, *inputFile, *outputFile); err != nil {
		log.Fatal(err)
	}
}

func processFile(generateThrift bool, inputFile string, outputFile string) error {
	if generateThrift {
		if outFile, err := runThrift(inputFile, *apacheThriftImport); err != nil {
			return fmt.Errorf("Could not generate thrift output: %v", err)
		} else if outputFile == "" {
			outputFile = outFile
		}
	}

	parser := &parser.Parser{}
	parsed, _, err := parser.ParseFile(inputFile)
	if err != nil {
		return fmt.Errorf("Could not parse .thrift file: %v", err)
	}

	goTmpl := parseTemplate()
	for filename, v := range parsed {
		if err := generateCode(outputFile, goTmpl, packageName(filename), v); err != nil {
			return err
		}
		// TODO(prashant): Support multiple files / includes etc?
		return nil
	}

	return nil
}

func parseTemplate() *template.Template {
	funcs := map[string]interface{}{
		"contextType": contextType,
	}
	return template.Must(template.New("thrift-gen").Funcs(funcs).Parse(serviceTmpl))
}

func generateCode(outputFile string, tmpl *template.Template, pkg string, parsed *parser.Thrift) error {
	wrappedServices, err := wrapServices(parsed)
	if err != nil {
		log.Fatalf("Service parsing error: %v", err)
	}

	buf := &bytes.Buffer{}

	td := TemplateData{
		Package:        pkg,
		Services:       wrappedServices,
		ThriftImport:   *apacheThriftImport,
		TChannelImport: tchannelThriftImport,
	}
	if err := tmpl.Execute(buf, td); err != nil {
		return fmt.Errorf("failed to execute template: %v", err)
	}

	generated := cleanGeneratedCode(buf.Bytes())
	if err := ioutil.WriteFile(outputFile, generated, 0666); err != nil {
		return fmt.Errorf("cannot write output file %s: %v", outputFile, err)
	}

	// Run gofmt on the file (ignore any errors)
	exec.Command("gofmt", "-w", outputFile).Run()
	return nil
}

func packageName(fullPath string) string {
	// TODO(prashant): Remove any characters that are not valid in Go package names.
	_, filename := filepath.Split(fullPath)
	file := strings.TrimSuffix(filename, filepath.Ext(filename))
	return strings.ToLower(file)
}

func cleanGeneratedCode(generated []byte) []byte {
	generated = nlSpaceNL.ReplaceAll(generated, []byte("\n"))
	return generated
}

func contextType() string {
	return "thrift.Context"
}
<commit_msg>Throw error if no output file is specified<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// thrift-gen generates code for Thrift services that can be used with the
// uber/tchannel/thrift package. thrift-gen generated code relies on the
// Apache Thrift generated code for serialization/deserialization, and should
// be a part of the generated code's package.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"text/template"

	"github.com/samuel/go-thrift/parser"
)

const tchannelThriftImport = "github.com/uber/tchannel/golang/thrift"

var (
	generateThrift     = flag.Bool("generateThrift", false, "Whether to generate all Thrift go code")
	apacheThriftImport = flag.String("thriftImport", "github.com/apache/thrift/lib/go/thrift", "Go package to use for the Thrift import")
	inputFile          = flag.String("inputFile", "", "The .thrift file to generate a client for")
	outputFile         = flag.String("outputFile", "", "The output file to generate go code to")
	nlSpaceNL          = regexp.MustCompile(`\n[ \t]+\n`)
)

// TemplateData is the data passed to the template that generates code.
type TemplateData struct {
	Package        string
	Services       []*Service
	ThriftImport   string
	TChannelImport string
}

func main() {
	flag.Parse()
	if *inputFile == "" {
		log.Fatalf("Please specify an inputFile")
	}

	if err := processFile(*generateThrift, *inputFile, *outputFile); err != nil {
		log.Fatal(err)
	}
}

func processFile(generateThrift bool, inputFile string, outputFile string) error {
	if generateThrift {
		if outFile, err := runThrift(inputFile, *apacheThriftImport); err != nil {
			return fmt.Errorf("Could not generate thrift output: %v", err)
		} else if outputFile == "" {
			outputFile = outFile
		}
	}

	parser := &parser.Parser{}
	parsed, _, err := parser.ParseFile(inputFile)
	if err != nil {
		return fmt.Errorf("Could not parse .thrift file: %v", err)
	}

	goTmpl := parseTemplate()
	for filename, v := range parsed {
		if err := generateCode(outputFile, goTmpl, packageName(filename), v); err != nil {
			return err
		}
		// TODO(prashant): Support multiple files / includes etc?
		return nil
	}

	return nil
}

func parseTemplate() *template.Template {
	funcs := map[string]interface{}{
		"contextType": contextType,
	}
	return template.Must(template.New("thrift-gen").Funcs(funcs).Parse(serviceTmpl))
}

func generateCode(outputFile string, tmpl *template.Template, pkg string, parsed *parser.Thrift) error {
	if outputFile == "" {
		return fmt.Errorf("must speciy an output file")
	}

	wrappedServices, err := wrapServices(parsed)
	if err != nil {
		log.Fatalf("Service parsing error: %v", err)
	}

	buf := &bytes.Buffer{}

	td := TemplateData{
		Package:        pkg,
		Services:       wrappedServices,
		ThriftImport:   *apacheThriftImport,
		TChannelImport: tchannelThriftImport,
	}
	if err := tmpl.Execute(buf, td); err != nil {
		return fmt.Errorf("failed to execute template: %v", err)
	}

	generated := cleanGeneratedCode(buf.Bytes())
	if err := ioutil.WriteFile(outputFile, generated, 0666); err != nil {
		return fmt.Errorf("cannot write output file %s: %v", outputFile, err)
	}

	// Run gofmt on the file (ignore any errors)
	exec.Command("gofmt", "-w", outputFile).Run()
	return nil
}

func packageName(fullPath string) string {
	// TODO(prashant): Remove any characters that are not valid in Go package names.
	_, filename := filepath.Split(fullPath)
	file := strings.TrimSuffix(filename, filepath.Ext(filename))
	return strings.ToLower(file)
}

func cleanGeneratedCode(generated []byte) []byte {
	generated = nlSpaceNL.ReplaceAll(generated, []byte("\n"))
	return generated
}

func contextType() string {
	return "thrift.Context"
}
<|endoftext|>