<commit_before>package swaggman

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strings"

	"github.com/grokify/gotilla/net/httputilmore"
	"github.com/grokify/swaggman/postman2"
	"github.com/grokify/swaggman/postman2/simple"
	"github.com/grokify/swaggman/swagger2"
)

// Configuration is a Swaggman configuration that holds information on how
// to create the Postman 2.0 collection including overriding Swagger 2.0
// spec values.
type Configuration struct {
	PostmanURLBase     string            `json:"postmanURLBase,omitempty"`
	PostmanURLHostname string            `json:"postmanURLHostname,omitempty"`
	PostmanHeaders     []postman2.Header `json:"postmanHeaders,omitempty"`
}

// Converter is the struct that manages the conversion.
type Converter struct {
	Configuration Configuration
	Swagger       swagger2.Specification
}

// NewConverter instantiates a new converter.
func NewConverter(cfg Configuration) Converter {
	return Converter{Configuration: cfg}
}

// MergeConvert builds a Postman 2.0 spec using a base Postman 2.0 collection
// and a Swagger 2.0 spec.
func (conv *Converter) MergeConvert(swaggerFilepath string, pmanBaseFilepath string, pmanSpecFilepath string) error {
	swag, err := swagger2.ReadSwagger2SpecFile(swaggerFilepath)
	if err != nil {
		return err
	}

	pman, err := simple.ReadCanonicalCollection(pmanBaseFilepath)
	if err != nil {
		return err
	}
	pm := Merge(conv.Configuration, pman, swag)

	bytes, err := json.MarshalIndent(pm, "", "  ")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(pmanSpecFilepath, bytes, 0644)
}

// Convert builds a Postman 2.0 spec using a Swagger 2.0 spec.
func (conv *Converter) Convert(swaggerFilepath string, pmanSpecFilepath string) error {
	swag, err := swagger2.ReadSwagger2SpecFile(swaggerFilepath)
	if err != nil {
		return err
	}
	pm := Convert(conv.Configuration, swag)

	bytes, err := json.MarshalIndent(pm, "", "  ")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(pmanSpecFilepath, bytes, 0644)
}

// Convert creates a Postman 2.0 collection from a configuration and Swagger 2.0 spec
func Convert(cfg Configuration, swag swagger2.Specification) postman2.Collection {
	return Merge(cfg, postman2.Collection{}, swag)
}

// Merge creates a Postman 2.0 collection from a configuration, base Postman
// 2.0 collection and Swagger 2.0 spec
func Merge(cfg Configuration, pman postman2.Collection, swag swagger2.Specification) postman2.Collection {
	if len(pman.Info.Name) == 0 {
		pman.Info.Name = strings.TrimSpace(swag.Info.Title)
	}
	if len(pman.Info.Description) == 0 {
		pman.Info.Description = strings.TrimSpace(swag.Info.Description)
	}
	if len(pman.Info.Schema) == 0 {
		pman.Info.Schema = "https://schema.getpostman.com/json/collection/v2.0.0/collection.json"
	}

	urls := []string{}
	for u := range swag.Paths {
		urls = append(urls, u)
	}
	sort.Strings(urls)

	for _, u := range urls {
		path := swag.Paths[u]

		if path.HasMethodWithTag(http.MethodGet) {
			pman = postmanAddItemToFolder(pman,
				Swagger2PathToPostman2APIItem(cfg, swag, u, http.MethodGet, path.Get),
				strings.TrimSpace(path.Get.Tags[0]))
		}
		if path.HasMethodWithTag(http.MethodPatch) {
			pman = postmanAddItemToFolder(pman,
				Swagger2PathToPostman2APIItem(cfg, swag, u, http.MethodPatch, path.Patch),
				strings.TrimSpace(path.Patch.Tags[0]))
		}
		if path.HasMethodWithTag(http.MethodPost) {
			pman = postmanAddItemToFolder(pman,
				Swagger2PathToPostman2APIItem(cfg, swag, u, http.MethodPost, path.Post),
				strings.TrimSpace(path.Post.Tags[0]))
		}
		if path.HasMethodWithTag(http.MethodPut) {
			pman = postmanAddItemToFolder(pman,
				Swagger2PathToPostman2APIItem(cfg, swag, u, http.MethodPut, path.Put),
				strings.TrimSpace(path.Put.Tags[0]))
		}
		if path.HasMethodWithTag(http.MethodDelete) {
			pman = postmanAddItemToFolder(pman,
				Swagger2PathToPostman2APIItem(cfg, swag, u, http.MethodDelete, path.Delete),
				strings.TrimSpace(path.Delete.Tags[0]))
		}
	}

	return pman
}

func postmanAddItemToFolder(pman postman2.Collection, pmItem postman2.Item, pmFolderName string) postman2.Collection {
	pmFolder := pman.GetOrNewFolder(pmFolderName)
	pmFolder.Item = append(pmFolder.Item, pmItem)
	pman.SetFolder(pmFolder)
	return pman
}

// Swagger2PathToPostman2APIItem converts a Swagger 2.0 path to a
// Postman 2.0 API item
func Swagger2PathToPostman2APIItem(cfg Configuration, swag swagger2.Specification, url string, method string, endpoint *swagger2.Endpoint) postman2.Item {
	item := postman2.Item{
		Name: endpoint.Summary,
		Request: postman2.Request{
			Method: strings.ToUpper(method),
			URL:    BuildPostmanURL(cfg, swag, url, endpoint),
		},
	}
	/*
		item.Name = endpoint.Summary

		item.Request = postman2.Request{Method: strings.ToUpper(method)}

		item.Request.URL = BuildPostmanURL(cfg, swag, url, endpoint)
	*/
	headers := cfg.PostmanHeaders

	headers, requestContentType := postman2.AppendPostmanHeaderValueLower(
		headers,
		httputilmore.HeaderContentType,
		endpoint.Consumes,
		postman2.DefaultMediaTypePreferencesSlice())

	headers, _ = postman2.AppendPostmanHeaderValueLower(
		headers,
		httputilmore.HeaderAccept,
		endpoint.Produces,
		postman2.DefaultMediaTypePreferencesSlice())

	item.Request.Header = headers

	jsonCT := httputilmore.ContentTypeAppJsonUtf8
	indexAppJson := strings.Index(
		strings.ToLower(requestContentType), jsonCT)
	if indexAppJson > -1 {
		jsonExample, err := swagger2.GetJsonBodyParameterExampleForKey(
			endpoint.Parameters, jsonCT)
		if err == nil {
			jsonExample = strings.TrimSpace(jsonExample)
			if len(jsonExample) >= 0 {
				item.Request.Body.Mode = "raw"
				item.Request.Body.Raw = jsonExample
			}
		}
	}
	return item
}

// BuildPostmanURL creates a Postman 2.0 spec URL from a Swagger URL
func BuildPostmanURL(cfg Configuration, swag swagger2.Specification, swaggerURL string, endpoint *swagger2.Endpoint) postman2.URL {
	goUrl := url.URL{}

	URLParts := []string{}
	URLPathParts := []string{}

	cfg.PostmanURLBase = strings.TrimSpace(cfg.PostmanURLBase)
	// Set URL path parts
	if len(cfg.PostmanURLBase) > 0 {
		URLParts = append(URLParts, cfg.PostmanURLBase)
		goUrl.Host = cfg.PostmanURLBase
	} else if len(strings.TrimSpace(cfg.PostmanURLHostname)) > 0 {
		URLParts = append(URLParts, strings.TrimSpace(cfg.PostmanURLHostname))
		goUrl.Host = cfg.PostmanURLHostname
	} else if len(strings.TrimSpace(swag.Host)) > 0 {
		URLParts = append(URLParts, strings.TrimSpace(swag.Host))
		goUrl.Host = swag.Host
	}

	if len(strings.TrimSpace(swag.BasePath)) > 0 {
		URLParts = append(URLParts, strings.TrimSpace(swag.BasePath))
		URLPathParts = append(URLPathParts, strings.TrimSpace(swag.BasePath))
	}
	if len(strings.TrimSpace(swaggerURL)) > 0 {
		URLParts = append(URLParts, strings.TrimSpace(swaggerURL))
		URLPathParts = append(URLPathParts, strings.TrimSpace(swaggerURL))
	}

	// Create URL
	rawPostmanURL := strings.TrimSpace(strings.Join(URLParts, "/"))
	rx1 := regexp.MustCompile(`/+`)
	rawPostmanURL = rx1.ReplaceAllString(rawPostmanURL, "/")
	rx2 := regexp.MustCompile(`^/+`)
	rawPostmanURL = rx2.ReplaceAllString(rawPostmanURL, "")

	rawPostmanURLPath := strings.TrimSpace(strings.Join(URLPathParts, "/"))
	rawPostmanURLPath = rx1.ReplaceAllString(rawPostmanURLPath, "/")
	rawPostmanURLPath = rx2.ReplaceAllString(rawPostmanURLPath, "")
	goUrl.Path = rawPostmanURLPath

	// Add URL Scheme
	if len(cfg.PostmanURLBase) == 0 {
		if len(swag.Schemes) > 0 {
			for _, scheme := range swag.Schemes {
				if len(strings.TrimSpace(scheme)) > 0 {
					rawPostmanURL = strings.Join([]string{scheme, rawPostmanURL}, "://")
					goUrl.Scheme = scheme
					break
				}
			}
		}
	}

	/*
		rx3 := regexp.MustCompile(`(^|[^\{])\{([^\/\{\}]+)\}([^\}]|$)`)
		rawPostmanURL = rx3.ReplaceAllString(rawPostmanURL, "$1:$2$3")
		goUrl.Path = rx3.ReplaceAllString(goUrl.Path, "$1:$2$3")
	*/

	rawPostmanURL = postman2.ApiUrlOasToPostman(rawPostmanURL)
	goUrl.Path = postman2.ApiUrlOasToPostman(goUrl.Path)

	//postmanURL := postman2.NewURL(rawPostmanURL)
	postmanURL := postman2.NewURLForGoUrl(goUrl)

	// Set Default URL Path Parameters
	rx4 := regexp.MustCompile(`^\s*(:(.+))\s*$`)

	for _, part := range postmanURL.Path {
		rs4 := rx4.FindAllStringSubmatch(part, -1)
		if len(rs4) > 0 {
			baseVariable := rs4[0][2]
			var defaultValue interface{}
			for _, parameter := range endpoint.Parameters {
				if parameter.Name == baseVariable {
					defaultValue = parameter.Default
					break
				}
			}
			postmanURL.AddVariable(baseVariable, defaultValue)
		}
	}

	return postmanURL
}
<commit_msg>remove ineffassign<commit_after>package swaggman

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strings"

	"github.com/grokify/gotilla/net/httputilmore"
	"github.com/grokify/swaggman/postman2"
	"github.com/grokify/swaggman/postman2/simple"
	"github.com/grokify/swaggman/swagger2"
)

// Configuration is a Swaggman configuration that holds information on how
// to create the Postman 2.0 collection including overriding Swagger 2.0
// spec values.
type Configuration struct {
	PostmanURLBase     string            `json:"postmanURLBase,omitempty"`
	PostmanURLHostname string            `json:"postmanURLHostname,omitempty"`
	PostmanHeaders     []postman2.Header `json:"postmanHeaders,omitempty"`
}

// Converter is the struct that manages the conversion.
type Converter struct {
	Configuration Configuration
	Swagger       swagger2.Specification
}

// NewConverter instantiates a new converter.
func NewConverter(cfg Configuration) Converter {
	return Converter{Configuration: cfg}
}

// MergeConvert builds a Postman 2.0 spec using a base Postman 2.0 collection
// and a Swagger 2.0 spec.
func (conv *Converter) MergeConvert(swaggerFilepath string, pmanBaseFilepath string, pmanSpecFilepath string) error {
	swag, err := swagger2.ReadSwagger2SpecFile(swaggerFilepath)
	if err != nil {
		return err
	}

	pman, err := simple.ReadCanonicalCollection(pmanBaseFilepath)
	if err != nil {
		return err
	}
	pm := Merge(conv.Configuration, pman, swag)

	bytes, err := json.MarshalIndent(pm, "", "  ")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(pmanSpecFilepath, bytes, 0644)
}

// Convert builds a Postman 2.0 spec using a Swagger 2.0 spec.
func (conv *Converter) Convert(swaggerFilepath string, pmanSpecFilepath string) error {
	swag, err := swagger2.ReadSwagger2SpecFile(swaggerFilepath)
	if err != nil {
		return err
	}
	pm := Convert(conv.Configuration, swag)

	bytes, err := json.MarshalIndent(pm, "", "  ")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(pmanSpecFilepath, bytes, 0644)
}

// Convert creates a Postman 2.0 collection from a configuration and Swagger 2.0 spec
func Convert(cfg Configuration, swag swagger2.Specification) postman2.Collection {
	return Merge(cfg, postman2.Collection{}, swag)
}

// Merge creates a Postman 2.0 collection from a configuration, base Postman
// 2.0 collection and Swagger 2.0 spec
func Merge(cfg Configuration, pman postman2.Collection, swag swagger2.Specification) postman2.Collection {
	if len(pman.Info.Name) == 0 {
		pman.Info.Name = strings.TrimSpace(swag.Info.Title)
	}
	if len(pman.Info.Description) == 0 {
		pman.Info.Description = strings.TrimSpace(swag.Info.Description)
	}
	if len(pman.Info.Schema) == 0 {
		pman.Info.Schema = "https://schema.getpostman.com/json/collection/v2.0.0/collection.json"
	}

	urls := []string{}
	for u := range swag.Paths {
		urls = append(urls, u)
	}
	sort.Strings(urls)

	for _, u := range urls {
		path := swag.Paths[u]

		if path.HasMethodWithTag(http.MethodGet) {
			pman = postmanAddItemToFolder(pman,
				Swagger2PathToPostman2APIItem(cfg, swag, u, http.MethodGet, path.Get),
				strings.TrimSpace(path.Get.Tags[0]))
		}
		if path.HasMethodWithTag(http.MethodPatch) {
			pman = postmanAddItemToFolder(pman,
				Swagger2PathToPostman2APIItem(cfg, swag, u, http.MethodPatch, path.Patch),
				strings.TrimSpace(path.Patch.Tags[0]))
		}
		if path.HasMethodWithTag(http.MethodPost) {
			pman = postmanAddItemToFolder(pman,
				Swagger2PathToPostman2APIItem(cfg, swag, u, http.MethodPost, path.Post),
				strings.TrimSpace(path.Post.Tags[0]))
		}
		if path.HasMethodWithTag(http.MethodPut) {
			pman = postmanAddItemToFolder(pman,
				Swagger2PathToPostman2APIItem(cfg, swag, u, http.MethodPut, path.Put),
				strings.TrimSpace(path.Put.Tags[0]))
		}
		if path.HasMethodWithTag(http.MethodDelete) {
			pman = postmanAddItemToFolder(pman,
				Swagger2PathToPostman2APIItem(cfg, swag, u, http.MethodDelete, path.Delete),
				strings.TrimSpace(path.Delete.Tags[0]))
		}
	}

	return pman
}

func postmanAddItemToFolder(pman postman2.Collection, pmItem postman2.Item, pmFolderName string) postman2.Collection {
	pmFolder := pman.GetOrNewFolder(pmFolderName)
	pmFolder.Item = append(pmFolder.Item, pmItem)
	pman.SetFolder(pmFolder)
	return pman
}

// Swagger2PathToPostman2APIItem converts a Swagger 2.0 path to a
// Postman 2.0 API item
func Swagger2PathToPostman2APIItem(cfg Configuration, swag swagger2.Specification, url string, method string, endpoint *swagger2.Endpoint) postman2.Item {
	item := postman2.Item{
		Name: endpoint.Summary,
		Request: postman2.Request{
			Method: strings.ToUpper(method),
			URL:    BuildPostmanURL(cfg, swag, url, endpoint),
		},
	}
	/*
		item.Name = endpoint.Summary

		item.Request = postman2.Request{Method: strings.ToUpper(method)}

		item.Request.URL = BuildPostmanURL(cfg, swag, url, endpoint)
	*/
	headers := cfg.PostmanHeaders

	headers, requestContentType := postman2.AppendPostmanHeaderValueLower(
		headers,
		httputilmore.HeaderContentType,
		endpoint.Consumes,
		postman2.DefaultMediaTypePreferencesSlice())

	headers, _ = postman2.AppendPostmanHeaderValueLower(
		headers,
		httputilmore.HeaderAccept,
		endpoint.Produces,
		postman2.DefaultMediaTypePreferencesSlice())

	item.Request.Header = headers

	jsonCT := httputilmore.ContentTypeAppJsonUtf8
	indexAppJson := strings.Index(
		strings.ToLower(requestContentType), jsonCT)
	if indexAppJson > -1 {
		jsonExample, err := swagger2.GetJsonBodyParameterExampleForKey(
			endpoint.Parameters, jsonCT)
		if err == nil {
			jsonExample = strings.TrimSpace(jsonExample)
			if len(jsonExample) >= 0 {
				item.Request.Body.Mode = "raw"
				item.Request.Body.Raw = jsonExample
			}
		}
	}
	return item
}

// BuildPostmanURL creates a Postman 2.0 spec URL from a Swagger URL
func BuildPostmanURL(cfg Configuration, swag swagger2.Specification, swaggerURL string, endpoint *swagger2.Endpoint) postman2.URL {
	goUrl := url.URL{}

	URLParts := []string{}
	URLPathParts := []string{}

	cfg.PostmanURLBase = strings.TrimSpace(cfg.PostmanURLBase)
	// Set URL path parts
	if len(cfg.PostmanURLBase) > 0 {
		URLParts = append(URLParts, cfg.PostmanURLBase)
		goUrl.Host = cfg.PostmanURLBase
	} else if len(strings.TrimSpace(cfg.PostmanURLHostname)) > 0 {
		URLParts = append(URLParts, strings.TrimSpace(cfg.PostmanURLHostname))
		goUrl.Host = cfg.PostmanURLHostname
	} else if len(strings.TrimSpace(swag.Host)) > 0 {
		URLParts = append(URLParts, strings.TrimSpace(swag.Host))
		goUrl.Host = swag.Host
	}

	if len(strings.TrimSpace(swag.BasePath)) > 0 {
		URLParts = append(URLParts, strings.TrimSpace(swag.BasePath))
		URLPathParts = append(URLPathParts, strings.TrimSpace(swag.BasePath))
	}
	if len(strings.TrimSpace(swaggerURL)) > 0 {
		URLParts = append(URLParts, strings.TrimSpace(swaggerURL))
		URLPathParts = append(URLPathParts, strings.TrimSpace(swaggerURL))
	}

	// Create URL
	rawPostmanURL := strings.TrimSpace(strings.Join(URLParts, "/"))
	rx1 := regexp.MustCompile(`/+`)
	rawPostmanURL = rx1.ReplaceAllString(rawPostmanURL, "/")
	rx2 := regexp.MustCompile(`^/+`)
	rawPostmanURL = rx2.ReplaceAllString(rawPostmanURL, "")

	rawPostmanURLPath := strings.TrimSpace(strings.Join(URLPathParts, "/"))
	rawPostmanURLPath = rx1.ReplaceAllString(rawPostmanURLPath, "/")
	rawPostmanURLPath = rx2.ReplaceAllString(rawPostmanURLPath, "")
	goUrl.Path = rawPostmanURLPath

	// Add URL Scheme
	if len(cfg.PostmanURLBase) == 0 {
		if len(swag.Schemes) > 0 {
			for _, scheme := range swag.Schemes {
				if len(strings.TrimSpace(scheme)) > 0 {
					rawPostmanURL = strings.Join([]string{scheme, rawPostmanURL}, "://")
					goUrl.Scheme = scheme
					break
				}
			}
		}
	}

	/*
		rx3 := regexp.MustCompile(`(^|[^\{])\{([^\/\{\}]+)\}([^\}]|$)`)
		rawPostmanURL = rx3.ReplaceAllString(rawPostmanURL, "$1:$2$3")
		goUrl.Path = rx3.ReplaceAllString(goUrl.Path, "$1:$2$3")
	*/

	goUrl.Path = postman2.ApiUrlOasToPostman(goUrl.Path)
	postmanURL := postman2.NewURLForGoUrl(goUrl)

	// rawPostmanURL = postman2.ApiUrlOasToPostman(rawPostmanURL)
	// postmanURL := postman2.NewURL(rawPostmanURL)

	// Set Default URL Path Parameters
	rx4 := regexp.MustCompile(`^\s*(:(.+))\s*$`)

	for _, part := range postmanURL.Path {
		rs4 := rx4.FindAllStringSubmatch(part, -1)
		if len(rs4) > 0 {
			baseVariable := rs4[0][2]
			var defaultValue interface{}
			for _, parameter := range endpoint.Parameters {
				if parameter.Name == baseVariable {
					defaultValue = parameter.Default
					break
				}
			}
			postmanURL.AddVariable(baseVariable, defaultValue)
		}
	}

	return postmanURL
}
<|endoftext|>


<commit_before>// Copyright © 2016 Jip J. Dekker <jip@dekker.li>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package settings

// Settings provides a structure to interact with the settings
// of a Ponder library
type Settings struct {
	Name             string   // Name of the Ponder library
	IgnoreDirs       []string // Directories to be ignored on search
	LilypondIncludes []string // Directories to be included when running the lilypond compiler
	OutputDir        string   // Directory in which all complete file are stored
}
<commit_msg>Adds a FromFile function to settings<commit_after>// Copyright © 2016 Jip J. Dekker <jip@dekker.li>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package settings

import (
	"encoding/json"
	"io/ioutil"
)

// Settings provides a structure to interact with the settings
// of a Ponder library
type Settings struct {
	Name             string   // Name of the Ponder library
	IgnoreDirs       []string // Directories to be ignored on search
	LilypondIncludes []string // Directories to be included when running the lilypond compiler
	OutputDir        string   // Directory in which all complete file are stored
}

// FromFile reads a settings file in json format and returns the Settings struct
func FromFile(path string) (*Settings, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	var s *Settings
	err = json.Unmarshal(data, s)
	if err != nil {
		return nil, err
	}
	return s, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcm

import (
	"time"

	cadvisor "github.com/google/cadvisor/info"
)

type SupportedMetric struct {
	MetricDescriptor

	// Returns whether this metric is present.
	HasValue func(*cadvisor.ContainerSpec) bool

	// Returns the desired data point for this metric from the stats.
	GetValue func(*cadvisor.ContainerSpec, *cadvisor.ContainerStats) interface{}
}

func GetSupportedMetrics() []SupportedMetric {
	return allMetrics
}

// TODO(vmarmol): Add the rest of the metrics.
var allMetrics = []SupportedMetric{
	{
		MetricDescriptor: MetricDescriptor{
			Name:        "uptime",
			Description: "Number of milliseconds since the container was started",
			Type:        MetricCumulative,
			ValueType:   ValueInt64,
		},
		HasValue: func(spec *cadvisor.ContainerSpec) bool {
			return !spec.CreationTime.IsZero()
		},
		GetValue: func(spec *cadvisor.ContainerSpec, stat *cadvisor.ContainerStats) interface{} {
			return time.Since(spec.CreationTime).Nanoseconds() / time.Millisecond.Nanoseconds()
		},
	},
	{
		MetricDescriptor: MetricDescriptor{
			Name:        "cpu/usage",
			Description: "Cumulative CPU usage on all cores",
			Type:        MetricCumulative,
			ValueType:   ValueInt64,
		},
		HasValue: func(spec *cadvisor.ContainerSpec) bool {
			return spec.HasCpu
		},
		GetValue: func(spec *cadvisor.ContainerSpec, stat *cadvisor.ContainerStats) interface{} {
			return int64(stat.Cpu.Usage.Total)
		},
	},
	{
		MetricDescriptor: MetricDescriptor{
			Name:        "memory/usage",
			Description: "Total memory usage",
			Type:        MetricGauge,
			ValueType:   ValueInt64,
		},
		HasValue: func(spec *cadvisor.ContainerSpec) bool {
			return spec.HasMemory
		},
		GetValue: func(spec *cadvisor.ContainerSpec, stat *cadvisor.ContainerStats) interface{} {
			return int64(stat.Memory.Usage)
		},
	},
	{
		MetricDescriptor: MetricDescriptor{
			Name:        "memory/working_set",
			Description: "Total working set usage. Working set is the memory being used and not easily dropped by the kernel",
			Type:        MetricGauge,
			ValueType:   ValueInt64,
		},
		HasValue: func(spec *cadvisor.ContainerSpec) bool {
			return spec.HasMemory
		},
		GetValue: func(spec *cadvisor.ContainerSpec, stat *cadvisor.ContainerStats) interface{} {
			return int64(stat.Memory.WorkingSet)
		},
	},
}
<commit_msg>Adding network metrics to GCM sink.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcm

import (
	"time"

	cadvisor "github.com/google/cadvisor/info"
)

type SupportedMetric struct {
	MetricDescriptor

	// Returns whether this metric is present.
	HasValue func(*cadvisor.ContainerSpec) bool

	// Returns the desired data point for this metric from the stats.
	GetValue func(*cadvisor.ContainerSpec, *cadvisor.ContainerStats) interface{}
}

func GetSupportedMetrics() []SupportedMetric {
	return allMetrics
}

var allMetrics = []SupportedMetric{
	{
		MetricDescriptor: MetricDescriptor{
			Name:        "uptime",
			Description: "Number of milliseconds since the container was started",
			Type:        MetricCumulative,
			ValueType:   ValueInt64,
		},
		HasValue: func(spec *cadvisor.ContainerSpec) bool {
			return !spec.CreationTime.IsZero()
		},
		GetValue: func(spec *cadvisor.ContainerSpec, stat *cadvisor.ContainerStats) interface{} {
			return time.Since(spec.CreationTime).Nanoseconds() / time.Millisecond.Nanoseconds()
		},
	},
	{
		MetricDescriptor: MetricDescriptor{
			Name:        "cpu/usage",
			Description: "Cumulative CPU usage on all cores",
			Type:        MetricCumulative,
			ValueType:   ValueInt64,
		},
		HasValue: func(spec *cadvisor.ContainerSpec) bool {
			return spec.HasCpu
		},
		GetValue: func(spec *cadvisor.ContainerSpec, stat *cadvisor.ContainerStats) interface{} {
			return int64(stat.Cpu.Usage.Total)
		},
	},
	{
		MetricDescriptor: MetricDescriptor{
			Name:        "memory/usage",
			Description: "Total memory usage",
			Type:        MetricGauge,
			ValueType:   ValueInt64,
		},
		HasValue: func(spec *cadvisor.ContainerSpec) bool {
			return spec.HasMemory
		},
		GetValue: func(spec *cadvisor.ContainerSpec, stat *cadvisor.ContainerStats) interface{} {
			return int64(stat.Memory.Usage)
		},
	},
	{
		MetricDescriptor: MetricDescriptor{
			Name:        "memory/working_set",
			Description: "Total working set usage. Working set is the memory being used and not easily dropped by the kernel",
			Type:        MetricGauge,
			ValueType:   ValueInt64,
		},
		HasValue: func(spec *cadvisor.ContainerSpec) bool {
			return spec.HasMemory
		},
		GetValue: func(spec *cadvisor.ContainerSpec, stat *cadvisor.ContainerStats) interface{} {
			return int64(stat.Memory.WorkingSet)
		},
	},
	{
		MetricDescriptor: MetricDescriptor{
			Name:        "memory/page_faults",
			Description: "Number of major page faults",
			Type:        MetricGauge,
			ValueType:   ValueInt64,
		},
		HasValue: func(spec *cadvisor.ContainerSpec) bool {
			return spec.HasMemory
		},
		GetValue: func(spec *cadvisor.ContainerSpec, stat *cadvisor.ContainerStats) interface{} {
			return int64(stat.Memory.ContainerData.Pgfault)
		},
	},
	{
		MetricDescriptor: MetricDescriptor{
			Name:        "network/rx_bytes",
			Description: "Cumulative number of bytes received over the network",
			Type:        MetricCumulative,
			ValueType:   ValueInt64,
		},
		HasValue: func(spec *cadvisor.ContainerSpec) bool {
			return spec.HasNetwork
		},
		GetValue: func(spec *cadvisor.ContainerSpec, stat *cadvisor.ContainerStats) interface{} {
			return int64(stat.Network.RxBytes)
		},
	},
	{
		MetricDescriptor: MetricDescriptor{
			Name:        "network/rx_errors",
			Description: "Cumulative number of errors while receiving over the network",
			Type:        MetricCumulative,
			ValueType:   ValueInt64,
		},
		HasValue: func(spec *cadvisor.ContainerSpec) bool {
			return spec.HasNetwork
		},
		GetValue: func(spec *cadvisor.ContainerSpec, stat *cadvisor.ContainerStats) interface{} {
			return int64(stat.Network.RxErrors)
		},
	},
	{
		MetricDescriptor: MetricDescriptor{
			Name:        "network/tx_bytes",
			Description: "Cumulative number of bytes sent over the network",
			Type:        MetricCumulative,
			ValueType:   ValueInt64,
		},
		HasValue: func(spec *cadvisor.ContainerSpec) bool {
			return spec.HasNetwork
		},
		GetValue: func(spec *cadvisor.ContainerSpec, stat *cadvisor.ContainerStats) interface{} {
			return int64(stat.Network.TxBytes)
		},
	},
	{
		MetricDescriptor: MetricDescriptor{
			Name:        "network/tx_errors",
			Description: "Cumulative number of errors while sending over the network",
			Type:        MetricCumulative,
			ValueType:   ValueInt64,
		},
		HasValue: func(spec *cadvisor.ContainerSpec) bool {
			return spec.HasNetwork
		},
		GetValue: func(spec *cadvisor.ContainerSpec, stat *cadvisor.ContainerStats) interface{} {
			return int64(stat.Network.TxErrors)
		},
	},
	// TODO(vmarmol): DiskIO stats if we find those useful and know how to export them in a user-friendly way.
	// TODO(vmarmol): FS usage. That requires more labels that the rest here so we will need a new function.
}
<|endoftext|>


<commit_before>package sflow

import (
	"fmt"
)

type IPv4Header struct {
	Version  int    // protocol version
	TOS      int    // type-of-service
	TotalLen int    // packet total length
	ID       int    // identification
	Flags    int    // flags
	FragOff  int    // fragment offset
	TTL      int    // time-to-live
	Protocol int    // next protocol
	Checksum int    // checksum
	Src      string // source address
	Dst      string // destination address
	Options  []byte // options, extension headers
}

const (
	EtherTypeIPv4      = 0x0800
	EtherTypeIPv6      = 0x86DD
	EtherTypeIEEE8021Q = 0x8100
)

func decodeISO88023(b []byte) {

	if len(b) < 14 {
		return
	}

	etherType := uint16(b[13]) | uint16(b[12])<<8
	switch etherType {
	case EtherTypeIPv4:
		h := decodeIPv4Header(b[14:])
		fmt.Printf("IPv4: %#v\n", h)
	case EtherTypeIEEE8021Q:
		fmt.Println("802.1Q")
	}
}

func decodeIPv4Header(b []byte) IPv4Header {
	return IPv4Header{
		Version:  int(b[0] & 0xf0 >> 4),
		TOS:      int(b[1]),
		TotalLen: int(b[2])<<8 | int(b[3]),
		ID:       int(b[4])<<8 | int(b[5]),
		Flags:    int(b[6] & 0x07),
		TTL:      int(b[8]),
		Protocol: int(b[9]),
		Checksum: int(b[10])<<8 | int(b[11]),
		Src:      fmt.Sprintf("%d.%d.%d.%d", b[12], b[13], b[14], b[15]),
		Dst:      fmt.Sprintf("%d.%d.%d.%d", b[16], b[17], b[18], b[19]),
	}
}
<commit_msg>add ethernet, 802.1q and ipv4 decoder<commit_after>package sflow

import (
	"errors"
	"fmt"
)

// IPv4Header represents an IPv4 header
type IPv4Header struct {
	Version  int    // protocol version
	TOS      int    // type-of-service
	TotalLen int    // packet total length
	ID       int    // identification
	Flags    int    // flags
	FragOff  int    // fragment offset
	TTL      int    // time-to-live
	Protocol int    // next protocol
	Checksum int    // checksum
	Src      string // source address
	Dst      string // destination address
}

// IPv6Header represents an IPv6 header
type IPv6Header struct {
	Version      int    // protocol version
	TrafficClass int    // traffic class
	FlowLabel    int    // flow label
	PayloadLen   int    // payload length
	NextHeader   int    // next header
	HopLimit     int    // hop limit
	Src          string // source address
	Dst          string // destination address
}

type Datalink struct {
	SrcMAC    string
	DstMAC    string
	Vlan      int
	EtherType uint16
}

type Packet struct {
	L2 Datalink
	L3 interface{}
	L4 interface{}
}

const (
	EtherTypeIPv4      = 0x0800
	EtherTypeIPv6      = 0x86DD
	EtherTypeIEEE8021Q = 0x8100
)

var (
	errShortEthernetHeaderLength = errors.New("the ethernet header is too small")
	errShortIPv4HeaderLength     = errors.New("short ipv4 header length")
	errShortIPv6HeaderLength     = errors.New("short ipv6 header length")
	errShortEthernetLength       = errors.New("short ethernet header length")
	errUnknownEtherType          = errors.New("unknown ether type")
)

func decodeISO88023(b []byte) (Packet, error) {
	var (
		err    error
		packet Packet
	)

	if len(b) < 14 {
		return packet, errShortEthernetHeaderLength
	}

	packet.L2, err = decodeIEEE802(b)
	if err != nil {
		return packet, err
	}

	switch packet.L2.EtherType {
	case EtherTypeIPv4:
		b = b[14:]

		packet.L3, err = decodeIPv4Header(b)
		if err != nil {
			return packet, err
		}

		packet.L4, err = decodeTransportLayer(int(b[9]), b[20:])
		if err != nil {
			return packet, err
		}

		return packet, nil

	case EtherTypeIPv6:
	case EtherTypeIEEE8021Q:
		vlan := int(b[14])<<8 | int(b[15])
		b[12], b[13] = b[16], b[17]
		b = append(b[:14], b[18:]...)
		packet, err = decodeISO88023(b)
		if err != nil {
			return packet, err
		}
		packet.L2.Vlan = vlan
		return packet, nil

	default:
		return packet, errUnknownEtherType
	}

	return packet, nil
}

func decodeIPv6Header(b []byte) (IPv6Header, error) {
	if len(b) < 40 {
		return IPv6Header{}, errShortIPv6HeaderLength
	}

	return IPv6Header{}, nil
}

func decodeIPv4Header(b []byte) (IPv4Header, error) {
	if len(b) < 20 {
		return IPv4Header{}, errShortIPv4HeaderLength
	}

	return IPv4Header{
		Version:  int(b[0] & 0xf0 >> 4),
		TOS:      int(b[1]),
		TotalLen: int(b[2])<<8 | int(b[3]),
		ID:       int(b[4])<<8 | int(b[5]),
		Flags:    int(b[6] & 0x07),
		TTL:      int(b[8]),
		Protocol: int(b[9]),
		Checksum: int(b[10])<<8 | int(b[11]),
		Src:      fmt.Sprintf("%d.%d.%d.%d", b[12], b[13], b[14], b[15]),
		Dst:      fmt.Sprintf("%d.%d.%d.%d", b[16], b[17], b[18], b[19]),
	}, nil
}

func decodeIEEE802(b []byte) (Datalink, error) {
	var d Datalink

	if len(b) < 14 {
		return d, errShortEthernetLength
	}

	d.EtherType = uint16(b[13]) | uint16(b[12])<<8

	if d.EtherType != EtherTypeIEEE8021Q {
		d.SrcMAC = fmt.Sprintf("%0.2x:%0.2x:%0.2x:%0.2x:%0.2x:%0.2x", b[0], b[1], b[2], b[3], b[4], b[5])
		d.DstMAC = fmt.Sprintf("%0.2x:%0.2x:%0.2x:%0.2x:%0.2x:%0.2x", b[6], b[7], b[8], b[9], b[10], b[11])
	}

	return d, nil
}
<|endoftext|>


<commit_before>package sflow

import (
	"errors"
	"io"

	"git.edgecastcdn.net/vflow/packet"
)

const (
	SFDataRawHeader      = 1
	SFDataEthernetHeader = 2
	SFDataIPV4Header     = 3
	SFDataIPV6Header     = 4

	SFDataExtSwitch     = 1001
	SFDataExtRouter     = 1002
	SFDataExtGateway    = 1003
	SFDataExtUser       = 1004
	SFDataExtURL        = 1005
	SFDataExtMPLS       = 1006
	SFDataExtNAT        = 1007
	SFDataExtMPLSTunnel = 1008
	SFDataExtMPLSVC     = 1009
	SFDataExtMPLSFEC    = 1010
	SFDataExtMPLSLVPFEC = 1011
	SFDataExtVLANTunnel = 1012
)

type FlowSample struct {
	SequenceNo   uint32 // Incremented with each flow sample
	SourceId     byte   // fsSourceId
	SamplingRate uint32 // fsPacketSamplingRate
	SamplePool   uint32 // Total number of packets that could have been sampled
	Drops        uint32 // Number of times a packet was dropped due to lack of resources
	Input        uint32 // SNMP ifIndex of input interface
	Output       uint32 // SNMP ifIndex of input interface
	RecordsNo    uint32 // Number of records to follow
}

type SampledHeader struct {
	Protocol     uint32 // (enum SFLHeader_protocol)
	FrameLength  uint32 // Original length of packet before sampling
	Stripped     uint32 // Header/trailer bytes stripped by sender
	HeaderLength uint32 // Length of sampled header bytes to follow
	Header       []byte // Header bytes
}

var (
	maxOutEthernetLength = errors.New("the ethernet lenght is greater than 1500")
)

func decodeFlowSample(r io.ReadSeeker) (interface{}, error) {
	var (
		fs          = new(FlowSample)
		rTypeFormat uint32
		rTypeLength uint32
		err         error
	)

	if err = read(r, &fs.SequenceNo); err != nil {
		return nil, err
	}

	if err = read(r, &fs.SourceId); err != nil {
		return nil, err
	}

	r.Seek(3, 1) // skip counter sample decoding

	if err = read(r, &fs.SamplingRate); err != nil {
		return nil, err
	}

	if err = read(r, &fs.SamplePool); err != nil {
		return nil, err
	}

	if err = read(r, &fs.Drops); err != nil {
		return nil, err
	}

	if err = read(r, &fs.Input); err != nil {
		return nil, err
	}

	if err = read(r, &fs.Output); err != nil {
		return nil, err
	}

	if err = read(r, &fs.RecordsNo); err != nil {
		return nil, err
	}

	for i := uint32(0); i < fs.RecordsNo; i++ {
		if err = read(r, &rTypeFormat); err != nil {
			return nil, err
		}
		if err = read(r, &rTypeLength); err != nil {
			return nil, err
		}

		switch rTypeFormat {
		case SFDataRawHeader:
			h, err := decodeSampledHeader(r)
			return h, err
		case SFDataExtSwitch:
			r.Seek(int64(rTypeLength), 1)
		default:
			r.Seek(int64(rTypeLength), 1)
		}
	}

	return nil, nil
}

func decodeSampledHeader(r io.Reader) (*packet.Packet, error) {
	// TODO
	var (
		h   = new(SampledHeader)
		err error
	)
	if err = read(r, &h.Protocol); err != nil {
		return nil, err
	}

	if err = read(r, &h.FrameLength); err != nil {
		return nil, err
	}

	if err = read(r, &h.Stripped); err != nil {
		return nil, err
	}

	if err = read(r, &h.HeaderLength); err != nil {
		return nil, err
	}

	if h.HeaderLength > 1500 {
		return nil, maxOutEthernetLength
	}

	// TODO: make sure the padding works!!
	// cut off a header length mod 4 == 0 number of bytes
	tmp := (4 - h.HeaderLength) % 4
	if tmp < 0 {
		tmp += 4
	}

	h.Header = make([]byte, h.HeaderLength+tmp)
	if _, err = r.Read(h.Header); err != nil {
		return nil, err
	}

	h.Header = h.Header[:h.HeaderLength]

	p := packet.NewPacket()
	d, err := p.Decoder(h.Header)
	if err != nil {
		return nil, err
	}

	return d, nil
}
<commit_msg>add extended switch data decoder, add data as slice<commit_after>package sflow

import (
	"errors"
	"io"

	"git.edgecastcdn.net/vflow/packet"
)

const (
	SFDataRawHeader      = 1
	SFDataEthernetHeader = 2
	SFDataIPV4Header     = 3
	SFDataIPV6Header     = 4

	SFDataExtSwitch     = 1001
	SFDataExtRouter     = 1002
	SFDataExtGateway    = 1003
	SFDataExtUser       = 1004
	SFDataExtURL        = 1005
	SFDataExtMPLS       = 1006
	SFDataExtNAT        = 1007
	SFDataExtMPLSTunnel = 1008
	SFDataExtMPLSVC     = 1009
	SFDataExtMPLSFEC    = 1010
	SFDataExtMPLSLVPFEC = 1011
	SFDataExtVLANTunnel = 1012
)

type FlowSample struct {
	SequenceNo   uint32 // Incremented with each flow sample
	SourceId     byte   // fsSourceId
	SamplingRate uint32 // fsPacketSamplingRate
	SamplePool   uint32 // Total number of packets that could have been sampled
	Drops        uint32 // Number of times a packet was dropped due to lack of resources
	Input        uint32 // SNMP ifIndex of input interface
	Output       uint32 // SNMP ifIndex of input interface
	RecordsNo    uint32 // Number of records to follow
}

type SampledHeader struct {
	Protocol     uint32 // (enum SFLHeader_protocol)
	FrameLength  uint32 // Original length of packet before sampling
	Stripped     uint32 // Header/trailer bytes stripped by sender
	HeaderLength uint32 // Length of sampled header bytes to follow
	Header       []byte // Header bytes
}

type ExtSwitchData struct {
	SrcVlan     uint32
	SrcPriority uint32
	DstVlan     uint32
	DstPriority uint32
}

var (
	maxOutEthernetLength = errors.New("the ethernet lenght is greater than 1500")
)

func decodeFlowSample(r io.ReadSeeker) ([]interface{}, error) {
	var (
		fs          = new(FlowSample)
		data        []interface{}
		rTypeFormat uint32
		rTypeLength uint32
		err         error
	)

	if err = read(r, &fs.SequenceNo); err != nil {
		return nil, err
	}

	if err = read(r, &fs.SourceId); err != nil {
		return nil, err
	}

	r.Seek(3, 1) // skip counter sample decoding

	if err = read(r, &fs.SamplingRate); err != nil {
		return nil, err
	}

	if err = read(r, &fs.SamplePool); err != nil {
		return nil, err
	}

	if err = read(r, &fs.Drops); err != nil {
		return nil, err
	}

	if err = read(r, &fs.Input); err != nil {
		return nil, err
	}

	if err = read(r, &fs.Output); err != nil {
		return nil, err
	}

	if err = read(r, &fs.RecordsNo); err != nil {
		return nil, err
	}

	data = append(data, fs)

	for i := uint32(0); i < fs.RecordsNo; i++ {
		if err = read(r, &rTypeFormat); err != nil {
			return nil, err
		}
		if err = read(r, &rTypeLength); err != nil {
			return nil, err
		}

		switch rTypeFormat {
		case SFDataRawHeader:
			h, err := decodeSampledHeader(r)
			if err != nil {
				return data, err
			}
			data = append(data, h)
		case SFDataExtSwitch:
			d, err := decodeExtSwitchData(r)
			if err != nil {
				return data, err
			}
			data = append(data, d)
		default:
			r.Seek(int64(rTypeLength), 1)
		}
	}

	return data, nil
}

func decodeSampledHeader(r io.Reader) (*packet.Packet, error) {
	// TODO
	var (
		h   = new(SampledHeader)
		err error
	)
	if err = read(r, &h.Protocol); err != nil {
		return nil, err
	}

	if err = read(r, &h.FrameLength); err != nil {
		return nil, err
	}

	if err = read(r, &h.Stripped); err != nil {
		return nil, err
	}

	if err = read(r, &h.HeaderLength); err != nil {
		return nil, err
	}

	if h.HeaderLength > 1500 {
		return nil, maxOutEthernetLength
	}

	// TODO: make sure the padding works!!
	// cut off a header length mod 4 == 0 number of bytes
	tmp := (4 - h.HeaderLength) % 4
	if tmp < 0 {
		tmp += 4
	}

	h.Header = make([]byte, h.HeaderLength+tmp)
	if _, err = r.Read(h.Header); err != nil {
		return nil, err
	}

	h.Header = h.Header[:h.HeaderLength]

	p := packet.NewPacket()
	d, err := p.Decoder(h.Header)
	if err != nil {
		return nil, err
	}

	return d, nil
}

func decodeExtSwitchData(r io.Reader) (*ExtSwitchData, error) {
	var (
		d   = new(ExtSwitchData)
		err error
	)

	if err = read(r, &d.SrcVlan); err != nil {
		return nil, err
	}

	if err = read(r, &d.SrcPriority); err != nil {
		return nil, err
	}

	if err = read(r, &d.DstVlan); err != nil {
		return nil, err
	}

	if err = read(r, &d.SrcPriority); err != nil {
		return nil, err
	}

	return d, nil
}
<|endoftext|>


<commit_before>package yarn

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/bitly/go-simplejson"
	"github.com/rootsongjc/magpie/docker"
	"github.com/rootsongjc/magpie/utils"
	"github.com/samalba/dockerclient"
	"github.com/spf13/viper"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"sort"
	"strings"
	"sync"
	"time"
)

type YarnStatus struct {
	appsPending           string
	reservedVirtualCores  string
	availableVirtualCores string
	allocatedVirtualCores string
	totalVirtualCores     string
	lostNodes             int64
	activeNodes           int64
	appsRunning           string
	appsFailed            string
	appsKilled            string
	availableMB           string
	allocatedMB           string
	containersPending     string
	totalMB               string
	totalNodes            int64
	rebootedNodes         string
	appsSubmitted         string
	appsCompleted         string
	containersAllocated   string
	reservedMB            string
	containersReserved    string
	unhealthyNodes        int64
	decommissionedNodes   int64
}

//Get yarn cluster node status
func Get_yarn_status(cluster_names []string) {
	fmt.Println("======================YARN CLSUTER STATUS===========================")
	fmt.Println("CLUSTER\tTOTAL\tACTIVE\tDECOM\tLOST\tUNHEALTHY\tUsed")
	var total_nodes, total_active, total_decom, total_lost, total_unhealthy int64
	for i := range cluster_names {
		name := cluster_names[i]
		url := "http://" + utils.Clustername2ip(name) + ":8088/ws/v1/cluster/metrics"
		resp, err := http.Get(url)
		if err != nil {
			fmt.Println("Yarn cluster ", name, " not found.")
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		js, err := simplejson.NewJson(body)
		if err != nil {
			panic(err.Error())
		}
		nodes, _ := js.Get("clusterMetrics").Map()
		activeNodes, _ := nodes["activeNodes"].(json.Number).Int64()
		totalNodes, _ := nodes["totalNodes"].(json.Number).Int64()
		decommissionedNodes, _ := nodes["decommissionedNodes"].(json.Number).Int64()
		lostNodes, _ := nodes["lostNodes"].(json.Number).Int64()
		unhealthyNodes, _ := nodes["unhealthyNodes"].(json.Number).Int64()
		usage := get_yarn_resource_usage(name)
		total_active += activeNodes
		total_decom += decommissionedNodes
		total_nodes += totalNodes
		total_lost += lostNodes
		total_unhealthy += unhealthyNodes
		fmt.Println(name, "\t", totalNodes, "\t", activeNodes, "\t", decommissionedNodes,
			"\t", lostNodes, "\t", unhealthyNodes, "\t", usage)
	}
	fmt.Println("--------------------------------------------------------------------")
	fmt.Println("TOTAL", "\t", total_nodes, "\t", total_active, "\t", total_decom, "\t", total_lost, "\t", total_unhealthy)
}

//Get yarn cluster resource usage percent
func get_yarn_resource_usage(clustername string) float64 {
	url := "http://" + utils.Clustername2ip(clustername) + ":8088/ws/v1/cluster/scheduler"
	resp, err := http.Get(url)
	if err != nil {
		fmt.Println("Yarn cluster ", clustername, " not found.")
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	js, err := simplejson.NewJson(body)
	if err != nil {
		panic(err.Error())
	}
	used, _ := js.Get("scheduler").Get("schedulerInfo").Get("usedCapacity").Float64()
	return used
}

//Show the yarn nodemanagers distribution.
func Yarn_distribution(clustername string) {
	fmt.Println("====================RUNNING DOCKERS DISTRIBUTION====================")
	fmt.Println("HOSTNAME\tNUM")
	containers, err := docker.Get_running_docker_containers()
	if err != nil {
		panic(err)
	} else {
		distribution := make(map[string]int, len(containers))
		for i := range containers {
			c := docker.Get_container_name(containers[i].Names[0])
			if strings.HasPrefix(c, clustername) {
				h := docker.Get_nodemanager_host(containers[i].Names[0])
				distribution[h] += 1
			}
		}
		// sorted as the hostname
		sorted_keys := make([]string, 0)
		for k, _ := range distribution {
			sorted_keys = append(sorted_keys, k)
		}

		sort.Strings(sorted_keys)

		for _, k := range sorted_keys {
			fmt.Println(k, "\t", distribution[k])
		}
	}

}

//Inspect the yarn cluster container view
func Yarn_view(clustername string) {
	containers := docker.Get_all_yarn_containers()
	flag := false
	fmt.Println("ID\tCLUSTER\tNAME\tSTATUS\tSTATE\tIP\tHOST")
	fmt.Println("=======================================================================================================================================")
	for i := range containers {
		c := containers[i]
		if c.Clustername == clustername {
			flag = true
			fmt.Println(c.ID, "\t", c.Clustername, "\t", c.Name, "\t", c.Status, "\t", c.State, "\t", c.Ip, "\t", c.Host)
		}
	}
	if flag == false {
		fmt.Println("The cluster does not exited.")
	}

}

//Decommising the nodemanagers on each resourcemanagers.
//No matter which yarn clusters the nodemanager belonging to,
//you can put the in the same nodefile togethter.
//Magpie will recognize the yarn cluster automatically.
func Decommis_nodemanagers(nodemanagers []string) {
	nms := make([]docker.Yarn_docker_container, 0)
	containers := docker.Get_all_yarn_containers()
	for _, n := range nodemanagers {
		found := false
		for _, c := range containers {
			cid := c.ID
			if cid == n {
				found = true
				nms = append(nms, c)
			}
		}
		if found == false {
			fmt.Println("The nodemanager can not be found:", n)
		}
	}

	yarns := make(map[string]string, len(nodemanagers))
	for _, n := range nms {
		yarns[n.Clustername] = yarns[n.Clustername] + "\n" + n.ID
	}
	//	yarns := map[string]string{
	//		"yarn1": `3cd9494cdc80
	//0ebc7d9cf054`,
	//		"yarn2": "falj2ljfao3k",
	//	}
	fmt.Println("Decommising the following nodemangers...")
	var wg sync.WaitGroup
	//traverse all the yarn clusters
	for k, v := range yarns {
		wg.Add(1)
		go decommis_yarn_nodes(k, v, &wg)
	}
	wg.Wait()

}

// Decommising the nodemanager of the nodefile
func Decommis_nodemanagers_through_file(nodefile string) {
	fi, err := os.Open(nodefile)
	if err != nil {
		panic(err)
	}
	defer fi.Close()
	buff := bufio.NewReader(fi)
	nodes := make([]string, 0)
	for {
		line, err := buff.ReadString('\n')
		if err != nil || io.EOF == err {
			break
		}
		node := strings.Trim(line, "\n")
		nodes = append(nodes, node)
	}
	Decommis_nodemanagers(nodes)
}

//Decommising the nodemanagers of a yarn
func decommis_yarn_nodes(clustername string, nodemanagers string, wg *sync.WaitGroup) {
	if wg != nil {
		defer wg.Done()
	}
	logger := utils.Logger()
	fmt.Println(clustername, nodemanagers)
	logger.WithFields(logrus.Fields{"Time": time.Now(), "Cluster": clustername, "Nodemanagers": strings.Replace(nodemanagers, "\n", ",", -1), "Action": "DECOM"}).Info("Decomissing nodemanagers " + nodemanagers)
	resource_manager_ip := utils.Clustername2ip(clustername)
	nodemanger_exclude_file := viper.GetString("clusters.nodemanager_exclude_file")
	//command := "ssh -n root@" + resource_manager_ip + ` "echo -e '` + nodemanagers + `'>>/usr/local/hadoop/etc/hadoop/test.txt"`
	command := "ssh -n jingchao.song@" + "172.20.0.6" + ` "echo -e '` + nodemanagers + `'>>` + nodemanger_exclude_file + `"`
	//TODO should return result and error handler
	utils.Run_command(command)
	command = "ssh -n root@" + resource_manager_ip + ` 'su - hadoop -c "yarn rmadmin -refreshNodes"'`
	utils.Run_command(command)

}

//Offline the host, decommsing the nodemanagers and then delete the docker contianers.
func Offline_host(hostname string) {
	fmt.Println("Offline host", hostname, "...")
	containers := docker.Get_all_yarn_containers()
	nms := make([]string, 0)
	for _, c := range containers {
		host := c.Host
		if host == hostname {
			nms = append(nms, c.Name)
		}
	}
	Decommis_nodemanagers(nms)
	docker.Delete_containers_on_host(hostname)
}

//Create a new nodemanager and start it
func Create_new_nodemanager(nm_config Nodemanager_config) {
	swarm_master_ip := viper.GetString("clusters.swarm_master_ip")
	swarm_master_port := viper.GetString("clusters.swarm_master_port")
	endpoint := "tcp://" + swarm_master_ip + ":" + swarm_master_port
	client, err := dockerclient.NewDockerClient(endpoint, nil)
	logger := utils.Logger()
	if err != nil {
		panic(err)
	}
	if err != nil {
		fmt.Println("Cannot connect to the swarm master.")
	}
	fmt.Println("Creating new nodemanager container...")
	env := []string{
		"HA=" + get_nodemanager_config(nm_config.HA, "HA"),
		"NAMESERVICE=" + get_nodemanager_config(nm_config.NAMESERVICE, "NAMESERVICE"),
		"ACTIVE_NAMENODE_IP=" + get_nodemanager_config(nm_config.ACTIVE_NAMENODE_IP, "ACTIVE_NAMENODE_IP"),
		"STANDBY_NAMENODE_IP=" + get_nodemanager_config(nm_config.STANDBY_NAMENODE_IP, "STANDBY_NAMENODE_IP"),
		"ACTIVE_NAMENODE_ID=" + get_nodemanager_config(nm_config.ACTIVE_NAMENODE_ID, "ACTIVE_NAMENODE_ID"),
		"STANDBY_NAMENODE_ID=" + get_nodemanager_config(nm_config.STANDBY_NAMENODE_ID, "STANDBY_NAMENODE_ID"),
		"HA_ZOOKEEPER_QUORUM=" + get_nodemanager_config(nm_config.HA_ZOOKEEPER_QUORUM, "HA_ZOOKEEPER_QUORUM"),
		"NAMENODE_IP=" + get_nodemanager_config(nm_config.NAMENODE_IP, "NAMENODE_IP"),
		"RESOURCEMANAGER_IP=" + get_nodemanager_config(nm_config.RESOURCEMANAGER_IP, "RESOURCEMANAGER_IP"),
		"YARN_RM1_IP=" + nm_config.YARN_RM1_IP,
		"YARN_RM2_IP=" + nm_config.YARN_RM2_IP,
		"YARN_JOBHISTORY_IP=" + nm_config.YARN_JOBHISTORY_IP,
		"CPU_CORE_NUM=" + get_nodemanager_config(nm_config.CPU_CORE_NUM, "CPU_CORE_NUM"),
		"NODEMANAGER_MEMORY_MB=" + get_nodemanager_config(nm_config.NODEMANAGER_MEMORY_MB, "NODEMANAGER_MEMORY_MB"),
		"YARN_CLUSTER_ID=" + nm_config.YARN_CLUSTER_ID,
		"YARN_ZK_DIR=" + nm_config.YARN_ZK_DIR,
		//"PATH=/usr/local/hadoop/bin:/usr/local/hadoop/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/java/bin://usr/local/java/jre/bin",
	}
	hostConifg := dockerclient.HostConfig{
		CpuShares:   nm_config.Limit_cpus,
		Memory:      nm_config.Limit_memory_mb * 1024 * 1024, //transform to Byte
		NetworkMode: nm_config.Network_mode,
	}
	var config *dockerclient.ContainerConfig
	config = new(dockerclient.ContainerConfig)
	config.Image = get_nodemanager_config(nm_config.Image, "image")
	config.Env = env

	//inherit Cmd and Entrypoint settings from docker Image or set them on config file
	config.Cmd = viper.GetStringSlice("nodemanager.cmd")
	config.Entrypoint = viper.GetStringSlice("nodemanager.entrypoint")
	config.HostConfig = hostConifg
	id, err := client.CreateContainer(config, "", nil)
	if err != nil {
		logger.WithFields(logrus.Fields{"Time": time.Now(), "ContainerID": "-", "Action": "CREATE"}).Error(err)
		panic(err)
	}
	container_name := id[0:12]
	fmt.Println("Container", container_name, "created.")
	logger.WithFields(logrus.Fields{"Time": time.Now(), "ContainerID": container_name, "Action": "CREATE"}).Info("Create a new nodemanager docker container")

	if nm_config.Container_name != "" {
		err = client.RenameContainer(container_name, nm_config.Container_name)
		if err != nil {
			logger.WithFields(logrus.Fields{"Time": time.Now(), "ContainerID": container_name, "Action": "RENAME"}).Error(err)
			panic(err)
		}
		fmt.Println("Rename container name to", nm_config.Container_name)
		logger.WithFields(logrus.Fields{"Time": time.Now(), "ContainerID": container_name, "Action": "RENAME"}).Info("Rename container "+container_name+" name to ", nm_config.Container_name)
	}
	err = client.StartContainer(id, nil)
	if err != nil {
		logger.WithFields(logrus.Fields{"Time": time.Now(), "ContainerID": container_name, "Action": "START"}).Error(err)
		panic(err)
	}
	fmt.Println("Started.")
	logger.WithFields(logrus.Fields{"Time": time.Now(), "ContainerID": container_name, "Action": "START"}).Info("Start container " + container_name)

}

//Get nodemanger configuration item from config file
//If no config specify throught the command line,use the default settings
func get_nodemanager_config(nm string, config string) string {
	if nm == "" {
		return viper.GetString("nodemanager." + config)
	}
	return nm
}
<commit_msg>fix decomis bug<commit_after>package yarn

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/bitly/go-simplejson"
	"github.com/rootsongjc/magpie/docker"
	"github.com/rootsongjc/magpie/utils"
	"github.com/samalba/dockerclient"
	"github.com/spf13/viper"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"sort"
	"strings"
	"sync"
	"time"
)

type YarnStatus struct {
	appsPending           string
	reservedVirtualCores  string
	availableVirtualCores string
	allocatedVirtualCores string
	totalVirtualCores     string
	lostNodes             int64
	activeNodes           int64
	appsRunning           string
	appsFailed            string
	appsKilled            string
	availableMB           string
	allocatedMB           string
	containersPending     string
	totalMB               string
	totalNodes            int64
	rebootedNodes         string
	appsSubmitted         string
	appsCompleted         string
	containersAllocated   string
	reservedMB            string
	containersReserved    string
	unhealthyNodes        int64
	decommissionedNodes   int64
}

//Get yarn cluster node status
func Get_yarn_status(cluster_names []string) {
	fmt.Println("======================YARN CLSUTER STATUS===========================")
	fmt.Println("CLUSTER\tTOTAL\tACTIVE\tDECOM\tLOST\tUNHEALTHY\tUsed")
	var total_nodes, total_active, total_decom, total_lost, total_unhealthy int64
	for i := range cluster_names {
		name := cluster_names[i]
		url := "http://" + utils.Clustername2ip(name) + ":8088/ws/v1/cluster/metrics"
		resp, err := http.Get(url)
		if err != nil {
			fmt.Println("Yarn cluster ", name, " not found.")
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		js, err := simplejson.NewJson(body)
		if err != nil {
			panic(err.Error())
		}
		nodes, _ := js.Get("clusterMetrics").Map()
		activeNodes, _ := nodes["activeNodes"].(json.Number).Int64()
		totalNodes, _ := nodes["totalNodes"].(json.Number).Int64()
		decommissionedNodes, _ := nodes["decommissionedNodes"].(json.Number).Int64()
		lostNodes, _ := nodes["lostNodes"].(json.Number).Int64()
		unhealthyNodes, _ := nodes["unhealthyNodes"].(json.Number).Int64()
		usage := get_yarn_resource_usage(name)
		total_active += activeNodes
		total_decom += decommissionedNodes
		total_nodes += totalNodes
		total_lost += lostNodes
		total_unhealthy += unhealthyNodes
		fmt.Println(name, "\t", totalNodes, "\t", activeNodes, "\t", decommissionedNodes,
			"\t", lostNodes, "\t", unhealthyNodes, "\t", usage)
	}
	fmt.Println("--------------------------------------------------------------------")
	fmt.Println("TOTAL", "\t", total_nodes, "\t", total_active, "\t", total_decom, "\t", total_lost, "\t", total_unhealthy)
}

//Get yarn cluster resource usage percent
func get_yarn_resource_usage(clustername string) float64 {
	url := "http://" + utils.Clustername2ip(clustername) + ":8088/ws/v1/cluster/scheduler"
	resp, err := http.Get(url)
	if err != nil {
		fmt.Println("Yarn cluster ", clustername, " not found.")
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	js, err := simplejson.NewJson(body)
	if err != nil {
		panic(err.Error())
	}
	used, _ := js.Get("scheduler").Get("schedulerInfo").Get("usedCapacity").Float64()
	return used
}

//Show the yarn nodemanagers distribution.
func Yarn_distribution(clustername string) {
	fmt.Println("====================RUNNING DOCKERS DISTRIBUTION====================")
	fmt.Println("HOSTNAME\tNUM")
	containers, err := docker.Get_running_docker_containers()
	if err != nil {
		panic(err)
	} else {
		distribution := make(map[string]int, len(containers))
		for i := range containers {
			c := docker.Get_container_name(containers[i].Names[0])
			if strings.HasPrefix(c, clustername) {
				h := docker.Get_nodemanager_host(containers[i].Names[0])
				distribution[h] += 1
			}
		}
		// sorted as the hostname
		sorted_keys := make([]string, 0)
		for k, _ := range distribution {
			sorted_keys = append(sorted_keys, k)
		}

		sort.Strings(sorted_keys)

		for _, k := range sorted_keys {
			fmt.Println(k, "\t", distribution[k])
		}
	}

}

//Inspect the yarn cluster container view
func Yarn_view(clustername string) {
	containers := docker.Get_all_yarn_containers()
	flag := false
	fmt.Println("ID\tCLUSTER\tNAME\tSTATUS\tSTATE\tIP\tHOST")
	fmt.Println("=======================================================================================================================================")
	for i := range containers {
		c := containers[i]
		if c.Clustername == clustername {
			flag = true
			fmt.Println(c.ID, "\t", c.Clustername, "\t", c.Name, "\t", c.Status, "\t", c.State, "\t", c.Ip, "\t", c.Host)
		}
	}
	if flag == false {
		fmt.Println("The cluster does not exited.")
	}

}

//Decommising the nodemanagers on each resourcemanagers.
//No matter which yarn clusters the nodemanager belonging to,
//you can put the in the same nodefile togethter.
//Magpie will recognize the yarn cluster automatically.
func Decommis_nodemanagers(nodemanagers []string) {
	nms := make([]docker.Yarn_docker_container, 0)
	containers := docker.Get_all_yarn_containers()
	for _, n := range nodemanagers {
		found := false
		for _, c := range containers {
			cid := c.ID
			if cid == n {
				found = true
				nms = append(nms, c)
			}
		}
		if found == false {
			fmt.Println("The nodemanager can not be found:", n)
		}
	}

	yarns := make(map[string]string, len(nodemanagers))
	for _, n := range nms {
		yarns[n.Clustername] = yarns[n.Clustername] + "\n" + n.ID
	}
	//	yarns := map[string]string{
	//		"yarn1": "3cd9494cdc80\n0ebc7d9cf054",
	//		"yarn2": "falj2ljfao3k",
	//	}
	fmt.Println("Decommising the following nodemangers...")
	var wg sync.WaitGroup
	//traverse all the yarn clusters
	for k, v := range yarns {
		wg.Add(1)
		go decommis_yarn_nodes(k, v, &wg)
	}
	wg.Wait()

}

// Decommising the nodemanager of the nodefile
func Decommis_nodemanagers_through_file(nodefile string) {
	fi, err := os.Open(nodefile)
	if err != nil {
		panic(err)
	}
	defer fi.Close()
	buff := bufio.NewReader(fi)
	nodes := make([]string, 0)
	for {
		line, err := buff.ReadString('\n')
		if err != nil || io.EOF == err {
			break
		}
		node := strings.Trim(line, "\n")
		nodes = append(nodes, node)
	}
	Decommis_nodemanagers(nodes)
}

//Decommising the nodemanagers of a yarn
func decommis_yarn_nodes(clustername string, nodemanagers string, wg *sync.WaitGroup) {
	if wg != nil {
		defer wg.Done()
	}
	logger := utils.Logger()
	fmt.Println(clustername, nodemanagers)
	logger.WithFields(logrus.Fields{"Time": time.Now(), "Cluster": clustername, "Nodemanagers": strings.Replace(nodemanagers, "\n", ",", -1), "Action": "DECOM"}).Info("Decomissing nodemanagers " + nodemanagers)
	resource_manager_ip := utils.Clustername2ip(clustername)
	nodemanger_exclude_file := viper.GetString("clusters.nodemanager_exclude_file")
	command := "ssh -n root@" + resource_manager_ip + ` "echo -e '` + nodemanagers + `'>>` + nodemanger_exclude_file + `"`
	//command := "ssh -n jingchao.song@" + "172.20.0.6" + ` "echo -e '` + nodemanagers + `'>>` + "/home/jingchao.song/docker/test.txt"+ `"`
	//TODO should return result and error handler
	utils.Run_command(command)
	command = "ssh -n root@" + resource_manager_ip + ` 'su - hadoop -c "yarn rmadmin -refreshNodes"'`
	utils.Run_command(command)

}

//Offline the host, decommsing the nodemanagers and then delete the docker contianers.
func Offline_host(hostname string) {
	fmt.Println("Offline host", hostname, "...")
	containers := docker.Get_all_yarn_containers()
	nms := make([]string, 0)
	for _, c := range containers {
		host := c.Host
		if host == hostname {
			nms = append(nms, c.Name)
		}
	}
	Decommis_nodemanagers(nms)
	docker.Delete_containers_on_host(hostname)
}

//Create a new nodemanager and start it
func Create_new_nodemanager(nm_config Nodemanager_config) {
	swarm_master_ip := viper.GetString("clusters.swarm_master_ip")
	swarm_master_port := viper.GetString("clusters.swarm_master_port")
	endpoint := "tcp://" + swarm_master_ip + ":" + swarm_master_port
	client, err := dockerclient.NewDockerClient(endpoint, nil)
	logger := utils.Logger()
	if err != nil {
		panic(err)
	}
	if err != nil {
		fmt.Println("Cannot connect to the swarm master.")
	}
	fmt.Println("Creating new nodemanager container...")
	env := []string{
		"HA=" + get_nodemanager_config(nm_config.HA, "HA"),
		"NAMESERVICE=" + get_nodemanager_config(nm_config.NAMESERVICE, "NAMESERVICE"),
		"ACTIVE_NAMENODE_IP=" + get_nodemanager_config(nm_config.ACTIVE_NAMENODE_IP, "ACTIVE_NAMENODE_IP"),
		"STANDBY_NAMENODE_IP=" + get_nodemanager_config(nm_config.STANDBY_NAMENODE_IP, "STANDBY_NAMENODE_IP"),
		"ACTIVE_NAMENODE_ID=" + get_nodemanager_config(nm_config.ACTIVE_NAMENODE_ID, "ACTIVE_NAMENODE_ID"),
		"STANDBY_NAMENODE_ID=" + get_nodemanager_config(nm_config.STANDBY_NAMENODE_ID, "STANDBY_NAMENODE_ID"),
		"HA_ZOOKEEPER_QUORUM=" + get_nodemanager_config(nm_config.HA_ZOOKEEPER_QUORUM, "HA_ZOOKEEPER_QUORUM"),
		"NAMENODE_IP=" + get_nodemanager_config(nm_config.NAMENODE_IP, "NAMENODE_IP"),
		"RESOURCEMANAGER_IP=" + get_nodemanager_config(nm_config.RESOURCEMANAGER_IP, "RESOURCEMANAGER_IP"),
		"YARN_RM1_IP=" + nm_config.YARN_RM1_IP,
		"YARN_RM2_IP=" + nm_config.YARN_RM2_IP,
		"YARN_JOBHISTORY_IP=" + nm_config.YARN_JOBHISTORY_IP,
		"CPU_CORE_NUM=" + get_nodemanager_config(nm_config.CPU_CORE_NUM, "CPU_CORE_NUM"),
		"NODEMANAGER_MEMORY_MB=" + get_nodemanager_config(nm_config.NODEMANAGER_MEMORY_MB, "NODEMANAGER_MEMORY_MB"),
		"YARN_CLUSTER_ID=" + nm_config.YARN_CLUSTER_ID,
		"YARN_ZK_DIR=" + nm_config.YARN_ZK_DIR,
		//"PATH=/usr/local/hadoop/bin:/usr/local/hadoop/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/java/bin://usr/local/java/jre/bin",
	}
	hostConifg := dockerclient.HostConfig{
		CpuShares:   nm_config.Limit_cpus,
		Memory:      nm_config.Limit_memory_mb * 1024 * 1024, //transform to Byte
		NetworkMode: nm_config.Network_mode,
	}
	var config *dockerclient.ContainerConfig
	config = new(dockerclient.ContainerConfig)
	config.Image = get_nodemanager_config(nm_config.Image, "image")
	config.Env = env

	//inherit Cmd and Entrypoint settings from docker Image or set them on config file
	config.Cmd = viper.GetStringSlice("nodemanager.cmd")
	config.Entrypoint = viper.GetStringSlice("nodemanager.entrypoint")
	config.HostConfig = hostConifg
	id, err := client.CreateContainer(config, "", nil)
	if err != nil {
		logger.WithFields(logrus.Fields{"Time": time.Now(), "ContainerID": "-", "Action": "CREATE"}).Error(err)
		panic(err)
	}
	container_name := id[0:12]
	fmt.Println("Container", container_name, "created.")
	logger.WithFields(logrus.Fields{"Time": time.Now(), "ContainerID": container_name, "Action": "CREATE"}).Info("Create a new nodemanager docker container")

	if nm_config.Container_name != "" {
		err = client.RenameContainer(container_name, nm_config.Container_name)
		if err != nil {
			logger.WithFields(logrus.Fields{"Time": time.Now(), "ContainerID": container_name, "Action": "RENAME"}).Error(err)
			panic(err)
		}
		fmt.Println("Rename container name to", nm_config.Container_name)
		logger.WithFields(logrus.Fields{"Time": time.Now(), "ContainerID": container_name, "Action": "RENAME"}).Info("Rename container "+container_name+" name to ", nm_config.Container_name)
	}
	err = client.StartContainer(id, nil)
	if err != nil {
		logger.WithFields(logrus.Fields{"Time": time.Now(), "ContainerID": container_name, "Action": "START"}).Error(err)
		panic(err)
	}
	fmt.Println("Started.")
	logger.WithFields(logrus.Fields{"Time": time.Now(), "ContainerID": container_name, "Action": "START"}).Info("Start container " + container_name)

}

//Get nodemanger configuration item from config file
//If no config specify throught the command line,use the default settings
func get_nodemanager_config(nm string, config string) string {
	if nm == "" {
		return viper.GetString("nodemanager." + config)
	}
	return nm
}
<|endoftext|>


<commit_before>package client

import (
	"fmt"
	"io"
	"os"
	"time"

	"strings"

	"github.com/ncodes/cocoon/core/orderer"
	proto "github.com/ncodes/cocoon/core/runtime/golang/proto"
	"github.com/ncodes/cocoon/core/types"
	logging "github.com/op/go-logging"
	cmap "github.com/orcaman/concurrent-map"
	context "golang.org/x/net/context"
	"google.golang.org/grpc"
)

var log = logging.MustGetLogger("connector.client")

// txChannels holds the channels to send transaction responses to
var txRespChannels = cmap.New()

// Client represents a cocoon code GRPC client
// that interacts with a cocoon code.
type Client struct {
	ccodeAddr        string
	stub             proto.StubClient
	conCtx           context.Context
	conCancel        context.CancelFunc
	orderDiscoTicker *time.Ticker
	ordererAddrs     []string
	stream           proto.Stub_TransactClient
	cocoonID         string
}

// NewClient creates a new cocoon code client
func NewClient() *Client {
	return &Client{}
}

// SetCocoonID sets the cocoon id
func (c *Client) SetCocoonID(id string) {
	c.cocoonID = id
}

// SetCocoonCodeAddr sets the cocoon code bind address
func (c *Client) SetCocoonCodeAddr(ccAddr string) {
	c.ccodeAddr = ccAddr
}

// getCCAddr returns the cocoon code bind address.
// For development, if DEV_COCOON_ADDR is set, it connects to it.
func (c *Client) getCCAddr() string {
	if devCCodeAddr := os.Getenv("DEV_COCOON_ADDR"); len(devCCodeAddr) > 0 {
		return devCCodeAddr
	}
	return c.ccodeAddr
}

// Close the stream and cancel connections
func (c *Client) Close() {
	if c.stream != nil {
		c.stream.CloseSend()
	}
	if c.conCancel != nil {
		c.conCancel()
	}
}

// GetStream returns the grpc stream connected to the grpc cocoon code server
func (c *Client) GetStream() proto.Stub_TransactClient {
	return c.stream
}

// Connect connects to a cocoon code server
// running on a known port
func (c *Client) Connect() error {

	log.Info("Starting cocoon code client")

	// start a ticker to continously discover orderer addreses
	go func() {
		c.orderDiscoTicker = time.NewTicker(60 * time.Second)
		for _ = range c.orderDiscoTicker.C {
			var ordererAddrs []string
			ordererAddrs, err := orderer.DiscoverOrderers()
			if err != nil {
				log.Error(err.Error())
				continue
			}
			c.ordererAddrs = ordererAddrs
		}
	}()

	var ordererAddrs []string
	ordererAddrs, err := orderer.DiscoverOrderers()
	if err != nil {
		return err
	}
	c.ordererAddrs = ordererAddrs

	if len(c.ordererAddrs) > 0 {
		log.Infof("Orderer address list updated. Contains %d orderer address(es)", len(c.ordererAddrs))
	} else {
		log.Warning("No orderer address was found. We won't be able to reach the orderer. ")
	}

	conn, err := grpc.Dial(c.getCCAddr(), grpc.WithInsecure())
	if err != nil {
		return fmt.Errorf("failed to connect to cocoon code server. %s", err)
	}
	defer conn.Close()

	log.Debugf("Now connected to cocoon code at port=%s", strings.Split(c.getCCAddr(), ":")[1])

	c.stub = proto.NewStubClient(conn)

	if err = c.Do(conn); err != nil {
		return err
	}

	return nil
}

// Do starts a request loop that continously asks the
// server for transactions. When it receives a transaction,
// it processes it and returns the result.
func (c *Client) Do(conn *grpc.ClientConn) error {

	var err error

	// create a context so we have complete controll of the connection
	c.conCtx, c.conCancel = context.WithCancel(context.Background())

	// connect to the cocoon code
	c.stream, err = c.stub.Transact(c.conCtx)
	if err != nil {
		return fmt.Errorf("failed to start transaction stream with cocoon code. %s", err)
	}

	for {

		in, err := c.stream.Recv()
		if err == io.EOF {
			return fmt.Errorf("connection with cocoon code has ended")
		}
		if err != nil {
			if strings.Contains(err.Error(), "context canceled") {
				log.Info("Connection to cocoon code closed")
				return nil
			}
			return fmt.Errorf("failed to read message from cocoon code. %s", err)
		}

		switch in.Invoke {
		case true:
			go func() {
				log.Debugf("New invoke transaction (%s) from cocoon code", in.GetId())
				if err = c.handleInvokeTransaction(in); err != nil {
					log.Error(err.Error())
					c.stream.Send(&proto.Tx{
						Response: true,
						Id:       in.GetId(),
						Status:   500,
						Body:     []byte(err.Error()),
					})
				}
			}()
		case false:
			log.Debugf("New response transaction (%s) from cocoon code", in.GetId())
			go func() {
				if err = c.handleRespTransaction(in); err != nil {
					log.Error(err.Error())
					c.stream.Send(&proto.Tx{
						Response: true,
						Id:       in.GetId(),
						Status:   500,
						Body:     []byte(err.Error()),
					})
				}
			}()
		}
	}
}

// handleInvokeTransaction processes invoke transaction requests
func (c *Client) handleInvokeTransaction(tx *proto.Tx) error {
	switch tx.GetName() {
	case types.TxCreateLedger:
		return c.createLedger(tx)
	case types.TxPut:
		return c.put(tx)
	case types.TxGetLedger:
		return c.getLedger(tx)
	case types.TxGet:
		return c.get(tx, false)
	case types.TxGetByID:
		return c.get(tx, true)
	case types.TxGetBlockByID:
		return c.getBlock(tx)
	case types.TxRangeGet:
		return c.getRange(tx)
	default:
		return c.stream.Send(&proto.Tx{
			Id:       tx.GetId(),
			Response: true,
			Status:   500,
			Body:     []byte(fmt.Sprintf("unsupported transaction name (%s)", tx.GetName())),
		})
	}
}

// handleRespTransaction passes the transaction to a response
// channel with a matching transaction id and deletes the channel afterwards.
func (c *Client) handleRespTransaction(tx *proto.Tx) error {
	if !txRespChannels.Has(tx.GetId()) {
		return fmt.Errorf("response transaction (%s) does not have a corresponding response channel", tx.GetId())
	}

	txRespCh, _ := txRespChannels.Get(tx.GetId())
	txRespCh.(chan *proto.Tx) <- tx
	txRespChannels.Remove(tx.GetId())
	return nil
}

// SendTx sends a transaction to the cocoon code
// and saves the response channel. The response channel will
// be passed a response when it is available in the Transact loop.
func (c *Client) SendTx(tx *proto.Tx, respCh chan *proto.Tx) error {
	txRespChannels.Set(tx.GetId(), respCh)
	if err := c.stream.Send(tx); err != nil {
		txRespChannels.Remove(tx.GetId())
		return err
	}
	log.Debugf("Successfully sent transaction (%s) to cocoon code", tx.GetId())
	return nil
}
<commit_msg>Prevent exit of connector when `transport is closing` error is received. This appears to be a grpc bug (see: https://github.com/GoogleCloudPlatform/google-cloud-go/issues/479)<commit_after>package client

import (
	"fmt"
	"io"
	"os"
	"time"

	"strings"

	"github.com/ncodes/cocoon/core/orderer"
	proto "github.com/ncodes/cocoon/core/runtime/golang/proto"
	"github.com/ncodes/cocoon/core/types"
	logging "github.com/op/go-logging"
	cmap "github.com/orcaman/concurrent-map"
	context "golang.org/x/net/context"
	"google.golang.org/grpc"
)

var log = logging.MustGetLogger("connector.client")

// txChannels holds the channels to send transaction responses to
var txRespChannels = cmap.New()

// Client represents a cocoon code GRPC client
// that interacts with a cocoon code.
type Client struct {
	ccodeAddr        string
	stub             proto.StubClient
	conCtx           context.Context
	conCancel        context.CancelFunc
	orderDiscoTicker *time.Ticker
	ordererAddrs     []string
	stream           proto.Stub_TransactClient
	cocoonID         string
}

// NewClient creates a new cocoon code client
func NewClient() *Client {
	return &Client{}
}

// SetCocoonID sets the cocoon id
func (c *Client) SetCocoonID(id string) {
	c.cocoonID = id
}

// SetCocoonCodeAddr sets the cocoon code bind address
func (c *Client) SetCocoonCodeAddr(ccAddr string) {
	c.ccodeAddr = ccAddr
}

// getCCAddr returns the cocoon code bind address.
// For development, if DEV_COCOON_ADDR is set, it connects to it.
func (c *Client) getCCAddr() string {
	if devCCodeAddr := os.Getenv("DEV_COCOON_ADDR"); len(devCCodeAddr) > 0 {
		return devCCodeAddr
	}
	return c.ccodeAddr
}

// Close the stream and cancel connections
func (c *Client) Close() {
	if c.stream != nil {
		c.stream.CloseSend()
	}
	if c.conCancel != nil {
		c.conCancel()
	}
}

// GetStream returns the grpc stream connected to the grpc cocoon code server
func (c *Client) GetStream() proto.Stub_TransactClient {
	return c.stream
}

// Connect connects to a cocoon code server
// running on a known port
func (c *Client) Connect() error {

	log.Info("Starting cocoon code client")

	// start a ticker to continously discover orderer addreses
	go func() {
		c.orderDiscoTicker = time.NewTicker(60 * time.Second)
		for _ = range c.orderDiscoTicker.C {
			var ordererAddrs []string
			ordererAddrs, err := orderer.DiscoverOrderers()
			if err != nil {
				log.Error(err.Error())
				continue
			}
			c.ordererAddrs = ordererAddrs
		}
	}()

	var ordererAddrs []string
	ordererAddrs, err := orderer.DiscoverOrderers()
	if err != nil {
		return err
	}
	c.ordererAddrs = ordererAddrs

	if len(c.ordererAddrs) > 0 {
		log.Infof("Orderer address list updated. Contains %d orderer address(es)", len(c.ordererAddrs))
	} else {
		log.Warning("No orderer address was found. We won't be able to reach the orderer. ")
	}

	conn, err := grpc.Dial(c.getCCAddr(), grpc.WithInsecure())
	if err != nil {
		return fmt.Errorf("failed to connect to cocoon code server. %s", err)
	}
	defer conn.Close()

	log.Debugf("Now connected to cocoon code at port=%s", strings.Split(c.getCCAddr(), ":")[1])

	c.stub = proto.NewStubClient(conn)

	if err = c.Do(conn); err != nil {
		return err
	}

	return nil
}

// Do starts a request loop that continously asks the
// server for transactions. When it receives a transaction,
// it processes it and returns the result.
func (c *Client) Do(conn *grpc.ClientConn) error {

	var err error

	// create a context so we have complete controll of the connection
	c.conCtx, c.conCancel = context.WithCancel(context.Background())

	// connect to the cocoon code
	c.stream, err = c.stub.Transact(c.conCtx)
	if err != nil {
		return fmt.Errorf("failed to start transaction stream with cocoon code. %s", err)
	}

	for {

		in, err := c.stream.Recv()
		if err == io.EOF {
			return fmt.Errorf("connection with cocoon code has ended")
		}
		if err != nil {
			if strings.Contains(err.Error(), "context canceled") {
				log.Info("Connection to cocoon code closed")
				return nil
			}
			if grpc.ErrorDesc(err) == "transport is closing" {
				log.Error(err.Error())
				c.stream.Send(&proto.Tx{
					Response: true,
					Id:       "error",
					Status:   500,
					Body:     []byte("server error"),
				})
				continue
			} else {
				return fmt.Errorf("failed to read message from cocoon code. %s", err)
			}
		}

		switch in.Invoke {
		case true:
			go func() {
				log.Debugf("New invoke transaction (%s) from cocoon code", in.GetId())
				if err = c.handleInvokeTransaction(in); err != nil {
					log.Error(err.Error())
					c.stream.Send(&proto.Tx{
						Response: true,
						Id:       in.GetId(),
						Status:   500,
						Body:     []byte(err.Error()),
					})
				}
			}()
		case false:
			log.Debugf("New response transaction (%s) from cocoon code", in.GetId())
			go func() {
				if err = c.handleRespTransaction(in); err != nil {
					log.Error(err.Error())
					c.stream.Send(&proto.Tx{
						Response: true,
						Id:       in.GetId(),
						Status:   500,
						Body:     []byte(err.Error()),
					})
				}
			}()
		}
	}
}

// handleInvokeTransaction processes invoke transaction requests
func (c *Client) handleInvokeTransaction(tx *proto.Tx) error {
	switch tx.GetName() {
	case types.TxCreateLedger:
		return c.createLedger(tx)
	case types.TxPut:
		return c.put(tx)
	case types.TxGetLedger:
		return c.getLedger(tx)
	case types.TxGet:
		return c.get(tx, false)
	case types.TxGetByID:
		return c.get(tx, true)
	case types.TxGetBlockByID:
		return c.getBlock(tx)
	case types.TxRangeGet:
		return c.getRange(tx)
	default:
		return c.stream.Send(&proto.Tx{
			Id:       tx.GetId(),
			Response: true,
			Status:   500,
			Body:     []byte(fmt.Sprintf("unsupported transaction name (%s)", tx.GetName())),
		})
	}
}

// handleRespTransaction passes the transaction to a response
// channel with a matching transaction id and deletes the channel afterwards.
func (c *Client) handleRespTransaction(tx *proto.Tx) error {
	if !txRespChannels.Has(tx.GetId()) {
		return fmt.Errorf("response transaction (%s) does not have a corresponding response channel", tx.GetId())
	}

	txRespCh, _ := txRespChannels.Get(tx.GetId())
	txRespCh.(chan *proto.Tx) <- tx
	txRespChannels.Remove(tx.GetId())
	return nil
}

// SendTx sends a transaction to the cocoon code
// and saves the response channel. The response channel will
// be passed a response when it is available in the Transact loop.
func (c *Client) SendTx(tx *proto.Tx, respCh chan *proto.Tx) error {
	txRespChannels.Set(tx.GetId(), respCh)
	if err := c.stream.Send(tx); err != nil {
		txRespChannels.Remove(tx.GetId())
		return err
	}
	log.Debugf("Successfully sent transaction (%s) to cocoon code", tx.GetId())
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bitbucket.org/lpabon/filecc/caches"
	"bitbucket.org/lpabon/filecc/zipfworkload"
	"bufio"
	"fmt"
	"github.com/lpabon/godbc"
	"math/rand"
	"os"
	"strconv"
	"time"
)

type SimFile struct {
	iogen *zipfworkload.ZipfWorkload
	size  uint64
}

func main() {

	var filezipf *zipfworkload.ZipfWorkload

	// Setup seed for random numbers
	r := rand.New(rand.NewSource(time.Now().UnixNano()))

	// Config
	chunksize := 256 * 1024
	maxfilesize := int64(1*1024*1024*1024*1024) / int64(chunksize) // Up to 1 TB in 256k chunks
	cachesize := uint64(64*1024*1024*1024) / uint64(chunksize)     // 16 GB divided into 256k chunks
	numfiles := 100000
	numios := 5000000
	deletion_chance := 15 // percent
	read_chance := 65     // percent
	writethrough := true
	filedistribution_zipf := true

	// Determine distribution type
	if filedistribution_zipf {
		filezipf = zipfworkload.NewZipfWorkload(uint64(numfiles), 0)
	}

	// Create simulated files
	files := make([]*SimFile, numfiles)
	for file := 0; file < numfiles; file++ {
		files[file] = &SimFile{}
		files[file].size = uint64(r.Int63n(maxfilesize)) + uint64(1) // in case we get 0
		files[file].iogen = zipfworkload.NewZipfWorkload(files[file].size, read_chance)
	}

	// Print here Simulation information, also Mean file size and std deviation

	// Setup file to write cache metrics
	fp, err := os.Create("cache.data")
	godbc.Check(err == nil)
	defer fp.Close()
	metrics := bufio.NewWriter(fp)

	// Create the cache
	cache := caches.NewCache(cachesize, writethrough)
	cache_prev := cache.Copy()

	for io := 0; io < numios; io++ {

		// Save metrics
		if (io % 100) == 0 {
			_, err := metrics.WriteString(fmt.Sprintf("%d,", io) + cache.DumpDelta(cache_prev))
			godbc.Check(err == nil)

			// Now copy the data
			*cache_prev = *cache
		}

		// Get the file
		var file uint64
		if filedistribution_zipf {
			file, _ = filezipf.ZipfGenerate()
		} else {
			// Random Distribution
			file = uint64(r.Int63n(int64(numfiles)))
		}
		godbc.Check(int(file) <= numfiles, fmt.Sprintf("file = %v", file))

		// Check if we need to delete this file
		if rand.Intn(100) < deletion_chance {
			cache.Delete(strconv.FormatUint(file, 10))
			continue
		}

		// Which block on the file
		chunk, isread := files[file].iogen.ZipfGenerate()
		if isread {
			cache.Read(strconv.FormatUint(file, 10), strconv.FormatUint(chunk, 10))
		} else {
			cache.Write(strconv.FormatUint(file, 10), strconv.FormatUint(chunk, 10))
		}
	}
	metrics.Flush()
	fmt.Print(cache)
}
<commit_msg>Bring back the zipfdemo<commit_after>package main

import (
	"bitbucket.org/lpabon/filecc/zipfworkload"
	"bufio"
	"fmt"
	"github.com/lpabon/godbc"
	"os"
)

type LoadInfo struct {
	numaccess, reads, writes int
}

func main() {
	z := zipfworkload.NewZipfWorkload(1*1024*1024*1024, 90)
	h := make(map[uint64]*LoadInfo)

	for i := 0; i < 20000000; i++ {
		obj, isread := z.ZipfGenerate()
		if nil == h[obj] {
			h[obj] = &LoadInfo{}
		}
		h[obj].numaccess += 1
		if isread {
			h[obj].reads += 1
		} else {
			h[obj].writes += 1
		}
	}

	fp, err := os.Create("filecc.data")
	godbc.Check(err == nil)
	defer fp.Close()
	w := bufio.NewWriter(fp)

	for k, v := range h {
		_, err := w.WriteString(fmt.Sprintf("%v %d %d %d\n", k, v.numaccess, v.reads, v.writes))
		godbc.Check(err == nil)
	}

}
<|endoftext|>


<commit_before>/*
 * http://www.apache.org/licenses/LICENSE-2.0.txt
 *
 * Copyright 2017 OpsVision Solutions
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package signalfx

// Imports
import (
	"bytes"
	"fmt"
	"github.com/intelsdi-x/snap-plugin-lib-go/v1/plugin"
	"github.com/signalfx/golib/datapoint"
	"github.com/signalfx/golib/sfxclient"
	"golang.org/x/net/context"
	"log"
	"os"
	"strings"
)

// Constants
const (
	NS_VENDOR = "opsvision" // plugin vendor
	NS_PLUGIN = "signalfx"  // plugin name
	VERSION   = 1           // plugin version
)

// SignalFx object
type SignalFx struct {
	token     string // SignalFx API token
	hostname  string // Hostname
	namespace string // Metric namespace
}

// New - Constructor
func New() *SignalFx {
	return new(SignalFx)
}

// GetConfigPolicy - Returns the configPolicy for the plugin
func (s *SignalFx) GetConfigPolicy() (plugin.ConfigPolicy, error) {
	policy := plugin.NewConfigPolicy()

	// The SignalFx token
	policy.AddNewStringRule([]string{NS_VENDOR, NS_PLUGIN},
		"token",
		true)

	// The hostname to use (defaults to local hostname)
	policy.AddNewStringRule([]string{NS_VENDOR, NS_PLUGIN},
		"hostname",
		false)

	// The file name to use when debugging
	policy.AddNewStringRule([]string{NS_VENDOR, NS_PLUGIN},
		"debug-file",
		false)

	return *policy, nil
}

// Publish - Publishes metrics to SignalFx using the TOKEN found in the config
func (s *SignalFx) Publish(mts []plugin.Metric, cfg plugin.Config) error {
	// Enable debugging if the debug-file config property was set
	fileName, err := cfg.GetString("debug-file")
	if err != nil {
		// Open the output file
		f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			return err
		}
		defer f.Close()

		// Set logging output for debugging
		log.SetOutput(f)
	}

	// Fetch the token
	token, err := cfg.GetString("token")
	if err != nil {
		return err
	}
	s.token = token

	// Attempt to set the hostname
	hostname, err := cfg.GetString("hostname")
	if err != nil {
		hostname, err = os.Hostname()
		if err != nil {
			hostname = "localhost"
		}
	}
	s.hostname = hostname

	// Iterate over the supplied metrics
	for _, m := range mts {
		var buffer bytes.Buffer

		// Convert the namespace to dot notation
		fmt.Fprintf(&buffer, "snap.%s", strings.Join(m.Namespace.Strings(), "."))
		s.namespace = buffer.String()

		// Do some type conversion and send the data
		switch v := m.Data.(type) {
		case uint:
			s.sendIntValue(int64(v))
		case uint32:
			s.sendIntValue(int64(v))
		case uint64:
			s.sendIntValue(int64(v))
		case int:
			s.sendIntValue(int64(v))
		case int32:
			s.sendIntValue(int64(v))
		case int64:
			s.sendIntValue(int64(v))
		case float32:
			s.sendFloatValue(float64(v))
		case float64:
			s.sendFloatValue(float64(v))
		default:
			log.Printf("Ignoring %T: %v\n", v, v)
			log.Printf("Contact the plugin author if you think this is an error")
		}
	}

	return nil
}

// sendIntValue - Method for sending int64 values to SignalFx
func (s *SignalFx) sendIntValue(value int64) {
	log.Printf("Sending [int64] %s -> %v", s.namespace, value)

	client := sfxclient.NewHTTPDatapointSink()
	client.AuthToken = s.token
	ctx := context.Background()
	client.AddDatapoints(ctx, []*datapoint.Datapoint{
		sfxclient.Gauge(s.namespace, map[string]string{
			"host": s.hostname,
		}, value),
	})
}

// sendFloatValue - Method for sending float64 values to SignalFx
func (s *SignalFx) sendFloatValue(value float64) {
	log.Printf("Sending [float64] %s -> %v", s.namespace, value)

	client := sfxclient.NewHTTPDatapointSink()
	client.AuthToken = s.token
	ctx := context.Background()
	client.AddDatapoints(ctx, []*datapoint.Datapoint{
		sfxclient.GaugeF(s.namespace, map[string]string{
			"host": s.hostname,
		}, value),
	})
}
<commit_msg>Refactored code per gocyclo<commit_after>/*
 * http://www.apache.org/licenses/LICENSE-2.0.txt
 *
 * Copyright 2017 OpsVision Solutions
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package signalfx

// Imports
import (
	"bytes"
	"fmt"
	"github.com/intelsdi-x/snap-plugin-lib-go/v1/plugin"
	"github.com/signalfx/golib/datapoint"
	"github.com/signalfx/golib/sfxclient"
	"golang.org/x/net/context"
	"log"
	"os"
	"strings"
)

// Constants
const (
	NS_VENDOR = "opsvision" // plugin vendor
	NS_PLUGIN = "signalfx"  // plugin name
	VERSION   = 1           // plugin version
)

// SignalFx object
type SignalFx struct {
	token     string // SignalFx API token
	hostname  string // Hostname
	namespace string // Metric namespace
}

// New - Constructor
func New() *SignalFx {
	return new(SignalFx)
}

// GetConfigPolicy - Returns the configPolicy for the plugin
func (s *SignalFx) GetConfigPolicy() (plugin.ConfigPolicy, error) {
	policy := plugin.NewConfigPolicy()

	// The SignalFx token
	policy.AddNewStringRule([]string{NS_VENDOR, NS_PLUGIN},
		"token",
		true)

	// The hostname to use (defaults to local hostname)
	policy.AddNewStringRule([]string{NS_VENDOR, NS_PLUGIN},
		"hostname",
		false)

	// The file name to use when debugging
	policy.AddNewStringRule([]string{NS_VENDOR, NS_PLUGIN},
		"debug-file",
		false)

	return *policy, nil
}

// Publish - Publishes metrics to SignalFx using the TOKEN found in the config
func (s *SignalFx) Publish(mts []plugin.Metric, cfg plugin.Config) error {
	// Enable debugging
	s.ConfigDebugLog(cfg)

	// Set our SignalFx API token
	s.SetToken(cfg)

	// Set the hostname
	s.SetHostname(cfg)

	// Iterate over the supplied metrics
	for _, m := range mts {
		var buffer bytes.Buffer

		// Convert the namespace to dot notation
		fmt.Fprintf(&buffer, "snap.%s", strings.Join(m.Namespace.Strings(), "."))
		s.namespace = buffer.String()

		// Do some type conversion and send the data
		switch v := m.Data.(type) {
		case uint:
			s.SendIntValue(int64(v))
		case uint32:
			s.SendIntValue(int64(v))
		case uint64:
			s.SendIntValue(int64(v))
		case int:
			s.SendIntValue(int64(v))
		case int32:
			s.SendIntValue(int64(v))
		case int64:
			s.SendIntValue(int64(v))
		case float32:
			s.SendFloatValue(float64(v))
		case float64:
			s.SendFloatValue(float64(v))
		default:
			log.Printf("Ignoring %T: %v\n", v, v)
			log.Printf("Contact the plugin author if you think this is an error")
		}
	}

	return nil
}

// ConfigDebugLog will configure logging if the debug-file config
// setting is present in the task file
func (s *SignalFx) ConfigDebugLog(cfg plugin.Config) {
	fileName, err := cfg.GetString("debug-file")
	if err == nil {
		// Open the output file
		f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			log.Panic(err)
		}
		defer f.Close()

		// Set logging output for debugging
		log.SetOutput(f)
	}
}

// SetToken will set the token required by the SignalFx API
func (s *SignalFx) SetToken(cfg plugin.Config) {
	// Fetch the token
	token, err := cfg.GetString("token")
	if err != nil {
		log.Panic(err)
	}
	s.token = token
}

// SetHostname will set the hostname from the config file, or, if absent,
// will attempt to figure out the hostname. As a last resort, we default
// to using localhost.
func (s *SignalFx) SetHostname(cfg plugin.Config) {
	hostname, err := cfg.GetString("hostname")
	if err != nil {
		hostname, err = os.Hostname()
		if err != nil {
			hostname = "localhost"
		}
	}
	s.hostname = hostname
}

// SendIntValue - Method for sending int64 values to SignalFx
func (s *SignalFx) SendIntValue(value int64) {
	log.Printf("Sending [int64] %s -> %v", s.namespace, value)

	client := sfxclient.NewHTTPDatapointSink()
	client.AuthToken = s.token
	ctx := context.Background()
	client.AddDatapoints(ctx, []*datapoint.Datapoint{
		sfxclient.Gauge(s.namespace, map[string]string{
			"host": s.hostname,
		}, value),
	})
}

// SendFloatValue - Method for sending float64 values to SignalFx
func (s *SignalFx) SendFloatValue(value float64) {
	log.Printf("Sending [float64] %s -> %v", s.namespace, value)

	client := sfxclient.NewHTTPDatapointSink()
	client.AuthToken = s.token
	ctx := context.Background()
	client.AddDatapoints(ctx, []*datapoint.Datapoint{
		sfxclient.GaugeF(s.namespace, map[string]string{
			"host": s.hostname,
		}, value),
	})
}
<|endoftext|>


<commit_before>// Copyright 2014 tsuru-client authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/cmd/tsuru-base"
)

type deploy struct {
	tsuru.GuessingCommand
}

func (c *deploy) Info() *cmd.Info {
	desc := `Deploys set of files and/or directories to tsuru server. Some examples of calls are:

tsuru deploy .
tsuru deploy myfile.jar Procfile
`
	return &cmd.Info{
		Name:    "deploy",
		Usage:   "deploy [-a/--app <appname>] <file-or-dir-1> [file-or-dir-2] ... [file-or-dir-n]",
		Desc:    desc,
		MinArgs: 1,
	}
}

func (c *deploy) Run(context *cmd.Context, client *cmd.Client) error {
	var body bytes.Buffer
	writer := multipart.NewWriter(&body)
	file, err := writer.CreateFormFile("file", "archive.tar.gz")
	if err != nil {
		return err
	}
	err = targz(context, file, context.Args...)
	if err != nil {
		return err
	}
	writer.Close()
	appName, err := c.Guess()
	if err != nil {
		return err
	}
	url, err := cmd.GetURL("/apps/" + appName + "/deploy")
	if err != nil {
		return err
	}
	request, err := http.NewRequest("POST", url, &body)
	if err != nil {
		return err
	}
	request.Header.Set("Content-Type", "multipart/form-data; boundary="+writer.Boundary())
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var respBody bytes.Buffer
	_, err = io.Copy(io.MultiWriter(&respBody, context.Stdout), resp.Body)
	if err != nil {
		return err
	}
	if strings.HasSuffix(respBody.String(), "\nOK\n") {
		return nil
	}
	return cmd.ErrAbortCommand
}

func targz(ctx *cmd.Context, destination io.Writer, filepaths ...string) error {
	var buf bytes.Buffer
	tarWriter := tar.NewWriter(&buf)
	for _, path := range filepaths {
		if path == ".." {
			fmt.Fprintf(ctx.Stderr, "Warning: skipping %q", path)
			continue
		}
		fi, err := os.Stat(path)
		if err != nil {
			return err
		}
		if fi.IsDir() {
			err = addDir(tarWriter, path)
		} else {
			err = addFile(tarWriter, path)
		}
		if err != nil {
			return err
		}
	}
	err := tarWriter.Close()
	if err != nil {
		return err
	}
	gzipWriter := gzip.NewWriter(destination)
	defer gzipWriter.Close()
	_, err = io.Copy(gzipWriter, &buf)
	return err
}

func addDir(writer *tar.Writer, path string) error {
	dir, err := os.Open(path)
	if err != nil {
		return err
	}
	defer dir.Close()
	fi, err := dir.Stat()
	if err != nil {
		return err
	}
	header, err := tar.FileInfoHeader(fi, "")
	if err != nil {
		return err
	}
	header.Name = path
	err = writer.WriteHeader(header)
	if err != nil {
		return err
	}
	fis, err := dir.Readdir(0)
	if err != nil {
		return err
	}
	for _, fi := range fis {
		if fi.IsDir() {
			err = addDir(writer, filepath.Join(path, fi.Name()))
		} else {
			err = addFile(writer, filepath.Join(path, fi.Name()))
		}
		if err != nil {
			return err
		}
	}
	return nil
}

func addFile(writer *tar.Writer, path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	fi, err := f.Stat()
	if err != nil {
		return err
	}
	header, err := tar.FileInfoHeader(fi, "")
	if err != nil {
		return err
	}
	header.Name = path
	err = writer.WriteHeader(header)
	if err != nil {
		return err
	}
	n, err := io.Copy(writer, f)
	if err != nil {
		return err
	}
	if n != fi.Size() {
		return io.ErrShortWrite
	}
	return nil
}
<commit_msg>tsuru: add some status to deploy<commit_after>// Copyright 2014 tsuru-client authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/cmd/tsuru-base"
)

type deploy struct {
	tsuru.GuessingCommand
}

func (c *deploy) Info() *cmd.Info {
	desc := `Deploys set of files and/or directories to tsuru server. Some examples of calls are:

tsuru deploy .
tsuru deploy myfile.jar Procfile
`
	return &cmd.Info{
		Name:    "deploy",
		Usage:   "deploy [-a/--app <appname>] <file-or-dir-1> [file-or-dir-2] ... [file-or-dir-n]",
		Desc:    desc,
		MinArgs: 1,
	}
}

func (c *deploy) Run(context *cmd.Context, client *cmd.Client) error {
	var body bytes.Buffer
	writer := multipart.NewWriter(&body)
	file, err := writer.CreateFormFile("file", "archive.tar.gz")
	if err != nil {
		return err
	}
	err = targz(context, file, context.Args...)
	if err != nil {
		return err
	}
	writer.Close()
	appName, err := c.Guess()
	if err != nil {
		return err
	}
	url, err := cmd.GetURL("/apps/" + appName + "/deploy")
	if err != nil {
		return err
	}
	request, err := http.NewRequest("POST", url, &body)
	if err != nil {
		return err
	}
	request.Header.Set("Content-Type", "multipart/form-data; boundary="+writer.Boundary())
	var buf bytes.Buffer
	respBody := firstWriter{Writer: io.MultiWriter(context.Stdout, &buf)}
	go func() {
		fmt.Fprint(context.Stdout, "Uploading files...")
		for buf.Len() == 0 {
			fmt.Fprint(context.Stdout, ".")
			time.Sleep(2e9)
		}
	}()
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	_, err = io.Copy(&respBody, resp.Body)
	if err != nil {
		return err
	}
	if strings.HasSuffix(buf.String(), "\nOK\n") {
		return nil
	}
	return cmd.ErrAbortCommand
}

func targz(ctx *cmd.Context, destination io.Writer, filepaths ...string) error {
	var buf bytes.Buffer
	tarWriter := tar.NewWriter(&buf)
	for _, path := range filepaths {
		if path == ".." {
			fmt.Fprintf(ctx.Stderr, "Warning: skipping %q", path)
			continue
		}
		fi, err := os.Stat(path)
		if err != nil {
			return err
		}
		if fi.IsDir() {
			err = addDir(tarWriter, path)
		} else {
			err = addFile(tarWriter, path)
		}
		if err != nil {
			return err
		}
	}
	err := tarWriter.Close()
	if err != nil {
		return err
	}
	gzipWriter := gzip.NewWriter(destination)
	defer gzipWriter.Close()
	_, err = io.Copy(gzipWriter, &buf)
	return err
}

func addDir(writer *tar.Writer, path string) error {
	dir, err := os.Open(path)
	if err != nil {
		return err
	}
	defer dir.Close()
	fi, err := dir.Stat()
	if err != nil {
		return err
	}
	header, err := tar.FileInfoHeader(fi, "")
	if err != nil {
		return err
	}
	header.Name = path
	err = writer.WriteHeader(header)
	if err != nil {
		return err
	}
	fis, err := dir.Readdir(0)
	if err != nil {
		return err
	}
	for _, fi := range fis {
		if fi.IsDir() {
			err = addDir(writer, filepath.Join(path, fi.Name()))
		} else {
			err = addFile(writer, filepath.Join(path, fi.Name()))
		}
		if err != nil {
			return err
		}
	}
	return nil
}

func addFile(writer *tar.Writer, path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	fi, err := f.Stat()
	if err != nil {
		return err
	}
	header, err := tar.FileInfoHeader(fi, "")
	if err != nil {
		return err
	}
	header.Name = path
	err = writer.WriteHeader(header)
	if err != nil {
		return err
	}
	n, err := io.Copy(writer, f)
	if err != nil {
		return err
	}
	if n != fi.Size() {
		return io.ErrShortWrite
	}
	return nil
}

type firstWriter struct {
	io.Writer
	once sync.Once
}

func (w *firstWriter) Write(p []byte) (int, error) {
	w.once.Do(func() {
		w.Writer.Write([]byte{'\n'})
	})
	return w.Writer.Write(p)
}
<|endoftext|>