<commit_before>package rest

import (
	"errors"
	"fmt"

	"github.com/azer/snakecase"

	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server/router"

	"github.com/rs/xid"

	"bytes"
	"io/ioutil"
	"net/http"
	"strings"

	log "github.com/opencontainers/runc/Godeps/_workspace/src/github.com/Sirupsen/logrus"
)

const (
	xHeaderPrefix     = "x-guble-"
	filterPrefix      = "filter"
	subscribersPrefix = "/subscribers"
)

var errNotFound = errors.New("Not Found.")

// RestMessageAPI is a struct representing a router's connector for a REST API.
type RestMessageAPI struct {
	router router.Router
	prefix string
}

// NewRestMessageAPI returns a new RestMessageAPI.
func NewRestMessageAPI(router router.Router, prefix string) *RestMessageAPI {
	return &RestMessageAPI{router, prefix}
}

// GetPrefix returns the prefix.
// It is a part of the service.endpoint implementation.
func (api *RestMessageAPI) GetPrefix() string {
	return api.prefix
}

// ServeHTTP is an http.Handler.
// It is a part of the service.endpoint implementation.
func (api *RestMessageAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method == http.MethodHead {
		return
	}

	if r.Method == http.MethodGet {
		log.WithField("url", r.URL.Path).Info("Get ")
		topic, err := api.extractTopic(r.URL.Path, subscribersPrefix)
		log.WithField("topic", topic).WithField("err", err).Debug("Extract")
		if err != nil {
			log.WithField("err", err).Error("Extracting topic failed")
			if err == errNotFound {
				http.NotFound(w, r)
				return
			}
			http.Error(w, "Server error.", http.StatusInternalServerError)
			return
		}
		resp, err := api.router.GetSubscribersForTopic(topic)
		w.Header().Set("Content-Type", "application/json")
		i, err := w.Write(resp)
		log.WithField("noOfBytes", i).WithField("topic", topic).Debug("Wrote as  response for GetSubscribersForTopic")
		if err != nil {
			log.WithField("Err", err).Error("Writing to byte stream failed")
			http.Error(w, "Server error.", http.StatusInternalServerError)
			return
		}
		return
	}

	if r.Method != http.MethodPost {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, `Can not read body`, http.StatusBadRequest)
		return
	}
	topic, err := api.extractTopic(r.URL.Path, "/message")
	if err != nil {
		if err == errNotFound {
			http.NotFound(w, r)
			return
		}
		http.Error(w, "Server error.", http.StatusInternalServerError)
		return
	}
	msg := &protocol.Message{
		Path:          protocol.Path(topic),
		Body:          body,
		UserID:        q(r, `userId`),
		ApplicationID: xid.New().String(),
		HeaderJSON:    headersToJSON(r.Header),
	}

	// add filters
	api.setFilters(r, msg)

	api.router.HandleMessage(msg)
	fmt.Fprintf(w, "OK")
}

func (api *RestMessageAPI) extractTopic(path string, requestTypeTopicPrefix string) (string, error) {
	p := removeTrailingSlash(api.prefix) + requestTypeTopicPrefix
	if !strings.HasPrefix(path, p) {
		return "", errNotFound
	}
	// Remove "`api.prefix` + /message" and we remain with the topic
	topic := strings.TrimPrefix(path, p)
	if topic == "/" || topic == "" {
		return "", errNotFound
	}
	return topic, nil
}

// setFilters sets a field found in the format `filterCamelCaseField` in the
// query of the request to underscore format on the message filters
func (api *RestMessageAPI) setFilters(r *http.Request, msg *protocol.Message) {
	for name, values := range r.URL.Query() {
		if strings.HasPrefix(name, filterPrefix) && len(values) > 0 {
			msg.SetFilter(filterName(name), values[0])
		}
	}
}

// returns a query parameter
func q(r *http.Request, name string) string {
	params := r.URL.Query()[name]
	if len(params) > 0 {
		return params[0]
	}
	return ""
}

// transform from filterCamelCase to camel_case
func filterName(name string) string {
	return snakecase.SnakeCase(strings.TrimPrefix(name, filterPrefix))
}

func headersToJSON(header http.Header) string {
	buff := &bytes.Buffer{}
	buff.WriteString("{")
	count := 0
	for key, valueList := range header {
		if strings.HasPrefix(strings.ToLower(key), xHeaderPrefix) && len(valueList) > 0 {
			if count > 0 {
				buff.WriteString(",")
			}
			buff.WriteString(`"`)
			buff.WriteString(key[len(xHeaderPrefix):])
			buff.WriteString(`":`)
			buff.WriteString(`"`)
			buff.WriteString(valueList[0])
			buff.WriteString(`"`)
			count++
		}
	}
	buff.WriteString("}")
	return string(buff.Bytes())
}

func removeTrailingSlash(path string) string {
	if len(path) > 1 && path[len(path)-1] == '/' {
		return path[:len(path)-1]
	}
	return path
}
<commit_msg>Remove improper import<commit_after>package rest

import (
	"errors"
	"fmt"

	"github.com/azer/snakecase"

	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server/router"

	"github.com/rs/xid"

	"bytes"
	"io/ioutil"
	"net/http"
	"strings"

	log "github.com/Sirupsen/logrus"
)

const (
	xHeaderPrefix     = "x-guble-"
	filterPrefix      = "filter"
	subscribersPrefix = "/subscribers"
)

var errNotFound = errors.New("Not Found.")

// RestMessageAPI is a struct representing a router's connector for a REST API.
type RestMessageAPI struct {
	router router.Router
	prefix string
}

// NewRestMessageAPI returns a new RestMessageAPI.
func NewRestMessageAPI(router router.Router, prefix string) *RestMessageAPI {
	return &RestMessageAPI{router, prefix}
}

// GetPrefix returns the prefix.
// It is a part of the service.endpoint implementation.
func (api *RestMessageAPI) GetPrefix() string {
	return api.prefix
}

// ServeHTTP is an http.Handler.
// It is a part of the service.endpoint implementation.
func (api *RestMessageAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method == http.MethodHead {
		return
	}

	if r.Method == http.MethodGet {
		log.WithField("url", r.URL.Path).Info("Get ")
		topic, err := api.extractTopic(r.URL.Path, subscribersPrefix)
		log.WithField("topic", topic).WithField("err", err).Debug("Extract")
		if err != nil {
			log.WithField("err", err).Error("Extracting topic failed")
			if err == errNotFound {
				http.NotFound(w, r)
				return
			}
			http.Error(w, "Server error.", http.StatusInternalServerError)
			return
		}
		resp, err := api.router.GetSubscribersForTopic(topic)
		w.Header().Set("Content-Type", "application/json")
		i, err := w.Write(resp)
		log.WithField("noOfBytes", i).WithField("topic", topic).Debug("Wrote as  response for GetSubscribersForTopic")
		if err != nil {
			log.WithField("Err", err).Error("Writing to byte stream failed")
			http.Error(w, "Server error.", http.StatusInternalServerError)
			return
		}
		return
	}

	if r.Method != http.MethodPost {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, `Can not read body`, http.StatusBadRequest)
		return
	}
	topic, err := api.extractTopic(r.URL.Path, "/message")
	if err != nil {
		if err == errNotFound {
			http.NotFound(w, r)
			return
		}
		http.Error(w, "Server error.", http.StatusInternalServerError)
		return
	}
	msg := &protocol.Message{
		Path:          protocol.Path(topic),
		Body:          body,
		UserID:        q(r, `userId`),
		ApplicationID: xid.New().String(),
		HeaderJSON:    headersToJSON(r.Header),
	}

	// add filters
	api.setFilters(r, msg)

	api.router.HandleMessage(msg)
	fmt.Fprintf(w, "OK")
}

func (api *RestMessageAPI) extractTopic(path string, requestTypeTopicPrefix string) (string, error) {
	p := removeTrailingSlash(api.prefix) + requestTypeTopicPrefix
	if !strings.HasPrefix(path, p) {
		return "", errNotFound
	}
	// Remove "`api.prefix` + /message" and we remain with the topic
	topic := strings.TrimPrefix(path, p)
	if topic == "/" || topic == "" {
		return "", errNotFound
	}
	return topic, nil
}

// setFilters sets a field found in the format `filterCamelCaseField` in the
// query of the request to underscore format on the message filters
func (api *RestMessageAPI) setFilters(r *http.Request, msg *protocol.Message) {
	for name, values := range r.URL.Query() {
		if strings.HasPrefix(name, filterPrefix) && len(values) > 0 {
			msg.SetFilter(filterName(name), values[0])
		}
	}
}

// returns a query parameter
func q(r *http.Request, name string) string {
	params := r.URL.Query()[name]
	if len(params) > 0 {
		return params[0]
	}
	return ""
}

// transform from filterCamelCase to camel_case
func filterName(name string) string {
	return snakecase.SnakeCase(strings.TrimPrefix(name, filterPrefix))
}

func headersToJSON(header http.Header) string {
	buff := &bytes.Buffer{}
	buff.WriteString("{")
	count := 0
	for key, valueList := range header {
		if strings.HasPrefix(strings.ToLower(key), xHeaderPrefix) && len(valueList) > 0 {
			if count > 0 {
				buff.WriteString(",")
			}
			buff.WriteString(`"`)
			buff.WriteString(key[len(xHeaderPrefix):])
			buff.WriteString(`":`)
			buff.WriteString(`"`)
			buff.WriteString(valueList[0])
			buff.WriteString(`"`)
			count++
		}
	}
	buff.WriteString("}")
	return string(buff.Bytes())
}

func removeTrailingSlash(path string) string {
	if len(path) > 1 && path[len(path)-1] == '/' {
		return path[:len(path)-1]
	}
	return path
}
<|endoftext|>


<commit_before>package rest

import (
	"errors"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server/router"

	"github.com/rs/xid"

	"bytes"
	"io/ioutil"
	"net/http"
	"strings"
)

const xHeaderPrefix = "x-guble-"

var errNotFound = errors.New("Not Found.")

type RestMessageAPI struct {
	router router.Router
	prefix string
}

func NewRestMessageAPI(router router.Router, prefix string) *RestMessageAPI {
	return &RestMessageAPI{router, prefix}
}

// GetPrefix returns the prefix.
// It is a part of the service.endpoint implementation.
func (api *RestMessageAPI) GetPrefix() string {
	return api.prefix
}

// ServeHTTP is an http.Handler.
// It is a part of the service.endpoint implementation.
func (api *RestMessageAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, `Can not read body`, http.StatusBadRequest)
		return
	}
	topic, err := api.extractTopic(r.URL.Path)
	if err != nil {
		if err == errNotFound {
			http.NotFound(w, r)
			return
		}
		http.Error(w, "Server error.", http.StatusInternalServerError)
		return
	}
	msg := &protocol.Message{
		Path:          protocol.Path(topic),
		Body:          body,
		UserID:        q(r, `userId`),
		ApplicationID: xid.New().String(),
		OptionalID:    q(r, `messageId`),
		HeaderJSON:    headersToJSON(r.Header),
	}
	api.router.HandleMessage(msg)
}

func (api *RestMessageAPI) extractTopic(path string) (string, error) {
	p := removeTrailingSlash(api.prefix) + "/message"
	if !strings.HasPrefix(path, p) {
		return "", errNotFound
	}
	// Remove "`api.prefix` + /message" and we remain with the topic
	topic := strings.TrimPrefix(path, p)
	if topic == "/" || topic == "" {
		return "", errNotFound
	}
	return topic, nil
}

// returns a query parameter
func q(r *http.Request, name string) string {
	params := r.URL.Query()[name]
	if len(params) > 0 {
		return params[0]
	}
	return ""
}

func headersToJSON(header http.Header) string {
	buff := &bytes.Buffer{}
	buff.WriteString("{")
	count := 0
	for key, valueList := range header {
		if strings.HasPrefix(strings.ToLower(key), xHeaderPrefix) && len(valueList) > 0 {
			if count > 0 {
				buff.WriteString(",")
			}
			buff.WriteString(`"`)
			buff.WriteString(key[len(xHeaderPrefix):])
			buff.WriteString(`":`)
			buff.WriteString(`"`)
			buff.WriteString(valueList[0])
			buff.WriteString(`"`)
			count++
		}
	}
	buff.WriteString("}")
	return string(buff.Bytes())
}

func removeTrailingSlash(path string) string {
	if len(path) > 1 && path[len(path)-1] == '/' {
		return path[:len(path)-1]
	}
	return path
}
<commit_msg>go doc<commit_after>package rest

import (
	"errors"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server/router"

	"github.com/rs/xid"

	"bytes"
	"io/ioutil"
	"net/http"
	"strings"
)

const xHeaderPrefix = "x-guble-"

var errNotFound = errors.New("Not Found.")

// RestMessageAPI is a struct representing a router's connector for a REST API.
type RestMessageAPI struct {
	router router.Router
	prefix string
}

// NewRestMessageAPI returns a new RestMessageAPI.
func NewRestMessageAPI(router router.Router, prefix string) *RestMessageAPI {
	return &RestMessageAPI{router, prefix}
}

// GetPrefix returns the prefix.
// It is a part of the service.endpoint implementation.
func (api *RestMessageAPI) GetPrefix() string {
	return api.prefix
}

// ServeHTTP is an http.Handler.
// It is a part of the service.endpoint implementation.
func (api *RestMessageAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, `Can not read body`, http.StatusBadRequest)
		return
	}
	topic, err := api.extractTopic(r.URL.Path)
	if err != nil {
		if err == errNotFound {
			http.NotFound(w, r)
			return
		}
		http.Error(w, "Server error.", http.StatusInternalServerError)
		return
	}
	msg := &protocol.Message{
		Path:          protocol.Path(topic),
		Body:          body,
		UserID:        q(r, `userId`),
		ApplicationID: xid.New().String(),
		OptionalID:    q(r, `messageId`),
		HeaderJSON:    headersToJSON(r.Header),
	}
	api.router.HandleMessage(msg)
}

func (api *RestMessageAPI) extractTopic(path string) (string, error) {
	p := removeTrailingSlash(api.prefix) + "/message"
	if !strings.HasPrefix(path, p) {
		return "", errNotFound
	}
	// Remove "`api.prefix` + /message" and we remain with the topic
	topic := strings.TrimPrefix(path, p)
	if topic == "/" || topic == "" {
		return "", errNotFound
	}
	return topic, nil
}

// returns a query parameter
func q(r *http.Request, name string) string {
	params := r.URL.Query()[name]
	if len(params) > 0 {
		return params[0]
	}
	return ""
}

func headersToJSON(header http.Header) string {
	buff := &bytes.Buffer{}
	buff.WriteString("{")
	count := 0
	for key, valueList := range header {
		if strings.HasPrefix(strings.ToLower(key), xHeaderPrefix) && len(valueList) > 0 {
			if count > 0 {
				buff.WriteString(",")
			}
			buff.WriteString(`"`)
			buff.WriteString(key[len(xHeaderPrefix):])
			buff.WriteString(`":`)
			buff.WriteString(`"`)
			buff.WriteString(valueList[0])
			buff.WriteString(`"`)
			count++
		}
	}
	buff.WriteString("}")
	return string(buff.Bytes())
}

func removeTrailingSlash(path string) string {
	if len(path) > 1 && path[len(path)-1] == '/' {
		return path[:len(path)-1]
	}
	return path
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/spf13/cobra"

	"github.com/NebulousLabs/Sia/api"
)

var (
	updateCmd = &cobra.Command{
		Use:   "update",
		Short: "Update Sia",
		Long:  "Check for (and/or download) available updates for Sia.",
		Run:   wrap(updatecmd),
	}

	updateCheckCmd = &cobra.Command{
		Use:   "check",
		Short: "Check for available updates",
		Long:  "Check for available updates.",
		Run:   wrap(updatecheckcmd),
	}

	updateApplyCmd = &cobra.Command{
		Use:   "apply [version]",
		Short: "Apply an update",
		Long:  "Update Sia to 'version'. To use the latest version, run 'update apply current', or simply 'update'.",
		Run:   wrap(updateapplycmd),
	}

	stopCmd = &cobra.Command{
		Use:   "stop",
		Short: "Stop the Sia daemon",
		Long:  "Stop the Sia daemon.",
		Run:   wrap(stopcmd),
	}
)

func updatecmd() {
	var update api.UpdateInfo
	err := getAPI("/daemon/updates/check", &update)
	if err != nil {
		fmt.Println("Could not check for update:", err)
		return
	}
	if !update.Available {
		fmt.Println("Already up to date.")
		return
	}
	err = get("/daemon/update/apply?version=" + update.Version)
	if err != nil {
		fmt.Println("Could not apply update:", err)
		return
	}
	fmt.Printf("Updated to version %s! Restart siad now.\n", update.Version)
}

func updatecheckcmd() {
	var update api.UpdateInfo
	err := getAPI("/daemon/updates/check", &update)
	if err != nil {
		fmt.Println("Could not check for update:", err)
		return
	}
	if !update.Available {
		fmt.Println("Up to date!")
		return
	}
	fmt.Printf("Update %s is available! Run 'siac update apply %s' to install it.\n", update.Version, update.Version)
}

func updateapplycmd(version string) {
	err := post("/daemon/updates/apply", "version="+version)
	if err != nil {
		fmt.Println("Could not apply update:", err)
		return
	}
	fmt.Printf("Updated to version %s! Restart siad now.\n", version)
}

func stopcmd() {
	err := post("/daemon/stop", "")
	if err != nil {
		fmt.Println("Could not stop daemon:", err)
		return
	}
	fmt.Println("Sia daemon stopped.")
}
<commit_msg>fix siac update cmd<commit_after>package main

import (
	"fmt"

	"github.com/spf13/cobra"

	"github.com/NebulousLabs/Sia/api"
)

var (
	updateCmd = &cobra.Command{
		Use:   "update",
		Short: "Update Sia",
		Long:  "Check for (and/or download) available updates for Sia.",
		Run:   wrap(updatecmd),
	}

	updateCheckCmd = &cobra.Command{
		Use:   "check",
		Short: "Check for available updates",
		Long:  "Check for available updates.",
		Run:   wrap(updatecheckcmd),
	}

	updateApplyCmd = &cobra.Command{
		Use:   "apply [version]",
		Short: "Apply an update",
		Long:  "Update Sia to 'version'. To use the latest version, run 'update apply current', or simply 'update'.",
		Run:   wrap(updateapplycmd),
	}

	stopCmd = &cobra.Command{
		Use:   "stop",
		Short: "Stop the Sia daemon",
		Long:  "Stop the Sia daemon.",
		Run:   wrap(stopcmd),
	}
)

func updatecmd() {
	var update api.UpdateInfo
	err := getAPI("/daemon/updates/check", &update)
	if err != nil {
		fmt.Println("Could not check for update:", err)
		return
	}
	if !update.Available {
		fmt.Println("Already up to date.")
		return
	}
	err = get("/daemon/updates/apply?version=" + update.Version)
	if err != nil {
		fmt.Println("Could not apply update:", err)
		return
	}
	fmt.Printf("Updated to version %s! Restart siad now.\n", update.Version)
}

func updatecheckcmd() {
	var update api.UpdateInfo
	err := getAPI("/daemon/updates/check", &update)
	if err != nil {
		fmt.Println("Could not check for update:", err)
		return
	}
	if !update.Available {
		fmt.Println("Up to date!")
		return
	}
	fmt.Printf("Update %s is available! Run 'siac update apply %s' to install it.\n", update.Version, update.Version)
}

func updateapplycmd(version string) {
	err := post("/daemon/updates/apply", "version="+version)
	if err != nil {
		fmt.Println("Could not apply update:", err)
		return
	}
	fmt.Printf("Updated to version %s! Restart siad now.\n", version)
}

func stopcmd() {
	err := post("/daemon/stop", "")
	if err != nil {
		fmt.Println("Could not stop daemon:", err)
		return
	}
	fmt.Println("Sia daemon stopped.")
}
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package rest

import (
	"fmt"
	"github.com/ServiceComb/service-center/pkg/rest"
	"github.com/ServiceComb/service-center/server/core"
	"github.com/prometheus/client_golang/prometheus"
	dto "github.com/prometheus/client_model/go"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var (
	incomingRequests = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "service_center",
			Subsystem: "http",
			Name:      "request_total",
			Help:      "Counter of requests received into ROA handler",
		}, []string{"method", "code", "instance", "api"})

	successfulRequests = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "service_center",
			Subsystem: "http",
			Name:      "success_total",
			Help:      "Counter of successful requests processed by ROA handler",
		}, []string{"method", "code", "instance", "api"})

	reqDurations = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace:  "service_center",
			Subsystem:  "http",
			Name:       "request_durations_microseconds",
			Help:       "HTTP request latency summary of ROA handler",
			Objectives: prometheus.DefObjectives,
		}, []string{"method", "instance", "api"})
)

func init() {
	prometheus.MustRegister(incomingRequests, successfulRequests, reqDurations)

	http.Handle("/metrics", prometheus.Handler())
}

func ReportRequestCompleted(w http.ResponseWriter, r *http.Request, start time.Time) {
	instance := fmt.Sprint(core.Instance.Endpoints)
	elapsed := float64(time.Since(start).Nanoseconds()) / 1000
	route := r.Context().Value(rest.CTX_MATCH_PATTERN).(string)

	if strings.Index(r.Method, "WATCH") != 0 {
		reqDurations.WithLabelValues(r.Method, instance, route).Observe(elapsed)
	}

	success, code := codeOf(w.Header())

	incomingRequests.WithLabelValues(r.Method, code, instance, route).Inc()

	if success {
		successfulRequests.WithLabelValues(r.Method, code, instance, route).Inc()
	}
}

func codeOf(h http.Header) (bool, string) {
	statusCode := h.Get("X-Response-Status")
	if statusCode == "" {
		return true, "200"
	}

	if code, _ := strconv.Atoi(statusCode); code >= http.StatusOK && code <= http.StatusAccepted {
		return true, statusCode
	}

	return false, statusCode
}

// Get value of metricFamily
func MetricValueOf(mf *dto.MetricFamily) float64 {
	if len(mf.GetMetric()) == 0 {
		return 0
	}

	switch mf.GetType() {
	case dto.MetricType_GAUGE:
		return mf.GetMetric()[0].GetGauge().GetValue()
	case dto.MetricType_COUNTER:
		return metricCounterOf(mf.GetMetric())
	case dto.MetricType_SUMMARY:
		return metricSummaryOf(mf.GetMetric())
	default:
		return 0
	}
}

func metricCounterOf(m []*dto.Metric) float64 {
	var sum float64 = 0
	for _, d := range m {
		sum += d.GetCounter().GetValue()
	}
	return sum
}

func metricSummaryOf(m []*dto.Metric) float64 {
	var (
		count uint64  = 0
		sum   float64 = 0
	)
	for _, d := range m {
		count += d.GetSummary().GetSampleCount()
		sum += d.GetSummary().GetSampleSum()
	}

	if count == 0 {
		return 0
	}

	return sum / float64(count)
}
<commit_msg>Bug fix: metrics panic when request is invalid. (#211)<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package rest

import (
	"fmt"
	"github.com/ServiceComb/service-center/pkg/rest"
	"github.com/ServiceComb/service-center/server/core"
	"github.com/prometheus/client_golang/prometheus"
	dto "github.com/prometheus/client_model/go"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var (
	incomingRequests = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "service_center",
			Subsystem: "http",
			Name:      "request_total",
			Help:      "Counter of requests received into ROA handler",
		}, []string{"method", "code", "instance", "api"})

	successfulRequests = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "service_center",
			Subsystem: "http",
			Name:      "success_total",
			Help:      "Counter of successful requests processed by ROA handler",
		}, []string{"method", "code", "instance", "api"})

	reqDurations = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace:  "service_center",
			Subsystem:  "http",
			Name:       "request_durations_microseconds",
			Help:       "HTTP request latency summary of ROA handler",
			Objectives: prometheus.DefObjectives,
		}, []string{"method", "instance", "api"})
)

func init() {
	prometheus.MustRegister(incomingRequests, successfulRequests, reqDurations)

	http.Handle("/metrics", prometheus.Handler())
}

func ReportRequestCompleted(w http.ResponseWriter, r *http.Request, start time.Time) {
	instance := fmt.Sprint(core.Instance.Endpoints)
	elapsed := float64(time.Since(start).Nanoseconds()) / 1000
	route, _ := r.Context().Value(rest.CTX_MATCH_PATTERN).(string)

	if strings.Index(r.Method, "WATCH") != 0 {
		reqDurations.WithLabelValues(r.Method, instance, route).Observe(elapsed)
	}

	success, code := codeOf(w.Header())

	incomingRequests.WithLabelValues(r.Method, code, instance, route).Inc()

	if success {
		successfulRequests.WithLabelValues(r.Method, code, instance, route).Inc()
	}
}

func codeOf(h http.Header) (bool, string) {
	statusCode := h.Get("X-Response-Status")
	if statusCode == "" {
		return true, "200"
	}

	if code, _ := strconv.Atoi(statusCode); code >= http.StatusOK && code <= http.StatusAccepted {
		return true, statusCode
	}

	return false, statusCode
}

// Get value of metricFamily
func MetricValueOf(mf *dto.MetricFamily) float64 {
	if len(mf.GetMetric()) == 0 {
		return 0
	}

	switch mf.GetType() {
	case dto.MetricType_GAUGE:
		return mf.GetMetric()[0].GetGauge().GetValue()
	case dto.MetricType_COUNTER:
		return metricCounterOf(mf.GetMetric())
	case dto.MetricType_SUMMARY:
		return metricSummaryOf(mf.GetMetric())
	default:
		return 0
	}
}

func metricCounterOf(m []*dto.Metric) float64 {
	var sum float64 = 0
	for _, d := range m {
		sum += d.GetCounter().GetValue()
	}
	return sum
}

func metricSummaryOf(m []*dto.Metric) float64 {
	var (
		count uint64  = 0
		sum   float64 = 0
	)
	for _, d := range m {
		count += d.GetSummary().GetSampleCount()
		sum += d.GetSummary().GetSampleSum()
	}

	if count == 0 {
		return 0
	}

	return sum / float64(count)
}
<|endoftext|>


<commit_before>// errorcheck

// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package p

type S struct{}

func (S) _[_ any]() {} // ERROR "method must have no type parameters"

type _ interface {
	m[_ any]() // ERROR "method must have no type parameters"
}
<commit_msg>test/typeparam: adjust test preamble (fix longtests)<commit_after>// errorcheck -G=3

// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package p

type S struct{}

func (S) _[_ any]() {} // ERROR "method must have no type parameters"

type _ interface {
	m[_ any]() // ERROR "method must have no type parameters"
}
<|endoftext|>


<commit_before>package disk

import (
	"os"
	"testing"
)

func Test_SwarmStoring(t *testing.T) {
	i, err := CreateSwarmSystem("f")
    if err!=nil{
        t.Error(err.Error())
    }
	i.CreateFile("0", 1000)
	i.CreateFile("1", 10)
	i.SaveSwarm()
	i, err = CreateSwarmSystem("f")
	if err != nil {
		t.Error(err.Error())
	}
	os.Remove("f")
	os.Remove("f.conf")

}
func Test_Parallel(t *testing.T) {
	i, _ := CreateSwarmSystem("files")
	for c := 0; c < 100; c++ {
		go i.CreateFile(string(c), uint64(c))
	}
}
func Test_Swarm(t *testing.T) {
	i, err := CreateSwarmSystem("SW1")
	defer os.Remove("SW1")
	if err != nil {
		t.Error(err.Error())
	}
	if i == nil {
		t.Fatal("first returned object is nil")
	}

	b, err := CreateSwarmSystem("SW2")
	defer os.Remove("SW2")
	if err != nil {
		if b == nil {
			t.Error("Failed to create second swarm")
		}
	}
	_, err = i.CreateFile("hi_there", 1000)
	if err != nil {
		t.Error(err.Error())
		t.Fatal("Failed to Create file")

	}
	b.CreateFile("Hello", 1000)
	f1 := []byte{12, 34, 51, 23, 51, 12, 51}
	f2 := []byte{24, 34, 51, 25}
	err = i.WriteFile("hi_there", 10, f1)
	if nil != err {
		if os.IsPermission(err) {
			t.Error("Permission bits are set so that things don't work.")
		} else {
			t.Error("Failed to write file 1")
		}
	}

	if nil != b.WriteFile("Hello", 0, f2) {
		t.Error("Error. File 2 failed to write.")
	}
	if nil != i.DeleteFile("hi_there") {
		t.Error("Failed to destroy file")
	}
	if nil != b.DeleteFile("Hello") {
		t.Error("Failed to destroy second file")
	}
	if nil == b.DeleteFile("hello") {
		t.Error("Swarm did not cause error when incorrect name applied.")
	}

}
<commit_msg>Fix diststorage formatting<commit_after>package disk

import (
	"os"
	"testing"
)

func Test_SwarmStoring(t *testing.T) {
	i, err := CreateSwarmSystem("f")
	if err != nil {
		t.Error(err.Error())
	}
	i.CreateFile("0", 1000)
	i.CreateFile("1", 10)
	i.SaveSwarm()
	i, err = CreateSwarmSystem("f")
	if err != nil {
		t.Error(err.Error())
	}
	os.Remove("f")
	os.Remove("f.conf")

}
func Test_Parallel(t *testing.T) {
	i, _ := CreateSwarmSystem("files")
	for c := 0; c < 100; c++ {
		go i.CreateFile(string(c), uint64(c))
	}
}
func Test_Swarm(t *testing.T) {
	i, err := CreateSwarmSystem("SW1")
	defer os.Remove("SW1")
	if err != nil {
		t.Error(err.Error())
	}
	if i == nil {
		t.Fatal("first returned object is nil")
	}

	b, err := CreateSwarmSystem("SW2")
	defer os.Remove("SW2")
	if err != nil {
		if b == nil {
			t.Error("Failed to create second swarm")
		}
	}
	_, err = i.CreateFile("hi_there", 1000)
	if err != nil {
		t.Error(err.Error())
		t.Fatal("Failed to Create file")

	}
	b.CreateFile("Hello", 1000)
	f1 := []byte{12, 34, 51, 23, 51, 12, 51}
	f2 := []byte{24, 34, 51, 25}
	err = i.WriteFile("hi_there", 10, f1)
	if nil != err {
		if os.IsPermission(err) {
			t.Error("Permission bits are set so that things don't work.")
		} else {
			t.Error("Failed to write file 1")
		}
	}

	if nil != b.WriteFile("Hello", 0, f2) {
		t.Error("Error. File 2 failed to write.")
	}
	if nil != i.DeleteFile("hi_there") {
		t.Error("Failed to destroy file")
	}
	if nil != b.DeleteFile("Hello") {
		t.Error("Failed to destroy second file")
	}
	if nil == b.DeleteFile("hello") {
		t.Error("Swarm did not cause error when incorrect name applied.")
	}

}
<|endoftext|>


<commit_before>package testhelper

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"
	"testing"
)

// TestFormValues ensures that all the URL parameters given to the http.Request are the same as values.
func TestFormValues(t *testing.T, r *http.Request, values map[string]string) {
	want := url.Values{}
	for k, v := range values {
		want.Add(k, v)
	}

	r.ParseForm()
	if !reflect.DeepEqual(want, r.Form) {
		t.Errorf("Request parameters = %v, want %v", r.Form, want)
	}
}

// TestMethod checks that the Request has the expected method (e.g. GET, POST).
func TestMethod(t *testing.T, r *http.Request, expected string) {
	if expected != r.Method {
		t.Errorf("Request method = %v, expected %v", r.Method, expected)
	}
}

// TestHeader checks that the header on the http.Request matches the expected value.
func TestHeader(t *testing.T, r *http.Request, header string, expected string) {
	if actual := r.Header.Get(header); expected != actual {
		t.Errorf("Header %s = %s, expected %s", header, actual, expected)
	}
}

// TestBody verifies that the request body matches an expected body.
func TestBody(t *testing.T, r *http.Request, expected string) {
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		t.Errorf("Unable to read body: %v", err)
	}
	str := string(b)
	if expected != str {
		t.Errorf("Body = %s, expected %s", str, expected)
	}
}

// TestJSONRequest verifies that the JSON payload of a request matches an expected structure, without asserting things about
// whitespace or ordering.
func TestJSONRequest(t *testing.T, r *http.Request, expected string) {
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		t.Errorf("Unable to read request body: %v", err)
	}

	var expectedJSON interface{}
	err = json.Unmarshal([]byte(expected), &expectedJSON)
	if err != nil {
		t.Errorf("Unable to parse expected value as JSON: %v", err)
	}

	var actualJSON interface{}
	err = json.Unmarshal(b, &actualJSON)
	if err != nil {
		t.Errorf("Unable to parse request body as JSON: %v", err)
	}

	if !reflect.DeepEqual(expectedJSON, actualJSON) {
		t.Errorf("Response body did not contain the correct JSON.")
	}
}
<commit_msg>Pretty-print JSON when inconsistencies arise.<commit_after>package testhelper

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"
	"testing"
)

// TestFormValues ensures that all the URL parameters given to the http.Request are the same as values.
func TestFormValues(t *testing.T, r *http.Request, values map[string]string) {
	want := url.Values{}
	for k, v := range values {
		want.Add(k, v)
	}

	r.ParseForm()
	if !reflect.DeepEqual(want, r.Form) {
		t.Errorf("Request parameters = %v, want %v", r.Form, want)
	}
}

// TestMethod checks that the Request has the expected method (e.g. GET, POST).
func TestMethod(t *testing.T, r *http.Request, expected string) {
	if expected != r.Method {
		t.Errorf("Request method = %v, expected %v", r.Method, expected)
	}
}

// TestHeader checks that the header on the http.Request matches the expected value.
func TestHeader(t *testing.T, r *http.Request, header string, expected string) {
	if actual := r.Header.Get(header); expected != actual {
		t.Errorf("Header %s = %s, expected %s", header, actual, expected)
	}
}

// TestBody verifies that the request body matches an expected body.
func TestBody(t *testing.T, r *http.Request, expected string) {
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		t.Errorf("Unable to read body: %v", err)
	}
	str := string(b)
	if expected != str {
		t.Errorf("Body = %s, expected %s", str, expected)
	}
}

// TestJSONRequest verifies that the JSON payload of a request matches an expected structure, without asserting things about
// whitespace or ordering.
func TestJSONRequest(t *testing.T, r *http.Request, expected string) {
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		t.Errorf("Unable to read request body: %v", err)
	}

	var expectedJSON interface{}
	err = json.Unmarshal([]byte(expected), &expectedJSON)
	if err != nil {
		t.Errorf("Unable to parse expected value as JSON: %v", err)
	}

	var actualJSON interface{}
	err = json.Unmarshal(b, &actualJSON)
	if err != nil {
		t.Errorf("Unable to parse request body as JSON: %v", err)
	}

	if !reflect.DeepEqual(expectedJSON, actualJSON) {
		prettyExpected, err := json.MarshalIndent(expectedJSON, "", "  ")
		if err != nil {
			t.Logf("Unable to pretty-print expected JSON: %v\n%s", err, expected)
		} else {
			t.Logf("Expected JSON:\n%s", prettyExpected)
		}

		prettyActual, err := json.MarshalIndent(actualJSON, "", "  ")
		if err != nil {
			t.Logf("Unable to pretty-print actual JSON: %v\n%s", err, b)
		} else {
			t.Logf("Actual JSON:\n%s", prettyActual)
		}

		t.Errorf("Response body did not contain the correct JSON.")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/zenoss/glog"
	//"github.com/zenoss/serviced/dao"

	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"syscall"
)

// exePaths returns the full path to the given executables in a map
func exePaths(exes []string) (map[string]string, error) {
	exeMap := map[string]string{}

	for _, exe := range exes {
		path, err := exec.LookPath(exe)
		if err != nil {
			glog.Errorf("exe:'%v' not found err: %v\n", exe, err)
			return nil, err
		}

		exeMap[exe] = path
	}

	return exeMap, nil
}

// attachContainerAndExec connects to a container and executes an arbitrary bash command
func attachContainerAndExec(containerId string, cmd []string) error {
	exeMap, err := exePaths([]string{"sudo", "nsinit"})
	if err != nil {
		return err
	}

	NSINIT_ROOT := "/var/lib/docker/execdriver/native" // has container.json

	attachCmd := fmt.Sprintf("cd %s/%s && %s exec %s", NSINIT_ROOT, containerId,
		exeMap["nsinit"], strings.Join(cmd, " "))
	fullCmd := []string{exeMap["sudo"], "--", "/bin/bash", "-c", attachCmd}
	glog.V(1).Infof("exec cmd: %v\n", fullCmd)
	return syscall.Exec(fullCmd[0], fullCmd[0:], os.Environ())
}

func findContainerIdFromDocker(pattern string) (string, error) {
	// docker ps --no-trunc| awk '/serviced.*redis/{print $1;exit}'
	exeMap, err := exePaths([]string{"docker"})
	if err != nil {
		return "", err
	}

	CMD := exeMap["docker"]
	argv := []string{"ps", "--no-trunc"}
	glog.V(1).Infof("running command: '%s %s'", CMD, argv)
	cmd := exec.Command(CMD, argv...)

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		glog.Errorf("Error setting up command: '%v %v'  err: %s", CMD, argv, err)
		return "", err
	}

	err = cmd.Start()
	if err != nil {
		glog.Errorf("Error starting command: '%v %v'  err: %s", CMD, argv, err)
		return "", err
	}

	glog.V(1).Infof("looking for container matching pattern: %s", pattern)
	reader := bufio.NewReader(stdout)
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF && len(line) == 0 {
			break
		}
		matched, err := regexp.MatchString(pattern, line)
		if matched {
			return strings.Split(line, " ")[0], nil
		}
	}

	newError := fmt.Sprintf("could not find container from pattern: %v", pattern)
	return "", errors.New(newError)
}

// CmdAttach attaches to a service container and runs the given arbitrary bash command
func (cli *ServicedCli) CmdAttach(args ...string) error {

	cmd := Subcmd("attach", "BASH_CMD ...", "attach to a service container and run command")

	var containerID string
	cmd.StringVar(&containerID, "containerID", "", "attach to container given containerID")

	var pattern string
	cmd.StringVar(&pattern, "pattern", "", "attach to container found by matching pattern pattern in docker ps")

	if err := cmd.Parse(args); err != nil {
		return err
	}

	if len(cmd.Args()) < 1 {
		glog.Errorf("len(args) = %d; missing bash command to run\n", len(cmd.Args()))
		os.Exit(253)
	}

	if len(containerID) <= 0 {
		if len(pattern) > 0 {
			var err error
			// TODO: find containerID from service state that matches user supplied pattern
			containerID, err = findContainerIdFromDocker(pattern)
			if err != nil {
				return err
			}
		} else {
			glog.Errorf("neither containerID nor pattern is specified\n")
			os.Exit(254)
		}
	}

	bashCommand := cmd.Args()[0:]

	if err := attachContainerAndExec(containerID, bashCommand); err != nil {
		glog.Errorf("error running bash command:%v  err:%v\n", cmd.Args(), err)
		os.Exit(255)
	}

	return nil
}
<commit_msg>fixed comments from review<commit_after>package main

import (
	"github.com/zenoss/glog"
	//"github.com/zenoss/serviced/dao"

	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"syscall"
)

// exePaths returns the full path to the given executables in a map
func exePaths(exes []string) (map[string]string, error) {
	exeMap := map[string]string{}

	for _, exe := range exes {
		path, err := exec.LookPath(exe)
		if err != nil {
			glog.Errorf("exe:'%v' not found err: %v\n", exe, err)
			return nil, err
		}

		exeMap[exe] = path
	}

	return exeMap, nil
}

// attachContainerAndExec connects to a container and executes an arbitrary bash command
func attachContainerAndExec(containerId string, cmd []string) error {
	exeMap, err := exePaths([]string{"sudo", "nsinit"})
	if err != nil {
		return err
	}

	NSINIT_ROOT := "/var/lib/docker/execdriver/native" // has container.json

	attachCmd := fmt.Sprintf("cd %s/%s && %s exec %s", NSINIT_ROOT, containerId,
		exeMap["nsinit"], strings.Join(cmd, " "))
	fullCmd := []string{exeMap["sudo"], "--", "/bin/bash", "-c", attachCmd}
	glog.V(1).Infof("exec cmd: %v\n", fullCmd)
	return syscall.Exec(fullCmd[0], fullCmd[0:], os.Environ())
}

// findContainerIdFromDocker returns the containerID that matches docker ps output
func findContainerIdFromDocker(pattern string) (string, error) {
	// algorithm - perform following shell one-liner in this short 40 line go function :)
	//   docker ps --no-trunc | awk '/serviced.*redis/{print $1;exit}'
	exeMap, err := exePaths([]string{"docker"})
	if err != nil {
		return "", err
	}

	CMD := exeMap["docker"]
	argv := []string{"ps", "--no-trunc"}
	glog.V(1).Infof("running command: '%s %s'", CMD, argv)
	cmd := exec.Command(CMD, argv...)

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		glog.Errorf("Error setting up command: '%v %v'  err: %s", CMD, argv, err)
		return "", err
	}

	if err := cmd.Start(); err != nil {
		glog.Errorf("Error starting command: '%v %v'  err: %s", CMD, argv, err)
		return "", err
	}

	glog.V(1).Infof("looking for container matching pattern: %s", pattern)
	reader := bufio.NewReader(stdout)
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF && len(line) == 0 {
			break
		}
		matched, err := regexp.MatchString(pattern, line)
		if matched {
			return strings.Split(line, " ")[0], nil
		}
	}

	return "", errors.New(fmt.Sprintf("could not find container from pattern: %v", pattern))
}

// CmdAttach attaches to a service container and runs the given arbitrary bash command
func (cli *ServicedCli) CmdAttach(args ...string) error {
	cmd := Subcmd("attach", "BASH_CMD ...", "attach to a service container and run command")

	var containerID string
	cmd.StringVar(&containerID, "containerID", "", "attach to container given containerID")

	var pattern string
	cmd.StringVar(&pattern, "pattern", "", "attach to first container found by matching pattern in docker ps")

	if err := cmd.Parse(args); err != nil {
		return err
	}

	if len(cmd.Args()) < 1 {
		return errors.New(fmt.Sprintf("missing bash command to run\n"))
	}

	if len(containerID) <= 0 {
		if len(pattern) > 0 {
			var err error
			// TODO: find containerID from service state that matches user supplied pattern
			containerID, err = findContainerIdFromDocker(pattern)
			if err != nil {
				return err
			}
		} else {
			return errors.New(fmt.Sprintf("neither containerID nor pattern is specified\n"))
		}
	}

	bashCommand := cmd.Args()[0:]

	if err := attachContainerAndExec(containerID, bashCommand); err != nil {
		return errors.New(fmt.Sprintf("error running bash command:%v  err:%v\n", cmd.Args(), err))
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"sync"

	"github.com/colinmarc/hdfs"
)

// These are all the tokens that look like noise when I look at logs.
var logsToSkip = [][]byte{
	[]byte("INFO"),
	[]byte("WARN"),
	[]byte("SLF4J"),
	[]byte("log4j:ERROR"),
	[]byte("Container killed on request"),
	[]byte("Container exited with"),
	[]byte("RMCommunicator Allocator"),
	[]byte("RMContainerAllocator"),
	[]byte("TaskAttemptListenerImpl"),
	[]byte("AM com"),
	[]byte("PM com"),
}

func (jt *jobTracker) testLogsDir() error {
	client, err := hdfs.New(*namenodeAddress)
	if err != nil {
		return err
	}

	_, err = client.ReadDir(*yarnLogDir)
	return err
}

func (jt *jobTracker) fetchLogs(id string) ([]string, error) {
	client, err := hdfs.New(*namenodeAddress)
	if err != nil {
		return nil, err
	}

	appID, jobID := hadoopIDs(id)

	jt.jobsLock.Lock()
	user := jt.jobs[jobID].Details.User
	jt.jobsLock.Lock()

	dir := fmt.Sprintf("%s/%s/logs/%s/", *yarnLogDir, user, appID)

	files, err := client.ReadDir(dir)
	if err != nil {
		return nil, err
	}

	filenames := make(chan string, len(files))
	results := make(chan string, len(files))

	var wg sync.WaitGroup

	for x := 1; x <= 5; x++ {
		go func() {
			for name := range filenames {
				result, err := findStacktrace(client, name)
				if err != nil {
					log.Println("Error reading", name, err)
				} else if result != "" {
					results <- result
				}
				wg.Done()
			}
		}()
	}

	var logs []string

	wg.Add(len(files))
	for _, file := range files {
		filenames <- dir + file.Name()
	}
	close(filenames)

	wg.Wait()
	close(results)

	for result := range results {
		logs = append(logs, result)
	}

	return logs, nil
}

func findStacktrace(client *hdfs.Client, name string) (string, error) {
	log.Println("Reading", name)
	file, err := client.Open(name)
	if err != nil {
		return "", err
	}

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return "", err
	}

	var logs [][]byte

	lines := bytes.SplitAfter(data, []byte("\n"))

	for _, line := range lines {
		matched := false
		for _, token := range logsToSkip {
			if bytes.Contains(line, token) {
				matched = true
				break
			}
		}
		if !matched {
			logs = append(logs, line)
		}
	}
	log.Println("Finished", name)

	return string(bytes.Join(logs, nil)), nil
}
<commit_msg>Fix a typo where we try to grab a lock twice in a row<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"sync"

	"github.com/colinmarc/hdfs"
)

// These are all the tokens that look like noise when I look at logs.
var logsToSkip = [][]byte{
	[]byte("INFO"),
	[]byte("WARN"),
	[]byte("SLF4J"),
	[]byte("log4j:ERROR"),
	[]byte("Container killed on request"),
	[]byte("Container exited with"),
	[]byte("RMCommunicator Allocator"),
	[]byte("RMContainerAllocator"),
	[]byte("TaskAttemptListenerImpl"),
	[]byte("AM com"),
	[]byte("PM com"),
}

func (jt *jobTracker) testLogsDir() error {
	client, err := hdfs.New(*namenodeAddress)
	if err != nil {
		return err
	}

	_, err = client.ReadDir(*yarnLogDir)
	return err
}

func (jt *jobTracker) fetchLogs(id string) ([]string, error) {
	client, err := hdfs.New(*namenodeAddress)
	if err != nil {
		return nil, err
	}

	appID, jobID := hadoopIDs(id)

	jt.jobsLock.Lock()
	user := jt.jobs[jobID].Details.User
	jt.jobsLock.Unlock()

	dir := fmt.Sprintf("%s/%s/logs/%s/", *yarnLogDir, user, appID)

	files, err := client.ReadDir(dir)
	if err != nil {
		return nil, err
	}

	filenames := make(chan string, len(files))
	results := make(chan string, len(files))

	var wg sync.WaitGroup

	for x := 1; x <= 5; x++ {
		go func() {
			for name := range filenames {
				result, err := findStacktrace(client, name)
				if err != nil {
					log.Println("Error reading", name, err)
				} else if result != "" {
					results <- result
				}
				wg.Done()
			}
		}()
	}

	var logs []string

	wg.Add(len(files))
	for _, file := range files {
		filenames <- dir + file.Name()
	}
	close(filenames)

	wg.Wait()
	close(results)

	for result := range results {
		logs = append(logs, result)
	}

	return logs, nil
}

func findStacktrace(client *hdfs.Client, name string) (string, error) {
	log.Println("Reading", name)
	file, err := client.Open(name)
	if err != nil {
		return "", err
	}

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return "", err
	}

	var logs [][]byte

	lines := bytes.SplitAfter(data, []byte("\n"))

	for _, line := range lines {
		matched := false
		for _, token := range logsToSkip {
			if bytes.Contains(line, token) {
				matched = true
				break
			}
		}
		if !matched {
			logs = append(logs, line)
		}
	}
	log.Println("Finished", name)

	return string(bytes.Join(logs, nil)), nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file is adapted from https://github.com/robpike/ivy/blob/master/value/loop.go.

package apd

import (
	"math"

	"github.com/pkg/errors"
)

type loop struct {
	c             *Context
	name          string // The name of the function we are evaluating.
	i             uint64 // Loop count.
	precision     int32
	maxIterations uint64   // When to give up.
	arg           *Decimal // original argument to function; only used for diagnostic.
	prevZ         *Decimal // Result from the previous iteration.
	delta         *Decimal // |Change| from previous iteration.
}

const digitsToBitsRatio = math.Ln10 / math.Ln2

// newLoop returns a new loop checker. Arguments:
// 	 - name: name of the function being calculated (for error messages)
// 	 - arg: argument to the function (for error messages)
// 	 - precision: desired precision; the loop ends when consecutive estimates
// 	              differ less than the desired precision. Note that typically
// 	              the inner computations in an iteration need higher precision,
// 	              so this is normally lower than the precision in the context.
// 	 - maxItersPerDigit: after this many iterations per digit of precision, the
// 	                     loop ends in error.
func (c *Context) newLoop(name string, arg *Decimal, precision uint32, maxItersPerDigit int) *loop {
	return &loop{
		c:             c,
		name:          name,
		arg:           new(Decimal).Set(arg),
		precision:     int32(precision),
		maxIterations: 10 + uint64(maxItersPerDigit*int(precision)),
		prevZ:         new(Decimal),
		delta:         new(Decimal),
	}
}

// done reports whether the loop is done. If it does not converge
// after the maximum number of iterations, it returns an error.
func (l *loop) done(z *Decimal) (bool, error) {
	l.c.Sub(l.delta, l.prevZ, z)
	sign := l.delta.Sign()
	if sign == 0 {
		return true, nil
	}
	if sign < 0 {
		// Convergence can oscillate when the calculation is nearly
		// done and we're running out of bits. This stops that.
		// See next comment.
		l.delta.Neg(l.delta)
	}

	// We stop if the delta is smaller than a change of 1 in the
	// (l.precision)-th digit of z. Examples:
	//
	//   p   = 4
	//   z   = 12345.678 = 12345678 * 10^-3
	//   eps =    10.000 = 10^(-4+8-3)
	//
	//   p   = 3
	//   z   = 0.001234 = 1234 * 10^-6
	//   eps = 0.00001  = 10^(-3+4-6)
	eps := Decimal{Coeff: *bigOne, Exponent: -l.precision + int32(z.NumDigits()) + z.Exponent}
	if l.delta.Cmp(&eps) <= 0 {
		return true, nil
	}
	l.i++
	if l.i == l.maxIterations {
		return false, errors.Errorf(
			"%s %s: did not converge after %d iterations; prev,last result %s,%s delta %s precision: %d",
			l.name, l.arg.String(), l.maxIterations, z, l.prevZ, l.delta, l.precision,
		)
	}
	l.prevZ.Set(z)
	return false, nil
}
<commit_msg>check for error<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file is adapted from https://github.com/robpike/ivy/blob/master/value/loop.go.

package apd

import (
	"math"

	"github.com/pkg/errors"
)

type loop struct {
	c             *Context
	name          string // The name of the function we are evaluating.
	i             uint64 // Loop count.
	precision     int32
	maxIterations uint64   // When to give up.
	arg           *Decimal // original argument to function; only used for diagnostic.
	prevZ         *Decimal // Result from the previous iteration.
	delta         *Decimal // |Change| from previous iteration.
}

const digitsToBitsRatio = math.Ln10 / math.Ln2

// newLoop returns a new loop checker. Arguments:
// 	 - name: name of the function being calculated (for error messages)
// 	 - arg: argument to the function (for error messages)
// 	 - precision: desired precision; the loop ends when consecutive estimates
// 	              differ less than the desired precision. Note that typically
// 	              the inner computations in an iteration need higher precision,
// 	              so this is normally lower than the precision in the context.
// 	 - maxItersPerDigit: after this many iterations per digit of precision, the
// 	                     loop ends in error.
func (c *Context) newLoop(name string, arg *Decimal, precision uint32, maxItersPerDigit int) *loop {
	return &loop{
		c:             c,
		name:          name,
		arg:           new(Decimal).Set(arg),
		precision:     int32(precision),
		maxIterations: 10 + uint64(maxItersPerDigit*int(precision)),
		prevZ:         new(Decimal),
		delta:         new(Decimal),
	}
}

// done reports whether the loop is done. If it does not converge
// after the maximum number of iterations, it returns an error.
func (l *loop) done(z *Decimal) (bool, error) {
	if _, err := l.c.Sub(l.delta, l.prevZ, z); err != nil {
		return false, err
	}
	sign := l.delta.Sign()
	if sign == 0 {
		return true, nil
	}
	if sign < 0 {
		// Convergence can oscillate when the calculation is nearly
		// done and we're running out of bits. This stops that.
		// See next comment.
		l.delta.Neg(l.delta)
	}

	// We stop if the delta is smaller than a change of 1 in the
	// (l.precision)-th digit of z. Examples:
	//
	//   p   = 4
	//   z   = 12345.678 = 12345678 * 10^-3
	//   eps =    10.000 = 10^(-4+8-3)
	//
	//   p   = 3
	//   z   = 0.001234 = 1234 * 10^-6
	//   eps = 0.00001  = 10^(-3+4-6)
	eps := Decimal{Coeff: *bigOne, Exponent: -l.precision + int32(z.NumDigits()) + z.Exponent}
	if l.delta.Cmp(&eps) <= 0 {
		return true, nil
	}
	l.i++
	if l.i == l.maxIterations {
		return false, errors.Errorf(
			"%s %s: did not converge after %d iterations; prev,last result %s,%s delta %s precision: %d",
			l.name, l.arg.String(), l.maxIterations, z, l.prevZ, l.delta, l.precision,
		)
	}
	l.prevZ.Set(z)
	return false, nil
}
<|endoftext|>


<commit_before>package main

import (
	"c-14/grue/config"
	"fmt"
	"github.com/mmcdole/gofeed"
	"gopkg.in/gomail.v2"
	"strconv"
	"strings"
	"time"
)

type Email struct {
	FromName    string
	FromAddress string
	Recipient   string
	Date        time.Time
	Subject     string
	ItemURI     string
	Body        string
}

func (email *Email) setFrom(feedName string, feedTitle string, account config.AccountConfig, conf *config.GrueConfig) {
	r := strings.NewReplacer("{name}", feedName, "{title}", feedTitle)
	if account.NameFormat != nil {
		email.FromName = r.Replace(*account.NameFormat)
	} else {
		email.FromName = r.Replace(conf.NameFormat)
	}
	email.FromAddress = conf.FromAddress
}

func (email *Email) Send(ch chan *gomail.Message) {
	m := gomail.NewMessage()
	m.SetAddressHeader("From", email.FromAddress, email.FromName)
	m.SetHeader("To", email.Recipient)
	m.SetHeader("Subject", email.Subject)
	m.SetDateHeader("Date", email.Date)
	m.SetDateHeader("X-Date", time.Now())
	m.SetHeader("X-RSS-URI", email.ItemURI)
	m.SetBody("text/plain", email.Body)
	ch <- m
}

func createEmail(feedName string, feedTitle string, item *gofeed.Item, date time.Time, account config.AccountConfig, conf *config.GrueConfig) *Email {
	email := new(Email)
	email.setFrom(feedName, feedTitle, account, conf)
	email.Recipient = conf.Recipient
	email.Subject = item.Title
	email.Date = date
	email.ItemURI = item.Link
	email.Body = item.Description
	return email
}

func startDialing(messages chan *gomail.Message, ret chan error, conf *config.GrueConfig) {
	var d *gomail.Dialer
	var hostname string
	var port int
	var err error
	if conf.SmtpServer != nil {
		parts := strings.Split(*conf.SmtpServer, ":")
		if len(parts) > 2 {
			ret <- fmt.Errorf("%s not a valid hostname\n", *conf.SmtpServer)
			return
		} else if len(parts) == 1 {
			hostname = parts[0]
			port = 587
		} else {
			hostname = parts[0]
			port, err = strconv.Atoi(parts[1])
			if err != nil {
				ret <- fmt.Errorf("Failed to parse port: %v\n", err)
				return
			}
		}
	} else {
		hostname = "localhost"
		port = 25
	}
	if conf.SmtpUser != nil {
		d = gomail.NewDialer(hostname, port, *conf.SmtpUser, *conf.SmtpPass)
	} else {
		d = gomail.NewDialer(hostname, port, "", "")
	}

	s, err := d.Dial()
	if err != nil {
		ret <- err
		return
	}
	for m := range messages {
		err = gomail.Send(s, m)
		if err != nil {
			ret <- err
		}
	}

	ret <- s.Close()
}
<commit_msg>Add basic html2text functionality via jaytaylor/html2text<commit_after>package main

import (
	"c-14/grue/config"
	"fmt"
	"github.com/jaytaylor/html2text"
	"github.com/mmcdole/gofeed"
	"gopkg.in/gomail.v2"
	"strconv"
	"strings"
	"time"
)

type Email struct {
	FromName    string
	FromAddress string
	Recipient   string
	Date        time.Time
	Subject     string
	ItemURI     string
	Body        string
}

func (email *Email) setFrom(feedName string, feedTitle string, account config.AccountConfig, conf *config.GrueConfig) {
	r := strings.NewReplacer("{name}", feedName, "{title}", feedTitle)
	if account.NameFormat != nil {
		email.FromName = r.Replace(*account.NameFormat)
	} else {
		email.FromName = r.Replace(conf.NameFormat)
	}
	email.FromAddress = conf.FromAddress
}

func (email *Email) Send(ch chan *gomail.Message) {
	m := gomail.NewMessage()
	m.SetAddressHeader("From", email.FromAddress, email.FromName)
	m.SetHeader("To", email.Recipient)
	m.SetHeader("Subject", email.Subject)
	m.SetDateHeader("Date", email.Date)
	m.SetDateHeader("X-Date", time.Now())
	m.SetHeader("X-RSS-URI", email.ItemURI)
	bodyPlain, err := html2text.FromString(email.Body)
	if err != nil {
		fmt.Printf("Failed to parse text as HTML: %v", email.Subject)
		m.SetBody("text/html", email.Body)
	} else {
		m.SetBody("text/plain", bodyPlain)
	}
	ch <- m
}

func createEmail(feedName string, feedTitle string, item *gofeed.Item, date time.Time, account config.AccountConfig, conf *config.GrueConfig) *Email {
	email := new(Email)
	email.setFrom(feedName, feedTitle, account, conf)
	email.Recipient = conf.Recipient
	email.Subject = item.Title
	email.Date = date
	email.ItemURI = item.Link
	email.Body = item.Description
	return email
}

func startDialing(messages chan *gomail.Message, ret chan error, conf *config.GrueConfig) {
	var d *gomail.Dialer
	var hostname string
	var port int
	var err error
	if conf.SmtpServer != nil {
		parts := strings.Split(*conf.SmtpServer, ":")
		if len(parts) > 2 {
			ret <- fmt.Errorf("%s not a valid hostname\n", *conf.SmtpServer)
			return
		} else if len(parts) == 1 {
			hostname = parts[0]
			port = 587
		} else {
			hostname = parts[0]
			port, err = strconv.Atoi(parts[1])
			if err != nil {
				ret <- fmt.Errorf("Failed to parse port: %v\n", err)
				return
			}
		}
	} else {
		hostname = "localhost"
		port = 25
	}
	if conf.SmtpUser != nil {
		d = gomail.NewDialer(hostname, port, *conf.SmtpUser, *conf.SmtpPass)
	} else {
		d = gomail.NewDialer(hostname, port, "", "")
	}

	s, err := d.Dial()
	if err != nil {
		ret <- err
		return
	}
	for m := range messages {
		err = gomail.Send(s, m)
		if err != nil {
			ret <- err
		}
	}

	ret <- s.Close()
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package protokube

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"strings"

	cinderv2 "github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach"
	"k8s.io/klog"
	"k8s.io/kops/protokube/pkg/etcd"
	"k8s.io/kops/protokube/pkg/gossip"
	gossipos "k8s.io/kops/protokube/pkg/gossip/openstack"
	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
)

const MetadataLatest string = "http://169.254.169.254/openstack/latest/meta_data.json"

type Metadata struct {
	// Matches openstack.TagClusterName
	ClusterName string `json:"KubernetesCluster"`
}

type InstanceMetadata struct {
	Name             string    `json:"name"`
	UserMeta         *Metadata `json:"meta"`
	ProjectID        string    `json:"project_id"`
	AvailabilityZone string    `json:"availability_zone"`
	Hostname         string    `json:"hostname"`
	ServerID         string    `json:"uuid"`
}

// GCEVolumes is the Volumes implementation for GCE
type OpenstackVolumes struct {
	cloud openstack.OpenstackCloud

	meta *InstanceMetadata

	clusterName  string
	project      string
	instanceName string
	internalIP   net.IP
	storageZone  string
}

var _ Volumes = &OpenstackVolumes{}

func getLocalMetadata() (*InstanceMetadata, error) {
	var meta InstanceMetadata
	var client http.Client
	resp, err := client.Get(MetadataLatest)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusOK {
		bodyBytes, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
		err = json.Unmarshal(bodyBytes, &meta)
		if err != nil {
			return nil, err
		}
		return &meta, nil
	}
	return nil, err
}

// NewOpenstackVolumes builds a OpenstackVolume
func NewOpenstackVolumes() (*OpenstackVolumes, error) {

	metadata, err := getLocalMetadata()
	if err != nil {
		return nil, fmt.Errorf("Failed to get server metadata: %v", err)
	}

	tags := make(map[string]string)
	// Cluster name needed to bypass missing designate options
	tags[openstack.TagClusterName] = metadata.UserMeta.ClusterName

	oscloud, err := openstack.NewOpenstackCloud(tags, nil)
	if err != nil {
		return nil, fmt.Errorf("Failed to initialize OpenstackVolumes: %v", err)
	}

	a := &OpenstackVolumes{
		cloud: oscloud,
		meta:  metadata,
	}

	err = a.discoverTags()
	if err != nil {
		return nil, err
	}

	return a, nil
}

// ClusterID implements Volumes ClusterID
func (a *OpenstackVolumes) ClusterID() string {
	return a.meta.UserMeta.ClusterName
}

// Project returns the current GCE project
func (a *OpenstackVolumes) Project() string {
	return a.meta.ProjectID
}

// InternalIP implements Volumes InternalIP
func (a *OpenstackVolumes) InternalIP() net.IP {
	return a.internalIP
}

func (a *OpenstackVolumes) discoverTags() error {

	// Cluster Name
	{
		a.clusterName = strings.TrimSpace(string(a.meta.UserMeta.ClusterName))
		if a.clusterName == "" {
			return fmt.Errorf("cluster name metadata was empty")
		}
		klog.Infof("Found cluster name=%q", a.clusterName)
	}

	// Project ID
	{
		a.project = strings.TrimSpace(a.meta.ProjectID)
		if a.project == "" {
			return fmt.Errorf("project metadata was empty")
		}
		klog.Infof("Found project=%q", a.project)
	}

	// Storage Availability Zone
	az, err := a.cloud.GetStorageAZFromCompute(a.meta.AvailabilityZone)
	if err != nil {
		return fmt.Errorf("Could not establish storage availability zone: %v", err)
	}
	a.storageZone = az.ZoneName
	klog.Infof("Found zone=%q", a.storageZone)

	// Instance Name
	{
		a.instanceName = strings.TrimSpace(a.meta.Name)
		if a.instanceName == "" {
			return fmt.Errorf("instance name metadata was empty")
		}
		klog.Infof("Found instanceName=%q", a.instanceName)
	}

	// Internal IP
	{
		server, err := a.cloud.GetInstance(strings.TrimSpace(a.meta.ServerID))
		ip, err := openstack.GetServerFixedIP(server, a.clusterName)
		if err != nil {
			return fmt.Errorf("error querying InternalIP from name: %v", err)
		}
		a.internalIP = net.ParseIP(ip)
		klog.Infof("Found internalIP=%q", a.internalIP)
	}

	return nil
}

func (v *OpenstackVolumes) buildOpenstackVolume(d *cinderv2.Volume) (*Volume, error) {
	volumeName := d.Name
	vol := &Volume{
		ID: d.ID,
		Info: VolumeInfo{
			Description: volumeName,
		},
	}

	vol.Status = d.Status

	for _, attachedTo := range d.Attachments {
		vol.AttachedTo = attachedTo.HostName
		if attachedTo.ServerID == v.meta.ServerID {
			vol.LocalDevice = attachedTo.Device
		}
	}

	// FIXME: Zone matters, broken in my env

	for k, v := range d.Metadata {
		if strings.HasPrefix(k, openstack.TagNameEtcdClusterPrefix) {
			etcdClusterName := k[len(openstack.TagNameEtcdClusterPrefix):]
			spec, err := etcd.ParseEtcdClusterSpec(etcdClusterName, v)
			if err != nil {
				return nil, fmt.Errorf("error parsing etcd cluster meta %q on volume %q: %v", v, d.Name, err)
			}
			vol.Info.EtcdClusters = append(vol.Info.EtcdClusters, spec)
		}
	}

	return vol, nil
}

func (v *OpenstackVolumes) FindVolumes() ([]*Volume, error) {
	var volumes []*Volume

	klog.V(2).Infof("Listing Openstack disks in %s/%s", v.project, v.meta.AvailabilityZone)

	vols, err := v.cloud.ListVolumes(cinderv2.ListOpts{
		TenantID: v.project,
	})
	if err != nil {
		return volumes, fmt.Errorf("FindVolumes: Failed to list volume.")
	}

	for _, volume := range vols {
		if clusterName, ok := volume.Metadata[openstack.TagClusterName]; ok && clusterName == v.clusterName {
			if _, isMasterRole := volume.Metadata[openstack.TagNameRolePrefix+"master"]; isMasterRole {
				vol, err := v.buildOpenstackVolume(&volume)
				if err != nil {
					klog.Errorf("FindVolumes: Failed to build openstack volume %s: %v", volume.Name, err)
					continue
				}
				volumes = append(volumes, vol)
			}
		}
	}

	return volumes, nil
}

// FindMountedVolume implements Volumes::FindMountedVolume
func (v *OpenstackVolumes) FindMountedVolume(volume *Volume) (string, error) {
	device := volume.LocalDevice

	_, err := os.Stat(pathFor(device))
	if err == nil {
		return device, nil
	}
	if os.IsNotExist(err) {
		return "", nil
	}
	return "", fmt.Errorf("error checking for device %q: %v", device, err)
}

// AttachVolume attaches the specified volume to this instance, returning the mountpoint & nil if successful
func (v *OpenstackVolumes) AttachVolume(volume *Volume) error {
	opts := volumeattach.CreateOpts{
		VolumeID: volume.ID,
	}
	attachment, err := v.cloud.AttachVolume(v.meta.ServerID, opts)
	if err != nil {
		return fmt.Errorf("AttachVolume: failed to attach volume: %s", err)
	}
	volume.LocalDevice = attachment.Device
	return nil
}

func (g *OpenstackVolumes) GossipSeeds() (gossip.SeedProvider, error) {
	return gossipos.NewSeedProvider(g.cloud.ComputeClient(), g.clusterName, g.project)
}

func (g *OpenstackVolumes) InstanceName() string {
	return g.instanceName
}
<commit_msg>Check for error when describing openstack instances<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package protokube

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"strings"

	cinderv2 "github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach"
	"k8s.io/klog"
	"k8s.io/kops/protokube/pkg/etcd"
	"k8s.io/kops/protokube/pkg/gossip"
	gossipos "k8s.io/kops/protokube/pkg/gossip/openstack"
	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
)

const MetadataLatest string = "http://169.254.169.254/openstack/latest/meta_data.json"

type Metadata struct {
	// Matches openstack.TagClusterName
	ClusterName string `json:"KubernetesCluster"`
}

type InstanceMetadata struct {
	Name             string    `json:"name"`
	UserMeta         *Metadata `json:"meta"`
	ProjectID        string    `json:"project_id"`
	AvailabilityZone string    `json:"availability_zone"`
	Hostname         string    `json:"hostname"`
	ServerID         string    `json:"uuid"`
}

// GCEVolumes is the Volumes implementation for GCE
type OpenstackVolumes struct {
	cloud openstack.OpenstackCloud

	meta *InstanceMetadata

	clusterName  string
	project      string
	instanceName string
	internalIP   net.IP
	storageZone  string
}

var _ Volumes = &OpenstackVolumes{}

func getLocalMetadata() (*InstanceMetadata, error) {
	var meta InstanceMetadata
	var client http.Client
	resp, err := client.Get(MetadataLatest)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusOK {
		bodyBytes, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
		err = json.Unmarshal(bodyBytes, &meta)
		if err != nil {
			return nil, err
		}
		return &meta, nil
	}
	return nil, err
}

// NewOpenstackVolumes builds a OpenstackVolume
func NewOpenstackVolumes() (*OpenstackVolumes, error) {

	metadata, err := getLocalMetadata()
	if err != nil {
		return nil, fmt.Errorf("Failed to get server metadata: %v", err)
	}

	tags := make(map[string]string)
	// Cluster name needed to bypass missing designate options
	tags[openstack.TagClusterName] = metadata.UserMeta.ClusterName

	oscloud, err := openstack.NewOpenstackCloud(tags, nil)
	if err != nil {
		return nil, fmt.Errorf("Failed to initialize OpenstackVolumes: %v", err)
	}

	a := &OpenstackVolumes{
		cloud: oscloud,
		meta:  metadata,
	}

	err = a.discoverTags()
	if err != nil {
		return nil, err
	}

	return a, nil
}

// ClusterID implements Volumes ClusterID
func (a *OpenstackVolumes) ClusterID() string {
	return a.meta.UserMeta.ClusterName
}

// Project returns the current GCE project
func (a *OpenstackVolumes) Project() string {
	return a.meta.ProjectID
}

// InternalIP implements Volumes InternalIP
func (a *OpenstackVolumes) InternalIP() net.IP {
	return a.internalIP
}

func (a *OpenstackVolumes) discoverTags() error {

	// Cluster Name
	{
		a.clusterName = strings.TrimSpace(string(a.meta.UserMeta.ClusterName))
		if a.clusterName == "" {
			return fmt.Errorf("cluster name metadata was empty")
		}
		klog.Infof("Found cluster name=%q", a.clusterName)
	}

	// Project ID
	{
		a.project = strings.TrimSpace(a.meta.ProjectID)
		if a.project == "" {
			return fmt.Errorf("project metadata was empty")
		}
		klog.Infof("Found project=%q", a.project)
	}

	// Storage Availability Zone
	az, err := a.cloud.GetStorageAZFromCompute(a.meta.AvailabilityZone)
	if err != nil {
		return fmt.Errorf("Could not establish storage availability zone: %v", err)
	}
	a.storageZone = az.ZoneName
	klog.Infof("Found zone=%q", a.storageZone)

	// Instance Name
	{
		a.instanceName = strings.TrimSpace(a.meta.Name)
		if a.instanceName == "" {
			return fmt.Errorf("instance name metadata was empty")
		}
		klog.Infof("Found instanceName=%q", a.instanceName)
	}

	// Internal IP
	{
		server, err := a.cloud.GetInstance(strings.TrimSpace(a.meta.ServerID))
		if err != nil {
			return fmt.Errorf("error getting instance from ID: %v", err)
		}
		ip, err := openstack.GetServerFixedIP(server, a.clusterName)
		if err != nil {
			return fmt.Errorf("error querying InternalIP from name: %v", err)
		}
		a.internalIP = net.ParseIP(ip)
		klog.Infof("Found internalIP=%q", a.internalIP)
	}

	return nil
}

func (v *OpenstackVolumes) buildOpenstackVolume(d *cinderv2.Volume) (*Volume, error) {
	volumeName := d.Name
	vol := &Volume{
		ID: d.ID,
		Info: VolumeInfo{
			Description: volumeName,
		},
	}

	vol.Status = d.Status

	for _, attachedTo := range d.Attachments {
		vol.AttachedTo = attachedTo.HostName
		if attachedTo.ServerID == v.meta.ServerID {
			vol.LocalDevice = attachedTo.Device
		}
	}

	// FIXME: Zone matters, broken in my env

	for k, v := range d.Metadata {
		if strings.HasPrefix(k, openstack.TagNameEtcdClusterPrefix) {
			etcdClusterName := k[len(openstack.TagNameEtcdClusterPrefix):]
			spec, err := etcd.ParseEtcdClusterSpec(etcdClusterName, v)
			if err != nil {
				return nil, fmt.Errorf("error parsing etcd cluster meta %q on volume %q: %v", v, d.Name, err)
			}
			vol.Info.EtcdClusters = append(vol.Info.EtcdClusters, spec)
		}
	}

	return vol, nil
}

func (v *OpenstackVolumes) FindVolumes() ([]*Volume, error) {
	var volumes []*Volume

	klog.V(2).Infof("Listing Openstack disks in %s/%s", v.project, v.meta.AvailabilityZone)

	vols, err := v.cloud.ListVolumes(cinderv2.ListOpts{
		TenantID: v.project,
	})
	if err != nil {
		return volumes, fmt.Errorf("FindVolumes: Failed to list volume.")
	}

	for _, volume := range vols {
		if clusterName, ok := volume.Metadata[openstack.TagClusterName]; ok && clusterName == v.clusterName {
			if _, isMasterRole := volume.Metadata[openstack.TagNameRolePrefix+"master"]; isMasterRole {
				vol, err := v.buildOpenstackVolume(&volume)
				if err != nil {
					klog.Errorf("FindVolumes: Failed to build openstack volume %s: %v", volume.Name, err)
					continue
				}
				volumes = append(volumes, vol)
			}
		}
	}

	return volumes, nil
}

// FindMountedVolume implements Volumes::FindMountedVolume
func (v *OpenstackVolumes) FindMountedVolume(volume *Volume) (string, error) {
	device := volume.LocalDevice

	_, err := os.Stat(pathFor(device))
	if err == nil {
		return device, nil
	}
	if os.IsNotExist(err) {
		return "", nil
	}
	return "", fmt.Errorf("error checking for device %q: %v", device, err)
}

// AttachVolume attaches the specified volume to this instance, returning the mountpoint & nil if successful
func (v *OpenstackVolumes) AttachVolume(volume *Volume) error {
	opts := volumeattach.CreateOpts{
		VolumeID: volume.ID,
	}
	attachment, err := v.cloud.AttachVolume(v.meta.ServerID, opts)
	if err != nil {
		return fmt.Errorf("AttachVolume: failed to attach volume: %s", err)
	}
	volume.LocalDevice = attachment.Device
	return nil
}

func (g *OpenstackVolumes) GossipSeeds() (gossip.SeedProvider, error) {
	return gossipos.NewSeedProvider(g.cloud.ComputeClient(), g.clusterName, g.project)
}

func (g *OpenstackVolumes) InstanceName() string {
	return g.instanceName
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package gateway

import (
	"errors"
	"fmt"
	"github.com/thethingsnetwork/core/lorawan/semtech"
	"net"
	"strings"
	"time"
)

const (
	LISTENER_BUF_SIZE = 1024
	LISTENER_TIMEOUT  = 4 * time.Second
)

type Forwarder interface {
	Forward(packet semtech.Packet) error
	Start() (<-chan semtech.Packet, <-chan error, error)
	Stop() error
}

// Start a Gateway. This will create several udp connections - one per associated router.
// Then, incoming streams of packets are merged into a single one.
// Same for errors.
func (g *Gateway) Start() (<-chan semtech.Packet, <-chan error, error) {
	// Ensure not already started
	if g.quit != nil {
		return nil, nil, errors.New("Try to start a started gateway")
	}

	// Open all UDP connections
	connections := make([]*net.UDPConn, 0)
	var err error
	for _, addr := range g.routers {
		var conn *net.UDPConn
		conn, err = net.ListenUDP("udp", addr)
		if err != nil {
			break
		}
		connections = append(connections, conn)
	}

	// On error, close all opened UDP connection and leave
	if err != nil {
		for _, conn := range connections {
			conn.Close()
		}
		return nil, nil, err
	}

	// Create communication channels and launch goroutines to handle connections
	chout := make(chan semtech.Packet)
	cherr := make(chan error)
	quit := make(chan chan error)
	cmd := make(chan command)
	go reduceCmd(g, cmd)
	for _, conn := range connections {
		go listen(conn, chout, cherr, cmd, quit)
	}

	// Keep a reference to the quit channel, and return the others
	g.quit = quit
	g.cmd = cmd
	return chout, cherr, nil
}

// listen materialize the goroutine handling incoming packet from routers
func listen(conn *net.UDPConn, chout chan<- semtech.Packet, cherr chan<- error, cmd chan<- command, quit <-chan chan error) {
	connIn, connErr := asChannel(conn)
	errBuf := make([]error, 0)
	outBuf := make([]semtech.Packet, 0)
	for {
		var safeChout chan<- semtech.Packet
		var packet semtech.Packet
		if len(outBuf) > 0 {
			safeChout = chout
			packet = outBuf[0]
		}

		var safeCherr chan<- error
		var err error
		if len(errBuf) > 0 {
			safeCherr = cherr
			err = errBuf[0]
		}

		select {
		case ack := <-quit: // quit event, the gateway is stoppped
			e := conn.Close()
			ack <- e
			return
		case buf := <-connIn: // connIn event, a packet has been received by the listener goroutine
			cmd <- cmd_RECD_PACKET
			packet, err := semtech.Unmarshal(buf)
			if err != nil {
				errBuf = append(errBuf, err)
				continue
			}
			outBuf = append(outBuf, *packet)
		case safeChout <- packet: // emit an available packet to chout
			outBuf = outBuf[1:]
		case err := <-connErr: // connErr event, an error has been triggered by the listener goroutine
			errBuf = append(errBuf, err)
		case safeCherr <- err: // emit an existing error to cherr
			errBuf = errBuf[1:]
		}
	}
}

// as channel is actually a []byte generator that listen to a given udp connection.
// It is used to prevent the listen() function from blocking on ReadFromUDP() and then,
// still be available for a quit event that could come at any time.
// This function is thereby nothing more than a mapping of incoming connection to channels of
// communication.
func asChannel(conn *net.UDPConn) (<-chan []byte, <-chan error) {
	cherr := make(chan error)
	chout := make(chan []byte)
	go func() {
		buf := make([]byte, LISTENER_BUF_SIZE)
		for {
			n, _, err := conn.ReadFromUDP(buf)
			if err != nil {
				if strings.Contains(err.Error(), "use of closed network connection") {
					return
				}
				select {
				case cherr <- err:
					continue
				case <-time.After(LISTENER_TIMEOUT):
					close(cherr)
					close(chout)
					return
				}
			}
			select {
			case chout <- buf[:n]:
			case <-time.After(LISTENER_TIMEOUT):
				close(cherr)
				close(chout)
				return
			}
		}
	}()
	return chout, cherr
}

// reduceCmd handle all updates made on the gateway statistics
func reduceCmd(gateway *Gateway, commands <-chan command) {
	for command := range commands {
		switch command {
		case cmd_ACKN_PACKET:
			gateway.ackr += 1
		case cmd_EMIT_PACKET:
			gateway.txnb += 1
		case cmd_FORW_PACKET:
			gateway.rxfw += 1
		case cmd_RECU_PACKET:
			gateway.rxnb += 1
		case cmd_RECD_PACKET:
			gateway.dwnb += 1
		}
	}
}

// Stop remove all previously created connection.
func (g *Gateway) Stop() error {
	if g.quit == nil || g.cmd == nil {
		return errors.New("Try to stop a non-started gateway")
	}

	for range g.routers {
		errc := make(chan error)
		g.quit <- errc
		err := <-errc
		if err != nil {
			fmt.Printf("%+v\n", err)
		}
	}
	close(g.quit)
	close(g.cmd)
	g.routers = make([]*net.UDPAddr, 0)
	g.quit = nil
	return nil
}

// Forward transfer a packet to all known routers.
// It fails if the gateway hasn't been started beforehand.
func (g *Gateway) Forward(packet semtech.Packet) error {
	if g.quit == nil || g.cmd == nil {
		return errors.New("Unable to forward on a non-started gateway")
	}

	g.cmd <- cmd_RECU_PACKET

	connections := make([]*net.UDPConn, 0)
	var err error
	for _, addr := range g.routers {
		var conn *net.UDPConn
		conn, err = net.DialUDP("udp", nil, addr)
		if err != nil {
			break
		}
		defer conn.Close()
		connections = append(connections, conn)
	}
	raw, err := semtech.Marshal(packet)

	if err != nil {
		return errors.New(fmt.Sprintf("Unable to forward the packet. %v\n", err))
	}

	g.cmd <- cmd_FORW_PACKET

	for _, conn := range connections {
		_, err = conn.Write(raw)
		g.cmd <- cmd_EMIT_PACKET
	}

	if err != nil {
		return errors.New(fmt.Sprintf("Something went wrong during forwarding. %v\n", err))
	}

	return nil
}
<commit_msg>[simulators.gateway] Add comment about concurrent accesses in forwarder<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package gateway

import (
	"errors"
	"fmt"
	"github.com/thethingsnetwork/core/lorawan/semtech"
	"net"
	"strings"
	"time"
)

const (
	LISTENER_BUF_SIZE = 1024
	LISTENER_TIMEOUT  = 4 * time.Second
)

type Forwarder interface {
	Forward(packet semtech.Packet) error
	Start() (<-chan semtech.Packet, <-chan error, error)
	Stop() error
}

// Start a Gateway. This will create several udp connections - one per associated router.
// Then, incoming streams of packets are merged into a single one.
// Same for errors.
func (g *Gateway) Start() (<-chan semtech.Packet, <-chan error, error) {
	// Ensure not already started
	if g.quit != nil {
		return nil, nil, errors.New("Try to start a started gateway")
	}

	// Open all UDP connections
	connections := make([]*net.UDPConn, 0)
	var err error
	for _, addr := range g.routers {
		var conn *net.UDPConn
		conn, err = net.ListenUDP("udp", addr)
		if err != nil {
			break
		}
		connections = append(connections, conn)
	}

	// On error, close all opened UDP connection and leave
	if err != nil {
		for _, conn := range connections {
			conn.Close()
		}
		return nil, nil, err
	}

	// Create communication channels and launch goroutines to handle connections
	chout := make(chan semtech.Packet)
	cherr := make(chan error)
	quit := make(chan chan error)
	cmd := make(chan command)
	go reduceCmd(g, cmd)
	for _, conn := range connections {
		go listen(conn, chout, cherr, cmd, quit)
	}

	// Keep a reference to the quit channel, and return the others
	g.quit = quit
	g.cmd = cmd
	return chout, cherr, nil
}

// listen materialize the goroutine handling incoming packet from routers
func listen(conn *net.UDPConn, chout chan<- semtech.Packet, cherr chan<- error, cmd chan<- command, quit <-chan chan error) {
	connIn, connErr := asChannel(conn)
	errBuf := make([]error, 0)
	outBuf := make([]semtech.Packet, 0)
	for {
		var safeChout chan<- semtech.Packet
		var packet semtech.Packet
		if len(outBuf) > 0 {
			safeChout = chout
			packet = outBuf[0]
		}

		var safeCherr chan<- error
		var err error
		if len(errBuf) > 0 {
			safeCherr = cherr
			err = errBuf[0]
		}

		select {
		case ack := <-quit: // quit event, the gateway is stoppped
			e := conn.Close()
			ack <- e
			return
		case buf := <-connIn: // connIn event, a packet has been received by the listener goroutine
			cmd <- cmd_RECD_PACKET
			packet, err := semtech.Unmarshal(buf)
			if err != nil {
				errBuf = append(errBuf, err)
				continue
			}
			outBuf = append(outBuf, *packet)
		case safeChout <- packet: // emit an available packet to chout
			outBuf = outBuf[1:]
		case err := <-connErr: // connErr event, an error has been triggered by the listener goroutine
			errBuf = append(errBuf, err)
		case safeCherr <- err: // emit an existing error to cherr
			errBuf = errBuf[1:]
		}
	}
}

// as channel is actually a []byte generator that listen to a given udp connection.
// It is used to prevent the listen() function from blocking on ReadFromUDP() and then,
// still be available for a quit event that could come at any time.
// This function is thereby nothing more than a mapping of incoming connection to channels of
// communication.
func asChannel(conn *net.UDPConn) (<-chan []byte, <-chan error) {
	cherr := make(chan error)
	chout := make(chan []byte)
	go func() {
		buf := make([]byte, LISTENER_BUF_SIZE)
		for {
			n, _, err := conn.ReadFromUDP(buf)
			if err != nil {
				if strings.Contains(err.Error(), "use of closed network connection") {
					return
				}
				select {
				case cherr <- err:
					continue
				case <-time.After(LISTENER_TIMEOUT):
					close(cherr)
					close(chout)
					return
				}
			}
			select {
			case chout <- buf[:n]:
			case <-time.After(LISTENER_TIMEOUT):
				close(cherr)
				close(chout)
				return
			}
		}
	}()
	return chout, cherr
}

// reduceCmd handle all updates made on the gateway statistics
func reduceCmd(gateway *Gateway, commands <-chan command) {
	//TODO beware of concurrent access on gateway properties.
	for command := range commands {
		switch command {
		case cmd_ACKN_PACKET:
			gateway.ackr += 1
		case cmd_EMIT_PACKET:
			gateway.txnb += 1
		case cmd_FORW_PACKET:
			gateway.rxfw += 1
		case cmd_RECU_PACKET:
			gateway.rxnb += 1
		case cmd_RECD_PACKET:
			gateway.dwnb += 1
		}
	}
}

// Stop remove all previously created connection.
func (g *Gateway) Stop() error {
	if g.quit == nil || g.cmd == nil {
		return errors.New("Try to stop a non-started gateway")
	}

	for range g.routers {
		errc := make(chan error)
		g.quit <- errc
		err := <-errc
		if err != nil {
			fmt.Printf("%+v\n", err)
		}
	}
	close(g.quit)
	close(g.cmd)
	g.routers = make([]*net.UDPAddr, 0)
	g.quit = nil
	return nil
}

// Forward transfer a packet to all known routers.
// It fails if the gateway hasn't been started beforehand.
func (g *Gateway) Forward(packet semtech.Packet) error {
	if g.quit == nil || g.cmd == nil {
		return errors.New("Unable to forward on a non-started gateway")
	}

	g.cmd <- cmd_RECU_PACKET

	connections := make([]*net.UDPConn, 0)
	var err error
	for _, addr := range g.routers {
		var conn *net.UDPConn
		conn, err = net.DialUDP("udp", nil, addr)
		if err != nil {
			break
		}
		defer conn.Close()
		connections = append(connections, conn)
	}
	raw, err := semtech.Marshal(packet)

	if err != nil {
		return errors.New(fmt.Sprintf("Unable to forward the packet. %v\n", err))
	}

	g.cmd <- cmd_FORW_PACKET

	for _, conn := range connections {
		_, err = conn.Write(raw)
		g.cmd <- cmd_EMIT_PACKET
	}

	if err != nil {
		return errors.New(fmt.Sprintf("Something went wrong during forwarding. %v\n", err))
	}

	return nil
}
<|endoftext|>