<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"k8s.io/klog/v2"

	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

// AppendFunc is used to add a matching item to whatever list the caller is using
type AppendFunc func(interface{})

// ListAll calls appendFn with each value retrieved from store which matches the selector.
func ListAll(store Store, selector labels.Selector, appendFn AppendFunc) error {
	selectAll := selector.Empty()
	for _, m := range store.List() {
		if selectAll {
			// Avoid computing labels of the objects to speed up common flows
			// of listing all objects.
			appendFn(m)
			continue
		}
		metadata, err := meta.Accessor(m)
		if err != nil {
			return err
		}
		if selector.Matches(labels.Set(metadata.GetLabels())) {
			appendFn(m)
		}
	}
	return nil
}

// ListAllByNamespace used to list items belongs to namespace from Indexer.
func ListAllByNamespace(indexer Indexer, namespace string, selector labels.Selector, appendFn AppendFunc) error {
	selectAll := selector.Empty()
	if namespace == metav1.NamespaceAll {
		for _, m := range indexer.List() {
			if selectAll {
				// Avoid computing labels of the objects to speed up common flows
				// of listing all objects.
				appendFn(m)
				continue
			}
			metadata, err := meta.Accessor(m)
			if err != nil {
				return err
			}
			if selector.Matches(labels.Set(metadata.GetLabels())) {
				appendFn(m)
			}
		}
		return nil
	}

	items, err := indexer.Index(NamespaceIndex, &metav1.ObjectMeta{Namespace: namespace})
	if err != nil {
		// Ignore error; do slow search without index.
		klog.Warningf("can not retrieve list of objects using index : %v", err)
		for _, m := range indexer.List() {
			metadata, err := meta.Accessor(m)
			if err != nil {
				return err
			}
			if metadata.GetNamespace() == namespace && selector.Matches(labels.Set(metadata.GetLabels())) {
				appendFn(m)
			}

		}
		return nil
	}
	for _, m := range items {
		if selectAll {
			// Avoid computing labels of the objects to speed up common flows
			// of listing all objects.
			appendFn(m)
			continue
		}
		metadata, err := meta.Accessor(m)
		if err != nil {
			return err
		}
		if selector.Matches(labels.Set(metadata.GetLabels())) {
			appendFn(m)
		}
	}

	return nil
}

// GenericLister is a lister skin on a generic Indexer
type GenericLister interface {
	// List will return all objects across namespaces
	List(selector labels.Selector) (ret []runtime.Object, err error)
	// Get will attempt to retrieve assuming that name==key
	Get(name string) (runtime.Object, error)
	// ByNamespace will give you a GenericNamespaceLister for one namespace
	ByNamespace(namespace string) GenericNamespaceLister
}

// GenericNamespaceLister is a lister skin on a generic Indexer
type GenericNamespaceLister interface {
	// List will return all objects in this namespace
	List(selector labels.Selector) (ret []runtime.Object, err error)
	// Get will attempt to retrieve by namespace and name
	Get(name string) (runtime.Object, error)
}

// NewGenericLister creates a new instance for the genericLister.
func NewGenericLister(indexer Indexer, resource schema.GroupResource) GenericLister {
	return &genericLister{indexer: indexer, resource: resource}
}

type genericLister struct {
	indexer  Indexer
	resource schema.GroupResource
}

func (s *genericLister) List(selector labels.Selector) (ret []runtime.Object, err error) {
	err = ListAll(s.indexer, selector, func(m interface{}) {
		ret = append(ret, m.(runtime.Object))
	})
	return ret, err
}

func (s *genericLister) ByNamespace(namespace string) GenericNamespaceLister {
	return &genericNamespaceLister{indexer: s.indexer, namespace: namespace, resource: s.resource}
}

func (s *genericLister) Get(name string) (runtime.Object, error) {
	obj, exists, err := s.indexer.GetByKey(name)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, errors.NewNotFound(s.resource, name)
	}
	return obj.(runtime.Object), nil
}

type genericNamespaceLister struct {
	indexer   Indexer
	namespace string
	resource  schema.GroupResource
}

func (s *genericNamespaceLister) List(selector labels.Selector) (ret []runtime.Object, err error) {
	err = ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) {
		ret = append(ret, m.(runtime.Object))
	})
	return ret, err
}

func (s *genericNamespaceLister) Get(name string) (runtime.Object, error) {
	obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, errors.NewNotFound(s.resource, name)
	}
	return obj.(runtime.Object), nil
}
<commit_msg>Fix duplicate code block of ListAll function<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"k8s.io/klog/v2"

	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

// AppendFunc is used to add a matching item to whatever list the caller is using
type AppendFunc func(interface{})

// ListAll calls appendFn with each value retrieved from store which matches the selector.
func ListAll(store Store, selector labels.Selector, appendFn AppendFunc) error {
	selectAll := selector.Empty()
	for _, m := range store.List() {
		if selectAll {
			// Avoid computing labels of the objects to speed up common flows
			// of listing all objects.
			appendFn(m)
			continue
		}
		metadata, err := meta.Accessor(m)
		if err != nil {
			return err
		}
		if selector.Matches(labels.Set(metadata.GetLabels())) {
			appendFn(m)
		}
	}
	return nil
}

// ListAllByNamespace used to list items belongs to namespace from Indexer.
func ListAllByNamespace(indexer Indexer, namespace string, selector labels.Selector, appendFn AppendFunc) error {
	if namespace == metav1.NamespaceAll {
		return ListAll(indexer, selector, appendFn)
	}

	items, err := indexer.Index(NamespaceIndex, &metav1.ObjectMeta{Namespace: namespace})
	if err != nil {
		// Ignore error; do slow search without index.
		klog.Warningf("can not retrieve list of objects using index : %v", err)
		for _, m := range indexer.List() {
			metadata, err := meta.Accessor(m)
			if err != nil {
				return err
			}
			if metadata.GetNamespace() == namespace && selector.Matches(labels.Set(metadata.GetLabels())) {
				appendFn(m)
			}

		}
		return nil
	}

	selectAll := selector.Empty()
	for _, m := range items {
		if selectAll {
			// Avoid computing labels of the objects to speed up common flows
			// of listing all objects.
			appendFn(m)
			continue
		}
		metadata, err := meta.Accessor(m)
		if err != nil {
			return err
		}
		if selector.Matches(labels.Set(metadata.GetLabels())) {
			appendFn(m)
		}
	}

	return nil
}

// GenericLister is a lister skin on a generic Indexer
type GenericLister interface {
	// List will return all objects across namespaces
	List(selector labels.Selector) (ret []runtime.Object, err error)
	// Get will attempt to retrieve assuming that name==key
	Get(name string) (runtime.Object, error)
	// ByNamespace will give you a GenericNamespaceLister for one namespace
	ByNamespace(namespace string) GenericNamespaceLister
}

// GenericNamespaceLister is a lister skin on a generic Indexer
type GenericNamespaceLister interface {
	// List will return all objects in this namespace
	List(selector labels.Selector) (ret []runtime.Object, err error)
	// Get will attempt to retrieve by namespace and name
	Get(name string) (runtime.Object, error)
}

// NewGenericLister creates a new instance for the genericLister.
func NewGenericLister(indexer Indexer, resource schema.GroupResource) GenericLister {
	return &genericLister{indexer: indexer, resource: resource}
}

type genericLister struct {
	indexer  Indexer
	resource schema.GroupResource
}

func (s *genericLister) List(selector labels.Selector) (ret []runtime.Object, err error) {
	err = ListAll(s.indexer, selector, func(m interface{}) {
		ret = append(ret, m.(runtime.Object))
	})
	return ret, err
}

func (s *genericLister) ByNamespace(namespace string) GenericNamespaceLister {
	return &genericNamespaceLister{indexer: s.indexer, namespace: namespace, resource: s.resource}
}

func (s *genericLister) Get(name string) (runtime.Object, error) {
	obj, exists, err := s.indexer.GetByKey(name)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, errors.NewNotFound(s.resource, name)
	}
	return obj.(runtime.Object), nil
}

type genericNamespaceLister struct {
	indexer   Indexer
	namespace string
	resource  schema.GroupResource
}

func (s *genericNamespaceLister) List(selector labels.Selector) (ret []runtime.Object, err error) {
	err = ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) {
		ret = append(ret, m.(runtime.Object))
	})
	return ret, err
}

func (s *genericNamespaceLister) Get(name string) (runtime.Object, error) {
	obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, errors.NewNotFound(s.resource, name)
	}
	return obj.(runtime.Object), nil
}
<|endoftext|>


<commit_before>package admin

import (
	"strconv"

	"github.com/13pinj/todoapp/Godeps/_workspace/src/github.com/gin-gonic/gin"
	"github.com/13pinj/todoapp/controllers"
	"github.com/13pinj/todoapp/models/user"
)

const (
	usersPerPage = 15
)

// Index возвращает главную страницу админской панели.
// GET /admin
func Index(c *gin.Context) {
	if !assertAccess(c) {
		return
	}
	page, err := strconv.Atoi(c.Query("p"))
	if err != nil {
		page = 1
	}
	pcount := user.Pages(usersPerPage)
	ctl.RenderHTML(c, "admin_index.tmpl", gin.H{
		"Users": user.FindPage(page, usersPerPage, user.ByVisitedAtDesc),
		"Pager": gin.H{
			"Cur":      page,
			"Max":      pcount,
			"PathTmpl": "/admin?p=",
		},
	})
}

// User возвращает страницу с информацией о пользователе.
// GET /admin/u/:name
func User(c *gin.Context) {
	if !assertAccess(c) {
		return
	}
	// Шаблон ожидает заполненую структуру пользователя
	ctl.RenderHTML(c, "admin_user.tmpl", gin.H{
		"User": &user.User{},
	})
}

// UserUpdate обновляет информацию о пользователе.
// POST /admin/u/:name
func UserUpdate(c *gin.Context) {

}

// UserDestroy стирает пользователя из базы.
// POST /admin/u/:name/destroy
func UserDestroy(c *gin.Context) {

}

func assertAccess(c *gin.Context) bool {
	u, ok := user.FromContext(c)
	if !ok || !u.Admin() {
		ctl.Render403(c)
		return false
	}
	return true
}
<commit_msg>Done admin.User<commit_after>package admin

import (
	"strconv"

	"github.com/13pinj/todoapp/Godeps/_workspace/src/github.com/gin-gonic/gin"
	"github.com/13pinj/todoapp/controllers"
	"github.com/13pinj/todoapp/models/user"
)

const (
	usersPerPage = 15
)

// Index возвращает главную страницу админской панели.
// GET /admin
func Index(c *gin.Context) {
	if !assertAccess(c) {
		return
	}
	page, err := strconv.Atoi(c.Query("p"))
	if err != nil {
		page = 1
	}
	pcount := user.Pages(usersPerPage)
	ctl.RenderHTML(c, "admin_index.tmpl", gin.H{
		"Users": user.FindPage(page, usersPerPage, user.ByVisitedAtDesc),
		"Pager": gin.H{
			"Cur":      page,
			"Max":      pcount,
			"PathTmpl": "/admin?p=",
		},
	})
}

// User возвращает страницу с информацией о пользователе.
// GET /admin/u/:name
func User(c *gin.Context) {
	if !assertAccess(c) {
		return
	}
	u, ok := user.Find(c.Param("name"))
	if !ok {
		ctl.Render404(c)
		return
	}
	u.LoadLists()
	for _, i := range u.Lists {
		i.LoadTodos()
	}
	// Шаблон ожидает заполненую структуру пользователя
	ctl.RenderHTML(c, "admin_user.tmpl", gin.H{
		"User": u,
	})
}

// UserUpdate обновляет информацию о пользователе.
// POST /admin/u/:name
func UserUpdate(c *gin.Context) {

}

// UserDestroy стирает пользователя из базы.
// POST /admin/u/:name/destroy
func UserDestroy(c *gin.Context) {

}

func assertAccess(c *gin.Context) bool {
	u, ok := user.FromContext(c)
	if !ok || !u.Admin() {
		ctl.Render403(c)
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package genhandler

import (
	"bytes"
	"strings"
	"text/template"

	pbdescriptor "github.com/golang/protobuf/protoc-gen-go/descriptor"
	"github.com/golang/protobuf/protoc-gen-go/generator"
	"github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/descriptor"
	"github.com/pkg/errors"
)

var (
	errNoTargetService = errors.New("no target service defined in the file")
)

type param struct {
	*descriptor.File
	Imports    []descriptor.GoPackage
	SwagBuffer []byte
}

func applyTemplate(p param) (string, error) {
	// r := &http.Request{}
	// r.URL.Query()
	w := bytes.NewBuffer(nil)
	if err := headerTemplate.Execute(w, p); err != nil {
		return "", err
	}

	if err := regTemplate.ExecuteTemplate(w, "base", p); err != nil {
		return "", err
	}

	type swaggerTmpl struct {
		FileName string
		Swagger  string
	}

	if err := footerTemplate.Execute(w, p); err != nil {
		return "", err
	}
	if err := clientTemplate.Execute(w, p); err != nil {
		return "", err
	}

	if err := patternsTemplate.ExecuteTemplate(w, "base", p); err != nil {
		return "", err
	}

	return w.String(), nil
}

var (
	varNameReplacer = strings.NewReplacer(
		".", "_",
		"/", "_",
		"-", "_",
	)
	funcMap = template.FuncMap{
		"varName":         func(s string) string { return varNameReplacer.Replace(s) },
		"goTypeName":      func(s string) string { return generator.CamelCase(s) },
		"byteStr":         func(b []byte) string { return string(b) },
		"escapeBackTicks": func(s string) string { return strings.Replace(s, "`", "` + \"``\" + `", -1) },
		"toGoType":        func(t pbdescriptor.FieldDescriptorProto_Type) string { return primitiveTypeToGo(t) },
		// arrayToPathInterp replaces chi-style path to fmt.Sprint-style path.
		"arrayToPathInterp": func(tpl string) string {
			vv := strings.Split(tpl, "/")
			ret := []string{}
			for _, v := range vv {
				if strings.HasPrefix(v, "{") {
					ret = append(ret, "%v")
					continue
				}
				ret = append(ret, v)
			}
			return strings.Join(ret, "/")
		},
	}

	headerTemplate = template.Must(template.New("header").Parse(`
// Code generated by protoc-gen-goclay
// source: {{.GetName}}
// DO NOT EDIT!

/*
Package {{.GoPkg.Name}} is a self-registering gRPC and JSON+Swagger service definition.

It conforms to the github.com/utrack/clay Service interface.
*/
package {{.GoPkg.Name}}
import (
	{{range $i := .Imports}}{{if $i.Standard}}{{$i | printf "%s\n"}}{{end}}{{end}}

	{{range $i := .Imports}}{{if not $i.Standard}}{{$i | printf "%s\n"}}{{end}}{{end}}
)

// Update your shared lib or downgrade generator to v1 if there's an error
var _ = transport.IsVersion2

var _ chi.Router
var _ runtime.Marshaler
`))
	regTemplate = template.Must(template.New("svc-reg").Funcs(funcMap).Parse(`
{{define "base"}}
{{range $svc := .Services}}
// {{$svc.GetName}}Desc is a descriptor/registrator for the {{$svc.GetName}}Server.
type {{$svc.GetName}}Desc struct {
      svc {{$svc.GetName}}Server
}

// New{{$svc.GetName}}ServiceDesc creates new registrator for the {{$svc.GetName}}Server.
func New{{$svc.GetName}}ServiceDesc(svc {{$svc.GetName}}Server) *{{$svc.GetName}}Desc {
      return &{{$svc.GetName}}Desc{svc:svc}
}

// RegisterGRPC implements service registrator interface.
func (d *{{$svc.GetName}}Desc) RegisterGRPC(s *grpc.Server) {
      Register{{$svc.GetName}}Server(s,d.svc)
}

// SwaggerDef returns this file's Swagger definition.
func (d *{{$svc.GetName}}Desc) SwaggerDef() []byte {
      return _swaggerDef_{{varName $.GetName}}
}

// RegisterHTTP registers this service's HTTP handlers/bindings.
func (d *{{$svc.GetName}}Desc) RegisterHTTP(mux transport.Router) {
	{{range $m := $svc.Methods}}
	// Handlers for {{$m.GetName}}
	{{range $b := $m.Bindings}}
	mux.MethodFunc("{{$b.HTTPMethod}}",pattern_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}}, func(w http.ResponseWriter, r *http.Request) {
          defer r.Body.Close()

	  var req {{$m.RequestType.GetName}}
          err := unmarshaler_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}}(r,&req)
	  if err != nil {
	    httpruntime.SetError(r.Context(),r,w,errors.Wrap(err,"couldn't parse request"))
	    return
	  }

	  ret,err := d.svc.{{$m.GetName}}(r.Context(),&req)
	  if err != nil {
	    httpruntime.SetError(r.Context(),r,w,errors.Wrap(err,"returned from handler"))
	    return
	  }

          _,outbound := httpruntime.MarshalerForRequest(r)
          w.Header().Set("Content-Type", outbound.ContentType())
	  err = outbound.Marshal(w, ret)
	  if err != nil {
	    httpruntime.SetError(r.Context(),r,w,errors.Wrap(err,"couldn't write response"))
	    return
	  }
      })
      {{end}}
      {{end}}
}
{{end}}
{{end}} // base service handler ended
`))

	footerTemplate = template.Must(template.New("footer").Funcs(funcMap).Parse(`
var _swaggerDef_{{varName .GetName}} = []byte(` + "`" + `{{escapeBackTicks (byteStr .SwagBuffer)}}` + `
` + "`)" + `
`))

	patternsTemplate = template.Must(template.New("patterns").Funcs(funcMap).Parse(`
{{define "base"}}
var (
{{range $svc := .Services}}
{{range $m := $svc.Methods}}
{{range $b := $m.Bindings}}
	pattern_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}} = "{{$b.PathTmpl.Template}}"
	pattern_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}}_builder = func(
{{range $p := $b.PathParams}}{{$p.Target.GetName}} {{toGoType $p.Target.GetType}},
{{end}}) string {
return fmt.Sprintf("{{arrayToPathInterp $b.PathTmpl.Template}}",{{range $p := $b.PathParams}}{{$p.Target.GetName}},{{end}})
}
        unmarshaler_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}} = func(r *http.Request,req *{{$m.RequestType.GetName}}) error {

        var err error
        {{if $b.Body}}
          {{template "unmbody" .}}
        {{end}}
        {{if $b.PathParams}}
          {{template "unmpath" .}}
        {{end}}

        return err
        }
{{end}}
{{end}}
{{end}}
)
{{end}}
{{define "unmbody"}}
          inbound,_ := httpruntime.MarshalerForRequest(r)
	  err = errors.Wrap(inbound.Unmarshal(r.Body,req),"couldn't read request JSON")
          if err != nil {
            return err
          }
{{end}}
{{define "unmpath"}}
	  rctx := chi.RouteContext(r.Context())
          if rctx == nil {
            panic("Only chi router is supported for GETs atm")
	  }
          for pos,k := range rctx.URLParams.Keys {
	    runtime.PopulateFieldFromPath(req, k, rctx.URLParams.Values[pos])
          }
{{end}}
`))
	clientTemplate = template.Must(template.New("http-client").Funcs(funcMap).Parse(`
{{range $svc := .Services}}
type {{$svc.GetName}}_httpClient struct {
c *http.Client
host string
}

// New{{$svc.GetName}}HTTPClient creates new HTTP client for {{$svc.GetName}}Server.
// Pass addr in format "http://host[:port]".
func New{{$svc.GetName}}HTTPClient(c *http.Client,addr string) {{$svc.GetName}}Client {
	if strings.HasSuffix(addr, "/") {
		addr = addr[:len(addr)-1]
	}
        return &{{$svc.GetName}}_httpClient{c:c,host:addr}
}
{{range $m := $svc.Methods}}
{{range $b := $m.Bindings}}
func (c *{{$svc.GetName}}_httpClient) {{$m.GetName}}(ctx context.Context,in *{{$m.RequestType.GetName}},_ ...grpc.CallOption) (*{{$m.ResponseType.GetName}},error) {

        //TODO path params aren't supported atm
        path := pattern_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}}_builder({{range $p := $b.PathParams}}in.{{goTypeName $p.String}},{{end}})

	buf := bytes.NewBuffer(nil)

	m := httpruntime.DefaultMarshaler(nil)
	err := m.Marshal(buf, in)
	if err != nil {
		return nil, errors.Wrap(err, "can't marshal request")
	}

	req, err := http.NewRequest("{{$b.HTTPMethod}}", c.host+path, buf)
	if err != nil {
		return nil, errors.Wrap(err, "can't initiate HTTP request")
	}

	req.Header.Add("Accept", m.ContentType())

	rsp, err := c.c.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, "error from client")
	}
	defer rsp.Body.Close()

	if rsp.StatusCode>= 400 {
		b,_ := ioutil.ReadAll(rsp.Body)
		return nil,errors.Errorf("%v %v: server returned HTTP %v: '%v'",req.Method,req.URL.String(),rsp.StatusCode,string(b))
	}

	ret := &{{$m.ResponseType.GetName}}{}
	err = m.Unmarshal(rsp.Body, ret)
	return ret, errors.Wrap(err, "can't unmarshal response")
}
{{end}}
{{end}}
{{end}}
`))
)
<commit_msg>fixed HTTP path builder vs dot-delimited paths<commit_after>package genhandler

import (
	"bytes"
	"strings"
	"text/template"

	pbdescriptor "github.com/golang/protobuf/protoc-gen-go/descriptor"
	"github.com/golang/protobuf/protoc-gen-go/generator"
	"github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/descriptor"
	"github.com/pkg/errors"
)

var (
	errNoTargetService = errors.New("no target service defined in the file")
)

type param struct {
	*descriptor.File
	Imports    []descriptor.GoPackage
	SwagBuffer []byte
}

func applyTemplate(p param) (string, error) {
	// r := &http.Request{}
	// r.URL.Query()
	w := bytes.NewBuffer(nil)
	if err := headerTemplate.Execute(w, p); err != nil {
		return "", err
	}

	if err := regTemplate.ExecuteTemplate(w, "base", p); err != nil {
		return "", err
	}

	type swaggerTmpl struct {
		FileName string
		Swagger  string
	}

	if err := footerTemplate.Execute(w, p); err != nil {
		return "", err
	}
	if err := clientTemplate.Execute(w, p); err != nil {
		return "", err
	}

	if err := patternsTemplate.ExecuteTemplate(w, "base", p); err != nil {
		return "", err
	}

	return w.String(), nil
}

var (
	varNameReplacer = strings.NewReplacer(
		".", "_",
		"/", "_",
		"-", "_",
	)
	funcMap = template.FuncMap{
		"varName": func(s string) string { return varNameReplacer.Replace(s) },
		"goTypeName": func(s string) string {
			toks := strings.Split(s, ".")
			for pos := range toks {
				toks[pos] = generator.CamelCase(toks[pos])
			}
			return strings.Join(toks, ".")
		},
		"byteStr":         func(b []byte) string { return string(b) },
		"escapeBackTicks": func(s string) string { return strings.Replace(s, "`", "` + \"``\" + `", -1) },
		"toGoType":        func(t pbdescriptor.FieldDescriptorProto_Type) string { return primitiveTypeToGo(t) },
		// arrayToPathInterp replaces chi-style path to fmt.Sprint-style path.
		"arrayToPathInterp": func(tpl string) string {
			vv := strings.Split(tpl, "/")
			ret := []string{}
			for _, v := range vv {
				if strings.HasPrefix(v, "{") {
					ret = append(ret, "%v")
					continue
				}
				ret = append(ret, v)
			}
			return strings.Join(ret, "/")
		},
	}

	headerTemplate = template.Must(template.New("header").Parse(`
// Code generated by protoc-gen-goclay
// source: {{.GetName}}
// DO NOT EDIT!

/*
Package {{.GoPkg.Name}} is a self-registering gRPC and JSON+Swagger service definition.

It conforms to the github.com/utrack/clay Service interface.
*/
package {{.GoPkg.Name}}
import (
	{{range $i := .Imports}}{{if $i.Standard}}{{$i | printf "%s\n"}}{{end}}{{end}}

	{{range $i := .Imports}}{{if not $i.Standard}}{{$i | printf "%s\n"}}{{end}}{{end}}
)

// Update your shared lib or downgrade generator to v1 if there's an error
var _ = transport.IsVersion2

var _ chi.Router
var _ runtime.Marshaler
`))
	regTemplate = template.Must(template.New("svc-reg").Funcs(funcMap).Parse(`
{{define "base"}}
{{range $svc := .Services}}
// {{$svc.GetName}}Desc is a descriptor/registrator for the {{$svc.GetName}}Server.
type {{$svc.GetName}}Desc struct {
      svc {{$svc.GetName}}Server
}

// New{{$svc.GetName}}ServiceDesc creates new registrator for the {{$svc.GetName}}Server.
func New{{$svc.GetName}}ServiceDesc(svc {{$svc.GetName}}Server) *{{$svc.GetName}}Desc {
      return &{{$svc.GetName}}Desc{svc:svc}
}

// RegisterGRPC implements service registrator interface.
func (d *{{$svc.GetName}}Desc) RegisterGRPC(s *grpc.Server) {
      Register{{$svc.GetName}}Server(s,d.svc)
}

// SwaggerDef returns this file's Swagger definition.
func (d *{{$svc.GetName}}Desc) SwaggerDef() []byte {
      return _swaggerDef_{{varName $.GetName}}
}

// RegisterHTTP registers this service's HTTP handlers/bindings.
func (d *{{$svc.GetName}}Desc) RegisterHTTP(mux transport.Router) {
	{{range $m := $svc.Methods}}
	// Handlers for {{$m.GetName}}
	{{range $b := $m.Bindings}}
	mux.MethodFunc("{{$b.HTTPMethod}}",pattern_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}}, func(w http.ResponseWriter, r *http.Request) {
          defer r.Body.Close()

	  var req {{$m.RequestType.GetName}}
          err := unmarshaler_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}}(r,&req)
	  if err != nil {
	    httpruntime.SetError(r.Context(),r,w,errors.Wrap(err,"couldn't parse request"))
	    return
	  }

	  ret,err := d.svc.{{$m.GetName}}(r.Context(),&req)
	  if err != nil {
	    httpruntime.SetError(r.Context(),r,w,errors.Wrap(err,"returned from handler"))
	    return
	  }

          _,outbound := httpruntime.MarshalerForRequest(r)
          w.Header().Set("Content-Type", outbound.ContentType())
	  err = outbound.Marshal(w, ret)
	  if err != nil {
	    httpruntime.SetError(r.Context(),r,w,errors.Wrap(err,"couldn't write response"))
	    return
	  }
      })
      {{end}}
      {{end}}
}
{{end}}
{{end}} // base service handler ended
`))

	footerTemplate = template.Must(template.New("footer").Funcs(funcMap).Parse(`
var _swaggerDef_{{varName .GetName}} = []byte(` + "`" + `{{escapeBackTicks (byteStr .SwagBuffer)}}` + `
` + "`)" + `
`))

	patternsTemplate = template.Must(template.New("patterns").Funcs(funcMap).Parse(`
{{define "base"}}
var (
{{range $svc := .Services}}
{{range $m := $svc.Methods}}
{{range $b := $m.Bindings}}
	pattern_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}} = "{{$b.PathTmpl.Template}}"
	pattern_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}}_builder = func(
{{range $p := $b.PathParams}}{{$p.Target.GetName}} {{toGoType $p.Target.GetType}},
{{end}}) string {
return fmt.Sprintf("{{arrayToPathInterp $b.PathTmpl.Template}}",{{range $p := $b.PathParams}}{{$p.Target.GetName}},{{end}})
}
        unmarshaler_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}} = func(r *http.Request,req *{{$m.RequestType.GetName}}) error {

        var err error
        {{if $b.Body}}
          {{template "unmbody" .}}
        {{end}}
        {{if $b.PathParams}}
          {{template "unmpath" .}}
        {{end}}

        return err
        }
{{end}}
{{end}}
{{end}}
)
{{end}}
{{define "unmbody"}}
          inbound,_ := httpruntime.MarshalerForRequest(r)
	  err = errors.Wrap(inbound.Unmarshal(r.Body,req),"couldn't read request JSON")
          if err != nil {
            return err
          }
{{end}}
{{define "unmpath"}}
	  rctx := chi.RouteContext(r.Context())
          if rctx == nil {
            panic("Only chi router is supported for GETs atm")
	  }
          for pos,k := range rctx.URLParams.Keys {
	    runtime.PopulateFieldFromPath(req, k, rctx.URLParams.Values[pos])
          }
{{end}}
`))
	clientTemplate = template.Must(template.New("http-client").Funcs(funcMap).Parse(`
{{range $svc := .Services}}
type {{$svc.GetName}}_httpClient struct {
c *http.Client
host string
}

// New{{$svc.GetName}}HTTPClient creates new HTTP client for {{$svc.GetName}}Server.
// Pass addr in format "http://host[:port]".
func New{{$svc.GetName}}HTTPClient(c *http.Client,addr string) {{$svc.GetName}}Client {
	if strings.HasSuffix(addr, "/") {
		addr = addr[:len(addr)-1]
	}
        return &{{$svc.GetName}}_httpClient{c:c,host:addr}
}
{{range $m := $svc.Methods}}
{{range $b := $m.Bindings}}
func (c *{{$svc.GetName}}_httpClient) {{$m.GetName}}(ctx context.Context,in *{{$m.RequestType.GetName}},_ ...grpc.CallOption) (*{{$m.ResponseType.GetName}},error) {

        //TODO path params aren't supported atm
        path := pattern_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}}_builder({{range $p := $b.PathParams}}in.{{goTypeName $p.String}},{{end}})

	buf := bytes.NewBuffer(nil)

	m := httpruntime.DefaultMarshaler(nil)
	err := m.Marshal(buf, in)
	if err != nil {
		return nil, errors.Wrap(err, "can't marshal request")
	}

	req, err := http.NewRequest("{{$b.HTTPMethod}}", c.host+path, buf)
	if err != nil {
		return nil, errors.Wrap(err, "can't initiate HTTP request")
	}

	req.Header.Add("Accept", m.ContentType())

	rsp, err := c.c.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, "error from client")
	}
	defer rsp.Body.Close()

	if rsp.StatusCode>= 400 {
		b,_ := ioutil.ReadAll(rsp.Body)
		return nil,errors.Errorf("%v %v: server returned HTTP %v: '%v'",req.Method,req.URL.String(),rsp.StatusCode,string(b))
	}

	ret := &{{$m.ResponseType.GetName}}{}
	err = m.Unmarshal(rsp.Body, ret)
	return ret, errors.Wrap(err, "can't unmarshal response")
}
{{end}}
{{end}}
{{end}}
`))
)
<|endoftext|>


<commit_before>package collectors

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/StackExchange/scollector/metadata"
	"github.com/StackExchange/scollector/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_procstats_linux})
}

var uptimeRE = regexp.MustCompile(`(\S+)\s+(\S+)`)
var meminfoRE = regexp.MustCompile(`(\w+):\s+(\d+)\s+(\w+)`)
var vmstatRE = regexp.MustCompile(`(\w+)\s+(\d+)`)
var statRE = regexp.MustCompile(`(\w+)\s+(.*)`)
var statCpuRE = regexp.MustCompile(`cpu(\d+)`)
var loadavgRE = regexp.MustCompile(`(\S+)\s+(\S+)\s+(\S+)\s+(\d+)/(\d+)\s+`)
var inoutRE = regexp.MustCompile(`(.*)(in|out)`)

var CPU_FIELDS = []string{
	"user",
	"nice",
	"system",
	"idle",
	"iowait",
	"irq",
	"softirq",
	"steal",
	"guest",
	"guest_nice",
}

func c_procstats_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	var Error error
	if err := readLine("/proc/uptime", func(s string) error {
		m := uptimeRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		Add(&md, "linux.uptime_total", m[1], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.uptime_now", m[2], nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	mem := make(map[string]float64)
	if err := readLine("/proc/meminfo", func(s string) error {
		m := meminfoRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		i, err := strconv.ParseFloat(m[2], 64)
		if err != nil {
			return err
		}
		mem[m[1]] = i
		Add(&md, "linux.mem."+strings.ToLower(m[1]), m[2], nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	Add(&md, osMemTotal, int(mem["MemTotal"])*1024, nil, metadata.Unknown, metadata.None, "")
	Add(&md, osMemFree, int(mem["MemFree"])*1024, nil, metadata.Unknown, metadata.None, "")
	Add(&md, osMemUsed, (int(mem["MemTotal"])-(int(mem["MemFree"])+int(mem["Buffers"])+int(mem["Cached"])))*1024, nil, metadata.Unknown, metadata.None, "")
	if mem["MemTotal"] != 0 {
		Add(&md, osMemPctFree, (mem["MemFree"]+mem["Buffers"]+mem["Cached"])/mem["MemTotal"]*100, nil, metadata.Unknown, metadata.None, "")
	}

	if err := readLine("/proc/vmstat", func(s string) error {
		m := vmstatRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}

		switch m[1] {
		case "pgpgin", "pgpgout", "pswpin", "pswpout", "pgfault", "pgmajfault":
			mio := inoutRE.FindStringSubmatch(m[1])
			if mio != nil {
				Add(&md, "linux.mem."+mio[1], m[2], opentsdb.TagSet{"direction": mio[2]}, metadata.Unknown, metadata.None, "")
			} else {
				Add(&md, "linux.mem."+m[1], m[2], nil, metadata.Unknown, metadata.None, "")
			}
		}
		return nil
	}); err != nil {
		Error = err
	}
	num_cores := 0
	var t_util float64
	if err := readLine("/proc/stat", func(s string) error {
		m := statRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		if strings.HasPrefix(m[1], "cpu") {
			metric_percpu := ""
			tag_cpu := ""
			cpu_m := statCpuRE.FindStringSubmatch(m[1])
			if cpu_m != nil {
				num_cores += 1
				metric_percpu = ".percpu"
				tag_cpu = cpu_m[1]
			}
			fields := strings.Fields(m[2])
			for i, value := range fields {
				if i >= len(CPU_FIELDS) {
					break
				}
				tags := opentsdb.TagSet{
					"type": CPU_FIELDS[i],
				}
				if tag_cpu != "" {
					tags["cpu"] = tag_cpu
				}
				Add(&md, "linux.cpu"+metric_percpu, value, tags, metadata.Unknown, metadata.None, "")
			}
			if metric_percpu == "" {
				if len(fields) < 3 {
					return nil
				}
				user, err := strconv.ParseFloat(fields[0], 64)
				if err != nil {
					return nil
				}
				nice, err := strconv.ParseFloat(fields[1], 64)
				if err != nil {
					return nil
				}
				system, err := strconv.ParseFloat(fields[2], 64)
				if err != nil {
					return nil
				}
				t_util = user + nice + system
			}
		} else if m[1] == "intr" {
			Add(&md, "linux.intr", strings.Fields(m[2])[0], nil, metadata.Unknown, metadata.None, "")
		} else if m[1] == "ctxt" {
			Add(&md, "linux.ctxt", m[2], nil, metadata.Unknown, metadata.None, "")
		} else if m[1] == "processes" {
			Add(&md, "linux.processes", m[2], nil, metadata.Unknown, metadata.None, "")
		} else if m[1] == "procs_blocked" {
			Add(&md, "linux.procs_blocked", m[2], nil, metadata.Unknown, metadata.None, "")
		}
		return nil
	}); err != nil {
		Error = err
	}
	if num_cores != 0 && t_util != 0 {
		Add(&md, osCPU, t_util/float64(num_cores), nil, metadata.Unknown, metadata.None, "")
	}
	if err := readLine("/proc/loadavg", func(s string) error {
		m := loadavgRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		Add(&md, "linux.loadavg_1_min", m[1], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_5_min", m[2], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_15_min", m[3], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_runnable", m[4], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_total_threads", m[5], nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	if err := readLine("/proc/sys/kernel/random/entropy_avail", func(s string) error {
		Add(&md, "linux.entropy_avail", strings.TrimSpace(s), nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	num_cpus := 0
	if err := readLine("/proc/interrupts", func(s string) error {
		cols := strings.Fields(s)
		if num_cpus == 0 {
			num_cpus = len(cols)
			return nil
		} else if len(cols) < 2 {
			return nil
		}
		irq_type := strings.TrimRight(cols[0], ":")
		if !IsAlNum(irq_type) {
			return nil
		}
		if IsDigit(irq_type) {
			if cols[len(cols)-2] == "PCI-MSI-edge" && strings.Contains(cols[len(cols)-1], "eth") {
				irq_type = cols[len(cols)-1]
			} else {
				// Interrupt type is just a number, ignore.
				return nil
			}
		}
		for i, val := range cols[1:] {
			if i >= num_cpus {
				// All values read, remaining cols contain textual description.
				break
			}
			if !IsDigit(val) {
				// Something is weird, there should only be digit values.
				return fmt.Errorf("interrupts: unexpected value: %v", val)
				break
			}
			Add(&md, "linux.interrupts", val, opentsdb.TagSet{"type": irq_type, "cpu": strconv.Itoa(i)}, metadata.Unknown, metadata.None, "")
		}
		return nil
	}); err != nil {
		Error = err
	}
	if err := readLine("/proc/net/sockstat", func(s string) error {
		cols := strings.Fields(s)
		switch cols[0] {
		case "sockets:":
			if len(cols) < 3 {
				return fmt.Errorf("sockstat: error parsing sockets line")
			}
			Add(&md, "linux.net.sockets.used", cols[2], nil, metadata.Unknown, metadata.None, "")
		case "TCP:":
			if len(cols) < 11 {
				return fmt.Errorf("sockstat: error parsing tcp line")
			}
			Add(&md, "linux.net.sockets.tcp_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_orphaned", cols[4], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_time_wait", cols[6], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_allocated", cols[8], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_mem", cols[10], nil, metadata.Unknown, metadata.None, "")
		case "UDP:":
			if len(cols) < 5 {
				return fmt.Errorf("sockstat: error parsing udp line")
			}
			Add(&md, "linux.net.sockets.udp_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.udp_mem", cols[4], nil, metadata.Unknown, metadata.None, "")
		case "UDPLITE:":
			if len(cols) < 3 {
				return fmt.Errorf("sockstat: error parsing udplite line")
			}
			Add(&md, "linux.net.sockets.udplite_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
		case "RAW:":
			if len(cols) < 3 {
				return fmt.Errorf("sockstat: error parsing raw line")
			}
			Add(&md, "linux.net.sockets.raw_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
		case "FRAG:":
			if len(cols) < 5 {
				return fmt.Errorf("sockstat: error parsing frag line")
			}
			Add(&md, "linux.net.sockets.frag_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.frag_mem", cols[4], nil, metadata.Unknown, metadata.None, "")
		}
		return nil
	}); err != nil {
		Error = err
	}
	ln := 0
	var headers []string
	if err := readLine("/proc/net/netstat", func(s string) error {
		cols := strings.Fields(s)
		if ln%2 == 0 {
			headers = cols
		} else {
			if len(cols) < 1 || len(cols) != len(headers) {
				return fmt.Errorf("netstat: parsing failed")
			}
			root := strings.ToLower(strings.TrimSuffix(headers[0], "Ext:"))
			for i, v := range cols[1:] {
				i += 1
				m := "linux.net.stat." + root + "." + strings.TrimPrefix(strings.ToLower(headers[i]), "tcp")
				Add(&md, m, v, nil, metadata.Unknown, metadata.None, "")
			}
		}
		ln += 1
		return nil
	}); err != nil {
		Error = err
	}
	ln = 0
	metric := "linux.net.stat."
	if err := readLine("/proc/net/snmp", func(s string) error {
		ln++
		if ln%2 != 0 {
			f := strings.Fields(s)
			if len(f) < 2 {
				return fmt.Errorf("Failed to parse header line")
			}
			headers = f
		} else {
			values := strings.Fields(s)
			if len(values) != len(headers) {
				return fmt.Errorf("Mismatched header and value length")
			}
			proto := strings.ToLower(strings.TrimSuffix(values[0], ":"))
			for i, v := range values {
				if i == 0 {
					continue
				}
				stat := strings.ToLower(headers[i])
				if strings.HasPrefix(stat, "rto") {
					Add(&md, metric+proto+"."+stat, v, nil, metadata.Gauge, metadata.None, "")
					continue
				}
				Add(&md, metric+proto+"."+stat, v, nil, metadata.Counter, metadata.None, "")
			}
		}
		return nil
	}); err != nil {
		Error = err
	}
	return md, Error
}
<commit_msg>cmd/scollector: Set var for rate type<commit_after>package collectors

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/StackExchange/scollector/metadata"
	"github.com/StackExchange/scollector/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_procstats_linux})
}

var uptimeRE = regexp.MustCompile(`(\S+)\s+(\S+)`)
var meminfoRE = regexp.MustCompile(`(\w+):\s+(\d+)\s+(\w+)`)
var vmstatRE = regexp.MustCompile(`(\w+)\s+(\d+)`)
var statRE = regexp.MustCompile(`(\w+)\s+(.*)`)
var statCpuRE = regexp.MustCompile(`cpu(\d+)`)
var loadavgRE = regexp.MustCompile(`(\S+)\s+(\S+)\s+(\S+)\s+(\d+)/(\d+)\s+`)
var inoutRE = regexp.MustCompile(`(.*)(in|out)`)

var CPU_FIELDS = []string{
	"user",
	"nice",
	"system",
	"idle",
	"iowait",
	"irq",
	"softirq",
	"steal",
	"guest",
	"guest_nice",
}

func c_procstats_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	var Error error
	if err := readLine("/proc/uptime", func(s string) error {
		m := uptimeRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		Add(&md, "linux.uptime_total", m[1], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.uptime_now", m[2], nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	mem := make(map[string]float64)
	if err := readLine("/proc/meminfo", func(s string) error {
		m := meminfoRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		i, err := strconv.ParseFloat(m[2], 64)
		if err != nil {
			return err
		}
		mem[m[1]] = i
		Add(&md, "linux.mem."+strings.ToLower(m[1]), m[2], nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	Add(&md, osMemTotal, int(mem["MemTotal"])*1024, nil, metadata.Unknown, metadata.None, "")
	Add(&md, osMemFree, int(mem["MemFree"])*1024, nil, metadata.Unknown, metadata.None, "")
	Add(&md, osMemUsed, (int(mem["MemTotal"])-(int(mem["MemFree"])+int(mem["Buffers"])+int(mem["Cached"])))*1024, nil, metadata.Unknown, metadata.None, "")
	if mem["MemTotal"] != 0 {
		Add(&md, osMemPctFree, (mem["MemFree"]+mem["Buffers"]+mem["Cached"])/mem["MemTotal"]*100, nil, metadata.Unknown, metadata.None, "")
	}

	if err := readLine("/proc/vmstat", func(s string) error {
		m := vmstatRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}

		switch m[1] {
		case "pgpgin", "pgpgout", "pswpin", "pswpout", "pgfault", "pgmajfault":
			mio := inoutRE.FindStringSubmatch(m[1])
			if mio != nil {
				Add(&md, "linux.mem."+mio[1], m[2], opentsdb.TagSet{"direction": mio[2]}, metadata.Unknown, metadata.None, "")
			} else {
				Add(&md, "linux.mem."+m[1], m[2], nil, metadata.Unknown, metadata.None, "")
			}
		}
		return nil
	}); err != nil {
		Error = err
	}
	num_cores := 0
	var t_util float64
	if err := readLine("/proc/stat", func(s string) error {
		m := statRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		if strings.HasPrefix(m[1], "cpu") {
			metric_percpu := ""
			tag_cpu := ""
			cpu_m := statCpuRE.FindStringSubmatch(m[1])
			if cpu_m != nil {
				num_cores += 1
				metric_percpu = ".percpu"
				tag_cpu = cpu_m[1]
			}
			fields := strings.Fields(m[2])
			for i, value := range fields {
				if i >= len(CPU_FIELDS) {
					break
				}
				tags := opentsdb.TagSet{
					"type": CPU_FIELDS[i],
				}
				if tag_cpu != "" {
					tags["cpu"] = tag_cpu
				}
				Add(&md, "linux.cpu"+metric_percpu, value, tags, metadata.Unknown, metadata.None, "")
			}
			if metric_percpu == "" {
				if len(fields) < 3 {
					return nil
				}
				user, err := strconv.ParseFloat(fields[0], 64)
				if err != nil {
					return nil
				}
				nice, err := strconv.ParseFloat(fields[1], 64)
				if err != nil {
					return nil
				}
				system, err := strconv.ParseFloat(fields[2], 64)
				if err != nil {
					return nil
				}
				t_util = user + nice + system
			}
		} else if m[1] == "intr" {
			Add(&md, "linux.intr", strings.Fields(m[2])[0], nil, metadata.Unknown, metadata.None, "")
		} else if m[1] == "ctxt" {
			Add(&md, "linux.ctxt", m[2], nil, metadata.Unknown, metadata.None, "")
		} else if m[1] == "processes" {
			Add(&md, "linux.processes", m[2], nil, metadata.Unknown, metadata.None, "")
		} else if m[1] == "procs_blocked" {
			Add(&md, "linux.procs_blocked", m[2], nil, metadata.Unknown, metadata.None, "")
		}
		return nil
	}); err != nil {
		Error = err
	}
	if num_cores != 0 && t_util != 0 {
		Add(&md, osCPU, t_util/float64(num_cores), nil, metadata.Unknown, metadata.None, "")
	}
	if err := readLine("/proc/loadavg", func(s string) error {
		m := loadavgRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		Add(&md, "linux.loadavg_1_min", m[1], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_5_min", m[2], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_15_min", m[3], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_runnable", m[4], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_total_threads", m[5], nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	if err := readLine("/proc/sys/kernel/random/entropy_avail", func(s string) error {
		Add(&md, "linux.entropy_avail", strings.TrimSpace(s), nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	num_cpus := 0
	if err := readLine("/proc/interrupts", func(s string) error {
		cols := strings.Fields(s)
		if num_cpus == 0 {
			num_cpus = len(cols)
			return nil
		} else if len(cols) < 2 {
			return nil
		}
		irq_type := strings.TrimRight(cols[0], ":")
		if !IsAlNum(irq_type) {
			return nil
		}
		if IsDigit(irq_type) {
			if cols[len(cols)-2] == "PCI-MSI-edge" && strings.Contains(cols[len(cols)-1], "eth") {
				irq_type = cols[len(cols)-1]
			} else {
				// Interrupt type is just a number, ignore.
				return nil
			}
		}
		for i, val := range cols[1:] {
			if i >= num_cpus {
				// All values read, remaining cols contain textual description.
				break
			}
			if !IsDigit(val) {
				// Something is weird, there should only be digit values.
				return fmt.Errorf("interrupts: unexpected value: %v", val)
				break
			}
			Add(&md, "linux.interrupts", val, opentsdb.TagSet{"type": irq_type, "cpu": strconv.Itoa(i)}, metadata.Unknown, metadata.None, "")
		}
		return nil
	}); err != nil {
		Error = err
	}
	if err := readLine("/proc/net/sockstat", func(s string) error {
		cols := strings.Fields(s)
		switch cols[0] {
		case "sockets:":
			if len(cols) < 3 {
				return fmt.Errorf("sockstat: error parsing sockets line")
			}
			Add(&md, "linux.net.sockets.used", cols[2], nil, metadata.Unknown, metadata.None, "")
		case "TCP:":
			if len(cols) < 11 {
				return fmt.Errorf("sockstat: error parsing tcp line")
			}
			Add(&md, "linux.net.sockets.tcp_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_orphaned", cols[4], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_time_wait", cols[6], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_allocated", cols[8], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_mem", cols[10], nil, metadata.Unknown, metadata.None, "")
		case "UDP:":
			if len(cols) < 5 {
				return fmt.Errorf("sockstat: error parsing udp line")
			}
			Add(&md, "linux.net.sockets.udp_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.udp_mem", cols[4], nil, metadata.Unknown, metadata.None, "")
		case "UDPLITE:":
			if len(cols) < 3 {
				return fmt.Errorf("sockstat: error parsing udplite line")
			}
			Add(&md, "linux.net.sockets.udplite_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
		case "RAW:":
			if len(cols) < 3 {
				return fmt.Errorf("sockstat: error parsing raw line")
			}
			Add(&md, "linux.net.sockets.raw_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
		case "FRAG:":
			if len(cols) < 5 {
				return fmt.Errorf("sockstat: error parsing frag line")
			}
			Add(&md, "linux.net.sockets.frag_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.frag_mem", cols[4], nil, metadata.Unknown, metadata.None, "")
		}
		return nil
	}); err != nil {
		Error = err
	}
	ln := 0
	var headers []string
	if err := readLine("/proc/net/netstat", func(s string) error {
		cols := strings.Fields(s)
		if ln%2 == 0 {
			headers = cols
		} else {
			if len(cols) < 1 || len(cols) != len(headers) {
				return fmt.Errorf("netstat: parsing failed")
			}
			root := strings.ToLower(strings.TrimSuffix(headers[0], "Ext:"))
			for i, v := range cols[1:] {
				i += 1
				m := "linux.net.stat." + root + "." + strings.TrimPrefix(strings.ToLower(headers[i]), "tcp")
				Add(&md, m, v, nil, metadata.Unknown, metadata.None, "")
			}
		}
		ln += 1
		return nil
	}); err != nil {
		Error = err
	}
	ln = 0
	metric := "linux.net.stat."
	if err := readLine("/proc/net/snmp", func(s string) error {
		ln++
		if ln%2 != 0 {
			f := strings.Fields(s)
			if len(f) < 2 {
				return fmt.Errorf("Failed to parse header line")
			}
			headers = f
		} else {
			values := strings.Fields(s)
			if len(values) != len(headers) {
				return fmt.Errorf("Mismatched header and value length")
			}
			proto := strings.ToLower(strings.TrimSuffix(values[0], ":"))
			for i, v := range values {
				if i == 0 {
					continue
				}
				var stype metadata.RateType = metadata.Counter
				stat := strings.ToLower(headers[i])
				if strings.HasPrefix(stat, "rto") {
					stype = metadata.Gauge
					continue
				}
				Add(&md, metric+proto+"."+stat, v, nil, stype, metadata.None, "")
			}
		}
		return nil
	}); err != nil {
		Error = err
	}
	return md, Error
}
<|endoftext|>


<commit_before>package arn

import (
	"encoding/json"

	"github.com/aerogo/aero"
)

// // Add adds an anime to the list if it hasn't been added yet.
// func (list *AnimeList) Add(animeID string) error {
// 	animeID := id.(string)

// 	if list.Contains(animeID) {
// 		return errors.New("Anime " + animeID + " has already been added")
// 	}

// 	creationDate := DateTimeUTC()

// 	newItem := &AnimeListItem{
// 		AnimeID: animeID,
// 		Status:  AnimeListStatusPlanned,
// 		Rating:  &AnimeRating{},
// 		Created: creationDate,
// 		Edited:  creationDate,
// 	}

// 	list.Items = append(list.Items, newItem)

// 	return nil
// }

// Remove removes the anime ID from the list.
func (list *AnimeList) Remove(animeID string) bool {
	for index, item := range list.Items {
		if item.AnimeID == animeID {
			list.Items = append(list.Items[:index], list.Items[index+1:]...)
			return true
		}
	}

	return false
}

// Contains checks if the list contains the anime ID already.
func (list *AnimeList) Contains(animeID string) bool {
	for _, item := range list.Items {
		if item.AnimeID == animeID {
			return true
		}
	}

	return false
}

// Authorize returns an error if the given API request is not authorized.
func (list *AnimeList) Authorize(ctx *aero.Context, action string) error {
	return AuthorizeIfLoggedInAndOwnData(ctx, "id")
}

// PostBody returns an item that is passed to methods like Add, Remove, etc.
func (list *AnimeList) PostBody(body []byte) interface{} {
	if len(body) > 0 && body[0] == '{' {
		var updates interface{}
		PanicOnError(json.Unmarshal(body, &updates))
		return updates.(map[string]interface{})
	}

	return string(body)
}

// Save saves the anime list in the database.
func (list *AnimeList) Save() error {
	return DB.Set("AnimeList", list.UserID, list)
}
<commit_msg>Cleanup<commit_after>package arn

import (
	"github.com/aerogo/aero"
)

// // Add adds an anime to the list if it hasn't been added yet.
// func (list *AnimeList) Add(animeID string) error {
// 	animeID := id.(string)

// 	if list.Contains(animeID) {
// 		return errors.New("Anime " + animeID + " has already been added")
// 	}

// 	creationDate := DateTimeUTC()

// 	newItem := &AnimeListItem{
// 		AnimeID: animeID,
// 		Status:  AnimeListStatusPlanned,
// 		Rating:  &AnimeRating{},
// 		Created: creationDate,
// 		Edited:  creationDate,
// 	}

// 	list.Items = append(list.Items, newItem)

// 	return nil
// }

// Remove removes the anime ID from the list.
func (list *AnimeList) Remove(animeID string) bool {
	for index, item := range list.Items {
		if item.AnimeID == animeID {
			list.Items = append(list.Items[:index], list.Items[index+1:]...)
			return true
		}
	}

	return false
}

// Contains checks if the list contains the anime ID already.
func (list *AnimeList) Contains(animeID string) bool {
	for _, item := range list.Items {
		if item.AnimeID == animeID {
			return true
		}
	}

	return false
}

// Authorize returns an error if the given API request is not authorized.
func (list *AnimeList) Authorize(ctx *aero.Context, action string) error {
	return AuthorizeIfLoggedInAndOwnData(ctx, "id")
}

// Save saves the anime list in the database.
func (list *AnimeList) Save() error {
	return DB.Set("AnimeList", list.UserID, list)
}
<|endoftext|>


<commit_before><commit_msg>slightly simplify python grapher Dockerfile<commit_after><|endoftext|>


<commit_before>package inspect

import (
	"flag"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"

	"github.com/hashicorp/consul/agent/consul/fsm"
	"github.com/hashicorp/consul/agent/structs"
	"github.com/hashicorp/consul/command/flags"
	"github.com/hashicorp/consul/snapshot"
	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/go-msgpack/codec"
	"github.com/hashicorp/raft"
	"github.com/mitchellh/cli"
)

func New(ui cli.Ui) *cmd {
	c := &cmd{UI: ui}
	c.init()
	return c
}

type cmd struct {
	UI     cli.Ui
	flags  *flag.FlagSet
	help   string
	format string

	// flags
	kvDetails bool
	kvDepth   int
	kvFilter  string
}

func (c *cmd) init() {
	c.flags = flag.NewFlagSet("", flag.ContinueOnError)
	c.flags.BoolVar(&c.kvDetails, "kvdetails", false,
		"Provides a detailed KV space usage breakdown for any KV data that's been stored.")
	c.flags.IntVar(&c.kvDepth, "kvdepth", 0,
		"Can only be used with -kvdetails. The key prefix depth used to breakdown KV store data. Defaults to 2.")
	c.flags.StringVar(&c.kvFilter, "kvfilter", "",
		"Can only be used with -kvdetails. Limits KV key breakdown using this prefix filter.")
	c.flags.StringVar(
		&c.format,
		"format",
		PrettyFormat,
		fmt.Sprintf("Output format {%s}", strings.Join(GetSupportedFormats(), "|")))

	c.help = flags.Usage(help, c.flags)
}

// MetadataInfo is used for passing information
// through the formatter
type MetadataInfo struct {
	ID      string
	Size    int64
	Index   uint64
	Term    uint64
	Version raft.SnapshotVersion
}

// SnapshotInfo is used for passing snapshot stat
// information between functions
type SnapshotInfo struct {
	Meta        MetadataInfo
	Stats       map[structs.MessageType]typeStats
	StatsKV     map[string]typeStats
	TotalSize   int
	TotalSizeKV int
}

// OutputFormat is used for passing information
// through the formatter
type OutputFormat struct {
	Meta        *MetadataInfo
	Stats       []typeStats
	StatsKV     []typeStats
	TotalSize   int
	TotalSizeKV int
}

func (c *cmd) Run(args []string) int {
	if err := c.flags.Parse(args); err != nil {
		c.UI.Error(err.Error())
		return 1
	}

	var file string
	args = c.flags.Args()

	switch len(args) {
	case 0:
		c.UI.Error("Missing FILE argument")
		return 1
	case 1:
		file = args[0]
	default:
		c.UI.Error(fmt.Sprintf("Too many arguments (expected 1, got %d)", len(args)))
		return 1
	}

	// Open the file.
	f, err := os.Open(file)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error opening snapshot file: %s", err))
		return 1
	}
	defer f.Close()

	readFile, meta, err := snapshot.Read(hclog.New(nil), f)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error reading snapshot: %s", err))
	}
	defer func() {
		if err := readFile.Close(); err != nil {
			c.UI.Error(fmt.Sprintf("Failed to close temp snapshot: %v", err))
		}
		if err := os.Remove(readFile.Name()); err != nil {
			c.UI.Error(fmt.Sprintf("Failed to clean up temp snapshot: %v", err))
		}
	}()

	info, err := c.enhance(readFile)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error extracting snapshot data: %s", err))
		return 1
	}

	formatter, err := NewFormatter(c.format)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error outputting enhanced snapshot data: %s", err))
		return 1
	}
	//Generate structs for the formatter with information we read in
	metaformat := &MetadataInfo{
		ID:      meta.ID,
		Size:    meta.Size,
		Index:   meta.Index,
		Term:    meta.Term,
		Version: meta.Version,
	}

	//Restructures stats given above to be human readable
	formattedStats := generateStats(info)
	formattedStatsKV := generateKVStats(info)

	in := &OutputFormat{
		Meta:        metaformat,
		Stats:       formattedStats,
		StatsKV:     formattedStatsKV,
		TotalSize:   info.TotalSize,
		TotalSizeKV: info.TotalSizeKV,
	}

	out, err := formatter.Format(in)
	if err != nil {
		c.UI.Error(err.Error())
		return 1
	}

	c.UI.Output(out)
	return 0
}

type typeStats struct {
	Name  string
	Sum   int
	Count int
}

// Generate the stats for the output struct that's
// used to produce the printed output the user sees.
func generateStats(info SnapshotInfo) []typeStats {
	ss := make([]typeStats, 0, len(info.Stats))

	for _, s := range info.Stats {
		ss = append(ss, s)
	}

	ss = sortTypeStats(ss)

	return ss
}

// Generate the KV stats for the output struct that's
// used to produce the printed output the user sees.
func generateKVStats(info SnapshotInfo) []typeStats {
	if info.StatsKV != nil {
		ks := make([]typeStats, 0, len(info.StatsKV))

		for _, s := range info.StatsKV {
			ks = append(ks, s)
		}

		ks = sortTypeStats(ks)

		return ks
	}

	return nil
}

// Sort the stat slice
func sortTypeStats(stats []typeStats) []typeStats {
	sort.Slice(stats, func(i, j int) bool {
		// sort alphabetically if size is equal
		if stats[i].Sum == stats[j].Sum {
			return stats[i].Name < stats[j].Name
		}

		return stats[i].Sum > stats[j].Sum
	})

	return stats
}

// countingReader helps keep track of the bytes we have read
// when reading snapshots
type countingReader struct {
	wrappedReader io.Reader
	read          int
}

func (r *countingReader) Read(p []byte) (n int, err error) {
	n, err = r.wrappedReader.Read(p)
	if err == nil {
		r.read += n
	}
	return n, err
}

// enhance utilizes ReadSnapshot to populate the struct with
// all of the snapshot's itemized data
func (c *cmd) enhance(file io.Reader) (SnapshotInfo, error) {
	info := SnapshotInfo{
		Stats:       make(map[structs.MessageType]typeStats),
		StatsKV:     make(map[string]typeStats),
		TotalSize:   0,
		TotalSizeKV: 0,
	}
	cr := &countingReader{wrappedReader: file}
	handler := func(header *fsm.SnapshotHeader, msg structs.MessageType, dec *codec.Decoder) error {
		name := structs.MessageType.String(msg)
		s := info.Stats[msg]
		if s.Name == "" {
			s.Name = name
		}

		var val interface{}
		err := dec.Decode(&val)
		if err != nil {
			return fmt.Errorf("failed to decode msg type %v, error %v", name, err)
		}

		size := cr.read - info.TotalSize
		s.Sum += size
		s.Count++
		info.TotalSize = cr.read
		info.Stats[msg] = s

		c.kvEnhance(s.Name, val, size, &info)

		return nil
	}
	if err := fsm.ReadSnapshot(cr, handler); err != nil {
		return info, err
	}
	return info, nil

}

// kvEnhance populates the struct with all of the snapshot's
// size information for KV data stored in it
func (c *cmd) kvEnhance(keyType string, val interface{}, size int, info *SnapshotInfo) {
	// automatically set kvDetails to true if a depth or filter
	// is provided. this allows the user to omit the -kvdetails
	// flag if they prefer.
	if c.kvDepth != 0 || c.kvFilter != "" {
		c.kvDetails = true
	}

	// set the default depth if one wasn't specified with -kvdepth.
	// this is used rather than the flag default to facilitate the
	// above shortcut.
	if c.kvDetails && c.kvDepth == 0 {
		c.kvDepth = 2
	}

	if c.kvDetails {
		if keyType != "KVS" {
			return
		}

		// have to coerce this into a usable type here or this won't work
		keyVal := val.(map[string]interface{})
		for k, v := range keyVal {
			// we only care about the entry on the key specifically
			// related to the key name, so skip all others
			if k != "Key" {
				continue
			}

			// check for whether a filter is specified. if it is, skip
			// any keys that don't match.
			if len(c.kvFilter) > 0 && !strings.HasPrefix(v.(string), c.kvFilter) {
				break
			}

			split := strings.Split(v.(string), "/")

			// handle the situation where the key is shorter than
			// the specified depth.
			actualDepth := c.kvDepth
			if c.kvDepth > len(split) {
				actualDepth = len(split)
			}
			prefix := strings.Join(split[0:actualDepth], "/")
			kvs := info.StatsKV[prefix]
			if kvs.Name == "" {
				kvs.Name = prefix
			}

			kvs.Sum += size
			kvs.Count++
			info.TotalSizeKV += size
			info.StatsKV[prefix] = kvs
		}
	}
}

func (c *cmd) Synopsis() string {
	return synopsis
}

func (c *cmd) Help() string {
	return c.help
}

const synopsis = "Displays information about a Consul snapshot file"
const help = `
Usage: consul snapshot inspect [options] FILE

  Displays information about a snapshot file on disk.

  To inspect the file "backup.snap":

    $ consul snapshot inspect backup.snap

  For a full list of options and examples, please see the Consul documentation.
`
<commit_msg>Length check is required here<commit_after>package inspect

import (
	"flag"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"

	"github.com/hashicorp/consul/agent/consul/fsm"
	"github.com/hashicorp/consul/agent/structs"
	"github.com/hashicorp/consul/command/flags"
	"github.com/hashicorp/consul/snapshot"
	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/go-msgpack/codec"
	"github.com/hashicorp/raft"
	"github.com/mitchellh/cli"
)

func New(ui cli.Ui) *cmd {
	c := &cmd{UI: ui}
	c.init()
	return c
}

type cmd struct {
	UI     cli.Ui
	flags  *flag.FlagSet
	help   string
	format string

	// flags
	kvDetails bool
	kvDepth   int
	kvFilter  string
}

func (c *cmd) init() {
	c.flags = flag.NewFlagSet("", flag.ContinueOnError)
	c.flags.BoolVar(&c.kvDetails, "kvdetails", false,
		"Provides a detailed KV space usage breakdown for any KV data that's been stored.")
	c.flags.IntVar(&c.kvDepth, "kvdepth", 0,
		"Can only be used with -kvdetails. The key prefix depth used to breakdown KV store data. Defaults to 2.")
	c.flags.StringVar(&c.kvFilter, "kvfilter", "",
		"Can only be used with -kvdetails. Limits KV key breakdown using this prefix filter.")
	c.flags.StringVar(
		&c.format,
		"format",
		PrettyFormat,
		fmt.Sprintf("Output format {%s}", strings.Join(GetSupportedFormats(), "|")))

	c.help = flags.Usage(help, c.flags)
}

// MetadataInfo is used for passing information
// through the formatter
type MetadataInfo struct {
	ID      string
	Size    int64
	Index   uint64
	Term    uint64
	Version raft.SnapshotVersion
}

// SnapshotInfo is used for passing snapshot stat
// information between functions
type SnapshotInfo struct {
	Meta        MetadataInfo
	Stats       map[structs.MessageType]typeStats
	StatsKV     map[string]typeStats
	TotalSize   int
	TotalSizeKV int
}

// OutputFormat is used for passing information
// through the formatter
type OutputFormat struct {
	Meta        *MetadataInfo
	Stats       []typeStats
	StatsKV     []typeStats
	TotalSize   int
	TotalSizeKV int
}

func (c *cmd) Run(args []string) int {
	if err := c.flags.Parse(args); err != nil {
		c.UI.Error(err.Error())
		return 1
	}

	var file string
	args = c.flags.Args()

	switch len(args) {
	case 0:
		c.UI.Error("Missing FILE argument")
		return 1
	case 1:
		file = args[0]
	default:
		c.UI.Error(fmt.Sprintf("Too many arguments (expected 1, got %d)", len(args)))
		return 1
	}

	// Open the file.
	f, err := os.Open(file)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error opening snapshot file: %s", err))
		return 1
	}
	defer f.Close()

	readFile, meta, err := snapshot.Read(hclog.New(nil), f)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error reading snapshot: %s", err))
	}
	defer func() {
		if err := readFile.Close(); err != nil {
			c.UI.Error(fmt.Sprintf("Failed to close temp snapshot: %v", err))
		}
		if err := os.Remove(readFile.Name()); err != nil {
			c.UI.Error(fmt.Sprintf("Failed to clean up temp snapshot: %v", err))
		}
	}()

	info, err := c.enhance(readFile)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error extracting snapshot data: %s", err))
		return 1
	}

	formatter, err := NewFormatter(c.format)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error outputting enhanced snapshot data: %s", err))
		return 1
	}
	//Generate structs for the formatter with information we read in
	metaformat := &MetadataInfo{
		ID:      meta.ID,
		Size:    meta.Size,
		Index:   meta.Index,
		Term:    meta.Term,
		Version: meta.Version,
	}

	//Restructures stats given above to be human readable
	formattedStats := generateStats(info)
	formattedStatsKV := generateKVStats(info)

	in := &OutputFormat{
		Meta:        metaformat,
		Stats:       formattedStats,
		StatsKV:     formattedStatsKV,
		TotalSize:   info.TotalSize,
		TotalSizeKV: info.TotalSizeKV,
	}

	out, err := formatter.Format(in)
	if err != nil {
		c.UI.Error(err.Error())
		return 1
	}

	c.UI.Output(out)
	return 0
}

type typeStats struct {
	Name  string
	Sum   int
	Count int
}

// Generate the stats for the output struct that's
// used to produce the printed output the user sees.
func generateStats(info SnapshotInfo) []typeStats {
	ss := make([]typeStats, 0, len(info.Stats))

	for _, s := range info.Stats {
		ss = append(ss, s)
	}

	ss = sortTypeStats(ss)

	return ss
}

// Generate the KV stats for the output struct that's
// used to produce the printed output the user sees.
func generateKVStats(info SnapshotInfo) []typeStats {
	kvLen := len(info.StatsKV)
	if kvLen > 0 {
		ks := make([]typeStats, 0, kvLen)

		for _, s := range info.StatsKV {
			ks = append(ks, s)
		}

		ks = sortTypeStats(ks)

		return ks
	}

	return nil
}

// Sort the stat slice
func sortTypeStats(stats []typeStats) []typeStats {
	sort.Slice(stats, func(i, j int) bool {
		// sort alphabetically if size is equal
		if stats[i].Sum == stats[j].Sum {
			return stats[i].Name < stats[j].Name
		}

		return stats[i].Sum > stats[j].Sum
	})

	return stats
}

// countingReader helps keep track of the bytes we have read
// when reading snapshots
type countingReader struct {
	wrappedReader io.Reader
	read          int
}

func (r *countingReader) Read(p []byte) (n int, err error) {
	n, err = r.wrappedReader.Read(p)
	if err == nil {
		r.read += n
	}
	return n, err
}

// enhance utilizes ReadSnapshot to populate the struct with
// all of the snapshot's itemized data
func (c *cmd) enhance(file io.Reader) (SnapshotInfo, error) {
	info := SnapshotInfo{
		Stats:       make(map[structs.MessageType]typeStats),
		StatsKV:     make(map[string]typeStats),
		TotalSize:   0,
		TotalSizeKV: 0,
	}
	cr := &countingReader{wrappedReader: file}
	handler := func(header *fsm.SnapshotHeader, msg structs.MessageType, dec *codec.Decoder) error {
		name := structs.MessageType.String(msg)
		s := info.Stats[msg]
		if s.Name == "" {
			s.Name = name
		}

		var val interface{}
		err := dec.Decode(&val)
		if err != nil {
			return fmt.Errorf("failed to decode msg type %v, error %v", name, err)
		}

		size := cr.read - info.TotalSize
		s.Sum += size
		s.Count++
		info.TotalSize = cr.read
		info.Stats[msg] = s

		c.kvEnhance(s.Name, val, size, &info)

		return nil
	}
	if err := fsm.ReadSnapshot(cr, handler); err != nil {
		return info, err
	}
	return info, nil

}

// kvEnhance populates the struct with all of the snapshot's
// size information for KV data stored in it
func (c *cmd) kvEnhance(keyType string, val interface{}, size int, info *SnapshotInfo) {
	// automatically set kvDetails to true if a depth or filter
	// is provided. this allows the user to omit the -kvdetails
	// flag if they prefer.
	if c.kvDepth != 0 || c.kvFilter != "" {
		c.kvDetails = true
	}

	// set the default depth if one wasn't specified with -kvdepth.
	// this is used rather than the flag default to facilitate the
	// above shortcut.
	if c.kvDetails && c.kvDepth == 0 {
		c.kvDepth = 2
	}

	if c.kvDetails {
		if keyType != "KVS" {
			return
		}

		// have to coerce this into a usable type here or this won't work
		keyVal := val.(map[string]interface{})
		for k, v := range keyVal {
			// we only care about the entry on the key specifically
			// related to the key name, so skip all others
			if k != "Key" {
				continue
			}

			// check for whether a filter is specified. if it is, skip
			// any keys that don't match.
			if len(c.kvFilter) > 0 && !strings.HasPrefix(v.(string), c.kvFilter) {
				break
			}

			split := strings.Split(v.(string), "/")

			// handle the situation where the key is shorter than
			// the specified depth.
			actualDepth := c.kvDepth
			if c.kvDepth > len(split) {
				actualDepth = len(split)
			}
			prefix := strings.Join(split[0:actualDepth], "/")
			kvs := info.StatsKV[prefix]
			if kvs.Name == "" {
				kvs.Name = prefix
			}

			kvs.Sum += size
			kvs.Count++
			info.TotalSizeKV += size
			info.StatsKV[prefix] = kvs
		}
	}
}

func (c *cmd) Synopsis() string {
	return synopsis
}

func (c *cmd) Help() string {
	return c.help
}

const synopsis = "Displays information about a Consul snapshot file"
const help = `
Usage: consul snapshot inspect [options] FILE

  Displays information about a snapshot file on disk.

  To inspect the file "backup.snap":

    $ consul snapshot inspect backup.snap

  For a full list of options and examples, please see the Consul documentation.
`
<|endoftext|>


<commit_before>package config

import "testing"

func TestHello(t *testing.T) {
	if val := Hello(); val != "hello" {
		t.Fatalf("Expected 'hello', got %s instead.", val)
	}
}
<commit_msg>Goveralls fixes.<commit_after>package config

import "testing"

func TestHello(t *testing.T) {
	if val := Hello(); val != "hello123" {
		t.Fatalf("Expected 'hello', got %s instead.", val)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package syslog_test

import (
	"io/ioutil"
	"path/filepath"
	stdtesting "testing"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/agent"
	"launchpad.net/juju-core/utils/syslog"
	syslogtesting "launchpad.net/juju-core/utils/syslog/testing"
)

func Test(t *stdtesting.T) {
	gc.TestingT(t)
}

type SyslogConfigSuite struct {
	configDir string
}

var _ = gc.Suite(&SyslogConfigSuite{})

func (s *SyslogConfigSuite) SetUpTest(c *gc.C) {
	s.configDir = c.MkDir()
}

func (s *SyslogConfigSuite) assertRsyslogConfigPath(c *gc.C, slConfig *syslog.SyslogConfig) {
	slConfig.ConfigDir = s.configDir
	slConfig.ConfigFileName = "rsyslog.conf"
	c.Assert(slConfig.ConfigFilePath(), gc.Equals, filepath.Join(s.configDir, "rsyslog.conf"))
}

func (s *SyslogConfigSuite) assertRsyslogConfigContents(c *gc.C, slConfig *syslog.SyslogConfig,
	expectedConf string) {
	data, err := slConfig.Render()
	c.Assert(err, gc.IsNil)
	c.Assert(string(data), gc.Equals, expectedConf)
}

func (s *SyslogConfigSuite) TestAccumulateConfigRender(c *gc.C) {
	syslogConfigRenderer := syslog.NewAccumulateConfig("some-machine", agent.DefaultLogDir, 8888, "")
	s.assertRsyslogConfigContents(
		c,
		syslogConfigRenderer,
		syslogtesting.ExpectedAccumulateSyslogConf(c, "some-machine", "", 8888),
	)
}

func (s *SyslogConfigSuite) TestAccumulateConfigWrite(c *gc.C) {
	syslogConfigRenderer := syslog.NewAccumulateConfig("some-machine", agent.DefaultLogDir, 8888, "")
	syslogConfigRenderer.ConfigDir = s.configDir
	syslogConfigRenderer.ConfigFileName = "rsyslog.conf"
	s.assertRsyslogConfigPath(c, syslogConfigRenderer)
	err := syslogConfigRenderer.Write()
	c.Assert(err, gc.IsNil)
	syslogConfData, err := ioutil.ReadFile(syslogConfigRenderer.ConfigFilePath())
	c.Assert(err, gc.IsNil)
	c.Assert(
		string(syslogConfData),
		gc.Equals,
		syslogtesting.ExpectedAccumulateSyslogConf(c, "some-machine", "", 8888),
	)
}

func (s *SyslogConfigSuite) TestAccumulateConfigRenderWithNamespace(c *gc.C) {
	syslogConfigRenderer := syslog.NewAccumulateConfig("some-machine", agent.DefaultLogDir, 8888, "namespace")
	syslogConfigRenderer.LogDir += "-namespace"
	s.assertRsyslogConfigContents(
		c, syslogConfigRenderer, syslogtesting.ExpectedAccumulateSyslogConf(
			c, "some-machine", "namespace", 8888,
		),
	)
}

func (s *SyslogConfigSuite) TestForwardConfigRender(c *gc.C) {
	syslogConfigRenderer := syslog.NewForwardConfig(
		"some-machine", agent.DefaultLogDir, 999, "", []string{"server"},
	)
	s.assertRsyslogConfigContents(
		c, syslogConfigRenderer, syslogtesting.ExpectedForwardSyslogConf(
			c, "some-machine", agent.DefaultLogDir, "", "server", 999,
		),
	)
}

func (s *SyslogConfigSuite) TestForwardConfigRenderWithNamespace(c *gc.C) {
	syslogConfigRenderer := syslog.NewForwardConfig(
		"some-machine", agent.DefaultLogDir, 999, "namespace", []string{"server"},
	)
	s.assertRsyslogConfigContents(
		c, syslogConfigRenderer, syslogtesting.ExpectedForwardSyslogConf(
			c, "some-machine", agent.DefaultLogDir, "namespace", "server", 999,
		),
	)
}

func (s *SyslogConfigSuite) TestForwardConfigWrite(c *gc.C) {
	syslogConfigRenderer := syslog.NewForwardConfig(
		"some-machine", agent.DefaultLogDir, 999, "", []string{"server"},
	)
	syslogConfigRenderer.ConfigDir = s.configDir
	syslogConfigRenderer.ConfigFileName = "rsyslog.conf"
	s.assertRsyslogConfigPath(c, syslogConfigRenderer)
	err := syslogConfigRenderer.Write()
	c.Assert(err, gc.IsNil)
	syslogConfData, err := ioutil.ReadFile(syslogConfigRenderer.ConfigFilePath())
	c.Assert(err, gc.IsNil)
	c.Assert(
		string(syslogConfData),
		gc.Equals,
		syslogtesting.ExpectedForwardSyslogConf(
			c, "some-machine", agent.DefaultLogDir, "", "server", 999,
		),
	)
}
<commit_msg>Add state addresses to config test<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package syslog_test

import (
	"io/ioutil"
	"path/filepath"
	stdtesting "testing"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/agent"
	"launchpad.net/juju-core/utils/syslog"
	syslogtesting "launchpad.net/juju-core/utils/syslog/testing"
)

func Test(t *stdtesting.T) {
	gc.TestingT(t)
}

type SyslogConfigSuite struct {
	configDir string
}

var _ = gc.Suite(&SyslogConfigSuite{})

func (s *SyslogConfigSuite) SetUpTest(c *gc.C) {
	s.configDir = c.MkDir()
}

func (s *SyslogConfigSuite) assertRsyslogConfigPath(c *gc.C, slConfig *syslog.SyslogConfig) {
	slConfig.ConfigDir = s.configDir
	slConfig.ConfigFileName = "rsyslog.conf"
	c.Assert(slConfig.ConfigFilePath(), gc.Equals, filepath.Join(s.configDir, "rsyslog.conf"))
}

func (s *SyslogConfigSuite) assertRsyslogConfigContents(c *gc.C, slConfig *syslog.SyslogConfig,
	expectedConf string) {
	data, err := slConfig.Render()
	c.Assert(err, gc.IsNil)
	c.Assert(string(data), gc.Equals, expectedConf)
}

func (s *SyslogConfigSuite) TestAccumulateConfigRender(c *gc.C) {
	syslogConfigRenderer := syslog.NewAccumulateConfig("some-machine", agent.DefaultLogDir, 8888, "", []string{"foo:80"})
	s.assertRsyslogConfigContents(
		c,
		syslogConfigRenderer,
		syslogtesting.ExpectedAccumulateSyslogConf(c, "some-machine", "", 8888),
	)
}

func (s *SyslogConfigSuite) TestAccumulateConfigWrite(c *gc.C) {
	syslogConfigRenderer := syslog.NewAccumulateConfig("some-machine", agent.DefaultLogDir, 8888, "", []string{"foo:80"})
	syslogConfigRenderer.ConfigDir = s.configDir
	syslogConfigRenderer.ConfigFileName = "rsyslog.conf"
	s.assertRsyslogConfigPath(c, syslogConfigRenderer)
	err := syslogConfigRenderer.Write()
	c.Assert(err, gc.IsNil)
	syslogConfData, err := ioutil.ReadFile(syslogConfigRenderer.ConfigFilePath())
	c.Assert(err, gc.IsNil)
	c.Assert(
		string(syslogConfData),
		gc.Equals,
		syslogtesting.ExpectedAccumulateSyslogConf(c, "some-machine", "", 8888),
	)
}

func (s *SyslogConfigSuite) TestAccumulateConfigRenderWithNamespace(c *gc.C) {
	syslogConfigRenderer := syslog.NewAccumulateConfig("some-machine", agent.DefaultLogDir, 8888, "namespace", []string{"foo:80"})
	syslogConfigRenderer.LogDir += "-namespace"
	s.assertRsyslogConfigContents(
		c, syslogConfigRenderer, syslogtesting.ExpectedAccumulateSyslogConf(
			c, "some-machine", "namespace", 8888,
		),
	)
}

func (s *SyslogConfigSuite) TestForwardConfigRender(c *gc.C) {
	syslogConfigRenderer := syslog.NewForwardConfig(
		"some-machine", agent.DefaultLogDir, 999, "", []string{"server"},
	)
	s.assertRsyslogConfigContents(
		c, syslogConfigRenderer, syslogtesting.ExpectedForwardSyslogConf(
			c, "some-machine", agent.DefaultLogDir, "", "server", 999,
		),
	)
}

func (s *SyslogConfigSuite) TestForwardConfigRenderWithNamespace(c *gc.C) {
	syslogConfigRenderer := syslog.NewForwardConfig(
		"some-machine", agent.DefaultLogDir, 999, "namespace", []string{"server"},
	)
	s.assertRsyslogConfigContents(
		c, syslogConfigRenderer, syslogtesting.ExpectedForwardSyslogConf(
			c, "some-machine", agent.DefaultLogDir, "namespace", "server", 999,
		),
	)
}

func (s *SyslogConfigSuite) TestForwardConfigWrite(c *gc.C) {
	syslogConfigRenderer := syslog.NewForwardConfig(
		"some-machine", agent.DefaultLogDir, 999, "", []string{"server"},
	)
	syslogConfigRenderer.ConfigDir = s.configDir
	syslogConfigRenderer.ConfigFileName = "rsyslog.conf"
	s.assertRsyslogConfigPath(c, syslogConfigRenderer)
	err := syslogConfigRenderer.Write()
	c.Assert(err, gc.IsNil)
	syslogConfData, err := ioutil.ReadFile(syslogConfigRenderer.ConfigFilePath())
	c.Assert(err, gc.IsNil)
	c.Assert(
		string(syslogConfData),
		gc.Equals,
		syslogtesting.ExpectedForwardSyslogConf(
			c, "some-machine", agent.DefaultLogDir, "", "server", 999,
		),
	)
}
<|endoftext|>


<commit_before>// +build integ
//  Copyright Istio Authors
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package externalca

import (
	"testing"

	"istio.io/istio/pkg/test/framework"
	"istio.io/istio/pkg/test/framework/components/echo"
	"istio.io/istio/pkg/test/framework/components/echo/echoboot"
	"istio.io/istio/pkg/test/framework/components/istio"
	"istio.io/istio/pkg/test/framework/components/namespace"
	"istio.io/istio/pkg/test/framework/label"
	"istio.io/istio/pkg/test/framework/resource"
	"istio.io/istio/tests/integration/security/util"
)

const (
	ASvc = "a"
	BSvc = "b"
)

type EchoDeployments struct {
	Namespace namespace.Instance
	// workloads for TestSecureNaming
	A, B echo.Instances
}

var (
	inst istio.Instance
	apps = &EchoDeployments{}
)

func SetupApps(ctx resource.Context, apps *EchoDeployments) error {
	var err error
	apps.Namespace, err = namespace.New(ctx, namespace.Config{
		Prefix: "test-ns",
		Inject: true,
	})
	if err != nil {
		return err
	}

	builder := echoboot.NewBuilder(ctx)
	builder.
		WithClusters(ctx.Clusters()...).
		WithConfig(util.EchoConfig(ASvc, apps.Namespace, false, nil)).
		WithConfig(util.EchoConfig(BSvc, apps.Namespace, false, nil))

	echos, err := builder.Build()
	if err != nil {
		return err
	}
	apps.A = echos.Match(echo.Service(ASvc))
	apps.B = echos.Match(echo.Service(BSvc))
	return nil
}

func TestMain(m *testing.M) {
	// Integration test for testing interoperability with external CA's that are integrated with K8s CSR API
	// Refer to https://kubernetes.io/docs/reference/access-authn-authz/certificate-signing-requests/
	framework.NewSuite(m).
		Label(label.CustomSetup).
		Setup(istio.Setup(&inst, setupConfig)).
		Setup(func(ctx resource.Context) error {
			return SetupApps(ctx, apps)
		}).
		Run()
}

func setupConfig(_ resource.Context, cfg *istio.Config) {
	if cfg == nil {
		return
	}

	// TODO: Replace K8s legacy signer by deploying external signer common to all clusters with a known root
	cfg.ControlPlaneValues = `
components:
  pilot:
    k8s:
      env:
      - name: EXTERNAL_CA
        value: ISTIOD_RA_KUBERNETES_API
      - name: K8S_SIGNER
        value: kubernetes.io/legacy-unknown
values:
  meshConfig:
    trustDomainAliases: [some-other, trust-domain-foo]
`
}
<commit_msg>External CA: Ensure integration tests only run after K8s 1.18 (#30525)<commit_after>// +build integ
//  Copyright Istio Authors
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package externalca

import (
	"testing"

	"istio.io/istio/pkg/test/framework"
	"istio.io/istio/pkg/test/framework/components/echo"
	"istio.io/istio/pkg/test/framework/components/echo/echoboot"
	"istio.io/istio/pkg/test/framework/components/istio"
	"istio.io/istio/pkg/test/framework/components/namespace"
	"istio.io/istio/pkg/test/framework/label"
	"istio.io/istio/pkg/test/framework/resource"
	"istio.io/istio/tests/integration/security/util"
)

const (
	ASvc = "a"
	BSvc = "b"
)

type EchoDeployments struct {
	Namespace namespace.Instance
	// workloads for TestSecureNaming
	A, B echo.Instances
}

var (
	inst istio.Instance
	apps = &EchoDeployments{}
)

func SetupApps(ctx resource.Context, apps *EchoDeployments) error {
	var err error
	apps.Namespace, err = namespace.New(ctx, namespace.Config{
		Prefix: "test-ns",
		Inject: true,
	})
	if err != nil {
		return err
	}

	builder := echoboot.NewBuilder(ctx)
	builder.
		WithClusters(ctx.Clusters()...).
		WithConfig(util.EchoConfig(ASvc, apps.Namespace, false, nil)).
		WithConfig(util.EchoConfig(BSvc, apps.Namespace, false, nil))

	echos, err := builder.Build()
	if err != nil {
		return err
	}
	apps.A = echos.Match(echo.Service(ASvc))
	apps.B = echos.Match(echo.Service(BSvc))
	return nil
}

func TestMain(m *testing.M) {
	// Integration test for testing interoperability with external CA's that are integrated with K8s CSR API
	// Refer to https://kubernetes.io/docs/reference/access-authn-authz/certificate-signing-requests/
	framework.NewSuite(m).
		Label(label.CustomSetup).
		RequireEnvironmentVersion("1.18").
		Setup(istio.Setup(&inst, setupConfig)).
		Setup(func(ctx resource.Context) error {
			return SetupApps(ctx, apps)
		}).
		Run()
}

func setupConfig(_ resource.Context, cfg *istio.Config) {
	if cfg == nil {
		return
	}

	// TODO: Replace K8s legacy signer by deploying external signer common to all clusters with a known root
	cfg.ControlPlaneValues = `
components:
  pilot:
    k8s:
      env:
      - name: EXTERNAL_CA
        value: ISTIOD_RA_KUBERNETES_API
      - name: K8S_SIGNER
        value: kubernetes.io/legacy-unknown
values:
  meshConfig:
    trustDomainAliases: [some-other, trust-domain-foo]
`
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dhcp

import (
	"errors"
	"io"
	"net"
	"time"

	"golang.org/x/net/ipv4"
)

// defined as a var so tests can override it.
var (
	dhcpClientPort = 68
	platformConn   func(string) (conn, error)
)

// txType describes how a Packet should be sent on the wire.
type txType int

// The various transmission strategies described in RFC 2131. "MUST",
// "MUST NOT", "SHOULD" and "MAY" are as specified in RFC 2119.
const (
	// Packet MUST be broadcast.
	txBroadcast txType = iota
	// Packet MUST be unicasted to port 67 of RelayAddr
	txRelayAddr
	// Packet MUST be unicasted to port 68 of ClientAddr
	txClientAddr
	// Packet SHOULD be unicasted to port 68 of YourAddr, with the
	// link-layer destination explicitly set to HardwareAddr. You MUST
	// NOT rely on ARP resolution to discover the link-layer
	// destination address.
	//
	// Conn implementations that cannot explicitly set the link-layer
	// destination address MAY instead broadcast the packet.
	txHardwareAddr
)

type conn interface {
	io.Closer
	Recv([]byte) (b []byte, addr *net.UDPAddr, ifidx int, err error)
	Send(b []byte, addr *net.UDPAddr, ifidx int) error
	SetReadDeadline(t time.Time) error
	SetWriteDeadline(t time.Time) error
}

// Conn is a DHCP-oriented packet socket.
//
// Multiple goroutines may invoke methods on a Conn simultaneously.
type Conn struct {
	conn conn
}

// NewConn creates a Conn bound to the given UDP ip:port.
func NewConn(addr string) (*Conn, error) {
	if platformConn != nil {
		c, err := platformConn(addr)
		if err == nil {
			return &Conn{c}, nil
		}
	}
	// Always try falling back to the portable implementation
	c, err := newPortableConn(addr)
	if err != nil {
		return nil, err
	}
	return &Conn{c}, nil
}

func (c *Conn) Close() error {
	return c.conn.Close()
}

func (c *Conn) RecvDHCP() (*Packet, *net.Interface, error) {
	var buf [1500]byte
	for {
		b, _, ifidx, err := c.conn.Recv(buf[:])
		if err != nil {
			return nil, nil, err
		}
		pkt, err := Unmarshal(b)
		if err != nil {
			continue
		}
		intf, err := net.InterfaceByIndex(ifidx)
		if err != nil {
			return nil, nil, err
		}
		// TODO: possibly more validation that the source lines up
		// with what the packet says.
		return pkt, intf, nil
	}
}

func (c *Conn) SendDHCP(pkt *Packet, intf *net.Interface) error {
	b, err := pkt.Marshal()
	if err != nil {
		return err
	}

	switch pkt.txType() {
	case txBroadcast, txHardwareAddr:
		addr := net.UDPAddr{
			IP:   net.IPv4bcast,
			Port: dhcpClientPort,
		}
		return c.conn.Send(b, &addr, intf.Index)
	case txRelayAddr:
		addr := net.UDPAddr{
			IP:   pkt.RelayAddr,
			Port: 67,
		}
		return c.conn.Send(b, &addr, 0)
	case txClientAddr:
		addr := net.UDPAddr{
			IP:   pkt.ClientAddr,
			Port: dhcpClientPort,
		}
		return c.conn.Send(b, &addr, 0)
	default:
		return errors.New("unknown TX type for packet")
	}
}

func (c *Conn) SetReadDeadline(t time.Time) error {
	return c.conn.SetReadDeadline(t)
}

func (c *Conn) SetWriteDeadline(t time.Time) error {
	return c.conn.SetReadDeadline(t)
}

type portableConn struct {
	conn *ipv4.PacketConn
}

func newPortableConn(addr string) (conn, error) {
	c, err := net.ListenPacket("udp4", addr)
	if err != nil {
		return nil, err
	}
	l := ipv4.NewPacketConn(c)
	if err = l.SetControlMessage(ipv4.FlagInterface, true); err != nil {
		l.Close()
		return nil, err
	}
	return &portableConn{l}, nil
}

func (c *portableConn) Close() error {
	return c.conn.Close()
}

func (c *portableConn) Recv(b []byte) (rb []byte, addr *net.UDPAddr, ifidx int, err error) {
	n, cm, a, err := c.conn.ReadFrom(b)
	if err != nil {
		return nil, nil, 0, err
	}
	return b[:n], a.(*net.UDPAddr), cm.IfIndex, nil
}

func (c *portableConn) Send(b []byte, addr *net.UDPAddr, ifidx int) error {
	if ifidx > 0 {
		_, err := c.conn.WriteTo(b, nil, addr)
		return err
	}
	cm := ipv4.ControlMessage{
		IfIndex: ifidx,
	}
	_, err := c.conn.WriteTo(b, &cm, addr)
	return err
}

func (c *portableConn) SetReadDeadline(t time.Time) error {
	return c.conn.SetReadDeadline(t)
}

func (c *portableConn) SetWriteDeadline(t time.Time) error {
	return c.conn.SetWriteDeadline(t)
}
<commit_msg>Document the new non-interface Conn type.<commit_after>// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dhcp

import (
	"errors"
	"io"
	"net"
	"time"

	"golang.org/x/net/ipv4"
)

// defined as a var so tests can override it.
var (
	dhcpClientPort = 68
	platformConn   func(string) (conn, error)
)

// txType describes how a Packet should be sent on the wire.
type txType int

// The various transmission strategies described in RFC 2131. "MUST",
// "MUST NOT", "SHOULD" and "MAY" are as specified in RFC 2119.
const (
	// Packet MUST be broadcast.
	txBroadcast txType = iota
	// Packet MUST be unicasted to port 67 of RelayAddr
	txRelayAddr
	// Packet MUST be unicasted to port 68 of ClientAddr
	txClientAddr
	// Packet SHOULD be unicasted to port 68 of YourAddr, with the
	// link-layer destination explicitly set to HardwareAddr. You MUST
	// NOT rely on ARP resolution to discover the link-layer
	// destination address.
	//
	// Conn implementations that cannot explicitly set the link-layer
	// destination address MAY instead broadcast the packet.
	txHardwareAddr
)

type conn interface {
	io.Closer
	Recv([]byte) (b []byte, addr *net.UDPAddr, ifidx int, err error)
	Send(b []byte, addr *net.UDPAddr, ifidx int) error
	SetReadDeadline(t time.Time) error
	SetWriteDeadline(t time.Time) error
}

// Conn is a DHCP-oriented packet socket.
//
// Multiple goroutines may invoke methods on a Conn simultaneously.
type Conn struct {
	conn conn
}

// NewConn creates a Conn bound to the given UDP ip:port.
func NewConn(addr string) (*Conn, error) {
	if platformConn != nil {
		c, err := platformConn(addr)
		if err == nil {
			return &Conn{c}, nil
		}
	}
	// Always try falling back to the portable implementation
	c, err := newPortableConn(addr)
	if err != nil {
		return nil, err
	}
	return &Conn{c}, nil
}

// Close closes the DHCP socket.
// Any blocked Read or Write operations will be unblocked and return errors.
func (c *Conn) Close() error {
	return c.conn.Close()
}

// RecvDHCP reads a Packet from the connection. It returns the
// packet and the interface it was received on, which may be nil
// if interface information cannot be obtained.
func (c *Conn) RecvDHCP() (*Packet, *net.Interface, error) {
	var buf [1500]byte
	for {
		b, _, ifidx, err := c.conn.Recv(buf[:])
		if err != nil {
			return nil, nil, err
		}
		pkt, err := Unmarshal(b)
		if err != nil {
			continue
		}
		intf, err := net.InterfaceByIndex(ifidx)
		if err != nil {
			return nil, nil, err
		}
		// TODO: possibly more validation that the source lines up
		// with what the packet says.
		return pkt, intf, nil
	}
}

// SendDHCP sends pkt. The precise transmission mechanism depends
// on pkt.txType(). intf should be the net.Interface returned by
// RecvDHCP if responding to a DHCP client, or the interface for
// which configuration is desired if acting as a client.
func (c *Conn) SendDHCP(pkt *Packet, intf *net.Interface) error {
	b, err := pkt.Marshal()
	if err != nil {
		return err
	}

	switch pkt.txType() {
	case txBroadcast, txHardwareAddr:
		addr := net.UDPAddr{
			IP:   net.IPv4bcast,
			Port: dhcpClientPort,
		}
		return c.conn.Send(b, &addr, intf.Index)
	case txRelayAddr:
		addr := net.UDPAddr{
			IP:   pkt.RelayAddr,
			Port: 67,
		}
		return c.conn.Send(b, &addr, 0)
	case txClientAddr:
		addr := net.UDPAddr{
			IP:   pkt.ClientAddr,
			Port: dhcpClientPort,
		}
		return c.conn.Send(b, &addr, 0)
	default:
		return errors.New("unknown TX type for packet")
	}
}

// SetReadDeadline sets the deadline for future Read calls.  If the
// deadline is reached, Read will fail with a timeout (see net.Error)
// instead of blocking.  A zero value for t means Read will not time
// out.
func (c *Conn) SetReadDeadline(t time.Time) error {
	return c.conn.SetReadDeadline(t)
}

// SetWriteDeadline sets the deadline for future Write calls.  If the
// deadline is reached, Write will fail with a timeout (see net.Error)
// instead of blocking.  A zero value for t means Write will not time
// out.
func (c *Conn) SetWriteDeadline(t time.Time) error {
	return c.conn.SetWriteDeadline(t)
}

type portableConn struct {
	conn *ipv4.PacketConn
}

func newPortableConn(addr string) (conn, error) {
	c, err := net.ListenPacket("udp4", addr)
	if err != nil {
		return nil, err
	}
	l := ipv4.NewPacketConn(c)
	if err = l.SetControlMessage(ipv4.FlagInterface, true); err != nil {
		l.Close()
		return nil, err
	}
	return &portableConn{l}, nil
}

func (c *portableConn) Close() error {
	return c.conn.Close()
}

func (c *portableConn) Recv(b []byte) (rb []byte, addr *net.UDPAddr, ifidx int, err error) {
	n, cm, a, err := c.conn.ReadFrom(b)
	if err != nil {
		return nil, nil, 0, err
	}
	return b[:n], a.(*net.UDPAddr), cm.IfIndex, nil
}

func (c *portableConn) Send(b []byte, addr *net.UDPAddr, ifidx int) error {
	if ifidx > 0 {
		_, err := c.conn.WriteTo(b, nil, addr)
		return err
	}
	cm := ipv4.ControlMessage{
		IfIndex: ifidx,
	}
	_, err := c.conn.WriteTo(b, &cm, addr)
	return err
}

func (c *portableConn) SetReadDeadline(t time.Time) error {
	return c.conn.SetReadDeadline(t)
}

func (c *portableConn) SetWriteDeadline(t time.Time) error {
	return c.conn.SetWriteDeadline(t)
}
<|endoftext|>


<commit_before>package wats

import (
	"fmt"
	"io"
	"os"
	"strconv"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
)

var _ = Describe("Application Lifecycle", func() {
	reportedComputerNames := func(instances int) map[string]bool {
		timer := time.NewTimer(time.Second * 120)
		defer timer.Stop()
		run := true
		go func() {
			<-timer.C
			run = false
		}()

		seenComputerNames := map[string]bool{}
		for len(seenComputerNames) != instances && run == true {
			seenComputerNames[helpers.CurlApp(appName, "/ENV/CF_INSTANCE_IP")] = true
		}

		return seenComputerNames
	}

	Describe("An app staged on Diego and running on Diego", func() {
		It("attempts to forkbomb the environment", func() {
			numWinCells, err := strconv.Atoi(os.Getenv("NUM_WIN_CELLS"))
			Expect(err).NotTo(HaveOccurred(), "Please provide NUM_WIN_CELLS (The number of windows cells in tested deployment)")

			if numWinCells > 2 {
				Skip(fmt.Sprintf("Fork bomb test cannot run on more than 2 cells: found: %d\n"+
					"To run set the NUM_WIN_CELLS environment to 2 or less", numWinCells))
			}

			src, err := os.Open("../../assets/greenhouse-security-fixtures/bin/BreakoutBomb.exe")
			Expect(err).NotTo(HaveOccurred())
			defer src.Close()
			dst, err := os.Create("../../assets/nora/NoraPublished/bin/breakoutbomb.exe")
			Expect(err).NotTo(HaveOccurred())
			defer dst.Close()
			_, err = io.Copy(dst, src)
			Expect(err).NotTo(HaveOccurred())
			dst.Close()

			By("pushing it", func() {
				Eventually(pushNoraWithOptions(appName, numWinCells*2, "2G"), CF_PUSH_TIMEOUT).Should(Succeed())
			})

			By("staging and running it on Diego", func() {
				enableDiego(appName)
				Eventually(runCf("start", appName), CF_PUSH_TIMEOUT).Should(Succeed())
			})

			By("verifying it's up", func() {
				Eventually(helpers.CurlingAppRoot(appName)).Should(ContainSubstring("hello i am nora"))
			})

			By("storing the current computer names")
			computerNames := reportedComputerNames(numWinCells)
			Expect(len(computerNames)).To(Equal(numWinCells))

			By("Running fork bomb", func() {
				helpers.CurlApp(appName, "/run", "-f", "-X", "POST", "-d", "bin/breakoutbomb.exe")
			})

			time.Sleep(3 * time.Second)

			By("Making sure the bomb did not take down the machine", func() {
				newComputerNames := reportedComputerNames(numWinCells)
				Expect(newComputerNames).To(Equal(computerNames))
			})
		})
	})
})
<commit_msg>Ensure all instances are running for fork_bomb<commit_after>package wats

import (
	"fmt"
	"io"
	"os"
	"strconv"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
)

var _ = Describe("Application Lifecycle", func() {
	reportedComputerNames := func(instances int) map[string]bool {
		timer := time.NewTimer(time.Second * 120)
		defer timer.Stop()
		run := true
		go func() {
			<-timer.C
			run = false
		}()

		seenComputerNames := map[string]bool{}
		for len(seenComputerNames) != instances && run == true {
			seenComputerNames[helpers.CurlApp(appName, "/ENV/CF_INSTANCE_IP")] = true
		}

		return seenComputerNames
	}

	Describe("An app staged on Diego and running on Diego", func() {
		It("attempts to forkbomb the environment", func() {
			numWinCells, err := strconv.Atoi(os.Getenv("NUM_WIN_CELLS"))
			Expect(err).NotTo(HaveOccurred(), "Please provide NUM_WIN_CELLS (The number of windows cells in tested deployment)")

			if numWinCells > 2 {
				Skip(fmt.Sprintf("Fork bomb test cannot run on more than 2 cells: found: %d\n"+
					"To run set the NUM_WIN_CELLS environment to 2 or less", numWinCells))
			}

			src, err := os.Open("../../assets/greenhouse-security-fixtures/bin/BreakoutBomb.exe")
			Expect(err).NotTo(HaveOccurred())
			defer src.Close()
			dst, err := os.Create("../../assets/nora/NoraPublished/bin/breakoutbomb.exe")
			Expect(err).NotTo(HaveOccurred())
			defer dst.Close()
			_, err = io.Copy(dst, src)
			Expect(err).NotTo(HaveOccurred())
			dst.Close()

			By("pushing it", func() {
				Eventually(pushNoraWithOptions(appName, numWinCells*2, "2G"), CF_PUSH_TIMEOUT).Should(Succeed())
			})

			By("staging and running it on Diego", func() {
				enableDiego(appName)
				Eventually(runCf("start", appName), CF_PUSH_TIMEOUT).Should(Succeed())
			})

			By("verifying it's up", func() {
				Eventually(appRunning(appName, numWinCells*2, CF_PUSH_TIMEOUT), CF_PUSH_TIMEOUT).Should(Succeed())
				Eventually(helpers.CurlingAppRoot(appName)).Should(ContainSubstring("hello i am nora"))
			})

			By("storing the current computer names")
			computerNames := reportedComputerNames(numWinCells)
			Expect(len(computerNames)).To(Equal(numWinCells))

			By("Running fork bomb", func() {
				helpers.CurlApp(appName, "/run", "-f", "-X", "POST", "-d", "bin/breakoutbomb.exe")
			})

			time.Sleep(3 * time.Second)

			By("Making sure the bomb did not take down the machine", func() {
				newComputerNames := reportedComputerNames(numWinCells)
				Expect(newComputerNames).To(Equal(computerNames))
			})
		})
	})
})
<|endoftext|>