<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package latest

import (
	"fmt"

	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/api/registered"
	"k8s.io/kubernetes/pkg/runtime"
)

var (
	allGroups = GroupMetaMap{}
	// Group is a shortcut to allGroups.Group.
	Group = allGroups.Group
	// RegisterGroup is a shortcut to allGroups.RegisterGroup.
	RegisterGroup = allGroups.RegisterGroup
	// GroupOrDie is a shortcut to allGroups.GroupOrDie.
	GroupOrDie = allGroups.GroupOrDie
)

// GroupMetaMap is a map between group names and their metadata.
type GroupMetaMap map[string]*GroupMeta

// RegisterGroup registers a group to GroupMetaMap.
func (g GroupMetaMap) RegisterGroup(group string) (*GroupMeta, error) {
	_, found := g[group]
	if found {
		return nil, fmt.Errorf("group %v is already registered", g)
	}
	if len(registered.GroupVersionsForGroup(group)) == 0 {
		return nil, fmt.Errorf("No version is registered for group %v", group)
	}
	g[group] = &GroupMeta{}
	return g[group], nil
}

// Group returns the metadata of a group if the gruop is registered, otherwise
// an erorr is returned.
func (g GroupMetaMap) Group(group string) (*GroupMeta, error) {
	groupMeta, found := g[group]
	if !found {
		return nil, fmt.Errorf("no version is registered for group %v", group)
	}
	return groupMeta, nil
}

// TODO: This is an expedient function, because we don't check if a Group is
// supported throughout the code base. We will abandon this function and
// checking the error returned by the Group() function.
func (g GroupMetaMap) GroupOrDie(group string) *GroupMeta {
	groupMeta, found := g[group]
	if !found {
		panic(fmt.Sprintf("no version is registered for group %v", group))
	}
	return groupMeta
}

// GroupMeta stores the metadata of a group, such as the latest supported version.
type GroupMeta struct {
	// GroupVersion represents the current external default version of the group. It
	// is in the form of "group/version".
	GroupVersion string

	// Version represents the current external default version of the group.
	// It equals to the "version" part of GroupVersion.
	Version string

	// Group represents the name of the group
	Group string

	// Versions is the list of versions that are recognized in code. The order
	// provided is assumed to be from the oldest to the newest, e.g.,
	// Versions[0] == oldest and Versions[N-1] == newest.
	// Clients may choose to prefer the latter items in the list over the former
	// items when presented with a set of versions to choose.
	Versions []string

	// Codec is the default codec for serializing output that should use
	// the latest supported version.  Use this Codec when writing to
	// disk, a data store that is not dynamically versioned, or in tests.
	// This codec can decode any object that Kubernetes is aware of.
	Codec runtime.Codec

	// SelfLinker can set or get the SelfLink field of all API types.
	// TODO: when versioning changes, make this part of each API definition.
	// TODO(lavalamp): Combine SelfLinker & ResourceVersioner interfaces, force all uses
	// to go through the InterfacesFor method below.
	SelfLinker runtime.SelfLinker

	// RESTMapper provides the default mapping between REST paths and the objects declared in api.Scheme and all known
	// Kubernetes versions.
	RESTMapper meta.RESTMapper

	// InterfacesFor returns the default Codec and ResourceVersioner for a given version
	// string, or an error if the version is not known.
	InterfacesFor func(version string) (*meta.VersionInterfaces, error)
}
<commit_msg>improve the panic message of GroupOrDie<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package latest

import (
	"fmt"

	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/api/registered"
	"k8s.io/kubernetes/pkg/runtime"
)

var (
	allGroups = GroupMetaMap{}
	// Group is a shortcut to allGroups.Group.
	Group = allGroups.Group
	// RegisterGroup is a shortcut to allGroups.RegisterGroup.
	RegisterGroup = allGroups.RegisterGroup
	// GroupOrDie is a shortcut to allGroups.GroupOrDie.
	GroupOrDie = allGroups.GroupOrDie
)

// GroupMetaMap is a map between group names and their metadata.
type GroupMetaMap map[string]*GroupMeta

// RegisterGroup registers a group to GroupMetaMap.
func (g GroupMetaMap) RegisterGroup(group string) (*GroupMeta, error) {
	_, found := g[group]
	if found {
		return nil, fmt.Errorf("group %v is already registered", g)
	}
	if len(registered.GroupVersionsForGroup(group)) == 0 {
		return nil, fmt.Errorf("No version is registered for group %v", group)
	}
	g[group] = &GroupMeta{}
	return g[group], nil
}

// Group returns the metadata of a group if the gruop is registered, otherwise
// an erorr is returned.
func (g GroupMetaMap) Group(group string) (*GroupMeta, error) {
	groupMeta, found := g[group]
	if !found {
		return nil, fmt.Errorf("no version is registered for group %v", group)
	}
	return groupMeta, nil
}

// TODO: This is an expedient function, because we don't check if a Group is
// supported throughout the code base. We will abandon this function and
// checking the error returned by the Group() function.
func (g GroupMetaMap) GroupOrDie(group string) *GroupMeta {
	groupMeta, found := g[group]
	if !found {
		const msg = "Please check the KUBE_API_VERSIONS environment variable."
		if group == "" {
			panic("The legacy v1 API is not registered. " + msg)
		} else {
			panic(fmt.Sprintf("No version is registered for group %s. ", group) + msg)
		}
	}
	return groupMeta
}

// GroupMeta stores the metadata of a group, such as the latest supported version.
type GroupMeta struct {
	// GroupVersion represents the current external default version of the group. It
	// is in the form of "group/version".
	GroupVersion string

	// Version represents the current external default version of the group.
	// It equals to the "version" part of GroupVersion.
	Version string

	// Group represents the name of the group
	Group string

	// Versions is the list of versions that are recognized in code. The order
	// provided is assumed to be from the oldest to the newest, e.g.,
	// Versions[0] == oldest and Versions[N-1] == newest.
	// Clients may choose to prefer the latter items in the list over the former
	// items when presented with a set of versions to choose.
	Versions []string

	// Codec is the default codec for serializing output that should use
	// the latest supported version.  Use this Codec when writing to
	// disk, a data store that is not dynamically versioned, or in tests.
	// This codec can decode any object that Kubernetes is aware of.
	Codec runtime.Codec

	// SelfLinker can set or get the SelfLink field of all API types.
	// TODO: when versioning changes, make this part of each API definition.
	// TODO(lavalamp): Combine SelfLinker & ResourceVersioner interfaces, force all uses
	// to go through the InterfacesFor method below.
	SelfLinker runtime.SelfLinker

	// RESTMapper provides the default mapping between REST paths and the objects declared in api.Scheme and all known
	// Kubernetes versions.
	RESTMapper meta.RESTMapper

	// InterfacesFor returns the default Codec and ResourceVersioner for a given version
	// string, or an error if the version is not known.
	InterfacesFor func(version string) (*meta.VersionInterfaces, error)
}
<|endoftext|>


<commit_before>package frontend

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	"github.com/mchudgins/go-service-helper/pkg/loggingWriter"
	"github.com/mchudgins/go-service-helper/pkg/serveSwagger"
	"github.com/mchudgins/golang-service-starter/pkg/healthz"
	pb "github.com/mchudgins/golang-service-starter/pkg/service"
	"github.com/mchudgins/golang-service-starter/pkg/utils"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/spf13/cobra"
	"google.golang.org/grpc"
)

// frontendCmd represents the frontend command
var (
	swagger = MustAsset("pkg/service/service.swagger.json")
)

func serveSwaggerData(w http.ResponseWriter, r *http.Request) {
	if !strings.HasSuffix(r.URL.Path, ".swagger.json") {
		log.Printf("Not Found: %s", r.URL.Path)
		http.NotFound(w, r)
		return
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Write(swagger)
}

// allowCORS allows Cross Origin Resource Sharing from any origin.
// Don't do this without consideration in production systems.
func allowCORS(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if origin := r.Header.Get("Origin"); origin != "" {
			log.Printf("Origin: %s", origin)
			w.Header().Set("Access-Control-Allow-Origin", origin)
			if r.Method == "OPTIONS" && r.Header.Get("Access-Control-Request-Method") != "" {
				preflightHandler(w, r)
				return
			}
		}
		h.ServeHTTP(w, r)
	})
}

func preflightHandler(w http.ResponseWriter, r *http.Request) {
	headers := []string{"Content-Type", "Accept"}
	w.Header().Set("Access-Control-Allow-Headers", strings.Join(headers, ","))
	methods := []string{"GET", "HEAD", "POST", "PUT", "DELETE"}
	w.Header().Set("Access-Control-Allow-Methods", strings.Join(methods, ","))
	log.Printf("preflight request for %s", r.URL.Path)
	return
}

// Run the frontend command
func Run(cmd *cobra.Command, args []string) {
	utils.StartUpMessage()

	cfg, err := utils.NewAppConfig(cmd)
	if err != nil {
		log.Printf("Unable to initialize the application (%s).  Exiting now.", err)
	}

	// make a channel to listen on events,
	// then launch the servers.

	errc := make(chan error)

	// interrupt handler
	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
		errc <- fmt.Errorf("%s", <-c)
	}()

	// http server
	go func() {
		ctx := context.Background()
		ctx, cancel := context.WithCancel(ctx)
		defer cancel()

		mux := http.NewServeMux()
		gw := runtime.NewServeMux()

		hc, err := healthz.NewConfig(cfg)
		healthzHandler, err := healthz.Handler(hc)
		if err != nil {
			log.Panic(err)
		}

		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			fmt.Fprintf(w, `
{
"endpoints" :
  [
  "/api/v1/echo",
  "/healthz",
  "/metrics",
  "/swagger/service.swagger.json",
  "/swagger-ui/"
  ]
}
`)
		})

		secProxy, err := NewSecurityProxy("https://auth.dstcorp.io/login")
		if err != nil {
			log.Panic(err)
		}

		circuitBreaker, err := utils.NewHystrixHelper("grpc")
		if err != nil {
			log.Fatalf("Error creating circuitBreaker: %s", err)
		}

		mux.Handle("/api/v1/", secProxy.Handler(circuitBreaker.Handler(gw)))
		mux.Handle("/healthz", healthzHandler)
		mux.Handle("/metrics", prometheus.Handler())
		mux.HandleFunc("/swagger/", serveSwaggerData)

		swaggerProxy, _ := serveSwagger.NewSwaggerProxy("/swagger-ui/")
		mux.HandleFunc("/swagger-ui/", swaggerProxy.ServeHTTP)

		/*
		   http.HandleFunc("/v1/", func(w http.ResponseWriter, r *http.Request) {
		     log.Printf("/api+")
		     defer log.Printf("/api-")

		     gw.ServeHTTP(w, r)
		   })
		*/
		opts := []grpc.DialOption{grpc.WithInsecure()}
		err = pb.RegisterGreeterHandlerFromEndpoint(ctx, gw, cfg.GRPCListenAddress, opts)
		if err != nil {
			errc <- err
			return
		}

		log.Printf("HTTP service listening on %s", cfg.HTTPListenAddress)
		correlator := utils.NewCoreRequest()
		errc <- http.ListenAndServe(
			cfg.HTTPListenAddress,
			correlator.CorrelateRequest(loggingWriter.HttpLogger(allowCORS(mux))))
	}()

	// wait for somthin'
	log.Printf("exit: %s", <-errc)
}
<commit_msg>added hystrix metrics on /hystrix url<commit_after>package frontend

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/afex/hystrix-go/hystrix"
	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	"github.com/mchudgins/go-service-helper/pkg/loggingWriter"
	"github.com/mchudgins/go-service-helper/pkg/serveSwagger"
	"github.com/mchudgins/golang-service-starter/pkg/healthz"
	pb "github.com/mchudgins/golang-service-starter/pkg/service"
	"github.com/mchudgins/golang-service-starter/pkg/utils"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/spf13/cobra"
	"google.golang.org/grpc"
)

// frontendCmd represents the frontend command
var (
	swagger = MustAsset("pkg/service/service.swagger.json")
)

func serveSwaggerData(w http.ResponseWriter, r *http.Request) {
	if !strings.HasSuffix(r.URL.Path, ".swagger.json") {
		log.Printf("Not Found: %s", r.URL.Path)
		http.NotFound(w, r)
		return
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Write(swagger)
}

// allowCORS allows Cross Origin Resource Sharing from any origin.
// Don't do this without consideration in production systems.
func allowCORS(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if origin := r.Header.Get("Origin"); origin != "" {
			log.Printf("Origin: %s", origin)
			w.Header().Set("Access-Control-Allow-Origin", origin)
			if r.Method == "OPTIONS" && r.Header.Get("Access-Control-Request-Method") != "" {
				preflightHandler(w, r)
				return
			}
		}
		h.ServeHTTP(w, r)
	})
}

func preflightHandler(w http.ResponseWriter, r *http.Request) {
	headers := []string{"Content-Type", "Accept"}
	w.Header().Set("Access-Control-Allow-Headers", strings.Join(headers, ","))
	methods := []string{"GET", "HEAD", "POST", "PUT", "DELETE"}
	w.Header().Set("Access-Control-Allow-Methods", strings.Join(methods, ","))
	log.Printf("preflight request for %s", r.URL.Path)
	return
}

// Run the frontend command
func Run(cmd *cobra.Command, args []string) {
	utils.StartUpMessage()

	cfg, err := utils.NewAppConfig(cmd)
	if err != nil {
		log.Printf("Unable to initialize the application (%s).  Exiting now.", err)
	}

	// make a channel to listen on events,
	// then launch the servers.

	errc := make(chan error)

	// interrupt handler
	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
		errc <- fmt.Errorf("%s", <-c)
	}()

	// http server
	go func() {
		ctx := context.Background()
		ctx, cancel := context.WithCancel(ctx)
		defer cancel()

		mux := http.NewServeMux()
		gw := runtime.NewServeMux()

		hc, err := healthz.NewConfig(cfg)
		healthzHandler, err := healthz.Handler(hc)
		if err != nil {
			log.Panic(err)
		}

		// set up the backend's proxy
		secProxy, err := NewSecurityProxy("https://auth.dstcorp.io/login")
		if err != nil {
			log.Panic(err)
		}

		circuitBreaker, err := utils.NewHystrixHelper("grpc")
		if err != nil {
			log.Fatalf("Error creating circuitBreaker: %s", err)
		}

		mux.Handle("/api/v1/", secProxy.Handler(circuitBreaker.Handler(gw)))

		// now set up all the other, supporting url handlers for the frontend

		// this handler gives 'em a sitemap
		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			fmt.Fprintf(w, `
{
"endpoints" :
  [
  "/api/v1/echo",
  "/healthz",
  "/metrics",
  "/swagger/service.swagger.json",
  "/swagger-ui/"
  ]
}
`)
		})

		// add a /healthz endpoint to monitor this instance's health
		mux.Handle("/healthz", healthzHandler)

		// prometheus for metrics
		mux.Handle("/metrics", prometheus.Handler())

		// swagger for discovery
		mux.HandleFunc("/swagger/", serveSwaggerData)
		swaggerProxy, _ := serveSwagger.NewSwaggerProxy("/swagger-ui/")
		mux.HandleFunc("/swagger-ui/", swaggerProxy.ServeHTTP)

		// Enable hystrix dashboard metrics
		hystrixStreamHandler := hystrix.NewStreamHandler()
		hystrixStreamHandler.Start()

		mux.Handle("/hystrix", hystrixStreamHandler)

		/*
		   http.HandleFunc("/v1/", func(w http.ResponseWriter, r *http.Request) {
		     log.Printf("/api+")
		     defer log.Printf("/api-")

		     gw.ServeHTTP(w, r)
		   })
		*/
		opts := []grpc.DialOption{grpc.WithInsecure()}
		err = pb.RegisterGreeterHandlerFromEndpoint(ctx, gw, cfg.GRPCListenAddress, opts)
		if err != nil {
			errc <- err
			return
		}

		log.Printf("HTTP service listening on %s", cfg.HTTPListenAddress)
		correlator := utils.NewCoreRequest()
		errc <- http.ListenAndServe(
			cfg.HTTPListenAddress,
			correlator.CorrelateRequest(loggingWriter.HttpLogger(allowCORS(mux))))
	}()

	// wait for somethin'
	log.Printf("exit: %s", <-errc)
}
<|endoftext|>


<commit_before>package httpclient

import (
	"context"
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"time"

	"github.com/sirupsen/logrus"
)

// A Client is an HTTP client.
type Client struct {
	baseURL    string
	baseClient *http.Client
	opts       Options
}

// Option is a functional option for an HTTP client.
type Option func(opts *Options)

// Options are configuration options for an HTTP client.
type Options struct {
	Timeout         time.Duration
	TLS             *tls.Config
	Logger          *logrus.Logger
	ACSToken        string
	CheckRedirect   func(req *http.Request, via []*http.Request) error
	FailOnErrStatus bool
}

// ctxKey is a custom type to set values in request contexts.
type ctxKey int

// ctxKeyFailOnErrStatus is a request context key which, when sets, indicates that
// the HTTP client should return in error when it encounters an HTTP error (4XX / 5XX).
const ctxKeyFailOnErrStatus ctxKey = 0

// TLS sets the TLS configuration for the HTTP client transport.
func TLS(tlsConfig *tls.Config) Option {
	return func(opts *Options) {
		opts.TLS = tlsConfig
	}
}

// ACSToken sets the authentication token for HTTP requests.
func ACSToken(token string) Option {
	return func(opts *Options) {
		opts.ACSToken = token
	}
}

// Timeout sets the timeout for HTTP requests.
func Timeout(timeout time.Duration) Option {
	return func(opts *Options) {
		opts.Timeout = timeout
	}
}

// Logger sets the logger for the HTTP client.
func Logger(logger *logrus.Logger) Option {
	return func(opts *Options) {
		opts.Logger = logger
	}
}

// NoFollow prevents the client to follow redirect responses.
func NoFollow() Option {
	return func(opts *Options) {
		noFollow := func(req *http.Request, via []*http.Request) error {
			return http.ErrUseLastResponse
		}
		opts.CheckRedirect = noFollow
	}
}

// FailOnErrStatus specifies whether or not the client should fail on HTTP error response (4XX / 5XX).
func FailOnErrStatus(failOnErrStatus bool) Option {
	return func(opts *Options) {
		opts.FailOnErrStatus = failOnErrStatus
	}
}

// New returns a new HTTP client for a given baseURL and functional options.
func New(baseURL string, opts ...Option) *Client {
	// Default request timeout to 3 minutes. We don't use http.Client.Timeout on purpose as the
	// current approach allows to change the timeout on a per-request basis. The same client can
	// be shared for requests with different timeouts.
	options := Options{Timeout: 3 * time.Minute}

	for _, opt := range opts {
		opt(&options)
	}

	return &Client{
		baseURL: baseURL,
		baseClient: &http.Client{
			Transport: &http.Transport{

				// Allow http_proxy, https_proxy, and no_proxy.
				Proxy: http.ProxyFromEnvironment,

				// Set a 10 seconds timeout for the connection to be established.
				DialContext: (&net.Dialer{
					Timeout: 10 * time.Second,
				}).DialContext,

				// Set it to 10 seconds as well for the TLS handshake when using HTTPS.
				TLSHandshakeTimeout: 10 * time.Second,

				// The client will be dealing with a single host (the one in baseURL),
				// set max idle connections to 30 regardless of the host.
				MaxIdleConns:        30,
				MaxIdleConnsPerHost: 30,

				// Set the TLS configuration as specified in the context.
				TLSClientConfig: options.TLS,
			},

			// Specify the redirect policy for the client.
			CheckRedirect: options.CheckRedirect,
		},
		opts: options,
	}
}

// Get issues a GET to the specified DC/OS cluster path.
func (c *Client) Get(path string, opts ...Option) (*http.Response, error) {
	req, err := c.NewRequest("GET", path, nil, opts...)
	if err != nil {
		return nil, err
	}
	return c.Do(req)
}

// Post issues a POST to the specified DC/OS cluster path.
func (c *Client) Post(path string, contentType string, body io.Reader, opts ...Option) (*http.Response, error) {
	req, err := c.NewRequest("POST", path, body, opts...)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", contentType)
	return c.Do(req)
}

// Delete issues a DELETE to the specified DC/OS cluster path.
func (c *Client) Delete(path string, opts ...Option) (*http.Response, error) {
	req, err := c.NewRequest("DELETE", path, nil, opts...)
	if err != nil {
		return nil, err
	}
	return c.Do(req)
}

// NewRequest returns a new Request given a method, path, and optional body.
// Also adds the authorization header with the ACS token to work with the
// DC/OS cluster we are linked to if it has been set.
func (c *Client) NewRequest(method, path string, body io.Reader, opts ...Option) (*http.Request, error) {
	req, err := http.NewRequest(method, c.baseURL+path, body)
	if err != nil {
		return nil, err
	}

	options := c.opts
	for _, opt := range opts {
		opt(&options)
	}

	if options.ACSToken != "" {
		req.Header.Add("Authorization", "token="+options.ACSToken)
	}

	if options.FailOnErrStatus {
		ctx := context.WithValue(req.Context(), ctxKeyFailOnErrStatus, struct{}{})
		req = req.WithContext(ctx)
	}

	if options.Timeout > 0 {
		var cancel context.CancelFunc
		ctx, cancel := context.WithTimeout(req.Context(), options.Timeout)
		go func() {
			<-ctx.Done()
			cancel()
		}()
		req = req.WithContext(ctx)
	}
	return req, nil
}

// Do sends an HTTP request and returns an HTTP response, following
// policy (such as redirects, cookies, auth) as configured on the
// client.
func (c *Client) Do(req *http.Request) (*http.Response, error) {
	logger := c.opts.Logger

	if logger != nil && logger.Level >= logrus.DebugLevel {
		reqDump, err := httputil.DumpRequestOut(req, true)
		if err != nil {
			logger.Debugf("Couldn't dump request: %s", err)
		} else {
			logger.Debug(string(reqDump))
		}
	}

	resp, err := c.baseClient.Do(req)

	if logger != nil && logger.Level >= logrus.DebugLevel {
		if err == nil {
			respDump, err := httputil.DumpResponse(resp, true)
			if err != nil {
				logger.Debugf("Couldn't dump response: %s", err)
			} else {
				logger.Debug(string(respDump))
			}
		} else {
			logger.Debug(err)
		}
	}

	if err == nil {
		_, failOnErrStatus := req.Context().Value(ctxKeyFailOnErrStatus).(struct{})

		if failOnErrStatus && resp.StatusCode >= 400 && resp.StatusCode < 600 {
			return nil, fmt.Errorf("HTTP %d error", resp.StatusCode)
		}
	}
	return resp, err
}

// BaseURL returns the HTTP client's base URL.
func (c *Client) BaseURL() *url.URL {
	baseURL, err := url.Parse(c.baseURL)
	if err != nil && c.opts.Logger != nil {
		// We don't return error-out to keep the method signature clean.
		// If an http client contains an invalid URL it is broken anyway and
		// it is not this method responsibility to check this.
		c.opts.Logger.Debug(err)
	}
	return baseURL
}
<commit_msg>Only dump textual HTTP entity bodies<commit_after>package httpclient

import (
	"context"
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"time"

	"github.com/sirupsen/logrus"
)

// A Client is an HTTP client.
type Client struct {
	baseURL    string
	baseClient *http.Client
	opts       Options
}

// Option is a functional option for an HTTP client.
type Option func(opts *Options)

// Options are configuration options for an HTTP client.
type Options struct {
	Timeout         time.Duration
	TLS             *tls.Config
	Logger          *logrus.Logger
	ACSToken        string
	CheckRedirect   func(req *http.Request, via []*http.Request) error
	FailOnErrStatus bool
}

// ctxKey is a custom type to set values in request contexts.
type ctxKey int

// ctxKeyFailOnErrStatus is a request context key which, when sets, indicates that
// the HTTP client should return in error when it encounters an HTTP error (4XX / 5XX).
const ctxKeyFailOnErrStatus ctxKey = 0

// TLS sets the TLS configuration for the HTTP client transport.
func TLS(tlsConfig *tls.Config) Option {
	return func(opts *Options) {
		opts.TLS = tlsConfig
	}
}

// ACSToken sets the authentication token for HTTP requests.
func ACSToken(token string) Option {
	return func(opts *Options) {
		opts.ACSToken = token
	}
}

// Timeout sets the timeout for HTTP requests.
func Timeout(timeout time.Duration) Option {
	return func(opts *Options) {
		opts.Timeout = timeout
	}
}

// Logger sets the logger for the HTTP client.
func Logger(logger *logrus.Logger) Option {
	return func(opts *Options) {
		opts.Logger = logger
	}
}

// NoFollow prevents the client to follow redirect responses.
func NoFollow() Option {
	return func(opts *Options) {
		noFollow := func(req *http.Request, via []*http.Request) error {
			return http.ErrUseLastResponse
		}
		opts.CheckRedirect = noFollow
	}
}

// FailOnErrStatus specifies whether or not the client should fail on HTTP error response (4XX / 5XX).
func FailOnErrStatus(failOnErrStatus bool) Option {
	return func(opts *Options) {
		opts.FailOnErrStatus = failOnErrStatus
	}
}

// New returns a new HTTP client for a given baseURL and functional options.
func New(baseURL string, opts ...Option) *Client {
	// Default request timeout to 3 minutes. We don't use http.Client.Timeout on purpose as the
	// current approach allows to change the timeout on a per-request basis. The same client can
	// be shared for requests with different timeouts.
	options := Options{Timeout: 3 * time.Minute}

	for _, opt := range opts {
		opt(&options)
	}

	return &Client{
		baseURL: baseURL,
		baseClient: &http.Client{
			Transport: &http.Transport{

				// Allow http_proxy, https_proxy, and no_proxy.
				Proxy: http.ProxyFromEnvironment,

				// Set a 10 seconds timeout for the connection to be established.
				DialContext: (&net.Dialer{
					Timeout: 10 * time.Second,
				}).DialContext,

				// Set it to 10 seconds as well for the TLS handshake when using HTTPS.
				TLSHandshakeTimeout: 10 * time.Second,

				// The client will be dealing with a single host (the one in baseURL),
				// set max idle connections to 30 regardless of the host.
				MaxIdleConns:        30,
				MaxIdleConnsPerHost: 30,

				// Set the TLS configuration as specified in the context.
				TLSClientConfig: options.TLS,
			},

			// Specify the redirect policy for the client.
			CheckRedirect: options.CheckRedirect,
		},
		opts: options,
	}
}

// Get issues a GET to the specified DC/OS cluster path.
func (c *Client) Get(path string, opts ...Option) (*http.Response, error) {
	req, err := c.NewRequest("GET", path, nil, opts...)
	if err != nil {
		return nil, err
	}
	return c.Do(req)
}

// Post issues a POST to the specified DC/OS cluster path.
func (c *Client) Post(path string, contentType string, body io.Reader, opts ...Option) (*http.Response, error) {
	req, err := c.NewRequest("POST", path, body, opts...)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", contentType)
	return c.Do(req)
}

// Delete issues a DELETE to the specified DC/OS cluster path.
func (c *Client) Delete(path string, opts ...Option) (*http.Response, error) {
	req, err := c.NewRequest("DELETE", path, nil, opts...)
	if err != nil {
		return nil, err
	}
	return c.Do(req)
}

// NewRequest returns a new Request given a method, path, and optional body.
// Also adds the authorization header with the ACS token to work with the
// DC/OS cluster we are linked to if it has been set.
func (c *Client) NewRequest(method, path string, body io.Reader, opts ...Option) (*http.Request, error) {
	req, err := http.NewRequest(method, c.baseURL+path, body)
	if err != nil {
		return nil, err
	}

	options := c.opts
	for _, opt := range opts {
		opt(&options)
	}

	if options.ACSToken != "" {
		req.Header.Add("Authorization", "token="+options.ACSToken)
	}

	if options.FailOnErrStatus {
		ctx := context.WithValue(req.Context(), ctxKeyFailOnErrStatus, struct{}{})
		req = req.WithContext(ctx)
	}

	if options.Timeout > 0 {
		var cancel context.CancelFunc
		ctx, cancel := context.WithTimeout(req.Context(), options.Timeout)
		go func() {
			<-ctx.Done()
			cancel()
		}()
		req = req.WithContext(ctx)
	}
	return req, nil
}

// Do sends an HTTP request and returns an HTTP response, following
// policy (such as redirects, cookies, auth) as configured on the
// client.
func (c *Client) Do(req *http.Request) (*http.Response, error) {
	logger := c.opts.Logger

	if logger != nil && logger.Level >= logrus.DebugLevel {
		dumpBody := c.isText(req.Header.Get("Content-Type"))
		reqDump, err := httputil.DumpRequestOut(req, dumpBody)
		if err != nil {
			logger.Debugf("Couldn't dump request: %s", err)
		} else {
			logger.Debug(string(reqDump))
		}
	}

	resp, err := c.baseClient.Do(req)

	if logger != nil && logger.Level >= logrus.DebugLevel {
		if err == nil {
			dumpBody := c.isText(resp.Header.Get("Content-Type"))
			respDump, err := httputil.DumpResponse(resp, dumpBody)
			if err != nil {
				logger.Debugf("Couldn't dump response: %s", err)
			} else {
				logger.Debug(string(respDump))
			}
		} else {
			logger.Debug(err)
		}
	}

	if err == nil {
		_, failOnErrStatus := req.Context().Value(ctxKeyFailOnErrStatus).(struct{})

		if failOnErrStatus && resp.StatusCode >= 400 && resp.StatusCode < 600 {
			return nil, fmt.Errorf("HTTP %d error", resp.StatusCode)
		}
	}
	return resp, err
}

// BaseURL returns the HTTP client's base URL.
func (c *Client) BaseURL() *url.URL {
	baseURL, err := url.Parse(c.baseURL)
	if err != nil && c.opts.Logger != nil {
		// We don't return error-out to keep the method signature clean.
		// If an http client contains an invalid URL it is broken anyway and
		// it is not this method responsibility to check this.
		c.opts.Logger.Debug(err)
	}
	return baseURL
}

// isText returns whether the Content-type header refers to a textual body.
func (c *Client) isText(contentType string) bool {
	return contentType == "application/json" || strings.HasPrefix(contentType, "text/")
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package job

import (
	"github.com/spf13/cobra"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	arbv1 "github.com/kubernetes-incubator/kube-arbitrator/pkg/apis/v1"
	"github.com/kubernetes-incubator/kube-arbitrator/pkg/client/clientset"
)

type launchFlags struct {
	Name         string
	Namespace    string
	Image        string
	Master       string
	MinAvailable int
	Replicas     int
	Requests     string
}

var launchJobFlags = &launchFlags{}

func InitLaunchFlags(cmd *cobra.Command) {
	cmd.Flags().StringVarP(&launchJobFlags.Image, "image", "i", "busybox", "the container image of job")
	cmd.Flags().StringVarP(&launchJobFlags.Namespace, "namespace", "", "default", "the namespace of job")
	cmd.Flags().StringVarP(&launchJobFlags.Name, "name", "n", "test", "the name of job")
	cmd.Flags().IntVarP(&launchJobFlags.MinAvailable, "min", "m", 1, "the minimal available tasks of job")
	cmd.Flags().IntVarP(&launchJobFlags.Replicas, "replicas", "r", 1, "the total tasks of job")
	cmd.Flags().StringVarP(&launchJobFlags.Requests, "requests", "", "cpu=1000m,memory=100Mi", "the resource request of the task")
	cmd.Flags().StringVarP(&launchJobFlags.Master, "master", "s", "localhost:8080", "the address of api server")
}

func LaunchJob() {
	config, err := buildConfig(launchJobFlags.Master, "")
	if err != nil {
		panic(err)
	}

	queueClient := clientset.NewForConfigOrDie(config)

	req, err := populateResourceListV1(launchJobFlags.Requests)
	if err != nil {
		panic(err)
	}

	qj := &arbv1.QueueJob{
		ObjectMeta: metav1.ObjectMeta{
			Name:      launchJobFlags.Name,
			Namespace: launchJobFlags.Namespace,
		},
		Spec: arbv1.QueueJobSpec{
			Replicas: int32(launchJobFlags.Replicas),
			SchedSpec: arbv1.SchedulingSpecTemplate{
				MinAvailable: launchJobFlags.MinAvailable,
			},
			Template: v1.PodTemplateSpec{
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						{
							Image: launchJobFlags.Image,
							Name:  launchJobFlags.Name,
							Resources: v1.ResourceRequirements{
								Requests: req,
							},
						},
					},
				},
			},
		},
	}

	if _, err := queueClient.ArbV1().QueueJobs(launchJobFlags.Namespace).Create(qj); err != nil {
		panic(err)
	}
}
<commit_msg>Added scheduler name parameter to cli.<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package job

import (
	"github.com/spf13/cobra"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	arbv1 "github.com/kubernetes-incubator/kube-arbitrator/pkg/apis/v1"
	"github.com/kubernetes-incubator/kube-arbitrator/pkg/client/clientset"
)

type launchFlags struct {
	Name          string
	Namespace     string
	Image         string
	Master        string
	MinAvailable  int
	Replicas      int
	Requests      string
	SchedulerName string
}

var launchJobFlags = &launchFlags{}

func InitLaunchFlags(cmd *cobra.Command) {
	cmd.Flags().StringVarP(&launchJobFlags.Image, "image", "i", "busybox", "the container image of job")
	cmd.Flags().StringVarP(&launchJobFlags.Namespace, "namespace", "", "default", "the namespace of job")
	cmd.Flags().StringVarP(&launchJobFlags.Name, "name", "n", "test", "the name of job")
	cmd.Flags().IntVarP(&launchJobFlags.MinAvailable, "min", "m", 1, "the minimal available tasks of job")
	cmd.Flags().IntVarP(&launchJobFlags.Replicas, "replicas", "r", 1, "the total tasks of job")
	cmd.Flags().StringVarP(&launchJobFlags.Requests, "requests", "", "cpu=1000m,memory=100Mi", "the resource request of the task")
	cmd.Flags().StringVarP(&launchJobFlags.SchedulerName, "scheduler", "", "kar-scheduler", "the scheduler for this job")
	cmd.Flags().StringVarP(&launchJobFlags.Master, "master", "s", "localhost:8080", "the address of api server")
}

func LaunchJob() {
	config, err := buildConfig(launchJobFlags.Master, "")
	if err != nil {
		panic(err)
	}

	queueClient := clientset.NewForConfigOrDie(config)

	req, err := populateResourceListV1(launchJobFlags.Requests)
	if err != nil {
		panic(err)
	}

	qj := &arbv1.QueueJob{
		ObjectMeta: metav1.ObjectMeta{
			Name:      launchJobFlags.Name,
			Namespace: launchJobFlags.Namespace,
		},
		Spec: arbv1.QueueJobSpec{
			Replicas: int32(launchJobFlags.Replicas),
			SchedSpec: arbv1.SchedulingSpecTemplate{
				MinAvailable: launchJobFlags.MinAvailable,
			},
			Template: v1.PodTemplateSpec{
				Spec: v1.PodSpec{
					SchedulerName: launchJobFlags.SchedulerName,
					Containers: []v1.Container{
						{
							Image: launchJobFlags.Image,
							Name:  launchJobFlags.Name,
							Resources: v1.ResourceRequirements{
								Requests: req,
							},
						},
					},
				},
			},
		},
	}

	if _, err := queueClient.ArbV1().QueueJobs(launchJobFlags.Namespace).Create(qj); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package middlewares

import (
	"fmt"
	"runtime"

	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
)

type (
	// RecoverConfig defines the config for Recover middleware.
	RecoverConfig struct {
		// Skipper defines a function to skip middleware.
		Skipper middleware.Skipper

		// Size of the stack to be printed.
		// Optional. Default value 4KB.
		StackSize int `json:"stack_size"`
	}
)

// RecoverWithConfig returns a Recover middleware with config.
// See: `Recover()`.
func RecoverWithConfig(config RecoverConfig) echo.MiddlewareFunc {
	// Defaults
	if config.Skipper == nil {
		config.Skipper = middleware.DefaultSkipper
	}
	if config.StackSize == 0 {
		config.StackSize = 4 << 10 // 4 KB
	}

	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) error {
			if config.Skipper(c) {
				return next(c)
			}

			defer func() {
				if r := recover(); r != nil {
					var err error
					switch r := r.(type) {
					case error:
						err = r
					default:
						err = fmt.Errorf("%v", r)
					}
					stack := make([]byte, config.StackSize)
					length := runtime.Stack(stack, false)
					log := logger.WithDomain(c.Request().Host).WithField("panic", true)
					log.Errorf("PANIC RECOVER %s: %s", err.Error(), stack[:length])
					c.Error(err)
				}
			}()
			return next(c)
		}
	}
}
<commit_msg>Indents<commit_after>package middlewares

import (
	"fmt"
	"runtime"

	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
)

// RecoverConfig defines the config for Recover middleware.
type RecoverConfig struct {
	// Skipper defines a function to skip middleware.
	Skipper middleware.Skipper

	// Size of the stack to be printed.
	// Optional. Default value 4KB.
	StackSize int `json:"stack_size"`
}

// RecoverWithConfig returns a Recover middleware with config.
func RecoverWithConfig(config RecoverConfig) echo.MiddlewareFunc {
	// Defaults
	if config.Skipper == nil {
		config.Skipper = middleware.DefaultSkipper
	}
	if config.StackSize == 0 {
		config.StackSize = 4 << 10 // 4 KB
	}

	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) error {
			if config.Skipper(c) {
				return next(c)
			}

			defer func() {
				if r := recover(); r != nil {
					var err error
					switch r := r.(type) {
					case error:
						err = r
					default:
						err = fmt.Errorf("%v", r)
					}
					stack := make([]byte, config.StackSize)
					length := runtime.Stack(stack, false)
					log := logger.WithDomain(c.Request().Host).WithField("panic", true)
					log.Errorf("PANIC RECOVER %s: %s", err.Error(), stack[:length])
					c.Error(err)
				}
			}()
			return next(c)
		}
	}
}
<|endoftext|>


<commit_before>package models

import (
	"testing"

	"github.com/grafana/grafana/pkg/components/simplejson"
	. "github.com/smartystreets/goconvey/convey"
)

func TestAlertingModelTest(t *testing.T) {
	Convey("Testing Alerting model", t, func() {
		json1, err := simplejson.NewJson([]byte(`{ "field": "value" }`))
		So(err, ShouldBeNil)
		json2, err := simplejson.NewJson([]byte(`{ "field": "value" }`))
		So(err, ShouldBeNil)

		rule1 := &Alert{
			Settings: json1,
			Name:     "Namn",
			Message:  "Message",
		}

		rule2 := &Alert{
			Settings: json2,
			Name:     "Namn",
			Message:  "Message",
		}

		Convey("Testing AlertRule equals", func() {
			So(rule1.ContainsUpdates(rule2), ShouldBeFalse)
		})

		Convey("Changing the expression should contain update", func() {
			json2, err := simplejson.NewJson([]byte(`{ "field": "newValue" }`))
			So(err, ShouldBeNil)
			rule1.Settings = json2
			So(rule1.ContainsUpdates(rule2), ShouldBeTrue)
		})

		Convey("Should parse alertRule tags correctly", func() {
			json2, err := simplejson.NewJson([]byte(`{
				"field": "value",
				"alertRuleTags": {
					"foo": "bar",
					"waldo": "fred",
					"tagMap": { "mapValue": "value" }
				}
			}`))
			So(err, ShouldBeNil)
			rule1.Settings = json2
			expectedTags := []*Tag{
				{Id: 0, Key: "foo", Value: "bar"},
				{Id: 0, Key: "waldo", Value: "fred"},
				{Id: 0, Key: "tagMap", Value: ""},
			}
			actualTags := rule1.GetTagsFromSettings()

			So(len(actualTags), ShouldEqual, len(expectedTags))
			for _, tag := range expectedTags {
				So(ContainsTag(actualTags, tag), ShouldBeTrue)
			}
		})
	})
}
<commit_msg>Chore: Rewrite models alert test to standard library (#30021)<commit_after>package models

import (
	"testing"

	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestAlert_ContainsUpdates(t *testing.T) {
	settings, err := simplejson.NewJson([]byte(`{ "field": "value" }`))
	require.NoError(t, err)

	alert1 := &Alert{
		Settings: settings,
		Name:     "Name",
		Message:  "Message",
	}

	alert2 := &Alert{
		Settings: settings,
		Name:     "Name",
		Message:  "Message",
	}

	assert.False(t, alert1.ContainsUpdates(alert2))

	settingsUpdated, err := simplejson.NewJson([]byte(`{ "field": "newValue" }`))
	require.NoError(t, err)

	alert2.Settings = settingsUpdated

	assert.True(t, alert1.ContainsUpdates(alert2))
}

func TestAlert_GetTagsFromSettings(t *testing.T) {
	settings, err := simplejson.NewJson([]byte(`{
		"field": "value",
		"alertRuleTags": {
			"foo": "bar",
			"waldo": "fred",
			"tagMap": { "mapValue": "value" }
		}
	}`))
	require.NoError(t, err)

	alert := &Alert{
		Settings: settings,
		Name:     "Name",
		Message:  "Message",
	}

	expectedTags := []*Tag{
		{Id: 0, Key: "foo", Value: "bar"},
		{Id: 0, Key: "waldo", Value: "fred"},
		{Id: 0, Key: "tagMap", Value: ""},
	}
	actualTags := alert.GetTagsFromSettings()

	assert.ElementsMatch(t, actualTags, expectedTags)
}
<|endoftext|>


<commit_before>// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package arch

import (
	"gvisor.dev/gvisor/pkg/context"
	"gvisor.dev/gvisor/pkg/hostarch"
	"gvisor.dev/gvisor/pkg/marshal/primitive"

	"gvisor.dev/gvisor/pkg/usermem"
)

// Stack is a simple wrapper around a hostarch.IO and an address. Stack
// implements marshal.CopyContext, and marshallable values can be pushed or
// popped from the stack through the marshal.Marshallable interface.
//
// Stack is not thread-safe.
type Stack struct {
	// Our arch info.
	// We use this for automatic Native conversion of hostarch.Addrs during
	// Push() and Pop().
	Arch Context

	// The interface used to actually copy user memory.
	IO usermem.IO

	// Our current stack bottom.
	Bottom hostarch.Addr

	// Scratch buffer used for marshalling to avoid having to repeatedly
	// allocate scratch memory.
	scratchBuf []byte
}

// scratchBufLen is the default length of Stack.scratchBuf. The
// largest structs the stack regularly serializes are linux.SignalInfo
// and arch.UContext64. We'll set the default size as the larger of
// the two, arch.UContext64.
var scratchBufLen = (*UContext64)(nil).SizeBytes()

// CopyScratchBuffer implements marshal.CopyContext.CopyScratchBuffer.
func (s *Stack) CopyScratchBuffer(size int) []byte {
	if len(s.scratchBuf) < size {
		s.scratchBuf = make([]byte, size)
	}
	return s.scratchBuf[:size]
}

// StackBottomMagic is the special address callers must past to all stack
// marshalling operations to cause the src/dst address to be computed based on
// the current end of the stack.
const StackBottomMagic = ^hostarch.Addr(0) // hostarch.Addr(-1)

// CopyOutBytes implements marshal.CopyContext.CopyOutBytes. CopyOutBytes
// computes an appropriate address based on the current end of the
// stack. Callers use the sentinel address StackBottomMagic to marshal methods
// to indicate this.
func (s *Stack) CopyOutBytes(sentinel hostarch.Addr, b []byte) (int, error) {
	if sentinel != StackBottomMagic {
		panic("Attempted to copy out to stack with absolute address")
	}
	c := len(b)
	n, err := s.IO.CopyOut(context.Background(), s.Bottom-hostarch.Addr(c), b, usermem.IOOpts{})
	if err == nil && n == c {
		s.Bottom -= hostarch.Addr(n)
	}
	return n, err
}

// CopyInBytes implements marshal.CopyContext.CopyInBytes. CopyInBytes computes
// an appropriate address based on the current end of the stack. Callers must
// use the sentinel address StackBottomMagic to marshal methods to indicate
// this.
func (s *Stack) CopyInBytes(sentinel hostarch.Addr, b []byte) (int, error) {
	if sentinel != StackBottomMagic {
		panic("Attempted to copy in from stack with absolute address")
	}
	n, err := s.IO.CopyIn(context.Background(), s.Bottom, b, usermem.IOOpts{})
	if err == nil {
		s.Bottom += hostarch.Addr(n)
	}
	return n, err
}

// Align aligns the stack to the given offset.
func (s *Stack) Align(offset int) {
	if s.Bottom%hostarch.Addr(offset) != 0 {
		s.Bottom -= (s.Bottom % hostarch.Addr(offset))
	}
}

// PushNullTerminatedByteSlice writes bs to the stack, followed by an extra null
// byte at the end. On error, the contents of the stack and the bottom cursor
// are undefined.
func (s *Stack) PushNullTerminatedByteSlice(bs []byte) (int, error) {
	// Note: Stack grows up, so write the terminal null byte first.
	nNull, err := primitive.CopyUint8Out(s, StackBottomMagic, 0)
	if err != nil {
		return 0, err
	}
	n, err := primitive.CopyByteSliceOut(s, StackBottomMagic, bs)
	if err != nil {
		return 0, err
	}
	return n + nNull, nil
}

// StackLayout describes the location of the arguments and environment on the
// stack.
type StackLayout struct {
	// ArgvStart is the beginning of the argument vector.
	ArgvStart hostarch.Addr

	// ArgvEnd is the end of the argument vector.
	ArgvEnd hostarch.Addr

	// EnvvStart is the beginning of the environment vector.
	EnvvStart hostarch.Addr

	// EnvvEnd is the end of the environment vector.
	EnvvEnd hostarch.Addr
}

// Load pushes the given args, env and aux vector to the stack using the
// well-known format for a new executable. It returns the start and end
// of the argument and environment vectors.
func (s *Stack) Load(args []string, env []string, aux Auxv) (StackLayout, error) {
	l := StackLayout{}

	// Make sure we start with a 16-byte alignment.
	s.Align(16)

	// Push the environment vector so the end of the argument vector is adjacent to
	// the beginning of the environment vector.
	// While the System V abi for x86_64 does not specify an ordering to the
	// Information Block (the block holding the arg, env, and aux vectors),
	// support features like setproctitle(3) naturally expect these segments
	// to be in this order. See: https://www.uclibc.org/docs/psABI-x86_64.pdf
	// page 29.
	l.EnvvEnd = s.Bottom
	envAddrs := make([]hostarch.Addr, len(env))
	for i := len(env) - 1; i >= 0; i-- {
		if _, err := s.PushNullTerminatedByteSlice([]byte(env[i])); err != nil {
			return StackLayout{}, err
		}
		envAddrs[i] = s.Bottom
	}
	l.EnvvStart = s.Bottom

	// Push our strings.
	l.ArgvEnd = s.Bottom
	argAddrs := make([]hostarch.Addr, len(args))
	for i := len(args) - 1; i >= 0; i-- {
		if _, err := s.PushNullTerminatedByteSlice([]byte(args[i])); err != nil {
			return StackLayout{}, err
		}
		argAddrs[i] = s.Bottom
	}
	l.ArgvStart = s.Bottom

	// We need to align the arguments appropriately.
	//
	// We must finish on a 16-byte alignment, but we'll play it
	// conservatively and finish at 32-bytes. It would be nice to be able
	// to call Align here, but unfortunately we need to align the stack
	// with all the variable sized arrays pushed. So we just need to do
	// some calculations.
	argvSize := s.Arch.Width() * uint(len(args)+1)
	envvSize := s.Arch.Width() * uint(len(env)+1)
	auxvSize := s.Arch.Width() * 2 * uint(len(aux)+1)
	total := hostarch.Addr(argvSize) + hostarch.Addr(envvSize) + hostarch.Addr(auxvSize) + hostarch.Addr(s.Arch.Width())
	expectedBottom := s.Bottom - total
	if expectedBottom%32 != 0 {
		s.Bottom -= expectedBottom % 32
	}

	// Push our auxvec.
	// NOTE: We need an extra zero here per spec.
	// The Push function will automatically terminate
	// strings and arrays with a single null value.
	auxv := make([]hostarch.Addr, 0, len(aux))
	for _, a := range aux {
		auxv = append(auxv, hostarch.Addr(a.Key), a.Value)
	}
	auxv = append(auxv, hostarch.Addr(0))
	_, err := s.pushAddrSliceAndTerminator(auxv)
	if err != nil {
		return StackLayout{}, err
	}

	// Push environment.
	_, err = s.pushAddrSliceAndTerminator(envAddrs)
	if err != nil {
		return StackLayout{}, err
	}

	// Push args.
	_, err = s.pushAddrSliceAndTerminator(argAddrs)
	if err != nil {
		return StackLayout{}, err
	}

	// Push arg count.
	lenP := s.Arch.Native(uintptr(len(args)))
	if _, err = lenP.CopyOut(s, StackBottomMagic); err != nil {
		return StackLayout{}, err
	}

	return l, nil
}
<commit_msg>Allocate auxv with better capacity while loading stack<commit_after>// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package arch

import (
	"gvisor.dev/gvisor/pkg/context"
	"gvisor.dev/gvisor/pkg/hostarch"
	"gvisor.dev/gvisor/pkg/marshal/primitive"

	"gvisor.dev/gvisor/pkg/usermem"
)

// Stack is a simple wrapper around a hostarch.IO and an address. Stack
// implements marshal.CopyContext, and marshallable values can be pushed or
// popped from the stack through the marshal.Marshallable interface.
//
// Stack is not thread-safe.
type Stack struct {
	// Our arch info.
	// We use this for automatic Native conversion of hostarch.Addrs during
	// Push() and Pop().
	Arch Context

	// The interface used to actually copy user memory.
	IO usermem.IO

	// Our current stack bottom.
	Bottom hostarch.Addr

	// Scratch buffer used for marshalling to avoid having to repeatedly
	// allocate scratch memory.
	scratchBuf []byte
}

// scratchBufLen is the default length of Stack.scratchBuf. The
// largest structs the stack regularly serializes are linux.SignalInfo
// and arch.UContext64. We'll set the default size as the larger of
// the two, arch.UContext64.
var scratchBufLen = (*UContext64)(nil).SizeBytes()

// CopyScratchBuffer implements marshal.CopyContext.CopyScratchBuffer.
func (s *Stack) CopyScratchBuffer(size int) []byte {
	if len(s.scratchBuf) < size {
		s.scratchBuf = make([]byte, size)
	}
	return s.scratchBuf[:size]
}

// StackBottomMagic is the special address callers must past to all stack
// marshalling operations to cause the src/dst address to be computed based on
// the current end of the stack.
const StackBottomMagic = ^hostarch.Addr(0) // hostarch.Addr(-1)

// CopyOutBytes implements marshal.CopyContext.CopyOutBytes. CopyOutBytes
// computes an appropriate address based on the current end of the
// stack. Callers use the sentinel address StackBottomMagic to marshal methods
// to indicate this.
func (s *Stack) CopyOutBytes(sentinel hostarch.Addr, b []byte) (int, error) {
	if sentinel != StackBottomMagic {
		panic("Attempted to copy out to stack with absolute address")
	}
	c := len(b)
	n, err := s.IO.CopyOut(context.Background(), s.Bottom-hostarch.Addr(c), b, usermem.IOOpts{})
	if err == nil && n == c {
		s.Bottom -= hostarch.Addr(n)
	}
	return n, err
}

// CopyInBytes implements marshal.CopyContext.CopyInBytes. CopyInBytes computes
// an appropriate address based on the current end of the stack. Callers must
// use the sentinel address StackBottomMagic to marshal methods to indicate
// this.
func (s *Stack) CopyInBytes(sentinel hostarch.Addr, b []byte) (int, error) {
	if sentinel != StackBottomMagic {
		panic("Attempted to copy in from stack with absolute address")
	}
	n, err := s.IO.CopyIn(context.Background(), s.Bottom, b, usermem.IOOpts{})
	if err == nil {
		s.Bottom += hostarch.Addr(n)
	}
	return n, err
}

// Align aligns the stack to the given offset.
func (s *Stack) Align(offset int) {
	if s.Bottom%hostarch.Addr(offset) != 0 {
		s.Bottom -= (s.Bottom % hostarch.Addr(offset))
	}
}

// PushNullTerminatedByteSlice writes bs to the stack, followed by an extra null
// byte at the end. On error, the contents of the stack and the bottom cursor
// are undefined.
func (s *Stack) PushNullTerminatedByteSlice(bs []byte) (int, error) {
	// Note: Stack grows up, so write the terminal null byte first.
	nNull, err := primitive.CopyUint8Out(s, StackBottomMagic, 0)
	if err != nil {
		return 0, err
	}
	n, err := primitive.CopyByteSliceOut(s, StackBottomMagic, bs)
	if err != nil {
		return 0, err
	}
	return n + nNull, nil
}

// StackLayout describes the location of the arguments and environment on the
// stack.
type StackLayout struct {
	// ArgvStart is the beginning of the argument vector.
	ArgvStart hostarch.Addr

	// ArgvEnd is the end of the argument vector.
	ArgvEnd hostarch.Addr

	// EnvvStart is the beginning of the environment vector.
	EnvvStart hostarch.Addr

	// EnvvEnd is the end of the environment vector.
	EnvvEnd hostarch.Addr
}

// Load pushes the given args, env and aux vector to the stack using the
// well-known format for a new executable. It returns the start and end
// of the argument and environment vectors.
func (s *Stack) Load(args []string, env []string, aux Auxv) (StackLayout, error) {
	l := StackLayout{}

	// Make sure we start with a 16-byte alignment.
	s.Align(16)

	// Push the environment vector so the end of the argument vector is adjacent to
	// the beginning of the environment vector.
	// While the System V abi for x86_64 does not specify an ordering to the
	// Information Block (the block holding the arg, env, and aux vectors),
	// support features like setproctitle(3) naturally expect these segments
	// to be in this order. See: https://www.uclibc.org/docs/psABI-x86_64.pdf
	// page 29.
	l.EnvvEnd = s.Bottom
	envAddrs := make([]hostarch.Addr, len(env))
	for i := len(env) - 1; i >= 0; i-- {
		if _, err := s.PushNullTerminatedByteSlice([]byte(env[i])); err != nil {
			return StackLayout{}, err
		}
		envAddrs[i] = s.Bottom
	}
	l.EnvvStart = s.Bottom

	// Push our strings.
	l.ArgvEnd = s.Bottom
	argAddrs := make([]hostarch.Addr, len(args))
	for i := len(args) - 1; i >= 0; i-- {
		if _, err := s.PushNullTerminatedByteSlice([]byte(args[i])); err != nil {
			return StackLayout{}, err
		}
		argAddrs[i] = s.Bottom
	}
	l.ArgvStart = s.Bottom

	// We need to align the arguments appropriately.
	//
	// We must finish on a 16-byte alignment, but we'll play it
	// conservatively and finish at 32-bytes. It would be nice to be able
	// to call Align here, but unfortunately we need to align the stack
	// with all the variable sized arrays pushed. So we just need to do
	// some calculations.
	argvSize := s.Arch.Width() * uint(len(args)+1)
	envvSize := s.Arch.Width() * uint(len(env)+1)
	auxvSize := s.Arch.Width() * 2 * uint(len(aux)+1)
	total := hostarch.Addr(argvSize) + hostarch.Addr(envvSize) + hostarch.Addr(auxvSize) + hostarch.Addr(s.Arch.Width())
	expectedBottom := s.Bottom - total
	if expectedBottom%32 != 0 {
		s.Bottom -= expectedBottom % 32
	}

	// Push our auxvec.
	// NOTE: We need an extra zero here per spec.
	// The Push function will automatically terminate
	// strings and arrays with a single null value.
	auxv := make([]hostarch.Addr, 0, len(aux)*2+1)
	for _, a := range aux {
		auxv = append(auxv, hostarch.Addr(a.Key), a.Value)
	}
	auxv = append(auxv, hostarch.Addr(0))
	_, err := s.pushAddrSliceAndTerminator(auxv)
	if err != nil {
		return StackLayout{}, err
	}

	// Push environment.
	_, err = s.pushAddrSliceAndTerminator(envAddrs)
	if err != nil {
		return StackLayout{}, err
	}

	// Push args.
	_, err = s.pushAddrSliceAndTerminator(argAddrs)
	if err != nil {
		return StackLayout{}, err
	}

	// Push arg count.
	lenP := s.Arch.Native(uintptr(len(args)))
	if _, err = lenP.CopyOut(s, StackBottomMagic); err != nil {
		return StackLayout{}, err
	}

	return l, nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

// Plan is a description of a sequence of steps to produce a correct
// result for a query.

package simple

import (
	"log"

	"github.com/couchbaselabs/tuqtng/ast"
	"github.com/couchbaselabs/tuqtng/catalog"
	"github.com/couchbaselabs/tuqtng/plan"
)

type SimplePlanner struct {
	pool catalog.Pool
}

func NewSimplePlanner(pool catalog.Pool) *SimplePlanner {
	return &SimplePlanner{
		pool: pool,
	}
}

func (this *SimplePlanner) Plan(stmt ast.Statement) plan.PlanChannel {
	rv := make(plan.PlanChannel)
	go this.buildPlans(stmt, rv)
	return rv
}

func (this *SimplePlanner) buildPlans(stmt ast.Statement, pc plan.PlanChannel) {
	froms := stmt.GetFroms()

	var lastStep plan.PlanElement

	if froms == nil {
		// simple expression evaluation
		lastStep = plan.NewExpressionEvaluator()

	} else {
		from := froms[0]

		// see if the bucket exists
		if this.pool != nil {
			bucket, err := this.pool.Bucket(from.Bucket)
			if err != nil {
				log.Printf("no bucket named %v", from.Bucket)
				// no bucket, no plan
				close(pc)
				return
			}
			defer bucket.Release()

			// find all docs scanner
			scanners, err := bucket.Scanners()
			if err != nil {
				// no scanner, no plan
				close(pc)
				return
			}

			foundUsableScanner := false
			for _, scanner := range scanners {
				switch scanner.(type) {
				case catalog.FullScanner:
					lastStep = plan.NewScan(bucket.Name(), scanner.Name())
					lastStep = plan.NewFetch(lastStep, bucket.Name())
					if from.Projection != nil {
						if from.As == "" {
							// insert a project-inline phase to extrac the sub-document
							lastStep = plan.NewProjectorInline(lastStep, ast.NewResultExpression(from.Projection))
						} else {
							lastStep = plan.NewProjector(lastStep, ast.ResultExpressionList{ast.NewResultExpressionWithAlias(from.Projection, from.As)}, false)
						}
					}
					if len(froms) > 1 {
						// add document joins
						for _, from := range froms[1:] {
							lastStep = plan.NewDocumentJoin(lastStep, from.Projection, from.As)
						}
					}
					foundUsableScanner = true
					break
				}
			}

			if !foundUsableScanner {
				// no useable scanner
				close(pc)
				return
			}
		} else {
			// there is no pool
			close(pc)
			return
		}
	}

	if stmt.GetWhere() != nil {
		lastStep = plan.NewFilter(lastStep, stmt.GetWhere())
	}

	if stmt.GetOrderBy() != nil {
		lastStep = plan.NewOrder(lastStep, stmt.GetOrderBy())
	}

	if stmt.GetOffset() != 0 {
		lastStep = plan.NewOffset(lastStep, stmt.GetOffset())
	}

	if stmt.GetLimit() >= 0 {
		lastStep = plan.NewLimit(lastStep, stmt.GetLimit())
	}

	lastStep = plan.NewProjector(lastStep, stmt.GetResultExpressionList(), true)

	if stmt.IsDistinct() {
		lastStep = plan.NewEliminateDuplicates(lastStep)
	}

	pc <- plan.Plan{Root: lastStep}

	close(pc)
}
<commit_msg>fix bug where LIMIT/OFFSET were applied before distinct now they are applied last closes MB-8697<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

// Plan is a description of a sequence of steps to produce a correct
// result for a query.

package simple

import (
	"log"

	"github.com/couchbaselabs/tuqtng/ast"
	"github.com/couchbaselabs/tuqtng/catalog"
	"github.com/couchbaselabs/tuqtng/plan"
)

type SimplePlanner struct {
	pool catalog.Pool
}

func NewSimplePlanner(pool catalog.Pool) *SimplePlanner {
	return &SimplePlanner{
		pool: pool,
	}
}

func (this *SimplePlanner) Plan(stmt ast.Statement) plan.PlanChannel {
	rv := make(plan.PlanChannel)
	go this.buildPlans(stmt, rv)
	return rv
}

func (this *SimplePlanner) buildPlans(stmt ast.Statement, pc plan.PlanChannel) {
	froms := stmt.GetFroms()

	var lastStep plan.PlanElement

	if froms == nil {
		// simple expression evaluation
		lastStep = plan.NewExpressionEvaluator()

	} else {
		from := froms[0]

		// see if the bucket exists
		if this.pool != nil {
			bucket, err := this.pool.Bucket(from.Bucket)
			if err != nil {
				log.Printf("no bucket named %v", from.Bucket)
				// no bucket, no plan
				close(pc)
				return
			}
			defer bucket.Release()

			// find all docs scanner
			scanners, err := bucket.Scanners()
			if err != nil {
				// no scanner, no plan
				close(pc)
				return
			}

			foundUsableScanner := false
			for _, scanner := range scanners {
				switch scanner.(type) {
				case catalog.FullScanner:
					lastStep = plan.NewScan(bucket.Name(), scanner.Name())
					lastStep = plan.NewFetch(lastStep, bucket.Name())
					if from.Projection != nil {
						if from.As == "" {
							// insert a project-inline phase to extrac the sub-document
							lastStep = plan.NewProjectorInline(lastStep, ast.NewResultExpression(from.Projection))
						} else {
							lastStep = plan.NewProjector(lastStep, ast.ResultExpressionList{ast.NewResultExpressionWithAlias(from.Projection, from.As)}, false)
						}
					}
					if len(froms) > 1 {
						// add document joins
						for _, from := range froms[1:] {
							lastStep = plan.NewDocumentJoin(lastStep, from.Projection, from.As)
						}
					}
					foundUsableScanner = true
					break
				}
			}

			if !foundUsableScanner {
				// no useable scanner
				close(pc)
				return
			}
		} else {
			// there is no pool
			close(pc)
			return
		}
	}

	if stmt.GetWhere() != nil {
		lastStep = plan.NewFilter(lastStep, stmt.GetWhere())
	}

	if stmt.GetOrderBy() != nil {
		lastStep = plan.NewOrder(lastStep, stmt.GetOrderBy())
	}

	lastStep = plan.NewProjector(lastStep, stmt.GetResultExpressionList(), true)

	if stmt.IsDistinct() {
		lastStep = plan.NewEliminateDuplicates(lastStep)
	}

	if stmt.GetOffset() != 0 {
		lastStep = plan.NewOffset(lastStep, stmt.GetOffset())
	}

	if stmt.GetLimit() >= 0 {
		lastStep = plan.NewLimit(lastStep, stmt.GetLimit())
	}

	pc <- plan.Plan{Root: lastStep}

	close(pc)
}
<|endoftext|>


<commit_before>package irc

import (
	"fmt"

	"github.com/drone/drone/shared/model"
	"github.com/thoj/go-ircevent"
)

const (
	MessageStarted = "Building: %s, commit %s, author %s"
	MessageSuccess = "Success: %s, commit %s, author %s"
	MessageFailure = "Failed: %s, commit %s, author %s"
)

type IRC struct {
	Channel string
	Nick    string
	Server  string
	Started *bool `yaml:"on_started,omitempty"`
	Success *bool `yaml:"on_success,omitempty"`
	Failure *bool `yaml:"on_failure,omitempty"`
}

func (i *IRC) Send(req *model.Request) error {
	switch {
	case req.Commit.Status == model.StatusStarted && i.Started != nil && *i.Started == true:
		return i.sendStarted(req)
	case req.Commit.Status == model.StatusSuccess && i.Success != nil && *i.Success == true:
		return i.sendSuccess(req)
	case req.Commit.Status == model.StatusFailure && i.Failure != nil && *i.Failure == true:
		return i.sendFailure(req)
	}
	return nil
}

func (i *IRC) sendStarted(req *model.Request) error {
	msg := fmt.Sprintf(MessageStarted, req.Repo.Name, req.Commit.ShaShort(), req.Commit.Author)
	return i.send(i.Channel, msg)
}

func (i *IRC) sendFailure(req *model.Request) error {
	msg := fmt.Sprintf(MessageFailure, req.Repo.Name, req.Commit.ShaShort(), req.Commit.Author)
	return i.send(i.Channel, msg)
}

func (i *IRC) sendSuccess(req *model.Request) error {
	msg := fmt.Sprintf(MessageSuccess, req.Repo.Name, req.Commit.ShaShort(), req.Commit.Author)
	return i.send(i.Channel, msg)
}

// send is a helper function that will send notice messages
// to the connected IRC client
func (i *IRC) send(channel string, message string) error {
	client := irc.IRC(i.Nick, i.Nick)

	if client == nil {
		return fmt.Errorf("Error creating IRC client")
	}

	err := client.Connect(i.Server)

	if err != nil {
		return fmt.Errorf("Error connecting to IRC server: %v", err)
	}

	client.AddCallback("001", func(_ *irc.Event) {
		client.Notice(channel, message)
		client.Disconnect()
	})

	go client.Loop()

	return nil
}
<commit_msg>Fix IRC notification to use .Quit() instead of .Disconnect()<commit_after>package irc

import (
	"fmt"

	"github.com/drone/drone/shared/model"
	"github.com/thoj/go-ircevent"
)

const (
	MessageStarted = "Building: %s, commit %s, author %s"
	MessageSuccess = "Success: %s, commit %s, author %s"
	MessageFailure = "Failed: %s, commit %s, author %s"
)

type IRC struct {
	Channel string
	Nick    string
	Server  string
	Started *bool `yaml:"on_started,omitempty"`
	Success *bool `yaml:"on_success,omitempty"`
	Failure *bool `yaml:"on_failure,omitempty"`
}

func (i *IRC) Send(req *model.Request) error {
	switch {
	case req.Commit.Status == model.StatusStarted && i.Started != nil && *i.Started == true:
		return i.sendStarted(req)
	case req.Commit.Status == model.StatusSuccess && i.Success != nil && *i.Success == true:
		return i.sendSuccess(req)
	case req.Commit.Status == model.StatusFailure && i.Failure != nil && *i.Failure == true:
		return i.sendFailure(req)
	}
	return nil
}

func (i *IRC) sendStarted(req *model.Request) error {
	msg := fmt.Sprintf(MessageStarted, req.Repo.Name, req.Commit.ShaShort(), req.Commit.Author)
	return i.send(i.Channel, msg)
}

func (i *IRC) sendFailure(req *model.Request) error {
	msg := fmt.Sprintf(MessageFailure, req.Repo.Name, req.Commit.ShaShort(), req.Commit.Author)
	return i.send(i.Channel, msg)
}

func (i *IRC) sendSuccess(req *model.Request) error {
	msg := fmt.Sprintf(MessageSuccess, req.Repo.Name, req.Commit.ShaShort(), req.Commit.Author)
	return i.send(i.Channel, msg)
}

// send is a helper function that will send notice messages
// to the connected IRC client
func (i *IRC) send(channel string, message string) error {
	client := irc.IRC(i.Nick, i.Nick)

	if client == nil {
		return fmt.Errorf("Error creating IRC client")
	}

	err := client.Connect(i.Server)

	if err != nil {
		return fmt.Errorf("Error connecting to IRC server: %v", err)
	}

	client.AddCallback("001", func(_ *irc.Event) {
		client.Notice(channel, message)
		client.Quit()
	})

	go client.Loop()

	return nil
}
<|endoftext|>


<commit_before>/*
 * Copyright 2016 Frank Wessels <fwessels@xs4all.nl>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package kv

import (
	"encoding/hex"
	"fmt"
	"github.com/s3git/s3git-go/internal/config"
	"github.com/bmatsuo/lmdb-go/lmdb"
	"os"
	"path"
)

// TODO: Use new transaction style for lmdb

var env *lmdb.Env

// KV databases containing root level digests for different object types
// When a particular key is present, the value is as follows:
//   - when empty, underlying chunk(s) are not cached locally
//   - when set, it is the concatenation of the leaf level digests for all nodes
//     (and thus necessarily needs to correspond when BLAKE2'd to its key)
//
// If you know the type of the key, you can fetch it directly for the corresponding database
// If you do not know the type, you need to search all stores

var dbiLevel1Blobs lmdb.DBI
var dbiLevel1Commits lmdb.DBI
var dbiLevel1Prefixes lmdb.DBI
var dbiLevel1Trees lmdb.DBI
var dbiLevel1Snapshots lmdb.DBI

var dbiLevel0CacheSize lmdb.DBI
var dbiLevel0StageSize lmdb.DBI

// KV database containing overview of added/removed blobs in stage
var dbiStage lmdb.DBI

// KV database that marks commits objects as being a parent commit
var dbiLevel1CommitsIsParent lmdb.DBI

func OpenDatabase() error {

	mdbDir := path.Join(config.Config.BasePath, config.S3GIT_DIR, "mdb")
	err := os.MkdirAll(mdbDir, os.ModePerm)
	if err != nil {
		return err
	}

	env, _ = lmdb.NewEnv()
	// TODO: Figure out proper size for lmdb
	// TODO: Windows: max size is capped at 32
	env.SetMapSize(1 << 36) // max file size
	env.SetMaxDBs(10)       // up to 10 named databases
	env.Open(mdbDir, 0, 0664)

	err = env.Update(func(txn *lmdb.Txn) (err error) {

		// overview of blobs in stage
		dbiStage, err = txn.OpenDBI("stage", lmdb.Create)
		if err != nil {
			return err
		}

		// Level 1 databases
		dbiLevel1Blobs, err = txn.OpenDBI("l1blobs", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel1Commits, err = txn.OpenDBI("l1commits", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel1Prefixes, err = txn.OpenDBI("l1prefixes", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel1Trees, err = txn.OpenDBI("l1trees", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel1Snapshots, err = txn.OpenDBI("l1snapshots", lmdb.Create)
		if err != nil {
			return err
		}

		// Level 0 leaf databases
		dbiLevel0CacheSize, err = txn.OpenDBI("l0cache", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel0StageSize, err = txn.OpenDBI("l0stage", lmdb.Create)
		if err != nil {
			return err
		}

		// list of top most commits
		dbiLevel1CommitsIsParent, err = txn.OpenDBI("l1commitsisparent", lmdb.Create)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return err
	}

	// From https://godoc.org/github.com/bmatsuo/lmdb-go/lmdb
	//   A database is referenced by an opaque handle known as its DBI which
	//   must be opened inside a transaction with the OpenDBI or OpenRoot methods.
	//   DBIs may be closed but it is not required. Typically, applications acquire
	//   handles for all their databases immediately after opening an environment
	//   and retain them for the lifetime of the process.

	return nil
}

func AddToStage(key string) error {

	hx, _ := hex.DecodeString(key)

	err := env.Update(func(txn *lmdb.Txn) (err error) {
		return txn.Put(dbiStage, hx, nil, 0)
	})
	if err != nil {
		return err
	}

	return nil
}

func ClearStage() error {

	list, err := listMdb(&dbiStage, "")
	if err != nil {
		return err
	}

	txn, _ := env.BeginTxn(nil, 0)
	for k := range list {
		txn.Del(dbiStage, k, nil)
	}
	txn.Commit()

	return nil
}

func ListStage() (<-chan []byte, error) {

	return listMdb(&dbiStage, "")
}

func MarkCommitAsParent(key string) error {

	hx, _ := hex.DecodeString(key)

	txn, _ := env.BeginTxn(nil, 0)
	txn.Put(dbiLevel1CommitsIsParent, hx, nil, 0)
	txn.Commit()

	return nil
}

func CommitIsParent(key []byte) (bool, error) {

	txn, _ := env.BeginTxn(nil, lmdb.Readonly)
	defer txn.Abort()

	_, err := txn.Get(dbiLevel1CommitsIsParent, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return false, err
	} else if !lmdb.IsNotFound(err) {
		return true, err
	}

	return false, lmdb.NotFound
}

func ListTopMostCommits() (<-chan []byte, error) {

	list, err := ListLevel1Commits()
	if err != nil {
		return nil, err
	}

	result := make(chan []byte)

	go func() {
		// make sure we always close the channel
		defer close(result)

		for l := range list {
			isParent, err := CommitIsParent(l)
			if err != nil && !lmdb.IsNotFound(err) {
				return
			}

			if !isParent { // In case this commit is not a parent --> output it as a top most commit
				result <- l
			}
		}
	}()

	return result, nil
}

func ListLevel1Commits() (<-chan []byte, error) {

	return listMdb(&dbiLevel1Commits, "")
}

func ListLevel1Prefixes() (<-chan []byte, error) {

	return listMdb(&dbiLevel1Prefixes, "")
}

func ListLevel1Trees() (<-chan []byte, error) {

	return listMdb(&dbiLevel1Trees, "")
}

func ListLevel1Snapshots() (<-chan []byte, error) {

	return listMdb(&dbiLevel1Snapshots, "")
}

func ListLevel1Blobs(query string) (<-chan []byte, error) {

	return listMdb(&dbiLevel1Blobs, query)
}

func GetLevel1BlobsStats() (uint64, error) {

	txn, _ := env.BeginTxn(nil, lmdb.Readonly)
	defer txn.Abort()
	stats, err := txn.Stat(dbiLevel1Blobs)
	if err != nil {
		return 0, err
	}

	return stats.Entries, nil
}

const BLOB = "blob"
const COMMIT = "commit"
const PREFIX = "prefix"
const TREE = "tree"
const SNAPSHOT = "snapshot"

func getDbForObjectType(objType string) *lmdb.DBI {

	var dbi *lmdb.DBI
	switch objType {
	case BLOB:
		dbi = &dbiLevel1Blobs
	case COMMIT:
		dbi = &dbiLevel1Commits
	case PREFIX:
		dbi = &dbiLevel1Prefixes
	case TREE:
		dbi = &dbiLevel1Trees
	case SNAPSHOT:
		dbi = &dbiLevel1Snapshots
	default:
		panic(fmt.Sprintf("Bad type: %s", objType))
	}
	return dbi
}

func AddToLevel1(key, value []byte, objType string) error {

	dbi := getDbForObjectType(objType)
	txn, _ := env.BeginTxn(nil, 0)
	txn.Put(*dbi, key, value, 0)
	txn.Commit()

	return nil
}

func AddMultiToLevel1(keys, values [][]byte, objType string) error {

	dbi := getDbForObjectType(objType)
	txn, _ := env.BeginTxn(nil, 0)
	for index, key := range keys {
		txn.Put(*dbi, key, values[index], 0)
	}
	txn.Commit()

	return nil
}

// Get object of any type, return value and type
func GetLevel1(key []byte) ([]byte, string, error) {

	txn, _ := env.BeginTxn(nil, lmdb.Readonly)
	defer txn.Abort()

	val, err := txn.Get(dbiLevel1Blobs, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, BLOB, err
	}

	val, err = txn.Get(dbiLevel1Commits, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, COMMIT, err
	}

	val, err = txn.Get(dbiLevel1Prefixes, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, PREFIX, err
	}

	val, err = txn.Get(dbiLevel1Trees, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, TREE, err
	}

	val, err = txn.Get(dbiLevel1Snapshots, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, SNAPSHOT, err
	}

	return nil, "", lmdb.NotFound
}

func listMdb(dbi *lmdb.DBI, query string) (<-chan []byte, error) {

	result := make(chan []byte)

	go func() {
		// make sure we always close the channel
		defer close(result)

		// scan the database
		txn, _ := env.BeginTxn(nil, lmdb.Readonly)
		defer txn.Abort()
		cursor, _ := txn.OpenCursor(*dbi)
		defer cursor.Close()

		setRangeUponStart := len(query) > 0
		var queryKey []byte
		if setRangeUponStart {
			q := query
			if len(q)%2 == 1 {
				q = q + "0"
			}
			queryKey, _ = hex.DecodeString(q)
		}

		for {

			var bkey []byte
			if setRangeUponStart {
				var err error
				bkey, _, err = cursor.Get(queryKey, nil, lmdb.SetRange)
				if lmdb.IsNotFound(err) {
					break
				}
				if err != nil {
					fmt.Fprint(os.Stderr, err)
					return
				}

				setRangeUponStart = false
			} else {
				var err error
				bkey, _, err = cursor.Get(nil, nil, lmdb.Next)
				if lmdb.IsNotFound(err) {
					break
				}
				if err != nil {
					fmt.Fprint(os.Stderr, err)
					return
				}
			}

			// break early if start of key is not longer
			if hex.EncodeToString(bkey)[:len(query)] != query {
				break
			}
			result <- bkey
		}

	}()

	return result, nil
}
<commit_msg>Allow querying for commits<commit_after>/*
 * Copyright 2016 Frank Wessels <fwessels@xs4all.nl>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package kv

import (
	"encoding/hex"
	"fmt"
	"github.com/s3git/s3git-go/internal/config"
	"github.com/bmatsuo/lmdb-go/lmdb"
	"os"
	"path"
)

// TODO: Use new transaction style for lmdb

var env *lmdb.Env

// KV databases containing root level digests for different object types
// When a particular key is present, the value is as follows:
//   - when empty, underlying chunk(s) are not cached locally
//   - when set, it is the concatenation of the leaf level digests for all nodes
//     (and thus necessarily needs to correspond when BLAKE2'd to its key)
//
// If you know the type of the key, you can fetch it directly for the corresponding database
// If you do not know the type, you need to search all stores

var dbiLevel1Blobs lmdb.DBI
var dbiLevel1Commits lmdb.DBI
var dbiLevel1Prefixes lmdb.DBI
var dbiLevel1Trees lmdb.DBI
var dbiLevel1Snapshots lmdb.DBI

var dbiLevel0CacheSize lmdb.DBI
var dbiLevel0StageSize lmdb.DBI

// KV database containing overview of added/removed blobs in stage
var dbiStage lmdb.DBI

// KV database that marks commits objects as being a parent commit
var dbiLevel1CommitsIsParent lmdb.DBI

func OpenDatabase() error {

	mdbDir := path.Join(config.Config.BasePath, config.S3GIT_DIR, "mdb")
	err := os.MkdirAll(mdbDir, os.ModePerm)
	if err != nil {
		return err
	}

	env, _ = lmdb.NewEnv()
	// TODO: Figure out proper size for lmdb
	// TODO: Windows: max size is capped at 32
	env.SetMapSize(1 << 36) // max file size
	env.SetMaxDBs(10)       // up to 10 named databases
	env.Open(mdbDir, 0, 0664)

	err = env.Update(func(txn *lmdb.Txn) (err error) {

		// overview of blobs in stage
		dbiStage, err = txn.OpenDBI("stage", lmdb.Create)
		if err != nil {
			return err
		}

		// Level 1 databases
		dbiLevel1Blobs, err = txn.OpenDBI("l1blobs", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel1Commits, err = txn.OpenDBI("l1commits", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel1Prefixes, err = txn.OpenDBI("l1prefixes", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel1Trees, err = txn.OpenDBI("l1trees", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel1Snapshots, err = txn.OpenDBI("l1snapshots", lmdb.Create)
		if err != nil {
			return err
		}

		// Level 0 leaf databases
		dbiLevel0CacheSize, err = txn.OpenDBI("l0cache", lmdb.Create)
		if err != nil {
			return err
		}
		dbiLevel0StageSize, err = txn.OpenDBI("l0stage", lmdb.Create)
		if err != nil {
			return err
		}

		// list of top most commits
		dbiLevel1CommitsIsParent, err = txn.OpenDBI("l1commitsisparent", lmdb.Create)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return err
	}

	// From https://godoc.org/github.com/bmatsuo/lmdb-go/lmdb
	//   A database is referenced by an opaque handle known as its DBI which
	//   must be opened inside a transaction with the OpenDBI or OpenRoot methods.
	//   DBIs may be closed but it is not required. Typically, applications acquire
	//   handles for all their databases immediately after opening an environment
	//   and retain them for the lifetime of the process.

	return nil
}

func AddToStage(key string) error {

	hx, _ := hex.DecodeString(key)

	err := env.Update(func(txn *lmdb.Txn) (err error) {
		return txn.Put(dbiStage, hx, nil, 0)
	})
	if err != nil {
		return err
	}

	return nil
}

func ClearStage() error {

	list, err := listMdb(&dbiStage, "")
	if err != nil {
		return err
	}

	txn, _ := env.BeginTxn(nil, 0)
	for k := range list {
		txn.Del(dbiStage, k, nil)
	}
	txn.Commit()

	return nil
}

func ListStage() (<-chan []byte, error) {

	return listMdb(&dbiStage, "")
}

func MarkCommitAsParent(key string) error {

	hx, _ := hex.DecodeString(key)

	txn, _ := env.BeginTxn(nil, 0)
	txn.Put(dbiLevel1CommitsIsParent, hx, nil, 0)
	txn.Commit()

	return nil
}

func CommitIsParent(key []byte) (bool, error) {

	txn, _ := env.BeginTxn(nil, lmdb.Readonly)
	defer txn.Abort()

	_, err := txn.Get(dbiLevel1CommitsIsParent, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return false, err
	} else if !lmdb.IsNotFound(err) {
		return true, err
	}

	return false, lmdb.NotFound
}

func ListTopMostCommits() (<-chan []byte, error) {

	list, err := ListLevel1Commits("")
	if err != nil {
		return nil, err
	}

	result := make(chan []byte)

	go func() {
		// make sure we always close the channel
		defer close(result)

		for l := range list {
			isParent, err := CommitIsParent(l)
			if err != nil && !lmdb.IsNotFound(err) {
				return
			}

			if !isParent { // In case this commit is not a parent --> output it as a top most commit
				result <- l
			}
		}
	}()

	return result, nil
}

func ListLevel1Commits(query string) (<-chan []byte, error) {

	return listMdb(&dbiLevel1Commits, query)
}

func ListLevel1Prefixes() (<-chan []byte, error) {

	return listMdb(&dbiLevel1Prefixes, "")
}

func ListLevel1Trees() (<-chan []byte, error) {

	return listMdb(&dbiLevel1Trees, "")
}

func ListLevel1Snapshots() (<-chan []byte, error) {

	return listMdb(&dbiLevel1Snapshots, "")
}

func ListLevel1Blobs(query string) (<-chan []byte, error) {

	return listMdb(&dbiLevel1Blobs, query)
}

func GetLevel1BlobsStats() (uint64, error) {

	txn, _ := env.BeginTxn(nil, lmdb.Readonly)
	defer txn.Abort()
	stats, err := txn.Stat(dbiLevel1Blobs)
	if err != nil {
		return 0, err
	}

	return stats.Entries, nil
}

const BLOB = "blob"
const COMMIT = "commit"
const PREFIX = "prefix"
const TREE = "tree"
const SNAPSHOT = "snapshot"

func getDbForObjectType(objType string) *lmdb.DBI {

	var dbi *lmdb.DBI
	switch objType {
	case BLOB:
		dbi = &dbiLevel1Blobs
	case COMMIT:
		dbi = &dbiLevel1Commits
	case PREFIX:
		dbi = &dbiLevel1Prefixes
	case TREE:
		dbi = &dbiLevel1Trees
	case SNAPSHOT:
		dbi = &dbiLevel1Snapshots
	default:
		panic(fmt.Sprintf("Bad type: %s", objType))
	}
	return dbi
}

func AddToLevel1(key, value []byte, objType string) error {

	dbi := getDbForObjectType(objType)
	txn, _ := env.BeginTxn(nil, 0)
	txn.Put(*dbi, key, value, 0)
	txn.Commit()

	return nil
}

func AddMultiToLevel1(keys, values [][]byte, objType string) error {

	dbi := getDbForObjectType(objType)
	txn, _ := env.BeginTxn(nil, 0)
	for index, key := range keys {
		txn.Put(*dbi, key, values[index], 0)
	}
	txn.Commit()

	return nil
}

// Get object of any type, return value and type
func GetLevel1(key []byte) ([]byte, string, error) {

	txn, _ := env.BeginTxn(nil, lmdb.Readonly)
	defer txn.Abort()

	val, err := txn.Get(dbiLevel1Blobs, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, BLOB, err
	}

	val, err = txn.Get(dbiLevel1Commits, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, COMMIT, err
	}

	val, err = txn.Get(dbiLevel1Prefixes, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, PREFIX, err
	}

	val, err = txn.Get(dbiLevel1Trees, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, TREE, err
	}

	val, err = txn.Get(dbiLevel1Snapshots, key)
	if err != nil && !lmdb.IsNotFound(err) {
		return nil, "", err
	} else if !lmdb.IsNotFound(err) {
		return val, SNAPSHOT, err
	}

	return nil, "", lmdb.NotFound
}

func listMdb(dbi *lmdb.DBI, query string) (<-chan []byte, error) {

	result := make(chan []byte)

	go func() {
		// make sure we always close the channel
		defer close(result)

		// scan the database
		txn, _ := env.BeginTxn(nil, lmdb.Readonly)
		defer txn.Abort()
		cursor, _ := txn.OpenCursor(*dbi)
		defer cursor.Close()

		setRangeUponStart := len(query) > 0
		var queryKey []byte
		if setRangeUponStart {
			q := query
			if len(q)%2 == 1 {
				q = q + "0"
			}
			queryKey, _ = hex.DecodeString(q)
		}

		for {

			var bkey []byte
			if setRangeUponStart {
				var err error
				bkey, _, err = cursor.Get(queryKey, nil, lmdb.SetRange)
				if lmdb.IsNotFound(err) {
					break
				}
				if err != nil {
					fmt.Fprint(os.Stderr, err)
					return
				}

				setRangeUponStart = false
			} else {
				var err error
				bkey, _, err = cursor.Get(nil, nil, lmdb.Next)
				if lmdb.IsNotFound(err) {
					break
				}
				if err != nil {
					fmt.Fprint(os.Stderr, err)
					return
				}
			}

			// break early if start of key is not longer
			if hex.EncodeToString(bkey)[:len(query)] != query {
				break
			}
			result <- bkey
		}

	}()

	return result, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Bobby Powers. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package xmile

import (
	"crypto/rand"
	"encoding/xml"
	"fmt"
)

// the standard XML declaration, declared as a constant for easy
// reuse.
const XMLDeclaration = `<?xml version="1.0" encoding="utf-8" ?>`

// File represents the entire contents of a XMILE document.
type File struct {
	XMLName xml.Name `xml:"xmile"`
	Version string   `xml:"version,attr"`
	Level   int      `xml:"level,attr"`
	Header  Header   `xml:"header"`
	SimSpec SimSpec  `xml:"sim_specs"`
	Models  []*Model `xml:",omitempty"`
}

// Header contains metadata about a given XMILE File.
type Header struct {
	Name    string  `xml:"name"`
	UUID    string  `xml:"uuid"`
	Vendor  string  `xml:"vendor"`
	Product Product `xml:"product"`
}

// Product contains information about the software that created this
// XMILE document.
type Product struct {
	Name    string `xml:",chardata"`
	Version string `xml:"version,attr"`
	Lang    string `xml:"lang,attr"`
}

// SimSpec defines the time parameters a given model should be
// simulated with, or the defaults for all models defined in a given
// file.
type SimSpec struct {
	TimeUnits string  `xml:"time_units,attr,omitempty"`
	Start     float64 `xml:"start"`
	Stop      float64 `xml:"stop"`
	DT        float64 `xml:"dt"`
	Method    string  `xml:"method,omitempty"`
}

// Model represents a container for both the computational definition
// of a system dynamics model, as well as the visual representations
// of that model.
type Model struct {
	XMLName   xml.Name    `xml:"model"`
	Name      string      `xml:"name,attr,omitempty"`
	Variables []*Variable `xml:"variables>variable,omitempty"`
	Views     []*View     `xml:"views,omitempty>view,omitempty"`
}

// View is a collection of objects representing the visual structure
// of a model, such as a stock and flow diagram, a causal loop
// diagram, or the iThink interface layer.
type View struct {
	XMLName xml.Name `xml:"view"`
	Name    string   `xml:"name,attr,omitempty"`
}

// Variable is the definition of a model entity.  Some fields, such as
// Inflows and Outflows are only applicable for certain variable
// types.  The type is determined by the tag name and is stored in
// XMLName.Name.
type Variable struct {
	XMLName  xml.Name
	Name     string   `xml:"name,attr"`
	Equation string   `xml:"eqn"`
	Inflows  []string `xml:"inflow,omitempty"`  // empty for non-stocks
	Outflows []string `xml:"outflow,omitempty"` // empty for non-stocks
	Units    string   `xml:"units"`
}

// UUIDv4 returns a version 4 (random) variant of a UUID, or an error
// if it can not.
func UUIDv4() (string, error) {
	const uuidBytes = 16
	b := make([]byte, uuidBytes)

	n, err := rand.Read(b)
	if err != nil {
		return "", err
	} else if n != uuidBytes {
		return "", fmt.Errorf("rand.Read(): short read  of %d (wanted %d)", n, uuidBytes)
	}

	return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]), nil
}

// NewFile returns a new File object of the given XMILE compliance
// level and name, along with a new UUID.
func NewFile(level int, name string) *File {
	id, err := UUIDv4()
	if err != nil {
		// this is pretty frowned upon, but I don't want
		// NewFile's interface to potentially fail, and if
		// rand.Read fails we have bigger issues.
		panic(err)
	}

	f := &File{Version: "1.0", Level: level}
	f.Header = Header{
		Name:   "hello, xworld",
		UUID:   id,
		Vendor: "XMILE TC",
		Product: Product{
			Name:    "go-xmile",
			Version: "0.1",
			Lang:    "en",
		},
	}
	return f
}
<commit_msg>correctly use name in NewFile<commit_after>// Copyright 2013 Bobby Powers. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package xmile

import (
	"crypto/rand"
	"encoding/xml"
	"fmt"
)

// the standard XML declaration, declared as a constant for easy
// reuse.
const XMLDeclaration = `<?xml version="1.0" encoding="utf-8" ?>`

// File represents the entire contents of a XMILE document.
type File struct {
	XMLName xml.Name `xml:"xmile"`
	Version string   `xml:"version,attr"`
	Level   int      `xml:"level,attr"`
	Header  Header   `xml:"header"`
	SimSpec SimSpec  `xml:"sim_specs"`
	Models  []*Model `xml:",omitempty"`
}

// Header contains metadata about a given XMILE File.
type Header struct {
	Name    string  `xml:"name"`
	UUID    string  `xml:"uuid"`
	Vendor  string  `xml:"vendor"`
	Product Product `xml:"product"`
}

// Product contains information about the software that created this
// XMILE document.
type Product struct {
	Name    string `xml:",chardata"`
	Version string `xml:"version,attr"`
	Lang    string `xml:"lang,attr"`
}

// SimSpec defines the time parameters a given model should be
// simulated with, or the defaults for all models defined in a given
// file.
type SimSpec struct {
	TimeUnits string  `xml:"time_units,attr,omitempty"`
	Start     float64 `xml:"start"`
	Stop      float64 `xml:"stop"`
	DT        float64 `xml:"dt"`
	Method    string  `xml:"method,omitempty"`
}

// Model represents a container for both the computational definition
// of a system dynamics model, as well as the visual representations
// of that model.
type Model struct {
	XMLName   xml.Name    `xml:"model"`
	Name      string      `xml:"name,attr,omitempty"`
	Variables []*Variable `xml:"variables>variable,omitempty"`
	Views     []*View     `xml:"views,omitempty>view,omitempty"`
}

// View is a collection of objects representing the visual structure
// of a model, such as a stock and flow diagram, a causal loop
// diagram, or the iThink interface layer.
type View struct {
	XMLName xml.Name `xml:"view"`
	Name    string   `xml:"name,attr,omitempty"`
}

// Variable is the definition of a model entity.  Some fields, such as
// Inflows and Outflows are only applicable for certain variable
// types.  The type is determined by the tag name and is stored in
// XMLName.Name.
type Variable struct {
	XMLName  xml.Name
	Name     string   `xml:"name,attr"`
	Equation string   `xml:"eqn"`
	Inflows  []string `xml:"inflow,omitempty"`  // empty for non-stocks
	Outflows []string `xml:"outflow,omitempty"` // empty for non-stocks
	Units    string   `xml:"units"`
}

// UUIDv4 returns a version 4 (random) variant of a UUID, or an error
// if it can not.
func UUIDv4() (string, error) {
	const uuidBytes = 16
	b := make([]byte, uuidBytes)

	n, err := rand.Read(b)
	if err != nil {
		return "", err
	} else if n != uuidBytes {
		return "", fmt.Errorf("rand.Read(): short read  of %d (wanted %d)", n, uuidBytes)
	}

	return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]), nil
}

// NewFile returns a new File object of the given XMILE compliance
// level and name, along with a new UUID.
func NewFile(level int, name string) *File {
	id, err := UUIDv4()
	if err != nil {
		// this is pretty frowned upon, but I don't want
		// NewFile's interface to potentially fail, and if
		// rand.Read fails we have bigger issues.
		panic(err)
	}

	f := &File{Version: "1.0", Level: level}
	f.Header = Header{
		Name:   name,
		UUID:   id,
		Vendor: "XMILE TC",
		Product: Product{
			Name:    "go-xmile",
			Version: "0.1",
			Lang:    "en",
		},
	}
	return f
}
<|endoftext|>


<commit_before>// +build !windows

/*
 * Copyright 2017 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package y

import (
	"os"

	"golang.org/x/sys/unix"
)

// Mmap uses the mmap system call to memory-map a file. If writable is true,
// memory protection of the pages is set so that they may be written to as well.
func Mmap(fd *os.File, writable bool, size int64) ([]byte, error) {
	mtype := unix.PROT_READ
	if writable {
		mtype |= unix.PROT_WRITE
	}
	return unix.Mmap(int(fd.Fd()), 0, int(size), mtype, unix.MAP_SHARED)
}

// Munmap unmaps a previously mapped slice.
func Munmap(b []byte) error {
	return unix.Munmap(b)
}

// Madvise uses the madvise system call to give advise about the use of memory
// when using a slice that is memory-mapped to a file. Set the readahead flag to
// false if page references are expected in random order.
func Madvise(b []byte, readahead bool) error {
	flags := unix.MADV_NORMAL
	if !readahead {
		flags = unix.MADV_RANDOM
	}
	return unix.Madvise(b, flags)
}
<commit_msg>Use syscall interface to directly call madvise.<commit_after>// +build !windows

/*
 * Copyright 2017 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package y

import (
	"os"
	"syscall"
	"unsafe"

	"golang.org/x/sys/unix"
)

// Mmap uses the mmap system call to memory-map a file. If writable is true,
// memory protection of the pages is set so that they may be written to as well.
func Mmap(fd *os.File, writable bool, size int64) ([]byte, error) {
	mtype := unix.PROT_READ
	if writable {
		mtype |= unix.PROT_WRITE
	}
	return unix.Mmap(int(fd.Fd()), 0, int(size), mtype, unix.MAP_SHARED)
}

// Munmap unmaps a previously mapped slice.
func Munmap(b []byte) error {
	return unix.Munmap(b)
}

// Madvise uses the madvise system call to give advise about the use of memory
// when using a slice that is memory-mapped to a file. Set the readahead flag to
// false if page references are expected in random order.
func Madvise(b []byte, readahead bool) error {
	flags := unix.MADV_NORMAL
	if !readahead {
		flags = unix.MADV_RANDOM
	}
	return madvise(b, flags)
}

// This is required because the unix package does not support the madvise system call on OS X.
func madvise(b []byte, advice int) (err error) {
	_, _, e1 := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&b[0])),
		uintptr(len(b)), uintptr(advice))
	if e1 != 0 {
		err = e1
	}
	return
}
<|endoftext|>