<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
	"fmt"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/spf13/pflag"

	genericapiserver "k8s.io/apiserver/pkg/server"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	"k8s.io/kubernetes/pkg/kubeapiserver/authenticator"
	authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
)

type BuiltInAuthenticationOptions struct {
	Anonymous       *AnonymousAuthenticationOptions
	BootstrapToken  *BootstrapTokenAuthenticationOptions
	ClientCert      *genericoptions.ClientCertAuthenticationOptions
	Keystone        *KeystoneAuthenticationOptions
	OIDC            *OIDCAuthenticationOptions
	PasswordFile    *PasswordFileAuthenticationOptions
	RequestHeader   *genericoptions.RequestHeaderAuthenticationOptions
	ServiceAccounts *ServiceAccountAuthenticationOptions
	TokenFile       *TokenFileAuthenticationOptions
	WebHook         *WebHookAuthenticationOptions
}

type AnonymousAuthenticationOptions struct {
	Allow bool
}

type BootstrapTokenAuthenticationOptions struct {
	Allow bool
}

type KeystoneAuthenticationOptions struct {
	URL    string
	CAFile string
}

type OIDCAuthenticationOptions struct {
	CAFile        string
	ClientID      string
	IssuerURL     string
	UsernameClaim string
	GroupsClaim   string
}

type PasswordFileAuthenticationOptions struct {
	BasicAuthFile string
}

type ServiceAccountAuthenticationOptions struct {
	KeyFiles []string
	Lookup   bool
}

type TokenFileAuthenticationOptions struct {
	TokenFile string
}

type WebHookAuthenticationOptions struct {
	ConfigFile string
	CacheTTL   time.Duration
}

func NewBuiltInAuthenticationOptions() *BuiltInAuthenticationOptions {
	return &BuiltInAuthenticationOptions{}
}

func (s *BuiltInAuthenticationOptions) WithAll() *BuiltInAuthenticationOptions {
	return s.
		WithAnyonymous().
		WithBootstrapToken().
		WithClientCert().
		WithKeystone().
		WithOIDC().
		WithPasswordFile().
		WithRequestHeader().
		WithServiceAccounts().
		WithTokenFile().
		WithWebHook()
}

func (s *BuiltInAuthenticationOptions) WithAnyonymous() *BuiltInAuthenticationOptions {
	s.Anonymous = &AnonymousAuthenticationOptions{Allow: true}
	return s
}

func (s *BuiltInAuthenticationOptions) WithBootstrapToken() *BuiltInAuthenticationOptions {
	s.BootstrapToken = &BootstrapTokenAuthenticationOptions{}
	return s
}

func (s *BuiltInAuthenticationOptions) WithClientCert() *BuiltInAuthenticationOptions {
	s.ClientCert = &genericoptions.ClientCertAuthenticationOptions{}
	return s
}

func (s *BuiltInAuthenticationOptions) WithKeystone() *BuiltInAuthenticationOptions {
	s.Keystone = &KeystoneAuthenticationOptions{}
	return s
}

func (s *BuiltInAuthenticationOptions) WithOIDC() *BuiltInAuthenticationOptions {
	s.OIDC = &OIDCAuthenticationOptions{}
	return s
}

func (s *BuiltInAuthenticationOptions) WithPasswordFile() *BuiltInAuthenticationOptions {
	s.PasswordFile = &PasswordFileAuthenticationOptions{}
	return s
}

func (s *BuiltInAuthenticationOptions) WithRequestHeader() *BuiltInAuthenticationOptions {
	s.RequestHeader = &genericoptions.RequestHeaderAuthenticationOptions{}
	return s
}

func (s *BuiltInAuthenticationOptions) WithServiceAccounts() *BuiltInAuthenticationOptions {
	s.ServiceAccounts = &ServiceAccountAuthenticationOptions{Lookup: true}
	return s
}

func (s *BuiltInAuthenticationOptions) WithTokenFile() *BuiltInAuthenticationOptions {
	s.TokenFile = &TokenFileAuthenticationOptions{}
	return s
}

func (s *BuiltInAuthenticationOptions) WithWebHook() *BuiltInAuthenticationOptions {
	s.WebHook = &WebHookAuthenticationOptions{
		CacheTTL: 2 * time.Minute,
	}
	return s
}

// Validate checks invalid config combination
func (s *BuiltInAuthenticationOptions) Validate() []error {
	allErrors := []error{}

	if s.OIDC != nil && (len(s.OIDC.IssuerURL) > 0) != (len(s.OIDC.ClientID) > 0) {
		allErrors = append(allErrors, fmt.Errorf("oidc-issuer-url and oidc-client-id should be specified together"))
	}

	return allErrors
}

func (s *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
	if s.Anonymous != nil {
		fs.BoolVar(&s.Anonymous.Allow, "anonymous-auth", s.Anonymous.Allow, ""+
			"Enables anonymous requests to the secure port of the API server. "+
			"Requests that are not rejected by another authentication method are treated as anonymous requests. "+
			"Anonymous requests have a username of system:anonymous, and a group name of system:unauthenticated.")
	}

	if s.BootstrapToken != nil {
		fs.BoolVar(&s.BootstrapToken.Allow, "experimental-bootstrap-token-auth", s.BootstrapToken.Allow, ""+
			"Enable to allow secrets of type 'bootstrap.kubernetes.io/token' in the 'kube-system' "+
			"namespace to be used for TLS bootstrapping authentication.")
	}

	if s.ClientCert != nil {
		s.ClientCert.AddFlags(fs)
	}

	if s.Keystone != nil {
		fs.StringVar(&s.Keystone.URL, "experimental-keystone-url", s.Keystone.URL,
			"If passed, activates the keystone authentication plugin.")

		fs.StringVar(&s.Keystone.CAFile, "experimental-keystone-ca-file", s.Keystone.CAFile, ""+
			"If set, the Keystone server's certificate will be verified by one of the authorities "+
			"in the experimental-keystone-ca-file, otherwise the host's root CA set will be used.")
	}

	if s.OIDC != nil {
		fs.StringVar(&s.OIDC.IssuerURL, "oidc-issuer-url", s.OIDC.IssuerURL, ""+
			"The URL of the OpenID issuer, only HTTPS scheme will be accepted. "+
			"If set, it will be used to verify the OIDC JSON Web Token (JWT).")

		fs.StringVar(&s.OIDC.ClientID, "oidc-client-id", s.OIDC.ClientID,
			"The client ID for the OpenID Connect client, must be set if oidc-issuer-url is set.")

		fs.StringVar(&s.OIDC.CAFile, "oidc-ca-file", s.OIDC.CAFile, ""+
			"If set, the OpenID server's certificate will be verified by one of the authorities "+
			"in the oidc-ca-file, otherwise the host's root CA set will be used.")

		fs.StringVar(&s.OIDC.UsernameClaim, "oidc-username-claim", "sub", ""+
			"The OpenID claim to use as the user name. Note that claims other than the default ('sub') "+
			"is not guaranteed to be unique and immutable. This flag is experimental, please see "+
			"the authentication documentation for further details.")

		fs.StringVar(&s.OIDC.GroupsClaim, "oidc-groups-claim", "", ""+
			"If provided, the name of a custom OpenID Connect claim for specifying user groups. "+
			"The claim value is expected to be a string or array of strings. This flag is experimental, "+
			"please see the authentication documentation for further details.")
	}

	if s.PasswordFile != nil {
		fs.StringVar(&s.PasswordFile.BasicAuthFile, "basic-auth-file", s.PasswordFile.BasicAuthFile, ""+
			"If set, the file that will be used to admit requests to the secure port of the API server "+
			"via http basic authentication.")
	}

	if s.RequestHeader != nil {
		s.RequestHeader.AddFlags(fs)
	}

	if s.ServiceAccounts != nil {
		fs.StringArrayVar(&s.ServiceAccounts.KeyFiles, "service-account-key-file", s.ServiceAccounts.KeyFiles, ""+
			"File containing PEM-encoded x509 RSA or ECDSA private or public keys, used to verify "+
			"ServiceAccount tokens. If unspecified, --tls-private-key-file is used. "+
			"The specified file can contain multiple keys, and the flag can be specified multiple times with different files.")

		fs.BoolVar(&s.ServiceAccounts.Lookup, "service-account-lookup", s.ServiceAccounts.Lookup,
			"If true, validate ServiceAccount tokens exist in etcd as part of authentication.")
	}

	if s.TokenFile != nil {
		fs.StringVar(&s.TokenFile.TokenFile, "token-auth-file", s.TokenFile.TokenFile, ""+
			"If set, the file that will be used to secure the secure port of the API server "+
			"via token authentication.")
	}

	if s.WebHook != nil {
		fs.StringVar(&s.WebHook.ConfigFile, "authentication-token-webhook-config-file", s.WebHook.ConfigFile, ""+
			"File with webhook configuration for token authentication in kubeconfig format. "+
			"The API server will query the remote service to determine authentication for bearer tokens.")

		fs.DurationVar(&s.WebHook.CacheTTL, "authentication-token-webhook-cache-ttl", s.WebHook.CacheTTL,
			"The duration to cache responses from the webhook token authenticator.")
	}
}

func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() authenticator.AuthenticatorConfig {
	ret := authenticator.AuthenticatorConfig{}

	if s.Anonymous != nil {
		ret.Anonymous = s.Anonymous.Allow
	}

	if s.BootstrapToken != nil {
		ret.BootstrapToken = s.BootstrapToken.Allow
	}

	if s.ClientCert != nil {
		ret.ClientCAFile = s.ClientCert.ClientCA
	}

	if s.Keystone != nil {
		ret.KeystoneURL = s.Keystone.URL
		ret.KeystoneCAFile = s.Keystone.CAFile
	}

	if s.OIDC != nil {
		ret.OIDCCAFile = s.OIDC.CAFile
		ret.OIDCClientID = s.OIDC.ClientID
		ret.OIDCGroupsClaim = s.OIDC.GroupsClaim
		ret.OIDCIssuerURL = s.OIDC.IssuerURL
		ret.OIDCUsernameClaim = s.OIDC.UsernameClaim
	}

	if s.PasswordFile != nil {
		ret.BasicAuthFile = s.PasswordFile.BasicAuthFile
	}

	if s.RequestHeader != nil {
		ret.RequestHeaderConfig = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
	}

	if s.ServiceAccounts != nil {
		ret.ServiceAccountKeyFiles = s.ServiceAccounts.KeyFiles
		ret.ServiceAccountLookup = s.ServiceAccounts.Lookup
	}

	if s.TokenFile != nil {
		ret.TokenAuthFile = s.TokenFile.TokenFile
	}

	if s.WebHook != nil {
		ret.WebhookTokenAuthnConfigFile = s.WebHook.ConfigFile
		ret.WebhookTokenAuthnCacheTTL = s.WebHook.CacheTTL
	}

	return ret
}

func (o *BuiltInAuthenticationOptions) ApplyTo(c *genericapiserver.Config) error {
	if o == nil {
		return nil
	}

	var err error
	if o.ClientCert != nil {
		c, err = c.ApplyClientCert(o.ClientCert.ClientCA)
		if err != nil {
			return fmt.Errorf("unable to load client CA file: %v", err)
		}
	}
	if o.RequestHeader != nil {
		c, err = c.ApplyClientCert(o.RequestHeader.ClientCAFile)
		if err != nil {
			return fmt.Errorf("unable to load client CA file: %v", err)
		}
	}

	c.SupportsBasicAuth = o.PasswordFile != nil && len(o.PasswordFile.BasicAuthFile) > 0

	return nil
}

// ApplyAuthorization will conditionally modify the authentication options based on the authorization options
func (o *BuiltInAuthenticationOptions) ApplyAuthorization(authorization *BuiltInAuthorizationOptions) {
	if o == nil || authorization == nil || o.Anonymous == nil {
		return
	}

	// authorization ModeAlwaysAllow cannot be combined with AnonymousAuth.
	// in such a case the AnonymousAuth is stomped to false and you get a message
	if o.Anonymous.Allow {
		found := false
		for _, mode := range strings.Split(authorization.Mode, ",") {
			if mode == authzmodes.ModeAlwaysAllow {
				found = true
				break
			}
		}
		if found {
			glog.Warningf("AnonymousAuth is not allowed with the AllowAll authorizer.  Resetting AnonymousAuth to false. You should use a different authorizer")
			o.Anonymous.Allow = false
		}
	}
}
<commit_msg>Fix the method name of BuiltInAuthenticationOptions<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
	"fmt"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/spf13/pflag"

	genericapiserver "k8s.io/apiserver/pkg/server"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	"k8s.io/kubernetes/pkg/kubeapiserver/authenticator"
	authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
)

type BuiltInAuthenticationOptions struct {
	Anonymous       *AnonymousAuthenticationOptions
	BootstrapToken  *BootstrapTokenAuthenticationOptions
	ClientCert      *genericoptions.ClientCertAuthenticationOptions
	Keystone        *KeystoneAuthenticationOptions
	OIDC            *OIDCAuthenticationOptions
	PasswordFile    *PasswordFileAuthenticationOptions
	RequestHeader   *genericoptions.RequestHeaderAuthenticationOptions
	ServiceAccounts *ServiceAccountAuthenticationOptions
	TokenFile       *TokenFileAuthenticationOptions
	WebHook         *WebHookAuthenticationOptions
}

type AnonymousAuthenticationOptions struct {
	Allow bool
}

type BootstrapTokenAuthenticationOptions struct {
	Allow bool
}

type KeystoneAuthenticationOptions struct {
	URL    string
	CAFile string
}

type OIDCAuthenticationOptions struct {
	CAFile        string
	ClientID      string
	IssuerURL     string
	UsernameClaim string
	GroupsClaim   string
}

type PasswordFileAuthenticationOptions struct {
	BasicAuthFile string
}

type ServiceAccountAuthenticationOptions struct {
	KeyFiles []string
	Lookup   bool
}

type TokenFileAuthenticationOptions struct {
	TokenFile string
}

type WebHookAuthenticationOptions struct {
	ConfigFile string
	CacheTTL   time.Duration
}

func NewBuiltInAuthenticationOptions() *BuiltInAuthenticationOptions {
	return &BuiltInAuthenticationOptions{}
}

func (s *BuiltInAuthenticationOptions) WithAll() *BuiltInAuthenticationOptions {
	return s.
		WithAnonymous().
		WithBootstrapToken().
		WithClientCert().
		WithKeystone().
		WithOIDC().
		WithPasswordFile().
		WithRequestHeader().
		WithServiceAccounts().
		WithTokenFile().
		WithWebHook()
}

func (s *BuiltInAuthenticationOptions) WithAnonymous() *BuiltInAuthenticationOptions {
	s.Anonymous = &AnonymousAuthenticationOptions{Allow: true}
	return s
}

func (s *BuiltInAuthenticationOptions) WithBootstrapToken() *BuiltInAuthenticationOptions {
	s.BootstrapToken = &BootstrapTokenAuthenticationOptions{}
	return s
}

func (s *BuiltInAuthenticationOptions) WithClientCert() *BuiltInAuthenticationOptions {
	s.ClientCert = &genericoptions.ClientCertAuthenticationOptions{}
	return s
}

func (s *BuiltInAuthenticationOptions) WithKeystone() *BuiltInAuthenticationOptions {
	s.Keystone = &KeystoneAuthenticationOptions{}
	return s
}

func (s *BuiltInAuthenticationOptions) WithOIDC() *BuiltInAuthenticationOptions {
	s.OIDC = &OIDCAuthenticationOptions{}
	return s
}

func (s *BuiltInAuthenticationOptions) WithPasswordFile() *BuiltInAuthenticationOptions {
	s.PasswordFile = &PasswordFileAuthenticationOptions{}
	return s
}

func (s *BuiltInAuthenticationOptions) WithRequestHeader() *BuiltInAuthenticationOptions {
	s.RequestHeader = &genericoptions.RequestHeaderAuthenticationOptions{}
	return s
}

func (s *BuiltInAuthenticationOptions) WithServiceAccounts() *BuiltInAuthenticationOptions {
	s.ServiceAccounts = &ServiceAccountAuthenticationOptions{Lookup: true}
	return s
}

func (s *BuiltInAuthenticationOptions) WithTokenFile() *BuiltInAuthenticationOptions {
	s.TokenFile = &TokenFileAuthenticationOptions{}
	return s
}

func (s *BuiltInAuthenticationOptions) WithWebHook() *BuiltInAuthenticationOptions {
	s.WebHook = &WebHookAuthenticationOptions{
		CacheTTL: 2 * time.Minute,
	}
	return s
}

// Validate checks invalid config combination
func (s *BuiltInAuthenticationOptions) Validate() []error {
	allErrors := []error{}

	if s.OIDC != nil && (len(s.OIDC.IssuerURL) > 0) != (len(s.OIDC.ClientID) > 0) {
		allErrors = append(allErrors, fmt.Errorf("oidc-issuer-url and oidc-client-id should be specified together"))
	}

	return allErrors
}

func (s *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
	if s.Anonymous != nil {
		fs.BoolVar(&s.Anonymous.Allow, "anonymous-auth", s.Anonymous.Allow, ""+
			"Enables anonymous requests to the secure port of the API server. "+
			"Requests that are not rejected by another authentication method are treated as anonymous requests. "+
			"Anonymous requests have a username of system:anonymous, and a group name of system:unauthenticated.")
	}

	if s.BootstrapToken != nil {
		fs.BoolVar(&s.BootstrapToken.Allow, "experimental-bootstrap-token-auth", s.BootstrapToken.Allow, ""+
			"Enable to allow secrets of type 'bootstrap.kubernetes.io/token' in the 'kube-system' "+
			"namespace to be used for TLS bootstrapping authentication.")
	}

	if s.ClientCert != nil {
		s.ClientCert.AddFlags(fs)
	}

	if s.Keystone != nil {
		fs.StringVar(&s.Keystone.URL, "experimental-keystone-url", s.Keystone.URL,
			"If passed, activates the keystone authentication plugin.")

		fs.StringVar(&s.Keystone.CAFile, "experimental-keystone-ca-file", s.Keystone.CAFile, ""+
			"If set, the Keystone server's certificate will be verified by one of the authorities "+
			"in the experimental-keystone-ca-file, otherwise the host's root CA set will be used.")
	}

	if s.OIDC != nil {
		fs.StringVar(&s.OIDC.IssuerURL, "oidc-issuer-url", s.OIDC.IssuerURL, ""+
			"The URL of the OpenID issuer, only HTTPS scheme will be accepted. "+
			"If set, it will be used to verify the OIDC JSON Web Token (JWT).")

		fs.StringVar(&s.OIDC.ClientID, "oidc-client-id", s.OIDC.ClientID,
			"The client ID for the OpenID Connect client, must be set if oidc-issuer-url is set.")

		fs.StringVar(&s.OIDC.CAFile, "oidc-ca-file", s.OIDC.CAFile, ""+
			"If set, the OpenID server's certificate will be verified by one of the authorities "+
			"in the oidc-ca-file, otherwise the host's root CA set will be used.")

		fs.StringVar(&s.OIDC.UsernameClaim, "oidc-username-claim", "sub", ""+
			"The OpenID claim to use as the user name. Note that claims other than the default ('sub') "+
			"is not guaranteed to be unique and immutable. This flag is experimental, please see "+
			"the authentication documentation for further details.")

		fs.StringVar(&s.OIDC.GroupsClaim, "oidc-groups-claim", "", ""+
			"If provided, the name of a custom OpenID Connect claim for specifying user groups. "+
			"The claim value is expected to be a string or array of strings. This flag is experimental, "+
			"please see the authentication documentation for further details.")
	}

	if s.PasswordFile != nil {
		fs.StringVar(&s.PasswordFile.BasicAuthFile, "basic-auth-file", s.PasswordFile.BasicAuthFile, ""+
			"If set, the file that will be used to admit requests to the secure port of the API server "+
			"via http basic authentication.")
	}

	if s.RequestHeader != nil {
		s.RequestHeader.AddFlags(fs)
	}

	if s.ServiceAccounts != nil {
		fs.StringArrayVar(&s.ServiceAccounts.KeyFiles, "service-account-key-file", s.ServiceAccounts.KeyFiles, ""+
			"File containing PEM-encoded x509 RSA or ECDSA private or public keys, used to verify "+
			"ServiceAccount tokens. If unspecified, --tls-private-key-file is used. "+
			"The specified file can contain multiple keys, and the flag can be specified multiple times with different files.")

		fs.BoolVar(&s.ServiceAccounts.Lookup, "service-account-lookup", s.ServiceAccounts.Lookup,
			"If true, validate ServiceAccount tokens exist in etcd as part of authentication.")
	}

	if s.TokenFile != nil {
		fs.StringVar(&s.TokenFile.TokenFile, "token-auth-file", s.TokenFile.TokenFile, ""+
			"If set, the file that will be used to secure the secure port of the API server "+
			"via token authentication.")
	}

	if s.WebHook != nil {
		fs.StringVar(&s.WebHook.ConfigFile, "authentication-token-webhook-config-file", s.WebHook.ConfigFile, ""+
			"File with webhook configuration for token authentication in kubeconfig format. "+
			"The API server will query the remote service to determine authentication for bearer tokens.")

		fs.DurationVar(&s.WebHook.CacheTTL, "authentication-token-webhook-cache-ttl", s.WebHook.CacheTTL,
			"The duration to cache responses from the webhook token authenticator.")
	}
}

func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() authenticator.AuthenticatorConfig {
	ret := authenticator.AuthenticatorConfig{}

	if s.Anonymous != nil {
		ret.Anonymous = s.Anonymous.Allow
	}

	if s.BootstrapToken != nil {
		ret.BootstrapToken = s.BootstrapToken.Allow
	}

	if s.ClientCert != nil {
		ret.ClientCAFile = s.ClientCert.ClientCA
	}

	if s.Keystone != nil {
		ret.KeystoneURL = s.Keystone.URL
		ret.KeystoneCAFile = s.Keystone.CAFile
	}

	if s.OIDC != nil {
		ret.OIDCCAFile = s.OIDC.CAFile
		ret.OIDCClientID = s.OIDC.ClientID
		ret.OIDCGroupsClaim = s.OIDC.GroupsClaim
		ret.OIDCIssuerURL = s.OIDC.IssuerURL
		ret.OIDCUsernameClaim = s.OIDC.UsernameClaim
	}

	if s.PasswordFile != nil {
		ret.BasicAuthFile = s.PasswordFile.BasicAuthFile
	}

	if s.RequestHeader != nil {
		ret.RequestHeaderConfig = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
	}

	if s.ServiceAccounts != nil {
		ret.ServiceAccountKeyFiles = s.ServiceAccounts.KeyFiles
		ret.ServiceAccountLookup = s.ServiceAccounts.Lookup
	}

	if s.TokenFile != nil {
		ret.TokenAuthFile = s.TokenFile.TokenFile
	}

	if s.WebHook != nil {
		ret.WebhookTokenAuthnConfigFile = s.WebHook.ConfigFile
		ret.WebhookTokenAuthnCacheTTL = s.WebHook.CacheTTL
	}

	return ret
}

func (o *BuiltInAuthenticationOptions) ApplyTo(c *genericapiserver.Config) error {
	if o == nil {
		return nil
	}

	var err error
	if o.ClientCert != nil {
		c, err = c.ApplyClientCert(o.ClientCert.ClientCA)
		if err != nil {
			return fmt.Errorf("unable to load client CA file: %v", err)
		}
	}
	if o.RequestHeader != nil {
		c, err = c.ApplyClientCert(o.RequestHeader.ClientCAFile)
		if err != nil {
			return fmt.Errorf("unable to load client CA file: %v", err)
		}
	}

	c.SupportsBasicAuth = o.PasswordFile != nil && len(o.PasswordFile.BasicAuthFile) > 0

	return nil
}

// ApplyAuthorization will conditionally modify the authentication options based on the authorization options
func (o *BuiltInAuthenticationOptions) ApplyAuthorization(authorization *BuiltInAuthorizationOptions) {
	if o == nil || authorization == nil || o.Anonymous == nil {
		return
	}

	// authorization ModeAlwaysAllow cannot be combined with AnonymousAuth.
	// in such a case the AnonymousAuth is stomped to false and you get a message
	if o.Anonymous.Allow {
		found := false
		for _, mode := range strings.Split(authorization.Mode, ",") {
			if mode == authzmodes.ModeAlwaysAllow {
				found = true
				break
			}
		}
		if found {
			glog.Warningf("AnonymousAuth is not allowed with the AllowAll authorizer.  Resetting AnonymousAuth to false. You should use a different authorizer")
			o.Anonymous.Allow = false
		}
	}
}
<|endoftext|>


<commit_before>package planner

import (
	"encoding/base64"
	"encoding/json"
	"fmt"

	rkev1 "github.com/rancher/rancher/pkg/apis/rke.cattle.io/v1"
	"github.com/rancher/rancher/pkg/apis/rke.cattle.io/v1/plan"
	v1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
	corev1 "k8s.io/api/core/v1"
)

func (p *Planner) addRegistryConfig(config map[string]interface{}, controlPlane *rkev1.RKEControlPlane) ([]plan.File, error) {
	registry := controlPlane.Spec.Registries
	if registry == nil {
		return nil, nil
	}

	registryConfig, files, err := toRegistryConfig(p.secretCache, GetRuntime(controlPlane.Spec.KubernetesVersion), controlPlane.Namespace, registry)
	if err != nil {
		return nil, err
	}

	config["private-registry"] = string(registryConfig)
	return files, nil
}

func toRegistryConfig(secrets v1.SecretCache, runtime, namespace string, registry *rkev1.Registry) ([]byte, []plan.File, error) {
	var (
		files   []plan.File
		configs = map[string]interface{}{}
	)

	for registryName, config := range registry.Configs {
		registryConfig := &registryConfig{}
		if config.TLSSecretName == "" && config.InsecureSkipVerify {
			registryConfig.TLS = &tlsConfig{
				InsecureSkipVerify: true,
			}
		} else if config.TLSSecretName != "" {
			secret, err := secrets.Get(namespace, config.TLSSecretName)
			if err != nil {
				return nil, nil, err
			}
			if secret.Type != corev1.SecretTypeTLS {
				return nil, nil, fmt.Errorf("secret [%s] must be of type [%s]", config.TLSSecretName, corev1.SecretTypeTLS)
			}
			registryConfig.TLS = &tlsConfig{
				InsecureSkipVerify: config.InsecureSkipVerify,
			}

			if cert := secret.Data[corev1.TLSCertKey]; len(cert) != 0 {
				file := toFile(runtime, fmt.Sprintf("tls/registries/%s/tls.crt", registryName), cert)
				registryConfig.TLS.CertFile = file.Path
				files = append(files, file)
			}

			if key := secret.Data[corev1.TLSPrivateKeyKey]; len(key) != 0 {
				file := toFile(runtime, fmt.Sprintf("tls/registries/%s/tls.key", registryName), key)
				registryConfig.TLS.KeyFile = file.Path
				files = append(files, file)
			}
		}

		if len(config.CABundle) > 0 {
			file := toFile(runtime, fmt.Sprintf("tls/registries/%s/ca.crt", registryName), config.CABundle)
			registryConfig.TLS.CAFile = file.Path
			files = append(files, file)
		}

		if config.AuthConfigSecretName != "" {
			secret, err := secrets.Get(namespace, config.AuthConfigSecretName)
			if err != nil {
				return nil, nil, err
			}
			if secret.Type != rkev1.AuthConfigSecretType {
				return nil, nil, fmt.Errorf("secret [%s] must be of type [%s]", config.TLSSecretName, rkev1.AuthConfigSecretType)
			}
			registryConfig.Auth = &authConfig{
				Username:      string(secret.Data[rkev1.UsernameAuthConfigSecretKey]),
				Password:      string(secret.Data[rkev1.PasswordAuthConfigSecretKey]),
				Auth:          string(secret.Data[rkev1.AuthAuthConfigSecretKey]),
				IdentityToken: string(secret.Data[rkev1.IdentityTokenAuthConfigSecretKey]),
			}
		}

		configs[registryName] = registryConfig
	}

	data, err := json.Marshal(map[string]interface{}{
		"mirrors": registry.Mirrors,
		"configs": configs,
	})
	if err != nil {
		return nil, nil, err
	}

	return data, files, nil
}

func toFile(runtime, path string, content []byte) plan.File {
	return plan.File{
		Content: base64.StdEncoding.EncodeToString(content),
		Path:    fmt.Sprintf("/var/lib/rancher/%s/etc/%s", runtime, path),
	}
}

type registryConfig struct {
	Auth *authConfig `json:"auth"`
	TLS  *tlsConfig  `json:"tls"`
}

type tlsConfig struct {
	CAFile             string `json:"ca_file"`
	CertFile           string `json:"cert_file"`
	KeyFile            string `json:"key_file"`
	InsecureSkipVerify bool   `json:"insecure_skip_verify"`
}

type authConfig struct {
	Username      string `json:"username"`
	Password      string `json:"password"`
	Auth          string `json:"auth"`
	IdentityToken string `json:"identity_token"`
}
<commit_msg>Support basic auth secrets for registry config<commit_after>package planner

import (
	"encoding/base64"
	"encoding/json"
	"fmt"

	rkev1 "github.com/rancher/rancher/pkg/apis/rke.cattle.io/v1"
	"github.com/rancher/rancher/pkg/apis/rke.cattle.io/v1/plan"
	v1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
	corev1 "k8s.io/api/core/v1"
)

func (p *Planner) addRegistryConfig(config map[string]interface{}, controlPlane *rkev1.RKEControlPlane) ([]plan.File, error) {
	registry := controlPlane.Spec.Registries
	if registry == nil {
		return nil, nil
	}

	registryConfig, files, err := toRegistryConfig(p.secretCache, GetRuntime(controlPlane.Spec.KubernetesVersion), controlPlane.Namespace, registry)
	if err != nil {
		return nil, err
	}

	config["private-registry"] = string(registryConfig)
	return files, nil
}

func toRegistryConfig(secrets v1.SecretCache, runtime, namespace string, registry *rkev1.Registry) ([]byte, []plan.File, error) {
	var (
		files   []plan.File
		configs = map[string]interface{}{}
	)

	for registryName, config := range registry.Configs {
		registryConfig := &registryConfig{}
		if config.TLSSecretName == "" && config.InsecureSkipVerify {
			registryConfig.TLS = &tlsConfig{
				InsecureSkipVerify: true,
			}
		} else if config.TLSSecretName != "" {
			secret, err := secrets.Get(namespace, config.TLSSecretName)
			if err != nil {
				return nil, nil, err
			}
			if secret.Type != corev1.SecretTypeTLS {
				return nil, nil, fmt.Errorf("secret [%s] must be of type [%s]", config.TLSSecretName, corev1.SecretTypeTLS)
			}
			registryConfig.TLS = &tlsConfig{
				InsecureSkipVerify: config.InsecureSkipVerify,
			}

			if cert := secret.Data[corev1.TLSCertKey]; len(cert) != 0 {
				file := toFile(runtime, fmt.Sprintf("tls/registries/%s/tls.crt", registryName), cert)
				registryConfig.TLS.CertFile = file.Path
				files = append(files, file)
			}

			if key := secret.Data[corev1.TLSPrivateKeyKey]; len(key) != 0 {
				file := toFile(runtime, fmt.Sprintf("tls/registries/%s/tls.key", registryName), key)
				registryConfig.TLS.KeyFile = file.Path
				files = append(files, file)
			}
		}

		if len(config.CABundle) > 0 {
			file := toFile(runtime, fmt.Sprintf("tls/registries/%s/ca.crt", registryName), config.CABundle)
			registryConfig.TLS.CAFile = file.Path
			files = append(files, file)
		}

		if config.AuthConfigSecretName != "" {
			secret, err := secrets.Get(namespace, config.AuthConfigSecretName)
			if err != nil {
				return nil, nil, err
			}
			if secret.Type != rkev1.AuthConfigSecretType && secret.Type != corev1.SecretTypeBasicAuth {
				return nil, nil, fmt.Errorf("secret [%s] must be of type [%s] or [%s]",
					config.AuthConfigSecretName, rkev1.AuthConfigSecretType, corev1.SecretTypeBasicAuth)
			}
			registryConfig.Auth = &authConfig{
				Username:      string(secret.Data[rkev1.UsernameAuthConfigSecretKey]),
				Password:      string(secret.Data[rkev1.PasswordAuthConfigSecretKey]),
				Auth:          string(secret.Data[rkev1.AuthAuthConfigSecretKey]),
				IdentityToken: string(secret.Data[rkev1.IdentityTokenAuthConfigSecretKey]),
			}
		}

		configs[registryName] = registryConfig
	}

	data, err := json.Marshal(map[string]interface{}{
		"mirrors": registry.Mirrors,
		"configs": configs,
	})
	if err != nil {
		return nil, nil, err
	}

	return data, files, nil
}

func toFile(runtime, path string, content []byte) plan.File {
	return plan.File{
		Content: base64.StdEncoding.EncodeToString(content),
		Path:    fmt.Sprintf("/var/lib/rancher/%s/etc/%s", runtime, path),
	}
}

type registryConfig struct {
	Auth *authConfig `json:"auth"`
	TLS  *tlsConfig  `json:"tls"`
}

type tlsConfig struct {
	CAFile             string `json:"ca_file"`
	CertFile           string `json:"cert_file"`
	KeyFile            string `json:"key_file"`
	InsecureSkipVerify bool   `json:"insecure_skip_verify"`
}

type authConfig struct {
	Username      string `json:"username"`
	Password      string `json:"password"`
	Auth          string `json:"auth"`
	IdentityToken string `json:"identity_token"`
}
<|endoftext|>


<commit_before>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package validate

import (
	"reflect"
	"regexp"
	"strings"

	"k8s.io/kube-openapi/pkg/validation/errors"
	"k8s.io/kube-openapi/pkg/validation/spec"
	"k8s.io/kube-openapi/pkg/validation/strfmt"
)

type objectValidator struct {
	Path                 string
	In                   string
	MaxProperties        *int64
	MinProperties        *int64
	Required             []string
	Properties           map[string]spec.Schema
	AdditionalProperties *spec.SchemaOrBool
	PatternProperties    map[string]spec.Schema
	Root                 interface{}
	KnownFormats         strfmt.Registry
	Options              SchemaValidatorOptions
}

func (o *objectValidator) SetPath(path string) {
	o.Path = path
}

func (o *objectValidator) Applies(source interface{}, kind reflect.Kind) bool {
	// TODO: this should also work for structs
	// there is a problem in the type validator where it will be unhappy about null values
	// so that requires more testing
	r := reflect.TypeOf(source) == specSchemaType && (kind == reflect.Map || kind == reflect.Struct)
	debugLog("object validator for %q applies %t for %T (kind: %v)\n", o.Path, r, source, kind)
	return r
}

func (o *objectValidator) isProperties() bool {
	p := strings.Split(o.Path, ".")
	return len(p) > 1 && p[len(p)-1] == jsonProperties && p[len(p)-2] != jsonProperties
}

func (o *objectValidator) isDefault() bool {
	p := strings.Split(o.Path, ".")
	return len(p) > 1 && p[len(p)-1] == jsonDefault && p[len(p)-2] != jsonDefault
}

func (o *objectValidator) isExample() bool {
	p := strings.Split(o.Path, ".")
	return len(p) > 1 && (p[len(p)-1] == swaggerExample || p[len(p)-1] == swaggerExamples) && p[len(p)-2] != swaggerExample
}

func (o *objectValidator) checkArrayMustHaveItems(res *Result, val map[string]interface{}) {
	// for swagger 2.0 schemas, there is an additional constraint to have array items defined explicitly.
	// with pure jsonschema draft 4, one may have arrays with undefined items (i.e. any type).
	if t, typeFound := val[jsonType]; typeFound {
		if tpe, ok := t.(string); ok && tpe == arrayType {
			if _, itemsKeyFound := val[jsonItems]; !itemsKeyFound {
				res.AddErrors(errors.Required(jsonItems, o.Path))
			}
		}
	}
}

func (o *objectValidator) checkItemsMustBeTypeArray(res *Result, val map[string]interface{}) {
	if !o.isProperties() && !o.isDefault() && !o.isExample() {
		if _, itemsKeyFound := val[jsonItems]; itemsKeyFound {
			t, typeFound := val[jsonType]
			if typeFound {
				if tpe, ok := t.(string); !ok || tpe != arrayType {
					res.AddErrors(errors.InvalidType(o.Path, o.In, arrayType, nil))
				}
			} else {
				// there is no type
				res.AddErrors(errors.Required(jsonType, o.Path))
			}
		}
	}
}

func (o *objectValidator) precheck(res *Result, val map[string]interface{}) {
	if o.Options.EnableArrayMustHaveItemsCheck {
		o.checkArrayMustHaveItems(res, val)
	}
	if o.Options.EnableObjectArrayTypeCheck {
		o.checkItemsMustBeTypeArray(res, val)
	}
}

func (o *objectValidator) Validate(data interface{}) *Result {
	val := data.(map[string]interface{})
	// TODO: guard against nil data
	numKeys := int64(len(val))

	if o.MinProperties != nil && numKeys < *o.MinProperties {
		return errorHelp.sErr(errors.TooFewProperties(o.Path, o.In, *o.MinProperties))
	}
	if o.MaxProperties != nil && numKeys > *o.MaxProperties {
		return errorHelp.sErr(errors.TooManyProperties(o.Path, o.In, *o.MaxProperties))
	}

	res := new(Result)

	o.precheck(res, val)

	// check validity of field names
	if o.AdditionalProperties != nil && !o.AdditionalProperties.Allows {
		// Case: additionalProperties: false
		for k := range val {
			_, regularProperty := o.Properties[k]
			matched := false

			for pk := range o.PatternProperties {
				if matches, _ := regexp.MatchString(pk, k); matches {
					matched = true
					break
				}
			}

			if !regularProperty && k != "$schema" && k != "id" && !matched {
				// Special properties "$schema" and "id" are ignored
				res.AddErrors(errors.PropertyNotAllowed(o.Path, o.In, k))

				// BUG(fredbi): This section should move to a part dedicated to spec validation as
				// it will conflict with regular schemas where a property "headers" is defined.

				//
				// Croaks a more explicit message on top of the standard one
				// on some recognized cases.
				//
				// NOTE: edge cases with invalid type assertion are simply ignored here.
				// NOTE: prefix your messages here by "IMPORTANT!" so there are not filtered
				// by higher level callers (the IMPORTANT! tag will be eventually
				// removed).
				if k == "headers" && val[k] != nil {
					// $ref is forbidden in header
					if headers, mapOk := val[k].(map[string]interface{}); mapOk {
						for headerKey, headerBody := range headers {
							if headerBody != nil {
								if headerSchema, mapOfMapOk := headerBody.(map[string]interface{}); mapOfMapOk {
									if _, found := headerSchema["$ref"]; found {
										var msg string
										if refString, stringOk := headerSchema["$ref"].(string); stringOk {
											msg = strings.Join([]string{", one may not use $ref=\":", refString, "\""}, "")
										}
										res.AddErrors(refNotAllowedInHeaderMsg(o.Path, headerKey, msg))
									}
								}
							}
						}
					}
					/*
						case "$ref":
							if val[k] != nil {
								// TODO: check context of that ref: warn about siblings, check against invalid context
							}
					*/
				}
			}
		}
	} else {
		// Cases: no additionalProperties (implying: true), or additionalProperties: true, or additionalProperties: { <<schema>> }
		for key, value := range val {
			_, regularProperty := o.Properties[key]

			// Validates property against "patternProperties" if applicable
			// BUG(fredbi): succeededOnce is always false

			// NOTE: how about regular properties which do not match patternProperties?
			matched, succeededOnce, _ := o.validatePatternProperty(key, value, res)

			if !(regularProperty || matched || succeededOnce) {

				// Cases: properties which are not regular properties and have not been matched by the PatternProperties validator
				if o.AdditionalProperties != nil && o.AdditionalProperties.Schema != nil {
					// AdditionalProperties as Schema
					r := NewSchemaValidator(o.AdditionalProperties.Schema, o.Root, o.Path+"."+key, o.KnownFormats, o.Options.Options()...).Validate(value)
					res.mergeForField(data.(map[string]interface{}), key, r)
				} else if regularProperty && !(matched || succeededOnce) {
					// TODO: this is dead code since regularProperty=false here
					res.AddErrors(errors.FailedAllPatternProperties(o.Path, o.In, key))
				}
			}
		}
		// Valid cases: additionalProperties: true or undefined
	}

	createdFromDefaults := map[string]bool{}

	// Property types:
	// - regular Property
	for pName := range o.Properties {
		pSchema := o.Properties[pName] // one instance per iteration
		rName := pName
		if o.Path != "" {
			rName = o.Path + "." + pName
		}

		// Recursively validates each property against its schema
		if v, ok := val[pName]; ok {
			r := NewSchemaValidator(&pSchema, o.Root, rName, o.KnownFormats, o.Options.Options()...).Validate(v)
			res.mergeForField(data.(map[string]interface{}), pName, r)
		} else if pSchema.Default != nil {
			// If a default value is defined, creates the property from defaults
			// NOTE: JSON schema does not enforce default values to be valid against schema. Swagger does.
			createdFromDefaults[pName] = true
			res.addPropertySchemata(data.(map[string]interface{}), pName, &pSchema)
		}
	}

	// Check required properties
	if len(o.Required) > 0 {
		for _, k := range o.Required {
			if _, ok := val[k]; !ok && !createdFromDefaults[k] {
				res.AddErrors(errors.Required(o.Path+"."+k, o.In))
				continue
			}
		}
	}

	// Check patternProperties
	// TODO: it looks like we have done that twice in many cases
	for key, value := range val {
		_, regularProperty := o.Properties[key]
		matched, _ /*succeededOnce*/, patterns := o.validatePatternProperty(key, value, res)
		if !regularProperty && (matched /*|| succeededOnce*/) {
			for _, pName := range patterns {
				if v, ok := o.PatternProperties[pName]; ok {
					r := NewSchemaValidator(&v, o.Root, o.Path+"."+key, o.KnownFormats, o.Options.Options()...).Validate(value)
					res.mergeForField(data.(map[string]interface{}), key, r)
				}
			}
		}
	}
	return res
}

// TODO: succeededOnce is not used anywhere
func (o *objectValidator) validatePatternProperty(key string, value interface{}, result *Result) (bool, bool, []string) {
	matched := false
	succeededOnce := false
	var patterns []string

	for k, schema := range o.PatternProperties {
		sch := schema
		if match, _ := regexp.MatchString(k, key); match {
			patterns = append(patterns, k)
			matched = true
			validator := NewSchemaValidator(&sch, o.Root, o.Path+"."+key, o.KnownFormats, o.Options.Options()...)

			res := validator.Validate(value)
			result.Merge(res)
		}
	}

	// BUG(fredbi): can't get to here. Should remove dead code (commented out).

	//if succeededOnce {
	//	result.Inc()
	//}

	return matched, succeededOnce, patterns
}
<commit_msg>pkg/validation/validate: delete misplaced $schema and id logic<commit_after>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package validate

import (
	"reflect"
	"regexp"
	"strings"

	"k8s.io/kube-openapi/pkg/validation/errors"
	"k8s.io/kube-openapi/pkg/validation/spec"
	"k8s.io/kube-openapi/pkg/validation/strfmt"
)

type objectValidator struct {
	Path                 string
	In                   string
	MaxProperties        *int64
	MinProperties        *int64
	Required             []string
	Properties           map[string]spec.Schema
	AdditionalProperties *spec.SchemaOrBool
	PatternProperties    map[string]spec.Schema
	Root                 interface{}
	KnownFormats         strfmt.Registry
	Options              SchemaValidatorOptions
}

func (o *objectValidator) SetPath(path string) {
	o.Path = path
}

func (o *objectValidator) Applies(source interface{}, kind reflect.Kind) bool {
	// TODO: this should also work for structs
	// there is a problem in the type validator where it will be unhappy about null values
	// so that requires more testing
	r := reflect.TypeOf(source) == specSchemaType && (kind == reflect.Map || kind == reflect.Struct)
	debugLog("object validator for %q applies %t for %T (kind: %v)\n", o.Path, r, source, kind)
	return r
}

func (o *objectValidator) isProperties() bool {
	p := strings.Split(o.Path, ".")
	return len(p) > 1 && p[len(p)-1] == jsonProperties && p[len(p)-2] != jsonProperties
}

func (o *objectValidator) isDefault() bool {
	p := strings.Split(o.Path, ".")
	return len(p) > 1 && p[len(p)-1] == jsonDefault && p[len(p)-2] != jsonDefault
}

func (o *objectValidator) isExample() bool {
	p := strings.Split(o.Path, ".")
	return len(p) > 1 && (p[len(p)-1] == swaggerExample || p[len(p)-1] == swaggerExamples) && p[len(p)-2] != swaggerExample
}

func (o *objectValidator) checkArrayMustHaveItems(res *Result, val map[string]interface{}) {
	// for swagger 2.0 schemas, there is an additional constraint to have array items defined explicitly.
	// with pure jsonschema draft 4, one may have arrays with undefined items (i.e. any type).
	if t, typeFound := val[jsonType]; typeFound {
		if tpe, ok := t.(string); ok && tpe == arrayType {
			if _, itemsKeyFound := val[jsonItems]; !itemsKeyFound {
				res.AddErrors(errors.Required(jsonItems, o.Path))
			}
		}
	}
}

func (o *objectValidator) checkItemsMustBeTypeArray(res *Result, val map[string]interface{}) {
	if !o.isProperties() && !o.isDefault() && !o.isExample() {
		if _, itemsKeyFound := val[jsonItems]; itemsKeyFound {
			t, typeFound := val[jsonType]
			if typeFound {
				if tpe, ok := t.(string); !ok || tpe != arrayType {
					res.AddErrors(errors.InvalidType(o.Path, o.In, arrayType, nil))
				}
			} else {
				// there is no type
				res.AddErrors(errors.Required(jsonType, o.Path))
			}
		}
	}
}

func (o *objectValidator) precheck(res *Result, val map[string]interface{}) {
	if o.Options.EnableArrayMustHaveItemsCheck {
		o.checkArrayMustHaveItems(res, val)
	}
	if o.Options.EnableObjectArrayTypeCheck {
		o.checkItemsMustBeTypeArray(res, val)
	}
}

func (o *objectValidator) Validate(data interface{}) *Result {
	val := data.(map[string]interface{})
	// TODO: guard against nil data
	numKeys := int64(len(val))

	if o.MinProperties != nil && numKeys < *o.MinProperties {
		return errorHelp.sErr(errors.TooFewProperties(o.Path, o.In, *o.MinProperties))
	}
	if o.MaxProperties != nil && numKeys > *o.MaxProperties {
		return errorHelp.sErr(errors.TooManyProperties(o.Path, o.In, *o.MaxProperties))
	}

	res := new(Result)

	o.precheck(res, val)

	// check validity of field names
	if o.AdditionalProperties != nil && !o.AdditionalProperties.Allows {
		// Case: additionalProperties: false
		for k := range val {
			_, regularProperty := o.Properties[k]
			matched := false

			for pk := range o.PatternProperties {
				if matches, _ := regexp.MatchString(pk, k); matches {
					matched = true
					break
				}
			}

			if !regularProperty && !matched {
				// Special properties "$schema" and "id" are ignored
				res.AddErrors(errors.PropertyNotAllowed(o.Path, o.In, k))
			}
		}
	} else {
		// Cases: no additionalProperties (implying: true), or additionalProperties: true, or additionalProperties: { <<schema>> }
		for key, value := range val {
			_, regularProperty := o.Properties[key]

			// Validates property against "patternProperties" if applicable
			// BUG(fredbi): succeededOnce is always false

			// NOTE: how about regular properties which do not match patternProperties?
			matched, succeededOnce, _ := o.validatePatternProperty(key, value, res)

			if !(regularProperty || matched || succeededOnce) {

				// Cases: properties which are not regular properties and have not been matched by the PatternProperties validator
				if o.AdditionalProperties != nil && o.AdditionalProperties.Schema != nil {
					// AdditionalProperties as Schema
					r := NewSchemaValidator(o.AdditionalProperties.Schema, o.Root, o.Path+"."+key, o.KnownFormats, o.Options.Options()...).Validate(value)
					res.mergeForField(data.(map[string]interface{}), key, r)
				} else if regularProperty && !(matched || succeededOnce) {
					// TODO: this is dead code since regularProperty=false here
					res.AddErrors(errors.FailedAllPatternProperties(o.Path, o.In, key))
				}
			}
		}
		// Valid cases: additionalProperties: true or undefined
	}

	createdFromDefaults := map[string]bool{}

	// Property types:
	// - regular Property
	for pName := range o.Properties {
		pSchema := o.Properties[pName] // one instance per iteration
		rName := pName
		if o.Path != "" {
			rName = o.Path + "." + pName
		}

		// Recursively validates each property against its schema
		if v, ok := val[pName]; ok {
			r := NewSchemaValidator(&pSchema, o.Root, rName, o.KnownFormats, o.Options.Options()...).Validate(v)
			res.mergeForField(data.(map[string]interface{}), pName, r)
		} else if pSchema.Default != nil {
			// If a default value is defined, creates the property from defaults
			// NOTE: JSON schema does not enforce default values to be valid against schema. Swagger does.
			createdFromDefaults[pName] = true
			res.addPropertySchemata(data.(map[string]interface{}), pName, &pSchema)
		}
	}

	// Check required properties
	if len(o.Required) > 0 {
		for _, k := range o.Required {
			if _, ok := val[k]; !ok && !createdFromDefaults[k] {
				res.AddErrors(errors.Required(o.Path+"."+k, o.In))
				continue
			}
		}
	}

	// Check patternProperties
	// TODO: it looks like we have done that twice in many cases
	for key, value := range val {
		_, regularProperty := o.Properties[key]
		matched, _ /*succeededOnce*/, patterns := o.validatePatternProperty(key, value, res)
		if !regularProperty && (matched /*|| succeededOnce*/) {
			for _, pName := range patterns {
				if v, ok := o.PatternProperties[pName]; ok {
					r := NewSchemaValidator(&v, o.Root, o.Path+"."+key, o.KnownFormats, o.Options.Options()...).Validate(value)
					res.mergeForField(data.(map[string]interface{}), key, r)
				}
			}
		}
	}
	return res
}

// TODO: succeededOnce is not used anywhere
func (o *objectValidator) validatePatternProperty(key string, value interface{}, result *Result) (bool, bool, []string) {
	matched := false
	succeededOnce := false
	var patterns []string

	for k, schema := range o.PatternProperties {
		sch := schema
		if match, _ := regexp.MatchString(k, key); match {
			patterns = append(patterns, k)
			matched = true
			validator := NewSchemaValidator(&sch, o.Root, o.Path+"."+key, o.KnownFormats, o.Options.Options()...)

			res := validator.Validate(value)
			result.Merge(res)
		}
	}

	// BUG(fredbi): can't get to here. Should remove dead code (commented out).

	//if succeededOnce {
	//	result.Inc()
	//}

	return matched, succeededOnce, patterns
}
<|endoftext|>


<commit_before>// +build integration

package integration

import (
	log "github.com/Sirupsen/logrus"
	"github.com/intelsdi-x/swan/pkg/executor"
	"github.com/intelsdi-x/swan/pkg/workloads/memcached"
	. "github.com/smartystreets/goconvey/convey"
	"os"
	"testing"
)

const (
	netstatCommand       = "echo stats | nc -w 1 127.0.0.1 11211"
	defaultMemcachedPath = "workloads/data_caching/memcached/memcached-1.4.25/build/memcached"
	swanPkg              = "github.com/intelsdi-x/swan"
)

// TestMemcachedWithExecutor is an integration test with local executor.
// See README for setup items.
func TestMemcachedWithExecutor(t *testing.T) {
	log.SetLevel(log.ErrorLevel)

	// Get optional custom Memcached path from MEMCACHED_PATH.
	memcachedPath := os.Getenv("MEMCACHED_BIN")

	if memcachedPath == "" {
		// If custom path does not exists use default path for built memcached.
		memcachedPath = os.Getenv("GOPATH") + "/src/" + swanPkg + "/" + defaultMemcachedPath
	}

	Convey("While using Local Shell in Memcached launcher", t, func() {
		l := executor.NewLocal()
		memcachedLauncher := memcached.New(
			l, memcached.DefaultMemcachedConfig(memcachedPath))

		Convey("When memcached is launched", func() {
			task, err := memcachedLauncher.Launch()

			Convey("There should be no error", func() {
				task.Stop()

				So(err, ShouldBeNil)
			})

			Convey("Wait 1 second for memcached to init", func() {
				isTerminated := task.Wait(1)

				Convey("Memcached should be still running", func() {
					task.Stop()

					So(isTerminated, ShouldBeFalse)
				})

				Convey("When we check the memcached endpoint for stats after 1 second", func() {

					netstatTask, netstatErr := l.Execute(netstatCommand)

					Convey("There should be no error", func() {
						task.Stop()
						netstatTask.Stop()

						So(netstatErr, ShouldBeNil)

					})

					Convey("When we wait for netstat ", func() {
						netstatTask.Wait(0)

						Convey("The netstat task should be terminated, the task status should be 0"+
							" and output resultes with a STAT information", func() {
							netstatTaskState, netstatTaskStatus := netstatTask.Status()

							task.Stop()
							netstatTask.Stop()

							So(netstatTaskState, ShouldEqual, executor.TERMINATED)
							So(netstatTaskStatus.ExitCode, ShouldEqual, 0)
							So(netstatTaskStatus.Stdout, ShouldStartWith, "STAT")

						})
					})
				})

				Convey("When we stop the memcached task", func() {
					err := task.Stop()

					Convey("There should be no error", func() {
						So(err, ShouldBeNil)
					})

					Convey("The task should be terminated and the task status should be -15 or 0", func() {
						taskState, taskStatus := task.Status()
						So(taskState, ShouldEqual, executor.TERMINATED)
						// Memcached on CentOS returns 0 (successful code) after SIGTERM.
						So(taskStatus.ExitCode, ShouldBeIn, -15, 0)
					})
				})
			})
		})
	})
}
<commit_msg>Give memcached one second to init.<commit_after>// +build integration

package integration

import (
	log "github.com/Sirupsen/logrus"
	"github.com/intelsdi-x/swan/pkg/executor"
	"github.com/intelsdi-x/swan/pkg/workloads/memcached"
	. "github.com/smartystreets/goconvey/convey"
	"os"
	"testing"
)

const (
	netstatCommand       = "echo stats | nc -w 1 127.0.0.1 11211"
	defaultMemcachedPath = "workloads/data_caching/memcached/memcached-1.4.25/build/memcached"
	swanPkg              = "github.com/intelsdi-x/swan"
)

// TestMemcachedWithExecutor is an integration test with local executor.
// See README for setup items.
func TestMemcachedWithExecutor(t *testing.T) {
	log.SetLevel(log.ErrorLevel)

	// Get optional custom Memcached path from MEMCACHED_PATH.
	memcachedPath := os.Getenv("MEMCACHED_BIN")

	if memcachedPath == "" {
		// If custom path does not exists use default path for built memcached.
		memcachedPath = os.Getenv("GOPATH") + "/src/" + swanPkg + "/" + defaultMemcachedPath
	}

	Convey("While using Local Shell in Memcached launcher", t, func() {
		l := executor.NewLocal()
		memcachedLauncher := memcached.New(
			l, memcached.DefaultMemcachedConfig(memcachedPath))

		Convey("When memcached is launched", func() {
			task, err := memcachedLauncher.Launch()

			Convey("There should be no error", func() {
				task.Stop()

				So(err, ShouldBeNil)
			})

			Convey("Wait 1 second for memcached to init", func() {
				isTerminated := task.Wait(1000)

				Convey("Memcached should be still running", func() {
					task.Stop()

					So(isTerminated, ShouldBeFalse)
				})

				Convey("When we check the memcached endpoint for stats after 1 second", func() {

					netstatTask, netstatErr := l.Execute(netstatCommand)

					Convey("There should be no error", func() {
						task.Stop()
						netstatTask.Stop()

						So(netstatErr, ShouldBeNil)

					})

					Convey("When we wait for netstat ", func() {
						netstatTask.Wait(0)

						Convey("The netstat task should be terminated, the task status should be 0"+
							" and output resultes with a STAT information", func() {
							netstatTaskState, netstatTaskStatus := netstatTask.Status()

							task.Stop()
							netstatTask.Stop()

							So(netstatTaskState, ShouldEqual, executor.TERMINATED)
							So(netstatTaskStatus.ExitCode, ShouldEqual, 0)
							So(netstatTaskStatus.Stdout, ShouldStartWith, "STAT")

						})
					})
				})

				Convey("When we stop the memcached task", func() {
					err := task.Stop()

					Convey("There should be no error", func() {
						So(err, ShouldBeNil)
					})

					Convey("The task should be terminated and the task status should be -15 or 0", func() {
						taskState, taskStatus := task.Status()
						So(taskState, ShouldEqual, executor.TERMINATED)
						// Memcached on CentOS returns 0 (successful code) after SIGTERM.
						So(taskStatus.ExitCode, ShouldBeIn, -15, 0)
					})
				})
			})
		})
	})
}
<|endoftext|>


<commit_before>package qshell

import (
	"qiniu/api.v6/conf"
)

type ZoneConfig struct {
	UpHost    string
	RsHost    string
	RsfHost   string
	IovipHost string
	ApiHost   string
}

var (
	DEFAULT_API_HOST = ZoneNBConfig.ApiHost
)

const (
	ZoneNB  = "nb"
	ZoneBC  = "bc"
	ZoneAWS = "aws"
)

//zone all defaults to the service source site

var ZoneNBConfig = ZoneConfig{
	UpHost:    "http://up.qiniu.com",
	RsHost:    "http://rs.qiniu.com",
	RsfHost:   "http://rsf.qbox.me",
	IovipHost: "http://iovip.qbox.me",
	ApiHost:   "http://api.qiniu.com",
}

var ZoneBCConfig = ZoneConfig{
	UpHost:    "http://up-z1.qiniu.com",
	RsHost:    "http://rs.qiniu.com",
	RsfHost:   "http://rsf-z1.qbox.me",
	IovipHost: "http://iovip-z1.qbox.me",
	ApiHost:   "http://api-z1.qiniu.com",
}

var ZoneAWSConfig = ZoneConfig{
	UpHost:    "http://up.gdipper.com",
	RsHost:    "http://rs.gdipper.com",
	RsfHost:   "http://rsf.gdipper.com",
	IovipHost: "http://iovip.gdipper.com",
	ApiHost:   "http://api.gdipper.com",
}

func SetZone(zoneConfig ZoneConfig) {
	conf.UP_HOST = zoneConfig.UpHost
	conf.RS_HOST = zoneConfig.RsHost
	conf.RSF_HOST = zoneConfig.RsfHost
	conf.IO_HOST = zoneConfig.IovipHost
	DEFAULT_API_HOST = zoneConfig.ApiHost
}

func IsValidZone(zone string) (valid bool) {
	switch zone {
	case ZoneNB, ZoneBC, ZoneAWS:
		valid = true
	default:
		valid = false
	}
	return
}
<commit_msg>fix the overseas io domain error<commit_after>package qshell

import (
	"qiniu/api.v6/conf"
)

type ZoneConfig struct {
	UpHost    string
	RsHost    string
	RsfHost   string
	IovipHost string
	ApiHost   string
}

var (
	DEFAULT_API_HOST = ZoneNBConfig.ApiHost
)

const (
	ZoneNB  = "nb"
	ZoneBC  = "bc"
	ZoneAWS = "aws"
)

//zone all defaults to the service source site

var ZoneNBConfig = ZoneConfig{
	UpHost:    "http://up.qiniu.com",
	RsHost:    "http://rs.qiniu.com",
	RsfHost:   "http://rsf.qbox.me",
	IovipHost: "http://iovip.qbox.me",
	ApiHost:   "http://api.qiniu.com",
}

var ZoneBCConfig = ZoneConfig{
	UpHost:    "http://up-z1.qiniu.com",
	RsHost:    "http://rs.qiniu.com",
	RsfHost:   "http://rsf-z1.qbox.me",
	IovipHost: "http://iovip-z1.qbox.me",
	ApiHost:   "http://api-z1.qiniu.com",
}

var ZoneAWSConfig = ZoneConfig{
	UpHost:    "http://up.gdipper.com",
	RsHost:    "http://rs.gdipper.com",
	RsfHost:   "http://rsf.gdipper.com",
	IovipHost: "http://io.gdipper.com",
	ApiHost:   "http://api.gdipper.com",
}

func SetZone(zoneConfig ZoneConfig) {
	conf.UP_HOST = zoneConfig.UpHost
	conf.RS_HOST = zoneConfig.RsHost
	conf.RSF_HOST = zoneConfig.RsfHost
	conf.IO_HOST = zoneConfig.IovipHost
	DEFAULT_API_HOST = zoneConfig.ApiHost
}

func IsValidZone(zone string) (valid bool) {
	switch zone {
	case ZoneNB, ZoneBC, ZoneAWS:
		valid = true
	default:
		valid = false
	}
	return
}
<|endoftext|>


<commit_before>package eventsource

import (
	"fmt"
	"net/http"
)

/*
See :
http://html5doctor.com/server-sent-events/
http://www.html5rocks.com/en/tutorials/eventsource/basics/
http://cjihrig.com/blog/the-server-side-of-server-sent-events/
*/

type Conn struct {
	writer http.ResponseWriter
	http.Flusher
	http.CloseNotifier
}

func (c Conn) Write(message string) {
	c.writer.Write([]byte(fmt.Sprintf("data: %s", message)))
	c.writer.Write([]byte("\n\n"))
	c.Flush()
}

type Handler func(*Conn)

func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	f, f_ok := w.(http.Flusher)

	if !f_ok {
		panic("ResponseWriter is not a Flusher")
	}

	cn, cn_ok := w.(http.CloseNotifier)

	if !cn_ok {
		panic("ResponseWriter is not a CloseNotifier")
	}

	w.Header().Set("Content-Type", "text/event-stream")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Connection", "keep-alive")
	w.Header().Set("Transfer-Encoding", "chunked")

	f.Flush()

	h(&Conn{w, f, cn})
}
<commit_msg>Add a package comment for better visibility on godoc.org<commit_after>/*
Package eventsouce provides a basic interface for
serving server sent events (aka EventSource).

It is patterned after the Go Websocket library:
https://code.google.com/p/go/source/browse?repo=net#hg%2Fwebsocket

For more destails about Server Sent Events see:
http://html5doctor.com/server-sent-events/
http://www.html5rocks.com/en/tutorials/eventsource/basics/
http://cjihrig.com/blog/the-server-side-of-server-sent-events/
*/

package eventsource

import (
	"fmt"
	"net/http"
)

type Conn struct {
	writer http.ResponseWriter
	http.Flusher
	http.CloseNotifier
}

func (c Conn) Write(message string) {
	c.writer.Write([]byte(fmt.Sprintf("data: %s", message)))
	c.writer.Write([]byte("\n\n"))
	c.Flush()
}

type Handler func(*Conn)

func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	f, f_ok := w.(http.Flusher)

	if !f_ok {
		panic("ResponseWriter is not a Flusher")
	}

	cn, cn_ok := w.(http.CloseNotifier)

	if !cn_ok {
		panic("ResponseWriter is not a CloseNotifier")
	}

	w.Header().Set("Content-Type", "text/event-stream")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Connection", "keep-alive")
	w.Header().Set("Transfer-Encoding", "chunked")

	f.Flush()

	h(&Conn{w, f, cn})
}
<|endoftext|>


<commit_before>// Copyright 2017-2019 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package http implements HTTP probe type.
package http

import (
	"bytes"
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptrace"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/google/cloudprober/common/oauth"
	"github.com/google/cloudprober/common/tlsconfig"
	"github.com/google/cloudprober/logger"
	"github.com/google/cloudprober/metrics"
	configpb "github.com/google/cloudprober/probes/http/proto"
	"github.com/google/cloudprober/probes/options"
	"github.com/google/cloudprober/targets/endpoint"
	"github.com/google/cloudprober/validators"
	"golang.org/x/oauth2"
)

const (
	maxResponseSizeForMetrics = 128
	targetsUpdateInterval     = 1 * time.Minute
	largeBodyThreshold        = bytes.MinRead // 512.
)

// Probe holds aggregate information about all probe runs, per-target.
type Probe struct {
	name   string
	opts   *options.Options
	c      *configpb.ProbeConf
	l      *logger.Logger
	client *http.Client

	// book-keeping params
	targets      []endpoint.Endpoint
	httpRequests map[string]*http.Request
	results      map[string]*probeResult
	protocol     string
	method       string
	url          string
	oauthTS      oauth2.TokenSource
	bearerToken  string

	// Run counter, used to decide when to update targets or export
	// stats.
	runCnt int64

	// How often to resolve targets (in probe counts), initialized to
	// targetsUpdateInterval / p.opts.Interval. Targets and associated data
	// structures are updated when (runCnt % targetsUpdateFrequency) == 0
	targetsUpdateFrequency int64

	// How often to export metrics (in probe counts), initialized to
	// statsExportInterval / p.opts.Interval. Metrics are exported when
	// (runCnt % statsExportFrequency) == 0
	statsExportFrequency int64

	requestBody []byte
}

type probeResult struct {
	total, success, timeouts int64
	connEvent                int64
	latency                  metrics.Value
	respCodes                *metrics.Map
	respBodies               *metrics.Map
	validationFailure        *metrics.Map
}

// Init initializes the probe with the given params.
func (p *Probe) Init(name string, opts *options.Options) error {
	c, ok := opts.ProbeConf.(*configpb.ProbeConf)
	if !ok {
		return fmt.Errorf("not http config")
	}
	p.name = name
	p.opts = opts
	if p.l = opts.Logger; p.l == nil {
		p.l = &logger.Logger{}
	}
	p.c = c

	p.protocol = strings.ToLower(p.c.GetProtocol().String())
	p.method = p.c.GetMethod().String()

	p.url = p.c.GetRelativeUrl()
	if len(p.url) > 0 && p.url[0] != '/' {
		return fmt.Errorf("Invalid Relative URL: %s, must begin with '/'", p.url)
	}

	p.requestBody = []byte(p.c.GetBody())

	// Create a transport for our use. This is mostly based on
	// http.DefaultTransport with some timeouts changed.
	// TODO(manugarg): Considering cloning DefaultTransport once
	// https://github.com/golang/go/issues/26013 is fixed.
	dialer := &net.Dialer{
		Timeout:   p.opts.Timeout,
		KeepAlive: 30 * time.Second, // TCP keep-alive
	}

	if p.opts.SourceIP != nil {
		dialer.LocalAddr = &net.TCPAddr{
			IP: p.opts.SourceIP,
		}
	}

	transport := &http.Transport{
		Proxy:               http.ProxyFromEnvironment,
		DialContext:         dialer.DialContext,
		MaxIdleConns:        256, // http.DefaultTransport.MaxIdleConns: 100.
		TLSHandshakeTimeout: p.opts.Timeout,
	}

	if p.c.GetProxyUrl() != "" {
		url, err := url.Parse(p.c.GetProxyUrl())
		if err != nil {
			return fmt.Errorf("error parsing proxy URL (%s): %v", p.c.GetProxyUrl(), err)
		}
		transport.Proxy = http.ProxyURL(url)
	}

	if p.c.GetDisableCertValidation() || p.c.GetTlsConfig() != nil {
		if transport.TLSClientConfig == nil {
			transport.TLSClientConfig = &tls.Config{}
		}

		if p.c.GetDisableCertValidation() {
			p.l.Warning("disable_cert_validation is deprecated as of v0.10.6. Instead of this, please use \"tls_config {disable_cert_validation: true}\"")
			transport.TLSClientConfig.InsecureSkipVerify = true
		}

		if p.c.GetTlsConfig() != nil {
			if err := tlsconfig.UpdateTLSConfig(transport.TLSClientConfig, p.c.GetTlsConfig(), false); err != nil {
				return err
			}
		}
	}

	// If HTTP keep-alives are not enabled (default), disable HTTP keep-alive in
	// transport.
	if !p.c.GetKeepAlive() {
		transport.DisableKeepAlives = true
	} else {
		// If it's been more than 2 probe intervals since connection was used, close it.
		transport.IdleConnTimeout = 2 * p.opts.Interval
	}

	if p.c.GetOauthConfig() != nil {
		oauthTS, err := oauth.TokenSourceFromConfig(p.c.GetOauthConfig(), p.l)
		if err != nil {
			return err
		}
		p.oauthTS = oauthTS
	}

	if p.c.GetDisableHttp2() {
		// HTTP/2 is enabled by default if server supports it. Setting TLSNextProto
		// to an empty dict is the only to disable it.
		transport.TLSNextProto = make(map[string]func(string, *tls.Conn) http.RoundTripper)
	}

	// Clients are safe for concurrent use by multiple goroutines.
	p.client = &http.Client{
		Transport: transport,
	}

	p.statsExportFrequency = p.opts.StatsExportInterval.Nanoseconds() / p.opts.Interval.Nanoseconds()
	if p.statsExportFrequency == 0 {
		p.statsExportFrequency = 1
	}

	// Update targets and associated data structures (requests and results) once
	// in Init(). It's also called periodically in Start(), at
	// targetsUpdateInterval.
	p.updateTargets()
	p.targetsUpdateFrequency = targetsUpdateInterval.Nanoseconds() / p.opts.Interval.Nanoseconds()
	if p.targetsUpdateFrequency == 0 {
		p.targetsUpdateFrequency = 1
	}

	return nil
}

// Return true if the underlying error indicates a http.Client timeout.
//
// Use for errors returned from http.Client methods (Get, Post).
func isClientTimeout(err error) bool {
	if uerr, ok := err.(*url.Error); ok {
		if nerr, ok := uerr.Err.(net.Error); ok && nerr.Timeout() {
			return true
		}
	}
	return false
}

// httpRequest executes an HTTP request and updates the provided result struct.
func (p *Probe) doHTTPRequest(req *http.Request, result *probeResult, resultMu *sync.Mutex) {

	if len(p.requestBody) >= largeBodyThreshold {
		req = req.Clone(req.Context())
		req.Body = ioutil.NopCloser(bytes.NewReader(p.requestBody))
	}

	if p.c.GetKeepAlive() {
		trace := &httptrace.ClientTrace{
			ConnectDone: func(_, addr string, err error) {
				result.connEvent++
				if err != nil {
					p.l.Warning("Error establishing a new connection to: ", addr, ". Err: ", err.Error())
					return
				}
				p.l.Info("Established a new connection to: ", addr)
			},
		}
		req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
	}

	start := time.Now()
	resp, err := p.client.Do(req)
	latency := time.Since(start)

	// Note that we take lock on result object outside of the actual request.
	resultMu.Lock()
	defer resultMu.Unlock()

	result.total++

	if err != nil {
		if isClientTimeout(err) {
			p.l.Warning("Target:", req.Host, ", URL:", req.URL.String(), ", http.doHTTPRequest: timeout error: ", err.Error())
			result.timeouts++
			return
		}
		p.l.Warning("Target:", req.Host, ", URL:", req.URL.String(), ", http.doHTTPRequest: ", err.Error())
		return
	}

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		p.l.Warning("Target:", req.Host, ", URL:", req.URL.String(), ", http.doHTTPRequest: ", err.Error())
		return
	}

	p.l.Debug("Target:", req.Host, ", URL:", req.URL.String(), ", response: ", string(respBody))

	// Calling Body.Close() allows the TCP connection to be reused.
	resp.Body.Close()
	result.respCodes.IncKey(strconv.FormatInt(int64(resp.StatusCode), 10))

	if p.opts.Validators != nil {
		failedValidations := validators.RunValidators(p.opts.Validators, &validators.Input{Response: resp, ResponseBody: respBody}, result.validationFailure, p.l)

		// If any validation failed, return now, leaving the success and latency
		// counters unchanged.
		if len(failedValidations) > 0 {
			p.l.Debug("Target:", req.Host, ", URL:", req.URL.String(), ", http.doHTTPRequest: failed validations: ", strings.Join(failedValidations, ","))
			return
		}
	}

	result.success++
	result.latency.AddFloat64(latency.Seconds() / p.opts.LatencyUnit.Seconds())
	if p.c.GetExportResponseAsMetrics() {
		if len(respBody) <= maxResponseSizeForMetrics {
			result.respBodies.IncKey(string(respBody))
		}
	}
}

func (p *Probe) updateTargets() {
	p.targets = p.opts.Targets.ListEndpoints()

	if p.httpRequests == nil {
		p.httpRequests = make(map[string]*http.Request, len(p.targets))
	}

	if p.results == nil {
		p.results = make(map[string]*probeResult, len(p.targets))
	}

	// OAuth token is target independent.
	if p.oauthTS != nil {
		tok, err := p.oauthTS.Token()
		if err != nil {
			p.l.Error("Error getting OAuth token: ", err.Error(), ". Skipping updating the token.")
		} else {
			p.l.Debug("Got OAuth token, len: ", strconv.FormatInt(int64(len(tok.AccessToken)), 10), ", expirationTime: ", tok.Expiry.String())
			p.bearerToken = tok.AccessToken
		}
	}

	for _, target := range p.targets {
		// Update HTTP request
		req := p.httpRequestForTarget(target, nil)
		if req != nil {
			p.httpRequests[target.Name] = req
		}

		for _, al := range p.opts.AdditionalLabels {
			al.UpdateForTarget(target.Name, target.Labels)
		}

		// Add missing result objects
		if p.results[target.Name] == nil {
			var latencyValue metrics.Value
			if p.opts.LatencyDist != nil {
				latencyValue = p.opts.LatencyDist.Clone()
			} else {
				latencyValue = metrics.NewFloat(0)
			}
			p.results[target.Name] = &probeResult{
				latency:           latencyValue,
				respCodes:         metrics.NewMap("code", metrics.NewInt(0)),
				respBodies:        metrics.NewMap("resp", metrics.NewInt(0)),
				validationFailure: validators.ValidationFailureMap(p.opts.Validators),
			}
		}
	}
}

func (p *Probe) runProbe(ctx context.Context) {
	reqCtx, cancelReqCtx := context.WithTimeout(ctx, p.opts.Timeout)
	defer cancelReqCtx()

	wg := sync.WaitGroup{}
	for _, target := range p.targets {
		req, result := p.httpRequests[target.Name], p.results[target.Name]
		if req == nil {
			continue
		}

		// We launch a separate goroutine for each HTTP request. Since there can be
		// multiple requests per probe per target, we use a mutex to protect access
		// to per-target result object in doHTTPRequest. Note that result object is
		// not accessed concurrently anywhere else -- export of the metrics happens
		// when probe is not running.
		var resultMu sync.Mutex

		for numReq := int32(0); numReq < p.c.GetRequestsPerProbe(); numReq++ {
			wg.Add(1)

			go func(req *http.Request, result *probeResult) {
				defer wg.Done()
				p.doHTTPRequest(req.WithContext(reqCtx), result, &resultMu)
			}(req, result)
		}
	}

	// Wait until all probes are done.
	wg.Wait()
}

// Start starts and runs the probe indefinitely.
func (p *Probe) Start(ctx context.Context, dataChan chan *metrics.EventMetrics) {
	ticker := time.NewTicker(p.opts.Interval)
	defer ticker.Stop()

	for ts := range ticker.C {
		// Don't run another probe if context is canceled already.
		select {
		case <-ctx.Done():
			return
		default:
		}

		// Update targets if its the turn for that.
		if (p.runCnt % p.targetsUpdateFrequency) == 0 {
			p.updateTargets()
		}
		p.runCnt++

		p.runProbe(ctx)

		if (p.runCnt % p.statsExportFrequency) == 0 {
			for _, target := range p.targets {
				result := p.results[target.Name]
				em := metrics.NewEventMetrics(ts).
					AddMetric("total", metrics.NewInt(result.total)).
					AddMetric("success", metrics.NewInt(result.success)).
					AddMetric("latency", result.latency).
					AddMetric("timeouts", metrics.NewInt(result.timeouts)).
					AddMetric("resp-code", result.respCodes).
					AddMetric("resp-body", result.respBodies).
					AddLabel("ptype", "http").
					AddLabel("probe", p.name).
					AddLabel("dst", target.Name)

				if p.c.GetKeepAlive() {
					em.AddMetric("connect_event", metrics.NewInt(result.connEvent))
				}

				for _, al := range p.opts.AdditionalLabels {
					em.AddLabel(al.KeyValueForTarget(target.Name))
				}

				if p.opts.Validators != nil {
					em.AddMetric("validation_failure", result.validationFailure)
				}

				p.opts.LogMetrics(em)
				dataChan <- em
			}
		}
	}
}
<commit_msg>Use target name for logging. (#474)<commit_after>// Copyright 2017-2019 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package http implements HTTP probe type.
package http

import (
	"bytes"
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptrace"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/google/cloudprober/common/oauth"
	"github.com/google/cloudprober/common/tlsconfig"
	"github.com/google/cloudprober/logger"
	"github.com/google/cloudprober/metrics"
	configpb "github.com/google/cloudprober/probes/http/proto"
	"github.com/google/cloudprober/probes/options"
	"github.com/google/cloudprober/targets/endpoint"
	"github.com/google/cloudprober/validators"
	"golang.org/x/oauth2"
)

const (
	maxResponseSizeForMetrics = 128
	targetsUpdateInterval     = 1 * time.Minute
	largeBodyThreshold        = bytes.MinRead // 512.
)

// Probe holds aggregate information about all probe runs, per-target.
type Probe struct {
	name   string
	opts   *options.Options
	c      *configpb.ProbeConf
	l      *logger.Logger
	client *http.Client

	// book-keeping params
	targets      []endpoint.Endpoint
	httpRequests map[string]*http.Request
	results      map[string]*probeResult
	protocol     string
	method       string
	url          string
	oauthTS      oauth2.TokenSource
	bearerToken  string

	// Run counter, used to decide when to update targets or export
	// stats.
	runCnt int64

	// How often to resolve targets (in probe counts), initialized to
	// targetsUpdateInterval / p.opts.Interval. Targets and associated data
	// structures are updated when (runCnt % targetsUpdateFrequency) == 0
	targetsUpdateFrequency int64

	// How often to export metrics (in probe counts), initialized to
	// statsExportInterval / p.opts.Interval. Metrics are exported when
	// (runCnt % statsExportFrequency) == 0
	statsExportFrequency int64

	requestBody []byte
}

type probeResult struct {
	total, success, timeouts int64
	connEvent                int64
	latency                  metrics.Value
	respCodes                *metrics.Map
	respBodies               *metrics.Map
	validationFailure        *metrics.Map
}

// Init initializes the probe with the given params.
func (p *Probe) Init(name string, opts *options.Options) error {
	c, ok := opts.ProbeConf.(*configpb.ProbeConf)
	if !ok {
		return fmt.Errorf("not http config")
	}
	p.name = name
	p.opts = opts
	if p.l = opts.Logger; p.l == nil {
		p.l = &logger.Logger{}
	}
	p.c = c

	p.protocol = strings.ToLower(p.c.GetProtocol().String())
	p.method = p.c.GetMethod().String()

	p.url = p.c.GetRelativeUrl()
	if len(p.url) > 0 && p.url[0] != '/' {
		return fmt.Errorf("Invalid Relative URL: %s, must begin with '/'", p.url)
	}

	p.requestBody = []byte(p.c.GetBody())

	// Create a transport for our use. This is mostly based on
	// http.DefaultTransport with some timeouts changed.
	// TODO(manugarg): Considering cloning DefaultTransport once
	// https://github.com/golang/go/issues/26013 is fixed.
	dialer := &net.Dialer{
		Timeout:   p.opts.Timeout,
		KeepAlive: 30 * time.Second, // TCP keep-alive
	}

	if p.opts.SourceIP != nil {
		dialer.LocalAddr = &net.TCPAddr{
			IP: p.opts.SourceIP,
		}
	}

	transport := &http.Transport{
		Proxy:               http.ProxyFromEnvironment,
		DialContext:         dialer.DialContext,
		MaxIdleConns:        256, // http.DefaultTransport.MaxIdleConns: 100.
		TLSHandshakeTimeout: p.opts.Timeout,
	}

	if p.c.GetProxyUrl() != "" {
		url, err := url.Parse(p.c.GetProxyUrl())
		if err != nil {
			return fmt.Errorf("error parsing proxy URL (%s): %v", p.c.GetProxyUrl(), err)
		}
		transport.Proxy = http.ProxyURL(url)
	}

	if p.c.GetDisableCertValidation() || p.c.GetTlsConfig() != nil {
		if transport.TLSClientConfig == nil {
			transport.TLSClientConfig = &tls.Config{}
		}

		if p.c.GetDisableCertValidation() {
			p.l.Warning("disable_cert_validation is deprecated as of v0.10.6. Instead of this, please use \"tls_config {disable_cert_validation: true}\"")
			transport.TLSClientConfig.InsecureSkipVerify = true
		}

		if p.c.GetTlsConfig() != nil {
			if err := tlsconfig.UpdateTLSConfig(transport.TLSClientConfig, p.c.GetTlsConfig(), false); err != nil {
				return err
			}
		}
	}

	// If HTTP keep-alives are not enabled (default), disable HTTP keep-alive in
	// transport.
	if !p.c.GetKeepAlive() {
		transport.DisableKeepAlives = true
	} else {
		// If it's been more than 2 probe intervals since connection was used, close it.
		transport.IdleConnTimeout = 2 * p.opts.Interval
	}

	if p.c.GetOauthConfig() != nil {
		oauthTS, err := oauth.TokenSourceFromConfig(p.c.GetOauthConfig(), p.l)
		if err != nil {
			return err
		}
		p.oauthTS = oauthTS
	}

	if p.c.GetDisableHttp2() {
		// HTTP/2 is enabled by default if server supports it. Setting TLSNextProto
		// to an empty dict is the only to disable it.
		transport.TLSNextProto = make(map[string]func(string, *tls.Conn) http.RoundTripper)
	}

	// Clients are safe for concurrent use by multiple goroutines.
	p.client = &http.Client{
		Transport: transport,
	}

	p.statsExportFrequency = p.opts.StatsExportInterval.Nanoseconds() / p.opts.Interval.Nanoseconds()
	if p.statsExportFrequency == 0 {
		p.statsExportFrequency = 1
	}

	// Update targets and associated data structures (requests and results) once
	// in Init(). It's also called periodically in Start(), at
	// targetsUpdateInterval.
	p.updateTargets()
	p.targetsUpdateFrequency = targetsUpdateInterval.Nanoseconds() / p.opts.Interval.Nanoseconds()
	if p.targetsUpdateFrequency == 0 {
		p.targetsUpdateFrequency = 1
	}

	return nil
}

// Return true if the underlying error indicates a http.Client timeout.
//
// Use for errors returned from http.Client methods (Get, Post).
func isClientTimeout(err error) bool {
	if uerr, ok := err.(*url.Error); ok {
		if nerr, ok := uerr.Err.(net.Error); ok && nerr.Timeout() {
			return true
		}
	}
	return false
}

// httpRequest executes an HTTP request and updates the provided result struct.
func (p *Probe) doHTTPRequest(req *http.Request, targetName string, result *probeResult, resultMu *sync.Mutex) {

	if len(p.requestBody) >= largeBodyThreshold {
		req = req.Clone(req.Context())
		req.Body = ioutil.NopCloser(bytes.NewReader(p.requestBody))
	}

	if p.c.GetKeepAlive() {
		trace := &httptrace.ClientTrace{
			ConnectDone: func(_, addr string, err error) {
				result.connEvent++
				if err != nil {
					p.l.Warning("Error establishing a new connection to: ", addr, ". Err: ", err.Error())
					return
				}
				p.l.Info("Established a new connection to: ", addr)
			},
		}
		req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
	}

	start := time.Now()
	resp, err := p.client.Do(req)
	latency := time.Since(start)

	// Note that we take lock on result object outside of the actual request.
	resultMu.Lock()
	defer resultMu.Unlock()

	result.total++

	if err != nil {
		if isClientTimeout(err) {
			p.l.Warning("Target:", targetName, ", URL:", req.URL.String(), ", http.doHTTPRequest: timeout error: ", err.Error())
			result.timeouts++
			return
		}
		p.l.Warning("Target:", targetName, ", URL:", req.URL.String(), ", http.doHTTPRequest: ", err.Error())
		return
	}

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		p.l.Warning("Target:", targetName, ", URL:", req.URL.String(), ", http.doHTTPRequest: ", err.Error())
		return
	}

	p.l.Debug("Target:", targetName, ", URL:", req.URL.String(), ", response: ", string(respBody))

	// Calling Body.Close() allows the TCP connection to be reused.
	resp.Body.Close()
	result.respCodes.IncKey(strconv.FormatInt(int64(resp.StatusCode), 10))

	if p.opts.Validators != nil {
		failedValidations := validators.RunValidators(p.opts.Validators, &validators.Input{Response: resp, ResponseBody: respBody}, result.validationFailure, p.l)

		// If any validation failed, return now, leaving the success and latency
		// counters unchanged.
		if len(failedValidations) > 0 {
			p.l.Debug("Target:", targetName, ", URL:", req.URL.String(), ", http.doHTTPRequest: failed validations: ", strings.Join(failedValidations, ","))
			return
		}
	}

	result.success++
	result.latency.AddFloat64(latency.Seconds() / p.opts.LatencyUnit.Seconds())
	if p.c.GetExportResponseAsMetrics() {
		if len(respBody) <= maxResponseSizeForMetrics {
			result.respBodies.IncKey(string(respBody))
		}
	}
}

func (p *Probe) updateTargets() {
	p.targets = p.opts.Targets.ListEndpoints()

	if p.httpRequests == nil {
		p.httpRequests = make(map[string]*http.Request, len(p.targets))
	}

	if p.results == nil {
		p.results = make(map[string]*probeResult, len(p.targets))
	}

	// OAuth token is target independent.
	if p.oauthTS != nil {
		tok, err := p.oauthTS.Token()
		if err != nil {
			p.l.Error("Error getting OAuth token: ", err.Error(), ". Skipping updating the token.")
		} else {
			p.l.Debug("Got OAuth token, len: ", strconv.FormatInt(int64(len(tok.AccessToken)), 10), ", expirationTime: ", tok.Expiry.String())
			p.bearerToken = tok.AccessToken
		}
	}

	for _, target := range p.targets {
		// Update HTTP request
		req := p.httpRequestForTarget(target, nil)
		if req != nil {
			p.httpRequests[target.Name] = req
		}

		for _, al := range p.opts.AdditionalLabels {
			al.UpdateForTarget(target.Name, target.Labels)
		}

		// Add missing result objects
		if p.results[target.Name] == nil {
			var latencyValue metrics.Value
			if p.opts.LatencyDist != nil {
				latencyValue = p.opts.LatencyDist.Clone()
			} else {
				latencyValue = metrics.NewFloat(0)
			}
			p.results[target.Name] = &probeResult{
				latency:           latencyValue,
				respCodes:         metrics.NewMap("code", metrics.NewInt(0)),
				respBodies:        metrics.NewMap("resp", metrics.NewInt(0)),
				validationFailure: validators.ValidationFailureMap(p.opts.Validators),
			}
		}
	}
}

func (p *Probe) runProbe(ctx context.Context) {
	reqCtx, cancelReqCtx := context.WithTimeout(ctx, p.opts.Timeout)
	defer cancelReqCtx()

	wg := sync.WaitGroup{}
	for _, target := range p.targets {
		req, result := p.httpRequests[target.Name], p.results[target.Name]
		if req == nil {
			continue
		}

		// We launch a separate goroutine for each HTTP request. Since there can be
		// multiple requests per probe per target, we use a mutex to protect access
		// to per-target result object in doHTTPRequest. Note that result object is
		// not accessed concurrently anywhere else -- export of the metrics happens
		// when probe is not running.
		var resultMu sync.Mutex

		for numReq := int32(0); numReq < p.c.GetRequestsPerProbe(); numReq++ {
			wg.Add(1)

			go func(req *http.Request, targetName string, result *probeResult) {
				defer wg.Done()
				p.doHTTPRequest(req.WithContext(reqCtx), targetName, result, &resultMu)
			}(req, target.Name, result)
		}
	}

	// Wait until all probes are done.
	wg.Wait()
}

// Start starts and runs the probe indefinitely.
func (p *Probe) Start(ctx context.Context, dataChan chan *metrics.EventMetrics) {
	ticker := time.NewTicker(p.opts.Interval)
	defer ticker.Stop()

	for ts := range ticker.C {
		// Don't run another probe if context is canceled already.
		select {
		case <-ctx.Done():
			return
		default:
		}

		// Update targets if its the turn for that.
		if (p.runCnt % p.targetsUpdateFrequency) == 0 {
			p.updateTargets()
		}
		p.runCnt++

		p.runProbe(ctx)

		if (p.runCnt % p.statsExportFrequency) == 0 {
			for _, target := range p.targets {
				result := p.results[target.Name]
				em := metrics.NewEventMetrics(ts).
					AddMetric("total", metrics.NewInt(result.total)).
					AddMetric("success", metrics.NewInt(result.success)).
					AddMetric("latency", result.latency).
					AddMetric("timeouts", metrics.NewInt(result.timeouts)).
					AddMetric("resp-code", result.respCodes).
					AddMetric("resp-body", result.respBodies).
					AddLabel("ptype", "http").
					AddLabel("probe", p.name).
					AddLabel("dst", target.Name)

				if p.c.GetKeepAlive() {
					em.AddMetric("connect_event", metrics.NewInt(result.connEvent))
				}

				for _, al := range p.opts.AdditionalLabels {
					em.AddLabel(al.KeyValueForTarget(target.Name))
				}

				if p.opts.Validators != nil {
					em.AddMetric("validation_failure", result.validationFailure)
				}

				p.opts.LogMetrics(em)
				dataChan <- em
			}
		}
	}
}
<|endoftext|>


<commit_before>//
// Face-meltingly fast, thread-safe, marshalable, unionable, probability- and optimal-size-calculating Bloom filter in go
//
// https://github.com/steakknife/bloomfilter
//
// Copyright © 2014, 2015 Barry Allard
//
// MIT license
//
package bloomfilter

import (
	"bytes"
	"crypto/sha512"
	"encoding/binary"
)

// conforms to encoding.BinaryMarshaler

// marshalled binary layout (Little Endian):
//
//   k      1 uint64
//   n      1 uint64
//   m      1 uint64
//   keys   [k]uint64
//   bits   [(m+63)/64]uint64
//   hash   sha384 (384 bits == 48 bytes)
//
//   size = (3 + k + (m+63)/64) * 8 bytes
//

func (f *Filter) marshal() (buf *bytes.Buffer, hash [sha512.Size384]byte, err error) {
	f.lock.RLock()
	defer f.lock.RUnlock()

	debug("write bf k=%d n=%d m=%d\n", f.K(), f.n, f.m)

	buf = new(bytes.Buffer)

	err = binary.Write(buf, binary.LittleEndian, f.K())
	if err != nil {
		return
	}

	err = binary.Write(buf, binary.LittleEndian, f.n)
	if err != nil {
		return
	}

	err = binary.Write(buf, binary.LittleEndian, f.m)
	if err != nil {
		return
	}

	kLen := uint64(len(f.keys))
	debug("write bf kLen%d", kLen)
	err = binary.Write(buf, binary.LittleEndian, kLen)
	if err != nil {
		return
	}
	err = binary.Write(buf, binary.LittleEndian, f.keys)
	if err != nil {
		return
	}

	bLen := uint64(len(f.bits))
	debug("write bf bLen%d", bLen)
	err = binary.Write(buf, binary.LittleEndian, bLen)
	if err != nil {
		return
	}
	err = binary.Write(buf, binary.LittleEndian, f.bits)
	if err != nil {
		return
	}

	hash = sha512.Sum384(buf.Bytes())
	err = binary.Write(buf, binary.LittleEndian, hash)
	return
}

func (f *Filter) MarshalBinary() (data []byte, err error) {
	buf, hash, err := f.marshal()
	if err != nil {
		return
	}

	data = buf.Bytes()

	debug("bloomfilter.MarshalBinary: Successfully wrote %d byte(s), sha384 %v", buf.Len(), hash)

	return
}
<commit_msg>Update binarymarshaler.go<commit_after>//
// Face-meltingly fast, thread-safe, marshalable, unionable, probability- and optimal-size-calculating Bloom filter in go
//
// https://github.com/steakknife/bloomfilter
//
// Copyright © 2014, 2015 Barry Allard
//
// MIT license
//
package bloomfilter

import (
	"bytes"
	"crypto/sha512"
	"encoding/binary"
)

// conforms to encoding.BinaryMarshaler

// marshalled binary layout (Little Endian):
//
//   k      1 uint64
//   n      1 uint64
//   m      1 uint64
//   keys   [k]uint64
//   bits   [(m+63)/64]uint64
//   hash   sha384 (384 bits == 48 bytes)
//
//   size = (3 + k + (m+63)/64) * 8 bytes
//

func (f *Filter) marshal() (buf *bytes.Buffer, hash [sha512.Size384]byte, err error) {
	f.lock.RLock()
	defer f.lock.RUnlock()

	debug("write bf k=%d n=%d m=%d\n", f.K(), f.n, f.m)

	buf = new(bytes.Buffer)

	err = binary.Write(buf, binary.LittleEndian, f.K())
	if err != nil {
		return
	}

	err = binary.Write(buf, binary.LittleEndian, f.n)
	if err != nil {
		return
	}

	err = binary.Write(buf, binary.LittleEndian, f.m)
	if err != nil {
		return
	}

	err = binary.Write(buf, binary.LittleEndian, f.keys)
	if err != nil {
		return
	}

	err = binary.Write(buf, binary.LittleEndian, f.bits)
	if err != nil {
		return
	}

	hash = sha512.Sum384(buf.Bytes())
	err = binary.Write(buf, binary.LittleEndian, hash)
	return
}

func (f *Filter) MarshalBinary() (data []byte, err error) {
	buf, hash, err := f.marshal()
	if err != nil {
		return
	}

	data = buf.Bytes()

	debug("bloomfilter.MarshalBinary: Successfully wrote %d byte(s), sha384 %v", buf.Len(), hash)

	return
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package binding

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"net/http"
	"reflect"
	"strconv"
	"strings"
)

type (
	Binding interface {
		Bind(*http.Request, interface{}) error
	}

	// JSON binding
	jsonBinding struct{}

	// XML binding
	xmlBinding struct{}

	// // form binding
	formBinding struct{}
)

var (
	JSON = jsonBinding{}
	XML  = xmlBinding{}
	Form = formBinding{} // todo
)

func (_ jsonBinding) Bind(req *http.Request, obj interface{}) error {
	decoder := json.NewDecoder(req.Body)
	if err := decoder.Decode(obj); err == nil {
		return Validate(obj)
	} else {
		return err
	}
}

func (_ xmlBinding) Bind(req *http.Request, obj interface{}) error {
	decoder := xml.NewDecoder(req.Body)
	if err := decoder.Decode(obj); err == nil {
		return Validate(obj)
	} else {
		return err
	}
}

func (_ formBinding) Bind(req *http.Request, obj interface{}) error {
	if err := req.ParseForm(); err != nil {
		return err
	}
	if err := mapForm(obj, req.Form); err != nil {
		return err
	}
	return Validate(obj)
}

func mapForm(ptr interface{}, form map[string][]string) error {
	typ := reflect.TypeOf(ptr).Elem()
	formStruct := reflect.ValueOf(ptr).Elem()
	for i := 0; i < typ.NumField(); i++ {
		typeField := typ.Field(i)
		if inputFieldName := typeField.Tag.Get("form"); inputFieldName != "" {
			structField := formStruct.Field(i)
			if !structField.CanSet() {
				continue
			}

			inputValue, exists := form[inputFieldName]
			if !exists {
				continue
			}
			numElems := len(inputValue)
			if structField.Kind() == reflect.Slice && numElems > 0 {
				sliceOf := structField.Type().Elem().Kind()
				slice := reflect.MakeSlice(structField.Type(), numElems, numElems)
				for i := 0; i < numElems; i++ {
					if err := setWithProperType(sliceOf, inputValue[i], slice.Index(i)); err != nil {
						return err
					}
				}
				formStruct.Elem().Field(i).Set(slice)
			} else {
				if err := setWithProperType(typeField.Type.Kind(), inputValue[0], structField); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func setWithProperType(valueKind reflect.Kind, val string, structField reflect.Value) error {
	switch valueKind {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		if val == "" {
			val = "0"
		}
		intVal, err := strconv.Atoi(val)
		if err != nil {
			return err
		} else {
			structField.SetInt(int64(intVal))
		}
	case reflect.Bool:
		if val == "" {
			val = "false"
		}
		boolVal, err := strconv.ParseBool(val)
		if err != nil {
			return err
		} else {
			structField.SetBool(boolVal)
		}
	case reflect.Float32:
		if val == "" {
			val = "0.0"
		}
		floatVal, err := strconv.ParseFloat(val, 32)
		if err != nil {
			return err
		} else {
			structField.SetFloat(floatVal)
		}
	case reflect.Float64:
		if val == "" {
			val = "0.0"
		}
		floatVal, err := strconv.ParseFloat(val, 64)
		if err != nil {
			return err
		} else {
			structField.SetFloat(floatVal)
		}
	case reflect.String:
		structField.SetString(val)
	}
	return nil
}

// Don't pass in pointers to bind to. Can lead to bugs. See:
// https://github.com/codegangsta/martini-contrib/issues/40
// https://github.com/codegangsta/martini-contrib/pull/34#issuecomment-29683659
func ensureNotPointer(obj interface{}) {
	if reflect.TypeOf(obj).Kind() == reflect.Ptr {
		panic("Pointers are not accepted as binding models")
	}
}

func Validate(obj interface{}) error {
	typ := reflect.TypeOf(obj)
	val := reflect.ValueOf(obj)

	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
		val = val.Elem()
	}

	switch typ.Kind() {
	case reflect.Struct:
		for i := 0; i < typ.NumField(); i++ {
			field := typ.Field(i)

			// Allow ignored and unexported fields in the struct
			if field.Tag.Get("form") == "-" || field.PkgPath != "" {
				continue
			}

			fieldValue := val.Field(i).Interface()
			zero := reflect.Zero(field.Type).Interface()

			if strings.Index(field.Tag.Get("binding"), "required") > -1 {
				fieldType := field.Type.Kind()
				if fieldType == reflect.Struct {
					err := Validate(fieldValue)
					if err != nil {
						return err
					}
				} else if reflect.DeepEqual(zero, fieldValue) {
					return errors.New("Required " + field.Name)
				} else if fieldType == reflect.Slice && field.Type.Elem().Kind() == reflect.Struct {
					err := Validate(fieldValue)
					if err != nil {
						return err
					}
				}
			}
		}
	case reflect.Slice:
		for i := 0; i < val.Len(); i++ {
			fieldValue := val.Index(i).Interface()
			err := Validate(fieldValue)
			if err != nil {
				return err
			}
		}
	default:
		return nil
	}
	return nil
}
<commit_msg>Fix binding.go panic<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package binding

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"net/http"
	"reflect"
	"strconv"
	"strings"
)

type (
	Binding interface {
		Bind(*http.Request, interface{}) error
	}

	// JSON binding
	jsonBinding struct{}

	// XML binding
	xmlBinding struct{}

	// // form binding
	formBinding struct{}
)

var (
	JSON = jsonBinding{}
	XML  = xmlBinding{}
	Form = formBinding{} // todo
)

func (_ jsonBinding) Bind(req *http.Request, obj interface{}) error {
	decoder := json.NewDecoder(req.Body)
	if err := decoder.Decode(obj); err == nil {
		return Validate(obj)
	} else {
		return err
	}
}

func (_ xmlBinding) Bind(req *http.Request, obj interface{}) error {
	decoder := xml.NewDecoder(req.Body)
	if err := decoder.Decode(obj); err == nil {
		return Validate(obj)
	} else {
		return err
	}
}

func (_ formBinding) Bind(req *http.Request, obj interface{}) error {
	if err := req.ParseForm(); err != nil {
		return err
	}
	if err := mapForm(obj, req.Form); err != nil {
		return err
	}
	return Validate(obj)
}

func mapForm(ptr interface{}, form map[string][]string) error {
	typ := reflect.TypeOf(ptr).Elem()
	formStruct := reflect.ValueOf(ptr).Elem()
	for i := 0; i < typ.NumField(); i++ {
		typeField := typ.Field(i)
		if inputFieldName := typeField.Tag.Get("form"); inputFieldName != "" {
			structField := formStruct.Field(i)
			if !structField.CanSet() {
				continue
			}

			inputValue, exists := form[inputFieldName]
			if !exists {
				continue
			}
			numElems := len(inputValue)
			if structField.Kind() == reflect.Slice && numElems > 0 {
				sliceOf := structField.Type().Elem().Kind()
				slice := reflect.MakeSlice(structField.Type(), numElems, numElems)
				for i := 0; i < numElems; i++ {
					if err := setWithProperType(sliceOf, inputValue[i], slice.Index(i)); err != nil {
						return err
					}
				}
				formStruct.Field(i).Set(slice)
			} else {
				if err := setWithProperType(typeField.Type.Kind(), inputValue[0], structField); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func setWithProperType(valueKind reflect.Kind, val string, structField reflect.Value) error {
	switch valueKind {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		if val == "" {
			val = "0"
		}
		intVal, err := strconv.Atoi(val)
		if err != nil {
			return err
		} else {
			structField.SetInt(int64(intVal))
		}
	case reflect.Bool:
		if val == "" {
			val = "false"
		}
		boolVal, err := strconv.ParseBool(val)
		if err != nil {
			return err
		} else {
			structField.SetBool(boolVal)
		}
	case reflect.Float32:
		if val == "" {
			val = "0.0"
		}
		floatVal, err := strconv.ParseFloat(val, 32)
		if err != nil {
			return err
		} else {
			structField.SetFloat(floatVal)
		}
	case reflect.Float64:
		if val == "" {
			val = "0.0"
		}
		floatVal, err := strconv.ParseFloat(val, 64)
		if err != nil {
			return err
		} else {
			structField.SetFloat(floatVal)
		}
	case reflect.String:
		structField.SetString(val)
	}
	return nil
}

// Don't pass in pointers to bind to. Can lead to bugs. See:
// https://github.com/codegangsta/martini-contrib/issues/40
// https://github.com/codegangsta/martini-contrib/pull/34#issuecomment-29683659
func ensureNotPointer(obj interface{}) {
	if reflect.TypeOf(obj).Kind() == reflect.Ptr {
		panic("Pointers are not accepted as binding models")
	}
}

func Validate(obj interface{}) error {
	typ := reflect.TypeOf(obj)
	val := reflect.ValueOf(obj)

	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
		val = val.Elem()
	}

	switch typ.Kind() {
	case reflect.Struct:
		for i := 0; i < typ.NumField(); i++ {
			field := typ.Field(i)

			// Allow ignored and unexported fields in the struct
			if field.Tag.Get("form") == "-" || field.PkgPath != "" {
				continue
			}

			fieldValue := val.Field(i).Interface()
			zero := reflect.Zero(field.Type).Interface()

			if strings.Index(field.Tag.Get("binding"), "required") > -1 {
				fieldType := field.Type.Kind()
				if fieldType == reflect.Struct {
					err := Validate(fieldValue)
					if err != nil {
						return err
					}
				} else if reflect.DeepEqual(zero, fieldValue) {
					return errors.New("Required " + field.Name)
				} else if fieldType == reflect.Slice && field.Type.Elem().Kind() == reflect.Struct {
					err := Validate(fieldValue)
					if err != nil {
						return err
					}
				}
			}
		}
	case reflect.Slice:
		for i := 0; i < val.Len(); i++ {
			fieldValue := val.Index(i).Interface()
			err := Validate(fieldValue)
			if err != nil {
				return err
			}
		}
	default:
		return nil
	}
	return nil
}
<|endoftext|>