<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package alpha

import (
	"fmt"
	"io"
	"text/tabwriter"

	"github.com/lithammer/dedent"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"

	"k8s.io/apimachinery/pkg/util/duration"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
	kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal"
	"k8s.io/kubernetes/cmd/kubeadm/app/phases/copycerts"
	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
)

var (
	genericCertRenewLongDesc = cmdutil.LongDesc(`
	Renew the %s.

	Renewals run unconditionally, regardless of certificate expiration date; extra attributes such as SANs will
	be based on the existing file/certificates, there is no need to resupply them.

	Renewal by default tries to use the certificate authority in the local PKI managed by kubeadm; as alternative
	it is possible to use K8s certificate API for certificate renewal, or as a last option, to generate a CSR request.

	After renewal, in order to make changes effective, is required to restart control-plane components and
	eventually re-distribute the renewed certificate in case the file is used elsewhere.
`)

	allLongDesc = cmdutil.LongDesc(`
    Renew all known certificates necessary to run the control plane. Renewals are run unconditionally, regardless
    of expiration date. Renewals can also be run individually for more control.
`)

	expirationLongDesc = cmdutil.LongDesc(`
	Checks expiration for the certificates in the local PKI managed by kubeadm.
`)

	certificateKeyLongDesc = dedent.Dedent(`
	This command will print out a secure randomly-generated certificate key that can be used with
	the "init" command.

	You can also use "kubeadm init --upload-certs" without specifying a certificate key and it will
	generate and print one for you.
`)
)

// newCmdCertsUtility returns main command for certs phase
func newCmdCertsUtility(out io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use:     "certs",
		Aliases: []string{"certificates"},
		Short:   "Commands related to handling kubernetes certificates",
	}

	cmd.AddCommand(newCmdCertsRenewal(out))
	cmd.AddCommand(newCmdCertsExpiration(out, constants.KubernetesDir))
	cmd.AddCommand(NewCmdCertificateKey())
	return cmd
}

// NewCmdCertificateKey returns cobra.Command for certificate key generate
func NewCmdCertificateKey() *cobra.Command {
	return &cobra.Command{
		Use:   "certificate-key",
		Short: "Generate certificate keys",
		Long:  certificateKeyLongDesc,

		RunE: func(cmd *cobra.Command, args []string) error {
			key, err := copycerts.CreateCertificateKey()
			if err != nil {
				return err
			}
			fmt.Println(key)
			return nil
		},
	}
}

// newCmdCertsRenewal creates a new `cert renew` command.
func newCmdCertsRenewal(out io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "renew",
		Short: "Renew certificates for a Kubernetes cluster",
		Long:  cmdutil.MacroCommandLongDescription,
		RunE:  cmdutil.SubCmdRunE("renew"),
	}

	cmd.AddCommand(getRenewSubCommands(out, constants.KubernetesDir)...)

	return cmd
}

type renewFlags struct {
	cfgPath        string
	kubeconfigPath string
	cfg            kubeadmapiv1beta2.ClusterConfiguration
	useAPI         bool
	csrOnly        bool
	csrPath        string
}

func getRenewSubCommands(out io.Writer, kdir string) []*cobra.Command {
	flags := &renewFlags{
		cfg: kubeadmapiv1beta2.ClusterConfiguration{
			// Setting kubernetes version to a default value in order to allow a not necessary internet lookup
			KubernetesVersion: constants.CurrentKubernetesVersion.String(),
		},
		kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(),
	}
	// Default values for the cobra help text
	kubeadmscheme.Scheme.Default(&flags.cfg)

	// Get a renewal manager for a generic Cluster configuration, that is used only for getting
	// the list of certificates for building subcommands
	rm, err := renewal.NewManager(&kubeadmapi.ClusterConfiguration{}, "")
	if err != nil {
		return nil
	}

	cmdList := []*cobra.Command{}
	for _, handler := range rm.Certificates() {
		// get the cobra.Command skeleton for this command
		cmd := &cobra.Command{
			Use:   handler.Name,
			Short: fmt.Sprintf("Renew the %s", handler.LongName),
			Long:  fmt.Sprintf(genericCertRenewLongDesc, handler.LongName),
		}
		addRenewFlags(cmd, flags)
		// get the implementation of renewing this certificate
		renewalFunc := func(handler *renewal.CertificateRenewHandler) func() error {
			return func() error {
				// Get cluster configuration (from --config, kubeadm-config ConfigMap, or default as a fallback)
				internalcfg, err := getInternalCfg(flags.cfgPath, flags.kubeconfigPath, flags.cfg, out, "renew")
				if err != nil {
					return err
				}

				return renewCert(flags, kdir, internalcfg, handler)
			}
		}(handler)
		// install the implementation into the command
		cmd.RunE = func(*cobra.Command, []string) error { return renewalFunc() }
		cmdList = append(cmdList, cmd)
	}

	allCmd := &cobra.Command{
		Use:   "all",
		Short: "Renew all available certificates",
		Long:  allLongDesc,
		RunE: func(*cobra.Command, []string) error {
			// Get cluster configuration (from --config, kubeadm-config ConfigMap, or default as a fallback)
			internalcfg, err := getInternalCfg(flags.cfgPath, flags.kubeconfigPath, flags.cfg, out, "renew")
			if err != nil {
				return err
			}

			// Get a renewal manager for a actual Cluster configuration
			rm, err := renewal.NewManager(&internalcfg.ClusterConfiguration, kdir)
			if err != nil {
				return nil
			}

			// Renew certificates
			for _, handler := range rm.Certificates() {
				if err := renewCert(flags, kdir, internalcfg, handler); err != nil {
					return err
				}
			}
			return nil
		},
	}
	addRenewFlags(allCmd, flags)

	cmdList = append(cmdList, allCmd)
	return cmdList
}

func addRenewFlags(cmd *cobra.Command, flags *renewFlags) {
	options.AddConfigFlag(cmd.Flags(), &flags.cfgPath)
	options.AddCertificateDirFlag(cmd.Flags(), &flags.cfg.CertificatesDir)
	options.AddKubeConfigFlag(cmd.Flags(), &flags.kubeconfigPath)
	options.AddCSRFlag(cmd.Flags(), &flags.csrOnly)
	options.AddCSRDirFlag(cmd.Flags(), &flags.csrPath)
	cmd.Flags().BoolVar(&flags.useAPI, "use-api", flags.useAPI, "Use the Kubernetes certificate API to renew certificates")
}

func renewCert(flags *renewFlags, kdir string, internalcfg *kubeadmapi.InitConfiguration, handler *renewal.CertificateRenewHandler) error {
	// Get a renewal manager for the given cluster configuration
	rm, err := renewal.NewManager(&internalcfg.ClusterConfiguration, kdir)
	if err != nil {
		return err
	}

	if ok, _ := rm.CertificateExists(handler.Name); !ok {
		fmt.Printf("MISSING! %s\n", handler.LongName)
		return nil
	}

	// if the renewal operation is set to generate CSR request only
	if flags.csrOnly {
		// checks a path for storing CSR request is given
		if flags.csrPath == "" {
			return errors.New("please provide a path where CSR request should be stored")
		}
		return rm.CreateRenewCSR(handler.Name, flags.csrPath)
	}

	// otherwise, the renewal operation has to actually renew a certificate

	// renew the certificate using the requested renew method
	if flags.useAPI {
		// renew using K8s certificate API
		kubeConfigPath := cmdutil.GetKubeConfigPath(flags.kubeconfigPath)
		client, err := kubeconfigutil.ClientSetFromFile(kubeConfigPath)
		if err != nil {
			return err
		}

		if err := rm.RenewUsingCSRAPI(handler.Name, client); err != nil {
			return err
		}
	} else {
		// renew using local certificate authorities.
		// this operation can't complete in case the certificate key is not provided (external CA)
		renewed, err := rm.RenewUsingLocalCA(handler.Name)
		if err != nil {
			return err
		}
		if !renewed {
			fmt.Printf("Detected external %s, %s can't be renewed\n", handler.CABaseName, handler.LongName)
			return nil
		}
	}
	fmt.Printf("%s renewed\n", handler.LongName)
	return nil
}

func getInternalCfg(cfgPath string, kubeconfigPath string, cfg kubeadmapiv1beta2.ClusterConfiguration, out io.Writer, logPrefix string) (*kubeadmapi.InitConfiguration, error) {
	// In case the user is not providing a custom config, try to get current config from the cluster.
	// NB. this operation should not block, because we want to allow certificate renewal also in case of not-working clusters
	if cfgPath == "" {
		client, err := kubeconfigutil.ClientSetFromFile(kubeconfigPath)
		if err == nil {
			internalcfg, err := configutil.FetchInitConfigurationFromCluster(client, out, logPrefix, false)
			if err == nil {
				fmt.Println() // add empty line to separate the FetchInitConfigurationFromCluster output from the command output
				return internalcfg, nil
			}
			fmt.Printf("[%s] Error reading configuration from the Cluster. Falling back to default configuration\n\n", logPrefix)
		}
	}

	// Otherwise read config from --config if provided, otherwise use default configuration
	return configutil.LoadOrDefaultInitConfiguration(cfgPath, &kubeadmapiv1beta2.InitConfiguration{}, &cfg)
}

// newCmdCertsExpiration creates a new `cert check-expiration` command.
func newCmdCertsExpiration(out io.Writer, kdir string) *cobra.Command {
	flags := &expirationFlags{
		cfg: kubeadmapiv1beta2.ClusterConfiguration{
			// Setting kubernetes version to a default value in order to allow a not necessary internet lookup
			KubernetesVersion: constants.CurrentKubernetesVersion.String(),
		},
		kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(),
	}
	// Default values for the cobra help text
	kubeadmscheme.Scheme.Default(&flags.cfg)

	cmd := &cobra.Command{
		Use:   "check-expiration",
		Short: "Check certificates expiration for a Kubernetes cluster",
		Long:  expirationLongDesc,
		RunE: func(cmd *cobra.Command, args []string) error {
			// Get cluster configuration (from --config, kubeadm-config ConfigMap, or default as a fallback)
			internalcfg, err := getInternalCfg(flags.cfgPath, flags.kubeconfigPath, flags.cfg, out, "check-expiration")
			if err != nil {
				return err
			}

			// Get a renewal manager for the given cluster configuration
			rm, err := renewal.NewManager(&internalcfg.ClusterConfiguration, kdir)
			if err != nil {
				return err
			}

			// Get all the certificate expiration info
			yesNo := func(b bool) string {
				if b {
					return "yes"
				}
				return "no"
			}
			w := tabwriter.NewWriter(out, 10, 4, 3, ' ', 0)
			fmt.Fprintln(w, "CERTIFICATE\tEXPIRES\tRESIDUAL TIME\tCERTIFICATE AUTHORITY\tEXTERNALLY MANAGED")
			for _, handler := range rm.Certificates() {
				if ok, _ := rm.CertificateExists(handler.Name); ok {
					e, err := rm.GetCertificateExpirationInfo(handler.Name)
					if err != nil {
						return err
					}

					s := fmt.Sprintf("%s\t%s\t%s\t%s\t%-8v",
						e.Name,
						e.ExpirationDate.Format("Jan 02, 2006 15:04 MST"),
						duration.ShortHumanDuration(e.ResidualTime()),
						handler.CAName,
						yesNo(e.ExternallyManaged),
					)

					fmt.Fprintln(w, s)
					continue
				}

				// the certificate does not exist (for any reason)
				s := fmt.Sprintf("!MISSING! %s\t\t\t\t",
					handler.Name,
				)
				fmt.Fprintln(w, s)
			}
			fmt.Fprintln(w)
			fmt.Fprintln(w, "CERTIFICATE AUTHORITY\tEXPIRES\tRESIDUAL TIME\tEXTERNALLY MANAGED")
			for _, handler := range rm.CAs() {
				if ok, _ := rm.CAExists(handler.Name); ok {
					e, err := rm.GetCAExpirationInfo(handler.Name)
					if err != nil {
						return err
					}

					s := fmt.Sprintf("%s\t%s\t%s\t%-8v",
						e.Name,
						e.ExpirationDate.Format("Jan 02, 2006 15:04 MST"),
						duration.ShortHumanDuration(e.ResidualTime()),
						yesNo(e.ExternallyManaged),
					)

					fmt.Fprintln(w, s)
					continue
				}

				// the CA does not exist (for any reason)
				s := fmt.Sprintf("!MISSING! %s\t\t\t",
					handler.Name,
				)
				fmt.Fprintln(w, s)
			}
			w.Flush()
			return nil
		},
	}
	addExpirationFlags(cmd, flags)

	return cmd
}

type expirationFlags struct {
	cfgPath        string
	kubeconfigPath string
	cfg            kubeadmapiv1beta2.ClusterConfiguration
}

func addExpirationFlags(cmd *cobra.Command, flags *expirationFlags) {
	options.AddConfigFlag(cmd.Flags(), &flags.cfgPath)
	options.AddCertificateDirFlag(cmd.Flags(), &flags.cfg.CertificatesDir)
	options.AddKubeConfigFlag(cmd.Flags(), &flags.kubeconfigPath)
}
<commit_msg>kubeadm: deprecate the flag --use-api for cert renewal<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package alpha

import (
	"fmt"
	"io"
	"text/tabwriter"

	"github.com/lithammer/dedent"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"

	"k8s.io/apimachinery/pkg/util/duration"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
	kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal"
	"k8s.io/kubernetes/cmd/kubeadm/app/phases/copycerts"
	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
)

var (
	genericCertRenewLongDesc = cmdutil.LongDesc(`
	Renew the %s.

	Renewals run unconditionally, regardless of certificate expiration date; extra attributes such as SANs will
	be based on the existing file/certificates, there is no need to resupply them.

	Renewal by default tries to use the certificate authority in the local PKI managed by kubeadm; as alternative
	it is possible to use K8s certificate API for certificate renewal, or as a last option, to generate a CSR request.

	After renewal, in order to make changes effective, is required to restart control-plane components and
	eventually re-distribute the renewed certificate in case the file is used elsewhere.
`)

	allLongDesc = cmdutil.LongDesc(`
    Renew all known certificates necessary to run the control plane. Renewals are run unconditionally, regardless
    of expiration date. Renewals can also be run individually for more control.
`)

	expirationLongDesc = cmdutil.LongDesc(`
	Checks expiration for the certificates in the local PKI managed by kubeadm.
`)

	certificateKeyLongDesc = dedent.Dedent(`
	This command will print out a secure randomly-generated certificate key that can be used with
	the "init" command.

	You can also use "kubeadm init --upload-certs" without specifying a certificate key and it will
	generate and print one for you.
`)
)

// newCmdCertsUtility returns main command for certs phase
func newCmdCertsUtility(out io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use:     "certs",
		Aliases: []string{"certificates"},
		Short:   "Commands related to handling kubernetes certificates",
	}

	cmd.AddCommand(newCmdCertsRenewal(out))
	cmd.AddCommand(newCmdCertsExpiration(out, constants.KubernetesDir))
	cmd.AddCommand(NewCmdCertificateKey())
	return cmd
}

// NewCmdCertificateKey returns cobra.Command for certificate key generate
func NewCmdCertificateKey() *cobra.Command {
	return &cobra.Command{
		Use:   "certificate-key",
		Short: "Generate certificate keys",
		Long:  certificateKeyLongDesc,

		RunE: func(cmd *cobra.Command, args []string) error {
			key, err := copycerts.CreateCertificateKey()
			if err != nil {
				return err
			}
			fmt.Println(key)
			return nil
		},
	}
}

// newCmdCertsRenewal creates a new `cert renew` command.
func newCmdCertsRenewal(out io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "renew",
		Short: "Renew certificates for a Kubernetes cluster",
		Long:  cmdutil.MacroCommandLongDescription,
		RunE:  cmdutil.SubCmdRunE("renew"),
	}

	cmd.AddCommand(getRenewSubCommands(out, constants.KubernetesDir)...)

	return cmd
}

type renewFlags struct {
	cfgPath        string
	kubeconfigPath string
	cfg            kubeadmapiv1beta2.ClusterConfiguration
	useAPI         bool
	csrOnly        bool
	csrPath        string
}

func getRenewSubCommands(out io.Writer, kdir string) []*cobra.Command {
	flags := &renewFlags{
		cfg: kubeadmapiv1beta2.ClusterConfiguration{
			// Setting kubernetes version to a default value in order to allow a not necessary internet lookup
			KubernetesVersion: constants.CurrentKubernetesVersion.String(),
		},
		kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(),
	}
	// Default values for the cobra help text
	kubeadmscheme.Scheme.Default(&flags.cfg)

	// Get a renewal manager for a generic Cluster configuration, that is used only for getting
	// the list of certificates for building subcommands
	rm, err := renewal.NewManager(&kubeadmapi.ClusterConfiguration{}, "")
	if err != nil {
		return nil
	}

	cmdList := []*cobra.Command{}
	for _, handler := range rm.Certificates() {
		// get the cobra.Command skeleton for this command
		cmd := &cobra.Command{
			Use:   handler.Name,
			Short: fmt.Sprintf("Renew the %s", handler.LongName),
			Long:  fmt.Sprintf(genericCertRenewLongDesc, handler.LongName),
		}
		addRenewFlags(cmd, flags)
		// get the implementation of renewing this certificate
		renewalFunc := func(handler *renewal.CertificateRenewHandler) func() error {
			return func() error {
				// Get cluster configuration (from --config, kubeadm-config ConfigMap, or default as a fallback)
				internalcfg, err := getInternalCfg(flags.cfgPath, flags.kubeconfigPath, flags.cfg, out, "renew")
				if err != nil {
					return err
				}

				return renewCert(flags, kdir, internalcfg, handler)
			}
		}(handler)
		// install the implementation into the command
		cmd.RunE = func(*cobra.Command, []string) error { return renewalFunc() }
		cmdList = append(cmdList, cmd)
	}

	allCmd := &cobra.Command{
		Use:   "all",
		Short: "Renew all available certificates",
		Long:  allLongDesc,
		RunE: func(*cobra.Command, []string) error {
			// Get cluster configuration (from --config, kubeadm-config ConfigMap, or default as a fallback)
			internalcfg, err := getInternalCfg(flags.cfgPath, flags.kubeconfigPath, flags.cfg, out, "renew")
			if err != nil {
				return err
			}

			// Get a renewal manager for a actual Cluster configuration
			rm, err := renewal.NewManager(&internalcfg.ClusterConfiguration, kdir)
			if err != nil {
				return nil
			}

			// Renew certificates
			for _, handler := range rm.Certificates() {
				if err := renewCert(flags, kdir, internalcfg, handler); err != nil {
					return err
				}
			}
			return nil
		},
	}
	addRenewFlags(allCmd, flags)

	cmdList = append(cmdList, allCmd)
	return cmdList
}

func addRenewFlags(cmd *cobra.Command, flags *renewFlags) {
	options.AddConfigFlag(cmd.Flags(), &flags.cfgPath)
	options.AddCertificateDirFlag(cmd.Flags(), &flags.cfg.CertificatesDir)
	options.AddKubeConfigFlag(cmd.Flags(), &flags.kubeconfigPath)
	options.AddCSRFlag(cmd.Flags(), &flags.csrOnly)
	options.AddCSRDirFlag(cmd.Flags(), &flags.csrPath)
	// TODO: remove the flag and related logic once legacy signers are removed,
	// potentially with the release of certificates.k8s.io/v1:
	//   https://github.com/kubernetes/kubeadm/issues/2047
	cmd.Flags().BoolVar(&flags.useAPI, "use-api", flags.useAPI, "Use the Kubernetes certificate API to renew certificates")
	cmd.Flags().MarkDeprecated("use-api", "certificate renewal from kubeadm using the Kubernetes API "+
		"is deprecated and will be removed when 'certificates.k8s.io/v1' releases.")
}

func renewCert(flags *renewFlags, kdir string, internalcfg *kubeadmapi.InitConfiguration, handler *renewal.CertificateRenewHandler) error {
	// Get a renewal manager for the given cluster configuration
	rm, err := renewal.NewManager(&internalcfg.ClusterConfiguration, kdir)
	if err != nil {
		return err
	}

	if ok, _ := rm.CertificateExists(handler.Name); !ok {
		fmt.Printf("MISSING! %s\n", handler.LongName)
		return nil
	}

	// if the renewal operation is set to generate CSR request only
	if flags.csrOnly {
		// checks a path for storing CSR request is given
		if flags.csrPath == "" {
			return errors.New("please provide a path where CSR request should be stored")
		}
		return rm.CreateRenewCSR(handler.Name, flags.csrPath)
	}

	// otherwise, the renewal operation has to actually renew a certificate

	// renew the certificate using the requested renew method
	if flags.useAPI {
		// renew using K8s certificate API
		kubeConfigPath := cmdutil.GetKubeConfigPath(flags.kubeconfigPath)
		client, err := kubeconfigutil.ClientSetFromFile(kubeConfigPath)
		if err != nil {
			return err
		}

		if err := rm.RenewUsingCSRAPI(handler.Name, client); err != nil {
			return err
		}
	} else {
		// renew using local certificate authorities.
		// this operation can't complete in case the certificate key is not provided (external CA)
		renewed, err := rm.RenewUsingLocalCA(handler.Name)
		if err != nil {
			return err
		}
		if !renewed {
			fmt.Printf("Detected external %s, %s can't be renewed\n", handler.CABaseName, handler.LongName)
			return nil
		}
	}
	fmt.Printf("%s renewed\n", handler.LongName)
	return nil
}

func getInternalCfg(cfgPath string, kubeconfigPath string, cfg kubeadmapiv1beta2.ClusterConfiguration, out io.Writer, logPrefix string) (*kubeadmapi.InitConfiguration, error) {
	// In case the user is not providing a custom config, try to get current config from the cluster.
	// NB. this operation should not block, because we want to allow certificate renewal also in case of not-working clusters
	if cfgPath == "" {
		client, err := kubeconfigutil.ClientSetFromFile(kubeconfigPath)
		if err == nil {
			internalcfg, err := configutil.FetchInitConfigurationFromCluster(client, out, logPrefix, false)
			if err == nil {
				fmt.Println() // add empty line to separate the FetchInitConfigurationFromCluster output from the command output
				return internalcfg, nil
			}
			fmt.Printf("[%s] Error reading configuration from the Cluster. Falling back to default configuration\n\n", logPrefix)
		}
	}

	// Otherwise read config from --config if provided, otherwise use default configuration
	return configutil.LoadOrDefaultInitConfiguration(cfgPath, &kubeadmapiv1beta2.InitConfiguration{}, &cfg)
}

// newCmdCertsExpiration creates a new `cert check-expiration` command.
func newCmdCertsExpiration(out io.Writer, kdir string) *cobra.Command {
	flags := &expirationFlags{
		cfg: kubeadmapiv1beta2.ClusterConfiguration{
			// Setting kubernetes version to a default value in order to allow a not necessary internet lookup
			KubernetesVersion: constants.CurrentKubernetesVersion.String(),
		},
		kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(),
	}
	// Default values for the cobra help text
	kubeadmscheme.Scheme.Default(&flags.cfg)

	cmd := &cobra.Command{
		Use:   "check-expiration",
		Short: "Check certificates expiration for a Kubernetes cluster",
		Long:  expirationLongDesc,
		RunE: func(cmd *cobra.Command, args []string) error {
			// Get cluster configuration (from --config, kubeadm-config ConfigMap, or default as a fallback)
			internalcfg, err := getInternalCfg(flags.cfgPath, flags.kubeconfigPath, flags.cfg, out, "check-expiration")
			if err != nil {
				return err
			}

			// Get a renewal manager for the given cluster configuration
			rm, err := renewal.NewManager(&internalcfg.ClusterConfiguration, kdir)
			if err != nil {
				return err
			}

			// Get all the certificate expiration info
			yesNo := func(b bool) string {
				if b {
					return "yes"
				}
				return "no"
			}
			w := tabwriter.NewWriter(out, 10, 4, 3, ' ', 0)
			fmt.Fprintln(w, "CERTIFICATE\tEXPIRES\tRESIDUAL TIME\tCERTIFICATE AUTHORITY\tEXTERNALLY MANAGED")
			for _, handler := range rm.Certificates() {
				if ok, _ := rm.CertificateExists(handler.Name); ok {
					e, err := rm.GetCertificateExpirationInfo(handler.Name)
					if err != nil {
						return err
					}

					s := fmt.Sprintf("%s\t%s\t%s\t%s\t%-8v",
						e.Name,
						e.ExpirationDate.Format("Jan 02, 2006 15:04 MST"),
						duration.ShortHumanDuration(e.ResidualTime()),
						handler.CAName,
						yesNo(e.ExternallyManaged),
					)

					fmt.Fprintln(w, s)
					continue
				}

				// the certificate does not exist (for any reason)
				s := fmt.Sprintf("!MISSING! %s\t\t\t\t",
					handler.Name,
				)
				fmt.Fprintln(w, s)
			}
			fmt.Fprintln(w)
			fmt.Fprintln(w, "CERTIFICATE AUTHORITY\tEXPIRES\tRESIDUAL TIME\tEXTERNALLY MANAGED")
			for _, handler := range rm.CAs() {
				if ok, _ := rm.CAExists(handler.Name); ok {
					e, err := rm.GetCAExpirationInfo(handler.Name)
					if err != nil {
						return err
					}

					s := fmt.Sprintf("%s\t%s\t%s\t%-8v",
						e.Name,
						e.ExpirationDate.Format("Jan 02, 2006 15:04 MST"),
						duration.ShortHumanDuration(e.ResidualTime()),
						yesNo(e.ExternallyManaged),
					)

					fmt.Fprintln(w, s)
					continue
				}

				// the CA does not exist (for any reason)
				s := fmt.Sprintf("!MISSING! %s\t\t\t",
					handler.Name,
				)
				fmt.Fprintln(w, s)
			}
			w.Flush()
			return nil
		},
	}
	addExpirationFlags(cmd, flags)

	return cmd
}

type expirationFlags struct {
	cfgPath        string
	kubeconfigPath string
	cfg            kubeadmapiv1beta2.ClusterConfiguration
}

func addExpirationFlags(cmd *cobra.Command, flags *expirationFlags) {
	options.AddConfigFlag(cmd.Flags(), &flags.cfgPath)
	options.AddCertificateDirFlag(cmd.Flags(), &flags.cfg.CertificatesDir)
	options.AddKubeConfigFlag(cmd.Flags(), &flags.kubeconfigPath)
}
<|endoftext|>


<commit_before>package coordinator

import (
	"crypto/rand"
	"encoding/binary"
	"fmt"
	"sync"
	"time"

	"github.com/privacylab/talek/bloom"
	"github.com/privacylab/talek/common"
	"github.com/privacylab/talek/cuckoo"
	"golang.org/x/net/trace"
)

// Server is the main logic for the central coordinator
type Server struct {
	/** Private State **/
	// Static
	log               *common.Logger
	name              string
	snapshotThreshold uint64
	snapshotInterval  time.Duration

	// Thread-safe (locked)
	lock          *sync.Mutex
	config        common.Config // Config
	servers       []NotifyInterface
	commitLog     []*CommitArgs // Append and read only
	numNewCommits uint64
	snapshotCount uint64
	lastLayout    []uint64
	intVec        []uint64
	cuckooData    []byte
	cuckooTable   *cuckoo.Table

	// Channels
	notifyChan chan bool
	closeChan  chan bool
}

// NewServer creates a new Centralized talek server.
func NewServer(name string, config common.Config, servers []NotifyInterface, snapshotThreshold uint64, snapshotInterval time.Duration) (*Server, error) {
	s := &Server{}
	s.log = common.NewLogger(name)
	s.name = name
	s.snapshotThreshold = snapshotThreshold
	s.snapshotInterval = snapshotInterval

	s.lock = &sync.Mutex{}
	s.config = config
	if servers == nil {
		s.servers = make([]NotifyInterface, 0)
	} else {
		s.servers = servers
	}
	s.commitLog = make([]*CommitArgs, 0)
	s.numNewCommits = 0
	s.snapshotCount = 0
	s.lastLayout = make([]uint64, config.NumBuckets*config.BucketDepth)
	s.intVec = buildInterestVector(config.WindowSize(), config.BloomFalsePositive, s.commitLog[:]).Bytes()
	s.cuckooData = make([]byte, config.NumBuckets*config.BucketDepth*uint64(common.IDSize))

	// Choose a random seed for the cuckoo table
	seedBytes := make([]byte, 8)
	_, err := rand.Read(seedBytes)
	if err != nil {
		s.log.Error.Printf("coordinator.NewServer(%v) error: %v", name, err)
		return nil, err
	}
	seed, _ := binary.Varint(seedBytes)
	s.cuckooTable = cuckoo.NewTable(name, config.NumBuckets, config.BucketDepth, uint64(common.IDSize), s.cuckooData, seed)
	// Should not be possible
	if s.cuckooTable == nil {
		err := fmt.Errorf("Invalid cuckoo table parameters")
		s.log.Error.Printf("coordinator.NewServer(%v) error: %v", name, err)
		return nil, err
	}
	s.notifyChan = make(chan bool)
	s.closeChan = make(chan bool)

	go s.loop()

	s.log.Info.Printf("coordinator.NewServer(%v) success\n", name)
	return s, nil
}

/**********************************
 * PUBLIC RPC METHODS (threadsafe)
 **********************************/

// GetInfo returns information about this server
func (s *Server) GetInfo(args *interface{}, reply *GetInfoReply) error {
	tr := trace.New("Coordinator", "GetInfo")
	defer tr.Finish()
	s.lock.Lock()

	reply.Err = ""
	reply.Name = s.name
	reply.SnapshotID = s.snapshotCount

	s.lock.Unlock()
	return nil
}

// GetCommonConfig returns the common global config
func (s *Server) GetCommonConfig(args *interface{}, reply *common.Config) error {
	tr := trace.New("Coordinator", "GetCommonConfig")
	defer tr.Finish()
	s.lock.Lock()

	*reply = s.config

	s.lock.Unlock()
	return nil
}

// GetLayout returns the layout for a shard
func (s *Server) GetLayout(args *GetLayoutArgs, reply *GetLayoutReply) error {
	tr := trace.New("Coordinator", "GetLayout")
	defer tr.Finish()
	s.lock.Lock()

	// Check for correct snapshot ID
	reply.SnapshotID = s.snapshotCount
	if args.SnapshotID != s.snapshotCount {
		reply.Err = "Invalid SnapshotID"
		s.lock.Unlock()
		return nil
	}

	// Check non-zero NumShards
	if args.NumShards < 1 {
		reply.Err = "NumShards must be > 0"
		s.lock.Unlock()
		return nil
	}

	shardSize := uint64(len(s.lastLayout)) / args.NumShards
	idx := args.ShardID * shardSize
	if idx < 0 || (idx+shardSize) > uint64(len(s.lastLayout)) {
		reply.Err = "Out of bounds ShardID"
	} else {
		reply.Err = ""
		reply.Layout = s.lastLayout[idx:(idx + shardSize)]
	}

	s.lock.Unlock()
	return nil
}

// GetIntVec returns the global interest vector
func (s *Server) GetIntVec(args *GetIntVecArgs, reply *GetIntVecReply) error {
	tr := trace.New("Coordinator", "GetIntVec")
	defer tr.Finish()
	s.lock.Lock()

	// Check for correct snapshot ID
	reply.SnapshotID = s.snapshotCount
	if args.SnapshotID != s.snapshotCount {
		reply.Err = "Invalid SnapshotID"
		s.lock.Unlock()
		return nil
	}

	reply.Err = ""
	reply.IntVec = s.intVec[:]

	s.lock.Unlock()
	return nil
}

// Commit accepts a single Write to commit. The
func (s *Server) Commit(args *CommitArgs, reply *CommitReply) error {
	tr := trace.New("Coordinator", "Commit")
	defer tr.Finish()
	reply.Err = ""

	s.lock.Lock()

	windowSize := s.config.WindowSize()
	// Garbage Collect old elements
	for uint64(len(s.commitLog)) >= windowSize {
		_ = s.cuckooTable.Remove(asCuckooItem(s.config.NumBuckets, s.commitLog[0]))
		s.commitLog = s.commitLog[1:]
	}

	// Insert new item
	s.numNewCommits++
	s.commitLog = append(s.commitLog, args)
	ok, _ := s.cuckooTable.Insert(asCuckooItem(s.config.NumBuckets, args))
	if !ok {
		s.log.Error.Fatalf("%v.processCommit failed to insert new element", s.name)
		return fmt.Errorf("Error inserting into cuckoo table")
	}

	s.lock.Unlock()

	// Do notifications in loop()
	s.notifyChan <- false
	return nil
}

/**********************************
 * PUBLIC LOCAL METHODS (threadsafe)
 **********************************/

// Close shuts down the server
func (s *Server) Close() {
	s.log.Info.Printf("%v.Close: success", s.name)
	s.closeChan <- true
}

// AddServer adds a server to the list that is notified on snapshot changes
func (s *Server) AddServer(server NotifyInterface) {
	s.lock.Lock()
	s.servers = append(s.servers, server)
	s.lock.Unlock()
}

// NotifySnapshot notifies the current cuckoo layout out
// If `force` is false, ignore when under a threshold
// Returns: true if snapshot was built, false if ignored
func (s *Server) NotifySnapshot(force bool) bool {
	s.lock.Lock()

	// Ignore if under threshold and not forcing
	if !force {
		if s.numNewCommits < s.snapshotThreshold {
			s.lock.Unlock()
			return false
		}
	}

	// Reset state
	s.numNewCommits = 0
	s.snapshotCount++

	// Construct global interest vector
	s.intVec = buildInterestVector(s.config.WindowSize(), s.config.BloomFalsePositive, s.commitLog[:]).Bytes()

	// Copy the layout
	for i := 0; i < len(s.lastLayout); i++ {
		idx := i * 8
		s.lastLayout[i], _ = binary.Uvarint(s.cuckooData[idx:(idx + 8)])
	}

	// Sync with buildGlobalInterestVector goroutine
	// @todo when this happens in parallel

	// Send notifications only if there are servers
	if s.servers != nil && len(s.servers) > 0 {
		go sendNotification(s.log, s.servers[:], s.snapshotCount)
	}
	s.lock.Unlock()

	return true
}

/**********************************
 * PRIVATE METHODS (single-threaded)
 **********************************/
// Periodically call NotifySnapshot
func (s *Server) loop() {
	tick := time.After(s.snapshotInterval)
	for {
		select {
		// Called after Commit
		case <-s.notifyChan:
			ok := s.NotifySnapshot(false)
			if ok { // Re-up timer if snapshot built
				tick = time.After(s.snapshotInterval)
			}
			continue
		// Periodically trigger a snapshot
		case <-tick:
			s.NotifySnapshot(true)
			tick = time.After(s.snapshotInterval) // Re-up timer
			continue
		// Stop the loop
		case <-s.closeChan:
			return
		}
	}
}

/**********************************
 * HELPER FUNCTIONS
 **********************************/

// Converts a CommitArgs to a cuckoo.Item
func asCuckooItem(numBuckets uint64, args *CommitArgs) *cuckoo.Item {
	itemData := make([]byte, common.IDSize)
	binary.PutUvarint(itemData, args.ID)
	return &cuckoo.Item{
		ID:      args.ID,
		Data:    itemData,
		Bucket1: args.Bucket1 % numBuckets,
		Bucket2: args.Bucket2 % numBuckets,
	}
}

// buildInterestVector traverses the commitLog and creates a global interest vector
// representing the elements.
// Bloom filter stores n elements with fp false positive rate
func buildInterestVector(n uint64, fp float64, commitLog []*CommitArgs) *bloom.BitSet {
	bits, numHash := bloom.EstimateParameters(n, fp)
	intVec := bloom.NewBitSet(bits)
	for _, c := range commitLog {
		// Truncate interest vectors to first numHash bits
		if uint64(len(c.IntVecLoc)) > numHash {
			intVec = bloom.SetLocations(intVec, c.IntVecLoc[:numHash])
		} else {
			intVec = bloom.SetLocations(intVec, c.IntVecLoc[:])
		}
	}
	return intVec
}

func sendNotification(log *common.Logger, servers []NotifyInterface, snapshotID uint64) {
	args := &NotifyArgs{
		SnapshotID: snapshotID,
	}
	doNotify := func(l *common.Logger, s NotifyInterface, args *NotifyArgs) {
		err := s.Notify(args, &NotifyReply{})
		if err != nil {
			l.Error.Printf("sendNotification failed: %v", err)
		}
	}
	for _, s := range servers {
		go doNotify(log, s, args)
	}
}
<commit_msg>minor logs<commit_after>package coordinator

import (
	"crypto/rand"
	"encoding/binary"
	"fmt"
	"sync"
	"time"

	"github.com/privacylab/talek/bloom"
	"github.com/privacylab/talek/common"
	"github.com/privacylab/talek/cuckoo"
	"golang.org/x/net/trace"
)

// Server is the main logic for the central coordinator
type Server struct {
	/** Private State **/
	// Static
	log               *common.Logger
	name              string
	snapshotThreshold uint64
	snapshotInterval  time.Duration

	// Thread-safe (locked)
	lock          *sync.Mutex
	config        common.Config // Config
	servers       []NotifyInterface
	commitLog     []*CommitArgs // Append and read only
	numNewCommits uint64
	snapshotCount uint64
	lastLayout    []uint64
	intVec        []uint64
	cuckooData    []byte
	cuckooTable   *cuckoo.Table

	// Channels
	notifyChan chan bool
	closeChan  chan bool
}

// NewServer creates a new Centralized talek server.
func NewServer(name string, config common.Config, servers []NotifyInterface, snapshotThreshold uint64, snapshotInterval time.Duration) (*Server, error) {
	s := &Server{}
	s.log = common.NewLogger(name)
	s.name = name
	s.snapshotThreshold = snapshotThreshold
	s.snapshotInterval = snapshotInterval

	s.lock = &sync.Mutex{}
	s.config = config
	if servers == nil {
		s.servers = make([]NotifyInterface, 0)
	} else {
		s.servers = servers
	}
	s.commitLog = make([]*CommitArgs, 0)
	s.numNewCommits = 0
	s.snapshotCount = 0
	s.lastLayout = make([]uint64, config.NumBuckets*config.BucketDepth)
	s.intVec = buildInterestVector(config.WindowSize(), config.BloomFalsePositive, s.commitLog[:]).Bytes()
	s.cuckooData = make([]byte, config.NumBuckets*config.BucketDepth*uint64(common.IDSize))

	// Choose a random seed for the cuckoo table
	seedBytes := make([]byte, 8)
	_, err := rand.Read(seedBytes)
	if err != nil {
		s.log.Error.Printf("coordinator.NewServer(%v) error: %v", name, err)
		return nil, err
	}
	seed, _ := binary.Varint(seedBytes)
	s.cuckooTable = cuckoo.NewTable(name, config.NumBuckets, config.BucketDepth, uint64(common.IDSize), s.cuckooData, seed)
	// Should not be possible
	if s.cuckooTable == nil {
		err := fmt.Errorf("Invalid cuckoo table parameters")
		s.log.Error.Printf("coordinator.NewServer(%v) error: %v", name, err)
		return nil, err
	}
	s.notifyChan = make(chan bool)
	s.closeChan = make(chan bool)

	go s.loop()

	s.log.Info.Printf("coordinator.NewServer(%v) success\n", name)
	return s, nil
}

/**********************************
 * PUBLIC RPC METHODS (threadsafe)
 **********************************/

// GetInfo returns information about this server
func (s *Server) GetInfo(args *interface{}, reply *GetInfoReply) error {
	tr := trace.New("Coordinator", "GetInfo")
	defer tr.Finish()
	s.lock.Lock()

	reply.Err = ""
	reply.Name = s.name
	reply.SnapshotID = s.snapshotCount

	s.lock.Unlock()
	return nil
}

// GetCommonConfig returns the common global config
func (s *Server) GetCommonConfig(args *interface{}, reply *common.Config) error {
	tr := trace.New("Coordinator", "GetCommonConfig")
	defer tr.Finish()
	s.lock.Lock()

	*reply = s.config

	s.lock.Unlock()
	return nil
}

// GetLayout returns the layout for a shard
func (s *Server) GetLayout(args *GetLayoutArgs, reply *GetLayoutReply) error {
	tr := trace.New("Coordinator", "GetLayout")
	defer tr.Finish()
	s.lock.Lock()

	// Check for correct snapshot ID
	reply.SnapshotID = s.snapshotCount
	if args.SnapshotID != s.snapshotCount {
		reply.Err = "Invalid SnapshotID"
		s.lock.Unlock()
		return nil
	}

	// Check non-zero NumShards
	if args.NumShards < 1 {
		reply.Err = "NumShards must be > 0"
		s.lock.Unlock()
		return nil
	}

	shardSize := uint64(len(s.lastLayout)) / args.NumShards
	idx := args.ShardID * shardSize
	if idx < 0 || (idx+shardSize) > uint64(len(s.lastLayout)) {
		reply.Err = "Out of bounds ShardID"
	} else {
		reply.Err = ""
		reply.Layout = s.lastLayout[idx:(idx + shardSize)]
	}

	s.lock.Unlock()
	return nil
}

// GetIntVec returns the global interest vector
func (s *Server) GetIntVec(args *GetIntVecArgs, reply *GetIntVecReply) error {
	tr := trace.New("Coordinator", "GetIntVec")
	defer tr.Finish()
	s.lock.Lock()

	// Check for correct snapshot ID
	reply.SnapshotID = s.snapshotCount
	if args.SnapshotID != s.snapshotCount {
		reply.Err = "Invalid SnapshotID"
		s.lock.Unlock()
		return nil
	}

	reply.Err = ""
	reply.IntVec = s.intVec[:]

	s.lock.Unlock()
	return nil
}

// Commit accepts a single Write to commit. The
func (s *Server) Commit(args *CommitArgs, reply *CommitReply) error {
	tr := trace.New("Coordinator", "Commit")
	defer tr.Finish()
	reply.Err = ""

	s.lock.Lock()

	windowSize := s.config.WindowSize()
	// Garbage Collect old elements
	for uint64(len(s.commitLog)) >= windowSize {
		_ = s.cuckooTable.Remove(asCuckooItem(s.config.NumBuckets, s.commitLog[0]))
		s.commitLog = s.commitLog[1:]
	}

	// Insert new item
	s.numNewCommits++
	s.commitLog = append(s.commitLog, args)
	ok, _ := s.cuckooTable.Insert(asCuckooItem(s.config.NumBuckets, args))
	if !ok {
		s.log.Error.Fatalf("%v.processCommit failed to insert new element", s.name)
		return fmt.Errorf("Error inserting into cuckoo table")
	}

	s.lock.Unlock()

	// Do notifications in loop()
	s.notifyChan <- false
	return nil
}

/**********************************
 * PUBLIC LOCAL METHODS (threadsafe)
 **********************************/

// Close shuts down the server
func (s *Server) Close() {
	s.log.Info.Printf("%v.Close: success", s.name)
	s.closeChan <- true
}

// AddServer adds a server to the list that is notified on snapshot changes
func (s *Server) AddServer(server NotifyInterface) {
	s.lock.Lock()
	s.servers = append(s.servers, server)
	s.log.Info.Printf("%v.AddServer() success\n", s.name)
	s.lock.Unlock()
}

// NotifySnapshot notifies the current cuckoo layout out
// If `force` is false, ignore when under a threshold
// Returns: true if snapshot was built, false if ignored
func (s *Server) NotifySnapshot(force bool) bool {
	s.lock.Lock()

	// Ignore if under threshold and not forcing
	if !force {
		if s.numNewCommits < s.snapshotThreshold {
			s.lock.Unlock()
			return false
		}
	}

	// Reset state
	s.numNewCommits = 0
	s.snapshotCount++

	// Construct global interest vector
	s.intVec = buildInterestVector(s.config.WindowSize(), s.config.BloomFalsePositive, s.commitLog[:]).Bytes()

	// Copy the layout
	for i := 0; i < len(s.lastLayout); i++ {
		idx := i * 8
		s.lastLayout[i], _ = binary.Uvarint(s.cuckooData[idx:(idx + 8)])
	}

	// Sync with buildGlobalInterestVector goroutine
	// @todo when this happens in parallel

	// Send notifications only if there are servers
	if s.servers != nil && len(s.servers) > 0 {
		go sendNotification(s.log, s.servers[:], s.snapshotCount)
	}

	s.log.Info.Printf("%v.NotifySnapshot() success\n", s.name)
	s.lock.Unlock()
	return true
}

/**********************************
 * PRIVATE METHODS (single-threaded)
 **********************************/
// Periodically call NotifySnapshot
func (s *Server) loop() {
	tick := time.After(s.snapshotInterval)
	for {
		select {
		// Called after Commit
		case <-s.notifyChan:
			ok := s.NotifySnapshot(false)
			if ok { // Re-up timer if snapshot built
				tick = time.After(s.snapshotInterval)
			}
			continue
		// Periodically trigger a snapshot
		case <-tick:
			s.NotifySnapshot(true)
			tick = time.After(s.snapshotInterval) // Re-up timer
			continue
		// Stop the loop
		case <-s.closeChan:
			return
		}
	}
}

/**********************************
 * HELPER FUNCTIONS
 **********************************/

// Converts a CommitArgs to a cuckoo.Item
func asCuckooItem(numBuckets uint64, args *CommitArgs) *cuckoo.Item {
	itemData := make([]byte, common.IDSize)
	binary.PutUvarint(itemData, args.ID)
	return &cuckoo.Item{
		ID:      args.ID,
		Data:    itemData,
		Bucket1: args.Bucket1 % numBuckets,
		Bucket2: args.Bucket2 % numBuckets,
	}
}

// buildInterestVector traverses the commitLog and creates a global interest vector
// representing the elements.
// Bloom filter stores n elements with fp false positive rate
func buildInterestVector(n uint64, fp float64, commitLog []*CommitArgs) *bloom.BitSet {
	bits, numHash := bloom.EstimateParameters(n, fp)
	intVec := bloom.NewBitSet(bits)
	for _, c := range commitLog {
		// Truncate interest vectors to first numHash bits
		if uint64(len(c.IntVecLoc)) > numHash {
			intVec = bloom.SetLocations(intVec, c.IntVecLoc[:numHash])
		} else {
			intVec = bloom.SetLocations(intVec, c.IntVecLoc[:])
		}
	}
	return intVec
}

func sendNotification(log *common.Logger, servers []NotifyInterface, snapshotID uint64) {
	args := &NotifyArgs{
		SnapshotID: snapshotID,
	}
	doNotify := func(l *common.Logger, s NotifyInterface, args *NotifyArgs) {
		err := s.Notify(args, &NotifyReply{})
		if err != nil {
			l.Error.Printf("sendNotification failed: %v", err)
		}
	}
	for _, s := range servers {
		go doNotify(log, s, args)
	}
}
<|endoftext|>


<commit_before>package levant

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"path"
	"text/template"

	"github.com/jrasell/levant/helper"
	"github.com/jrasell/levant/logging"
	yaml "gopkg.in/yaml.v2"

	nomad "github.com/hashicorp/nomad/api"
	"github.com/hashicorp/nomad/jobspec"
	"github.com/hashicorp/terraform/config"
)

const (
	terraformVarExtention = "tf"
	yamlVarExtension      = "yaml"
	ymlVarExtension       = "yml"
)

// RenderTemplate is the main entry point to render the template based on the
// passed variables file.
func RenderTemplate(templateFile, variableFile string, flagVars *map[string]string) (job *nomad.Job, err error) {

	var tpl *bytes.Buffer
	ext := path.Ext(variableFile)

	src, err := ioutil.ReadFile(templateFile)
	if err != nil {
		return
	}

	switch ext {
	case terraformVarExtention:
		logging.Debug("levant/templater: detected .tf variable file extension")
		tpl, err = renderTFTemplte(string(src), variableFile, flagVars)
	case yamlVarExtension, ymlVarExtension:
		logging.Debug("levant/templater: detected .yaml or .yml variable file extension")
		tpl, err = renderYAMLVarsTemplate(string(src), variableFile, flagVars)
	case "":
		if len(*flagVars) == 0 {
			logging.Debug("levant/template: no variables file or var flags, skipping templating")
			tpl = bytes.NewBuffer(src)
			break
		}

		logging.Debug("levant/templater: variable file not passed, using any passed CLI variables")
		tpl, err = readJobFile(string(src), flagVars)
	default:
		err = fmt.Errorf("variables file extension %v not supported", ext)
	}

	if err != nil {
		return
	}

	job, err = jobspec.Parse(tpl)
	return
}

func renderTFTemplte(src, variableFile string, flagVars *map[string]string) (tpl *bytes.Buffer, err error) {

	c := &config.Config{}

	// Setup our variables map and template
	variables := make(map[string]interface{})
	tpl = &bytes.Buffer{}

	// Load the variables file
	if c, err = config.LoadFile(variableFile); err != nil {
		return
	}

	// Populate our map with the parsed variables
	for _, variable := range c.Variables {
		variables[variable.Name] = variable.Default
	}

	// Merge variables passed on the CLI with those passed through a variables
	// file.
	mergedVars := helper.VariableMerge(&variables, flagVars)

	// Setup the template file for rendering
	t := template.New("jobTemplate")

	if t, err = t.Parse(src); err != nil {
		return
	}

	if err = t.Execute(tpl, mergedVars); err != nil {
		return
	}
	return tpl, nil
}

func renderYAMLVarsTemplate(src, variableFile string, flagVars *map[string]string) (tpl *bytes.Buffer, err error) {

	// Setup our variables map and template
	variables := make(map[string]interface{})
	tpl = &bytes.Buffer{}

	yamlFile, err := ioutil.ReadFile(variableFile)
	if err != nil {
		return
	}

	if err = yaml.Unmarshal(yamlFile, &variables); err != nil {
		return
	}

	// Merge variables passed on the CLI with those passed through a variables
	// file.
	mergedVars := helper.VariableMerge(&variables, flagVars)

	// Setup the template file for rendering
	t := template.New("jobTemplate")
	tempF, _ := ioutil.ReadFile(src)
	if t, err = t.Parse(string(tempF)); err != nil {
		return
	}

	if err = t.Execute(tpl, mergedVars); err != nil {
		return
	}

	return tpl, nil
}

func readJobFile(src string, flagVars *map[string]string) (tpl *bytes.Buffer, err error) {

	tpl = &bytes.Buffer{}

	// Setup the template file for rendering
	t := template.New("jobTemplate")
	if t, err = t.Parse(src); err != nil {
		return
	}

	if err = t.Execute(tpl, flagVars); err != nil {
		return
	}

	return tpl, nil
}
<commit_msg>Fix issue with file extension parsing and yaml file rendering.<commit_after>package levant

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"path"
	"text/template"

	"github.com/jrasell/levant/helper"
	"github.com/jrasell/levant/logging"
	yaml "gopkg.in/yaml.v2"

	nomad "github.com/hashicorp/nomad/api"
	"github.com/hashicorp/nomad/jobspec"
	"github.com/hashicorp/terraform/config"
)

const (
	terraformVarExtention = ".tf"
	yamlVarExtension      = ".yaml"
	ymlVarExtension       = ".yml"
)

// RenderTemplate is the main entry point to render the template based on the
// passed variables file.
func RenderTemplate(templateFile, variableFile string, flagVars *map[string]string) (job *nomad.Job, err error) {

	var tpl *bytes.Buffer
	ext := path.Ext(variableFile)

	src, err := ioutil.ReadFile(templateFile)
	if err != nil {
		return
	}

	switch ext {
	case terraformVarExtention:
		logging.Debug("levant/templater: detected .tf variable file extension")
		tpl, err = renderTFTemplte(string(src), variableFile, flagVars)
	case yamlVarExtension, ymlVarExtension:
		logging.Debug("levant/templater: detected .yaml or .yml variable file extension")
		tpl, err = renderYAMLVarsTemplate(string(src), variableFile, flagVars)
	case "":
		if len(*flagVars) == 0 {
			logging.Debug("levant/template: no variables file or var flags, skipping templating")
			tpl = bytes.NewBuffer(src)
			break
		}

		logging.Debug("levant/templater: variable file not passed, using any passed CLI variables")
		tpl, err = readJobFile(string(src), flagVars)
	default:
		err = fmt.Errorf("variables file extension %v not supported", ext)
	}

	if err != nil {
		return
	}

	job, err = jobspec.Parse(tpl)
	return
}

func renderTFTemplte(src, variableFile string, flagVars *map[string]string) (tpl *bytes.Buffer, err error) {

	c := &config.Config{}

	// Setup our variables map and template
	variables := make(map[string]interface{})
	tpl = &bytes.Buffer{}

	// Load the variables file
	if c, err = config.LoadFile(variableFile); err != nil {
		return
	}

	// Populate our map with the parsed variables
	for _, variable := range c.Variables {
		variables[variable.Name] = variable.Default
	}

	// Merge variables passed on the CLI with those passed through a variables
	// file.
	mergedVars := helper.VariableMerge(&variables, flagVars)

	// Setup the template file for rendering
	t := template.New("jobTemplate")

	if t, err = t.Parse(src); err != nil {
		return
	}

	if err = t.Execute(tpl, mergedVars); err != nil {
		return
	}
	return tpl, nil
}

func renderYAMLVarsTemplate(src, variableFile string, flagVars *map[string]string) (tpl *bytes.Buffer, err error) {

	// Setup our variables map and template
	variables := make(map[string]interface{})
	tpl = &bytes.Buffer{}

	yamlFile, err := ioutil.ReadFile(variableFile)
	if err != nil {
		return
	}

	if err = yaml.Unmarshal(yamlFile, &variables); err != nil {
		return
	}

	// Merge variables passed on the CLI with those passed through a variables
	// file.
	mergedVars := helper.VariableMerge(&variables, flagVars)

	// Setup the template file for rendering
	t := template.New("jobTemplate")
	if t, err = t.Parse(string(src)); err != nil {
		return
	}

	if err = t.Execute(tpl, mergedVars); err != nil {
		return
	}

	return tpl, nil
}

func readJobFile(src string, flagVars *map[string]string) (tpl *bytes.Buffer, err error) {

	tpl = &bytes.Buffer{}

	// Setup the template file for rendering
	t := template.New("jobTemplate")
	if t, err = t.Parse(src); err != nil {
		return
	}

	if err = t.Execute(tpl, flagVars); err != nil {
		return
	}

	return tpl, nil
}
<|endoftext|>


<commit_before><commit_msg>blockmanager: extract common filter reconcilation into detectBadPeers<commit_after><|endoftext|>


<commit_before>package etf

func exclusiveTime(n int, logs []string) []int {
	return nil
}
<commit_msg>solve 636 using stack<commit_after>package etf

import (
	"strconv"
	"strings"
)

func exclusiveTime(n int, logs []string) []int {
	return useStack(n, logs)
}

// useStack time complexity O(M) M is the length of logs, space complexity is O(M)
func useStack(n int, logs []string) []int {
	m := len(logs)
	res := make([]int, n)
	st := make([]int, 0, 2000)
	id, _, prev := helper(logs[0])
	st = append(st, id)
	for i := 1; i < m; i++ {
		cid, status, cts := helper(logs[i])
		ns := len(st)
		if status == "start" {
			if ns > 0 {
				// update the top of the stack's time
				res[st[ns-1]] += cts - prev
			}
			st = append(st, cid)
			prev = cts
		} else {
			res[st[ns-1]] += cts - prev + 1
			st = st[:ns-1]
			prev = cts + 1
		}
	}
	return res
}

// helper parse the log in `id:status:ts` format, and returns the fields
func helper(log string) (id int, status string, ts int) {
	arr := strings.FieldsFunc(log, func(r rune) bool {
		return r == ':'
	})
	id, _ = strconv.Atoi(arr[0])
	ts, _ = strconv.Atoi(arr[2])
	status = arr[1]
	return
}
<|endoftext|>


<commit_before><commit_msg>gxz: improved file name handling<commit_after><|endoftext|>


<commit_before><commit_msg>add serialise and unserialise function to Entry<commit_after><|endoftext|>


<commit_before><commit_msg>Refactor cleanup<commit_after><|endoftext|>


<commit_before><commit_msg>Added error checks<commit_after><|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package builder

import (
	"os"
	"path/filepath"
	"regexp"

	"mynewt.apache.org/newt/newt/newtutil"
	"mynewt.apache.org/newt/newt/pkg"
	"mynewt.apache.org/newt/newt/resolve"
	"mynewt.apache.org/newt/newt/syscfg"
	"mynewt.apache.org/newt/newt/toolchain"
	"mynewt.apache.org/newt/util"
)

type BuildPackage struct {
	rpkg              *resolve.ResolvePackage
	SourceDirectories []string
	ci                *toolchain.CompilerInfo
}

func NewBuildPackage(rpkg *resolve.ResolvePackage) *BuildPackage {
	bpkg := &BuildPackage{
		rpkg: rpkg,
	}

	return bpkg
}

// Recursively iterates through an pkg's dependencies, adding each pkg
// encountered to the supplied set.
func (bpkg *BuildPackage) collectDepsAux(b *Builder,
	set *map[*BuildPackage]bool) error {

	if (*set)[bpkg] {
		return nil
	}

	(*set)[bpkg] = true

	for _, dep := range bpkg.rpkg.Deps {
		dbpkg := b.PkgMap[dep.Rpkg]
		if dbpkg == nil {
			return util.FmtNewtError("Package not found %s; required by %s",
				dbpkg.rpkg.Lpkg.Name(), bpkg.rpkg.Lpkg.Name())
		}

		if err := dbpkg.collectDepsAux(b, set); err != nil {
			return err
		}
	}

	return nil
}

// Recursively iterates through an pkg's dependencies.  The resulting array
// contains a pointer to each encountered pkg.
func (bpkg *BuildPackage) collectDeps(b *Builder) ([]*BuildPackage, error) {
	set := map[*BuildPackage]bool{}

	err := bpkg.collectDepsAux(b, &set)
	if err != nil {
		return nil, err
	}

	arr := []*BuildPackage{}
	for p, _ := range set {
		arr = append(arr, p)
	}

	return arr, nil
}

// Calculates the include paths exported by the specified pkg and all of
// its recursive dependencies.
func (bpkg *BuildPackage) recursiveIncludePaths(
	b *Builder) ([]string, error) {

	deps, err := bpkg.collectDeps(b)
	if err != nil {
		return nil, err
	}

	incls := []string{}
	for _, p := range deps {
		incls = append(incls, p.publicIncludeDirs(b.targetBuilder.bspPkg)...)
	}

	return incls, nil
}

// Replaces instances of "@<repo-name>" with repo paths.
func expandFlags(flags []string) {
	for i, f := range flags {
		newFlag, changed := newtutil.ReplaceRepoDesignators(f)
		if changed {
			flags[i] = newFlag
		}
	}
}

func (bpkg *BuildPackage) CompilerInfo(
	b *Builder) (*toolchain.CompilerInfo, error) {

	// If this package's compiler info has already been generated, returned the
	// cached copy.
	if bpkg.ci != nil {
		return bpkg.ci, nil
	}

	ci := toolchain.NewCompilerInfo()
	settings := b.cfg.SettingsForLpkg(bpkg.rpkg.Lpkg)

	// Read each set of flags and expand repo designators ("@<repo-nme>") into
	// paths.
	ci.Cflags = bpkg.rpkg.Lpkg.PkgY.GetValStringSlice("pkg.cflags", settings)
	expandFlags(ci.Cflags)

	ci.Lflags = bpkg.rpkg.Lpkg.PkgY.GetValStringSlice("pkg.lflags", settings)
	expandFlags(ci.Lflags)

	ci.Aflags = bpkg.rpkg.Lpkg.PkgY.GetValStringSlice("pkg.aflags", settings)
	expandFlags(ci.Aflags)

	// Package-specific injected settings get specified as C flags on the
	// command line.
	for k, _ := range bpkg.rpkg.Lpkg.InjectedSettings() {
		ci.Cflags = append(ci.Cflags, syscfg.FeatureToCflag(k))
	}

	ci.IgnoreFiles = []*regexp.Regexp{}
	ignPats := bpkg.rpkg.Lpkg.PkgY.GetValStringSlice("pkg.ign_files", settings)
	for _, str := range ignPats {
		re, err := regexp.Compile(str)
		if err != nil {
			return nil, util.NewNewtError(
				"Ignore files, unable to compile re: " + err.Error())
		}
		ci.IgnoreFiles = append(ci.IgnoreFiles, re)
	}

	ci.IgnoreDirs = []*regexp.Regexp{}
	ignPats = bpkg.rpkg.Lpkg.PkgY.GetValStringSlice("pkg.ign_dirs", settings)
	for _, str := range ignPats {
		re, err := regexp.Compile(str)
		if err != nil {
			return nil, util.NewNewtError(
				"Ignore dirs, unable to compile re: " + err.Error())
		}
		ci.IgnoreDirs = append(ci.IgnoreDirs, re)
	}

	bpkg.SourceDirectories = bpkg.rpkg.Lpkg.PkgY.GetValStringSlice(
		"pkg.src_dirs", settings)

	includePaths, err := bpkg.recursiveIncludePaths(b)
	if err != nil {
		return nil, err
	}

	ci.Includes = append(bpkg.privateIncludeDirs(b), includePaths...)
	bpkg.ci = ci

	return bpkg.ci, nil
}

func (bpkg *BuildPackage) findSdkIncludes() []string {
	sdkDir := bpkg.rpkg.Lpkg.BasePath() + "/src/ext/"

	sdkPathList := []string{}
	err := filepath.Walk(sdkDir,
		func(path string, info os.FileInfo, err error) error {
			if !info.IsDir() {
				return nil
			}

			sdkPathList = append(sdkPathList, path)
			return nil
		})
	if err != nil {
		return []string{}
	}

	return sdkPathList
}

func (bpkg *BuildPackage) publicIncludeDirs(bspPkg *pkg.BspPackage) []string {
	pkgBase := filepath.Base(bpkg.rpkg.Lpkg.Name())
	bp := bpkg.rpkg.Lpkg.BasePath()

	incls := []string{
		bp + "/include",
		bp + "/include/" + pkgBase + "/arch/" + bspPkg.Arch,
	}

	if bpkg.rpkg.Lpkg.Type() == pkg.PACKAGE_TYPE_SDK {
		incls = append(incls, bspPkg.BasePath()+"/include/bsp/")

		sdkIncls := bpkg.findSdkIncludes()
		incls = append(incls, sdkIncls...)
	}

	return incls
}

func (bpkg *BuildPackage) privateIncludeDirs(b *Builder) []string {
	srcDir := bpkg.rpkg.Lpkg.BasePath() + "/src/"

	incls := []string{}
	incls = append(incls, srcDir)
	incls = append(incls, srcDir+"/arch/"+b.targetBuilder.bspPkg.Arch)

	switch bpkg.rpkg.Lpkg.Type() {
	case pkg.PACKAGE_TYPE_SDK:
		// If pkgType == SDK, include all the items in "ext" directly into the
		// include path
		incls = append(incls, b.bspPkg.rpkg.Lpkg.BasePath()+"/include/bsp/")

		sdkIncls := bpkg.findSdkIncludes()
		incls = append(incls, sdkIncls...)

	case pkg.PACKAGE_TYPE_UNITTEST:
		// A unittest package gets access to its parent package's private
		// includes.
		parentPkg := b.testOwner(bpkg)
		if parentPkg != nil {
			parentIncls := parentPkg.privateIncludeDirs(b)
			incls = append(incls, parentIncls...)
		}

	default:
	}

	return incls
}
<commit_msg>newt - Fix null pointer dereference.<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package builder

import (
	"os"
	"path/filepath"
	"regexp"

	"mynewt.apache.org/newt/newt/newtutil"
	"mynewt.apache.org/newt/newt/pkg"
	"mynewt.apache.org/newt/newt/resolve"
	"mynewt.apache.org/newt/newt/syscfg"
	"mynewt.apache.org/newt/newt/toolchain"
	"mynewt.apache.org/newt/util"
)

type BuildPackage struct {
	rpkg              *resolve.ResolvePackage
	SourceDirectories []string
	ci                *toolchain.CompilerInfo
}

func NewBuildPackage(rpkg *resolve.ResolvePackage) *BuildPackage {
	bpkg := &BuildPackage{
		rpkg: rpkg,
	}

	return bpkg
}

// Recursively iterates through an pkg's dependencies, adding each pkg
// encountered to the supplied set.
func (bpkg *BuildPackage) collectDepsAux(b *Builder,
	set *map[*BuildPackage]bool) error {

	if (*set)[bpkg] {
		return nil
	}

	(*set)[bpkg] = true

	for _, dep := range bpkg.rpkg.Deps {
		dbpkg := b.PkgMap[dep.Rpkg]
		if dbpkg == nil {
			return util.FmtNewtError("Package not found %s; required by %s",
				dep.Rpkg.Lpkg.Name(), bpkg.rpkg.Lpkg.Name())
		}

		if err := dbpkg.collectDepsAux(b, set); err != nil {
			return err
		}
	}

	return nil
}

// Recursively iterates through an pkg's dependencies.  The resulting array
// contains a pointer to each encountered pkg.
func (bpkg *BuildPackage) collectDeps(b *Builder) ([]*BuildPackage, error) {
	set := map[*BuildPackage]bool{}

	err := bpkg.collectDepsAux(b, &set)
	if err != nil {
		return nil, err
	}

	arr := []*BuildPackage{}
	for p, _ := range set {
		arr = append(arr, p)
	}

	return arr, nil
}

// Calculates the include paths exported by the specified pkg and all of
// its recursive dependencies.
func (bpkg *BuildPackage) recursiveIncludePaths(
	b *Builder) ([]string, error) {

	deps, err := bpkg.collectDeps(b)
	if err != nil {
		return nil, err
	}

	incls := []string{}
	for _, p := range deps {
		incls = append(incls, p.publicIncludeDirs(b.targetBuilder.bspPkg)...)
	}

	return incls, nil
}

// Replaces instances of "@<repo-name>" with repo paths.
func expandFlags(flags []string) {
	for i, f := range flags {
		newFlag, changed := newtutil.ReplaceRepoDesignators(f)
		if changed {
			flags[i] = newFlag
		}
	}
}

func (bpkg *BuildPackage) CompilerInfo(
	b *Builder) (*toolchain.CompilerInfo, error) {

	// If this package's compiler info has already been generated, returned the
	// cached copy.
	if bpkg.ci != nil {
		return bpkg.ci, nil
	}

	ci := toolchain.NewCompilerInfo()
	settings := b.cfg.SettingsForLpkg(bpkg.rpkg.Lpkg)

	// Read each set of flags and expand repo designators ("@<repo-nme>") into
	// paths.
	ci.Cflags = bpkg.rpkg.Lpkg.PkgY.GetValStringSlice("pkg.cflags", settings)
	expandFlags(ci.Cflags)

	ci.Lflags = bpkg.rpkg.Lpkg.PkgY.GetValStringSlice("pkg.lflags", settings)
	expandFlags(ci.Lflags)

	ci.Aflags = bpkg.rpkg.Lpkg.PkgY.GetValStringSlice("pkg.aflags", settings)
	expandFlags(ci.Aflags)

	// Package-specific injected settings get specified as C flags on the
	// command line.
	for k, _ := range bpkg.rpkg.Lpkg.InjectedSettings() {
		ci.Cflags = append(ci.Cflags, syscfg.FeatureToCflag(k))
	}

	ci.IgnoreFiles = []*regexp.Regexp{}
	ignPats := bpkg.rpkg.Lpkg.PkgY.GetValStringSlice("pkg.ign_files", settings)
	for _, str := range ignPats {
		re, err := regexp.Compile(str)
		if err != nil {
			return nil, util.NewNewtError(
				"Ignore files, unable to compile re: " + err.Error())
		}
		ci.IgnoreFiles = append(ci.IgnoreFiles, re)
	}

	ci.IgnoreDirs = []*regexp.Regexp{}
	ignPats = bpkg.rpkg.Lpkg.PkgY.GetValStringSlice("pkg.ign_dirs", settings)
	for _, str := range ignPats {
		re, err := regexp.Compile(str)
		if err != nil {
			return nil, util.NewNewtError(
				"Ignore dirs, unable to compile re: " + err.Error())
		}
		ci.IgnoreDirs = append(ci.IgnoreDirs, re)
	}

	bpkg.SourceDirectories = bpkg.rpkg.Lpkg.PkgY.GetValStringSlice(
		"pkg.src_dirs", settings)

	includePaths, err := bpkg.recursiveIncludePaths(b)
	if err != nil {
		return nil, err
	}

	ci.Includes = append(bpkg.privateIncludeDirs(b), includePaths...)
	bpkg.ci = ci

	return bpkg.ci, nil
}

func (bpkg *BuildPackage) findSdkIncludes() []string {
	sdkDir := bpkg.rpkg.Lpkg.BasePath() + "/src/ext/"

	sdkPathList := []string{}
	err := filepath.Walk(sdkDir,
		func(path string, info os.FileInfo, err error) error {
			if !info.IsDir() {
				return nil
			}

			sdkPathList = append(sdkPathList, path)
			return nil
		})
	if err != nil {
		return []string{}
	}

	return sdkPathList
}

func (bpkg *BuildPackage) publicIncludeDirs(bspPkg *pkg.BspPackage) []string {
	pkgBase := filepath.Base(bpkg.rpkg.Lpkg.Name())
	bp := bpkg.rpkg.Lpkg.BasePath()

	incls := []string{
		bp + "/include",
		bp + "/include/" + pkgBase + "/arch/" + bspPkg.Arch,
	}

	if bpkg.rpkg.Lpkg.Type() == pkg.PACKAGE_TYPE_SDK {
		incls = append(incls, bspPkg.BasePath()+"/include/bsp/")

		sdkIncls := bpkg.findSdkIncludes()
		incls = append(incls, sdkIncls...)
	}

	return incls
}

func (bpkg *BuildPackage) privateIncludeDirs(b *Builder) []string {
	srcDir := bpkg.rpkg.Lpkg.BasePath() + "/src/"

	incls := []string{}
	incls = append(incls, srcDir)
	incls = append(incls, srcDir+"/arch/"+b.targetBuilder.bspPkg.Arch)

	switch bpkg.rpkg.Lpkg.Type() {
	case pkg.PACKAGE_TYPE_SDK:
		// If pkgType == SDK, include all the items in "ext" directly into the
		// include path
		incls = append(incls, b.bspPkg.rpkg.Lpkg.BasePath()+"/include/bsp/")

		sdkIncls := bpkg.findSdkIncludes()
		incls = append(incls, sdkIncls...)

	case pkg.PACKAGE_TYPE_UNITTEST:
		// A unittest package gets access to its parent package's private
		// includes.
		parentPkg := b.testOwner(bpkg)
		if parentPkg != nil {
			parentIncls := parentPkg.privateIncludeDirs(b)
			incls = append(incls, parentIncls...)
		}

	default:
	}

	return incls
}
<|endoftext|>


<commit_before>package nlp_math

import "testing"

func TestMultiCountCreation(t *testing.T) {
	mc := MultiCount{}
	mc.Count()

	if mc.counts[0] != 0 {
		t.Log("Expected key[0] to have value 0")
		t.Fail()
	}
}

func TestMultiCountIncrement(t *testing.T) {
	mc := MultiCount{}
	mc.Count()

	for i := 0; i < 200; i++ {
		mc.Increment(i)
	}

	z := mc.counts[1]
	if z != 1 {
		t.Log("Expected key[0] to be 1 but got: ", z)
		t.Fail()
	}

	tf := mc.counts[25]
	if tf != 1 {
		t.Log("Expected key[25] to be 201, but got: ", tf)
		t.Fail()
	}

	numKeys := len(mc.counts)
	if numKeys != 200 {
		t.Log("Expected Number of Kyes to be 201, but got: ", numKeys)
		t.Fail()
	}
}
<commit_msg>add more to multinomial tests<commit_after>package nlp_math

import "testing"

func TestMaximumLikelihood(t *testing.T) {
	mn := Multinomial{}
	mc := MultiCount{}
	mc.Count()

	for i := 0; i < 200; i++ {
		mc.Increment(i)
	}
	mn.MaximumLikelihood(mc)

	sixProb := mn.Probability(6)
	if sixProb != 0.005 {
		t.Log("Expected probability to be 0.005 but got: ", sixProb)
		t.Fail()
	}

	for i := 0; i < 200; i++ {
		mn.distribution[12]++
	}
	twelveProb := mn.Probability(12)
	if twelveProb != 200.005 {
		t.Log("Expected probability to be 0.005 but got: ", twelveProb)
		t.Fail()
	}

	for i := 0; i < 800; i++ {
		mn.distribution[99]++
	}
	nineNineProb := mn.Probability(99)
	if nineNineProb != 800.005 {
		t.Log("Expected probability to be 0.005 but got: ", nineNineProb)
		t.Fail()
	}

}

func TestMultiCountIncrement(t *testing.T) {
	mc := MultiCount{}
	mc.Count()

	for i := 0; i < 200; i++ {
		mc.Increment(i)
	}

	z := mc.counts[1]
	if z != 1 {
		t.Log("Expected key[0] to be 1 but got: ", z)
		t.Fail()
	}

	tf := mc.counts[25]
	if tf != 1 {
		t.Log("Expected key[25] to be 201, but got: ", tf)
		t.Fail()
	}

	numKeys := len(mc.counts)
	if numKeys != 200 {
		t.Log("Expected Number of Kyes to be 201, but got: ", numKeys)
		t.Fail()
	}
}

func TestMultiNomial(t *testing.T) {
	mn := Multinomial{
		distribution: make(map[int]float64),
	}

	for i := 0; i < 25; i++ {
		mn.distribution[i]++
	}

	threeProb := mn.Probability(3)
	if threeProb != 1 {
		t.Log("Expected probability to be 1.0 but got: ", threeProb)
		t.Fail()
	}

	for i := 0; i < 200; i++ {
		mn.distribution[6]++
	}
	sixProb := mn.Probability(6)
	if sixProb != 201 {
		t.Log("Expected probability to be 0.0 but got: ", sixProb)
		t.Fail()
	}
}
<|endoftext|>


<commit_before>/**
 * (C) Copyright 2014, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"fmt"
	"strings"
	"reflect"
	"strconv"
	"net/http"
	"io/ioutil"
	"encoding/json"
	"labix.org/v2/mgo/bson"
	"github.com/gorilla/mux"
)

type HttpParamDataType int8
type HttpParamType int8

const(
	HttpIntParam = HttpParamDataType(0)
	HttpStringParam = HttpParamDataType(1)
	HttpFloatParam = HttpParamDataType(2)
	HttpBoolParam = HttpParamDataType(3) // Boolean types include: 1, t, T, TRUE, true, True, 0, f, F, FALSE, false
	HttpObjectIdParam = HttpParamDataType(4)

	// All of the param types only support single values (i.e., no slices). If multiple values are present, the
	// first is taken.
	HttpParamPost = HttpParamType(0)
	HttpParamJsonPost = HttpParamType(1) // When the content body is posted in json format. This only supports one level
	HttpParamQuery = HttpParamType(2)
	HttpParamHeader = HttpParamType(3)
	HttpParamPath = HttpParamType(4) // This must be declared as {someName} in the path mapping
)

type HttpContext struct {
	Response http.ResponseWriter
	Request *http.Request
	Params map[string]*HttpParam
	ErrorCodes []string
	Errors []error

	postJson map[string]interface{}

	body []byte
}

type HttpParam struct {
	Name string
	InvalidErrorCode string
	DataType HttpParamDataType
	Type HttpParamType
	Required bool
	MinLength int
	MaxLength int
	Post bool
	Value interface{}
	Raw string
	Valid bool
	Present bool // If value is present and parsed properly
}

// Make sure your params are present and valid before trying to access.
func (self *HttpParam) Int() int { return self.Value.(int) }

func (self *HttpParam) Float() float64 { return self.Value.(float64) }

func (self *HttpParam) String() string { return self.Value.(string) }

func (self *HttpParam) Bool() bool { return self.Value.(bool) }

func (self *HttpParam) ObjectId() *bson.ObjectId { return self.Value.(*bson.ObjectId) }

// Set a valid value for a param. Missing can be valid, but not present.
func (self *HttpParam) setPresentValue(value interface{}) {
	self.Present = true
	self.Value = value
}

// Validate the params. If any of the params are invalid, false is returned. You must call
// this first before calling the ErrorCodes []string. If not params are defined, this always
// returns "true". If there are raw data extraction errors, this is always false (e.g., body missing or incorrect).
func (self *HttpContext) ParamsAreValid() bool {

	if len(self.Errors) != 0 { return false }

	if len(self.Params) == 0 { return true }

	for _, param := range self.Params {
		switch param.DataType {
			case HttpIntParam: validateIntParam(self, param)
			case HttpStringParam: validateStringParam(self, param)
			case HttpFloatParam: validateFloatParam(self, param)
			case HttpBoolParam: validateBoolParam(self, param)
			case HttpObjectIdParam: validateObjectIdParam(self, param)
		}
	}

	return len(self.ErrorCodes) == 0
}

func (self *HttpContext) ParamInt(name string) int { return self.Params[name].Int() }

func (self *HttpContext) ParamFloat(name string) float64 { return self.Params[name].Float() }

func (self *HttpContext) ParamString(name string) string { return self.Params[name].String() }

func (self *HttpContext) HasParam(name string) bool {
	if val, found := self.Params[name]; !found || val == nil || val.Value == nil { return false }
	return true
}

func (self *HttpContext) ParamBool(name string) bool { return self.Params[name].Bool() }

func (self *HttpContext) ParamObjectId(name string) *bson.ObjectId { return self.Params[name].ObjectId() }

func (self *HttpContext) HasRawErrors() bool { return len(self.Errors) > 0 }

// This returns the param value as a string. If the param is missing or empty,
// the string will be len == 0.
func retrieveParamValue(ctx *HttpContext, param *HttpParam) string {
	switch param.Type {
		case HttpParamPost: return strings.TrimSpace(ctx.Request.PostFormValue(param.Name))
		case HttpParamJsonPost: return retrieveJsonParamValue(ctx, param)
		case HttpParamQuery: return strings.TrimSpace(ctx.Request.FormValue(param.Name))
		case HttpParamHeader: return strings.TrimSpace(ctx.Request.Header.Get(param.Name))
		case HttpParamPath: return strings.TrimSpace(mux.Vars(ctx.Request)[param.Name])
	}
	return nadaStr
}

func retrieveJsonParamValue(ctx *HttpContext, param *HttpParam) string {

	if len(ctx.Errors) > 0 {
		return nadaStr
	}

	// If this is the first access, read the body
	if len(ctx.body) == 0 {
		var err error
		ctx.body, err = ioutil.ReadAll(ctx.Request.Body)
		if err != nil {
			ctx.Errors = append(ctx.Errors, NewStackError("Error in raw data extraction - error: %v", err))
			return nadaStr
		}
	}

	if ctx.postJson == nil {
		var genJson interface{}
		err := json.Unmarshal(ctx.body, &genJson)
		if err != nil {
			ctx.Errors = append(ctx.Errors, NewStackError("Error in raw json data extraction - error: %v", err))
			return nadaStr
		}

		ctx.postJson = genJson.(map[string]interface{})
	}

	// Look for the value in the json. The json may hold the data in a variety
	// of formats. Convert back to a string to deal with the other data types :-(
	val, found := ctx.postJson[param.Name]
	if !found {
		return nadaStr
	}

	valType := reflect.TypeOf(val)

	if valType == nil { return nadaStr }

	switch valType.Kind() {
		case reflect.Invalid: return nadaStr
		case reflect.Bool: return fmt.Sprintf("%t", val.(bool))
		case reflect.Float64: return fmt.Sprintf("%g", val.(float64))
		case reflect.String: return val.(string)
		default: return nadaStr
	}

	return nadaStr
}

func appendInvalidErrorCode(ctx *HttpContext, param *HttpParam) {
	// Do not dupplicate error codes.
	for i := range ctx.ErrorCodes { if ctx.ErrorCodes[i] == param.InvalidErrorCode { return } }

	if len(param.InvalidErrorCode) == 0 {
		panic(fmt.Sprintf("We do not have an error code defined for param: %s - required: %t", param.Name, param.Required))
	}

	ctx.ErrorCodes = append(ctx.ErrorCodes, param.InvalidErrorCode)
	param.Valid = false
}

func validateIntParam(ctx *HttpContext, param *HttpParam) {

	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required { appendInvalidErrorCode(ctx, param); return }

	if len(param.Raw) == 0 { return }

	if val, err := strconv.Atoi(param.Raw); err != nil { appendInvalidErrorCode(ctx, param)
	} else { param.setPresentValue(val) }
}

func validateStringParam(ctx *HttpContext, param *HttpParam) {

	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if param.Required && param.MinLength > 0 && len(param.Raw) < param.MinLength {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if param.Required && param.MaxLength > 0 && len(param.Raw) > param.MaxLength {
		appendInvalidErrorCode(ctx, param)
		return
	}

	param.setPresentValue(param.Raw)
}

func validateFloatParam(ctx *HttpContext, param *HttpParam) {
	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if len(param.Raw) == 0 {
		return
	}

	if val, err := strconv.ParseFloat(param.Raw, 64); err != nil {
		appendInvalidErrorCode(ctx, param)
	} else {
		param.setPresentValue(val)
	}
}

func validateObjectIdParam(ctx *HttpContext, param *HttpParam) {

	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required { appendInvalidErrorCode(ctx, param); return }

	if len(param.Raw) == 0 { return }

	if !bson.IsObjectIdHex(param.Raw) { appendInvalidErrorCode(ctx, param); return }

	value := bson.ObjectIdHex(param.Raw)
	param.setPresentValue(&value)
}

// Boolean types include: 1, t, T, TRUE, true, True, 0, f, F, FALSE, false
func validateBoolParam(ctx *HttpContext, param *HttpParam) {

	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if len(param.Raw) == 0 { return }

	if val, err := strconv.ParseBool(param.Raw); err != nil {
		appendInvalidErrorCode(ctx, param)
	} else {
		param.setPresentValue(val)
	}
}

func (self *HttpContext) DefineIntParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpIntParam, Required: required, Type: paramType, Valid: true }
}

// Boolean types include: 1, t, T, TRUE, true, True, 0, f, F, FALSE, false
func (self *HttpContext) DefineBoolParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpBoolParam, Required: required, Type: paramType, Valid: true }
}

func (self *HttpContext) DefineFloatParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpFloatParam, Required: required, Type: paramType, Valid: true }
}

func (self *HttpContext) DefineObjectIdParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpObjectIdParam, Required: required, Type: paramType, Valid: true }
}

func (self *HttpContext) DefineStringParam(name, invalidErrorCode string, paramType HttpParamType, required bool, minLength, maxLength int) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpStringParam, Required: required, Type: paramType, Valid: true }
}

// Call this method to init the http context struct.
func NewHttpContext(response http.ResponseWriter, request *http.Request) *HttpContext {
	return &HttpContext{ Response: response, Request: request, Params: make(map[string]*HttpParam) }
}

// This method returns true if the http request method is a HTTP post. If the
// field missing or incorrect, false is returned. This method will panic if
// the request is nil.
func IsHttpMethodPost(request *http.Request) bool {
	if request == nil {
		panic("request param is nil")
	}
	return len(request.Method) > 0 && strings.ToUpper(request.Method) == HttpPostMethod
}

// Write an http ok response string. The content type is text/plain.
func WriteOkResponseString(response http.ResponseWriter, msg string) error {
	if response == nil {
		return NewStackError("response param is nil")
	}

	msgLength := len(msg)

	if msgLength == 0 {
		return NewStackError("Response message is an empty string")
	}

	response.Header().Set(ContentTypeHeader, ContentTypeTextPlain)

	written, err := response.Write([]byte(msg))

	if err != nil {
		return err
	}

	if written != msgLength {
		return NewStackError("Did not write full message - bytes written %d - expected %d", written, msgLength)
	}

	return nil
}

// Encode and write a json response. If there is a problem encoding an http 500 is sent and an
// error is returned. If there are problems writting the response an error is returned.
func JsonEncodeAndWriteResponse(response http.ResponseWriter, value interface{}) error {

	if value == nil {
		return NewStackError("Nil value passed")
	}

	rawJson, err := json.Marshal(value)
	if err != nil {
		http.Error(response, "Error", 500)
		return NewStackError("Unable to marshal json: %v", err)
	}

	response.Header().Set(ContentTypeHeader, ContentTypeJson)

	written, err := response.Write(rawJson)
	if err != nil {
		return NewStackError("Unable to write response: %v", err)
	}

	if written != len(rawJson) {
		return NewStackError("Unable to write full response - wrote: %d - expected: %d", written, len(rawJson))
	}

	return nil
}

<commit_msg>fixed nil ObjectId bug<commit_after>/**
 * (C) Copyright 2014, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"fmt"
	"strings"
	"reflect"
	"strconv"
	"net/http"
	"io/ioutil"
	"encoding/json"
	"labix.org/v2/mgo/bson"
	"github.com/gorilla/mux"
)

type HttpParamDataType int8
type HttpParamType int8

const(
	HttpIntParam = HttpParamDataType(0)
	HttpStringParam = HttpParamDataType(1)
	HttpFloatParam = HttpParamDataType(2)
	HttpBoolParam = HttpParamDataType(3) // Boolean types include: 1, t, T, TRUE, true, True, 0, f, F, FALSE, false
	HttpObjectIdParam = HttpParamDataType(4)

	// All of the param types only support single values (i.e., no slices). If multiple values are present, the
	// first is taken.
	HttpParamPost = HttpParamType(0)
	HttpParamJsonPost = HttpParamType(1) // When the content body is posted in json format. This only supports one level
	HttpParamQuery = HttpParamType(2)
	HttpParamHeader = HttpParamType(3)
	HttpParamPath = HttpParamType(4) // This must be declared as {someName} in the path mapping
)

type HttpContext struct {
	Response http.ResponseWriter
	Request *http.Request
	Params map[string]*HttpParam
	ErrorCodes []string
	Errors []error

	postJson map[string]interface{}

	body []byte
}

type HttpParam struct {
	Name string
	InvalidErrorCode string
	DataType HttpParamDataType
	Type HttpParamType
	Required bool
	MinLength int
	MaxLength int
	Post bool
	Value interface{}
	Raw string
	Valid bool
	Present bool // If value is present and parsed properly
}

// Make sure your params are present and valid before trying to access.
func (self *HttpParam) Int() int { return self.Value.(int) }

func (self *HttpParam) Float() float64 { return self.Value.(float64) }

func (self *HttpParam) String() string { return self.Value.(string) }

func (self *HttpParam) Bool() bool { return self.Value.(bool) }

func (self *HttpParam) ObjectId() *bson.ObjectId { if self.Value != nil { return self.Value.(*bson.ObjectId) } else { return nil } }

// Set a valid value for a param. Missing can be valid, but not present.
func (self *HttpParam) setPresentValue(value interface{}) {
	self.Present = true
	self.Value = value
}

// Validate the params. If any of the params are invalid, false is returned. You must call
// this first before calling the ErrorCodes []string. If not params are defined, this always
// returns "true". If there are raw data extraction errors, this is always false (e.g., body missing or incorrect).
func (self *HttpContext) ParamsAreValid() bool {

	if len(self.Errors) != 0 { return false }

	if len(self.Params) == 0 { return true }

	for _, param := range self.Params {
		switch param.DataType {
			case HttpIntParam: validateIntParam(self, param)
			case HttpStringParam: validateStringParam(self, param)
			case HttpFloatParam: validateFloatParam(self, param)
			case HttpBoolParam: validateBoolParam(self, param)
			case HttpObjectIdParam: validateObjectIdParam(self, param)
		}
	}

	return len(self.ErrorCodes) == 0
}

func (self *HttpContext) ParamInt(name string) int { return self.Params[name].Int() }

func (self *HttpContext) ParamFloat(name string) float64 { return self.Params[name].Float() }

func (self *HttpContext) ParamString(name string) string { return self.Params[name].String() }

func (self *HttpContext) HasParam(name string) bool {
	if val, found := self.Params[name]; !found || val == nil || val.Value == nil { return false }
	return true
}

func (self *HttpContext) ParamBool(name string) bool { return self.Params[name].Bool() }

func (self *HttpContext) ParamObjectId(name string) *bson.ObjectId { return self.Params[name].ObjectId() }

func (self *HttpContext) HasRawErrors() bool { return len(self.Errors) > 0 }

// This returns the param value as a string. If the param is missing or empty,
// the string will be len == 0.
func retrieveParamValue(ctx *HttpContext, param *HttpParam) string {
	switch param.Type {
		case HttpParamPost: return strings.TrimSpace(ctx.Request.PostFormValue(param.Name))
		case HttpParamJsonPost: return retrieveJsonParamValue(ctx, param)
		case HttpParamQuery: return strings.TrimSpace(ctx.Request.FormValue(param.Name))
		case HttpParamHeader: return strings.TrimSpace(ctx.Request.Header.Get(param.Name))
		case HttpParamPath: return strings.TrimSpace(mux.Vars(ctx.Request)[param.Name])
	}
	return nadaStr
}

func retrieveJsonParamValue(ctx *HttpContext, param *HttpParam) string {

	if len(ctx.Errors) > 0 {
		return nadaStr
	}

	// If this is the first access, read the body
	if len(ctx.body) == 0 {
		var err error
		ctx.body, err = ioutil.ReadAll(ctx.Request.Body)
		if err != nil {
			ctx.Errors = append(ctx.Errors, NewStackError("Error in raw data extraction - error: %v", err))
			return nadaStr
		}
	}

	if ctx.postJson == nil {
		var genJson interface{}
		err := json.Unmarshal(ctx.body, &genJson)
		if err != nil {
			ctx.Errors = append(ctx.Errors, NewStackError("Error in raw json data extraction - error: %v", err))
			return nadaStr
		}

		ctx.postJson = genJson.(map[string]interface{})
	}

	// Look for the value in the json. The json may hold the data in a variety
	// of formats. Convert back to a string to deal with the other data types :-(
	val, found := ctx.postJson[param.Name]
	if !found {
		return nadaStr
	}

	valType := reflect.TypeOf(val)

	if valType == nil { return nadaStr }

	switch valType.Kind() {
		case reflect.Invalid: return nadaStr
		case reflect.Bool: return fmt.Sprintf("%t", val.(bool))
		case reflect.Float64: return fmt.Sprintf("%g", val.(float64))
		case reflect.String: return val.(string)
		default: return nadaStr
	}

	return nadaStr
}

func appendInvalidErrorCode(ctx *HttpContext, param *HttpParam) {
	// Do not dupplicate error codes.
	for i := range ctx.ErrorCodes { if ctx.ErrorCodes[i] == param.InvalidErrorCode { return } }

	if len(param.InvalidErrorCode) == 0 {
		panic(fmt.Sprintf("We do not have an error code defined for param: %s - required: %t", param.Name, param.Required))
	}

	ctx.ErrorCodes = append(ctx.ErrorCodes, param.InvalidErrorCode)
	param.Valid = false
}

func validateIntParam(ctx *HttpContext, param *HttpParam) {

	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required { appendInvalidErrorCode(ctx, param); return }

	if len(param.Raw) == 0 { return }

	if val, err := strconv.Atoi(param.Raw); err != nil { appendInvalidErrorCode(ctx, param)
	} else { param.setPresentValue(val) }
}

func validateStringParam(ctx *HttpContext, param *HttpParam) {

	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if param.Required && param.MinLength > 0 && len(param.Raw) < param.MinLength {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if param.Required && param.MaxLength > 0 && len(param.Raw) > param.MaxLength {
		appendInvalidErrorCode(ctx, param)
		return
	}

	param.setPresentValue(param.Raw)
}

func validateFloatParam(ctx *HttpContext, param *HttpParam) {
	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if len(param.Raw) == 0 {
		return
	}

	if val, err := strconv.ParseFloat(param.Raw, 64); err != nil {
		appendInvalidErrorCode(ctx, param)
	} else {
		param.setPresentValue(val)
	}
}

func validateObjectIdParam(ctx *HttpContext, param *HttpParam) {

	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required { appendInvalidErrorCode(ctx, param); return }

	if len(param.Raw) == 0 { return }

	if !bson.IsObjectIdHex(param.Raw) { appendInvalidErrorCode(ctx, param); return }

	value := bson.ObjectIdHex(param.Raw)
	param.setPresentValue(&value)
}

// Boolean types include: 1, t, T, TRUE, true, True, 0, f, F, FALSE, false
func validateBoolParam(ctx *HttpContext, param *HttpParam) {

	param.Raw = retrieveParamValue(ctx, param)

	if len(param.Raw) == 0 && param.Required {
		appendInvalidErrorCode(ctx, param)
		return
	}

	if len(param.Raw) == 0 { return }

	if val, err := strconv.ParseBool(param.Raw); err != nil {
		appendInvalidErrorCode(ctx, param)
	} else {
		param.setPresentValue(val)
	}
}

func (self *HttpContext) DefineIntParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpIntParam, Required: required, Type: paramType, Valid: true }
}

// Boolean types include: 1, t, T, TRUE, true, True, 0, f, F, FALSE, false
func (self *HttpContext) DefineBoolParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpBoolParam, Required: required, Type: paramType, Valid: true }
}

func (self *HttpContext) DefineFloatParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpFloatParam, Required: required, Type: paramType, Valid: true }
}

func (self *HttpContext) DefineObjectIdParam(name, invalidErrorCode string, paramType HttpParamType, required bool) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpObjectIdParam, Required: required, Type: paramType, Valid: true }
}

func (self *HttpContext) DefineStringParam(name, invalidErrorCode string, paramType HttpParamType, required bool, minLength, maxLength int) {
	self.Params[name] = &HttpParam{ Name: name, InvalidErrorCode: invalidErrorCode, DataType: HttpStringParam, Required: required, Type: paramType, Valid: true }
}

// Call this method to init the http context struct.
func NewHttpContext(response http.ResponseWriter, request *http.Request) *HttpContext {
	return &HttpContext{ Response: response, Request: request, Params: make(map[string]*HttpParam) }
}

// This method returns true if the http request method is a HTTP post. If the
// field missing or incorrect, false is returned. This method will panic if
// the request is nil.
func IsHttpMethodPost(request *http.Request) bool {
	if request == nil {
		panic("request param is nil")
	}
	return len(request.Method) > 0 && strings.ToUpper(request.Method) == HttpPostMethod
}

// Write an http ok response string. The content type is text/plain.
func WriteOkResponseString(response http.ResponseWriter, msg string) error {
	if response == nil {
		return NewStackError("response param is nil")
	}

	msgLength := len(msg)

	if msgLength == 0 {
		return NewStackError("Response message is an empty string")
	}

	response.Header().Set(ContentTypeHeader, ContentTypeTextPlain)

	written, err := response.Write([]byte(msg))

	if err != nil {
		return err
	}

	if written != msgLength {
		return NewStackError("Did not write full message - bytes written %d - expected %d", written, msgLength)
	}

	return nil
}

// Encode and write a json response. If there is a problem encoding an http 500 is sent and an
// error is returned. If there are problems writting the response an error is returned.
func JsonEncodeAndWriteResponse(response http.ResponseWriter, value interface{}) error {

	if value == nil {
		return NewStackError("Nil value passed")
	}

	rawJson, err := json.Marshal(value)
	if err != nil {
		http.Error(response, "Error", 500)
		return NewStackError("Unable to marshal json: %v", err)
	}

	response.Header().Set(ContentTypeHeader, ContentTypeJson)

	written, err := response.Write(rawJson)
	if err != nil {
		return NewStackError("Unable to write response: %v", err)
	}

	if written != len(rawJson) {
		return NewStackError("Unable to write full response - wrote: %d - expected: %d", written, len(rawJson))
	}

	return nil
}

<|endoftext|>


<commit_before>package fs

import (
	"io"
	"log"

	"bazil.org/bazil/cas/blobs"
	wirecas "bazil.org/bazil/cas/wire"
	"bazil.org/bazil/fs/wire"
	"bazil.org/bazil/util/env"
	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"github.com/boltdb/bolt"
)

type file struct {
	fs.NodeRef

	inode  uint64
	name   string
	parent *dir
	blob   *blobs.Blob

	// when was this entry last changed
	// TODO: written time.Time
}

var _ = node(&file{})

func (f *file) getName() string {
	return f.name
}

func (f *file) setName(name string) {
	f.name = name
}

func (f *file) marshal() (*wire.Dirent, error) {
	de := &wire.Dirent{
		Inode: f.inode,
	}
	manifest, err := f.blob.Save()
	if err != nil {
		return nil, err
	}
	de.Type.File = &wire.File{
		Manifest: wirecas.FromBlob(manifest),
	}
	return de, nil
}

func (f *file) Attr() fuse.Attr {
	return fuse.Attr{
		Inode: f.inode,
		Mode:  0644,
		Nlink: 1,
		Uid:   env.MyUID,
		Gid:   env.MyGID,
		Size:  f.blob.Size(),
	}
}

func (f *file) Forget() {
	f.parent.fs.mu.Lock()
	defer f.parent.fs.mu.Unlock()

	f.parent.forgetChild(f)
}

func (f *file) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.Intr) fuse.Error {
	f.parent.fs.mu.Lock()
	defer f.parent.fs.mu.Unlock()

	n, err := f.blob.WriteAt(req.Data, req.Offset)
	resp.Size = n
	if err != nil {
		log.Printf("write error: %v", err)
		return fuse.EIO
	}
	return nil
}

func (f *file) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error {
	f.parent.fs.mu.Lock()
	defer f.parent.fs.mu.Unlock()

	// TODO only if dirty
	err := f.parent.fs.db.Update(func(tx *bolt.Tx) error {
		return f.parent.save(tx, f)
	})
	return err
}

const maxInt64 = 9223372036854775807

func (f *file) Read(req *fuse.ReadRequest, resp *fuse.ReadResponse, intr fs.Intr) fuse.Error {
	f.parent.fs.mu.Lock()
	defer f.parent.fs.mu.Unlock()

	if req.Offset < 0 {
		panic("unreachable")
	}
	if req.Offset > maxInt64 {
		log.Printf("offset is past int64 max: %d", req.Offset)
		return fuse.EIO
	}
	resp.Data = resp.Data[:req.Size]
	n, err := f.blob.ReadAt(resp.Data, int64(req.Offset))
	if err != nil && err != io.EOF {
		log.Printf("read error: %v", err)
		return fuse.EIO
	}
	resp.Data = resp.Data[:n]

	return nil
}

func (f *file) Setattr(req *fuse.SetattrRequest, resp *fuse.SetattrResponse, intr fs.Intr) fuse.Error {
	f.parent.fs.mu.Lock()
	defer f.parent.fs.mu.Unlock()

	valid := req.Valid
	if valid.Size() {
		err := f.blob.Truncate(req.Size)
		if err != nil {
			return err
		}
		valid &^= fuse.SetattrSize
	}

	// things we don't need to explicitly handle
	valid &^= fuse.SetattrLockOwner | fuse.SetattrHandle

	if valid != 0 {
		// don't let an unhandled operation slip by without error
		log.Printf("Setattr did not handle %v", valid)
		return fuse.ENOSYS
	}
	return nil
}
<commit_msg>fs: Allow kernel to use buffer cache to speed up reads and writes<commit_after>package fs

import (
	"io"
	"log"

	"bazil.org/bazil/cas/blobs"
	wirecas "bazil.org/bazil/cas/wire"
	"bazil.org/bazil/fs/wire"
	"bazil.org/bazil/util/env"
	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"github.com/boltdb/bolt"
)

type file struct {
	fs.NodeRef

	inode  uint64
	name   string
	parent *dir
	blob   *blobs.Blob

	// when was this entry last changed
	// TODO: written time.Time
}

var _ = node(&file{})

func (f *file) getName() string {
	return f.name
}

func (f *file) setName(name string) {
	f.name = name
}

func (f *file) marshal() (*wire.Dirent, error) {
	de := &wire.Dirent{
		Inode: f.inode,
	}
	manifest, err := f.blob.Save()
	if err != nil {
		return nil, err
	}
	de.Type.File = &wire.File{
		Manifest: wirecas.FromBlob(manifest),
	}
	return de, nil
}

func (f *file) Attr() fuse.Attr {
	return fuse.Attr{
		Inode: f.inode,
		Mode:  0644,
		Nlink: 1,
		Uid:   env.MyUID,
		Gid:   env.MyGID,
		Size:  f.blob.Size(),
	}
}

func (f *file) Forget() {
	f.parent.fs.mu.Lock()
	defer f.parent.fs.mu.Unlock()

	f.parent.forgetChild(f)
}

func (f *file) Open(req *fuse.OpenRequest, resp *fuse.OpenResponse, intr fs.Intr) (fs.Handle, fuse.Error) {
	// allow kernel to use buffer cache
	resp.Flags &^= fuse.OpenDirectIO
	return f, nil
}

func (f *file) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.Intr) fuse.Error {
	f.parent.fs.mu.Lock()
	defer f.parent.fs.mu.Unlock()

	n, err := f.blob.WriteAt(req.Data, req.Offset)
	resp.Size = n
	if err != nil {
		log.Printf("write error: %v", err)
		return fuse.EIO
	}
	return nil
}

func (f *file) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error {
	f.parent.fs.mu.Lock()
	defer f.parent.fs.mu.Unlock()

	// TODO only if dirty
	err := f.parent.fs.db.Update(func(tx *bolt.Tx) error {
		return f.parent.save(tx, f)
	})
	return err
}

const maxInt64 = 9223372036854775807

func (f *file) Read(req *fuse.ReadRequest, resp *fuse.ReadResponse, intr fs.Intr) fuse.Error {
	f.parent.fs.mu.Lock()
	defer f.parent.fs.mu.Unlock()

	if req.Offset < 0 {
		panic("unreachable")
	}
	if req.Offset > maxInt64 {
		log.Printf("offset is past int64 max: %d", req.Offset)
		return fuse.EIO
	}
	resp.Data = resp.Data[:req.Size]
	n, err := f.blob.ReadAt(resp.Data, int64(req.Offset))
	if err != nil && err != io.EOF {
		log.Printf("read error: %v", err)
		return fuse.EIO
	}
	resp.Data = resp.Data[:n]

	return nil
}

func (f *file) Setattr(req *fuse.SetattrRequest, resp *fuse.SetattrResponse, intr fs.Intr) fuse.Error {
	f.parent.fs.mu.Lock()
	defer f.parent.fs.mu.Unlock()

	valid := req.Valid
	if valid.Size() {
		err := f.blob.Truncate(req.Size)
		if err != nil {
			return err
		}
		valid &^= fuse.SetattrSize
	}

	// things we don't need to explicitly handle
	valid &^= fuse.SetattrLockOwner | fuse.SetattrHandle

	if valid != 0 {
		// don't let an unhandled operation slip by without error
		log.Printf("Setattr did not handle %v", valid)
		return fuse.ENOSYS
	}
	return nil
}
<|endoftext|>