<commit_before>package main

import (
	goflag "flag"
	"fmt"
	"os"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"k8s.io/kops/upup/pkg/api"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/vfs"
	"k8s.io/kops/upup/pkg/kutil"
)

type RootCmd struct {
	configFile string

	clusterRegistry *api.ClusterRegistry

	stateLocation string
	clusterName   string

	cobraCommand *cobra.Command
}

var rootCommand = RootCmd{
	cobraCommand: &cobra.Command{
		Use:   "kops",
		Short: "kops is kubernetes ops",
		Long: `kops is kubernetes ops.
It allows you to create, destroy, upgrade and maintain clusters.`,
	},
}

func Execute() {
	goflag.CommandLine.Parse([]string{})
	if err := rootCommand.cobraCommand.Execute(); err != nil {
		exitWithError(err)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	cmd := rootCommand.cobraCommand

	cmd.PersistentFlags().AddGoFlagSet(goflag.CommandLine)

	cmd.PersistentFlags().StringVar(&rootCommand.configFile, "config", "", "config file (default is $HOME/.kops.yaml)")

	defaultStateStore := os.Getenv("KOPS_STATE_STORE")
	cmd.PersistentFlags().StringVarP(&rootCommand.stateLocation, "state", "", defaultStateStore, "Location of state storage")

	cmd.PersistentFlags().StringVarP(&rootCommand.clusterName, "name", "", "", "Name of cluster")
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if rootCommand.configFile != "" {
		// enable ability to specify config file via flag
		viper.SetConfigFile(rootCommand.configFile)
	}

	viper.SetConfigName(".kops") // name of config file (without extension)
	viper.AddConfigPath("$HOME") // adding home directory as first search path
	viper.AutomaticEnv()         // read in environment variables that match

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}
}

func (c *RootCmd) AddCommand(cmd *cobra.Command) {
	c.cobraCommand.AddCommand(cmd)
}

// ProcessArgs will parse the positional args.  It assumes one of these formats:
//  * <no arguments at all>
//  * <clustername> (and --name not specified)
// Everything else is an error.
func (c *RootCmd) ProcessArgs(args []string) error {
	if len(args) == 0 {
		return nil
	}
	if len(args) == 1 {
		// Assume <clustername>
		if c.clusterName != "" {
			return fmt.Errorf("Cannot specify cluster via --name and positional argument")
		}
		c.clusterName = args[0]
		return nil
	}

	return fmt.Errorf("expected a single <clustername> to be passed as an argument")
}

func (c *RootCmd) ClusterName() string {
	if c.clusterName != "" {
		return c.clusterName
	}

	config, err := readKubectlClusterConfig()
	if err != nil {
		glog.Warningf("error reading kubecfg: %v", err)
	} else if config != nil && config.Name != "" {
		fmt.Fprintf(os.Stderr, "Using cluster from kubectl context: %s\n\n", config.Name)
		c.clusterName = config.Name
	}
	return c.clusterName
}

func readKubectlClusterConfig() (*kutil.KubectlClusterWithName, error) {
	kubectl := &kutil.Kubectl{}
	context, err := kubectl.GetCurrentContext()
	if err != nil {
		return nil, fmt.Errorf("error getting current context from kubectl: %v", err)
	}
	glog.V(4).Infof("context = %q", context)

	config, err := kubectl.GetConfig(true)
	if err != nil {
		return nil, fmt.Errorf("error getting current config from kubectl: %v", err)
	}

	// Minify should have done this
	if len(config.Clusters) != 1 {
		return nil, fmt.Errorf("expected exactly one cluster in kubectl config, found %d", len(config.Clusters))
	}

	return config.Clusters[0], nil
}

func (c *RootCmd) ClusterRegistry() (*api.ClusterRegistry, error) {
	if c.clusterRegistry != nil {
		return c.clusterRegistry, nil
	}

	if c.stateLocation == "" {
		return nil, fmt.Errorf("--state is required (or export KOPS_STATE_STORE)")
	}

	basePath, err := vfs.Context.BuildVfsPath(c.stateLocation)
	if err != nil {
		return nil, fmt.Errorf("error building state store path for %q: %v", c.stateLocation, err)
	}

	if !vfs.IsClusterReadable(basePath) {
		return nil, fmt.Errorf("State store %q is not cloud-reachable - please use an S3 bucket", c.stateLocation)
	}

	clusterRegistry := api.NewClusterRegistry(basePath)
	c.clusterRegistry = clusterRegistry
	return clusterRegistry, nil
}

func (c *RootCmd) Cluster() (*api.ClusterRegistry, *api.Cluster, error) {
	clusterRegistry, err := c.ClusterRegistry()
	if err != nil {
		return nil, nil, err
	}

	clusterName := c.ClusterName()
	if clusterName == "" {
		return nil, nil, fmt.Errorf("--name is required")
	}

	cluster, err := clusterRegistry.Find(clusterName)
	if err != nil {
		return nil, nil, fmt.Errorf("error reading cluster configuration: %v", err)
	}
	if cluster == nil {
		return nil, nil, fmt.Errorf("cluster %q not found", clusterName)
	}

	if clusterName != cluster.Name {
		return nil, nil, fmt.Errorf("cluster name did not match expected name: %v vs %v", clusterName, cluster.Name)
	}
	return clusterRegistry, cluster, nil
}

func (c *RootCmd) InstanceGroupRegistry() (*api.InstanceGroupRegistry, error) {
	clusterRegistry, err := c.ClusterRegistry()
	if err != nil {
		return nil, err
	}

	clusterName := c.ClusterName()
	if clusterName == "" {
		return nil, fmt.Errorf("--name is required")
	}

	return clusterRegistry.InstanceGroups(clusterName)
}

func (c *RootCmd) SecretStore() (fi.SecretStore, error) {
	clusterRegistry, err := c.ClusterRegistry()
	if err != nil {
		return nil, err
	}

	clusterName := c.ClusterName()
	if clusterName == "" {
		return nil, fmt.Errorf("--name is required")
	}

	return clusterRegistry.SecretStore(clusterName), nil
}

func (c *RootCmd) KeyStore() (fi.CAStore, error) {
	clusterRegistry, err := c.ClusterRegistry()
	if err != nil {
		return nil, err
	}

	clusterName := c.ClusterName()
	if clusterName == "" {
		return nil, fmt.Errorf("--name is required")
	}

	return clusterRegistry.KeyStore(clusterName), nil
}
<commit_msg>Always set logtostderr<commit_after>package main

import (
	goflag "flag"
	"fmt"
	"os"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"k8s.io/kops/upup/pkg/api"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/vfs"
	"k8s.io/kops/upup/pkg/kutil"
)

type RootCmd struct {
	configFile string

	clusterRegistry *api.ClusterRegistry

	stateLocation string
	clusterName   string

	cobraCommand *cobra.Command
}

var rootCommand = RootCmd{
	cobraCommand: &cobra.Command{
		Use:   "kops",
		Short: "kops is kubernetes ops",
		Long: `kops is kubernetes ops.
It allows you to create, destroy, upgrade and maintain clusters.`,
	},
}

func Execute() {
	goflag.Set("logtostderr", "true")
	goflag.CommandLine.Parse([]string{})
	if err := rootCommand.cobraCommand.Execute(); err != nil {
		exitWithError(err)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	cmd := rootCommand.cobraCommand

	cmd.PersistentFlags().AddGoFlagSet(goflag.CommandLine)

	cmd.PersistentFlags().StringVar(&rootCommand.configFile, "config", "", "config file (default is $HOME/.kops.yaml)")

	defaultStateStore := os.Getenv("KOPS_STATE_STORE")
	cmd.PersistentFlags().StringVarP(&rootCommand.stateLocation, "state", "", defaultStateStore, "Location of state storage")

	cmd.PersistentFlags().StringVarP(&rootCommand.clusterName, "name", "", "", "Name of cluster")
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if rootCommand.configFile != "" {
		// enable ability to specify config file via flag
		viper.SetConfigFile(rootCommand.configFile)
	}

	viper.SetConfigName(".kops") // name of config file (without extension)
	viper.AddConfigPath("$HOME") // adding home directory as first search path
	viper.AutomaticEnv()         // read in environment variables that match

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}
}

func (c *RootCmd) AddCommand(cmd *cobra.Command) {
	c.cobraCommand.AddCommand(cmd)
}

// ProcessArgs will parse the positional args.  It assumes one of these formats:
//  * <no arguments at all>
//  * <clustername> (and --name not specified)
// Everything else is an error.
func (c *RootCmd) ProcessArgs(args []string) error {
	if len(args) == 0 {
		return nil
	}
	if len(args) == 1 {
		// Assume <clustername>
		if c.clusterName != "" {
			return fmt.Errorf("Cannot specify cluster via --name and positional argument")
		}
		c.clusterName = args[0]
		return nil
	}

	return fmt.Errorf("expected a single <clustername> to be passed as an argument")
}

func (c *RootCmd) ClusterName() string {
	if c.clusterName != "" {
		return c.clusterName
	}

	config, err := readKubectlClusterConfig()
	if err != nil {
		glog.Warningf("error reading kubecfg: %v", err)
	} else if config != nil && config.Name != "" {
		fmt.Fprintf(os.Stderr, "Using cluster from kubectl context: %s\n\n", config.Name)
		c.clusterName = config.Name
	}
	return c.clusterName
}

func readKubectlClusterConfig() (*kutil.KubectlClusterWithName, error) {
	kubectl := &kutil.Kubectl{}
	context, err := kubectl.GetCurrentContext()
	if err != nil {
		return nil, fmt.Errorf("error getting current context from kubectl: %v", err)
	}
	glog.V(4).Infof("context = %q", context)

	config, err := kubectl.GetConfig(true)
	if err != nil {
		return nil, fmt.Errorf("error getting current config from kubectl: %v", err)
	}

	// Minify should have done this
	if len(config.Clusters) != 1 {
		return nil, fmt.Errorf("expected exactly one cluster in kubectl config, found %d", len(config.Clusters))
	}

	return config.Clusters[0], nil
}

func (c *RootCmd) ClusterRegistry() (*api.ClusterRegistry, error) {
	if c.clusterRegistry != nil {
		return c.clusterRegistry, nil
	}

	if c.stateLocation == "" {
		return nil, fmt.Errorf("--state is required (or export KOPS_STATE_STORE)")
	}

	basePath, err := vfs.Context.BuildVfsPath(c.stateLocation)
	if err != nil {
		return nil, fmt.Errorf("error building state store path for %q: %v", c.stateLocation, err)
	}

	if !vfs.IsClusterReadable(basePath) {
		return nil, fmt.Errorf("State store %q is not cloud-reachable - please use an S3 bucket", c.stateLocation)
	}

	clusterRegistry := api.NewClusterRegistry(basePath)
	c.clusterRegistry = clusterRegistry
	return clusterRegistry, nil
}

func (c *RootCmd) Cluster() (*api.ClusterRegistry, *api.Cluster, error) {
	clusterRegistry, err := c.ClusterRegistry()
	if err != nil {
		return nil, nil, err
	}

	clusterName := c.ClusterName()
	if clusterName == "" {
		return nil, nil, fmt.Errorf("--name is required")
	}

	cluster, err := clusterRegistry.Find(clusterName)
	if err != nil {
		return nil, nil, fmt.Errorf("error reading cluster configuration: %v", err)
	}
	if cluster == nil {
		return nil, nil, fmt.Errorf("cluster %q not found", clusterName)
	}

	if clusterName != cluster.Name {
		return nil, nil, fmt.Errorf("cluster name did not match expected name: %v vs %v", clusterName, cluster.Name)
	}
	return clusterRegistry, cluster, nil
}

func (c *RootCmd) InstanceGroupRegistry() (*api.InstanceGroupRegistry, error) {
	clusterRegistry, err := c.ClusterRegistry()
	if err != nil {
		return nil, err
	}

	clusterName := c.ClusterName()
	if clusterName == "" {
		return nil, fmt.Errorf("--name is required")
	}

	return clusterRegistry.InstanceGroups(clusterName)
}

func (c *RootCmd) SecretStore() (fi.SecretStore, error) {
	clusterRegistry, err := c.ClusterRegistry()
	if err != nil {
		return nil, err
	}

	clusterName := c.ClusterName()
	if clusterName == "" {
		return nil, fmt.Errorf("--name is required")
	}

	return clusterRegistry.SecretStore(clusterName), nil
}

func (c *RootCmd) KeyStore() (fi.CAStore, error) {
	clusterRegistry, err := c.ClusterRegistry()
	if err != nil {
		return nil, err
	}

	clusterName := c.ClusterName()
	if clusterName == "" {
		return nil, fmt.Errorf("--name is required")
	}

	return clusterRegistry.KeyStore(clusterName), nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"

	"github.com/gorilla/mux"
	"github.com/pakerfeldt/lovi/pkg/api"
	"github.com/pakerfeldt/lovi/pkg/bindings"
	"github.com/pakerfeldt/lovi/pkg/config"
	"github.com/pakerfeldt/lovi/pkg/core"
)

func main() {
	settings := config.Settings()
	router := mux.NewRouter()

	conf := config.Parse(settings.ConfigFile)
	for _, policy := range conf.Policies {
		core.AddPolicy(policy)
	}

	for _, transport := range conf.Transports {
		factory, exists := bindings.Factories()[transport.ID]
		if !exists {
			panic(errors.New("Transport '" + transport.ID + "' does not exist."))
		}
		core.AddTransport(factory(router, transport.Configuration, core.HandleAck))
	}
	config.Print(conf)

	api.Init(router, settings)
}
<commit_msg>Remove config dump<commit_after>package main

import (
	"errors"

	"github.com/gorilla/mux"
	"github.com/pakerfeldt/lovi/pkg/api"
	"github.com/pakerfeldt/lovi/pkg/bindings"
	"github.com/pakerfeldt/lovi/pkg/config"
	"github.com/pakerfeldt/lovi/pkg/core"
)

func main() {
	settings := config.Settings()
	router := mux.NewRouter()

	conf := config.Parse(settings.ConfigFile)
	for _, policy := range conf.Policies {
		core.AddPolicy(policy)
	}

	for _, transport := range conf.Transports {
		factory, exists := bindings.Factories()[transport.ID]
		if !exists {
			panic(errors.New("Transport '" + transport.ID + "' does not exist."))
		}
		core.AddTransport(factory(router, transport.Configuration, core.HandleAck))
	}

	api.Init(router, settings)
}
<|endoftext|>


<commit_before>package size

import (
	"fmt"

	"github.com/ncw/rclone/cmd"
	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/operations"
	"github.com/spf13/cobra"
)

func init() {
	cmd.Root.AddCommand(commandDefintion)
}

var commandDefintion = &cobra.Command{
	Use:   "size remote:path",
	Short: `Prints the total size and number of objects in remote:path.`,
	Run: func(command *cobra.Command, args []string) {
		cmd.CheckArgs(1, 1, command, args)
		fsrc := cmd.NewFsSrc(args)
		cmd.Run(false, false, command, func() error {
			objects, size, err := operations.Count(fsrc)
			if err != nil {
				return err
			}
			fmt.Printf("Total objects: %d\n", objects)
			fmt.Printf("Total size: %s (%d Bytes)\n", fs.SizeSuffix(size).Unit("Bytes"), size)
			return nil
		})
	},
}
<commit_msg>size: Add --json flag<commit_after>package size

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/ncw/rclone/cmd"
	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/operations"
	"github.com/spf13/cobra"
)

var jsonOutput bool

func init() {
	cmd.Root.AddCommand(commandDefinition)
	commandDefinition.Flags().BoolVar(&jsonOutput, "json", false, "format output as JSON")
}

var commandDefinition = &cobra.Command{
	Use:   "size remote:path",
	Short: `Prints the total size and number of objects in remote:path.`,
	Run: func(command *cobra.Command, args []string) {
		cmd.CheckArgs(1, 1, command, args)
		fsrc := cmd.NewFsSrc(args)
		cmd.Run(false, false, command, func() error {
			var err error
			var results struct {
				Count int64 `json:"count"`
				Bytes int64 `json:"bytes"`
			}

			results.Count, results.Bytes, err = operations.Count(fsrc)
			if err != nil {
				return err
			}

			if jsonOutput {
				return json.NewEncoder(os.Stdout).Encode(results)
			}

			fmt.Printf("Total objects: %d\n", results.Count)
			fmt.Printf("Total size: %s (%d Bytes)\n", fs.SizeSuffix(results.Bytes).Unit("Bytes"), results.Bytes)

			return nil
		})
	},
}
<|endoftext|>


<commit_before>// Copyright © 2017 Stream
//

package cmd

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/GetStream/vg/utils"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// uninstallCmd represents the uninstall command
var uninstallCmd = &cobra.Command{
	Use:   "uninstall <package> [otherPackages]",
	Short: "Uninstall a package from the active workspace",
	Long: `To remove github.com/pkg/errors:

	vg uninstall github.com/pkg/errors`,
	PreRunE: func(cmd *cobra.Command, args []string) error {
		if len(args) < 1 {
			return errors.New("No package specified")
		}
		return nil
	},
	RunE: func(cmd *cobra.Command, args []string) error {
		workspace := os.Getenv("VIRTUALGO")
		if workspace == "" {
			return errors.New("A virtualgo workspace should be activated first by using `vg activate [workspaceName]`")
		}

		for _, pkg := range args {
			// pkgComponents := strings.Split(pkg, hello
			fmt.Printf("Uninstalling %s from workspace\n", pkg)
			err := os.RemoveAll(filepath.Join(utils.VirtualgoDir(), workspace, "src", pkg))
			if err != nil {
				return errors.Wrapf(err, "Couldn't remove package src '%s'", workspace)
			}

			pkgInstalledDirs, err := filepath.Glob(filepath.Join(utils.VirtualgoDir(), workspace, "pkg", "*", pkg))
			if err != nil {
				return errors.Wrapf(err, "Something went wrong when globbing files for '%s'", pkg)
			}

			for _, path := range pkgInstalledDirs {
				fmt.Println("Removing", path)

				err = os.RemoveAll(path)
				if err != nil {
					return errors.Wrapf(err, "Couldn't remove installed package files for '%s'", pkg)
				}
			}

			pkgInstalledFiles, err := filepath.Glob(filepath.Join(utils.VirtualgoDir(), workspace, "pkg", "*", pkg+".a"))
			if err != nil {
				return errors.Wrapf(err, "Something went wrong when globbing files for '%s'", pkg)
			}

			for _, path := range pkgInstalledFiles {
				fmt.Println("Removing", path)

				err = os.RemoveAll(path)
				if err != nil {
					return errors.Wrapf(err, "Couldn't remove installed package files for '%s'", pkg)
				}
			}

		}
		return nil
	},
}

func init() {
	RootCmd.AddCommand(uninstallCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// uninstallCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// uninstallCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")

}
<commit_msg>Uninstall: Also remove local install from settings<commit_after>// Copyright © 2017 Stream
//

package cmd

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/GetStream/vg/utils"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// uninstallCmd represents the uninstall command
var uninstallCmd = &cobra.Command{
	Use:   "uninstall <package> [otherPackages]",
	Short: "Uninstall a package from the active workspace",
	Long: `To remove github.com/pkg/errors:

	vg uninstall github.com/pkg/errors`,
	PreRunE: func(cmd *cobra.Command, args []string) error {
		if len(args) < 1 {
			return errors.New("No package specified")
		}
		return nil
	},
	RunE: func(cmd *cobra.Command, args []string) error {
		workspace := os.Getenv("VIRTUALGO")
		if workspace == "" {
			return errors.New("A virtualgo workspace should be activated first by using `vg activate [workspaceName]`")
		}

		for _, pkg := range args {
			// pkgComponents := strings.Split(pkg, hello
			pkgDir := utils.PkgToDir(pkg)
			fmt.Printf("Uninstalling %q from workspace\n", pkg)
			err := os.RemoveAll(filepath.Join(utils.VirtualgoDir(), workspace, "src", pkgDir))
			if err != nil {
				return errors.Wrapf(err, "Couldn't remove package src '%s'", workspace)
			}

			pkgInstalledDirs, err := filepath.Glob(filepath.Join(utils.VirtualgoDir(), workspace, "pkg", "*", pkgDir))
			if err != nil {
				return errors.Wrapf(err, "Something went wrong when globbing files for '%s'", pkg)
			}

			for _, path := range pkgInstalledDirs {
				fmt.Println("Removing", path)

				err = os.RemoveAll(path)
				if err != nil {
					return errors.Wrapf(err, "Couldn't remove installed package files for '%s'", pkg)
				}
			}

			pkgInstalledFiles, err := filepath.Glob(filepath.Join(utils.VirtualgoDir(), workspace, "pkg", "*", pkgDir+".a"))
			if err != nil {
				return errors.Wrapf(err, "Something went wrong when globbing files for '%s'", pkg)
			}

			for _, path := range pkgInstalledFiles {
				fmt.Println("Removing", path)

				err = os.RemoveAll(path)
				if err != nil {
					return errors.Wrapf(err, "Couldn't remove installed package files for '%s'", pkg)
				}
			}

			settings, err := utils.CurrentSettings()
			if err != nil {
				return err
			}
			if _, ok := settings.LocalInstalls[pkg]; ok {
				fmt.Printf("Removing %q from persistent local installs\n", pkg)
				delete(settings.LocalInstalls, pkg)

				err = utils.SaveCurrentSettings(settings)
				if err != nil {
					return err
				}
			}

		}
		return nil
	},
}

func init() {
	RootCmd.AddCommand(uninstallCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// uninstallCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// uninstallCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")

}
<|endoftext|>


<commit_before>package models

import (
	"errors"

	"github.com/koding/bongo"
	"github.com/nu7hatch/gouuid"
)

func NewAccount() *Account {
	return &Account{}
}

func (a Account) GetId() int64 {
	return a.Id
}

func (a Account) BongoName() string {
	return "api.account"
}

func (a *Account) BeforeCreate() error {
	return a.createToken()
}

func (a *Account) BeforeUpdate() error {
	return a.createToken()
}

func (a *Account) createToken() error {
	if a.Token == "" {
		token, err := uuid.NewV4()
		if err != nil {
			return err
		}
		a.Token = token.String()
	}

	return nil
}

func (a *Account) AfterUpdate() {
	SetAccountToCache(a)
}

func (a *Account) AfterCreate() {
	SetAccountToCache(a)
	bongo.B.AfterCreate(a)
}

func (a *Account) One(q *bongo.Query) error {
	return bongo.B.One(a, a, q)
}

func (a *Account) ById(id int64) error {
	return bongo.B.ById(a, id)
}

func (a *Account) Update() error {
	return bongo.B.Update(a)
}

func (a *Account) Create() error {
	if a.OldId == "" {
		return errors.New("old id is not set")
	}

	if a.Nick == "guestuser" {
		return ErrGuestsAreNotAllowed
	}

	return bongo.B.Create(a)
}

func (a *Account) Some(data interface{}, q *bongo.Query) error {
	return bongo.B.Some(a, data, q)
}
<commit_msg>Socialapi: added a new bongo function for fetching the accounts by their ids<commit_after>package models

import (
	"errors"

	"github.com/koding/bongo"
	"github.com/nu7hatch/gouuid"
)

func NewAccount() *Account {
	return &Account{}
}

func (a Account) GetId() int64 {
	return a.Id
}

func (a Account) BongoName() string {
	return "api.account"
}

func (a *Account) BeforeCreate() error {
	return a.createToken()
}

func (a *Account) BeforeUpdate() error {
	return a.createToken()
}

func (a *Account) createToken() error {
	if a.Token == "" {
		token, err := uuid.NewV4()
		if err != nil {
			return err
		}
		a.Token = token.String()
	}

	return nil
}

func (a *Account) AfterUpdate() {
	SetAccountToCache(a)
}

func (a *Account) AfterCreate() {
	SetAccountToCache(a)
	bongo.B.AfterCreate(a)
}

func (a *Account) One(q *bongo.Query) error {
	return bongo.B.One(a, a, q)
}

func (a *Account) ById(id int64) error {
	return bongo.B.ById(a, id)
}

func (a *Account) Update() error {
	return bongo.B.Update(a)
}

func (a *Account) Create() error {
	if a.OldId == "" {
		return errors.New("old id is not set")
	}

	if a.Nick == "guestuser" {
		return ErrGuestsAreNotAllowed
	}

	return bongo.B.Create(a)
}

func (a *Account) Some(data interface{}, q *bongo.Query) error {
	return bongo.B.Some(a, data, q)
}

func (a *Account) FetchByIds(ids []int64) ([]Account, error) {
	var accounts []Account

	if len(ids) == 0 {
		return accounts, nil
	}

	if err := bongo.B.FetchByIds(a, &accounts, ids); err != nil {
		return nil, err
	}

	return accounts, nil
}
<|endoftext|>


<commit_before>// Copyright 2015, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package endtoend

import (
	"flag"
	"fmt"
	"os"
	"testing"

	"github.com/youtube/vitess/go/sqldb"
	"github.com/youtube/vitess/go/vt/tabletserver"
	"github.com/youtube/vitess/go/vt/tabletserver/endtoend/framework"
	"github.com/youtube/vitess/go/vt/vttest"

	// import mysql to register mysql connection function

	// import memcache to register memcache connection function
	_ "github.com/youtube/vitess/go/memcache"
)

var (
	connParams sqldb.ConnParams
)

func TestMain(m *testing.M) {
	flag.Parse()
	tabletserver.Init()

	exitCode := func() int {
		hdl, err := vttest.LaunchMySQL("vttest", schema, testing.Verbose())
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not launch mysql: %v\n", err)
			return 1
		}
		defer hdl.TearDown()
		connParams, err = hdl.MySQLConnParams()
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not fetch mysql params: %v\n", err)
			return 1
		}
		err = framework.StartDefaultServer(connParams)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%v", err)
			return 1
		}
		defer framework.StopDefaultServer()

		return m.Run()
	}()
	os.Exit(exitCode)
}

var schema = `create table vtocc_test(intval int, floatval float, charval varchar(256), binval varbinary(256), primary key(intval)) comment 'vtocc_nocache';
delete from vtocc_test;
insert into vtocc_test values(1, 1.12345, 0xC2A2, 0x00FF), (2, null, '', null), (3, null, null, null);

create table vtocc_a(eid bigint default 1, id int default 1, name varchar(128), foo varbinary(128), primary key(eid, id)) comment 'vtocc_nocache';
create table vtocc_b(eid bigint, id int, primary key(eid, id)) comment 'vtocc_nocache';
create table vtocc_c(eid bigint, name varchar(128), foo varbinary(128), primary key(eid, name)) comment 'vtocc_nocache';
create table vtocc_d(eid bigint, id int) comment 'vtocc_nocache';
create table vtocc_e(eid bigint auto_increment, id int default 1, name varchar(128) default 'name', foo varchar(128), primary key(eid, id, name)) comment 'vtocc_nocache';
create table vtocc_f(vb varbinary(16) default 'ab', id int, primary key(vb)) comment 'vtocc_nocache';
create table upsert_test(id1 int, id2 int, primary key (id1)) comment 'vtocc_nocache';
create unique index id2_idx on upsert_test(id2);
delete from vtocc_a;
delete from vtocc_c;
insert into vtocc_a(eid, id, name, foo) values(1, 1, 'abcd', 'efgh'), (1, 2, 'bcde', 'fghi');
insert into vtocc_b(eid, id) values(1, 1), (1, 2);
insert into vtocc_c(eid, name, foo) values(10, 'abcd', '20'), (11, 'bcde', '30');
delete from upsert_test;

create table vtocc_cached1(eid bigint, name varchar(128), foo varbinary(128), primary key(eid));
create index aname1 on vtocc_cached1(name);
delete from vtocc_cached1;
insert into vtocc_cached1 values (1, 'a', 'abcd');
insert into vtocc_cached1 values (2, 'a', 'abcd');
insert into vtocc_cached1 values (3, 'c', 'abcd');
insert into vtocc_cached1 values (4, 'd', 'abcd');
insert into vtocc_cached1 values (5, 'e', 'efgh');
insert into vtocc_cached1 values (9, 'i', 'ijkl');

create table vtocc_cached2(eid bigint, bid varbinary(16), name varchar(128), foo varbinary(128), primary key(eid, bid));
create index aname2 on vtocc_cached2(eid, name);
delete from vtocc_cached2;
insert into vtocc_cached2 values (1, 'foo', 'abcd1', 'efgh');
insert into vtocc_cached2 values (1, 'bar', 'abcd1', 'efgh');
insert into vtocc_cached2 values (2, 'foo', 'abcd2', 'efgh');
insert into vtocc_cached2 values (2, 'bar', 'abcd2', 'efgh');

create table vtocc_big(id int, string1 varchar(128), string2 varchar(100), string3 char(1), string4 varchar(50), string5 varchar(50), date1 date, string6 varchar(16), string7 varchar(120), bigint1 bigint(20), bigint2 bigint(20), date2 date, integer1 int, tinyint1 tinyint(4), primary key(id)) comment 'vtocc_big';

create table vtocc_ints(tiny tinyint, tinyu tinyint unsigned, small smallint, smallu smallint unsigned, medium mediumint, mediumu mediumint unsigned, normal int, normalu int unsigned, big bigint, bigu bigint unsigned, y year, primary key(tiny)) comment 'vtocc_nocache';
create table vtocc_fracts(id int, deci decimal(5,2), num numeric(5,2), f float, d double, primary key(id)) comment 'vtocc_nocache';
create table vtocc_strings(vb varbinary(16), c char(16), vc varchar(16), b binary(4), tb tinyblob, bl blob, ttx tinytext, tx text, en enum('a','b'), s set('a','b'), primary key(vb)) comment 'vtocc_nocache';
create table vtocc_misc(id int, b bit(8), d date, dt datetime, t time, primary key(id)) comment 'vtocc_nocache';

create table vtocc_part1(key1 bigint, key2 bigint, data1 int, primary key(key1, key2));
create unique index vtocc_key2 on vtocc_part1(key2);
create table vtocc_part2(key3 bigint, data2 int, primary key(key3));
create view vtocc_view as select key2, key1, data1, data2 from vtocc_part1, vtocc_part2 where key2=key3;
delete from vtocc_part1;
delete from vtocc_part2;
insert into vtocc_part1 values(10, 1, 1);
insert into vtocc_part1 values(10, 2, 2);
insert into vtocc_part2 values(1, 3);
insert into vtocc_part2 values(2, 4);

create table vtocc_acl_no_access(key1 bigint, key2 bigint, primary key(key1));
create table vtocc_acl_read_only(key1 bigint, key2 bigint, primary key(key1));
create table vtocc_acl_read_write(key1 bigint, key2 bigint, primary key(key1));
create table vtocc_acl_admin(key1 bigint, key2 bigint, primary key(key1));
create table vtocc_acl_unmatched(key1 bigint, key2 bigint, primary key(key1));
create table vtocc_acl_all_user_read_only(key1 bigint, key2 bigint, primary key(key1));`
<commit_msg>Adding comment to fix import into google3.<commit_after>// Copyright 2015, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package endtoend

import (
	"flag"
	"fmt"
	"os"
	"testing"

	"github.com/youtube/vitess/go/sqldb"
	"github.com/youtube/vitess/go/vt/tabletserver"
	"github.com/youtube/vitess/go/vt/tabletserver/endtoend/framework"
	"github.com/youtube/vitess/go/vt/vttest"

	// import mysql to register mysql connection function

	// import memcache to register memcache connection function
	_ "github.com/youtube/vitess/go/memcache"
)

var (
	connParams sqldb.ConnParams
)

func TestMain(m *testing.M) {
	flag.Parse() // Do not remove this comment, import into google3 depends on it
	tabletserver.Init()

	exitCode := func() int {
		hdl, err := vttest.LaunchMySQL("vttest", schema, testing.Verbose())
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not launch mysql: %v\n", err)
			return 1
		}
		defer hdl.TearDown()
		connParams, err = hdl.MySQLConnParams()
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not fetch mysql params: %v\n", err)
			return 1
		}
		err = framework.StartDefaultServer(connParams)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%v", err)
			return 1
		}
		defer framework.StopDefaultServer()

		return m.Run()
	}()
	os.Exit(exitCode)
}

var schema = `create table vtocc_test(intval int, floatval float, charval varchar(256), binval varbinary(256), primary key(intval)) comment 'vtocc_nocache';
delete from vtocc_test;
insert into vtocc_test values(1, 1.12345, 0xC2A2, 0x00FF), (2, null, '', null), (3, null, null, null);

create table vtocc_a(eid bigint default 1, id int default 1, name varchar(128), foo varbinary(128), primary key(eid, id)) comment 'vtocc_nocache';
create table vtocc_b(eid bigint, id int, primary key(eid, id)) comment 'vtocc_nocache';
create table vtocc_c(eid bigint, name varchar(128), foo varbinary(128), primary key(eid, name)) comment 'vtocc_nocache';
create table vtocc_d(eid bigint, id int) comment 'vtocc_nocache';
create table vtocc_e(eid bigint auto_increment, id int default 1, name varchar(128) default 'name', foo varchar(128), primary key(eid, id, name)) comment 'vtocc_nocache';
create table vtocc_f(vb varbinary(16) default 'ab', id int, primary key(vb)) comment 'vtocc_nocache';
create table upsert_test(id1 int, id2 int, primary key (id1)) comment 'vtocc_nocache';
create unique index id2_idx on upsert_test(id2);
delete from vtocc_a;
delete from vtocc_c;
insert into vtocc_a(eid, id, name, foo) values(1, 1, 'abcd', 'efgh'), (1, 2, 'bcde', 'fghi');
insert into vtocc_b(eid, id) values(1, 1), (1, 2);
insert into vtocc_c(eid, name, foo) values(10, 'abcd', '20'), (11, 'bcde', '30');
delete from upsert_test;

create table vtocc_cached1(eid bigint, name varchar(128), foo varbinary(128), primary key(eid));
create index aname1 on vtocc_cached1(name);
delete from vtocc_cached1;
insert into vtocc_cached1 values (1, 'a', 'abcd');
insert into vtocc_cached1 values (2, 'a', 'abcd');
insert into vtocc_cached1 values (3, 'c', 'abcd');
insert into vtocc_cached1 values (4, 'd', 'abcd');
insert into vtocc_cached1 values (5, 'e', 'efgh');
insert into vtocc_cached1 values (9, 'i', 'ijkl');

create table vtocc_cached2(eid bigint, bid varbinary(16), name varchar(128), foo varbinary(128), primary key(eid, bid));
create index aname2 on vtocc_cached2(eid, name);
delete from vtocc_cached2;
insert into vtocc_cached2 values (1, 'foo', 'abcd1', 'efgh');
insert into vtocc_cached2 values (1, 'bar', 'abcd1', 'efgh');
insert into vtocc_cached2 values (2, 'foo', 'abcd2', 'efgh');
insert into vtocc_cached2 values (2, 'bar', 'abcd2', 'efgh');

create table vtocc_big(id int, string1 varchar(128), string2 varchar(100), string3 char(1), string4 varchar(50), string5 varchar(50), date1 date, string6 varchar(16), string7 varchar(120), bigint1 bigint(20), bigint2 bigint(20), date2 date, integer1 int, tinyint1 tinyint(4), primary key(id)) comment 'vtocc_big';

create table vtocc_ints(tiny tinyint, tinyu tinyint unsigned, small smallint, smallu smallint unsigned, medium mediumint, mediumu mediumint unsigned, normal int, normalu int unsigned, big bigint, bigu bigint unsigned, y year, primary key(tiny)) comment 'vtocc_nocache';
create table vtocc_fracts(id int, deci decimal(5,2), num numeric(5,2), f float, d double, primary key(id)) comment 'vtocc_nocache';
create table vtocc_strings(vb varbinary(16), c char(16), vc varchar(16), b binary(4), tb tinyblob, bl blob, ttx tinytext, tx text, en enum('a','b'), s set('a','b'), primary key(vb)) comment 'vtocc_nocache';
create table vtocc_misc(id int, b bit(8), d date, dt datetime, t time, primary key(id)) comment 'vtocc_nocache';

create table vtocc_part1(key1 bigint, key2 bigint, data1 int, primary key(key1, key2));
create unique index vtocc_key2 on vtocc_part1(key2);
create table vtocc_part2(key3 bigint, data2 int, primary key(key3));
create view vtocc_view as select key2, key1, data1, data2 from vtocc_part1, vtocc_part2 where key2=key3;
delete from vtocc_part1;
delete from vtocc_part2;
insert into vtocc_part1 values(10, 1, 1);
insert into vtocc_part1 values(10, 2, 2);
insert into vtocc_part2 values(1, 3);
insert into vtocc_part2 values(2, 4);

create table vtocc_acl_no_access(key1 bigint, key2 bigint, primary key(key1));
create table vtocc_acl_read_only(key1 bigint, key2 bigint, primary key(key1));
create table vtocc_acl_read_write(key1 bigint, key2 bigint, primary key(key1));
create table vtocc_acl_admin(key1 bigint, key2 bigint, primary key(key1));
create table vtocc_acl_unmatched(key1 bigint, key2 bigint, primary key(key1));
create table vtocc_acl_all_user_read_only(key1 bigint, key2 bigint, primary key(key1));`
<|endoftext|>


<commit_before>package moves

import (
	"errors"

	"github.com/jkomoros/boardgame"
	"github.com/jkomoros/boardgame/moves/interfaces"
)

//SeatPlayer is a game that seats a new player into an open seat in the game. It
//is a special interface point for the server library to interact with your game
//logic. The core engine has no notion of whether or not a real user is
//associated with any given player slot. The server package does distinguish
//this, keeping track of which player slots need to be filled by real users. But
//by default your core game logic can't detect which player slots haven't been
//filled, or when they are filled. SeatPlayer, when used in conjunction with
//behaviors.Seat, introduces the notion of a Seat to each player slot. Those
//properties communicate whether the seat is filled with a physical player, and
//whether it is open to having a player sit in it. SeatPlayer is a special type
//of move that will be proposed by the server engine when it has a player
//waiting to be seated. Your core game logic can decide when it should be legal
//based on which phases it is configured to be legal in. If you do not
//explicitly configure SeatPlayer (or a move that derives from it) in your game
//then the server will not alert you when a player has been seated.
//
//You may use this move directly, or embed it in a move of your own that
//overrides some logic, like for example DefaultsForState to override where the
//next player is seated.
//
//If you don't want a seat to have players seated in it, even if it's not yet
//filled, then you can call SetSeatClosed() method on the player state. The move
//CloseEmptySeats will automatically mark all currently unfilled seats as
//closed, so no new players will be accepted.
//
//It is NOT a FixUp move; it is a special move that is only every proposed by
//the server itself.
//
//boardgame:codegen
type SeatPlayer struct {
	Default
	TargetPlayerIndex boardgame.PlayerIndex
}

//DefaultsForState sets TargetPlayerIndex to the next player who is neither
//filled nor closed.
func (s *SeatPlayer) DefaultsForState(state boardgame.ImmutableState) {
	var index int
	for index = 0; index < len(state.ImmutablePlayerStates()); index++ {
		player := state.ImmutablePlayerStates()[index]
		if seat, ok := player.(interfaces.Seater); ok {
			if seat.SeatIsClosed() || seat.SeatIsFilled() {
				continue
			}
			s.TargetPlayerIndex = boardgame.PlayerIndex(index)
			return
		}
	}
}

//Legal verifies that TargetPlayerIndex is set to a player who is both not
//filled and not closed, and that the proposer is the admin, since only server
//should propose this move.
func (s *SeatPlayer) Legal(state boardgame.ImmutableState, proposer boardgame.PlayerIndex) error {
	if err := s.Default.Legal(state, proposer); err != nil {
		return err
	}
	if proposer != boardgame.AdminPlayerIndex {
		return errors.New("This move may only be proposed by an admin")
	}
	if s.TargetPlayerIndex < 0 || int(s.TargetPlayerIndex) >= len(state.ImmutablePlayerStates()) {
		return errors.New("TargetPlayerIndex is invalid")
	}
	seat, ok := state.ImmutablePlayerStates()[s.TargetPlayerIndex].(interfaces.Seater)
	if !ok {
		return errors.New("The selected player did not implement interfaces.Seater")
	}
	if seat.SeatIsClosed() {
		return errors.New("The selected seat was closed")
	}
	if seat.SeatIsFilled() {
		return errors.New("The selected seat was already filled")
	}
	return nil
}

//Apply sets the targeted player to be Filled.
func (s *SeatPlayer) Apply(state boardgame.State) error {
	player := state.ImmutablePlayerStates()[s.TargetPlayerIndex]
	seat, ok := player.(interfaces.Seater)
	if !ok {
		return errors.New("Player state didn't implement interfaces.Seater")
	}
	seat.SetSeatFilled()
	return nil
}

//ValidConfiguration checks that player states implement interfaces.Seater
func (s *SeatPlayer) ValidConfiguration(exampleState boardgame.State) error {
	player := exampleState.ImmutablePlayerStates()[0]
	_, ok := player.(interfaces.Seater)
	if !ok {
		return errors.New("Player state didn't implement interfaces.Seater. behaviors.Seat implements it for free")
	}
	return nil
}

//FallbackHelpText returns "Marks the next available seat as seated, which when
//done will mean the next player is part of the game"
func (s *SeatPlayer) FallbackHelpText() string {
	return "Marks the next available seat as seated, which when done will mean the next player is part of the game"
}

//FallbackName returns "Seat Player"
func (s *SeatPlayer) FallbackName() string {
	return "Seat Player"
}
<commit_msg>Make it so SeatPlayer.Apply also marks the seat as inactive if it supports that behavior. Part of #755.<commit_after>package moves

import (
	"errors"

	"github.com/jkomoros/boardgame"
	"github.com/jkomoros/boardgame/moves/interfaces"
)

//SeatPlayer is a game that seats a new player into an open seat in the game. It
//is a special interface point for the server library to interact with your game
//logic. The core engine has no notion of whether or not a real user is
//associated with any given player slot. The server package does distinguish
//this, keeping track of which player slots need to be filled by real users. But
//by default your core game logic can't detect which player slots haven't been
//filled, or when they are filled. SeatPlayer, when used in conjunction with
//behaviors.Seat, introduces the notion of a Seat to each player slot. Those
//properties communicate whether the seat is filled with a physical player, and
//whether it is open to having a player sit in it. SeatPlayer is a special type
//of move that will be proposed by the server engine when it has a player
//waiting to be seated. Your core game logic can decide when it should be legal
//based on which phases it is configured to be legal in. If you do not
//explicitly configure SeatPlayer (or a move that derives from it) in your game
//then the server will not alert you when a player has been seated.
//
//You may use this move directly, or embed it in a move of your own that
//overrides some logic, like for example DefaultsForState to override where the
//next player is seated.
//
//If you don't want a seat to have players seated in it, even if it's not yet
//filled, then you can call SetSeatClosed() method on the player state. The move
//CloseEmptySeats will automatically mark all currently unfilled seats as
//closed, so no new players will be accepted.
//
//It is NOT a FixUp move; it is a special move that is only every proposed by
//the server itself.
//
//boardgame:codegen
type SeatPlayer struct {
	Default
	TargetPlayerIndex boardgame.PlayerIndex
}

//DefaultsForState sets TargetPlayerIndex to the next player who is neither
//filled nor closed.
func (s *SeatPlayer) DefaultsForState(state boardgame.ImmutableState) {
	var index int
	for index = 0; index < len(state.ImmutablePlayerStates()); index++ {
		player := state.ImmutablePlayerStates()[index]
		if seat, ok := player.(interfaces.Seater); ok {
			if seat.SeatIsClosed() || seat.SeatIsFilled() {
				continue
			}
			s.TargetPlayerIndex = boardgame.PlayerIndex(index)
			return
		}
	}
}

//Legal verifies that TargetPlayerIndex is set to a player who is both not
//filled and not closed, and that the proposer is the admin, since only server
//should propose this move.
func (s *SeatPlayer) Legal(state boardgame.ImmutableState, proposer boardgame.PlayerIndex) error {
	if err := s.Default.Legal(state, proposer); err != nil {
		return err
	}
	if proposer != boardgame.AdminPlayerIndex {
		return errors.New("This move may only be proposed by an admin")
	}
	if s.TargetPlayerIndex < 0 || int(s.TargetPlayerIndex) >= len(state.ImmutablePlayerStates()) {
		return errors.New("TargetPlayerIndex is invalid")
	}
	seat, ok := state.ImmutablePlayerStates()[s.TargetPlayerIndex].(interfaces.Seater)
	if !ok {
		return errors.New("The selected player did not implement interfaces.Seater")
	}
	if seat.SeatIsClosed() {
		return errors.New("The selected seat was closed")
	}
	if seat.SeatIsFilled() {
		return errors.New("The selected seat was already filled")
	}
	return nil
}

//Apply sets the targeted player to be Filled. If the player state also
//implements interfaces.Inactiver (for example because it implements
//behaviors.PlayerInactive), then it will also set the player to inactive. This
//is often the behavior you want; if you're in the middle of a round you
//typically don't want a new player to be active in the middle of it. But if you
//do use behaviors.PlayerInactive, remember to implement ActivateInactivePlayer
//at the beginning of rounds to activate any new seated players.
func (s *SeatPlayer) Apply(state boardgame.State) error {
	player := state.ImmutablePlayerStates()[s.TargetPlayerIndex]
	seat, ok := player.(interfaces.Seater)
	if !ok {
		return errors.New("Player state didn't implement interfaces.Seater")
	}
	seat.SetSeatFilled()
	if inactiver, ok := player.(interfaces.PlayerInactiver); ok {
		inactiver.SetPlayerInactive()
	}
	return nil
}

//ValidConfiguration checks that player states implement interfaces.Seater
func (s *SeatPlayer) ValidConfiguration(exampleState boardgame.State) error {
	player := exampleState.ImmutablePlayerStates()[0]
	_, ok := player.(interfaces.Seater)
	if !ok {
		return errors.New("Player state didn't implement interfaces.Seater. behaviors.Seat implements it for free")
	}
	return nil
}

//FallbackHelpText returns "Marks the next available seat as seated, which when
//done will mean the next player is part of the game"
func (s *SeatPlayer) FallbackHelpText() string {
	return "Marks the next available seat as seated, which when done will mean the next player is part of the game"
}

//FallbackName returns "Seat Player"
func (s *SeatPlayer) FallbackName() string {
	return "Seat Player"
}
<|endoftext|>


<commit_before>package vindexes

import (
	"bytes"
	"fmt"

	"golang.org/x/text/collate"
	"golang.org/x/text/language"
)

// UnicodeLooseMD5 is a vindex that normalizes and hashes unicode strings
// to a keyspace id. It conservatively converts the string to its base
// characters before hashing. This is also known as UCA level 1.
// Ref: http://www.unicode.org/reports/tr10/#Multi_Level_Comparison.
// This is compatible with MySQL's utf8_unicode_ci collation.
type UnicodeLooseMD5 struct {
	name string
}

// MewUnicodeLooseMD5 creates a new UnicodeLooseMD5.
func MewUnicodeLooseMD5(name string, _ map[string]string) (Vindex, error) {
	return &UnicodeLooseMD5{name: name}, nil
}

// String returns the name of the vindex.
func (vind *UnicodeLooseMD5) String() string {
	return vind.name
}

// Cost returns the cost as 1.
func (vind *UnicodeLooseMD5) Cost() int {
	return 1
}

// Verify returns true if id maps to ksid.
func (vind *UnicodeLooseMD5) Verify(_ VCursor, id interface{}, ksid []byte) (bool, error) {
	data, err := unicodeHash(id)
	if err != nil {
		return false, fmt.Errorf("UnicodeLooseMD5.Verify: %v", err)
	}
	return bytes.Compare(data, ksid) == 0, nil
}

// Map returns the corresponding keyspace id values for the given ids.
func (vind *UnicodeLooseMD5) Map(_ VCursor, ids []interface{}) ([][]byte, error) {
	out := make([][]byte, 0, len(ids))
	for _, id := range ids {
		data, err := unicodeHash(id)
		if err != nil {
			return nil, fmt.Errorf("UnicodeLooseMD5.Map :%v", err)
		}
		out = append(out, data)
	}
	return out, nil
}

func unicodeHash(key interface{}) ([]byte, error) {
	source, err := getBytes(key)
	if err != nil {
		return nil, err
	}
	return binHash(normalize(source)), nil
}

func normalize(in []byte) []byte {
	// Ref: http://dev.mysql.com/doc/refman/5.6/en/char.html.
	// Trailing spaces are ignored by MySQL.
	in = bytes.TrimRight(in, " ")

	// We use the collation key which can be used to
	// perform lexical comparisons.
	return normalizer.Key(new(collate.Buffer), in)
}

var normalizer *collate.Collator

func init() {
	// Ref: http://www.unicode.org/reports/tr10/#Introduction.
	// Unicode seems to define a universal (or default) order.
	// But various locales have conflicting order,
	// which they have the right to override.
	// Unfortunately, the Go library requires you to specify a locale.
	// So, I chose English assuming that it won't override
	// the Unicode universal order. But I couldn't find an easy
	// way to verify this.
	// Also, the locale differences are not an issue for level 1,
	// because the conservative comparison makes them all equal.
	normalizer = collate.New(language.English, collate.Loose)
	Register("unicode_loose_md5", MewUnicodeLooseMD5)
}
<commit_msg>vindexes: fix data race bug with collator<commit_after>package vindexes

import (
	"bytes"
	"fmt"

	"golang.org/x/text/collate"
	"golang.org/x/text/language"
)

// UnicodeLooseMD5 is a vindex that normalizes and hashes unicode strings
// to a keyspace id. It conservatively converts the string to its base
// characters before hashing. This is also known as UCA level 1.
// Ref: http://www.unicode.org/reports/tr10/#Multi_Level_Comparison.
// This is compatible with MySQL's utf8_unicode_ci collation.
type UnicodeLooseMD5 struct {
	name string
}

// MewUnicodeLooseMD5 creates a new UnicodeLooseMD5.
func MewUnicodeLooseMD5(name string, _ map[string]string) (Vindex, error) {
	return &UnicodeLooseMD5{name: name}, nil
}

// String returns the name of the vindex.
func (vind *UnicodeLooseMD5) String() string {
	return vind.name
}

// Cost returns the cost as 1.
func (vind *UnicodeLooseMD5) Cost() int {
	return 1
}

// Verify returns true if id maps to ksid.
func (vind *UnicodeLooseMD5) Verify(_ VCursor, id interface{}, ksid []byte) (bool, error) {
	data, err := unicodeHash(id)
	if err != nil {
		return false, fmt.Errorf("UnicodeLooseMD5.Verify: %v", err)
	}
	return bytes.Compare(data, ksid) == 0, nil
}

// Map returns the corresponding keyspace id values for the given ids.
func (vind *UnicodeLooseMD5) Map(_ VCursor, ids []interface{}) ([][]byte, error) {
	out := make([][]byte, 0, len(ids))
	for _, id := range ids {
		data, err := unicodeHash(id)
		if err != nil {
			return nil, fmt.Errorf("UnicodeLooseMD5.Map :%v", err)
		}
		out = append(out, data)
	}
	return out, nil
}

func unicodeHash(key interface{}) ([]byte, error) {
	source, err := getBytes(key)
	if err != nil {
		return nil, err
	}
	return binHash(normalize(source)), nil
}

func normalize(in []byte) []byte {
	// Ref: http://dev.mysql.com/doc/refman/5.6/en/char.html.
	// Trailing spaces are ignored by MySQL.
	in = bytes.TrimRight(in, " ")

	// Ref: http://www.unicode.org/reports/tr10/#Introduction.
	// Unicode seems to define a universal (or default) order.
	// But various locales have conflicting order,
	// which they have the right to override.
	// Unfortunately, the Go library requires you to specify a locale.
	// So, I chose English assuming that it won't override
	// the Unicode universal order. But I couldn't find an easy
	// way to verify this.
	// Also, the locale differences are not an issue for level 1,
	// because the conservative comparison makes them all equal.
	normalizer := collate.New(language.English, collate.Loose)

	// We use the collation key which can be used to
	// perform lexical comparisons.
	return normalizer.Key(new(collate.Buffer), in)
}

func init() {
	Register("unicode_loose_md5", MewUnicodeLooseMD5)
}
<|endoftext|>


<commit_before>package movingmedian

import (
	"github.com/wangjohn/quickselect"
	"math"
	"math/rand"
	"testing"
)

func TestMedian(t *testing.T) {
	var windowSize = 10
	data := getData(100)
	m := NewMovingMedian(windowSize)
	for i, v := range data {
		want := median(data, i, windowSize)

		m.Push(v)
		actual := m.Median()
		if want != actual {
			t.Errorf("median failed on index %v: item %v want %v actual %v", i, v, want, actual)
		}
	}
}

func BenchmarkMovingMedianOptimized(b *testing.B) {
	var windowSize = int(4e2)
	data := getData(int(16e4))

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		m := NewMovingMedian(windowSize)
		for _, v := range data {
			m.Push(v)
			m.Median()
		}
	}
}

func BenchmarkMovingMedian(b *testing.B) {
	var windowSize = int(4e2)
	data := getData(int(16e4))

	b.ResetTimer()

	for j := 0; j < b.N; j++ {
		for i := range data {
			median(data, i, windowSize)
		}
	}
}

func getData(rangeSize int) []float64 {
	var data = make([]float64, rangeSize)
	var r = rand.New(rand.NewSource(99))
	for i, _ := range data {
		data[i] = r.Float64()
	}

	return data
}

func median(data []float64, i, windowSize int) float64 {
	min := 1 + i - windowSize
	if min < 0 {
		min = 0
	}

	window := make([]float64, 1+i-min)
	copy(window, data[min:i+1])
	return percentile(window, 50, true)
}

func percentile(data []float64, percent float64, interpolate bool) float64 {
	if len(data) == 0 || percent < 0 || percent > 100 {
		return math.NaN()
	}
	if len(data) == 1 {
		return data[0]
	}

	k := (float64(len(data)-1) * percent) / 100
	length := int(math.Ceil(k)) + 1
	quickselect.Float64QuickSelect(data, length)
	top, secondTop := math.Inf(-1), math.Inf(-1)
	for _, val := range data[0:length] {
		if val > top {
			secondTop = top
			top = val
		} else if val > secondTop {
			secondTop = val
		}
	}
	remainder := k - float64(int(k))
	if remainder == 0 || !interpolate {
		return top
	}
	return (top * remainder) + (secondTop * (1 - remainder))
}
<commit_msg>benchmark multiple window sizes<commit_after>package movingmedian

import (
	"github.com/wangjohn/quickselect"
	"math"
	"math/rand"
	"testing"
)

func TestMedian(t *testing.T) {
	var windowSize = 10
	data := getData(100)
	m := NewMovingMedian(windowSize)
	for i, v := range data {
		want := median(data, i, windowSize)

		m.Push(v)
		actual := m.Median()
		if want != actual {
			t.Errorf("median failed on index %v: item %v want %v actual %v", i, v, want, actual)
		}
	}
}

func Benchmark_10values_windowsize1(b *testing.B) {
	benchmark(b, 10, 1)
}

func Benchmark_100values_windowsize10(b *testing.B) {
	benchmark(b, 100, 10)
}

func Benchmark_10Kvalues_windowsize100(b *testing.B) {
	benchmark(b, 10000, 100)
}

func Benchmark_10Kvalues_windowsize1000(b *testing.B) {
	benchmark(b, 10000, 1000)
}

func benchmark(b *testing.B, numberOfValues, windowSize int) {
	data := getData(numberOfValues)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		m := NewMovingMedian(windowSize)
		for _, v := range data {
			m.Push(v)
			m.Median()
		}
	}
}

func getData(rangeSize int) []float64 {
	var data = make([]float64, rangeSize)
	var r = rand.New(rand.NewSource(99))
	for i, _ := range data {
		data[i] = r.Float64()
	}

	return data
}

func median(data []float64, i, windowSize int) float64 {
	min := 1 + i - windowSize
	if min < 0 {
		min = 0
	}

	window := make([]float64, 1+i-min)
	copy(window, data[min:i+1])
	return percentile(window, 50, true)
}

func percentile(data []float64, percent float64, interpolate bool) float64 {
	if len(data) == 0 || percent < 0 || percent > 100 {
		return math.NaN()
	}
	if len(data) == 1 {
		return data[0]
	}

	k := (float64(len(data)-1) * percent) / 100
	length := int(math.Ceil(k)) + 1
	quickselect.Float64QuickSelect(data, length)
	top, secondTop := math.Inf(-1), math.Inf(-1)
	for _, val := range data[0:length] {
		if val > top {
			secondTop = top
			top = val
		} else if val > secondTop {
			secondTop = val
		}
	}
	remainder := k - float64(int(k))
	if remainder == 0 || !interpolate {
		return top
	}
	return (top * remainder) + (secondTop * (1 - remainder))
}
<|endoftext|>


<commit_before>package report

import (
	"encoding/csv"
	"os"

	"github.com/luistm/banksaurus/app"
	infraCSV "github.com/luistm/banksaurus/infrastructure/csv"
	"github.com/luistm/banksaurus/infrastructure/sqlite"
	"github.com/luistm/banksaurus/lib/seller"
	"github.com/luistm/banksaurus/lib/transaction"
	"github.com/luistm/banksaurus/next/adapter/CGDcsv"
	"github.com/luistm/banksaurus/services"
	"github.com/luistm/banksaurus/services/reportgrouped"
)

// Command handles reports
type Command struct{}

// Execute the report command
func (rc *Command) Execute(arguments map[string]interface{}) error {
	var grouped bool

	if arguments["--grouped"].(bool) {
		grouped = true
	}

	if grouped {
		CSVStorage, err := infraCSV.New(arguments["<file>"].(string))
		if err != nil {
			return err
		}
		defer CSVStorage.Close()

		dbName, dbPath := app.DatabasePath()
		SQLStorage, err := sqlite.New(dbPath, dbName, false)
		if err != nil {
			return err
		}
		defer SQLStorage.Close()

		transactionRepository := transaction.NewRepository(CSVStorage, SQLStorage)
		sellersRepository := seller.NewRepository(SQLStorage)
		presenter := NewPresenter(os.Stdout)

		var rfr services.Servicer

		rfr, err = reportgrouped.New(transactionRepository, sellersRepository, presenter)
		if err != nil {
			return err
		}

		if err := rfr.Execute(); err != nil {
			return nil
		}

	} else {
		// rfr, err = report.New(transactionRepository, sellersRepository, presenter)
		filePath := arguments["<file>"].(string)
		_, err := os.Stat(filePath)
		if err != nil {
			return err
		}

		file, err := os.Open(filePath)
		if err != nil {
			return err
		}
		defer file.Close()

		reader := csv.NewReader(file)
		reader.Comma = ';'
		reader.FieldsPerRecord = -1

		lines, err := reader.ReadAll()
		if err != nil {
			return err
		}

		_, err = CGDcsv.New(lines)
		if err != nil {
			return err
		}

	}

	return nil
}
<commit_msg>Report calls the 'next' interactor<commit_after>package report

import (
	"encoding/csv"
	"os"

	"github.com/luistm/banksaurus/app"
	infraCSV "github.com/luistm/banksaurus/infrastructure/csv"
	"github.com/luistm/banksaurus/infrastructure/sqlite"
	"github.com/luistm/banksaurus/lib/seller"
	"github.com/luistm/banksaurus/lib/transaction"
	"github.com/luistm/banksaurus/next/adapter/CGDcsv"
	"github.com/luistm/banksaurus/next/adapter/transactionpresenter"
	"github.com/luistm/banksaurus/next/report"
	"github.com/luistm/banksaurus/services"
	"github.com/luistm/banksaurus/services/reportgrouped"
)

// Command handles reports
type Command struct{}

// Execute the report command
func (rc *Command) Execute(arguments map[string]interface{}) error {
	var grouped bool

	if arguments["--grouped"].(bool) {
		grouped = true
	}

	if grouped {
		CSVStorage, err := infraCSV.New(arguments["<file>"].(string))
		if err != nil {
			return err
		}
		defer CSVStorage.Close()

		dbName, dbPath := app.DatabasePath()
		SQLStorage, err := sqlite.New(dbPath, dbName, false)
		if err != nil {
			return err
		}
		defer SQLStorage.Close()

		transactionRepository := transaction.NewRepository(CSVStorage, SQLStorage)
		sellersRepository := seller.NewRepository(SQLStorage)
		presenter := NewPresenter(os.Stdout)

		var rfr services.Servicer

		rfr, err = reportgrouped.New(transactionRepository, sellersRepository, presenter)
		if err != nil {
			return err
		}

		if err := rfr.Execute(); err != nil {
			return nil
		}

	} else {
		filePath := arguments["<file>"].(string)
		_, err := os.Stat(filePath)
		if err != nil {
			return err
		}

		file, err := os.Open(filePath)
		if err != nil {
			return err
		}
		defer file.Close()

		reader := csv.NewReader(file)
		reader.Comma = ';'
		reader.FieldsPerRecord = -1

		lines, err := reader.ReadAll()
		if err != nil {
			return err
		}

		inputGateway, err := CGDcsv.New(lines)
		if err != nil {
			return err
		}

		p, err := transactionpresenter.NewPresenter()
		if err != nil {
			return err
		}

		i, err := report.NewInteractor(p, inputGateway)
		if err != nil {
			return err
		}

		r, _ := report.NewRequest()
		err = i.Execute(r)
		if err != nil {
			return err
		}

		vm, err := p.ViewModel()
		if err != nil {
			return err
		}

		vm.Write(os.Stdout)
	}

	return nil
}
<|endoftext|>


<commit_before>package influx

import (
	"reflect"
	"testing"

	"github.com/influxdata/chronograf"
)

func TestConvert(t *testing.T) {
	tests := []struct {
		name     string
		influxQL string
		RawText  string
		want     chronograf.QueryConfig
		wantErr  bool
	}{
		{
			name:     "Test named count field",
			influxQL: `SELECT moving_average(mean("count"),14) FROM "usage_computed"."autogen".unique_clusters_by_day WHERE time > now() - 90d AND product = 'influxdb' group by time(1d)`,
			RawText:  `SELECT moving_average(mean("count"),14) FROM "usage_computed"."autogen".unique_clusters_by_day WHERE time > now() - 90d AND product = 'influxdb' group by time(1d)`,
			want: chronograf.QueryConfig{
				Fields: []chronograf.Field{},
				Tags:   map[string][]string{},
				GroupBy: chronograf.GroupBy{
					Tags: []string{},
				},
			},
		},
		{
			name:     "Test math",
			influxQL: `SELECT count("event_id")/3 as "event_count_id" from discource.autogen.discourse_events where time > now() - 7d group by time(1d), "event_type"`,
			RawText:  `SELECT count("event_id")/3 as "event_count_id" from discource.autogen.discourse_events where time > now() - 7d group by time(1d), "event_type"`,
			want: chronograf.QueryConfig{
				Fields: []chronograf.Field{},
				Tags:   map[string][]string{},
				GroupBy: chronograf.GroupBy{
					Tags: []string{},
				},
			},
		},
		{
			name:     "Test range",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where "host" != 'myhost' and time > now() - 15m`,
			want: chronograf.QueryConfig{
				Database:        "telegraf",
				Measurement:     "cpu",
				RetentionPolicy: "autogen",
				Fields: []chronograf.Field{
					chronograf.Field{
						Field: "usage_user",
						Funcs: []string{},
					},
				},
				Tags: map[string][]string{"host": []string{"myhost"}},
				GroupBy: chronograf.GroupBy{
					Time: "",
					Tags: []string{},
				},
				AreTagsAccepted: false,
				Range: &chronograf.DurationRange{
					Lower: "15m",
					Upper: "now",
				},
			},
		},
		{
			name:     "Test invalid range",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where "host" != 'myhost' and time > now() - 15`,
			RawText:  `SELECT usage_user from telegraf.autogen.cpu where "host" != 'myhost' and time > now() - 15`,
			want: chronograf.QueryConfig{
				Fields: []chronograf.Field{},
				Tags:   map[string][]string{},
				GroupBy: chronograf.GroupBy{
					Tags: []string{},
				},
			},
		},
		{
			name:     "Test range with no duration",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where "host" != 'myhost' and time > now()`,
			want: chronograf.QueryConfig{
				Database:        "telegraf",
				Measurement:     "cpu",
				RetentionPolicy: "autogen",
				Fields: []chronograf.Field{
					chronograf.Field{
						Field: "usage_user",
						Funcs: []string{},
					},
				},
				Tags: map[string][]string{"host": []string{"myhost"}},
				GroupBy: chronograf.GroupBy{
					Time: "",
					Tags: []string{},
				},
				AreTagsAccepted: false,
				Range: &chronograf.DurationRange{
					Lower: "0s",
					Upper: "now",
				},
			},
		},
		{
			name:     "Test range with no tags",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where time > now() - 15m`,
			want: chronograf.QueryConfig{
				Database:        "telegraf",
				Measurement:     "cpu",
				RetentionPolicy: "autogen",
				Tags:            map[string][]string{},
				Fields: []chronograf.Field{
					chronograf.Field{
						Field: "usage_user",
						Funcs: []string{},
					},
				},
				GroupBy: chronograf.GroupBy{
					Time: "",
					Tags: []string{},
				},
				AreTagsAccepted: false,
				Range: &chronograf.DurationRange{
					Lower: "15m",
					Upper: "now",
				},
			},
		},
		{
			name:     "Test range with no tags nor duration",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where time`,
			RawText:  `SELECT usage_user from telegraf.autogen.cpu where time`,
			want: chronograf.QueryConfig{
				Fields: []chronograf.Field{},
				Tags:   map[string][]string{},
				GroupBy: chronograf.GroupBy{
					Tags: []string{},
				},
			},
		},
		{
			name:     "Test with no time range",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where "host" != 'myhost' and time`,
			RawText:  `SELECT usage_user from telegraf.autogen.cpu where "host" != 'myhost' and time`,
			want: chronograf.QueryConfig{
				Fields: []chronograf.Field{},
				Tags:   map[string][]string{},
				GroupBy: chronograf.GroupBy{
					Tags: []string{},
				},
			},
		},
		{
			name:     "Test tags accepted",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where "host" = 'myhost' and time > now() - 15m`,
			want: chronograf.QueryConfig{
				Database:        "telegraf",
				Measurement:     "cpu",
				RetentionPolicy: "autogen",
				Fields: []chronograf.Field{
					chronograf.Field{
						Field: "usage_user",
						Funcs: []string{},
					},
				},
				Tags: map[string][]string{"host": []string{"myhost"}},
				GroupBy: chronograf.GroupBy{
					Time: "",
					Tags: []string{},
				},
				AreTagsAccepted: true,
				Range: &chronograf.DurationRange{
					Lower: "15m",
					Upper: "now",
				},
			},
		},
		{
			name:     "Test mixed tag logic",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where ("host" = 'myhost' or "this" = 'those') and ("howdy" != 'doody') and time > now() - 15m`,
			RawText:  `SELECT usage_user from telegraf.autogen.cpu where ("host" = 'myhost' or "this" = 'those') and ("howdy" != 'doody') and time > now() - 15m`,
			want: chronograf.QueryConfig{
				Fields: []chronograf.Field{},
				Tags:   map[string][]string{},
				GroupBy: chronograf.GroupBy{
					Tags: []string{},
				},
			},
		},
		{
			name:     "Test tags accepted",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where ("host" = 'myhost' OR "host" = 'yourhost') and ("these" = 'those') and time > now() - 15m`,
			want: chronograf.QueryConfig{
				Database:        "telegraf",
				Measurement:     "cpu",
				RetentionPolicy: "autogen",
				Fields: []chronograf.Field{
					chronograf.Field{
						Field: "usage_user",
						Funcs: []string{},
					},
				},
				Tags: map[string][]string{
					"host":  []string{"myhost", "yourhost"},
					"these": []string{"those"},
				},
				GroupBy: chronograf.GroupBy{
					Time: "",
					Tags: []string{},
				},
				AreTagsAccepted: true,
				Range: &chronograf.DurationRange{
					Lower: "15m",
					Upper: "now",
				},
			},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got, err := Convert(tt.influxQL)
			if (err != nil) != tt.wantErr {
				t.Errorf("Convert() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if tt.RawText != "" {
				tt.want.RawText = &tt.RawText
				if got.RawText == nil {
					t.Errorf("Convert() = nil, want %s", tt.RawText)
				} else if *got.RawText != tt.RawText {
					t.Errorf("Convert() = %s, want %s", *got.RawText, tt.RawText)
				}
			}
			if !reflect.DeepEqual(got, tt.want) {
				t.Errorf("Convert() = %#v, want %#v", got, tt.want)
			}
		})
	}
}
<commit_msg>Add test to influxql AST parsing of durations with no WHERE clause<commit_after>package influx

import (
	"reflect"
	"testing"

	"github.com/influxdata/chronograf"
)

func TestConvert(t *testing.T) {
	tests := []struct {
		name     string
		influxQL string
		RawText  string
		want     chronograf.QueryConfig
		wantErr  bool
	}{
		{
			name:     "Test named count field",
			influxQL: `SELECT moving_average(mean("count"),14) FROM "usage_computed"."autogen".unique_clusters_by_day WHERE time > now() - 90d AND product = 'influxdb' group by time(1d)`,
			RawText:  `SELECT moving_average(mean("count"),14) FROM "usage_computed"."autogen".unique_clusters_by_day WHERE time > now() - 90d AND product = 'influxdb' group by time(1d)`,
			want: chronograf.QueryConfig{
				Fields: []chronograf.Field{},
				Tags:   map[string][]string{},
				GroupBy: chronograf.GroupBy{
					Tags: []string{},
				},
			},
		},
		{
			name:     "Test math",
			influxQL: `SELECT count("event_id")/3 as "event_count_id" from discource.autogen.discourse_events where time > now() - 7d group by time(1d), "event_type"`,
			RawText:  `SELECT count("event_id")/3 as "event_count_id" from discource.autogen.discourse_events where time > now() - 7d group by time(1d), "event_type"`,
			want: chronograf.QueryConfig{
				Fields: []chronograf.Field{},
				Tags:   map[string][]string{},
				GroupBy: chronograf.GroupBy{
					Tags: []string{},
				},
			},
		},
		{
			name:     "Test range",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where "host" != 'myhost' and time > now() - 15m`,
			want: chronograf.QueryConfig{
				Database:        "telegraf",
				Measurement:     "cpu",
				RetentionPolicy: "autogen",
				Fields: []chronograf.Field{
					chronograf.Field{
						Field: "usage_user",
						Funcs: []string{},
					},
				},
				Tags: map[string][]string{"host": []string{"myhost"}},
				GroupBy: chronograf.GroupBy{
					Time: "",
					Tags: []string{},
				},
				AreTagsAccepted: false,
				Range: &chronograf.DurationRange{
					Lower: "15m",
					Upper: "now",
				},
			},
		},
		{
			name:     "Test invalid range",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where "host" != 'myhost' and time > now() - 15`,
			RawText:  `SELECT usage_user from telegraf.autogen.cpu where "host" != 'myhost' and time > now() - 15`,
			want: chronograf.QueryConfig{
				Fields: []chronograf.Field{},
				Tags:   map[string][]string{},
				GroupBy: chronograf.GroupBy{
					Tags: []string{},
				},
			},
		},
		{
			name:     "Test range with no duration",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where "host" != 'myhost' and time > now()`,
			want: chronograf.QueryConfig{
				Database:        "telegraf",
				Measurement:     "cpu",
				RetentionPolicy: "autogen",
				Fields: []chronograf.Field{
					chronograf.Field{
						Field: "usage_user",
						Funcs: []string{},
					},
				},
				Tags: map[string][]string{"host": []string{"myhost"}},
				GroupBy: chronograf.GroupBy{
					Time: "",
					Tags: []string{},
				},
				AreTagsAccepted: false,
				Range: &chronograf.DurationRange{
					Lower: "0s",
					Upper: "now",
				},
			},
		},
		{
			name:     "Test range with no tags",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where time > now() - 15m`,
			want: chronograf.QueryConfig{
				Database:        "telegraf",
				Measurement:     "cpu",
				RetentionPolicy: "autogen",
				Tags:            map[string][]string{},
				Fields: []chronograf.Field{
					chronograf.Field{
						Field: "usage_user",
						Funcs: []string{},
					},
				},
				GroupBy: chronograf.GroupBy{
					Time: "",
					Tags: []string{},
				},
				AreTagsAccepted: false,
				Range: &chronograf.DurationRange{
					Lower: "15m",
					Upper: "now",
				},
			},
		},
		{
			name:     "Test range with no tags nor duration",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where time`,
			RawText:  `SELECT usage_user from telegraf.autogen.cpu where time`,
			want: chronograf.QueryConfig{
				Fields: []chronograf.Field{},
				Tags:   map[string][]string{},
				GroupBy: chronograf.GroupBy{
					Tags: []string{},
				},
			},
		},
		{
			name:     "Test with no time range",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where "host" != 'myhost' and time`,
			RawText:  `SELECT usage_user from telegraf.autogen.cpu where "host" != 'myhost' and time`,
			want: chronograf.QueryConfig{
				Fields: []chronograf.Field{},
				Tags:   map[string][]string{},
				GroupBy: chronograf.GroupBy{
					Tags: []string{},
				},
			},
		},
		{
			name:     "Test with no where clauses",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu`,
			want: chronograf.QueryConfig{
				Database:        "telegraf",
				Measurement:     "cpu",
				RetentionPolicy: "autogen",
				Fields: []chronograf.Field{
					chronograf.Field{
						Field: "usage_user",
						Funcs: []string{},
					},
				},
				Tags: map[string][]string{},
				GroupBy: chronograf.GroupBy{
					Time: "",
					Tags: []string{},
				},
			},
		},
		{
			name:     "Test tags accepted",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where "host" = 'myhost' and time > now() - 15m`,
			want: chronograf.QueryConfig{
				Database:        "telegraf",
				Measurement:     "cpu",
				RetentionPolicy: "autogen",
				Fields: []chronograf.Field{
					chronograf.Field{
						Field: "usage_user",
						Funcs: []string{},
					},
				},
				Tags: map[string][]string{"host": []string{"myhost"}},
				GroupBy: chronograf.GroupBy{
					Time: "",
					Tags: []string{},
				},
				AreTagsAccepted: true,
				Range: &chronograf.DurationRange{
					Lower: "15m",
					Upper: "now",
				},
			},
		},
		{
			name:     "Test mixed tag logic",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where ("host" = 'myhost' or "this" = 'those') and ("howdy" != 'doody') and time > now() - 15m`,
			RawText:  `SELECT usage_user from telegraf.autogen.cpu where ("host" = 'myhost' or "this" = 'those') and ("howdy" != 'doody') and time > now() - 15m`,
			want: chronograf.QueryConfig{
				Fields: []chronograf.Field{},
				Tags:   map[string][]string{},
				GroupBy: chronograf.GroupBy{
					Tags: []string{},
				},
			},
		},
		{
			name:     "Test tags accepted",
			influxQL: `SELECT usage_user from telegraf.autogen.cpu where ("host" = 'myhost' OR "host" = 'yourhost') and ("these" = 'those') and time > now() - 15m`,
			want: chronograf.QueryConfig{
				Database:        "telegraf",
				Measurement:     "cpu",
				RetentionPolicy: "autogen",
				Fields: []chronograf.Field{
					chronograf.Field{
						Field: "usage_user",
						Funcs: []string{},
					},
				},
				Tags: map[string][]string{
					"host":  []string{"myhost", "yourhost"},
					"these": []string{"those"},
				},
				GroupBy: chronograf.GroupBy{
					Time: "",
					Tags: []string{},
				},
				AreTagsAccepted: true,
				Range: &chronograf.DurationRange{
					Lower: "15m",
					Upper: "now",
				},
			},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got, err := Convert(tt.influxQL)
			if (err != nil) != tt.wantErr {
				t.Errorf("Convert() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if tt.RawText != "" {
				tt.want.RawText = &tt.RawText
				if got.RawText == nil {
					t.Errorf("Convert() = nil, want %s", tt.RawText)
				} else if *got.RawText != tt.RawText {
					t.Errorf("Convert() = %s, want %s", *got.RawText, tt.RawText)
				}
			}
			if !reflect.DeepEqual(got, tt.want) {
				t.Errorf("Convert() = %#v, want %#v", got, tt.want)
			}
		})
	}
}
<|endoftext|>


<commit_before>package input

import (
	certstream "github.com/CaliDog/certstream-go"
	log "github.com/Sirupsen/logrus"
	"github.com/jmoiron/jsonq"
)

// CertStreamInput provides a stream of Certificate Transparency Logs https://www.certificate-transparency.org/
// It's used for example purposes only and should NOT be used in production
// Since the stream is a websocket slow processing would result in memory pressure
// Instead the data should be pushed into a queue, such as Kinesis, and GoFish should read from that stream
type CertStreamInput struct {
	stream chan jsonq.JsonQuery
}

func (c *CertStreamInput) Init() error {
	c.stream = certstream.CertStreamEventStream(false)
	return nil
}

func (c *CertStreamInput) Retrieve(output *chan []byte) {
	defer close(*output)
	for i := range c.stream {
		json, err := i.String()
		if err != nil {
			log.Errorf("Invalid data from Cert Stream: %s", err)
			continue
		}
		*output <- []byte(json)
	}
}
<commit_msg>Horrible hack, unmarshal the event so we can re-marshal it<commit_after>package input

import (
	"encoding/json"

	certstream "github.com/CaliDog/certstream-go"
	log "github.com/Sirupsen/logrus"
	"github.com/jmoiron/jsonq"
)

// CertStreamInput provides a stream of Certificate Transparency Logs https://www.certificate-transparency.org/
// It's used for example purposes only and should NOT be used in production
// Since the stream is a websocket slow processing would result in memory pressure
// Instead the data should be pushed into a queue, such as Kinesis, and GoFish should read from that stream
type CertStreamInput struct {
	stream chan jsonq.JsonQuery
}

func (c *CertStreamInput) Init() error {
	c.stream = certstream.CertStreamEventStream(false)
	return nil
}

func (c *CertStreamInput) Retrieve(output *chan []byte) {
	defer close(*output)
	for i := range c.stream {
		j, err := i.Object()
		if err != nil {
			log.Errorf("Invalid data from Cert Stream: %s", err)
			continue
		}
		data, err := json.Marshal(j)
		if err != nil {
			log.Errorf("Unable to Marshal Cert Stream: %s", err)
		}
		*output <- data
	}
}
<|endoftext|>


<commit_before>package restic

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/restic/restic/backend"
	"github.com/restic/restic/debug"
	"github.com/restic/restic/repository"
)

// Cache is used to locally cache items from a repository.
type Cache struct {
	base string
}

// NewCache returns a new cache at cacheDir. If it is the empty string, the
// default cache location is chosen.
func NewCache(repo *repository.Repository, cacheDir string) (*Cache, error) {
	var err error

	if cacheDir == "" {
		cacheDir, err = getCacheDir()
		if err != nil {
			return nil, err
		}
	}

	basedir := filepath.Join(cacheDir, repo.Config.ID)
	debug.Log("Cache.New", "opened cache at %v", basedir)

	return &Cache{base: basedir}, nil
}

// Has checks if the local cache has the id.
func (c *Cache) Has(t backend.Type, subtype string, id backend.ID) (bool, error) {
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return false, err
	}
	fd, err := os.Open(filename)
	defer fd.Close()

	if err != nil {
		if os.IsNotExist(err) {
			debug.Log("Cache.Has", "test for file %v: not cached", filename)
			return false, nil
		}

		debug.Log("Cache.Has", "test for file %v: error %v", filename, err)
		return false, err
	}

	debug.Log("Cache.Has", "test for file %v: is cached", filename)
	return true, nil
}

// Store returns an io.WriteCloser that is used to save new information to the
// cache. The returned io.WriteCloser must be closed by the caller after all
// data has been written.
func (c *Cache) Store(t backend.Type, subtype string, id backend.ID) (io.WriteCloser, error) {
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return nil, err
	}

	dirname := filepath.Dir(filename)
	err = os.MkdirAll(dirname, 0700)
	if err != nil {
		return nil, err
	}

	file, err := os.Create(filename)
	if err != nil {
		debug.Log("Cache.Store", "error creating file %v: %v", filename, err)
		return nil, err
	}

	debug.Log("Cache.Store", "created file %v", filename)
	return file, nil
}

// Load returns information from the cache. The returned io.ReadCloser must be
// closed by the caller.
func (c *Cache) Load(t backend.Type, subtype string, id backend.ID) (io.ReadCloser, error) {
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return nil, err
	}

	return os.Open(filename)
}

func (c *Cache) purge(t backend.Type, subtype string, id backend.ID) error {
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return err
	}

	err = os.Remove(filename)
	debug.Log("Cache.purge", "Remove file %v: %v", filename, err)

	if err != nil && os.IsNotExist(err) {
		return nil
	}

	return err
}

// Clear removes information from the cache that isn't present in the repository any more.
func (c *Cache) Clear(repo *repository.Repository) error {
	list, err := c.list(backend.Snapshot)
	if err != nil {
		return err
	}

	for _, entry := range list {
		debug.Log("Cache.Clear", "found entry %v", entry)

		if ok, err := repo.Backend().Test(backend.Snapshot, entry.ID.String()); !ok || err != nil {
			debug.Log("Cache.Clear", "snapshot %v doesn't exist any more, removing %v", entry.ID, entry)

			err = c.purge(backend.Snapshot, entry.Subtype, entry.ID)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

type cacheEntry struct {
	ID      backend.ID
	Subtype string
}

func (c cacheEntry) String() string {
	if c.Subtype != "" {
		return c.ID.Str() + "." + c.Subtype
	}
	return c.ID.Str()
}

func (c *Cache) list(t backend.Type) ([]cacheEntry, error) {
	var dir string

	switch t {
	case backend.Snapshot:
		dir = filepath.Join(c.base, "snapshots")
	default:
		return nil, fmt.Errorf("cache not supported for type %v", t)
	}

	fd, err := os.Open(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return []cacheEntry{}, nil
		}
		return nil, err
	}
	defer fd.Close()

	fis, err := fd.Readdir(-1)
	if err != nil {
		return nil, err
	}

	entries := make([]cacheEntry, 0, len(fis))

	for _, fi := range fis {
		parts := strings.SplitN(fi.Name(), ".", 2)

		id, err := backend.ParseID(parts[0])
		// ignore invalid cache entries for now
		if err != nil {
			debug.Log("Cache.List", "unable to parse name %v as id: %v", parts[0], err)
			continue
		}

		e := cacheEntry{ID: id}

		if len(parts) == 2 {
			e.Subtype = parts[1]
		}

		entries = append(entries, e)
	}

	return entries, nil
}

func (c *Cache) filename(t backend.Type, subtype string, id backend.ID) (string, error) {
	filename := id.String()
	if subtype != "" {
		filename += "." + subtype
	}

	switch t {
	case backend.Snapshot:
		return filepath.Join(c.base, "snapshots", filename), nil
	}

	return "", fmt.Errorf("cache not supported for type %v", t)
}

func getCacheDir() (string, error) {
	if dir := os.Getenv("RESTIC_CACHE"); dir != "" {
		return dir, nil
	}
	if runtime.GOOS == "windows" {
		return getWindowsCacheDir()
	}

	return getXDGCacheDir()
}

// getWindowsCacheDir will return %APPDATA%\restic or create
// a folder in the temporary folder called "restic".
func getWindowsCacheDir() (string, error) {
	cachedir := os.Getenv("APPDATA")
	if cachedir == "" {
		cachedir = os.TempDir()
	}
	cachedir = filepath.Join(cachedir, "restic")
	fi, err := os.Stat(cachedir)

	if os.IsNotExist(err) {
		err = os.MkdirAll(cachedir, 0700)
		if err != nil {
			return "", err
		}
	}

	if err != nil {
		return "", err
	}

	if !fi.IsDir() {
		return "", fmt.Errorf("cache dir %v is not a directory", cachedir)
	}
	return cachedir, nil
}

// getXDGCacheDir returns the cache directory according to XDG basedir spec, see
// http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
func getXDGCacheDir() (string, error) {
	xdgcache := os.Getenv("XDG_CACHE_HOME")
	home := os.Getenv("HOME")

	if xdgcache == "" && home == "" {
		return "", errors.New("unable to locate cache directory (XDG_CACHE_HOME and HOME unset)")
	}

	cachedir := ""
	if xdgcache != "" {
		cachedir = filepath.Join(xdgcache, "restic")
	} else if home != "" {
		cachedir = filepath.Join(home, ".cache", "restic")
	}

	fi, err := os.Stat(cachedir)
	if os.IsNotExist(err) {
		err = os.MkdirAll(cachedir, 0700)
		if err != nil {
			return "", err
		}

		fi, err = os.Stat(cachedir)
		debug.Log("getCacheDir", "create cache dir %v", cachedir)
	}

	if err != nil {
		return "", err
	}

	if !fi.IsDir() {
		return "", fmt.Errorf("cache dir %v is not a directory", cachedir)
	}

	return cachedir, nil
}
<commit_msg>windows: fix nil pointer reference<commit_after>package restic

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/restic/restic/backend"
	"github.com/restic/restic/debug"
	"github.com/restic/restic/repository"
)

// Cache is used to locally cache items from a repository.
type Cache struct {
	base string
}

// NewCache returns a new cache at cacheDir. If it is the empty string, the
// default cache location is chosen.
func NewCache(repo *repository.Repository, cacheDir string) (*Cache, error) {
	var err error

	if cacheDir == "" {
		cacheDir, err = getCacheDir()
		if err != nil {
			return nil, err
		}
	}

	basedir := filepath.Join(cacheDir, repo.Config.ID)
	debug.Log("Cache.New", "opened cache at %v", basedir)

	return &Cache{base: basedir}, nil
}

// Has checks if the local cache has the id.
func (c *Cache) Has(t backend.Type, subtype string, id backend.ID) (bool, error) {
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return false, err
	}
	fd, err := os.Open(filename)
	defer fd.Close()

	if err != nil {
		if os.IsNotExist(err) {
			debug.Log("Cache.Has", "test for file %v: not cached", filename)
			return false, nil
		}

		debug.Log("Cache.Has", "test for file %v: error %v", filename, err)
		return false, err
	}

	debug.Log("Cache.Has", "test for file %v: is cached", filename)
	return true, nil
}

// Store returns an io.WriteCloser that is used to save new information to the
// cache. The returned io.WriteCloser must be closed by the caller after all
// data has been written.
func (c *Cache) Store(t backend.Type, subtype string, id backend.ID) (io.WriteCloser, error) {
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return nil, err
	}

	dirname := filepath.Dir(filename)
	err = os.MkdirAll(dirname, 0700)
	if err != nil {
		return nil, err
	}

	file, err := os.Create(filename)
	if err != nil {
		debug.Log("Cache.Store", "error creating file %v: %v", filename, err)
		return nil, err
	}

	debug.Log("Cache.Store", "created file %v", filename)
	return file, nil
}

// Load returns information from the cache. The returned io.ReadCloser must be
// closed by the caller.
func (c *Cache) Load(t backend.Type, subtype string, id backend.ID) (io.ReadCloser, error) {
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return nil, err
	}

	return os.Open(filename)
}

func (c *Cache) purge(t backend.Type, subtype string, id backend.ID) error {
	filename, err := c.filename(t, subtype, id)
	if err != nil {
		return err
	}

	err = os.Remove(filename)
	debug.Log("Cache.purge", "Remove file %v: %v", filename, err)

	if err != nil && os.IsNotExist(err) {
		return nil
	}

	return err
}

// Clear removes information from the cache that isn't present in the repository any more.
func (c *Cache) Clear(repo *repository.Repository) error {
	list, err := c.list(backend.Snapshot)
	if err != nil {
		return err
	}

	for _, entry := range list {
		debug.Log("Cache.Clear", "found entry %v", entry)

		if ok, err := repo.Backend().Test(backend.Snapshot, entry.ID.String()); !ok || err != nil {
			debug.Log("Cache.Clear", "snapshot %v doesn't exist any more, removing %v", entry.ID, entry)

			err = c.purge(backend.Snapshot, entry.Subtype, entry.ID)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

type cacheEntry struct {
	ID      backend.ID
	Subtype string
}

func (c cacheEntry) String() string {
	if c.Subtype != "" {
		return c.ID.Str() + "." + c.Subtype
	}
	return c.ID.Str()
}

func (c *Cache) list(t backend.Type) ([]cacheEntry, error) {
	var dir string

	switch t {
	case backend.Snapshot:
		dir = filepath.Join(c.base, "snapshots")
	default:
		return nil, fmt.Errorf("cache not supported for type %v", t)
	}

	fd, err := os.Open(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return []cacheEntry{}, nil
		}
		return nil, err
	}
	defer fd.Close()

	fis, err := fd.Readdir(-1)
	if err != nil {
		return nil, err
	}

	entries := make([]cacheEntry, 0, len(fis))

	for _, fi := range fis {
		parts := strings.SplitN(fi.Name(), ".", 2)

		id, err := backend.ParseID(parts[0])
		// ignore invalid cache entries for now
		if err != nil {
			debug.Log("Cache.List", "unable to parse name %v as id: %v", parts[0], err)
			continue
		}

		e := cacheEntry{ID: id}

		if len(parts) == 2 {
			e.Subtype = parts[1]
		}

		entries = append(entries, e)
	}

	return entries, nil
}

func (c *Cache) filename(t backend.Type, subtype string, id backend.ID) (string, error) {
	filename := id.String()
	if subtype != "" {
		filename += "." + subtype
	}

	switch t {
	case backend.Snapshot:
		return filepath.Join(c.base, "snapshots", filename), nil
	}

	return "", fmt.Errorf("cache not supported for type %v", t)
}

func getCacheDir() (string, error) {
	if dir := os.Getenv("RESTIC_CACHE"); dir != "" {
		return dir, nil
	}
	if runtime.GOOS == "windows" {
		return getWindowsCacheDir()
	}

	return getXDGCacheDir()
}

// getWindowsCacheDir will return %APPDATA%\restic or create
// a folder in the temporary folder called "restic".
func getWindowsCacheDir() (string, error) {
	cachedir := os.Getenv("APPDATA")
	if cachedir == "" {
		cachedir = os.TempDir()
	}
	cachedir = filepath.Join(cachedir, "restic")
	fi, err := os.Stat(cachedir)

	if os.IsNotExist(err) {
		err = os.MkdirAll(cachedir, 0700)
		if err != nil {
			return "", err
		}

		return cachedir, nil
	}

	if err != nil {
		return "", err
	}

	if !fi.IsDir() {
		return "", fmt.Errorf("cache dir %v is not a directory", cachedir)
	}
	return cachedir, nil
}

// getXDGCacheDir returns the cache directory according to XDG basedir spec, see
// http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
func getXDGCacheDir() (string, error) {
	xdgcache := os.Getenv("XDG_CACHE_HOME")
	home := os.Getenv("HOME")

	if xdgcache == "" && home == "" {
		return "", errors.New("unable to locate cache directory (XDG_CACHE_HOME and HOME unset)")
	}

	cachedir := ""
	if xdgcache != "" {
		cachedir = filepath.Join(xdgcache, "restic")
	} else if home != "" {
		cachedir = filepath.Join(home, ".cache", "restic")
	}

	fi, err := os.Stat(cachedir)
	if os.IsNotExist(err) {
		err = os.MkdirAll(cachedir, 0700)
		if err != nil {
			return "", err
		}

		fi, err = os.Stat(cachedir)
		debug.Log("getCacheDir", "create cache dir %v", cachedir)
	}

	if err != nil {
		return "", err
	}

	if !fi.IsDir() {
		return "", fmt.Errorf("cache dir %v is not a directory", cachedir)
	}

	return cachedir, nil
}
<|endoftext|>


<commit_before>package cache // import "gopkg.in/go-redis/cache.v5"

import (
	"errors"
	"log"
	"sync/atomic"
	"time"

	"gopkg.in/go-redis/cache.v5/lrucache"

	"go4.org/syncutil/singleflight"
	"gopkg.in/redis.v5"
)

var ErrCacheMiss = errors.New("cache: key is missing")

type rediser interface {
	Set(key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	Get(key string) *redis.StringCmd
	Del(keys ...string) *redis.IntCmd
}

type Codec struct {
	Redis rediser

	// Local LRU cache for super hot items.
	LocalCache *lrucache.Cache

	Marshal   func(interface{}) ([]byte, error)
	Unmarshal func([]byte, interface{}) error

	group        singleflight.Group
	hits, misses int64
}

type Item struct {
	Key    string
	Object interface{}

	// Func returns object to cache.
	Func func() (interface{}, error)

	// Expiration is the cache expiration time.
	// Default expiration is 1 hour.
	Expiration time.Duration
}

func (item *Item) object() (interface{}, error) {
	if item.Object != nil {
		return item.Object, nil
	}
	if item.Func != nil {
		return item.Func()
	}
	return nil, nil
}

// Set caches the item.
func (cd *Codec) Set(item *Item) error {
	if item.Expiration >= 0 && item.Expiration < time.Second {
		item.Expiration = time.Hour
	} else if item.Expiration == -1 {
		item.Expiration = 0
	}

	object, err := item.object()
	if err != nil {
		return err
	}

	b, err := cd.Marshal(object)
	if err != nil {
		log.Printf("cache: Marshal failed: %s", err)
		return err
	}

	if cd.LocalCache != nil {
		cd.LocalCache.Set(item.Key, b)
	}

	err = cd.Redis.Set(item.Key, b, item.Expiration).Err()
	if err != nil {
		log.Printf("cache: Set key=%q failed: %s", item.Key, err)
	}
	return err
}

// Get gets the object for the given key.
func (cd *Codec) Get(key string, object interface{}) error {
	b, err := cd.getBytes(key)
	if err != nil {
		return err
	}

	if object == nil || len(b) == 0 {
		return nil
	}

	if err := cd.Unmarshal(b, object); err != nil {
		log.Printf("cache: Unmarshal(%v) failed: %s", object, err)
		return err
	}

	return nil
}

func (cd *Codec) getBytes(key string) ([]byte, error) {
	if cd.LocalCache != nil {
		v, ok := cd.LocalCache.Get(key)
		if ok {
			b, ok := v.([]byte)
			if ok {
				atomic.AddInt64(&cd.hits, 1)
				return b, nil
			}
		}
	}

	b, err := cd.Redis.Get(key).Bytes()
	if err != nil {
		atomic.AddInt64(&cd.misses, 1)
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		log.Printf("cache: Get key=%q failed: %s", key, err)
		return nil, err
	}

	if cd.LocalCache != nil {
		cd.LocalCache.Set(key, b)
	}
	return b, nil
}

// Do gets the item.Object for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Codec) Do(item *Item) (interface{}, error) {
	if err := cd.getItem(item); err == nil {
		return item.Object, nil
	}

	return cd.group.Do(item.Key, func() (interface{}, error) {
		if err := cd.getItem(item); err == nil {
			return item.Object, nil
		}

		obj, err := item.Func()
		if err != nil {
			return nil, err
		}

		item.Object = obj
		item.Func = nil
		cd.Set(item)

		return obj, nil
	})
}

func (cd *Codec) getItem(item *Item) error {
	if item.Object != nil {
		return cd.Get(item.Key, item.Object)
	} else {
		return cd.Get(item.Key, &item.Object)
	}
}

func (cd *Codec) Delete(key string) error {
	if cd.LocalCache != nil {
		cd.LocalCache.Delete(key)
	}

	deleted, err := cd.Redis.Del(key).Result()
	if err != nil {
		log.Printf("cache: Del key=%q failed: %s", key, err)
		return err
	}
	if deleted == 0 {
		return ErrCacheMiss
	}
	return nil
}

func (cd *Codec) Hits() int {
	return int(atomic.LoadInt64(&cd.hits))
}

func (cd *Codec) Misses() int {
	return int(atomic.LoadInt64(&cd.misses))
}
<commit_msg>Improve log message.<commit_after>package cache // import "gopkg.in/go-redis/cache.v5"

import (
	"errors"
	"log"
	"sync/atomic"
	"time"

	"gopkg.in/go-redis/cache.v5/lrucache"

	"go4.org/syncutil/singleflight"
	"gopkg.in/redis.v5"
)

var ErrCacheMiss = errors.New("cache: key is missing")

type rediser interface {
	Set(key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	Get(key string) *redis.StringCmd
	Del(keys ...string) *redis.IntCmd
}

type Codec struct {
	Redis rediser

	// Local LRU cache for super hot items.
	LocalCache *lrucache.Cache

	Marshal   func(interface{}) ([]byte, error)
	Unmarshal func([]byte, interface{}) error

	group        singleflight.Group
	hits, misses int64
}

type Item struct {
	Key    string
	Object interface{}

	// Func returns object to cache.
	Func func() (interface{}, error)

	// Expiration is the cache expiration time.
	// Default expiration is 1 hour.
	Expiration time.Duration
}

func (item *Item) object() (interface{}, error) {
	if item.Object != nil {
		return item.Object, nil
	}
	if item.Func != nil {
		return item.Func()
	}
	return nil, nil
}

// Set caches the item.
func (cd *Codec) Set(item *Item) error {
	if item.Expiration >= 0 && item.Expiration < time.Second {
		item.Expiration = time.Hour
	} else if item.Expiration == -1 {
		item.Expiration = 0
	}

	object, err := item.object()
	if err != nil {
		return err
	}

	b, err := cd.Marshal(object)
	if err != nil {
		log.Printf("cache: Marshal failed: %s", err)
		return err
	}

	if cd.LocalCache != nil {
		cd.LocalCache.Set(item.Key, b)
	}

	err = cd.Redis.Set(item.Key, b, item.Expiration).Err()
	if err != nil {
		log.Printf("cache: Set key=%q failed: %s", item.Key, err)
	}
	return err
}

// Get gets the object for the given key.
func (cd *Codec) Get(key string, object interface{}) error {
	b, err := cd.getBytes(key)
	if err != nil {
		return err
	}

	if object == nil || len(b) == 0 {
		return nil
	}

	if err := cd.Unmarshal(b, object); err != nil {
		log.Printf("cache: key=%q Unmarshal(%T) failed: %s", key, object, err)
		return err
	}

	return nil
}

func (cd *Codec) getBytes(key string) ([]byte, error) {
	if cd.LocalCache != nil {
		v, ok := cd.LocalCache.Get(key)
		if ok {
			b, ok := v.([]byte)
			if ok {
				atomic.AddInt64(&cd.hits, 1)
				return b, nil
			}
		}
	}

	b, err := cd.Redis.Get(key).Bytes()
	if err != nil {
		atomic.AddInt64(&cd.misses, 1)
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		log.Printf("cache: Get key=%q failed: %s", key, err)
		return nil, err
	}

	if cd.LocalCache != nil {
		cd.LocalCache.Set(key, b)
	}
	return b, nil
}

// Do gets the item.Object for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Codec) Do(item *Item) (interface{}, error) {
	if err := cd.getItem(item); err == nil {
		return item.Object, nil
	}

	return cd.group.Do(item.Key, func() (interface{}, error) {
		if err := cd.getItem(item); err == nil {
			return item.Object, nil
		}

		obj, err := item.Func()
		if err != nil {
			return nil, err
		}

		item.Object = obj
		item.Func = nil
		cd.Set(item)

		return obj, nil
	})
}

func (cd *Codec) getItem(item *Item) error {
	if item.Object != nil {
		return cd.Get(item.Key, item.Object)
	} else {
		return cd.Get(item.Key, &item.Object)
	}
}

func (cd *Codec) Delete(key string) error {
	if cd.LocalCache != nil {
		cd.LocalCache.Delete(key)
	}

	deleted, err := cd.Redis.Del(key).Result()
	if err != nil {
		log.Printf("cache: Del key=%q failed: %s", key, err)
		return err
	}
	if deleted == 0 {
		return ErrCacheMiss
	}
	return nil
}

func (cd *Codec) Hits() int {
	return int(atomic.LoadInt64(&cd.hits))
}

func (cd *Codec) Misses() int {
	return int(atomic.LoadInt64(&cd.misses))
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"strconv"

	log "github.com/Sirupsen/logrus"
	"github.com/garyburd/redigo/redis"
)

// getRedisPool returns thread safe Redis connection pool
func getRedisPool() *redis.Pool {

	// getting redis connection
	maxConnections := 10
	mc := os.Getenv("MaxConnections")
	if mc != "" {
		maxCons, err := strconv.Atoi(mc)
		if err != nil {
			maxConnections = 10
		} else {
			maxConnections = maxCons
		}
	}
	// getting redis client for state storing
	redisPool := redis.NewPool(func() (redis.Conn, error) {
		c, err := redis.Dial("tcp", AppConfig.redisAddress)

		if err != nil {
			log.WithFields(log.Fields{"Error": err.Error()}).Panic("Failed to create Redis connection pool!")
			return nil, err
		}
		if AppConfig.redisPassword != "" {
			if _, err := c.Do("AUTH", AppConfig.redisPassword); err != nil {
				log.WithFields(log.Fields{
					"Error":        err.Error(),
					"PasswordUsed": AppConfig.redisPassword,
				}).Panic("Failed to authenticate to Redis!")
				c.Close()
				return nil, err
			} else {
				log.Info("Authenticated to Redis successfully! ")
			}
		}

		return c, err
	}, maxConnections)

	defer redisPool.Close()

	return redisPool
}
<commit_msg>added record to cache function<commit_after>package main

import (
	"os"
	"strconv"

	log "github.com/Sirupsen/logrus"
	"github.com/garyburd/redigo/redis"
)

type Cache struct {
	pool *redis.Pool
}

// set records a key in cache (redis)
func (c *Cache) set(key string, value []byte) error {
	client := c.pool.Get()
	defer client.Close()

	_, err := client.Do("SET", key, value)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to record request...")
	} else {
		log.WithFields(log.Fields{}).Info("Request recorded!")
	}
}

// getRedisPool returns thread safe Redis connection pool
func getRedisPool() *redis.Pool {

	// getting redis connection
	maxConnections := 10
	mc := os.Getenv("MaxConnections")
	if mc != "" {
		maxCons, err := strconv.Atoi(mc)
		if err != nil {
			maxConnections = 10
		} else {
			maxConnections = maxCons
		}
	}
	// getting redis client for state storing
	redisPool := redis.NewPool(func() (redis.Conn, error) {
		c, err := redis.Dial("tcp", AppConfig.redisAddress)

		if err != nil {
			log.WithFields(log.Fields{"Error": err.Error()}).Panic("Failed to create Redis connection pool!")
			return nil, err
		}
		if AppConfig.redisPassword != "" {
			if _, err := c.Do("AUTH", AppConfig.redisPassword); err != nil {
				log.WithFields(log.Fields{
					"Error":        err.Error(),
					"PasswordUsed": AppConfig.redisPassword,
				}).Panic("Failed to authenticate to Redis!")
				c.Close()
				return nil, err
			} else {
				log.Info("Authenticated to Redis successfully! ")
			}
		}

		return c, err
	}, maxConnections)

	defer redisPool.Close()

	return redisPool
}
<|endoftext|>