<commit_before>/*
Copyright 2017 The OpenEBS Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package command

import (
	"errors"
	"fmt"
	"os"
	"os/signal"
	"reflect"
	"sort"

	"strings"
	"syscall"
	"time"

	"github.com/golang/glog"

	"github.com/openebs/maya/cmd/maya-apiserver/app/config"
	"github.com/openebs/maya/cmd/maya-apiserver/app/server"
	k8sapi "github.com/openebs/maya/pkg/client/k8s/v1alpha1"
	env "github.com/openebs/maya/pkg/env/v1alpha1"
	install "github.com/openebs/maya/pkg/install/v1alpha1"
	"github.com/openebs/maya/pkg/usage"
	"github.com/openebs/maya/pkg/util"
	"github.com/openebs/maya/pkg/version"

	"github.com/openebs/maya/cmd/maya-apiserver/spc-watcher"
	"github.com/spf13/cobra"
)

var (
	helpText = `
Usage: m-apiserver start [options]

  Starts maya-apiserver and runs until an interrupt is received.

  The maya apiserver's configuration primarily comes from the config
  files used, but a subset of the options may also be passed directly
  as CLI arguments, listed below.

General Options :

  -bind=<addr>
    The address the server will bind to for all of its various network
    services. The individual services that run bind to individual
    ports on this address. Defaults to the loopback 127.0.0.1.

  -config=<path>
    The path to either a single config file or a directory of config
    files to use for configuring maya api server. This option may be
    specified multiple times. If multiple config files are used, the
    values from each will be merged together. During merging, values
    from files found later in the list are merged over values from
    previously parsed files.

  -log-level=<level>
    Specify the verbosity level of maya api server's logs. Valid values include
    DEBUG, INFO, and WARN, in decreasing order of verbosity. The
    default is INFO.
 `
)

// gracefulTimeout controls how long we wait before forcefully terminating
const gracefulTimeout = 5 * time.Second

// CmdStartOptions is a cli implementation that runs a maya apiserver.
// The command will not end unless a shutdown message is sent on the
// ShutdownCh. If two messages are sent on the ShutdownCh it will forcibly
// exit.
type CmdStartOptions struct {
	BindAddr   string
	LogLevel   string
	ConfigPath string
	ShutdownCh <-chan struct{}
	args       []string

	// TODO
	// Check if both maya & httpServer instances are required ?
	// Can httpServer or maya embed one of the other ?
	// Need to take care of shuting down & graceful exit scenarios !!
	maya       *server.MayaApiServer
	httpServer *server.HTTPServer
}

// NewCmdStart creates start command for maya-apiserver
func NewCmdStart() *cobra.Command {
	options := CmdStartOptions{}

	cmd := &cobra.Command{
		Use:   "start",
		Short: "start maya-apiserver",
		Long:  helpText,

		Run: func(cmd *cobra.Command, args []string) {
			util.CheckErr(Run(cmd, &options), util.Fatal)
		},
	}

	cmd.Flags().StringVarP(&options.BindAddr, "bind", "", options.BindAddr,
		"IP Address to bind for maya apiserver.")

	cmd.Flags().StringVarP(&options.LogLevel, "log-level", "", options.LogLevel,
		"Log level for maya apiserver DEBUG INFO WARN.")

	cmd.Flags().StringVarP(&options.ConfigPath, "config", "", options.ConfigPath,
		"Path to a single config file or directory.")

	return cmd
}

// Run does tasks related to mayaserver.
func Run(cmd *cobra.Command, c *CmdStartOptions) error {
	glog.Infof("Initializing maya-apiserver...")

	// Read and merge with default configuration
	mconfig := c.readMayaConfig()
	if mconfig == nil {
		return errors.New("Unable to load the configuration.")
	}
	go func() {
		err := spc.Start()
		if err != nil {
			glog.Errorf("Could not start controller: %s", err.Error())
		}
	}()
	//TODO Setup Log Level

	// Setup Maya server
	if err := c.setupMayaServer(mconfig); err != nil {
		return err
	}
	defer c.maya.Shutdown()

	// Check and shut down at the end
	defer func() {
		if c.httpServer != nil {
			c.httpServer.Shutdown()
		}
	}()

	// Compile Maya server information for output later
	info := make(map[string]string)
	info["version"] = fmt.Sprintf("%s%s", mconfig.Version, mconfig.VersionPrerelease)
	info["log level"] = mconfig.LogLevel
	info["region"] = fmt.Sprintf("%s (DC: %s)", mconfig.Region, mconfig.Datacenter)

	// Sort the keys for output
	infoKeys := make([]string, 0, len(info))
	for key := range info {
		infoKeys = append(infoKeys, key)
	}
	sort.Strings(infoKeys)

	// Maya server configuration output
	padding := 18
	glog.Info("Maya api server configuration:\n")
	for _, k := range infoKeys {
		glog.Infof(
			"%s%s: %s",
			strings.Repeat(" ", padding-len(k)),
			strings.Title(k),
			info[k])
	}
	glog.Infof("")

	// Output the header that the server has started
	glog.Info("Maya api server started! Log data will stream in below:\n")

	if env.Truthy(env.OpenEBSEnableAnalytics) {
		clusterSize, _ := k8sapi.NumberOfNodes()
		event := usage.NewEvent("install", "running", "nodes", int64(clusterSize))
		event.Send()
	}
	// Wait for exit
	if c.handleSignals(mconfig) > 0 {
		return errors.New("Ungraceful exit ...")
	}

	return nil
}

func (c *CmdStartOptions) readMayaConfig() *config.MayaConfig {
	// Load the configuration
	mconfig := config.DefaultMayaConfig()

	if c.ConfigPath != "" {
		current, err := config.LoadMayaConfig(c.ConfigPath)
		if err != nil {
			glog.Errorf(
				"Error loading configuration from %s: %s", c.ConfigPath, err)
			return nil
		}

		// The user asked us to load some config here but we didn't find any,
		// so we'll complain but continue.
		if current == nil || reflect.DeepEqual(current, &config.MayaConfig{}) {
			glog.Warningf("No configuration loaded from %s", c.ConfigPath)
		}

		if mconfig == nil {
			mconfig = current
		} else {
			mconfig = mconfig.Merge(current)
		}
	}

	// Merge any CLI options over config file options

	// Set the version info
	mconfig.Revision = version.GetGitCommit()
	mconfig.Version = version.GetVersion()
	mconfig.VersionPrerelease = version.GetBuildMeta()

	// Set the details from command line
	if c.BindAddr != "" {
		mconfig.BindAddr = c.BindAddr
	}
	if c.LogLevel != "" {
		mconfig.LogLevel = c.LogLevel
	}

	// Normalize binds, ports, addresses, and advertise
	if err := mconfig.NormalizeAddrs(); err != nil {
		glog.Errorf(err.Error())
		return nil
	}

	return mconfig
}

// setupMayaServer is used to start Maya server
func (c *CmdStartOptions) setupMayaServer(mconfig *config.MayaConfig) error {
	glog.Info("Starting maya api server ...")

	// run maya installer
	installErrs := install.SimpleInstaller().Install()
	if len(installErrs) != 0 {
		glog.Errorf("Install errors were found: %+v", installErrs)
		return fmt.Errorf("Failed to install resources")
	}

	// Setup maya service i.e. maya api server
	maya, err := server.NewMayaApiServer(mconfig, os.Stdout)
	if err != nil {
		glog.Errorf("Error starting maya api server: %s", err)
		return err
	}

	c.maya = maya

	// Setup the HTTP server
	http, err := server.NewHTTPServer(maya, mconfig, os.Stdout)
	if err != nil {
		maya.Shutdown()
		glog.Errorf("Error starting http server: %s", err)
		return err
	}

	c.httpServer = http

	return nil
}

// handleSignals blocks until we get an exit-causing signal
func (c *CmdStartOptions) handleSignals(mconfig *config.MayaConfig) int {
	signalCh := make(chan os.Signal, 4)
	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGPIPE)

	// Wait for a signal
WAIT:
	var sig os.Signal
	select {
	case s := <-signalCh:
		sig = s
	case <-c.ShutdownCh:
		sig = os.Interrupt
	}
	glog.Infof("Caught signal: %v", sig)

	// Skip any SIGPIPE signal (See issue #1798)
	if sig == syscall.SIGPIPE {
		goto WAIT
	}

	// Check if this is a SIGHUP
	if sig == syscall.SIGHUP {
		if conf := c.handleReload(mconfig); conf != nil {
			// Update the value only, not address
			*mconfig = *conf
		}
		goto WAIT
	}

	// Check if we should do a graceful leave
	graceful := false
	if sig == os.Interrupt && mconfig.LeaveOnInt {
		graceful = true
	} else if sig == syscall.SIGTERM && mconfig.LeaveOnTerm {
		graceful = true
	}

	// Bail fast if not doing a graceful leave
	if !graceful {
		return 1
	}

	// Attempt a graceful leave
	gracefulCh := make(chan struct{})
	glog.Info("Gracefully shutting maya api server...")
	go func() {
		if err := c.maya.Leave(); err != nil {
			glog.Errorf("Error: %s", err)
			return
		}
		close(gracefulCh)
	}()

	// Wait for leave or another signal
	select {
	case <-signalCh:
		return 1
	case <-time.After(gracefulTimeout):
		return 1
	case <-gracefulCh:
		return 0
	}
}

// handleReload is invoked when we should reload our configs, e.g. SIGHUP
// TODO
// The current reload code is very basic.
// Add ways to reload the orchestrator & plugins without shuting down the
// process
func (c *CmdStartOptions) handleReload(mconfig *config.MayaConfig) *config.MayaConfig {

	glog.Info("Reloading maya api server configuration...")

	newConf := c.readMayaConfig()
	if newConf == nil {
		glog.Error("Failed to reload config")
		return mconfig
	}

	//TODO Change the log level dynamically
	glog.Infof("Log level is : %s", strings.ToUpper(newConf.LogLevel))

	return newConf
}
<commit_msg>fix(spc, install, startup): move spc start logic after API service starts<commit_after>/*
Copyright 2017 The OpenEBS Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package command

import (
	"errors"
	"fmt"
	"os"
	"os/signal"
	"reflect"
	"sort"

	"strings"
	"syscall"
	"time"

	"github.com/golang/glog"

	"github.com/openebs/maya/cmd/maya-apiserver/app/config"
	"github.com/openebs/maya/cmd/maya-apiserver/app/server"
	k8sapi "github.com/openebs/maya/pkg/client/k8s/v1alpha1"
	env "github.com/openebs/maya/pkg/env/v1alpha1"
	install "github.com/openebs/maya/pkg/install/v1alpha1"
	"github.com/openebs/maya/pkg/usage"
	"github.com/openebs/maya/pkg/util"
	"github.com/openebs/maya/pkg/version"

	"github.com/openebs/maya/cmd/maya-apiserver/spc-watcher"
	"github.com/spf13/cobra"
)

var (
	helpText = `
Usage: m-apiserver start [options]

  Starts maya-apiserver and runs until an interrupt is received.

  The maya apiserver's configuration primarily comes from the config
  files used, but a subset of the options may also be passed directly
  as CLI arguments, listed below.

General Options :

  -bind=<addr>
    The address the server will bind to for all of its various network
    services. The individual services that run bind to individual
    ports on this address. Defaults to the loopback 127.0.0.1.

  -config=<path>
    The path to either a single config file or a directory of config
    files to use for configuring maya api server. This option may be
    specified multiple times. If multiple config files are used, the
    values from each will be merged together. During merging, values
    from files found later in the list are merged over values from
    previously parsed files.

  -log-level=<level>
    Specify the verbosity level of maya api server's logs. Valid values include
    DEBUG, INFO, and WARN, in decreasing order of verbosity. The
    default is INFO.
 `
)

// gracefulTimeout controls how long we wait before forcefully terminating
const gracefulTimeout = 5 * time.Second

// CmdStartOptions is a cli implementation that runs a maya apiserver.
// The command will not end unless a shutdown message is sent on the
// ShutdownCh. If two messages are sent on the ShutdownCh it will forcibly
// exit.
type CmdStartOptions struct {
	BindAddr   string
	LogLevel   string
	ConfigPath string
	ShutdownCh <-chan struct{}
	args       []string

	// TODO
	// Check if both maya & httpServer instances are required ?
	// Can httpServer or maya embed one of the other ?
	// Need to take care of shuting down & graceful exit scenarios !!
	maya       *server.MayaApiServer
	httpServer *server.HTTPServer
}

// NewCmdStart creates start command for maya-apiserver
func NewCmdStart() *cobra.Command {
	options := CmdStartOptions{}

	cmd := &cobra.Command{
		Use:   "start",
		Short: "start maya-apiserver",
		Long:  helpText,

		Run: func(cmd *cobra.Command, args []string) {
			util.CheckErr(Run(cmd, &options), util.Fatal)
		},
	}

	cmd.Flags().StringVarP(&options.BindAddr, "bind", "", options.BindAddr,
		"IP Address to bind for maya apiserver.")

	cmd.Flags().StringVarP(&options.LogLevel, "log-level", "", options.LogLevel,
		"Log level for maya apiserver DEBUG INFO WARN.")

	cmd.Flags().StringVarP(&options.ConfigPath, "config", "", options.ConfigPath,
		"Path to a single config file or directory.")

	return cmd
}

// Run does tasks related to mayaserver.
func Run(cmd *cobra.Command, c *CmdStartOptions) error {
	glog.Infof("Initializing maya-apiserver...")

	// Read and merge with default configuration
	mconfig := c.readMayaConfig()
	if mconfig == nil {
		return errors.New("Unable to load the configuration.")
	}

	//TODO Setup Log Level

	// Setup Maya server
	if err := c.setupMayaServer(mconfig); err != nil {
		return err
	}
	defer c.maya.Shutdown()

	// Check and shut down at the end
	defer func() {
		if c.httpServer != nil {
			c.httpServer.Shutdown()
		}
	}()

	// Compile Maya server information for output later
	info := make(map[string]string)
	info["version"] = fmt.Sprintf("%s%s", mconfig.Version, mconfig.VersionPrerelease)
	info["log level"] = mconfig.LogLevel
	info["region"] = fmt.Sprintf("%s (DC: %s)", mconfig.Region, mconfig.Datacenter)

	// Sort the keys for output
	infoKeys := make([]string, 0, len(info))
	for key := range info {
		infoKeys = append(infoKeys, key)
	}
	sort.Strings(infoKeys)

	// Maya server configuration output
	padding := 18
	glog.Info("Maya api server configuration:\n")
	for _, k := range infoKeys {
		glog.Infof(
			"%s%s: %s",
			strings.Repeat(" ", padding-len(k)),
			strings.Title(k),
			info[k])
	}
	glog.Infof("")

	// Output the header that the server has started
	glog.Info("Maya api server started! Log data will stream in below:\n")

	// start storage pool controller
	go func() {
		err := spc.Start()
		if err != nil {
			glog.Errorf("Failed to start storage pool controller: %s", err.Error())
		}
	}()

	if env.Truthy(env.OpenEBSEnableAnalytics) {
		clusterSize, _ := k8sapi.NumberOfNodes()
		event := usage.NewEvent("install", "running", "nodes", int64(clusterSize))
		event.Send()
	}

	// Wait for exit
	if c.handleSignals(mconfig) > 0 {
		return errors.New("Ungraceful exit ...")
	}

	return nil
}

func (c *CmdStartOptions) readMayaConfig() *config.MayaConfig {
	// Load the configuration
	mconfig := config.DefaultMayaConfig()

	if c.ConfigPath != "" {
		current, err := config.LoadMayaConfig(c.ConfigPath)
		if err != nil {
			glog.Errorf(
				"Error loading configuration from %s: %s", c.ConfigPath, err)
			return nil
		}

		// The user asked us to load some config here but we didn't find any,
		// so we'll complain but continue.
		if current == nil || reflect.DeepEqual(current, &config.MayaConfig{}) {
			glog.Warningf("No configuration loaded from %s", c.ConfigPath)
		}

		if mconfig == nil {
			mconfig = current
		} else {
			mconfig = mconfig.Merge(current)
		}
	}

	// Merge any CLI options over config file options

	// Set the version info
	mconfig.Revision = version.GetGitCommit()
	mconfig.Version = version.GetVersion()
	mconfig.VersionPrerelease = version.GetBuildMeta()

	// Set the details from command line
	if c.BindAddr != "" {
		mconfig.BindAddr = c.BindAddr
	}
	if c.LogLevel != "" {
		mconfig.LogLevel = c.LogLevel
	}

	// Normalize binds, ports, addresses, and advertise
	if err := mconfig.NormalizeAddrs(); err != nil {
		glog.Errorf(err.Error())
		return nil
	}

	return mconfig
}

// setupMayaServer is used to start Maya server
func (c *CmdStartOptions) setupMayaServer(mconfig *config.MayaConfig) error {
	glog.Info("Starting maya api server ...")

	// run maya installer
	installErrs := install.SimpleInstaller().Install()
	if len(installErrs) != 0 {
		glog.Errorf("Install errors were found: %+v", installErrs)
		return fmt.Errorf("Failed to install resources")
	}

	// Setup maya service i.e. maya api server
	maya, err := server.NewMayaApiServer(mconfig, os.Stdout)
	if err != nil {
		glog.Errorf("Error starting maya api server: %s", err)
		return err
	}

	c.maya = maya

	// Setup the HTTP server
	http, err := server.NewHTTPServer(maya, mconfig, os.Stdout)
	if err != nil {
		maya.Shutdown()
		glog.Errorf("Error starting http server: %s", err)
		return err
	}

	c.httpServer = http

	return nil
}

// handleSignals blocks until we get an exit-causing signal
func (c *CmdStartOptions) handleSignals(mconfig *config.MayaConfig) int {
	signalCh := make(chan os.Signal, 4)
	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGPIPE)

	// Wait for a signal
WAIT:
	var sig os.Signal
	select {
	case s := <-signalCh:
		sig = s
	case <-c.ShutdownCh:
		sig = os.Interrupt
	}
	glog.Infof("Caught signal: %v", sig)

	// Skip any SIGPIPE signal (See issue #1798)
	if sig == syscall.SIGPIPE {
		goto WAIT
	}

	// Check if this is a SIGHUP
	if sig == syscall.SIGHUP {
		if conf := c.handleReload(mconfig); conf != nil {
			// Update the value only, not address
			*mconfig = *conf
		}
		goto WAIT
	}

	// Check if we should do a graceful leave
	graceful := false
	if sig == os.Interrupt && mconfig.LeaveOnInt {
		graceful = true
	} else if sig == syscall.SIGTERM && mconfig.LeaveOnTerm {
		graceful = true
	}

	// Bail fast if not doing a graceful leave
	if !graceful {
		return 1
	}

	// Attempt a graceful leave
	gracefulCh := make(chan struct{})
	glog.Info("Gracefully shutting maya api server...")
	go func() {
		if err := c.maya.Leave(); err != nil {
			glog.Errorf("Error: %s", err)
			return
		}
		close(gracefulCh)
	}()

	// Wait for leave or another signal
	select {
	case <-signalCh:
		return 1
	case <-time.After(gracefulTimeout):
		return 1
	case <-gracefulCh:
		return 0
	}
}

// handleReload is invoked when we should reload our configs, e.g. SIGHUP
// TODO
// The current reload code is very basic.
// Add ways to reload the orchestrator & plugins without shuting down the
// process
func (c *CmdStartOptions) handleReload(mconfig *config.MayaConfig) *config.MayaConfig {

	glog.Info("Reloading maya api server configuration...")

	newConf := c.readMayaConfig()
	if newConf == nil {
		glog.Error("Failed to reload config")
		return mconfig
	}

	//TODO Change the log level dynamically
	glog.Infof("Log level is : %s", strings.ToUpper(newConf.LogLevel))

	return newConf
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"sync/atomic"
	"time"

	"github.com/gosuri/uilive"
	"github.com/grafana/metrictank/conf"
	"github.com/grafana/metrictank/logger"
	log "github.com/sirupsen/logrus"
)

func init() {
	formatter := &logger.TextFormatter{}
	formatter.TimestampFormat = "2006-01-02 15:04:05.000"
	log.SetFormatter(formatter)
	log.SetLevel(log.InfoLevel)
}

func perror(err error) {
	if err != nil {
		log.Fatal(err.Error())
	}
}

func main() {
	var indexRulesFile string
	flag.StringVar(&indexRulesFile, "index-rules-file", "/etc/metrictank/index-rules.conf", "name of file which defines the max-stale times")
	flag.Parse()

	indexRules, err := conf.ReadIndexRules(indexRulesFile)
	if os.IsNotExist(err) {
		log.Fatalf("Index-rules.conf file %s does not exist; exiting", indexRulesFile)
		os.Exit(1)
	}
	scanner := bufio.NewScanner(os.Stdin)

	counts := make([]uint64, len(indexRules.Rules)+1)

	writer := uilive.New()
	writer.Start()

	go func() {
		tick := time.NewTicker(100 * time.Millisecond)
		for range tick.C {
			for i := range counts {
				fmt.Fprintf(writer, "Patt [%d]: %s    -- %d\n", i, indexRules.Get(uint16(i)), atomic.LoadUint64(&counts[i]))
			}
			writer.Flush()
		}

	}()

	for scanner.Scan() {
		word := scanner.Text()
		ruleID, _ := indexRules.Match(word)
		atomic.AddUint64(&counts[ruleID], 1)
	}
	writer.Stop()

	if err := scanner.Err(); err != nil {
		log.Println(err)
	}

}
<commit_msg>show stats better and make sure to show last update before exit<commit_after>package main

import (
	"bufio"
	"context"
	"flag"
	"fmt"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/gosuri/uilive"
	"github.com/grafana/metrictank/conf"
	"github.com/grafana/metrictank/logger"
	log "github.com/sirupsen/logrus"
)

func init() {
	formatter := &logger.TextFormatter{}
	formatter.TimestampFormat = "2006-01-02 15:04:05.000"
	log.SetFormatter(formatter)
	log.SetLevel(log.InfoLevel)
}

func perror(err error) {
	if err != nil {
		log.Fatal(err.Error())
	}
}

func showStats(indexRules conf.IndexRules, counts []uint64, wg *sync.WaitGroup, ctx context.Context) {

	start := time.Now()

	writer := uilive.New()
	writer.Start()

	printTable := func() {
		fmt.Fprintln(writer, "Count        RuleID Pattern")
		for i := range counts {
			fmt.Fprintf(writer, "%12d %6d %s\n", atomic.LoadUint64(&counts[i]), i, indexRules.Get(uint16(i)))
		}
	}

	tick := time.NewTicker(time.Second)
	for {
		select {
		case <-tick.C:
			printTable()
			writer.Flush()
		case <-ctx.Done():
			printTable()
			fmt.Fprintf(writer, "Finished in %v\n", time.Since(start))
			writer.Stop()
			wg.Done()
			return
		}
	}

}

func main() {
	var indexRulesFile string
	flag.StringVar(&indexRulesFile, "index-rules-file", "/etc/metrictank/index-rules.conf", "name of file which defines the max-stale times")
	flag.Parse()

	indexRules, err := conf.ReadIndexRules(indexRulesFile)
	if os.IsNotExist(err) {
		log.Fatalf("Index-rules.conf file %s does not exist; exiting", indexRulesFile)
		os.Exit(1)
	}
	scanner := bufio.NewScanner(os.Stdin)

	counts := make([]uint64, len(indexRules.Rules)+1)

	ctx, cancel := context.WithCancel(context.Background())
	wg := &sync.WaitGroup{}

	wg.Add(1)
	go showStats(indexRules, counts, wg, ctx)

	for scanner.Scan() {
		word := scanner.Text()
		ruleID, _ := indexRules.Match(word)
		atomic.AddUint64(&counts[ruleID], 1)
	}
	cancel()
	wg.Wait()

	if err := scanner.Err(); err != nil {
		log.Println(err)
	}

}
<|endoftext|>


<commit_before>// query_benchmarker_cassandra speed tests Cassandra servers using request
// data from stdin.
//
// It reads encoded HLQuery objects from stdin, and makes concurrent requests
// to the provided Cassandra cluster. This program is a 'heavy client', i.e.
// it builds a client-side index of table metadata before beginning the
// benchmarking.
//
// TODO(rw): On my machine, this only decodes 700k/sec messages from stdin.
package main

import (
	"bufio"
	"encoding/gob"
	"flag"
	"fmt"
	"github.com/influxdata/influxdb-comparisons/util/report"
	"io"
	"log"
	"os"
	"runtime/pprof"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	BucketDuration   time.Duration = 24 * time.Hour
	BucketTimeLayout string        = "2006-01-02"
	BlessedKeyspace  string        = "measurements"
)

// Blessed tables that hold benchmark data:
var (
	BlessedTables []string = []string{
		"series_bigint",
		"series_float",
		"series_double",
		"series_boolean",
		"series_blob",
	}
)

// Program option vars:
var (
	daemonUrl            string
	workers              int
	aggrPlanLabel        string
	subQueryParallelism  int
	requestTimeout       time.Duration
	csiTimeout           time.Duration
	debug                int
	prettyPrintResponses bool
	limit                int64
	burnIn               uint64
	printInterval        uint64
	memProfile           string
	reportDatabase       string
	reportHost           string
	reportUser           string
	reportPassword       string
	reportTagsCSV        string
)

// Helpers for choice-like flags:
var (
	aggrPlanChoices map[string]int = map[string]int{
		"server": AggrPlanTypeWithServerAggregation,
		"client": AggrPlanTypeWithoutServerAggregation,
	}
)

// Global vars:
var (
	queryPool       sync.Pool
	hlQueryChan     chan *HLQuery
	statPool        sync.Pool
	statChan        chan *Stat
	workersGroup    sync.WaitGroup
	statGroup       sync.WaitGroup
	aggrPlan        int
	reportTags      [][2]string
	reportHostname  string
	reportQueryStat StatGroup
)

type statsMap map[string]*StatGroup

// Parse args:
func init() {
	flag.StringVar(&daemonUrl, "url", "localhost:9042", "Cassandra URL.")
	flag.IntVar(&workers, "workers", 1, "Number of concurrent requests to make.")
	flag.StringVar(&aggrPlanLabel, "aggregation-plan", "", "Aggregation plan (choices: server, client)")
	flag.IntVar(&subQueryParallelism, "subquery-workers", 1, "Number of concurrent subqueries to make (because the client does a scatter+gather operation).")
	flag.DurationVar(&requestTimeout, "request-timeout", 1*time.Second, "Maximum request timeout.")
	flag.DurationVar(&csiTimeout, "client-side-index-timeout", 10*time.Second, "Maximum client-side index timeout (only used at initialization).")
	flag.IntVar(&debug, "debug", 0, "Whether to print debug messages.")
	flag.Int64Var(&limit, "limit", -1, "Limit the number of queries to send.")
	flag.Uint64Var(&burnIn, "burn-in", 0, "Number of queries to ignore before collecting statistics.")
	flag.Uint64Var(&printInterval, "print-interval", 100, "Print timing stats to stderr after this many queries (0 to disable)")
	flag.BoolVar(&prettyPrintResponses, "print-responses", false, "Pretty print response bodies (for correctness checking) (default false).")
	flag.StringVar(&memProfile, "memprofile", "", "Write a memory profile to this file.")
	flag.StringVar(&reportDatabase, "report-database", "database_benchmarks", "Database name where to store result metrics.")
	flag.StringVar(&reportHost, "report-host", "", "Host to send result metrics.")
	flag.StringVar(&reportUser, "report-user", "", "User for Host to send result metrics.")
	flag.StringVar(&reportPassword, "report-password", "", "User password for Host to send result metrics.")
	flag.StringVar(&reportTagsCSV, "report-tags", "", "Comma separated k:v tags to send  alongside result metrics.")

	flag.Parse()

	if _, ok := aggrPlanChoices[aggrPlanLabel]; !ok {
		log.Fatal("invalid aggregation plan")
	}
	aggrPlan = aggrPlanChoices[aggrPlanLabel]

	if reportHost != "" {
		fmt.Printf("results report destination: %v\n", reportHost)
		fmt.Printf("results report database: %v\n", reportDatabase)

		var err error
		reportHostname, err = os.Hostname()
		if err != nil {
			log.Fatalf("os.Hostname() error: %s", err.Error())
		}
		fmt.Printf("hostname for results report: %v\n", reportHostname)

		if reportTagsCSV != "" {
			pairs := strings.Split(reportTagsCSV, ",")
			for _, pair := range pairs {
				fields := strings.SplitN(pair, ":", 2)
				tagpair := [2]string{fields[0], fields[1]}
				reportTags = append(reportTags, tagpair)
			}
		}
		fmt.Printf("results report tags: %v\n", reportTags)
	}
}

func main() {
	// Make pools to minimize heap usage:
	queryPool = sync.Pool{
		New: func() interface{} {
			return &HLQuery{
				HumanLabel:       make([]byte, 0, 1024),
				HumanDescription: make([]byte, 0, 1024),
				MeasurementName:  make([]byte, 0, 1024),
				FieldName:        make([]byte, 0, 1024),
				AggregationType:  make([]byte, 0, 1024),
				TagSets:          make([][]string, 0, 10),
			}
		},
	}

	statPool = sync.Pool{
		New: func() interface{} {
			return &Stat{
				Label: make([]byte, 0, 1024),
			}
		},
	}

	// Make client-side index:
	csi := NewClientSideIndex(FetchSeriesCollection(daemonUrl, csiTimeout))

	// Make database connection pool:
	session := NewCassandraSession(daemonUrl, requestTimeout)
	defer session.Close()

	// Make data and stat channels:
	hlQueryChan = make(chan *HLQuery, workers)
	statChan = make(chan *Stat, workers)

	// Launch the stats processor:
	statGroup.Add(1)
	go processStats()

	// Launch the query processors:
	qe := NewHLQueryExecutor(session, csi, debug)
	for i := 0; i < workers; i++ {
		workersGroup.Add(1)
		go processQueries(qe)
	}

	// Read in jobs, closing the job channel when done:
	input := bufio.NewReaderSize(os.Stdin, 1<<20)
	wallStart := time.Now()
	scan(input)
	close(hlQueryChan)

	// Block for workers to finish sending requests, closing the stats
	// channel when done:
	workersGroup.Wait()
	close(statChan)

	// Wait on the stat collector to finish (and print its results):
	statGroup.Wait()

	wallEnd := time.Now()
	wallTook := wallEnd.Sub(wallStart)
	_, err := fmt.Printf("wall clock time: %fsec\n", float64(wallTook.Nanoseconds())/1e9)
	if err != nil {
		log.Fatal(err)
	}

	// (Optional) create a memory profile:
	if memProfile != "" {
		f, err := os.Create(memProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.WriteHeapProfile(f)
		f.Close()
	}

	if reportHost != "" {
		//append db specific tags to custom tags
		reportTags = append(reportTags, [2]string{"aggregation_plan", aggrPlanLabel})
		reportTags = append(reportTags, [2]string{"subquery_workers", strconv.Itoa(subQueryParallelism)})
		reportTags = append(reportTags, [2]string{"request_timeout", strconv.Itoa(int(requestTimeout.Seconds()))})
		reportTags = append(reportTags, [2]string{"client_side_index_timeout", strconv.Itoa(int(csiTimeout.Seconds()))})

		reportParams := &report.QueryReportParams{
			ReportParams: report.ReportParams{
				DBType:             "Cassandra",
				ReportDatabaseName: reportDatabase,
				ReportHost:         reportHost,
				ReportUser:         reportUser,
				ReportPassword:     reportPassword,
				ReportTags:         reportTags,
				Hostname:           reportHostname,
				DestinationUrl:     daemonUrl,
				Workers:            workers,
				ItemLimit:          int(limit),
			},
			BurnIn: int64(burnIn),
		}
		err = report.ReportQueryResult(reportParams, reportQueryStat.Min, reportQueryStat.Mean, reportQueryStat.Max, reportQueryStat.Count, wallTook)

		if err != nil {
			log.Fatal(err)
		}
	}
}

// scan reads encoded Queries and places them onto the workqueue.
func scan(r io.Reader) {
	dec := gob.NewDecoder(r)

	n := int64(0)
	for {
		if limit >= 0 && n >= limit {
			break
		}

		q := queryPool.Get().(*HLQuery)
		err := dec.Decode(q)
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}

		q.ID = n
		q.ForceUTC()

		hlQueryChan <- q

		n++
	}
}

// processQueries reads byte buffers from hlQueryChan and writes them to the
// target server, while tracking latency.
func processQueries(qc *HLQueryExecutor) {
	opts := HLQueryExecutorDoOptions{
		AggregationPlan:      aggrPlan,
		Debug:                debug,
		PrettyPrintResponses: prettyPrintResponses,
	}
	labels := map[string][][]byte{}
	for q := range hlQueryChan {
		qpLagMs, reqLagMs, err := qc.Do(q, opts)

		// if needed, prepare stat labels:
		if _, ok := labels[string(q.HumanLabel)]; !ok {
			labels[string(q.HumanLabel)] = [][]byte{
				q.HumanLabel,
				[]byte(fmt.Sprintf("%s-qp", q.HumanLabel)),
				[]byte(fmt.Sprintf("%s-req", q.HumanLabel)),
			}
		}
		ls := labels[string(q.HumanLabel)]

		// total lag stat:
		stat := statPool.Get().(*Stat)
		stat.Init(ls[0], qpLagMs+reqLagMs, true)
		statChan <- stat

		// qp lag stat:
		stat = statPool.Get().(*Stat)
		stat.Init(ls[1], qpLagMs, false)
		statChan <- stat

		// req lag stat:
		stat = statPool.Get().(*Stat)
		stat.Init(ls[2], reqLagMs, false)
		statChan <- stat

		queryPool.Put(q)
		if err != nil {
			log.Fatalf("Error during request: %s\n", err.Error())
		}
	}
	workersGroup.Done()
}

// processStats collects latency results, aggregating them into summary
// statistics. Optionally, they are printed to stderr at regular intervals.
func processStats() {
	statMapping := statsMap{}

	i := uint64(0)
	for stat := range statChan {
		if i < burnIn {
			i++
			statPool.Put(stat)
			continue
		} else if i == burnIn && burnIn > 0 {
			_, err := fmt.Fprintf(os.Stderr, "burn-in complete after %d queries with %d workers\n", burnIn, workers)
			if err != nil {
				log.Fatal(err)
			}
		}
		if _, ok := statMapping[string(stat.Label)]; !ok {
			statMapping[string(stat.Label)] = &StatGroup{}
		}

		statMapping[string(stat.Label)].Push(stat.Value)

		if stat.IsActual {
			i++
			reportQueryStat.Push(stat.Value)
		}

		statPool.Put(stat)

		// print stats to stderr (if printInterval is greater than zero):
		if printInterval > 0 && i > 0 && i%printInterval == 0 && (int64(i) < limit || limit < 0) {
			_, err := fmt.Fprintf(os.Stderr, "after %d queries with %d workers:\n", i-burnIn, workers)
			if err != nil {
				log.Fatal(err)
			}
			fprintStats(os.Stderr, statMapping)
			_, err = fmt.Fprintf(os.Stderr, "\n")
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	// the final stats output goes to stdout:
	_, err := fmt.Printf("run complete after %d queries with %d workers:\n", i-burnIn, workers)
	if err != nil {
		log.Fatal(err)
	}
	fprintStats(os.Stdout, statMapping)
	statGroup.Done()
}

// fprintStats pretty-prints stats to the given writer.
func fprintStats(w io.Writer, statGroups statsMap) {
	maxKeyLength := 0
	keys := make([]string, 0, len(statGroups))
	for k := range statGroups {
		if len(k) > maxKeyLength {
			maxKeyLength = len(k)
		}
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		v := statGroups[k]
		minRate := 1e3 / v.Min
		meanRate := 1e3 / v.Mean
		maxRate := 1e3 / v.Max
		paddedKey := fmt.Sprintf("%s", k)
		for len(paddedKey) < maxKeyLength {
			paddedKey += " "
		}
		_, err := fmt.Fprintf(w, "%s : min: %8.2fms (%7.2f/sec), mean: %8.2fms (%7.2f/sec), max: %7.2fms (%6.2f/sec), count: %8d, sum: %5.1fsec \n", paddedKey, v.Min, minRate, v.Mean, meanRate, v.Max, maxRate, v.Count, v.Sum/1e3)
		if err != nil {
			log.Fatal(err)
		}
	}

}
<commit_msg>Added support for reading input from file by program argument<commit_after>// query_benchmarker_cassandra speed tests Cassandra servers using request
// data from stdin.
//
// It reads encoded HLQuery objects from stdin, and makes concurrent requests
// to the provided Cassandra cluster. This program is a 'heavy client', i.e.
// it builds a client-side index of table metadata before beginning the
// benchmarking.
//
// TODO(rw): On my machine, this only decodes 700k/sec messages from stdin.
package main

import (
	"bufio"
	"encoding/gob"
	"flag"
	"fmt"
	"github.com/influxdata/influxdb-comparisons/util/report"
	"io"
	"log"
	"os"
	"runtime/pprof"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	BucketDuration   time.Duration = 24 * time.Hour
	BucketTimeLayout string        = "2006-01-02"
	BlessedKeyspace  string        = "measurements"
)

// Blessed tables that hold benchmark data:
var (
	BlessedTables []string = []string{
		"series_bigint",
		"series_float",
		"series_double",
		"series_boolean",
		"series_blob",
	}
)

// Program option vars:
var (
	daemonUrl            string
	workers              int
	aggrPlanLabel        string
	subQueryParallelism  int
	requestTimeout       time.Duration
	csiTimeout           time.Duration
	debug                int
	prettyPrintResponses bool
	limit                int64
	burnIn               uint64
	printInterval        uint64
	memProfile           string
	reportDatabase       string
	reportHost           string
	reportUser           string
	reportPassword       string
	reportTagsCSV        string
	file                 string
)

// Helpers for choice-like flags:
var (
	aggrPlanChoices map[string]int = map[string]int{
		"server": AggrPlanTypeWithServerAggregation,
		"client": AggrPlanTypeWithoutServerAggregation,
	}
)

// Global vars:
var (
	queryPool       sync.Pool
	hlQueryChan     chan *HLQuery
	statPool        sync.Pool
	statChan        chan *Stat
	workersGroup    sync.WaitGroup
	statGroup       sync.WaitGroup
	aggrPlan        int
	reportTags      [][2]string
	reportHostname  string
	reportQueryStat StatGroup
	sourceReader    *os.File
)

type statsMap map[string]*StatGroup

// Parse args:
func init() {
	flag.StringVar(&file, "file", "", "Input file")
	flag.StringVar(&daemonUrl, "url", "localhost:9042", "Cassandra URL.")
	flag.IntVar(&workers, "workers", 1, "Number of concurrent requests to make.")
	flag.StringVar(&aggrPlanLabel, "aggregation-plan", "", "Aggregation plan (choices: server, client)")
	flag.IntVar(&subQueryParallelism, "subquery-workers", 1, "Number of concurrent subqueries to make (because the client does a scatter+gather operation).")
	flag.DurationVar(&requestTimeout, "request-timeout", 1*time.Second, "Maximum request timeout.")
	flag.DurationVar(&csiTimeout, "client-side-index-timeout", 10*time.Second, "Maximum client-side index timeout (only used at initialization).")
	flag.IntVar(&debug, "debug", 0, "Whether to print debug messages.")
	flag.Int64Var(&limit, "limit", -1, "Limit the number of queries to send.")
	flag.Uint64Var(&burnIn, "burn-in", 0, "Number of queries to ignore before collecting statistics.")
	flag.Uint64Var(&printInterval, "print-interval", 100, "Print timing stats to stderr after this many queries (0 to disable)")
	flag.BoolVar(&prettyPrintResponses, "print-responses", false, "Pretty print response bodies (for correctness checking) (default false).")
	flag.StringVar(&memProfile, "memprofile", "", "Write a memory profile to this file.")
	flag.StringVar(&reportDatabase, "report-database", "database_benchmarks", "Database name where to store result metrics.")
	flag.StringVar(&reportHost, "report-host", "", "Host to send result metrics.")
	flag.StringVar(&reportUser, "report-user", "", "User for Host to send result metrics.")
	flag.StringVar(&reportPassword, "report-password", "", "User password for Host to send result metrics.")
	flag.StringVar(&reportTagsCSV, "report-tags", "", "Comma separated k:v tags to send  alongside result metrics.")

	flag.Parse()

	if _, ok := aggrPlanChoices[aggrPlanLabel]; !ok {
		log.Fatal("invalid aggregation plan")
	}
	aggrPlan = aggrPlanChoices[aggrPlanLabel]

	if reportHost != "" {
		fmt.Printf("results report destination: %v\n", reportHost)
		fmt.Printf("results report database: %v\n", reportDatabase)

		var err error
		reportHostname, err = os.Hostname()
		if err != nil {
			log.Fatalf("os.Hostname() error: %s", err.Error())
		}
		fmt.Printf("hostname for results report: %v\n", reportHostname)

		if reportTagsCSV != "" {
			pairs := strings.Split(reportTagsCSV, ",")
			for _, pair := range pairs {
				fields := strings.SplitN(pair, ":", 2)
				tagpair := [2]string{fields[0], fields[1]}
				reportTags = append(reportTags, tagpair)
			}
		}
		fmt.Printf("results report tags: %v\n", reportTags)
	}

	if file != "" {
		if f, err := os.Open(file); err == nil {
			sourceReader = f
		} else {
			log.Fatalf("Error opening %s: %v\n", file, err)
		}
	}
	if sourceReader == nil {
		sourceReader = os.Stdin
	}
}

func main() {
	// Make pools to minimize heap usage:
	queryPool = sync.Pool{
		New: func() interface{} {
			return &HLQuery{
				HumanLabel:       make([]byte, 0, 1024),
				HumanDescription: make([]byte, 0, 1024),
				MeasurementName:  make([]byte, 0, 1024),
				FieldName:        make([]byte, 0, 1024),
				AggregationType:  make([]byte, 0, 1024),
				TagSets:          make([][]string, 0, 10),
			}
		},
	}

	statPool = sync.Pool{
		New: func() interface{} {
			return &Stat{
				Label: make([]byte, 0, 1024),
			}
		},
	}

	// Make client-side index:
	csi := NewClientSideIndex(FetchSeriesCollection(daemonUrl, csiTimeout))

	// Make database connection pool:
	session := NewCassandraSession(daemonUrl, requestTimeout)
	defer session.Close()

	// Make data and stat channels:
	hlQueryChan = make(chan *HLQuery, workers)
	statChan = make(chan *Stat, workers)

	// Launch the stats processor:
	statGroup.Add(1)
	go processStats()

	// Launch the query processors:
	qe := NewHLQueryExecutor(session, csi, debug)
	for i := 0; i < workers; i++ {
		workersGroup.Add(1)
		go processQueries(qe)
	}

	// Read in jobs, closing the job channel when done:
	input := bufio.NewReaderSize(sourceReader, 1<<20)
	wallStart := time.Now()
	scan(input)
	close(hlQueryChan)

	// Block for workers to finish sending requests, closing the stats
	// channel when done:
	workersGroup.Wait()
	close(statChan)

	// Wait on the stat collector to finish (and print its results):
	statGroup.Wait()

	wallEnd := time.Now()
	wallTook := wallEnd.Sub(wallStart)
	_, err := fmt.Printf("wall clock time: %fsec\n", float64(wallTook.Nanoseconds())/1e9)
	if err != nil {
		log.Fatal(err)
	}

	// (Optional) create a memory profile:
	if memProfile != "" {
		f, err := os.Create(memProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.WriteHeapProfile(f)
		f.Close()
	}

	if reportHost != "" {
		//append db specific tags to custom tags
		reportTags = append(reportTags, [2]string{"aggregation_plan", aggrPlanLabel})
		reportTags = append(reportTags, [2]string{"subquery_workers", strconv.Itoa(subQueryParallelism)})
		reportTags = append(reportTags, [2]string{"request_timeout", strconv.Itoa(int(requestTimeout.Seconds()))})
		reportTags = append(reportTags, [2]string{"client_side_index_timeout", strconv.Itoa(int(csiTimeout.Seconds()))})

		reportParams := &report.QueryReportParams{
			ReportParams: report.ReportParams{
				DBType:             "Cassandra",
				ReportDatabaseName: reportDatabase,
				ReportHost:         reportHost,
				ReportUser:         reportUser,
				ReportPassword:     reportPassword,
				ReportTags:         reportTags,
				Hostname:           reportHostname,
				DestinationUrl:     daemonUrl,
				Workers:            workers,
				ItemLimit:          int(limit),
			},
			BurnIn: int64(burnIn),
		}
		err = report.ReportQueryResult(reportParams, reportQueryStat.Min, reportQueryStat.Mean, reportQueryStat.Max, reportQueryStat.Count, wallTook)

		if err != nil {
			log.Fatal(err)
		}
	}
}

// scan reads encoded Queries and places them onto the workqueue.
func scan(r io.Reader) {
	dec := gob.NewDecoder(r)

	n := int64(0)
	for {
		if limit >= 0 && n >= limit {
			break
		}

		q := queryPool.Get().(*HLQuery)
		err := dec.Decode(q)
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}

		q.ID = n
		q.ForceUTC()

		hlQueryChan <- q

		n++
	}
}

// processQueries reads byte buffers from hlQueryChan and writes them to the
// target server, while tracking latency.
func processQueries(qc *HLQueryExecutor) {
	opts := HLQueryExecutorDoOptions{
		AggregationPlan:      aggrPlan,
		Debug:                debug,
		PrettyPrintResponses: prettyPrintResponses,
	}
	labels := map[string][][]byte{}
	for q := range hlQueryChan {
		qpLagMs, reqLagMs, err := qc.Do(q, opts)

		// if needed, prepare stat labels:
		if _, ok := labels[string(q.HumanLabel)]; !ok {
			labels[string(q.HumanLabel)] = [][]byte{
				q.HumanLabel,
				[]byte(fmt.Sprintf("%s-qp", q.HumanLabel)),
				[]byte(fmt.Sprintf("%s-req", q.HumanLabel)),
			}
		}
		ls := labels[string(q.HumanLabel)]

		// total lag stat:
		stat := statPool.Get().(*Stat)
		stat.Init(ls[0], qpLagMs+reqLagMs, true)
		statChan <- stat

		// qp lag stat:
		stat = statPool.Get().(*Stat)
		stat.Init(ls[1], qpLagMs, false)
		statChan <- stat

		// req lag stat:
		stat = statPool.Get().(*Stat)
		stat.Init(ls[2], reqLagMs, false)
		statChan <- stat

		queryPool.Put(q)
		if err != nil {
			log.Fatalf("Error during request: %s\n", err.Error())
		}
	}
	workersGroup.Done()
}

// processStats collects latency results, aggregating them into summary
// statistics. Optionally, they are printed to stderr at regular intervals.
func processStats() {
	statMapping := statsMap{}

	i := uint64(0)
	for stat := range statChan {
		if i < burnIn {
			i++
			statPool.Put(stat)
			continue
		} else if i == burnIn && burnIn > 0 {
			_, err := fmt.Fprintf(os.Stderr, "burn-in complete after %d queries with %d workers\n", burnIn, workers)
			if err != nil {
				log.Fatal(err)
			}
		}
		if _, ok := statMapping[string(stat.Label)]; !ok {
			statMapping[string(stat.Label)] = &StatGroup{}
		}

		statMapping[string(stat.Label)].Push(stat.Value)

		if stat.IsActual {
			i++
			reportQueryStat.Push(stat.Value)
		}

		statPool.Put(stat)

		// print stats to stderr (if printInterval is greater than zero):
		if printInterval > 0 && i > 0 && i%printInterval == 0 && (int64(i) < limit || limit < 0) {
			_, err := fmt.Fprintf(os.Stderr, "after %d queries with %d workers:\n", i-burnIn, workers)
			if err != nil {
				log.Fatal(err)
			}
			fprintStats(os.Stderr, statMapping)
			_, err = fmt.Fprintf(os.Stderr, "\n")
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	// the final stats output goes to stdout:
	_, err := fmt.Printf("run complete after %d queries with %d workers:\n", i-burnIn, workers)
	if err != nil {
		log.Fatal(err)
	}
	fprintStats(os.Stdout, statMapping)
	statGroup.Done()
}

// fprintStats pretty-prints stats to the given writer.
func fprintStats(w io.Writer, statGroups statsMap) {
	maxKeyLength := 0
	keys := make([]string, 0, len(statGroups))
	for k := range statGroups {
		if len(k) > maxKeyLength {
			maxKeyLength = len(k)
		}
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		v := statGroups[k]
		minRate := 1e3 / v.Min
		meanRate := 1e3 / v.Mean
		maxRate := 1e3 / v.Max
		paddedKey := fmt.Sprintf("%s", k)
		for len(paddedKey) < maxKeyLength {
			paddedKey += " "
		}
		_, err := fmt.Fprintf(w, "%s : min: %8.2fms (%7.2f/sec), mean: %8.2fms (%7.2f/sec), max: %7.2fms (%6.2f/sec), count: %8d, sum: %5.1fsec \n", paddedKey, v.Min, minRate, v.Mean, meanRate, v.Max, maxRate, v.Count, v.Sum/1e3)
		if err != nil {
			log.Fatal(err)
		}
	}

}
<|endoftext|>


<commit_before>package tunnel

import (
	"bytes"
	"fmt"
	"net/http"
	"os"
	"reflect"
	"strconv"
	"strings"
	"time"

	"gopkg.in/inconshreveable/log15.v2"
)

var VV string

func SetVV(vv string) { VV = vv }

func writePid(file string) {
	if file != "" {
		_ = os.Remove(file)
		pid := os.Getpid()
		f, err := os.Create(file)
		if err != nil {
			log15.Crit("Can't create pidfile", "file", file, "err", err.Error())
			os.Exit(1)
		}
		_, err = f.WriteString(strconv.Itoa(pid) + "\n")
		if err != nil {
			log15.Crit("Can't write to pidfile", "file", file, "err", err.Error())
			os.Exit(1)
		}
		f.Close()
	}
}

// Set logging to use the file or syslog, one of the them must be "" else an error ensues
func makeLogger(pkg, file, facility string) log15.Logger {
	log := log15.New("pkg", pkg)
	if file != "" {
		if facility != "" {
			log.Crit("Can't log to syslog and logfile simultaneously")
			os.Exit(1)
		}
		log.Info("Switching logging", "file", file)
		h, err := log15.FileHandler(file, SimpleFormat())
		if err != nil {
			log.Crit("Can't create log file", "file", file, "err", err.Error())
			os.Exit(1)
		}
		log15.Root().SetHandler(h)
		log.Info("Started logging here")
	} else if facility != "" {
		log.Info("Switching logging to syslog", "facility", facility)
		h, err := log15.SyslogHandler(facility, SimpleFormat())
		if err != nil {
			log.Crit("Can't connect to syslog", "err", err.Error())
			os.Exit(1)
		}
		log15.Root().SetHandler(h)
		log.Info("Started logging here")
	} else {
		log.Info("WStunnel starting")
	}
	return log
}

const simpleTimeFormat = "2006-01-02 15:04:05"
const simpleMsgJust = 40

func SimpleFormat() log15.Format {
	return log15.FormatFunc(func(r *log15.Record) []byte {
		b := &bytes.Buffer{}
		lvl := strings.ToUpper(r.Lvl.String())
		fmt.Fprintf(b, "[%s] %s %s ", r.Time.Format(simpleTimeFormat), lvl, r.Msg)

		// try to justify the log output for short messages
		if len(r.Ctx) > 0 && len(r.Msg) < simpleMsgJust {
			b.Write(bytes.Repeat([]byte{' '}, simpleMsgJust-len(r.Msg)))
		}
		// print the keys logfmt style
		for i := 0; i < len(r.Ctx); i += 2 {
			if i != 0 {
				b.WriteByte(' ')
			}

			k, ok := r.Ctx[i].(string)
			v := formatLogfmtValue(r.Ctx[i+1])
			if !ok {
				k, v = "LOG_ERR", formatLogfmtValue(k)
			}

			// XXX: we should probably check that all of your key bytes aren't invalid
			fmt.Fprintf(b, "%s=%s", k, v)
		}

		b.WriteByte('\n')
		return b.Bytes()
	})
}

// copied from log15 https://github.com/inconshreveable/log15/blob/master/format.go#L203-L223
func formatLogfmtValue(value interface{}) string {
	if value == nil {
		return "nil"
	}

	value = formatShared(value)
	switch v := value.(type) {
	case bool:
		return strconv.FormatBool(v)
	case float32:
		return strconv.FormatFloat(float64(v), 'f', 3, 64)
	case float64:
		return strconv.FormatFloat(v, 'f', 3, 64)
	case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
		return fmt.Sprintf("%d", value)
	case string:
		return escapeString(v)
	default:
		return escapeString(fmt.Sprintf("%+v", value))
	}
}

// copied from log15 https://github.com/inconshreveable/log15/blob/master/format.go
func formatShared(value interface{}) (result interface{}) {
	defer func() {
		if err := recover(); err != nil {
			if v := reflect.ValueOf(value); v.Kind() == reflect.Ptr && v.IsNil() {
				result = "nil"
			} else {
				panic(err)
			}
		}
	}()

	switch v := value.(type) {
	case time.Time:
		return v.Format(simpleTimeFormat)

	case error:
		return v.Error()

	case fmt.Stringer:
		return v.String()

	default:
		return v
	}
}

// copied from log15 https://github.com/inconshreveable/log15/blob/master/format.go
func escapeString(s string) string {
	needQuotes := false
	e := bytes.Buffer{}
	e.WriteByte('"')
	for _, r := range s {
		if r <= ' ' || r == '=' || r == '"' {
			needQuotes = true
		}

		switch r {
		case '\\', '"':
			e.WriteByte('\\')
			e.WriteByte(byte(r))
		case '\n':
			e.WriteByte('\\')
			e.WriteByte('n')
		case '\r':
			e.WriteByte('\\')
			e.WriteByte('r')
		case '\t':
			e.WriteByte('\\')
			e.WriteByte('t')
		default:
			e.WriteRune(r)
		}
	}
	e.WriteByte('"')
	start, stop := 0, e.Len()
	if !needQuotes {
		start, stop = 1, stop-1
	}
	return string(e.Bytes()[start:stop])
}

func calcWsTimeout(tout int) time.Duration {
	var wsTimeout time.Duration
	if tout < 3 {
		wsTimeout = 3 * time.Second
	} else if tout > 600 {
		wsTimeout = 600 * time.Second
	} else {
		wsTimeout = time.Duration(tout) * time.Second
	}
	log15.Info("Websocket keep-alive timeout", "timeout", wsTimeout)
	return wsTimeout
}

// copy http headers over
func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}
<commit_msg>don't log timestamp with syslog<commit_after>package tunnel

import (
	"bytes"
	"fmt"
	"net/http"
	"os"
	"reflect"
	"strconv"
	"strings"
	"time"

	"gopkg.in/inconshreveable/log15.v2"
)

var VV string

func SetVV(vv string) { VV = vv }

func writePid(file string) {
	if file != "" {
		_ = os.Remove(file)
		pid := os.Getpid()
		f, err := os.Create(file)
		if err != nil {
			log15.Crit("Can't create pidfile", "file", file, "err", err.Error())
			os.Exit(1)
		}
		_, err = f.WriteString(strconv.Itoa(pid) + "\n")
		if err != nil {
			log15.Crit("Can't write to pidfile", "file", file, "err", err.Error())
			os.Exit(1)
		}
		f.Close()
	}
}

// Set logging to use the file or syslog, one of the them must be "" else an error ensues
func makeLogger(pkg, file, facility string) log15.Logger {
	log := log15.New("pkg", pkg)
	if file != "" {
		if facility != "" {
			log.Crit("Can't log to syslog and logfile simultaneously")
			os.Exit(1)
		}
		log.Info("Switching logging", "file", file)
		h, err := log15.FileHandler(file, SimpleFormat(true))
		if err != nil {
			log.Crit("Can't create log file", "file", file, "err", err.Error())
			os.Exit(1)
		}
		log15.Root().SetHandler(h)
		log.Info("Started logging here")
	} else if facility != "" {
		log.Info("Switching logging to syslog", "facility", facility)
		h, err := log15.SyslogHandler(facility, SimpleFormat(false))
		if err != nil {
			log.Crit("Can't connect to syslog", "err", err.Error())
			os.Exit(1)
		}
		log15.Root().SetHandler(h)
		log.Info("Started logging here")
	} else {
		log.Info("WStunnel starting")
	}
	return log
}

const simpleTimeFormat = "2006-01-02 15:04:05"
const simpleMsgJust = 40

func SimpleFormat(timestamps bool) log15.Format {
	return log15.FormatFunc(func(r *log15.Record) []byte {
		b := &bytes.Buffer{}
		lvl := strings.ToUpper(r.Lvl.String())
		if timestamps {
			fmt.Fprintf(b, "[%s] %s %s ", r.Time.Format(simpleTimeFormat), lvl, r.Msg)
		} else {
			fmt.Fprintf(b, "%s %s ", lvl, r.Msg)
		}

		// try to justify the log output for short messages
		if len(r.Ctx) > 0 && len(r.Msg) < simpleMsgJust {
			b.Write(bytes.Repeat([]byte{' '}, simpleMsgJust-len(r.Msg)))
		}
		// print the keys logfmt style
		for i := 0; i < len(r.Ctx); i += 2 {
			if i != 0 {
				b.WriteByte(' ')
			}

			k, ok := r.Ctx[i].(string)
			v := formatLogfmtValue(r.Ctx[i+1])
			if !ok {
				k, v = "LOG_ERR", formatLogfmtValue(k)
			}

			// XXX: we should probably check that all of your key bytes aren't invalid
			fmt.Fprintf(b, "%s=%s", k, v)
		}

		b.WriteByte('\n')
		return b.Bytes()
	})
}

// copied from log15 https://github.com/inconshreveable/log15/blob/master/format.go#L203-L223
func formatLogfmtValue(value interface{}) string {
	if value == nil {
		return "nil"
	}

	value = formatShared(value)
	switch v := value.(type) {
	case bool:
		return strconv.FormatBool(v)
	case float32:
		return strconv.FormatFloat(float64(v), 'f', 3, 64)
	case float64:
		return strconv.FormatFloat(v, 'f', 3, 64)
	case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
		return fmt.Sprintf("%d", value)
	case string:
		return escapeString(v)
	default:
		return escapeString(fmt.Sprintf("%+v", value))
	}
}

// copied from log15 https://github.com/inconshreveable/log15/blob/master/format.go
func formatShared(value interface{}) (result interface{}) {
	defer func() {
		if err := recover(); err != nil {
			if v := reflect.ValueOf(value); v.Kind() == reflect.Ptr && v.IsNil() {
				result = "nil"
			} else {
				panic(err)
			}
		}
	}()

	switch v := value.(type) {
	case time.Time:
		return v.Format(simpleTimeFormat)

	case error:
		return v.Error()

	case fmt.Stringer:
		return v.String()

	default:
		return v
	}
}

// copied from log15 https://github.com/inconshreveable/log15/blob/master/format.go
func escapeString(s string) string {
	needQuotes := false
	e := bytes.Buffer{}
	e.WriteByte('"')
	for _, r := range s {
		if r <= ' ' || r == '=' || r == '"' {
			needQuotes = true
		}

		switch r {
		case '\\', '"':
			e.WriteByte('\\')
			e.WriteByte(byte(r))
		case '\n':
			e.WriteByte('\\')
			e.WriteByte('n')
		case '\r':
			e.WriteByte('\\')
			e.WriteByte('r')
		case '\t':
			e.WriteByte('\\')
			e.WriteByte('t')
		default:
			e.WriteRune(r)
		}
	}
	e.WriteByte('"')
	start, stop := 0, e.Len()
	if !needQuotes {
		start, stop = 1, stop-1
	}
	return string(e.Bytes()[start:stop])
}

func calcWsTimeout(tout int) time.Duration {
	var wsTimeout time.Duration
	if tout < 3 {
		wsTimeout = 3 * time.Second
	} else if tout > 600 {
		wsTimeout = 600 * time.Second
	} else {
		wsTimeout = time.Duration(tout) * time.Second
	}
	log15.Info("Websocket keep-alive timeout", "timeout", wsTimeout)
	return wsTimeout
}

// copy http headers over
func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}
<|endoftext|>


<commit_before>package repository_test

import (
	"testing"

	"github.com/syou6162/go-active-learning/lib/example"
	"github.com/syou6162/go-active-learning/lib/model"
	"github.com/syou6162/go-active-learning/lib/repository"
)

func TestUpdateRelatedExamples(t *testing.T) {
	repo, err := repository.New()
	if err != nil {
		t.Errorf(err.Error())
	}
	defer repo.Close()

	e1 := example.NewExample("http://hoge1.com", model.POSITIVE)
	e2 := example.NewExample("http://hoge2.com", model.NEGATIVE)
	e3 := example.NewExample("http://hoge3.com", model.UNLABELED)
	examples := model.Examples{e1, e2, e3}
	for _, e := range examples {
		err = repo.UpdateOrCreateExample(e)
		if err != nil {
			t.Error(err)
		}
	}
	related := model.RelatedExamples{ExampleId: e1.Id, RelatedExampleIds: []int{e2.Id, e3.Id}}
	err = repo.UpdateRelatedExamples(related)
	if err != nil {
		t.Error(err)
	}

	{
		related, err := repo.FindRelatedExamples(e1)
		if err != nil {
			t.Error(err)
		}
		if len(related.RelatedExampleIds) != 2 {
			t.Error("len(related.RelatedExampleIds) must be 2")
		}
	}
}

func TestUpdateRelatedExamplesMyOwn(t *testing.T) {
	repo, err := repository.New()
	if err != nil {
		t.Errorf(err.Error())
	}
	defer repo.Close()

	e1 := example.NewExample("http://hoge1.com", model.POSITIVE)
	e2 := example.NewExample("http://hoge2.com", model.NEGATIVE)
	e3 := example.NewExample("http://hoge3.com", model.UNLABELED)
	examples := model.Examples{e1, e2, e3}
	for _, e := range examples {
		err = repo.UpdateOrCreateExample(e)
		if err != nil {
			t.Error(err)
		}
	}
	related := model.RelatedExamples{ExampleId: e1.Id, RelatedExampleIds: []int{e1.Id, e2.Id, e3.Id}}
	err = repo.UpdateRelatedExamples(related)
	if err == nil {
		t.Error("自身と同一のexample_idを持つ事例はrelated_example_idに追加できない")
	}
}
<commit_msg>関連エントリが0件の場合<commit_after>package repository_test

import (
	"testing"

	"github.com/syou6162/go-active-learning/lib/example"
	"github.com/syou6162/go-active-learning/lib/model"
	"github.com/syou6162/go-active-learning/lib/repository"
)

func TestUpdateRelatedExamples(t *testing.T) {
	repo, err := repository.New()
	if err != nil {
		t.Errorf(err.Error())
	}
	defer repo.Close()

	e1 := example.NewExample("http://hoge1.com", model.POSITIVE)
	e2 := example.NewExample("http://hoge2.com", model.NEGATIVE)
	e3 := example.NewExample("http://hoge3.com", model.UNLABELED)
	examples := model.Examples{e1, e2, e3}
	for _, e := range examples {
		err = repo.UpdateOrCreateExample(e)
		if err != nil {
			t.Error(err)
		}
	}
	related := model.RelatedExamples{ExampleId: e1.Id, RelatedExampleIds: []int{e2.Id, e3.Id}}
	err = repo.UpdateRelatedExamples(related)
	if err != nil {
		t.Error(err)
	}

	{
		related, err := repo.FindRelatedExamples(e1)
		if err != nil {
			t.Error(err)
		}
		if len(related.RelatedExampleIds) != 2 {
			t.Error("len(related.RelatedExampleIds) must be 2")
		}
	}
	{
		related, err := repo.FindRelatedExamples(e2)
		if err != nil {
			t.Error(err)
		}
		if len(related.RelatedExampleIds) != 0 {
			t.Error("len(related.RelatedExampleIds) must be 0")
		}
	}
}

func TestUpdateRelatedExamplesMyOwn(t *testing.T) {
	repo, err := repository.New()
	if err != nil {
		t.Errorf(err.Error())
	}
	defer repo.Close()

	e1 := example.NewExample("http://hoge1.com", model.POSITIVE)
	e2 := example.NewExample("http://hoge2.com", model.NEGATIVE)
	e3 := example.NewExample("http://hoge3.com", model.UNLABELED)
	examples := model.Examples{e1, e2, e3}
	for _, e := range examples {
		err = repo.UpdateOrCreateExample(e)
		if err != nil {
			t.Error(err)
		}
	}
	related := model.RelatedExamples{ExampleId: e1.Id, RelatedExampleIds: []int{e1.Id, e2.Id, e3.Id}}
	err = repo.UpdateRelatedExamples(related)
	if err == nil {
		t.Error("自身と同一のexample_idを持つ事例はrelated_example_idに追加できない")
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Red Hat, Inc, and individual contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"strings"

	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

type CommandCompleter struct {
	Root *cobra.Command
}

func (cc *CommandCompleter) Do(line []rune, pos int) (newLine [][]rune, offset int) {
	cmd := cc.Root
	index := strings.LastIndex(string(line[:pos]), " ") + 1
	word := string(line[:pos])
	if index > 0 {
		word = word[index:pos]
		var err error
		cmd, _, err = cc.Root.Find(strings.Split(string(line), " "))
		if err != nil {
			return
		}
	}
	for _, completion := range completions(word, cmd) {
		if len(word) >= len(completion) {
			if len(word) == len(completion) {
				newLine = append(newLine, []rune{' '})
			} else {
				newLine = append(newLine, []rune(completion))
			}
			offset = len(completion)
		} else {
			newLine = append(newLine, []rune(completion)[len(word):])
			offset = len(word)
		}

	}
	return
}

func completions(prefix string, cmd *cobra.Command) []string {
	candidates := []string{}
	if strings.HasPrefix(prefix, "-") {
		candidates = flags(cmd)
	} else {
		candidates = subCommands(cmd)
		if len(candidates) == 0 {
			candidates = resourceTypes(cmd)
		}
	}
	return complete(prefix, candidates)
}

func complete(prefix string, candidates []string) (results []string) {
	for _, s := range candidates {
		if strings.HasPrefix(s, prefix) {
			results = append(results, s)
		}
	}
	return
}

func subCommands(cmd *cobra.Command) []string {
	prefixes := make([]string, len(cmd.Commands()))
	for i, c := range cmd.Commands() {
		prefixes[i] = c.Name()
	}
	return prefixes
}

func resourceTypes(cmd *cobra.Command) []string {
	return cmd.ValidArgs
}

func flags(cmd *cobra.Command) []string {
	flags := []string{}
	fn := func(f *pflag.Flag) {
		flag := "--" + f.Name
		if len(f.NoOptDefVal) == 0 {
			flag += "="
		}
		flags = append(flags, flag)
	}
	cmd.NonInheritedFlags().VisitAll(fn)
	cmd.InheritedFlags().VisitAll(fn)
	return flags
}
<commit_msg>Don't suggest deprecated or hidden flags for completion<commit_after>// Copyright 2016 Red Hat, Inc, and individual contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"strings"

	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

type CommandCompleter struct {
	Root *cobra.Command
}

func (cc *CommandCompleter) Do(line []rune, pos int) (newLine [][]rune, offset int) {
	cmd := cc.Root
	index := strings.LastIndex(string(line[:pos]), " ") + 1
	word := string(line[:pos])
	if index > 0 {
		word = word[index:pos]
		var err error
		cmd, _, err = cc.Root.Find(strings.Split(string(line), " "))
		if err != nil {
			return
		}
	}
	for _, completion := range completions(word, cmd) {
		if len(word) >= len(completion) {
			if len(word) == len(completion) {
				newLine = append(newLine, []rune{' '})
			} else {
				newLine = append(newLine, []rune(completion))
			}
			offset = len(completion)
		} else {
			newLine = append(newLine, []rune(completion)[len(word):])
			offset = len(word)
		}

	}
	return
}

func completions(prefix string, cmd *cobra.Command) []string {
	candidates := []string{}
	if strings.HasPrefix(prefix, "-") {
		candidates = flags(cmd)
	} else {
		candidates = subCommands(cmd)
		if len(candidates) == 0 {
			candidates = resourceTypes(cmd)
		}
	}
	return complete(prefix, candidates)
}

func complete(prefix string, candidates []string) (results []string) {
	for _, s := range candidates {
		if strings.HasPrefix(s, prefix) {
			results = append(results, s)
		}
	}
	return
}

func subCommands(cmd *cobra.Command) []string {
	prefixes := make([]string, len(cmd.Commands()))
	for i, c := range cmd.Commands() {
		prefixes[i] = c.Name()
	}
	return prefixes
}

func resourceTypes(cmd *cobra.Command) []string {
	return cmd.ValidArgs
}

func flags(cmd *cobra.Command) []string {
	flags := []string{}
	fn := func(f *pflag.Flag) {
		if len(f.Deprecated) > 0 || f.Hidden {
			return
		}
		flag := "--" + f.Name
		if len(f.NoOptDefVal) == 0 {
			flag += "="
		}
		flags = append(flags, flag)
	}
	cmd.NonInheritedFlags().VisitAll(fn)
	cmd.InheritedFlags().VisitAll(fn)
	return flags
}
<|endoftext|>


<commit_before>package conf

import (
	"fmt"
	"github.com/spf13/viper"
)

func GetConf() map[string]interface{} {
	viper.SetConfigName("microb_dev_config_pg")
	viper.AddConfigPath(".")
	viper.SetDefault("http_host", ":8080")
	viper.SetDefault("centrifugo_host", "localhost")
	viper.SetDefault("centrifugo_port", 8001)
	viper.SetDefault("db_type", "rethinkdb")
	viper.SetDefault("db_host", "localhost")
	viper.SetDefault("db_port", 28015)
	viper.SetDefault("db_user", "admin")
	viper.SetDefault("db_password", "")
	viper.SetDefault("hits_log", true)
	viper.SetDefault("hits_monitor", true)
	hits_channels := []string{"$microb_hits"}
	viper.SetDefault("hits_channels", hits_channels)
	viper.SetDefault("commands_transport", []string{"default"})
	err := viper.ReadInConfig()
	if err != nil {
	    panic(fmt.Errorf("Fatal error config file: %s \n", err))
	}
	conf := make(map[string]interface{})
	conf["http_host"] = viper.Get("http_host")
	conf["centrifugo_host"] = viper.Get("centrifugo_host")
	conf["centrifugo_port"] = viper.Get("centrifugo_port")
	conf["centrifugo_secret_key"] = viper.Get("centrifugo_secret_key")
	conf["db_type"] = viper.Get("db_type")
	conf["db_host"] = viper.Get("db_host")
	conf["db_port"] = viper.Get("db_port")
	conf["db_user"] = viper.Get("db_user")
	conf["db_password"] = viper.Get("db_password")
	conf["domain"] = viper.Get("domain")
	conf["hits_log"] = viper.Get("hits_log")
	conf["hits_monitor"] = viper.Get("hits_monitor")
	conf["hits_channels"] = viper.Get("hits_channels")
	conf["commands_transport"] = viper.Get("commands_transport")
	return conf
}

var Config = conf.GetConf()

func commandsTransports() []string {
	var ts []string
	cts := Config["commands_transport"].([]string)
	// check for defaults
	is_default := false
	for _, transp := range cts {
		if transp == "default" {
			is_default = true
			if Config["db_type"].(string) == "rethinkdb" {
				ts = []string{"changefeeds"}
			}
		}
	}
	if is_default == false {
		ts = cts
	}
	return ts
}

func ListenToChangefeeds() bool {
	listen := false
	transports := commandsTransports()
	for _, val := range transports {
		if val == "changefeeds" {
			listen = true
			break
		}
	}
	return listen
}
<commit_msg>Moved some functions in conf<commit_after>package conf

import (
	"fmt"
	"github.com/spf13/viper"
)

func GetConf() map[string]interface{} {
	viper.SetConfigName("microb_dev_config_pg")
	viper.AddConfigPath(".")
	viper.SetDefault("http_host", ":8080")
	viper.SetDefault("centrifugo_host", "localhost")
	viper.SetDefault("centrifugo_port", 8001)
	viper.SetDefault("db_type", "rethinkdb")
	viper.SetDefault("db_host", "localhost")
	viper.SetDefault("db_port", 28015)
	viper.SetDefault("db_user", "admin")
	viper.SetDefault("db_password", "")
	viper.SetDefault("hits_log", true)
	viper.SetDefault("hits_monitor", true)
	hits_channels := []string{"$microb_hits"}
	viper.SetDefault("hits_channels", hits_channels)
	viper.SetDefault("commands_transport", []string{"default"})
	err := viper.ReadInConfig()
	if err != nil {
	    panic(fmt.Errorf("Fatal error config file: %s \n", err))
	}
	conf := make(map[string]interface{})
	conf["http_host"] = viper.Get("http_host")
	conf["centrifugo_host"] = viper.Get("centrifugo_host")
	conf["centrifugo_port"] = viper.Get("centrifugo_port")
	conf["centrifugo_secret_key"] = viper.Get("centrifugo_secret_key")
	conf["db_type"] = viper.Get("db_type")
	conf["db_host"] = viper.Get("db_host")
	conf["db_port"] = viper.Get("db_port")
	conf["db_user"] = viper.Get("db_user")
	conf["db_password"] = viper.Get("db_password")
	conf["domain"] = viper.Get("domain")
	conf["hits_log"] = viper.Get("hits_log")
	conf["hits_monitor"] = viper.Get("hits_monitor")
	conf["hits_channels"] = viper.Get("hits_channels")
	conf["commands_transport"] = viper.Get("commands_transport")
	return conf
}

var Config = GetConf()

func commandsTransports() []string {
	var ts []string
	cts := Config["commands_transport"].([]string)
	// check for defaults
	is_default := false
	for _, transp := range cts {
		if transp == "default" {
			is_default = true
			if Config["db_type"].(string) == "rethinkdb" {
				ts = []string{"changefeeds"}
			}
		}
	}
	if is_default == false {
		ts = cts
	}
	return ts
}

func ListenToChangefeeds() bool {
	listen := false
	transports := commandsTransports()
	for _, val := range transports {
		if val == "changefeeds" {
			listen = true
			break
		}
	}
	return listen
}
<|endoftext|>


<commit_before>package smtpapi

import (
	"encoding/json"
	"reflect"
	"testing"
)

func Test_Adds(t *testing.T) {
	validHeader, _ := json.Marshal([]byte(`{"to":["test@email.com"],
	"sub":{"subKey":["subValue"]},
	"section":{"testSection":"sectionValue"},
	"category":["testCategory"],
	"unique_args":{"testUnique":"uniqueValue"},
	"filters":{"testFilter":{"settings":{"filter":"filterValue"}}}}`))

	headers := NewSMTPAPIHeader()
	headers.AddTo("test@email.com")
	headers.AddSubstitution("subKey", "subValue")
	headers.AddSection("testSection", "sectionValue")
	headers.AddCategory("testCategory")
	headers.AddUniqueArg("testUnique", "uniqueValue")
	headers.AddFilter("testFilter", "filter", "filterValue")
	if h, e := headers.JsonString(); e != nil {
		t.Errorf("Error! %s", e)
	} else {
		testHeader, _ := json.Marshal([]byte(h))
		if reflect.DeepEqual(testHeader, validHeader) {
			t.Logf("Success")
		} else {
			t.Errorf("Invalid output")
		}
	}
}
<commit_msg>added tests to sets<commit_after>package smtpapi

import (
	"encoding/json"
	"reflect"
	"testing"
)

func Test_Adds(t *testing.T) {
	validHeader, _ := json.Marshal([]byte(`{"to":["test@email.com"],"sub":{"subKey":["subValue"]},"section":{"testSection":"sectionValue"},"category":["testCategory"],"unique_args":{"testUnique":"uniqueValue"},"filters":{"testFilter":{"settings":{"filter":"filterValue"}}}}`))
	headers := NewSMTPAPIHeader()
	headers.AddTo("test@email.com")
	headers.AddSubstitution("subKey", "subValue")
	headers.AddSection("testSection", "sectionValue")
	headers.AddCategory("testCategory")
	headers.AddUniqueArg("testUnique", "uniqueValue")
	headers.AddFilter("testFilter", "filter", "filterValue")
	if h, e := headers.JsonString(); e != nil {
		t.Errorf("Error! %s", e)
	} else {
		testHeader, _ := json.Marshal([]byte(h))
		if reflect.DeepEqual(testHeader, validHeader) {
			t.Logf("Success")
		} else {
			t.Errorf("Invalid headers")
		}
	}
}

func Test_Sets(t *testing.T) {
	validHeader, _ := json.Marshal([]byte(`{"to":["test@email.com"],"sub":{"subKey":["subValue"]},"section":{"testSection":"sectionValue"},"category":["testCategory"],"unique_args":{"testUnique":"uniqueValue"},"filters":{"testFilter":{"settings":{"filter":"filterValue"}}}}`))
	headers := NewSMTPAPIHeader()
	headers.SetTos([]string{"test@email.com"})
	sub := make(map[string][]string)
	sub["subKey"] = []string{"subValue"}
	headers.SetSubstitutions(sub)
	sections := make(map[string]string)
	sections["testSection"] = "sectionValue"
	headers.SetSections(sections)
	headers.SetCategories([]string{"testCategory"})
	unique := make(map[string]string)
	unique["testUnique"] = "uniqueValue"
	headers.SetUniqueArgs(unique)
	headers.AddFilter("testFilter", "filter", "filterValue")
	if h, e := headers.JsonString(); e != nil {
		t.Errorf("Error! %s", e)
	} else {
		testHeader, _ := json.Marshal([]byte(h))
		if reflect.DeepEqual(testHeader, validHeader) {
			t.Logf("Success")
		} else {
			t.Errorf("Invalid headers")
		}
	}
}
<|endoftext|>


<commit_before>package generic

import (
	"encoding/json"
	"testing"
	"time"
)

func TestTimeJsonMarshal(t *testing.T) {
	v := time.Now()
	tt := Time{
		ValidFlag: true,
		time:      v,
	}
	expected := `"` + v.Format(time.RFC3339Nano) + `"`
	actual, err := json.Marshal(tt)
	if err != nil {
		t.Errorf("Not Expected error when json.Marshal. error:%v", err.Error())
	}
	if string(actual) != expected {
		t.Errorf("actual:%s, expected:%s", string(actual), expected)
	}
}

func TestTimeJsonMarshalValidFalse(t *testing.T) {
	tt := Time{
		ValidFlag: false,
		time:      time.Now(),
	}
	expected := []byte("null")
	actual, err := json.Marshal(tt)
	if err != nil {
		t.Errorf("Not Expected error when json.Marshal. error:%v", err.Error())
	}
	if string(actual) != string(expected) {
		t.Errorf("actual:%v, expected:%v", actual, expected)
	}
}

func TestTimeJsonUnmarshal(t *testing.T) {
	v := time.Now()
	in, _ := v.MarshalJSON()
	tt := Time{}
	if err := tt.UnmarshalJSON(in); err != nil {
		t.Errorf("Not Expected error when json.Unmarshal. error:%v", err.Error())
	}
	if !tt.Valid() {
		t.Error("ValidFlag should be TRUE")
	}
	if tt.Time() != v {
		t.Errorf("actual:%v, expected:%v", tt.Time(), v)
	}
}

func TestTimeJsonUnmarshalNil(t *testing.T) {
	tt := Time{}
	if err := tt.UnmarshalJSON(nil); err != nil {
		t.Errorf("Not Expected error when json.Unmarshal. error:%v", err.Error())
	}
	if tt.Valid() {
		t.Error("ValidFlag should be FALSE")
	}
	if tt.Time() != time.Unix(0, 0) {
		t.Errorf("actual:%v, expected:%v", tt.Time(), time.Unix(0, 0))
	}
}

func TestTimeSetNil(t *testing.T) {
	tt := Time{}
	err := tt.Set(nil)
	if err != nil {
		t.Errorf("Not Expected error. error:%s", err.Error())
	}
	if _, err = tt.Value(); err != nil {
		t.Errorf("This value should return nil. error:%s", err.Error())
	}
}

func TestTimeSetTime(t *testing.T) {
	v := time.Now()
	tt := Time{}
	err := tt.Set(v)
	if err != nil {
		t.Errorf("Not Expected error")
	}
	if tt.Weak() == nil {
		t.Errorf("This value should return nil. error:%#v", tt.Weak())
	}
}

func TestTimeSetInt64(t *testing.T) {
	var v int64 = 1367059792
	tt := Time{}
	err := tt.Set(v)
	if err == nil {
		t.Errorf("Not Expected error")
	}
	if tt.Weak() != nil {
		t.Errorf("This value should return nil. error:%#v", tt.Weak())
	}
}

func TestTimeSetNumericString(t *testing.T) {
	v := "1467059792"
	tt := Time{}
	err := tt.Set(v)
	if err == nil {
		t.Errorf("Expected error.")
	}
	if tt.Weak() != nil {
		t.Errorf("This value should return nil. error:%#v", tt.Weak())
	}
}

func TestTimeSetNonNumericString(t *testing.T) {
	v := "a"
	tt := Time{}
	err := tt.Set(v)
	if err == nil {
		t.Error("Expected error.")
	}
	if tt.Weak() != nil {
		t.Errorf("This value should return nil. error:%#v", tt.Weak())
	}
}

func TestTimeSetBool(t *testing.T) {
	v := true
	tt := Time{}
	err := tt.Set(v)
	if err == nil {
		t.Error("Expected error.")
	}
	if tt.Weak() != nil {
		t.Errorf("This value should return nil. error:%#v", tt.Weak())
	}
}

func TestTimeString(t *testing.T) {
	var expected = time.Now()
	tt := Time{}
	tt.Set(expected)
	if tt.String() != expected.String() {
		t.Errorf("actual:%s, expected:%s", tt.String(), expected.String())
	}
}

func TestTimeStringInvalid(t *testing.T) {
	tt := Time{
		ValidFlag: false,
		time:      time.Now(),
	}
	if tt.String() != "" {
		t.Errorf("expected empty string, actual:%s", tt.String())
	}
}
<commit_msg>fix test<commit_after>package generic

import (
	"encoding/json"
	"testing"
	"time"
)

func TestTimeJsonMarshal(t *testing.T) {
	v := time.Now()
	tt := Time{
		ValidFlag: true,
		time:      v,
	}
	expected := `"` + v.Format(time.RFC3339Nano) + `"`
	actual, err := json.Marshal(tt)
	if err != nil {
		t.Errorf("Not Expected error when json.Marshal. error:%v", err.Error())
	}
	if string(actual) != expected {
		t.Errorf("actual:%s, expected:%s", string(actual), expected)
	}
}

func TestTimeJsonMarshalValidFalse(t *testing.T) {
	tt := Time{
		ValidFlag: false,
		time:      time.Now(),
	}
	expected := []byte("null")
	actual, err := json.Marshal(tt)
	if err != nil {
		t.Errorf("Not Expected error when json.Marshal. error:%v", err.Error())
	}
	if string(actual) != string(expected) {
		t.Errorf("actual:%v, expected:%v", actual, expected)
	}
}

func TestTimeJsonUnmarshal(t *testing.T) {
	v := time.Now()
	in, _ := v.MarshalJSON()
	tt := Time{}
	if err := tt.UnmarshalJSON(in); err != nil {
		t.Errorf("Not Expected error when json.Unmarshal. error:%v", err.Error())
	}
	if !tt.Valid() {
		t.Error("ValidFlag should be TRUE")
	}
	if tt.Time().String() != v.String() {
		t.Errorf("actual:%v, expected:%v", tt.Time(), v)
	}
}

func TestTimeJsonUnmarshalNil(t *testing.T) {
	tt := Time{}
	if err := tt.UnmarshalJSON(nil); err != nil {
		t.Errorf("Not Expected error when json.Unmarshal. error:%v", err.Error())
	}
	if tt.Valid() {
		t.Error("ValidFlag should be FALSE")
	}
	if tt.Time() != time.Unix(0, 0) {
		t.Errorf("actual:%v, expected:%v", tt.Time(), time.Unix(0, 0))
	}
}

func TestTimeSetNil(t *testing.T) {
	tt := Time{}
	err := tt.Set(nil)
	if err != nil {
		t.Errorf("Not Expected error. error:%s", err.Error())
	}
	if _, err = tt.Value(); err != nil {
		t.Errorf("This value should return nil. error:%s", err.Error())
	}
}

func TestTimeSetTime(t *testing.T) {
	v := time.Now()
	tt := Time{}
	err := tt.Set(v)
	if err != nil {
		t.Errorf("Not Expected error")
	}
	if tt.Weak() == nil {
		t.Errorf("This value should return nil. error:%#v", tt.Weak())
	}
}

func TestTimeSetInt64(t *testing.T) {
	var v int64 = 1367059792
	tt := Time{}
	err := tt.Set(v)
	if err == nil {
		t.Errorf("Not Expected error")
	}
	if tt.Weak() != nil {
		t.Errorf("This value should return nil. error:%#v", tt.Weak())
	}
}

func TestTimeSetNumericString(t *testing.T) {
	v := "1467059792"
	tt := Time{}
	err := tt.Set(v)
	if err == nil {
		t.Errorf("Expected error.")
	}
	if tt.Weak() != nil {
		t.Errorf("This value should return nil. error:%#v", tt.Weak())
	}
}

func TestTimeSetNonNumericString(t *testing.T) {
	v := "a"
	tt := Time{}
	err := tt.Set(v)
	if err == nil {
		t.Error("Expected error.")
	}
	if tt.Weak() != nil {
		t.Errorf("This value should return nil. error:%#v", tt.Weak())
	}
}

func TestTimeSetBool(t *testing.T) {
	v := true
	tt := Time{}
	err := tt.Set(v)
	if err == nil {
		t.Error("Expected error.")
	}
	if tt.Weak() != nil {
		t.Errorf("This value should return nil. error:%#v", tt.Weak())
	}
}

func TestTimeString(t *testing.T) {
	var expected = time.Now()
	tt := Time{}
	tt.Set(expected)
	if tt.String() != expected.String() {
		t.Errorf("actual:%s, expected:%s", tt.String(), expected.String())
	}
}

func TestTimeStringInvalid(t *testing.T) {
	tt := Time{
		ValidFlag: false,
		time:      time.Now(),
	}
	if tt.String() != "" {
		t.Errorf("expected empty string, actual:%s", tt.String())
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package boot

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"
)

var createResourceCmd = &cobra.Command{
	Use:   "create-resource",
	Short: "Creates an API resource",
	Long:  `Creates an API resource`,
	Run:   RunCreateResource,
}

func AddCreateResource(cmd *cobra.Command) {
	createResourceCmd.Flags().StringVar(&groupName, "group", "", "name of the API group")
	createResourceCmd.Flags().StringVar(&versionName, "version", "", "name of the API version")
	createResourceCmd.Flags().StringVar(&kindName, "kind", "", "name of the API kind to create")
	createResourceCmd.Flags().StringVar(&resourceName, "resource", "", "name of the API resource to create, plural name of the kind")
	createResourceCmd.Flags().StringVar(&copyright, "copyright", "", "path to copyright file.  defaults to boilerplate.go.txt")
	createResourceCmd.Flags().StringVar(&domain, "domain", "", "domain the api group lives under")
	cmd.AddCommand(createResourceCmd)
}

func RunCreateResource(cmd *cobra.Command, args []string) {
	if len(domain) == 0 {
		fmt.Fprintf(os.Stderr, "apiserver-boot create-resource requires the --domain flag\n")
		os.Exit(-1)
	}
	if len(groupName) == 0 {
		fmt.Fprintf(os.Stderr, "apiserver-boot create-resource requires the --group flag\n")
		os.Exit(-1)
	}
	if len(versionName) == 0 {
		fmt.Fprintf(os.Stderr, "apiserver-boot create-resource requires the --version flag\n")
		os.Exit(-1)
	}
	if len(kindName) == 0 {
		fmt.Fprintf(os.Stderr, "apiserver-boot create-resource requires the --kind flag\n")
		os.Exit(-1)
	}
	if len(resourceName) == 0 {
		fmt.Fprintf(os.Stderr, "apiserver-boot create-resource requires the --resource flag\n")
		os.Exit(-1)
	}

	cr := getCopyright()
	createResource(cr)
}

func createResource(boilerplate string) {
	dir, err := os.Getwd()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(-1)
	}
	typesFileName := fmt.Sprintf("%s_types.go", strings.ToLower(kindName))
	path := filepath.Join(dir, "pkg", "apis", groupName, versionName, typesFileName)
	a := resourceTemplateArgs{
		boilerplate,
		domain,
		groupName,
		versionName,
		kindName,
		resourceName,
		Repo,
	}
	created := writeIfNotFound(path, "resource-template", resourceTemplate, a)
	if !created {
		fmt.Fprintf(os.Stderr,
			"API group version kind %s/%s/%s already exists.\n", groupName, versionName, kindName)
		os.Exit(-1)
	}

	typesFileName = fmt.Sprintf("%s_types_test.go", strings.ToLower(kindName))
	path = filepath.Join(dir, "pkg", "apis", groupName, versionName, typesFileName)
	created = writeIfNotFound(path, "resource-test-template", resourceTestTemplate, a)
	if !created {
		fmt.Fprintf(os.Stderr,
			"API group version kind %s/%s/%s already exists.\n", groupName, versionName, kindName)
		os.Exit(-1)
	}
}

type resourceTemplateArgs struct {
	BoilerPlate string
	Domain      string
	Group       string
	Version     string
	Kind        string
	Resource    string
	Repo        string
}

var resourceTemplate = `
{{.BoilerPlate}}

package {{.Version}}

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// +genclient=true

// +k8s:openapi-gen=true
// +resource={{.Resource}}
// {{.Kind}}
type {{.Kind}} struct {
	metav1.TypeMeta   ` + "`" + `json:",inline"` + "`" + `
	metav1.ObjectMeta ` + "`" + `json:"metadata,omitempty"` + "`" + `

	Spec   {{.Kind}}Spec   ` + "`" + `json:"spec,omitempty"` + "`" + `
	Status {{.Kind}}Status ` + "`" + `json:"status,omitempty"` + "`" + `
}

// {{.Kind}}Spec defines the desired state of {{.Kind}}
type {{.Kind}}Spec struct {
}

// {{.Kind}}Status defines the observed state of {{.Kind}}
type {{.Kind}}Status struct {
}

`

var resourceTestTemplate = `
{{.BoilerPlate}}

package {{.Version}}

import (
	"os"
	"testing"

	"k8s.io/client-go/rest"
	"github.com/kubernetes-incubator/apiserver-builder/pkg/test"

	"{{.Repo}}/pkg/apis"
	"{{.Repo}}/pkg/client/clientset_generated/clientset"
	"{{.Repo}}/pkg/openapi"
)

var testenv *test.TestEnvironment
var config *rest.Config
var client *clientset.Clientset

// Do Test Suite setup / teardown
func TestMain(m *testing.M) {
	testenv = test.NewTestEnvironment()
	config = testenv.Start(apis.GetAllApiBuilders(), openapi.GetOpenAPIDefinitions)
	client = clientset.NewForConfigOrDie(config)
	retCode := m.Run()
	testenv.Stop()
	os.Exit(retCode)
}

func TestCreateDelete{{.Kind}}(t *testing.T) {
}
`
<commit_msg>Update apiserver-boot create resource template to use new resource format.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package boot

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"
)

var createResourceCmd = &cobra.Command{
	Use:   "create-resource",
	Short: "Creates an API resource",
	Long:  `Creates an API resource`,
	Run:   RunCreateResource,
}

func AddCreateResource(cmd *cobra.Command) {
	createResourceCmd.Flags().StringVar(&groupName, "group", "", "name of the API group")
	createResourceCmd.Flags().StringVar(&versionName, "version", "", "name of the API version")
	createResourceCmd.Flags().StringVar(&kindName, "kind", "", "name of the API kind to create")
	createResourceCmd.Flags().StringVar(&resourceName, "resource", "", "name of the API resource to create, plural name of the kind")
	createResourceCmd.Flags().StringVar(&copyright, "copyright", "", "path to copyright file.  defaults to boilerplate.go.txt")
	createResourceCmd.Flags().StringVar(&domain, "domain", "", "domain the api group lives under")
	cmd.AddCommand(createResourceCmd)
}

func RunCreateResource(cmd *cobra.Command, args []string) {
	if len(domain) == 0 {
		fmt.Fprintf(os.Stderr, "apiserver-boot create-resource requires the --domain flag\n")
		os.Exit(-1)
	}
	if len(groupName) == 0 {
		fmt.Fprintf(os.Stderr, "apiserver-boot create-resource requires the --group flag\n")
		os.Exit(-1)
	}
	if len(versionName) == 0 {
		fmt.Fprintf(os.Stderr, "apiserver-boot create-resource requires the --version flag\n")
		os.Exit(-1)
	}
	if len(kindName) == 0 {
		fmt.Fprintf(os.Stderr, "apiserver-boot create-resource requires the --kind flag\n")
		os.Exit(-1)
	}
	if len(resourceName) == 0 {
		fmt.Fprintf(os.Stderr, "apiserver-boot create-resource requires the --resource flag\n")
		os.Exit(-1)
	}

	cr := getCopyright()
	createResource(cr)
}

func createResource(boilerplate string) {
	dir, err := os.Getwd()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(-1)
	}
	typesFileName := fmt.Sprintf("%s_types.go", strings.ToLower(kindName))
	path := filepath.Join(dir, "pkg", "apis", groupName, versionName, typesFileName)
	a := resourceTemplateArgs{
		boilerplate,
		domain,
		groupName,
		versionName,
		kindName,
		resourceName,
		Repo,
	}
	created := writeIfNotFound(path, "resource-template", resourceTemplate, a)
	if !created {
		fmt.Fprintf(os.Stderr,
			"API group version kind %s/%s/%s already exists.\n", groupName, versionName, kindName)
		os.Exit(-1)
	}

	typesFileName = fmt.Sprintf("%s_types_test.go", strings.ToLower(kindName))
	path = filepath.Join(dir, "pkg", "apis", groupName, versionName, typesFileName)
	created = writeIfNotFound(path, "resource-test-template", resourceTestTemplate, a)
	if !created {
		fmt.Fprintf(os.Stderr,
			"API group version kind %s/%s/%s already exists.\n", groupName, versionName, kindName)
		os.Exit(-1)
	}
}

type resourceTemplateArgs struct {
	BoilerPlate string
	Domain      string
	Group       string
	Version     string
	Kind        string
	Resource    string
	Repo        string
}

var resourceTemplate = `
{{.BoilerPlate}}

package {{.Version}}

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// +genclient=true

// +k8s:openapi-gen=true
// +resource:path={{.Resource}}
// {{.Kind}}
type {{.Kind}} struct {
	metav1.TypeMeta   ` + "`" + `json:",inline"` + "`" + `
	metav1.ObjectMeta ` + "`" + `json:"metadata,omitempty"` + "`" + `

	Spec   {{.Kind}}Spec   ` + "`" + `json:"spec,omitempty"` + "`" + `
	Status {{.Kind}}Status ` + "`" + `json:"status,omitempty"` + "`" + `
}

// {{.Kind}}Spec defines the desired state of {{.Kind}}
type {{.Kind}}Spec struct {
}

// {{.Kind}}Status defines the observed state of {{.Kind}}
type {{.Kind}}Status struct {
}

`

var resourceTestTemplate = `
{{.BoilerPlate}}

package {{.Version}}_test

import (
	"os"
	"testing"

	"k8s.io/client-go/rest"
	"github.com/kubernetes-incubator/apiserver-builder/pkg/test"

	"{{.Repo}}/pkg/apis"
	"{{.Repo}}/pkg/client/clientset_generated/clientset"
	"{{.Repo}}/pkg/openapi"
)

var testenv *test.TestEnvironment
var config *rest.Config
var client *clientset.Clientset

// Do Test Suite setup / teardown
func TestMain(m *testing.M) {
	testenv = test.NewTestEnvironment()
	config = testenv.Start(apis.GetAllApiBuilders(), openapi.GetOpenAPIDefinitions)
	client = clientset.NewForConfigOrDie(config)
	retCode := m.Run()
	testenv.Stop()
	os.Exit(retCode)
}

func TestCreateDelete{{.Kind}}(t *testing.T) {
}
`
<|endoftext|>