<commit_before>package cli

import (
	"fmt"
	"io"
	"sort"
	"strings"
	"text/template"

	cmds "github.com/ipfs/go-ipfs/commands"
)

const (
	requiredArg = "<%v>"
	optionalArg = "[<%v>]"
	variadicArg = "%v..."
	shortFlag   = "-%v"
	longFlag    = "--%v"
	optionType  = "(%v)"

	whitespace = "\r\n\t "

	indentStr = "  "
)

type helpFields struct {
	Indent      string
	Usage       string
	Path        string
	ArgUsage    string
	Tagline     string
	Arguments   string
	Options     string
	Synopsis    string
	Subcommands string
	Description string
	MoreHelp    bool
}

// TrimNewlines removes extra newlines from fields. This makes aligning
// commands easier. Below, the leading + tralining newlines are removed:
//	Synopsis: `
//	    ipfs config <key>          - Get value of <key>
//	    ipfs config <key> <value>  - Set value of <key> to <value>
//	    ipfs config --show         - Show config file
//	    ipfs config --edit         - Edit config file in $EDITOR
//	`
func (f *helpFields) TrimNewlines() {
	f.Path = strings.Trim(f.Path, "\n")
	f.ArgUsage = strings.Trim(f.ArgUsage, "\n")
	f.Tagline = strings.Trim(f.Tagline, "\n")
	f.Arguments = strings.Trim(f.Arguments, "\n")
	f.Options = strings.Trim(f.Options, "\n")
	f.Synopsis = strings.Trim(f.Synopsis, "\n")
	f.Subcommands = strings.Trim(f.Subcommands, "\n")
	f.Description = strings.Trim(f.Description, "\n")
}

// Indent adds whitespace the lines of fields.
func (f *helpFields) IndentAll() {
	indent := func(s string) string {
		if s == "" {
			return s
		}
		return indentString(s, indentStr)
	}

	f.Arguments = indent(f.Arguments)
	f.Options = indent(f.Options)
	f.Synopsis = indent(f.Synopsis)
	f.Subcommands = indent(f.Subcommands)
	f.Description = indent(f.Description)
}

const usageFormat = "{{if .Usage}}{{.Usage}}{{else}}{{.Path}}{{if .ArgUsage}} {{.ArgUsage}}{{end}} - {{.Tagline}}{{end}}"

const longHelpFormat = `USAGE
{{.Indent}}{{template "usage" .}}

{{if .Synopsis}}SYNOPSIS
{{.Synopsis}}

{{end}}{{if .Arguments}}ARGUMENTS

{{.Arguments}}

{{end}}{{if .Options}}OPTIONS

{{.Options}}

{{end}}{{if .Description}}DESCRIPTION

{{.Description}}

{{end}}{{if .Subcommands}}SUBCOMMANDS
{{.Subcommands}}

{{.Indent}}Use '{{.Path}} <subcmd> --help' for more information about each command.
{{end}}
`
const shortHelpFormat = `USAGE
{{.Indent}}{{template "usage" .}}
{{if .Synopsis}}
{{.Synopsis}}
{{end}}{{if .Description}}
{{.Description}}
{{end}}{{if .Subcommands}}
SUBCOMMANDS
{{.Subcommands}}
{{end}}{{if .MoreHelp}}
Use '{{.Path}} --help' for more information about this command.
{{end}}
`

var usageTemplate *template.Template
var longHelpTemplate *template.Template
var shortHelpTemplate *template.Template

func init() {
	usageTemplate = template.Must(template.New("usage").Parse(usageFormat))
	longHelpTemplate = template.Must(usageTemplate.New("longHelp").Parse(longHelpFormat))
	shortHelpTemplate = template.Must(usageTemplate.New("shortHelp").Parse(shortHelpFormat))
}

// LongHelp writes a formatted CLI helptext string to a Writer for the given command
func LongHelp(rootName string, root *cmds.Command, path []string, out io.Writer) error {
	cmd, err := root.Get(path)
	if err != nil {
		return err
	}

	pathStr := rootName
	if len(path) > 0 {
		pathStr += " " + strings.Join(path, " ")
	}

	fields := helpFields{
		Indent:      indentStr,
		Path:        pathStr,
		ArgUsage:    usageText(cmd),
		Tagline:     cmd.Helptext.Tagline,
		Arguments:   cmd.Helptext.Arguments,
		Options:     cmd.Helptext.Options,
		Synopsis:    cmd.Helptext.Synopsis,
		Subcommands: cmd.Helptext.Subcommands,
		Description: cmd.Helptext.ShortDescription,
		Usage:       cmd.Helptext.Usage,
		MoreHelp:    (cmd != root),
	}

	if len(cmd.Helptext.LongDescription) > 0 {
		fields.Description = cmd.Helptext.LongDescription
	}

	// autogen fields that are empty
	if len(fields.Arguments) == 0 {
		fields.Arguments = strings.Join(argumentText(cmd), "\n")
	}
	if len(fields.Options) == 0 {
		fields.Options = strings.Join(optionText(cmd), "\n")
	}
	if len(fields.Subcommands) == 0 {
		fields.Subcommands = strings.Join(subcommandText(cmd, rootName, path), "\n")
	}
	if len(fields.Synopsis) == 0 {
		fields.Synopsis = generateSynopsis(cmd, pathStr)
	}

	// trim the extra newlines (see TrimNewlines doc)
	fields.TrimNewlines()

	// indent all fields that have been set
	fields.IndentAll()

	return longHelpTemplate.Execute(out, fields)
}

// ShortHelp writes a formatted CLI helptext string to a Writer for the given command
func ShortHelp(rootName string, root *cmds.Command, path []string, out io.Writer) error {
	cmd, err := root.Get(path)
	if err != nil {
		return err
	}

	// default cmd to root if there is no path
	if path == nil && cmd == nil {
		cmd = root
	}

	pathStr := rootName
	if len(path) > 0 {
		pathStr += " " + strings.Join(path, " ")
	}

	fields := helpFields{
		Indent:      indentStr,
		Path:        pathStr,
		ArgUsage:    usageText(cmd),
		Tagline:     cmd.Helptext.Tagline,
		Synopsis:    cmd.Helptext.Synopsis,
		Description: cmd.Helptext.ShortDescription,
		Subcommands: cmd.Helptext.Subcommands,
		Usage:       cmd.Helptext.Usage,
		MoreHelp:    (cmd != root),
	}

	// autogen fields that are empty
	if len(fields.Subcommands) == 0 {
		fields.Subcommands = strings.Join(subcommandText(cmd, rootName, path), "\n")
	}
	if len(fields.Synopsis) == 0 {
		fields.Synopsis = generateSynopsis(cmd, pathStr)
	}

	// trim the extra newlines (see TrimNewlines doc)
	fields.TrimNewlines()

	// indent all fields that have been set
	fields.IndentAll()

	return shortHelpTemplate.Execute(out, fields)
}

func generateSynopsis(cmd *cmds.Command, path string) string {
	res := path
	for _, opt := range cmd.Options {
		valopt, ok := cmd.Helptext.SynopsisOptionsValues[opt.Names()[0]]
		if !ok {
			valopt = opt.Names()[0]
		}
		sopt := ""
		for i, n := range opt.Names() {
			pre := "-"
			if len(n) > 1 {
				pre = "--"
			}
			if i == 0 {
				sopt = fmt.Sprintf("%s%s=<%s>", pre, n, valopt)
			} else {
				sopt = fmt.Sprintf("%s | %s%s", sopt, pre, n)
			}
		}
		res = fmt.Sprintf("%s [%s]", res, sopt)
	}
	if len(cmd.Arguments) > 0 {
		res = fmt.Sprintf("%s [--]", res)
	}
	for _, arg := range cmd.Arguments {
		sarg := fmt.Sprintf("<%s>", arg.Name)
		if arg.Variadic {
			sarg = sarg + "..."
		}

		if !arg.Required {
			sarg = fmt.Sprintf("[%s]", sarg)
		}
		res = fmt.Sprintf("%s %s", res, sarg)
	}
	return strings.Trim(res, " ")
}

func argumentText(cmd *cmds.Command) []string {
	lines := make([]string, len(cmd.Arguments))

	for i, arg := range cmd.Arguments {
		lines[i] = argUsageText(arg)
	}
	lines = align(lines)
	for i, arg := range cmd.Arguments {
		lines[i] += " - " + arg.Description
	}

	return lines
}

func optionFlag(flag string) string {
	if len(flag) == 1 {
		return fmt.Sprintf(shortFlag, flag)
	} else {
		return fmt.Sprintf(longFlag, flag)
	}
}

func optionText(cmd ...*cmds.Command) []string {
	// get a slice of the options we want to list out
	options := make([]cmds.Option, 0)
	for _, c := range cmd {
		for _, opt := range c.Options {
			options = append(options, opt)
		}
	}

	// add option names to output (with each name aligned)
	lines := make([]string, 0)
	j := 0
	for {
		done := true
		i := 0
		for _, opt := range options {
			if len(lines) < i+1 {
				lines = append(lines, "")
			}

			names := sortByLength(opt.Names())
			if len(names) >= j+1 {
				lines[i] += optionFlag(names[j])
			}
			if len(names) > j+1 {
				lines[i] += ", "
				done = false
			}

			i++
		}

		if done {
			break
		}

		lines = align(lines)
		j++
	}
	lines = align(lines)

	// add option types to output
	for i, opt := range options {
		lines[i] += " " + fmt.Sprintf("%v", opt.Type())
	}
	lines = align(lines)

	// add option descriptions to output
	for i, opt := range options {
		lines[i] += " - " + opt.Description()
	}

	return lines
}

func subcommandText(cmd *cmds.Command, rootName string, path []string) []string {
	prefix := fmt.Sprintf("%v %v", rootName, strings.Join(path, " "))
	if len(path) > 0 {
		prefix += " "
	}
	subcmds := make([]*cmds.Command, len(cmd.Subcommands))
	lines := make([]string, len(cmd.Subcommands))

	i := 0
	for name, sub := range cmd.Subcommands {
		usage := usageText(sub)
		if len(usage) > 0 {
			usage = " " + usage
		}
		lines[i] = prefix + name + usage
		subcmds[i] = sub
		i++
	}

	lines = align(lines)
	for i, sub := range subcmds {
		lines[i] += " - " + sub.Helptext.Tagline
	}

	return lines
}

func usageText(cmd *cmds.Command) string {
	s := ""
	for i, arg := range cmd.Arguments {
		if i != 0 {
			s += " "
		}
		s += argUsageText(arg)
	}

	return s
}

func argUsageText(arg cmds.Argument) string {
	s := arg.Name

	if arg.Required {
		s = fmt.Sprintf(requiredArg, s)
	} else {
		s = fmt.Sprintf(optionalArg, s)
	}

	if arg.Variadic {
		s = fmt.Sprintf(variadicArg, s)
	}

	return s
}

func align(lines []string) []string {
	longest := 0
	for _, line := range lines {
		length := len(line)
		if length > longest {
			longest = length
		}
	}

	for i, line := range lines {
		length := len(line)
		if length > 0 {
			lines[i] += strings.Repeat(" ", longest-length)
		}
	}

	return lines
}

func indent(lines []string, prefix string) []string {
	for i, line := range lines {
		lines[i] = prefix + indentString(line, prefix)
	}
	return lines
}

func indentString(line string, prefix string) string {
	return prefix + strings.Replace(line, "\n", "\n"+prefix, -1)
}

type lengthSlice []string

func (ls lengthSlice) Len() int {
	return len(ls)
}
func (ls lengthSlice) Swap(a, b int) {
	ls[a], ls[b] = ls[b], ls[a]
}
func (ls lengthSlice) Less(a, b int) bool {
	return len(ls[a]) < len(ls[b])
}

func sortByLength(slice []string) []string {
	output := make(lengthSlice, len(slice))
	for i, val := range slice {
		output[i] = val
	}
	sort.Sort(output)
	return []string(output)
}
<commit_msg>Add default true options handling<commit_after>package cli

import (
	"fmt"
	"io"
	"sort"
	"strings"
	"text/template"

	cmds "github.com/ipfs/go-ipfs/commands"
)

const (
	requiredArg = "<%v>"
	optionalArg = "[<%v>]"
	variadicArg = "%v..."
	shortFlag   = "-%v"
	longFlag    = "--%v"
	optionType  = "(%v)"

	whitespace = "\r\n\t "

	indentStr = "  "
)

type helpFields struct {
	Indent      string
	Usage       string
	Path        string
	ArgUsage    string
	Tagline     string
	Arguments   string
	Options     string
	Synopsis    string
	Subcommands string
	Description string
	MoreHelp    bool
}

// TrimNewlines removes extra newlines from fields. This makes aligning
// commands easier. Below, the leading + tralining newlines are removed:
//	Synopsis: `
//	    ipfs config <key>          - Get value of <key>
//	    ipfs config <key> <value>  - Set value of <key> to <value>
//	    ipfs config --show         - Show config file
//	    ipfs config --edit         - Edit config file in $EDITOR
//	`
func (f *helpFields) TrimNewlines() {
	f.Path = strings.Trim(f.Path, "\n")
	f.ArgUsage = strings.Trim(f.ArgUsage, "\n")
	f.Tagline = strings.Trim(f.Tagline, "\n")
	f.Arguments = strings.Trim(f.Arguments, "\n")
	f.Options = strings.Trim(f.Options, "\n")
	f.Synopsis = strings.Trim(f.Synopsis, "\n")
	f.Subcommands = strings.Trim(f.Subcommands, "\n")
	f.Description = strings.Trim(f.Description, "\n")
}

// Indent adds whitespace the lines of fields.
func (f *helpFields) IndentAll() {
	indent := func(s string) string {
		if s == "" {
			return s
		}
		return indentString(s, indentStr)
	}

	f.Arguments = indent(f.Arguments)
	f.Options = indent(f.Options)
	f.Synopsis = indent(f.Synopsis)
	f.Subcommands = indent(f.Subcommands)
	f.Description = indent(f.Description)
}

const usageFormat = "{{if .Usage}}{{.Usage}}{{else}}{{.Path}}{{if .ArgUsage}} {{.ArgUsage}}{{end}} - {{.Tagline}}{{end}}"

const longHelpFormat = `USAGE
{{.Indent}}{{template "usage" .}}

{{if .Synopsis}}SYNOPSIS
{{.Synopsis}}

{{end}}{{if .Arguments}}ARGUMENTS

{{.Arguments}}

{{end}}{{if .Options}}OPTIONS

{{.Options}}

{{end}}{{if .Description}}DESCRIPTION

{{.Description}}

{{end}}{{if .Subcommands}}SUBCOMMANDS
{{.Subcommands}}

{{.Indent}}Use '{{.Path}} <subcmd> --help' for more information about each command.
{{end}}
`
const shortHelpFormat = `USAGE
{{.Indent}}{{template "usage" .}}
{{if .Synopsis}}
{{.Synopsis}}
{{end}}{{if .Description}}
{{.Description}}
{{end}}{{if .Subcommands}}
SUBCOMMANDS
{{.Subcommands}}
{{end}}{{if .MoreHelp}}
Use '{{.Path}} --help' for more information about this command.
{{end}}
`

var usageTemplate *template.Template
var longHelpTemplate *template.Template
var shortHelpTemplate *template.Template

func init() {
	usageTemplate = template.Must(template.New("usage").Parse(usageFormat))
	longHelpTemplate = template.Must(usageTemplate.New("longHelp").Parse(longHelpFormat))
	shortHelpTemplate = template.Must(usageTemplate.New("shortHelp").Parse(shortHelpFormat))
}

// LongHelp writes a formatted CLI helptext string to a Writer for the given command
func LongHelp(rootName string, root *cmds.Command, path []string, out io.Writer) error {
	cmd, err := root.Get(path)
	if err != nil {
		return err
	}

	pathStr := rootName
	if len(path) > 0 {
		pathStr += " " + strings.Join(path, " ")
	}

	fields := helpFields{
		Indent:      indentStr,
		Path:        pathStr,
		ArgUsage:    usageText(cmd),
		Tagline:     cmd.Helptext.Tagline,
		Arguments:   cmd.Helptext.Arguments,
		Options:     cmd.Helptext.Options,
		Synopsis:    cmd.Helptext.Synopsis,
		Subcommands: cmd.Helptext.Subcommands,
		Description: cmd.Helptext.ShortDescription,
		Usage:       cmd.Helptext.Usage,
		MoreHelp:    (cmd != root),
	}

	if len(cmd.Helptext.LongDescription) > 0 {
		fields.Description = cmd.Helptext.LongDescription
	}

	// autogen fields that are empty
	if len(fields.Arguments) == 0 {
		fields.Arguments = strings.Join(argumentText(cmd), "\n")
	}
	if len(fields.Options) == 0 {
		fields.Options = strings.Join(optionText(cmd), "\n")
	}
	if len(fields.Subcommands) == 0 {
		fields.Subcommands = strings.Join(subcommandText(cmd, rootName, path), "\n")
	}
	if len(fields.Synopsis) == 0 {
		fields.Synopsis = generateSynopsis(cmd, pathStr)
	}

	// trim the extra newlines (see TrimNewlines doc)
	fields.TrimNewlines()

	// indent all fields that have been set
	fields.IndentAll()

	return longHelpTemplate.Execute(out, fields)
}

// ShortHelp writes a formatted CLI helptext string to a Writer for the given command
func ShortHelp(rootName string, root *cmds.Command, path []string, out io.Writer) error {
	cmd, err := root.Get(path)
	if err != nil {
		return err
	}

	// default cmd to root if there is no path
	if path == nil && cmd == nil {
		cmd = root
	}

	pathStr := rootName
	if len(path) > 0 {
		pathStr += " " + strings.Join(path, " ")
	}

	fields := helpFields{
		Indent:      indentStr,
		Path:        pathStr,
		ArgUsage:    usageText(cmd),
		Tagline:     cmd.Helptext.Tagline,
		Synopsis:    cmd.Helptext.Synopsis,
		Description: cmd.Helptext.ShortDescription,
		Subcommands: cmd.Helptext.Subcommands,
		Usage:       cmd.Helptext.Usage,
		MoreHelp:    (cmd != root),
	}

	// autogen fields that are empty
	if len(fields.Subcommands) == 0 {
		fields.Subcommands = strings.Join(subcommandText(cmd, rootName, path), "\n")
	}
	if len(fields.Synopsis) == 0 {
		fields.Synopsis = generateSynopsis(cmd, pathStr)
	}

	// trim the extra newlines (see TrimNewlines doc)
	fields.TrimNewlines()

	// indent all fields that have been set
	fields.IndentAll()

	return shortHelpTemplate.Execute(out, fields)
}

func generateSynopsis(cmd *cmds.Command, path string) string {
	res := path
	for _, opt := range cmd.Options {
		valopt, ok := cmd.Helptext.SynopsisOptionsValues[opt.Names()[0]]
		if !ok {
			valopt = opt.Names()[0]
		}
		sopt := ""
		for i, n := range opt.Names() {
			pre := "-"
			if len(n) > 1 {
				pre = "--"
			}
			if opt.Type() == cmds.Bool && opt.DefaultVal() == true {
				pre = "--"
				sopt = fmt.Sprintf("%s%s=false", pre, n)
				break
			} else {
				if i == 0 {
					if opt.Type() == cmds.Bool {
						sopt = fmt.Sprintf("%s%s", pre, n)
					} else {
						sopt = fmt.Sprintf("%s%s=<%s>", pre, n, valopt)
					}
				} else {
					sopt = fmt.Sprintf("%s | %s%s", sopt, pre, n)
				}
			}
		}
		res = fmt.Sprintf("%s [%s]", res, sopt)
	}
	if len(cmd.Arguments) > 0 {
		res = fmt.Sprintf("%s [--]", res)
	}
	for _, arg := range cmd.Arguments {
		sarg := fmt.Sprintf("<%s>", arg.Name)
		if arg.Variadic {
			sarg = sarg + "..."
		}

		if !arg.Required {
			sarg = fmt.Sprintf("[%s]", sarg)
		}
		res = fmt.Sprintf("%s %s", res, sarg)
	}
	return strings.Trim(res, " ")
}

func argumentText(cmd *cmds.Command) []string {
	lines := make([]string, len(cmd.Arguments))

	for i, arg := range cmd.Arguments {
		lines[i] = argUsageText(arg)
	}
	lines = align(lines)
	for i, arg := range cmd.Arguments {
		lines[i] += " - " + arg.Description
	}

	return lines
}

func optionFlag(flag string) string {
	if len(flag) == 1 {
		return fmt.Sprintf(shortFlag, flag)
	} else {
		return fmt.Sprintf(longFlag, flag)
	}
}

func optionText(cmd ...*cmds.Command) []string {
	// get a slice of the options we want to list out
	options := make([]cmds.Option, 0)
	for _, c := range cmd {
		for _, opt := range c.Options {
			options = append(options, opt)
		}
	}

	// add option names to output (with each name aligned)
	lines := make([]string, 0)
	j := 0
	for {
		done := true
		i := 0
		for _, opt := range options {
			if len(lines) < i+1 {
				lines = append(lines, "")
			}

			names := sortByLength(opt.Names())
			if len(names) >= j+1 {
				lines[i] += optionFlag(names[j])
			}
			if len(names) > j+1 {
				lines[i] += ", "
				done = false
			}

			i++
		}

		if done {
			break
		}

		lines = align(lines)
		j++
	}
	lines = align(lines)

	// add option types to output
	for i, opt := range options {
		lines[i] += " " + fmt.Sprintf("%v", opt.Type())
	}
	lines = align(lines)

	// add option descriptions to output
	for i, opt := range options {
		lines[i] += " - " + opt.Description()
	}

	return lines
}

func subcommandText(cmd *cmds.Command, rootName string, path []string) []string {
	prefix := fmt.Sprintf("%v %v", rootName, strings.Join(path, " "))
	if len(path) > 0 {
		prefix += " "
	}
	subcmds := make([]*cmds.Command, len(cmd.Subcommands))
	lines := make([]string, len(cmd.Subcommands))

	i := 0
	for name, sub := range cmd.Subcommands {
		usage := usageText(sub)
		if len(usage) > 0 {
			usage = " " + usage
		}
		lines[i] = prefix + name + usage
		subcmds[i] = sub
		i++
	}

	lines = align(lines)
	for i, sub := range subcmds {
		lines[i] += " - " + sub.Helptext.Tagline
	}

	return lines
}

func usageText(cmd *cmds.Command) string {
	s := ""
	for i, arg := range cmd.Arguments {
		if i != 0 {
			s += " "
		}
		s += argUsageText(arg)
	}

	return s
}

func argUsageText(arg cmds.Argument) string {
	s := arg.Name

	if arg.Required {
		s = fmt.Sprintf(requiredArg, s)
	} else {
		s = fmt.Sprintf(optionalArg, s)
	}

	if arg.Variadic {
		s = fmt.Sprintf(variadicArg, s)
	}

	return s
}

func align(lines []string) []string {
	longest := 0
	for _, line := range lines {
		length := len(line)
		if length > longest {
			longest = length
		}
	}

	for i, line := range lines {
		length := len(line)
		if length > 0 {
			lines[i] += strings.Repeat(" ", longest-length)
		}
	}

	return lines
}

func indent(lines []string, prefix string) []string {
	for i, line := range lines {
		lines[i] = prefix + indentString(line, prefix)
	}
	return lines
}

func indentString(line string, prefix string) string {
	return prefix + strings.Replace(line, "\n", "\n"+prefix, -1)
}

type lengthSlice []string

func (ls lengthSlice) Len() int {
	return len(ls)
}
func (ls lengthSlice) Swap(a, b int) {
	ls[a], ls[b] = ls[b], ls[a]
}
func (ls lengthSlice) Less(a, b int) bool {
	return len(ls[a]) < len(ls[b])
}

func sortByLength(slice []string) []string {
	output := make(lengthSlice, len(slice))
	for i, val := range slice {
		output[i] = val
	}
	sort.Sort(output)
	return []string(output)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"context"
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/spf13/cobra"

	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apiserver/pkg/server"
	"k8s.io/apiserver/pkg/server/healthz"
	apiserverflag "k8s.io/apiserver/pkg/util/flag"
	"k8s.io/apiserver/pkg/util/globalflag"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	cloudprovider "k8s.io/cloud-provider"
	"k8s.io/klog"
	cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config"
	"k8s.io/kubernetes/cmd/cloud-controller-manager/app/options"
	genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
	"k8s.io/kubernetes/pkg/util/configz"
	utilflag "k8s.io/kubernetes/pkg/util/flag"
	"k8s.io/kubernetes/pkg/version"
	"k8s.io/kubernetes/pkg/version/verflag"
)

const (
	// ControllerStartJitter is the jitter value used when starting controller managers.
	ControllerStartJitter = 1.0
	// ConfigzName is the name used for register cloud-controller manager /configz, same with GroupName.
	ConfigzName = "cloudcontrollermanager.config.k8s.io"
)

// NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters
func NewCloudControllerManagerCommand() *cobra.Command {
	s, err := options.NewCloudControllerManagerOptions()
	if err != nil {
		klog.Fatalf("unable to initialize command options: %v", err)
	}

	cmd := &cobra.Command{
		Use: "cloud-controller-manager",
		Long: `The Cloud controller manager is a daemon that embeds
the cloud specific control loops shipped with Kubernetes.`,
		Run: func(cmd *cobra.Command, args []string) {
			verflag.PrintAndExitIfRequested()
			utilflag.PrintFlags(cmd.Flags())

			c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List())
			if err != nil {
				fmt.Fprintf(os.Stderr, "%v\n", err)
				os.Exit(1)
			}

			if err := Run(c.Complete(), wait.NeverStop); err != nil {
				fmt.Fprintf(os.Stderr, "%v\n", err)
				os.Exit(1)
			}

		},
	}

	fs := cmd.Flags()
	namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List())
	verflag.AddFlags(namedFlagSets.FlagSet("global"))
	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
	cmoptions.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
	for _, f := range namedFlagSets.FlagSets {
		fs.AddFlagSet(f)
	}
	usageFmt := "Usage:\n  %s\n"
	cols, _, _ := apiserverflag.TerminalSize(cmd.OutOrStdout())
	cmd.SetUsageFunc(func(cmd *cobra.Command) error {
		fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine())
		apiserverflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols)
		return nil
	})
	cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
		fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine())
		apiserverflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols)
	})

	return cmd
}

// Run runs the ExternalCMServer.  This should never exit.
func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error {
	// To help debugging, immediately log version
	klog.Infof("Version: %+v", version.Get())

	cloud, err := cloudprovider.InitCloudProvider(c.ComponentConfig.KubeCloudShared.CloudProvider.Name, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile)
	if err != nil {
		klog.Fatalf("Cloud provider could not be initialized: %v", err)
	}
	if cloud == nil {
		klog.Fatalf("cloud provider is nil")
	}

	if cloud.HasClusterID() == false {
		if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud == true {
			klog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
		} else {
			klog.Fatalf("no ClusterID found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
		}
	}

	// setup /configz endpoint
	if cz, err := configz.New(ConfigzName); err == nil {
		cz.Set(c.ComponentConfig)
	} else {
		klog.Errorf("unable to register configz: %c", err)
	}

	// Setup any healthz checks we will want to use.
	var checks []healthz.HealthzChecker
	var electionChecker *leaderelection.HealthzAdaptor
	if c.ComponentConfig.Generic.LeaderElection.LeaderElect {
		electionChecker = leaderelection.NewLeaderHealthzAdaptor(time.Second * 20)
		checks = append(checks, electionChecker)
	}

	// Start the controller manager HTTP server
	if c.SecureServing != nil {
		unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...)
		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication)
		// TODO: handle stoppedCh returned by c.SecureServing.Serve
		if _, err := c.SecureServing.Serve(handler, 0, stopCh); err != nil {
			return err
		}
	}
	if c.InsecureServing != nil {
		unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...)
		insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}}
		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn)
		if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil {
			return err
		}
	}

	run := func(ctx context.Context) {
		if err := startControllers(c, ctx.Done(), cloud, newControllerInitializers()); err != nil {
			klog.Fatalf("error running controllers: %v", err)
		}
	}

	if !c.ComponentConfig.Generic.LeaderElection.LeaderElect {
		run(context.TODO())
		panic("unreachable")
	}

	// Identity used to distinguish between multiple cloud controller manager instances
	id, err := os.Hostname()
	if err != nil {
		return err
	}
	// add a uniquifier so that two processes on the same host don't accidentally both become active
	id = id + "_" + string(uuid.NewUUID())

	// Lock required for leader election
	rl, err := resourcelock.New(c.ComponentConfig.Generic.LeaderElection.ResourceLock,
		"kube-system",
		"cloud-controller-manager",
		c.LeaderElectionClient.CoreV1(),
		c.LeaderElectionClient.CoordinationV1(),
		resourcelock.ResourceLockConfig{
			Identity:      id,
			EventRecorder: c.EventRecorder,
		})
	if err != nil {
		klog.Fatalf("error creating lock: %v", err)
	}

	// Try and become the leader and start cloud controller manager loops
	leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{
		Lock:          rl,
		LeaseDuration: c.ComponentConfig.Generic.LeaderElection.LeaseDuration.Duration,
		RenewDeadline: c.ComponentConfig.Generic.LeaderElection.RenewDeadline.Duration,
		RetryPeriod:   c.ComponentConfig.Generic.LeaderElection.RetryPeriod.Duration,
		Callbacks: leaderelection.LeaderCallbacks{
			OnStartedLeading: run,
			OnStoppedLeading: func() {
				klog.Fatalf("leaderelection lost")
			},
		},
		WatchDog: electionChecker,
		Name:     "cloud-controller-manager",
	})
	panic("unreachable")
}

// startControllers starts the cloud specific controller loops.
func startControllers(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}, cloud cloudprovider.Interface, controllers map[string]initFunc) error {
	if cloud != nil {
		// Initialize the cloud provider with a reference to the clientBuilder
		cloud.Initialize(c.ClientBuilder, stopCh)
		// Set the informer on the user cloud object
		if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
			informerUserCloud.SetInformers(c.SharedInformers)
		}
	}

	for controllerName, initFn := range controllers {
		if !genericcontrollermanager.IsControllerEnabled(controllerName, ControllersDisabledByDefault, c.ComponentConfig.Generic.Controllers) {
			klog.Warningf("%q is disabled", controllerName)
			continue
		}

		klog.V(1).Infof("Starting %q", controllerName)
		_, started, err := initFn(c, cloud, stopCh)
		if err != nil {
			klog.Errorf("Error starting %q", controllerName)
			return err
		}
		if !started {
			klog.Warningf("Skipping %q", controllerName)
			continue
		}
		klog.Infof("Started %q", controllerName)

		time.Sleep(wait.Jitter(c.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter))
	}

	// If apiserver is not running we should wait for some time and fail only then. This is particularly
	// important when we start apiserver and controller manager at the same time.
	if err := genericcontrollermanager.WaitForAPIServer(c.VersionedClient, 10*time.Second); err != nil {
		klog.Fatalf("Failed to wait for apiserver being healthy: %v", err)
	}

	c.SharedInformers.Start(stopCh)

	select {}
}

// initFunc is used to launch a particular controller.  It may run additional "should I activate checks".
// Any error returned will cause the controller process to `Fatal`
// The bool indicates whether the controller was enabled.
type initFunc func(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stop <-chan struct{}) (debuggingHandler http.Handler, enabled bool, err error)

// KnownControllers indicate the default controller we are known.
func KnownControllers() []string {
	ret := sets.StringKeySet(newControllerInitializers())
	return ret.List()
}

// ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers.
var ControllersDisabledByDefault = sets.NewString()

// newControllerInitializers is a private map of named controller groups (you can start more than one in an init func)
// paired to their initFunc.  This allows for structured downstream composition and subdivision.
func newControllerInitializers() map[string]initFunc {
	controllers := map[string]initFunc{}
	controllers["cloud-node"] = startCloudNodeController
	controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleController
	controllers["persistentvolume-binder"] = startPersistentVolumeLabelController
	controllers["service"] = startServiceController
	controllers["route"] = startRouteController
	return controllers
}
<commit_msg>remove unnecessary nil check for cloud<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"context"
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/spf13/cobra"

	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apiserver/pkg/server"
	"k8s.io/apiserver/pkg/server/healthz"
	apiserverflag "k8s.io/apiserver/pkg/util/flag"
	"k8s.io/apiserver/pkg/util/globalflag"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	cloudprovider "k8s.io/cloud-provider"
	"k8s.io/klog"
	cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config"
	"k8s.io/kubernetes/cmd/cloud-controller-manager/app/options"
	genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
	"k8s.io/kubernetes/pkg/util/configz"
	utilflag "k8s.io/kubernetes/pkg/util/flag"
	"k8s.io/kubernetes/pkg/version"
	"k8s.io/kubernetes/pkg/version/verflag"
)

const (
	// ControllerStartJitter is the jitter value used when starting controller managers.
	ControllerStartJitter = 1.0
	// ConfigzName is the name used for register cloud-controller manager /configz, same with GroupName.
	ConfigzName = "cloudcontrollermanager.config.k8s.io"
)

// NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters
func NewCloudControllerManagerCommand() *cobra.Command {
	s, err := options.NewCloudControllerManagerOptions()
	if err != nil {
		klog.Fatalf("unable to initialize command options: %v", err)
	}

	cmd := &cobra.Command{
		Use: "cloud-controller-manager",
		Long: `The Cloud controller manager is a daemon that embeds
the cloud specific control loops shipped with Kubernetes.`,
		Run: func(cmd *cobra.Command, args []string) {
			verflag.PrintAndExitIfRequested()
			utilflag.PrintFlags(cmd.Flags())

			c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List())
			if err != nil {
				fmt.Fprintf(os.Stderr, "%v\n", err)
				os.Exit(1)
			}

			if err := Run(c.Complete(), wait.NeverStop); err != nil {
				fmt.Fprintf(os.Stderr, "%v\n", err)
				os.Exit(1)
			}

		},
	}

	fs := cmd.Flags()
	namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List())
	verflag.AddFlags(namedFlagSets.FlagSet("global"))
	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
	cmoptions.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
	for _, f := range namedFlagSets.FlagSets {
		fs.AddFlagSet(f)
	}
	usageFmt := "Usage:\n  %s\n"
	cols, _, _ := apiserverflag.TerminalSize(cmd.OutOrStdout())
	cmd.SetUsageFunc(func(cmd *cobra.Command) error {
		fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine())
		apiserverflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols)
		return nil
	})
	cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
		fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine())
		apiserverflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols)
	})

	return cmd
}

// Run runs the ExternalCMServer.  This should never exit.
func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error {
	// To help debugging, immediately log version
	klog.Infof("Version: %+v", version.Get())

	cloud, err := cloudprovider.InitCloudProvider(c.ComponentConfig.KubeCloudShared.CloudProvider.Name, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile)
	if err != nil {
		klog.Fatalf("Cloud provider could not be initialized: %v", err)
	}
	if cloud == nil {
		klog.Fatalf("cloud provider is nil")
	}

	if cloud.HasClusterID() == false {
		if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud == true {
			klog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
		} else {
			klog.Fatalf("no ClusterID found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
		}
	}

	// setup /configz endpoint
	if cz, err := configz.New(ConfigzName); err == nil {
		cz.Set(c.ComponentConfig)
	} else {
		klog.Errorf("unable to register configz: %c", err)
	}

	// Setup any healthz checks we will want to use.
	var checks []healthz.HealthzChecker
	var electionChecker *leaderelection.HealthzAdaptor
	if c.ComponentConfig.Generic.LeaderElection.LeaderElect {
		electionChecker = leaderelection.NewLeaderHealthzAdaptor(time.Second * 20)
		checks = append(checks, electionChecker)
	}

	// Start the controller manager HTTP server
	if c.SecureServing != nil {
		unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...)
		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication)
		// TODO: handle stoppedCh returned by c.SecureServing.Serve
		if _, err := c.SecureServing.Serve(handler, 0, stopCh); err != nil {
			return err
		}
	}
	if c.InsecureServing != nil {
		unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...)
		insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}}
		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn)
		if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil {
			return err
		}
	}

	run := func(ctx context.Context) {
		if err := startControllers(c, ctx.Done(), cloud, newControllerInitializers()); err != nil {
			klog.Fatalf("error running controllers: %v", err)
		}
	}

	if !c.ComponentConfig.Generic.LeaderElection.LeaderElect {
		run(context.TODO())
		panic("unreachable")
	}

	// Identity used to distinguish between multiple cloud controller manager instances
	id, err := os.Hostname()
	if err != nil {
		return err
	}
	// add a uniquifier so that two processes on the same host don't accidentally both become active
	id = id + "_" + string(uuid.NewUUID())

	// Lock required for leader election
	rl, err := resourcelock.New(c.ComponentConfig.Generic.LeaderElection.ResourceLock,
		"kube-system",
		"cloud-controller-manager",
		c.LeaderElectionClient.CoreV1(),
		c.LeaderElectionClient.CoordinationV1(),
		resourcelock.ResourceLockConfig{
			Identity:      id,
			EventRecorder: c.EventRecorder,
		})
	if err != nil {
		klog.Fatalf("error creating lock: %v", err)
	}

	// Try and become the leader and start cloud controller manager loops
	leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{
		Lock:          rl,
		LeaseDuration: c.ComponentConfig.Generic.LeaderElection.LeaseDuration.Duration,
		RenewDeadline: c.ComponentConfig.Generic.LeaderElection.RenewDeadline.Duration,
		RetryPeriod:   c.ComponentConfig.Generic.LeaderElection.RetryPeriod.Duration,
		Callbacks: leaderelection.LeaderCallbacks{
			OnStartedLeading: run,
			OnStoppedLeading: func() {
				klog.Fatalf("leaderelection lost")
			},
		},
		WatchDog: electionChecker,
		Name:     "cloud-controller-manager",
	})
	panic("unreachable")
}

// startControllers starts the cloud specific controller loops.
func startControllers(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}, cloud cloudprovider.Interface, controllers map[string]initFunc) error {
	// Initialize the cloud provider with a reference to the clientBuilder
	cloud.Initialize(c.ClientBuilder, stopCh)
	// Set the informer on the user cloud object
	if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
		informerUserCloud.SetInformers(c.SharedInformers)
	}

	for controllerName, initFn := range controllers {
		if !genericcontrollermanager.IsControllerEnabled(controllerName, ControllersDisabledByDefault, c.ComponentConfig.Generic.Controllers) {
			klog.Warningf("%q is disabled", controllerName)
			continue
		}

		klog.V(1).Infof("Starting %q", controllerName)
		_, started, err := initFn(c, cloud, stopCh)
		if err != nil {
			klog.Errorf("Error starting %q", controllerName)
			return err
		}
		if !started {
			klog.Warningf("Skipping %q", controllerName)
			continue
		}
		klog.Infof("Started %q", controllerName)

		time.Sleep(wait.Jitter(c.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter))
	}

	// If apiserver is not running we should wait for some time and fail only then. This is particularly
	// important when we start apiserver and controller manager at the same time.
	if err := genericcontrollermanager.WaitForAPIServer(c.VersionedClient, 10*time.Second); err != nil {
		klog.Fatalf("Failed to wait for apiserver being healthy: %v", err)
	}

	c.SharedInformers.Start(stopCh)

	select {}
}

// initFunc is used to launch a particular controller.  It may run additional "should I activate checks".
// Any error returned will cause the controller process to `Fatal`
// The bool indicates whether the controller was enabled.
type initFunc func(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stop <-chan struct{}) (debuggingHandler http.Handler, enabled bool, err error)

// KnownControllers indicate the default controller we are known.
func KnownControllers() []string {
	ret := sets.StringKeySet(newControllerInitializers())
	return ret.List()
}

// ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers.
var ControllersDisabledByDefault = sets.NewString()

// newControllerInitializers is a private map of named controller groups (you can start more than one in an init func)
// paired to their initFunc.  This allows for structured downstream composition and subdivision.
func newControllerInitializers() map[string]initFunc {
	controllers := map[string]initFunc{}
	controllers["cloud-node"] = startCloudNodeController
	controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleController
	controllers["persistentvolume-binder"] = startPersistentVolumeLabelController
	controllers["service"] = startServiceController
	controllers["route"] = startRouteController
	return controllers
}
<|endoftext|>


<commit_before>
package sre2

import (
  "fmt"
  "testing"
)

// Check the given state to be true.
func checkState(t *testing.T, state bool, err string) {
  if !state {
    t.Error(err)
  }
}

// Check the equality of two []int slices.
func checkIntSlice(t *testing.T, expected []int, result []int, err string) {
  match := true
  if len(expected) != len(result) {
    match = false
  } else {
    for i := 0; i < len(expected); i++ {
      if expected[i] != result[i] {
        match = false
      }
    }
  }
  checkState(t, match, fmt.Sprintf("%s: got %s, expected %s", err, result, expected))
}

// Run a selection of basic regular expressions against this package.
func TestSimpleRe(t *testing.T) {
  r := MustParse("")
  checkState(t, r.RunSimple(""), "everything should match")
  checkState(t, r.RunSimple("fadsnjkflsdafnas"), "everything should match")

  r = MustParse("^(a|b)+c*$")
  checkState(t, !r.RunSimple("abd"), "not a valid match")
  checkState(t, r.RunSimple("a"), "basic string should match")
  checkState(t, !r.RunSimple(""), "empty string should not match")
  checkState(t, r.RunSimple("abcccc"), "longer string should match")

  r = MustParse("(\\w*)\\s*(\\w*)")
  ok, res := r.RunSubMatch("zing hello there")
  checkState(t, ok, "should match generally")
  checkIntSlice(t, []int{0, 10, 0, 4, 5, 10}, res, "did not match first two words as expected")

  r = MustParse(".*?(\\w+)$")
  ok, res = r.RunSubMatch("zing hello there")
  checkState(t, ok, "should match generally")
  checkIntSlice(t, []int{0, 16, 11, 16}, res, "did not match last word as expected")
}

// Test parsing an invalid RE returns an error.
func TestInvalidRe(t *testing.T) {
  r, err := Parse("a**")
  checkState(t, err != nil, "must fail parsing")
  checkState(t, r == nil, "regexp must be nil")

  pass := false
  func() {
    defer func() {
      if r := recover(); r != nil {
        pass = true
      }
    }()
    MustParse("z(((a")
  }()
  checkState(t, pass, "should panic")
}

// Test behaviour related to character classes expressed within [...].
func TestCharClass(t *testing.T) {
  r := MustParse("^[\t[:word:]]+$") // Match tabs and word characters.
  checkState(t, r.RunSimple("c"), "non-space should match")
  checkState(t, !r.RunSimple("c t"), "space should not match")
  checkState(t, r.RunSimple("c\tt"), "tab should match")

  r = MustParse("^[:ascii:]*$")
  checkState(t, r.RunSimple(""), "nothing should match")
  checkState(t, r.RunSimple("c"), "ascii should match")
  checkState(t, !r.RunSimple("Π"), "unicode should not match")

  r = MustParse("^\\pN$")
  checkState(t, r.RunSimple("〩"), "character from Nl should match")
  checkState(t, r.RunSimple("¾"), "character from Nu should match")

  r = MustParse("^\\p{Nl}$")
  checkState(t, r.RunSimple("〩"), "character from Nl should match")
  checkState(t, !r.RunSimple("¾"), "character from Nu should not match")
}

// Test regexp generated by escape sequences (e.g. \n, \. etc).
func TestEscapeSequences(t *testing.T) {
  r := MustParse("^\\.\n\\044$") // Match '.\n$'
  checkState(t, r.RunSimple(".\n$"), "should match")
  checkState(t, !r.RunSimple(" \n$"), "space should not match")
  checkState(t, !r.RunSimple(".\n"), "# should not be treated as end char")

  r = MustParse("^\\x{03a0}\\x25$") // Match 'Π%'.
  checkState(t, r.RunSimple("Π%"), "should match pi+percent")

  r, err := Parse("^\\Π$")
  checkState(t, err != nil && r == nil,
      "should have failed on trying to escape Π, not punctuation")
}

// Tests string literals between \Q...\E.
func TestStringLiteral(t *testing.T) {
  r := MustParse("^\\Qhello\\E$")
  checkState(t, r.RunSimple("hello"), "should match hello")

  r = MustParse("^\\Q.$\\\\E$") // match ".$\\"
  checkState(t, r.RunSimple(".$\\"), "should match")
  checkState(t, !r.RunSimple(" $\\"), "should not match")

  r = MustParse("^a\\Q\\E*b$") // match absolutely nothing between 'ab'
  checkState(t, r.RunSimple("ab"), "should match")
  checkState(t, !r.RunSimple("acb"), "should not match")
}

// Test closure expansion types, such as {..}, ?, +, * etc.
func TestClosureExpansion(t *testing.T) {
  r := MustParse("^za?$")
  checkState(t, r.RunSimple("z"), "should match none")
  checkState(t, r.RunSimple("za"), "should match single")
  checkState(t, !r.RunSimple("zaa"), "should not match more")

  r = MustParse("^a{2,2}$")
  checkState(t, !r.RunSimple(""), "0 should fail")
  checkState(t, !r.RunSimple("a"), "1 should fail")
  checkState(t, r.RunSimple("aa"), "2 should succeed")
  checkState(t, r.RunSimple("aaa"), "3 should succeed")
  checkState(t, r.RunSimple("aaaa"), "4 should succeed")
  checkState(t, !r.RunSimple("aaaaa"), "5 should fail")

  r = MustParse("^a{2}$")
  checkState(t, !r.RunSimple(""), "0 should fail")
  checkState(t, !r.RunSimple("a"), "1 should fail")
  checkState(t, r.RunSimple("aa"), "2 should succeed")
  checkState(t, !r.RunSimple("aaa"), "3 should fail")

  r = MustParse("^a{3,}$")
  checkState(t, !r.RunSimple("aa"), "2 should fail")
  checkState(t, r.RunSimple("aaa"), "3 should succeed")
  checkState(t, r.RunSimple("aaaaaa"), "more should succeed")
}

// Test specific greedy/non-greedy closure types.
func TestClosureGreedy(t *testing.T) {
  r := MustParse("^(a{0,2}?)(a*)$")
  ok, res := r.RunSubMatch("aaa")
  checkState(t, ok, "should match")
  checkIntSlice(t, []int{0, 3, 0, 0, 0, 3}, res, "did not match expected")

  r = MustParse("^(a{0,2})?(a*)$")
  ok, res = r.RunSubMatch("aaa")
  checkState(t, ok, "should match")
  checkIntSlice(t, []int{0, 3, 0, 2, 2, 3}, res, "did not match expected")

  r = MustParse("^(a{2,}?)(a*)$")
  ok, res = r.RunSubMatch("aaa")
  checkState(t, ok, "should match")
  checkIntSlice(t, []int{0, 3, 0, 2, 2, 3}, res, "did not match expected")
}

// Test simple left/right matchers.
func TestLeftRight(t *testing.T) {
  r := MustParse("^.\\b.$")
  checkState(t, r.RunSimple("a "), "left char is word")
  checkState(t, r.RunSimple(" a"), "right char is word")
  checkState(t, !r.RunSimple("  "), "not a boundary")
  checkState(t, !r.RunSimple("aa"), "not a boundary")
}

// Test general flags in sre2.
func TestFlags(t *testing.T) {
  r := MustParse("^(?i:AbC)zz$")
  checkState(t, r.RunSimple("abczz"), "success")
  checkState(t, !r.RunSimple("abcZZ"), "fail, flag should not escape")
  ok, res := r.RunSubMatch("ABCzz")
  checkState(t, ok, "should pass")
  checkIntSlice(t, []int{0,5}, res, "should just have a single outer paren")

  r = MustParse("^(?U)(a+)(.+)$")
  ok, res = r.RunSubMatch("aaaabb")
  checkState(t, ok, "should pass")
  checkIntSlice(t, []int{0,6,0,1,1,6}, res, "should be ungreedy")

  r = MustParse("^(?i)a*(?-i)b*$")
  checkState(t, r.RunSimple("AAaaAAaabbbbb"), "success")
  checkState(t, !r.RunSimple("AAaaAAaaBBBa"), "should fail, flag should not escape")
}

// Test the behaviour of rune classes.
func TestRuneClass(t *testing.T) {
  c := NewRuneClass()
  checkState(t, c.MatchRune('B'), "should implicitly match everything")

  c.AddRune(false, '#')
  checkState(t, !c.MatchRune('B'), "should no longer implicitly match everything")
  checkState(t, c.MatchRune('#'), "should match added rune")

  c.AddRuneRange(false, 'A', 'Z')
  checkState(t, c.MatchRune('A'), "should match rune 'A' in range")
  checkState(t, c.MatchRune('B'), "should match rune 'B' in range")

  c.AddRuneRange(true, 'B', 'C')
  checkState(t, c.MatchRune('A'), "should match rune 'A' in range")
  checkState(t, !c.MatchRune('B'), "should not match rune 'B' in range")

  c = NewRuneClass()
  c.AddUnicodeClass(false, "Greek")
  c.AddRune(true, 'Π')
  c.AddRune(false, 'A')
  checkState(t, !c.MatchRune('Π'), "should not match pi")
  checkState(t, c.MatchRune('Ω'), "should match omega")
  checkState(t, !c.MatchRune('Z'), "should not match regular latin char")
  checkState(t, c.MatchRune('A'), "should match included latin char")

  c = NewRuneClass()
  c.AddUnicodeClass(true, "Cyrillic")
  checkState(t, c.MatchRune('%'), "should match random char, class is exclude-only")
  c.AddRune(false, '')
  checkState(t, !c.MatchRune('%'), "should no longer match random char")
  checkState(t, c.MatchRune(''), "should match single opt-in char")
}

// Test the SafeParser used by much of the code.
func TestStringParser(t *testing.T) {
  src := NewSafeReader("a{bc}d")

  checkState(t, src.curr() == -1, "should not yet be parsing")
  checkState(t, src.nextCh() == 'a', "first char should be a")
  checkState(t, src.nextCh() == '{', "second char should be {")
  lit := src.literal("{", "}")
  checkState(t, lit == "bc", "should equal contained value, got: " + lit)
  checkState(t, src.curr() == 'd', "should now rest on d")
  checkState(t, src.nextCh() == -1, "should be done now")
}
<commit_msg>added expected tests<commit_after>
package sre2

import (
  "fmt"
  "testing"
)

// Check the given state to be true.
func checkState(t *testing.T, state bool, err string) {
  if !state {
    t.Error(err)
  }
}

// Check the equality of two []int slices.
func checkIntSlice(t *testing.T, expected []int, result []int, err string) {
  match := true
  if len(expected) != len(result) {
    match = false
  } else {
    for i := 0; i < len(expected); i++ {
      if expected[i] != result[i] {
        match = false
      }
    }
  }
  checkState(t, match, fmt.Sprintf("%s: got %s, expected %s", err, result, expected))
}

// Run a selection of basic regular expressions against this package.
func TestSimpleRe(t *testing.T) {
  r := MustParse("")
  checkState(t, r.RunSimple(""), "everything should match")
  checkState(t, r.RunSimple("fadsnjkflsdafnas"), "everything should match")

  r = MustParse("^(a|b)+c*$")
  checkState(t, !r.RunSimple("abd"), "not a valid match")
  checkState(t, r.RunSimple("a"), "basic string should match")
  checkState(t, !r.RunSimple(""), "empty string should not match")
  checkState(t, r.RunSimple("abcccc"), "longer string should match")

  r = MustParse("(\\w*)\\s*(\\w*)")
  ok, res := r.RunSubMatch("zing hello there")
  checkState(t, ok, "should match generally")
  checkIntSlice(t, []int{0, 10, 0, 4, 5, 10}, res, "did not match first two words as expected")

  r = MustParse(".*?(\\w+)$")
  ok, res = r.RunSubMatch("zing hello there")
  checkState(t, ok, "should match generally")
  checkIntSlice(t, []int{0, 16, 11, 16}, res, "did not match last word as expected")
}

// Test parsing an invalid RE returns an error.
func TestInvalidRe(t *testing.T) {
  r, err := Parse("a**")
  checkState(t, err != nil, "must fail parsing")
  checkState(t, r == nil, "regexp must be nil")

  pass := false
  func() {
    defer func() {
      if r := recover(); r != nil {
        pass = true
      }
    }()
    MustParse("z(((a")
  }()
  checkState(t, pass, "should panic")
}

// Test behaviour related to character classes expressed within [...].
func TestCharClass(t *testing.T) {
  r := MustParse("^[\t[:word:]]+$") // Match tabs and word characters.
  checkState(t, r.RunSimple("c"), "non-space should match")
  checkState(t, !r.RunSimple("c t"), "space should not match")
  checkState(t, r.RunSimple("c\tt"), "tab should match")

  r = MustParse("^[:ascii:]*$")
  checkState(t, r.RunSimple(""), "nothing should match")
  checkState(t, r.RunSimple("c"), "ascii should match")
  checkState(t, !r.RunSimple("Π"), "unicode should not match")

  r = MustParse("^\\pN$")
  checkState(t, r.RunSimple("〩"), "character from Nl should match")
  checkState(t, r.RunSimple("¾"), "character from Nu should match")

  r = MustParse("^\\p{Nl}$")
  checkState(t, r.RunSimple("〩"), "character from Nl should match")
  checkState(t, !r.RunSimple("¾"), "character from Nu should not match")

  r = MustParse("^[^.]$")
  r.DebugOut()
  checkState(t, r.RunSimple("\n"), "should match \\n")
  checkState(t, !r.RunSimple(" "), "should match only \\n")
}

// Test regexp generated by escape sequences (e.g. \n, \. etc).
func TestEscapeSequences(t *testing.T) {
  r := MustParse("^\\.\n\\044$") // Match '.\n$'
  checkState(t, r.RunSimple(".\n$"), "should match")
  checkState(t, !r.RunSimple(" \n$"), "space should not match")
  checkState(t, !r.RunSimple("\n\n$"), ". does not match \n by default")
  checkState(t, !r.RunSimple(".\n"), "# should not be treated as end char")

  r = MustParse("^\\x{03a0}\\x25$") // Match 'Π%'.
  checkState(t, r.RunSimple("Π%"), "should match pi+percent")

  r, err := Parse("^\\Π$")
  checkState(t, err != nil && r == nil,
      "should have failed on trying to escape Π, not punctuation")
}

// Tests string literals between \Q...\E.
func TestStringLiteral(t *testing.T) {
  r := MustParse("^\\Qhello\\E$")
  checkState(t, r.RunSimple("hello"), "should match hello")

  r = MustParse("^\\Q.$\\\\E$") // match ".$\\"
  checkState(t, r.RunSimple(".$\\"), "should match")
  checkState(t, !r.RunSimple(" $\\"), "should not match")

  r = MustParse("^a\\Q\\E*b$") // match absolutely nothing between 'ab'
  checkState(t, r.RunSimple("ab"), "should match")
  checkState(t, !r.RunSimple("acb"), "should not match")
}

// Test closure expansion types, such as {..}, ?, +, * etc.
func TestClosureExpansion(t *testing.T) {
  r := MustParse("^za?$")
  checkState(t, r.RunSimple("z"), "should match none")
  checkState(t, r.RunSimple("za"), "should match single")
  checkState(t, !r.RunSimple("zaa"), "should not match more")

  r = MustParse("^a{2,2}$")
  checkState(t, !r.RunSimple(""), "0 should fail")
  checkState(t, !r.RunSimple("a"), "1 should fail")
  checkState(t, r.RunSimple("aa"), "2 should succeed")
  checkState(t, r.RunSimple("aaa"), "3 should succeed")
  checkState(t, r.RunSimple("aaaa"), "4 should succeed")
  checkState(t, !r.RunSimple("aaaaa"), "5 should fail")

  r = MustParse("^a{2}$")
  checkState(t, !r.RunSimple(""), "0 should fail")
  checkState(t, !r.RunSimple("a"), "1 should fail")
  checkState(t, r.RunSimple("aa"), "2 should succeed")
  checkState(t, !r.RunSimple("aaa"), "3 should fail")

  r = MustParse("^a{3,}$")
  checkState(t, !r.RunSimple("aa"), "2 should fail")
  checkState(t, r.RunSimple("aaa"), "3 should succeed")
  checkState(t, r.RunSimple("aaaaaa"), "more should succeed")
}

// Test specific greedy/non-greedy closure types.
func TestClosureGreedy(t *testing.T) {
  r := MustParse("^(a{0,2}?)(a*)$")
  ok, res := r.RunSubMatch("aaa")
  checkState(t, ok, "should match")
  checkIntSlice(t, []int{0, 3, 0, 0, 0, 3}, res, "did not match expected")

  r = MustParse("^(a{0,2})?(a*)$")
  ok, res = r.RunSubMatch("aaa")
  checkState(t, ok, "should match")
  checkIntSlice(t, []int{0, 3, 0, 2, 2, 3}, res, "did not match expected")

  r = MustParse("^(a{2,}?)(a*)$")
  ok, res = r.RunSubMatch("aaa")
  checkState(t, ok, "should match")
  checkIntSlice(t, []int{0, 3, 0, 2, 2, 3}, res, "did not match expected")
}

// Test simple left/right matchers.
func TestLeftRight(t *testing.T) {
  r := MustParse("^.\\b.$")
  checkState(t, r.RunSimple("a "), "left char is word")
  checkState(t, r.RunSimple(" a"), "right char is word")
  checkState(t, !r.RunSimple("  "), "not a boundary")
  checkState(t, !r.RunSimple("aa"), "not a boundary")
}

// Test general flags in sre2.
func TestFlags(t *testing.T) {
  r := MustParse("^(?i:AbC)zz$")
  checkState(t, r.RunSimple("abczz"), "success")
  checkState(t, !r.RunSimple("abcZZ"), "fail, flag should not escape")
  ok, res := r.RunSubMatch("ABCzz")
  checkState(t, ok, "should pass")
  checkIntSlice(t, []int{0,5}, res, "should just have a single outer paren")

  r = MustParse("^(?U)(a+)(.+)$")
  ok, res = r.RunSubMatch("aaaabb")
  checkState(t, ok, "should pass")
  checkIntSlice(t, []int{0,6,0,1,1,6}, res, "should be ungreedy")

  r = MustParse("^(?i)a*(?-i)b*$")
  checkState(t, r.RunSimple("AAaaAAaabbbbb"), "success")
  checkState(t, !r.RunSimple("AAaaAAaaBBBa"), "should fail, flag should not escape")
}

// Test the behaviour of rune classes.
func TestRuneClass(t *testing.T) {
  c := NewRuneClass()
  checkState(t, c.MatchRune('B'), "should implicitly match everything")

  c.AddRune(false, '#')
  checkState(t, !c.MatchRune('B'), "should no longer implicitly match everything")
  checkState(t, c.MatchRune('#'), "should match added rune")

  c.AddRuneRange(false, 'A', 'Z')
  checkState(t, c.MatchRune('A'), "should match rune 'A' in range")
  checkState(t, c.MatchRune('B'), "should match rune 'B' in range")

  c.AddRuneRange(true, 'B', 'C')
  checkState(t, c.MatchRune('A'), "should match rune 'A' in range")
  checkState(t, !c.MatchRune('B'), "should not match rune 'B' in range")

  c = NewRuneClass()
  c.AddUnicodeClass(false, "Greek")
  c.AddRune(true, 'Π')
  c.AddRune(false, 'A')
  checkState(t, !c.MatchRune('Π'), "should not match pi")
  checkState(t, c.MatchRune('Ω'), "should match omega")
  checkState(t, !c.MatchRune('Z'), "should not match regular latin char")
  checkState(t, c.MatchRune('A'), "should match included latin char")

  c = NewRuneClass()
  c.AddUnicodeClass(true, "Cyrillic")
  checkState(t, c.MatchRune('%'), "should match random char, class is exclude-only")
  c.AddRune(false, '')
  checkState(t, !c.MatchRune('%'), "should no longer match random char")
  checkState(t, c.MatchRune(''), "should match single opt-in char")
}

// Test the SafeParser used by much of the code.
func TestStringParser(t *testing.T) {
  src := NewSafeReader("a{bc}d")

  checkState(t, src.curr() == -1, "should not yet be parsing")
  checkState(t, src.nextCh() == 'a', "first char should be a")
  checkState(t, src.nextCh() == '{', "second char should be {")
  lit := src.literal("{", "}")
  checkState(t, lit == "bc", "should equal contained value, got: " + lit)
  checkState(t, src.curr() == 'd', "should now rest on d")
  checkState(t, src.nextCh() == -1, "should be done now")
}
<|endoftext|>


<commit_before>package library

import (
	"errors"
	"github.com/mjibson/go-dsp/fft"            // fft
	"github.com/nytlabs/gojee"                 // jee
	"github.com/nytlabs/streamtools/st/blocks" // blocks
	"github.com/nytlabs/streamtools/st/util"   // util
	"time"
)

// specify those channels we're going to use to communicate with streamtools
type FFT struct {
	blocks.Block
	queryrule chan chan interface{}
	queryfft  chan chan interface{}
	inrule    chan interface{}
	inpoll    chan interface{}
	in        chan interface{}
	out       chan interface{}
	quit      chan interface{}
}

func buildFFT(data tsData) [][]float64 {
	x := make([]float64, len(data.Values))
	for i, d := range data.Values {
		x[i] = d.Value
	}
	X := fft.FFTReal(x)
	Xout := make([][]float64, len(X))
	for i, Xi := range X {
		Xout[i] = make([]float64, 2)
		Xout[i][0] = real(Xi)
		Xout[i][1] = imag(Xi)
	}
	return Xout
}

// we need to build a simple factory so that streamtools can make new blocks of this kind
func NewFFT() blocks.BlockInterface {
	return &FFT{}
}

// Setup is called once before running the block. We build up the channels and specify what kind of block this is.
func (b *FFT) Setup() {
	b.Kind = "FFT"
	b.in = b.InRoute("in")
	b.inrule = b.InRoute("rule")
	b.queryrule = b.QueryRoute("rule")
	b.queryfft = b.QueryRoute("fft")
	b.inpoll = b.InRoute("poll")
	b.quit = b.Quit()
	b.out = b.Broadcast()
}

// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *FFT) Run() {

	var err error
	//var path, lagStr string
	var path string
	var tree *jee.TokenTree
	//var lag time.Duration
	var data tsData
	var numSamples float64

	// defaults
	numSamples = 512

	for {
		select {
		case ruleI := <-b.inrule:
			// set a parameter of the block
			rule, ok := ruleI.(map[string]interface{})
			if !ok {
				b.Error(errors.New("could not assert rule to map"))
			}
			path, err = util.ParseString(rule, "Path")
			if err != nil {
				b.Error(err)
				continue
			}
			tree, err = util.BuildTokenTree(path)
			if err != nil {
				b.Error(err)
				continue
			}
			numSamples, err = util.ParseFloat(rule, "NumSamples")
			if err != nil {
				b.Error(err)
				continue
			}
			data = tsData{
				Values: make([]tsDataPoint, int(numSamples)),
			}

		case <-b.quit:
			// quit * time.Second the block
			return
		case msg := <-b.in:
			if tree == nil {
				continue
			}
			if data.Values == nil {
				continue
			}
			// deal with inbound data
			v, err := jee.Eval(tree, msg)
			if err != nil {
				b.Error(err)
				continue
			}
			var val float64
			switch v := v.(type) {
			case float32:
				val = float64(v)
			case int:
				val = float64(v)
			case float64:
				val = v
			}

			//t := float64(time.Now().Add(-lag).Unix())
			t := float64(time.Now().Unix())

			d := tsDataPoint{
				Timestamp: t,
				Value:     val,
			}
			data.Values = append(data.Values[1:], d)
		case respChan := <-b.queryrule:
			// deal with a query request
			respChan <- map[string]interface{}{
				//"Window":     lagStr,
				"Path":       path,
				"NumSamples": numSamples,
			}
		case respChan := <-b.queryfft:
			out := map[string]interface{}{
				"fft": buildFFT(data),
			}
			respChan <- out
		case <-b.inpoll:
			out := map[string]interface{}{
				"fft": buildFFT(data),
			}
			b.out <- out
		}
	}
}
<commit_msg>FFT fix for the route change<commit_after>package library

import (
	"errors"
	"github.com/mjibson/go-dsp/fft"            // fft
	"github.com/nytlabs/gojee"                 // jee
	"github.com/nytlabs/streamtools/st/blocks" // blocks
	"github.com/nytlabs/streamtools/st/util"   // util
	"time"
)

// specify those channels we're going to use to communicate with streamtools
type FFT struct {
	blocks.Block
	queryrule chan blocks.MsgChan
	queryfft  chan blocks.MsgChan
	inrule    blocks.MsgChan
	inpoll    blocks.MsgChan
	in        blocks.MsgChan
	out       blocks.MsgChan
	quit      blocks.MsgChan
}

func buildFFT(data tsData) [][]float64 {
	x := make([]float64, len(data.Values))
	for i, d := range data.Values {
		x[i] = d.Value
	}
	X := fft.FFTReal(x)
	Xout := make([][]float64, len(X))
	for i, Xi := range X {
		Xout[i] = make([]float64, 2)
		Xout[i][0] = real(Xi)
		Xout[i][1] = imag(Xi)
	}
	return Xout
}

// we need to build a simple factory so that streamtools can make new blocks of this kind
func NewFFT() blocks.BlockInterface {
	return &FFT{}
}

// Setup is called once before running the block. We build up the channels and specify what kind of block this is.
func (b *FFT) Setup() {
	b.Kind = "FFT"
	b.in = b.InRoute("in")
	b.inrule = b.InRoute("rule")
	b.queryrule = b.QueryRoute("rule")
	b.queryfft = b.QueryRoute("fft")
	b.inpoll = b.InRoute("poll")
	b.quit = b.Quit()
	b.out = b.Broadcast()
}

// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *FFT) Run() {

	var err error
	//var path, lagStr string
	var path string
	var tree *jee.TokenTree
	//var lag time.Duration
	var data tsData
	var numSamples float64

	// defaults
	numSamples = 512

	for {
		select {
		case ruleI := <-b.inrule:
			// set a parameter of the block
			rule, ok := ruleI.(map[string]interface{})
			if !ok {
				b.Error(errors.New("could not assert rule to map"))
			}
			path, err = util.ParseString(rule, "Path")
			if err != nil {
				b.Error(err)
				continue
			}
			tree, err = util.BuildTokenTree(path)
			if err != nil {
				b.Error(err)
				continue
			}
			numSamples, err = util.ParseFloat(rule, "NumSamples")
			if err != nil {
				b.Error(err)
				continue
			}
			data = tsData{
				Values: make([]tsDataPoint, int(numSamples)),
			}

		case <-b.quit:
			// quit * time.Second the block
			return
		case msg := <-b.in:
			if tree == nil {
				continue
			}
			if data.Values == nil {
				continue
			}
			// deal with inbound data
			v, err := jee.Eval(tree, msg)
			if err != nil {
				b.Error(err)
				continue
			}
			var val float64
			switch v := v.(type) {
			case float32:
				val = float64(v)
			case int:
				val = float64(v)
			case float64:
				val = v
			}

			//t := float64(time.Now().Add(-lag).Unix())
			t := float64(time.Now().Unix())

			d := tsDataPoint{
				Timestamp: t,
				Value:     val,
			}
			data.Values = append(data.Values[1:], d)
		case respChan := <-b.queryrule:
			// deal with a query request
			respChan <- map[string]interface{}{
				//"Window":     lagStr,
				"Path":       path,
				"NumSamples": numSamples,
			}
		case respChan := <-b.queryfft:
			out := map[string]interface{}{
				"fft": buildFFT(data),
			}
			respChan <- out
		case <-b.inpoll:
			out := map[string]interface{}{
				"fft": buildFFT(data),
			}
			b.out <- out
		}
	}
}
<|endoftext|>


<commit_before>package CSVWriter

import (
	"encoding/binary"
	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/Layout"
	"github.com/xitongsys/parquet-go/ParquetFile"
	"github.com/xitongsys/parquet-go/ParquetType"
	"github.com/xitongsys/parquet-go/SchemaHandler"
	"github.com/xitongsys/parquet-go/parquet"
	"reflect"
	"strings"
)

//Write handler for CSV data
type CSVWriter struct {
	SchemaHandler *SchemaHandler.SchemaHandler
	NP            int64
	Footer        *parquet.FileMetaData
	RowGroups     []*Layout.RowGroup

	PFile ParquetFile.ParquetFile

	PageSize     int64
	RowGroupSize int64
	Offset       int64

	Objs              [][]interface{}
	ObjsSize          int64
	ObjSize           int64
	CheckSizeCritical int64
	Metadata          []MetadataType

	PagesMapBuf map[string][]*Layout.Page
	Size        int64
	NumRows     int64
}

//Create CSV writer
func NewCSVWriter(md []MetadataType, pfile ParquetFile.ParquetFile, np int64) (*CSVWriter, error) {
	res := new(CSVWriter)
	res.SchemaHandler = NewSchemaHandlerFromMetadata(md)
	res.Metadata = md
	res.PFile = pfile
	res.PageSize = 8 * 1024              //8K
	res.RowGroupSize = 128 * 1024 * 1024 //128M
	res.PagesMapBuf = make(map[string][]*Layout.Page)
	res.NP = np
	res.Footer = parquet.NewFileMetaData()
	res.Footer.Version = 1
	res.Footer.Schema = append(res.Footer.Schema, res.SchemaHandler.SchemaElements...)
	res.Offset = 4
	_, err := res.PFile.Write([]byte("PAR1"))
	return res, err
}

//Convert the column names to lowercase
func (self *CSVWriter) NameToLower() {
	for _, schema := range self.Footer.Schema {
		schema.Name = strings.ToLower(schema.Name)
	}
	for _, rowGroup := range self.Footer.RowGroups {
		for _, chunk := range rowGroup.Columns {
			ln := len(chunk.MetaData.PathInSchema)
			for i := 0; i < ln; i++ {
				chunk.MetaData.PathInSchema[i] = strings.ToLower(chunk.MetaData.PathInSchema[i])
			}
		}
	}
}

//Write string values to parquet file
func (self *CSVWriter) WriteString(recs []*string) {
	lr := len(recs)
	rec := make([]interface{}, lr)
	for i := 0; i < lr; i++ {
		rec[i] = nil
		if recs[i] != nil {
			rec[i] = ParquetType.StrToParquetType(*recs[i], self.Metadata[i].Type)
		}
	}

	ln := int64(len(self.Objs))
	if self.CheckSizeCritical <= ln {
		self.ObjSize = Common.SizeOf(reflect.ValueOf(rec))
	}
	self.ObjsSize += self.ObjSize
	self.Objs = append(self.Objs, rec)

	criSize := self.NP * self.PageSize * self.SchemaHandler.GetColumnNum()

	if self.ObjsSize > criSize {
		self.Flush(false)
	} else {
		dln := (criSize - self.ObjsSize + self.ObjSize - 1) / self.ObjSize / 2
		self.CheckSizeCritical = dln + ln
	}
}

//Write parquet values to parquet file
func (self *CSVWriter) Write(rec []interface{}) {
	ln := int64(len(self.Objs))
	if self.CheckSizeCritical <= ln {
		self.ObjSize = Common.SizeOf(reflect.ValueOf(rec))
	}

	self.ObjsSize += self.ObjSize
	self.Objs = append(self.Objs, rec)

	criSize := self.NP * self.PageSize * self.SchemaHandler.GetColumnNum()

	if self.ObjsSize > criSize {
		self.Flush(false)
	} else {
		dln := (criSize - self.ObjsSize + self.ObjSize - 1) / self.ObjSize / 2
		self.CheckSizeCritical = dln + ln
	}
}

//Write footer to parquet file and stop writing
func (self *CSVWriter) WriteStop() {
	//self.Flush()
	ts := thrift.NewTSerializer()
	ts.Protocol = thrift.NewTCompactProtocolFactory().GetProtocol(ts.Transport)
	footerBuf, _ := ts.Write(self.Footer)

	self.PFile.Write(footerBuf)
	footerSizeBuf := make([]byte, 4)
	binary.LittleEndian.PutUint32(footerSizeBuf, uint32(len(footerBuf)))
	self.PFile.Write(footerSizeBuf)
	self.PFile.Write([]byte("PAR1"))
}

//Flush the write buffer to parquet file
func (self *CSVWriter) Flush(flag bool) {
	pagesMapList := make([]map[string][]*Layout.Page, self.NP)
	for i := 0; i < int(self.NP); i++ {
		pagesMapList[i] = make(map[string][]*Layout.Page)
	}

	doneChan := make(chan int)
	l := int64(len(self.Objs))
	var c int64 = 0
	delta := (l + self.NP - 1) / self.NP
	for c = 0; c < self.NP; c++ {
		bgn := c * delta
		end := bgn + delta
		if end > l {
			end = l
		}
		if bgn >= l {
			bgn, end = l, l
		}

		go func(b, e int, index int64) {
			if e <= b {
				doneChan <- 0
				return
			}
			tableMap := MarshalCSV(self.Objs, b, e, self.Metadata, self.SchemaHandler)
			for name, table := range *tableMap {
				pagesMapList[index][name], _ = Layout.TableToDataPages(table, int32(self.PageSize),
					parquet.CompressionCodec_SNAPPY)
			}

			doneChan <- 0
		}(int(bgn), int(end), c)
	}

	for c = 0; c < self.NP; c++ {
		<-doneChan
	}

	for _, pagesMap := range pagesMapList {
		for name, pages := range pagesMap {
			if _, ok := self.PagesMapBuf[name]; !ok {
				self.PagesMapBuf[name] = pages
			} else {
				self.PagesMapBuf[name] = append(self.PagesMapBuf[name], pages...)
			}
			for _, page := range pages {
				self.Size += int64(len(page.RawData))
				page.DataTable = nil
			}
		}
	}

	self.NumRows += int64(len(self.Objs))

	if self.Size+self.ObjsSize >= self.RowGroupSize || flag {
		//pages -> chunk
		chunkMap := make(map[string]*Layout.Chunk)
		for name, pages := range self.PagesMapBuf {
			chunkMap[name] = Layout.PagesToChunk(pages)
		}

		//chunks -> rowGroup
		rowGroup := Layout.NewRowGroup()
		rowGroup.RowGroupHeader.Columns = make([]*parquet.ColumnChunk, 0)

		for k := 0; k < len(self.SchemaHandler.SchemaElements); k++ {
			//for _, chunk := range chunkMap {
			schema := self.SchemaHandler.SchemaElements[k]
			if schema.GetNumChildren() > 0 {
				continue
			}
			chunk := chunkMap[self.SchemaHandler.IndexMap[int32(k)]]
			rowGroup.Chunks = append(rowGroup.Chunks, chunk)
			rowGroup.RowGroupHeader.TotalByteSize += chunk.ChunkHeader.MetaData.TotalCompressedSize
			rowGroup.RowGroupHeader.Columns = append(rowGroup.RowGroupHeader.Columns, chunk.ChunkHeader)
		}
		rowGroup.RowGroupHeader.NumRows = self.NumRows
		self.NumRows = 0

		for k := 0; k < len(rowGroup.Chunks); k++ {
			rowGroup.Chunks[k].ChunkHeader.MetaData.DataPageOffset = self.Offset
			rowGroup.Chunks[k].ChunkHeader.FileOffset = self.Offset

			for l := 0; l < len(rowGroup.Chunks[k].Pages); l++ {
				data := rowGroup.Chunks[k].Pages[l].RawData
				self.PFile.Write(data)
				self.Offset += int64(len(data))
			}
		}
		self.Footer.RowGroups = append(self.Footer.RowGroups, rowGroup.RowGroupHeader)
		self.Size = 0
		self.PagesMapBuf = make(map[string][]*Layout.Page)
	}
	self.Footer.NumRows += int64(len(self.Objs))
	self.Objs = self.Objs[:0]
	self.ObjsSize = 0
}
<commit_msg>fix csvwriter bug<commit_after>package CSVWriter

import (
	"encoding/binary"
	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/Layout"
	"github.com/xitongsys/parquet-go/ParquetFile"
	"github.com/xitongsys/parquet-go/ParquetType"
	"github.com/xitongsys/parquet-go/SchemaHandler"
	"github.com/xitongsys/parquet-go/parquet"
	"reflect"
)

//Write handler for CSV data
type CSVWriter struct {
	SchemaHandler *SchemaHandler.SchemaHandler
	NP            int64
	Footer        *parquet.FileMetaData
	RowGroups     []*Layout.RowGroup

	PFile ParquetFile.ParquetFile

	PageSize     int64
	RowGroupSize int64
	Offset       int64

	Objs              [][]interface{}
	ObjsSize          int64
	ObjSize           int64
	CheckSizeCritical int64
	Metadata          []MetadataType

	PagesMapBuf map[string][]*Layout.Page
	Size        int64
	NumRows     int64
}

//Create CSV writer
func NewCSVWriter(md []MetadataType, pfile ParquetFile.ParquetFile, np int64) (*CSVWriter, error) {
	res := new(CSVWriter)
	res.SchemaHandler = NewSchemaHandlerFromMetadata(md)
	res.Metadata = md
	res.PFile = pfile
	res.PageSize = 8 * 1024              //8K
	res.RowGroupSize = 128 * 1024 * 1024 //128M
	res.PagesMapBuf = make(map[string][]*Layout.Page)
	res.NP = np
	res.Footer = parquet.NewFileMetaData()
	res.Footer.Version = 1
	res.Footer.Schema = append(res.Footer.Schema, res.SchemaHandler.SchemaElements...)
	res.Offset = 4
	_, err := res.PFile.Write([]byte("PAR1"))
	return res, err
}

//Rename schema name to exname
func (self *CSVWriter) RenameSchema() {
	for _, rowGroup := range self.Footer.RowGroups {
		for _, chunk := range rowGroup.Columns {
			chunk.MetaData.PathInSchema = chunk.MetaData.PathInSchema[1:]
		}
	}
}

//Write string values to parquet file
func (self *CSVWriter) WriteString(recs []*string) {
	lr := len(recs)
	rec := make([]interface{}, lr)
	for i := 0; i < lr; i++ {
		rec[i] = nil
		if recs[i] != nil {
			rec[i] = ParquetType.StrToParquetType(*recs[i], self.Metadata[i].Type)
		}
	}

	ln := int64(len(self.Objs))
	if self.CheckSizeCritical <= ln {
		self.ObjSize = Common.SizeOf(reflect.ValueOf(rec))
	}
	self.ObjsSize += self.ObjSize
	self.Objs = append(self.Objs, rec)

	criSize := self.NP * self.PageSize * self.SchemaHandler.GetColumnNum()

	if self.ObjsSize > criSize {
		self.Flush(false)
	} else {
		dln := (criSize - self.ObjsSize + self.ObjSize - 1) / self.ObjSize / 2
		self.CheckSizeCritical = dln + ln
	}
}

//Write parquet values to parquet file
func (self *CSVWriter) Write(rec []interface{}) {
	ln := int64(len(self.Objs))
	if self.CheckSizeCritical <= ln {
		self.ObjSize = Common.SizeOf(reflect.ValueOf(rec))
	}

	self.ObjsSize += self.ObjSize
	self.Objs = append(self.Objs, rec)

	criSize := self.NP * self.PageSize * self.SchemaHandler.GetColumnNum()

	if self.ObjsSize > criSize {
		self.Flush(false)
	} else {
		dln := (criSize - self.ObjsSize + self.ObjSize - 1) / self.ObjSize / 2
		self.CheckSizeCritical = dln + ln
	}
}

//Write footer to parquet file and stop writing
func (self *CSVWriter) WriteStop() {
	//self.Flush()
	ts := thrift.NewTSerializer()
	ts.Protocol = thrift.NewTCompactProtocolFactory().GetProtocol(ts.Transport)
	self.RenameSchema()
	footerBuf, _ := ts.Write(self.Footer)

	self.PFile.Write(footerBuf)
	footerSizeBuf := make([]byte, 4)
	binary.LittleEndian.PutUint32(footerSizeBuf, uint32(len(footerBuf)))
	self.PFile.Write(footerSizeBuf)
	self.PFile.Write([]byte("PAR1"))
}

//Flush the write buffer to parquet file
func (self *CSVWriter) Flush(flag bool) {
	pagesMapList := make([]map[string][]*Layout.Page, self.NP)
	for i := 0; i < int(self.NP); i++ {
		pagesMapList[i] = make(map[string][]*Layout.Page)
	}

	doneChan := make(chan int)
	l := int64(len(self.Objs))
	var c int64 = 0
	delta := (l + self.NP - 1) / self.NP
	for c = 0; c < self.NP; c++ {
		bgn := c * delta
		end := bgn + delta
		if end > l {
			end = l
		}
		if bgn >= l {
			bgn, end = l, l
		}

		go func(b, e int, index int64) {
			if e <= b {
				doneChan <- 0
				return
			}
			tableMap := MarshalCSV(self.Objs, b, e, self.Metadata, self.SchemaHandler)
			for name, table := range *tableMap {
				pagesMapList[index][name], _ = Layout.TableToDataPages(table, int32(self.PageSize),
					parquet.CompressionCodec_SNAPPY)
			}

			doneChan <- 0
		}(int(bgn), int(end), c)
	}

	for c = 0; c < self.NP; c++ {
		<-doneChan
	}

	for _, pagesMap := range pagesMapList {
		for name, pages := range pagesMap {
			if _, ok := self.PagesMapBuf[name]; !ok {
				self.PagesMapBuf[name] = pages
			} else {
				self.PagesMapBuf[name] = append(self.PagesMapBuf[name], pages...)
			}
			for _, page := range pages {
				self.Size += int64(len(page.RawData))
				page.DataTable = nil
			}
		}
	}

	self.NumRows += int64(len(self.Objs))

	if self.Size+self.ObjsSize >= self.RowGroupSize || flag {
		//pages -> chunk
		chunkMap := make(map[string]*Layout.Chunk)
		for name, pages := range self.PagesMapBuf {
			chunkMap[name] = Layout.PagesToChunk(pages)
		}

		//chunks -> rowGroup
		rowGroup := Layout.NewRowGroup()
		rowGroup.RowGroupHeader.Columns = make([]*parquet.ColumnChunk, 0)

		for k := 0; k < len(self.SchemaHandler.SchemaElements); k++ {
			//for _, chunk := range chunkMap {
			schema := self.SchemaHandler.SchemaElements[k]
			if schema.GetNumChildren() > 0 {
				continue
			}
			chunk := chunkMap[self.SchemaHandler.IndexMap[int32(k)]]
			rowGroup.Chunks = append(rowGroup.Chunks, chunk)
			rowGroup.RowGroupHeader.TotalByteSize += chunk.ChunkHeader.MetaData.TotalCompressedSize
			rowGroup.RowGroupHeader.Columns = append(rowGroup.RowGroupHeader.Columns, chunk.ChunkHeader)
		}
		rowGroup.RowGroupHeader.NumRows = self.NumRows
		self.NumRows = 0

		for k := 0; k < len(rowGroup.Chunks); k++ {
			rowGroup.Chunks[k].ChunkHeader.MetaData.DataPageOffset = self.Offset
			rowGroup.Chunks[k].ChunkHeader.FileOffset = self.Offset

			for l := 0; l < len(rowGroup.Chunks[k].Pages); l++ {
				data := rowGroup.Chunks[k].Pages[l].RawData
				self.PFile.Write(data)
				self.Offset += int64(len(data))
			}
		}
		self.Footer.RowGroups = append(self.Footer.RowGroups, rowGroup.RowGroupHeader)
		self.Size = 0
		self.PagesMapBuf = make(map[string][]*Layout.Page)
	}
	self.Footer.NumRows += int64(len(self.Objs))
	self.Objs = self.Objs[:0]
	self.ObjsSize = 0
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "net/http"
    "os"
    "reflect"
    "strings"

    "github.com/op/go-logging"
    "github.com/gorilla/websocket"
    "github.com/dghubble/go-twitter/twitter"
	"github.com/dghubble/oauth1"
)

// Get Consumer key/secret and Access token/secret from enviroment variables
var consumerKey = os.Getenv("TWITTER_CONSUMER_KEY")
var consumerSecret = os.Getenv("TWITTER_CONSUMER_SECRET")
var accessToken = os.Getenv("TWITTER_ACCESS_TOKEN")
var accessSecret = os.Getenv("TWITTER_ACCESS_SECRET")

var config = oauth1.NewConfig(consumerKey, consumerSecret)
var token = oauth1.NewToken(accessToken, accessSecret)

// Auto Oauth1
var httpClient = config.Client(oauth1.NoContext, token)

// Twitter Client
var client = twitter.NewClient(httpClient)

// Websocket Upgrade buffer
// Allow any domain can access stream API
var upgrader = websocket.Upgrader {
    ReadBufferSize:  1024,
    WriteBufferSize: 1024,
    CheckOrigin: func(r *http.Request) bool { return true },
}

var filters = []string{};
var conns = make(map[string]*websocket.Conn)

var log = logging.MustGetLogger("Twitter Tweeds Log")

var globalStream *twitter.Stream

// Start the app then Listen and Serve
func main() {
    http.HandleFunc("/", home)
    http.HandleFunc("/twitter/stream", twitterStream)

    http.ListenAndServe(":8080",nil)
}

// Serve home page
func home(res http.ResponseWriter, req *http.Request) {
    res.Header().Set("Access-Control-Allow-Origin", "*")
    http.ServeFile(res,req, "../public/index.html")
}

// Serve Websocket Twitter stream
func twitterStream(res http.ResponseWriter, req *http.Request) {
    log.Info("WS Twitter Request: ",req.URL.Query())
    filter := req.URL.Query()["filter"]
    if filter != nil {

        // Upgrades the http server connection to the websocket protocol
        conn, _ := upgrader.Upgrade(res, req, nil)

        fmt.Println("CONN:",reflect.TypeOf(conn))
        filters = append(filters,filter[0]);
        conns[filter[0]] = conn;
        fmt.Println("Filters",filters)
        fmt.Println("Connections:",conns)

        if globalStream != nil {
            globalStream.Stop()
        }

    	// Read only tweet from twitter stream
    	demux := twitter.NewSwitchDemux()
    	demux.Tweet = func(tweet *twitter.Tweet) {
    	    text := tweet.Text

    	    for _, v := range filters {
    	       if strings.Contains(text,v) && conns[v] != nil {
    	           wsWriter(conns[v],text)
    	       }
    	    }
    	}

    	fmt.Println("Starting Stream...")

    	// Filter
    	filterParams := &twitter.StreamFilterParams{
    		Track:         filters,
    		StallWarnings: twitter.Bool(true),
    	}

    	stream, err := client.Streams.Filter(filterParams)

    	globalStream = stream

    	fmt.Println(reflect.TypeOf(stream))
    	if err != nil {
    		log.Error(err)
    	}

        go wsReader(conn,filter[0],stream)      // Read messages from the websocket or stop websoket and twitter stream
    	go demux.HandleChan(stream.Messages)    // Receive twitter messages until stream quits

    } else {
        fmt.Fprintf(res, "Warning: Filter param is required to stream the data from Twitter")
    }
}

// Read messages from the websocket
func wsReader(conn *websocket.Conn,filter string,stream *twitter.Stream) {
    for {
        _, _, err := conn.ReadMessage()

        //Close the websocket connection
        if err != nil {

            for index, value := range filters {
                if value == filter {
                    // remove filtered word from filter slice
                    filters = append(filters[:index], filters[index+1:]...)
                    break
                }
            }

             // Remove connections from connection list
            delete(conns, filter)

            // Close connection
            conn.Close()
            log.Info("wsReader/WS Connection Closed: "+filter)
            break
        } else {
            log.Error(err)
        }
    }
}

// Write messages to the websocket
func wsWriter(conn *websocket.Conn,filter string) {
    conn.WriteJSON(tweetStruct{
        Tweet: filter,
    })
}

type tweetStruct struct {
    Tweet string
}<commit_msg>Remove tweet filter word from the list<commit_after>package main

import (
    "fmt"
    "net/http"
    "os"
    "reflect"
    "strings"

    "github.com/op/go-logging"
    "github.com/gorilla/websocket"
    "github.com/dghubble/go-twitter/twitter"
	"github.com/dghubble/oauth1"
)

// Get Consumer key/secret and Access token/secret from enviroment variables
var consumerKey = os.Getenv("TWITTER_CONSUMER_KEY")
var consumerSecret = os.Getenv("TWITTER_CONSUMER_SECRET")
var accessToken = os.Getenv("TWITTER_ACCESS_TOKEN")
var accessSecret = os.Getenv("TWITTER_ACCESS_SECRET")

var config = oauth1.NewConfig(consumerKey, consumerSecret)
var token = oauth1.NewToken(accessToken, accessSecret)

// Auto Oauth1
var httpClient = config.Client(oauth1.NoContext, token)

// Twitter Client
var client = twitter.NewClient(httpClient)

// Websocket Upgrade buffer
// Allow any domain can access stream API
var upgrader = websocket.Upgrader {
    ReadBufferSize:  1024,
    WriteBufferSize: 1024,
    CheckOrigin: func(r *http.Request) bool { return true },
}

var filters = []string{};
var conns = make(map[string]*websocket.Conn)

var log = logging.MustGetLogger("Twitter Tweeds Log")

var globalTwitterStream *twitter.Stream

// Start the app then Listen and Serve
func main() {

    http.HandleFunc("/", home)
    http.HandleFunc("/twitter/stream", twitterStream)
    fmt.Println("TweetDeck started and ready to rock");
    http.ListenAndServe(":8080",nil)
}

// Serve home page
func home(res http.ResponseWriter, req *http.Request) {
    res.Header().Set("Access-Control-Allow-Origin", "*")
    http.ServeFile(res,req, "../public/index.html")
}

// Serve Websocket Twitter stream
func twitterStream(res http.ResponseWriter, req *http.Request) {
    log.Info("WS Twitter Request: ",req.URL.Query())
    filter := req.URL.Query()["filter"]
    if filter != nil {

        // Upgrades the http server connection to the websocket protocol
        conn, _ := upgrader.Upgrade(res, req, nil)

        fmt.Println("CONN:",reflect.TypeOf(conn))
        filters = append(filters,filter[0]);
        conns[filter[0]] = conn;
        fmt.Println("Filters",filters)
        fmt.Println("Connections:",conns)

        // Stop the twitter stream in order to stream again with new filter
        stopGlobTwitterStream()

    	// Read only tweet from twitter stream
    	demux := twitter.NewSwitchDemux()
    	demux.Tweet = func(tweet *twitter.Tweet) {
    	    text := tweet.Text

    	    for _, v := range filters {
    	       if strings.Contains(text,v) && conns[v] != nil {
    	           wsWriter(conns[v],text)
    	       }
    	    }
    	}

    	fmt.Println("Starting Stream...")

    	// Filter
    	filterParams := &twitter.StreamFilterParams{
    		Track:         filters,
    		StallWarnings: twitter.Bool(true),
    	}

    	stream, err := client.Streams.Filter(filterParams)

    	globalTwitterStream = stream

    	fmt.Println(reflect.TypeOf(stream))
    	if err != nil {
    		log.Error(err)
    	}

        go wsReader(conn,filter[0],stream)      // Read messages from the websocket or stop websoket and twitter stream
    	go demux.HandleChan(stream.Messages)    // Receive twitter messages until stream quits

    } else {
        fmt.Fprintf(res, "Warning: Filter param is required to stream the data from Twitter")
    }
}

func stopGlobTwitterStream() {
    if globalTwitterStream != nil {
            globalTwitterStream.Stop()
    }
}

// Read messages from the websocket
func wsReader(conn *websocket.Conn,filter string,stream *twitter.Stream) {
    for {
        _, _, err := conn.ReadMessage()

        // Close the websocket connection
        if err != nil {
            // Close connection
            conn.Close()
             // Remove connections from connection list
            delete(conns, filter)

            for index, value := range filters {
                if value == filter {
                    // Remove filtered word from filter slice
                    filters = append(filters[:index], filters[index+1:]...)
                    break
                }
            }

            log.Info("wsReader/WS Connection Closed: "+filter)
            break
        } else {
            log.Error(err)
        }
    }
}

// Write messages to the websocket
func wsWriter(conn *websocket.Conn,filter string) {
    conn.WriteJSON(tweetStruct{
        Tweet: filter,
    })
}

type tweetStruct struct {
    Tweet string
}<|endoftext|>


<commit_before>package controllers

import "fmt"
import "net/http"
import "encoding/json"
import "github.com/julienschmidt/httprouter"
import "xyz/projectplay/onigokko/database"
import "xyz/projectplay/onigokko/models"

type PlayerController struct{}

var dao database.DatabaseAccessObject

func NewPlayerController() *PlayerController {
	dao = database.NewSQLDao()
	return &PlayerController{}
}

func (pc PlayerController) GetPlayer(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	token := r.Header.Get("Token")
	if dao.IsValidToken(token) {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)
		id := p.ByName("id")
		fmt.Fprintf(w, "{\"player\": \""+id+"\"}")
	} else {
		w.WriteHeader(http.StatusUnauthorized)
	}
}

func (pc PlayerController) CreatePlayer(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	decoder := json.NewDecoder(r.Body)
	var p models.Player
	err := decoder.Decode(&p)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "%s\n", err)
		return
	}
	defer r.Body.Close()
	err = dao.InsertPlayer(p)
	if err != nil {
		w.WriteHeader(http.StatusConflict)
		fmt.Fprintf(w, "This record already exists.\n")
		return
	}
}
<commit_msg>Remove unnecessary close on body and update .gitignore<commit_after>package controllers

import "fmt"
import "net/http"
import "encoding/json"
import "github.com/julienschmidt/httprouter"
import "xyz/projectplay/onigokko/database"
import "xyz/projectplay/onigokko/models"

type PlayerController struct{}

var dao database.DatabaseAccessObject

func NewPlayerController() *PlayerController {
	dao = database.NewSQLDao()
	return &PlayerController{}
}

func (pc PlayerController) GetPlayer(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	token := r.Header.Get("Token")
	if dao.IsValidToken(token) {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)
		id := p.ByName("id")
		fmt.Fprintf(w, "{\"player\": \""+id+"\"}")
	} else {
		w.WriteHeader(http.StatusUnauthorized)
	}
}

func (pc PlayerController) CreatePlayer(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	decoder := json.NewDecoder(r.Body)
	var p models.Player
	err := decoder.Decode(&p)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "%s\n", err)
		return
	}
	err = dao.InsertPlayer(p)
	if err != nil {
		w.WriteHeader(http.StatusConflict)
		fmt.Fprintf(w, "This record already exists.\n")
		return
	}
}
<|endoftext|>


<commit_before>package db

import "github.com/jinzhu/gorm"

//go:generate counterfeiter . CredentialRepository

type CredentialRepository interface {
	ForScanWithID(int) ([]Credential, error)
}

type credentialRepository struct {
	db *gorm.DB
}

func NewCredentialRepository(db *gorm.DB) CredentialRepository {
	return &credentialRepository{db: db}
}

func (r *credentialRepository) ForScanWithID(scanID int) ([]Credential, error) {
	rows, err := r.db.DB().Query(`
    SELECT c.owner,
           c.repository,
           c.sha,
           c.path,
           c.line_number,
           c.match_start,
           c.match_end,
           c.private
    FROM credentials c
    WHERE c.scan_id = ?`, scanID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var (
		credentials []Credential

		owner      string
		repository string
		sha        string
		path       string
		lineNumber int
		matchStart int
		matchEnd   int
		private    bool
	)

	for rows.Next() {
		scanErr := rows.Scan(
			&owner,
			&repository,
			&sha,
			&path,
			&lineNumber,
			&matchStart,
			&matchEnd,
			&private,
		)
		if scanErr != nil {
			return nil, err
		}

		credentials = append(credentials, NewCredential(
			owner,
			repository,
			sha,
			path,
			lineNumber,
			matchStart,
			matchEnd,
			private,
		))
	}

	return credentials, nil
}
<commit_msg>Return correct error<commit_after>package db

import "github.com/jinzhu/gorm"

//go:generate counterfeiter . CredentialRepository

type CredentialRepository interface {
	ForScanWithID(int) ([]Credential, error)
}

type credentialRepository struct {
	db *gorm.DB
}

func NewCredentialRepository(db *gorm.DB) CredentialRepository {
	return &credentialRepository{db: db}
}

func (r *credentialRepository) ForScanWithID(scanID int) ([]Credential, error) {
	rows, err := r.db.DB().Query(`
    SELECT c.owner,
           c.repository,
           c.sha,
           c.path,
           c.line_number,
           c.match_start,
           c.match_end,
           c.private
    FROM credentials c
    WHERE c.scan_id = ?`, scanID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var (
		credentials []Credential

		owner      string
		repository string
		sha        string
		path       string
		lineNumber int
		matchStart int
		matchEnd   int
		private    bool
	)

	for rows.Next() {
		scanErr := rows.Scan(
			&owner,
			&repository,
			&sha,
			&path,
			&lineNumber,
			&matchStart,
			&matchEnd,
			&private,
		)
		if scanErr != nil {
			return nil, scanErr
		}

		credentials = append(credentials, NewCredential(
			owner,
			repository,
			sha,
			path,
			lineNumber,
			matchStart,
			matchEnd,
			private,
		))
	}

	return credentials, nil
}
<|endoftext|>


<commit_before>package sniff

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const knownSHA = "a144a5b6a69d5911853e09884954ae360f56a184f7e9c2f6c007db730716f5a9"

func shouldScan(fi os.FileInfo) bool {
	if fi.IsDir() {
		return false
	}

	if strings.HasSuffix(fi.Name(), "_test.go") {
		return false
	}

	return true
}

var _ = Describe("Updating the RulesVersion", func() {
	It("checks if the RulesVersion needs to be updated", func() {
		pathsToScan := []string{}

		sniffFiles, err := ioutil.ReadDir(".")
		Expect(err).NotTo(HaveOccurred())

		for _, file := range sniffFiles {
			if !shouldScan(file) {
				continue
			}

			if file.Name() == "version.go" {
				continue
			}

			pathsToScan = append(pathsToScan, "./"+file.Name())
		}

		matcherFiles, err := ioutil.ReadDir("./matchers")
		Expect(err).NotTo(HaveOccurred())

		for _, file := range matcherFiles {
			if !shouldScan(file) {
				continue
			}

			pathsToScan = append(pathsToScan, "./matchers/"+file.Name())
		}

		hasher := sha256.New()

		for _, path := range pathsToScan {
			fh, err := os.Open(path)
			Expect(err).NotTo(HaveOccurred())

			_, err = io.Copy(hasher, fh)
			Expect(err).NotTo(HaveOccurred())

			fh.Close()
		}

		currentSHA := hex.EncodeToString(hasher.Sum(nil))

		helpfulMessage := fmt.Sprintf(`Uh oh! It looks the the scanning logic may have changed!

It's important that the sniff.RulesVersion is changed if a change has been made
to the scanning logic. However, if none of the scanning logic/heuristics has
been modified and you pinky swear it, then you just need to update the expected
SHA to: %s

If the scanning logic has changed then you need to update the SHA as above but
also increment the RulesVersion in sniff/version.go.
		`, currentSHA)

		Expect(knownSHA).To(Equal(currentSHA), helpfulMessage)
	})
})
<commit_msg>Update version_test<commit_after>package sniff

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const expectedSHA = "a144a5b6a69d5911853e09884954ae360f56a184f7e9c2f6c007db730716f5a9"

func shouldNotScan(fi os.FileInfo) bool {
	return fi.IsDir() || strings.HasSuffix(fi.Name(), "_test.go")
}

var _ = Describe("RulesVersion", func() {
	It("checks if the RulesVersion needs to be updated", func() {
		pathsToScan := []string{}

		sniffFiles, err := ioutil.ReadDir(".")
		Expect(err).NotTo(HaveOccurred())

		for _, file := range sniffFiles {
			if shouldNotScan(file) {
				continue
			}

			if file.Name() == "version.go" {
				continue
			}

			pathsToScan = append(pathsToScan, file.Name())
		}

		matcherFiles, err := ioutil.ReadDir("./matchers")
		Expect(err).NotTo(HaveOccurred())

		for _, file := range matcherFiles {
			if shouldNotScan(file) {
				continue
			}

			pathsToScan = append(pathsToScan, filepath.Join("matchers", file.Name()))
		}

		hasher := sha256.New()

		for _, path := range pathsToScan {
			fh, err := os.Open(path)
			Expect(err).NotTo(HaveOccurred())

			_, err = io.Copy(hasher, fh)
			Expect(err).NotTo(HaveOccurred())

			fh.Close()
		}

		currentSHA := hex.EncodeToString(hasher.Sum(nil))

		helpfulMessage := fmt.Sprintf(`Uh oh! It looks the the scanning logic may have changed!

It's important that the sniff.RulesVersion is changed if a change has been made
to the scanning logic. However, if none of the scanning logic/heuristics has
been modified and you pinky swear it, then you just need to update the expected
SHA to: %s

If the scanning logic has changed then you need to update the SHA as above but
also increment the RulesVersion in sniff/version.go.
		`, currentSHA)

		Expect(expectedSHA).To(Equal(currentSHA), helpfulMessage)
	})
})
<|endoftext|>


<commit_before>package worker_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"cred-alert/queue/queuefakes"
	"cred-alert/worker"
)

var _ = Describe("Worker", func() {
	var (
		logger  *lagertest.TestLogger
		foreman *queuefakes.FakeForeman
		queue   *queuefakes.FakeQueue

		process ifrit.Process
	)

	BeforeEach(func() {
		logger = lagertest.NewTestLogger("worker")
		foreman = &queuefakes.FakeForeman{}
		queue = &queuefakes.FakeQueue{}

		job := &queuefakes.FakeJob{}
		job.RunReturns(nil)

		task := &queuefakes.FakeAckTask{}
		queue.DequeueReturns(task, nil)

		foreman.BuildJobReturns(job, nil)
	})

	JustBeforeEach(func() {
		runner := worker.New(logger, foreman, queue)
		process = ginkgomon.Invoke(runner)
	})

	AfterEach(func() {
		ginkgomon.Interrupt(process)
	})

	It("works", func() {
		Expect(true).To(BeTrue())
	})
})
<commit_msg>[#123984511] Add tests for worker<commit_after>package worker_test

import (
	"errors"
	"os"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"cred-alert/queue"
	"cred-alert/queue/queuefakes"
	"cred-alert/worker"
)

var _ = Describe("Worker", func() {
	var (
		logger    *lagertest.TestLogger
		foreman   *queuefakes.FakeForeman
		fakeQueue *queuefakes.FakeQueue

		process ifrit.Process
		job     *queuefakes.FakeJob
		task    *queuefakes.FakeAckTask
		runner  ifrit.Runner
	)

	BeforeEach(func() {
		logger = lagertest.NewTestLogger("worker")
		foreman = &queuefakes.FakeForeman{}
		job = &queuefakes.FakeJob{}
		job.RunReturns(nil)
		task = &queuefakes.FakeAckTask{}
		foreman.BuildJobReturns(job, nil)
		fakeQueue = &queuefakes.FakeQueue{}
	})

	JustBeforeEach(func() {
		runner = worker.New(logger, foreman, fakeQueue)
		process = ginkgomon.Invoke(runner)
	})

	AfterEach(func() {
		ginkgomon.Interrupt(process)
	})

	Context("Everything's working fine", func() {
		BeforeEach(func() {
			fakeQueue.DequeueStub = func() (queue.AckTask, error) {
				// comparing with 1 becase call count was already incremented by this point
				for fakeQueue.DequeueCallCount() > 1 {
				}
				return task, nil
			}
		})

		It("Runs the job", func() {
			Eventually(job.RunCallCount).Should(Equal(1))
		})

		It("Acks the task", func() {
			Eventually(task.AckCallCount).Should(Equal(1))
		})

	})

	Context("Dequeue returns an error", func() {
		BeforeEach(func() {
			fakeQueue.DequeueStub = func() (queue.AckTask, error) {
				// comparing with 1 becase call count was already incremented by this point
				for fakeQueue.DequeueCallCount() > 1 {
				}
				return nil, errors.New("error dequeuing")
			}
		})

		It("logs an error", func() {
			Eventually(logger.LogMessages).Should(HaveLen(1))
			Expect(logger.LogMessages()[0]).To(ContainSubstring("got-error"))
		})

	})

	Context("Returns of shutdown", func() {
		BeforeEach(func() {
			fakeQueue.DequeueStub = func() (queue.AckTask, error) {
				for {
				}
			}
		})

		It("Shuts down on receiving signal", func() {
			signal := os.Interrupt
			signals := make(chan os.Signal)
			go func() {
				signals <- signal
			}()

			ready := make(chan<- struct{})
			Expect(runner.Run(signals, ready)).To(BeNil())
		})
	})
})
<|endoftext|>