<commit_before>package main

import (
	"github.com/gobs/args"
	"github.com/gobs/cmd"
	"github.com/gobs/httpclient"
	"github.com/gobs/simplejson"

	"fmt"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

var (
	completion_words = []string{}
)

func CompletionFunction(text, line string) (matches []string) {
	// for the "ls" command we let readline show real file names
	if strings.HasPrefix(line, "ls ") {
		return
	}

	// for all other commands, we pick from our list of completion words
	for _, w := range completion_words {
		if strings.HasPrefix(w, text) {
			matches = append(matches, w)
		}
	}

	return
}

func request(client *httpclient.HttpClient, method, params string) *httpclient.HttpResponse {
	options := []httpclient.RequestOption{client.Method(method)}
	args := args.ParseArgs(params)

	if len(args.Arguments) > 0 {
		options = append(options, client.Path(args.Arguments[0]))
	}

	if len(args.Arguments) > 1 {
		data := strings.Join(args.Arguments[1:], " ")
		options = append(options, client.Body(strings.NewReader(data)))
	}

	res, err := client.SendRequest(options...)
	if err == nil {
		err = res.ResponseError()
	}
	if err != nil {
		fmt.Println("ERROR:", err)
	} else {
		body := res.Content()
		fmt.Println(string(body))
	}

	return res
}

func main() {
	var interrupted bool
	var client = httpclient.NewHttpClient("")

	client.UserAgent = "httpclient/0.1"

	commander := &cmd.Cmd{
		HistoryFile: ".httpclient_history",
		Complete:    CompletionFunction,
		EnableShell: true,
		Interrupt:   func(sig os.Signal) bool { interrupted = true; return false },
	}

	commander.Init()

	commander.Vars = map[string]string{}

	commander.Add(cmd.Command{
		"base",
		`base [url]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := url.Parse(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.BaseURL = val
				commander.Prompt = fmt.Sprintf("%v> ", client.BaseURL)
			}

			return
		},
		nil})

	commander.Add(cmd.Command{
		"insecure",
		`insecure [true|false]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := strconv.ParseBool(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.AllowInsecure(val)
			}

			return
		},
		nil})

	commander.Add(cmd.Command{
		"timeout",
		`timeout [duration]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := time.ParseDuration(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.SetTimeout(val)
			}

			return
		},
		nil})

	commander.Add(cmd.Command{
		"verbose",
		`verbose [true|false]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := strconv.ParseBool(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.Verbose = val
			}

			fmt.Println("Verbose", client.Verbose)
			return
		},
		nil})

	commander.Add(cmd.Command{
		"agent",
		`agent user-agent-string`,
		func(line string) (stop bool) {
			if line != "" {
				client.UserAgent = line
			}

			fmt.Println("User-Agent:", client.UserAgent)
			return
		},
		nil})

	commander.Add(cmd.Command{
		"header",
		`header [name [value]]`,
		func(line string) (stop bool) {
			if line == "" {
				if len(client.Headers) == 0 {
					fmt.Println("No headers")
				} else {
					fmt.Println("Headers:")
					for k, v := range client.Headers {
						fmt.Printf("  %v: %v\n", k, v)
					}
				}

				return
			}

			parts := strings.SplitN(line, " ", 2)
			if len(parts) == 2 {
				client.Headers[parts[0]] = parts[1]
			}

			fmt.Printf("%v: %v\n", parts[0], client.Headers[parts[0]])
			return
		},
		nil})

	commander.Add(cmd.Command{"get",
		`
                get [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "get", line)
			return
		},
		nil})

	commander.Add(cmd.Command{"head",
		`
                head [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			res := request(client, "head", line)
			if res != nil {
				fmt.Println(simplejson.MustDumpString(res.Header, simplejson.Indent("  ")))
			}
			return
		},
		nil})

	commander.Add(cmd.Command{"post",
		`
                post [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "post", line)
			return
		},
		nil})

	commander.Add(cmd.Command{"put",
		`
                put [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "put", line)
			return
		},
		nil})

	commander.Add(cmd.Command{"delete",
		`
                delete [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "delete", line)
			return
		},
		nil})

	commander.Add(cmd.Command{
		"exit",
		`exit script`,
		func(line string) (stop bool) {
			fmt.Println("goodbye!")
			return true
		},
		nil})

	switch len(os.Args) {
	case 1: // program name only
		break

	case 2: // one arg - expect URL
		commander.OneCmd("base " + os.Args[1])

	default:
		fmt.Println("usage:", os.Args[0], "[base-url]")
		return
	}

	commander.CmdLoop()
}
<commit_msg>Normalize header names (to Mixed-Case) and format JSON responses.<commit_after>package main

import (
	"github.com/gobs/args"
	"github.com/gobs/cmd"
	"github.com/gobs/httpclient"
	"github.com/gobs/simplejson"

	"fmt"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

var (
	completion_words = []string{}
)

func CompletionFunction(text, line string) (matches []string) {
	// for the "ls" command we let readline show real file names
	if strings.HasPrefix(line, "ls ") {
		return
	}

	// for all other commands, we pick from our list of completion words
	for _, w := range completion_words {
		if strings.HasPrefix(w, text) {
			matches = append(matches, w)
		}
	}

	return
}

func request(client *httpclient.HttpClient, method, params string) *httpclient.HttpResponse {
	options := []httpclient.RequestOption{client.Method(method)}
	args := args.ParseArgs(params)

	if len(args.Arguments) > 0 {
		options = append(options, client.Path(args.Arguments[0]))
	}

	if len(args.Arguments) > 1 {
		data := strings.Join(args.Arguments[1:], " ")
		options = append(options, client.Body(strings.NewReader(data)))
	}

	res, err := client.SendRequest(options...)
	if err == nil {
		err = res.ResponseError()
	}
	if err != nil {
		fmt.Println("ERROR:", err)
	}

	body := res.Content()
	if len(body) > 0 {
		if strings.Contains(res.Header.Get("Content-Type"), "json") {
			jbody, err := simplejson.LoadBytes(body)
			if err != nil {
				fmt.Println(err)
			} else {
				fmt.Println(simplejson.MustDumpString(jbody.Data(), simplejson.Indent("  ")))
			}
		} else {
			fmt.Println(string(body))
		}
	}

	return res
}

func headerName(s string) string {
	s = strings.ToLower(s)
	parts := strings.Split(s, "-")
	for i, p := range parts {
		if len(p) > 0 {
			parts[i] = strings.ToUpper(p[0:1]) + p[1:]
		}
	}
	return strings.Join(parts, "-")
}

func unquote(s string) string {
	if res, err := strconv.Unquote(s); err == nil {
		return res
	}

	return s
}

func main() {
	var interrupted bool
	var client = httpclient.NewHttpClient("")

	client.UserAgent = "httpclient/0.1"

	commander := &cmd.Cmd{
		HistoryFile: ".httpclient_history",
		Complete:    CompletionFunction,
		EnableShell: true,
		Interrupt:   func(sig os.Signal) bool { interrupted = true; return false },
	}

	commander.Init()

	commander.Vars = map[string]string{}

	commander.Add(cmd.Command{
		"base",
		`base [url]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := url.Parse(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.BaseURL = val
				commander.Prompt = fmt.Sprintf("%v> ", client.BaseURL)
			}

			return
		},
		nil})

	commander.Add(cmd.Command{
		"insecure",
		`insecure [true|false]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := strconv.ParseBool(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.AllowInsecure(val)
			}

			return
		},
		nil})

	commander.Add(cmd.Command{
		"timeout",
		`timeout [duration]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := time.ParseDuration(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.SetTimeout(val)
			}

			return
		},
		nil})

	commander.Add(cmd.Command{
		"verbose",
		`verbose [true|false]`,
		func(line string) (stop bool) {
			if line != "" {
				val, err := strconv.ParseBool(line)
				if err != nil {
					fmt.Println(err)
					return
				}

				client.Verbose = val
			}

			fmt.Println("Verbose", client.Verbose)
			return
		},
		nil})

	commander.Add(cmd.Command{
		"agent",
		`agent user-agent-string`,
		func(line string) (stop bool) {
			if line != "" {
				client.UserAgent = line
			}

			fmt.Println("User-Agent:", client.UserAgent)
			return
		},
		nil})

	commander.Add(cmd.Command{
		"header",
		`header [name [value]]`,
		func(line string) (stop bool) {
			if line == "" {
				if len(client.Headers) == 0 {
					fmt.Println("No headers")
				} else {
					fmt.Println("Headers:")
					for k, v := range client.Headers {
						fmt.Printf("  %v: %v\n", k, v)
					}
				}

				return
			}

			parts := args.GetArgsN(line, 2)
			name := headerName(parts[0])

			if len(parts) == 2 {
				client.Headers[name] = unquote(parts[1])
			}

			fmt.Printf("%v: %v\n", name, client.Headers[name])
			return
		},
		nil})

	commander.Add(cmd.Command{"get",
		`
                get [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "get", line)
			return
		},
		nil})

	commander.Add(cmd.Command{"head",
		`
                head [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			res := request(client, "head", line)
			if res != nil {
				fmt.Println(simplejson.MustDumpString(res.Header, simplejson.Indent("  ")))
			}
			return
		},
		nil})

	commander.Add(cmd.Command{"post",
		`
                post [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "post", line)
			return
		},
		nil})

	commander.Add(cmd.Command{"put",
		`
                put [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "put", line)
			return
		},
		nil})

	commander.Add(cmd.Command{"delete",
		`
                delete [url-path] [short-data]
                `,
		func(line string) (stop bool) {
			request(client, "delete", line)
			return
		},
		nil})

	commander.Add(cmd.Command{
		"exit",
		`exit script`,
		func(line string) (stop bool) {
			fmt.Println("goodbye!")
			return true
		},
		nil})

	switch len(os.Args) {
	case 1: // program name only
		break

	case 2: // one arg - expect URL
		commander.OneCmd("base " + os.Args[1])

	default:
		fmt.Println("usage:", os.Args[0], "[base-url]")
		return
	}

	commander.CmdLoop()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"text/tabwriter"
	"time"

	"github.com/influxdb/influxdb/cmd/influx_tsm/b1"
	"github.com/influxdb/influxdb/cmd/influx_tsm/bz1"
	"github.com/influxdb/influxdb/cmd/influx_tsm/tsdb"
)

type ShardReader interface {
	KeyIterator
	Open() error
	Close() error
}

const (
	backupExt = "bak"
	tsmExt    = "tsm"
)

var description = fmt.Sprintf(`
Convert a database from b1 or bz1 format to tsm1 format.

This tool will backup any directory before conversion. It is up to the
end-user to delete the backup on the disk, once the end-user is happy
with the converted data. Backups are named by suffixing the database
name with '.%s'. The backups will be ignored by the system since they
are not registered with the cluster.

To restore a backup, delete the tsm1 version, rename the backup directory
restart the node.`, backupExt)

var dataPath string
var ds string
var tsmSz uint64
var parallel bool
var disBack bool

const maxTSMSz = 2 * 1000 * 1000 * 1000

func init() {
	flag.StringVar(&ds, "dbs", "", "Comma-delimited list of databases to convert. Default is to convert all databases.")
	flag.Uint64Var(&tsmSz, "sz", maxTSMSz, "Maximum size of individual TSM files.")
	flag.BoolVar(&parallel, "parallel", false, "Perform parallel conversion.")
	flag.BoolVar(&disBack, "nobackup", false, "Disable database backups. Not recommended.")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [options] <data-path> \n", os.Args[0])
		fmt.Fprintf(os.Stderr, "%s\n\n", description)
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\n")
	}
}

func main() {
	pg := NewParallelGroup(1)

	flag.Parse()
	if len(flag.Args()) < 1 {
		fmt.Fprintf(os.Stderr, "No data directory specified\n")
		os.Exit(1)
	}
	dataPath = flag.Args()[0]

	if tsmSz > maxTSMSz {
		fmt.Fprintf(os.Stderr, "Maximum TSM file size is %d\n", maxTSMSz)
		os.Exit(1)
	}

	// Check if specific directories were requested.
	reqDs := strings.Split(ds, ",")
	if len(reqDs) == 1 && reqDs[0] == "" {
		reqDs = nil
	}

	// Determine the list of databases
	dbs, err := ioutil.ReadDir(dataPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to access data directory at %s: %s\n", dataPath, err.Error())
		os.Exit(1)
	}
	fmt.Println() // Cleanly separate output from start of program.

	// Dump summary of what is about to happen.
	fmt.Println("b1 and bz1 shard conversion.")
	fmt.Println("-----------------------------------")
	fmt.Println("Data directory is:       ", dataPath)
	fmt.Println("Databases specified:     ", allDBs(reqDs))
	fmt.Println("Database backups enabled:", yesno(!disBack))
	fmt.Println("Parallel mode enabled:   ", yesno(parallel))
	fmt.Println()

	// Get the list of shards for conversion.
	var shards []*tsdb.ShardInfo
	for _, db := range dbs {
		if strings.HasSuffix(db.Name(), backupExt) {
			fmt.Printf("Skipping %s as it looks like a backup.\n", db.Name())
			continue
		}

		d := tsdb.NewDatabase(filepath.Join(dataPath, db.Name()))
		shs, err := d.Shards()
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to access shards for database %s: %s\n", d.Name(), err.Error())
			os.Exit(1)
		}
		shards = append(shards, shs...)
	}
	sort.Sort(tsdb.ShardInfos(shards))
	usl := len(shards)
	shards = tsdb.ShardInfos(shards).FilterFormat(tsdb.TSM1).ExclusiveDatabases(reqDs)
	sl := len(shards)

	// Anything to convert?
	fmt.Printf("\n%d shard(s) detected, %d non-TSM shards detected.\n", usl, sl)
	if len(shards) == 0 {
		fmt.Printf("Nothing to do.\n")
		os.Exit(0)
	}

	// Display list of convertible shards.
	fmt.Println()
	w := new(tabwriter.Writer)
	w.Init(os.Stdout, 0, 8, 1, '\t', 0)
	fmt.Fprintln(w, "Database\tRetention\tPath\tEngine\tSize")
	for _, si := range shards {
		fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\n", si.Database, si.RetentionPolicy, si.FullPath(dataPath), si.FormatAsString(), si.Size)
	}
	w.Flush()

	// Get confirmation from user.
	fmt.Printf("\nThese shards will be converted. Proceed? y/N: ")
	liner := bufio.NewReader(os.Stdin)
	yn, err := liner.ReadString('\n')
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to read response: %s", err.Error())
		os.Exit(1)
	}
	yn = strings.TrimRight(strings.ToLower(yn), "\n")
	if yn != "y" {
		fmt.Println("Conversion aborted.")
		os.Exit(1)
	}
	fmt.Println("Conversion starting....")

	// Backup each directory.
	conversionStart := time.Now()
	if !disBack {
		databases := tsdb.ShardInfos(shards).Databases()
		fmt.Printf("Backing up %d databases...\n", len(databases))
		if parallel {
			pg = NewParallelGroup(len(databases))
		}
		for _, db := range databases {
			pg.Request()
			go func(db string) {
				defer pg.Release()

				start := time.Now()
				err := backupDatabase(filepath.Join(dataPath, db))
				if err != nil {
					fmt.Fprintf(os.Stderr, "Backup of database %s failed: %s\n", db, err.Error())
					os.Exit(1)
				}
				fmt.Printf("Database %s backed up (%v)\n", db, time.Now().Sub(start))
			}(db)
		}
		pg.Wait()
	} else {
		fmt.Println("Database backup disabled.")
	}

	// Convert each shard.
	if parallel {
		pg = NewParallelGroup(len(shards))
	}
	for _, si := range shards {
		pg.Request()
		go func(si *tsdb.ShardInfo) {
			defer pg.Release()

			start := time.Now()
			if err := convertShard(si); err != nil {
				fmt.Fprintf(os.Stderr, "Failed to convert %s: %s\n", si.FullPath(dataPath), err.Error())
				os.Exit(1)
			}
			fmt.Printf("Conversion of %s successful (%s)\n", si.FullPath(dataPath), time.Now().Sub(start))
		}(si)
	}
	pg.Wait()

	// Dump stats.
	preSize := tsdb.ShardInfos(shards).Size()
	postSize := TsmBytesWritten
	fmt.Printf("\nSummary statistics\n========================================\n")
	fmt.Printf("Databases converted:                 %d\n", len(tsdb.ShardInfos(shards).Databases()))
	fmt.Printf("Shards converted:                    %d\n", len(shards))
	fmt.Printf("TSM files created:                   %d\n", TsmFilesCreated)
	fmt.Printf("Points read:                         %d\n", PointsRead)
	fmt.Printf("Points written:                      %d\n", PointsWritten)
	fmt.Printf("NaN filtered:                        %d\n", NanFiltered)
	fmt.Printf("Inf filtered:                        %d\n", InfFiltered)
	fmt.Printf("Points without fields filtered:      %d\n", b1.NoFieldsFiltered+bz1.NoFieldsFiltered)
	fmt.Printf("Disk usage pre-conversion (bytes):   %d\n", preSize)
	fmt.Printf("Disk usage post-conversion (bytes):  %d\n", postSize)
	fmt.Printf("Reduction factor:                    %d%%\n", (100*preSize-postSize)/preSize)
	fmt.Printf("Bytes per TSM point:                 %.2f\n", float64(postSize)/float64(PointsWritten))
	fmt.Printf("Total conversion time:               %v\n", time.Now().Sub(conversionStart))
	fmt.Println()
}

// backupDatabase backs up the database at src.
func backupDatabase(src string) error {
	dest := filepath.Join(src + "." + backupExt)
	if _, err := os.Stat(dest); !os.IsNotExist(err) {
		return fmt.Errorf("backup of %s already exists", src)
	}
	return copyDir(dest, src)
}

// copyDir copies the directory at src to dest. If dest does not exist it
// will be created. It is up to the caller to ensure the paths don't overlap.
func copyDir(dest, src string) error {
	copyFile := func(path string, info os.FileInfo, err error) error {
		// Strip the src from the path and replace with dest.
		toPath := strings.Replace(path, src, dest, 1)

		// Copy it.
		if info.IsDir() {
			if err := os.MkdirAll(toPath, info.Mode()); err != nil {
				return err
			}
		} else {
			err := func() error {
				in, err := os.Open(path)
				if err != nil {
					return err
				}
				defer in.Close()

				out, err := os.OpenFile(toPath, os.O_CREATE|os.O_WRONLY, info.Mode())
				if err != nil {
					return err
				}
				defer out.Close()

				_, err = io.Copy(out, in)
				return err
			}()
			if err != nil {
				return err
			}
		}
		return nil
	}

	return filepath.Walk(src, copyFile)
}

// convertShard converts the shard in-place.
func convertShard(si *tsdb.ShardInfo) error {
	src := si.FullPath(dataPath)
	dst := fmt.Sprintf("%s.%s", src, tsmExt)

	var reader ShardReader
	switch si.Format {
	case tsdb.BZ1:
		reader = bz1.NewReader(src)
	case tsdb.B1:
		reader = b1.NewReader(src)
	default:
		return fmt.Errorf("Unsupported shard format: %s", si.FormatAsString())
	}
	defer reader.Close()

	// Open the shard, and create a converter.
	if err := reader.Open(); err != nil {
		return fmt.Errorf("Failed to open %s for conversion: %s", src, err.Error())
	}
	converter := NewConverter(dst, uint32(tsmSz))

	// Perform the conversion.
	if err := converter.Process(reader); err != nil {
		return fmt.Errorf("Conversion of %s failed: %s", src, err.Error())
	}

	// Delete source shard, and rename new tsm1 shard.
	if err := reader.Close(); err != nil {
		return fmt.Errorf("Conversion of %s failed due to close: %s", src, err.Error())
	}

	if err := os.RemoveAll(si.FullPath(dataPath)); err != nil {
		return fmt.Errorf("Deletion of %s failed: %s", src, err.Error())
	}
	if err := os.Rename(dst, src); err != nil {
		return fmt.Errorf("Rename of %s to %s failed: %s", dst, src, err.Error())
	}

	return nil
}

// ParallelGroup allows the maximum parrallelism of a set of operations to be controlled.
type ParallelGroup struct {
	c  chan struct{}
	wg sync.WaitGroup
}

// NewParallelGroup returns a group which allows n operations to run in parallel. A value of 0
// means no operations will ever run.
func NewParallelGroup(n int) *ParallelGroup {
	return &ParallelGroup{
		c: make(chan struct{}, n),
	}
}

// Request requests permission to start an operation. It will block unless and until
// the parallel requirements would not be violated.
func (p *ParallelGroup) Request() {
	p.wg.Add(1)
	p.c <- struct{}{}
}

// Release informs the group that a previoulsy requested operation has completed.
func (p *ParallelGroup) Release() {
	<-p.c
	p.wg.Done()
}

// Wait blocks until the ParallelGroup has no unreleased operations.
func (p *ParallelGroup) Wait() {
	p.wg.Wait()
}

// yesno returns "yes" for true, "no" for false.
func yesno(b bool) string {
	if b {
		return "yes"
	}
	return "no"
}

// allDBs returns "all" if all databases are requested for conversion.
func allDBs(dbs []string) string {
	if dbs == nil {
		return "all"
	}
	return fmt.Sprintf("%v", dbs)
}
<commit_msg>Limit parallelism for 'influx_tsm -parallel'<commit_after>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"text/tabwriter"
	"time"

	"github.com/influxdb/influxdb/cmd/influx_tsm/b1"
	"github.com/influxdb/influxdb/cmd/influx_tsm/bz1"
	"github.com/influxdb/influxdb/cmd/influx_tsm/tsdb"
)

type ShardReader interface {
	KeyIterator
	Open() error
	Close() error
}

const (
	backupExt = "bak"
	tsmExt    = "tsm"
)

var description = fmt.Sprintf(`
Convert a database from b1 or bz1 format to tsm1 format.

This tool will backup any directory before conversion. It is up to the
end-user to delete the backup on the disk, once the end-user is happy
with the converted data. Backups are named by suffixing the database
name with '.%v'. The backups will be ignored by the system since they
are not registered with the cluster.

To restore a backup, delete the tsm1 version, rename the backup directory
restart the node.`, backupExt)

type options struct {
	DataPath       string
	DBs            []string
	TSMSize        uint64
	Parallel       bool
	SkipBackup     bool
	UpdateInterval time.Duration
	Quiet          bool
}

func (o *options) Parse() error {
	fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)

	var dbs string

	fs.StringVar(&dbs, "dbs", "", "Comma-delimited list of databases to convert. Default is to convert all databases.")
	fs.Uint64Var(&opts.TSMSize, "sz", maxTSMSz, "Maximum size of individual TSM files.")
	fs.BoolVar(&opts.Parallel, "parallel", false, "Perform parallel conversion. (up to GOMAXPROCS shards at once)")
	fs.BoolVar(&opts.SkipBackup, "nobackup", false, "Disable database backups. Not recommended.")
	fs.BoolVar(&opts.Quiet, "quiet", false, "Suppresses the regular status updates.")
	fs.DurationVar(&opts.UpdateInterval, "interval", 5*time.Second, "How often status updates are printed.")
	fs.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %v [options] <data-path> \n", os.Args[0])
		fmt.Fprintf(os.Stderr, "%v\n\n", description)
		fs.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\n")
	}

	if err := fs.Parse(os.Args[1:]); err != nil {
		return err
	}

	if len(fs.Args()) < 1 {
		return errors.New("no data directory specified")
	}
	o.DataPath = fs.Args()[0]

	if o.TSMSize > maxTSMSz {
		return fmt.Errorf("bad TSM file size, maximum TSM file size is %d", maxTSMSz)
	}

	// Check if specific databases were requested.
	o.DBs = strings.Split(dbs, ",")
	if len(o.DBs) == 1 && o.DBs[0] == "" {
		o.DBs = nil
	}

	return nil
}

var opts options

const maxTSMSz = 2 * 1000 * 1000 * 1000

func init() {
	log.SetOutput(os.Stderr)
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
}

func main() {
	if err := opts.Parse(); err != nil {
		log.Fatal(err)
	}

	// Determine the list of databases
	dbs, err := ioutil.ReadDir(opts.DataPath)
	if err != nil {
		log.Fatalf("failed to access data directory at %v: %v\n", opts.DataPath, err)
	}
	fmt.Println() // Cleanly separate output from start of program.

	if opts.Parallel {
		if !isEnvSet("GOMAXPROCS") {
			// Only modify GOMAXPROCS if it wasn't set in the environment
			// This means 'GOMAXPROCS=1 influx_tsm -parallel' will not actually
			// run in parallel
			runtime.GOMAXPROCS(runtime.NumCPU())
		}
	}

	// Dump summary of what is about to happen.
	fmt.Println("b1 and bz1 shard conversion.")
	fmt.Println("-----------------------------------")
	fmt.Println("Data directory is:       ", opts.DataPath)
	fmt.Println("Databases specified:     ", allDBs(opts.DBs))
	fmt.Println("Database backups enabled:", yesno(!opts.SkipBackup))
	fmt.Println("Parallel mode enabled:   ", yesno(opts.Parallel), runtime.GOMAXPROCS(0))
	fmt.Println()

	shards := collectShards(dbs)

	// Anything to convert?
	fmt.Printf("\nFound %d shards that will be converted.\n", len(shards))
	if len(shards) == 0 {
		fmt.Println("Nothing to do.")
		os.Exit(0)
	}

	// Display list of convertible shards.
	fmt.Println()
	w := new(tabwriter.Writer)
	w.Init(os.Stdout, 0, 8, 1, '\t', 0)
	fmt.Fprintln(w, "Database\tRetention\tPath\tEngine\tSize")
	for _, si := range shards {
		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%d\n", si.Database, si.RetentionPolicy, si.FullPath(opts.DataPath), si.FormatAsString(), si.Size)
	}
	w.Flush()

	// Get confirmation from user.
	fmt.Printf("\nThese shards will be converted. Proceed? y/N: ")
	liner := bufio.NewReader(os.Stdin)
	yn, err := liner.ReadString('\n')
	if err != nil {
		log.Printf("failed to read response: %v", err)
		os.Exit(1)
	}
	yn = strings.TrimRight(strings.ToLower(yn), "\n")
	if yn != "y" {
		fmt.Println("Conversion aborted.")
		os.Exit(1)
	}
	fmt.Println("Conversion starting....")

	// GOMAXPROCS(0) just queires the current value
	pg := NewParallelGroup(runtime.GOMAXPROCS(0))
	var wg sync.WaitGroup

	conversionStart := time.Now()

	// Backup each directory.
	if !opts.SkipBackup {
		databases := shards.Databases()
		fmt.Printf("Backing up %d databases...\n", len(databases))
		wg.Add(len(databases))
		for i := range databases {
			db := databases[i]
			go pg.Do(func() {
				defer wg.Done()

				start := time.Now()
				log.Printf("Backup of databse '%v' started", db)
				err := backupDatabase(filepath.Join(opts.DataPath, db))
				if err != nil {
					log.Printf("Backup of database %v failed: %v\n", db, err)
					os.Exit(1)
				}
				log.Printf("Database %v backed up (%v)\n", db, time.Now().Sub(start))
			})
		}
		wg.Wait()
	} else {
		fmt.Println("Database backup disabled.")
	}

	wg.Add(len(shards))
	for i := range shards {
		si := shards[i]
		go pg.Do(func() {
			defer wg.Done()

			start := time.Now()
			log.Printf("Starting conversion of shard: %v", si.FullPath(opts.DataPath))
			if err := convertShard(si); err != nil {
				log.Printf("Failed to convert %v: %v\n", si.FullPath(opts.DataPath), err)
				os.Exit(1)
			}
			log.Printf("Conversion of %v successful (%v)\n", si.FullPath(opts.DataPath), time.Since(start))
		})
	}
	wg.Wait()

	// Dump stats.
	preSize := shards.Size()
	postSize := TsmBytesWritten
	totalTime := time.Since(conversionStart)

	fmt.Printf("\nSummary statistics\n========================================\n")
	fmt.Printf("Databases converted:                 %d\n", len(shards.Databases()))
	fmt.Printf("Shards converted:                    %d\n", len(shards))
	fmt.Printf("TSM files created:                   %d\n", TsmFilesCreated)
	fmt.Printf("Points read:                         %d\n", PointsRead)
	fmt.Printf("Points written:                      %d\n", PointsWritten)
	fmt.Printf("NaN filtered:                        %d\n", NanFiltered)
	fmt.Printf("Inf filtered:                        %d\n", InfFiltered)
	fmt.Printf("Points without fields filtered:      %d\n", b1.NoFieldsFiltered+bz1.NoFieldsFiltered)
	fmt.Printf("Disk usage pre-conversion (bytes):   %d\n", preSize)
	fmt.Printf("Disk usage post-conversion (bytes):  %d\n", postSize)
	fmt.Printf("Reduction factor:                    %d%%\n", 100*(preSize-postSize)/preSize)
	fmt.Printf("Bytes per TSM point:                 %.2f\n", float64(postSize)/float64(PointsWritten))
	fmt.Printf("Total conversion time:               %v\n", totalTime)
	fmt.Println()
}

func collectShards(dbs []os.FileInfo) tsdb.ShardInfos {
	// Get the list of shards for conversion.
	var shards tsdb.ShardInfos
	for _, db := range dbs {
		if strings.HasSuffix(db.Name(), backupExt) {
			log.Printf("Skipping %v as it looks like a backup.\n", db.Name())
			continue
		}

		d := tsdb.NewDatabase(filepath.Join(opts.DataPath, db.Name()))
		shs, err := d.Shards()
		if err != nil {
			log.Fatalf("Failed to access shards for database %v: %v\n", d.Name(), err)
		}
		shards = append(shards, shs...)
	}

	sort.Sort(shards)
	shards = shards.FilterFormat(tsdb.TSM1)
	if len(dbs) > 0 {
		shards = shards.ExclusiveDatabases(opts.DBs)
	}

	return shards
}

// backupDatabase backs up the database at src.
func backupDatabase(src string) error {
	dest := filepath.Join(src + "." + backupExt)
	return copyDir(dest, src)
}

// copyDir copies the directory at src to dest. If dest does not exist it
// will be created. It is up to the caller to ensure the paths don't overlap.
func copyDir(dest, src string) error {
	copyFile := func(path string, info os.FileInfo, err error) error {
		// Strip the src from the path and replace with dest.
		toPath := strings.Replace(path, src, dest, 1)

		// Copy it.
		if info.IsDir() {
			if err := os.MkdirAll(toPath, info.Mode()); err != nil {
				return err
			}
		} else {
			err := func() error {
				in, err := os.Open(path)
				if err != nil {
					return err
				}
				defer in.Close()

				out, err := os.OpenFile(toPath, os.O_CREATE|os.O_WRONLY, info.Mode())
				if err != nil {
					return err
				}
				defer out.Close()

				_, err = io.Copy(out, in)
				return err
			}()
			if err != nil {
				return err
			}
		}
		return nil
	}

	return filepath.Walk(src, copyFile)
}

// convertShard converts the shard in-place.
func convertShard(si *tsdb.ShardInfo) error {
	src := si.FullPath(opts.DataPath)
	dst := fmt.Sprintf("%v.%v", src, tsmExt)

	var reader ShardReader
	switch si.Format {
	case tsdb.BZ1:
		reader = bz1.NewReader(src)
	case tsdb.B1:
		reader = b1.NewReader(src)
	default:
		return fmt.Errorf("Unsupported shard format: %v", si.FormatAsString())
	}
	defer reader.Close()

	// Open the shard, and create a converter.
	if err := reader.Open(); err != nil {
		return fmt.Errorf("Failed to open %v for conversion: %v", src, err)
	}
	converter := NewConverter(dst, uint32(opts.TSMSize))

	// Perform the conversion.
	if err := converter.Process(reader); err != nil {
		return fmt.Errorf("Conversion of %v failed: %v", src, err)
	}

	// Delete source shard, and rename new tsm1 shard.
	if err := reader.Close(); err != nil {
		return fmt.Errorf("Conversion of %v failed due to close: %v", src, err)
	}

	if err := os.RemoveAll(si.FullPath(opts.DataPath)); err != nil {
		return fmt.Errorf("Deletion of %v failed: %v", src, err)
	}
	if err := os.Rename(dst, src); err != nil {
		return fmt.Errorf("Rename of %v to %v failed: %v", dst, src, err)
	}

	return nil
}

// ParallelGroup allows the maximum parrallelism of a set of operations to be controlled.
type ParallelGroup chan struct{}

// NewParallelGroup returns a group which allows n operations to run in parallel. A value of 0
// means no operations will ever run.
func NewParallelGroup(n int) ParallelGroup {
	return make(chan struct{}, n)
}

func (p ParallelGroup) Do(f func()) {
	p <- struct{}{} // acquire working slot
	defer func() { <-p }()

	f()
}

// yesno returns "yes" for true, "no" for false.
func yesno(b bool) string {
	if b {
		return "yes"
	}
	return "no"
}

// allDBs returns "all" if all databases are requested for conversion.
func allDBs(dbs []string) string {
	if dbs == nil {
		return "all"
	}
	return fmt.Sprintf("%v", dbs)
}

// isEnvSet checks to see if a variable was set in the environment
func isEnvSet(name string) bool {
	for _, s := range os.Environ() {
		if strings.SplitN(s, "=", 2)[0] == name {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/zyedidia/clipboard"
	"github.com/zyedidia/tcell"
)

// TermMessage sends a message to the user in the terminal. This usually occurs before
// micro has been fully initialized -- ie if there is an error in the syntax highlighting
// regular expressions
// The function must be called when the screen is not initialized
// This will write the message, and wait for the user
// to press and key to continue
func TermMessage(msg ...interface{}) {
	screenWasNil := screen == nil
	if !screenWasNil {
		screen.Fini()
	}

	fmt.Println(msg...)
	fmt.Print("\nPress enter to continue")

	reader := bufio.NewReader(os.Stdin)
	reader.ReadString('\n')

	if !screenWasNil {
		InitScreen()
	}
}

// TermError sends an error to the user in the terminal. Like TermMessage except formatted
// as an error
func TermError(filename string, lineNum int, err string) {
	TermMessage(filename + ", " + strconv.Itoa(lineNum) + ": " + err)
}

// Messenger is an object that makes it easy to send messages to the user
// and get input from the user
type Messenger struct {
	// Are we currently prompting the user?
	hasPrompt bool
	// Is there a message to print
	hasMessage bool

	// Message to print
	message string
	// The user's response to a prompt
	response string
	// style to use when drawing the message
	style tcell.Style

	// We have to keep track of the cursor for prompting
	cursorx int

	// This map stores the history for all the different kinds of uses Prompt has
	// It's a map of history type -> history array
	history    map[string][]string
	historyNum int

	// Is the current message a message from the gutter
	gutterMessage bool
}

// Message sends a message to the user
func (m *Messenger) Message(msg ...interface{}) {
	buf := new(bytes.Buffer)
	fmt.Fprint(buf, msg...)
	m.message = buf.String()
	m.style = defStyle

	if _, ok := colorscheme["message"]; ok {
		m.style = colorscheme["message"]
	}
	m.hasMessage = true
}

// Error sends an error message to the user
func (m *Messenger) Error(msg ...interface{}) {
	buf := new(bytes.Buffer)
	fmt.Fprint(buf, msg...)
	m.message = buf.String()
	m.style = defStyle.
		Foreground(tcell.ColorBlack).
		Background(tcell.ColorMaroon)

	if _, ok := colorscheme["error-message"]; ok {
		m.style = colorscheme["error-message"]
	}
	m.hasMessage = true
}

// YesNoPrompt asks the user a yes or no question (waits for y or n) and returns the result
func (m *Messenger) YesNoPrompt(prompt string) (bool, bool) {
	m.Message(prompt)

	_, h := screen.Size()
	for {
		m.Clear()
		m.Display()
		screen.ShowCursor(Count(m.message), h-1)
		screen.Show()
		event := <-events

		switch e := event.(type) {
		case *tcell.EventKey:
			switch e.Key() {
			case tcell.KeyRune:
				if e.Rune() == 'y' {
					return true, false
				} else if e.Rune() == 'n' {
					return false, false
				}
			case tcell.KeyCtrlC, tcell.KeyCtrlQ, tcell.KeyEscape:
				return false, true
			}
		}
	}
}

// LetterPrompt gives the user a prompt and waits for a one letter response
func (m *Messenger) LetterPrompt(prompt string, responses ...rune) (rune, bool) {
	m.Message(prompt)

	_, h := screen.Size()
	for {
		m.Clear()
		m.Display()
		screen.ShowCursor(Count(m.message), h-1)
		screen.Show()
		event := <-events

		switch e := event.(type) {
		case *tcell.EventKey:
			switch e.Key() {
			case tcell.KeyRune:
				for _, r := range responses {
					if e.Rune() == r {
						m.Reset()
						return r, false
					}
				}
			case tcell.KeyCtrlC, tcell.KeyCtrlQ, tcell.KeyEscape:
				return ' ', true
			}
		}
	}
}

type Completion int

const (
	NoCompletion Completion = iota
	FileCompletion
	CommandCompletion
	HelpCompletion
	OptionCompletion
)

// Prompt sends the user a message and waits for a response to be typed in
// This function blocks the main loop while waiting for input
func (m *Messenger) Prompt(prompt, historyType string, completionTypes ...Completion) (string, bool) {
	m.hasPrompt = true
	m.Message(prompt)
	if _, ok := m.history[historyType]; !ok {
		m.history[historyType] = []string{""}
	} else {
		m.history[historyType] = append(m.history[historyType], "")
	}
	m.historyNum = len(m.history[historyType]) - 1

	response, canceled := "", true

	RedrawAll()
	for m.hasPrompt {
		var suggestions []string
		m.Clear()

		event := <-events

		switch e := event.(type) {
		case *tcell.EventKey:
			switch e.Key() {
			case tcell.KeyCtrlQ, tcell.KeyCtrlC, tcell.KeyEscape:
				// Cancel
				m.hasPrompt = false
			case tcell.KeyEnter:
				// User is done entering their response
				m.hasPrompt = false
				response, canceled = m.response, false
				m.history[historyType][len(m.history[historyType])-1] = response
			case tcell.KeyTab:
				args := strings.Split(m.response, " ")
				currentArgNum := len(args) - 1
				currentArg := args[currentArgNum]
				var completionType Completion

				if completionTypes[0] == CommandCompletion && currentArgNum > 0 {
					if command, ok := commands[args[0]]; ok {
						completionTypes = append([]Completion{CommandCompletion}, command.completions...)
					}
				}

				if currentArgNum >= len(completionTypes) {
					completionType = completionTypes[len(completionTypes)-1]
				} else {
					completionType = completionTypes[currentArgNum]
				}

				var chosen string
				if completionType == FileCompletion {
					chosen, suggestions = FileComplete(currentArg)
				} else if completionType == CommandCompletion {
					chosen, suggestions = CommandComplete(currentArg)
				} else if completionType == HelpCompletion {
					chosen, suggestions = HelpComplete(currentArg)
				} else if completionType == OptionCompletion {
					chosen, suggestions = OptionComplete(currentArg)
				}

				if len(suggestions) > 1 {
					chosen = chosen + CommonSubstring(suggestions...)
				}

				if chosen != "" {
					if len(args) > 1 {
						chosen = " " + chosen
					}
					m.response = strings.Join(args[:len(args)-1], " ") + chosen
					m.cursorx = Count(m.response)
				}
			}
		}

		m.HandleEvent(event, m.history[historyType])

		messenger.Clear()
		for _, v := range tabs[curTab].views {
			v.Display()
		}
		DisplayTabs()
		messenger.Display()
		if len(suggestions) > 1 {
			m.DisplaySuggestions(suggestions)
		}
		screen.Show()
	}

	m.Reset()
	return response, canceled
}

// HandleEvent handles an event for the prompter
func (m *Messenger) HandleEvent(event tcell.Event, history []string) {
	switch e := event.(type) {
	case *tcell.EventKey:
		switch e.Key() {
		case tcell.KeyUp:
			if m.historyNum > 0 {
				m.historyNum--
				m.response = history[m.historyNum]
				m.cursorx = Count(m.response)
			}
		case tcell.KeyDown:
			if m.historyNum < len(history)-1 {
				m.historyNum++
				m.response = history[m.historyNum]
				m.cursorx = Count(m.response)
			}
		case tcell.KeyLeft:
			if m.cursorx > 0 {
				m.cursorx--
			}
		case tcell.KeyRight:
			if m.cursorx < Count(m.response) {
				m.cursorx++
			}
		case tcell.KeyBackspace2, tcell.KeyBackspace:
			if m.cursorx > 0 {
				m.response = string([]rune(m.response)[:m.cursorx-1]) + string([]rune(m.response)[m.cursorx:])
				m.cursorx--
			}
		case tcell.KeyCtrlV:
			clip, _ := clipboard.ReadAll()
			m.response = Insert(m.response, m.cursorx, clip)
			m.cursorx += Count(clip)
		case tcell.KeyRune:
			m.response = Insert(m.response, m.cursorx, string(e.Rune()))
			m.cursorx++
		}
		history[m.historyNum] = m.response

	case *tcell.EventPaste:
		clip := e.Text()
		m.response = Insert(m.response, m.cursorx, clip)
		m.cursorx += Count(clip)
	}
}

// Reset resets the messenger's cursor, message and response
func (m *Messenger) Reset() {
	m.cursorx = 0
	m.message = ""
	m.response = ""
}

// Clear clears the line at the bottom of the editor
func (m *Messenger) Clear() {
	w, h := screen.Size()
	for x := 0; x < w; x++ {
		screen.SetContent(x, h-1, ' ', nil, defStyle)
	}
}

func (m *Messenger) DisplaySuggestions(suggestions []string) {
	w, screenH := screen.Size()

	y := screenH - 2

	statusLineStyle := defStyle.Reverse(true)
	if style, ok := colorscheme["statusline"]; ok {
		statusLineStyle = style
	}

	for x := 0; x < w; x++ {
		screen.SetContent(x, y, ' ', nil, statusLineStyle)
	}

	x := 0
	for _, suggestion := range suggestions {
		for _, c := range suggestion {
			screen.SetContent(x, y, c, nil, statusLineStyle)
			x++
		}
		screen.SetContent(x, y, ' ', nil, statusLineStyle)
		x++
	}
}

// Display displays messages or prompts
func (m *Messenger) Display() {
	_, h := screen.Size()
	if m.hasMessage {
		if !m.hasPrompt {
			return
		}
		runes := []rune(m.message + m.response)
		for x := 0; x < len(runes); x++ {
			screen.SetContent(x, h-1, runes[x], nil, m.style)
		}
	}
	if m.hasPrompt {
		screen.ShowCursor(Count(m.message)+m.cursorx, h-1)
		screen.Show()
	}
}

// A GutterMessage is a message displayed on the side of the editor
type GutterMessage struct {
	lineNum int
	msg     string
	kind    int
}

// These are the different types of messages
const (
	// GutterInfo represents a simple info message
	GutterInfo = iota
	// GutterWarning represents a compiler warning
	GutterWarning
	// GutterError represents a compiler error
	GutterError
)
<commit_msg>Fix prompts not displaying because of infobar<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/zyedidia/clipboard"
	"github.com/zyedidia/tcell"
)

// TermMessage sends a message to the user in the terminal. This usually occurs before
// micro has been fully initialized -- ie if there is an error in the syntax highlighting
// regular expressions
// The function must be called when the screen is not initialized
// This will write the message, and wait for the user
// to press and key to continue
func TermMessage(msg ...interface{}) {
	screenWasNil := screen == nil
	if !screenWasNil {
		screen.Fini()
	}

	fmt.Println(msg...)
	fmt.Print("\nPress enter to continue")

	reader := bufio.NewReader(os.Stdin)
	reader.ReadString('\n')

	if !screenWasNil {
		InitScreen()
	}
}

// TermError sends an error to the user in the terminal. Like TermMessage except formatted
// as an error
func TermError(filename string, lineNum int, err string) {
	TermMessage(filename + ", " + strconv.Itoa(lineNum) + ": " + err)
}

// Messenger is an object that makes it easy to send messages to the user
// and get input from the user
type Messenger struct {
	// Are we currently prompting the user?
	hasPrompt bool
	// Is there a message to print
	hasMessage bool

	// Message to print
	message string
	// The user's response to a prompt
	response string
	// style to use when drawing the message
	style tcell.Style

	// We have to keep track of the cursor for prompting
	cursorx int

	// This map stores the history for all the different kinds of uses Prompt has
	// It's a map of history type -> history array
	history    map[string][]string
	historyNum int

	// Is the current message a message from the gutter
	gutterMessage bool
}

// Message sends a message to the user
func (m *Messenger) Message(msg ...interface{}) {
	buf := new(bytes.Buffer)
	fmt.Fprint(buf, msg...)
	m.message = buf.String()
	m.style = defStyle

	if _, ok := colorscheme["message"]; ok {
		m.style = colorscheme["message"]
	}
	m.hasMessage = true
}

// Error sends an error message to the user
func (m *Messenger) Error(msg ...interface{}) {
	buf := new(bytes.Buffer)
	fmt.Fprint(buf, msg...)
	m.message = buf.String()
	m.style = defStyle.
		Foreground(tcell.ColorBlack).
		Background(tcell.ColorMaroon)

	if _, ok := colorscheme["error-message"]; ok {
		m.style = colorscheme["error-message"]
	}
	m.hasMessage = true
}

// YesNoPrompt asks the user a yes or no question (waits for y or n) and returns the result
func (m *Messenger) YesNoPrompt(prompt string) (bool, bool) {
	m.Message(prompt)

	_, h := screen.Size()
	for {
		m.Clear()
		m.Display()
		screen.ShowCursor(Count(m.message), h-1)
		screen.Show()
		event := <-events

		switch e := event.(type) {
		case *tcell.EventKey:
			switch e.Key() {
			case tcell.KeyRune:
				if e.Rune() == 'y' {
					return true, false
				} else if e.Rune() == 'n' {
					return false, false
				}
			case tcell.KeyCtrlC, tcell.KeyCtrlQ, tcell.KeyEscape:
				return false, true
			}
		}
	}
}

// LetterPrompt gives the user a prompt and waits for a one letter response
func (m *Messenger) LetterPrompt(prompt string, responses ...rune) (rune, bool) {
	m.Message(prompt)

	_, h := screen.Size()
	for {
		m.Clear()
		m.Display()
		screen.ShowCursor(Count(m.message), h-1)
		screen.Show()
		event := <-events

		switch e := event.(type) {
		case *tcell.EventKey:
			switch e.Key() {
			case tcell.KeyRune:
				for _, r := range responses {
					if e.Rune() == r {
						m.Reset()
						return r, false
					}
				}
			case tcell.KeyCtrlC, tcell.KeyCtrlQ, tcell.KeyEscape:
				return ' ', true
			}
		}
	}
}

type Completion int

const (
	NoCompletion Completion = iota
	FileCompletion
	CommandCompletion
	HelpCompletion
	OptionCompletion
)

// Prompt sends the user a message and waits for a response to be typed in
// This function blocks the main loop while waiting for input
func (m *Messenger) Prompt(prompt, historyType string, completionTypes ...Completion) (string, bool) {
	m.hasPrompt = true
	m.Message(prompt)
	if _, ok := m.history[historyType]; !ok {
		m.history[historyType] = []string{""}
	} else {
		m.history[historyType] = append(m.history[historyType], "")
	}
	m.historyNum = len(m.history[historyType]) - 1

	response, canceled := "", true

	RedrawAll()
	for m.hasPrompt {
		var suggestions []string
		m.Clear()

		event := <-events

		switch e := event.(type) {
		case *tcell.EventKey:
			switch e.Key() {
			case tcell.KeyCtrlQ, tcell.KeyCtrlC, tcell.KeyEscape:
				// Cancel
				m.hasPrompt = false
			case tcell.KeyEnter:
				// User is done entering their response
				m.hasPrompt = false
				response, canceled = m.response, false
				m.history[historyType][len(m.history[historyType])-1] = response
			case tcell.KeyTab:
				args := strings.Split(m.response, " ")
				currentArgNum := len(args) - 1
				currentArg := args[currentArgNum]
				var completionType Completion

				if completionTypes[0] == CommandCompletion && currentArgNum > 0 {
					if command, ok := commands[args[0]]; ok {
						completionTypes = append([]Completion{CommandCompletion}, command.completions...)
					}
				}

				if currentArgNum >= len(completionTypes) {
					completionType = completionTypes[len(completionTypes)-1]
				} else {
					completionType = completionTypes[currentArgNum]
				}

				var chosen string
				if completionType == FileCompletion {
					chosen, suggestions = FileComplete(currentArg)
				} else if completionType == CommandCompletion {
					chosen, suggestions = CommandComplete(currentArg)
				} else if completionType == HelpCompletion {
					chosen, suggestions = HelpComplete(currentArg)
				} else if completionType == OptionCompletion {
					chosen, suggestions = OptionComplete(currentArg)
				}

				if len(suggestions) > 1 {
					chosen = chosen + CommonSubstring(suggestions...)
				}

				if chosen != "" {
					if len(args) > 1 {
						chosen = " " + chosen
					}
					m.response = strings.Join(args[:len(args)-1], " ") + chosen
					m.cursorx = Count(m.response)
				}
			}
		}

		m.HandleEvent(event, m.history[historyType])

		messenger.Clear()
		for _, v := range tabs[curTab].views {
			v.Display()
		}
		DisplayTabs()
		messenger.Display()
		if len(suggestions) > 1 {
			m.DisplaySuggestions(suggestions)
		}
		screen.Show()
	}

	m.Reset()
	return response, canceled
}

// HandleEvent handles an event for the prompter
func (m *Messenger) HandleEvent(event tcell.Event, history []string) {
	switch e := event.(type) {
	case *tcell.EventKey:
		switch e.Key() {
		case tcell.KeyUp:
			if m.historyNum > 0 {
				m.historyNum--
				m.response = history[m.historyNum]
				m.cursorx = Count(m.response)
			}
		case tcell.KeyDown:
			if m.historyNum < len(history)-1 {
				m.historyNum++
				m.response = history[m.historyNum]
				m.cursorx = Count(m.response)
			}
		case tcell.KeyLeft:
			if m.cursorx > 0 {
				m.cursorx--
			}
		case tcell.KeyRight:
			if m.cursorx < Count(m.response) {
				m.cursorx++
			}
		case tcell.KeyBackspace2, tcell.KeyBackspace:
			if m.cursorx > 0 {
				m.response = string([]rune(m.response)[:m.cursorx-1]) + string([]rune(m.response)[m.cursorx:])
				m.cursorx--
			}
		case tcell.KeyCtrlV:
			clip, _ := clipboard.ReadAll()
			m.response = Insert(m.response, m.cursorx, clip)
			m.cursorx += Count(clip)
		case tcell.KeyRune:
			m.response = Insert(m.response, m.cursorx, string(e.Rune()))
			m.cursorx++
		}
		history[m.historyNum] = m.response

	case *tcell.EventPaste:
		clip := e.Text()
		m.response = Insert(m.response, m.cursorx, clip)
		m.cursorx += Count(clip)
	}
}

// Reset resets the messenger's cursor, message and response
func (m *Messenger) Reset() {
	m.cursorx = 0
	m.message = ""
	m.response = ""
}

// Clear clears the line at the bottom of the editor
func (m *Messenger) Clear() {
	w, h := screen.Size()
	for x := 0; x < w; x++ {
		screen.SetContent(x, h-1, ' ', nil, defStyle)
	}
}

func (m *Messenger) DisplaySuggestions(suggestions []string) {
	w, screenH := screen.Size()

	y := screenH - 2

	statusLineStyle := defStyle.Reverse(true)
	if style, ok := colorscheme["statusline"]; ok {
		statusLineStyle = style
	}

	for x := 0; x < w; x++ {
		screen.SetContent(x, y, ' ', nil, statusLineStyle)
	}

	x := 0
	for _, suggestion := range suggestions {
		for _, c := range suggestion {
			screen.SetContent(x, y, c, nil, statusLineStyle)
			x++
		}
		screen.SetContent(x, y, ' ', nil, statusLineStyle)
		x++
	}
}

// Display displays messages or prompts
func (m *Messenger) Display() {
	_, h := screen.Size()
	if m.hasMessage {
		if !m.hasPrompt && !globalSettings["infobar"].(bool) {
			return
		}
		runes := []rune(m.message + m.response)
		for x := 0; x < len(runes); x++ {
			screen.SetContent(x, h-1, runes[x], nil, m.style)
		}
	}
	if m.hasPrompt {
		screen.ShowCursor(Count(m.message)+m.cursorx, h-1)
		screen.Show()
	}
}

// A GutterMessage is a message displayed on the side of the editor
type GutterMessage struct {
	lineNum int
	msg     string
	kind    int
}

// These are the different types of messages
const (
	// GutterInfo represents a simple info message
	GutterInfo = iota
	// GutterWarning represents a compiler warning
	GutterWarning
	// GutterError represents a compiler error
	GutterError
)
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net"
	"os"
	"strconv"
	"strings"

	"k8s.io/dns/cmd/node-cache/app"

	corednsmain "github.com/coredns/coredns/coremain"
	clog "github.com/coredns/coredns/plugin/pkg/log"
	utilnet "k8s.io/utils/net"

	// blank imports to make sure the plugin code is pulled in from vendor when building node-cache image
	"github.com/caddyserver/caddy"
	_ "github.com/coredns/coredns/plugin/bind"
	_ "github.com/coredns/coredns/plugin/cache"
	_ "github.com/coredns/coredns/plugin/debug"
	_ "github.com/coredns/coredns/plugin/errors"
	_ "github.com/coredns/coredns/plugin/forward"
	_ "github.com/coredns/coredns/plugin/health"
	_ "github.com/coredns/coredns/plugin/hosts"
	_ "github.com/coredns/coredns/plugin/loadbalance"
	_ "github.com/coredns/coredns/plugin/log"
	_ "github.com/coredns/coredns/plugin/loop"
	_ "github.com/coredns/coredns/plugin/metrics"
	_ "github.com/coredns/coredns/plugin/pprof"
	_ "github.com/coredns/coredns/plugin/reload"
	_ "github.com/coredns/coredns/plugin/template"
	_ "github.com/coredns/coredns/plugin/whoami"
	"k8s.io/dns/pkg/version"
)

var cache *app.CacheApp

func init() {
	clog.Infof("Starting node-cache image: %+v", version.VERSION)
	params, err := parseAndValidateFlags()
	if err != nil {
		clog.Fatalf("Error parsing flags - %s, Exiting", err)
	}
	cache, err = app.NewCacheApp(params)
	if err != nil {
		clog.Fatalf("Failed to obtain CacheApp instance, err %v", err)
	}
	cache.Init()
	if !params.SkipTeardown {
		caddy.OnProcessExit = append(caddy.OnProcessExit, func() { cache.TeardownNetworking() })
	}
}

func parseAndValidateFlags() (*app.ConfigParams, error) {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "Runs CoreDNS v%s as a nodelocal cache listening on the specified ip:port\n\n", corednsmain.CoreVersion)
		flag.PrintDefaults()
	}

	params := &app.ConfigParams{LocalPort: "53"}

	flag.StringVar(&params.LocalIPStr, "localip", "", "comma-separated string of ip addresses to bind dnscache to")
	flag.BoolVar(&params.SetupInterface, "setupinterface", true, "indicates whether network interface should be setup")
	flag.StringVar(&params.InterfaceName, "interfacename", "nodelocaldns", "name of the interface to be created")
	flag.DurationVar(&params.Interval, "syncinterval", 60, "interval(in seconds) to check for iptables rules")
	flag.StringVar(&params.MetricsListenAddress, "metrics-listen-address", "0.0.0.0:9353", "address to serve metrics on")
	flag.BoolVar(&params.SetupIptables, "setupiptables", true, "indicates whether iptables rules should be setup")
	flag.BoolVar(&params.SetupEbtables, "setupebtables", false, "indicates whether ebtables rules should be setup")
	flag.StringVar(&params.BaseCoreFile, "basecorefile", "/etc/coredns/Corefile.base", "Path to the template Corefile for node-cache")
	flag.StringVar(&params.CoreFile, "corefile", "/etc/Corefile", "Path to the Corefile to be used by node-cache")
	flag.StringVar(&params.KubednsCMPath, "kubednscm", "", "Path where the kube-dns configmap will be mounted")
	flag.StringVar(&params.UpstreamSvcName, "upstreamsvc", "kube-dns", "Service name whose cluster IP is upstream for node-cache")
	flag.StringVar(&params.HealthPort, "health-port", "8080", "port used by health plugin")
	flag.BoolVar(&params.SkipTeardown, "skipteardown", false, "indicates whether iptables rules should be torn down on exit")
	flag.Parse()

	for _, ipstr := range strings.Split(params.LocalIPStr, ",") {
		newIP := net.ParseIP(ipstr)
		if newIP == nil {
			return params, fmt.Errorf("Invalid localip specified - %q", ipstr)
		}
		params.LocalIPs = append(params.LocalIPs, newIP)
	}

	// validate all the IPs have the same IP family
	for _, ip := range params.LocalIPs {
		if utilnet.IsIPv6(params.LocalIPs[0]) != utilnet.IsIPv6(ip) {
			return params, fmt.Errorf("Unexpected IP Family for localIP - %q, want IPv6=%v", ip, utilnet.IsIPv6(params.LocalIPs[0]))
		}
	}
	// lookup specified dns port
	f := flag.Lookup("dns.port")
	if f == nil {
		return nil, fmt.Errorf("Failed to lookup \"dns.port\" parameter")
	}
	params.LocalPort = f.Value.String()
	if _, err := strconv.Atoi(params.LocalPort); err != nil {
		return nil, fmt.Errorf("Invalid port specified - %q", params.LocalPort)
	}
	if _, err := strconv.Atoi(params.HealthPort); err != nil {
		return nil, fmt.Errorf("Invalid healthcheck port specified - %q", params.HealthPort)
	}
	if f = flag.Lookup("conf"); f != nil {
		params.CoreFile = f.Value.String()
		clog.Infof("Using Corefile %s", params.CoreFile)
	}
	return params, nil
}

func main() {
	cache.RunApp()
}
<commit_msg>Add support for rewrite plugin in node-cache<commit_after>package main

import (
	"flag"
	"fmt"
	"net"
	"os"
	"strconv"
	"strings"

	"k8s.io/dns/cmd/node-cache/app"

	corednsmain "github.com/coredns/coredns/coremain"
	clog "github.com/coredns/coredns/plugin/pkg/log"
	utilnet "k8s.io/utils/net"

	// blank imports to make sure the plugin code is pulled in from vendor when building node-cache image
	"github.com/caddyserver/caddy"
	_ "github.com/coredns/coredns/plugin/bind"
	_ "github.com/coredns/coredns/plugin/cache"
	_ "github.com/coredns/coredns/plugin/debug"
	_ "github.com/coredns/coredns/plugin/errors"
	_ "github.com/coredns/coredns/plugin/forward"
	_ "github.com/coredns/coredns/plugin/health"
	_ "github.com/coredns/coredns/plugin/hosts"
	_ "github.com/coredns/coredns/plugin/loadbalance"
	_ "github.com/coredns/coredns/plugin/log"
	_ "github.com/coredns/coredns/plugin/loop"
	_ "github.com/coredns/coredns/plugin/metrics"
	_ "github.com/coredns/coredns/plugin/pprof"
	_ "github.com/coredns/coredns/plugin/reload"
	_ "github.com/coredns/coredns/plugin/rewrite"
	_ "github.com/coredns/coredns/plugin/template"
	_ "github.com/coredns/coredns/plugin/whoami"
	"k8s.io/dns/pkg/version"
)

var cache *app.CacheApp

func init() {
	clog.Infof("Starting node-cache image: %+v", version.VERSION)
	params, err := parseAndValidateFlags()
	if err != nil {
		clog.Fatalf("Error parsing flags - %s, Exiting", err)
	}
	cache, err = app.NewCacheApp(params)
	if err != nil {
		clog.Fatalf("Failed to obtain CacheApp instance, err %v", err)
	}
	cache.Init()
	if !params.SkipTeardown {
		caddy.OnProcessExit = append(caddy.OnProcessExit, func() { cache.TeardownNetworking() })
	}
}

func parseAndValidateFlags() (*app.ConfigParams, error) {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "Runs CoreDNS v%s as a nodelocal cache listening on the specified ip:port\n\n", corednsmain.CoreVersion)
		flag.PrintDefaults()
	}

	params := &app.ConfigParams{LocalPort: "53"}

	flag.StringVar(&params.LocalIPStr, "localip", "", "comma-separated string of ip addresses to bind dnscache to")
	flag.BoolVar(&params.SetupInterface, "setupinterface", true, "indicates whether network interface should be setup")
	flag.StringVar(&params.InterfaceName, "interfacename", "nodelocaldns", "name of the interface to be created")
	flag.DurationVar(&params.Interval, "syncinterval", 60, "interval(in seconds) to check for iptables rules")
	flag.StringVar(&params.MetricsListenAddress, "metrics-listen-address", "0.0.0.0:9353", "address to serve metrics on")
	flag.BoolVar(&params.SetupIptables, "setupiptables", true, "indicates whether iptables rules should be setup")
	flag.BoolVar(&params.SetupEbtables, "setupebtables", false, "indicates whether ebtables rules should be setup")
	flag.StringVar(&params.BaseCoreFile, "basecorefile", "/etc/coredns/Corefile.base", "Path to the template Corefile for node-cache")
	flag.StringVar(&params.CoreFile, "corefile", "/etc/Corefile", "Path to the Corefile to be used by node-cache")
	flag.StringVar(&params.KubednsCMPath, "kubednscm", "", "Path where the kube-dns configmap will be mounted")
	flag.StringVar(&params.UpstreamSvcName, "upstreamsvc", "kube-dns", "Service name whose cluster IP is upstream for node-cache")
	flag.StringVar(&params.HealthPort, "health-port", "8080", "port used by health plugin")
	flag.BoolVar(&params.SkipTeardown, "skipteardown", false, "indicates whether iptables rules should be torn down on exit")
	flag.Parse()

	for _, ipstr := range strings.Split(params.LocalIPStr, ",") {
		newIP := net.ParseIP(ipstr)
		if newIP == nil {
			return params, fmt.Errorf("Invalid localip specified - %q", ipstr)
		}
		params.LocalIPs = append(params.LocalIPs, newIP)
	}

	// validate all the IPs have the same IP family
	for _, ip := range params.LocalIPs {
		if utilnet.IsIPv6(params.LocalIPs[0]) != utilnet.IsIPv6(ip) {
			return params, fmt.Errorf("Unexpected IP Family for localIP - %q, want IPv6=%v", ip, utilnet.IsIPv6(params.LocalIPs[0]))
		}
	}
	// lookup specified dns port
	f := flag.Lookup("dns.port")
	if f == nil {
		return nil, fmt.Errorf("Failed to lookup \"dns.port\" parameter")
	}
	params.LocalPort = f.Value.String()
	if _, err := strconv.Atoi(params.LocalPort); err != nil {
		return nil, fmt.Errorf("Invalid port specified - %q", params.LocalPort)
	}
	if _, err := strconv.Atoi(params.HealthPort); err != nil {
		return nil, fmt.Errorf("Invalid healthcheck port specified - %q", params.HealthPort)
	}
	if f = flag.Lookup("conf"); f != nil {
		params.CoreFile = f.Value.String()
		clog.Infof("Using Corefile %s", params.CoreFile)
	}
	return params, nil
}

func main() {
	cache.RunApp()
}
<|endoftext|>


<commit_before>/*
	This file is part of go-ethereum

	go-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	go-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
package main

import (
	"crypto/elliptic"
	"flag"
	"log"
	"os"

	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/p2p"
)

var (
	natType    = flag.String("nat", "", "NAT traversal implementation")
	pmpGateway = flag.String("gateway", "", "gateway address for NAT-PMP")
	listenAddr = flag.String("addr", ":30301", "listen address")
)

func main() {
	flag.Parse()
	nat, err := p2p.ParseNAT(*natType, *pmpGateway)
	if err != nil {
		log.Fatal("invalid nat:", err)
	}

	logger.AddLogSystem(logger.NewStdLogSystem(os.Stdout, log.LstdFlags, logger.InfoLevel))
	key, _ := crypto.GenerateKey()
	marshaled := elliptic.Marshal(crypto.S256(), key.PublicKey.X, key.PublicKey.Y)

	srv := p2p.Server{
		MaxPeers:   100,
		Identity:   p2p.NewSimpleClientIdentity("Ethereum(G)", "0.1", "Peer Server Two", marshaled),
		ListenAddr: *listenAddr,
		NAT:        nat,
		NoDial:     true,
	}
	if err := srv.Start(); err != nil {
		log.Fatal("could not start server:", err)
	}
	select {}
}
<commit_msg>cmd/peerserver: is gone<commit_after><|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net/http"
	"path"

	"github.com/DeedleFake/signage"
)

func marshalRSS(t string, bills []signage.Bill) (io.Reader, error) {
	var buf bytes.Buffer
	err := tmpl.ExecuteTemplate(&buf, "rss", map[string]interface{}{
		"Type":  t,
		"Bills": bills,
	})
	return &buf, err
}

func marshalJSON(t string, bills []signage.Bill) (io.Reader, error) {
	panic("Not implemented.")
}

func handleSigned(rw http.ResponseWriter, req *http.Request) {
	mode := path.Ext(req.URL.Path)
	if mode == "" {
		mode = ".rss"
	}

	var marshal func(t string, bills []signage.Bill) (io.Reader, error)
	switch mode {
	case ".rss":
		marshal = marshalRSS
	case ".json":
		marshal = marshalJSON
	default:
		http.Error(rw, fmt.Sprintf("Unknown format: %q", mode), http.StatusBadRequest)
		return
	}

	bills, err := signage.GetSigned()
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	buf, err := marshal("Signed", bills)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	_, err = io.Copy(rw, buf)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
}

func main() {
	http.HandleFunc("/", handleSigned)
	log.Fatalln(http.ListenAndServe(":8080", nil))
}
<commit_msg>cmd/signage: Add JSON support.<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"path"

	"github.com/DeedleFake/signage"
)

func marshalRSS(t string, bills []signage.Bill) (io.Reader, error) {
	var buf bytes.Buffer
	err := tmpl.ExecuteTemplate(&buf, "rss", map[string]interface{}{
		"Type":  t,
		"Bills": bills,
	})
	return &buf, err
}

func marshalJSON(t string, bills []signage.Bill) (io.Reader, error) {
	buf, err := json.Marshal(map[string]interface{}{
		"type":  t,
		"bills": bills,
	})
	return bytes.NewReader(buf), err
}

func handleSigned(rw http.ResponseWriter, req *http.Request) {
	mode := path.Ext(req.URL.Path)
	if mode == "" {
		mode = ".rss"
	}

	var marshal func(t string, bills []signage.Bill) (io.Reader, error)
	switch mode {
	case ".rss":
		marshal = marshalRSS
	case ".json":
		marshal = marshalJSON
	default:
		http.Error(rw, fmt.Sprintf("Unknown format: %q", mode), http.StatusBadRequest)
		return
	}

	bills, err := signage.GetSigned()
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	buf, err := marshal("Signed", bills)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	_, err = io.Copy(rw, buf)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
}

func main() {
	http.HandleFunc("/", handleSigned)
	log.Fatalln(http.ListenAndServe(":8080", nil))
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"github.com/smira/aptly/deb"
	"github.com/smira/aptly/query"
	"github.com/smira/commander"
	"github.com/smira/flag"
)

func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error {
	var err error
	if len(args) != 2 {
		cmd.Usage()
		return commander.ErrCommandError
	}

	name := args[0]
	command := cmd.Parent.Name()

	var reflist *deb.PackageRefList

	if command == "snapshot" {
		snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
		if err != nil {
			return fmt.Errorf("unable to search: %s", err)
		}

		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
		if err != nil {
			return fmt.Errorf("unable to search: %s", err)
		}

		reflist = snapshot.RefList()
	} else if command == "mirror" {
		repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
		if err != nil {
			return fmt.Errorf("unable to search: %s", err)
		}

		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
		if err != nil {
			return fmt.Errorf("unable to search: %s", err)
		}

		reflist = repo.RefList()
	} else if command == "repo" {
		repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
		if err != nil {
			return fmt.Errorf("unable to search: %s", err)
		}

		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
		if err != nil {
			return fmt.Errorf("unable to search: %s", err)
		}

		reflist = repo.RefList()
	} else {
		panic("unknown command")
	}

	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
	if err != nil {
		return fmt.Errorf("unable to search: %s", err)
	}

	list.PrepareIndex()

	q, err := query.Parse(args[1])
	if err != nil {
		return fmt.Errorf("unable to search: %s", err)
	}

	result, err := list.Filter([]deb.PackageQuery{q}, context.flags.Lookup("with-deps").Value.Get().(bool),
		nil, context.DependencyOptions(), context.ArchitecturesList())
	if err != nil {
		return fmt.Errorf("unable to search: %s", err)
	}

	result.ForEach(func(p *deb.Package) error {
		context.Progress().Printf("%s\n", p)
		return nil
	})

	return err
}

func makeCmdSnapshotSearch() *commander.Command {
	cmd := &commander.Command{
		Run:       aptlySnapshotMirrorRepoSearch,
		UsageLine: "search <name> <package-query>",
		Short:     "search snapshot for packages matching query",
		Long: `
Command search displays list of packages in snapshot that match package query

Example:

    $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'
`,
		Flag: *flag.NewFlagSet("aptly-snapshot-search", flag.ExitOnError),
	}

	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")

	return cmd
}
<commit_msg>Fix -with-deps searching. #81<commit_after>package cmd

import (
	"fmt"
	"github.com/smira/aptly/deb"
	"github.com/smira/aptly/query"
	"github.com/smira/commander"
	"github.com/smira/flag"
	"sort"
)

func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error {
	var err error
	if len(args) != 2 {
		cmd.Usage()
		return commander.ErrCommandError
	}

	name := args[0]
	command := cmd.Parent.Name()

	var reflist *deb.PackageRefList

	if command == "snapshot" {
		snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
		if err != nil {
			return fmt.Errorf("unable to search: %s", err)
		}

		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
		if err != nil {
			return fmt.Errorf("unable to search: %s", err)
		}

		reflist = snapshot.RefList()
	} else if command == "mirror" {
		repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
		if err != nil {
			return fmt.Errorf("unable to search: %s", err)
		}

		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
		if err != nil {
			return fmt.Errorf("unable to search: %s", err)
		}

		reflist = repo.RefList()
	} else if command == "repo" {
		repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
		if err != nil {
			return fmt.Errorf("unable to search: %s", err)
		}

		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
		if err != nil {
			return fmt.Errorf("unable to search: %s", err)
		}

		reflist = repo.RefList()
	} else {
		panic("unknown command")
	}

	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
	if err != nil {
		return fmt.Errorf("unable to search: %s", err)
	}

	list.PrepareIndex()

	q, err := query.Parse(args[1])
	if err != nil {
		return fmt.Errorf("unable to search: %s", err)
	}

	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
	architecturesList := []string{}

	if withDeps {
		if len(context.ArchitecturesList()) > 0 {
			architecturesList = context.ArchitecturesList()
		} else {
			architecturesList = list.Architectures(false)
		}

		sort.Strings(architecturesList)

		if len(architecturesList) == 0 {
			return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
		}
	}

	result, err := list.Filter([]deb.PackageQuery{q}, withDeps,
		nil, context.DependencyOptions(), architecturesList)
	if err != nil {
		return fmt.Errorf("unable to search: %s", err)
	}

	result.ForEach(func(p *deb.Package) error {
		context.Progress().Printf("%s\n", p)
		return nil
	})

	return err
}

func makeCmdSnapshotSearch() *commander.Command {
	cmd := &commander.Command{
		Run:       aptlySnapshotMirrorRepoSearch,
		UsageLine: "search <name> <package-query>",
		Short:     "search snapshot for packages matching query",
		Long: `
Command search displays list of packages in snapshot that match package query

Example:

    $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'
`,
		Flag: *flag.NewFlagSet("aptly-snapshot-search", flag.ExitOnError),
	}

	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")

	return cmd
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	goflag "flag"
	"fmt"
	"math/rand"
	"os"
	"time"

	"github.com/golang/glog"
	flag "github.com/spf13/pflag"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/Mirantis/virtlet/pkg/api/virtlet.k8s/v1"
	"github.com/Mirantis/virtlet/pkg/cni"
	"github.com/Mirantis/virtlet/pkg/config"
	"github.com/Mirantis/virtlet/pkg/diag"
	"github.com/Mirantis/virtlet/pkg/libvirttools"
	"github.com/Mirantis/virtlet/pkg/manager"
	"github.com/Mirantis/virtlet/pkg/nsfix"
	"github.com/Mirantis/virtlet/pkg/tapmanager"
	"github.com/Mirantis/virtlet/pkg/utils"
	"github.com/Mirantis/virtlet/pkg/version"
)

const (
	wantTapManagerEnv = "WANT_TAP_MANAGER"
	nodeNameEnv       = "KUBE_NODE_NAME"
	diagSocket        = "/run/virtlet-diag.sock"
	netnsDiagCommand  = `if [ -d /var/run/netns ]; then cd /var/run/netns; for ns in *; do echo "*** ${ns} ***"; ip netns exec "${ns}" ip a; ip netns exec "${ns}" ip r; echo; done; fi`
	qemuLogDir        = "/var/log/libvirt/qemu"
)

var (
	dumpConfig     = flag.Bool("dump-config", false, "Dump node-specific Virtlet config as a shell script and exit")
	dumpDiag       = flag.Bool("diag", false, "Dump diagnostics as JSON and exit")
	displayVersion = flag.Bool("version", false, "Display version and exit")
	versionFormat  = flag.String("version-format", "text", "Version format to use (text, short, json, yaml)")
)

func configWithDefaults(cfg *v1.VirtletConfig) *v1.VirtletConfig {
	r := config.GetDefaultConfig()
	config.Override(r, cfg)
	return r
}

func runVirtlet(config *v1.VirtletConfig, clientCfg clientcmd.ClientConfig, diagSet *diag.DiagSet) {
	manager := manager.NewVirtletManager(config, nil, clientCfg, diagSet)
	if err := manager.Run(); err != nil {
		glog.Errorf("Error: %v", err)
		os.Exit(1)
	}
}

func runTapManager(config *v1.VirtletConfig) {
	cniClient, err := cni.NewClient(*config.CNIPluginDir, *config.CNIConfigDir)
	if err != nil {
		glog.Errorf("Error initializing CNI client: %v", err)
		os.Exit(1)
	}
	src, err := tapmanager.NewTapFDSource(cniClient, *config.EnableSriov, *config.CalicoSubnetSize)
	if err != nil {
		glog.Errorf("Error creating tap fd source: %v", err)
		os.Exit(1)
	}
	os.Remove(*config.FDServerSocketPath) // FIXME
	s := tapmanager.NewFDServer(*config.FDServerSocketPath, src)
	if err = s.Serve(); err != nil {
		glog.Errorf("FD server returned error: %v", err)
		os.Exit(1)
	}
	if err := libvirttools.ChownForEmulator(*config.FDServerSocketPath); err != nil {
		glog.Warningf("Couldn't set tapmanager socket permissions: %v", err)
	}
	for {
		time.Sleep(1000 * time.Hour)
	}
}

func printVersion() {
	out, err := version.Get().ToBytes(*versionFormat)
	if err == nil {
		_, err = os.Stdout.Write(out)
	}
	if err != nil {
		glog.Errorf("Error printing version info: %v", err)
		os.Exit(1)
	}
}

func setLogLevel(config *v1.VirtletConfig) {
	goflag.CommandLine.Parse([]string{
		fmt.Sprintf("-v=%d", config.LogLevel),
		"-logtostderr=true",
	})
}

func runDiagServer() *diag.DiagSet {
	diagSet := diag.NewDiagSet()
	diagSet.RegisterDiagSource("ip-a", diag.NewCommandSource("txt", []string{"ip", "a"}))
	diagSet.RegisterDiagSource("ip-r", diag.NewCommandSource("txt", []string{"ip", "r"}))
	diagSet.RegisterDiagSource("netns", diag.NewCommandSource("txt", []string{"/bin/bash", "-c", netnsDiagCommand}))
	diagSet.RegisterDiagSource("libvirt-logs", diag.NewLogDirSource(qemuLogDir))
	diagSet.RegisterDiagSource("stack", diag.StackDumpSource)
	server := diag.NewServer(diagSet)
	go func() {
		err := server.Serve(diagSocket, nil)
		glog.V(1).Infof("Diag server returned: %v", err)
	}()
	return diagSet
}

func doDiag() {
	dr, err := diag.RetrieveDiagnostics(diagSocket)
	if err != nil {
		glog.Errorf("Failed to retrieve diagnostics: %v", err)
		os.Exit(1)
	}
	os.Stdout.Write(dr.ToJSON())
}

func main() {
	nsfix.HandleReexec()
	clientCfg := utils.BindFlags(flag.CommandLine)
	var cb *config.Binder
	cb = config.NewBinder(flag.CommandLine)
	flag.Parse()
	localConfig := cb.GetConfig()

	rand.Seed(time.Now().UnixNano())
	setLogLevel(configWithDefaults(localConfig))
	switch {
	case *displayVersion:
		printVersion()
	case *dumpConfig:
		nodeConfig := config.NewNodeConfig(clientCfg)
		nodeName := os.Getenv(nodeNameEnv)
		cfg, err := nodeConfig.LoadConfig(localConfig, nodeName)
		if err != nil {
			glog.Warningf("Failed to load per-node configs, using local config only: %v", err)
			cfg = localConfig
		}
		if _, err := os.Stdout.Write([]byte(config.DumpEnv(cfg))); err != nil {
			glog.Errorf("Error writing config: %v", err)
			os.Exit(1)
		}
	case *dumpDiag:
		doDiag()
	default:
		localConfig = configWithDefaults(localConfig)
		go runTapManager(localConfig)
		diagSet := runDiagServer()
		runVirtlet(localConfig, clientCfg, diagSet)
	}
}
<commit_msg>Include CRI Proxy logs in the diagnostics<commit_after>/*
Copyright 2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	goflag "flag"
	"fmt"
	"math/rand"
	"os"
	"time"

	"github.com/golang/glog"
	flag "github.com/spf13/pflag"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/Mirantis/virtlet/pkg/api/virtlet.k8s/v1"
	"github.com/Mirantis/virtlet/pkg/cni"
	"github.com/Mirantis/virtlet/pkg/config"
	"github.com/Mirantis/virtlet/pkg/diag"
	"github.com/Mirantis/virtlet/pkg/libvirttools"
	"github.com/Mirantis/virtlet/pkg/manager"
	"github.com/Mirantis/virtlet/pkg/nsfix"
	"github.com/Mirantis/virtlet/pkg/tapmanager"
	"github.com/Mirantis/virtlet/pkg/utils"
	"github.com/Mirantis/virtlet/pkg/version"
)

const (
	wantTapManagerEnv  = "WANT_TAP_MANAGER"
	nodeNameEnv        = "KUBE_NODE_NAME"
	diagSocket         = "/run/virtlet-diag.sock"
	netnsDiagCommand   = `if [ -d /var/run/netns ]; then cd /var/run/netns; for ns in *; do echo "*** ${ns} ***"; ip netns exec "${ns}" ip a; ip netns exec "${ns}" ip r; echo; done; fi`
	criproxyLogCommand = `nsenter -t 1 -m -u -i journalctl -xe -u criproxy -n 20000 --no-pager || true`
	qemuLogDir         = "/var/log/libvirt/qemu"
)

var (
	dumpConfig     = flag.Bool("dump-config", false, "Dump node-specific Virtlet config as a shell script and exit")
	dumpDiag       = flag.Bool("diag", false, "Dump diagnostics as JSON and exit")
	displayVersion = flag.Bool("version", false, "Display version and exit")
	versionFormat  = flag.String("version-format", "text", "Version format to use (text, short, json, yaml)")
)

func configWithDefaults(cfg *v1.VirtletConfig) *v1.VirtletConfig {
	r := config.GetDefaultConfig()
	config.Override(r, cfg)
	return r
}

func runVirtlet(config *v1.VirtletConfig, clientCfg clientcmd.ClientConfig, diagSet *diag.DiagSet) {
	manager := manager.NewVirtletManager(config, nil, clientCfg, diagSet)
	if err := manager.Run(); err != nil {
		glog.Errorf("Error: %v", err)
		os.Exit(1)
	}
}

func runTapManager(config *v1.VirtletConfig) {
	cniClient, err := cni.NewClient(*config.CNIPluginDir, *config.CNIConfigDir)
	if err != nil {
		glog.Errorf("Error initializing CNI client: %v", err)
		os.Exit(1)
	}
	src, err := tapmanager.NewTapFDSource(cniClient, *config.EnableSriov, *config.CalicoSubnetSize)
	if err != nil {
		glog.Errorf("Error creating tap fd source: %v", err)
		os.Exit(1)
	}
	os.Remove(*config.FDServerSocketPath) // FIXME
	s := tapmanager.NewFDServer(*config.FDServerSocketPath, src)
	if err = s.Serve(); err != nil {
		glog.Errorf("FD server returned error: %v", err)
		os.Exit(1)
	}
	if err := libvirttools.ChownForEmulator(*config.FDServerSocketPath); err != nil {
		glog.Warningf("Couldn't set tapmanager socket permissions: %v", err)
	}
	for {
		time.Sleep(1000 * time.Hour)
	}
}

func printVersion() {
	out, err := version.Get().ToBytes(*versionFormat)
	if err == nil {
		_, err = os.Stdout.Write(out)
	}
	if err != nil {
		glog.Errorf("Error printing version info: %v", err)
		os.Exit(1)
	}
}

func setLogLevel(config *v1.VirtletConfig) {
	goflag.CommandLine.Parse([]string{
		fmt.Sprintf("-v=%d", config.LogLevel),
		"-logtostderr=true",
	})
}

func runDiagServer() *diag.DiagSet {
	diagSet := diag.NewDiagSet()
	diagSet.RegisterDiagSource("ip-a", diag.NewCommandSource("txt", []string{"ip", "a"}))
	diagSet.RegisterDiagSource("ip-r", diag.NewCommandSource("txt", []string{"ip", "r"}))
	diagSet.RegisterDiagSource("netns", diag.NewCommandSource("txt", []string{"/bin/bash", "-c", netnsDiagCommand}))
	diagSet.RegisterDiagSource("criproxy", diag.NewCommandSource("log", []string{"/bin/bash", "-c", criproxyLogCommand}))
	diagSet.RegisterDiagSource("libvirt-logs", diag.NewLogDirSource(qemuLogDir))
	diagSet.RegisterDiagSource("stack", diag.StackDumpSource)
	server := diag.NewServer(diagSet)
	go func() {
		err := server.Serve(diagSocket, nil)
		glog.V(1).Infof("Diag server returned: %v", err)
	}()
	return diagSet
}

func doDiag() {
	dr, err := diag.RetrieveDiagnostics(diagSocket)
	if err != nil {
		glog.Errorf("Failed to retrieve diagnostics: %v", err)
		os.Exit(1)
	}
	os.Stdout.Write(dr.ToJSON())
}

func main() {
	nsfix.HandleReexec()
	clientCfg := utils.BindFlags(flag.CommandLine)
	var cb *config.Binder
	cb = config.NewBinder(flag.CommandLine)
	flag.Parse()
	localConfig := cb.GetConfig()

	rand.Seed(time.Now().UnixNano())
	setLogLevel(configWithDefaults(localConfig))
	switch {
	case *displayVersion:
		printVersion()
	case *dumpConfig:
		nodeConfig := config.NewNodeConfig(clientCfg)
		nodeName := os.Getenv(nodeNameEnv)
		cfg, err := nodeConfig.LoadConfig(localConfig, nodeName)
		if err != nil {
			glog.Warningf("Failed to load per-node configs, using local config only: %v", err)
			cfg = localConfig
		}
		if _, err := os.Stdout.Write([]byte(config.DumpEnv(cfg))); err != nil {
			glog.Errorf("Error writing config: %v", err)
			os.Exit(1)
		}
	case *dumpDiag:
		doDiag()
	default:
		localConfig = configWithDefaults(localConfig)
		go runTapManager(localConfig)
		diagSet := runDiagServer()
		runVirtlet(localConfig, clientCfg, diagSet)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/mrspock/godocsis"
	"net"
	"os"
)

func main() {
	//var ip string
	flag.Parse()
	if len(flag.Args()) == 0 {
		Help(os.Args[0])
		return
	}
	for _, host := range flag.Args() {
		ip, err := net.LookupHost(host)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Host lookup error:", err)
			//os.Exit(1)
			continue
		}
		for _, address := range ip {
			//fmt.Println(address)
			err := godocsis.ResetCm(address)
			if err != nil {
				fmt.Fprintln(os.Stderr, "NG: Wystąpił błąd komunikacji z modemem", address, ":", err)
				//os.Exit(1)
				continue
			} else {
				fmt.Fprintln(os.Stdout, "OK: Modem", address, "w trakcie restartu..")
				//os.Exit(0)
			}
		}
	}
	os.Exit(1)
}

func Help(name string) {
	fmt.Fprintf(os.Stderr, "======= Cable Modem restarter by Spock (BSD) ========\nUsage: %s cm1_ipaddr cm2_ipaddr\n============================================\n", name)
}
<commit_msg>correct exit code<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/mrspock/godocsis"
	"net"
	"os"
)

func main() {
	//var ip string
	flag.Parse()
	if len(flag.Args()) == 0 {
		Help(os.Args[0])
		return
	}
	for _, host := range flag.Args() {
		ip, err := net.LookupHost(host)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Host lookup error:", err)
			//os.Exit(1)
			continue
		}
		for _, address := range ip {
			//fmt.Println(address)
			err := godocsis.ResetCm(address)
			if err != nil {
				fmt.Fprintln(os.Stderr, "NG: Wystąpił błąd komunikacji z modemem", address, ":", err)
				//os.Exit(1)
				continue
			} else {
				fmt.Fprintln(os.Stdout, "OK: Modem", address, "w trakcie restartu..")
				//os.Exit(0)
			}
		}
	}
	os.Exit(0)
}

func Help(name string) {
	fmt.Fprintf(os.Stderr, "======= Cable Modem restarter by Spock (BSD) ========\nUsage: %s cm1_ipaddr cm2_ipaddr\n============================================\n", name)
}
<|endoftext|>


<commit_before>package proto

type Message struct {
	Data []byte
}

func (m Message) MarshalJSON() ([]byte, error) {
	return m.Data, nil
}

func (m *Message) UnmarshalJSON(data []byte) error {
	m.Data = data
	return nil
}

func (m *Message) ProtoMessage() {}

func (m *Message) Reset() {
	*m = Message{}
}

func (m *Message) String() string {
	return string(m.Data)
}

func (m *Message) Marshal() ([]byte, error) {
	return m.Data, nil
}

func (m *Message) Unmarshal(data []byte) error {
	m.Data = data
	return nil
}

func NewMessage(data []byte) *Message {
	return &Message{data}
}
<commit_msg>fix missing pointer<commit_after>package proto

type Message struct {
	Data []byte
}

func (m *Message) MarshalJSON() ([]byte, error) {
	return m.Data, nil
}

func (m *Message) UnmarshalJSON(data []byte) error {
	m.Data = data
	return nil
}

func (m *Message) ProtoMessage() {}

func (m *Message) Reset() {
	*m = Message{}
}

func (m *Message) String() string {
	return string(m.Data)
}

func (m *Message) Marshal() ([]byte, error) {
	return m.Data, nil
}

func (m *Message) Unmarshal(data []byte) error {
	m.Data = data
	return nil
}

func NewMessage(data []byte) *Message {
	return &Message{data}
}
<|endoftext|>


<commit_before>package collector

import (
	"fmt"
	"github.com/timeredbull/tsuru/api/app"
	"github.com/timeredbull/tsuru/db"
	"launchpad.net/goyaml"
	"launchpad.net/mgo/bson"
	"os/exec"
)

type Collector struct{}

type Unit struct {
	Machine int
	State   string
}

type Service struct {
	Units map[string]Unit
}

type output struct {
	Services map[string]Service
	Machines map[int]interface{}
}

func (c *Collector) Collect() ([]byte, error) {
	fmt.Println("collecting status")
	return exec.Command("juju", "status").Output()
}

func (c *Collector) Parse(data []byte) *output {
	fmt.Println("parsing yaml")
	raw := new(output)
	_ = goyaml.Unmarshal(data, raw)
	return raw
}

func (c *Collector) Update(out *output) {
	fmt.Println("updating status")

	for serviceName, service := range out.Services {
		for _, unit := range service.Units {
			appUnit := app.App{Name: serviceName}
			appUnit.Get()
			if unit.State == "started" {
				appUnit.State = "STARTED"
			} else {
				appUnit.State = "STOPPED"
			}
			appUnit.Ip = out.Machines[unit.Machine].(map[interface{}]interface{})["dns-name"].(string)
			db.Session.Apps().Update(bson.M{"name": appUnit.Name}, appUnit)
		}
	}
}
<commit_msg>using log in collector<commit_after>package collector

import (
	"github.com/timeredbull/tsuru/api/app"
	"github.com/timeredbull/tsuru/db"
	"github.com/timeredbull/tsuru/log"
	"launchpad.net/goyaml"
	"launchpad.net/mgo/bson"
	"os/exec"
)

type Collector struct{}

type Unit struct {
	Machine int
	State   string
}

type Service struct {
	Units map[string]Unit
}

type output struct {
	Services map[string]Service
	Machines map[int]interface{}
}

func (c *Collector) Collect() ([]byte, error) {
	log.Print("collecting status")
	return exec.Command("juju", "status").Output()
}

func (c *Collector) Parse(data []byte) *output {
	log.Print("parsing yaml")
	raw := new(output)
	_ = goyaml.Unmarshal(data, raw)
	return raw
}

func (c *Collector) Update(out *output) {
	log.Print("updating status")
	for serviceName, service := range out.Services {
		for _, unit := range service.Units {
			appUnit := app.App{Name: serviceName}
			appUnit.Get()
			if unit.State == "started" {
				appUnit.State = "STARTED"
			} else {
				appUnit.State = "STOPPED"
			}
			appUnit.Ip = out.Machines[unit.Machine].(map[interface{}]interface{})["dns-name"].(string)
			db.Session.Apps().Update(bson.M{"name": appUnit.Name}, appUnit)
		}
	}
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package sse

import (
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var url string
var srv *Server
var server *httptest.Server

func setup(empty bool) {
	// New Server
	srv = New()

	mux := http.NewServeMux()
	mux.HandleFunc("/events", srv.HTTPHandler)
	server = httptest.NewServer(mux)
	url = server.URL + "/events"

	srv.CreateStream("test")

	// Send continuous string of events to the client
	go func(s *Server) {
		for {
			if empty {
				s.Publish("test", &Event{Data: []byte("\n")})
			} else {
				s.Publish("test", &Event{Data: []byte("ping")})
			}
			time.Sleep(time.Millisecond * 50)
		}
	}(srv)
}

func cleanup() {
	server.CloseClientConnections()
	server.Close()
	srv.Close()
}

func TestClientSubscribe(t *testing.T) {
	setup(false)
	defer cleanup()

	c := NewClient(url)

	events := make(chan *Event)
	var cErr error
	go func() {
		cErr = c.Subscribe("test", func(msg *Event) {
			if msg.Data != nil {
				events <- msg
				return
			}
		})
	}()

	for i := 0; i < 5; i++ {
		msg, err := wait(events, time.Second*1)
		require.Nil(t, err)
		assert.Equal(t, []byte(`ping`), msg)
	}

	assert.Nil(t, cErr)
}

func TestClientChanSubscribeEmptyMessage(t *testing.T) {
	setup(true)
	defer cleanup()

	c := NewClient(url)

	events := make(chan *Event)
	err := c.SubscribeChan("test", events)
	require.Nil(t, err)

	for i := 0; i < 5; i++ {
		_, err := waitEvent(events, time.Second)
		require.Nil(t, err)
	}
}

func TestClientChanSubscribe(t *testing.T) {
	setup(false)
	defer cleanup()

	c := NewClient(url)

	events := make(chan *Event)
	err := c.SubscribeChan("test", events)
	require.Nil(t, err)

	for i := 0; i < 5; i++ {
		msg, merr := wait(events, time.Second*1)
		if msg == nil {
			i--
			continue
		}
		assert.Nil(t, merr)
		assert.Equal(t, []byte(`ping`), msg)
	}
	c.Unsubscribe(events)
}

func TestClientOnDisconnect(t *testing.T) {
	setup(false)
	defer cleanup()

	c := NewClient(url)

	called := make(chan bool)
	c.OnDisconnect(func(client *Client) {
		called <- true
	})

	go c.Subscribe("test", func(msg *Event) {})

	time.Sleep(time.Second)
	server.CloseClientConnections()

	assert.True(t, <-called)
}

func TestClientChanReconnect(t *testing.T) {
	setup(false)
	defer cleanup()

	c := NewClient(url)

	events := make(chan *Event)
	err := c.SubscribeChan("test", events)
	require.Nil(t, err)

	for i := 0; i < 10; i++ {
		if i == 5 {
			// kill connection
			server.CloseClientConnections()
		}
		msg, merr := wait(events, time.Second*1)
		if msg == nil {
			i--
			continue
		}
		assert.Nil(t, merr)
		assert.Equal(t, []byte(`ping`), msg)
	}
	c.Unsubscribe(events)
}

func TestClientUnsubscribe(t *testing.T) {
	setup(false)
	defer cleanup()

	c := NewClient(url)

	events := make(chan *Event)
	err := c.SubscribeChan("test", events)
	require.Nil(t, err)

	time.Sleep(time.Millisecond * 500)

	go c.Unsubscribe(events)
	go c.Unsubscribe(events)
}
<commit_msg>Test for unsubscribe blocking<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package sse

import (
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var url string
var srv *Server
var server *httptest.Server

func setup(empty bool) {
	// New Server
	srv = newServer()
	// Send almost-continuous string of events to the client
	go publishMsgs(srv, empty, 100000000)
}

func setupCount(empty bool, count int) {
	srv = newServer()
	go publishMsgs(srv, empty, count)
}

func newServer() *Server {
	srv = New()

	mux := http.NewServeMux()
	mux.HandleFunc("/events", srv.HTTPHandler)
	server = httptest.NewServer(mux)
	url = server.URL + "/events"

	srv.CreateStream("test")

	return srv
}

func publishMsgs(s *Server, empty bool, count int) {
	for a := 0; a < count; a++ {
		if empty {
			s.Publish("test", &Event{Data: []byte("\n")})
		} else {
			s.Publish("test", &Event{Data: []byte("ping")})
		}
		time.Sleep(time.Millisecond * 50)
	}
}

func cleanup() {
	server.CloseClientConnections()
	server.Close()
	srv.Close()
}

func TestClientSubscribe(t *testing.T) {
	setup(false)
	defer cleanup()

	c := NewClient(url)

	events := make(chan *Event)
	var cErr error
	go func() {
		cErr = c.Subscribe("test", func(msg *Event) {
			if msg.Data != nil {
				events <- msg
				return
			}
		})
	}()

	for i := 0; i < 5; i++ {
		msg, err := wait(events, time.Second*1)
		require.Nil(t, err)
		assert.Equal(t, []byte(`ping`), msg)
	}

	assert.Nil(t, cErr)
}

func TestClientChanSubscribeEmptyMessage(t *testing.T) {
	setup(true)
	defer cleanup()

	c := NewClient(url)

	events := make(chan *Event)
	err := c.SubscribeChan("test", events)
	require.Nil(t, err)

	for i := 0; i < 5; i++ {
		_, err := waitEvent(events, time.Second)
		require.Nil(t, err)
	}
}

func TestClientChanSubscribe(t *testing.T) {
	setup(false)
	defer cleanup()

	c := NewClient(url)

	events := make(chan *Event)
	err := c.SubscribeChan("test", events)
	require.Nil(t, err)

	for i := 0; i < 5; i++ {
		msg, merr := wait(events, time.Second*1)
		if msg == nil {
			i--
			continue
		}
		assert.Nil(t, merr)
		assert.Equal(t, []byte(`ping`), msg)
	}
	c.Unsubscribe(events)
}

func TestClientOnDisconnect(t *testing.T) {
	setup(false)
	defer cleanup()

	c := NewClient(url)

	called := make(chan bool)
	c.OnDisconnect(func(client *Client) {
		called <- true
	})

	go c.Subscribe("test", func(msg *Event) {})

	time.Sleep(time.Second)
	server.CloseClientConnections()

	assert.True(t, <-called)
}

func TestClientChanReconnect(t *testing.T) {
	setup(false)
	defer cleanup()

	c := NewClient(url)

	events := make(chan *Event)
	err := c.SubscribeChan("test", events)
	require.Nil(t, err)

	for i := 0; i < 10; i++ {
		if i == 5 {
			// kill connection
			server.CloseClientConnections()
		}
		msg, merr := wait(events, time.Second*1)
		if msg == nil {
			i--
			continue
		}
		assert.Nil(t, merr)
		assert.Equal(t, []byte(`ping`), msg)
	}
	c.Unsubscribe(events)
}

func TestClientUnsubscribe(t *testing.T) {
	setup(false)
	defer cleanup()

	c := NewClient(url)

	events := make(chan *Event)
	err := c.SubscribeChan("test", events)
	require.Nil(t, err)

	time.Sleep(time.Millisecond * 500)

	go c.Unsubscribe(events)
	go c.Unsubscribe(events)
}

func TestClientUnsubscribeNonBlock(t *testing.T) {
	count := 2
	setupCount(false, count)
	defer cleanup()

	c := NewClient(url)

	events := make(chan *Event)
	err := c.SubscribeChan("test", events)
	require.Nil(t, err)

	// Read count messages from the channel
	for i := 0; i < count; i++ {
		msg, merr := wait(events, time.Second*1)
		assert.Nil(t, merr)
		assert.Equal(t, []byte(`ping`), msg)
	}
	//No more data is available to be read in the channel
	// Make sure Unsubscribe returns quickly
	doneCh := make(chan *Event)
	go func() {
		var e Event
		c.Unsubscribe(events)
		doneCh <- &e
	}()
	_, merr := wait(doneCh, time.Millisecond*100)
	assert.Nil(t, merr)
}
<|endoftext|>


<commit_before>package twitch

import (
	"bufio"
	"net"
	"reflect"
	"strings"
	"testing"
	"time"
)

func TestCanCreateClient(t *testing.T) {
	client := NewClient("justinfan123123", "oauth:1123123")

	if reflect.TypeOf(*client) != reflect.TypeOf(Client{}) {
		t.Error("client is not of type Client")
	}
}

func TestCanConnectAndAuthenticate(t *testing.T) {
	var nicknameMsg string
	var oauthMsg string

	go func() {
		ln, _ := net.Listen("tcp", ":4321")
		conn, _ := ln.Accept()

		for {
			message, _ := bufio.NewReader(conn).ReadString('\n')
			message = strings.Replace(message, "\r\n", "", 1)
			if strings.HasPrefix(message, "NICK") {
				nicknameMsg = message
			}
			if strings.HasPrefix(message, "PASS") {
				oauthMsg = message
			}
			if nicknameMsg != "" && oauthMsg != "" {
				ln.Close()
			}
		}
	}()

	client := NewClient("justinfan123123", "oauth:123123132")
	client.SetIrcAddress("127.0.0.1:4321")
	go client.Connect()

	// wait for client to connect and server to read messages
	time.Sleep(time.Second)

	if nicknameMsg != "NICK justinfan123123" || oauthMsg != "PASS oauth:123123132" {
		t.Fatalf("invalid authentication data: username: %s, oauth: %s", nicknameMsg, oauthMsg)
	}
}
<commit_msg>change ip address<commit_after>package twitch

import (
	"bufio"
	"net"
	"reflect"
	"strings"
	"testing"
	"time"
)

func TestCanCreateClient(t *testing.T) {
	client := NewClient("justinfan123123", "oauth:1123123")

	if reflect.TypeOf(*client) != reflect.TypeOf(Client{}) {
		t.Error("client is not of type Client")
	}
}

func TestCanConnectAndAuthenticate(t *testing.T) {
	var nicknameMsg string
	var oauthMsg string

	go func() {
		ln, _ := net.Listen("tcp", ":4321")
		conn, _ := ln.Accept()

		for {
			message, _ := bufio.NewReader(conn).ReadString('\n')
			message = strings.Replace(message, "\r\n", "", 1)
			if strings.HasPrefix(message, "NICK") {
				nicknameMsg = message
			}
			if strings.HasPrefix(message, "PASS") {
				oauthMsg = message
			}
			if nicknameMsg != "" && oauthMsg != "" {
				ln.Close()
			}
		}
	}()

	client := NewClient("justinfan123123", "oauth:123123132")
	client.SetIrcAddress(":4321")
	go client.Connect()

	// wait for client to connect and server to read messages
	time.Sleep(time.Second)

	if nicknameMsg != "NICK justinfan123123" || oauthMsg != "PASS oauth:123123132" {
		t.Fatalf("invalid authentication data: username: %s, oauth: %s", nicknameMsg, oauthMsg)
	}
}
<|endoftext|>


<commit_before>package main

import "fmt"

type versionCommand struct{}

func (cmd *versionCommand) Execute(args []string) error {
	return runInContext(func(current *executionContext) error {
		version := buildVersion
		if version == "" {
			version = "0.0.0-development"
		}

		fmt.Printf("%s %s\n", ProjectId, version)
		return nil
	})
}

func init() {
	commandParser.AddCommand("version", "", "", &versionCommand{})
}
<commit_msg>remove old version cmd<commit_after><|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/develed/develed/slackbot"
	"github.com/nlopes/slack"
)

func main() {
	bot := slackbot.New(os.Getenv("SLACK_BOT_TOKEN"))

	bot.DefaultResponse(func(b *slackbot.Bot, msg *slack.Msg) {
		bot.Message(msg.Channel, "Non ho capito")
	})

	bot.RespondTo("ciao", func(b *slackbot.Bot, msg *slack.Msg, args ...string) {
		bot.Message(msg.Channel, "Olà!")
	})

	bot.RespondTo("echo (.*)", func(b *slackbot.Bot, msg *slack.Msg, args ...string) {
		bot.Message(msg.Channel, "Hai scritto: "+args[1])
	})

	bot.Start()
}
<commit_msg>bot: support for textd functionalities<commit_after>package main

import (
	"flag"
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/develed/develed/config"
	srv "github.com/develed/develed/services"
	"github.com/develed/develed/slackbot"
	"github.com/nlopes/slack"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

var (
	cfg = flag.String("config", "/etc/develed.toml", "configuration file")
)

func main() {
	flag.Parse()

	conf, err := config.Load(*cfg)
	if err != nil {
		log.Fatalln(err)
	}

	bot := slackbot.New(os.Getenv("SLACK_BOT_TOKEN"))

	conn, err := grpc.Dial(conf.Textd.GRPCServerAddress, grpc.WithInsecure())
	if err != nil {
		log.Fatalln(err)
	}
	defer conn.Close()

	textd := srv.NewTextdClient(conn)

	bot.DefaultResponse(func(b *slackbot.Bot, msg *slack.Msg) {
		bot.Message(msg.Channel, "Non ho capito")
	})

	bot.RespondTo("scrivi (.*)", func(b *slackbot.Bot, msg *slack.Msg, args ...string) {
		text := args[1]

		_, err := textd.Write(context.Background(), &srv.TextRequest{
			Text: text,
		})
		if err != nil {
			log.Errorln(err)
		} else {
			bot.Message(msg.Channel, "Hai scritto: "+text)
		}
	})

	bot.Start()
}
<|endoftext|>