<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/cenkalti/log"
	"github.com/cheggaaa/pb"
        "github.com/urfave/cli"
	dupes "github.com/danmarg/undupes/libdupes"
	"github.com/dustin/go-humanize"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// Horrible hack to convert -v to log Levels.
var orderedLevels []log.Level = []log.Level{log.DEBUG, log.INFO, log.NOTICE, log.WARNING, log.ERROR, log.CRITICAL}

func setLogLevel(l int) error {
	if l >= 0 && l < len(orderedLevels) {
		log.SetLevel(orderedLevels[l])
		return nil
	}
	return fmt.Errorf("invalid log level specified")
}

func main() {
	app := cli.NewApp()
	app.Name = "undupes"
	app.Usage = "manage duplicate files"
	app.Version = "0.1"
	app.Commands = []*cli.Command{
		&cli.Command{
			Name:      "interactive",
			Aliases: []string{"i"},
			Usage:     "interactive mode",
			Flags: []cli.Flag{
				&cli.StringSliceFlag{
					Name:  "directory, d",
					Usage: "directory in which to find duplicates (required)",
				},
				&cli.IntFlag{
					Name:  "v",
					Usage: "log level",
					Value: 3, // Don't print as much in interactive mode.
				},
			},
			Action: func(c *cli.Context) error {
				if err := setLogLevel(c.Int("v")); err != nil {
					return err
				}
				if err := runInteractive(c.Bool("dry_run"), c.StringSlice("directory")); err != nil {
                                        return err
				}
                                return nil
			},
		},
		{
			Name:      "print",
			Aliases: []string{"p"},
			Usage:     "print duplicates",
			Flags: []cli.Flag{
				&cli.StringSliceFlag{
					Name:  "directory, d",
					Usage: "directory in which to find duplicates (required)",
				},
				&cli.StringFlag{
					Name:  "output, o",
					Usage: "output file",
				},
			},
			Action: func(c *cli.Context) error {
				if len(c.StringSlice("directory")) == 0 {
					return fmt.Errorf("--directory required")
				}
				if c.String("output") == "" {
					return fmt.Errorf("--output required")
				}
				return runPrint(c.StringSlice("directory"), c.String("output"))
			},
		},
		{
			Name:      "auto",
			Aliases: []string{"a"},
			Usage:     "automatically resolve duplicates",
			Flags: []cli.Flag{
				&cli.StringSliceFlag{
					Name:  "directory, d",
					Usage: "directory in which to find duplicates (required)",
				},
				&cli.StringFlag{
					Name:  "prefer, p",
					Usage: "prefer to keep files matching this pattern",
				},
				&cli.StringFlag{
					Name:  "over, o",
					Usage: "used with --prefer; will restrict preferred files to those where the duplicate matches --over",
				},
				&cli.BoolFlag{
					Name:  "invert, i",
					Usage: "invert matching logic; preferred files with be prioritized for deletion rather than retention",
				},
				&cli.BoolFlag{
					Name:  "dry_run",
					Usage: "simulate (log but don't delete files)",
				},
				&cli.IntFlag{
					Name:  "v",
					Usage: "log level",
					Value: 1,
				},
			},
			Action: func(c *cli.Context) error {
				// Check for required arguments.
				if len(c.StringSlice("directory")) == 0 {
					return fmt.Errorf("--directory is required")
				}
				if c.String("prefer") == "" {
					return fmt.Errorf("--prefer is required")
				}
				if err := setLogLevel(c.Int("v")); err != nil {
					return err
				}
				// Compile regexps.
				var prefer, over *regexp.Regexp
				var err error
				if prefer, err = regexp.Compile(c.String("prefer")); err != nil {
					return fmt.Errorf("invalid regexp specified for --prefer")
				}
				if c.String("over") != "" {
					if over, err = regexp.Compile(c.String("over")); err != nil {
						return fmt.Errorf("invalid regexp specified for --over")
					}
				}
				// Do deduplication.
				return runAutomatic(c.Bool("dry_run"), c.StringSlice("directory"), prefer, over, c.Bool("invert"))
			},
		},
	}
	app.RunAndExitOnError()
}

func remove(dryRun bool, file string) {
	if dryRun {
		log.Noticef("DRY RUN: delete %s", file)
	} else {
		if err := os.Remove(file); err != nil {
			log.Warningf("error deleting %n: %v", file, err)
		} else {
			log.Noticef("deleted %s", file)
		}
	}

}

func getInput(prompt string, validator func(string) bool) (string, error) {
	// Reader to read from user input.
	reader := bufio.NewReader(os.Stdin)
	var (
		val string
		err error
	)
	for {
		fmt.Printf(prompt)
		val, err = reader.ReadString('\n')
		if err != nil {
			return "", err
		}
		val = strings.TrimSpace(val)
		if validator(val) {
			break
		}
	}
	return val, nil
}

func getDupesAndPrintSummary(roots []string) ([]dupes.Info, error) {
	fmt.Printf("Indexing...")
	var b *pb.ProgressBar
	dupes, err := dupes.Dupes(roots, func(cur int, outof int) {
		if b == nil {
			b = pb.StartNew(outof)
		}
		b.Set(cur)
	})
	b.Finish()
	if err != nil {
		return dupes, err
	}
	fcount := 0
	tsize := int64(0)
	for _, i := range dupes {
		fcount += len(i.Names) - 1
		tsize += i.Size * int64(len(i.Names)-1)
	}

	fmt.Printf("\rFound %d sets of duplicate files\n", len(dupes))
	fmt.Printf("Total file count: %d\n", fcount)
	fmt.Printf("Total size used: %s\n", humanize.Bytes(uint64(tsize)))
	return dupes, err
}

func runInteractive(dryRun bool, roots []string) error {
	var err error
	if len(roots) == 0 {
		// Get parent dir.
                root, err := getInput("Enter parent directory to scan for duplicates in: ", func(f string) bool {
			i, err := os.Stat(f)
			if err != nil {
				return false
			}
			return i.IsDir()
		})
		if err != nil {
			return err
		}
                roots = []string{root}
	}
	dupes, err := getDupesAndPrintSummary(roots)
	if err != nil {
		return err
	}

	fmt.Printf("\nReviewing results:\nFor each duplicate fileset, select 'f' to delete all but the first file, 'a' to keep all files, or 'n' (e.g. 2) to delete all except the second file.\n")

	for i, dupe := range dupes {
		keep := -1
		names := ""
		for j, n := range dupe.Names {
			names += fmt.Sprintf("%d: %s\n", j+1, n)
		}
		_, err := getInput(fmt.Sprintf("\n%d of %d  %s:\n%s\nKeep [F]irst/[a]ll/[n]th? ", i+1, len(dupes), humanize.Bytes(uint64(dupe.Size*int64(len(dupe.Names)-1))), names), func(v string) bool {
			switch strings.ToLower(v) {
			case "f", "":
				keep = 0
				return true
			case "a":
				return true
			default:
				k, err := strconv.ParseInt(v, 10, 32)
				if err != nil {
					return false
				}
				if k < 1 || int(k) > len(dupe.Names) {
					return false
				}
				keep = int(k) - 1
				return true
			}
		})
		if err != nil {
			log.Fatalln(err)
		}
		if keep >= 0 {
			for i, n := range dupe.Names {
				if i != keep {
					remove(dryRun, n)
				}
			}
		}
	}
	return nil
}

func runPrint(roots []string, output string) error {
	dupes, err := getDupesAndPrintSummary(roots)
	if err != nil {
		return err
	}

	var f *os.File
	if output != "" {
		if f, err = os.Create(output); err != nil {
			return err
		}
	}
	defer func() {
		if f != nil {
			if err := f.Close(); err != nil {
				log.Warningln(err)
			}
		}
	}()
	for _, dupe := range dupes {
		l := fmt.Sprintf("%s * %d => %s\n", humanize.Bytes(uint64(dupe.Size)), len(dupe.Names), strings.Join(dupe.Names, ", "))
		if f != nil {
			f.Write([]byte(l))
		} else {
			fmt.Printf(l)
		}
	}
	return nil
}

func runAutomatic(dryRun bool, roots []string, prefer *regexp.Regexp, over *regexp.Regexp, invert bool) error {
	dupes, err := getDupesAndPrintSummary(roots)
	if err != nil {
		return err
	}

	for _, dupe := range dupes {
		p := make(map[string]struct{})
		o := make(map[string]struct{})
		for _, n := range dupe.Names {
			nb := []byte(n)
			pm := prefer.Match(nb)
			om := false
			if over != nil {
				om = over.Match(nb)
			}
			if pm && om {
				log.Warningf("both --prefer and --over matched %s", n)
			}
			if pm {
				p[n] = struct{}{}
			} else if om {
				o[n] = struct{}{}
			}
		}
		if len(p) > 0 { // If we found a preferred match.
			// Generate debug line.
			dbg := fmt.Sprintf("processing %s\n\tprefer: ", strings.Join(dupe.Names, ", "))
			for k := range p {
				dbg += k + ", "
			}
			if len(o) > 0 {
				dbg += "\n\tover: "
				for k := range o {
					dbg += k + ", "
				}
			}
			log.Debugf("%s", dbg)
			// Logic is here.
			if over != nil {
				if len(o) > 0 {
					// If prefer and over are both specified, and both match, remove the non-preferred matches.
					if invert {
						for n := range p {
							remove(dryRun, n)
						}
					} else {
						for n := range o {
							remove(dryRun, n)
						}
					}
				}
			} else {
				// If over is not specified, keep only the preferred, but (for the case of --invert) only when preferred is not everything.
				if len(p) < len(dupe.Names) {
					for _, n := range dupe.Names {
						if _, ok := p[n]; ok && invert || !ok && !invert {
							remove(dryRun, n)
						}
					}
				}

			}
		}
	}

	return nil
}
<commit_msg>Fix help flag typo.<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/cenkalti/log"
	"github.com/cheggaaa/pb"
        "github.com/urfave/cli"
	dupes "github.com/danmarg/undupes/libdupes"
	"github.com/dustin/go-humanize"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// Horrible hack to convert -v to log Levels.
var orderedLevels []log.Level = []log.Level{log.DEBUG, log.INFO, log.NOTICE, log.WARNING, log.ERROR, log.CRITICAL}

func setLogLevel(l int) error {
	if l >= 0 && l < len(orderedLevels) {
		log.SetLevel(orderedLevels[l])
		return nil
	}
	return fmt.Errorf("invalid log level specified")
}

func main() {
	app := cli.NewApp()
	app.Name = "undupes"
	app.Usage = "manage duplicate files"
	app.Version = "0.1"
	app.Commands = []*cli.Command{
		&cli.Command{
			Name:      "interactive",
			Aliases: []string{"i"},
			Usage:     "interactive mode",
			Flags: []cli.Flag{
				&cli.StringSliceFlag{
					Name:  "directory, d",
					Usage: "directory in which to find duplicates (required)",
				},
				&cli.IntFlag{
					Name:  "v",
					Usage: "log level",
					Value: 3, // Don't print as much in interactive mode.
				},
			},
			Action: func(c *cli.Context) error {
				if err := setLogLevel(c.Int("v")); err != nil {
					return err
				}
				if err := runInteractive(c.Bool("dry_run"), c.StringSlice("directory")); err != nil {
                                        return err
				}
                                return nil
			},
		},
		{
			Name:      "print",
			Aliases: []string{"p"},
			Usage:     "print duplicates",
			Flags: []cli.Flag{
				&cli.StringSliceFlag{
					Name:  "directory, d",
					Usage: "directory in which to find duplicates (required)",
				},
				&cli.StringFlag{
					Name:  "output, o",
					Usage: "output file",
				},
			},
			Action: func(c *cli.Context) error {
				if len(c.StringSlice("directory")) == 0 {
					return fmt.Errorf("--directory required")
				}
				if c.String("output") == "" {
					return fmt.Errorf("--output required")
				}
				return runPrint(c.StringSlice("directory"), c.String("output"))
			},
		},
		{
			Name:      "auto",
			Aliases: []string{"a"},
			Usage:     "automatically resolve duplicates",
			Flags: []cli.Flag{
				&cli.StringSliceFlag{
					Name:  "directory, d",
					Usage: "directory in which to find duplicates (required)",
				},
				&cli.StringFlag{
					Name:  "prefer, p",
					Usage: "prefer to keep files matching this pattern",
				},
				&cli.StringFlag{
					Name:  "over, o",
					Usage: "used with --prefer; will restrict preferred files to those where the duplicate matches --over",
				},
				&cli.BoolFlag{
					Name:  "invert, i",
					Usage: "invert matching logic; preferred files will be prioritized for deletion rather than retention",
				},
				&cli.BoolFlag{
					Name:  "dry_run",
					Usage: "simulate (log but don't delete files)",
				},
				&cli.IntFlag{
					Name:  "v",
					Usage: "log level",
					Value: 1,
				},
			},
			Action: func(c *cli.Context) error {
				// Check for required arguments.
				if len(c.StringSlice("directory")) == 0 {
					return fmt.Errorf("--directory is required")
				}
				if c.String("prefer") == "" {
					return fmt.Errorf("--prefer is required")
				}
				if err := setLogLevel(c.Int("v")); err != nil {
					return err
				}
				// Compile regexps.
				var prefer, over *regexp.Regexp
				var err error
				if prefer, err = regexp.Compile(c.String("prefer")); err != nil {
					return fmt.Errorf("invalid regexp specified for --prefer")
				}
				if c.String("over") != "" {
					if over, err = regexp.Compile(c.String("over")); err != nil {
						return fmt.Errorf("invalid regexp specified for --over")
					}
				}
				// Do deduplication.
				return runAutomatic(c.Bool("dry_run"), c.StringSlice("directory"), prefer, over, c.Bool("invert"))
			},
		},
	}
	app.RunAndExitOnError()
}

func remove(dryRun bool, file string) {
	if dryRun {
		log.Noticef("DRY RUN: delete %s", file)
	} else {
		if err := os.Remove(file); err != nil {
			log.Warningf("error deleting %n: %v", file, err)
		} else {
			log.Noticef("deleted %s", file)
		}
	}

}

func getInput(prompt string, validator func(string) bool) (string, error) {
	// Reader to read from user input.
	reader := bufio.NewReader(os.Stdin)
	var (
		val string
		err error
	)
	for {
		fmt.Printf(prompt)
		val, err = reader.ReadString('\n')
		if err != nil {
			return "", err
		}
		val = strings.TrimSpace(val)
		if validator(val) {
			break
		}
	}
	return val, nil
}

func getDupesAndPrintSummary(roots []string) ([]dupes.Info, error) {
	fmt.Printf("Indexing...")
	var b *pb.ProgressBar
	dupes, err := dupes.Dupes(roots, func(cur int, outof int) {
		if b == nil {
			b = pb.StartNew(outof)
		}
		b.Set(cur)
	})
	b.Finish()
	if err != nil {
		return dupes, err
	}
	fcount := 0
	tsize := int64(0)
	for _, i := range dupes {
		fcount += len(i.Names) - 1
		tsize += i.Size * int64(len(i.Names)-1)
	}

	fmt.Printf("\rFound %d sets of duplicate files\n", len(dupes))
	fmt.Printf("Total file count: %d\n", fcount)
	fmt.Printf("Total size used: %s\n", humanize.Bytes(uint64(tsize)))
	return dupes, err
}

func runInteractive(dryRun bool, roots []string) error {
	var err error
	if len(roots) == 0 {
		// Get parent dir.
                root, err := getInput("Enter parent directory to scan for duplicates in: ", func(f string) bool {
			i, err := os.Stat(f)
			if err != nil {
				return false
			}
			return i.IsDir()
		})
		if err != nil {
			return err
		}
                roots = []string{root}
	}
	dupes, err := getDupesAndPrintSummary(roots)
	if err != nil {
		return err
	}

	fmt.Printf("\nReviewing results:\nFor each duplicate fileset, select 'f' to delete all but the first file, 'a' to keep all files, or 'n' (e.g. 2) to delete all except the second file.\n")

	for i, dupe := range dupes {
		keep := -1
		names := ""
		for j, n := range dupe.Names {
			names += fmt.Sprintf("%d: %s\n", j+1, n)
		}
		_, err := getInput(fmt.Sprintf("\n%d of %d  %s:\n%s\nKeep [F]irst/[a]ll/[n]th? ", i+1, len(dupes), humanize.Bytes(uint64(dupe.Size*int64(len(dupe.Names)-1))), names), func(v string) bool {
			switch strings.ToLower(v) {
			case "f", "":
				keep = 0
				return true
			case "a":
				return true
			default:
				k, err := strconv.ParseInt(v, 10, 32)
				if err != nil {
					return false
				}
				if k < 1 || int(k) > len(dupe.Names) {
					return false
				}
				keep = int(k) - 1
				return true
			}
		})
		if err != nil {
			log.Fatalln(err)
		}
		if keep >= 0 {
			for i, n := range dupe.Names {
				if i != keep {
					remove(dryRun, n)
				}
			}
		}
	}
	return nil
}

func runPrint(roots []string, output string) error {
	dupes, err := getDupesAndPrintSummary(roots)
	if err != nil {
		return err
	}

	var f *os.File
	if output != "" {
		if f, err = os.Create(output); err != nil {
			return err
		}
	}
	defer func() {
		if f != nil {
			if err := f.Close(); err != nil {
				log.Warningln(err)
			}
		}
	}()
	for _, dupe := range dupes {
		l := fmt.Sprintf("%s * %d => %s\n", humanize.Bytes(uint64(dupe.Size)), len(dupe.Names), strings.Join(dupe.Names, ", "))
		if f != nil {
			f.Write([]byte(l))
		} else {
			fmt.Printf(l)
		}
	}
	return nil
}

func runAutomatic(dryRun bool, roots []string, prefer *regexp.Regexp, over *regexp.Regexp, invert bool) error {
	dupes, err := getDupesAndPrintSummary(roots)
	if err != nil {
		return err
	}

	for _, dupe := range dupes {
		p := make(map[string]struct{})
		o := make(map[string]struct{})
		for _, n := range dupe.Names {
			nb := []byte(n)
			pm := prefer.Match(nb)
			om := false
			if over != nil {
				om = over.Match(nb)
			}
			if pm && om {
				log.Warningf("both --prefer and --over matched %s", n)
			}
			if pm {
				p[n] = struct{}{}
			} else if om {
				o[n] = struct{}{}
			}
		}
		if len(p) > 0 { // If we found a preferred match.
			// Generate debug line.
			dbg := fmt.Sprintf("processing %s\n\tprefer: ", strings.Join(dupe.Names, ", "))
			for k := range p {
				dbg += k + ", "
			}
			if len(o) > 0 {
				dbg += "\n\tover: "
				for k := range o {
					dbg += k + ", "
				}
			}
			log.Debugf("%s", dbg)
			// Logic is here.
			if over != nil {
				if len(o) > 0 {
					// If prefer and over are both specified, and both match, remove the non-preferred matches.
					if invert {
						for n := range p {
							remove(dryRun, n)
						}
					} else {
						for n := range o {
							remove(dryRun, n)
						}
					}
				}
			} else {
				// If over is not specified, keep only the preferred, but (for the case of --invert) only when preferred is not everything.
				if len(p) < len(dupe.Names) {
					for _, n := range dupe.Names {
						if _, ok := p[n]; ok && invert || !ok && !invert {
							remove(dryRun, n)
						}
					}
				}

			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/gcfg"
	"compress/gzip"
	"flag"
	"fmt"
	"github.com/boredomist/mixport/exports"
	"github.com/boredomist/mixport/mixpanel"
	kinesis "github.com/sendgridlabs/go-kinesis"
	"io"
	"log"
	"os"
	"path"
	"runtime"
	"strings"
	"sync"
	"syscall"
	"time"
)

// Mixpanel API credentials, used by the configuration parser.
type mixpanelCredentials struct {
	Key    string
	Secret string
	Token  string
}

// Configuration options common to the file export streams (CSV and JSON)
type fileExportConfig struct {
	State     bool
	Gzip      bool
	Fifo      bool
	Directory string
}

// configFormat is the in-memory representation of the mixport configuration
// file.
//
// - `Product` is Mixpanel API credential information for each product that will be
//   exported.
// - `Kinesis` is access keys and configuration for Amazon Kinesis exporter.
// - `JSON` and `CSV` are the configuration setups for the `JSON` and `CSV`
//   exporters, respectively.
type configFormat struct {
	Product map[string]*mixpanelCredentials
	Kinesis struct {
		State     bool
		Keyid     string
		Secretkey string
		Stream    string
		Region    string
	}

	JSON fileExportConfig
	CSV  fileExportConfig
}

var (
	configFile  string
	dateString  string
	rangeString string
	maxProcs    int
)

func init() {
	const (
		confUsage  = "path to configuration file"
		dateUsage  = "date (YYYY/MM/DD) of data to pull, default is yesterday"
		rangeUsage = "date range (YYYY/MM/DD-YYYY/MM/DD) of data to pull."
		procUsage  = "maximum number of OS threads to spawn. These will be IO bound."
	)

	// TODO: Tune this.
	defaultProcs := runtime.NumCPU() * 4
	defaultConfig := "./mixport.conf"

	flag.StringVar(&configFile, "config", defaultConfig, confUsage)
	flag.StringVar(&configFile, "c", defaultConfig, confUsage)
	flag.StringVar(&dateString, "date", "", dateUsage)
	flag.StringVar(&dateString, "d", "", dateUsage)
	flag.StringVar(&rangeString, "range", "", rangeUsage)
	flag.StringVar(&rangeString, "r", "", rangeUsage)
	flag.IntVar(&maxProcs, "procs", defaultProcs, procUsage)
	flag.IntVar(&maxProcs, "p", defaultProcs, procUsage)
}

var cfg = configFormat{}

func main() {
	flag.Parse()

	runtime.GOMAXPROCS(maxProcs)

	if err := gcfg.ReadFileInto(&cfg, configFile); err != nil {
		log.Fatalf("Failed to load %s: %s", configFile, err)
	}

	var exportStart, exportEnd time.Time

	// Default to yesterday (should be newest available data)
	if dateString == "" {
		year, month, day := time.Now().UTC().AddDate(0, 0, -1).Date()

		exportStart = time.Date(year, month, day, 0, 0, 0, 0, time.UTC)
		exportEnd = exportStart
	} else {
		if d, err := time.Parse("2006/01/02", dateString); err != nil {
			log.Fatalf("Invalid date: %s, should be in YYYY/MM/DD format",
				dateString)
		} else {
			exportStart = d
			exportEnd = exportStart
		}
	}

	if rangeString != "" {
		formatError := func() {
			log.Fatalf("Invalid range: %s, should be YYYY/MM/DD-YYYY/MM/DD.", rangeString)
		}

		parts := strings.Split(rangeString, "-")

		if len(parts) != 2 {
			formatError()
		}

		var err error

		if exportStart, err = time.Parse("2006/01/02", parts[0]); err != nil {
			formatError()
		}

		if exportEnd, err = time.Parse("2006/01/02", parts[1]); err != nil {
			formatError()
		}

	}

	// WaitGroup will hold the process open until all of the child
	// goroutines have completed execution.
	var wg sync.WaitGroup
	wg.Add(len(cfg.Product))

	for product, creds := range cfg.Product {
		// Run each individual product in a new thread.
		go exportProduct(exportStart, exportEnd, product, *creds, &wg)
	}

	// Wait for all our goroutines to finish up
	wg.Wait()
}

func exportProduct(start, end time.Time, product string, creds mixpanelCredentials, wg *sync.WaitGroup) {
	defer wg.Done()

	client := mixpanel.New(product, creds.Key, creds.Secret)
	eventData := make(chan mixpanel.EventData)

	// We need to mux eventData into multiple channels to
	// ensure all export funcs have a chance to see each event
	// instance.
	var chans []chan mixpanel.EventData

	if cfg.Kinesis.State {
		ch := make(chan mixpanel.EventData)
		chans = append(chans, ch)

		ksis := kinesis.New(cfg.Kinesis.Keyid, cfg.Kinesis.Secretkey)
		if cfg.Kinesis.Region != "" {
			ksis.Region = "us-east-1"
		}

		wg.Add(1)
		go func() {
			exports.KinesisStreamer(ksis, cfg.Kinesis.Stream, ch)
			defer wg.Done()
		}()
	}

	type streamFunc func(io.Writer, <-chan mixpanel.EventData)

	// Generalize setup of JSON and CSV streams into a single function.
	setupFileExportStream := func(conf fileExportConfig, ext string, streamer streamFunc) {
		wg.Add(1)
		defer wg.Done()

		ch := make(chan mixpanel.EventData)
		chans = append(chans, ch)

		if conf.Gzip {
			ext += ".gz"
		}

		timeFmt := "20060102"
		stamp := start.Format(timeFmt)

		// Append end date to timestamp if we're using a date range.
		if start != end {
			stamp += fmt.Sprintf("-%s", end.Format(timeFmt))
		}

		name := path.Join(conf.Directory, fmt.Sprintf("%s-%s.%s", product, stamp, ext))

		if conf.Fifo {
			if err := syscall.Mkfifo(name, syscall.S_IRWXU); err != nil {
				log.Fatalf("Couldn't create named pipe: %s", err)
			}
		}

		fp, err := os.Create(name)
		if err != nil {
			log.Fatalf("Couldn't create file: %s", err)
		}

		if conf.Fifo {
			defer os.Remove(name)
		}
		defer fp.Close()

		writer := (io.Writer)(fp)
		if conf.Gzip {
			writer = gzip.NewWriter(fp)
			defer (writer).(*gzip.Writer).Close()
		} else {
			writer = fp
		}

		streamer(writer, ch)
	}

	if cfg.JSON.State {
		go setupFileExportStream(cfg.JSON, "json", exports.JSONStreamer)
	}

	if cfg.CSV.State {
		go setupFileExportStream(cfg.CSV, "csv", exports.CSVStreamer)
	}

	go func() {
		defer close(eventData)

		// We want it to be start-end inclusive, so add one day to end date.
		end = end.AddDate(0, 0, 1)

		for date := start; date.Before(end); date = date.AddDate(0, 0, 1) {
			client.ExportDate(date, eventData, nil)
		}
	}()

	// Multiplex each received event to each of the active export
	// functions.
	for data := range eventData {
		for _, ch := range chans {
			ch <- data
		}
	}

	// Closing all the channels will signal the streaming export functions
	// that they've reached the end of the stream and should terminate as
	// soon as the channel is drained.
	for _, ch := range chans {
		close(ch)
	}
}
<commit_msg>Add -prof flag to generate pprof data<commit_after>package main

import (
	"code.google.com/p/gcfg"
	"compress/gzip"
	"flag"
	"fmt"
	"github.com/boredomist/mixport/exports"
	"github.com/boredomist/mixport/mixpanel"
	kinesis "github.com/sendgridlabs/go-kinesis"
	"io"
	"log"
	"os"
	"path"
	"runtime"
	"runtime/pprof"
	"strings"
	"sync"
	"syscall"
	"time"
)

// Mixpanel API credentials, used by the configuration parser.
type mixpanelCredentials struct {
	Key    string
	Secret string
	Token  string
}

// Configuration options common to the file export streams (CSV and JSON)
type fileExportConfig struct {
	State     bool
	Gzip      bool
	Fifo      bool
	Directory string
}

// configFormat is the in-memory representation of the mixport configuration
// file.
//
// - `Product` is Mixpanel API credential information for each product that will be
//   exported.
// - `Kinesis` is access keys and configuration for Amazon Kinesis exporter.
// - `JSON` and `CSV` are the configuration setups for the `JSON` and `CSV`
//   exporters, respectively.
type configFormat struct {
	Product map[string]*mixpanelCredentials
	Kinesis struct {
		State     bool
		Keyid     string
		Secretkey string
		Stream    string
		Region    string
	}

	JSON fileExportConfig
	CSV  fileExportConfig
}

var (
	configFile  string
	dateString  string
	rangeString string
	cpuProfile  *string
	maxProcs    int
)

func init() {
	const (
		confUsage  = "path to configuration file"
		dateUsage  = "date (YYYY/MM/DD) of data to pull, default is yesterday"
		rangeUsage = "date range (YYYY/MM/DD-YYYY/MM/DD) of data to pull."
		procUsage  = "maximum number of OS threads to spawn. These will be IO bound."
	)

	// TODO: Tune this.
	defaultProcs := runtime.NumCPU() * 4
	defaultConfig := "./mixport.conf"

	flag.StringVar(&configFile, "config", defaultConfig, confUsage)
	flag.StringVar(&configFile, "c", defaultConfig, confUsage)
	flag.StringVar(&dateString, "date", "", dateUsage)
	flag.StringVar(&dateString, "d", "", dateUsage)
	flag.StringVar(&rangeString, "range", "", rangeUsage)
	flag.StringVar(&rangeString, "r", "", rangeUsage)
	flag.IntVar(&maxProcs, "procs", defaultProcs, procUsage)
	flag.IntVar(&maxProcs, "p", defaultProcs, procUsage)

	cpuProfile = flag.String("prof", "", "dump pprof info to a file.")
}

var cfg = configFormat{}

func main() {
	flag.Parse()

	runtime.GOMAXPROCS(maxProcs)

	if *cpuProfile != "" {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if err := gcfg.ReadFileInto(&cfg, configFile); err != nil {
		log.Fatalf("Failed to load %s: %s", configFile, err)
	}

	var exportStart, exportEnd time.Time

	// Default to yesterday (should be newest available data)
	if dateString == "" {
		year, month, day := time.Now().UTC().AddDate(0, 0, -1).Date()

		exportStart = time.Date(year, month, day, 0, 0, 0, 0, time.UTC)
		exportEnd = exportStart
	} else {
		if d, err := time.Parse("2006/01/02", dateString); err != nil {
			log.Fatalf("Invalid date: %s, should be in YYYY/MM/DD format",
				dateString)
		} else {
			exportStart = d
			exportEnd = exportStart
		}
	}

	if rangeString != "" {
		formatError := func() {
			log.Fatalf("Invalid range: %s, should be YYYY/MM/DD-YYYY/MM/DD.", rangeString)
		}

		parts := strings.Split(rangeString, "-")

		if len(parts) != 2 {
			formatError()
		}

		var err error

		if exportStart, err = time.Parse("2006/01/02", parts[0]); err != nil {
			formatError()
		}

		if exportEnd, err = time.Parse("2006/01/02", parts[1]); err != nil {
			formatError()
		}

	}

	// WaitGroup will hold the process open until all of the child
	// goroutines have completed execution.
	var wg sync.WaitGroup
	wg.Add(len(cfg.Product))

	for product, creds := range cfg.Product {
		// Run each individual product in a new thread.
		go exportProduct(exportStart, exportEnd, product, *creds, &wg)
	}

	// Wait for all our goroutines to finish up
	wg.Wait()
}

func exportProduct(start, end time.Time, product string, creds mixpanelCredentials, wg *sync.WaitGroup) {
	defer wg.Done()

	client := mixpanel.New(product, creds.Key, creds.Secret)
	eventData := make(chan mixpanel.EventData)

	// We need to mux eventData into multiple channels to
	// ensure all export funcs have a chance to see each event
	// instance.
	var chans []chan mixpanel.EventData

	if cfg.Kinesis.State {
		ch := make(chan mixpanel.EventData)
		chans = append(chans, ch)

		ksis := kinesis.New(cfg.Kinesis.Keyid, cfg.Kinesis.Secretkey)
		if cfg.Kinesis.Region != "" {
			ksis.Region = "us-east-1"
		}

		wg.Add(1)
		go func() {
			exports.KinesisStreamer(ksis, cfg.Kinesis.Stream, ch)
			defer wg.Done()
		}()
	}

	type streamFunc func(io.Writer, <-chan mixpanel.EventData)

	// Generalize setup of JSON and CSV streams into a single function.
	setupFileExportStream := func(conf fileExportConfig, ext string, streamer streamFunc) {
		wg.Add(1)
		defer wg.Done()

		ch := make(chan mixpanel.EventData)
		chans = append(chans, ch)

		if conf.Gzip {
			ext += ".gz"
		}

		timeFmt := "20060102"
		stamp := start.Format(timeFmt)

		// Append end date to timestamp if we're using a date range.
		if start != end {
			stamp += fmt.Sprintf("-%s", end.Format(timeFmt))
		}

		name := path.Join(conf.Directory, fmt.Sprintf("%s-%s.%s", product, stamp, ext))

		if conf.Fifo {
			if err := syscall.Mkfifo(name, syscall.S_IRWXU); err != nil {
				log.Fatalf("Couldn't create named pipe: %s", err)
			}
		}

		fp, err := os.Create(name)
		if err != nil {
			log.Fatalf("Couldn't create file: %s", err)
		}

		if conf.Fifo {
			defer os.Remove(name)
		}
		defer fp.Close()

		writer := (io.Writer)(fp)
		if conf.Gzip {
			writer = gzip.NewWriter(fp)
			defer (writer).(*gzip.Writer).Close()
		} else {
			writer = fp
		}

		streamer(writer, ch)
	}

	if cfg.JSON.State {
		go setupFileExportStream(cfg.JSON, "json", exports.JSONStreamer)
	}

	if cfg.CSV.State {
		go setupFileExportStream(cfg.CSV, "csv", exports.CSVStreamer)
	}

	go func() {
		defer close(eventData)

		// We want it to be start-end inclusive, so add one day to end date.
		end = end.AddDate(0, 0, 1)

		for date := start; date.Before(end); date = date.AddDate(0, 0, 1) {
			client.ExportDate(date, eventData, nil)
		}
	}()

	// Multiplex each received event to each of the active export
	// functions.
	for data := range eventData {
		for _, ch := range chans {
			ch <- data
		}
	}

	// Closing all the channels will signal the streaming export functions
	// that they've reached the end of the stream and should terminate as
	// soon as the channel is drained.
	for _, ch := range chans {
		close(ch)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"net"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/miekg/dns"
)

func rrsearch(ns string, q string, t uint16) *dns.Msg {
	c := new(dns.Client)
	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(q), t)

	r, _, err := c.Exchange(m, net.JoinHostPort(ns, "53"))

	if err != nil {
		fmt.Println(err.Error())
		panic("rrsearch Error")
	}
	return r
}

func splitRR(rr dns.RR) []string {
	rrary := strings.SplitN(rr.String(), "\t", 5)
	return rrary
}

func setNS(rrs []string, r *dns.Msg, qns string) (string, string) {
	var ns string
	typ := "A"

	if strings.Contains(rrs[4], rrs[0]) || strings.Compare(qns, "202.12.27.33") == 0 {
		for _, rr := range r.Extra {
			rrss := splitRR(rr)
			if strings.Compare(rrss[0], rrs[4]) == 0 {
				ns = rrss[4]
				typ = rrss[3]
			}
		}
	} else {
		ns = rrs[4]
	}

	return ns, typ
}

func noRecRsolve(dst, ns string, n int) string {
	var tab string
	for i := 0; i < n; i++ {
		tab = tab + "\t"
	}
	rand.Seed(time.Now().UnixNano())
	for {
		r := rrsearch(ns, dst, dns.TypeA)
		if len(r.Answer) == 0 {
			rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
			nss, typ := setNS(rrs, r, ns)
			for strings.Compare(typ, "AAAA") == 0 {
				rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
				nss, typ = setNS(rrs, r, ns)
			}
			fmt.Println(tab + ns + "=>")
			fmt.Printf(tab+"\t%s -> %s\n",
				rrs[0],
				rrs[4],
			)
			ns = nss
		} else {
			fmt.Println(tab + "Answer : " + ns + "=>")
			var ip string
			for _, ans := range r.Answer {
				anss := splitRR(ans)
				fmt.Printf(tab+"\t%s -> %s\n",
					dst,
					anss[4],
				)
				ip = anss[4]
			}
			return ip
		}
	}
}

func recRsolve(dst, ns string, n int) string {
	var tab string
	for i := 0; i < n; i++ {
		tab = tab + "\t"
	}
	rand.Seed(time.Now().UnixNano())
	for {
		checkNS, _ := regexp.MatchString("^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$", ns)
		if checkNS == false {
			ns = recRsolve(ns, "202.12.27.33", n+1)
		}
		r := rrsearch(ns, dst, dns.TypeA)
		if len(r.Answer) == 0 {
			rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
			nss, typ := setNS(rrs, r, ns)
			for strings.Compare(typ, "AAAA") == 0 {
				rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
				nss, typ = setNS(rrs, r, ns)
			}
			fmt.Println(tab + ns + "=>")
			fmt.Printf(tab+"\t%s -> %s\n",
				rrs[0],
				rrs[4],
			)
			ns = nss
		} else {
			fmt.Println(tab + "Answer : " + ns + "=>")
			var ip string
			for _, ans := range r.Answer {
				anss := splitRR(ans)
				fmt.Printf(tab+"\t%s -> %s\n",
					dst,
					anss[4],
				)
				ip = anss[4]
			}
			return ip
		}
	}
}

func main() {
	app := cli.NewApp()
	app.Usage = "It is a tool to see recursive search of the DNS is how to."
	app.Version = "0.0.1"

	app.Commands = []cli.Command{
		{
			Name:    "rec",
			Aliases: []string{"r"},
			Usage:   "Iterate Search",
			Action: func(c *cli.Context) {
				recRsolve(c.Args().First(), "202.12.27.33", 0)
			},
		},
		{
			Name:    "norec",
			Aliases: []string{"n"},
			Usage:   "No Iterate Search",
			Action: func(c *cli.Context) {
				noRecRsolve(c.Args().First(), "202.12.27.33", 0)
			},
		},
	}
	app.Run(os.Args)
}
<commit_msg>ルートDNSを指定するフラグを追加<commit_after>package main

import (
	"fmt"
	"math/rand"
	"net"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/miekg/dns"
)

const (
	Version = "0.0.1"
)

var (
	root string
)

func rrsearch(ns string, q string, t uint16) *dns.Msg {
	c := new(dns.Client)
	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(q), t)

	r, _, err := c.Exchange(m, net.JoinHostPort(ns, "53"))

	if err != nil {
		fmt.Println(err.Error())
		panic("rrsearch Error")
	}
	return r
}

func splitRR(rr dns.RR) []string {
	rrary := strings.SplitN(rr.String(), "\t", 5)
	return rrary
}

func setNS(rrs []string, r *dns.Msg, qns string) (string, string) {
	var ns string
	typ := "A"

	if strings.Contains(rrs[4], rrs[0]) || strings.Compare(qns, root) == 0 {
		for _, rr := range r.Extra {
			rrss := splitRR(rr)
			if strings.Compare(rrss[0], rrs[4]) == 0 {
				ns = rrss[4]
				typ = rrss[3]
			}
		}
	} else {
		ns = rrs[4]
	}

	return ns, typ
}

func noRecRsolve(dst, ns string) string {
	rand.Seed(time.Now().UnixNano())
	for {
		r := rrsearch(ns, dst, dns.TypeA)
		if len(r.Answer) == 0 {
			rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
			nss, typ := setNS(rrs, r, ns)
			for strings.Compare(typ, "AAAA") == 0 {
				rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
				nss, typ = setNS(rrs, r, ns)
			}
			fmt.Println(ns + "=>")
			fmt.Printf("\t%s -> %s\n",
				rrs[0],
				rrs[4],
			)
			ns = nss
		} else {
			fmt.Println("Answer : " + ns + "=>")
			var ip string
			for _, ans := range r.Answer {
				anss := splitRR(ans)
				fmt.Printf("\t%s -> %s\n",
					dst,
					anss[4],
				)
				ip = anss[4]
			}
			return ip
		}
	}
}

func recRsolve(dst, ns string, n int) string {
	var tab string
	for i := 0; i < n; i++ {
		tab = tab + "\t"
	}
	rand.Seed(time.Now().UnixNano())
	for {
		checkNS, _ := regexp.MatchString("^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$", ns)
		if checkNS == false {
			ns = recRsolve(ns, root, n+1)
		}
		r := rrsearch(ns, dst, dns.TypeA)
		if len(r.Answer) == 0 {
			rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
			nss, typ := setNS(rrs, r, ns)
			for strings.Compare(typ, "AAAA") == 0 {
				rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
				nss, typ = setNS(rrs, r, ns)
			}
			fmt.Println(tab + ns + "=>")
			fmt.Printf(tab+"\t%s -> %s\n",
				rrs[0],
				rrs[4],
			)
			ns = nss
		} else {
			fmt.Println(tab + "Answer : " + ns + "=>")
			var ip string
			for _, ans := range r.Answer {
				anss := splitRR(ans)
				fmt.Printf(tab+"\t%s -> %s\n",
					dst,
					anss[4],
				)
				ip = anss[4]
			}
			return ip
		}
	}
}

func main() {
	app := cli.NewApp()
	app.Usage = "It is a tool to see recursive search of the DNS is how to."
	app.Version = Version

	app.Commands = []cli.Command{
		{
			Name:    "rec",
			Aliases: []string{"r"},
			Usage:   "Iterate Search",
			Action: func(c *cli.Context) {
				recRsolve(c.Args().First(), root, 0)
			},
		},
		{
			Name:    "norec",
			Aliases: []string{"n"},
			Usage:   "No Iterate Search",
			Action: func(c *cli.Context) {
				noRecRsolve(c.Args().First(), root)
			},
		},
	}

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "root",
			Value:       "202.12.27.33",
			Usage:       "Root DNS Server's IP address",
			Destination: &root,
		},
	}
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>// Copyright 2016 bs authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"syscall"

	"github.com/google/gops/agent"
	"github.com/tsuru/bs/bslog"
	"github.com/tsuru/bs/config"
	"github.com/tsuru/bs/log"
	"github.com/tsuru/bs/metric"
	_ "github.com/tsuru/bs/metric/logstash"
	"github.com/tsuru/bs/status"
)

const (
	version = "v1.10-rc2"
)

var printVersion bool

type StopWaiter interface {
	Stop()
	Wait()
}

func init() {
	flag.BoolVar(&printVersion, "version", false, "Print version and exit")
}

func startSignalHandler(callback func(os.Signal), signals ...os.Signal) {
	sigChan := make(chan os.Signal, 4)
	go func() {
		if signal, ok := <-sigChan; ok {
			callback(signal)
		}
	}()
	signal.Notify(sigChan, signals...)
}

func main() {
	err := agent.Listen(&agent.Options{
		NoShutdownCleanup: true,
	})
	if err != nil {
		bslog.Fatalf("Unable to initialize gops agent: %s\n", err)
	}
	defer agent.Close()
	flag.Parse()
	if printVersion {
		fmt.Printf("bs version %s\n", version)
		return
	}
	lf := log.LogForwarder{
		BindAddress:     config.Config.SyslogListenAddress,
		DockerEndpoint:  config.Config.DockerEndpoint,
		EnabledBackends: config.Config.LogBackends,
	}
	err = lf.Start()
	if err != nil {
		bslog.Fatalf("Unable to initialize log forwarder: %s\n", err)
	}
	mRunner := metric.NewRunner(config.Config.DockerEndpoint, config.Config.MetricsInterval,
		config.Config.MetricsBackend)
	err = mRunner.Start()
	if err != nil {
		bslog.Warnf("Unable to initialize metrics runner: %s\n", err)
	}
	reporter, err := status.NewReporter(&status.ReporterConfig{
		TsuruEndpoint:  config.Config.TsuruEndpoint,
		TsuruToken:     config.Config.TsuruToken,
		DockerEndpoint: config.Config.DockerEndpoint,
		Interval:       config.Config.StatusInterval,
	})
	if err != nil {
		bslog.Warnf("Unable to initialize status reporter: %s\n", err)
	}
	monitorEl := []StopWaiter{&lf, mRunner}
	if reporter != nil {
		monitorEl = append(monitorEl, reporter)
	}
	var signaled bool
	startSignalHandler(func(signal os.Signal) {
		signaled = true
		for _, m := range monitorEl {
			go m.Stop()
		}
	}, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
	for _, m := range monitorEl {
		m.Wait()
	}
	if !signaled {
		bslog.Fatalf("Exiting bs because no service could be initialized.")
	}
}
<commit_msg>bump version to 1.10-rc4<commit_after>// Copyright 2016 bs authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"syscall"

	"github.com/google/gops/agent"
	"github.com/tsuru/bs/bslog"
	"github.com/tsuru/bs/config"
	"github.com/tsuru/bs/log"
	"github.com/tsuru/bs/metric"
	_ "github.com/tsuru/bs/metric/logstash"
	"github.com/tsuru/bs/status"
)

const (
	version = "v1.10-rc4"
)

var printVersion bool

type StopWaiter interface {
	Stop()
	Wait()
}

func init() {
	flag.BoolVar(&printVersion, "version", false, "Print version and exit")
}

func startSignalHandler(callback func(os.Signal), signals ...os.Signal) {
	sigChan := make(chan os.Signal, 4)
	go func() {
		if signal, ok := <-sigChan; ok {
			callback(signal)
		}
	}()
	signal.Notify(sigChan, signals...)
}

func main() {
	err := agent.Listen(&agent.Options{
		NoShutdownCleanup: true,
	})
	if err != nil {
		bslog.Fatalf("Unable to initialize gops agent: %s\n", err)
	}
	defer agent.Close()
	flag.Parse()
	if printVersion {
		fmt.Printf("bs version %s\n", version)
		return
	}
	lf := log.LogForwarder{
		BindAddress:     config.Config.SyslogListenAddress,
		DockerEndpoint:  config.Config.DockerEndpoint,
		EnabledBackends: config.Config.LogBackends,
	}
	err = lf.Start()
	if err != nil {
		bslog.Fatalf("Unable to initialize log forwarder: %s\n", err)
	}
	mRunner := metric.NewRunner(config.Config.DockerEndpoint, config.Config.MetricsInterval,
		config.Config.MetricsBackend)
	err = mRunner.Start()
	if err != nil {
		bslog.Warnf("Unable to initialize metrics runner: %s\n", err)
	}
	reporter, err := status.NewReporter(&status.ReporterConfig{
		TsuruEndpoint:  config.Config.TsuruEndpoint,
		TsuruToken:     config.Config.TsuruToken,
		DockerEndpoint: config.Config.DockerEndpoint,
		Interval:       config.Config.StatusInterval,
	})
	if err != nil {
		bslog.Warnf("Unable to initialize status reporter: %s\n", err)
	}
	monitorEl := []StopWaiter{&lf, mRunner}
	if reporter != nil {
		monitorEl = append(monitorEl, reporter)
	}
	var signaled bool
	startSignalHandler(func(signal os.Signal) {
		signaled = true
		for _, m := range monitorEl {
			go m.Stop()
		}
	}, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
	for _, m := range monitorEl {
		m.Wait()
	}
	if !signaled {
		bslog.Fatalf("Exiting bs because no service could be initialized.")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"io"
	"log"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/ewhal/pygments"
	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

const (
	ADDRESS  = "http://localhost:9900"
	LENGTH   = 6
	PORT     = ":9900"
	USERNAME = ""
	PASS     = ""
	NAME     = ""
	DATABASE = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

type Response struct {
	ID     string `json:"id"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}

func generateName() string {
	s := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	query, err := db.Query("select id from pastebin")
	for query.Next() {
		var id string
		err := query.Scan(&id)
		if err != nil {

		}
		if id == s {
			generateName()
		}
	}
	db.Close()

	return s

}
func hash(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

func save(raw string, lang string) []string {
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	sha := hash(raw)
	query, err := db.Query("select id, hash, data, delkey from pastebin")
	for query.Next() {
		var id, hash, paste, delkey string
		err := query.Scan(&id, &hash, &paste, &delkey)
		check(err)
		if hash == sha {
			url := ADDRESS + "/p/" + id
			return []string{id, hash, url, paste, delkey}
		}
	}
	id := generateName()
	var url string
	if lang == "" {
		url = ADDRESS + "/p/" + id
	} else {
		url = ADDRESS + "/p/" + id + "/" + lang
	}
	delKey := uniuri.NewLen(40)
	paste := html.EscapeString(raw)

	stmt, err := db.Prepare("INSERT INTO pastebin(id, hash, data, delkey) values(?,?,?,?)")
	check(err)
	_, err = stmt.Exec(id, sha, paste, delKey)
	check(err)
	db.Close()
	return []string{id, sha, url, paste, delKey}
}

func delHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	check(err)

	stmt, err := db.Prepare("delete from pastebin where delkey=?")
	check(err)

	res, err := stmt.Exec(html.EscapeString(delkey))
	check(err)

	_, err = res.RowsAffected()
	if err == sql.ErrNoRows {
		io.WriteString(w, "Error invalid paste")
	} else {
		io.WriteString(w, paste+" deleted")
	}
	db.Close()

}
func saveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		paste := r.FormValue("p")
		lang := r.FormValue("lang")
		if paste == "" {
			http.Error(w, "Empty paste", 500)
		}
		values := save(paste, lang)
		b := &Response{
			ID:     values[0],
			HASH:   values[1],
			URL:    values[2],
			SIZE:   len(values[3]),
			DELKEY: values[4],
		}

		switch output {
		case "json":

			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)
		case "html":
			w.Header().Set("Content-Type", "text/html")
			io.WriteString(w, "
URL: "+b.URL+"
")
			io.WriteString(w, "
Delete Key: "+b.DELKEY+"
")

		default:
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

func langHandler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]
	s := getPaste(paste)
	highlight, err := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "full, style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,", "utf-8")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	io.WriteString(w, highlight)

}

func getPaste(paste string) string {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	var s string
	err = db.QueryRow("select data from pastebin where id=?", param1).Scan(&s)
	db.Close()
	check(err)

	if err == sql.ErrNoRows {
		return "Error invalid paste"
	} else {
		return html.UnescapeString(s)
	}

}
func pasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s := getPaste(paste)
	io.WriteString(w, s)

}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/p/{pasteId}", pasteHandler)
	router.HandleFunc("/p/{pasteId}/{lang}", langHandler)
	router.HandleFunc("/save", saveHandler)
	router.HandleFunc("/save/{output}", saveHandler)
	router.HandleFunc("/del/{pasteId}/{delKey}", delHandler)
	router.PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.Dir("assets/"))))
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<commit_msg>Fix up empty paste bug<commit_after>package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"io"
	"log"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/ewhal/pygments"
	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

const (
	ADDRESS  = "http://localhost:9900"
	LENGTH   = 6
	PORT     = ":9900"
	USERNAME = ""
	PASS     = ""
	NAME     = ""
	DATABASE = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

type Response struct {
	ID     string `json:"id"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}

func generateName() string {
	s := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	query, err := db.Query("select id from pastebin")
	for query.Next() {
		var id string
		err := query.Scan(&id)
		if err != nil {

		}
		if id == s {
			generateName()
		}
	}
	db.Close()

	return s

}
func hash(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

func save(raw string, lang string) []string {
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	sha := hash(raw)
	query, err := db.Query("select id, hash, data, delkey from pastebin")
	for query.Next() {
		var id, hash, paste, delkey string
		err := query.Scan(&id, &hash, &paste, &delkey)
		check(err)
		if hash == sha {
			url := ADDRESS + "/p/" + id
			return []string{id, hash, url, paste, delkey}
		}
	}
	id := generateName()
	var url string
	if lang == "" {
		url = ADDRESS + "/p/" + id
	} else {
		url = ADDRESS + "/p/" + id + "/" + lang
	}
	delKey := uniuri.NewLen(40)
	paste := html.EscapeString(raw)

	stmt, err := db.Prepare("INSERT INTO pastebin(id, hash, data, delkey) values(?,?,?,?)")
	check(err)
	_, err = stmt.Exec(id, sha, paste, delKey)
	check(err)
	db.Close()
	return []string{id, sha, url, paste, delKey}
}

func delHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	check(err)

	stmt, err := db.Prepare("delete from pastebin where delkey=?")
	check(err)

	res, err := stmt.Exec(html.EscapeString(delkey))
	check(err)

	_, err = res.RowsAffected()
	if err == sql.ErrNoRows {
		io.WriteString(w, "Error invalid paste")
	} else {
		io.WriteString(w, paste+" deleted")
	}
	db.Close()

}
func saveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		paste := r.FormValue("p")
		lang := r.FormValue("lang")
		if paste == "" {
			http.Error(w, "Empty paste", 500)
			return
		}
		values := save(paste, lang)
		b := &Response{
			ID:     values[0],
			HASH:   values[1],
			URL:    values[2],
			SIZE:   len(values[3]),
			DELKEY: values[4],
		}

		switch output {
		case "json":

			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)
		case "html":
			w.Header().Set("Content-Type", "text/html")
			io.WriteString(w, "
URL: "+b.URL+"
")
			io.WriteString(w, "
Delete Key: "+b.DELKEY+"
")

		default:
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

func langHandler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]
	s := getPaste(paste)
	highlight, err := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "full, style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,", "utf-8")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	io.WriteString(w, highlight)

}

func getPaste(paste string) string {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	var s string
	err = db.QueryRow("select data from pastebin where id=?", param1).Scan(&s)
	db.Close()
	check(err)

	if err == sql.ErrNoRows {
		return "Error invalid paste"
	} else {
		return html.UnescapeString(s)
	}

}
func pasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s := getPaste(paste)
	io.WriteString(w, s)

}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/p/{pasteId}", pasteHandler)
	router.HandleFunc("/p/{pasteId}/{lang}", langHandler)
	router.HandleFunc("/save", saveHandler)
	router.HandleFunc("/save/{output}", saveHandler)
	router.HandleFunc("/del/{pasteId}/{delKey}", delHandler)
	router.PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.Dir("assets/"))))
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"runtime"

	"github.com/goldeneggg/ipcl/parser"
	"github.com/goldeneggg/ipcl/writer"
	"github.com/jessevdk/go-flags"
)

const (
	Version = "0.1.0"
)

// element names need to Uppercase
type options struct {
	Help    bool   `short:"h" long:"help" description:"Show help message"` // not "help" but "Help", because cause error using "-h" option
	File    string `short:"f" long:"file" description:"Filepath listed target CIDR"`
	IsCsv   bool   `short:"c" long:"csv" description:"Output format is csv"`
	IsTsv   bool   `short:"t" long:"tsv" description:"Output format is tsv"`
	Version bool   `short:"v" long:"version" description:"Print version"`
}

type optArgs struct {
	opts *options
	args []string
}

func main() {
	var status int
	// handler for return
	defer func() { os.Exit(status) }()

	// parse option args
	opts := &options{}
	parser := flags.NewParser(opts, flags.PrintErrors)
	args, err := parser.Parse()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		printHelp()
		status = 1
		return
	}

	// print help
	if opts.Help {
		printHelp()
		return
	}

	// print version
	if opts.Version {
		fmt.Fprintf(os.Stderr, "Ipcl: version %s (%s)\n", Version, runtime.GOARCH)
		return
	}

	// get source CIDRs
	oa := &optArgs{opts, args}
	cidrs, e := getCIDRs(oa)
	if e != nil {
		fmt.Printf("%s\n", e)
		status = 1
		return
	}

	// write
	write(cidrs, oa)
}

func getCIDRs(oa *optArgs) ([]parser.CIDRInfo, error) {
	var cidrStrs []string
	var cidrs []parser.CIDRInfo
	ac := len(oa.args)

	switch {
	case ac == 1:
		cidrStrs = append(cidrStrs, oa.args[0])
	case ac > 1:
		cidrStrs = oa.args
	case oa.opts.File != "":
		var e error
		cidrStrs, e = fromFile(oa)
		if e != nil {
			return cidrs, e
		}
	default:
		return cidrs, fmt.Errorf("Target CIDR(or CIDR list file) is not assigned\n")
	}

	for i, cs := range cidrStrs {
		c, e := parser.Parse(cs)
		if e != nil {
			fmt.Fprintf(os.Stderr, "CIDR string[%d] %s validate error: %s\n", i, cs, e)
		} else {
			cidrs = append(cidrs, c)
		}
	}

	return cidrs, nil
}

func fromFile(oa *optArgs) ([]string, error) {
	cidrs := make([]string, 0, 10)

	f, err := os.Open(oa.opts.File)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		return cidrs, err
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		cidrs = append(cidrs, scanner.Text())
	}
	if serr := scanner.Err(); serr != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		return cidrs, err
	}

	return cidrs, nil
}

func write(cidrs []parser.CIDRInfo, oa *optArgs) {
	w := writer.NewWriter(oa.opts.IsCsv, oa.opts.IsTsv)
	w.Write(cidrs)
}

func printHelp() {
	h := `
Usage:
  ipcl [OPTIONS] [CIDR TEXT]

Application Options:
  -f, --file=    Filepath listed target CIDR
  -c, --csv=     Output format is csv
  -t, --tsv=     Output format is tsv
  -v, --version  Print version

Help Options:
  -h, --help     Show this help message
`
	os.Stderr.Write([]byte(h))
}
<commit_msg>refactoring<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"runtime"

	"github.com/goldeneggg/ipcl/parser"
	"github.com/goldeneggg/ipcl/writer"
	"github.com/jessevdk/go-flags"
)

const (
	Version = "0.1.0"
)

// element names need to Uppercase
type options struct {
	Help    bool   `short:"h" long:"help" description:"Show help message"` // not "help" but "Help", because cause error using "-h" option
	File    string `short:"f" long:"file" description:"Filepath listed target CIDR"`
	IsCsv   bool   `short:"c" long:"csv" description:"Output format is csv"`
	IsTsv   bool   `short:"t" long:"tsv" description:"Output format is tsv"`
	Version bool   `short:"v" long:"version" description:"Print version"`
}

type optArgs struct {
	opts *options
	args []string
}

func main() {
	var status int
	// handler for return
	defer func() { os.Exit(status) }()

	// parse option args
	opts := &options{}
	parser := flags.NewParser(opts, flags.PrintErrors)
	args, err := parser.Parse()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		printHelp()
		status = 1
		return
	}

	// print help
	if opts.Help {
		printHelp()
		return
	}

	// print version
	if opts.Version {
		fmt.Fprintf(os.Stderr, "Ipcl: version %s (%s)\n", Version, runtime.GOARCH)
		return
	}

	// get source CIDRs
	oa := &optArgs{opts, args}
	cidrs, e := getCIDRs(oa)
	if e != nil {
		fmt.Printf("%s\n", e)
		printHelp()
		status = 1
		return
	}

	// write
	write(cidrs, oa)
}

func getCIDRs(oa *optArgs) ([]parser.CIDRInfo, error) {
	var cidrStrs []string
	var cidrs []parser.CIDRInfo
	ac := len(oa.args)

	switch {
	case ac == 1:
		cidrStrs = append(cidrStrs, oa.args[0])
	case ac > 1:
		cidrStrs = oa.args
	case oa.opts.File != "":
		var e error
		cidrStrs, e = fromFile(oa)
		if e != nil {
			return cidrs, e
		}
	default:
		return cidrs, fmt.Errorf("Target CIDR(or CIDR list file) is not assigned\n")
	}

	for i, cs := range cidrStrs {
		c, e := parser.Parse(cs)
		if e != nil {
			fmt.Fprintf(os.Stderr, "CIDR string[%d] %s validate error: %s\n", i, cs, e)
		} else {
			cidrs = append(cidrs, c)
		}
	}

	return cidrs, nil
}

func fromFile(oa *optArgs) ([]string, error) {
	cidrs := make([]string, 0, 10)

	f, err := os.Open(oa.opts.File)
	if err != nil {
		return cidrs, err
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		cidrs = append(cidrs, scanner.Text())
	}
	if serr := scanner.Err(); serr != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		return cidrs, err
	}

	return cidrs, nil
}

func write(cidrs []parser.CIDRInfo, oa *optArgs) {
	w := writer.NewWriter(oa.opts.IsCsv, oa.opts.IsTsv)
	w.Write(cidrs)
}

func printHelp() {
	h := `
Usage:
  ipcl [OPTIONS] <CIDR TEXT | -f <FILE>>

Application Options:
  -f, --file=    Filepath listed target CIDR
  -c, --csv=     Output format is csv
  -t, --tsv=     Output format is tsv
  -v, --version  Print version

Help Options:
  -h, --help     Show this help message
`
	os.Stderr.Write([]byte(h))
}
<|endoftext|>


<commit_before>package mock_godless

import (
	"reflect"
	"testing"

	"github.com/golang/mock/gomock"
	lib "github.com/johnny-morrice/godless"
	"github.com/pkg/errors"
)

func TestRunQueryReadSuccess(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	mock := NewMockKvNamespace(ctrl)
	query := &lib.Query{
		OpCode:   lib.SELECT,
		TableKey: "Table Key",
		Select: lib.QuerySelect{
			Limit: 1,
			Where: lib.QueryWhere{
				OpCode: lib.PREDICATE,
				Predicate: lib.QueryPredicate{
					OpCode:   lib.STR_EQ,
					Literals: []string{"Hi"},
					Keys:     []string{"Entry A"},
				},
			},
		},
	}

	mock.EXPECT().IsChanged().Return(false)
	mock.EXPECT().RunKvQuery(mtchkvqq(query))

	api, errch := lib.LaunchKeyValueStore(mock)
	resp, err := api.RunQuery(query)

	if err != nil {
		t.Error(err)
	}

	if resp == nil {
		t.Error("Response channel was nil")
	}

	api.Close()

	for err := range errch {
		t.Error(err)
	}
}

func TestRunQueryWriteSuccess(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	mock := NewMockKvNamespace(ctrl)
	query := &lib.Query{
		OpCode:   lib.JOIN,
		TableKey: "Table Key",
		Join: lib.QueryJoin{
			Rows: []lib.QueryRowJoin{
				lib.QueryRowJoin{
					RowKey: "Row thing",
					Entries: map[string]string{
						"Hello": "world",
					},
				},
			},
		},
	}

	mock.EXPECT().IsChanged().Return(true)
	mock.EXPECT().RunKvQuery(mtchkvqq(query))
	mock.EXPECT().Persist().Return(mock, nil)

	api, errch := lib.LaunchKeyValueStore(mock)
	resp, err := api.RunQuery(query)

	if err != nil {
		t.Error(err)
	}

	if resp == nil {
		t.Error("Response channel was nil")
	}

	api.Close()

	for err := range errch {
		t.Error(err)
	}
}

func TestRunQueryWriteFailure(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	mock := NewMockKvNamespace(ctrl)
	query := &lib.Query{
		OpCode:   lib.JOIN,
		TableKey: "Table Key",
		Join: lib.QueryJoin{
			Rows: []lib.QueryRowJoin{
				lib.QueryRowJoin{
					RowKey: "Row thing",
					Entries: map[string]string{
						"Hello": "world",
					},
				},
			},
		},
	}

	mock.EXPECT().IsChanged().Return(true)
	mock.EXPECT().RunKvQuery(mtchkvqq(query))
	mock.EXPECT().Persist().Return(nil, errors.New("Expected error"))

	api, errch := lib.LaunchKeyValueStore(mock)
	resp, qerr := api.RunQuery(query)

	if qerr != nil {
		t.Error(qerr)
	}

	if resp == nil {
		t.Error("Response channel was nil")
	}

	api.Close()

	if err := <-errch; err == nil {
		t.Error("err was nil")
	}

	empty := lib.APIResponse{}
	if r := <-resp; !reflect.DeepEqual(r, empty) {
		t.Error("Non zero APIResponse")
	}
}

// No EXPECT but still valid mock: verifies no calls.
func TestRunQueryInvalid(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	mock := NewMockKvNamespace(ctrl)
	query := &lib.Query{}

	api, _ := lib.LaunchKeyValueStore(mock)
	resp, err := api.RunQuery(query)

	if err == nil {
		t.Error("err was nil")
	}

	if resp != nil {
		t.Error("Response channel was not nil")
	}

	api.Close()
}

func mtchkvqq(q *lib.Query) gomock.Matcher {
	return kvqqmatcher{q}
}

type kvqqmatcher struct {
	q *lib.Query
}

func (kvqqm kvqqmatcher) String() string {
	return "is matching KvQuery"
}

func (kvqqm kvqqmatcher) Matches(v interface{}) bool {
	other, ok := v.(lib.KvQuery)

	if !ok {
		return false
	}

	return reflect.DeepEqual(*kvqqm.q, *other.Query)
}
<commit_msg>implement reflect webservice<commit_after>package mock_godless

import (
	"reflect"
	"testing"

	"github.com/golang/mock/gomock"
	lib "github.com/johnny-morrice/godless"
	"github.com/pkg/errors"
)

func TestRunQueryReadSuccess(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	mock := NewMockKvNamespace(ctrl)
	query := &lib.Query{
		OpCode:   lib.SELECT,
		TableKey: "Table Key",
		Select: lib.QuerySelect{
			Limit: 1,
			Where: lib.QueryWhere{
				OpCode: lib.PREDICATE,
				Predicate: lib.QueryPredicate{
					OpCode:   lib.STR_EQ,
					Literals: []string{"Hi"},
					Keys:     []lib.EntryName{"Entry A"},
				},
			},
		},
	}

	mock.EXPECT().IsChanged().Return(false)
	mock.EXPECT().RunKvQuery(query, mtchkvqq(query))

	api, errch := lib.LaunchKeyValueStore(mock)
	resp, err := api.RunQuery(query)

	if err != nil {
		t.Error(err)
	}

	if resp == nil {
		t.Error("Response channel was nil")
	}

	api.CloseAPI()

	for err := range errch {
		t.Error(err)
	}
}

func TestRunQueryWriteSuccess(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	mock := NewMockKvNamespace(ctrl)
	query := &lib.Query{
		OpCode:   lib.JOIN,
		TableKey: "Table Key",
		Join: lib.QueryJoin{
			Rows: []lib.QueryRowJoin{
				lib.QueryRowJoin{
					RowKey: "Row thing",
					Entries: map[lib.EntryName]lib.Point{
						"Hello": "world",
					},
				},
			},
		},
	}

	mock.EXPECT().IsChanged().Return(true)
	mock.EXPECT().RunKvQuery(query, mtchkvqq(query))
	mock.EXPECT().Persist().Return(mock, nil)

	api, errch := lib.LaunchKeyValueStore(mock)
	resp, err := api.RunQuery(query)

	if err != nil {
		t.Error(err)
	}

	if resp == nil {
		t.Error("Response channel was nil")
	}

	api.CloseAPI()

	for err := range errch {
		t.Error(err)
	}
}

func TestRunQueryWriteFailure(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	mock := NewMockKvNamespace(ctrl)
	query := &lib.Query{
		OpCode:   lib.JOIN,
		TableKey: "Table Key",
		Join: lib.QueryJoin{
			Rows: []lib.QueryRowJoin{
				lib.QueryRowJoin{
					RowKey: "Row thing",
					Entries: map[lib.EntryName]lib.Point{
						"Hello": "world",
					},
				},
			},
		},
	}

	mock.EXPECT().IsChanged().Return(true)
	mock.EXPECT().RunKvQuery(query, mtchkvqq(query))
	mock.EXPECT().Persist().Return(nil, errors.New("Expected error"))

	api, errch := lib.LaunchKeyValueStore(mock)
	resp, qerr := api.RunQuery(query)

	if qerr != nil {
		t.Error(qerr)
	}

	if resp == nil {
		t.Error("Response channel was nil")
	}

	api.CloseAPI()

	if err := <-errch; err == nil {
		t.Error("err was nil")
	}

	empty := lib.APIResponse{}
	if r := <-resp; !reflect.DeepEqual(r, empty) {
		t.Error("Non zero APIResponse")
	}
}

// No EXPECT but still valid mock: verifies no calls.
func TestRunQueryInvalid(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	mock := NewMockKvNamespace(ctrl)
	query := &lib.Query{}

	api, _ := lib.LaunchKeyValueStore(mock)
	resp, err := api.RunQuery(query)

	if err == nil {
		t.Error("err was nil")
	}

	if resp != nil {
		t.Error("Response channel was not nil")
	}

	api.CloseAPI()
}

func mtchkvqq(q *lib.Query) gomock.Matcher {
	return kvqqmatcher{q}
}

type kvqqmatcher struct {
	q *lib.Query
}

func (kvqqm kvqqmatcher) String() string {
	return "is matching KvQuery"
}

func (kvqqm kvqqmatcher) Matches(v interface{}) bool {
	other, ok := v.(lib.KvQuery)

	if !ok {
		return false
	}

	return reflect.DeepEqual(*kvqqm.q, *other.Query)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/yosisa/fluxion/message"
	"github.com/yosisa/fluxion/plugin"
)

const apiPath = "/api/alerts"

type Config struct {
	Tag  string
	Bind string
}

type InPrometheusAlert struct {
	env  *plugin.Env
	conf *Config
}

func (p *InPrometheusAlert) Init(env *plugin.Env) error {
	p.env = env
	p.conf = &Config{}
	return env.ReadConfig(p.conf)
}

func (p *InPrometheusAlert) Start() error {
	c := make(chan error, 1)
	http.Handle(apiPath, newAlertHandler(p.env, p.conf.Tag))
	go func() {
		c <- http.ListenAndServe(p.conf.Bind, nil)
	}()
	select {
	case err := <-c:
		return err
	case <-time.After(100 * time.Millisecond):
		return nil
	}
}

func (p *InPrometheusAlert) Close() error {
	return nil
}

type alertHandler struct {
	env *plugin.Env
	tag string
}

func newAlertHandler(env *plugin.Env, tag string) *alertHandler {
	return &alertHandler{
		env: env,
		tag: tag,
	}
}

func (h *alertHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	alerts := make([]map[string]interface{}, 0)
	if err := json.NewDecoder(r.Body).Decode(&alerts); err != nil {
		h.env.Log.Info(err)
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "%v", err)
		return
	}
	for _, alert := range alerts {
		h.env.Emit(message.NewEvent(h.tag, alert))
	}
}

func main() {
	plugin.New("in-prometheus-alert", func() plugin.Plugin {
		return &InPrometheusAlert{}
	}).Run()
}
<commit_msg>Add options: first_only and ttl<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/yosisa/fluxion/buffer"
	"github.com/yosisa/fluxion/message"
	"github.com/yosisa/fluxion/plugin"
)

const apiPath = "/api/alerts"

type Config struct {
	Tag       string
	Bind      string
	FirstOnly bool `toml:"first_only"`
	TTL       buffer.Duration
}

type InPrometheusAlert struct {
	env  *plugin.Env
	conf *Config
}

func (p *InPrometheusAlert) Init(env *plugin.Env) error {
	p.env = env
	p.conf = &Config{}
	return env.ReadConfig(p.conf)
}

func (p *InPrometheusAlert) Start() error {
	c := make(chan error, 1)
	http.Handle(apiPath, newAlertHandler(p.env, p.conf))
	go func() {
		c <- http.ListenAndServe(p.conf.Bind, nil)
	}()
	select {
	case err := <-c:
		return err
	case <-time.After(100 * time.Millisecond):
		return nil
	}
}

func (p *InPrometheusAlert) Close() error {
	return nil
}

type alertHandler struct {
	env       *plugin.Env
	tag       string
	firstOnly bool
	ttl       time.Duration
	seen      map[string]*time.Timer
	m         sync.Mutex
}

func newAlertHandler(env *plugin.Env, conf *Config) *alertHandler {
	return &alertHandler{
		env:       env,
		tag:       conf.Tag,
		firstOnly: conf.FirstOnly,
		ttl:       time.Duration(conf.TTL),
		seen:      make(map[string]*time.Timer),
	}
}

func (h *alertHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	alerts := make([]map[string]interface{}, 0)
	if err := json.NewDecoder(r.Body).Decode(&alerts); err != nil {
		h.env.Log.Info(err)
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "%v", err)
		return
	}
	for _, alert := range alerts {
		seen := h.handleAlert(alert)
		if !h.firstOnly || h.firstOnly && !seen {
			h.env.Emit(message.NewEvent(h.tag+".active", alert))
		}
	}
}

func (h *alertHandler) handleAlert(alert map[string]interface{}) (seen bool) {
	var timer *time.Timer
	id := h.makeID(alert)
	h.m.Lock()
	defer h.m.Unlock()
	timer, seen = h.seen[id]
	if seen {
		if timer != nil {
			timer.Reset(h.ttl)
		}
		return
	}
	if h.ttl > 0 {
		timer = time.AfterFunc(h.ttl, func() {
			h.m.Lock()
			defer h.m.Unlock()
			h.env.Emit(message.NewEvent(h.tag+".inactive", alert))
			delete(h.seen, id)
		})
	}
	h.seen[id] = timer
	return
}

func (h *alertHandler) makeID(alert map[string]interface{}) string {
	labels := alert["Labels"].(map[string]interface{})
	keys := make([]string, len(labels))
	var i int
	for key := range labels {
		keys[i] = key
		i++
	}
	sort.Strings(keys)
	vals := make([]string, len(labels))
	i = 0
	for _, key := range keys {
		vals[i] = fmt.Sprintf("%v", labels[key])
		i++
	}
	return strings.Join(vals, ":")
}

func main() {
	plugin.New("in-prometheus-alert", func() plugin.Plugin {
		return &InPrometheusAlert{}
	}).Run()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"strings"
)

var address = flag.String("address", ":8080", "address to listen on")
var repositories = flag.String("repositories", "scraperwiki/tang", "colon separated list of repositories to watch")

func check(err error) {
	if err != nil {
		panic(err)
	}
}

func main() {
	flag.Parse()

	configureHooks()

	http.HandleFunc("/hook", handleHook)

	log.Println("Listening on:", *address)
	log.Fatal(http.ListenAndServe(*address, nil))
}

func configureHooks() {
	github_user := os.Getenv("GITHUB_USER")
	github_password := os.Getenv("GITHUB_PASSWORD")

	json := `{
	"name": "web",
	"config": {"url": "http://services.scraperwiki.com/hook",
		"content_type": "json"},
	"events": ["push", "issues", "issue_comment",
		"commit_comment", "create", "delete",
		"pull_request", "pull_request_review_comment",
		"gollum", "watch", "release", "fork", "member",
		"public", "team_add", "status"],
	"active": true
	}`

	endpoint := "https://" + github_user + ":" + github_password + "@" + "api.github.com"

	repos := strings.Split(*repositories, ":")

	for _, repo := range repos {

		buffer := strings.NewReader(json)
		resp, err := http.Post(endpoint+"/repos/"+repo+"/hooks", "application/json", buffer)
		check(err)

		log.Println("Rate Limit:", resp.Header["X-Ratelimit-Remaining"][0])

		switch resp.StatusCode {
		default:
			response, err := ioutil.ReadAll(resp.Body)
			check(err)

			log.Print(string(response))

		case 422:
			log.Println("Already hooked for", repo)
		}
	}

}

func handleHook(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hello, world!\n")

	request, err := ioutil.ReadAll(r.Body)
	check(err)

	var dst bytes.Buffer
	r.Header.Write(&dst)
	log.Println("Incoming request headers: ", string(dst.Bytes()))

	dst.Reset()
	err = json.Indent(&dst, request, "", "  ")
	check(err)

	var doc_ interface{}
	err = json.Unmarshal(dst.Bytes(), &doc_)
	check(err)
	doc := doc_.(map[string]interface{})

	log.Println("Incoming request:", string(dst.Bytes()))

	switch eventType := r.Header["X-Github-Event"][0]; eventType {
	case "push":

		log.Println("Pushed")
		ref := doc["ref"].(string)
		url := doc["repository"].(map[string]interface{})["url"].(string)
		after := doc["after"].(string)

		log.Println("Push to", url, ref, "after", after)

		// The name of the subdirectory where the git
		// mirror is (or will appear, if it hasn't been
		// cloned yet).
		url_base := path.Base(url)
		git_dir := url_base
		if !strings.HasSuffix(git_dir, ".git") {
			git_dir = git_dir + ".git"
		}
		clone := exec.Command("git", "clone", "--mirror", url)
		clone.Stdout = os.Stdout
		clone.Stderr = os.Stderr
		err = clone.Run()
		if err == nil {
			log.Println("Cloned", url)
		} else if _, ok := err.(*exec.ExitError); ok {
			// Try "git remote update"
			remote := exec.Command("sh", "-c",
				"cd "+git_dir+" && git remote update")
			remote.Stdout = os.Stdout
			remote.Stderr = os.Stderr
			err = remote.Run()
			check(err)
			log.Println("Remote updated", url)
		} else {
			check(err)
		}
		prefix_dir := url_base + "-" + after + "/"
		log.Println("Creating", prefix_dir)
		archive := exec.Command("sh", "-c",
			"(cd "+git_dir+"&& git archive --prefix="+
				url_base+"-"+after+"/ "+after+
				") | tar xvf -")
		archive.Stderr = os.Stderr
		err = archive.Run()
		check(err)
		log.Println("Created", prefix_dir)

	default:
		log.Println("Unhandled event:", eventType)
	}

}
<commit_msg>Rearranging the deck chairs.<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"strings"
)

var address = flag.String("address", ":8080", "address to listen on")
var repositories = flag.String("repositories", "scraperwiki/tang", "colon separated list of repositories to watch")

func check(err error) {
	if err != nil {
		panic(err)
	}
}

func main() {
	flag.Parse()

	configureHooks()

	http.HandleFunc("/hook", handleHook)

	log.Println("Listening on:", *address)
	log.Fatal(http.ListenAndServe(*address, nil))
}

func configureHooks() {
	github_user := os.Getenv("GITHUB_USER")
	github_password := os.Getenv("GITHUB_PASSWORD")

	json := `{
	"name": "web",
	"config": {"url": "http://services.scraperwiki.com/hook",
		"content_type": "json"},
	"events": ["push", "issues", "issue_comment",
		"commit_comment", "create", "delete",
		"pull_request", "pull_request_review_comment",
		"gollum", "watch", "release", "fork", "member",
		"public", "team_add", "status"],
	"active": true
	}`

	endpoint := "https://" + github_user + ":" + github_password + "@" + "api.github.com"

	repos := strings.Split(*repositories, ":")

	for _, repo := range repos {

		buffer := strings.NewReader(json)
		resp, err := http.Post(endpoint+"/repos/"+repo+"/hooks", "application/json", buffer)
		check(err)

		log.Println("Rate Limit:", resp.Header["X-Ratelimit-Remaining"][0])

		switch resp.StatusCode {
		default:
			response, err := ioutil.ReadAll(resp.Body)
			check(err)

			log.Print(string(response))

		case 422:
			log.Println("Already hooked for", repo)
		}
	}

}

func handleHook(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hello, world!\n")

	request, err := ioutil.ReadAll(r.Body)
	check(err)

	var dst bytes.Buffer
	r.Header.Write(&dst)
	log.Println("Incoming request headers: ", string(dst.Bytes()))

	dst.Reset()
	err = json.Indent(&dst, request, "", "  ")
	check(err)

	var doc_ interface{}
	err = json.Unmarshal(dst.Bytes(), &doc_)
	check(err)
	doc := doc_.(map[string]interface{})

	log.Println("Incoming request:", string(dst.Bytes()))

	switch eventType := r.Header["X-Github-Event"][0]; eventType {
	case "push":
		eventPush(doc)

	default:
		log.Println("Unhandled event:", eventType)
	}

}

func eventPush(doc map[string]interface{}) {
	log.Println("Pushed")
	ref := doc["ref"].(string)
	url := doc["repository"].(map[string]interface{})["url"].(string)
	after := doc["after"].(string)

	log.Println("Push to", url, ref, "after", after)

	// The name of the subdirectory where the git
	// mirror is (or will appear, if it hasn't been
	// cloned yet).
	url_base := path.Base(url)
	git_dir := url_base
	if !strings.HasSuffix(git_dir, ".git") {
		git_dir = git_dir + ".git"
	}
	clone := exec.Command("git", "clone", "--mirror", url)
	clone.Stdout = os.Stdout
	clone.Stderr = os.Stderr
	err := clone.Run()
	if err == nil {
		log.Println("Cloned", url)
	} else if _, ok := err.(*exec.ExitError); ok {
		// Try "git remote update"
		remote := exec.Command("sh", "-c",
			"cd "+git_dir+" && git remote update")
		remote.Stdout = os.Stdout
		remote.Stderr = os.Stderr
		err = remote.Run()
		check(err)
		log.Println("Remote updated", url)
	} else {
		check(err)
	}
	prefix_dir := url_base + "-" + after + "/"
	log.Println("Creating", prefix_dir)
	archive := exec.Command("sh", "-c",
		"(cd "+git_dir+"&& git archive --prefix="+
			url_base+"-"+after+"/ "+after+
			") | tar xvf -")
	archive.Stderr = os.Stderr
	err = archive.Run()
	check(err)
	log.Println("Created", prefix_dir)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"strings"
	"syscall"

	log "github.com/golang/glog"
	"github.com/openshift/openshift-sdn/ovssubnet"
	"github.com/openshift/openshift-sdn/pkg/api"
	"github.com/openshift/openshift-sdn/pkg/registry"
)

type NetworkManager interface {
	StartMaster(sync bool, containerNetwork string, containerSubnetLength uint) error
	StartNode(sync, skipsetup bool) error
	Stop()
}

type CmdLineOpts struct {
	containerNetwork      string
	containerSubnetLength uint
	etcdEndpoints         string
	etcdPath              string
	etcdKeyfile           string
	etcdCertfile          string
	etcdCAFile            string
	ip                    string
	hostname              string
	master                bool
	minion                bool
	skipsetup             bool
	sync                  bool
	kube                  bool
	help                  bool
}

var opts CmdLineOpts

func init() {
	flag.StringVar(&opts.containerNetwork, "container-network", "10.1.0.0/16", "container network")
	flag.UintVar(&opts.containerSubnetLength, "container-subnet-length", 8, "container subnet length")
	flag.StringVar(&opts.etcdEndpoints, "etcd-endpoints", "http://127.0.0.1:4001", "a comma-delimited list of etcd endpoints")
	flag.StringVar(&opts.etcdPath, "etcd-path", "/registry/sdn/", "etcd path")
	flag.StringVar(&opts.etcdKeyfile, "etcd-keyfile", "", "SSL key file used to secure etcd communication")
	flag.StringVar(&opts.etcdCertfile, "etcd-certfile", "", "SSL certification file used to secure etcd communication")
	flag.StringVar(&opts.etcdCAFile, "etcd-cafile", "", "SSL Certificate Authority file used to secure etcd communication")

	flag.StringVar(&opts.ip, "public-ip", "", "Publicly reachable IP address of this host (for node mode).")
	flag.StringVar(&opts.hostname, "hostname", "", "Hostname as registered with master (for node mode), will default to 'hostname -f'")

	flag.BoolVar(&opts.master, "master", true, "Run in master mode")
	flag.BoolVar(&opts.minion, "minion", false, "Run in minion mode")
	flag.BoolVar(&opts.skipsetup, "skip-setup", false, "Skip the setup when in minion mode")
	flag.BoolVar(&opts.sync, "sync", false, "Sync the minions directly to etcd-path (Do not wait for PaaS to do so!)")
	flag.BoolVar(&opts.kube, "kube", false, "Use kubernetes hooks for optimal integration with OVS. This option bypasses the Linux bridge. Any docker containers started manually (not through OpenShift/Kubernetes) will stay local and not connect to the SDN.")

	flag.BoolVar(&opts.help, "help", false, "print this message")
}

func newNetworkManager() (NetworkManager, error) {
	sub, err := newSubnetRegistry()
	if err != nil {
		return nil, err
	}
	host := opts.hostname
	if host == "" {
		output, err := exec.Command("hostname", "-f").CombinedOutput()
		if err != nil {
			return nil, err
		}
		host = strings.TrimSpace(string(output))
	}

	if opts.kube {
		return ovssubnet.NewKubeController(sub, string(host), opts.ip)
	}
	// default OVS controller
	return ovssubnet.NewDefaultController(sub, string(host), opts.ip)
}

func newSubnetRegistry() (api.SubnetRegistry, error) {
	peers := strings.Split(opts.etcdEndpoints, ",")

	subnetPath := path.Join(opts.etcdPath, "subnets")
	subnetConfigPath := path.Join(opts.etcdPath, "config")
	minionPath := "/registry/minions/"
	if opts.sync {
		minionPath = path.Join(opts.etcdPath, "minions")
	}

	cfg := &registry.EtcdConfig{
		Endpoints:        peers,
		Keyfile:          opts.etcdKeyfile,
		Certfile:         opts.etcdCertfile,
		CAFile:           opts.etcdCAFile,
		SubnetPath:       subnetPath,
		SubnetConfigPath: subnetConfigPath,
		MinionPath:       minionPath,
	}

	return registry.NewEtcdSubnetRegistry(cfg)
}

func main() {
	// glog will log to tmp files by default. override so all entries
	// can flow into journald (if running under systemd)
	flag.Set("logtostderr", "true")

	// now parse command line args
	flag.Parse()

	if opts.help {
		fmt.Fprintf(os.Stderr, "Usage: %s [OPTION]...\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(0)
	}

	// Register for SIGINT and SIGTERM and wait for one of them to arrive
	log.Info("Installing signal handlers")
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)

	be, err := newNetworkManager()
	if err != nil {
		log.Fatalf("Failed to create new network manager: %v", err)
	}
	if opts.minion {
		err := be.StartNode(opts.sync, opts.skipsetup)
		if err != nil {
			log.Fatalf("Failed to start openshift sdn in node mode: %v", err)
		}
	} else if opts.master {
		err := be.StartMaster(opts.sync, opts.containerNetwork, opts.containerSubnetLength)
		if err != nil {
			log.Fatalf("Failed to start openshift sdn in master mode: %v", err)
		}
	}

	select {
	case <-sigs:
		// unregister to get default OS nuke behaviour in case we don't exit cleanly
		signal.Stop(sigs)

		log.Info("Exiting...")
		be.Stop()
	}
}
<commit_msg>add option for etcd path to watched for minions<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"strings"
	"syscall"

	log "github.com/golang/glog"
	"github.com/openshift/openshift-sdn/ovssubnet"
	"github.com/openshift/openshift-sdn/pkg/api"
	"github.com/openshift/openshift-sdn/pkg/registry"
)

type NetworkManager interface {
	StartMaster(sync bool, containerNetwork string, containerSubnetLength uint) error
	StartNode(sync, skipsetup bool) error
	Stop()
}

type CmdLineOpts struct {
	containerNetwork      string
	containerSubnetLength uint
	etcdEndpoints         string
	etcdPath              string
	etcdKeyfile           string
	etcdCertfile          string
	etcdCAFile            string
	ip                    string
	hostname              string
	minionPath            string
	master                bool
	minion                bool
	skipsetup             bool
	sync                  bool
	kube                  bool
	help                  bool
}

var opts CmdLineOpts

func init() {
	flag.StringVar(&opts.containerNetwork, "container-network", "10.1.0.0/16", "container network")
	flag.UintVar(&opts.containerSubnetLength, "container-subnet-length", 8, "container subnet length")
	flag.StringVar(&opts.etcdEndpoints, "etcd-endpoints", "http://127.0.0.1:4001", "a comma-delimited list of etcd endpoints")
	flag.StringVar(&opts.etcdPath, "etcd-path", "/registry/sdn/", "etcd path")
	flag.StringVar(&opts.minionPath, "minion-path", "/kubernetes.io/registry/minions/", "etcd path that will be watched for minion creation/deletion (Note: -sync flag will override this path with -etcd-path)")
	flag.StringVar(&opts.etcdKeyfile, "etcd-keyfile", "", "SSL key file used to secure etcd communication")
	flag.StringVar(&opts.etcdCertfile, "etcd-certfile", "", "SSL certification file used to secure etcd communication")
	flag.StringVar(&opts.etcdCAFile, "etcd-cafile", "", "SSL Certificate Authority file used to secure etcd communication")

	flag.StringVar(&opts.ip, "public-ip", "", "Publicly reachable IP address of this host (for node mode).")
	flag.StringVar(&opts.hostname, "hostname", "", "Hostname as registered with master (for node mode), will default to 'hostname -f'")

	flag.BoolVar(&opts.master, "master", true, "Run in master mode")
	flag.BoolVar(&opts.minion, "minion", false, "Run in minion mode")
	flag.BoolVar(&opts.skipsetup, "skip-setup", false, "Skip the setup when in minion mode")
	flag.BoolVar(&opts.sync, "sync", false, "Sync the minions directly to etcd-path (Do not wait for PaaS to do so!)")
	flag.BoolVar(&opts.kube, "kube", false, "Use kubernetes hooks for optimal integration with OVS. This option bypasses the Linux bridge. Any docker containers started manually (not through OpenShift/Kubernetes) will stay local and not connect to the SDN.")

	flag.BoolVar(&opts.help, "help", false, "print this message")
}

func newNetworkManager() (NetworkManager, error) {
	sub, err := newSubnetRegistry()
	if err != nil {
		return nil, err
	}
	host := opts.hostname
	if host == "" {
		output, err := exec.Command("hostname", "-f").CombinedOutput()
		if err != nil {
			return nil, err
		}
		host = strings.TrimSpace(string(output))
	}

	if opts.kube {
		return ovssubnet.NewKubeController(sub, string(host), opts.ip)
	}
	// default OVS controller
	return ovssubnet.NewDefaultController(sub, string(host), opts.ip)
}

func newSubnetRegistry() (api.SubnetRegistry, error) {
	peers := strings.Split(opts.etcdEndpoints, ",")

	subnetPath := path.Join(opts.etcdPath, "subnets")
	subnetConfigPath := path.Join(opts.etcdPath, "config")
	minionPath := opts.minionPath
	if opts.sync {
		minionPath = path.Join(opts.etcdPath, "minions")
	}

	cfg := &registry.EtcdConfig{
		Endpoints:        peers,
		Keyfile:          opts.etcdKeyfile,
		Certfile:         opts.etcdCertfile,
		CAFile:           opts.etcdCAFile,
		SubnetPath:       subnetPath,
		SubnetConfigPath: subnetConfigPath,
		MinionPath:       minionPath,
	}

	return registry.NewEtcdSubnetRegistry(cfg)
}

func main() {
	// glog will log to tmp files by default. override so all entries
	// can flow into journald (if running under systemd)
	flag.Set("logtostderr", "true")

	// now parse command line args
	flag.Parse()

	if opts.help {
		fmt.Fprintf(os.Stderr, "Usage: %s [OPTION]...\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(0)
	}

	// Register for SIGINT and SIGTERM and wait for one of them to arrive
	log.Info("Installing signal handlers")
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)

	be, err := newNetworkManager()
	if err != nil {
		log.Fatalf("Failed to create new network manager: %v", err)
	}
	if opts.minion {
		err := be.StartNode(opts.sync, opts.skipsetup)
		if err != nil {
			log.Fatalf("Failed to start openshift sdn in node mode: %v", err)
		}
	} else if opts.master {
		err := be.StartMaster(opts.sync, opts.containerNetwork, opts.containerSubnetLength)
		if err != nil {
			log.Fatalf("Failed to start openshift sdn in master mode: %v", err)
		}
	}

	select {
	case <-sigs:
		// unregister to get default OS nuke behaviour in case we don't exit cleanly
		signal.Stop(sigs)

		log.Info("Exiting...")
		be.Stop()
	}
}
<|endoftext|>


<commit_before>package sexpconv

import (
	"go/ast"
	"go/types"
	"lisp/function"
	"sexp"
)

func (conv *Converter) exprList(nodes []ast.Expr) []sexp.Form {
	forms := make([]sexp.Form, len(nodes))
	for i, node := range nodes {
		forms[i] = conv.Expr(node)
	}
	return forms
}

func (conv *Converter) stmtList(nodes []ast.Stmt) []sexp.Form {
	forms := make([]sexp.Form, len(nodes))
	for i, node := range nodes {
		forms[i] = conv.Stmt(node)
	}
	return forms
}

func (conv *Converter) valueCopyList(forms []sexp.Form) []sexp.Form {
	for i := range forms {
		forms[i] = conv.valueCopy(forms[i])
	}
	return forms
}

func (conv *Converter) valueCopy(form sexp.Form) sexp.Form {
	if isArray(form.Type()) && !isArrayLit(form) {
		return &sexp.Call{
			Fn:   function.CopySequence,
			Args: []sexp.Form{form},
		}
	}
	return form
}

func isArray(typ types.Type) bool {
	_, ok := typ.Underlying().(*types.Array)
	return ok
}

func isArrayLit(form sexp.Form) bool {
	_, ok := form.(*sexp.ArrayLit)
	if ok {
		return true
	}
	_, ok = form.(*sexp.SparseArrayLit)
	return ok
}
<commit_msg>isGlobal predicate<commit_after>package sexpconv

import (
	"go/ast"
	"go/types"
	"lisp/function"
	"sexp"
)

func (conv *Converter) exprList(nodes []ast.Expr) []sexp.Form {
	forms := make([]sexp.Form, len(nodes))
	for i, node := range nodes {
		forms[i] = conv.Expr(node)
	}
	return forms
}

func (conv *Converter) stmtList(nodes []ast.Stmt) []sexp.Form {
	forms := make([]sexp.Form, len(nodes))
	for i, node := range nodes {
		forms[i] = conv.Stmt(node)
	}
	return forms
}

func (conv *Converter) valueCopyList(forms []sexp.Form) []sexp.Form {
	for i := range forms {
		forms[i] = conv.valueCopy(forms[i])
	}
	return forms
}

func (conv *Converter) valueCopy(form sexp.Form) sexp.Form {
	if isArray(form.Type()) && !isArrayLit(form) {
		return &sexp.Call{
			Fn:   function.CopySequence,
			Args: []sexp.Form{form},
		}
	}
	return form
}

func isGlobal(obj types.Object) bool {
	objScope := obj.Parent()
	// If parent scope is Universe, then object scope
	// is Package => it is global.
	return objScope.Parent() == types.Universe
}

func isArray(typ types.Type) bool {
	_, ok := typ.Underlying().(*types.Array)
	return ok
}

func isArrayLit(form sexp.Form) bool {
	_, ok := form.(*sexp.ArrayLit)
	if ok {
		return true
	}
	_, ok = form.(*sexp.SparseArrayLit)
	return ok
}
<|endoftext|>