<commit_before>package main

import (
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/alienth/fastlyctl/util"
	"github.com/alienth/go-fastly"
	"github.com/urfave/cli"
	"gopkg.in/mcuadros/go-syslog.v2"

	_ "expvar"
	_ "net/http/pprof"
)

var client *fastly.Client
var ipLists IPLists
var hook hookService
var noop bool

var syslogChannel syslog.LogPartsChannel

// How many syslog logs we want to buffer, at max.
const syslogChannelBufferSize = 3000

// How many log reading workers we fire up
const workers = 1

var hits = hitMap{m: make(map[string]*ipRate)}

func main() {
	app := cli.NewApp()
	app.Name = "fastly-ratelimit"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config, c",
			Usage: "Read config `FILE`.",
			Value: "config.toml",
		},
		cli.StringFlag{
			Name:  "listen, l",
			Usage: "Specify listen `ADDRESS:PORT`.",
			Value: "0.0.0.0:514",
		},
		cli.StringFlag{
			Name:   "fastly-key, K",
			Usage:  "Fastly API Key. Can be read from 'fastly_key' file in CWD.",
			EnvVar: "FASTLY_KEY",
			Value:  util.GetFastlyKey(),
		},
		cli.BoolFlag{
			Name:  "noop, n",
			Usage: "Noop mode. Print what we'd do, but don't actually do anything.",
		},
	}
	app.Before = func(c *cli.Context) error {
		if len(c.Args()) > 0 {
			return cli.NewExitError("Invalid usage. More arguments received than expected.", -1)
		}
		return nil
	}
	app.Action = runServer

	app.Run(os.Args)
}

func runServer(c *cli.Context) error {
	http.HandleFunc("/", handler)
	go http.ListenAndServe(":80", nil)
	client = fastly.NewClient(nil, c.GlobalString("fastly-key"))
	syslogChannel = make(syslog.LogPartsChannel, syslogChannelBufferSize)
	handler := syslog.NewChannelHandler(syslogChannel)

	noop = c.GlobalBool("noop")

	config, err := readConfig(c.GlobalString("config"))
	if err != nil {
		return cli.NewExitError(fmt.Sprintf("Error reading config file:\n%s\n", err), -1)
	}
	ipLists = config.Lists
	hook = config.HookService
	hook.hookedIPs.m = make(map[string]bool)

	serviceDomains, err := getServiceDomains()
	if err != nil {
		return cli.NewExitError(fmt.Sprintf("Error fetching fasty domains:\n%s\n", err), -1)
	}

	if err := hits.importIPRates(serviceDomains); err != nil {
		return cli.NewExitError(fmt.Sprintf("Error importing existing IP rates: %s", err), -1)
	}
	go hits.expireRecords()
	go hits.expireLimits()
	go queueFanout()
	if hook.SyncIPsUri != "" {
		go hits.syncIPsWithHook()
	}

	server := syslog.NewServer()
	server.SetFormat(syslog.RFC3164)
	server.SetHandler(handler)
	if err := server.ListenUDP(c.GlobalString("listen")); err != nil {
		return cli.NewExitError(fmt.Sprintf("Unable to listen: %s\n", err), -1)
	}
	if err := server.Boot(); err != nil {
		return cli.NewExitError(fmt.Sprintf("Unable to start server: %s\n", err), -1)
	}

	for i := 1; i <= workers; i++ {
		go readLogs(syslogChannel, serviceDomains)
	}

	server.Wait()

	return nil
}

func readLogs(channel syslog.LogPartsChannel, serviceDomains ServiceDomains) {
	for logParts := range channel {
		var line string
		var ok bool
		if line, ok = logParts["content"].(string); !ok || line == "" {
			continue
		}
		log := parseLog(line)
		if log == nil || log.cdnIP == nil || log.clientIP == nil {
			continue
		}
		if time.Now().Sub(log.timestamp) > time.Duration(2)*time.Minute {
			fmt.Printf("Warning: old log line. Log TS: %s, Current time: %s\n", log.timestamp.String(), time.Now().String())
		}
		var ipr *ipRate
		var found bool
		hits.Lock()
		if ipr, found = hits.m[log.cdnIP.String()]; !found {
			ipr = ipLists.getRate(log.cdnIP)
			hits.m[log.cdnIP.String()] = ipr
		}
		hits.Unlock()
		service, err := serviceDomains.getServiceByHost(log.host.Value)
		if err != nil {
			fmt.Printf("Error while finding fastly service for domain %s: %s\n.", log.host.Value, err)
		}
		if service == nil {
			fmt.Printf("Found request for host %s which is not in fastly. Ignoring\n", log.host.Value)
			continue
		}
		dimension := ipr.list.getDimension(log, service)
		overLimit := ipr.Hit(log.timestamp, dimension)
		if overLimit {
			if err := ipr.Limit(service); err != nil {
				fmt.Printf("Error limiting IP: %s\n", err)
			}
		}

		if len(channel) == syslogChannelBufferSize {
			fmt.Println("Warning: log buffer full. We are dropping logs.")
		}
	}

}
<commit_msg>Add flag to specify webstats listen addr.<commit_after>package main

import (
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/alienth/fastlyctl/util"
	"github.com/alienth/go-fastly"
	"github.com/urfave/cli"
	"gopkg.in/mcuadros/go-syslog.v2"

	_ "expvar"
	_ "net/http/pprof"
)

var client *fastly.Client
var ipLists IPLists
var hook hookService
var noop bool

var syslogChannel syslog.LogPartsChannel

// How many syslog logs we want to buffer, at max.
const syslogChannelBufferSize = 3000

// How many log reading workers we fire up
const workers = 1

var hits = hitMap{m: make(map[string]*ipRate)}

func main() {
	app := cli.NewApp()
	app.Name = "fastly-ratelimit"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config, c",
			Usage: "Read config `FILE`.",
			Value: "config.toml",
		},
		cli.StringFlag{
			Name:  "listen, l",
			Usage: "Specify listen `ADDRESS:PORT`.",
			Value: "0.0.0.0:514",
		},
		cli.StringFlag{
			Name:  "stats-listen, s",
			Usage: "Specify listen for web stats interface, `ADDRESS:PORT`.",
			Value: "0.0.0.0:80",
		},
		cli.StringFlag{
			Name:   "fastly-key, K",
			Usage:  "Fastly API Key. Can be read from 'fastly_key' file in CWD.",
			EnvVar: "FASTLY_KEY",
			Value:  util.GetFastlyKey(),
		},
		cli.BoolFlag{
			Name:  "noop, n",
			Usage: "Noop mode. Print what we'd do, but don't actually do anything.",
		},
	}
	app.Before = func(c *cli.Context) error {
		if len(c.Args()) > 0 {
			return cli.NewExitError("Invalid usage. More arguments received than expected.", -1)
		}
		return nil
	}
	app.Action = runServer

	app.Run(os.Args)
}

func runServer(c *cli.Context) error {
	http.HandleFunc("/", handler)
	go http.ListenAndServe(c.GlobalString("stats-listen"), nil)
	client = fastly.NewClient(nil, c.GlobalString("fastly-key"))
	syslogChannel = make(syslog.LogPartsChannel, syslogChannelBufferSize)
	handler := syslog.NewChannelHandler(syslogChannel)

	noop = c.GlobalBool("noop")

	config, err := readConfig(c.GlobalString("config"))
	if err != nil {
		return cli.NewExitError(fmt.Sprintf("Error reading config file:\n%s\n", err), -1)
	}
	ipLists = config.Lists
	hook = config.HookService
	hook.hookedIPs.m = make(map[string]bool)

	serviceDomains, err := getServiceDomains()
	if err != nil {
		return cli.NewExitError(fmt.Sprintf("Error fetching fasty domains:\n%s\n", err), -1)
	}

	if err := hits.importIPRates(serviceDomains); err != nil {
		return cli.NewExitError(fmt.Sprintf("Error importing existing IP rates: %s", err), -1)
	}
	go hits.expireRecords()
	go hits.expireLimits()
	go queueFanout()
	if hook.SyncIPsUri != "" {
		go hits.syncIPsWithHook()
	}

	server := syslog.NewServer()
	server.SetFormat(syslog.RFC3164)
	server.SetHandler(handler)
	if err := server.ListenUDP(c.GlobalString("listen")); err != nil {
		return cli.NewExitError(fmt.Sprintf("Unable to listen: %s\n", err), -1)
	}
	if err := server.Boot(); err != nil {
		return cli.NewExitError(fmt.Sprintf("Unable to start server: %s\n", err), -1)
	}

	for i := 1; i <= workers; i++ {
		go readLogs(syslogChannel, serviceDomains)
	}

	server.Wait()

	return nil
}

func readLogs(channel syslog.LogPartsChannel, serviceDomains ServiceDomains) {
	for logParts := range channel {
		var line string
		var ok bool
		if line, ok = logParts["content"].(string); !ok || line == "" {
			continue
		}
		log := parseLog(line)
		if log == nil || log.cdnIP == nil || log.clientIP == nil {
			continue
		}
		if time.Now().Sub(log.timestamp) > time.Duration(2)*time.Minute {
			fmt.Printf("Warning: old log line. Log TS: %s, Current time: %s\n", log.timestamp.String(), time.Now().String())
		}
		var ipr *ipRate
		var found bool
		hits.Lock()
		if ipr, found = hits.m[log.cdnIP.String()]; !found {
			ipr = ipLists.getRate(log.cdnIP)
			hits.m[log.cdnIP.String()] = ipr
		}
		hits.Unlock()
		service, err := serviceDomains.getServiceByHost(log.host.Value)
		if err != nil {
			fmt.Printf("Error while finding fastly service for domain %s: %s\n.", log.host.Value, err)
		}
		if service == nil {
			fmt.Printf("Found request for host %s which is not in fastly. Ignoring\n", log.host.Value)
			continue
		}
		dimension := ipr.list.getDimension(log, service)
		overLimit := ipr.Hit(log.timestamp, dimension)
		if overLimit {
			if err := ipr.Limit(service); err != nil {
				fmt.Printf("Error limiting IP: %s\n", err)
			}
		}

		if len(channel) == syslogChannelBufferSize {
			fmt.Println("Warning: log buffer full. We are dropping logs.")
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/ninjasphere/go-ninja"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-openzwave"
	"github.com/ninjasphere/go-openzwave/LOG_LEVEL"
)

const driverName = "driver-zwave"

var log = logger.GetLogger(driverName)

func main() {

	log.Infof("Starting " + driverName)

	conn, err := ninja.Connect("com.ninjablocks.zwave")
	if err != nil {
		log.FatalError(err, "Could not connect to MQTT")
	}

	pwd, _ := os.Getwd()

	bus, err := conn.AnnounceDriver("com.ninjablocks.zwave", driverName, pwd)
	if err != nil {
		log.FatalError(err, "Could not get driver bus")
	}

	statusJob, err := ninja.CreateStatusJob(conn, driverName)

	if err != nil {
		log.FatalError(err, "Could not setup status job")
	}

	statusJob.Start()

	ipAddr, err := ninja.GetNetAddress()
	if err != nil {
		log.FatalError(err, "Could not get net address")
	}

	_ = bus
	_ = ipAddr

	loop := func(notifications chan openzwave.Notification) {
		for {
		    var notification openzwave.Notification;
		    notifications <- notification;
		    _ = notification
		}
	}

	os.Exit(openzwave.
		API("/usr/local/etc/openzwave", "", "").
		AddIntOption("SaveLogLevel", LOG_LEVEL.DETAIL).
		AddIntOption("QueueLogLevel", LOG_LEVEL.DEBUG).
		AddIntOption("DumpTrigger", LOG_LEVEL.ERROR).
		AddIntOption("PollInterval", 500).
		AddBoolOption("IntervalBetweenPolls", true).
		AddBoolOption("ValidateValueChanges", true).
		Run(loop));

}
<commit_msg>Add support for additional channels.<commit_after>package main

import (
       "fmt"
       "os"

       "github.com/ninjasphere/go-ninja"
       "github.com/ninjasphere/go-ninja/logger"
       "github.com/ninjasphere/go-openzwave"
       "github.com/ninjasphere/go-openzwave/LOG_LEVEL"
)

const driverName = "driver-zwave"

var log = logger.GetLogger(driverName)

func main() {

	log.Infof("Starting " + driverName)

	conn, err := ninja.Connect("com.ninjablocks.zwave")
	if err != nil {
		log.FatalError(err, "Could not connect to MQTT")
	}

	pwd, _ := os.Getwd()

	bus, err := conn.AnnounceDriver("com.ninjablocks.zwave", driverName, pwd)
	if err != nil {
		log.FatalError(err, "Could not get driver bus")
	}

	statusJob, err := ninja.CreateStatusJob(conn, driverName)

	if err != nil {
		log.FatalError(err, "Could not setup status job")
	}

	statusJob.Start()

	ipAddr, err := ninja.GetNetAddress()
	if err != nil {
		log.FatalError(err, "Could not get net address")
	}

	_ = bus
	_ = ipAddr

	loop := func(notifications chan openzwave.Notification, quit chan bool) {
		for {
		    select {
		    	case notification := <- notifications:
				_ = notification
			case quitReceived := <- quit:
			     _ = quitReceived; // TODO: something useful
			     fmt.Printf("TODO: quit received\n");
		    }
		}
	}

	os.Exit(openzwave.
		API("/usr/local/etc/openzwave", "", "").
		AddIntOption("SaveLogLevel", LOG_LEVEL.DETAIL).
		AddIntOption("QueueLogLevel", LOG_LEVEL.DEBUG).
		AddIntOption("DumpTrigger", LOG_LEVEL.ERROR).
		AddIntOption("PollInterval", 500).
		AddBoolOption("IntervalBetweenPolls", true).
		AddBoolOption("ValidateValueChanges", true).
		Run(loop));

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/Cepave/open-falcon-backend/cmd"
	"github.com/spf13/cobra"
)

var RootCmd = &cobra.Command{
	Use: "open-falcon",
}

func init() {
	RootCmd.AddCommand(cmd.Start)
	RootCmd.AddCommand(cmd.Stop)
	RootCmd.AddCommand(cmd.Restart)
	RootCmd.AddCommand(cmd.Check)
	RootCmd.AddCommand(cmd.Monitor)
	RootCmd.AddCommand(cmd.Reload)
	cmd.Start.Flags().BoolVar(&cmd.PreqOrderFlag, "preq-order", false, "start modules in the order of prerequisites")
	cmd.Start.Flags().BoolVar(&cmd.ConsoleOutputFlag, "console-output", false, "print the module's output to the console")
}

func main() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>[OWL-998][backend] Fix `-v` & `--version`<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/Cepave/open-falcon-backend/cmd"
	"github.com/Cepave/open-falcon-backend/g"
	"github.com/spf13/cobra"
	flag "github.com/spf13/pflag"
)

var versionFlag bool

var RootCmd = &cobra.Command{
	Use: "open-falcon",
}

func init() {
	RootCmd.AddCommand(cmd.Start)
	RootCmd.AddCommand(cmd.Stop)
	RootCmd.AddCommand(cmd.Restart)
	RootCmd.AddCommand(cmd.Check)
	RootCmd.AddCommand(cmd.Monitor)
	RootCmd.AddCommand(cmd.Reload)
	cmd.Start.Flags().BoolVar(&cmd.PreqOrderFlag, "preq-order", false, "start modules in the order of prerequisites")
	cmd.Start.Flags().BoolVar(&cmd.ConsoleOutputFlag, "console-output", false, "print the module's output to the console")
	flag.BoolVarP(&versionFlag, "version", "v", false, "show version")
	flag.Parse()
}

func main() {
	if versionFlag {
		fmt.Println(g.Version)
		os.Exit(0)
	}
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Package main starts the binary.
// Argument parsing, usage information and the actual execution can be found here.
// See package promplot for using piece directly from you own Go code.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"os"
	"runtime"
	"time"

	"qvl.io/promplot/flags"
	"qvl.io/promplot/promplot"
)

// Can be set in build step using -ldflags
var version string

const (
	usage = `
Usage: %s [flags...]

Create and deliver plots from your Prometheus metrics.

Save plot to file or send it right to a slack channel.
At least one of -slack or -file must be set.

Flags:
`
	more = "\nFor more visit: https://qvl.io/promplot"
)

// Number of data points for the plot
const step = 100

func main() {
	var (
		silent      = flag.Bool("silent", false, "Optional. Suppress all output.")
		versionFlag = flag.Bool("version", false, "Optional. Print binary version.")
		promServer  = flag.String("url", "", "Required. URL of Prometheus server.")
		query       = flag.String("query", "", "Required. PQL query.")
		queryTime   = flags.UnixTime("time", time.Now(), "Time for query (default is now). Format like the default format of the Unix date command.")
		duration    = flags.Duration("range", 0, "Required. Time to look back to. Format: 5d12h34m56s")
		title       = flag.String("title", "Prometheus metrics", "Optional. Title of graph.")
		//
		format = flag.String("format", "png", "Optional. Image format. For possible values see: https://godoc.org/github.com/gonum/plot/vg/draw#NewFormattedCanvas")
	)

	var (
		file = flag.String("file", "", "File to save image to. Should have same extension as specified -format. Set -file to - to write to stdout.")
	)

	var (
		slackToken = flag.String("slack", "", "Slack API token (https://api.slack.com/docs/oauth-test-tokens). Set to post plot to Slack.")
		channel    = flag.String("channel", "", "Required when -slack is set. Slack channel to post to.")
	)

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage, os.Args[0])
		flag.PrintDefaults()
		fmt.Fprintln(os.Stderr, more)
	}
	flag.Parse()

	if *versionFlag {
		fmt.Printf("promplot %s %s %s\n", version, runtime.GOOS, runtime.GOARCH)
		os.Exit(0)
	}

	// Required flags
	if *promServer == "" || *query == "" || *duration == 0 || (*file == "" && (*slackToken == "" || *channel == "")) {
		flag.Usage()
		os.Exit(1)
	}

	// Logging helper
	log := func(format string, a ...interface{}) {
		if !*silent {
			fmt.Fprintf(os.Stderr, format+"\n", a...)
		}
	}

	// Fetch from Prometheus
	log("Querying Prometheus %q", *query)
	metrics, err := promplot.Metrics(*promServer, *query, *queryTime, *duration, step)
	fatal(err, "failed to get metrics")

	// Plot
	log("Creating plot %q", *title)
	plot, err := promplot.Plot(metrics, *title, *format)
	fatal(err, "failed to create plot")

	// Write to file
	if *file != "" {
		// Copy plot to be able to use it for Slack after
		buf := new(bytes.Buffer)
		t := io.TeeReader(plot, buf)
		plot = buf

		var out *os.File
		if *file == "-" {
			log("Writing to stdout")
			out = os.Stdout
		} else {
			log("Writing to '%s'", *file)
			out, err = os.Create(*file)
			fatal(err, "failed to create file")
		}
		_, err = io.Copy(out, t)
		fatal(err, "failed to copy to file")
	}

	// Upload to Slack
	if *slackToken != "" {
		log("Uploading to Slack channel %q", *channel)
		fatal(promplot.Slack(*slackToken, *channel, *title, plot), "failed to upload to Slack")
	}

	log("Done")
}

func fatal(err error, msg string) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "msg: %v\n", err)
		os.Exit(1)
	}
}
<commit_msg>support only one output<commit_after>// Package main starts the binary.
// Argument parsing, usage information and the actual execution can be found here.
// See package promplot for using piece directly from you own Go code.
package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"runtime"
	"time"

	"qvl.io/promplot/flags"
	"qvl.io/promplot/promplot"
)

// Can be set in build step using -ldflags
var version string

const (
	usage = `
Usage: %s [flags...]

Create and deliver plots from your Prometheus metrics.

Save plot to file or send it right to a slack channel.
One of -slack or -file must be set.

Flags:
`
	more = "\nFor more visit: https://qvl.io/promplot"
)

// Number of data points for the plot
const step = 100

func main() {
	var (
		silent      = flag.Bool("silent", false, "Optional. Suppress all output.")
		versionFlag = flag.Bool("version", false, "Optional. Print binary version.")
		promServer  = flag.String("url", "", "Required. URL of Prometheus server.")
		query       = flag.String("query", "", "Required. PQL query.")
		queryTime   = flags.UnixTime("time", time.Now(), "Time for query (default is now). Format like the default format of the Unix date command.")
		duration    = flags.Duration("range", 0, "Required. Time to look back to. Format: 5d12h34m56s")
		title       = flag.String("title", "Prometheus metrics", "Optional. Title of graph.")
		//
		format = flag.String("format", "png", "Optional. Image format. For possible values see: https://godoc.org/github.com/gonum/plot/vg/draw#NewFormattedCanvas")
	)

	var (
		file = flag.String("file", "", "File to save image to. Should have same extension as specified -format. Set -file to - to write to stdout.")
	)

	var (
		slackToken = flag.String("slack", "", "Slack API token (https://api.slack.com/docs/oauth-test-tokens). Set to post plot to Slack.")
		channel    = flag.String("channel", "", "Required when -slack is set. Slack channel to post to.")
	)

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage, os.Args[0])
		flag.PrintDefaults()
		fmt.Fprintln(os.Stderr, more)
	}
	flag.Parse()

	if *versionFlag {
		fmt.Printf("promplot %s %s %s\n", version, runtime.GOOS, runtime.GOARCH)
		os.Exit(0)
	}

	// Required flags
	if *promServer == "" || *query == "" || *duration == 0 || (*file == "" && (*slackToken == "" || *channel == "")) || !(*file == "" || *slackToken == "") {
		flag.Usage()
		os.Exit(1)
	}

	// Logging helper
	log := func(format string, a ...interface{}) {
		if !*silent {
			fmt.Fprintf(os.Stderr, format+"\n", a...)
		}
	}

	// Fetch from Prometheus
	log("Querying Prometheus %q", *query)
	metrics, err := promplot.Metrics(*promServer, *query, *queryTime, *duration, step)
	fatal(err, "failed to get metrics")

	// Plot
	log("Creating plot %q", *title)
	plot, err := promplot.Plot(metrics, *title, *format)
	fatal(err, "failed to create plot")

	// Write to file
	if *file != "" {
		var out *os.File
		if *file == "-" {
			log("Writing to stdout")
			out = os.Stdout
		} else {
			log("Writing to '%s'", *file)
			out, err = os.Create(*file)
			fatal(err, "failed to create file")
		}
		_, err = io.Copy(out, plot)
		fatal(err, "failed to copy to file")

		// Upload to Slack
	} else {
		log("Uploading to Slack channel %q", *channel)
		fatal(promplot.Slack(*slackToken, *channel, *title, plot), "failed to upload to Slack")
	}

	log("Done")
}

func fatal(err error, msg string) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "msg: %v\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"

	"github.com/mitchellh/colorstring"

	"github.com/hashicorp/go-plugin"
	"github.com/hashicorp/terraform/command/format"
	"github.com/hashicorp/terraform/helper/logging"
	"github.com/hashicorp/terraform/terraform"
	"github.com/mattn/go-colorable"
	"github.com/mattn/go-shellwords"
	"github.com/mitchellh/cli"
	"github.com/mitchellh/panicwrap"
	"github.com/mitchellh/prefixedio"
)

const (
	// EnvCLI is the environment variable name to set additional CLI args.
	EnvCLI = "TF_CLI_ARGS"
)

func main() {
	// Override global prefix set by go-dynect during init()
	log.SetPrefix("")
	os.Exit(realMain())
}

func realMain() int {
	var wrapConfig panicwrap.WrapConfig

	// don't re-exec terraform as a child process for easier debugging
	if os.Getenv("TF_FORK") == "0" {
		return wrappedMain()
	}

	if !panicwrap.Wrapped(&wrapConfig) {
		// Determine where logs should go in general (requested by the user)
		logWriter, err := logging.LogOutput()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Couldn't setup log output: %s", err)
			return 1
		}

		// We always send logs to a temporary file that we use in case
		// there is a panic. Otherwise, we delete it.
		logTempFile, err := ioutil.TempFile("", "terraform-log")
		if err != nil {
			fmt.Fprintf(os.Stderr, "Couldn't setup logging tempfile: %s", err)
			return 1
		}
		defer os.Remove(logTempFile.Name())
		defer logTempFile.Close()

		// Setup the prefixed readers that send data properly to
		// stdout/stderr.
		doneCh := make(chan struct{})
		outR, outW := io.Pipe()
		go copyOutput(outR, doneCh)

		// Create the configuration for panicwrap and wrap our executable
		wrapConfig.Handler = panicHandler(logTempFile)
		wrapConfig.Writer = io.MultiWriter(logTempFile, logWriter)
		wrapConfig.Stdout = outW
		wrapConfig.IgnoreSignals = ignoreSignals
		wrapConfig.ForwardSignals = forwardSignals
		exitStatus, err := panicwrap.Wrap(&wrapConfig)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Couldn't start Terraform: %s", err)
			return 1
		}

		// If >= 0, we're the parent, so just exit
		if exitStatus >= 0 {
			// Close the stdout writer so that our copy process can finish
			outW.Close()

			// Wait for the output copying to finish
			<-doneCh

			return exitStatus
		}

		// We're the child, so just close the tempfile we made in order to
		// save file handles since the tempfile is only used by the parent.
		logTempFile.Close()
	}

	// Call the real main
	return wrappedMain()
}

func init() {
	Ui = &cli.PrefixedUi{
		AskPrefix:    OutputPrefix,
		OutputPrefix: OutputPrefix,
		InfoPrefix:   OutputPrefix,
		ErrorPrefix:  ErrorPrefix,
		Ui:           &cli.BasicUi{Writer: os.Stdout},
	}
}

func wrappedMain() int {
	var err error

	// We always need to close the DebugInfo before we exit.
	defer terraform.CloseDebugInfo()

	log.SetOutput(os.Stderr)
	log.Printf(
		"[INFO] Terraform version: %s %s %s",
		Version, VersionPrerelease, GitCommit)
	log.Printf("[INFO] Go runtime version: %s", runtime.Version())
	log.Printf("[INFO] CLI args: %#v", os.Args)

	config, diags := LoadConfig()
	if len(diags) > 0 {
		// Since we haven't instantiated a command.Meta yet, we need to do
		// some things manually here and use some "safe" defaults for things
		// that command.Meta could otherwise figure out in smarter ways.
		Ui.Error("There are some problems with the CLI configuration:")
		for _, diag := range diags {
			earlyColor := &colorstring.Colorize{
				Colors:  colorstring.DefaultColors,
				Disable: true, // Disable color to be conservative until we know better
				Reset:   true,
			}
			Ui.Error(format.Diagnostic(diag, earlyColor, 78))
		}
		if diags.HasErrors() {
			Ui.Error("As a result of the above problems, Terraform may not behave as intended.\n\n")
			// We continue to run anyway, since Terraform has reasonable defaults.
		}
	}
	log.Printf("[DEBUG] CLI config is %#v", config)

	// In tests, Commands may already be set to provide mock commands
	if Commands == nil {
		initCommands(config)
	}

	// Run checkpoint
	go runCheckpoint(config)

	// Make sure we clean up any managed plugins at the end of this
	defer plugin.CleanupClients()

	// Get the command line args.
	binName := filepath.Base(os.Args[0])
	args := os.Args[1:]

	// Build the CLI so far, we do this so we can query the subcommand.
	cliRunner := &cli.CLI{
		Args:       args,
		Commands:   Commands,
		HelpFunc:   helpFunc,
		HelpWriter: os.Stdout,
	}

	// Prefix the args with any args from the EnvCLI
	args, err = mergeEnvArgs(EnvCLI, cliRunner.Subcommand(), args)
	if err != nil {
		Ui.Error(err.Error())
		return 1
	}

	// Prefix the args with any args from the EnvCLI targeting this command
	suffix := strings.Replace(strings.Replace(
		cliRunner.Subcommand(), "-", "_", -1), " ", "_", -1)
	args, err = mergeEnvArgs(
		fmt.Sprintf("%s_%s", EnvCLI, suffix), cliRunner.Subcommand(), args)
	if err != nil {
		Ui.Error(err.Error())
		return 1
	}

	// We shortcut "--version" and "-v" to just show the version
	for _, arg := range args {
		if arg == "-v" || arg == "-version" || arg == "--version" {
			newArgs := make([]string, len(args)+1)
			newArgs[0] = "version"
			copy(newArgs[1:], args)
			args = newArgs
			break
		}
	}

	// Rebuild the CLI with any modified args.
	log.Printf("[INFO] CLI command args: %#v", args)
	cliRunner = &cli.CLI{
		Name:       binName,
		Args:       args,
		Commands:   Commands,
		HelpFunc:   helpFunc,
		HelpWriter: os.Stdout,

		Autocomplete:          true,
		AutocompleteInstall:   "install-autocomplete",
		AutocompleteUninstall: "uninstall-autocomplete",
	}

	// Pass in the overriding plugin paths from config
	PluginOverrides.Providers = config.Providers
	PluginOverrides.Provisioners = config.Provisioners

	exitCode, err := cliRunner.Run()
	if err != nil {
		Ui.Error(fmt.Sprintf("Error executing CLI: %s", err.Error()))
		return 1
	}

	return exitCode
}

func cliConfigFile() (string, error) {
	mustExist := true
	configFilePath := os.Getenv("TERRAFORM_CONFIG")
	if configFilePath == "" {
		var err error
		configFilePath, err = ConfigFile()
		mustExist = false

		if err != nil {
			log.Printf(
				"[ERROR] Error detecting default CLI config file path: %s",
				err)
		}
	}

	log.Printf("[DEBUG] Attempting to open CLI config file: %s", configFilePath)
	f, err := os.Open(configFilePath)
	if err == nil {
		f.Close()
		return configFilePath, nil
	}

	if mustExist || !os.IsNotExist(err) {
		return "", err
	}

	log.Println("[DEBUG] File doesn't exist, but doesn't need to. Ignoring.")
	return "", nil
}

// copyOutput uses output prefixes to determine whether data on stdout
// should go to stdout or stderr. This is due to panicwrap using stderr
// as the log and error channel.
func copyOutput(r io.Reader, doneCh chan<- struct{}) {
	defer close(doneCh)

	pr, err := prefixedio.NewReader(r)
	if err != nil {
		panic(err)
	}

	stderrR, err := pr.Prefix(ErrorPrefix)
	if err != nil {
		panic(err)
	}
	stdoutR, err := pr.Prefix(OutputPrefix)
	if err != nil {
		panic(err)
	}
	defaultR, err := pr.Prefix("")
	if err != nil {
		panic(err)
	}

	var stdout io.Writer = os.Stdout
	var stderr io.Writer = os.Stderr

	if runtime.GOOS == "windows" {
		stdout = colorable.NewColorableStdout()
		stderr = colorable.NewColorableStderr()

		// colorable is not concurrency-safe when stdout and stderr are the
		// same console, so we need to add some synchronization to ensure that
		// we can't be concurrently writing to both stderr and stdout at
		// once, or else we get intermingled writes that create gibberish
		// in the console.
		wrapped := synchronizedWriters(stdout, stderr)
		stdout = wrapped[0]
		stderr = wrapped[1]
	}

	var wg sync.WaitGroup
	wg.Add(3)
	go func() {
		defer wg.Done()
		io.Copy(stderr, stderrR)
	}()
	go func() {
		defer wg.Done()
		io.Copy(stdout, stdoutR)
	}()
	go func() {
		defer wg.Done()
		io.Copy(stdout, defaultR)
	}()

	wg.Wait()
}

func mergeEnvArgs(envName string, cmd string, args []string) ([]string, error) {
	v := os.Getenv(envName)
	if v == "" {
		return args, nil
	}

	log.Printf("[INFO] %s value: %q", envName, v)
	extra, err := shellwords.Parse(v)
	if err != nil {
		return nil, fmt.Errorf(
			"Error parsing extra CLI args from %s: %s",
			envName, err)
	}

	// Find the command to look for in the args. If there is a space,
	// we need to find the last part.
	search := cmd
	if idx := strings.LastIndex(search, " "); idx >= 0 {
		search = cmd[idx+1:]
	}

	// Find the index to place the flags. We put them exactly
	// after the first non-flag arg.
	idx := -1
	for i, v := range args {
		if v == search {
			idx = i
			break
		}
	}

	// idx points to the exact arg that isn't a flag. We increment
	// by one so that all the copying below expects idx to be the
	// insertion point.
	idx++

	// Copy the args
	newArgs := make([]string, len(args)+len(extra))
	copy(newArgs, args[:idx])
	copy(newArgs[idx:], extra)
	copy(newArgs[len(extra)+idx:], args[idx:])
	return newArgs, nil
}
<commit_msg>main: don't print the CLI config into the logs<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"

	"github.com/mitchellh/colorstring"

	"github.com/hashicorp/go-plugin"
	"github.com/hashicorp/terraform/command/format"
	"github.com/hashicorp/terraform/helper/logging"
	"github.com/hashicorp/terraform/terraform"
	"github.com/mattn/go-colorable"
	"github.com/mattn/go-shellwords"
	"github.com/mitchellh/cli"
	"github.com/mitchellh/panicwrap"
	"github.com/mitchellh/prefixedio"
)

const (
	// EnvCLI is the environment variable name to set additional CLI args.
	EnvCLI = "TF_CLI_ARGS"
)

func main() {
	// Override global prefix set by go-dynect during init()
	log.SetPrefix("")
	os.Exit(realMain())
}

func realMain() int {
	var wrapConfig panicwrap.WrapConfig

	// don't re-exec terraform as a child process for easier debugging
	if os.Getenv("TF_FORK") == "0" {
		return wrappedMain()
	}

	if !panicwrap.Wrapped(&wrapConfig) {
		// Determine where logs should go in general (requested by the user)
		logWriter, err := logging.LogOutput()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Couldn't setup log output: %s", err)
			return 1
		}

		// We always send logs to a temporary file that we use in case
		// there is a panic. Otherwise, we delete it.
		logTempFile, err := ioutil.TempFile("", "terraform-log")
		if err != nil {
			fmt.Fprintf(os.Stderr, "Couldn't setup logging tempfile: %s", err)
			return 1
		}
		defer os.Remove(logTempFile.Name())
		defer logTempFile.Close()

		// Setup the prefixed readers that send data properly to
		// stdout/stderr.
		doneCh := make(chan struct{})
		outR, outW := io.Pipe()
		go copyOutput(outR, doneCh)

		// Create the configuration for panicwrap and wrap our executable
		wrapConfig.Handler = panicHandler(logTempFile)
		wrapConfig.Writer = io.MultiWriter(logTempFile, logWriter)
		wrapConfig.Stdout = outW
		wrapConfig.IgnoreSignals = ignoreSignals
		wrapConfig.ForwardSignals = forwardSignals
		exitStatus, err := panicwrap.Wrap(&wrapConfig)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Couldn't start Terraform: %s", err)
			return 1
		}

		// If >= 0, we're the parent, so just exit
		if exitStatus >= 0 {
			// Close the stdout writer so that our copy process can finish
			outW.Close()

			// Wait for the output copying to finish
			<-doneCh

			return exitStatus
		}

		// We're the child, so just close the tempfile we made in order to
		// save file handles since the tempfile is only used by the parent.
		logTempFile.Close()
	}

	// Call the real main
	return wrappedMain()
}

func init() {
	Ui = &cli.PrefixedUi{
		AskPrefix:    OutputPrefix,
		OutputPrefix: OutputPrefix,
		InfoPrefix:   OutputPrefix,
		ErrorPrefix:  ErrorPrefix,
		Ui:           &cli.BasicUi{Writer: os.Stdout},
	}
}

func wrappedMain() int {
	var err error

	// We always need to close the DebugInfo before we exit.
	defer terraform.CloseDebugInfo()

	log.SetOutput(os.Stderr)
	log.Printf(
		"[INFO] Terraform version: %s %s %s",
		Version, VersionPrerelease, GitCommit)
	log.Printf("[INFO] Go runtime version: %s", runtime.Version())
	log.Printf("[INFO] CLI args: %#v", os.Args)

	config, diags := LoadConfig()
	if len(diags) > 0 {
		// Since we haven't instantiated a command.Meta yet, we need to do
		// some things manually here and use some "safe" defaults for things
		// that command.Meta could otherwise figure out in smarter ways.
		Ui.Error("There are some problems with the CLI configuration:")
		for _, diag := range diags {
			earlyColor := &colorstring.Colorize{
				Colors:  colorstring.DefaultColors,
				Disable: true, // Disable color to be conservative until we know better
				Reset:   true,
			}
			Ui.Error(format.Diagnostic(diag, earlyColor, 78))
		}
		if diags.HasErrors() {
			Ui.Error("As a result of the above problems, Terraform may not behave as intended.\n\n")
			// We continue to run anyway, since Terraform has reasonable defaults.
		}
	}

	// In tests, Commands may already be set to provide mock commands
	if Commands == nil {
		initCommands(config)
	}

	// Run checkpoint
	go runCheckpoint(config)

	// Make sure we clean up any managed plugins at the end of this
	defer plugin.CleanupClients()

	// Get the command line args.
	binName := filepath.Base(os.Args[0])
	args := os.Args[1:]

	// Build the CLI so far, we do this so we can query the subcommand.
	cliRunner := &cli.CLI{
		Args:       args,
		Commands:   Commands,
		HelpFunc:   helpFunc,
		HelpWriter: os.Stdout,
	}

	// Prefix the args with any args from the EnvCLI
	args, err = mergeEnvArgs(EnvCLI, cliRunner.Subcommand(), args)
	if err != nil {
		Ui.Error(err.Error())
		return 1
	}

	// Prefix the args with any args from the EnvCLI targeting this command
	suffix := strings.Replace(strings.Replace(
		cliRunner.Subcommand(), "-", "_", -1), " ", "_", -1)
	args, err = mergeEnvArgs(
		fmt.Sprintf("%s_%s", EnvCLI, suffix), cliRunner.Subcommand(), args)
	if err != nil {
		Ui.Error(err.Error())
		return 1
	}

	// We shortcut "--version" and "-v" to just show the version
	for _, arg := range args {
		if arg == "-v" || arg == "-version" || arg == "--version" {
			newArgs := make([]string, len(args)+1)
			newArgs[0] = "version"
			copy(newArgs[1:], args)
			args = newArgs
			break
		}
	}

	// Rebuild the CLI with any modified args.
	log.Printf("[INFO] CLI command args: %#v", args)
	cliRunner = &cli.CLI{
		Name:       binName,
		Args:       args,
		Commands:   Commands,
		HelpFunc:   helpFunc,
		HelpWriter: os.Stdout,

		Autocomplete:          true,
		AutocompleteInstall:   "install-autocomplete",
		AutocompleteUninstall: "uninstall-autocomplete",
	}

	// Pass in the overriding plugin paths from config
	PluginOverrides.Providers = config.Providers
	PluginOverrides.Provisioners = config.Provisioners

	exitCode, err := cliRunner.Run()
	if err != nil {
		Ui.Error(fmt.Sprintf("Error executing CLI: %s", err.Error()))
		return 1
	}

	return exitCode
}

func cliConfigFile() (string, error) {
	mustExist := true
	configFilePath := os.Getenv("TERRAFORM_CONFIG")
	if configFilePath == "" {
		var err error
		configFilePath, err = ConfigFile()
		mustExist = false

		if err != nil {
			log.Printf(
				"[ERROR] Error detecting default CLI config file path: %s",
				err)
		}
	}

	log.Printf("[DEBUG] Attempting to open CLI config file: %s", configFilePath)
	f, err := os.Open(configFilePath)
	if err == nil {
		f.Close()
		return configFilePath, nil
	}

	if mustExist || !os.IsNotExist(err) {
		return "", err
	}

	log.Println("[DEBUG] File doesn't exist, but doesn't need to. Ignoring.")
	return "", nil
}

// copyOutput uses output prefixes to determine whether data on stdout
// should go to stdout or stderr. This is due to panicwrap using stderr
// as the log and error channel.
func copyOutput(r io.Reader, doneCh chan<- struct{}) {
	defer close(doneCh)

	pr, err := prefixedio.NewReader(r)
	if err != nil {
		panic(err)
	}

	stderrR, err := pr.Prefix(ErrorPrefix)
	if err != nil {
		panic(err)
	}
	stdoutR, err := pr.Prefix(OutputPrefix)
	if err != nil {
		panic(err)
	}
	defaultR, err := pr.Prefix("")
	if err != nil {
		panic(err)
	}

	var stdout io.Writer = os.Stdout
	var stderr io.Writer = os.Stderr

	if runtime.GOOS == "windows" {
		stdout = colorable.NewColorableStdout()
		stderr = colorable.NewColorableStderr()

		// colorable is not concurrency-safe when stdout and stderr are the
		// same console, so we need to add some synchronization to ensure that
		// we can't be concurrently writing to both stderr and stdout at
		// once, or else we get intermingled writes that create gibberish
		// in the console.
		wrapped := synchronizedWriters(stdout, stderr)
		stdout = wrapped[0]
		stderr = wrapped[1]
	}

	var wg sync.WaitGroup
	wg.Add(3)
	go func() {
		defer wg.Done()
		io.Copy(stderr, stderrR)
	}()
	go func() {
		defer wg.Done()
		io.Copy(stdout, stdoutR)
	}()
	go func() {
		defer wg.Done()
		io.Copy(stdout, defaultR)
	}()

	wg.Wait()
}

func mergeEnvArgs(envName string, cmd string, args []string) ([]string, error) {
	v := os.Getenv(envName)
	if v == "" {
		return args, nil
	}

	log.Printf("[INFO] %s value: %q", envName, v)
	extra, err := shellwords.Parse(v)
	if err != nil {
		return nil, fmt.Errorf(
			"Error parsing extra CLI args from %s: %s",
			envName, err)
	}

	// Find the command to look for in the args. If there is a space,
	// we need to find the last part.
	search := cmd
	if idx := strings.LastIndex(search, " "); idx >= 0 {
		search = cmd[idx+1:]
	}

	// Find the index to place the flags. We put them exactly
	// after the first non-flag arg.
	idx := -1
	for i, v := range args {
		if v == search {
			idx = i
			break
		}
	}

	// idx points to the exact arg that isn't a flag. We increment
	// by one so that all the copying below expects idx to be the
	// insertion point.
	idx++

	// Copy the args
	newArgs := make([]string, len(args)+len(extra))
	copy(newArgs, args[:idx])
	copy(newArgs[idx:], extra)
	copy(newArgs[len(extra)+idx:], args[idx:])
	return newArgs, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/flosch/pongo2"
	"github.com/russross/blackfriday"
	"gopkg.in/yaml.v2"
)

// Config is a struct that will be used to store Blag's config.
type Config struct {
	Input            *string
	Theme            *string
	Output           *string
	PostsPerPage     *int
	StoryShortLength *int
}

// BlagPostMeta is a struct that will hold a blogpost metadata
type BlagPostMeta struct {
	Title     string
	Timestamp int
	Author    string
	Slug      string
}

// BlagPost is a struct that holds post's content (in html) and its metadata
type BlagPost struct {
	BlagPostMeta
	Content string
}

// Theme holds templates that will be used to render HTML
type Theme struct {
	Page *pongo2.Template
	Post *pongo2.Template
}

// LoadTheme loads pongo2 templates for both pages and posts.
// It will try to load templates from themeDir/page.html and
// themeDir/post.html, and it will panic if that will not succeed.
func LoadTheme(themeDir string) Theme {
	t := Theme{}
	t.Page = pongo2.Must(pongo2.FromFile(path.Join(themeDir, "page.html")))
	t.Post = pongo2.Must(pongo2.FromFile(path.Join(themeDir, "post.html")))
	return t
}

// LoadPost loads post file specified by path argument, and returns BlagPost
// object with data loaded from that file.
func LoadPost(path string) BlagPost {
	file, err := os.Open(path)
	if err != nil {
		panic(err)
	}
	buf := bufio.NewReader(file)
	yamlMeta := ""
	for !strings.HasSuffix(yamlMeta, "\n\n") {
		var s string
		s, err = buf.ReadString('\n')
		yamlMeta += s
	}

	var meta BlagPostMeta
	yaml.Unmarshal([]byte(yamlMeta), &meta)

	markdown, _ := ioutil.ReadAll(buf)
	html := string(blackfriday.MarkdownCommon(markdown))
	return BlagPost{
		meta,
		html,
	}
}

// LoadPosts loads all markdown files in inputDir (not recursive), and returns
// a slice []BlagPost, containing extracted metadata and HTML rendered from
// Markdown.
func LoadPosts(inputDir string) []BlagPost {
	var p []BlagPost
	filelist, err := ioutil.ReadDir(inputDir)
	if err != nil {
		panic(err)
	}
	for _, file := range filelist {
		p = append(p, LoadPost(path.Join(inputDir, file.Name())))
	}
	return p
}

// GenerateHTML generates page's static html and stores it in directory
// specified in config.
func GenerateHTML(config Config, theme Theme, posts []BlagPost) {
	for _, post := range posts {
		postFile, err := os.OpenFile(path.Join(*config.Output, post.Slug+".html"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		defer postFile.Close()
		if err != nil {
			panic(err)
		}
		theme.Post.ExecuteWriter(pongo2.Context{
			"title":     post.Title,
			"author":    post.Author,
			"timestamp": post.Timestamp,
			"content":   post.Content,
		}, postFile)
	}
}

func main() {
	var config Config
	config.Input = flag.String("input", "./input/", "Directory where blog posts are stored (in markdown format)")
	config.Output = flag.String("output", "./output/", "Directory where generated html should be stored")
	config.Theme = flag.String("theme", "./theme/", "Directory containing theme files (templates)")
	config.PostsPerPage = flag.Int("pps", 10, "Post count per page")
	config.StoryShortLength = flag.Int("short", 250, "Length of shortened versions of stories (-1 disables shortening)")
	flag.Parse()

	var theme Theme
	theme = LoadTheme(*config.Theme)

	var posts []BlagPost
	posts = LoadPosts(*config.Input)

	GenerateHTML(config, theme, posts)
}
<commit_msg>Changed default directories to not include platform specific path separator (/)<commit_after>package main

import (
	"bufio"
	"flag"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/flosch/pongo2"
	"github.com/russross/blackfriday"
	"gopkg.in/yaml.v2"
)

// Config is a struct that will be used to store Blag's config.
type Config struct {
	Input            *string
	Theme            *string
	Output           *string
	PostsPerPage     *int
	StoryShortLength *int
}

// BlagPostMeta is a struct that will hold a blogpost metadata
type BlagPostMeta struct {
	Title     string
	Timestamp int
	Author    string
	Slug      string
}

// BlagPost is a struct that holds post's content (in html) and its metadata
type BlagPost struct {
	BlagPostMeta
	Content string
}

// Theme holds templates that will be used to render HTML
type Theme struct {
	Page *pongo2.Template
	Post *pongo2.Template
}

// LoadTheme loads pongo2 templates for both pages and posts.
// It will try to load templates from themeDir/page.html and
// themeDir/post.html, and it will panic if that will not succeed.
func LoadTheme(themeDir string) Theme {
	t := Theme{}
	t.Page = pongo2.Must(pongo2.FromFile(path.Join(themeDir, "page.html")))
	t.Post = pongo2.Must(pongo2.FromFile(path.Join(themeDir, "post.html")))
	return t
}

// LoadPost loads post file specified by path argument, and returns BlagPost
// object with data loaded from that file.
func LoadPost(path string) BlagPost {
	file, err := os.Open(path)
	if err != nil {
		panic(err)
	}
	buf := bufio.NewReader(file)
	yamlMeta := ""
	for !strings.HasSuffix(yamlMeta, "\n\n") {
		var s string
		s, err = buf.ReadString('\n')
		yamlMeta += s
	}

	var meta BlagPostMeta
	yaml.Unmarshal([]byte(yamlMeta), &meta)

	markdown, _ := ioutil.ReadAll(buf)
	html := string(blackfriday.MarkdownCommon(markdown))
	return BlagPost{
		meta,
		html,
	}
}

// LoadPosts loads all markdown files in inputDir (not recursive), and returns
// a slice []BlagPost, containing extracted metadata and HTML rendered from
// Markdown.
func LoadPosts(inputDir string) []BlagPost {
	var p []BlagPost
	filelist, err := ioutil.ReadDir(inputDir)
	if err != nil {
		panic(err)
	}
	for _, file := range filelist {
		p = append(p, LoadPost(path.Join(inputDir, file.Name())))
	}
	return p
}

// GenerateHTML generates page's static html and stores it in directory
// specified in config.
func GenerateHTML(config Config, theme Theme, posts []BlagPost) {
	for _, post := range posts {
		postFile, err := os.OpenFile(path.Join(*config.Output, post.Slug+".html"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		defer postFile.Close()
		if err != nil {
			panic(err)
		}
		theme.Post.ExecuteWriter(pongo2.Context{
			"title":     post.Title,
			"author":    post.Author,
			"timestamp": post.Timestamp,
			"content":   post.Content,
		}, postFile)
	}
}

func main() {
	var config Config
	config.Input = flag.String("input", "input", "Directory where blog posts are stored (in markdown format)")
	config.Output = flag.String("output", "output", "Directory where generated html should be stored")
	config.Theme = flag.String("theme", "theme", "Directory containing theme files (templates)")
	config.PostsPerPage = flag.Int("pps", 10, "Post count per page")
	config.StoryShortLength = flag.Int("short", 250, "Length of shortened versions of stories (-1 disables shortening)")
	flag.Parse()

	var theme Theme
	theme = LoadTheme(*config.Theme)

	var posts []BlagPost
	posts = LoadPosts(*config.Input)

	GenerateHTML(config, theme, posts)
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"io"
	"log"
	"net"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/rneugeba/virtsock/go/vsock"
	"github.com/rneugeba/virtsock/go/hvsock"
)

func run(timeout time.Duration, w *tar.Writer, command string, args ...string) {
	log.Printf("Running %s", command)
	c := exec.Command(command, args...)
	stdoutPipe, err := c.StdoutPipe()
	if err != nil {
		log.Fatalf("Failed to create stdout pipe: %#v", err)
	}
	stderrPipe, err := c.StderrPipe()
	if err != nil {
		log.Fatalf("Failed to create stderr pipe: %#v", err)
	}
	var stdoutBuffer bytes.Buffer
	var stderrBuffer bytes.Buffer
	done := make(chan int)
	go func() {
		io.Copy(&stdoutBuffer, stdoutPipe)
		done <- 0
	}()
	go func() {
		io.Copy(&stderrBuffer, stderrPipe)
		done <- 0
	}()
	var timer *time.Timer
	timer = time.AfterFunc(timeout, func() {
		timer.Stop()
		if c.Process != nil {
			c.Process.Kill()
		}
	})
	_ = c.Run()
	<-done
	<-done
	timer.Stop()

	name := strings.Join(append([]string{path.Base(command)}, args...), " ")

	hdr := &tar.Header{
		Name: name + ".stdout",
		Mode: 0644,
		Size: int64(stdoutBuffer.Len()),
	}
	if err = w.WriteHeader(hdr); err != nil {
		log.Fatalln(err)
	}
	if _, err = w.Write(stdoutBuffer.Bytes()); err != nil {
		log.Fatalln(err)
	}
	hdr = &tar.Header{
		Name: name + ".stderr",
		Mode: 0644,
		Size: int64(stderrBuffer.Len()),
	}
	if err = w.WriteHeader(hdr); err != nil {
		log.Fatalln(err)
	}
	if _, err = w.Write(stderrBuffer.Bytes()); err != nil {
		log.Fatalln(err)
	}

}

func capture(w *tar.Writer) {
	t := 2 * time.Second

	run(t, w, "/bin/date")
	run(t, w, "/bin/uname", "-a")
	run(t, w, "/bin/ps", "uax")
	run(t, w, "/bin/netstat", "-tulpn")
	run(t, w, "/sbin/iptables-save")
	run(t, w, "/sbin/ifconfig", "-a")
	run(t, w, "/sbin/route", "-n")
	run(t, w, "/usr/sbin/brctl", "show")
	run(t, w, "/bin/dmesg")
	run(t, w, "/usr/bin/docker", "ps")
	run(t, w, "/usr/bin/tail", "-100", "/var/log/docker.log")
	run(t, w, "/usr/bin/tail", "-100", "/var/log/messages")
	run(t, w, "/usr/bin/tail", "-100", "/var/log/proxy-vsockd.log")
	run(t, w, "/usr/bin/tail", "-100", "/var/log/vsudd.log")
	run(t, w, "/bin/mount")
	run(t, w, "/bin/df")
	run(t, w, "/bin/ls", "-l", "/var")
	run(t, w, "/bin/ls", "-l", "/var/lib")
	run(t, w, "/bin/ls", "-l", "/var/lib/docker")
	run(t, w, "/usr/bin/diagnostics")
	run(t, w, "/bin/ping", "-w", "5", "8.8.8.8")
	run(t, w, "/bin/cat", "/etc/docker/daemon.json")
	run(t, w, "/bin/cat", "/etc/network/interfaces")
	run(t, w, "/bin/cat", "/etc/resolv.conf")
	run(t, w, "/bin/cat", "/etc/sysctl.conf")
	run(t, w, "/usr/bin/dig", "docker.com")
	run(t, w, "/usr/bin/wget", "-O", "-", "http://www.docker.com/")

	// Dump the database
	dbBase := "/Database/branch/master/ro"
	filepath.Walk(dbBase, func(path string, f os.FileInfo, err error) error {
		if f.Mode().IsRegular() {
			run(t, w, "/bin/cat", path)
		}
		return nil
	})
}

func main() {
	listeners := make([]net.Listener, 0)

	ip, err := net.Listen("tcp", ":62374")
	if err != nil {
		log.Printf("Failed to bind to TCP port 62374: %#v", err)
	} else {
		listeners = append(listeners, ip)
	}
	vsock, err := vsock.Listen(uint(62374))
	if err != nil {
		log.Printf("Failed to bind to vsock port 62374: %#v", err)
	} else {
		listeners = append(listeners, vsock)
	}
	svcid, _ := hvsock.GuidFromString("445BA2CB-E69B-4912-8B42-D7F494D007EA")
	hvsock, err := hvsock.Listen(hvsock.HypervAddr{VmId: hvsock.GUID_WILDCARD, ServiceId: svcid})
	if err != nil {
		log.Printf("Failed to bind to hvsock port: %#v", err)
	} else {
		listeners = append(listeners, hvsock)
	}

	for _, l := range listeners {
		go func(l net.Listener) {
			for {
				conn, err := l.Accept()
				if err != nil {
					log.Printf("Error accepting connection: %#v", err)
					return // no more listening
				}
				go func(conn net.Conn) {
					w := tar.NewWriter(conn)
					capture(w)
					if err := w.Close(); err != nil {
						log.Println(err)
					}
					conn.Close()
				}(conn)
			}
		}(l)
	}
	forever := make(chan int)
	<-forever
}
<commit_msg>diag: improve diagnostics<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"io"
	"log"
	"net"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/rneugeba/virtsock/go/vsock"
	"github.com/rneugeba/virtsock/go/hvsock"
)

func run(timeout time.Duration, w *tar.Writer, command string, args ...string) {
	log.Printf("Running %s", command)
	c := exec.Command(command, args...)
	stdoutPipe, err := c.StdoutPipe()
	if err != nil {
		log.Fatalf("Failed to create stdout pipe: %#v", err)
	}
	stderrPipe, err := c.StderrPipe()
	if err != nil {
		log.Fatalf("Failed to create stderr pipe: %#v", err)
	}
	var stdoutBuffer bytes.Buffer
	var stderrBuffer bytes.Buffer
	done := make(chan int)
	go func() {
		io.Copy(&stdoutBuffer, stdoutPipe)
		done <- 0
	}()
	go func() {
		io.Copy(&stderrBuffer, stderrPipe)
		done <- 0
	}()
	var timer *time.Timer
	timer = time.AfterFunc(timeout, func() {
		timer.Stop()
		if c.Process != nil {
			c.Process.Kill()
		}
	})
	_ = c.Run()
	<-done
	<-done
	timer.Stop()

	name := strings.Join(append([]string{path.Base(command)}, args...), " ")

	hdr := &tar.Header{
		Name: name + ".stdout",
		Mode: 0644,
		Size: int64(stdoutBuffer.Len()),
	}
	if err = w.WriteHeader(hdr); err != nil {
		log.Fatalln(err)
	}
	if _, err = w.Write(stdoutBuffer.Bytes()); err != nil {
		log.Fatalln(err)
	}
	hdr = &tar.Header{
		Name: name + ".stderr",
		Mode: 0644,
		Size: int64(stderrBuffer.Len()),
	}
	if err = w.WriteHeader(hdr); err != nil {
		log.Fatalln(err)
	}
	if _, err = w.Write(stderrBuffer.Bytes()); err != nil {
		log.Fatalln(err)
	}

}

func capture(w *tar.Writer) {
	t := 2 * time.Second

	run(t, w, "/bin/date")
	run(t, w, "/bin/uname", "-a")
	run(t, w, "/bin/ps", "uax")
	run(t, w, "/bin/netstat", "-tulpn")
	run(t, w, "/sbin/iptables-save")
	run(t, w, "/sbin/ifconfig", "-a")
	run(t, w, "/sbin/route", "-n")
	run(t, w, "/usr/sbin/brctl", "show")
	run(t, w, "/bin/dmesg")
	run(t, w, "/usr/bin/docker", "ps")
	run(t, w, "/usr/bin/tail", "-100", "/var/log/docker.log")
	run(t, w, "/usr/bin/tail", "-100", "/var/log/messages")
	run(t, w, "/usr/bin/tail", "-100", "/var/log/proxy-vsockd.log")
	run(t, w, "/usr/bin/tail", "-100", "/var/log/service-port-opener.log")
	run(t, w, "/usr/bin/tail", "-100", "/var/log/vsudd.log")
	run(t, w, "/bin/mount")
	run(t, w, "/bin/df")
	run(t, w, "/bin/ls", "-l", "/var")
	run(t, w, "/bin/ls", "-l", "/var/lib")
	run(t, w, "/bin/ls", "-l", "/var/lib/docker")
	run(t, w, "/usr/bin/diagnostics")
	run(t, w, "/bin/ping", "-w", "5", "8.8.8.8")
	run(t, w, "/bin/cat", "/etc/docker/daemon.json")
	run(t, w, "/bin/cat", "/etc/network/interfaces")
	run(t, w, "/bin/cat", "/etc/resolv.conf")
	run(t, w, "/bin/cat", "/etc/sysctl.conf")
	run(t, w, "/usr/bin/dig", "docker.com")
	run(t, w, "/usr/bin/dig", "@8.8.8.8", "docker.com")
	run(t, w, "/usr/bin/wget", "-O", "-", "http://www.docker.com/")
	run(t, w, "/usr/bin/wget", "-O", "-", "http://104.239.220.248/") // a www.docker.com address
	run(t, w, "/usr/bin/wget", "-O", "-", "http://216.58.213.68/") // a www.google.com address
	run(t, w, "/usr/bin/wget", "-O", "-", "http://91.198.174.192/") // a www.wikipedia.com address

	// Dump the database
	dbBase := "/Database/branch/master/ro"
	filepath.Walk(dbBase, func(path string, f os.FileInfo, err error) error {
		if f.Mode().IsRegular() {
			run(t, w, "/bin/cat", path)
		}
		return nil
	})
}

func main() {
	listeners := make([]net.Listener, 0)

	ip, err := net.Listen("tcp", ":62374")
	if err != nil {
		log.Printf("Failed to bind to TCP port 62374: %#v", err)
	} else {
		listeners = append(listeners, ip)
	}
	vsock, err := vsock.Listen(uint(62374))
	if err != nil {
		log.Printf("Failed to bind to vsock port 62374: %#v", err)
	} else {
		listeners = append(listeners, vsock)
	}
	svcid, _ := hvsock.GuidFromString("445BA2CB-E69B-4912-8B42-D7F494D007EA")
	hvsock, err := hvsock.Listen(hvsock.HypervAddr{VmId: hvsock.GUID_WILDCARD, ServiceId: svcid})
	if err != nil {
		log.Printf("Failed to bind to hvsock port: %#v", err)
	} else {
		listeners = append(listeners, hvsock)
	}

	for _, l := range listeners {
		go func(l net.Listener) {
			for {
				conn, err := l.Accept()
				if err != nil {
					log.Printf("Error accepting connection: %#v", err)
					return // no more listening
				}
				go func(conn net.Conn) {
					w := tar.NewWriter(conn)
					capture(w)
					if err := w.Close(); err != nil {
						log.Println(err)
					}
					conn.Close()
				}(conn)
			}
		}(l)
	}
	forever := make(chan int)
	<-forever
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/BurntSushi/toml"
	"github.com/yanc0/collectd-http-server/plugins"
	"io/ioutil"
	"log"
	"net/http"
	"os"
)

var pluginList []plugins.Plugin
var config Config

type BasicAuth struct {
	Active   bool     `toml:"active"`
	Accounts []string `toml:"accounts"`
}

type Config struct {
	Listen         string                        `toml:"listen"`
	Port           int                           `toml:"port"`
	BasicAuth      *BasicAuth                    `toml:"basic_auth"`
	GraphitePlugin *plugins.GraphitePluginConfig `toml:"graphite_plugin"`
	ConsolePlugin  *plugins.ConsolePluginConfig  `toml:"console_plugin"`
}

func loadConfig(configPath string) {
	configPath = os.ExpandEnv(configPath)

	configStr, err := ioutil.ReadFile(configPath)
	if err != nil {
		log.Fatalln(err.Error())
	}

	err = toml.Unmarshal(configStr, &config) //global config
	if err != nil {
		log.Fatalln(err.Error())
	}

	if config.Listen == "" {
		config.Listen = "127.0.0.1"
	}

	if config.Port == 0 {
		config.Port = 9223
	}
}

func loadPlugins(config *Config) {
	if config.GraphitePlugin != nil && config.GraphitePlugin.Active {
		pluginList = append(pluginList, plugins.NewGraphitePlugin(config.GraphitePlugin))
	}
	if config.ConsolePlugin != nil && config.ConsolePlugin.Active {
		pluginList = append(pluginList, plugins.NewConsolePlugin(config.ConsolePlugin))
	}

	if len(pluginList) < 1 {
		log.Println("[WARN] No plugins loaded")
	} else {
		log.Println("[INFO] Plugins loaded")
	}
}

func initPlugins() {
	for _, p := range pluginList {
		err := p.Init()
		if err != nil {
			log.Fatalln("[FATAL]", p.Name(), err.Error())
		}
	}
}

func main() {
	configPath := flag.String("configPath",
		"/etc/collectd-http-server/config.toml",
		"Config path")
	flag.Parse()

	loadConfig(*configPath)
	loadPlugins(&config)
	initPlugins()

	listen := fmt.Sprintf("%s:%d", config.Listen, config.Port)

	http.HandleFunc("/", auth(handlerMetricPost))
	log.Fatal(http.ListenAndServe(listen, nil))
}
<commit_msg>more precise log on plugin loading<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/BurntSushi/toml"
	"github.com/yanc0/collectd-http-server/plugins"
	"io/ioutil"
	"log"
	"net/http"
	"os"
)

var pluginList []plugins.Plugin
var config Config

type BasicAuth struct {
	Active   bool     `toml:"active"`
	Accounts []string `toml:"accounts"`
}

type Config struct {
	Listen         string                        `toml:"listen"`
	Port           int                           `toml:"port"`
	BasicAuth      *BasicAuth                    `toml:"basic_auth"`
	GraphitePlugin *plugins.GraphitePluginConfig `toml:"graphite_plugin"`
	ConsolePlugin  *plugins.ConsolePluginConfig  `toml:"console_plugin"`
}

func loadConfig(configPath string) {
	configPath = os.ExpandEnv(configPath)

	configStr, err := ioutil.ReadFile(configPath)
	if err != nil {
		log.Fatalln(err.Error())
	}

	err = toml.Unmarshal(configStr, &config) //global config
	if err != nil {
		log.Fatalln(err.Error())
	}

	if config.Listen == "" {
		config.Listen = "127.0.0.1"
	}

	if config.Port == 0 {
		config.Port = 9223
	}
}

func loadPlugins(config *Config) {
	if config.GraphitePlugin != nil && config.GraphitePlugin.Active {
		pluginList = append(pluginList, plugins.NewGraphitePlugin(config.GraphitePlugin))
	}
	if config.ConsolePlugin != nil && config.ConsolePlugin.Active {
		pluginList = append(pluginList, plugins.NewConsolePlugin(config.ConsolePlugin))
	}

	if len(pluginList) < 1 {
		log.Println("[WARN] No plugins loaded")
	} else {
		log.Println("[INFO]", len(pluginList), "Plugins loaded")
	}
}

func initPlugins() {
	for _, p := range pluginList {
		err := p.Init()
		if err != nil {
			log.Fatalln("[FATAL]", p.Name(), err.Error())
		}
	}
}

func main() {
	configPath := flag.String("configPath",
		"/etc/collectd-http-server/config.toml",
		"Config path")
	flag.Parse()

	loadConfig(*configPath)
	loadPlugins(&config)
	initPlugins()

	listen := fmt.Sprintf("%s:%d", config.Listen, config.Port)

	http.HandleFunc("/", auth(handlerMetricPost))
	log.Fatal(http.ListenAndServe(listen, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"regexp"
)

type tokenType int

const (
	tokenWord tokenType = iota
	tokenWhitespace
	tokenPunctuation
	tokenEof
)

var names = map[tokenType]string{
	tokenWord:        "WORD",
	tokenWhitespace:  "SPACE",
	tokenPunctuation: "PUNCTUATION",
	tokenEof:         "EOF",
}

var wordRegexp = regexp.MustCompile("[A-za-z]+")
var whitespaceRegexp = regexp.MustCompile("[\\s]+")
var punctuationRegexp = regexp.MustCompile("[\\p{P}\\p{S}]+")

type token struct {
	value     string
	pos       int
	tokenType tokenType
}

func (tok token) String() string {
	return fmt.Sprintf("{%s '%s' %d}", names[tok.tokenType], tok.value, tok.pos)
}

type stateFn func(*lexer) stateFn

type lexer struct {
	start  int // The position of the last emission
	pos    int // The current position of the lexer
	input  string
	tokens chan token
	state  stateFn
}

func (l *lexer) next() (val string) {
	if l.pos >= len(l.input) {
		l.pos++
		return ""
	}

	val = l.input[l.pos : l.pos+1]

	l.pos++

	return
}

func (l *lexer) backup() {
	l.pos--
}

func (l *lexer) peek() (val string) {
	val = l.next()

	l.backup()

	return
}

func (l *lexer) emit(t tokenType) {
	val := l.input[l.start:l.pos]
	tok := token{val, l.start, t}
	l.tokens <- tok
	l.start = l.pos
}

func (l *lexer) tokenize() {
	for l.state = lexData; l.state != nil; {
		l.state = l.state(l)
	}
}

func lexData(l *lexer) stateFn {
	v := l.peek()
	switch {
	case v == "":
		l.emit(tokenEof)
		return nil

	case punctuationRegexp.MatchString(v):
		return lexPunctuation

	case whitespaceRegexp.MatchString(v):
		return lexWhitespace
	}

	return lexWord
}

func lexPunctuation(l *lexer) stateFn {
	matched := punctuationRegexp.FindString(l.input[l.pos:])
	l.pos += len(matched)
	l.emit(tokenPunctuation)

	return lexData
}

func lexWhitespace(l *lexer) stateFn {
	matched := whitespaceRegexp.FindString(l.input[l.pos:])
	l.pos += len(matched)
	l.emit(tokenWhitespace)

	return lexData
}

func lexWord(l *lexer) stateFn {
	matched := wordRegexp.FindString(l.input[l.pos:])
	l.pos += len(matched)
	l.emit(tokenWord)

	return lexData
}

func newLexer(input string) *lexer {
	return &lexer{0, 0, input, make(chan token), nil}
}

func main() {
	lex := newLexer("This   is  a test-aculous test, sir...")

	go lex.tokenize()

	for {
		tok := <-lex.tokens
		fmt.Println(tok)
		if tok.tokenType == tokenEof {
			break
		}
	}
}
<commit_msg>Fixed typo in alpha regex<commit_after>package main

import (
	"fmt"
	"regexp"
)

type tokenType int

const (
	tokenWord tokenType = iota
	tokenWhitespace
	tokenPunctuation
	tokenEof
)

var names = map[tokenType]string{
	tokenWord:        "WORD",
	tokenWhitespace:  "SPACE",
	tokenPunctuation: "PUNCTUATION",
	tokenEof:         "EOF",
}

var wordRegexp = regexp.MustCompile("[A-Za-z]+")
var whitespaceRegexp = regexp.MustCompile("[\\s]+")
var punctuationRegexp = regexp.MustCompile("[\\p{P}\\p{S}]+")

type token struct {
	value     string
	pos       int
	tokenType tokenType
}

func (tok token) String() string {
	return fmt.Sprintf("{%s '%s' %d}", names[tok.tokenType], tok.value, tok.pos)
}

type stateFn func(*lexer) stateFn

type lexer struct {
	start  int // The position of the last emission
	pos    int // The current position of the lexer
	input  string
	tokens chan token
	state  stateFn
}

func (l *lexer) next() (val string) {
	if l.pos >= len(l.input) {
		l.pos++
		return ""
	}

	val = l.input[l.pos : l.pos+1]

	l.pos++

	return
}

func (l *lexer) backup() {
	l.pos--
}

func (l *lexer) peek() (val string) {
	val = l.next()

	l.backup()

	return
}

func (l *lexer) emit(t tokenType) {
	val := l.input[l.start:l.pos]
	tok := token{val, l.start, t}
	l.tokens <- tok
	l.start = l.pos
}

func (l *lexer) tokenize() {
	for l.state = lexData; l.state != nil; {
		l.state = l.state(l)
	}
}

func lexData(l *lexer) stateFn {
	v := l.peek()
	switch {
	case v == "":
		l.emit(tokenEof)
		return nil

	case punctuationRegexp.MatchString(v):
		return lexPunctuation

	case whitespaceRegexp.MatchString(v):
		return lexWhitespace
	}

	return lexWord
}

func lexPunctuation(l *lexer) stateFn {
	matched := punctuationRegexp.FindString(l.input[l.pos:])
	l.pos += len(matched)
	l.emit(tokenPunctuation)

	return lexData
}

func lexWhitespace(l *lexer) stateFn {
	matched := whitespaceRegexp.FindString(l.input[l.pos:])
	l.pos += len(matched)
	l.emit(tokenWhitespace)

	return lexData
}

func lexWord(l *lexer) stateFn {
	matched := wordRegexp.FindString(l.input[l.pos:])
	l.pos += len(matched)
	l.emit(tokenWord)

	return lexData
}

func newLexer(input string) *lexer {
	return &lexer{0, 0, input, make(chan token), nil}
}

func main() {
	lex := newLexer("This   is  a test-aculous test, sir...")

	go lex.tokenize()

	for {
		tok := <-lex.tokens
		fmt.Println(tok)
		if tok.tokenType == tokenEof {
			break
		}
	}
}
<|endoftext|>


<commit_before>package bookstore

import (
	"fmt"
	"net/http"

	"github.com/yhat/scrape"
	"golang.org/x/net/html"
	"google.golang.org/appengine"
	"google.golang.org/appengine/user"
)

// Match is a result returned from scraping
type Match struct {
	CourseCode string // PHYS-1030-FA
	Synonym    string // 643369
	Title      string // Intro Appl Phys I (Mechanics)
	Instructor string // Dr. Mark C. Gallagher
	Books      string // Link?
	Term       string // Fall
	Department string // Physics
	YearLevel  string // 1
}

func init() {
	http.HandleFunc("/", auth)
	http.HandleFunc("/scrape", HomeHandler)
}

// HomeHandler handles the home page
func HomeHandler(w http.ResponseWriter, r *http.Request) {

	fmt.Fprint(w, "Hello, Benny world!")
	Scrape("http://timetable.lakeheadu.ca/2015FW_UG_TBAY/phys.html", r)
}

func auth(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	u := user.Current(c)
	if u == nil {
		url, err := user.LoginURL(c, r.URL.String())
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		w.Header().Set("Location", url)
		w.WriteHeader(http.StatusFound)
		return
	}
	fmt.Fprintf(w, "Hello, %v!", u)
}

// Scrape finds and serializes the data from Lakehead's
// site.
func Scrape(url string, r *http.Request) (Matches []*Match, err error) {

	// Required for logging
	c := appengine.NewContext(r)

	res, err := http.Get(url)
	if err != nil {
		fmt.Println("Error :", err)
		return nil, err
	}

	root, err := html.Parse(res.Body)
	if err != nil {
		fmt.Println("Error :", err)
		return nil, err
	}

	// Yhat's package expects atomic values for tags, see
	// https://godoc.org/golang.org/x/net/html/atom if you
	// need a different tag.
	data := scrape.FindAll(root, scrape.ByTag(0x10502))
	// matches := make([]*Match, len(data))
	for _, match := range data {

		// c.Infof doesn't exist???
		c.Infof("Match: %v", match)
	}

	return nil, nil
}
<commit_msg>Add async scraping code<commit_after>package bookstore

import (
	"fmt"
	"net/http"

	"github.com/yhat/scrape"
	"golang.org/x/net/html"
	"google.golang.org/appengine"
	"google.golang.org/appengine/user"
)

// urls are all the absolute URLs that need to be scraped
var urls = []string{
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/anth.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/apbi.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/biol.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/busi.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/chem.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/clas.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/comp.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/crim.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/econ.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/educ.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/engi.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/finn.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/fren.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/geoa.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/geog.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/geol.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/gero.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/gsci.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/hist.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/indi.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/intd.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/ital.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/kine.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/lang.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/laws.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/ling.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/math.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/mdst.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/meds.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/musi.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/nacc.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/nort.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/nrmt.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/nurs.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/ojib.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/outd.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/phil.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/phys.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/poli.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/psyc.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/reli.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/soci.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/sowk.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/span.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/visu.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/wate.html",
	"http://timetable.lakeheadu.ca/2015FW_UG_TBAY/wome.html",
}

// Match is a result returned from scraping
type Match struct {
	CourseCode string // PHYS-1030-FA
	Synonym    string // 643369
	Title      string // Intro Appl Phys I (Mechanics)
	Instructor string // Dr. Mark C. Gallagher
	Books      string // Link?
	Term       string // Fall
	Department string // Physics
	YearLevel  string // 1
}

func init() {
	http.HandleFunc("/", AuthHandler)
	http.HandleFunc("/scrape", HomeHandler)
}

// HomeHandler handles the home page
func HomeHandler(w http.ResponseWriter, r *http.Request) {

	fmt.Fprint(w, "Home")
}

// AuthHandler allows a user to login and greets them
func AuthHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	u := user.Current(c)
	if u == nil {
		url, err := user.LoginURL(c, r.URL.String())
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		w.Header().Set("Location", url)
		w.WriteHeader(http.StatusFound)
		return
	}
	fmt.Fprintf(w, "Hello, %v!", u)
}

// HTTPResponse is the struct that holds our response
// data
type HTTPResponse struct {
	url      string
	response *http.Response
	err      error
}

// Scrape finds and serializes the data from Lakehead's
// site.
func Scrape(resp *HTTPResponse) {

	root, err := html.Parse(resp.response.Body)
	if err != nil {
		fmt.Println("Error :", err)
		return
	}

	// Yhat's package expects atomic values for tags, see
	// https://godoc.org/golang.org/x/net/html/atom if you
	// need a different tag.
	data := scrape.FindAll(root, scrape.ByTag(0x10502))
	for _, match := range data {
		fmt.Println(scrape.Text(match))
	}

	return
}

func asyncHTTPGet(urls []string) []*HTTPResponse {

	ch := make(chan *HTTPResponse)
	responses := []*HTTPResponse{}

	for _, url := range urls {
		go func(url string) {
			fmt.Printf("Fetching %s \n", url)
			resp, err := http.Get(url)
			if err != nil {
				fmt.Println("Error: ", err)
				ch <- &HTTPResponse{url, nil, err}
			} else {
				ch <- &HTTPResponse{url, resp, err}
			}
		}(url)
	}

	for {
		select {
		case r := <-ch:

			// Scrape
			Scrape(r)

			responses = append(responses, r)
			if len(responses) == len(urls) {
				return responses
			}

			defer r.response.Body.Close()
		}
	}

	return responses
}
<|endoftext|>


<commit_before>package main

import "fmt"

// Match-Nexter interface
// Allows for magic to happen inside the type.
type MatchNexter interface {
  // Is this rune a match?
  Match(rune) bool
  // Returns next possible states
  Next() []MatchNexter
}

// Matching function used by State type
type StateMatcherFunc func(rune) bool

// State type implements MatchNexter
type State struct {
  // Matcher Function returns true on a match.
  Matcher StateMatcherFunc
  // Possible next states
  // if this state is true.
  Nexts []MatchNexter
}

// Returns true on matching rune
func (s *State) Match(r rune) bool {
  return s.Matcher(r)
}

// Returns possible next states
func (s *State) Next() []MatchNexter {
  return s.Nexts
}

type Lexer struct {
  Runes <-chan rune
  States []MatchNexter
}

// Uses an array of possible MatchNexters
// Each MatchNexters' Match function is called
// exactly once per set of MatchNexters.
// If Match returns true, then Next is called.
func (l *Lexer) Lex(out chan rune) {
  defer close(out)
  for r := range l.Runes {
    var nextStates []MatchNexter
    for _, s := range l.States {
      if s.Match(r) {
        out <- r
        nextStates = append(nextStates, s.Next()...)
      }
    }
    l.States = nextStates
    if len(l.States) == 0 {
      return // No more possible states.
    }
  }
}

func main() {
  runes := make(chan rune)
  out := make(chan rune)
  loopMatcher := []MatchNexter{
    MatchNexter(&State{
      Matcher: func(r rune) bool {
        return r == 'a'
      },
      Nexts: []MatchNexter{
        MatchNexter(&State{
          Matcher: func (r rune) bool {
            return r == 'b'
          },
        }),
      },
    }),
  }
  // Generate loop to beginning of series,
  // acts like (ab)* in regex.
  loopMatcher[0].(*State).Nexts[0].(*State).Nexts = []MatchNexter{ loopMatcher[0] }
  l := Lexer{
    Runes: runes,
    States: loopMatcher,
  }
  // Asynchrounously lex input
  go l.Lex(out)
  // Asynchrounously create input and close channel.
  // On close of channel, l.Lex will close out.
  go func() {
    for _, r := range "abababc" {
      runes <- r
    }
    close(runes)
  }()
  // Keep-alive until out is closed.
  for r := range out {
    fmt.Println("Found match '"+string(r)+"'")
  }
}
<commit_msg>update<commit_after>package main

import (
  "fmt"
  "os"
  "bufio"
)

// Match-Nexter interface
// Allows for magic to happen inside the type.
type MatchNexter interface {
  // Returns next possible states
  Match(rune) []MatchNexter
}

// Matching function used by State type
type StateMatcherFunc func(rune) []MatchNexter

// State type implements MatchNexter
type State struct {
  // Matcher Function returns true on a match.
  Matcher StateMatcherFunc
}

// Returns true on matching rune
func (s *State) Match(r rune) []MatchNexter {
  return s.Matcher(r)
}

type Lexer struct {
  States []MatchNexter
  Hooks []Lexer
  Matched string
}

type Match struct {
   Success bool
   Match string
}

// Uses an array of possible MatchNexters
// Each MatchNexters' Match function is called
// exactly once per set of MatchNexters.
// If Match returns true, then Next is called.
func (l *Lexer) Lex(runes <-chan rune) chan Match {
  out := make(chan Match)
  go func(runes <-chan rune) {
    defer close(out)
    for r := range runes {
      var nextStates []MatchNexter
      for _, s := range l.States {
        if s != nil {
          nextStates = append(nextStates, s.Match(r)...)
        } else {
          // Have encountered a possible exit condition,
          // place a hook.
          l.Hooks = append([]Lexer{ *l }, l.Hooks...)
        }
      }
      l.States = nextStates
      if len(l.States) == 0 {
        if len(l.Hooks) > 0 {
          out <- Match{ Success: true, Match: l.Hooks[0].Matched }
        } else {
          out <- Match{ Success: false, Match: l.Matched }
        }
        return // No more possible states.
      } else {
        l.Matched += string(r)
      }
    }
  }(runes)
  return out
}

type RuneMatcher struct {
  MatchRune rune
  Nexts []MatchNexter
}

func (rm *RuneMatcher) Match(r rune) []MatchNexter {
  if r == rm.MatchRune {
    return rm.Nexts
  } else {
    return []MatchNexter{}
  }
}

// Creates chain of RuneMatchers to match
// an entire string.
func StringMatcher(match string) (first, last *RuneMatcher) {
  for _, r := range match {
    rm := &RuneMatcher{ MatchRune: r }
    if last != nil {
      last.Nexts = []MatchNexter{ MatchNexter(rm) }
    } else {
      first = rm
    }
    last = rm
  }
  return
}

func main() {
  runes := make(chan rune)

  // Generate loop to beginning of series,
  // acts like (ab)* in regex.
  first, last := StringMatcher("ab")
  last.Nexts = []MatchNexter{ first, nil }

  l := Lexer{
    States: []MatchNexter{ MatchNexter(first) },
  }

  // Asynchrounously lex input
  out := l.Lex(runes)

  // Asynchrounously create input and close channel.
  // On close of channel, l.Lex will close out.
  go func() {
    line, _ := bufio.NewReader(os.Stdin).ReadString('\n')
    for _, r := range line {
      runes <- r
    }
    close(runes)
  }()

  // Keep-alive until out is closed.
  for s := range out {
    if s.Success {
      fmt.Println("Found match '"+s.Match+"'")
    } else {
      fmt.Println("No match found, got: "+s.Match);
    }
  }
}
<|endoftext|>


<commit_before>package main

import "github.com/spf13/cobra"

func main() {
	var rootCommand = &cobra.Command{
		Use:   "spote-service",
		Short: "Spote is a headless media server for Spotify",
		Long:  `Spote is a headless media server for Spotify.`,
		Run: func(cmd *cobra.Command, args []string) {
		},
	}

	rootCommand.Execute()
}
<commit_msg>chore(main): init config and commands<commit_after>package main

func main() {
	initCommands()
	initConfig()

	spoteCommand.Execute()
}
<|endoftext|>


<commit_before>// Package c2go contains the main function for running the executable.
//
// Installation
//
//     go get -u github.com/elliotchance/c2go
//
// Usage
//
//     c2go myfile.c
//
package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"errors"

	"github.com/elliotchance/c2go/ast"
	"github.com/elliotchance/c2go/preprocessor"
	"github.com/elliotchance/c2go/program"
	"github.com/elliotchance/c2go/transpiler"
)

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.17.1 Samarium 2017-11-09"

var stderr io.Writer = os.Stderr

// ProgramArgs defines the options available when processing the program. There
// is no constructor since the zeroed out values are the appropriate defaults -
// you need only set the options you need.
//
// TODO: Better separation on CLI modes
// https://github.com/elliotchance/c2go/issues/134
//
// Do not instantiate this directly. Instead use DefaultProgramArgs(); then
// modify any specific attributes.
type ProgramArgs struct {
	verbose     bool
	ast         bool
	inputFiles  []string
	outputFile  string
	packageName string

	// A private option to output the Go as a *_test.go file.
	outputAsTest bool
}

// DefaultProgramArgs default value of ProgramArgs
func DefaultProgramArgs() ProgramArgs {
	return ProgramArgs{
		verbose:      false,
		ast:          false,
		packageName:  "main",
		outputAsTest: false,
	}
}

func readAST(data []byte) []string {
	return strings.Split(string(data), "\n")
}

type treeNode struct {
	indent int
	node   ast.Node
}

func convertLinesToNodes(lines []string) []treeNode {
	nodes := make([]treeNode, len(lines))
	var counter int
	for _, line := range lines {
		if strings.TrimSpace(line) == "" {
			continue
		}

		// It is tempting to discard null AST nodes, but these may
		// have semantic importance: for example, they represent omitted
		// for-loop conditions, as in for(;;).
		line = strings.Replace(line, "<<<NULL>>>", "NullStmt", 1)
		trimmed := strings.TrimLeft(line, "|\\- `")
		node := ast.Parse(trimmed)
		indentLevel := (len(line) - len(trimmed)) / 2
		nodes[counter] = treeNode{indentLevel, node}
		counter++
	}
	nodes = nodes[0:counter]

	return nodes
}

func convertLinesToNodesParallel(lines []string) []treeNode {
	// function f separate full list on 2 parts and
	// then each part can recursive run function f
	var f func([]string, int) []treeNode

	f = func(lines []string, deep int) []treeNode {
		deep = deep - 2
		part := len(lines) / 2

		var tr1 = make(chan []treeNode)
		var tr2 = make(chan []treeNode)

		go func(lines []string, deep int) {
			if deep <= 0 || len(lines) < deep {
				tr1 <- convertLinesToNodes(lines)
				return
			}
			tr1 <- f(lines, deep)
		}(lines[0:part], deep)

		go func(lines []string, deep int) {
			if deep <= 0 || len(lines) < deep {
				tr2 <- convertLinesToNodes(lines)
				return
			}
			tr2 <- f(lines, deep)
		}(lines[part:], deep)

		defer close(tr1)
		defer close(tr2)

		return append(<-tr1, <-tr2...)
	}

	// Parameter of deep - can be any, but effective to use
	// same amount of CPU
	return f(lines, runtime.NumCPU())
}

// buildTree converts an array of nodes, each prefixed with a depth into a tree.
func buildTree(nodes []treeNode, depth int) []ast.Node {
	if len(nodes) == 0 {
		return []ast.Node{}
	}

	// Split the list into sections, treat each section as a tree with its own
	// root.
	sections := [][]treeNode{}
	for _, node := range nodes {
		if node.indent == depth {
			sections = append(sections, []treeNode{node})
		} else {
			sections[len(sections)-1] = append(sections[len(sections)-1], node)
		}
	}

	results := []ast.Node{}
	for _, section := range sections {
		slice := []treeNode{}
		for _, n := range section {
			if n.indent > depth {
				slice = append(slice, n)
			}
		}

		children := buildTree(slice, depth+1)
		for _, child := range children {
			section[0].node.AddChild(child)
		}
		results = append(results, section[0].node)
	}

	return results
}

// Start begins transpiling an input file.
func Start(args ProgramArgs) (err error) {
	if args.verbose {
		fmt.Println("Start tanspiling ...")
	}

	if os.Getenv("GOPATH") == "" {
		return fmt.Errorf("The $GOPATH must be set")
	}

	// 1. Compile it first (checking for errors)
	for _, in := range args.inputFiles {
		_, err := os.Stat(in)
		if err != nil {
			return fmt.Errorf("Input file %s is not found", in)
		}
	}

	// 2. Preprocess
	if args.verbose {
		fmt.Println("Running clang preprocessor...")
	}

	pp, err := preprocessor.Analyze(args.inputFiles)
	if err != nil {
		return err
	}

	if args.verbose {
		fmt.Println("Writing preprocessor ...")
	}
	dir, err := ioutil.TempDir("", "c2go")
	if err != nil {
		return fmt.Errorf("Cannot create temp folder: %v", err)
	}
	defer os.RemoveAll(dir) // clean up

	ppFilePath := path.Join(dir, "pp.c")
	err = ioutil.WriteFile(ppFilePath, pp, 0644)
	if err != nil {
		return fmt.Errorf("writing to %s failed: %v", ppFilePath, err)
	}

	// 3. Generate JSON from AST
	if args.verbose {
		fmt.Println("Running clang for AST tree...")
	}
	astPP, err := exec.Command("clang", "-Xclang", "-ast-dump", "-fsyntax-only", "-fno-color-diagnostics", ppFilePath).Output()
	if err != nil {
		// If clang fails it still prints out the AST, so we have to run it
		// again to get the real error.
		errBody, _ := exec.Command("clang", ppFilePath).CombinedOutput()

		panic("clang failed: " + err.Error() + ":\n\n" + string(errBody))
	}

	if args.verbose {
		fmt.Println("Reading clang AST tree...")
	}
	lines := readAST(astPP)
	if args.ast {
		for _, l := range lines {
			fmt.Println(l)
		}
		fmt.Println()

		return nil
	}

	p := program.NewProgram()
	p.Verbose = args.verbose
	p.OutputAsTest = args.outputAsTest

	// Converting to nodes
	if args.verbose {
		fmt.Println("Converting to nodes...")
	}
	nodes := convertLinesToNodesParallel(lines)

	// build tree
	if args.verbose {
		fmt.Println("Building tree...")
	}
	tree := buildTree(nodes, 0)
	ast.FixPositions(tree)

	// Repair the floating literals. See RepairFloatingLiteralsFromSource for
	// more information.
	floatingErrors := ast.RepairFloatingLiteralsFromSource(tree[0], ppFilePath)

	for _, fErr := range floatingErrors {
		message := fmt.Sprintf("could not read exact floating literal: %s",
			fErr.Err.Error())
		p.AddMessage(p.GenerateWarningMessage(errors.New(message), fErr.Node))
	}

	outputFilePath := args.outputFile

	if outputFilePath == "" {
		// Choose inputFile for creating name of output file
		input := args.inputFiles[0]
		// We choose name for output Go code at the base
		// on filename for choosed input file
		cleanFileName := filepath.Clean(filepath.Base(input))
		extension := filepath.Ext(input)
		outputFilePath = cleanFileName[0:len(cleanFileName)-len(extension)] + ".go"
	}

	// transpile ast tree
	if args.verbose {
		fmt.Println("Transpiling tree...")
	}

	err = transpiler.TranspileAST(args.outputFile, args.packageName, p, tree[0].(ast.Node))
	if err != nil {
		return fmt.Errorf("cannot transpile AST : %v", err)
	}

	// write the output Go code
	if args.verbose {
		fmt.Println("Writing the output Go code...")
	}
	err = ioutil.WriteFile(outputFilePath, []byte(p.String()), 0644)
	if err != nil {
		return fmt.Errorf("writing Go output file failed: %v", err)
	}

	return nil
}

var (
	versionFlag       = flag.Bool("v", false, "print the version and exit")
	transpileCommand  = flag.NewFlagSet("transpile", flag.ContinueOnError)
	verboseFlag       = transpileCommand.Bool("V", false, "print progress as comments")
	outputFlag        = transpileCommand.String("o", "", "output Go generated code to the specified file")
	packageFlag       = transpileCommand.String("p", "main", "set the name of the generated package")
	transpileHelpFlag = transpileCommand.Bool("h", false, "print help information")
	astCommand        = flag.NewFlagSet("ast", flag.ContinueOnError)
	astHelpFlag       = astCommand.Bool("h", false, "print help information")
)

func main() {
	code := runCommand()
	if code != 0 {
		os.Exit(code)
	}
}

func runCommand() int {
	flag.Usage = func() {
		usage := "Usage: %s [-v] [<command>] [<flags>] file1.c ...\n\n"
		usage += "Commands:\n"
		usage += "  transpile\ttranspile an input C source file or files to Go\n"
		usage += "  ast\t\tprint AST before translated Go code\n\n"

		usage += "Flags:\n"
		fmt.Fprintf(stderr, usage, os.Args[0])
		flag.PrintDefaults()
	}

	transpileCommand.SetOutput(stderr)
	astCommand.SetOutput(stderr)

	flag.Parse()

	if *versionFlag {
		// Simply print out the version and exit.
		fmt.Println(Version)
		return 0
	}

	if flag.NArg() < 1 {
		flag.Usage()
		return 1
	}

	args := DefaultProgramArgs()

	switch os.Args[1] {
	case "ast":
		err := astCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("ast command cannot parse: %v", err)
			return 1
		}

		if *astHelpFlag || astCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s ast file.c\n", os.Args[0])
			astCommand.PrintDefaults()
			return 1
		}

		args.ast = true
		args.inputFiles = astCommand.Args()
	case "transpile":
		err := transpileCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("transpile command cannot parse: %v", err)
			return 1
		}

		if *transpileHelpFlag || transpileCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s transpile [-V] [-o file.go] [-p package] file1.c ...\n", os.Args[0])
			transpileCommand.PrintDefaults()
			return 1
		}

		args.inputFiles = transpileCommand.Args()
		args.outputFile = *outputFlag
		args.packageName = *packageFlag
		args.verbose = *verboseFlag
	default:
		flag.Usage()
		return 1
	}

	if err := Start(args); err != nil {
		fmt.Printf("Error: %v\n", err)
		return 1
	}

	return 0
}
<commit_msg>Bump version: v0.17.2 Samarium 2017-11-12<commit_after>// Package c2go contains the main function for running the executable.
//
// Installation
//
//     go get -u github.com/elliotchance/c2go
//
// Usage
//
//     c2go myfile.c
//
package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"errors"

	"github.com/elliotchance/c2go/ast"
	"github.com/elliotchance/c2go/preprocessor"
	"github.com/elliotchance/c2go/program"
	"github.com/elliotchance/c2go/transpiler"
)

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.17.2 Samarium 2017-11-12"

var stderr io.Writer = os.Stderr

// ProgramArgs defines the options available when processing the program. There
// is no constructor since the zeroed out values are the appropriate defaults -
// you need only set the options you need.
//
// TODO: Better separation on CLI modes
// https://github.com/elliotchance/c2go/issues/134
//
// Do not instantiate this directly. Instead use DefaultProgramArgs(); then
// modify any specific attributes.
type ProgramArgs struct {
	verbose     bool
	ast         bool
	inputFiles  []string
	outputFile  string
	packageName string

	// A private option to output the Go as a *_test.go file.
	outputAsTest bool
}

// DefaultProgramArgs default value of ProgramArgs
func DefaultProgramArgs() ProgramArgs {
	return ProgramArgs{
		verbose:      false,
		ast:          false,
		packageName:  "main",
		outputAsTest: false,
	}
}

func readAST(data []byte) []string {
	return strings.Split(string(data), "\n")
}

type treeNode struct {
	indent int
	node   ast.Node
}

func convertLinesToNodes(lines []string) []treeNode {
	nodes := make([]treeNode, len(lines))
	var counter int
	for _, line := range lines {
		if strings.TrimSpace(line) == "" {
			continue
		}

		// It is tempting to discard null AST nodes, but these may
		// have semantic importance: for example, they represent omitted
		// for-loop conditions, as in for(;;).
		line = strings.Replace(line, "<<<NULL>>>", "NullStmt", 1)
		trimmed := strings.TrimLeft(line, "|\\- `")
		node := ast.Parse(trimmed)
		indentLevel := (len(line) - len(trimmed)) / 2
		nodes[counter] = treeNode{indentLevel, node}
		counter++
	}
	nodes = nodes[0:counter]

	return nodes
}

func convertLinesToNodesParallel(lines []string) []treeNode {
	// function f separate full list on 2 parts and
	// then each part can recursive run function f
	var f func([]string, int) []treeNode

	f = func(lines []string, deep int) []treeNode {
		deep = deep - 2
		part := len(lines) / 2

		var tr1 = make(chan []treeNode)
		var tr2 = make(chan []treeNode)

		go func(lines []string, deep int) {
			if deep <= 0 || len(lines) < deep {
				tr1 <- convertLinesToNodes(lines)
				return
			}
			tr1 <- f(lines, deep)
		}(lines[0:part], deep)

		go func(lines []string, deep int) {
			if deep <= 0 || len(lines) < deep {
				tr2 <- convertLinesToNodes(lines)
				return
			}
			tr2 <- f(lines, deep)
		}(lines[part:], deep)

		defer close(tr1)
		defer close(tr2)

		return append(<-tr1, <-tr2...)
	}

	// Parameter of deep - can be any, but effective to use
	// same amount of CPU
	return f(lines, runtime.NumCPU())
}

// buildTree converts an array of nodes, each prefixed with a depth into a tree.
func buildTree(nodes []treeNode, depth int) []ast.Node {
	if len(nodes) == 0 {
		return []ast.Node{}
	}

	// Split the list into sections, treat each section as a tree with its own
	// root.
	sections := [][]treeNode{}
	for _, node := range nodes {
		if node.indent == depth {
			sections = append(sections, []treeNode{node})
		} else {
			sections[len(sections)-1] = append(sections[len(sections)-1], node)
		}
	}

	results := []ast.Node{}
	for _, section := range sections {
		slice := []treeNode{}
		for _, n := range section {
			if n.indent > depth {
				slice = append(slice, n)
			}
		}

		children := buildTree(slice, depth+1)
		for _, child := range children {
			section[0].node.AddChild(child)
		}
		results = append(results, section[0].node)
	}

	return results
}

// Start begins transpiling an input file.
func Start(args ProgramArgs) (err error) {
	if args.verbose {
		fmt.Println("Start tanspiling ...")
	}

	if os.Getenv("GOPATH") == "" {
		return fmt.Errorf("The $GOPATH must be set")
	}

	// 1. Compile it first (checking for errors)
	for _, in := range args.inputFiles {
		_, err := os.Stat(in)
		if err != nil {
			return fmt.Errorf("Input file %s is not found", in)
		}
	}

	// 2. Preprocess
	if args.verbose {
		fmt.Println("Running clang preprocessor...")
	}

	pp, err := preprocessor.Analyze(args.inputFiles)
	if err != nil {
		return err
	}

	if args.verbose {
		fmt.Println("Writing preprocessor ...")
	}
	dir, err := ioutil.TempDir("", "c2go")
	if err != nil {
		return fmt.Errorf("Cannot create temp folder: %v", err)
	}
	defer os.RemoveAll(dir) // clean up

	ppFilePath := path.Join(dir, "pp.c")
	err = ioutil.WriteFile(ppFilePath, pp, 0644)
	if err != nil {
		return fmt.Errorf("writing to %s failed: %v", ppFilePath, err)
	}

	// 3. Generate JSON from AST
	if args.verbose {
		fmt.Println("Running clang for AST tree...")
	}
	astPP, err := exec.Command("clang", "-Xclang", "-ast-dump", "-fsyntax-only", "-fno-color-diagnostics", ppFilePath).Output()
	if err != nil {
		// If clang fails it still prints out the AST, so we have to run it
		// again to get the real error.
		errBody, _ := exec.Command("clang", ppFilePath).CombinedOutput()

		panic("clang failed: " + err.Error() + ":\n\n" + string(errBody))
	}

	if args.verbose {
		fmt.Println("Reading clang AST tree...")
	}
	lines := readAST(astPP)
	if args.ast {
		for _, l := range lines {
			fmt.Println(l)
		}
		fmt.Println()

		return nil
	}

	p := program.NewProgram()
	p.Verbose = args.verbose
	p.OutputAsTest = args.outputAsTest

	// Converting to nodes
	if args.verbose {
		fmt.Println("Converting to nodes...")
	}
	nodes := convertLinesToNodesParallel(lines)

	// build tree
	if args.verbose {
		fmt.Println("Building tree...")
	}
	tree := buildTree(nodes, 0)
	ast.FixPositions(tree)

	// Repair the floating literals. See RepairFloatingLiteralsFromSource for
	// more information.
	floatingErrors := ast.RepairFloatingLiteralsFromSource(tree[0], ppFilePath)

	for _, fErr := range floatingErrors {
		message := fmt.Sprintf("could not read exact floating literal: %s",
			fErr.Err.Error())
		p.AddMessage(p.GenerateWarningMessage(errors.New(message), fErr.Node))
	}

	outputFilePath := args.outputFile

	if outputFilePath == "" {
		// Choose inputFile for creating name of output file
		input := args.inputFiles[0]
		// We choose name for output Go code at the base
		// on filename for choosed input file
		cleanFileName := filepath.Clean(filepath.Base(input))
		extension := filepath.Ext(input)
		outputFilePath = cleanFileName[0:len(cleanFileName)-len(extension)] + ".go"
	}

	// transpile ast tree
	if args.verbose {
		fmt.Println("Transpiling tree...")
	}

	err = transpiler.TranspileAST(args.outputFile, args.packageName, p, tree[0].(ast.Node))
	if err != nil {
		return fmt.Errorf("cannot transpile AST : %v", err)
	}

	// write the output Go code
	if args.verbose {
		fmt.Println("Writing the output Go code...")
	}
	err = ioutil.WriteFile(outputFilePath, []byte(p.String()), 0644)
	if err != nil {
		return fmt.Errorf("writing Go output file failed: %v", err)
	}

	return nil
}

var (
	versionFlag       = flag.Bool("v", false, "print the version and exit")
	transpileCommand  = flag.NewFlagSet("transpile", flag.ContinueOnError)
	verboseFlag       = transpileCommand.Bool("V", false, "print progress as comments")
	outputFlag        = transpileCommand.String("o", "", "output Go generated code to the specified file")
	packageFlag       = transpileCommand.String("p", "main", "set the name of the generated package")
	transpileHelpFlag = transpileCommand.Bool("h", false, "print help information")
	astCommand        = flag.NewFlagSet("ast", flag.ContinueOnError)
	astHelpFlag       = astCommand.Bool("h", false, "print help information")
)

func main() {
	code := runCommand()
	if code != 0 {
		os.Exit(code)
	}
}

func runCommand() int {
	flag.Usage = func() {
		usage := "Usage: %s [-v] [<command>] [<flags>] file1.c ...\n\n"
		usage += "Commands:\n"
		usage += "  transpile\ttranspile an input C source file or files to Go\n"
		usage += "  ast\t\tprint AST before translated Go code\n\n"

		usage += "Flags:\n"
		fmt.Fprintf(stderr, usage, os.Args[0])
		flag.PrintDefaults()
	}

	transpileCommand.SetOutput(stderr)
	astCommand.SetOutput(stderr)

	flag.Parse()

	if *versionFlag {
		// Simply print out the version and exit.
		fmt.Println(Version)
		return 0
	}

	if flag.NArg() < 1 {
		flag.Usage()
		return 1
	}

	args := DefaultProgramArgs()

	switch os.Args[1] {
	case "ast":
		err := astCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("ast command cannot parse: %v", err)
			return 1
		}

		if *astHelpFlag || astCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s ast file.c\n", os.Args[0])
			astCommand.PrintDefaults()
			return 1
		}

		args.ast = true
		args.inputFiles = astCommand.Args()
	case "transpile":
		err := transpileCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("transpile command cannot parse: %v", err)
			return 1
		}

		if *transpileHelpFlag || transpileCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s transpile [-V] [-o file.go] [-p package] file1.c ...\n", os.Args[0])
			transpileCommand.PrintDefaults()
			return 1
		}

		args.inputFiles = transpileCommand.Args()
		args.outputFile = *outputFlag
		args.packageName = *packageFlag
		args.verbose = *verboseFlag
	default:
		flag.Usage()
		return 1
	}

	if err := Start(args); err != nil {
		fmt.Printf("Error: %v\n", err)
		return 1
	}

	return 0
}
<|endoftext|>