<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"

	_ "github.com/go-sql-driver/mysql"
	"github.com/tedsuo/ifrit"

	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer/config"
	"github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer/database"
	"github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer/enforcer"
	"github.com/pivotal-cf-experimental/service-config"
	"github.com/pivotal-golang/lager"
)

type Config struct {
	Host         string
	Port         int
	User         string
	Password     string
	BrokerDBName string
}

func main() {
	serviceConfig := service_config.New()

	flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	runOnce := flags.Bool("runOnce", false, "Run only once instead of continuously")
	pidFile := flags.String("pidFile", "", "Location of pid file")
	serviceConfig.AddFlags(flags)
	cf_lager.AddFlags(flags)
	flags.Parse(os.Args[1:])
	logger, _ := cf_lager.New("Quota Enforcer")

	var config config.Config
	err := serviceConfig.Read(&config)
	if err != nil {
		logger.Fatal("Failed to read config", err)
	}

	err = config.Validate()
	if err != nil {
		logger.Fatal("Invalid config", err)
	}

	adminUser := config.User
	readOnlyUser := config.ReadOnlyUser

	db, err := database.NewConnection(config)
	if db != nil {
		defer db.Close()
	}

	if err != nil {
		logger.Fatal("Failed to open database connection", err)
	}

	logger.Info(
		"Database connection established.",
		lager.Data{
			"Host":         config.Host,
			"Port":         config.Port,
			"User":         adminUser,
			"DatabaseName": brokerDBName,
		})

	violatorRepo := database.NewViolatorRepo(brokerDBName, adminUser, readOnlyUser, db, logger)
	reformerRepo := database.NewReformerRepo(brokerDBName, adminUser, readOnlyUser, db, logger)

	e := enforcer.NewEnforcer(violatorRepo, reformerRepo, logger)
	r := enforcer.NewRunner(e, logger)

	if *runOnce {
		logger.Info("Running once")

		err := e.EnforceOnce()
		if err != nil {
			logger.Info(fmt.Sprintf("Quota Enforcing Failed: %s", err.Error()))
		}
	} else {
		process := ifrit.Invoke(r)
		logger.Info("Running continuously")

		// Write pid file once we are running continuously
		if *pidFile != "" {
			pid := os.Getpid()
			err = writePidFile(pid, *pidFile)
			if err != nil {
				logger.Fatal("Cannot write pid to file", err, lager.Data{"pidFile": pidFile, "pid": pid})
			}
			logger.Info("Wrote pid to file", lager.Data{"pidFile": pidFile, "pid": pid})
		}

		err := <-process.Wait()
		if err != nil {
			logger.Fatal("Quota Enforcing Failed", err)
		}
	}
}

func writePidFile(pid int, pidFile string) error {
	return ioutil.WriteFile(pidFile, []byte(strconv.Itoa(pid)), 0644)
}
<commit_msg>Cleanup unused Config struct in main<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"

	_ "github.com/go-sql-driver/mysql"
	"github.com/tedsuo/ifrit"

	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer/config"
	"github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer/database"
	"github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer/enforcer"
	"github.com/pivotal-cf-experimental/service-config"
	"github.com/pivotal-golang/lager"
)

func main() {
	serviceConfig := service_config.New()

	flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	runOnce := flags.Bool("runOnce", false, "Run only once instead of continuously")
	pidFile := flags.String("pidFile", "", "Location of pid file")
	serviceConfig.AddFlags(flags)
	cf_lager.AddFlags(flags)
	flags.Parse(os.Args[1:])
	logger, _ := cf_lager.New("Quota Enforcer")

	var config config.Config
	err := serviceConfig.Read(&config)
	if err != nil {
		logger.Fatal("Failed to read config", err)
	}

	err = config.Validate()
	if err != nil {
		logger.Fatal("Invalid config", err)
	}

	adminUser := config.User
	readOnlyUser := config.ReadOnlyUser
	brokerDBName := config.DBName

	db, err := database.NewConnection(config)
	if db != nil {
		defer db.Close()
	}

	if err != nil {
		logger.Fatal("Failed to open database connection", err)
	}

	logger.Info(
		"Database connection established.",
		lager.Data{
			"Host":         config.Host,
			"Port":         config.Port,
			"User":         adminUser,
			"DatabaseName": brokerDBName,
		})

	violatorRepo := database.NewViolatorRepo(brokerDBName, adminUser, readOnlyUser, db, logger)
	reformerRepo := database.NewReformerRepo(brokerDBName, adminUser, readOnlyUser, db, logger)

	e := enforcer.NewEnforcer(violatorRepo, reformerRepo, logger)
	r := enforcer.NewRunner(e, logger)

	if *runOnce {
		logger.Info("Running once")

		err := e.EnforceOnce()
		if err != nil {
			logger.Info(fmt.Sprintf("Quota Enforcing Failed: %s", err.Error()))
		}
	} else {
		process := ifrit.Invoke(r)
		logger.Info("Running continuously")

		// Write pid file once we are running continuously
		if *pidFile != "" {
			pid := os.Getpid()
			err = writePidFile(pid, *pidFile)
			if err != nil {
				logger.Fatal("Cannot write pid to file", err, lager.Data{"pidFile": pidFile, "pid": pid})
			}
			logger.Info("Wrote pid to file", lager.Data{"pidFile": pidFile, "pid": pid})
		}

		err := <-process.Wait()
		if err != nil {
			logger.Fatal("Quota Enforcing Failed", err)
		}
	}
}

func writePidFile(pid int, pidFile string) error {
	return ioutil.WriteFile(pidFile, []byte(strconv.Itoa(pid)), 0644)
}
<|endoftext|>


<commit_before>/*
 * Copyright 2016 Martin Helmich <kontakt@martin-helmich.de>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/martin-helmich/prometheus-nginxlog-exporter/config"
	"github.com/martin-helmich/prometheus-nginxlog-exporter/discovery"
	"github.com/martin-helmich/prometheus-nginxlog-exporter/relabeling"
	"github.com/martin-helmich/prometheus-nginxlog-exporter/tail"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/satyrius/gonx"
)

// Metrics is a struct containing pointers to all metrics that should be
// exposed to Prometheus
type Metrics struct {
	countTotal          *prometheus.CounterVec
	bytesTotal          *prometheus.CounterVec
	upstreamSeconds     *prometheus.SummaryVec
	upstreamSecondsHist *prometheus.HistogramVec
	responseSeconds     *prometheus.SummaryVec
	responseSecondsHist *prometheus.HistogramVec
	parseErrorsTotal    prometheus.Counter
}

// Init initializes a metrics struct
func (m *Metrics) Init(cfg *config.NamespaceConfig) {
	cfg.MustCompile()

	labels := cfg.OrderedLabelNames

	for _, r := range relabeling.DefaultRelabelings {
		labels = append(labels, r.TargetLabel)
	}

	for i := range cfg.RelabelConfigs {
		labels = append(labels, cfg.RelabelConfigs[i].TargetLabel)
	}

	m.countTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: cfg.Name,
		Name:      "http_response_count_total",
		Help:      "Amount of processed HTTP requests",
	}, labels)

	m.bytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: cfg.Name,
		Name:      "http_response_size_bytes",
		Help:      "Total amount of transferred bytes",
	}, labels)

	m.upstreamSeconds = prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Namespace: cfg.Name,
		Name:      "http_upstream_time_seconds",
		Help:      "Time needed by upstream servers to handle requests",
	}, labels)

	m.upstreamSecondsHist = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: cfg.Name,
		Name:      "http_upstream_time_seconds_hist",
		Help:      "Time needed by upstream servers to handle requests",
	}, labels)

	m.responseSeconds = prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Namespace: cfg.Name,
		Name:      "http_response_time_seconds",
		Help:      "Time needed by NGINX to handle requests",
	}, labels)

	m.responseSecondsHist = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: cfg.Name,
		Name:      "http_response_time_seconds_hist",
		Help:      "Time needed by NGINX to handle requests",
	}, labels)

	m.parseErrorsTotal = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: cfg.Name,
		Name:      "parse_errors_total",
		Help:      "Total number of log file lines that could not be parsed",
	})

	prometheus.MustRegister(m.countTotal)
	prometheus.MustRegister(m.bytesTotal)
	prometheus.MustRegister(m.upstreamSeconds)
	prometheus.MustRegister(m.upstreamSecondsHist)
	prometheus.MustRegister(m.responseSeconds)
	prometheus.MustRegister(m.responseSecondsHist)
	prometheus.MustRegister(m.parseErrorsTotal)
}

func main() {
	var opts config.StartupFlags
	var cfg = config.Config{
		Listen: config.ListenConfig{
			Port:    4040,
			Address: "0.0.0.0",
		},
	}

	flag.IntVar(&opts.ListenPort, "listen-port", 4040, "HTTP port to listen on")
	flag.StringVar(&opts.Format, "format", `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for"`, "NGINX access log format")
	flag.StringVar(&opts.Namespace, "namespace", "nginx", "namespace to use for metric names")
	flag.StringVar(&opts.ConfigFile, "config-file", "", "Configuration file to read from")
	flag.BoolVar(&opts.EnableExperimentalFeatures, "enable-experimental", false, "Set this flag to enable experimental features")
	flag.Parse()

	opts.Filenames = flag.Args()

	if opts.ConfigFile != "" {
		fmt.Printf("loading configuration file %s\n", opts.ConfigFile)
		if err := config.LoadConfigFromFile(&cfg, opts.ConfigFile); err != nil {
			panic(err)
		}
	} else if err := config.LoadConfigFromFlags(&cfg, &opts); err != nil {
		panic(err)
	}

	fmt.Printf("using configuration %s\n", cfg)

	if stabilityError := cfg.StabilityWarnings(); stabilityError != nil && !opts.EnableExperimentalFeatures {
		fmt.Fprintf(os.Stderr, "Your configuration file contains an option that is explicitly labeled as experimental feature:\n\n  %s\n\n", stabilityError.Error())
		fmt.Fprintln(os.Stderr, "Use the -enable-experimental flag or the enable_experimental option to enable these features. Use them at your own peril.")

		os.Exit(1)
	}

	if cfg.Consul.Enable {
		registrator, err := discovery.NewConsulRegistrator(&cfg)
		if err != nil {
			panic(err)
		}

		fmt.Printf("registering service in Consul\n")
		if err := registrator.RegisterConsul(); err != nil {
			panic(err)
		}

		exitChan := make(chan os.Signal, 1)
		signal.Notify(exitChan, os.Interrupt, syscall.SIGTERM)

		go func() {
			<-exitChan
			fmt.Printf("unregistering service in Consul\n")
			registrator.UnregisterConsul()
			os.Exit(0)
		}()
	}

	for _, ns := range cfg.Namespaces {
		fmt.Printf("starting listener for namespace %s\n", ns.Name)

		go func(nsCfg config.NamespaceConfig) {
			parser := gonx.NewParser(nsCfg.Format)

			metrics := Metrics{}
			metrics.Init(&nsCfg)

			for _, f := range nsCfg.SourceFiles {
				t, err := tail.NewFollower(f)
				if err != nil {
					panic(err)
				}

				t.OnError(func(err error) {
					panic(err)
				})

				go func(nsCfg config.NamespaceConfig) {
					relabelings := relabeling.NewRelabelings(nsCfg.RelabelConfigs)
					relabelings = append(relabeling.DefaultRelabelings, relabelings...)

					staticLabelValues := nsCfg.OrderedLabelValues

					totalLabelCount := len(staticLabelValues) + len(relabelings)
					relabelLabelOffset := len(staticLabelValues)
					labelValues := make([]string, totalLabelCount)

					for i := range staticLabelValues {
						labelValues[i] = staticLabelValues[i]
					}

					for line := range t.Lines() {
						entry, err := parser.ParseString(line.Text)
						if err != nil {
							fmt.Printf("error while parsing line '%s': %s\n", line.Text, err)
							metrics.parseErrorsTotal.Inc()
							continue
						}

						for i := range relabelings {
							if str, err := entry.Field(relabelings[i].SourceValue); err == nil {
								mapped, err := relabelings[i].Map(str)
								if err == nil {
									labelValues[i+relabelLabelOffset] = mapped
								}
							}
						}

						metrics.countTotal.WithLabelValues(labelValues...).Inc()

						if bytes, err := entry.FloatField("body_bytes_sent"); err == nil {
							metrics.bytesTotal.WithLabelValues(labelValues...).Add(bytes)
						}

						if upstreamTime, err := entry.FloatField("upstream_response_time"); err == nil {
							metrics.upstreamSeconds.WithLabelValues(labelValues...).Observe(upstreamTime)
							metrics.upstreamSecondsHist.WithLabelValues(labelValues...).Observe(upstreamTime)
						}

						if responseTime, err := entry.FloatField("request_time"); err == nil {
							metrics.responseSeconds.WithLabelValues(labelValues...).Observe(responseTime)
							metrics.responseSecondsHist.WithLabelValues(labelValues...).Observe(responseTime)
						}
					}
				}(nsCfg)
			}
		}(ns)
	}

	listenAddr := fmt.Sprintf("%s:%d", cfg.Listen.Address, cfg.Listen.Port)
	fmt.Printf("running HTTP server on address %s\n", listenAddr)

	http.Handle("/metrics", prometheus.Handler())
	http.ListenAndServe(listenAddr, nil)
}
<commit_msg>Reduce complexity of main function<commit_after>/*
 * Copyright 2016 Martin Helmich <kontakt@martin-helmich.de>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/martin-helmich/prometheus-nginxlog-exporter/config"
	"github.com/martin-helmich/prometheus-nginxlog-exporter/discovery"
	"github.com/martin-helmich/prometheus-nginxlog-exporter/relabeling"
	"github.com/martin-helmich/prometheus-nginxlog-exporter/tail"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/satyrius/gonx"
)

// Metrics is a struct containing pointers to all metrics that should be
// exposed to Prometheus
type Metrics struct {
	countTotal          *prometheus.CounterVec
	bytesTotal          *prometheus.CounterVec
	upstreamSeconds     *prometheus.SummaryVec
	upstreamSecondsHist *prometheus.HistogramVec
	responseSeconds     *prometheus.SummaryVec
	responseSecondsHist *prometheus.HistogramVec
	parseErrorsTotal    prometheus.Counter
}

// Init initializes a metrics struct
func (m *Metrics) Init(cfg *config.NamespaceConfig) {
	cfg.MustCompile()

	labels := cfg.OrderedLabelNames

	for _, r := range relabeling.DefaultRelabelings {
		labels = append(labels, r.TargetLabel)
	}

	for i := range cfg.RelabelConfigs {
		labels = append(labels, cfg.RelabelConfigs[i].TargetLabel)
	}

	m.countTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: cfg.Name,
		Name:      "http_response_count_total",
		Help:      "Amount of processed HTTP requests",
	}, labels)

	m.bytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: cfg.Name,
		Name:      "http_response_size_bytes",
		Help:      "Total amount of transferred bytes",
	}, labels)

	m.upstreamSeconds = prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Namespace: cfg.Name,
		Name:      "http_upstream_time_seconds",
		Help:      "Time needed by upstream servers to handle requests",
	}, labels)

	m.upstreamSecondsHist = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: cfg.Name,
		Name:      "http_upstream_time_seconds_hist",
		Help:      "Time needed by upstream servers to handle requests",
	}, labels)

	m.responseSeconds = prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Namespace: cfg.Name,
		Name:      "http_response_time_seconds",
		Help:      "Time needed by NGINX to handle requests",
	}, labels)

	m.responseSecondsHist = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: cfg.Name,
		Name:      "http_response_time_seconds_hist",
		Help:      "Time needed by NGINX to handle requests",
	}, labels)

	m.parseErrorsTotal = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: cfg.Name,
		Name:      "parse_errors_total",
		Help:      "Total number of log file lines that could not be parsed",
	})

	prometheus.MustRegister(m.countTotal)
	prometheus.MustRegister(m.bytesTotal)
	prometheus.MustRegister(m.upstreamSeconds)
	prometheus.MustRegister(m.upstreamSecondsHist)
	prometheus.MustRegister(m.responseSeconds)
	prometheus.MustRegister(m.responseSecondsHist)
	prometheus.MustRegister(m.parseErrorsTotal)
}

func main() {
	var opts config.StartupFlags
	var cfg = config.Config{
		Listen: config.ListenConfig{
			Port:    4040,
			Address: "0.0.0.0",
		},
	}

	flag.IntVar(&opts.ListenPort, "listen-port", 4040, "HTTP port to listen on")
	flag.StringVar(&opts.Format, "format", `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for"`, "NGINX access log format")
	flag.StringVar(&opts.Namespace, "namespace", "nginx", "namespace to use for metric names")
	flag.StringVar(&opts.ConfigFile, "config-file", "", "Configuration file to read from")
	flag.BoolVar(&opts.EnableExperimentalFeatures, "enable-experimental", false, "Set this flag to enable experimental features")
	flag.Parse()

	opts.Filenames = flag.Args()

	loadConfig(&opts, &cfg)

	fmt.Printf("using configuration %+v\n", cfg)

	if stabilityError := cfg.StabilityWarnings(); stabilityError != nil && !opts.EnableExperimentalFeatures {
		fmt.Fprintf(os.Stderr, "Your configuration file contains an option that is explicitly labeled as experimental feature:\n\n  %s\n\n", stabilityError.Error())
		fmt.Fprintln(os.Stderr, "Use the -enable-experimental flag or the enable_experimental option to enable these features. Use them at your own peril.")

		os.Exit(1)
	}

	if cfg.Consul.Enable {
		setupConsul(&cfg)
	}

	for _, ns := range cfg.Namespaces {
		fmt.Printf("starting listener for namespace %s\n", ns.Name)

		go processNamespace(ns)
	}

	listenAddr := fmt.Sprintf("%s:%d", cfg.Listen.Address, cfg.Listen.Port)
	fmt.Printf("running HTTP server on address %s\n", listenAddr)

	http.Handle("/metrics", prometheus.Handler())
	http.ListenAndServe(listenAddr, nil)
}

func loadConfig(opts *config.StartupFlags, cfg *config.Config) {
	if opts.ConfigFile != "" {
		fmt.Printf("loading configuration file %s\n", opts.ConfigFile)
		if err := config.LoadConfigFromFile(cfg, opts.ConfigFile); err != nil {
			panic(err)
		}
	} else if err := config.LoadConfigFromFlags(cfg, opts); err != nil {
		panic(err)
	}
}

func setupConsul(cfg *config.Config) {
	registrator, err := discovery.NewConsulRegistrator(cfg)
	if err != nil {
		panic(err)
	}

	fmt.Printf("registering service in Consul\n")
	if err := registrator.RegisterConsul(); err != nil {
		panic(err)
	}

	exitChan := make(chan os.Signal, 1)
	signal.Notify(exitChan, os.Interrupt, syscall.SIGTERM)

	go func() {
		<-exitChan
		fmt.Printf("unregistering service in Consul\n")
		registrator.UnregisterConsul()
		os.Exit(0)
	}()
}

func processNamespace(nsCfg config.NamespaceConfig) {
	parser := gonx.NewParser(nsCfg.Format)

	metrics := Metrics{}
	metrics.Init(&nsCfg)

	for _, f := range nsCfg.SourceFiles {
		t, err := tail.NewFollower(f)
		if err != nil {
			panic(err)
		}

		t.OnError(func(err error) {
			panic(err)
		})

		go processSourceFile(nsCfg, t, parser, &metrics)
	}
}

func processSourceFile(nsCfg config.NamespaceConfig, t tail.Follower, parser *gonx.Parser, metrics *Metrics) {
	relabelings := relabeling.NewRelabelings(nsCfg.RelabelConfigs)
	relabelings = append(relabeling.DefaultRelabelings, relabelings...)

	staticLabelValues := nsCfg.OrderedLabelValues

	totalLabelCount := len(staticLabelValues) + len(relabelings)
	relabelLabelOffset := len(staticLabelValues)
	labelValues := make([]string, totalLabelCount)

	for i := range staticLabelValues {
		labelValues[i] = staticLabelValues[i]
	}

	for line := range t.Lines() {
		entry, err := parser.ParseString(line.Text)
		if err != nil {
			fmt.Printf("error while parsing line '%s': %s\n", line.Text, err)
			metrics.parseErrorsTotal.Inc()
			continue
		}

		for i := range relabelings {
			if str, err := entry.Field(relabelings[i].SourceValue); err == nil {
				mapped, err := relabelings[i].Map(str)
				if err == nil {
					labelValues[i+relabelLabelOffset] = mapped
				}
			}
		}

		metrics.countTotal.WithLabelValues(labelValues...).Inc()

		if bytes, err := entry.FloatField("body_bytes_sent"); err == nil {
			metrics.bytesTotal.WithLabelValues(labelValues...).Add(bytes)
		}

		if upstreamTime, err := entry.FloatField("upstream_response_time"); err == nil {
			metrics.upstreamSeconds.WithLabelValues(labelValues...).Observe(upstreamTime)
			metrics.upstreamSecondsHist.WithLabelValues(labelValues...).Observe(upstreamTime)
		}

		if responseTime, err := entry.FloatField("request_time"); err == nil {
			metrics.responseSeconds.WithLabelValues(labelValues...).Observe(responseTime)
			metrics.responseSecondsHist.WithLabelValues(labelValues...).Observe(responseTime)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"io/ioutil"
	"flag"
	"log"
)

func main() {
	flag.Parse()
	var input []byte
	if flag.NArg() == 0 {
		var err os.Error
		input, err = ioutil.ReadAll(os.Stdin)
		if err != nil {
			log.Exitln(err)
		}
	} else {
		file, err := os.Open(flag.Arg(0), os.O_RDONLY, 0)
		if err != nil {
			log.Exitln(err)
		}
		defer file.Close()
		input, err = ioutil.ReadAll(file)
		if err != nil {
			log.Exitln(err)
		}
	}
	parsed := parse(input)
	fmt.Println(parsed)
}
<commit_msg>Handle invalid input.<commit_after>package main

import (
	"fmt"
	"os"
	"io/ioutil"
	"flag"
	"log"
)

func main() {
	flag.Parse()
	var input []byte
	if flag.NArg() == 0 {
		var err os.Error
		input, err = ioutil.ReadAll(os.Stdin)
		if err != nil {
			log.Exitln(err)
		}
	} else {
		file, err := os.Open(flag.Arg(0), os.O_RDONLY, 0)
		if err != nil {
			log.Exitln(err)
		}
		defer file.Close()
		input, err = ioutil.ReadAll(file)
		if err != nil {
			log.Exitln(err)
		}
	}
	if !valid(input) {
		log.Exitln("Syntax error.")
	}
	parsed := parse(input)
	fmt.Println(parsed)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"

	"github.com/jessevdk/go-flags"

	"github.com/ivanilves/lstags/auth"
	"github.com/ivanilves/lstags/docker"
	dockerclient "github.com/ivanilves/lstags/docker/client"
	dockerconfig "github.com/ivanilves/lstags/docker/config"
	"github.com/ivanilves/lstags/tag"
	"github.com/ivanilves/lstags/tag/local"
	"github.com/ivanilves/lstags/tag/remote"
	"github.com/ivanilves/lstags/util"
)

// Options represents configuration options we extract from passed command line arguments
type Options struct {
	DockerJSON         string `short:"j" long:"docker-json" default:"~/.docker/config.json" description:"JSON file with credentials" env:"DOCKER_JSON"`
	Pull               bool   `short:"P" long:"pull" description:"Pull Docker images matched by filter (will use local Docker deamon)" env:"PULL"`
	PushRegistry       string `short:"U" long:"push-registry" description:"[Re]Push pulled images to a specified remote registry" env:"PUSH_REGISTRY"`
	PushPrefix         string `short:"R" long:"push-prefix" description:"[Re]Push pulled images with a specified repo path prefix" env:"PUSH_PREFIX"`
	ConcurrentRequests int    `short:"c" long:"concurrent-requests" default:"32" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"`
	TraceRequests      bool   `short:"T" long:"trace-requests" description:"Trace Docker registry HTTP requests" env:"TRACE_REQUESTS"`
	DoNotFail          bool   `short:"N" long:"do-not-fail" description:"Do not fail on non-critical errors (could be dangerous!)" env:"DO_NOT_FAIL"`
	Version            bool   `short:"V" long:"version" description:"Show version and exit"`
	Positional         struct {
		Repositories []string `positional-arg-name:"REPO1 REPO2 REPOn" description:"Docker repositories to operate on, e.g.: alpine nginx~/1\\.13\\.5$/ busybox~/1.27.2/"`
	} `positional-args:"yes" required:"yes"`
}

var doNotFail = false

func suicide(err error, critical bool) {
	fmt.Printf("%s\n", err.Error())

	if doNotFail || critical {
		os.Exit(1)
	}
}

func parseFlags() (*Options, error) {
	var err error

	o := &Options{}

	_, err = flags.Parse(o)
	if err != nil {
		os.Exit(1) // YES! Just exit!
	}

	if o.Version {
		fmt.Printf("VERSION: %s\n", getVersion())
		os.Exit(0)
	}

	if len(o.Positional.Repositories) == 0 {
		return nil, errors.New("Need at least one repository name, e.g. 'nginx~/^1\\\\.13/' or 'mesosphere/chronos'")
	}

	if o.PushRegistry != "" {
		o.Pull = true
	}

	remote.TraceRequests = o.TraceRequests

	doNotFail = o.DoNotFail

	return o, nil
}

func getVersion() string {
	return VERSION
}

func main() {
	o, err := parseFlags()
	if err != nil {
		suicide(err, true)
	}

	dockerConfig, err := dockerconfig.Load(o.DockerJSON)
	if err != nil {
		suicide(err, true)
	}

	const format = "%-12s %-45s %-15s %-25s %s\n"
	fmt.Printf(format, "<STATE>", "<DIGEST>", "<(local) ID>", "<Created At>", "<TAG>")

	repoCount := len(o.Positional.Repositories)
	pullCount := 0
	pushCount := 0

	dc, err := dockerclient.New(dockerConfig)
	if err != nil {
		suicide(err, true)
	}

	type tagResult struct {
		Tags     []*tag.Tag
		Repo     string
		Path     string
		Registry string
	}

	trc := make(chan tagResult, repoCount)

	for _, r := range o.Positional.Repositories {
		go func(r string, o *Options, trc chan tagResult) {
			repository, filter, err := util.SeparateFilterAndRepo(r)
			if err != nil {
				suicide(err, true)
			}

			registry := docker.GetRegistry(repository)

			repoPath := docker.GetRepoPath(repository, registry)
			repoName := docker.GetRepoName(repository, registry)

			username, password, _ := dockerConfig.GetCredentials(registry)

			tr, err := auth.NewToken(registry, repoPath, username, password)
			if err != nil {
				suicide(err, true)
			}

			remoteTags, err := remote.FetchTags(registry, repoPath, tr.Header(), o.ConcurrentRequests)
			if err != nil {
				suicide(err, true)
			}

			imageSummaries, err := dc.ListImagesForRepo(repoName)
			if err != nil {
				suicide(err, true)
			}
			localTags, err := local.FetchTags(repoName, imageSummaries)
			if err != nil {
				suicide(err, true)
			}

			sortedKeys, names, joinedTags := tag.Join(remoteTags, localTags)

			tags := make([]*tag.Tag, 0)
			for _, key := range sortedKeys {
				name := names[key]

				tg := joinedTags[name]

				if !util.DoesMatch(tg.GetName(), filter) {
					continue
				}

				if tg.NeedsPull() {
					pullCount++
				}
				pushCount++

				tags = append(tags, tg)
			}

			trc <- tagResult{Tags: tags, Repo: repoName, Path: repoPath, Registry: registry}
		}(r, o, trc)
	}

	tagResults := make([]tagResult, repoCount)
	repoNumber := 0
	for tr := range trc {
		repoNumber++
		tagResults = append(tagResults, tr)
		if repoNumber >= repoCount {
			close(trc)
		}
	}

	for _, tr := range tagResults {
		for _, tg := range tr.Tags {
			fmt.Printf(
				format,
				tg.GetState(),
				tg.GetShortDigest(),
				tg.GetImageID(),
				tg.GetCreatedString(),
				tr.Repo+":"+tg.GetName(),
			)
		}
	}

	if o.Pull {
		done := make(chan bool, pullCount)

		for _, tr := range tagResults {
			go func(tags []*tag.Tag, repo string, done chan bool) {
				for _, tg := range tags {
					if tg.NeedsPull() {
						ref := repo + ":" + tg.GetName()

						fmt.Printf("PULLING %s\n", ref)
						err := dc.Pull(ref)
						if err != nil {
							suicide(err, false)
						}

						done <- true
					}

				}
			}(tr.Tags, tr.Repo, done)
		}

		pullNumber := 0
		if pullCount > 0 {
			for range done {
				pullNumber++

				if pullNumber >= pullCount {
					close(done)
				}
			}
		}
	}

	if o.Pull && o.PushRegistry != "" {
		done := make(chan bool, pullCount)

		for _, tr := range tagResults {
			go func(tags []*tag.Tag, repo, path, registry string, done chan bool) {
				for _, tg := range tags {
					prefix := o.PushPrefix
					if prefix == "" {
						prefix = util.GeneratePathFromHostname(registry)
					}

					srcRef := repo + ":" + tg.GetName()
					dstRef := o.PushRegistry + prefix + "/" + path + ":" + tg.GetName()

					fmt.Printf("PUSHING %s => %s\n", srcRef, dstRef)

					err := dc.Tag(srcRef, dstRef)
					if err != nil {
						suicide(err, true)
					}

					err = dc.Push(dstRef)
					if err != nil {
						suicide(err, false)
					}

					done <- true
				}
			}(tr.Tags, tr.Repo, tr.Path, tr.Registry, done)
		}

		pushNumber := 0
		if pushCount > 0 {
			for range done {
				pushNumber++

				if pushNumber >= pushCount {
					close(done)
				}
			}
		}
	}

}
<commit_msg>Change command line option names ... again :pig:<commit_after>package main

import (
	"errors"
	"fmt"
	"os"

	"github.com/jessevdk/go-flags"

	"github.com/ivanilves/lstags/auth"
	"github.com/ivanilves/lstags/docker"
	dockerclient "github.com/ivanilves/lstags/docker/client"
	dockerconfig "github.com/ivanilves/lstags/docker/config"
	"github.com/ivanilves/lstags/tag"
	"github.com/ivanilves/lstags/tag/local"
	"github.com/ivanilves/lstags/tag/remote"
	"github.com/ivanilves/lstags/util"
)

// Options represents configuration options we extract from passed command line arguments
type Options struct {
	DockerJSON         string `short:"j" long:"docker-json" default:"~/.docker/config.json" description:"JSON file with credentials" env:"DOCKER_JSON"`
	Pull               bool   `short:"p" long:"pull" description:"Pull Docker images matched by filter (will use local Docker deamon)" env:"PULL"`
	PushRegistry       string `short:"r" long:"push-registry" description:"[Re]Push pulled images to a specified remote registry" env:"PUSH_REGISTRY"`
	PushPrefix         string `short:"R" long:"push-prefix" description:"[Re]Push pulled images with a specified repo path prefix" env:"PUSH_PREFIX"`
	ConcurrentRequests int    `short:"c" long:"concurrent-requests" default:"32" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"`
	TraceRequests      bool   `short:"T" long:"trace-requests" description:"Trace Docker registry HTTP requests" env:"TRACE_REQUESTS"`
	DoNotFail          bool   `short:"N" long:"do-not-fail" description:"Do not fail on non-critical errors (could be dangerous!)" env:"DO_NOT_FAIL"`
	Version            bool   `short:"V" long:"version" description:"Show version and exit"`
	Positional         struct {
		Repositories []string `positional-arg-name:"REPO1 REPO2 REPOn" description:"Docker repositories to operate on, e.g.: alpine nginx~/1\\.13\\.5$/ busybox~/1.27.2/"`
	} `positional-args:"yes" required:"yes"`
}

var doNotFail = false

func suicide(err error, critical bool) {
	fmt.Printf("%s\n", err.Error())

	if doNotFail || critical {
		os.Exit(1)
	}
}

func parseFlags() (*Options, error) {
	var err error

	o := &Options{}

	_, err = flags.Parse(o)
	if err != nil {
		os.Exit(1) // YES! Just exit!
	}

	if o.Version {
		fmt.Printf("VERSION: %s\n", getVersion())
		os.Exit(0)
	}

	if len(o.Positional.Repositories) == 0 {
		return nil, errors.New("Need at least one repository name, e.g. 'nginx~/^1\\\\.13/' or 'mesosphere/chronos'")
	}

	if o.PushRegistry != "" {
		o.Pull = true
	}

	remote.TraceRequests = o.TraceRequests

	doNotFail = o.DoNotFail

	return o, nil
}

func getVersion() string {
	return VERSION
}

func main() {
	o, err := parseFlags()
	if err != nil {
		suicide(err, true)
	}

	dockerConfig, err := dockerconfig.Load(o.DockerJSON)
	if err != nil {
		suicide(err, true)
	}

	const format = "%-12s %-45s %-15s %-25s %s\n"
	fmt.Printf(format, "<STATE>", "<DIGEST>", "<(local) ID>", "<Created At>", "<TAG>")

	repoCount := len(o.Positional.Repositories)
	pullCount := 0
	pushCount := 0

	dc, err := dockerclient.New(dockerConfig)
	if err != nil {
		suicide(err, true)
	}

	type tagResult struct {
		Tags     []*tag.Tag
		Repo     string
		Path     string
		Registry string
	}

	trc := make(chan tagResult, repoCount)

	for _, r := range o.Positional.Repositories {
		go func(r string, o *Options, trc chan tagResult) {
			repository, filter, err := util.SeparateFilterAndRepo(r)
			if err != nil {
				suicide(err, true)
			}

			registry := docker.GetRegistry(repository)

			repoPath := docker.GetRepoPath(repository, registry)
			repoName := docker.GetRepoName(repository, registry)

			username, password, _ := dockerConfig.GetCredentials(registry)

			tr, err := auth.NewToken(registry, repoPath, username, password)
			if err != nil {
				suicide(err, true)
			}

			remoteTags, err := remote.FetchTags(registry, repoPath, tr.Header(), o.ConcurrentRequests)
			if err != nil {
				suicide(err, true)
			}

			imageSummaries, err := dc.ListImagesForRepo(repoName)
			if err != nil {
				suicide(err, true)
			}
			localTags, err := local.FetchTags(repoName, imageSummaries)
			if err != nil {
				suicide(err, true)
			}

			sortedKeys, names, joinedTags := tag.Join(remoteTags, localTags)

			tags := make([]*tag.Tag, 0)
			for _, key := range sortedKeys {
				name := names[key]

				tg := joinedTags[name]

				if !util.DoesMatch(tg.GetName(), filter) {
					continue
				}

				if tg.NeedsPull() {
					pullCount++
				}
				pushCount++

				tags = append(tags, tg)
			}

			trc <- tagResult{Tags: tags, Repo: repoName, Path: repoPath, Registry: registry}
		}(r, o, trc)
	}

	tagResults := make([]tagResult, repoCount)
	repoNumber := 0
	for tr := range trc {
		repoNumber++
		tagResults = append(tagResults, tr)
		if repoNumber >= repoCount {
			close(trc)
		}
	}

	for _, tr := range tagResults {
		for _, tg := range tr.Tags {
			fmt.Printf(
				format,
				tg.GetState(),
				tg.GetShortDigest(),
				tg.GetImageID(),
				tg.GetCreatedString(),
				tr.Repo+":"+tg.GetName(),
			)
		}
	}

	if o.Pull {
		done := make(chan bool, pullCount)

		for _, tr := range tagResults {
			go func(tags []*tag.Tag, repo string, done chan bool) {
				for _, tg := range tags {
					if tg.NeedsPull() {
						ref := repo + ":" + tg.GetName()

						fmt.Printf("PULLING %s\n", ref)
						err := dc.Pull(ref)
						if err != nil {
							suicide(err, false)
						}

						done <- true
					}

				}
			}(tr.Tags, tr.Repo, done)
		}

		pullNumber := 0
		if pullCount > 0 {
			for range done {
				pullNumber++

				if pullNumber >= pullCount {
					close(done)
				}
			}
		}
	}

	if o.Pull && o.PushRegistry != "" {
		done := make(chan bool, pullCount)

		for _, tr := range tagResults {
			go func(tags []*tag.Tag, repo, path, registry string, done chan bool) {
				for _, tg := range tags {
					prefix := o.PushPrefix
					if prefix == "" {
						prefix = util.GeneratePathFromHostname(registry)
					}

					srcRef := repo + ":" + tg.GetName()
					dstRef := o.PushRegistry + prefix + "/" + path + ":" + tg.GetName()

					fmt.Printf("PUSHING %s => %s\n", srcRef, dstRef)

					err := dc.Tag(srcRef, dstRef)
					if err != nil {
						suicide(err, true)
					}

					err = dc.Push(dstRef)
					if err != nil {
						suicide(err, false)
					}

					done <- true
				}
			}(tr.Tags, tr.Repo, tr.Path, tr.Registry, done)
		}

		pushNumber := 0
		if pushCount > 0 {
			for range done {
				pushNumber++

				if pushNumber >= pushCount {
					close(done)
				}
			}
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/ssbl/trago/db"
)

const (
	SERVFLAG = "-s"
	SERVCMD = "trago -s {dir}"
	serverUsage = "Run in server mode in the specified directory.\n"
)

var (
	server string
	serverDir string
	clientDir string
	flagDir string
	defaultDir = "(nil)"
)

func init() {
	flag.StringVar(&flagDir, "server", defaultDir, serverUsage)
	flag.StringVar(&flagDir, "s", defaultDir, "Shorthand for --server.")

	log.SetFlags(0)
	flag.Usage = usage
}

func main() {
	flag.Parse()

	if flagDir == defaultDir {
		server, serverDir, clientDir := parseArgs()
		log.Printf("%s:%s %s\n", server, serverDir, clientDir)
	} else {	  // running in server mode, so we ignore all other flags
		if err := os.Chdir(flagDir); err != nil {
			log.Fatalf("Error changing to directory: %s\n", err)
		}

		tradb, err := db.ParseFile()
		if err != nil {
			log.Fatalf("Error parsing db file: %s\n", err)
		}

		if err := tradb.Update(); err != nil {
			log.Fatalf("Error updating db file: %s\n", err)
		}

		if err := tradb.Write(); err != nil {
			log.Fatalf("Error writing to db file: %s\n", err)
		}

		bs, err := ioutil.ReadFile(db.TRADB)
		if err != nil {
			log.Fatalf("error reading file: %s\n", err)
		}

		fmt.Println(string(bs))		// send db to stdout
	}
}

func usage() {
	log.Printf("Usage: trago server:dir client-dir\n\n")

	log.Printf("-s <dir>, --server <dir>\n    %s\n", serverUsage);
}

func parseArgs() (string, string, string) {
	var s, sd, cd string

	if len(flag.Args()) != 2 {
		flag.Usage()
		os.Exit(1)
	}

	remote := strings.Split(flag.Arg(0), ":")
	if len(remote) != 2 {
		flag.Usage()
		os.Exit(1)
	} else {
		s = strings.TrimSpace(remote[0])
		sd = strings.TrimSpace(remote[1])
	}

	cd = strings.TrimSpace(flag.Arg(1))

	return s, sd, cd
}
<commit_msg>run server command over ssh<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/ssbl/trago/db"
)

const (
	SERVFLAG = "-s"
	SERVCMD = "trago -s {dir}"
	serverUsage = "Run in server mode in the specified directory.\n"
)

var (
	server string
	serverDir string
	clientDir string
	flagDir string
	defaultDir = "(nil)"
)

func init() {
	flag.StringVar(&flagDir, "server", defaultDir, serverUsage)
	flag.StringVar(&flagDir, "s", defaultDir, "Shorthand for --server.")

	log.SetFlags(0)
	flag.Usage = usage
}

func main() {
	flag.Parse()

	if flagDir == defaultDir {
		server, serverDir, clientDir := parseArgs()
		fmt.Printf("%s:%s %s\n", server, serverDir, clientDir)

		if err := os.Chdir(clientDir); err != nil {
			log.Fatalf("Error changing to directory: %s\n", err)
		}

		cmd := exec.Command("ssh", server,
			strings.Replace(SERVCMD, "{dir}", serverDir, 1))

		stdout := new(bytes.Buffer)
		stderr := new(bytes.Buffer)
		cmd.Stdout = stdout
		cmd.Stderr = stderr

		if err := cmd.Run(); err != nil {
			log.Fatalf(string(stderr.Bytes()))
		}

		parsed, err := db.Parse(string(stdout.Bytes()))
		if err != nil {
			log.Fatal("Error parsing server response")
		}

		fmt.Println(parsed)
	} else {	  // running in server mode, so we ignore all other flags
		if err := os.Chdir(flagDir); err != nil {
			log.Fatalf("Error changing to directory: %s\n", err)
		}

		tradb, err := db.ParseFile()
		if err != nil {
			log.Fatalf("Error parsing db file: %s\n", err)
		}

		if err := tradb.Update(); err != nil {
			log.Fatalf("Error updating db file: %s\n", err)
		}

		if err := tradb.Write(); err != nil {
			log.Fatalf("Error writing to db file: %s\n", err)
		}

		bs, err := ioutil.ReadFile(db.TRADB)
		if err != nil {
			log.Fatalf("error reading file: %s\n", err)
		}

		fmt.Println(string(bs))		// send db to stdout
	}
}

func usage() {
	log.Printf("Usage: trago server:dir client-dir\n\n")

	log.Printf("-s <dir>, --server <dir>\n    %s\n", serverUsage);
}

func parseArgs() (string, string, string) {
	var s, sd, cd string

	if len(flag.Args()) != 2 {
		flag.Usage()
		os.Exit(1)
	}

	remote := strings.Split(flag.Arg(0), ":")
	if len(remote) != 2 {
		flag.Usage()
		os.Exit(1)
	} else {
		s = strings.TrimSpace(remote[0])
		sd = strings.TrimSpace(remote[1])
	}

	cd = strings.TrimSpace(flag.Arg(1))

	return s, sd, cd
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"fmt"
	"log"
	"net/http"

	"gopkg.in/mgo.v2"

	"github.com/labstack/echo"

	"github.com/non-player-games/manager"
)

type Player struct {
	Name  string
	Token string
	Email string
}

func randToken() string {
	b := make([]byte, 42)
	rand.Read(b)
	return fmt.Sprintf("%x", b)
}

func signup(player *mgo.Collection) echo.HandlerFunc {
	return func(c echo.Context) error {
		u := &Player{}
		if err := c.Bind(u); err != nil {
			return err
		}
		u.Token = randToken()

		if err := player.Insert(u); err != nil {
			return err
		}

		log.Printf("Inserted new user %q\n", u)

		return c.JSON(http.StatusCreated, u)
	}
}

func main() {
	manager.RunGame("http://localhost:8080")

	// mongoAddr := os.Getenv("MONGO_CONNECTION")
	// mongoDbName := os.Getenv("MONGO_DB_NAME")
	// session, err := mgo.Dial(mongoAddr)
	// if err != nil {
	// 	panic(err)
	// }
	// defer session.Close()
	//
	// players := session.DB(mongoDbName).C("players")
	//
	// e := echo.New()
	// e.Use(middleware.Logger())
	// e.Use(middleware.Recover())
	// e.Use(middleware.CORS())
	//
	// e.Static("/", "public")
	//
	// e.GET("/api/hello", func(c echo.Context) error {
	// 	return c.String(http.StatusOK, "Hello, Contest Framework!\n")
	// })
	//
	// e.POST("/api/v1/user/signup", signup(players))
	//
	// log.Println("Contest Framework starting at port 9000")
	// e.Run(standard.New(":9000"))
}
<commit_msg>uncomment web server code<commit_after>package main

import (
	"crypto/rand"
	"fmt"
	"log"
	"net/http"
	"os"

	"gopkg.in/mgo.v2"

	"github.com/labstack/echo"
	"github.com/labstack/echo/engine/standard"
	"github.com/labstack/echo/middleware"
)

type Player struct {
	Name  string
	Token string
	Email string
}

func randToken() string {
	b := make([]byte, 42)
	rand.Read(b)
	return fmt.Sprintf("%x", b)
}

func signup(player *mgo.Collection) echo.HandlerFunc {
	return func(c echo.Context) error {
		u := &Player{}
		if err := c.Bind(u); err != nil {
			return err
		}
		u.Token = randToken()

		if err := player.Insert(u); err != nil {
			return err
		}

		log.Printf("Inserted new user %q\n", u)

		return c.JSON(http.StatusCreated, u)
	}
}

func main() {
	// manager.RunGame("http://localhost:8080")

	mongoAddr := os.Getenv("MONGO_CONNECTION")
	mongoDbName := os.Getenv("MONGO_DB_NAME")
	session, err := mgo.Dial(mongoAddr)
	if err != nil {
		panic(err)
	}
	defer session.Close()

	players := session.DB(mongoDbName).C("players")

	e := echo.New()
	e.Use(middleware.Logger())
	e.Use(middleware.Recover())
	e.Use(middleware.CORS())

	e.Static("/", "public")

	e.GET("/api/hello", func(c echo.Context) error {
		return c.String(http.StatusOK, "Hello, Contest Framework!\n")
	})

	e.POST("/api/v1/user/signup", signup(players))

	log.Println("Contest Framework starting at port 9000")
	e.Run(standard.New(":9000"))
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/aqatl/mal/mal"
	"github.com/urfave/cli"
	"os"
	"log"
	"sort"
	"strconv"
	"path/filepath"
	"github.com/skratchdot/open-golang/open"
	"fmt"
	"strings"
)

var dataDir = filepath.Join(homeDir(), ".mal")
var (
	CredentialsFile = filepath.Join(dataDir, "cred.dat")
	MalCacheFile    = filepath.Join(dataDir, "cache.xml")
	ConfigFile      = filepath.Join(dataDir, "config.json")
)

func main() {
	checkDataDir()

	app := cli.NewApp()
	app.Name = "mal"
	app.Usage = "App for managing your MAL"
	app.Version = "0.1"

	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "creds, prompt-credentials",
			Usage: "Prompt for username and password",
		},
		cli.BoolFlag{
			Name:  "sp, save-password",
			Usage: "save your password. Use with caution, your password can be decoded",
		},
		cli.BoolFlag{
			Name:  "r, refresh",
			Usage: "refreshes cache file",
		},
		cli.BoolFlag{
			Name:  "ver, verify",
			Usage: "verify credentials",
		},
		cli.IntFlag{
			Name:  "max",
			Usage: "visible entries threshold",
		},
		cli.StringFlag{
			Name:  "status",
			Usage: "display entries only with given status",
		},
	}

	app.Commands = []cli.Command{
		cli.Command{
			Name:      "inc",
			Aliases:   []string{"+1"},
			Category:  "Update",
			Usage:     "Increment selected entry by one",
			UsageText: "mal inc",
			Action:    incrementEntry,
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "n",
					Usage: "Specify exact episode to set the entry to",
				},
			},
		},
		cli.Command{
			Name:      "score",
			Category:  "Update",
			Usage:     "Set your rating for selected entry",
			UsageText: "mal score <0-10>",
			Action:    setEntryScore,
		},
		cli.Command{
			Name:      "status",
			Category:  "Update",
			Usage:     "Set your status for selected entry",
			UsageText: "mal status [watching|completed|onhold|dropped|plantowatch]",
			Action:    setEntryStatus,
		},
		cli.Command{
			Name:      "sel",
			Aliases:   []string{"select"},
			Category:  "Config",
			Usage:     "Select an entry",
			UsageText: "mal sel [entry ID]",
			Action:    selectEntry,
		},
		cli.Command{
			Name:     "cfg",
			Aliases:  []string{"config", "configuration"},
			Category: "Config",
			Usage:    "Change config values",
			Subcommands: cli.Commands{
				cli.Command{
					Name:      "max",
					Aliases:   []string{"visible"},
					Usage:     "Change amount of displayed entries",
					UsageText: "mal cfg max [number]",
					Action:    configChangeMax,
				},
				cli.Command{
					Name:      "status",
					Usage:     "Status value of displayed entries",
					UsageText: "mal cfg status [all|watching|completed|onhold|dropped|plantowatch]",
					Action:    configChangeStatus,
				},
				cli.Command{
					Name:      "status-auto-update",
					Usage:     "Allows entry to be automatically set to completed when number of all episodes is reached or exceeded",
					UsageText: "mal cfg status-auto-update [off|normal|after-threshold]",
					Action:    configChangeAutoUpdateMode,
				},
			},
		},
		cli.Command{
			Name:      "web",
			Aliases:   []string{"website", "open"},
			Category:  "Action",
			Usage:     "Open url associated with current entry",
			UsageText: "mal web",
			Action:    openWebsite,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "url",
					Usage: "Set url for current entry",
				},
				cli.BoolFlag{
					Name:  "clear",
					Usage: "Clear url for current entry",
				},
			},
			Subcommands: []cli.Command {
				cli.Command{
					Name: "get-all",
					Usage: "Print all set urls",
					UsageText: "mal web get-all",
					Action: printWebsites,
				},
			},
		},
	}

	app.Action = defaultAction

	if err := app.Run(os.Args); err != nil {
		log.Fatalf("Error: %v", err)
	}
}

func defaultAction(ctx *cli.Context) {
	creds := loadCredentials(ctx)
	if ctx.Bool("verify") && !mal.VerifyCredentials(creds) {
		log.Fatalln("Invalid credentials")
	}

	c := mal.NewClient(creds)
	if c == nil {
		os.Exit(1)
	}

	config := LoadConfig()

	list := loadList(c, ctx).FilterByStatus(statusFlag(ctx))
	sort.Sort(mal.AnimeSortByLastUpdated(list))

	var visibleEntries int
	if visibleEntries = ctx.Int("max"); visibleEntries == 0 {
		//`Max` flag not specified, get value from config
		visibleEntries = config.MaxVisibleEntries
	}
	if visibleEntries > len(list) {
		visibleEntries = len(list)
	}
	visibleList := list[:visibleEntries]
	reverseAnimeSlice(visibleList)

	PrettyList.Execute(os.Stdout, PrettyListData{visibleList, config.SelectedID})

	if ctx.GlobalBool("save-password") {
		saveCredentials(creds)
	}
}

func incrementEntry(ctx *cli.Context) error {
	creds := loadCredentials(ctx)
	if ctx.Bool("verify") && !mal.VerifyCredentials(creds) {
		return fmt.Errorf("invalid credentials")
	}

	c := mal.NewClient(creds)
	if c == nil {
		os.Exit(1)
	}
	cfg := LoadConfig()

	if cfg.SelectedID == 0 {
		return fmt.Errorf("no entry selected")
	}

	list := loadList(c, ctx)
	selectedEntry := list.GetByID(cfg.SelectedID)

	if selectedEntry == nil {
		return fmt.Errorf("no entry found")
	}

	if ctx.Int("n") > 0 {
		selectedEntry.WatchedEpisodes = ctx.Int("n")
	} else {
		selectedEntry.WatchedEpisodes++
	}

	statusAutoUpdate(cfg, selectedEntry)

	if selectedEntry.WatchedEpisodes > selectedEntry.Episodes {
		selectedEntry.WatchedEpisodes = selectedEntry.Episodes
	}

	if c.Update(selectedEntry) {
		log.Printf("Updated successfully")
		cacheList(list)
	}
	return nil
}

func setEntryScore(ctx *cli.Context) error {
	c := mal.NewClient(loadCredentials(ctx))
	cfg := LoadConfig()

	list := loadList(c, ctx)
	selectedEntry := list.GetByID(cfg.SelectedID)

	score, err := strconv.Atoi(ctx.Args().First())
	if err != nil {
		return fmt.Errorf("error parsing score: %v", err)
	}
	parsedScore, err := mal.ParseScore(score)
	if err != nil {
		return err
	}

	selectedEntry.MyScore = parsedScore
	if c.Update(selectedEntry) {
		log.Printf("Updated successfully")
		cacheList(list)
	}
	return nil
}

func setEntryStatus(ctx *cli.Context) error {
	c := mal.NewClient(loadCredentials(ctx))
	cfg := LoadConfig()

	list := loadList(c, ctx)
	selectedEntry := list.GetByID(cfg.SelectedID)

	status := mal.ParseStatus(ctx.Args().First())
	if status == mal.All {
		return fmt.Errorf("invalid status; possible values: watching, completed, " +
			"onhold, dropped, plantowatch")
	}

	selectedEntry.MyStatus = status
	if c.Update(selectedEntry) {
		log.Printf("Updated successfully")
		cacheList(list)
	}
	return nil
}

func selectEntry(ctx *cli.Context) error {
	cfg := LoadConfig()

	id, err := strconv.Atoi(ctx.Args().First())
	if err != nil {
		return fmt.Errorf("error parsing id: %v", err)
	}

	cfg.SelectedID = id
	cfg.Save()
	return nil
}

func openWebsite(ctx *cli.Context) error {
	cfg := LoadConfig()

	if url := ctx.String("url"); url != "" {
		cfg.Websites[cfg.SelectedID] = url
		log.Printf("Set url %s for entry %d", url, cfg.SelectedID)
		cfg.Save()
		return nil
	}

	if ctx.Bool("clear") {
		delete(cfg.Websites, cfg.SelectedID)
		log.Printf("Cleared url for entry %d", cfg.SelectedID)
		cfg.Save()
		return nil
	}

	if url, ok := cfg.Websites[cfg.SelectedID]; ok {
		open.Start(url)
	} else {
		log.Println("Nothing to open")
	}

	return nil
}

func printWebsites(ctx *cli.Context) error {
	cfg := LoadConfig()
	list := loadCachedList()

	for k, v := range cfg.Websites {
		url := fmt.Sprintf("\033[3%d;%dm%s\033[0m ", 3, 1, v)

		var title string
		if entry := list.GetByID(k); entry != nil {
			title = entry.Title
		}

		fmt.Printf("%6d (%s): %s\n", k, title, url)
	}

	return nil
}

func configChangeMax(ctx *cli.Context) error {
	cfg := LoadConfig()

	max, err := strconv.Atoi(ctx.Args().First())
	if err != nil || max < 0 {
		return fmt.Errorf("invalid value")
	}

	cfg.MaxVisibleEntries = max
	cfg.Save()
	return nil
}

func configChangeStatus(ctx *cli.Context) error {
	cfg := LoadConfig()

	status := mal.ParseStatus(ctx.Args().First())

	cfg.Status = status
	cfg.Save()
	return nil
}

func configChangeAutoUpdateMode(ctx *cli.Context) error {
	arg := strings.ToLower(ctx.Args().First())
	var mode StatusAutoUpdateMode

	if arg == "off" {
		mode = Off
	} else if arg == "normal" {
		mode = Normal
	} else if arg == "after-threshold" {
		mode = AfterThreshold
	} else {
		return fmt.Errorf("invalid option; possible values: off|normal|after-threshold")
	}

	cfg := LoadConfig()
	cfg.StatusAutoUpdateMode = mode
	cfg.Save()

	return nil
}
<commit_msg>Selecting entry by title<commit_after>package main

import (
	"github.com/aqatl/mal/mal"
	"github.com/urfave/cli"
	"os"
	"log"
	"sort"
	"strconv"
	"path/filepath"
	"github.com/skratchdot/open-golang/open"
	"fmt"
	"strings"
)

var dataDir = filepath.Join(homeDir(), ".mal")
var (
	CredentialsFile = filepath.Join(dataDir, "cred.dat")
	MalCacheFile    = filepath.Join(dataDir, "cache.xml")
	ConfigFile      = filepath.Join(dataDir, "config.json")
)

func main() {
	checkDataDir()

	app := cli.NewApp()
	app.Name = "mal"
	app.Usage = "App for managing your MAL"
	app.Version = "0.1"

	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "creds, prompt-credentials",
			Usage: "Prompt for username and password",
		},
		cli.BoolFlag{
			Name:  "sp, save-password",
			Usage: "save your password. Use with caution, your password can be decoded",
		},
		cli.BoolFlag{
			Name:  "r, refresh",
			Usage: "refreshes cache file",
		},
		cli.BoolFlag{
			Name:  "ver, verify",
			Usage: "verify credentials",
		},
		cli.IntFlag{
			Name:  "max",
			Usage: "visible entries threshold",
		},
		cli.StringFlag{
			Name:  "status",
			Usage: "display entries only with given status",
		},
	}

	app.Commands = []cli.Command{
		cli.Command{
			Name:      "inc",
			Aliases:   []string{"+1"},
			Category:  "Update",
			Usage:     "Increment selected entry by one",
			UsageText: "mal inc",
			Action:    incrementEntry,
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "n",
					Usage: "Specify exact episode to set the entry to",
				},
			},
		},
		cli.Command{
			Name:      "score",
			Category:  "Update",
			Usage:     "Set your rating for selected entry",
			UsageText: "mal score <0-10>",
			Action:    setEntryScore,
		},
		cli.Command{
			Name:      "status",
			Category:  "Update",
			Usage:     "Set your status for selected entry",
			UsageText: "mal status [watching|completed|onhold|dropped|plantowatch]",
			Action:    setEntryStatus,
		},
		cli.Command{
			Name:      "sel",
			Aliases:   []string{"select"},
			Category:  "Config",
			Usage:     "Select an entry",
			UsageText: "mal sel [entry ID]",
			Action:    selectEntry,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "by-title",
					Usage: "Select entry by name instead of by ID",
				},
			},
		},
		cli.Command{
			Name:     "cfg",
			Aliases:  []string{"config", "configuration"},
			Category: "Config",
			Usage:    "Change config values",
			Subcommands: cli.Commands{
				cli.Command{
					Name:      "max",
					Aliases:   []string{"visible"},
					Usage:     "Change amount of displayed entries",
					UsageText: "mal cfg max [number]",
					Action:    configChangeMax,
				},
				cli.Command{
					Name:      "status",
					Usage:     "Status value of displayed entries",
					UsageText: "mal cfg status [all|watching|completed|onhold|dropped|plantowatch]",
					Action:    configChangeStatus,
				},
				cli.Command{
					Name:      "status-auto-update",
					Usage:     "Allows entry to be automatically set to completed when number of all episodes is reached or exceeded",
					UsageText: "mal cfg status-auto-update [off|normal|after-threshold]",
					Action:    configChangeAutoUpdateMode,
				},
			},
		},
		cli.Command{
			Name:      "web",
			Aliases:   []string{"website", "open"},
			Category:  "Action",
			Usage:     "Open url associated with current entry",
			UsageText: "mal web",
			Action:    openWebsite,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "url",
					Usage: "Set url for current entry",
				},
				cli.BoolFlag{
					Name:  "clear",
					Usage: "Clear url for current entry",
				},
			},
			Subcommands: []cli.Command{
				cli.Command{
					Name:      "get-all",
					Usage:     "Print all set urls",
					UsageText: "mal web get-all",
					Action:    printWebsites,
				},
			},
		},
	}

	app.Action = defaultAction

	if err := app.Run(os.Args); err != nil {
		log.Fatalf("Error: %v", err)
	}
}

func defaultAction(ctx *cli.Context) {
	creds := loadCredentials(ctx)
	if ctx.Bool("verify") && !mal.VerifyCredentials(creds) {
		log.Fatalln("Invalid credentials")
	}

	c := mal.NewClient(creds)
	if c == nil {
		os.Exit(1)
	}

	config := LoadConfig()

	list := loadList(c, ctx).FilterByStatus(statusFlag(ctx))
	sort.Sort(mal.AnimeSortByLastUpdated(list))

	var visibleEntries int
	if visibleEntries = ctx.Int("max"); visibleEntries == 0 {
		//`Max` flag not specified, get value from config
		visibleEntries = config.MaxVisibleEntries
	}
	if visibleEntries > len(list) {
		visibleEntries = len(list)
	}
	visibleList := list[:visibleEntries]
	reverseAnimeSlice(visibleList)

	PrettyList.Execute(os.Stdout, PrettyListData{visibleList, config.SelectedID})

	if ctx.GlobalBool("save-password") {
		saveCredentials(creds)
	}
}

func incrementEntry(ctx *cli.Context) error {
	creds := loadCredentials(ctx)
	if ctx.Bool("verify") && !mal.VerifyCredentials(creds) {
		return fmt.Errorf("invalid credentials")
	}

	c := mal.NewClient(creds)
	if c == nil {
		os.Exit(1)
	}
	cfg := LoadConfig()

	if cfg.SelectedID == 0 {
		return fmt.Errorf("no entry selected")
	}

	list := loadList(c, ctx)
	selectedEntry := list.GetByID(cfg.SelectedID)

	if selectedEntry == nil {
		return fmt.Errorf("no entry found")
	}

	if ctx.Int("n") > 0 {
		selectedEntry.WatchedEpisodes = ctx.Int("n")
	} else {
		selectedEntry.WatchedEpisodes++
	}

	statusAutoUpdate(cfg, selectedEntry)

	if selectedEntry.WatchedEpisodes > selectedEntry.Episodes {
		selectedEntry.WatchedEpisodes = selectedEntry.Episodes
	}

	if c.Update(selectedEntry) {
		log.Printf("Updated successfully")
		cacheList(list)
	}
	return nil
}

func setEntryScore(ctx *cli.Context) error {
	c := mal.NewClient(loadCredentials(ctx))
	cfg := LoadConfig()

	list := loadList(c, ctx)
	selectedEntry := list.GetByID(cfg.SelectedID)

	score, err := strconv.Atoi(ctx.Args().First())
	if err != nil {
		return fmt.Errorf("error parsing score: %v", err)
	}
	parsedScore, err := mal.ParseScore(score)
	if err != nil {
		return err
	}

	selectedEntry.MyScore = parsedScore
	if c.Update(selectedEntry) {
		log.Printf("Updated successfully")
		cacheList(list)
	}
	return nil
}

func setEntryStatus(ctx *cli.Context) error {
	c := mal.NewClient(loadCredentials(ctx))
	cfg := LoadConfig()

	list := loadList(c, ctx)
	selectedEntry := list.GetByID(cfg.SelectedID)

	status := mal.ParseStatus(ctx.Args().First())
	if status == mal.All {
		return fmt.Errorf("invalid status; possible values: watching, completed, " +
			"onhold, dropped, plantowatch")
	}

	selectedEntry.MyStatus = status
	if c.Update(selectedEntry) {
		log.Printf("Updated successfully")
		cacheList(list)
	}
	return nil
}

func selectEntry(ctx *cli.Context) error {
	if ctx.Bool("by-title") {
		return selectByTitle(ctx)
	}

	id, err := strconv.Atoi(ctx.Args().First())
	if err != nil {
		return fmt.Errorf("error parsing id: %v", err)
	}

	cfg := LoadConfig()
	cfg.SelectedID = id
	cfg.Save()
	return nil
}

func selectByTitle(ctx *cli.Context) error {
	title := strings.ToLower(ctx.Args().First())
	list := loadList(mal.NewClient(loadCredentials(ctx)), ctx)

	found := make(mal.AnimeList, 0)
	for _, entry := range list {
		if strings.Contains(strings.ToLower(entry.Title), title) {
			found = append(found, entry)
		}
	}

	var id int

	if len(found) > 1 {
		fmt.Printf("Found more than 1 matching entry:\n")
		fmt.Printf("%3s%8s%7s\n", "No.", "ID", "Title")
		fmt.Println(strings.Repeat("=", 80))
		for i, entry := range found {
			fmt.Printf("%3d. %6d: %s\n", i, entry.ID, entry.Title)
		}

		fmt.Printf("Enter index of the selected entry: ")
		idx := 0
		_, err := fmt.Scanln(&idx)
		if err != nil || idx < 0 || idx > len(found)-1 {
			return fmt.Errorf("invalid input %v", err)
		}

		id = found[idx].ID
	} else if len(found) == 0 {
		return fmt.Errorf("no matches")
	} else {
		id = found[0].ID
	}

	cfg := LoadConfig()
	cfg.SelectedID = id
	cfg.Save()
	return nil
}

func openWebsite(ctx *cli.Context) error {
	cfg := LoadConfig()

	if url := ctx.String("url"); url != "" {
		cfg.Websites[cfg.SelectedID] = url
		log.Printf("Set url %s for entry %d", url, cfg.SelectedID)
		cfg.Save()
		return nil
	}

	if ctx.Bool("clear") {
		delete(cfg.Websites, cfg.SelectedID)
		log.Printf("Cleared url for entry %d", cfg.SelectedID)
		cfg.Save()
		return nil
	}

	if url, ok := cfg.Websites[cfg.SelectedID]; ok {
		open.Start(url)
	} else {
		log.Println("Nothing to open")
	}

	return nil
}

func printWebsites(ctx *cli.Context) error {
	cfg := LoadConfig()
	list := loadCachedList()

	for k, v := range cfg.Websites {
		url := fmt.Sprintf("\033[3%d;%dm%s\033[0m ", 3, 1, v)

		var title string
		if entry := list.GetByID(k); entry != nil {
			title = entry.Title
		}

		fmt.Printf("%6d (%s): %s\n", k, title, url)
	}

	return nil
}

func configChangeMax(ctx *cli.Context) error {
	cfg := LoadConfig()

	max, err := strconv.Atoi(ctx.Args().First())
	if err != nil || max < 0 {
		return fmt.Errorf("invalid value")
	}

	cfg.MaxVisibleEntries = max
	cfg.Save()
	return nil
}

func configChangeStatus(ctx *cli.Context) error {
	cfg := LoadConfig()

	status := mal.ParseStatus(ctx.Args().First())

	cfg.Status = status
	cfg.Save()
	return nil
}

func configChangeAutoUpdateMode(ctx *cli.Context) error {
	arg := strings.ToLower(ctx.Args().First())
	var mode StatusAutoUpdateMode

	if arg == "off" {
		mode = Off
	} else if arg == "normal" {
		mode = Normal
	} else if arg == "after-threshold" {
		mode = AfterThreshold
	} else {
		return fmt.Errorf("invalid option; possible values: off|normal|after-threshold")
	}

	cfg := LoadConfig()
	cfg.StatusAutoUpdateMode = mode
	cfg.Save()

	return nil
}
<|endoftext|>


<commit_before>// Wago (Watch, Go)
// A general purpose watch / build development tool.

// TODO: catch SIGINT and reset term
// see https://askubuntu.com/questions/171449/shell-does-not-show-typed-in-commands-reset-works-but-what-happened

package main

import (
	"flag"
	"fmt"
	"github.com/JonahBraun/dog"
	"github.com/howeyc/fsnotify"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"regexp"
	"runtime"
	"sync"
)

const VERSION = "1.0"

var (
	log     = dog.NewDog(dog.DEBUG)
	verbose = flag.Bool("v", false, "Verbose")
	quiet   = flag.Bool("q", false, "Quiet, only warnings and errors")

	buildCmd      = flag.String("cmd", "", "Run command, wait for it to complete.")
	daemonCmd     = flag.String("daemon", "", "Run command and leave running in the background.")
	daemonTimer   = flag.Int("timer", 0, "Wait miliseconds after starting daemon, then continue.")
	daemonTrigger = flag.String("trigger", "", "Wait for daemon to output this string, then continue.")
	exitWait      = flag.Int("exitwait", 50, "Max miliseconds a process has after a SIGTERM to exit before a SIGKILL.")
	fiddle        = flag.Bool("fiddle", false, "CLI fiddle mode! Start a web server, open browser to URL of targetDir/index.html")
	postCmd       = flag.String("pcmd", "", "Run command after daemon starts. Use this to kick off your test suite.")
	recursive     = flag.Bool("recursive", true, "Watch directory tree recursively.")
	targetDir     = flag.String("dir", "", "Directory to watch, defaults to current.")
	url           = flag.String("url", "", "Open browser to this URL after all commands are successful.")
	watchRegex    = flag.String("watch", `\w[\w\.]*": (CREATE|MODIFY$)`, "React to FS events matching regex. Use -v to see all events.")
	ignoreRegex   = flag.String("ignore", `\.(git|hg|svn)`, "Ignore directories matching regex.")
	webServer     = flag.String("web", "", "Start a web server at this address, e.g. :8420")
	shell         = flag.String("shell", "", "Shell used to run commands, defaults to $SHELL, fallback to /bin/sh")
)

type Watcher struct {
	Event chan fmt.Stringer
	Error chan error
}

func main() {
	// the following function calls merely serve to logically organize what
	// is otherwise a VERY lengthy setup

	// TODO: have configSetup return a config object so that the reliance on
	// config globals is removed
	configSetup()

	startWebServer()

	ManageStdin()

	runChain(newWatcher(), catchSignals())
}

func catchSignals() chan struct{} {
	// quit needs to inform multiple receivers, sig can't do that
	quit := make(chan struct{})
	sig := make(chan os.Signal, 1)

	// TODO add SIGTERM to this (need OS conditional)
	signal.Notify(sig, os.Interrupt, os.Kill)

	go func() {
		<-sig
		close(quit)
	}()

	return quit
}

// event loop and action chain happen here
func runChain(watcher *Watcher, quit chan struct{}) {
	chain := make([]Runnable, 0, 5)

	// build chain of runnables
	if len(*buildCmd) > 0 {
		chain = append(chain, NewRunWait(*buildCmd))
	}
	if len(*daemonCmd) > 0 {
		if len(*daemonTrigger) > 0 {
			chain = append(chain, NewDaemonTrigger(*daemonCmd, *daemonTrigger))
		} else {
			chain = append(chain, NewDaemonTimer(*daemonCmd, *daemonTimer))
		}
	}
	if len(*postCmd) > 0 {
		chain = append(chain, NewRunWait(*postCmd))
	}
	if *url != "" {
		chain = append(chain, NewBrowser(*url))
	}

	eventRegex, err := regexp.Compile(*watchRegex)
	if err != nil {
		log.Fatal("Watch regex compile error:", err)(1)
	}

	var wg sync.WaitGroup

	// main loop
	for {
		// kill is passed to all Runnable so they know when they should exit
		kill := make(chan struct{})

		var drain func()
		drain = func() {
			select {
			case ev := <-watcher.Event:
				log.Debug("Extra event ignored:", ev.String())
				drain()
			default:
			}
		}
		drain()

		// event loop
		go func() {
			for {
				select {
				case ev := <-watcher.Event:
					if eventRegex.MatchString(ev.String()) {
						log.Info("Matched event:", ev.String())
						close(kill)
						return
					} else {
						log.Debug("Ignored event:", ev.String())
					}
				case err = <-watcher.Error:
					log.Fatal("Watcher error:", err)(5)
				case <-quit:
					close(kill)
					return
				}
			}
		}()

	RunLoop:
		for _, runnable := range chain {
			done, dead := runnable(kill)
			wg.Add(1)

			go func() {
				<-dead
				wg.Done()
			}()

			select {
			case d := <-done:
				if !d {
					// Runnable's success metric failed, break out of the chain
					break RunLoop
				}
			case <-kill:
				break RunLoop
			}
		}

		// ensure an event has occured, we may be here because all runnables completed
		<-kill

		// ensure all runnables (procs) are dead before restarting the chain
		wg.Wait()

		// check if we should quit
		select {
		case <-quit:
			log.Debug("Quitting main event/action loop")
			return
		default:
		}
	}
}

func newWatcher() *Watcher {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		panic(err)
	}

	ignore, err := regexp.Compile(*ignoreRegex)
	if err != nil {
		log.Fatal("Ignore regex compile error:", err)(1)
	}

	watchDir := func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			return nil
		}

		if err != nil {
			log.Err("Error reading dir, skipping:", path, err)
			return filepath.SkipDir
		}

		if ignore.MatchString(path) {
			log.Debug("Ignoring dir:", path)
			return filepath.SkipDir
		}

		log.Debug("Watching dir:", path)
		err = watcher.Watch(path)
		if err != nil {
			log.Err("Error watching dir (path, error):", path, err)
		}

		return nil
	}

	if *recursive == true {
		// errors are handled in watchDir
		filepath.Walk(*targetDir, watchDir)
	} else {
		err = watcher.Watch(*targetDir)
		if err != nil {
			log.Fatal("Error watching dir (path, error):", *targetDir, err)(1)
		}
	}

	// To facilitate testing (which sends artifical events from a timer),
	// we have an abstracted struct Watcher that holds the applicable channels.
	// fsnotify.FileEvent is a fmt.Stringer, but channels cannot be converted.
	// Unfortunately, an extra channel is necessary to perform the conversion.
	event := make(chan fmt.Stringer)
	go func() {
		for {
			event <- <-watcher.Event
		}
	}()

	return &Watcher{event, watcher.Error}
}

func startWebServer() {
	if *webServer != "" {
		go func() {
			log.Info("Starting web server on port", *webServer)
			err := http.ListenAndServe(*webServer, http.FileServer(http.Dir(*targetDir)))
			if err != nil {
				log.Fatal("Error starting web server:", err)(2)
			}
		}()
	}
}

func configSetup() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.Usage = func() {
		fmt.Println("WaGo (Watch, Go) build tool. Version", VERSION)
		flag.PrintDefaults()
	}

	// TODO: this should check for actions
	if len(os.Args) < 2 {
		flag.Usage()
		log.Fatal("You must specify an action")(1)
	}

	flag.Parse()

	if *verbose {
		log = dog.NewDog(dog.DEBUG)
	} else if *quiet {
		log = dog.NewDog(dog.WARN)
	} else {
		log = dog.NewDog(dog.INFO)
	}

	if len(*shell) == 0 {
		*shell = os.Getenv("SHELL")
		if len(*shell) == 0 {
			*shell = "/bin/sh"
		}
	}
	log.Debug("Using shell", *shell)

	if (len(*daemonTrigger) > 0) && (*daemonTimer > 0) {
		log.Fatal("Both daemon trigger and timer specified, use only one")(1)
	}

	if (len(*daemonTrigger) > 0 || *daemonTimer > 0) && len(*daemonCmd) == 0 {
		log.Fatal("Specify a daemon command to use the trigger or timer")(1)
	}

	if len(*buildCmd) == 0 && len(*daemonCmd) == 0 && !*fiddle && len(*postCmd) == 0 && len(*url) == 0 && len(*webServer) == 0 {
		flag.Usage()
		log.Fatal("You must specify an action")(1)
	}

	if *fiddle {
		if *webServer == "" {
			*webServer = ":9933"
		}
		if *url == "" {
			*url = "http://localhost" + *webServer + "/"
		}
	}

	if *targetDir == "" {
		cwd, err := os.Getwd()
		if err != nil {
			panic(err)
		}
		targetDir = &cwd
	}
	log.Debug("Target dir:", *targetDir)
}
<commit_msg>fix watch regex<commit_after>// Wago (Watch, Go)
// A general purpose watch / build development tool.

// TODO: catch SIGINT and reset term
// see https://askubuntu.com/questions/171449/shell-does-not-show-typed-in-commands-reset-works-but-what-happened

package main

import (
	"flag"
	"fmt"
	"github.com/JonahBraun/dog"
	"github.com/howeyc/fsnotify"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"regexp"
	"runtime"
	"sync"
)

const VERSION = "1.0"

var (
	log     = dog.NewDog(dog.DEBUG)
	verbose = flag.Bool("v", false, "Verbose")
	quiet   = flag.Bool("q", false, "Quiet, only warnings and errors")

	buildCmd      = flag.String("cmd", "", "Run command, wait for it to complete.")
	daemonCmd     = flag.String("daemon", "", "Run command and leave running in the background.")
	daemonTimer   = flag.Int("timer", 0, "Wait miliseconds after starting daemon, then continue.")
	daemonTrigger = flag.String("trigger", "", "Wait for daemon to output this string, then continue.")
	exitWait      = flag.Int("exitwait", 50, "Max miliseconds a process has after a SIGTERM to exit before a SIGKILL.")
	fiddle        = flag.Bool("fiddle", false, "CLI fiddle mode! Start a web server, open browser to URL of targetDir/index.html")
	postCmd       = flag.String("pcmd", "", "Run command after daemon starts. Use this to kick off your test suite.")
	recursive     = flag.Bool("recursive", true, "Watch directory tree recursively.")
	targetDir     = flag.String("dir", "", "Directory to watch, defaults to current.")
	url           = flag.String("url", "", "Open browser to this URL after all commands are successful.")
	watchRegex    = flag.String("watch", `/\w[\w\.]*": (CREATE|MODIFY$)`, "React to FS events matching regex. Use -v to see all events.")
	ignoreRegex   = flag.String("ignore", `\.(git|hg|svn)`, "Ignore directories matching regex.")
	webServer     = flag.String("web", "", "Start a web server at this address, e.g. :8420")
	shell         = flag.String("shell", "", "Shell used to run commands, defaults to $SHELL, fallback to /bin/sh")
)

type Watcher struct {
	Event chan fmt.Stringer
	Error chan error
}

func main() {
	// the following function calls merely serve to logically organize what
	// is otherwise a VERY lengthy setup

	// TODO: have configSetup return a config object so that the reliance on
	// config globals is removed
	configSetup()

	startWebServer()

	ManageStdin()

	runChain(newWatcher(), catchSignals())
}

func catchSignals() chan struct{} {
	// quit needs to inform multiple receivers, sig can't do that
	quit := make(chan struct{})
	sig := make(chan os.Signal, 1)

	// TODO add SIGTERM to this (need OS conditional)
	signal.Notify(sig, os.Interrupt, os.Kill)

	go func() {
		<-sig
		close(quit)
	}()

	return quit
}

// event loop and action chain happen here
func runChain(watcher *Watcher, quit chan struct{}) {
	chain := make([]Runnable, 0, 5)

	// build chain of runnables
	if len(*buildCmd) > 0 {
		chain = append(chain, NewRunWait(*buildCmd))
	}
	if len(*daemonCmd) > 0 {
		if len(*daemonTrigger) > 0 {
			chain = append(chain, NewDaemonTrigger(*daemonCmd, *daemonTrigger))
		} else {
			chain = append(chain, NewDaemonTimer(*daemonCmd, *daemonTimer))
		}
	}
	if len(*postCmd) > 0 {
		chain = append(chain, NewRunWait(*postCmd))
	}
	if *url != "" {
		chain = append(chain, NewBrowser(*url))
	}

	eventRegex, err := regexp.Compile(*watchRegex)
	if err != nil {
		log.Fatal("Watch regex compile error:", err)(1)
	}

	var wg sync.WaitGroup

	// main loop
	for {
		// kill is passed to all Runnable so they know when they should exit
		kill := make(chan struct{})

		var drain func()
		drain = func() {
			select {
			case ev := <-watcher.Event:
				log.Debug("Extra event ignored:", ev.String())
				drain()
			default:
			}
		}
		drain()

		// event loop
		go func() {
			for {
				select {
				case ev := <-watcher.Event:
					if eventRegex.MatchString(ev.String()) {
						log.Info("Matched event:", ev.String())
						close(kill)
						return
					} else {
						log.Debug("Ignored event:", ev.String())
					}
				case err = <-watcher.Error:
					log.Fatal("Watcher error:", err)(5)
				case <-quit:
					close(kill)
					return
				}
			}
		}()

	RunLoop:
		for _, runnable := range chain {
			done, dead := runnable(kill)
			wg.Add(1)

			go func() {
				<-dead
				wg.Done()
			}()

			select {
			case d := <-done:
				if !d {
					// Runnable's success metric failed, break out of the chain
					break RunLoop
				}
			case <-kill:
				break RunLoop
			}
		}

		// ensure an event has occured, we may be here because all runnables completed
		<-kill

		// ensure all runnables (procs) are dead before restarting the chain
		wg.Wait()

		// check if we should quit
		select {
		case <-quit:
			log.Debug("Quitting main event/action loop")
			return
		default:
		}
	}
}

func newWatcher() *Watcher {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		panic(err)
	}

	ignore, err := regexp.Compile(*ignoreRegex)
	if err != nil {
		log.Fatal("Ignore regex compile error:", err)(1)
	}

	watchDir := func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			return nil
		}

		if err != nil {
			log.Err("Error reading dir, skipping:", path, err)
			return filepath.SkipDir
		}

		if ignore.MatchString(path) {
			log.Debug("Ignoring dir:", path)
			return filepath.SkipDir
		}

		log.Debug("Watching dir:", path)
		err = watcher.Watch(path)
		if err != nil {
			log.Err("Error watching dir (path, error):", path, err)
		}

		return nil
	}

	if *recursive == true {
		// errors are handled in watchDir
		filepath.Walk(*targetDir, watchDir)
	} else {
		err = watcher.Watch(*targetDir)
		if err != nil {
			log.Fatal("Error watching dir (path, error):", *targetDir, err)(1)
		}
	}

	// To facilitate testing (which sends artifical events from a timer),
	// we have an abstracted struct Watcher that holds the applicable channels.
	// fsnotify.FileEvent is a fmt.Stringer, but channels cannot be converted.
	// Unfortunately, an extra channel is necessary to perform the conversion.
	event := make(chan fmt.Stringer)
	go func() {
		for {
			event <- <-watcher.Event
		}
	}()

	return &Watcher{event, watcher.Error}
}

func startWebServer() {
	if *webServer != "" {
		go func() {
			log.Info("Starting web server on port", *webServer)
			err := http.ListenAndServe(*webServer, http.FileServer(http.Dir(*targetDir)))
			if err != nil {
				log.Fatal("Error starting web server:", err)(2)
			}
		}()
	}
}

func configSetup() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.Usage = func() {
		fmt.Println("WaGo (Watch, Go) build tool. Version", VERSION)
		flag.PrintDefaults()
	}

	// TODO: this should check for actions
	if len(os.Args) < 2 {
		flag.Usage()
		log.Fatal("You must specify an action")(1)
	}

	flag.Parse()

	if *verbose {
		log = dog.NewDog(dog.DEBUG)
	} else if *quiet {
		log = dog.NewDog(dog.WARN)
	} else {
		log = dog.NewDog(dog.INFO)
	}

	if len(*shell) == 0 {
		*shell = os.Getenv("SHELL")
		if len(*shell) == 0 {
			*shell = "/bin/sh"
		}
	}
	log.Debug("Using shell", *shell)

	if (len(*daemonTrigger) > 0) && (*daemonTimer > 0) {
		log.Fatal("Both daemon trigger and timer specified, use only one")(1)
	}

	if (len(*daemonTrigger) > 0 || *daemonTimer > 0) && len(*daemonCmd) == 0 {
		log.Fatal("Specify a daemon command to use the trigger or timer")(1)
	}

	if len(*buildCmd) == 0 && len(*daemonCmd) == 0 && !*fiddle && len(*postCmd) == 0 && len(*url) == 0 && len(*webServer) == 0 {
		flag.Usage()
		log.Fatal("You must specify an action")(1)
	}

	if *fiddle {
		if *webServer == "" {
			*webServer = ":9933"
		}
		if *url == "" {
			*url = "http://localhost" + *webServer + "/"
		}
	}

	if *targetDir == "" {
		cwd, err := os.Getwd()
		if err != nil {
			panic(err)
		}
		targetDir = &cwd
	}
	log.Debug("Target dir:", *targetDir)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/michenriksen/aquatone/agents"
	"github.com/michenriksen/aquatone/core"
	"github.com/michenriksen/aquatone/parsers"
)

var (
	sess *core.Session
	err  error
)

func isURL(s string) bool {
	u, err := url.ParseRequestURI(s)
	if err != nil {
		return false
	}
	if u.Scheme == "" {
		return false
	}
	return true
}

func hasSupportedScheme(s string) bool {
	u, err := url.ParseRequestURI(s)
	if err != nil {
		return false
	}
	if u.Scheme == "http" || u.Scheme == "https" {
		return true
	}
	return false
}

func main() {
	if sess, err = core.NewSession(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	fi, err := os.Stat(*sess.Options.OutDir)

	if os.IsNotExist(err) {
		sess.Out.Fatal("Output destination %s does not exist\n", *sess.Options.OutDir)
		os.Exit(1)
	}

	if !fi.IsDir() {
		sess.Out.Fatal("Output destination must be a directory\n")
		os.Exit(1)
	}

	sess.Out.Important("%s v%s started at %s\n\n", core.Name, core.Version, sess.Stats.StartedAt.Format(time.RFC3339))

	agents.NewTCPPortScanner().Register(sess)
	agents.NewURLPublisher().Register(sess)
	agents.NewURLRequester().Register(sess)
	agents.NewURLLogger().Register(sess)
	agents.NewURLScreenshotter().Register(sess)
	agents.NewURLTechnologyFingerprinter().Register(sess)
	agents.NewURLTakeoverDetector().Register(sess)

	reader := bufio.NewReader(os.Stdin)
	var targets []string

	if *sess.Options.Nmap {
		parser := parsers.NewNmapParser()
		targets, err = parser.Parse(reader)
		if err != nil {
			sess.Out.Fatal("Unable to parse input as Nmap/Masscan XML: %s\n", err)
			os.Exit(1)
		}
	} else {
		parser := parsers.NewRegexParser()
		targets, err = parser.Parse(reader)
		if err != nil {
			sess.Out.Fatal("Unable to parse input.\n")
			os.Exit(1)
		}
	}

	if len(targets) == 0 {
		sess.Out.Fatal("No targets found in input.\n")
		os.Exit(1)
	}

	sess.Out.Important("Targets    : %d\n", len(targets))
	sess.Out.Important("Threads    : %d\n", *sess.Options.Threads)
	sess.Out.Important("Ports      : %s\n", strings.Trim(strings.Replace(fmt.Sprint(sess.Ports), " ", ", ", -1), "[]"))
	sess.Out.Important("Output dir : %s\n\n", *sess.Options.OutDir)

	for _, target := range targets {
		if isURL(target) {
			if hasSupportedScheme(target) {
				sess.EventBus.Publish(core.URL, target)
			}
		} else {
			sess.EventBus.Publish(core.Host, target)
		}
	}

	sess.EventBus.WaitAsync()
	sess.WaitGroup.Wait()

	sess.Out.Important("\nClustering similar sites...")
	pageStructures := make(map[string][]string)
	var pageClusters [][]*core.ResponsiveURL

	f, _ := os.OpenFile(sess.GetFilePath("aquatone_urls.txt"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)

	for _, responsiveURL := range sess.ResponsiveURLs {
		filename := sess.GetFilePath(fmt.Sprintf("html/%s.html", agents.BaseFilenameFromURL(responsiveURL.URL)))
		body, err := os.Open(filename)
		if err != nil {
			continue
		}
		structure, _ := core.GetPageStructure(body)
		pageStructures[responsiveURL.URL] = structure
		f.WriteString(responsiveURL.URL + "\n")
	}
	f.Close()

	// Loop over URL and page structure pairs
	for url, structure := range pageStructures {
		foundCluster := false
		// Loop over existing page clusters
		for i, cluster := range pageClusters {
			addToCluster := true
			// Loop over pages in cluster and check if similarity for all are 0.80 or above
			for _, url2 := range cluster {
				if core.GetSimilarity(structure, pageStructures[url2.URL]) < 0.80 {
					addToCluster = false
				}
			}
			// Add to cluster if similarity between all pages are 0.80 or above
			if addToCluster {
				foundCluster = true
				pageClusters[i] = append(pageClusters[i], sess.ResponsiveURLs[url])
				break
			}
		}
		// If a cluster was not found for the page, create a new cluster for the page
		if !foundCluster {
			pageClusters = append(pageClusters, []*core.ResponsiveURL{sess.ResponsiveURLs[url]})
		}
	}

	sess.Out.Important(" done\n")
	sess.Out.Important("Generating HTML report...")

	reportData := core.ReportData{
		Session: sess,
	}

	for _, urls := range pageClusters {
		cluster, err := core.NewCluster(urls, sess)
		if err != nil {
			sess.Out.Fatal("Error during report generation: %s\n", err)
			os.Exit(1)
		}
		reportData.Clusters = append(reportData.Clusters, cluster)
	}

	report := core.NewReport(reportData)
	f, err = os.OpenFile(sess.GetFilePath("aquatone_report.html"), os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		sess.Out.Fatal("Error during report generation: %s\n", err)
		os.Exit(1)
	}
	err = report.Render(f)
	if err != nil {
		sess.Out.Fatal("Error during report generation: %s\n", err)
		os.Exit(1)
	}

	sess.Out.Important(" done\n\n")

	sess.End()

	sess.Out.Important("Time:\n")
	sess.Out.Info(" - Started at  : %v\n", sess.Stats.StartedAt.Format(time.RFC3339))
	sess.Out.Info(" - Finished at : %v\n", sess.Stats.FinishedAt.Format(time.RFC3339))
	sess.Out.Info(" - Duration    : %v\n\n", sess.Stats.Duration().Round(time.Second))

	sess.Out.Important("Requests:\n")
	sess.Out.Info(" - Successful : %v\n", sess.Stats.RequestSuccessful)
	sess.Out.Info(" - Failed     : %v\n\n", sess.Stats.RequestFailed)

	sess.Out.Info(" - 2xx : %v\n", sess.Stats.ResponseCode2xx)
	sess.Out.Info(" - 3xx : %v\n", sess.Stats.ResponseCode3xx)
	sess.Out.Info(" - 4xx : %v\n", sess.Stats.ResponseCode4xx)
	sess.Out.Info(" - 5xx : %v\n\n", sess.Stats.ResponseCode5xx)

	sess.Out.Important("Screenshots:\n")
	sess.Out.Info(" - Successful : %v\n", sess.Stats.ScreenshotSuccessful)
	sess.Out.Info(" - Failed     : %v\n\n", sess.Stats.ScreenshotFailed)

	sess.Out.Important("Wrote HTML report to: %s\n\n", sess.GetFilePath("aquatone_report.html"))
}
<commit_msg>add a delay before report writing to give agents time to publish new events<commit_after>package main

import (
	"bufio"
	"fmt"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/michenriksen/aquatone/agents"
	"github.com/michenriksen/aquatone/core"
	"github.com/michenriksen/aquatone/parsers"
)

var (
	sess *core.Session
	err  error
)

func isURL(s string) bool {
	u, err := url.ParseRequestURI(s)
	if err != nil {
		return false
	}
	if u.Scheme == "" {
		return false
	}
	return true
}

func hasSupportedScheme(s string) bool {
	u, err := url.ParseRequestURI(s)
	if err != nil {
		return false
	}
	if u.Scheme == "http" || u.Scheme == "https" {
		return true
	}
	return false
}

func main() {
	if sess, err = core.NewSession(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	fi, err := os.Stat(*sess.Options.OutDir)

	if os.IsNotExist(err) {
		sess.Out.Fatal("Output destination %s does not exist\n", *sess.Options.OutDir)
		os.Exit(1)
	}

	if !fi.IsDir() {
		sess.Out.Fatal("Output destination must be a directory\n")
		os.Exit(1)
	}

	sess.Out.Important("%s v%s started at %s\n\n", core.Name, core.Version, sess.Stats.StartedAt.Format(time.RFC3339))

	agents.NewTCPPortScanner().Register(sess)
	agents.NewURLPublisher().Register(sess)
	agents.NewURLRequester().Register(sess)
	agents.NewURLLogger().Register(sess)
	agents.NewURLScreenshotter().Register(sess)
	agents.NewURLTechnologyFingerprinter().Register(sess)
	agents.NewURLTakeoverDetector().Register(sess)

	reader := bufio.NewReader(os.Stdin)
	var targets []string

	if *sess.Options.Nmap {
		parser := parsers.NewNmapParser()
		targets, err = parser.Parse(reader)
		if err != nil {
			sess.Out.Fatal("Unable to parse input as Nmap/Masscan XML: %s\n", err)
			os.Exit(1)
		}
	} else {
		parser := parsers.NewRegexParser()
		targets, err = parser.Parse(reader)
		if err != nil {
			sess.Out.Fatal("Unable to parse input.\n")
			os.Exit(1)
		}
	}

	if len(targets) == 0 {
		sess.Out.Fatal("No targets found in input.\n")
		os.Exit(1)
	}

	sess.Out.Important("Targets    : %d\n", len(targets))
	sess.Out.Important("Threads    : %d\n", *sess.Options.Threads)
	sess.Out.Important("Ports      : %s\n", strings.Trim(strings.Replace(fmt.Sprint(sess.Ports), " ", ", ", -1), "[]"))
	sess.Out.Important("Output dir : %s\n\n", *sess.Options.OutDir)

	for _, target := range targets {
		if isURL(target) {
			if hasSupportedScheme(target) {
				sess.EventBus.Publish(core.URL, target)
			}
		} else {
			sess.EventBus.Publish(core.Host, target)
		}
	}

	time.Sleep(1 * time.Second)
	sess.EventBus.WaitAsync()
	sess.WaitGroup.Wait()

	sess.Out.Important("\nClustering similar sites...")
	pageStructures := make(map[string][]string)
	var pageClusters [][]*core.ResponsiveURL

	f, _ := os.OpenFile(sess.GetFilePath("aquatone_urls.txt"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)

	for _, responsiveURL := range sess.ResponsiveURLs {
		filename := sess.GetFilePath(fmt.Sprintf("html/%s.html", agents.BaseFilenameFromURL(responsiveURL.URL)))
		body, err := os.Open(filename)
		if err != nil {
			continue
		}
		structure, _ := core.GetPageStructure(body)
		pageStructures[responsiveURL.URL] = structure
		f.WriteString(responsiveURL.URL + "\n")
	}
	f.Close()

	// Loop over URL and page structure pairs
	for url, structure := range pageStructures {
		foundCluster := false
		// Loop over existing page clusters
		for i, cluster := range pageClusters {
			addToCluster := true
			// Loop over pages in cluster and check if similarity for all are 0.80 or above
			for _, url2 := range cluster {
				if core.GetSimilarity(structure, pageStructures[url2.URL]) < 0.80 {
					addToCluster = false
				}
			}
			// Add to cluster if similarity between all pages are 0.80 or above
			if addToCluster {
				foundCluster = true
				pageClusters[i] = append(pageClusters[i], sess.ResponsiveURLs[url])
				break
			}
		}
		// If a cluster was not found for the page, create a new cluster for the page
		if !foundCluster {
			pageClusters = append(pageClusters, []*core.ResponsiveURL{sess.ResponsiveURLs[url]})
		}
	}

	sess.Out.Important(" done\n")
	sess.Out.Important("Generating HTML report...")

	reportData := core.ReportData{
		Session: sess,
	}

	for _, urls := range pageClusters {
		cluster, err := core.NewCluster(urls, sess)
		if err != nil {
			sess.Out.Fatal("Error during report generation: %s\n", err)
			os.Exit(1)
		}
		reportData.Clusters = append(reportData.Clusters, cluster)
	}

	report := core.NewReport(reportData)
	f, err = os.OpenFile(sess.GetFilePath("aquatone_report.html"), os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		sess.Out.Fatal("Error during report generation: %s\n", err)
		os.Exit(1)
	}
	err = report.Render(f)
	if err != nil {
		sess.Out.Fatal("Error during report generation: %s\n", err)
		os.Exit(1)
	}

	sess.Out.Important(" done\n\n")

	sess.End()

	sess.Out.Important("Time:\n")
	sess.Out.Info(" - Started at  : %v\n", sess.Stats.StartedAt.Format(time.RFC3339))
	sess.Out.Info(" - Finished at : %v\n", sess.Stats.FinishedAt.Format(time.RFC3339))
	sess.Out.Info(" - Duration    : %v\n\n", sess.Stats.Duration().Round(time.Second))

	sess.Out.Important("Requests:\n")
	sess.Out.Info(" - Successful : %v\n", sess.Stats.RequestSuccessful)
	sess.Out.Info(" - Failed     : %v\n\n", sess.Stats.RequestFailed)

	sess.Out.Info(" - 2xx : %v\n", sess.Stats.ResponseCode2xx)
	sess.Out.Info(" - 3xx : %v\n", sess.Stats.ResponseCode3xx)
	sess.Out.Info(" - 4xx : %v\n", sess.Stats.ResponseCode4xx)
	sess.Out.Info(" - 5xx : %v\n\n", sess.Stats.ResponseCode5xx)

	sess.Out.Important("Screenshots:\n")
	sess.Out.Info(" - Successful : %v\n", sess.Stats.ScreenshotSuccessful)
	sess.Out.Info(" - Failed     : %v\n\n", sess.Stats.ScreenshotFailed)

	sess.Out.Important("Wrote HTML report to: %s\n\n", sess.GetFilePath("aquatone_report.html"))
}
<|endoftext|>