<commit_before>package adapter

import (
	"context"
	"encoding/hex"
	"flag"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/gorilla/securecookie"
	"github.com/lestrrat/go-server-starter/listener"
	"github.com/sirupsen/logrus"
)

// Version is the version of go-nginx-oauth2-adapter.
const Version = "0.3.0-rc2"

// Main starts the go-nginx-oauth2-adapter server.
func Main(args []string) int {

	flagSet := flag.NewFlagSet(args[0], flag.ContinueOnError)
	var configFile string
	var configtest bool
	var showVersion bool
	var showHelp bool
	var genKey bool
	flagSet.StringVar(&configFile, "c", "", "configuration file")
	flagSet.StringVar(&configFile, "config", "", "configuration file")
	flagSet.BoolVar(&configtest, "t", false, "test configuration and exit")
	flagSet.BoolVar(&configtest, "configtest", false, "test configuration and exit")
	flagSet.BoolVar(&showVersion, "v", false, "show version information")
	flagSet.BoolVar(&showVersion, "version", false, "show version information")
	flagSet.BoolVar(&showHelp, "h", false, "show help")
	flagSet.BoolVar(&showHelp, "help", false, "show help")
	flagSet.BoolVar(&genKey, "g", false, "shorthand of genkey")
	flagSet.BoolVar(&genKey, "genkey", false, "generate random key for cookie")
	err := flagSet.Parse(args[1:])
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"err": err.Error(),
		}).Error("error while parsing flags")
		return 2
	}

	if showVersion {
		fmt.Println("go-nginx-oauth2-adapter", Version)
		return 0
	}

	if showHelp {
		flagSet.Usage()
		return 0
	}

	if genKey {
		fmt.Println(hex.EncodeToString(securecookie.GenerateRandomKey(64)))
		fmt.Println(hex.EncodeToString(securecookie.GenerateRandomKey(32)))
		return 0
	}

	c, err := parseConfig(configFile)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"err": err.Error(),
		}).Error("error while parsing configure")
		return 1
	}
	c.ConfigTest = configtest

	s, err := NewServer(*c)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"err": err.Error(),
		}).Error("init server error")
		return 1
	}
	if configtest {
		return 0
	}

	l, err := getListener(c)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"err": err.Error(),
		}).Error("listen error")
		return 1
	}
	server := &http.Server{
		Handler: LoggingHandler(s),
	}
	go func() {
		err := server.Serve(l)
		if err == http.ErrServerClosed {
			return
		}
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"err": err.Error(),
			}).Fatal("serve error")
		}
	}()

	waitSignal()
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	err = server.Shutdown(ctx)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"err": err.Error(),
		}).Error("shutdown server error")
	}
	return 0
}

func parseConfig(configFile string) (*Config, error) {
	c := NewConfig()
	if err := c.LoadEnv(); err != nil {
		return nil, err
	}
	if configFile != "" {
		if err := c.LoadYaml(configFile); err != nil {
			return nil, err
		}
	}

	return c, nil
}

func waitSignal() {
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGTERM)
	for {
		s := <-signalChan
		if s == syscall.SIGTERM {
			logrus.Info("received SIGTERM")
			return
		}
	}
}

func getListener(c *Config) (net.Listener, error) {
	listeners, err := listener.ListenAll()
	if err != nil && err != listener.ErrNoListeningTarget {
		panic(err)
	}
	if err != listener.ErrNoListeningTarget {
		return listeners[0], nil
	}

	// Fallback if not running under Server::Starter
	return net.Listen("tcp", c.Address)
}
<commit_msg>bump version<commit_after>package adapter

import (
	"context"
	"encoding/hex"
	"flag"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/gorilla/securecookie"
	"github.com/lestrrat/go-server-starter/listener"
	"github.com/sirupsen/logrus"
)

// Version is the version of go-nginx-oauth2-adapter.
const Version = "0.3.0"

// Main starts the go-nginx-oauth2-adapter server.
func Main(args []string) int {

	flagSet := flag.NewFlagSet(args[0], flag.ContinueOnError)
	var configFile string
	var configtest bool
	var showVersion bool
	var showHelp bool
	var genKey bool
	flagSet.StringVar(&configFile, "c", "", "configuration file")
	flagSet.StringVar(&configFile, "config", "", "configuration file")
	flagSet.BoolVar(&configtest, "t", false, "test configuration and exit")
	flagSet.BoolVar(&configtest, "configtest", false, "test configuration and exit")
	flagSet.BoolVar(&showVersion, "v", false, "show version information")
	flagSet.BoolVar(&showVersion, "version", false, "show version information")
	flagSet.BoolVar(&showHelp, "h", false, "show help")
	flagSet.BoolVar(&showHelp, "help", false, "show help")
	flagSet.BoolVar(&genKey, "g", false, "shorthand of genkey")
	flagSet.BoolVar(&genKey, "genkey", false, "generate random key for cookie")
	err := flagSet.Parse(args[1:])
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"err": err.Error(),
		}).Error("error while parsing flags")
		return 2
	}

	if showVersion {
		fmt.Println("go-nginx-oauth2-adapter", Version)
		return 0
	}

	if showHelp {
		flagSet.Usage()
		return 0
	}

	if genKey {
		fmt.Println(hex.EncodeToString(securecookie.GenerateRandomKey(64)))
		fmt.Println(hex.EncodeToString(securecookie.GenerateRandomKey(32)))
		return 0
	}

	c, err := parseConfig(configFile)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"err": err.Error(),
		}).Error("error while parsing configure")
		return 1
	}
	c.ConfigTest = configtest

	s, err := NewServer(*c)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"err": err.Error(),
		}).Error("init server error")
		return 1
	}
	if configtest {
		return 0
	}

	l, err := getListener(c)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"err": err.Error(),
		}).Error("listen error")
		return 1
	}
	server := &http.Server{
		Handler: LoggingHandler(s),
	}
	go func() {
		err := server.Serve(l)
		if err == http.ErrServerClosed {
			return
		}
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"err": err.Error(),
			}).Fatal("serve error")
		}
	}()

	waitSignal()
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	err = server.Shutdown(ctx)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"err": err.Error(),
		}).Error("shutdown server error")
	}
	return 0
}

func parseConfig(configFile string) (*Config, error) {
	c := NewConfig()
	if err := c.LoadEnv(); err != nil {
		return nil, err
	}
	if configFile != "" {
		if err := c.LoadYaml(configFile); err != nil {
			return nil, err
		}
	}

	return c, nil
}

func waitSignal() {
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGTERM)
	for {
		s := <-signalChan
		if s == syscall.SIGTERM {
			logrus.Info("received SIGTERM")
			return
		}
	}
}

func getListener(c *Config) (net.Listener, error) {
	listeners, err := listener.ListenAll()
	if err != nil && err != listener.ErrNoListeningTarget {
		panic(err)
	}
	if err != listener.ErrNoListeningTarget {
		return listeners[0], nil
	}

	// Fallback if not running under Server::Starter
	return net.Listen("tcp", c.Address)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"time"
	"reflect"
	"encoding/json"

	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/pkg/api"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/pkg/apis/extensions/v1beta1"
	"strings"
	"k8s.io/client-go/rest"
)

type LogMessage struct {
	ObjectType string
	ObjectName string
	EventType string
	AssignedNode string
	Replicas int32
	Timestamp time.Time `json:"time"`
}

func main() {
	clientset := createClientSet()

	createWatcher(clientset.CoreV1().RESTClient(), &v1.Pod{}, "pods")
	createWatcher(clientset.CoreV1().RESTClient(), &v1.Service{}, "services")
	createWatcher(clientset.CoreV1().RESTClient(), &v1.Secret{}, "secrets")
	createWatcher(clientset.CoreV1().RESTClient(), &v1.ConfigMap{}, "configmaps")
	createWatcher(clientset.CoreV1().RESTClient(), &v1.Namespace{}, "namespaces")
	createWatcher(clientset.CoreV1().RESTClient(), &v1.ReplicationController{}, "replicationcontrollers")
	createWatcher(clientset.ExtensionsV1beta1().RESTClient(), &v1beta1.ReplicaSet{}, "replicasets")
	createWatcher(clientset.ExtensionsV1beta1().RESTClient(), &v1beta1.Ingress{}, "ingresses")
	createWatcher(clientset.ExtensionsV1beta1().RESTClient(), &v1beta1.Deployment{}, "deployments")

	for{
		time.Sleep(time.Second)
	}
}

func createWatcher(c cache.Getter, obj runtime.Object, resource string) cache.Controller {
	watchlist := cache.NewListWatchFromClient(c, resource, api.NamespaceAll, fields.Everything())
	resyncPeriod := 30 * time.Minute

	_, controller := cache.NewInformer(
		watchlist,
		obj,
		resyncPeriod,
		cache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				toJson(obj, "Created")
			},
			DeleteFunc: func(obj interface{}) {
				toJson(obj, "Deleted")
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
				toJson(newObj, "Updated")
			},
		},
	)

	go controller.Run(wait.NeverStop)
	return controller
}

func toJson(obj interface{}, eventType string) string {
	objType := reflect.TypeOf(obj)

	logMessage := &LogMessage{ObjectType: strings.TrimLeft(objType.String(), "*"), ObjectName: getName(obj), EventType: eventType, Timestamp: time.Now().UTC()}

	//Add additional information to log message
	switch t := obj.(type) {
	case *v1.Pod:
		logMessage.AssignedNode = t.Spec.NodeName
	case *v1beta1.Deployment:
		logMessage.Replicas = *t.Spec.Replicas
	case *v1beta1.ReplicaSet:
		logMessage.Replicas = *t.Spec.Replicas
	}

	b, err := json.Marshal(logMessage)
	if err != nil {
		fmt.Println(err)
		return ""
	}

	jsonString := string(b)
	fmt.Println(jsonString)
	return string(jsonString)
}

func getName(obj interface{}) string {
	switch t := obj.(type) {
	default:
		return "Unknown Name"
	case *v1.Namespace:
		return t.ObjectMeta.Name
	case *v1.Pod:
		return t.ObjectMeta.Name
	case *v1beta1.Deployment:
		return t.ObjectMeta.Name
	case *v1beta1.DaemonSet:
		return t.ObjectMeta.Name
	case *v1beta1.ReplicaSet:
		return t.ObjectMeta.Name
	case *v1.Secret:
		return t.ObjectMeta.Name
	case *v1.ConfigMap:
		return t.ObjectMeta.Name
	case *v1beta1.Ingress:
		return t.ObjectMeta.Name
	case *v1.Service:
		return t.ObjectMeta.Name
	}

	return "Unknown Name"
}

func createClientSet() *kubernetes.Clientset {
	var kubeconfig *string
	if home := homeDir(); home != "" {
		kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file")
	} else {
		kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
	}
	flag.Parse()

	//Default to using kubeconfig or commandline arg
	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
	if err != nil {
		// kubeconfig failed attempt in cluster config
		// creates the in-cluster config
		fmt.Println("Creating in cluster configuration...")
		config, err = rest.InClusterConfig()
		if err != nil {
			panic(err.Error())
		}
	}

	return buildClientSet(config)
}

func buildClientSet(config *rest.Config) *kubernetes.Clientset {
	// create the clientset
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		panic(err.Error())
	}

	return clientset
}

func homeDir() string {
	if h := os.Getenv("HOME"); h != "" {
		return h
	}
	return os.Getenv("USERPROFILE") // windows
}<commit_msg>Log some information from the old object on update<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"time"
	"reflect"
	"encoding/json"

	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/pkg/api"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/pkg/apis/extensions/v1beta1"
	"strings"
	"k8s.io/client-go/rest"
)

type LogMessage struct {
	ObjectType string
	ObjectName string
	EventType string
	AssignedNode string
	OldAssignedNode string
	Replicas int32
	OldReplicas int32
	Timestamp time.Time `json:"time"`
}

func main() {
	clientset := createClientSet()

	createWatcher(clientset.CoreV1().RESTClient(), &v1.Pod{}, "pods")
	createWatcher(clientset.CoreV1().RESTClient(), &v1.Service{}, "services")
	createWatcher(clientset.CoreV1().RESTClient(), &v1.Secret{}, "secrets")
	createWatcher(clientset.CoreV1().RESTClient(), &v1.ConfigMap{}, "configmaps")
	createWatcher(clientset.CoreV1().RESTClient(), &v1.Namespace{}, "namespaces")
	createWatcher(clientset.CoreV1().RESTClient(), &v1.ReplicationController{}, "replicationcontrollers")
	createWatcher(clientset.ExtensionsV1beta1().RESTClient(), &v1beta1.ReplicaSet{}, "replicasets")
	createWatcher(clientset.ExtensionsV1beta1().RESTClient(), &v1beta1.Ingress{}, "ingresses")
	createWatcher(clientset.ExtensionsV1beta1().RESTClient(), &v1beta1.Deployment{}, "deployments")

	for{
		time.Sleep(time.Second)
	}
}

func createWatcher(c cache.Getter, obj runtime.Object, resource string) cache.Controller {
	watchlist := cache.NewListWatchFromClient(c, resource, api.NamespaceAll, fields.Everything())
	resyncPeriod := 30 * time.Minute

	_, controller := cache.NewInformer(
		watchlist,
		obj,
		resyncPeriod,
		cache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				toJson(nil, obj, "Created")
			},
			DeleteFunc: func(obj interface{}) {
				toJson(nil, obj, "Deleted")
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
				toJson(oldObj, newObj, "Updated")
			},
		},
	)

	go controller.Run(wait.NeverStop)
	return controller
}

func toJson(oldObj interface{}, obj interface{}, eventType string) string {
	objType := reflect.TypeOf(obj)

	logMessage := &LogMessage{ObjectType: strings.TrimLeft(objType.String(), "*"), ObjectName: getName(obj), EventType: eventType, Timestamp: time.Now().UTC()}

	//Add additional information to log message
	switch t := obj.(type) {
	case *v1.Pod:
		logMessage.AssignedNode = t.Spec.NodeName
	case *v1beta1.Deployment:
		logMessage.Replicas = *t.Spec.Replicas
	case *v1beta1.ReplicaSet:
		logMessage.Replicas = *t.Spec.Replicas
	}

	//If this is an update add some information related to the old object
	if oldObj != nil {
		switch t := oldObj.(type) {
		case *v1.Pod:
			logMessage.OldAssignedNode = t.Spec.NodeName
		case *v1beta1.Deployment:
			logMessage.OldReplicas = *t.Spec.Replicas
		case *v1beta1.ReplicaSet:
			logMessage.OldReplicas = *t.Spec.Replicas
		}
	}

	b, err := json.Marshal(logMessage)
	if err != nil {
		fmt.Println(err)
		return ""
	}

	jsonString := string(b)
	fmt.Println(jsonString)
	return string(jsonString)
}

func getName(obj interface{}) string {
	switch t := obj.(type) {
	default:
		return "Unknown Name"
	case *v1.Namespace:
		return t.ObjectMeta.Name
	case *v1.Pod:
		return t.ObjectMeta.Name
	case *v1beta1.Deployment:
		return t.ObjectMeta.Name
	case *v1beta1.DaemonSet:
		return t.ObjectMeta.Name
	case *v1beta1.ReplicaSet:
		return t.ObjectMeta.Name
	case *v1.Secret:
		return t.ObjectMeta.Name
	case *v1.ConfigMap:
		return t.ObjectMeta.Name
	case *v1beta1.Ingress:
		return t.ObjectMeta.Name
	case *v1.Service:
		return t.ObjectMeta.Name
	}

	return "Unknown Name"
}

func createClientSet() *kubernetes.Clientset {
	var kubeconfig *string
	if home := homeDir(); home != "" {
		kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file")
	} else {
		kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
	}
	flag.Parse()

	//Default to using kubeconfig or commandline arg
	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
	if err != nil {
		// kubeconfig failed attempt in cluster config
		// creates the in-cluster config
		fmt.Println("Creating in cluster configuration...")
		config, err = rest.InClusterConfig()
		if err != nil {
			panic(err.Error())
		}
	}

	return buildClientSet(config)
}

func buildClientSet(config *rest.Config) *kubernetes.Clientset {
	// create the clientset
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		panic(err.Error())
	}

	return clientset
}

func homeDir() string {
	if h := os.Getenv("HOME"); h != "" {
		return h
	}
	return os.Getenv("USERPROFILE") // windows
}<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/icza/gowut/gwu"

	"github.com/udhos/jazigo/conf"
	"github.com/udhos/jazigo/dev"
	"github.com/udhos/jazigo/store"
)

const appName = "jazigo"
const appVersion = "0.0"

type app struct {
	configPathPrefix string
	repositoryPath   string

	table *dev.DeviceTable

	cfg *conf.Config

	apHome    gwu.Panel
	apAdmin   gwu.Panel
	apLogout  gwu.Panel
	winHome   gwu.Window
	winAdmin  gwu.Window
	winLogout gwu.Window

	cssPath string

	logger hasPrintf

	priority chan string
}

type hasPrintf interface {
	Printf(fmt string, v ...interface{})
}

func (a *app) logf(fmt string, v ...interface{}) {
	a.logger.Printf(fmt, v...)
}

func newApp(logger hasPrintf) *app {
	app := &app{
		table:    dev.NewDeviceTable(),
		logger:   logger,
		priority: make(chan string),
	}

	app.logf("%s %s starting", appName, appVersion)

	dev.RegisterModels(app.logger, app.table)

	return app
}

func defaultStaticDir() string {
	gopath := os.Getenv("GOPATH")
	pkgPath := filepath.Join("src", "github.com", "udhos", "jazigo") // from package github.com/udhos/jazigo
	return filepath.Join(gopath, pkgPath, "www")
}

func main() {

	logger := log.New(os.Stdout, "", log.LstdFlags)

	jaz := newApp(logger)

	var runOnce bool
	var staticDir string

	flag.StringVar(&jaz.configPathPrefix, "configPathPrefix", "/etc/jazigo/jazigo.conf.", "configuration path prefix")
	flag.StringVar(&jaz.repositoryPath, "repositoryPath", "/var/jazigo", "repository path")
	flag.StringVar(&staticDir, "wwwStaticPath", defaultStaticDir(), "directory for static www content")
	flag.BoolVar(&runOnce, "runOnce", false, "exit after scanning all devices once")
	flag.Parse()
	jaz.logf("config path prefix: %s", jaz.configPathPrefix)
	jaz.logf("repository path: %s", jaz.repositoryPath)
	jaz.logf("runOnce: %v", runOnce)

	lastConfig, configErr := store.FindLastConfig(jaz.configPathPrefix, logger)
	if configErr != nil {
		jaz.logf("error reading config: '%s': %v", jaz.configPathPrefix, configErr)
		jaz.cfg = conf.New()
	} else {
		jaz.logf("last config: %s", lastConfig)
		var loadErr error
		jaz.cfg, loadErr = conf.Load(lastConfig)
		if loadErr != nil {
			jaz.logf("could not load config: '%s': %v", lastConfig, loadErr)
			panic("main: could not load config")
		}
	}

	jaz.logf("scan interval: %s", jaz.cfg.ScanInterval)
	jaz.logf("holdtime: %s", jaz.cfg.Holdtime)

	//dev.CreateDevice(jaz.table, jaz.logger, "cisco-ios", "lab1", "localhost:2001", "telnet", "lab", "pass", "en")
	//dev.CreateDevice(jaz.table, jaz.logger, "cisco-ios", "lab1", "localhost:2001", "telnet", "lab", "pass", "en") // ugh
	//dev.CreateDevice(jaz, jaz.logger, "cisco-ios", "lab2", "localhost:2002", "ssh", "lab", "pass", "en")
	//dev.CreateDevice(jaz, jaz.logger, "cisco-ios", "lab3", "localhost:2003", "telnet,ssh", "lab", "pass", "en")
	//dev.CreateDevice(jaz, jaz.logger, "cisco-ios", "lab4", "localhost:2004", "ssh,telnet", "lab", "pass", "en")
	//dev.CreateDevice(jaz, jaz.logger, "cisco-ios", "lab5", "localhost", "telnet", "lab", "pass", "en")
	//dev.CreateDevice(jaz, jaz.logger, "cisco-ios", "lab6", "localhost", "ssh", "rat", "lab", "en")
	//dev.CreateDevice(jaz.table, jaz.logger, "http", "lab7", "localhost:2009", "telnet", "", "", "")

	dev.CreateDevice(jaz.table, jaz.logger, "linux", "lab8", "localhost", "ssh", "rat", "lab", "lab", false)
	dev.CreateDevice(jaz.table, jaz.logger, "junos", "lab9", "ex4200lab", "ssh", "test", "lab000", "lab", false)
	dev.CreateDevice(jaz.table, jaz.logger, "junos", "lab10", "ex4200lab", "telnet", "test", "lab000", "lab", false)

	//dev.CreateDevice(jaz.table, jaz.logger, "cisco-iosxr", "lab11", "192.168.56.1:2011", "telnet", "user", "pass", "pass", false)
	//dev.CreateDevice(jaz.table, jaz.logger, "cisco-iosxr", "lab12", "192.168.56.1:2012", "ssh", "user", "pass", "cisco8", true)

	dev.UpdateLastSuccess(jaz.table, jaz.logger, jaz.repositoryPath)

	saveConfig(jaz) // FIXME this is not the right place to save the config

	appAddr := "0.0.0.0:8080"
	serverName := fmt.Sprintf("%s application", appName)

	// Create GUI server
	server := gwu.NewServer(appName, appAddr)
	//folder := "./tls/"
	//server := gwu.NewServerTLS(appName, appAddr, folder+"cert.pem", folder+"key.pem")
	server.SetText(serverName)

	staticPath := "static"
	staticPathFull := fmt.Sprintf("/%s/%s", appName, staticPath)
	jaz.logf("static dir: path=[%s] mapped to dir=[%s]", staticPathFull, staticDir)
	jaz.cssPath = fmt.Sprintf("%s/jazigo.css", staticPathFull)
	jaz.logf("css path: %s", jaz.cssPath)

	server.AddStaticDir(staticPath, staticDir)

	// create account panel
	jaz.apHome = newAccPanel("")
	jaz.apAdmin = newAccPanel("")
	jaz.apLogout = newAccPanel("")
	accountPanelUpdate(jaz, "")

	buildHomeWin(jaz, server)
	buildLoginWin(jaz, server)

	if runOnce {
		dev.ScanDevices(jaz.table, jaz.table.ListDevices(), logger, jaz.cfg.MaxConcurrency, 50*time.Millisecond, 500*time.Millisecond, jaz.repositoryPath, jaz.cfg.MaxConfigFiles, jaz.cfg.Holdtime)
		jaz.logf("runOnce: exiting after single scan")
		return
	}

	go func() {
		for {
			begin := time.Now()
			dev.ScanDevices(jaz.table, jaz.table.ListDevices(), logger, jaz.cfg.MaxConcurrency, 50*time.Millisecond, 500*time.Millisecond, jaz.repositoryPath, jaz.cfg.MaxConfigFiles, jaz.cfg.Holdtime)

		SLEEP:
			for {
				elap := time.Since(begin)
				sleep := jaz.cfg.ScanInterval - elap
				if sleep < 1 {
					sleep = 0
				}
				jaz.logf("main: sleeping for %s (target: scanInterval=%s)", sleep, jaz.cfg.ScanInterval)
				select {
				case <-time.After(sleep):
					jaz.logf("main: sleep done")
					break SLEEP
				case id := <-jaz.priority:
					jaz.logf("main: sleep interrupted by priority: device %s", id)
					d, clearErr := dev.ClearDeviceStatus(jaz.table, id, logger, jaz.cfg.Holdtime)
					if clearErr != nil {
						jaz.logf("main: sleep interrupted by priority: device %s - error: %v", id, clearErr)
						continue SLEEP
					}
					singleDevice := []*dev.Device{d}
					dev.ScanDevices(jaz.table, singleDevice, logger, jaz.cfg.MaxConcurrency, 50*time.Millisecond, 500*time.Millisecond, jaz.repositoryPath, jaz.cfg.MaxConfigFiles, jaz.cfg.Holdtime)
				}
			}
		}
	}()

	// Start GUI server
	server.SetLogger(logger)
	if err := server.Start(); err != nil {
		jaz.logf("jazigo main: Cound not start GUI server: %s", err)
		return
	}
}

func saveConfig(jaz *app) {

	devices := jaz.table.ListDevices()
	jaz.cfg.Devices = make([]conf.DevConfig, len(devices))
	for i, d := range devices {
		jaz.cfg.Devices[i] = d.DevConfig
	}

	confWriteFunc := func(w store.HasWrite) error {
		b, err := jaz.cfg.Dump()
		if err != nil {
			return err
		}
		if _, wrErr := w.Write(b); wrErr != nil {
			return wrErr
		}
		return nil
	}

	_, saveErr := store.SaveNewConfig(jaz.configPathPrefix, jaz.cfg.MaxConfigFiles, jaz.logger, confWriteFunc)
	if saveErr != nil {
		jaz.logger.Printf("main: could not save config: %v", saveErr)
		panic("main: could not save config") // FIXME log only
	}
}
<commit_msg>Lock config and repository locations.<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/icza/gowut/gwu"
	"github.com/udhos/lockfile"

	"github.com/udhos/jazigo/conf"
	"github.com/udhos/jazigo/dev"
	"github.com/udhos/jazigo/store"
)

const appName = "jazigo"
const appVersion = "0.0"

type app struct {
	configPathPrefix string
	repositoryPath   string
	configLock       lockfile.Lockfile
	repositoryLock   lockfile.Lockfile

	table *dev.DeviceTable

	cfg *conf.Config

	apHome    gwu.Panel
	apAdmin   gwu.Panel
	apLogout  gwu.Panel
	winHome   gwu.Window
	winAdmin  gwu.Window
	winLogout gwu.Window

	cssPath string

	logger hasPrintf

	priority chan string
}

type hasPrintf interface {
	Printf(fmt string, v ...interface{})
}

func (a *app) logf(fmt string, v ...interface{}) {
	a.logger.Printf(fmt, v...)
}

func newApp(logger hasPrintf) *app {
	app := &app{
		table:    dev.NewDeviceTable(),
		logger:   logger,
		priority: make(chan string),
	}

	app.logf("%s %s starting", appName, appVersion)

	dev.RegisterModels(app.logger, app.table)

	return app
}

func defaultStaticDir() string {
	gopath := os.Getenv("GOPATH")
	pkgPath := filepath.Join("src", "github.com", "udhos", "jazigo") // from package github.com/udhos/jazigo
	return filepath.Join(gopath, pkgPath, "www")
}

func main() {

	logger := log.New(os.Stdout, "", log.LstdFlags)

	jaz := newApp(logger)

	var runOnce bool
	var staticDir string

	flag.StringVar(&jaz.configPathPrefix, "configPathPrefix", "/etc/jazigo/jazigo.conf.", "configuration path prefix")
	flag.StringVar(&jaz.repositoryPath, "repositoryPath", "/var/jazigo", "repository path")
	flag.StringVar(&staticDir, "wwwStaticPath", defaultStaticDir(), "directory for static www content")
	flag.BoolVar(&runOnce, "runOnce", false, "exit after scanning all devices once")
	flag.Parse()

	jaz.logf("runOnce: %v", runOnce)
	jaz.logf("config path prefix: %s", jaz.configPathPrefix)
	jaz.logf("repository path: %s", jaz.repositoryPath)

	if lockErr := exclusiveLock(jaz); lockErr != nil {
		jaz.logf("main: could not get exclusive lock: %v", lockErr)
		panic("main: refusing to run without exclusive lock")
	}
	defer exclusiveUnlock(jaz)

	// load config

	lastConfig, configErr := store.FindLastConfig(jaz.configPathPrefix, logger)
	if configErr != nil {
		jaz.logf("error reading config: '%s': %v", jaz.configPathPrefix, configErr)
		jaz.cfg = conf.New()
	} else {
		jaz.logf("last config: %s", lastConfig)
		var loadErr error
		jaz.cfg, loadErr = conf.Load(lastConfig)
		if loadErr != nil {
			jaz.logf("could not load config: '%s': %v", lastConfig, loadErr)
			panic("main: could not load config")
		}
	}

	jaz.logf("scan interval: %s", jaz.cfg.ScanInterval)
	jaz.logf("holdtime: %s", jaz.cfg.Holdtime)

	//dev.CreateDevice(jaz.table, jaz.logger, "cisco-ios", "lab1", "localhost:2001", "telnet", "lab", "pass", "en")
	//dev.CreateDevice(jaz.table, jaz.logger, "cisco-ios", "lab1", "localhost:2001", "telnet", "lab", "pass", "en") // ugh
	//dev.CreateDevice(jaz, jaz.logger, "cisco-ios", "lab2", "localhost:2002", "ssh", "lab", "pass", "en")
	//dev.CreateDevice(jaz, jaz.logger, "cisco-ios", "lab3", "localhost:2003", "telnet,ssh", "lab", "pass", "en")
	//dev.CreateDevice(jaz, jaz.logger, "cisco-ios", "lab4", "localhost:2004", "ssh,telnet", "lab", "pass", "en")
	//dev.CreateDevice(jaz, jaz.logger, "cisco-ios", "lab5", "localhost", "telnet", "lab", "pass", "en")
	//dev.CreateDevice(jaz, jaz.logger, "cisco-ios", "lab6", "localhost", "ssh", "rat", "lab", "en")
	//dev.CreateDevice(jaz.table, jaz.logger, "http", "lab7", "localhost:2009", "telnet", "", "", "")

	dev.CreateDevice(jaz.table, jaz.logger, "linux", "lab8", "localhost", "ssh", "rat", "lab", "lab", false)
	dev.CreateDevice(jaz.table, jaz.logger, "junos", "lab9", "ex4200lab", "ssh", "test", "lab000", "lab", false)
	dev.CreateDevice(jaz.table, jaz.logger, "junos", "lab10", "ex4200lab", "telnet", "test", "lab000", "lab", false)

	//dev.CreateDevice(jaz.table, jaz.logger, "cisco-iosxr", "lab11", "192.168.56.1:2011", "telnet", "user", "pass", "pass", false)
	//dev.CreateDevice(jaz.table, jaz.logger, "cisco-iosxr", "lab12", "192.168.56.1:2012", "ssh", "user", "pass", "cisco8", true)

	dev.UpdateLastSuccess(jaz.table, jaz.logger, jaz.repositoryPath)

	saveConfig(jaz) // FIXME this is not the right place to save the config

	appAddr := "0.0.0.0:8080"
	serverName := fmt.Sprintf("%s application", appName)

	// Create GUI server
	server := gwu.NewServer(appName, appAddr)
	//folder := "./tls/"
	//server := gwu.NewServerTLS(appName, appAddr, folder+"cert.pem", folder+"key.pem")
	server.SetText(serverName)

	staticPath := "static"
	staticPathFull := fmt.Sprintf("/%s/%s", appName, staticPath)
	jaz.logf("static dir: path=[%s] mapped to dir=[%s]", staticPathFull, staticDir)
	jaz.cssPath = fmt.Sprintf("%s/jazigo.css", staticPathFull)
	jaz.logf("css path: %s", jaz.cssPath)

	server.AddStaticDir(staticPath, staticDir)

	// create account panel
	jaz.apHome = newAccPanel("")
	jaz.apAdmin = newAccPanel("")
	jaz.apLogout = newAccPanel("")
	accountPanelUpdate(jaz, "")

	buildHomeWin(jaz, server)
	buildLoginWin(jaz, server)

	if runOnce {
		dev.ScanDevices(jaz.table, jaz.table.ListDevices(), logger, jaz.cfg.MaxConcurrency, 50*time.Millisecond, 500*time.Millisecond, jaz.repositoryPath, jaz.cfg.MaxConfigFiles, jaz.cfg.Holdtime)
		jaz.logf("runOnce: exiting after single scan")
		return
	}

	go func() {
		for {
			begin := time.Now()
			dev.ScanDevices(jaz.table, jaz.table.ListDevices(), logger, jaz.cfg.MaxConcurrency, 50*time.Millisecond, 500*time.Millisecond, jaz.repositoryPath, jaz.cfg.MaxConfigFiles, jaz.cfg.Holdtime)

		SLEEP:
			for {
				elap := time.Since(begin)
				sleep := jaz.cfg.ScanInterval - elap
				if sleep < 1 {
					sleep = 0
				}
				jaz.logf("main: sleeping for %s (target: scanInterval=%s)", sleep, jaz.cfg.ScanInterval)
				select {
				case <-time.After(sleep):
					jaz.logf("main: sleep done")
					break SLEEP
				case id := <-jaz.priority:
					jaz.logf("main: sleep interrupted by priority: device %s", id)
					d, clearErr := dev.ClearDeviceStatus(jaz.table, id, logger, jaz.cfg.Holdtime)
					if clearErr != nil {
						jaz.logf("main: sleep interrupted by priority: device %s - error: %v", id, clearErr)
						continue SLEEP
					}
					singleDevice := []*dev.Device{d}
					dev.ScanDevices(jaz.table, singleDevice, logger, jaz.cfg.MaxConcurrency, 50*time.Millisecond, 500*time.Millisecond, jaz.repositoryPath, jaz.cfg.MaxConfigFiles, jaz.cfg.Holdtime)
				}
			}
		}
	}()

	// Start GUI server
	server.SetLogger(logger)
	if err := server.Start(); err != nil {
		jaz.logf("jazigo main: Cound not start GUI server: %s", err)
		return
	}
}

func exclusiveLock(jaz *app) error {
	configLockPath := fmt.Sprintf("%slock", jaz.configPathPrefix)
	var newErr error
	if jaz.configLock, newErr = lockfile.New(configLockPath); newErr != nil {
		return fmt.Errorf("exclusiveLock: new failure: '%s': %v", configLockPath, newErr)
	}
	if err := jaz.configLock.TryLock(); err != nil {
		return fmt.Errorf("exclusiveLock: lock failure: '%s': %v", configLockPath, err)
	}

	repositoryLockPath := fmt.Sprintf("%slock", jaz.repositoryPath)
	if jaz.repositoryLock, newErr = lockfile.New(repositoryLockPath); newErr != nil {
		jaz.configLock.Unlock()
		return fmt.Errorf("exclusiveLock: new failure: '%s': %v", repositoryLockPath, newErr)
	}
	if err := jaz.repositoryLock.TryLock(); err != nil {
		jaz.configLock.Unlock()
		return fmt.Errorf("exclusiveLock: lock failure: '%s': %v", repositoryLockPath, err)
	}

	return nil
}

func exclusiveUnlock(jaz *app) {
	configLockPath := fmt.Sprintf("%slock", jaz.configPathPrefix)
	if err := jaz.configLock.Unlock(); err != nil {
		jaz.logger.Printf("exclusiveUnlock: '%s': %v", configLockPath, err)
	}

	repositoryLockPath := fmt.Sprintf("%slock", jaz.repositoryPath)
	if err := jaz.repositoryLock.Unlock(); err != nil {
		jaz.logger.Printf("exclusiveUnlock: '%s': %v", repositoryLockPath, err)
	}
}

func saveConfig(jaz *app) {

	devices := jaz.table.ListDevices()
	jaz.cfg.Devices = make([]conf.DevConfig, len(devices))
	for i, d := range devices {
		jaz.cfg.Devices[i] = d.DevConfig
	}

	confWriteFunc := func(w store.HasWrite) error {
		b, err := jaz.cfg.Dump()
		if err != nil {
			return err
		}
		if _, wrErr := w.Write(b); wrErr != nil {
			return wrErr
		}
		return nil
	}

	_, saveErr := store.SaveNewConfig(jaz.configPathPrefix, jaz.cfg.MaxConfigFiles, jaz.logger, confWriteFunc)
	if saveErr != nil {
		jaz.logger.Printf("main: could not save config: %v", saveErr)
		panic("main: could not save config") // FIXME log only
	}
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	gorillasession "github.com/gorilla/sessions"
	"github.com/ory-am/common/env"
	"github.com/ory-am/common/mgopath"
	"github.com/ory-am/event"
	"github.com/ory-am/gitdeploy/eco"
	"github.com/ory-am/gitdeploy/ip"
	"github.com/ory-am/gitdeploy/job"
	"github.com/ory-am/gitdeploy/job/deploy"
	gdLog "github.com/ory-am/gitdeploy/log"
	"github.com/ory-am/gitdeploy/public"
	"github.com/ory-am/gitdeploy/sse"
	"github.com/ory-am/gitdeploy/storage"
	"github.com/ory-am/gitdeploy/storage/mongo"
	"github.com/ory-am/gitdeploy/task"
	"github.com/ory-am/gitdeploy/task/flynn"
	"github.com/ory-am/google-json-style-response/responder"
	"gopkg.in/mgo.v2"
	"gopkg.in/validator.v2"
	"log"
	"net/http"
	"time"
)

const (
	sessionCurrentDeployment = "cdid"
	sessionName              = "gdp"
)

var (
	ApiVersion      = "1.0"
	host            = env.Getenv("HOST", "")
	port            = env.Getenv("PORT", "7654")
	envAppTtl       = env.Getenv("APP_TTL", "30m")
	envClusterConf  = env.Getenv("FLYNN_CLUSTER_CONFIG", "")
	corsAllowOrigin = env.Getenv("CORS_ALLOW_ORIGIN", "http://localhost:9000")
	sessionStore    = gorillasession.NewCookieStore([]byte(env.Getenv("SESSION_SECRET", "changme")))
	envMongoPath    = env.Getenv("MONGODB", "mongodb://localhost:27017/gitdeploy")
)

type deployRequest struct {
	Repository string `json:"repository",validate:"regex=^(https|http):\\/\\/github\\.com\\/[a-zA-Z0-9\\-\\_\\.]+/[a-zA-Z0-9\\-\\_\\.]+$"`
	Ref        string `json:"ref",validate:"regex=^(origin\\/.*|tags\\/.*|[a-z0-9]+)$"`
}

type appResponse struct {
	App string `json:"app"`
}

func main() {
	eco.IsGitAvailable()
	eco.IsFlynnAvailable()
	init := flag.Bool("init", false, "Initialize flynn")
	flag.Parse()
	if *init {
		eco.InitFlynn(envClusterConf)
		eco.InitGit()
	}

	db, dbName, err := mgopath.Connect(envMongoPath)
	if err != nil {
		log.Fatal(err)
	}
	eventManager := event.New()
	storage := mongo.New(db, dbName)
	sseBroker := sse.New(storage)
	eventManager.AttachListenerAggregate(storage)
	eventManager.AttachListenerAggregate(new(gdLog.Listener))

	// Mux router
	r := mux.NewRouter()
	r.HandleFunc("/config", configHandler).Methods("GET")
	r.HandleFunc("/deployments", deployWrapperAction(sseBroker, eventManager, storage)).Methods("POST")
	r.HandleFunc("/deployments", setCORSHeaders).Methods("OPTIONS")
	r.HandleFunc("/deployments/{app:.+}/events", eventWrapperAction(sseBroker)).Methods("GET")
	r.HandleFunc("/apps/{app:.+}", getAppHandler(storage)).Methods("GET")
	r.PathPrefix("/").HandlerFunc(public.HTML5ModeHandler("./app/dist", "index.html"))
	http.Handle("/", r)

	go job.KillAppsOnHitList(storage)

	listen := fmt.Sprintf("%s:%s", host, port)
	log.Printf("Listening on %s", listen)
	log.Fatal(http.ListenAndServe(listen, nil))
}

func eventWrapperAction(sseBroker *sse.Broker) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		setCORSHeaders(w, r)
		sseBroker.EventHandler(w, r)
	}
}

func deployWrapperAction(sseBroker *sse.Broker, em *event.EventManager, store *mongo.MongoStorage) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		setCORSHeaders(w, r)
		deployAction(w, r, sseBroker, em, store)
	}
}

func configHandler(w http.ResponseWriter, r *http.Request) {
	setCORSHeaders(w, r)
	responseSuccess(w, struct {
		Now time.Time `json:"now"`
	}{time.Now()})
}

func getAppHandler(store *mongo.MongoStorage) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if cleanUpSession(w, r) != nil {
			return
		}
		setCORSHeaders(w, r)
		vars := mux.Vars(r)
		id := vars["app"]
		app, err := store.GetApp(id)
		if err == mgo.ErrNotFound {
			responseError(w, http.StatusNotFound, fmt.Sprintf("App %s does not exist.", id))
			return
		} else if app.Killed {
			responseError(w, http.StatusNotFound, "App timed out and is no longer available.")
			return
		} else if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}

		logs, err := flynn.GetLogs(app.ID)
		if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}

		ps, err := flynn.GetProcs(app.ID)
		if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}

		deployLogs, err := store.FindDeployLogsForApp(id)
		if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}

		responseSuccess(w, struct {
			*storage.App
			Logs       string                 `json:"logs"`
			DeployLogs []*storage.DeployEvent `json:"deployLogs,inline"`
			PS         string                 `json:"ps"`
		}{app, logs, deployLogs, ps})
	}
}

func deployAction(w http.ResponseWriter, r *http.Request, sseBroker *sse.Broker, em *event.EventManager, store *mongo.MongoStorage) {
	app := uuid.NewRandom().String()
	log.Printf("Trying to fetch session: %s", app)
	session, err := sessionStore.Get(r, sessionName)
	if err != nil {
		responseError(w, http.StatusBadRequest, "Please delete your cookies.")
		return
	}

	// Check if the user is currently deploying an application and switch to that one.
	if v, ok := session.Values[sessionCurrentDeployment].(string); ok && len(v) > 0 {
		a, err := store.GetApp(v)
		if err != nil {
			cleanUpSession(w, r)
			log.Printf("Could not fetch app from cookie: %s", err.Error())
		} else if !sseBroker.IsChannelOpen(a.ID) {
			cleanUpSession(w, r)
			log.Printf("Channel %s does not exist any more", a.ID)
		} else {
			responseSuccess(w, a)
			return
		}
	}

	log.Printf("No deployment active for this session, starting new one: %s", app)
	dr := new(deployRequest)
	ttl, err := time.ParseDuration(envAppTtl)
	if err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		return
	} else if err := json.NewDecoder(r.Body).Decode(dr); err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		return
	} else if err := validator.Validate(dr); err != nil {
		responseError(w, http.StatusBadRequest, fmt.Sprintf("Validation failed: %s", err))
		return
	}

	log.Printf("Validation passed: %s", app)
	if dr.Repository[len(dr.Repository)-4:] != ".git" {
		dr.Repository = dr.Repository + ".git"
	}
	if len(dr.Ref) == 0 {
		dr.Ref = "master"
	}

	log.Printf("Storing app information: %s", app)
	appEntity, err := store.AddApp(app, time.Now().Add(ttl), dr.Repository, ip.GetRemoteAddr(r), dr.Ref)
	if err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		return
	}

	session.Values[sessionCurrentDeployment] = appEntity.ID
	if err := session.Save(r, w); err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		return
	}

	log.Printf("Storing app information: %s", app)
	responseSuccess(w, appEntity)
	go func() {
		sseBroker.OpenChannel(app)
		sseBroker.Start(app)
		defer func() {
			log.Printf("Waiting for slow clients: %s", app)
			// Give the client the chance to read the output...
			time.Sleep(5 * time.Second)
			log.Printf("Timeout for slow clients: %s", app)
			sseBroker.CloseChannel(app)
		}()
		log.Printf("Creating job: %s", app)
		tasks := deploy.CreateJob(store, appEntity)
		log.Printf("Run job: %s", app)
		err := task.RunJob(app, em, tasks)
		if err != nil {
			log.Printf("Error in task.RunJob: %s", err)
		}
	}()
}

func cleanUpSession(w http.ResponseWriter, r *http.Request) error {
	session, err := sessionStore.Get(r, sessionName)
	if err != nil {
		responseError(w, http.StatusBadRequest, "Please delete your cookies.")
		return err
	}
	delete(session.Values, sessionCurrentDeployment)
	if err := session.Save(r, w); err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		return err
	}
	return nil
}

// Set the different CORS headers required for CORS request
func setCORSHeaders(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
	w.Header().Set("Access-Control-Allow-Origin", corsAllowOrigin)
	w.Header().Set("Access-Control-Allow-Credentials", "true")
	w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
}

func responseError(w http.ResponseWriter, code int, message string) {
	log.Printf("Error %d: %s", code, message)
	response := responder.New(ApiVersion)
	response.Write(w, response.Error(code, message))
}

func responseSuccess(w http.ResponseWriter, data interface{}) {
	response := responder.New(ApiVersion)
	response.Write(w, response.Success(data))
}
<commit_msg>main: removed duplicate deployment prevention<commit_after>package main

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	gorillasession "github.com/gorilla/sessions"
	"github.com/ory-am/common/env"
	"github.com/ory-am/common/mgopath"
	"github.com/ory-am/event"
	"github.com/ory-am/gitdeploy/eco"
	"github.com/ory-am/gitdeploy/ip"
	"github.com/ory-am/gitdeploy/job"
	"github.com/ory-am/gitdeploy/job/deploy"
	gdLog "github.com/ory-am/gitdeploy/log"
	"github.com/ory-am/gitdeploy/public"
	"github.com/ory-am/gitdeploy/sse"
	"github.com/ory-am/gitdeploy/storage"
	"github.com/ory-am/gitdeploy/storage/mongo"
	"github.com/ory-am/gitdeploy/task"
	"github.com/ory-am/gitdeploy/task/flynn"
	"github.com/ory-am/google-json-style-response/responder"
	"gopkg.in/mgo.v2"
	"gopkg.in/validator.v2"
	"log"
	"net/http"
	"time"
)

const (
	sessionCurrentDeployment = "cdid"
	sessionName              = "gdp"
)

var (
	ApiVersion      = "1.0"
	host            = env.Getenv("HOST", "")
	port            = env.Getenv("PORT", "7654")
	envAppTtl       = env.Getenv("APP_TTL", "30m")
	envClusterConf  = env.Getenv("FLYNN_CLUSTER_CONFIG", "")
	corsAllowOrigin = env.Getenv("CORS_ALLOW_ORIGIN", "http://localhost:9000")
	sessionStore    = gorillasession.NewCookieStore([]byte(env.Getenv("SESSION_SECRET", "changme")))
	envMongoPath    = env.Getenv("MONGODB", "mongodb://localhost:27017/gitdeploy")
)

type deployRequest struct {
	Repository string `json:"repository",validate:"regex=^(https|http):\\/\\/github\\.com\\/[a-zA-Z0-9\\-\\_\\.]+/[a-zA-Z0-9\\-\\_\\.]+$"`
	Ref        string `json:"ref",validate:"regex=^(origin\\/.*|tags\\/.*|[a-z0-9]+)$"`
}

type appResponse struct {
	App string `json:"app"`
}

func main() {
	eco.IsGitAvailable()
	eco.IsFlynnAvailable()
	init := flag.Bool("init", false, "Initialize flynn")
	flag.Parse()
	if *init {
		eco.InitFlynn(envClusterConf)
		eco.InitGit()
	}

	db, dbName, err := mgopath.Connect(envMongoPath)
	if err != nil {
		log.Fatal(err)
	}
	eventManager := event.New()
	storage := mongo.New(db, dbName)
	sseBroker := sse.New(storage)
	eventManager.AttachListenerAggregate(storage)
	eventManager.AttachListenerAggregate(new(gdLog.Listener))

	// Mux router
	r := mux.NewRouter()
	r.HandleFunc("/config", configHandler).Methods("GET")
	r.HandleFunc("/deployments", deployWrapperAction(sseBroker, eventManager, storage)).Methods("POST")
	r.HandleFunc("/deployments", setCORSHeaders).Methods("OPTIONS")
	r.HandleFunc("/deployments/{app:.+}/events", eventWrapperAction(sseBroker)).Methods("GET")
	r.HandleFunc("/apps/{app:.+}", getAppHandler(storage)).Methods("GET")
	r.PathPrefix("/").HandlerFunc(public.HTML5ModeHandler("./app/dist", "index.html"))
	http.Handle("/", r)

	go job.KillAppsOnHitList(storage)

	listen := fmt.Sprintf("%s:%s", host, port)
	log.Printf("Listening on %s", listen)
	log.Fatal(http.ListenAndServe(listen, nil))
}

func eventWrapperAction(sseBroker *sse.Broker) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		setCORSHeaders(w, r)
		sseBroker.EventHandler(w, r)
	}
}

func deployWrapperAction(sseBroker *sse.Broker, em *event.EventManager, store *mongo.MongoStorage) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		setCORSHeaders(w, r)
		deployAction(w, r, sseBroker, em, store)
	}
}

func configHandler(w http.ResponseWriter, r *http.Request) {
	setCORSHeaders(w, r)
	responseSuccess(w, struct {
		Now time.Time `json:"now"`
	}{time.Now()})
}

func getAppHandler(store *mongo.MongoStorage) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if cleanUpSession(w, r) != nil {
			return
		}
		setCORSHeaders(w, r)
		vars := mux.Vars(r)
		id := vars["app"]
		app, err := store.GetApp(id)
		if err == mgo.ErrNotFound {
			responseError(w, http.StatusNotFound, fmt.Sprintf("App %s does not exist.", id))
			return
		} else if app.Killed {
			responseError(w, http.StatusNotFound, "App timed out and is no longer available.")
			return
		} else if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}

		logs, err := flynn.GetLogs(app.ID)
		if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}

		ps, err := flynn.GetProcs(app.ID)
		if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}

		deployLogs, err := store.FindDeployLogsForApp(id)
		if err != nil {
			responseError(w, http.StatusInternalServerError, err.Error())
			return
		}

		responseSuccess(w, struct {
			*storage.App
			Logs       string                 `json:"logs"`
			DeployLogs []*storage.DeployEvent `json:"deployLogs,inline"`
			PS         string                 `json:"ps"`
		}{app, logs, deployLogs, ps})
	}
}

func deployAction(w http.ResponseWriter, r *http.Request, sseBroker *sse.Broker, em *event.EventManager, store *mongo.MongoStorage) {
	app := uuid.NewRandom().String()
	log.Printf("Trying to fetch session: %s", app)
	session, err := sessionStore.Get(r, sessionName)
	if err != nil {
		responseError(w, http.StatusBadRequest, "Please delete your cookies.")
		return
	}

	// Check if the user is currently deploying an application and switch to that one.
	//	if v, ok := session.Values[sessionCurrentDeployment].(string); ok && len(v) > 0 {
	//		a, err := store.GetApp(v)
	//		if err != nil {
	//			cleanUpSession(w, r)
	//			log.Printf("Could not fetch app from cookie: %s", err.Error())
	//		} else if !sseBroker.IsChannelOpen(a.ID) {
	//			cleanUpSession(w, r)
	//			log.Printf("Channel %s does not exist any more", a.ID)
	//		} else if !a.Deployed {
	//			log.Printf("Channel %s found!", a.ID)
	//			responseSuccess(w, a)
	//			return
	//		}
	//	}

	log.Printf("No deployment active for this session, starting new one: %s", app)
	dr := new(deployRequest)
	ttl, err := time.ParseDuration(envAppTtl)
	if err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		return
	} else if err := json.NewDecoder(r.Body).Decode(dr); err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		return
	} else if err := validator.Validate(dr); err != nil {
		responseError(w, http.StatusBadRequest, fmt.Sprintf("Validation failed: %s", err))
		return
	}

	log.Printf("Validation passed: %s", app)
	if dr.Repository[len(dr.Repository)-4:] != ".git" {
		dr.Repository = dr.Repository + ".git"
	}
	if len(dr.Ref) == 0 {
		dr.Ref = "master"
	}

	log.Printf("Storing app information: %s", app)
	appEntity, err := store.AddApp(app, time.Now().Add(ttl), dr.Repository, ip.GetRemoteAddr(r), dr.Ref)
	if err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		return
	}

	session.Values[sessionCurrentDeployment] = appEntity.ID
	if err := session.Save(r, w); err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		return
	}

	log.Printf("Responding app information: %s", app)
	responseSuccess(w, appEntity)
	go func() {
		sseBroker.OpenChannel(app)
		sseBroker.Start(app)
		defer func() {
			log.Printf("Waiting for slow clients: %s", app)
			// Give the client the chance to read the output...
			time.Sleep(5 * time.Second)
			log.Printf("Timeout for slow clients: %s", app)
			//appEntity.Deployed = true;
			//if err := store.UpdateApp(appEntity); err != nil {
			//	log.Printf("Fatal error when trying to update app %s", app)
			//}
			sseBroker.CloseChannel(app)
		}()
		log.Printf("Creating job: %s", app)
		tasks := deploy.CreateJob(store, appEntity)
		log.Printf("Run job: %s", app)
		err := task.RunJob(app, em, tasks)
		if err != nil {
			log.Printf("Error in task.RunJob: %s", err)
		}
	}()
}

func cleanUpSession(w http.ResponseWriter, r *http.Request) error {
	session, err := sessionStore.Get(r, sessionName)
	if err != nil {
		responseError(w, http.StatusBadRequest, "Please delete your cookies.")
		return err
	}
	delete(session.Values, sessionCurrentDeployment)
	if err := session.Save(r, w); err != nil {
		responseError(w, http.StatusInternalServerError, err.Error())
		return err
	}
	return nil
}

// Set the different CORS headers required for CORS request
func setCORSHeaders(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
	w.Header().Set("Access-Control-Allow-Origin", corsAllowOrigin)
	w.Header().Set("Access-Control-Allow-Credentials", "true")
	w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
}

func responseError(w http.ResponseWriter, code int, message string) {
	log.Printf("Error %d: %s", code, message)
	response := responder.New(ApiVersion)
	response.Write(w, response.Error(code, message))
}

func responseSuccess(w http.ResponseWriter, data interface{}) {
	response := responder.New(ApiVersion)
	response.Write(w, response.Success(data))
}
<|endoftext|>


<commit_before>// Copyright 2021 the Cloud Run Proxy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package main is the entrypoint for cloud-run-proxy. It starts the proxy
// server.
package main

import (
	"context"
	"flag"
	"fmt"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"time"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
)

type contextKey string

const contextKeyError = contextKey("error")

const cloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform"

var (
	flagHost  = flag.String("host", "", "Cloud Run host for which to proxy")
	flagBind  = flag.String("bind", "127.0.0.1:8080", "local host:port on which to listen")
	flagToken = flag.String("token", "", "override OIDC token")
)

func main() {
	if err := realMain(); err != nil {
		fmt.Fprint(os.Stderr, err.Error(), "\n")
		os.Exit(1)
	}
}

func realMain() error {
	// Parse flags.
	flag.Parse()
	if *flagHost == "" {
		return fmt.Errorf("missing -host")
	}
	if *flagBind == "" {
		return fmt.Errorf("missing -bind")
	}

	// Get the best token source.
	tokenSource, err := findTokenSource(*flagToken)
	if err != nil {
		return fmt.Errorf("failed to find token source: %w", err)
	}

	// Build the remote host URL.
	host, err := smartBuildHost(*flagHost)
	if err != nil {
		return fmt.Errorf("failed to parse host URL: %w", err)
	}

	// Build the local bind URL.
	bindHost, bindPort, err := net.SplitHostPort(*flagBind)
	if err != nil {
		return fmt.Errorf("failed to parse bind address: %w", err)
	}
	bind := &url.URL{
		Scheme: "http",
		Host:   bindHost + ":" + bindPort,
	}

	// Construct the proxy.
	proxy := buildProxy(host, bind, tokenSource)

	// Create server.
	server := &http.Server{
		Addr:    bind.Host,
		Handler: proxy,
	}

	// Start server in background.
	errCh := make(chan error, 1)
	go func() {
		fmt.Fprintf(os.Stderr, "%s proxies to %s\n", bind, host)
		if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			select {
			case errCh <- err:
			default:
			}
		}
	}()

	// Signal on stop.
	stop := make(chan os.Signal, 1)
	signal.Notify(stop, os.Interrupt)

	// Wait for error or signal.
	select {
	case err := <-errCh:
		return fmt.Errorf("server error: %w", err)
	case <-stop:
		fmt.Fprint(os.Stderr, "\nserver is shutting down...\n")
	}

	// Attempt graceful shutdown.
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
	defer cancel()
	if err := server.Shutdown(ctx); err != nil {
		return fmt.Errorf("failed to shutdown server: %w", err)
	}
	return nil
}

// buildProxy builds the reverse proxy server, forwarding requests on bind to
// the provided host.
func buildProxy(host, bind *url.URL, tokenSource oauth2.TokenSource) *httputil.ReverseProxy {
	// Build and configure the proxy.
	proxy := httputil.NewSingleHostReverseProxy(host)

	// Configure the director.
	originalDirector := proxy.Director
	proxy.Director = func(r *http.Request) {
		// Call the original director, which configures most of the URL bits for us.
		originalDirector(r)

		// Override host - this is not done by the default director, but Cloud Run
		// requires it.
		r.Header.Set("Host", host.Host)
		r.Host = host.Host

		ctx := r.Context()

		// Get the oauth token.
		token, err := tokenSource.Token()
		if err != nil {
			*r = *r.WithContext(context.WithValue(ctx, contextKeyError,
				fmt.Errorf("failed to get token: %w", err)))
			return
		}

		// Get the id_token from the oauth token.
		idTokenRaw := token.Extra("id_token")
		if idTokenRaw == nil {
			*r = *r.WithContext(context.WithValue(ctx, contextKeyError,
				fmt.Errorf("missing id_token")))
			return
		}
		idToken, ok := idTokenRaw.(string)
		if !ok {
			*r = *r.WithContext(context.WithValue(ctx, contextKeyError,
				fmt.Errorf("id_token is not a string: %T", idTokenRaw)))
			return
		}

		// Set the bearer token to be the id token
		r.Header.Set("Authorization", "Bearer "+idToken)
	}

	// Configure error handling.
	proxy.ModifyResponse = func(r *http.Response) error {
		// In case of redirection, make sure the local address is still used for
		// host. If it has location header && the location url host is the proxied
		// host, change it to local address with http.
		location := r.Header.Get("Location")
		if location != "" {
			locationURL, err := url.Parse(location)
			if err == nil && locationURL.Host == host.Host {
				locationURL.Scheme = bind.Scheme
				locationURL.Host = bind.Host
				r.Header.Set("Location", locationURL.String())
			}
		}

		ctx := r.Request.Context()
		if err, ok := ctx.Value(contextKeyError).(error); ok && err != nil {
			return fmt.Errorf("[PROXY ERROR] %w", err)
		}

		return nil
	}
	proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	return proxy
}

// findTokenSource fetches the reusable/cached oauth2 token source. If t is
// provided, that token is used as a static value. Othwerise, this attempts to
// get the renewable token from the environment (including via Application
// Default Credentials).
func findTokenSource(t string) (oauth2.TokenSource, error) {
	// Prefer supplied value, usually from the flag.
	if t != "" {
		token := new(oauth2.Token).WithExtra(map[string]interface{}{
			"id_token": t,
		})
		return oauth2.StaticTokenSource(token), nil
	}

	// Try and find the default token from ADC.
	ctx := context.Background()
	tokenSource, err := google.DefaultTokenSource(ctx, cloudPlatformScope)
	if err != nil {
		return nil, fmt.Errorf("failed to get default token source: %w", err)
	}
	return oauth2.ReuseTokenSource(nil, tokenSource), nil
}

// smartBuildHost parses the URL, handling the case where it's a real URL
// (https://foo.bar) or just a host (foo.bar). If it's just a host, the URL is
// assumed to be TLS.
func smartBuildHost(host string) (*url.URL, error) {
	u, err := url.Parse(host)
	if err != nil {
		return nil, fmt.Errorf("failed to parse url: %w", err)
	}

	if u.Scheme == "" {
		u.Scheme = "https"

		parts := strings.SplitN(u.Path, "/", 2)
		switch len(parts) {
		case 0:
			u.Host = ""
			u.Path = ""
		case 1:
			u.Host = parts[0]
			u.Path = ""
		case 2:
			u.Host = parts[0]
			u.Path = parts[1]
		}
	}

	u.Host = strings.TrimSpace(u.Host)
	if u.Host == "" {
		return nil, fmt.Errorf("invalid url %q (missing host)", host)
	}

	u.Path = strings.TrimSpace(u.Path)
	if u.Path == "/" {
		u.RawPath = ""
		u.Path = ""
	}

	return u, nil
}
<commit_msg>Allow setting a custom user-agent string (#4)<commit_after>// Copyright 2021 the Cloud Run Proxy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package main is the entrypoint for cloud-run-proxy. It starts the proxy
// server.
package main

import (
	"context"
	"flag"
	"fmt"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/signal"
	"runtime"
	"strings"
	"time"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
)

type contextKey string

const contextKeyError = contextKey("error")

const cloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform"

const Version = "0.1.0"
const OSArch = runtime.GOOS + "/" + runtime.GOARCH
const UserAgent = "cloud-run-proxy/" + Version + " (" + OSArch + ")"

var (
	flagHost             = flag.String("host", "", "Cloud Run host for which to proxy")
	flagBind             = flag.String("bind", "127.0.0.1:8080", "local host:port on which to listen")
	flagToken            = flag.String("token", "", "override OIDC token")
	flagPrependUserAgent = flag.Bool("prepend-user-agent", true, "prepend a custom User-Agent header to requests")
)

func main() {
	if err := realMain(); err != nil {
		fmt.Fprint(os.Stderr, err.Error(), "\n")
		os.Exit(1)
	}
}

func realMain() error {
	// Parse flags.
	flag.Parse()
	if *flagHost == "" {
		return fmt.Errorf("missing -host")
	}
	if *flagBind == "" {
		return fmt.Errorf("missing -bind")
	}

	// Get the best token source.
	tokenSource, err := findTokenSource(*flagToken)
	if err != nil {
		return fmt.Errorf("failed to find token source: %w", err)
	}

	// Build the remote host URL.
	host, err := smartBuildHost(*flagHost)
	if err != nil {
		return fmt.Errorf("failed to parse host URL: %w", err)
	}

	// Build the local bind URL.
	bindHost, bindPort, err := net.SplitHostPort(*flagBind)
	if err != nil {
		return fmt.Errorf("failed to parse bind address: %w", err)
	}
	bind := &url.URL{
		Scheme: "http",
		Host:   bindHost + ":" + bindPort,
	}

	// Construct the proxy.
	proxy := buildProxy(host, bind, tokenSource)

	// Create server.
	server := &http.Server{
		Addr:    bind.Host,
		Handler: proxy,
	}

	// Start server in background.
	errCh := make(chan error, 1)
	go func() {
		fmt.Fprintf(os.Stderr, "%s proxies to %s\n", bind, host)
		if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			select {
			case errCh <- err:
			default:
			}
		}
	}()

	// Signal on stop.
	stop := make(chan os.Signal, 1)
	signal.Notify(stop, os.Interrupt)

	// Wait for error or signal.
	select {
	case err := <-errCh:
		return fmt.Errorf("server error: %w", err)
	case <-stop:
		fmt.Fprint(os.Stderr, "\nserver is shutting down...\n")
	}

	// Attempt graceful shutdown.
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
	defer cancel()
	if err := server.Shutdown(ctx); err != nil {
		return fmt.Errorf("failed to shutdown server: %w", err)
	}
	return nil
}

// buildProxy builds the reverse proxy server, forwarding requests on bind to
// the provided host.
func buildProxy(host, bind *url.URL, tokenSource oauth2.TokenSource) *httputil.ReverseProxy {
	// Build and configure the proxy.
	proxy := httputil.NewSingleHostReverseProxy(host)

	// Configure the director.
	originalDirector := proxy.Director
	proxy.Director = func(r *http.Request) {
		// Call the original director, which configures most of the URL bits for us.
		originalDirector(r)

		// Override host - this is not done by the default director, but Cloud Run
		// requires it.
		r.Header.Set("Host", host.Host)
		r.Host = host.Host

		ctx := r.Context()

		// Get the oauth token.
		token, err := tokenSource.Token()
		if err != nil {
			*r = *r.WithContext(context.WithValue(ctx, contextKeyError,
				fmt.Errorf("failed to get token: %w", err)))
			return
		}

		// Get the id_token from the oauth token.
		idTokenRaw := token.Extra("id_token")
		if idTokenRaw == nil {
			*r = *r.WithContext(context.WithValue(ctx, contextKeyError,
				fmt.Errorf("missing id_token")))
			return
		}
		idToken, ok := idTokenRaw.(string)
		if !ok {
			*r = *r.WithContext(context.WithValue(ctx, contextKeyError,
				fmt.Errorf("id_token is not a string: %T", idTokenRaw)))
			return
		}

		// Set a custom user-agent header.
		if *flagPrependUserAgent {
			ua := r.Header.Get("User-Agent")
			if ua == "" {
				ua = UserAgent
			} else {
				ua = UserAgent + " " + ua
			}

			r.Header.Set("User-Agent", ua)
		}

		// Set the bearer token to be the id token
		r.Header.Set("Authorization", "Bearer "+idToken)
	}

	// Configure error handling.
	proxy.ModifyResponse = func(r *http.Response) error {
		// In case of redirection, make sure the local address is still used for
		// host. If it has location header && the location url host is the proxied
		// host, change it to local address with http.
		location := r.Header.Get("Location")
		if location != "" {
			locationURL, err := url.Parse(location)
			if err == nil && locationURL.Host == host.Host {
				locationURL.Scheme = bind.Scheme
				locationURL.Host = bind.Host
				r.Header.Set("Location", locationURL.String())
			}
		}

		ctx := r.Request.Context()
		if err, ok := ctx.Value(contextKeyError).(error); ok && err != nil {
			return fmt.Errorf("[PROXY ERROR] %w", err)
		}

		return nil
	}
	proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	return proxy
}

// findTokenSource fetches the reusable/cached oauth2 token source. If t is
// provided, that token is used as a static value. Othwerise, this attempts to
// get the renewable token from the environment (including via Application
// Default Credentials).
func findTokenSource(t string) (oauth2.TokenSource, error) {
	// Prefer supplied value, usually from the flag.
	if t != "" {
		token := new(oauth2.Token).WithExtra(map[string]interface{}{
			"id_token": t,
		})
		return oauth2.StaticTokenSource(token), nil
	}

	// Try and find the default token from ADC.
	ctx := context.Background()
	tokenSource, err := google.DefaultTokenSource(ctx, cloudPlatformScope)
	if err != nil {
		return nil, fmt.Errorf("failed to get default token source: %w", err)
	}
	return oauth2.ReuseTokenSource(nil, tokenSource), nil
}

// smartBuildHost parses the URL, handling the case where it's a real URL
// (https://foo.bar) or just a host (foo.bar). If it's just a host, the URL is
// assumed to be TLS.
func smartBuildHost(host string) (*url.URL, error) {
	u, err := url.Parse(host)
	if err != nil {
		return nil, fmt.Errorf("failed to parse url: %w", err)
	}

	if u.Scheme == "" {
		u.Scheme = "https"

		parts := strings.SplitN(u.Path, "/", 2)
		switch len(parts) {
		case 0:
			u.Host = ""
			u.Path = ""
		case 1:
			u.Host = parts[0]
			u.Path = ""
		case 2:
			u.Host = parts[0]
			u.Path = parts[1]
		}
	}

	u.Host = strings.TrimSpace(u.Host)
	if u.Host == "" {
		return nil, fmt.Errorf("invalid url %q (missing host)", host)
	}

	u.Path = strings.TrimSpace(u.Path)
	if u.Path == "/" {
		u.RawPath = ""
		u.Path = ""
	}

	return u, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"runtime"

	"encoding/csv"
	"encoding/json"
	"io/ioutil"
	"path/filepath"

	"github.com/fatih/color"
	"github.com/sachaos/todoist/lib"
	"github.com/spf13/viper"
	"github.com/urfave/cli"
)

var (
	configPath, _      = os.UserHomeDir()
	default_cache_path = filepath.Join(configPath, ".todoist.cache.json")
	CommandFailed      = errors.New("command failed")
	IdNotFound         = errors.New("specified id not found")
	writer             Writer
)

const (
	configName = ".todoist.config"
	configType = "json"

	ShortDateTimeFormat = "06/01/02(Mon) 15:04"
	ShortDateFormat     = "06/01/02(Mon)"
)

func GetClient(c *cli.Context) *todoist.Client {
	return c.App.Metadata["client"].(*todoist.Client)
}

func main() {
	app := cli.NewApp()
	app.Name = "todoist"
	app.Usage = "Todoist CLI Client"
	app.Version = "0.14.0"

	contentFlag := cli.StringFlag{
		Name:  "content, c",
		Usage: "content",
	}
	priorityFlag := cli.IntFlag{
		Name:  "priority, p",
		Value: 1,
		Usage: "priority (1-4)",
	}
	labelIDsFlag := cli.StringFlag{
		Name:  "label-ids, L",
		Usage: "label ids (separated by ,)",
	}
	projectIDFlag := cli.IntFlag{
		Name:  "project-id, P",
		Usage: "project id",
	}
	projectNameFlag := cli.StringFlag{
		Name:  "project-name, N",
		Usage: "project name",
	}
	dateFlag := cli.StringFlag{
		Name:  "date, d",
		Usage: "date string (today, 2016/10/02, 2016/09/02 18:00)",
	}
	browseFlag := cli.BoolFlag{
		Name:  "browse, o",
		Usage: "when contain URL, open it",
	}
	filterFlag := cli.StringFlag{
		Name:  "filter, f",
		Usage: "filter expression",
	}
	reminderFlg := cli.BoolFlag{
		Name:  "reminder, r",
		Usage: "set reminder (only premium users)",
	}

	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "header",
			Usage: "output with header",
		},
		cli.BoolFlag{
			Name:  "color",
			Usage: "colorize output",
		},
		cli.BoolFlag{
			Name:  "csv",
			Usage: "output in CSV format",
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "output logs",
		},
		cli.BoolFlag{
			Name:  "namespace",
			Usage: "display parent task like namespace",
		},
		cli.BoolFlag{
			Name:  "indent",
			Usage: "display children task with indent",
		},
		cli.BoolFlag{
			Name:  "project-namespace",
			Usage: "display parent project like namespace",
		},
	}

	app.Before = func(c *cli.Context) error {
		var store todoist.Store

		if err := LoadCache(default_cache_path, &store); err != nil {
			return err
		}

		viper.SetConfigType(configType)
		viper.SetConfigName(configName)
		viper.AddConfigPath(configPath)
		viper.AddConfigPath(".")

		var token string

		if err := viper.ReadInConfig(); err != nil {
			fmt.Printf("Input API Token: ")
			fmt.Scan(&token)
			viper.Set("token", token)
			buf, err := json.MarshalIndent(viper.AllSettings(), "", "  ")
			if err != nil {
				panic(fmt.Errorf("Fatal error config file: %s \n", err))
			}
			err = ioutil.WriteFile(filepath.Join(configPath, configName+"."+configType), buf, os.ModePerm)
			if err != nil {
				panic(fmt.Errorf("Fatal error config file: %s \n", err))
			}
		}

		config := &todoist.Config{AccessToken: viper.GetString("token"), DebugMode: c.Bool("debug"), Color: viper.GetBool("color")}

		client := todoist.NewClient(config)
		client.Store = &store

		app.Metadata = map[string]interface{}{
			"client": client,
			"config": config,
		}

		if !c.Bool("color") && !config.Color {
			color.NoColor = true
		}

		if c.Bool("csv") {
			writer = csv.NewWriter(os.Stdout)
		} else if runtime.GOOS == "windows" && !color.NoColor {
			writer = NewTSVWriter(color.Output)
		} else {
			writer = NewTSVWriter(os.Stdout)
		}
		return nil
	}

	app.Commands = []cli.Command{
		{
			Name:    "list",
			Aliases: []string{"l"},
			Usage:   "Show all tasks",
			Action:  List,
			Flags: []cli.Flag{
				filterFlag,
			},
		},
		{
			Name:   "show",
			Usage:  "Show task detail",
			Action: Show,
			Flags: []cli.Flag{
				browseFlag,
			},
		},
		{
			Name:    "completed-list",
			Aliases: []string{"c-l", "cl"},
			Usage:   "Show all completed tasks (only premium user)",
			Action:  CompletedList,
			Flags: []cli.Flag{
				filterFlag,
			},
		},
		{
			Name:    "add",
			Aliases: []string{"a"},
			Usage:   "Add task",
			Action:  Add,
			Flags: []cli.Flag{
				priorityFlag,
				labelIDsFlag,
				projectIDFlag,
				projectNameFlag,
				dateFlag,
				reminderFlg,
			},
		},
		{
			Name:    "modify",
			Aliases: []string{"m"},
			Usage:   "Modify task",
			Action:  Modify,
			Flags: []cli.Flag{
				contentFlag,
				priorityFlag,
				labelIDsFlag,
				projectIDFlag,
				projectNameFlag,
				dateFlag,
			},
		},
		{
			Name:    "close",
			Aliases: []string{"c"},
			Usage:   "Close task",
			Action:  Close,
		},
		{
			Name:    "delete",
			Aliases: []string{"d"},
			Usage:   "Delete task",
			Action:  Delete,
		},
		{
			Name:   "labels",
			Usage:  "Show all labels",
			Action: Labels,
		},
		{
			Name:   "projects",
			Usage:  "Show all projects",
			Action: Projects,
		},
		{
			Name:   "karma",
			Usage:  "Show karma",
			Action: Karma,
		},
		{
			Name:    "sync",
			Aliases: []string{"s"},
			Usage:   "Sync cache",
			Action:  Sync,
		},
		{
			Name:    "quick",
			Aliases: []string{"q"},
			Usage:   "Quick add a task",
			Action:  Quick,
		},
	}
	if err := app.Run(os.Args); err != nil {
		fmt.Fprintln(os.Stderr, "Error:", err)
		os.Exit(1)
	}
}
<commit_msg>Change default priority value<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"runtime"

	"encoding/csv"
	"encoding/json"
	"io/ioutil"
	"path/filepath"

	"github.com/fatih/color"
	"github.com/sachaos/todoist/lib"
	"github.com/spf13/viper"
	"github.com/urfave/cli"
)

var (
	configPath, _      = os.UserHomeDir()
	default_cache_path = filepath.Join(configPath, ".todoist.cache.json")
	CommandFailed      = errors.New("command failed")
	IdNotFound         = errors.New("specified id not found")
	writer             Writer
)

const (
	configName = ".todoist.config"
	configType = "json"

	ShortDateTimeFormat = "06/01/02(Mon) 15:04"
	ShortDateFormat     = "06/01/02(Mon)"
)

func GetClient(c *cli.Context) *todoist.Client {
	return c.App.Metadata["client"].(*todoist.Client)
}

func main() {
	app := cli.NewApp()
	app.Name = "todoist"
	app.Usage = "Todoist CLI Client"
	app.Version = "0.14.0"

	contentFlag := cli.StringFlag{
		Name:  "content, c",
		Usage: "content",
	}
	priorityFlag := cli.IntFlag{
		Name:  "priority, p",
		Value: 4,
		Usage: "priority (1-4)",
	}
	labelIDsFlag := cli.StringFlag{
		Name:  "label-ids, L",
		Usage: "label ids (separated by ,)",
	}
	projectIDFlag := cli.IntFlag{
		Name:  "project-id, P",
		Usage: "project id",
	}
	projectNameFlag := cli.StringFlag{
		Name:  "project-name, N",
		Usage: "project name",
	}
	dateFlag := cli.StringFlag{
		Name:  "date, d",
		Usage: "date string (today, 2016/10/02, 2016/09/02 18:00)",
	}
	browseFlag := cli.BoolFlag{
		Name:  "browse, o",
		Usage: "when contain URL, open it",
	}
	filterFlag := cli.StringFlag{
		Name:  "filter, f",
		Usage: "filter expression",
	}
	reminderFlg := cli.BoolFlag{
		Name:  "reminder, r",
		Usage: "set reminder (only premium users)",
	}

	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "header",
			Usage: "output with header",
		},
		cli.BoolFlag{
			Name:  "color",
			Usage: "colorize output",
		},
		cli.BoolFlag{
			Name:  "csv",
			Usage: "output in CSV format",
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "output logs",
		},
		cli.BoolFlag{
			Name:  "namespace",
			Usage: "display parent task like namespace",
		},
		cli.BoolFlag{
			Name:  "indent",
			Usage: "display children task with indent",
		},
		cli.BoolFlag{
			Name:  "project-namespace",
			Usage: "display parent project like namespace",
		},
	}

	app.Before = func(c *cli.Context) error {
		var store todoist.Store

		if err := LoadCache(default_cache_path, &store); err != nil {
			return err
		}

		viper.SetConfigType(configType)
		viper.SetConfigName(configName)
		viper.AddConfigPath(configPath)
		viper.AddConfigPath(".")

		var token string

		if err := viper.ReadInConfig(); err != nil {
			fmt.Printf("Input API Token: ")
			fmt.Scan(&token)
			viper.Set("token", token)
			buf, err := json.MarshalIndent(viper.AllSettings(), "", "  ")
			if err != nil {
				panic(fmt.Errorf("Fatal error config file: %s \n", err))
			}
			err = ioutil.WriteFile(filepath.Join(configPath, configName+"."+configType), buf, os.ModePerm)
			if err != nil {
				panic(fmt.Errorf("Fatal error config file: %s \n", err))
			}
		}

		config := &todoist.Config{AccessToken: viper.GetString("token"), DebugMode: c.Bool("debug"), Color: viper.GetBool("color")}

		client := todoist.NewClient(config)
		client.Store = &store

		app.Metadata = map[string]interface{}{
			"client": client,
			"config": config,
		}

		if !c.Bool("color") && !config.Color {
			color.NoColor = true
		}

		if c.Bool("csv") {
			writer = csv.NewWriter(os.Stdout)
		} else if runtime.GOOS == "windows" && !color.NoColor {
			writer = NewTSVWriter(color.Output)
		} else {
			writer = NewTSVWriter(os.Stdout)
		}
		return nil
	}

	app.Commands = []cli.Command{
		{
			Name:    "list",
			Aliases: []string{"l"},
			Usage:   "Show all tasks",
			Action:  List,
			Flags: []cli.Flag{
				filterFlag,
			},
		},
		{
			Name:   "show",
			Usage:  "Show task detail",
			Action: Show,
			Flags: []cli.Flag{
				browseFlag,
			},
		},
		{
			Name:    "completed-list",
			Aliases: []string{"c-l", "cl"},
			Usage:   "Show all completed tasks (only premium user)",
			Action:  CompletedList,
			Flags: []cli.Flag{
				filterFlag,
			},
		},
		{
			Name:    "add",
			Aliases: []string{"a"},
			Usage:   "Add task",
			Action:  Add,
			Flags: []cli.Flag{
				priorityFlag,
				labelIDsFlag,
				projectIDFlag,
				projectNameFlag,
				dateFlag,
				reminderFlg,
			},
		},
		{
			Name:    "modify",
			Aliases: []string{"m"},
			Usage:   "Modify task",
			Action:  Modify,
			Flags: []cli.Flag{
				contentFlag,
				priorityFlag,
				labelIDsFlag,
				projectIDFlag,
				projectNameFlag,
				dateFlag,
			},
		},
		{
			Name:    "close",
			Aliases: []string{"c"},
			Usage:   "Close task",
			Action:  Close,
		},
		{
			Name:    "delete",
			Aliases: []string{"d"},
			Usage:   "Delete task",
			Action:  Delete,
		},
		{
			Name:   "labels",
			Usage:  "Show all labels",
			Action: Labels,
		},
		{
			Name:   "projects",
			Usage:  "Show all projects",
			Action: Projects,
		},
		{
			Name:   "karma",
			Usage:  "Show karma",
			Action: Karma,
		},
		{
			Name:    "sync",
			Aliases: []string{"s"},
			Usage:   "Sync cache",
			Action:  Sync,
		},
		{
			Name:    "quick",
			Aliases: []string{"q"},
			Usage:   "Quick add a task",
			Action:  Quick,
		},
	}
	if err := app.Run(os.Args); err != nil {
		fmt.Fprintln(os.Stderr, "Error:", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/oldspiceland/evenfib/fibgen"
	"flag"
)

func main() {
	fibgen.Generate()
}
<commit_msg>Set up main for testing findNext and GenerateSeq<commit_after>package main

import (
	"github.com/oldspiceland/evenfib/fibgen"
)

func main() {
	fibgen.GenerateSeq(1,2,4000000)
}
<|endoftext|>


<commit_before>// Copyright (C) 2016 Eneo Tecnologia S.L.
// Diego Fernández Barrera <bigomby@gmail.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/Shopify/sarama"
	"github.com/Sirupsen/logrus"
	"github.com/bsm/sarama-cluster"
	"github.com/redBorder/rbforwarder"
	"github.com/redBorder/rbforwarder/components/batch"
	"github.com/redBorder/rbforwarder/components/httpsender"
	"github.com/redBorder/rbforwarder/components/limiter"
	"github.com/x-cray/logrus-prefixed-formatter"
)

const (
	defaultQueueSize = 10000
	defaultWorkers   = 1
	defaultRetries   = 0
	defaultBackoff   = 2
)

// Logger is the main logger object
var Logger = logrus.New()
var logger *logrus.Entry

var (
	configFilename *string
	debug          *bool
	counter        *int
	version        string
)

func init() {
	configFilename = flag.String("config", "", "Config file")
	debug = flag.Bool("debug", false, "Show debug info")
	counter = flag.Int("counter", 0, "Display messages per second")
	versionFlag := flag.Bool("version", false, "Print version info")

	flag.Parse()

	if *versionFlag {
		displayVersion()
		os.Exit(0)
	}

	if len(*configFilename) == 0 {
		fmt.Println("No config file provided")
		flag.Usage()
		os.Exit(0)
	}

	Logger.Formatter = new(prefixed.TextFormatter)

	// Show debug info if required
	if *debug {
		Logger.Level = logrus.DebugLevel
	}

	if *debug {
		go func() {
			Logger.Debugln(http.ListenAndServe("localhost:6060", nil))
		}()
	}
}

func main() {
	var components []interface{}

	// Initialize logger
	logger = Logger.WithFields(logrus.Fields{
		"prefix": "k2http",
	})

	// Initialize forwarder and components
	f := rbforwarder.NewRBForwarder(loadForwarderConfig())
	components = append(components, &batcher.Batcher{Config: loadBatchConfig()})
	components = append(components, &httpsender.HTTPSender{Config: loadHTTPConfig()})
	components = append(components, &limiter.Limiter{Config: loadLimiterConfig()})
	f.PushComponents(components)

	// Initialize kafka
	kafka := &KafkaConsumer{Config: loadKafkaConfig()}

	// Set the forwarder on the kafka consumer
	kafka.forwarder = f

	// Wait for ctrl-c to close the consumer
	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)
	go func() {
		<-ctrlc
		kafka.Close()
	}()

	// Start getting messages
	f.Run()
	kafka.Start()
}

func displayVersion() {
	fmt.Println("K2HTTP VERSION:\t\t", version)
	fmt.Println("RBFORWARDER VERSION:\t", rbforwarder.Version)
}

func loadConfig(filename, component string) (config map[string]interface{}, err error) {
	generalConfig := make(map[string]interface{})
	config = make(map[string]interface{})

	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}

	yaml.Unmarshal([]byte(data), &generalConfig)
	if err != nil {
		return
	}

	for k, v := range generalConfig[component].(map[interface{}]interface{}) {
		config[k.(string)] = v
	}

	return
}

func loadForwarderConfig() rbforwarder.Config {
	pipelineConfig, err := loadConfig(*configFilename, "pipeline")
	if err != nil {
		logger.Fatal(err)
	}

	config := rbforwarder.Config{}
	if retries, ok := pipelineConfig["retries"].(int); ok {
		config.Retries = retries
	} else {
		logger.Fatal("Invalid 'retries' option")
	}
	if backoff, ok := pipelineConfig["backoff"].(int); ok {
		config.Backoff = backoff
	} else {
		logger.Fatal("Invalid 'backoff' option")
	}
	if queue, ok := pipelineConfig["queue"].(int); ok {
		config.QueueSize = queue
	} else {
		logger.Fatal("Invalid 'queue' option")
	}

	logger.WithFields(map[string]interface{}{
		"retries": config.Retries,
		"backoff": config.Backoff,
		"queue":   config.QueueSize,
	}).Info("Forwarder config")

	return config
}

func loadBatchConfig() batcher.Config {
	batchConfig, err := loadConfig(*configFilename, "batch")
	if err != nil {
		logger.Fatal(err)
	}

	config := batcher.Config{}
	if workers, ok := batchConfig["workers"].(int); ok {
		config.Workers = workers
	} else {
		config.Workers = 1
	}
	if TimeoutMillis, ok := batchConfig["timeoutMillis"].(int); ok {
		config.TimeoutMillis = uint(TimeoutMillis)
	} else {
		logger.Fatal("Invalid 'TimeoutMillis' option")
	}
	if size, ok := batchConfig["size"].(int); ok {
		config.Limit = uint64(size)
	} else {
		logger.Fatal("Invalid 'size' option")
	}
	if deflate, ok := batchConfig["deflate"].(bool); ok {
		config.Deflate = deflate
	}

	logger.WithFields(map[string]interface{}{
		"workers":       config.Workers,
		"timeoutMillis": config.TimeoutMillis,
		"size":          config.Limit,
		"deflate":       config.Deflate,
	}).Info("Batch config")

	return config
}

func loadHTTPConfig() httpsender.Config {
	httpConfig, err := loadConfig(*configFilename, "http")
	if err != nil {
		logger.Fatal(err)
	}

	config := httpsender.Config{}
	if workers, ok := httpConfig["workers"].(int); ok {
		config.Workers = workers
	} else {
		config.Workers = 1
	}
	if *debug {
		config.Logger = logger.WithField("prefix", "http sender")
		config.Debug = true
	}
	if url, ok := httpConfig["url"].(string); ok {
		config.URL = url
	} else {
		logger.Fatal("Invalid 'url' option")
	}

	logger.WithFields(map[string]interface{}{
		"workers": config.Workers,
		"debug":   config.Debug,
		"url":     config.URL,
	}).Info("HTTP config")

	return config
}

func loadLimiterConfig() limiter.Config {
	limiterConfig, err := loadConfig(*configFilename, "limiter")
	if err != nil {
		logger.Fatal(err)
	}

	config := limiter.Config{}
	if MessageLimit, ok := limiterConfig["max_messages"].(uint64); ok {
		config.MessageLimit = MessageLimit
	} else {
		config.MessageLimit = 0
	}
	if BytesLimit, ok := limiterConfig["max_bytes"].(uint64); ok {
		if config.MessageLimit > 0 {
			config.MessageLimit = 0
			logger.Warning("Ignoring \"max_messages\" option")
		}
		config.BytesLimit = BytesLimit
	} else {
		config.BytesLimit = 0
	}

	logger.WithFields(map[string]interface{}{
		"max_messages": config.MessageLimit,
	}).Info("Limiter config")

	return config
}

func loadKafkaConfig() KafkaConfig {
	kafkaConfig, err := loadConfig(*configFilename, "kafka")
	if err != nil {
		logger.Fatal(err)
	}

	config := KafkaConfig{}

	batchConfig, err := loadConfig(*configFilename, "batch")
	config.consumerGroupConfig = cluster.NewConfig()
	if err == nil {
		if deflate, ok := batchConfig["deflate"].(bool); ok {
			config.deflate = deflate
		}
	}
	if *debug {
		sarama.Logger = logger.WithField("prefix", "kafka-consumer")
	}
	if consumerGroup, ok := kafkaConfig["consumergroup"].(string); ok {
		config.consumergroup = consumerGroup
		config.consumerGroupConfig.ClientID = consumerGroup
	} else {
		config.consumergroup = "k2http"
		config.consumerGroupConfig.ClientID = "k2http"
	}
	if broker, ok := kafkaConfig["broker"].(string); ok {
		config.brokers = strings.Split(broker, ",")
	} else {
		logger.Fatal("Invalid 'broker' option")
	}
	if topics, ok := kafkaConfig["topics"].([]interface{}); ok {
		for _, topic := range topics {
			config.topics = append(config.topics, topic.(string))
		}
	}

	config.consumerGroupConfig.Config.Consumer.Offsets.CommitInterval = 1 * time.Second
	config.consumerGroupConfig.Consumer.Offsets.Initial = sarama.OffsetNewest
	// config.consumerGroupConfig.Consumer.MaxProcessingTime = 5 * time.Second

	return config
}
<commit_msg>:bug: Fix limiter component not working<commit_after>// Copyright (C) 2016 Eneo Tecnologia S.L.
// Diego Fernández Barrera <bigomby@gmail.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/Shopify/sarama"
	"github.com/Sirupsen/logrus"
	"github.com/bsm/sarama-cluster"
	"github.com/redBorder/rbforwarder"
	"github.com/redBorder/rbforwarder/components/batch"
	"github.com/redBorder/rbforwarder/components/httpsender"
	"github.com/redBorder/rbforwarder/components/limiter"
	"github.com/x-cray/logrus-prefixed-formatter"
)

const (
	defaultQueueSize = 10000
	defaultWorkers   = 1
	defaultRetries   = 0
	defaultBackoff   = 2
)

// Logger is the main logger object
var Logger = logrus.New()
var logger *logrus.Entry

var (
	configFilename *string
	debug          *bool
	counter        *int
	version        string
)

func init() {
	configFilename = flag.String("config", "", "Config file")
	debug = flag.Bool("debug", false, "Show debug info")
	counter = flag.Int("counter", 0, "Display messages per second")
	versionFlag := flag.Bool("version", false, "Print version info")

	flag.Parse()

	if *versionFlag {
		displayVersion()
		os.Exit(0)
	}

	if len(*configFilename) == 0 {
		fmt.Println("No config file provided")
		flag.Usage()
		os.Exit(0)
	}

	Logger.Formatter = new(prefixed.TextFormatter)

	// Show debug info if required
	if *debug {
		Logger.Level = logrus.DebugLevel
	}

	if *debug {
		go func() {
			Logger.Debugln(http.ListenAndServe("localhost:6060", nil))
		}()
	}
}

func main() {
	var components []interface{}

	// Initialize logger
	logger = Logger.WithFields(logrus.Fields{
		"prefix": "k2http",
	})

	// Initialize forwarder and components
	f := rbforwarder.NewRBForwarder(loadForwarderConfig())
	components = append(components, &limiter.Limiter{Config: loadLimiterConfig()})
	components = append(components, &batcher.Batcher{Config: loadBatchConfig()})
	components = append(components, &httpsender.HTTPSender{Config: loadHTTPConfig()})
	f.PushComponents(components)

	// Initialize kafka
	kafka := &KafkaConsumer{Config: loadKafkaConfig()}

	// Set the forwarder on the kafka consumer
	kafka.forwarder = f

	// Wait for ctrl-c to close the consumer
	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)
	go func() {
		<-ctrlc
		kafka.Close()
	}()

	// Start getting messages
	f.Run()
	kafka.Start()
}

func displayVersion() {
	fmt.Println("K2HTTP VERSION:\t\t", version)
	fmt.Println("RBFORWARDER VERSION:\t", rbforwarder.Version)
}

func loadConfig(filename, component string) (config map[string]interface{}, err error) {
	generalConfig := make(map[string]interface{})
	config = make(map[string]interface{})

	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}

	yaml.Unmarshal([]byte(data), &generalConfig)
	if err != nil {
		return
	}

	for k, v := range generalConfig[component].(map[interface{}]interface{}) {
		config[k.(string)] = v
	}

	return
}

func loadForwarderConfig() rbforwarder.Config {
	pipelineConfig, err := loadConfig(*configFilename, "pipeline")
	if err != nil {
		logger.Fatal(err)
	}

	config := rbforwarder.Config{}
	if retries, ok := pipelineConfig["retries"].(int); ok {
		config.Retries = retries
	} else {
		logger.Fatal("Invalid 'retries' option")
	}
	if backoff, ok := pipelineConfig["backoff"].(int); ok {
		config.Backoff = backoff
	} else {
		logger.Fatal("Invalid 'backoff' option")
	}
	if queue, ok := pipelineConfig["queue"].(int); ok {
		config.QueueSize = queue
	} else {
		logger.Fatal("Invalid 'queue' option")
	}

	logger.WithFields(map[string]interface{}{
		"retries": config.Retries,
		"backoff": config.Backoff,
		"queue":   config.QueueSize,
	}).Info("Forwarder config")

	return config
}

func loadBatchConfig() batcher.Config {
	batchConfig, err := loadConfig(*configFilename, "batch")
	if err != nil {
		logger.Fatal(err)
	}

	config := batcher.Config{}
	if workers, ok := batchConfig["workers"].(int); ok {
		config.Workers = workers
	} else {
		config.Workers = 1
	}
	if TimeoutMillis, ok := batchConfig["timeoutMillis"].(int); ok {
		config.TimeoutMillis = uint(TimeoutMillis)
	} else {
		logger.Fatal("Invalid 'TimeoutMillis' option")
	}
	if size, ok := batchConfig["size"].(int); ok {
		config.Limit = uint64(size)
	} else {
		logger.Fatal("Invalid 'size' option")
	}
	if deflate, ok := batchConfig["deflate"].(bool); ok {
		config.Deflate = deflate
	}

	logger.WithFields(map[string]interface{}{
		"workers":       config.Workers,
		"timeoutMillis": config.TimeoutMillis,
		"size":          config.Limit,
		"deflate":       config.Deflate,
	}).Info("Batch config")

	return config
}

func loadHTTPConfig() httpsender.Config {
	httpConfig, err := loadConfig(*configFilename, "http")
	if err != nil {
		logger.Fatal(err)
	}

	config := httpsender.Config{}
	if workers, ok := httpConfig["workers"].(int); ok {
		config.Workers = workers
	} else {
		config.Workers = 1
	}
	if *debug {
		config.Logger = logger.WithField("prefix", "http sender")
		config.Debug = true
	}
	if url, ok := httpConfig["url"].(string); ok {
		config.URL = url
	} else {
		logger.Fatal("Invalid 'url' option")
	}

	logger.WithFields(map[string]interface{}{
		"workers": config.Workers,
		"debug":   config.Debug,
		"url":     config.URL,
	}).Info("HTTP config")

	return config
}

func loadLimiterConfig() limiter.Config {
	limiterConfig, err := loadConfig(*configFilename, "limiter")
	if err != nil {
		logger.Fatal(err)
	}

	config := limiter.Config{}
	if MessageLimit, ok := limiterConfig["max_messages"].(int); ok {
		config.MessageLimit = uint64(MessageLimit)
	} else {
		config.MessageLimit = 0
	}
	if BytesLimit, ok := limiterConfig["max_bytes"].(int); ok {
		if config.MessageLimit > 0 {
			config.MessageLimit = 0
			logger.Warning("Ignoring \"max_messages\" option")
		}
		config.BytesLimit = uint64(BytesLimit)
	} else {
		config.BytesLimit = 0
	}

	logger.WithFields(map[string]interface{}{
		"max_messages": config.MessageLimit,
		"max_bytes":    config.BytesLimit,
	}).Info("Limiter config")

	return config
}

func loadKafkaConfig() KafkaConfig {
	kafkaConfig, err := loadConfig(*configFilename, "kafka")
	if err != nil {
		logger.Fatal(err)
	}

	config := KafkaConfig{}

	batchConfig, err := loadConfig(*configFilename, "batch")
	config.consumerGroupConfig = cluster.NewConfig()
	if err == nil {
		if deflate, ok := batchConfig["deflate"].(bool); ok {
			config.deflate = deflate
		}
	}
	if *debug {
		sarama.Logger = logger.WithField("prefix", "kafka-consumer")
	}
	if consumerGroup, ok := kafkaConfig["consumergroup"].(string); ok {
		config.consumergroup = consumerGroup
		config.consumerGroupConfig.ClientID = consumerGroup
	} else {
		config.consumergroup = "k2http"
		config.consumerGroupConfig.ClientID = "k2http"
	}
	if broker, ok := kafkaConfig["broker"].(string); ok {
		config.brokers = strings.Split(broker, ",")
	} else {
		logger.Fatal("Invalid 'broker' option")
	}
	if topics, ok := kafkaConfig["topics"].([]interface{}); ok {
		for _, topic := range topics {
			config.topics = append(config.topics, topic.(string))
		}
	}

	config.consumerGroupConfig.Config.Consumer.Offsets.CommitInterval = 1 * time.Second
	config.consumerGroupConfig.Consumer.Offsets.Initial = sarama.OffsetNewest
	// config.consumerGroupConfig.Consumer.MaxProcessingTime = 5 * time.Second

	return config
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/joho/godotenv"
	"github.com/lestrrat-go/server-starter/listener"
	"github.com/mono0x/my-scraper/lib/server"
	"github.com/pkg/errors"
	"golang.org/x/sync/errgroup"
)

func run() error {
	listeners, err := listener.ListenAll()
	if err != nil && err != listener.ErrNoListeningTarget {
		return errors.WithStack(err)
	}

	var l net.Listener
	if len(listeners) > 0 {
		l = listeners[0]
	} else {
		l, err = net.Listen("tcp", ":8080")
		if err != nil {
			return errors.WithStack(err)
		}
	}

	s := http.Server{Handler: server.NewHandler()}

	eg := errgroup.Group{}
	eg.Go(func() error {
		if err := s.Serve(l); err != nil && err != http.ErrServerClosed {
			return err
		}
		return nil
	})
	eg.Go(func() error {
		signalChan := make(chan os.Signal, 1)
		signal.Notify(signalChan, syscall.SIGTERM, os.Interrupt)
		<-signalChan

		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		return s.Shutdown(ctx)
	})
	return eg.Wait()
}

func main() {
	log.SetFlags(log.Lshortfile)

	_ = godotenv.Load()

	if err := run(); err != nil {
		log.Fatalf("%+v\n", err)
	}
}
<commit_msg>Use errors.WithStack<commit_after>package main

import (
	"context"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/joho/godotenv"
	"github.com/lestrrat-go/server-starter/listener"
	"github.com/mono0x/my-scraper/lib/server"
	"github.com/pkg/errors"
	"golang.org/x/sync/errgroup"
)

func run() error {
	listeners, err := listener.ListenAll()
	if err != nil && err != listener.ErrNoListeningTarget {
		return errors.WithStack(err)
	}

	var l net.Listener
	if len(listeners) > 0 {
		l = listeners[0]
	} else {
		l, err = net.Listen("tcp", ":8080")
		if err != nil {
			return errors.WithStack(err)
		}
	}

	s := http.Server{Handler: server.NewHandler()}

	eg := errgroup.Group{}
	eg.Go(func() error {
		if err := s.Serve(l); err != nil && err != http.ErrServerClosed {
			return errors.WithStack(err)
		}
		return nil
	})
	eg.Go(func() error {
		signalChan := make(chan os.Signal, 1)
		signal.Notify(signalChan, syscall.SIGTERM, os.Interrupt)
		<-signalChan

		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		if err := s.Shutdown(ctx); err != nil {
			return errors.WithStack(err)
		}
		return nil
	})
	return eg.Wait()
}

func main() {
	log.SetFlags(log.Lshortfile)

	_ = godotenv.Load()

	if err := run(); err != nil {
		log.Fatalf("%+v\n", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"fmt"
	"encoding/json"
	"net/http"
)

/*
 * Parsing of JSON
 */
type Route struct {
	Request  Request `json:"request"`
	Response Response `json:"response"`
}

type Request struct {
	Method string `json:"method"`
	Path   string `json:"path"`
}

type Response struct {
	Code int
	Body string `json:"body"`
}

/*
 * Slice of Route structs
 */
type Routes []Route

/*
 * Setting up HTTP server
 */
type ServerConfig struct {
	Address string
	Port    int
}

type Server struct {
	Config ServerConfig
	Routes Routes
}

func (server Server) getResponse(request *http.Request) Response {
	//getting our routes from JSON
	for _, route := range server.Routes {
		//if Path and Method Match
		if route.Request.Path == request.URL.Path && route.Request.Method == request.Method {
			//Respond with route response JSON
			return route.Response
		}
	}
	//else return bad response
	return Response{Code: 404, Body: "Not Found"}
}

func (server Server) Handle(writer http.ResponseWriter, request *http.Request) {
	response := server.getResponse(request)
	fmt.Fprintf(writer, response.Body)
}

func (server Server) Run() {
	//Handler Func Setup
	http.HandleFunc("/", server.Handle)
	//Setting up addr to listen & serve on, this feels a little crappy, cleanup s.c.s.a and s.c.s.p
	addr := fmt.Sprintf("%v:%v", server.Config.Address, server.Config.Port)
	panic(http.ListenAndServe(addr, nil))
}

/*
 * All Config
 */
type SetupRoutes struct {
	//Server ServerConfig
	Routes Routes `json:"routes"`
}

func main() {
	//Reading Routes FIle
	apiRoutes, err := ioutil.ReadFile("routes.json")

	//Panic at the Disco
	if err != nil {
		panic(err)
	}

	//Route Parsing Setup
	var routes SetupRoutes

	//Grabbing JSON, storing in Config Struct which parses the JSON automagically
	json.Unmarshal(apiRoutes, &routes)

	//@todo cleanup some of this, feels a little janky
	//Server Address and Port Setup
	serverConfig := ServerConfig{ Address: "127.0.0.1", Port: 8080 }
	server := Server{Config: serverConfig, Routes: routes.Routes}

	//Protect and Serve
	server.Run()

}
<commit_msg>a little more cleanup<commit_after>package main

import (
	"io/ioutil"
	"fmt"
	"encoding/json"
	"net/http"
)

/*
 * Parsing of JSON
 */
type Route struct {
	Request  Request `json:"request"`
	Response Response `json:"response"`
}

type Request struct {
	Method string `json:"method"`
	Path   string `json:"path"`
}

type Response struct {
	Code int
	Body string `json:"body"`
}

/*
 * Slice of Route structs
 */
type Routes []Route

/*
 * Setting up HTTP server
 */

type Server struct {
	Address string
	Port    int
	Routes  Routes
}

func (server Server) getResponse(request *http.Request) Response {
	//getting our routes from JSON
	for _, route := range server.Routes {
		//if Path and Method Match
		if route.Request.Path == request.URL.Path && route.Request.Method == request.Method {
			//Respond with route response JSON
			return route.Response
		}
	}
	//else return bad response
	return Response{Code: 404, Body: "Not Found"}
}

func (server Server) Handle(writer http.ResponseWriter, request *http.Request) {
	response := server.getResponse(request)
	fmt.Fprintf(writer, response.Body)
}

func (server Server) Run() {
	//Handler Func Setup
	http.HandleFunc("/", server.Handle)
	//Setting up addr to listen & serve on, this feels a little crappy, cleanup s.c.s.a and s.c.s.p
	addr := fmt.Sprintf("%v:%v", server.Address, server.Port)
	panic(http.ListenAndServe(addr, nil))
}

/*
 * All Config
 */
type ParsedRoutes struct {
	Routes Routes `json:"routes"`
}

func main() {
	//Reading Routes FIle
	apiRoutes, err := ioutil.ReadFile("routes.json")

	//Panic at the Disco
	if err != nil {
		panic(err)
	}

	//Route Parsing Setup
	var routes ParsedRoutes

	//Grabbing JSON, storing in Config Struct which parses the JSON automagically
	json.Unmarshal(apiRoutes, &routes)

	//Server Address and Port Setup
	server := Server{Address: "127.0.0.1", Port: 8080, Routes: routes.Routes}

	//Protect and Serve
	server.Run()

}
<|endoftext|>