<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/urfave/cli"
)

func main() {
	app := cli.NewApp()
	app.Name = "rubusy"
	app.Usage = "cron job schedule searcher"
	app.UsageText = "rubusy [-l line] [-fr from] [-p] [--line value] [--from value] [--plain] [file]"
	app.Version = "0.0.1"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "job, j",
			Usage: "specific job search of the input.",
		},
		cli.StringFlag{
			Name:  "from, fr",
			Usage: "specific schedule search from time formatted like 2006-01-02-15-04",
		},
		cli.BoolFlag{
			Name:  "plain, p",
			Usage: "only result is written to standard output.",
		},
	}
	app.Action = func(c *cli.Context) error {

		// TODO validation的なこと

		// 検索範囲
		t := time.Now()
		str := c.String("from")
		if str != "" {
			const layout = "2006-01-02-15-04"
			// TODO 時刻変換に失敗したらエラー
			t, _ = time.Parse(layout, str)
		}

		timeCondition := searchRange{
			from: t,
			to:   t.Add(time.Duration(24*365) * time.Hour),
		}

		plain := c.Bool("plain")
		if !plain {
			fmt.Println(timeCondition)
			fmt.Println("==============================================")
		}

		var jobs []CronJob
		// TODO ファイル名もJobも未指定の場合、エラー
		targetJob := c.String("line")
		if targetJob != "" {
			jobs = []CronJob{Parse(targetJob)}
		} else {
			fileName := c.Args().Get(0)
			var err error
			jobs, err = readCrontabFile(fileName)
			if err != nil {
				return cli.NewExitError("read crontab file failed", 1)
			}
		}

		if len(jobs) == 0 {
			fmt.Println("Probably nothing to do. no cron tasks.")
			return nil
		}

		for _, job := range jobs {
			from := timeCondition.from
			to := timeCondition.to
			cond := from
			for cond.Before(to) {
				ok, job := job.match(cond)
				if ok {
					fmt.Printf("%v : %s\n", job.schedule.Format("2006/01/02 15:04"), job.line)
					break
				}
				cond = cond.Add(time.Duration(1) * time.Minute)
			}
		}

		return nil
	}

	app.Run(os.Args)
}

// ErrCrontabFile はcrontabファイルが不正だった場合に発生するエラー
var ErrCrontabFile = errors.New("crontab file is something wrong")

func readCrontabFile(fileName string) ([]CronJob, error) {
	_, err := os.Stat(fileName)
	if err != nil {
		return nil, ErrCrontabFile
	}
	fp, err := os.Open(fileName)
	if err != nil {
		return nil, ErrCrontabFile
	}
	defer fp.Close()

	jobs := make([]CronJob, 0)
	scanner := bufio.NewScanner(fp)
	for scanner.Scan() {
		line := scanner.Text()
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		jobs = append(jobs, Parse(line))
	}

	return jobs, nil
}

// 実行されるcronの検索範囲を保持する構造体
type searchRange struct {
	from time.Time
	to   time.Time
}

func (s searchRange) String() string {
	const format = "2006/01/02 15:04"
	return fmt.Sprintf("from: %s - to: %s", s.from.Format(format), s.to.Format(format))
}
<commit_msg>modify message<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/urfave/cli"
)

func main() {
	app := cli.NewApp()
	app.Name = "rubusy"
	app.Usage = "cron job schedule searcher"
	app.UsageText = "rubusy [-l line] [-fr from] [-p] [--line value] [--from value] [--plain] [file]"
	app.Version = "0.0.1"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "job, j",
			Usage: "specific job search of the input.",
		},
		cli.StringFlag{
			Name:  "from, fr",
			Usage: "specific schedule search from time formatted like 2006-01-02-15-04",
		},
		cli.BoolFlag{
			Name:  "plain, p",
			Usage: "only result is written to standard output.",
		},
	}
	app.Action = func(c *cli.Context) error {

		// TODO validation的なこと

		// 検索範囲
		t := time.Now()
		str := c.String("from")
		if str != "" {
			const layout = "2006-01-02-15-04"
			// TODO 時刻変換に失敗したらエラー
			t, _ = time.Parse(layout, str)
		}

		timeCondition := searchRange{
			from: t,
			to:   t.Add(time.Duration(24*365) * time.Hour),
		}

		plain := c.Bool("plain")
		if !plain {
			fmt.Println(timeCondition)
			fmt.Println("==============================================")
		}

		var jobs []CronJob
		// TODO ファイル名もJobも未指定の場合、エラー
		targetJob := c.String("line")
		if targetJob != "" {
			jobs = []CronJob{Parse(targetJob)}
		} else {
			fileName := c.Args().Get(0)
			var err error
			jobs, err = readCrontabFile(fileName)
			if err != nil {
				return cli.NewExitError("error: read crontab file failed", 1)
			}
		}

		if len(jobs) == 0 {
			fmt.Println("Probably nothing to do. no cron tasks.")
			return nil
		}

		for _, job := range jobs {
			from := timeCondition.from
			to := timeCondition.to
			cond := from
			for cond.Before(to) {
				ok, job := job.match(cond)
				if ok {
					fmt.Printf("%v : %s\n", job.schedule.Format("2006/01/02 15:04"), job.line)
					break
				}
				cond = cond.Add(time.Duration(1) * time.Minute)
			}
		}

		return nil
	}

	app.Run(os.Args)
}

// ErrCrontabFile はcrontabファイルが不正だった場合に発生するエラー
var ErrCrontabFile = errors.New("crontab file is something wrong")

func readCrontabFile(fileName string) ([]CronJob, error) {
	_, err := os.Stat(fileName)
	if err != nil {
		return nil, ErrCrontabFile
	}
	fp, err := os.Open(fileName)
	if err != nil {
		return nil, ErrCrontabFile
	}
	defer fp.Close()

	jobs := make([]CronJob, 0)
	scanner := bufio.NewScanner(fp)
	for scanner.Scan() {
		line := scanner.Text()
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		jobs = append(jobs, Parse(line))
	}

	return jobs, nil
}

// 実行されるcronの検索範囲を保持する構造体
type searchRange struct {
	from time.Time
	to   time.Time
}

func (s searchRange) String() string {
	const format = "2006/01/02 15:04"
	return fmt.Sprintf("from: %s - to: %s", s.from.Format(format), s.to.Format(format))
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/fukata/golang-stats-api-handler"
	"github.com/gorilla/websocket"
	"github.com/masayukioguni/go-lgtm-front/config"
	"github.com/masayukioguni/go-lgtm-front/templates"
	"github.com/masayukioguni/go-lgtm-model"
	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web/middleware"
	"log"
	"net"
	"net/http"
	"os"
	"path"
	"sync"
	"text/template"
	"time"
)

// https://github.com/patcito/martini-gorilla-websocket-chat-example/blob/master/server.go
var ActiveClients = make(map[ClientConn]int)
var ActiveClientsRWMutex sync.RWMutex

type ClientConn struct {
	websocket *websocket.Conn
	clientIP  net.Addr
}

func addClient(cc ClientConn) {
	ActiveClientsRWMutex.Lock()
	ActiveClients[cc] = 0
	ActiveClientsRWMutex.Unlock()
}

func deleteClient(cc ClientConn) {
	ActiveClientsRWMutex.Lock()
	delete(ActiveClients, cc)
	ActiveClientsRWMutex.Unlock()
}

func broadcastMessage(result model.Image) {
	ActiveClientsRWMutex.RLock()
	defer ActiveClientsRWMutex.RUnlock()

	for client, _ := range ActiveClients {
		if err := client.websocket.WriteJSON(result); err != nil {
			return
		}
	}
}

type ImageChannel struct {
	Name string
}

type Front struct {
	ImageChannel chan *ImageChannel
	config       *config.Config
}

func main() {
	c, err := config.NewConfig(".env")
	if err != nil {
		log.Panic(err)
	}

	f := &Front{
		config: c,
	}

	LogPath := os.Getenv("LOG_PATH")

	file, err := os.OpenFile(LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		log.Fatal("Failed to os.OpenFile()")
	}
	defer file.Close()

	log.SetOutput(file)

	f.ImageChannel = make(chan *ImageChannel)

	goji.Use(middleware.Recoverer)
	goji.Use(middleware.NoCache)

	goji.Get("/assets/*", http.StripPrefix("/assets/", http.FileServer(http.Dir("assets"))))

	goji.Get("/", f.Index)
	goji.Get("/stats", stats_api.Handler)

	goji.Post("/command/image", f.CommandImage)
	goji.Get("/ws", f.WebSocket)

	goji.Serve()
}

type IndexView struct {
	WebSocketUrl string
	Names        []string
}

func (f *Front) Index(w http.ResponseWriter, r *http.Request) {
	s, _ := model.NewStore(f.config.MongoHost, f.config.MongoDatabase, f.config.MongoCollectionName)

	items, _ := s.All()
	names := []string{}

	for _, item := range items {
		names = append(names, path.Join(f.config.S3Url, item.Name))
	}

	indexView := &IndexView{
		WebSocketUrl: f.config.WebSocketUrl,
		Names:        names,
	}

	t := template.New("index")
	tmpl, _ := templates.Asset("assets/templates/index.tmpl")
	t, _ = t.Parse(string(tmpl))

	//t := template.Must(template.ParseFile("templates/index.tmpl"))
	err := t.Execute(w, indexView)
	if err != nil {
		log.Printf("template execution: %s\n", err)
	}

}

func (f *Front) CommandImage(w http.ResponseWriter, r *http.Request) {
	name := r.FormValue("Name")
	c := &ImageChannel{
		Name: name,
	}
	f.ImageChannel <- c
}

func (f *Front) WebSocket(w http.ResponseWriter, r *http.Request) {
	ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
	if _, ok := err.(websocket.HandshakeError); ok {
		http.Error(w, "Not a websocket handshae", 400)
		return
	} else if err != nil {
		log.Println(err)
		return
	}

	client := ws.RemoteAddr()
	clientConn := ClientConn{ws, client}
	addClient(clientConn)

	for {
		c := <-f.ImageChannel
		name := c.Name
		result := model.Image{
			Name: path.Join(f.config.S3Url, name),
		}

		time.Sleep(1000 * time.Millisecond)
		broadcastMessage(result)
	}

}
<commit_msg>update assets path<commit_after>package main

import (
	"github.com/fukata/golang-stats-api-handler"
	"github.com/gorilla/websocket"
	"github.com/masayukioguni/go-lgtm-front/config"
	"github.com/masayukioguni/go-lgtm-front/templates"
	"github.com/masayukioguni/go-lgtm-model"
	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web/middleware"
	"log"
	"net"
	"net/http"
	"os"
	"path"
	"sync"
	"text/template"
	"time"
)

// https://github.com/patcito/martini-gorilla-websocket-chat-example/blob/master/server.go
var ActiveClients = make(map[ClientConn]int)
var ActiveClientsRWMutex sync.RWMutex

type ClientConn struct {
	websocket *websocket.Conn
	clientIP  net.Addr
}

func addClient(cc ClientConn) {
	ActiveClientsRWMutex.Lock()
	ActiveClients[cc] = 0
	ActiveClientsRWMutex.Unlock()
}

func deleteClient(cc ClientConn) {
	ActiveClientsRWMutex.Lock()
	delete(ActiveClients, cc)
	ActiveClientsRWMutex.Unlock()
}

func broadcastMessage(result model.Image) {
	ActiveClientsRWMutex.RLock()
	defer ActiveClientsRWMutex.RUnlock()

	for client, _ := range ActiveClients {
		if err := client.websocket.WriteJSON(result); err != nil {
			return
		}
	}
}

type ImageChannel struct {
	Name string
}

type Front struct {
	ImageChannel chan *ImageChannel
	config       *config.Config
}

func main() {
	c, err := config.NewConfig(".env")
	if err != nil {
		log.Panic(err)
	}

	f := &Front{
		config: c,
	}

	LogPath := os.Getenv("LOG_PATH")

	file, err := os.OpenFile(LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		log.Fatal("Failed to os.OpenFile()")
	}
	defer file.Close()

	log.SetOutput(file)

	f.ImageChannel = make(chan *ImageChannel)

	goji.Use(middleware.Recoverer)
	goji.Use(middleware.NoCache)

	goji.Get("/assets/*", http.StripPrefix("/assets/", http.FileServer(http.Dir("./assets"))))

	goji.Get("/", f.Index)
	goji.Get("/stats", stats_api.Handler)

	goji.Post("/command/image", f.CommandImage)
	goji.Get("/ws", f.WebSocket)

	goji.Serve()
}

type IndexView struct {
	WebSocketUrl string
	Names        []string
}

func (f *Front) Index(w http.ResponseWriter, r *http.Request) {
	s, _ := model.NewStore(f.config.MongoHost, f.config.MongoDatabase, f.config.MongoCollectionName)

	items, _ := s.All()
	names := []string{}

	for _, item := range items {
		names = append(names, path.Join(f.config.S3Url, item.Name))
	}

	indexView := &IndexView{
		WebSocketUrl: f.config.WebSocketUrl,
		Names:        names,
	}

	t := template.New("index")
	tmpl, _ := templates.Asset("assets/templates/index.tmpl")
	t, _ = t.Parse(string(tmpl))

	//t := template.Must(template.ParseFile("templates/index.tmpl"))
	err := t.Execute(w, indexView)
	if err != nil {
		log.Printf("template execution: %s\n", err)
	}

}

func (f *Front) CommandImage(w http.ResponseWriter, r *http.Request) {
	name := r.FormValue("Name")
	c := &ImageChannel{
		Name: name,
	}
	f.ImageChannel <- c
}

func (f *Front) WebSocket(w http.ResponseWriter, r *http.Request) {
	ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
	if _, ok := err.(websocket.HandshakeError); ok {
		http.Error(w, "Not a websocket handshae", 400)
		return
	} else if err != nil {
		log.Println(err)
		return
	}

	client := ws.RemoteAddr()
	clientConn := ClientConn{ws, client}
	addClient(clientConn)

	for {
		c := <-f.ImageChannel
		name := c.Name
		result := model.Image{
			Name: path.Join(f.config.S3Url, name),
		}

		time.Sleep(1000 * time.Millisecond)
		broadcastMessage(result)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"strings"

	"github.com/42wim/matterircd/config"
	irckit "github.com/42wim/matterircd/mm-go-irckit"
	"github.com/google/gops/agent"
	"github.com/sirupsen/logrus"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

var (
	version = "0.23.1"
	githash string
	logger  *logrus.Entry
	v       *viper.Viper
)

func main() {
	ourlog := logrus.New()
	ourlog.Formatter = &logrus.TextFormatter{FullTimestamp: true}
	logger = ourlog.WithFields(logrus.Fields{"module": "matterircd"})
	config.Logger = logger

	// config related. instantiate a new config.Config to store flags
	flagConfig := flag.String("conf", "matterircd.toml", "config file")

	// bools for showing version/enabling debug
	flag.Bool("version", false, "show version")
	flag.Bool("debug", false, "enable debug logging")

	// bind related cfg
	flag.String("bind", "127.0.0.1:6667", "interface:port to bind to, or a path to bind to a Unix socket.")

	// TLS related cfg
	flag.String("tlsbind", "", "interface:port to bind to. (e.g 127.0.0.1:6697)")
	flag.String("tlsdir", ".", "directory to look for key.pem and cert.pem.")

	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
	pflag.Parse()

	// Attempt to load values from the config file
	if _, err := os.Stat(*flagConfig); err == nil {
		v, err = config.LoadConfig(*flagConfig)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		v = viper.New()
	}

	v.BindPFlags(pflag.CommandLine)

	if v.GetBool("debug") {
		logger.Info("enabling debug")
		ourlog.Level = logrus.DebugLevel
		irckit.SetLogLevel("debug")
	}

	if v.GetBool("trace") {
		logger.Info("enabling trace")
		ourlog.Level = logrus.TraceLevel
		irckit.SetLogLevel("trace")
	}

	if v.GetBool("gops") {
		if err := agent.Listen(agent.Options{}); err != nil {
			log.Fatal(err)
		}
	}

	if v.GetBool("version") {
		fmt.Printf("version: %s %s\n", version, githash)
		return
	}

	irckit.SetLogger(logger)

	logger.Infof("Running version %s %s", version, githash)
	if strings.Contains(version, "-dev") {
		logger.Infof("WARNING: THIS IS A DEVELOPMENT VERSION. Things may break.")
	}

	if v.GetString("tlsbind") != "" {
		go func() {
			logger.Infof("Listening on %s (TLS)", v.GetString("tlsbind"))
			socket := tlsbind()
			defer socket.Close()
			start(socket)
		}()
	}

	// backwards compatible

	if v.GetString("bind") != "" {
		go func() {
			var network string
			if strings.ContainsRune(v.GetString("bind"), os.PathSeparator) {
				network = "unix"
			} else {
				network = "tcp"
			}

			socket, err := net.Listen(network, v.GetString("bind"))
			if err != nil {
				logger.Errorf("Can not listen on %s: %v", v.GetString("bind"), err)
				os.Exit(1)
			}

			logger.Infof("Listening on %s", v.GetString("bind"))

			defer socket.Close()
			start(socket)
		}()
	}

	select {}
}

func tlsbind() net.Listener {
	certPath := v.GetString("tlsdir") + "/cert.pem"
	keyPath := v.GetString("tlsdir") + "/key.pem"

	if v.GetString("tlscert") != "" {
		certPath = v.GetString("tlscert")
	}

	if v.GetString("tlskey") != "" {
		keyPath = v.GetString("tlskey")
	}

	kpr, err := NewKeypairReloader(certPath, keyPath)
	if err != nil {
		logger.Errorf("could not load TLS, incorrect directory? Error: %s", err)
		os.Exit(1)
	}

	tlsConfig := tls.Config{
		GetCertificate: kpr.GetCertificateFunc(),
	}

	listenerTLS, err := tls.Listen("tcp", v.GetString("tlsbind"), &tlsConfig)
	if err != nil {
		logger.Errorf("Can not listen on %s: %v\n", v.GetString("tlsbind"), err)
		os.Exit(1)
	}

	logger.Info("TLS listening on ", v.GetString("tlsbind"))

	return listenerTLS
}

func start(socket net.Listener) {
	for {
		conn, err := socket.Accept()
		if err != nil {
			logger.Errorf("Failed to accept connection: %v", err)
			return
		}

		go func() {
			newsrv := irckit.ServerConfig{Name: "matterircd", Version: version}.Server()

			logger.Infof("New connection: %s", conn.RemoteAddr())

			user := irckit.NewUserBridge(conn, newsrv, v)
			err = newsrv.Connect(user)
			if err != nil {
				logger.Errorf("Failed to join: %v", err)
				return
			}
		}()
	}
}
<commit_msg>Bump version<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"strings"

	"github.com/42wim/matterircd/config"
	irckit "github.com/42wim/matterircd/mm-go-irckit"
	"github.com/google/gops/agent"
	"github.com/sirupsen/logrus"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

var (
	version = "0.23.2-dev"
	githash string
	logger  *logrus.Entry
	v       *viper.Viper
)

func main() {
	ourlog := logrus.New()
	ourlog.Formatter = &logrus.TextFormatter{FullTimestamp: true}
	logger = ourlog.WithFields(logrus.Fields{"module": "matterircd"})
	config.Logger = logger

	// config related. instantiate a new config.Config to store flags
	flagConfig := flag.String("conf", "matterircd.toml", "config file")

	// bools for showing version/enabling debug
	flag.Bool("version", false, "show version")
	flag.Bool("debug", false, "enable debug logging")

	// bind related cfg
	flag.String("bind", "127.0.0.1:6667", "interface:port to bind to, or a path to bind to a Unix socket.")

	// TLS related cfg
	flag.String("tlsbind", "", "interface:port to bind to. (e.g 127.0.0.1:6697)")
	flag.String("tlsdir", ".", "directory to look for key.pem and cert.pem.")

	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
	pflag.Parse()

	// Attempt to load values from the config file
	if _, err := os.Stat(*flagConfig); err == nil {
		v, err = config.LoadConfig(*flagConfig)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		v = viper.New()
	}

	v.BindPFlags(pflag.CommandLine)

	if v.GetBool("debug") {
		logger.Info("enabling debug")
		ourlog.Level = logrus.DebugLevel
		irckit.SetLogLevel("debug")
	}

	if v.GetBool("trace") {
		logger.Info("enabling trace")
		ourlog.Level = logrus.TraceLevel
		irckit.SetLogLevel("trace")
	}

	if v.GetBool("gops") {
		if err := agent.Listen(agent.Options{}); err != nil {
			log.Fatal(err)
		}
	}

	if v.GetBool("version") {
		fmt.Printf("version: %s %s\n", version, githash)
		return
	}

	irckit.SetLogger(logger)

	logger.Infof("Running version %s %s", version, githash)
	if strings.Contains(version, "-dev") {
		logger.Infof("WARNING: THIS IS A DEVELOPMENT VERSION. Things may break.")
	}

	if v.GetString("tlsbind") != "" {
		go func() {
			logger.Infof("Listening on %s (TLS)", v.GetString("tlsbind"))
			socket := tlsbind()
			defer socket.Close()
			start(socket)
		}()
	}

	// backwards compatible

	if v.GetString("bind") != "" {
		go func() {
			var network string
			if strings.ContainsRune(v.GetString("bind"), os.PathSeparator) {
				network = "unix"
			} else {
				network = "tcp"
			}

			socket, err := net.Listen(network, v.GetString("bind"))
			if err != nil {
				logger.Errorf("Can not listen on %s: %v", v.GetString("bind"), err)
				os.Exit(1)
			}

			logger.Infof("Listening on %s", v.GetString("bind"))

			defer socket.Close()
			start(socket)
		}()
	}

	select {}
}

func tlsbind() net.Listener {
	certPath := v.GetString("tlsdir") + "/cert.pem"
	keyPath := v.GetString("tlsdir") + "/key.pem"

	if v.GetString("tlscert") != "" {
		certPath = v.GetString("tlscert")
	}

	if v.GetString("tlskey") != "" {
		keyPath = v.GetString("tlskey")
	}

	kpr, err := NewKeypairReloader(certPath, keyPath)
	if err != nil {
		logger.Errorf("could not load TLS, incorrect directory? Error: %s", err)
		os.Exit(1)
	}

	tlsConfig := tls.Config{
		GetCertificate: kpr.GetCertificateFunc(),
	}

	listenerTLS, err := tls.Listen("tcp", v.GetString("tlsbind"), &tlsConfig)
	if err != nil {
		logger.Errorf("Can not listen on %s: %v\n", v.GetString("tlsbind"), err)
		os.Exit(1)
	}

	logger.Info("TLS listening on ", v.GetString("tlsbind"))

	return listenerTLS
}

func start(socket net.Listener) {
	for {
		conn, err := socket.Accept()
		if err != nil {
			logger.Errorf("Failed to accept connection: %v", err)
			return
		}

		go func() {
			newsrv := irckit.ServerConfig{Name: "matterircd", Version: version}.Server()

			logger.Infof("New connection: %s", conn.RemoteAddr())

			user := irckit.NewUserBridge(conn, newsrv, v)
			err = newsrv.Connect(user)
			if err != nil {
				logger.Errorf("Failed to join: %v", err)
				return
			}
		}()
	}
}
<|endoftext|>


<commit_before>/* httpstress-go is a CLI interface for httpstress library.
Use it for stress testing of HTTP servers with many concurrent connections.

Usage: httpstress-go -c {concurrent} -n {total} {URL list}
e.g. httpstress-go -c 1000 -n 2000 http://localhost http://google.com

{concurrent} defaults to 1, {total} is optional.

Returns 0 if no errors, 1 if some errors (see stdout), 2 on kill and 3 in case of invalid options.

Prints error count for each URL to stdout (does not count successful attempts).

Please note that this utility uses GOMAXPROCS environment variable if it's present.
If not, this defaults to CPU count + 1. */
package main

/* Copyright 2014 Chai Chillum

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

import (
	"flag"
	"fmt"
	"github.com/chillum/httpstress"
	"os"
	"runtime"
	"syscall"
)

func main() {
	var conn, max int
	flag.IntVar(&conn, "c", 1, "concurrent connections count")
	flag.IntVar(&max, "n", 0, "total connections (optional)")
	flag.Parse()

	urls := flag.Args()
	if len(urls) < 1 {
		fmt.Println("Usage:", os.Args[0], "<http://url1> [http://url2] ... [http://urlN]")
		os.Exit(3)
	}

	if os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(runtime.NumCPU() + 1)
	}

	// Set Unix limits. Should be a no-op on Windows.
	var rLimit syscall.Rlimit
	rLimit.Cur = uint64(conn + 6) // Magic. 1-5 does not work, 6 seems OK.
	rLimit.Max = rLimit.Cur
	syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)

	out, err := httpstress.Test(conn, max, urls)
	if err != nil {
		fmt.Println("ERROR:", err)
		os.Exit(3)
	}
	if len(out) > 0 {
		fmt.Println("Test finished. Failed requests:")
		for url, num := range out {
			fmt.Print(" ", url, ": ", num, "\n")
		}
		os.Exit(1)
	} else {
		fmt.Println("Test finished. No failed requests.")
	}
}
<commit_msg>Godoc update<commit_after>/* httpstress-go is a CLI interface for httpstress library.
Use it for stress testing of HTTP servers with many concurrent connections.

Usage: httpstress-go -c {concurrent} -n {total} {URL list}
e.g. httpstress-go -c 1000 -n 2000 http://localhost http://google.com

{concurrent} defaults to 1, {total} is optional.

Returns 0 if no errors, 1 if some errors (see stdout), 2 on kill and 3 in case of invalid options.

Prints error count for each URL to stdout (does not count successful attempts).

Please note that this utility uses GOMAXPROCS environment variable if it's present.
If not, this defaults to CPU count + 1.

Also this ulility sets `ulimit -n` on Unix systems (Mac OS X, Linux, FreeBSD).
*/
package main

/* Copyright 2014 Chai Chillum

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

import (
	"flag"
	"fmt"
	"github.com/chillum/httpstress"
	"os"
	"runtime"
	"syscall"
)

func main() {
	var conn, max int
	flag.IntVar(&conn, "c", 1, "concurrent connections count")
	flag.IntVar(&max, "n", 0, "total connections (optional)")
	flag.Parse()

	urls := flag.Args()
	if len(urls) < 1 {
		fmt.Println("Usage:", os.Args[0], "<http://url1> [http://url2] ... [http://urlN]")
		os.Exit(3)
	}

	if os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(runtime.NumCPU() + 1)
	}

	// Set Unix limits. Should be a no-op on Windows.
	var rLimit syscall.Rlimit
	rLimit.Cur = uint64(conn + 6) // Magic. 1-5 does not work, 6 seems OK.
	rLimit.Max = rLimit.Cur
	syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)

	out, err := httpstress.Test(conn, max, urls)
	if err != nil {
		fmt.Println("ERROR:", err)
		os.Exit(3)
	}
	if len(out) > 0 {
		fmt.Println("Test finished. Failed requests:")
		for url, num := range out {
			fmt.Print(" ", url, ": ", num, "\n")
		}
		os.Exit(1)
	} else {
		fmt.Println("Test finished. No failed requests.")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
)

var (
	envPath  = os.Getenv("PATH")
	envLevel = os.Getenv("LF_LEVEL")
)

type arrayFlag []string

var (
	gSingleMode    bool
	gClientID      int
	gHostname      string
	gLastDirPath   string
	gSelectionPath string
	gSocketProt    string
	gSocketPath    string
	gLogPath       string
	gSelect        string
	gConfigPath    string
	gCommands      arrayFlag
	gVersion       string
)

func (a *arrayFlag) Set(v string) error {
	*a = append(*a, v)
	return nil
}

func (a *arrayFlag) String() string {
	return strings.Join(*a, ", ")
}

func init() {
	h, err := os.Hostname()
	if err != nil {
		log.Printf("hostname: %s", err)
	}
	gHostname = h

	if envLevel == "" {
		envLevel = "0"
	}
}

func exportEnvVars() {
	os.Setenv("id", strconv.Itoa(gClientID))

	os.Setenv("OPENER", envOpener)
	os.Setenv("EDITOR", envEditor)
	os.Setenv("PAGER", envPager)
	os.Setenv("SHELL", envShell)

	dir, err := os.Getwd()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
	}
	os.Setenv("OLDPWD", dir)

	level, err := strconv.Atoi(envLevel)
	if err != nil {
		log.Printf("reading lf level: %s", err)
	}

	level++

	os.Setenv("LF_LEVEL", strconv.Itoa(level))
}

// used by exportOpts below
func fieldToString(field reflect.Value) string {
	kind := field.Kind()
	var value string

	switch kind {
	case reflect.Int:
		value = strconv.Itoa(int(field.Int()))
	case reflect.Bool:
		value = strconv.FormatBool(field.Bool())
	case reflect.Slice:
		for i := 0; i < field.Len(); i++ {
			element := field.Index(i)

			if i == 0 {
				value = fieldToString(element)
			} else {
				value += ":" + fieldToString(element)
			}
		}
	default:
		value = field.String()
	}

	return value
}

func exportOpts() {
	e := reflect.ValueOf(&gOpts).Elem()

	for i := 0; i < e.NumField(); i++ {
		// Get name and prefix it with lf_
		name := e.Type().Field(i).Name
		name = fmt.Sprintf("lf_%s", name)

		// Skip maps
		if name == "lf_keys" || name == "lf_cmdkeys" || name == "lf_cmds" {
			continue
		}

		// Get string representation of the value
		if name == "lf_sortType" {
			var sortby string

			switch gOpts.sortType.method {
			case naturalSort:
				sortby = "natural"
			case nameSort:
				sortby = "name"
			case sizeSort:
				sortby = "size"
			case timeSort:
				sortby = "time"
			case ctimeSort:
				sortby = "ctime"
			case atimeSort:
				sortby = "atime"
			case extSort:
				sortby = "ext"
			}

			os.Setenv("lf_sortby", sortby)

			reverse := strconv.FormatBool(gOpts.sortType.option&reverseSort != 0)
			os.Setenv("lf_reverse", reverse)

			hidden := strconv.FormatBool(gOpts.sortType.option&hiddenSort != 0)
			os.Setenv("lf_hidden", hidden)

			dirfirst := strconv.FormatBool(gOpts.sortType.option&dirfirstSort != 0)
			os.Setenv("lf_dirfirst", dirfirst)
		} else {
			field := e.Field(i)
			value := fieldToString(field)

			os.Setenv(name, value)
		}
	}
}

func startServer() {
	cmd := detachedCommand(os.Args[0], "-server")
	if err := cmd.Start(); err != nil {
		log.Printf("starting server: %s", err)
	}
}

func checkServer() {
	if gSocketProt == "unix" {
		if _, err := os.Stat(gSocketPath); os.IsNotExist(err) {
			startServer()
		} else if _, err := net.Dial(gSocketProt, gSocketPath); err != nil {
			os.Remove(gSocketPath)
			startServer()
		}
	} else {
		if _, err := net.Dial(gSocketProt, gSocketPath); err != nil {
			startServer()
		}
	}
}

func main() {
	flag.Usage = func() {
		f := flag.CommandLine.Output()
		fmt.Fprintln(f, "lf - Terminal file manager")
		fmt.Fprintln(f, "")
		fmt.Fprintf(f, "Usage:  %s [options] [select-path]\n\n", os.Args[0])
		fmt.Fprintln(f, "  select-path")
		fmt.Fprintln(f, "        set the initial file selection to the given argument")
		fmt.Fprintln(f, "")
		fmt.Fprintln(f, "Options:")
		flag.PrintDefaults()
	}

	showDoc := flag.Bool(
		"doc",
		false,
		"show documentation")

	showVersion := flag.Bool(
		"version",
		false,
		"show version")

	serverMode := flag.Bool(
		"server",
		false,
		"start server (automatic)")

	singleMode := flag.Bool(
		"single",
		false,
		"start a client without server")

	remoteCmd := flag.String(
		"remote",
		"",
		"send remote command to server")

	cpuprofile := flag.String(
		"cpuprofile",
		"",
		"path to the file to write the CPU profile")

	memprofile := flag.String(
		"memprofile",
		"",
		"path to the file to write the memory profile")

	flag.StringVar(&gLastDirPath,
		"last-dir-path",
		"",
		"path to the file to write the last dir on exit (to use for cd)")

	flag.StringVar(&gSelectionPath,
		"selection-path",
		"",
		"path to the file to write selected files on open (to use as open file dialog)")

	flag.StringVar(&gConfigPath,
		"config",
		"",
		"path to the config file (instead of the usual paths)")

	flag.Var(&gCommands,
		"command",
		"command to execute on client initialization")

	flag.StringVar(&gLogPath,
		"log",
		"",
		"path to the log file to write messages")

	flag.Parse()

	gSocketProt = gDefaultSocketProt
	gSocketPath = gDefaultSocketPath

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatalf("could not create CPU profile: %s", err)
		}
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatalf("could not start CPU profile: %s", err)
		}
		defer pprof.StopCPUProfile()
	}

	switch {
	case *showDoc:
		fmt.Print(genDocString)
	case *showVersion:
		fmt.Println(gVersion)
	case *remoteCmd != "":
		if err := remote(*remoteCmd); err != nil {
			log.Fatalf("remote command: %s", err)
		}
	case *serverMode:
		if gLogPath != "" && !filepath.IsAbs(gLogPath) {
			wd, err := os.Getwd()
			if err != nil {
				log.Fatalf("getting current directory: %s", err)
			} else {
				gLogPath = filepath.Join(wd, gLogPath)
			}
		}
		os.Chdir(gUser.HomeDir)
		serve()
	default:
		gSingleMode = *singleMode

		if !gSingleMode {
			checkServer()
		}

		gClientID = os.Getpid()

		switch flag.NArg() {
		case 0:
			_, err := os.Getwd()
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s\n", err)
				os.Exit(2)
			}
		case 1:
			gSelect = flag.Arg(0)
		default:
			fmt.Fprintf(os.Stderr, "only single file or directory is allowed\n")
			os.Exit(2)
		}

		exportEnvVars()

		run()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal("could not create memory profile: ", err)
		}
		runtime.GC()
		if err := pprof.WriteHeapProfile(f); err != nil {
			log.Fatal("could not write memory profile: ", err)
		}
		f.Close()
	}
}
<commit_msg>add missing error contexts for getwd calls<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
)

var (
	envPath  = os.Getenv("PATH")
	envLevel = os.Getenv("LF_LEVEL")
)

type arrayFlag []string

var (
	gSingleMode    bool
	gClientID      int
	gHostname      string
	gLastDirPath   string
	gSelectionPath string
	gSocketProt    string
	gSocketPath    string
	gLogPath       string
	gSelect        string
	gConfigPath    string
	gCommands      arrayFlag
	gVersion       string
)

func (a *arrayFlag) Set(v string) error {
	*a = append(*a, v)
	return nil
}

func (a *arrayFlag) String() string {
	return strings.Join(*a, ", ")
}

func init() {
	h, err := os.Hostname()
	if err != nil {
		log.Printf("hostname: %s", err)
	}
	gHostname = h

	if envLevel == "" {
		envLevel = "0"
	}
}

func exportEnvVars() {
	os.Setenv("id", strconv.Itoa(gClientID))

	os.Setenv("OPENER", envOpener)
	os.Setenv("EDITOR", envEditor)
	os.Setenv("PAGER", envPager)
	os.Setenv("SHELL", envShell)

	dir, err := os.Getwd()
	if err != nil {
		fmt.Fprintf(os.Stderr, "getting current directory: %s\n", err)
	}
	os.Setenv("OLDPWD", dir)

	level, err := strconv.Atoi(envLevel)
	if err != nil {
		log.Printf("reading lf level: %s", err)
	}

	level++

	os.Setenv("LF_LEVEL", strconv.Itoa(level))
}

// used by exportOpts below
func fieldToString(field reflect.Value) string {
	kind := field.Kind()
	var value string

	switch kind {
	case reflect.Int:
		value = strconv.Itoa(int(field.Int()))
	case reflect.Bool:
		value = strconv.FormatBool(field.Bool())
	case reflect.Slice:
		for i := 0; i < field.Len(); i++ {
			element := field.Index(i)

			if i == 0 {
				value = fieldToString(element)
			} else {
				value += ":" + fieldToString(element)
			}
		}
	default:
		value = field.String()
	}

	return value
}

func exportOpts() {
	e := reflect.ValueOf(&gOpts).Elem()

	for i := 0; i < e.NumField(); i++ {
		// Get name and prefix it with lf_
		name := e.Type().Field(i).Name
		name = fmt.Sprintf("lf_%s", name)

		// Skip maps
		if name == "lf_keys" || name == "lf_cmdkeys" || name == "lf_cmds" {
			continue
		}

		// Get string representation of the value
		if name == "lf_sortType" {
			var sortby string

			switch gOpts.sortType.method {
			case naturalSort:
				sortby = "natural"
			case nameSort:
				sortby = "name"
			case sizeSort:
				sortby = "size"
			case timeSort:
				sortby = "time"
			case ctimeSort:
				sortby = "ctime"
			case atimeSort:
				sortby = "atime"
			case extSort:
				sortby = "ext"
			}

			os.Setenv("lf_sortby", sortby)

			reverse := strconv.FormatBool(gOpts.sortType.option&reverseSort != 0)
			os.Setenv("lf_reverse", reverse)

			hidden := strconv.FormatBool(gOpts.sortType.option&hiddenSort != 0)
			os.Setenv("lf_hidden", hidden)

			dirfirst := strconv.FormatBool(gOpts.sortType.option&dirfirstSort != 0)
			os.Setenv("lf_dirfirst", dirfirst)
		} else {
			field := e.Field(i)
			value := fieldToString(field)

			os.Setenv(name, value)
		}
	}
}

func startServer() {
	cmd := detachedCommand(os.Args[0], "-server")
	if err := cmd.Start(); err != nil {
		log.Printf("starting server: %s", err)
	}
}

func checkServer() {
	if gSocketProt == "unix" {
		if _, err := os.Stat(gSocketPath); os.IsNotExist(err) {
			startServer()
		} else if _, err := net.Dial(gSocketProt, gSocketPath); err != nil {
			os.Remove(gSocketPath)
			startServer()
		}
	} else {
		if _, err := net.Dial(gSocketProt, gSocketPath); err != nil {
			startServer()
		}
	}
}

func main() {
	flag.Usage = func() {
		f := flag.CommandLine.Output()
		fmt.Fprintln(f, "lf - Terminal file manager")
		fmt.Fprintln(f, "")
		fmt.Fprintf(f, "Usage:  %s [options] [select-path]\n\n", os.Args[0])
		fmt.Fprintln(f, "  select-path")
		fmt.Fprintln(f, "        set the initial file selection to the given argument")
		fmt.Fprintln(f, "")
		fmt.Fprintln(f, "Options:")
		flag.PrintDefaults()
	}

	showDoc := flag.Bool(
		"doc",
		false,
		"show documentation")

	showVersion := flag.Bool(
		"version",
		false,
		"show version")

	serverMode := flag.Bool(
		"server",
		false,
		"start server (automatic)")

	singleMode := flag.Bool(
		"single",
		false,
		"start a client without server")

	remoteCmd := flag.String(
		"remote",
		"",
		"send remote command to server")

	cpuprofile := flag.String(
		"cpuprofile",
		"",
		"path to the file to write the CPU profile")

	memprofile := flag.String(
		"memprofile",
		"",
		"path to the file to write the memory profile")

	flag.StringVar(&gLastDirPath,
		"last-dir-path",
		"",
		"path to the file to write the last dir on exit (to use for cd)")

	flag.StringVar(&gSelectionPath,
		"selection-path",
		"",
		"path to the file to write selected files on open (to use as open file dialog)")

	flag.StringVar(&gConfigPath,
		"config",
		"",
		"path to the config file (instead of the usual paths)")

	flag.Var(&gCommands,
		"command",
		"command to execute on client initialization")

	flag.StringVar(&gLogPath,
		"log",
		"",
		"path to the log file to write messages")

	flag.Parse()

	gSocketProt = gDefaultSocketProt
	gSocketPath = gDefaultSocketPath

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatalf("could not create CPU profile: %s", err)
		}
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatalf("could not start CPU profile: %s", err)
		}
		defer pprof.StopCPUProfile()
	}

	switch {
	case *showDoc:
		fmt.Print(genDocString)
	case *showVersion:
		fmt.Println(gVersion)
	case *remoteCmd != "":
		if err := remote(*remoteCmd); err != nil {
			log.Fatalf("remote command: %s", err)
		}
	case *serverMode:
		if gLogPath != "" && !filepath.IsAbs(gLogPath) {
			wd, err := os.Getwd()
			if err != nil {
				log.Fatalf("getting current directory: %s", err)
			} else {
				gLogPath = filepath.Join(wd, gLogPath)
			}
		}
		os.Chdir(gUser.HomeDir)
		serve()
	default:
		gSingleMode = *singleMode

		if !gSingleMode {
			checkServer()
		}

		gClientID = os.Getpid()

		switch flag.NArg() {
		case 0:
			_, err := os.Getwd()
			if err != nil {
				fmt.Fprintf(os.Stderr, "getting current directory: %s\n", err)
				os.Exit(2)
			}
		case 1:
			gSelect = flag.Arg(0)
		default:
			fmt.Fprintf(os.Stderr, "only single file or directory is allowed\n")
			os.Exit(2)
		}

		exportEnvVars()

		run()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal("could not create memory profile: ", err)
		}
		runtime.GC()
		if err := pprof.WriteHeapProfile(f); err != nil {
			log.Fatal("could not write memory profile: ", err)
		}
		f.Close()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"

	"github.com/rs/cors"
	"github.com/urfave/negroni"

	"github.com/yuuki/diamondb/lib/config"
	"github.com/yuuki/diamondb/lib/env"
	"github.com/yuuki/diamondb/lib/log"
	"github.com/yuuki/diamondb/lib/storage"
	"github.com/yuuki/diamondb/lib/web"
)

// CLI is the command line object
type CLI struct {
	// outStream and errStream are the stdout and stderr
	// to write message from the CLI.
	outStream, errStream io.Writer
}

type Env struct {
	fetcher storage.Fetcher
}

func main() {
	cli := &CLI{outStream: os.Stdout, errStream: os.Stderr}
	os.Exit(cli.Run(os.Args))
}

// Run invokes the CLI with the given arguments.
func (cli *CLI) Run(args []string) int {
	if err := config.Load(); err != nil {
		log.Printf("Failed to load the config: %s", err)
		return 2
	}

	var (
		host    string
		port    string
		version bool
		debug   bool
	)

	flags := flag.NewFlagSet(Name, flag.ContinueOnError)
	flags.SetOutput(cli.errStream)
	flags.Usage = func() {
		fmt.Fprint(cli.errStream, helpText)
	}
	flags.StringVar(&host, "host", config.Config.Host, "")
	flags.StringVar(&host, "H", config.Config.Host, "")
	flags.StringVar(&port, "port", config.Config.Port, "")
	flags.StringVar(&port, "P", config.Config.Port, "")
	flags.BoolVar(&version, "version", false, "")
	flags.BoolVar(&version, "v", false, "")
	flags.BoolVar(&debug, "debug", config.Config.Debug, "")
	flags.BoolVar(&debug, "d", config.Config.Debug, "")

	if err := flags.Parse(args[1:]); err != nil {
		return 1
	}
	log.SetDebug(debug)

	if version {
		fmt.Fprintf(cli.errStream, "%s version %s, build %s \n", Name, Version, GitCommit)
		return 0
	}

	e := &env.Env{Fetcher: storage.NewStore()}

	mux := http.NewServeMux()
	mux.Handle("/render", web.RenderHandler(e))

	n := negroni.New()
	n.Use(negroni.NewRecovery())
	n.Use(negroni.NewLogger())
	n.Use(cors.New(cors.Options{
		AllowedOrigins: []string{"*"},
		AllowedMethods: []string{"GET", "POST"},
		AllowedHeaders: []string{"Origin", "Accept", "Content-Type"},
	}))
	n.UseHandler(mux)

	log.Printf("Listening %s:%s ...", host, port)
	if err := http.ListenAndServe(":"+port, n); err != nil {
		log.Println(err)
		return 3
	}

	return 0
}

var helpText = `
Usage: diamondb [options]

  A Reliable, Scalable, Cloud-Based Time Series Database.

Options:
  --host, -H           Bind host

  --port, -P           Listen port

  --debug, -d          Run with debug print
`
<commit_msg>Remove unused code<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"

	"github.com/rs/cors"
	"github.com/urfave/negroni"

	"github.com/yuuki/diamondb/lib/config"
	"github.com/yuuki/diamondb/lib/env"
	"github.com/yuuki/diamondb/lib/log"
	"github.com/yuuki/diamondb/lib/storage"
	"github.com/yuuki/diamondb/lib/web"
)

// CLI is the command line object
type CLI struct {
	// outStream and errStream are the stdout and stderr
	// to write message from the CLI.
	outStream, errStream io.Writer
}

func main() {
	cli := &CLI{outStream: os.Stdout, errStream: os.Stderr}
	os.Exit(cli.Run(os.Args))
}

// Run invokes the CLI with the given arguments.
func (cli *CLI) Run(args []string) int {
	if err := config.Load(); err != nil {
		log.Printf("Failed to load the config: %s", err)
		return 2
	}

	var (
		host    string
		port    string
		version bool
		debug   bool
	)

	flags := flag.NewFlagSet(Name, flag.ContinueOnError)
	flags.SetOutput(cli.errStream)
	flags.Usage = func() {
		fmt.Fprint(cli.errStream, helpText)
	}
	flags.StringVar(&host, "host", config.Config.Host, "")
	flags.StringVar(&host, "H", config.Config.Host, "")
	flags.StringVar(&port, "port", config.Config.Port, "")
	flags.StringVar(&port, "P", config.Config.Port, "")
	flags.BoolVar(&version, "version", false, "")
	flags.BoolVar(&version, "v", false, "")
	flags.BoolVar(&debug, "debug", config.Config.Debug, "")
	flags.BoolVar(&debug, "d", config.Config.Debug, "")

	if err := flags.Parse(args[1:]); err != nil {
		return 1
	}
	log.SetDebug(debug)

	if version {
		fmt.Fprintf(cli.errStream, "%s version %s, build %s \n", Name, Version, GitCommit)
		return 0
	}

	e := &env.Env{Fetcher: storage.NewStore()}

	mux := http.NewServeMux()
	mux.Handle("/render", web.RenderHandler(e))

	n := negroni.New()
	n.Use(negroni.NewRecovery())
	n.Use(negroni.NewLogger())
	n.Use(cors.New(cors.Options{
		AllowedOrigins: []string{"*"},
		AllowedMethods: []string{"GET", "POST"},
		AllowedHeaders: []string{"Origin", "Accept", "Content-Type"},
	}))
	n.UseHandler(mux)

	log.Printf("Listening %s:%s ...", host, port)
	if err := http.ListenAndServe(":"+port, n); err != nil {
		log.Println(err)
		return 3
	}

	return 0
}

var helpText = `
Usage: diamondb [options]

  A Reliable, Scalable, Cloud-Based Time Series Database.

Options:
  --host, -H           Bind host

  --port, -P           Listen port

  --debug, -d          Run with debug print
`
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"github.com/Shopify/sarama"
	"github.com/ugorji/go/codec"
	"os"
	"strings"
	"time"
)

var groupIdInput = flag.String("group-id", "", "Comma-separated consumer group ids")

var brokersInput = flag.String("brokers", "", "List of broker addresses with ports, comma separated")

var topicInput = flag.String("topic", "", "Topic name")

var commandInput = flag.String("command", "lag", "Command")

var msgpack codec.MsgpackHandle

type ConsumerOffset struct {
	Topic     string
	Offset    int64
	Partition int32
}

var brokers []string
var kafkaClient sarama.Client

func main() {
	flag.Parse()
	stat, _ := os.Stdin.Stat()
	if (stat.Mode() & os.ModeCharDevice) == 0 {
		processWithStdin()
	} else {
		processWithArgs()
	}
}

func processWithStdin() {
	setupClient()
	scanner := bufio.NewScanner(os.Stdin)
	var controlChannel = make(chan int)
	var lines int
	for scanner.Scan() {
		parts := strings.Split(scanner.Text(), " ")
		go func() {
			processGroup(*commandInput, parts[0], parts[1])
			controlChannel <- 1
		}()
		lines += 1
	}
	for i := 0; i < lines; i++ {
		_ = <-controlChannel
	}
}

func processWithArgs() {
	checkArgs()
	setupClient()
	processGroup(*commandInput, *groupIdInput, *topicInput)
}

func setupClient() {
	var err error
	if *brokersInput == "" {
		exit(errors.New("Brokers are not defined. Use --brokers argument to define it."))
	}
	brokers = strings.Split(*brokersInput, ",")
	kafkaClient, err = sarama.NewClient(brokers, nil)
	if err != nil {
		exit(err)
	}
}

func checkArgs() {
	if *topicInput == "" {
		exit(errors.New("Topic is not defined. Use --topic argument to define it."))
	}
	if *groupIdInput == "" {
		exit(errors.New("Consumer group is not defined. Use --group-id argument to define it."))
	}
}

func processGroup(command string, groupId string, topic string) {
	partitions, err := kafkaClient.Partitions(topic)
	if err != nil {
		exit(err)
	}
	offsetManager, err := sarama.NewOffsetManagerFromClient(groupId, kafkaClient)
	if err != nil {
		exit(err)
	}
	var lag int64
	var timestamps = make([]int64, len(partitions))
	var controlChannel = make(chan []int64)
	for _, partition := range partitions {
		go processPartition(command, topic, partition, controlChannel, offsetManager)
	}
	for _, partition := range partitions {
		response := <-controlChannel
		lag += response[0]
		if command == "lag_and_time" {
			timestamps[partition] = response[1]
		}
	}
	if command == "lag_and_time" {
		max := timestamps[0]
		for _, value := range timestamps {
			if value > max {
				max = value
			}
		}
		timelag := time.Now().Unix() - max
		fmt.Println(groupId, topic, lag, timelag)
	} else {
		fmt.Println(groupId, topic, lag)
	}
}

func processPartition(command string, topic string, partition int32, controlChannel chan []int64, offsetManager sarama.OffsetManager) {
	pom, err := offsetManager.ManagePartition(topic, int32(partition))
	if err != nil {
		exit(err)
	}
	consumerOffset, _ := pom.NextOffset()
	offset, err := kafkaClient.GetOffset(topic, int32(partition), sarama.OffsetNewest)
	//fmt.Println(topic, partition, consumerOffset, offset)
	if err != nil {
		exit(err)
	}
	var response = make([]int64, 0)
	var timelag int64
	lag := offset - consumerOffset + 1
	response = append(response, lag)
	if command == "lag_and_time" {
		broker, err := kafkaClient.Leader(topic, partition)
		if err != nil {
			exit(err)
		}
		fetchRequest := sarama.FetchRequest{MaxWaitTime: 10000, MinBytes: 0}
		fetchRequest.AddBlock(topic, partition, consumerOffset-2, 100000)
		fetchResponse, err := broker.Fetch(&fetchRequest)
		if err != nil {
			exit(err)
		}
		block := fetchResponse.GetBlock(topic, partition)
		messages := block.MsgSet.Messages
		if len(messages) > 0 {
			msg := messages[0].Messages()[0].Msg.Value
			var decodedData map[string]interface{}
			codec.NewDecoderBytes(msg, &msgpack).Decode(&decodedData)
			timestamp := decodedData["timestamp"]
			switch timestamp := timestamp.(type) {
			case uint64:
				timelag = int64(timestamp)
			default:
				fmt.Println(timestamp)
				exit(errors.New("message is missing timestamp"))
			}
		} else {
			timelag = 0
		}
		response = append(response, timelag)
	}
	controlChannel <- response
}

func exit(err error) {
	fmt.Println(err)
	os.Exit(1)
}
<commit_msg>Increase possible messages size.<commit_after>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"github.com/Shopify/sarama"
	"github.com/ugorji/go/codec"
	"os"
	"strings"
	"time"
)

var groupIdInput = flag.String("group-id", "", "Comma-separated consumer group ids")

var brokersInput = flag.String("brokers", "", "List of broker addresses with ports, comma separated")

var topicInput = flag.String("topic", "", "Topic name")

var commandInput = flag.String("command", "lag", "Command")

var msgpack codec.MsgpackHandle

type ConsumerOffset struct {
	Topic     string
	Offset    int64
	Partition int32
}

var brokers []string
var kafkaClient sarama.Client

func main() {
	flag.Parse()
	stat, _ := os.Stdin.Stat()
	if (stat.Mode() & os.ModeCharDevice) == 0 {
		processWithStdin()
	} else {
		processWithArgs()
	}
}

func processWithStdin() {
	setupClient()
	scanner := bufio.NewScanner(os.Stdin)
	var controlChannel = make(chan int)
	var lines int
	for scanner.Scan() {
		parts := strings.Split(scanner.Text(), " ")
		go func() {
			processGroup(*commandInput, parts[0], parts[1])
			controlChannel <- 1
		}()
		lines += 1
	}
	for i := 0; i < lines; i++ {
		_ = <-controlChannel
	}
}

func processWithArgs() {
	checkArgs()
	setupClient()
	processGroup(*commandInput, *groupIdInput, *topicInput)
}

func setupClient() {
	var err error
	if *brokersInput == "" {
		exit(errors.New("Brokers are not defined. Use --brokers argument to define it."))
	}
	brokers = strings.Split(*brokersInput, ",")
	kafkaClient, err = sarama.NewClient(brokers, nil)
	if err != nil {
		exit(err)
	}
}

func checkArgs() {
	if *topicInput == "" {
		exit(errors.New("Topic is not defined. Use --topic argument to define it."))
	}
	if *groupIdInput == "" {
		exit(errors.New("Consumer group is not defined. Use --group-id argument to define it."))
	}
}

func processGroup(command string, groupId string, topic string) {
	partitions, err := kafkaClient.Partitions(topic)
	if err != nil {
		exit(err)
	}
	offsetManager, err := sarama.NewOffsetManagerFromClient(groupId, kafkaClient)
	if err != nil {
		exit(err)
	}
	var lag int64
	var timestamps = make([]int64, len(partitions))
	var controlChannel = make(chan []int64)
	for _, partition := range partitions {
		go processPartition(command, topic, partition, controlChannel, offsetManager)
	}
	for _, partition := range partitions {
		response := <-controlChannel
		lag += response[0]
		if command == "lag_and_time" {
			timestamps[partition] = response[1]
		}
	}
	if command == "lag_and_time" {
		max := timestamps[0]
		for _, value := range timestamps {
			if value > max {
				max = value
			}
		}
		timelag := time.Now().Unix() - max
		fmt.Println(groupId, topic, lag, timelag)
	} else {
		fmt.Println(groupId, topic, lag)
	}
}

func processPartition(command string, topic string, partition int32, controlChannel chan []int64, offsetManager sarama.OffsetManager) {
	pom, err := offsetManager.ManagePartition(topic, int32(partition))
	if err != nil {
		exit(err)
	}
	consumerOffset, _ := pom.NextOffset()
	offset, err := kafkaClient.GetOffset(topic, int32(partition), sarama.OffsetNewest)
	//fmt.Println(topic, partition, consumerOffset, offset)
	if err != nil {
		exit(err)
	}
	var response = make([]int64, 0)
	var timelag int64
	lag := offset - consumerOffset + 1
	response = append(response, lag)
	if command == "lag_and_time" {
		broker, err := kafkaClient.Leader(topic, partition)
		if err != nil {
			exit(err)
		}
		fetchRequest := sarama.FetchRequest{MaxWaitTime: 10000, MinBytes: 0}
		fetchRequest.AddBlock(topic, partition, consumerOffset-2, 500000)
		fetchResponse, err := broker.Fetch(&fetchRequest)
		if err != nil {
			exit(err)
		}
		block := fetchResponse.GetBlock(topic, partition)
		messages := block.MsgSet.Messages
		if len(messages) > 0 {
			msg := messages[0].Messages()[0].Msg.Value
			var decodedData map[string]interface{}
			codec.NewDecoderBytes(msg, &msgpack).Decode(&decodedData)
			timestamp := decodedData["timestamp"]
			switch timestamp := timestamp.(type) {
			case uint64:
				timelag = int64(timestamp)
			default:
				fmt.Println(timestamp)
				exit(errors.New("message is missing timestamp"))
			}
		} else {
			timelag = 0
		}
		response = append(response, timelag)
	}
	controlChannel <- response
}

func exit(err error) {
	fmt.Println(err)
	os.Exit(1)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"

	"github.com/Shopify/sarama"
)

func main() {

	config := InitConfig()

	config.GetConfig()

	logFile, err := os.OpenFile(config.firehoseLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalln("Failed to open log file:", err)
	}
	defer logFile.Close()

	log.SetOutput(logFile)

	metricsFile, err := os.OpenFile(config.metricsLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalln("Failed to open log file:", err)
	}
	defer metricsFile.Close()

	log.Println("Getting the Kafka consumer")
	consumer, err := GetKafkaConsumer(config, metricsFile)
	if err != nil {
		log.Fatalln("Unable to create consumer", err)
	}

	log.Println("Getting the Kafka producer")
	producer, err := GetKafkaProducer(config, metricsFile)
	if err != nil {
		log.Fatalln("Unable to create producer", err)
	}

	defer CloseConsumer(&consumer)
	defer CloseProducer(&producer)

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT)

	var wg sync.WaitGroup

	configClient := NewClient(config)
	defer configClient.Close()

	transferChan := make(chan sarama.ProducerMessage, 100000)

	log.Println(configClient.GetNumPartitions())
	for partition := 0; partition < configClient.GetNumPartitions(); partition++ {

		syncChan := make(chan int64, 1)
		offset := sarama.OffsetNewest
		finalOffset := int64(-1)

		if config.historical {
			c := NewClient(config)
			offset, finalOffset = c.GetCustomOffset(lastFourHours)
			err := c.Close()
			if err != nil {
				log.Fatalln("Unable to close client. ", err)
			}

		}

		log.Println("Using offset ", offset, " for partition ", partition)
		partitionConsumer, err := consumer.ConsumePartition(config.topic, int32(partition), offset)
		if err != nil {
			log.Println("Unable to create partition consumer", err)
			c := NewClient(config)
			offset, finalOffset = c.GetCustomOffset(lastFourHours)
			err := c.Close()
			if err != nil {
				log.Fatalln("Unable to close client. ", err)
			}
			partitionConsumer, err = consumer.ConsumePartition(config.topic, int32(partition), offset)
			if err != nil {
				log.Println("Unable to create partition consumer", err)
				return
			}
		}

		wg.Add(1)

		go PullFromTopic(partitionConsumer, transferChan, signals, finalOffset, syncChan, &wg)
		log.Println("Started consumer for partition ", partition)

		wg.Add(2)
		go PushToTopic(producer, transferChan, signals, syncChan, &wg)
		log.Println("Started producer")
		go PushToTopic(producer, transferChan, signals, syncChan, &wg)
		log.Println("Started producer")
	}

	go MonitorChan(transferChan, signals, &wg)

	wg.Wait()

	log.Println("All threads done, closing clients and ending")
}
<commit_msg>Continue instead of return on partitionConsumer failure<commit_after>package main

import (
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"

	"github.com/Shopify/sarama"
)

func main() {

	config := InitConfig()

	config.GetConfig()

	logFile, err := os.OpenFile(config.firehoseLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalln("Failed to open log file:", err)
	}
	defer logFile.Close()

	log.SetOutput(logFile)

	metricsFile, err := os.OpenFile(config.metricsLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalln("Failed to open log file:", err)
	}
	defer metricsFile.Close()

	log.Println("Getting the Kafka consumer")
	consumer, err := GetKafkaConsumer(config, metricsFile)
	if err != nil {
		log.Fatalln("Unable to create consumer", err)
	}

	log.Println("Getting the Kafka producer")
	producer, err := GetKafkaProducer(config, metricsFile)
	if err != nil {
		log.Fatalln("Unable to create producer", err)
	}

	defer CloseConsumer(&consumer)
	defer CloseProducer(&producer)

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT)

	var wg sync.WaitGroup

	configClient := NewClient(config)
	defer configClient.Close()

	transferChan := make(chan sarama.ProducerMessage, 100000)

	log.Println(configClient.GetNumPartitions())
	for partition := 0; partition < configClient.GetNumPartitions(); partition++ {

		syncChan := make(chan int64, 1)
		offset := sarama.OffsetNewest
		finalOffset := int64(-1)

		if config.historical {
			c := NewClient(config)
			offset, finalOffset = c.GetCustomOffset(lastFourHours)
			err := c.Close()
			if err != nil {
				log.Fatalln("Unable to close client. ", err)
			}

		}

		log.Println("Using offset ", offset, " for partition ", partition)
		partitionConsumer, err := consumer.ConsumePartition(config.topic, int32(partition), offset)
		if err != nil {
			log.Println("Unable to create partition consumer", err)
			c := NewClient(config)
			offset, finalOffset = c.GetCustomOffset(lastFourHours)
			err := c.Close()
			if err != nil {
				log.Fatalln("Unable to close client. ", err)
			}
			partitionConsumer, err = consumer.ConsumePartition(config.topic, int32(partition), offset)
			if err != nil {
				log.Println("Unable to create partition consumer", err)
				continue
			}
		}

		wg.Add(1)

		go PullFromTopic(partitionConsumer, transferChan, signals, finalOffset, syncChan, &wg)
		log.Println("Started consumer for partition ", partition)

		wg.Add(2)
		go PushToTopic(producer, transferChan, signals, syncChan, &wg)
		log.Println("Started producer")
		go PushToTopic(producer, transferChan, signals, syncChan, &wg)
		log.Println("Started producer")
	}

	go MonitorChan(transferChan, signals, &wg)

	wg.Wait()

	log.Println("All threads done, closing clients and ending")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"runtime"
	"runtime/pprof"
	"time"
)

const (
	name    string = "godedupe"
	version string = "1.3.0"
)

type targetDirectories []string

func (i *targetDirectories) String() string {
	return ""
}

func (i *targetDirectories) Set(value string) error {
	*i = append(*i, value)
	return nil
}

// Options for start the program
type Options struct {
	cpuprofile         string
	targetDirs         targetDirectories
	fileExt            string
	jsonFile           string
	maxDepth           int
	showCurrentValues  bool
	excludeEmptyFiles  bool
	excludeHiddenFiles bool
	enableRecursion    bool
	followSymlinks     bool
	showSummary        bool
	quiet              bool
	showNotification   bool
	sameLine           bool
}

// Init the options to run the program
func initOptions() (opt Options) {
	fmt.Println()
	flag.StringVar(&opt.cpuprofile, "cpuprofile", "", "Enable profiling")
	flag.Var(&opt.targetDirs, "t", "Target directories where the program search for duplicated files")
	flag.StringVar(&opt.jsonFile, "json", "", "Export the list of duplicated files to the given json file")
	flag.StringVar(&opt.fileExt, "ext", "", "Only find duplicates for the given extension")
	flag.IntVar(&opt.maxDepth, "d", -1, "Max recursion depth, -1 = no limit. 1 = current directory")
	flag.BoolVar(&opt.excludeEmptyFiles, "z", true, "Exclude the zero length files")
	flag.BoolVar(&opt.excludeHiddenFiles, "h", true, "Exclude the hidden files")
	flag.BoolVar(&opt.showCurrentValues, "debug", false,
		"Show the current values of the program options")
	flag.BoolVar(&opt.enableRecursion, "r", true, "Follow subdirectories (recursion)")
	flag.BoolVar(&opt.followSymlinks, "s", false, "Follow symlinks")
	flag.BoolVar(&opt.showSummary, "m", false, "Show a summary")
	flag.BoolVar(&opt.quiet, "q", false, "Don't show progress info")
	flag.BoolVar(&opt.showNotification, "show-notification", false,
		"Show a desktop notification when the program finish")
	flag.BoolVar(&opt.sameLine, "1", false,
		"Show each set of duplicated files in one line (for scripting)."+
			"It implies -q (quiet) and ignores -m (show summary)")
	flag.Parse()

	return opt
}

// Header show the program name and current version
func header() {
	fmt.Println("------------------------")
	fmt.Printf("%s - version %s\n", name, version)
	fmt.Println("------------------------")
}

// ShowDebugInfo print all the current option values
func showDebugInfo(opt Options) {
	if opt.showCurrentValues {
		fmt.Println()
		fmt.Println("------------------------")
		fmt.Println("Current option values")
		fmt.Println("------------------------")
		fmt.Println("Target directory          :", opt.targetDirs)
		fmt.Println("Exclude zero length files :", opt.excludeEmptyFiles)
		fmt.Println("Exclude hidden files      :", opt.excludeHiddenFiles)
		fmt.Println("Ignore symlinks           :", opt.followSymlinks)
		fmt.Println("Recursive search          :", opt.enableRecursion)
		fmt.Println("Show a summary            :", opt.showSummary)
		fmt.Println("Quiet                     :", opt.quiet)
		fmt.Println("Show notification         :", opt.showNotification)
		fmt.Println("File extension            :", opt.fileExt)
		fmt.Println("Max depth                 :", opt.maxDepth)
		fmt.Println("Json file                 :", opt.jsonFile)
		fmt.Println("Profile output            :", opt.cpuprofile)
		fmt.Println("Same line                 :", opt.sameLine)
		fmt.Println("------------------------")
	}
}

func trackTime(now time.Time) {
	expired := time.Since(now)
	fmt.Printf("[+] Program terminated in %v\n", expired)
}

func executeCPUProfile(profile string) {
	f, err := os.Create(profile)
	if err != nil {
		panic(err)
	}
	pprof.StartCPUProfile(f)
}

func main() {
	options := initOptions()

	if options.sameLine {
		options.quiet = true
	}

	if !options.quiet {
		header()
	}

	showDebugInfo(options)

	if options.cpuprofile != "" {
		executeCPUProfile(options.cpuprofile)
		defer pprof.StopCPUProfile()
	}

	if !options.quiet {
		defer trackTime(time.Now())
	}

	runtime.GOMAXPROCS(runtime.NumCPU())
	Start(options)
}
<commit_msg>Version 1.3.1<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"runtime"
	"runtime/pprof"
	"time"
)

const (
	name    string = "godedupe"
	version string = "1.3.1"
)

type targetDirectories []string

func (i *targetDirectories) String() string {
	return ""
}

func (i *targetDirectories) Set(value string) error {
	*i = append(*i, value)
	return nil
}

// Options for start the program
type Options struct {
	cpuprofile         string
	targetDirs         targetDirectories
	fileExt            string
	jsonFile           string
	maxDepth           int
	showCurrentValues  bool
	excludeEmptyFiles  bool
	excludeHiddenFiles bool
	enableRecursion    bool
	followSymlinks     bool
	showSummary        bool
	quiet              bool
	showNotification   bool
	sameLine           bool
}

// Init the options to run the program
func initOptions() (opt Options) {
	fmt.Println()
	flag.StringVar(&opt.cpuprofile, "cpuprofile", "", "Enable profiling")
	flag.Var(&opt.targetDirs, "t", "Target directories where the program search for duplicated files")
	flag.StringVar(&opt.jsonFile, "json", "", "Export the list of duplicated files to the given json file")
	flag.StringVar(&opt.fileExt, "ext", "", "Only find duplicates for the given extension")
	flag.IntVar(&opt.maxDepth, "d", -1, "Max recursion depth, -1 = no limit. 1 = current directory")
	flag.BoolVar(&opt.excludeEmptyFiles, "z", true, "Exclude the zero length files")
	flag.BoolVar(&opt.excludeHiddenFiles, "h", true, "Exclude the hidden files")
	flag.BoolVar(&opt.showCurrentValues, "debug", false,
		"Show the current values of the program options")
	flag.BoolVar(&opt.enableRecursion, "r", true, "Follow subdirectories (recursion)")
	flag.BoolVar(&opt.followSymlinks, "s", false, "Follow symlinks")
	flag.BoolVar(&opt.showSummary, "m", false, "Show a summary")
	flag.BoolVar(&opt.quiet, "q", false, "Don't show progress info")
	flag.BoolVar(&opt.showNotification, "show-notification", false,
		"Show a desktop notification when the program finish")
	flag.BoolVar(&opt.sameLine, "1", false,
		"Show each set of duplicated files in one line (for scripting)."+
			"It implies -q (quiet) and ignores -m (show summary)")
	flag.Parse()

	return opt
}

// Header show the program name and current version
func header() {
	fmt.Println("------------------------")
	fmt.Printf("%s - version %s\n", name, version)
	fmt.Println("------------------------")
}

// ShowDebugInfo print all the current option values
func showDebugInfo(opt Options) {
	if opt.showCurrentValues {
		fmt.Println()
		fmt.Println("------------------------")
		fmt.Println("Current option values")
		fmt.Println("------------------------")
		fmt.Println("Target directory          :", opt.targetDirs)
		fmt.Println("Exclude zero length files :", opt.excludeEmptyFiles)
		fmt.Println("Exclude hidden files      :", opt.excludeHiddenFiles)
		fmt.Println("Ignore symlinks           :", opt.followSymlinks)
		fmt.Println("Recursive search          :", opt.enableRecursion)
		fmt.Println("Show a summary            :", opt.showSummary)
		fmt.Println("Quiet                     :", opt.quiet)
		fmt.Println("Show notification         :", opt.showNotification)
		fmt.Println("File extension            :", opt.fileExt)
		fmt.Println("Max depth                 :", opt.maxDepth)
		fmt.Println("Json file                 :", opt.jsonFile)
		fmt.Println("Profile output            :", opt.cpuprofile)
		fmt.Println("Same line                 :", opt.sameLine)
		fmt.Println("------------------------")
	}
}

func trackTime(now time.Time) {
	expired := time.Since(now)
	fmt.Printf("[+] Program terminated in %v\n", expired)
}

func executeCPUProfile(profile string) {
	f, err := os.Create(profile)
	if err != nil {
		panic(err)
	}
	pprof.StartCPUProfile(f)
}

func main() {
	options := initOptions()

	if options.sameLine {
		options.quiet = true
	}

	if !options.quiet {
		header()
	}

	showDebugInfo(options)

	if options.cpuprofile != "" {
		executeCPUProfile(options.cpuprofile)
		defer pprof.StopCPUProfile()
	}

	if !options.quiet {
		defer trackTime(time.Now())
	}

	runtime.GOMAXPROCS(runtime.NumCPU())
	Start(options)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"crypto"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/prometheus/client_golang/prometheus/promhttp"

	"github.com/xenolf/lego/acme"
	"github.com/xenolf/lego/providers/dns/cloudflare"

	"github.com/ericchiang/k8s"
	apiv1 "github.com/ericchiang/k8s/api/v1"
)

const annotationLetsEncryptCertificate string = "estafette.io/letsencrypt-certificate"
const annotationLetsEncryptCertificateHostnames string = "estafette.io/letsencrypt-certificate-hostnames"

const annotationLetsEncryptCertificateState string = "estafette.io/letsencrypt-certificate-state"

// KubeLetsEncryptCertificateState represents the state of the secret with respect to Let's Encrypt certificates
type KubeLetsEncryptCertificateState struct {
	Hostnames   string `json:"hostnames"`
	LastRenewed string `json:"lastRenewed"`
	LastAttempt string `json:"lastAttempt"`
}

var (
	addr = flag.String("listen-address", ":9101", "The address to listen on for HTTP requests.")

	// seed random number
	r = rand.New(rand.NewSource(time.Now().UnixNano()))
)

func main() {

	// create cloudflare api client
	cfAPIKey := os.Getenv("CF_API_KEY")
	if cfAPIKey == "" {
		log.Fatal("CF_API_KEY is required. Please set CF_API_KEY environment variable to your Cloudflare API key.")
	}
	cfAPIEmail := os.Getenv("CF_API_EMAIL")
	if cfAPIEmail == "" {
		log.Fatal("CF_API_EMAIL is required. Please set CF_API_KEY environment variable to your Cloudflare API email.")
	}

	// create kubernetes api client
	client, err := k8s.NewInClusterClient()
	if err != nil {
		log.Fatal(err)
	}

	// start prometheus
	go func() {
		fmt.Println("Serving Prometheus metrics at :9101/metrics...")
		flag.Parse()
		http.Handle("/metrics", promhttp.Handler())
		log.Fatal(http.ListenAndServe(*addr, nil))
	}()

	// watch secrets for all namespaces
	go func() {
		// loop indefinitely
		for {
			fmt.Println("Watching secrets for all namespaces...")
			watcher, err := client.CoreV1().WatchSecrets(context.Background(), k8s.AllNamespaces)
			if err != nil {
				log.Println(err)
			} else {
				// loop indefinitely, unless it errors
				for {
					event, secret, err := watcher.Next()
					if err != nil {
						log.Println(err)
						break
					}

					if *event.Type == k8s.EventAdded || *event.Type == k8s.EventModified {
						//fmt.Printf("Secret %v (namespace %v) has event of type %v, processing it...\n", *secret.Metadata.Name, *secret.Metadata.Namespace, *event.Type)
						err := processSecret(client, secret, fmt.Sprintf("[watcher:%v]", *event.Type))
						if err != nil {
							continue
						}
					} else {
						//fmt.Printf("Secret %v (namespace %v) has event of type %v, skipping it...\n", *secret.Metadata.Name, *secret.Metadata.Namespace, *event.Type)
					}
				}
			}

			// sleep random time between 22 and 37 seconds
			sleepTime := applyJitter(30)
			fmt.Printf("Sleeping for %v seconds...\n", sleepTime)
			time.Sleep(time.Duration(sleepTime) * time.Second)
		}
	}()

	// loop indefinitely
	for {

		// get secrets for all namespaces
		fmt.Println("Listing secrets for all namespaces...")
		secrets, err := client.CoreV1().ListSecrets(context.Background(), k8s.AllNamespaces)
		if err != nil {
			log.Println(err)
		}
		fmt.Printf("Cluster has %v secrets\n", len(secrets.Items))

		// loop all secrets
		if secrets != nil && secrets.Items != nil {
			for _, secret := range secrets.Items {

				err := processSecret(client, secret, "[poller]")
				if err != nil {
					continue
				}
			}
		}

		// sleep random time between 225 and 375 seconds
		sleepTime := applyJitter(300)
		fmt.Printf("Sleeping for %v seconds...\n", sleepTime)
		time.Sleep(time.Duration(sleepTime) * time.Second)
	}
}

func applyJitter(input int) (output int) {

	deviation := int(0.25 * float64(input))

	return input - deviation + r.Intn(2*deviation)
}

type LetsEncryptUser struct {
	Email        string
	Registration *acme.RegistrationResource
	key          crypto.PrivateKey
}

func (u LetsEncryptUser) GetEmail() string {
	return u.Email
}
func (u LetsEncryptUser) GetRegistration() *acme.RegistrationResource {
	return u.Registration
}
func (u LetsEncryptUser) GetPrivateKey() crypto.PrivateKey {
	return u.key
}

type Account struct {
	Email        string `json:"email"`
	key          crypto.PrivateKey
	Registration *acme.RegistrationResource `json:"registration"`
}

func processSecret(kubeclient *k8s.Client, secret *apiv1.Secret, initiator string) error {

	cfAPIKey := os.Getenv("CF_API_KEY")
	cfAPIEmail := os.Getenv("CF_API_EMAIL")

	if &secret != nil && &secret.Metadata != nil && &secret.Metadata.Annotations != nil {

		// get annotations or set default value
		letsEncryptCertificate, ok := secret.Metadata.Annotations[annotationLetsEncryptCertificate]
		if !ok {
			letsEncryptCertificate = "false"
		}
		letsEncryptCertificateHostnames, ok := secret.Metadata.Annotations[annotationLetsEncryptCertificateHostnames]
		if !ok {
			letsEncryptCertificateHostnames = ""
		}

		// get state stored in annotations if present or set to empty struct
		var kubeLetsEncryptCertificateState KubeLetsEncryptCertificateState
		kubeLetsEncryptCertificateStateString, ok := secret.Metadata.Annotations[annotationLetsEncryptCertificateState]
		if err := json.Unmarshal([]byte(kubeLetsEncryptCertificateStateString), &kubeLetsEncryptCertificateState); err != nil {
			// couldn't deserialize, setting to default struct
			kubeLetsEncryptCertificateState = KubeLetsEncryptCertificateState{}
		}

		// parse last renewed time from state
		lastRenewed := time.Time{}
		if kubeLetsEncryptCertificateState.LastRenewed != "" {
			var err error
			lastRenewed, err = time.Parse(time.RFC3339, kubeLetsEncryptCertificateState.LastRenewed)
			if err != nil {
				lastRenewed = time.Time{}
			}
		}

		lastAttempt := time.Time{}
		if kubeLetsEncryptCertificateState.LastAttempt != "" {
			var err error
			lastAttempt, err = time.Parse(time.RFC3339, kubeLetsEncryptCertificateState.LastAttempt)
			if err != nil {
				lastAttempt = time.Time{}
			}
		}

		// check if letsencrypt is enabled for this secret, hostnames are set and either the hostnames have changed or the certificate is older than 60 days and the last attempt was more than 15 minutes ago
		if letsEncryptCertificate == "true" && len(letsEncryptCertificateHostnames) > 0 && time.Since(lastAttempt).Minutes() > 15 && (letsEncryptCertificateHostnames != kubeLetsEncryptCertificateState.Hostnames || time.Since(lastRenewed).Hours() > float64(60*24)) {

			fmt.Printf("%v Certificates in secret %v (namespace %v) are more than 60 days old or hostnames have changed (%v), renewing them with Let's Encrypt...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace, letsEncryptCertificateHostnames)

			// 'lock' the secret for 15 minutes by storing the last attempt timestamp to prevent hitting the rate limit if the Let's Encrypt call fails and to prevent the watcher and the fallback polling to operate on the secret at the same time
			kubeLetsEncryptCertificateState.LastAttempt = time.Now().Format(time.RFC3339)

			// serialize state and store it in the annotation
			kubeLetsEncryptCertificateStateByteArray, err := json.Marshal(kubeLetsEncryptCertificateState)
			if err != nil {
				log.Println(err)
				return err
			}
			secret.Metadata.Annotations[annotationLetsEncryptCertificateState] = string(kubeLetsEncryptCertificateStateByteArray)

			// update secret, with last attempt; this will fire an event for the watcher, but this shouldn't lead to any action because storing the last attempt locks the secret for 15 minutes
			secret, err = kubeclient.CoreV1().UpdateSecret(context.Background(), secret)
			if err != nil {
				log.Println(err)
				return err
			}

			// load account.json
			fmt.Printf("%v Loading account.json for secret %v (namespace %v)...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace)
			fileBytes, err := ioutil.ReadFile("/account/account.json")
			if err != nil {
				log.Println(err)
				return err
			}
			var letsEncryptUser LetsEncryptUser
			err = json.Unmarshal(fileBytes, &letsEncryptUser)
			if err != nil {
				log.Println(err)
				return err
			}

			// load private key
			fmt.Printf("%v Loading account.key for secret %v (namespace %v)...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace)
			privateKey, err := loadPrivateKey("/account/account.key")
			if err != nil {
				log.Println(err)
				return err
			}
			letsEncryptUser.key = privateKey

			// set dns timeout
			fmt.Printf("%v Setting acme dns timeout to 600 seconds for secret %v (namespace %v)...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace)
			acme.DNSTimeout = time.Duration(600) * time.Second

			// create letsencrypt acme client
			fmt.Printf("%v Creating acme client for secret %v (namespace %v)...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace)
			client, err := acme.NewClient("https://acme-v01.api.letsencrypt.org/directory", letsEncryptUser, acme.RSA2048)
			if err != nil {
				log.Println(err)
				return err
			}

			// get dns challenge
			fmt.Printf("%v Creating cloudflare provider for secret %v (namespace %v)...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace)
			var provider acme.ChallengeProvider
			provider, err = cloudflare.NewDNSProviderCredentials(cfAPIEmail, cfAPIKey)
			if err != nil {
				log.Println(err)
				return err
			}

			// clean up acme challenge records in advance
			hostnames := strings.Split(letsEncryptCertificateHostnames, ",")
			for _, hostname := range hostnames {
				fmt.Printf("%v Cleaning up TXT record _acme-challenge.%v for secret %v (namespace %v)...\n", initiator, hostname, *secret.Metadata.Name, *secret.Metadata.Namespace)
				provider.CleanUp("_acme-challenge."+hostname, "", "123d==")
			}

			// set challenge and provider
			client.SetChallengeProvider(acme.DNS01, provider)
			client.ExcludeChallenges([]acme.Challenge{acme.HTTP01, acme.TLSSNI01})

			// get certificate
			fmt.Printf("%v Obtaining certificate for secret %v (namespace %v)...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace)
			var certificate acme.CertificateResource
			var failures map[string]error
			certificate, failures = client.ObtainCertificate(hostnames, true, nil, false)

			// clean up acme challenge records afterwards
			for _, hostname := range hostnames {
				fmt.Printf("%v Cleaning up TXT record _acme-challenge.%v for secret %v (namespace %v)...\n", initiator, hostname, *secret.Metadata.Name, *secret.Metadata.Namespace)
				provider.CleanUp("_acme-challenge."+hostname, "", "123d==")
			}

			// if obtaining secret failed exit and retry after more than 15 minutes
			if len(failures) > 0 {
				for k, v := range failures {
					log.Printf("[%s] Could not obtain certificates\n\t%s", k, v.Error())
				}

				return errors.New("Generating certificates has failed")
			}

			// update the secret
			kubeLetsEncryptCertificateState.Hostnames = letsEncryptCertificateHostnames
			kubeLetsEncryptCertificateState.LastRenewed = time.Now().Format(time.RFC3339)

			fmt.Printf("%v Updating secret %v (namespace %v) because new certificates have been obtained...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace)

			// serialize state and store it in the annotation
			kubeLetsEncryptCertificateStateByteArray, err = json.Marshal(kubeLetsEncryptCertificateState)
			if err != nil {
				log.Println(err)
				return err
			}
			secret.Metadata.Annotations[annotationLetsEncryptCertificateState] = string(kubeLetsEncryptCertificateStateByteArray)

			// store the certificates
			if secret.Data == nil {
				secret.Data = make(map[string][]byte)
			}

			fmt.Printf("%v Secret %v (namespace %v) has %v data items before writing the certificates...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace, len(secret.Data))

			secret.Data["ssl.crt"] = certificate.Certificate
			secret.Data["ssl.key"] = certificate.PrivateKey
			secret.Data["ssl.pem"] = bytes.Join([][]byte{certificate.Certificate, certificate.PrivateKey}, []byte{})

			fmt.Printf("%v Secret %v (namespace %v) has %v data items after writing the certificates...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace, len(secret.Data))

			// update secret, because the data and state annotation have changed
			secret, err = kubeclient.CoreV1().UpdateSecret(context.Background(), secret)
			if err != nil {
				log.Println(err)
				return err
			}
		}
	}

	return nil
}

func loadPrivateKey(file string) (crypto.PrivateKey, error) {
	keyBytes, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}

	keyBlock, _ := pem.Decode(keyBytes)

	switch keyBlock.Type {
	case "RSA PRIVATE KEY":
		return x509.ParsePKCS1PrivateKey(keyBlock.Bytes)
	case "EC PRIVATE KEY":
		return x509.ParseECPrivateKey(keyBlock.Bytes)
	}

	return nil, errors.New("Unknown private key type.")
}
<commit_msg>put secret name at start of log messages<commit_after>package main

import (
	"bytes"
	"context"
	"crypto"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/prometheus/client_golang/prometheus/promhttp"

	"github.com/xenolf/lego/acme"
	"github.com/xenolf/lego/providers/dns/cloudflare"

	"github.com/ericchiang/k8s"
	apiv1 "github.com/ericchiang/k8s/api/v1"
)

const annotationLetsEncryptCertificate string = "estafette.io/letsencrypt-certificate"
const annotationLetsEncryptCertificateHostnames string = "estafette.io/letsencrypt-certificate-hostnames"

const annotationLetsEncryptCertificateState string = "estafette.io/letsencrypt-certificate-state"

// KubeLetsEncryptCertificateState represents the state of the secret with respect to Let's Encrypt certificates
type KubeLetsEncryptCertificateState struct {
	Hostnames   string `json:"hostnames"`
	LastRenewed string `json:"lastRenewed"`
	LastAttempt string `json:"lastAttempt"`
}

var (
	addr = flag.String("listen-address", ":9101", "The address to listen on for HTTP requests.")

	// seed random number
	r = rand.New(rand.NewSource(time.Now().UnixNano()))
)

func main() {

	// create cloudflare api client
	cfAPIKey := os.Getenv("CF_API_KEY")
	if cfAPIKey == "" {
		log.Fatal("CF_API_KEY is required. Please set CF_API_KEY environment variable to your Cloudflare API key.")
	}
	cfAPIEmail := os.Getenv("CF_API_EMAIL")
	if cfAPIEmail == "" {
		log.Fatal("CF_API_EMAIL is required. Please set CF_API_KEY environment variable to your Cloudflare API email.")
	}

	// create kubernetes api client
	client, err := k8s.NewInClusterClient()
	if err != nil {
		log.Fatal(err)
	}

	// start prometheus
	go func() {
		fmt.Println("Serving Prometheus metrics at :9101/metrics...")
		flag.Parse()
		http.Handle("/metrics", promhttp.Handler())
		log.Fatal(http.ListenAndServe(*addr, nil))
	}()

	// watch secrets for all namespaces
	go func() {
		// loop indefinitely
		for {
			fmt.Println("Watching secrets for all namespaces...")
			watcher, err := client.CoreV1().WatchSecrets(context.Background(), k8s.AllNamespaces)
			if err != nil {
				log.Println(err)
			} else {
				// loop indefinitely, unless it errors
				for {
					event, secret, err := watcher.Next()
					if err != nil {
						log.Println(err)
						break
					}

					if *event.Type == k8s.EventAdded || *event.Type == k8s.EventModified {
						//fmt.Printf("Secret %v (namespace %v) has event of type %v, processing it...\n", *secret.Metadata.Name, *secret.Metadata.Namespace, *event.Type)
						err := processSecret(client, secret, fmt.Sprintf("[watcher:%v]", *event.Type))
						if err != nil {
							continue
						}
					} else {
						//fmt.Printf("Secret %v (namespace %v) has event of type %v, skipping it...\n", *secret.Metadata.Name, *secret.Metadata.Namespace, *event.Type)
					}
				}
			}

			// sleep random time between 22 and 37 seconds
			sleepTime := applyJitter(30)
			fmt.Printf("Sleeping for %v seconds...\n", sleepTime)
			time.Sleep(time.Duration(sleepTime) * time.Second)
		}
	}()

	// loop indefinitely
	for {

		// get secrets for all namespaces
		fmt.Println("Listing secrets for all namespaces...")
		secrets, err := client.CoreV1().ListSecrets(context.Background(), k8s.AllNamespaces)
		if err != nil {
			log.Println(err)
		}
		fmt.Printf("Cluster has %v secrets\n", len(secrets.Items))

		// loop all secrets
		if secrets != nil && secrets.Items != nil {
			for _, secret := range secrets.Items {

				err := processSecret(client, secret, "[poller]")
				if err != nil {
					continue
				}
			}
		}

		// sleep random time between 225 and 375 seconds
		sleepTime := applyJitter(300)
		fmt.Printf("Sleeping for %v seconds...\n", sleepTime)
		time.Sleep(time.Duration(sleepTime) * time.Second)
	}
}

func applyJitter(input int) (output int) {

	deviation := int(0.25 * float64(input))

	return input - deviation + r.Intn(2*deviation)
}

type LetsEncryptUser struct {
	Email        string
	Registration *acme.RegistrationResource
	key          crypto.PrivateKey
}

func (u LetsEncryptUser) GetEmail() string {
	return u.Email
}
func (u LetsEncryptUser) GetRegistration() *acme.RegistrationResource {
	return u.Registration
}
func (u LetsEncryptUser) GetPrivateKey() crypto.PrivateKey {
	return u.key
}

type Account struct {
	Email        string `json:"email"`
	key          crypto.PrivateKey
	Registration *acme.RegistrationResource `json:"registration"`
}

func processSecret(kubeclient *k8s.Client, secret *apiv1.Secret, initiator string) error {

	cfAPIKey := os.Getenv("CF_API_KEY")
	cfAPIEmail := os.Getenv("CF_API_EMAIL")

	if &secret != nil && &secret.Metadata != nil && &secret.Metadata.Annotations != nil {

		// get annotations or set default value
		letsEncryptCertificate, ok := secret.Metadata.Annotations[annotationLetsEncryptCertificate]
		if !ok {
			letsEncryptCertificate = "false"
		}
		letsEncryptCertificateHostnames, ok := secret.Metadata.Annotations[annotationLetsEncryptCertificateHostnames]
		if !ok {
			letsEncryptCertificateHostnames = ""
		}

		// get state stored in annotations if present or set to empty struct
		var kubeLetsEncryptCertificateState KubeLetsEncryptCertificateState
		kubeLetsEncryptCertificateStateString, ok := secret.Metadata.Annotations[annotationLetsEncryptCertificateState]
		if err := json.Unmarshal([]byte(kubeLetsEncryptCertificateStateString), &kubeLetsEncryptCertificateState); err != nil {
			// couldn't deserialize, setting to default struct
			kubeLetsEncryptCertificateState = KubeLetsEncryptCertificateState{}
		}

		// parse last renewed time from state
		lastRenewed := time.Time{}
		if kubeLetsEncryptCertificateState.LastRenewed != "" {
			var err error
			lastRenewed, err = time.Parse(time.RFC3339, kubeLetsEncryptCertificateState.LastRenewed)
			if err != nil {
				lastRenewed = time.Time{}
			}
		}

		lastAttempt := time.Time{}
		if kubeLetsEncryptCertificateState.LastAttempt != "" {
			var err error
			lastAttempt, err = time.Parse(time.RFC3339, kubeLetsEncryptCertificateState.LastAttempt)
			if err != nil {
				lastAttempt = time.Time{}
			}
		}

		// check if letsencrypt is enabled for this secret, hostnames are set and either the hostnames have changed or the certificate is older than 60 days and the last attempt was more than 15 minutes ago
		if letsEncryptCertificate == "true" && len(letsEncryptCertificateHostnames) > 0 && time.Since(lastAttempt).Minutes() > 15 && (letsEncryptCertificateHostnames != kubeLetsEncryptCertificateState.Hostnames || time.Since(lastRenewed).Hours() > float64(60*24)) {

			fmt.Printf("%v Secret %v.%v - Certificates are more than 60 days old or hostnames have changed (%v), renewing them with Let's Encrypt...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace, letsEncryptCertificateHostnames)

			// 'lock' the secret for 15 minutes by storing the last attempt timestamp to prevent hitting the rate limit if the Let's Encrypt call fails and to prevent the watcher and the fallback polling to operate on the secret at the same time
			kubeLetsEncryptCertificateState.LastAttempt = time.Now().Format(time.RFC3339)

			// serialize state and store it in the annotation
			kubeLetsEncryptCertificateStateByteArray, err := json.Marshal(kubeLetsEncryptCertificateState)
			if err != nil {
				log.Println(err)
				return err
			}
			secret.Metadata.Annotations[annotationLetsEncryptCertificateState] = string(kubeLetsEncryptCertificateStateByteArray)

			// update secret, with last attempt; this will fire an event for the watcher, but this shouldn't lead to any action because storing the last attempt locks the secret for 15 minutes
			secret, err = kubeclient.CoreV1().UpdateSecret(context.Background(), secret)
			if err != nil {
				log.Println(err)
				return err
			}

			// load account.json
			fmt.Printf("%v Secret %v.%v - Loading account.json...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace)
			fileBytes, err := ioutil.ReadFile("/account/account.json")
			if err != nil {
				log.Println(err)
				return err
			}
			var letsEncryptUser LetsEncryptUser
			err = json.Unmarshal(fileBytes, &letsEncryptUser)
			if err != nil {
				log.Println(err)
				return err
			}

			// load private key
			fmt.Printf("%v Secret %v.%v - Loading account.key...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace)
			privateKey, err := loadPrivateKey("/account/account.key")
			if err != nil {
				log.Println(err)
				return err
			}
			letsEncryptUser.key = privateKey

			// set dns timeout
			fmt.Printf("%v Secret %v.%v - Setting acme dns timeout to 600 seconds...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace)
			acme.DNSTimeout = time.Duration(600) * time.Second

			// create letsencrypt acme client
			fmt.Printf("%v Secret %v.%v - Creating acme client...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace)
			client, err := acme.NewClient("https://acme-v01.api.letsencrypt.org/directory", letsEncryptUser, acme.RSA2048)
			if err != nil {
				log.Println(err)
				return err
			}

			// get dns challenge
			fmt.Printf("%v Secret %v.%v - Creating cloudflare provider...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace)
			var provider acme.ChallengeProvider
			provider, err = cloudflare.NewDNSProviderCredentials(cfAPIEmail, cfAPIKey)
			if err != nil {
				log.Println(err)
				return err
			}

			// clean up acme challenge records in advance
			hostnames := strings.Split(letsEncryptCertificateHostnames, ",")
			for _, hostname := range hostnames {
				fmt.Printf("%v Secret %v.%v - Cleaning up TXT record _acme-challenge.%v...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace, hostname)
				provider.CleanUp("_acme-challenge."+hostname, "", "123d==")
			}

			// set challenge and provider
			client.SetChallengeProvider(acme.DNS01, provider)
			client.ExcludeChallenges([]acme.Challenge{acme.HTTP01, acme.TLSSNI01})

			// get certificate
			fmt.Printf("%v Secret %v.%v - Obtaining certificate...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace)
			var certificate acme.CertificateResource
			var failures map[string]error
			certificate, failures = client.ObtainCertificate(hostnames, true, nil, false)

			// clean up acme challenge records afterwards
			for _, hostname := range hostnames {
				fmt.Printf("%v Secret %v.%v - Cleaning up TXT record _acme-challenge.%v...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace, hostname)
				provider.CleanUp("_acme-challenge."+hostname, "", "123d==")
			}

			// if obtaining secret failed exit and retry after more than 15 minutes
			if len(failures) > 0 {
				for k, v := range failures {
					log.Printf("[%s] Could not obtain certificates\n\t%s", k, v.Error())
				}

				return errors.New("Generating certificates has failed")
			}

			// update the secret
			kubeLetsEncryptCertificateState.Hostnames = letsEncryptCertificateHostnames
			kubeLetsEncryptCertificateState.LastRenewed = time.Now().Format(time.RFC3339)

			fmt.Printf("%v Secret %v.%v - Updating secret because new certificates have been obtained...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace)

			// serialize state and store it in the annotation
			kubeLetsEncryptCertificateStateByteArray, err = json.Marshal(kubeLetsEncryptCertificateState)
			if err != nil {
				log.Println(err)
				return err
			}
			secret.Metadata.Annotations[annotationLetsEncryptCertificateState] = string(kubeLetsEncryptCertificateStateByteArray)

			// store the certificates
			if secret.Data == nil {
				secret.Data = make(map[string][]byte)
			}

			fmt.Printf("%v Secret %v.%v - Secret has %v data items before writing the certificates...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace, len(secret.Data))

			secret.Data["ssl.crt"] = certificate.Certificate
			secret.Data["ssl.key"] = certificate.PrivateKey
			secret.Data["ssl.pem"] = bytes.Join([][]byte{certificate.Certificate, certificate.PrivateKey}, []byte{})

			fmt.Printf("%v Secret %v.%v - Secret has %v data items after writing the certificates...\n", initiator, *secret.Metadata.Name, *secret.Metadata.Namespace, len(secret.Data))

			// update secret, because the data and state annotation have changed
			secret, err = kubeclient.CoreV1().UpdateSecret(context.Background(), secret)
			if err != nil {
				log.Println(err)
				return err
			}
		}
	}

	return nil
}

func loadPrivateKey(file string) (crypto.PrivateKey, error) {
	keyBytes, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}

	keyBlock, _ := pem.Decode(keyBytes)

	switch keyBlock.Type {
	case "RSA PRIVATE KEY":
		return x509.ParsePKCS1PrivateKey(keyBlock.Bytes)
	case "EC PRIVATE KEY":
		return x509.ParseECPrivateKey(keyBlock.Bytes)
	}

	return nil, errors.New("Unknown private key type.")
}
<|endoftext|>