<commit_before>package main

import (
	"fmt"
	"os"
)

// Version
const version = "v0.21.2-alpha"

func main() {
	if err := rootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}
<commit_msg>version bump<commit_after>package main

import (
	"fmt"
	"os"
)

// Version
const version = "v0.30.0-alpha"

func main() {
	if err := rootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"os"
	"encoding/json"
	"bytes"
	"fmt"
	"net/url"
	"log"
)

// Get it

type Data struct {
	WebhookEvent string
	User User
	Issue Issue
}

type User struct {
	Name string
	DisplayName string
}

type Issue struct {
	Self string
	Key string
	Fields Fields
}

type Fields struct {
	Summary string
}

// Send it

type Message struct {
	Text string
}

func index(w http.ResponseWriter, r *http.Request) {
	// Get mattermost URL
	mattermostHookURL := r.URL.Query().Get("mattermost_hook_url")
	//log.Println(mattermostHookURL) // FIXME Delete it

	// Parse JSON from JIRA
	decoder := json.NewDecoder(r.Body)
	var data Data
	decoder.Decode(&data)
	//log.Println(data) // FIXME Delete it

	// Get JIRA URL

	u, _ := url.Parse(data.Issue.Self)
	log.Println(u.Scheme + u.Host)

	// Send data to Mattermost

	// Create message
	text := fmt.Sprintf(
		"[[%s] %s](%s://%s/browse/%s) %s by [%s](%s://%s/secure/ViewProfile.jspa?name=%s)",
		data.Issue.Key,
		data.Issue.Fields.Summary,
		u.Scheme,
		u.Host,
		data.Issue.Key,
		data.WebhookEvent,
		data.User.DisplayName,
		u.Scheme,
		u.Host,
		data.User.Name,
	)

	message := Message{
		Text: text,
	}

	byteMessage, _ := json.Marshal(message)

	// Create http-client
	req, _ := http.NewRequest("POST", mattermostHookURL, bytes.NewBuffer(byteMessage))
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

}

func main() {

	port := os.Getenv("PORT")
	if port == "" {
		port = "5000"
	}
	http.HandleFunc("/", index)
	http.ListenAndServe(":" + port, nil)
}
<commit_msg>Comment log input<commit_after>package main

import (
	"net/http"
	"os"
	"encoding/json"
	"bytes"
	"fmt"
	"net/url"
	"log"
)

// Get it

type Data struct {
	WebhookEvent string
	User User
	Issue Issue
}

type User struct {
	Name string
	DisplayName string
}

type Issue struct {
	Self string
	Key string
	Fields Fields
}

type Fields struct {
	Summary string
}

// Send it

type Message struct {
	Text string
}

func index(w http.ResponseWriter, r *http.Request) {
	// Get mattermost URL
	mattermostHookURL := r.URL.Query().Get("mattermost_hook_url")
	//log.Println(mattermostHookURL) // FIXME Delete it

	// Parse JSON from JIRA
	decoder := json.NewDecoder(r.Body)
	var data Data
	decoder.Decode(&data)
	//log.Println(data) // FIXME Delete it

	// Get JIRA URL

	u, _ := url.Parse(data.Issue.Self)
	//log.Println(u.Scheme + u.Host)

	// Send data to Mattermost

	// Create message
	text := fmt.Sprintf(
		"[[%s] %s](%s://%s/browse/%s) %s by [%s](%s://%s/secure/ViewProfile.jspa?name=%s)",
		data.Issue.Key,
		data.Issue.Fields.Summary,
		u.Scheme,
		u.Host,
		data.Issue.Key,
		data.WebhookEvent,
		data.User.DisplayName,
		u.Scheme,
		u.Host,
		data.User.Name,
	)

	message := Message{
		Text: text,
	}

	byteMessage, _ := json.Marshal(message)

	// Create http-client
	req, _ := http.NewRequest("POST", mattermostHookURL, bytes.NewBuffer(byteMessage))
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

}

func main() {

	port := os.Getenv("PORT")
	if port == "" {
		port = "5000"
	}
	http.HandleFunc("/", index)
	http.ListenAndServe(":" + port, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/golang/groupcache"
	"github.com/gorilla/mux"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	"github.com/vokalinteractive/go-loggly"
	"log"
	"net/http"
	"os"
	"vip/fetch"
	"vip/peer"
	"vip/store"
)

var (
	cache     *groupcache.Group
	peers     peer.CachePool
	storage   store.ImageStore
	authToken string

	httpport *string = flag.String("httpport", "8080", "target port")
	secure   *bool   = flag.Bool("secure", false, "use SSL")
)

func listenHttp() {
	log.Printf("Listening on port :%s\n", *httpport)
	cert := os.Getenv("SSL_CERT")
	key := os.Getenv("SSL_KEY")

	port := fmt.Sprintf(":%s", *httpport)

	if cert != "" && key != "" {
		log.Println("Serving via SSL")
		if err := http.ListenAndServeTLS(port, cert, key, nil); err != nil {
			log.Fatalf("Error starting server: %s\n", err.Error())
		}
	} else {
		if err := http.ListenAndServe(port, nil); err != nil {
			log.Fatalf("Error starting server: %s\n", err.Error())
		}
	}
}

func getRegion() aws.Region {
	region := os.Getenv("AWS_REGION")
	aws_region, ok := aws.Regions[region]
	if ok {
		return aws_region
	} else {
		log.Printf(
			"\"%s\" is not a valid AWS_REGION parameter provided, defaulting to us-east-1",
			region)
		return aws.USEast
	}
}

func init() {
	flag.Parse()

	loggly_key := os.Getenv("LOGGLY_KEY")
	if loggly_key != "" {
		log.SetOutput(loggly.New(loggly_key, "vip"))
	}

	r := mux.NewRouter()

	authToken = os.Getenv("AUTH_TOKEN")
	if authToken == "" {
		log.Println("No AUTH_TOKEN parameter provided, uploads are insecure")
	}

	r.Handle("/upload/{bucket_id}", verifyAuth(handleUpload))
	r.HandleFunc("/{bucket_id}/{image_id}", handleImageRequest)
	r.HandleFunc("/ping", handlePing)
	http.Handle("/", r)
}

func main() {
	awsAuth, err := aws.EnvAuth()
	if err != nil {
		log.Fatalf(err.Error())
	}

	s3conn := s3.New(awsAuth, getRegion())
	storage = store.NewS3Store(s3conn)

	peers = peer.DebugPool()

	peers.SetContext(func(r *http.Request) groupcache.Context {
		return fetch.RequestContext(r)
	})

	cache = groupcache.NewGroup("ImageProxyCache", 64<<20, groupcache.GetterFunc(
		func(c groupcache.Context, key string, dest groupcache.Sink) error {
			log.Printf("Cache MISS for key -> %s", key)
			// Get image data from S3
			b, err := fetch.ImageData(storage, c)
			if err != nil {
				return err
			}

			return dest.SetBytes(b)
		}))

	go peers.Listen()
	go listenHttp()

	log.Println("Cache listening on port :" + peers.Port())
	s := &http.Server{
		Addr:    ":" + peers.Port(),
		Handler: peers,
	}
	s.ListenAndServe()
}
<commit_msg>changed to log to syslog<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/golang/groupcache"
	"github.com/gorilla/mux"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	"github.com/vokalinteractive/go-loggly"
	"log"
	"net/http"
	"os"
	"vip/fetch"
	"vip/peer"
	"vip/store"
)

var (
	cache     *groupcache.Group
	peers     peer.CachePool
	storage   store.ImageStore
	authToken string

	httpport *string = flag.String("httpport", "8080", "target port")
	secure   *bool   = flag.Bool("secure", false, "use SSL")
)

func listenHttp() {
	log.Printf("Listening on port :%s\n", *httpport)
	cert := os.Getenv("SSL_CERT")
	key := os.Getenv("SSL_KEY")

	port := fmt.Sprintf(":%s", *httpport)

	if cert != "" && key != "" {
		log.Println("Serving via SSL")
		if err := http.ListenAndServeTLS(port, cert, key, nil); err != nil {
			log.Fatalf("Error starting server: %s\n", err.Error())
		}
	} else {
		if err := http.ListenAndServe(port, nil); err != nil {
			log.Fatalf("Error starting server: %s\n", err.Error())
		}
	}
}

func getRegion() aws.Region {
	region := os.Getenv("AWS_REGION")
	aws_region, ok := aws.Regions[region]
	if ok {
		return aws_region
	} else {
		log.Printf(
			"\"%s\" is not a valid AWS_REGION parameter provided, defaulting to us-east-1",
			region)
		return aws.USEast
	}
}

func init() {
	flag.Parse()

	loggly_key := os.Getenv("LOGGLY_KEY")
	if loggly_key != "" {
		log.SetOutput(loggly.New(loggly_key, "vip"))
	}

	r := mux.NewRouter()

	authToken = os.Getenv("AUTH_TOKEN")
	if authToken == "" {
		log.Println("No AUTH_TOKEN parameter provided, uploads are insecure")
	}

	r.Handle("/upload/{bucket_id}", verifyAuth(handleUpload))
	r.HandleFunc("/{bucket_id}/{image_id}", handleImageRequest)
	r.HandleFunc("/ping", handlePing)
	http.Handle("/", r)
}

func main() {
	awsAuth, err := aws.EnvAuth()
	if err != nil {
		log.Fatalf(err.Error())
	}

	s3conn := s3.New(awsAuth, getRegion())
	storage = store.NewS3Store(s3conn)

	peers = peer.DebugPool()

	peers.SetContext(func(r *http.Request) groupcache.Context {
		return fetch.RequestContext(r)
	})

	cache = groupcache.NewGroup("ImageProxyCache", 64<<20, groupcache.GetterFunc(
		func(c groupcache.Context, key string, dest groupcache.Sink) error {
			log.Printf("Cache MISS for key -> %s", key)
			// Get image data from S3
			b, err := fetch.ImageData(storage, c)
			if err != nil {
				return err
			}

			return dest.SetBytes(b)
		}))

	logwriter, err := syslog.Dial("udp", "app_syslog:514", syslog.LOG_NOTICE, "vip")
	if err != nil {
		log.Fatal(err.Error())
	}
	log.SetOutput(logwriter)

	go peers.Listen()
	go listenHttp()

	log.Println("Cache listening on port :" + peers.Port())
	s := &http.Server{
		Addr:    ":" + peers.Port(),
		Handler: peers,
	}
	s.ListenAndServe()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
)

var version = "<not set>"

func main() {
	printHelp := flag.Bool("help", false, "print this help list")
	printVersion := flag.Bool("version", false, "print program version")
	verbose := flag.Bool("verbose", false, "print verbose output when run")
	filename := flag.String("config", "vangen.json", "vangen json configuration `filename`")
	outputDir := flag.String("out", "vangen/", "output `directory` that static files will be written to")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Vangen is a tool for generating static HTML for hosting Go repositories at a vanity import path.\n\n")
		fmt.Fprintf(os.Stderr, "Usage:\n\n")
		fmt.Fprintf(os.Stderr, "  vangen [-config=vangen.json] [-out=vangen/]\n\n")
		fmt.Fprintf(os.Stderr, "Flags:\n\n")
		flag.PrintDefaults()
	}
	flag.Parse()

	if *printHelp {
		flag.Usage()
		return
	}

	if *printVersion {
		fmt.Fprintln(os.Stderr, version)
		return
	}

	cf, err := os.Open(*filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return
	}

	c, err := parseConfig(cf)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return
	}

	err = os.MkdirAll(*outputDir, os.ModePerm)
	if err != nil {
		fmt.Fprintf(os.Stderr, "making dir %s: %v", *outputDir, err)
		return
	}

	if c.Index {
		pathOut := filepath.Join(*outputDir, "index.html")
		f, err := os.Create(pathOut)
		if err != nil {
			fmt.Fprintf(os.Stderr, "writing file %s: %v", pathOut, err)
			return
		}
		defer func() {
			err := f.Close()
			if err != nil {
				fmt.Fprintf(os.Stderr, "closing file %s: %v", pathOut, err)
			}
		}()

		if *verbose {
			fmt.Fprintf(os.Stderr, "Writing %s\n", pathOut)
		}
		err = generate_index(f, c.Domain, c.Repositories)
		if err != nil {
			fmt.Fprintf(os.Stderr, "generating index: %v", err)
			return
		}

		err = f.Sync()
		if err != nil {
			fmt.Fprintf(os.Stderr, "flushing file %s: %v", pathOut, err)
			return
		}
	}

	for _, r := range c.Repositories {
		for _, p := range r.Packages() {
			dirOut := filepath.Join(*outputDir, p)
			err = os.MkdirAll(dirOut, os.ModePerm)
			if err != nil {
				fmt.Fprintf(os.Stderr, "making dir %s: %v", dirOut, err)
				return
			}

			pathOut := filepath.Join(dirOut, "index.html")
			f, err := os.Create(pathOut)
			if err != nil {
				fmt.Fprintf(os.Stderr, "writing file %s: %v", pathOut, err)
				return
			}
			defer func() {
				err := f.Close()
				if err != nil {
					fmt.Fprintf(os.Stderr, "closing file %s: %v", pathOut, err)
				}
			}()

			if *verbose {
				fmt.Fprintf(os.Stderr, "Writing %s\n", pathOut)
			}
			err = generate_package(f, c.Domain, p, r)
			if err != nil {
				fmt.Fprintf(os.Stderr, "generating package %s: %v", p, err)
				return
			}

			err = f.Sync()
			if err != nil {
				fmt.Fprintf(os.Stderr, "flushing file %s: %v", pathOut, err)
				return
			}
		}
	}
}
<commit_msg>Add a --no-overwrite flag (#5)<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
)

var version = "<not set>"

func main() {
	printHelp := flag.Bool("help", false, "print this help list")
	printVersion := flag.Bool("version", false, "print program version")
	verbose := flag.Bool("verbose", false, "print verbose output when run")
	filename := flag.String("config", "vangen.json", "vangen json configuration `filename`")
	outputDir := flag.String("out", "vangen/", "output `directory` that static files will be written to")
	noOverwrite := flag.Bool("no-overwrite", false, "If an output file already exists, stops with a non-zero return code")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Vangen is a tool for generating static HTML for hosting Go repositories at a vanity import path.\n\n")
		fmt.Fprintf(os.Stderr, "Usage:\n\n")
		fmt.Fprintf(os.Stderr, "  vangen [-config=vangen.json] [-out=vangen/]\n\n")
		fmt.Fprintf(os.Stderr, "Flags:\n\n")
		flag.PrintDefaults()
	}
	flag.Parse()

	if *printHelp {
		flag.Usage()
		return
	}

	if *printVersion {
		fmt.Fprintln(os.Stderr, version)
		return
	}

	cf, err := os.Open(*filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return
	}

	c, err := parseConfig(cf)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return
	}

	err = os.MkdirAll(*outputDir, os.ModePerm)
	if err != nil {
		fmt.Fprintf(os.Stderr, "making dir %s: %v", *outputDir, err)
		return
	}

	if c.Index {
		pathOut := filepath.Join(*outputDir, "index.html")
		f, err := os.Create(pathOut)
		if err != nil {
			fmt.Fprintf(os.Stderr, "writing file %s: %v", pathOut, err)
			return
		}
		defer func() {
			err := f.Close()
			if err != nil {
				fmt.Fprintf(os.Stderr, "closing file %s: %v", pathOut, err)
			}
		}()

		if *verbose {
			fmt.Fprintf(os.Stderr, "Writing %s\n", pathOut)
		}
		err = generate_index(f, c.Domain, c.Repositories)
		if err != nil {
			fmt.Fprintf(os.Stderr, "generating index: %v", err)
			return
		}

		err = f.Sync()
		if err != nil {
			fmt.Fprintf(os.Stderr, "flushing file %s: %v", pathOut, err)
			return
		}
	}

	for _, r := range c.Repositories {
		for _, p := range r.Packages() {
			dirOut := filepath.Join(*outputDir, p)
			err = os.MkdirAll(dirOut, os.ModePerm)
			if err != nil {
				fmt.Fprintf(os.Stderr, "making dir %s: %v", dirOut, err)
				return
			}

			pathOut := filepath.Join(dirOut, "index.html")
			if *noOverwrite {
				if _, err := os.Stat(pathOut); !os.IsNotExist(err) {
					if err == nil {
						fmt.Fprintf(os.Stderr, "cannot overwrite output file %s\n", pathOut)
						os.Exit(1)
					} else {
						fmt.Fprintf(os.Stderr, "checking file %s: %v", pathOut, err)
					}
					return
				}
			}
			f, err := os.Create(pathOut)
			if err != nil {
				fmt.Fprintf(os.Stderr, "writing file %s: %v", pathOut, err)
				return
			}
			defer func() {
				err := f.Close()
				if err != nil {
					fmt.Fprintf(os.Stderr, "closing file %s: %v", pathOut, err)
				}
			}()

			if *verbose {
				fmt.Fprintf(os.Stderr, "Writing %s\n", pathOut)
			}
			err = generate_package(f, c.Domain, p, r)
			if err != nil {
				fmt.Fprintf(os.Stderr, "generating package %s: %v", p, err)
				return
			}

			err = f.Sync()
			if err != nil {
				fmt.Fprintf(os.Stderr, "flushing file %s: %v", pathOut, err)
				return
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"strings"

	"github.com/julienschmidt/httprouter"
)

func main() {
	http.ListenAndServe(":8080", GetApp())
}

func GetApp() http.Handler {
	r := httprouter.New()
	r.GET("/*file", FetcherFunc)
	return r
}

func FetcherFunc(rw http.ResponseWriter, r *http.Request, p httprouter.Params) {
	filename := removePrefixSlash(p.ByName("file"))

	if filename == "" {
		http.Error(rw, "No image url provided", http.StatusBadRequest)
		return
	}

	fmt.Fprintln(rw, fmt.Sprintf("Hello %s", filename))
}

func removePrefixSlash(s string) string {
	return strings.TrimPrefix(s, "/")
}
<commit_msg>Enhance errors when fetching images<commit_after>package main

import (
	"bytes"
	"io"
	"net/http"
	"strings"

	"github.com/julienschmidt/httprouter"
)

func main() {
	http.ListenAndServe(":8080", GetApp())
}

func GetApp() http.Handler {
	r := httprouter.New()
	r.GET("/*fileUri", FetcherFunc)
	return r
}

func FetcherFunc(rw http.ResponseWriter, r *http.Request, p httprouter.Params) {
	url := removePrefixSlash(p.ByName("fileUri"))

	if !isValid(url) {
		http.Error(rw, "Invalid image url provided", http.StatusBadRequest)
		return
	}

	response, err := getImage(&url)
	if err != nil {
		http.Error(rw, "", http.StatusInternalServerError)
		return
	}

	if externalImageNotFound(response) {
		http.Error(rw, "", http.StatusNotFound)
		return
	}

	defer response.Body.Close()

	imgAsBytes, err := decodeImageFromBody(&response.Body)
	if err != nil {
		http.Error(rw, "", http.StatusInternalServerError)
		return
	}

	format := response.Header.Get("Content-type")

	if len(format) == 0 {
		http.Error(rw, "Content-type not defined by upstream server", http.StatusInternalServerError)
		return
	}

	rw.Header().Add("Content-type", format)
	rw.Write(*imgAsBytes)
}

func decodeImageFromBody(body *io.ReadCloser) (*[]byte, error) {
	buf := &bytes.Buffer{}
	_, err := buf.ReadFrom(*body)

	if err != nil {
		return nil, err
	}

	b := buf.Bytes()
	return &b, nil
}

func externalImageNotFound(response *http.Response) bool {
	return response.StatusCode == http.StatusNotFound
}

func getImage(url *string) (*http.Response, error) {
	return http.Get(*url)
}

func isValid(s string) bool {
	if s == "" {
		return false
	}
	return true
}

func removePrefixSlash(s string) string {
	return strings.TrimPrefix(s, "/")
}
<|endoftext|>


<commit_before>package manifest

import (
	"errors"
	"fmt"
	"sync"

	"github.com/bpicode/fritzctl/console"
	"github.com/bpicode/fritzctl/fritz"
)

// AhaApiApplier is an Applier that performs changes to the AHA system via the HTTP API.
func AhaApiApplier(f fritz.HomeAutomationApi) Applier {
	return &ahaApiApplier{fritz: f}
}

type ahaApiApplier struct {
	fritz fritz.HomeAutomationApi
}

// Apply does only log the proposed changes.
func (a *ahaApiApplier) Apply(src, target *Plan) error {
	planner := TargetBasedPlanner(reconfigureSwitch, reconfigureThermostat)
	actions, err := planner.Plan(src, target)
	if err != nil {
		return err
	}
	var wg sync.WaitGroup
	fanOutChan := make(chan error)
	fanInChan := make(chan error)
	for _, action := range actions {
		wg.Add(1)
		go func(ac Action) {
			fanOutChan <- ac.Perform(a.fritz)
			wg.Done()
		}(action)
	}
	go func() {
		var msg string
		for e := range fanOutChan {
			if e != nil {
				msg += e.Error() + "\n"
			}
		}
		if msg != "" {
			fanInChan <- errors.New(msg)
		} else {
			fanInChan <- nil
		}
	}()
	wg.Wait()
	close(fanOutChan)
	err = <-fanInChan
	close(fanInChan)
	return err
}

type reconfigureSwitchAction struct {
	before Switch
	after  Switch
}

func reconfigureSwitch(before, after Switch) Action {
	return &reconfigureSwitchAction{before: before, after: after}
}

// Perform applies the target state to a switch by turning it on/off.
func (a *reconfigureSwitchAction) Perform(f fritz.HomeAutomationApi) (err error) {
	if a.before.State != a.after.State {
		if a.after.State {
			_, err = f.SwitchOn(a.before.ain)
		} else {
			_, err = f.SwitchOff(a.before.ain)
		}
		if err == nil {
			fmt.Printf("\tOK\t'%s'\t%s\t⟶\t%s\n", a.before.Name, console.Btoc(a.before.State), console.Btoc(a.after.State))
		}
	}
	return err
}

type reconfigureThermostatAction struct {
	before Thermostat
	after  Thermostat
}

func reconfigureThermostat(before, after Thermostat) Action {
	return &reconfigureThermostatAction{before: before, after: after}
}

// Perform applies the target state to a switch by turning it on/off.
func (a *reconfigureThermostatAction) Perform(f fritz.HomeAutomationApi) (err error) {
	if a.before.Temperature != a.after.Temperature {
		_, err = f.ApplyTemperature(a.after.Temperature, a.before.ain)
		if err == nil {
			fmt.Printf("\tOK\t'%s'\t%.1f°C\t⟶\t%.1f°C\n", a.before.Name, a.before.Temperature, a.after.Temperature)
		}
	}
	return err
}
<commit_msg>smaller methods for fan-out/fan-in goroutines<commit_after>package manifest

import (
	"errors"
	"fmt"
	"sync"

	"github.com/bpicode/fritzctl/console"
	"github.com/bpicode/fritzctl/fritz"
)

// AhaApiApplier is an Applier that performs changes to the AHA system via the HTTP API.
func AhaApiApplier(f fritz.HomeAutomationApi) Applier {
	return &ahaApiApplier{fritz: f}
}

type ahaApiApplier struct {
	fritz fritz.HomeAutomationApi
}

// Apply does only log the proposed changes.
func (a *ahaApiApplier) Apply(src, target *Plan) error {
	planner := TargetBasedPlanner(reconfigureSwitch, reconfigureThermostat)
	actions, err := planner.Plan(src, target)
	if err != nil {
		return err
	}
	fanOutChan, wg := a.fanOut(actions)
	fanInChan := a.fanIn(fanOutChan)
	wg.Wait()
	close(fanOutChan)
	err = <-fanInChan
	close(fanInChan)
	return err
}

func (a *ahaApiApplier) fanIn(fanOutChan chan error) chan error {
	fanInChan := make(chan error)
	go func() {
		var msg string
		for e := range fanOutChan {
			if e != nil {
				msg += e.Error() + "\n"
			}
		}
		if msg != "" {
			fanInChan <- errors.New(msg)
		} else {
			fanInChan <- nil
		}
	}()
	return fanInChan
}
func (a *ahaApiApplier) fanOut(actions []Action) (chan error, *sync.WaitGroup) {
	var wg sync.WaitGroup
	fanOutChan := make(chan error)
	for _, action := range actions {
		wg.Add(1)
		go func(ac Action) {
			fanOutChan <- ac.Perform(a.fritz)
			wg.Done()
		}(action)
	}
	return fanOutChan, &wg
}

type reconfigureSwitchAction struct {
	before Switch
	after  Switch
}

func reconfigureSwitch(before, after Switch) Action {
	return &reconfigureSwitchAction{before: before, after: after}
}

// Perform applies the target state to a switch by turning it on/off.
func (a *reconfigureSwitchAction) Perform(f fritz.HomeAutomationApi) (err error) {
	if a.before.State != a.after.State {
		if a.after.State {
			_, err = f.SwitchOn(a.before.ain)
		} else {
			_, err = f.SwitchOff(a.before.ain)
		}
		if err == nil {
			fmt.Printf("\tOK\t'%s'\t%s\t⟶\t%s\n", a.before.Name, console.Btoc(a.before.State), console.Btoc(a.after.State))
		}
	}
	return err
}

type reconfigureThermostatAction struct {
	before Thermostat
	after  Thermostat
}

func reconfigureThermostat(before, after Thermostat) Action {
	return &reconfigureThermostatAction{before: before, after: after}
}

// Perform applies the target state to a switch by turning it on/off.
func (a *reconfigureThermostatAction) Perform(f fritz.HomeAutomationApi) (err error) {
	if a.before.Temperature != a.after.Temperature {
		_, err = f.ApplyTemperature(a.after.Temperature, a.before.ain)
		if err == nil {
			fmt.Printf("\tOK\t'%s'\t%.1f°C\t⟶\t%.1f°C\n", a.before.Name, a.before.Temperature, a.after.Temperature)
		}
	}
	return err
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.  See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.

package main

import (
	"flag"
	"os"
	"runtime"

	commander "code.google.com/p/go-commander"
	"github.com/cockroachdb/cockroach/server"
	"github.com/golang/glog"
)

func init() {
	// If log directory has not been set, set -alsologtostderr to true.
	var hasLogDir, hasAlsoLogStderr bool
	for _, arg := range os.Args[1:] {
		switch arg {
		case "-log_dir", "--log_dir":
			hasLogDir = true
		case "-alsologtostderr", "--alsologtostderr":
			hasAlsoLogStderr = true
		}
	}
	if !hasLogDir && !hasAlsoLogStderr {
		flag.CommandLine.Set("alsologtostderr", "true")
	}
}

func main() {
	// Instruct Go to use all CPU cores.
	// TODO(spencer): this may be excessive and result in worse
	// performance. We should keep an eye on this as we move to
	// production workloads.
	numCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(numCPU)
	glog.Infof("running using %d processor cores", numCPU)

	c := commander.Commander{
		Name: "cockroach",
		Commands: []*commander.Command{
			server.CmdInit,
			server.CmdGetZone,
			server.CmdLsZones,
			server.CmdRmZone,
			server.CmdSetZone,
			server.CmdStart,
			&commander.Command{
				UsageLine: "listparams",
				Short:     "list all available parameters and their default values",
				Long: `
List all available parameters and their default values.
Note that parameter parsing stops after the first non-
option after the command name. Hence, the options need
to precede any additional arguments,

  cockroach <command> [options] [arguments].`,
				Run: func(cmd *commander.Command, args []string) {
					flag.CommandLine.PrintDefaults()
				},
			},
		},
	}

	if err := c.Run(os.Args[1:]); err != nil {
		glog.Errorf("Failed running command \"%s\": %v\n", os.Args[1:], err)
		os.Exit(1)
	}
}
<commit_msg>no error msg when called without args; omit cpu core info except when verbose<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.  See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.

package main

import (
	"flag"
	"os"
	"runtime"

	commander "code.google.com/p/go-commander"
	"github.com/cockroachdb/cockroach/server"
	"github.com/golang/glog"
)

func init() {
	// If log directory has not been set, set -alsologtostderr to true.
	var hasLogDir, hasAlsoLogStderr bool
	for _, arg := range os.Args[1:] {
		switch arg {
		case "-log_dir", "--log_dir":
			hasLogDir = true
		case "-alsologtostderr", "--alsologtostderr":
			hasAlsoLogStderr = true
		}
	}
	if !hasLogDir && !hasAlsoLogStderr {
		flag.CommandLine.Set("alsologtostderr", "true")
	}
}

func main() {
	// Instruct Go to use all CPU cores.
	// TODO(spencer): this may be excessive and result in worse
	// performance. We should keep an eye on this as we move to
	// production workloads.
	numCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(numCPU)
	glog.V(1).Infof("running using %d processor cores", numCPU)

	c := commander.Commander{
		Name: "cockroach",
		Commands: []*commander.Command{
			server.CmdInit,
			server.CmdGetZone,
			server.CmdLsZones,
			server.CmdRmZone,
			server.CmdSetZone,
			server.CmdStart,
			&commander.Command{
				UsageLine: "listparams",
				Short:     "list all available parameters and their default values",
				Long: `
List all available parameters and their default values.
Note that parameter parsing stops after the first non-
option after the command name. Hence, the options need
to precede any additional arguments,

  cockroach <command> [options] [arguments].`,
				Run: func(cmd *commander.Command, args []string) {
					flag.CommandLine.PrintDefaults()
				},
			},
		},
	}

	if len(os.Args) == 1 {
		os.Args = append(os.Args, "help")
	}
	if err := c.Run(os.Args[1:]); err != nil {
		glog.Errorf("Failed running command \"%s\": %v\n", os.Args[1:], err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/xml"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/f440/podcast-feed-server/config"
	"github.com/f440/podcast-feed-server/rss"
)

func escapeURL(str string) (string, error) {
	u, err := url.Parse(str)
	if err != nil {
		return "", err
	}
	return u.String(), nil
}

func main() {
	config := config.Config{}
	if err := config.Load("config.toml"); err != nil {
		panic(err)
	}

	fs := http.FileServer(http.Dir(config.Server.FileRoot))
	http.Handle("/", logHandler(userAgentHandler(fs)))

	http.Handle(config.Server.FeedPath, logHandler(userAgentHandler(http.HandlerFunc(feedHandler))))

	if err := http.ListenAndServe(config.Server.Listen, nil); err != nil {
		panic(err)
	}
}

func logHandler(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		rAddr := r.RemoteAddr
		method := r.Method
		path := r.URL.Path

		log.Printf("%s [%s] %s\n", rAddr, method, path)
		next.ServeHTTP(w, r)
	})
}

func userAgentHandler(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		config := config.Config{}
		if err := config.Load("config.toml"); err != nil {
			panic(err)
		}

		if config.Server.PermitUA != "" {
			ua := r.Header.Get("User-Agent")
			if strings.Index(ua, config.Server.PermitUA) == -1 {
				http.Error(w, "Forbidden", http.StatusForbidden)
				return
			}
		}

		next.ServeHTTP(w, r)
	})
}

func feedHandler(w http.ResponseWriter, r *http.Request) {
	config := config.Config{}
	if err := config.Load("config.toml"); err != nil {
		panic(err)
	}

	link, err := url.Parse(config.RSS.URL)
	if err != nil {
		panic(err)
	}
	feedURL, err := url.Parse(config.RSS.URL)
	if err != nil {
		panic(err)
	}
	feedURL.Path = config.Server.FeedPath

	feed := rss.RSS{
		XMLXmlnsAtom:   "http://www.w3.org/2005/Atom",
		XMLXmlnsItunes: "http://www.itunes.com/dtds/podcast-1.0.dtd",
		XMLVersion:     "2.0",
	}
	feed.Channel = &rss.Channel{
		Title:       config.RSS.Title,
		Description: config.RSS.Description,
		Link:        link.String(),
		AtomLink: &rss.AtomLink{
			Href: feedURL.String(),
			Rel:  "self",
			Type: "application/rss+xml",
		},
	}

	items := []*rss.Item{}
	err = filepath.Walk(config.Server.FileRoot, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		if !strings.HasSuffix(info.Name(), ".m4a") {
			return nil
		}

		title := strings.Replace(info.Name(), ".m4a", "", 1)
		pubDate := info.ModTime().Format(time.RFC1123)
		url, err := escapeURL(config.RSS.URL + strings.Replace(path, config.Server.FileRoot, "", 1))
		if err != nil {
			panic(err)
		}
		enclosure := rss.Enclosure{URL: url, Type: "audio/mpeg", Length: info.Size()}
		item := rss.Item{
			Title:     title,
			PubDate:   pubDate,
			Guid:      url,
			Enclosure: &enclosure,
		}

		items = append(items, &item)
		return nil
	})
	if err != nil {
		panic(err)
	}

	sort.Sort(sort.Reverse(rss.ByPubDate(items)))
	feed.Channel.Item = items

	buf, err := xml.MarshalIndent(feed, "", " ")
	if err != nil {
		panic(err)
	}

	w.Header().Set("Content-Type", "application/atom+xml")
	w.Header().Set("Last-Modified", items[0].PubDate)
	w.Write([]byte(xml.Header))
	w.Write(buf)
}
<commit_msg>obey staticcheck<commit_after>package main

import (
	"encoding/xml"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/f440/podcast-feed-server/config"
	"github.com/f440/podcast-feed-server/rss"
)

func escapeURL(str string) (string, error) {
	u, err := url.Parse(str)
	if err != nil {
		return "", err
	}
	return u.String(), nil
}

func main() {
	config := config.Config{}
	if err := config.Load("config.toml"); err != nil {
		panic(err)
	}

	fs := http.FileServer(http.Dir(config.Server.FileRoot))
	http.Handle("/", logHandler(userAgentHandler(fs)))

	http.Handle(config.Server.FeedPath, logHandler(userAgentHandler(http.HandlerFunc(feedHandler))))

	if err := http.ListenAndServe(config.Server.Listen, nil); err != nil {
		panic(err)
	}
}

func logHandler(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		rAddr := r.RemoteAddr
		method := r.Method
		path := r.URL.Path

		log.Printf("%s [%s] %s\n", rAddr, method, path)
		next.ServeHTTP(w, r)
	})
}

func userAgentHandler(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		config := config.Config{}
		if err := config.Load("config.toml"); err != nil {
			panic(err)
		}

		if config.Server.PermitUA != "" {
			ua := r.Header.Get("User-Agent")
			if !strings.Contains(ua, config.Server.PermitUA) {
				http.Error(w, "Forbidden", http.StatusForbidden)
				return
			}
		}

		next.ServeHTTP(w, r)
	})
}

func feedHandler(w http.ResponseWriter, r *http.Request) {
	config := config.Config{}
	if err := config.Load("config.toml"); err != nil {
		panic(err)
	}

	link, err := url.Parse(config.RSS.URL)
	if err != nil {
		panic(err)
	}
	feedURL, err := url.Parse(config.RSS.URL)
	if err != nil {
		panic(err)
	}
	feedURL.Path = config.Server.FeedPath

	feed := rss.RSS{
		XMLXmlnsAtom:   "http://www.w3.org/2005/Atom",
		XMLXmlnsItunes: "http://www.itunes.com/dtds/podcast-1.0.dtd",
		XMLVersion:     "2.0",
	}
	feed.Channel = &rss.Channel{
		Title:       config.RSS.Title,
		Description: config.RSS.Description,
		Link:        link.String(),
		AtomLink: &rss.AtomLink{
			Href: feedURL.String(),
			Rel:  "self",
			Type: "application/rss+xml",
		},
	}

	items := []*rss.Item{}
	err = filepath.Walk(config.Server.FileRoot, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		if !strings.HasSuffix(info.Name(), ".m4a") {
			return nil
		}

		title := strings.Replace(info.Name(), ".m4a", "", 1)
		pubDate := info.ModTime().Format(time.RFC1123)
		url, err := escapeURL(config.RSS.URL + strings.Replace(path, config.Server.FileRoot, "", 1))
		if err != nil {
			panic(err)
		}
		enclosure := rss.Enclosure{URL: url, Type: "audio/mpeg", Length: info.Size()}
		item := rss.Item{
			Title:     title,
			PubDate:   pubDate,
			Guid:      url,
			Enclosure: &enclosure,
		}

		items = append(items, &item)
		return nil
	})
	if err != nil {
		panic(err)
	}

	sort.Sort(sort.Reverse(rss.ByPubDate(items)))
	feed.Channel.Item = items

	buf, err := xml.MarshalIndent(feed, "", " ")
	if err != nil {
		panic(err)
	}

	w.Header().Set("Content-Type", "application/atom+xml")
	w.Header().Set("Last-Modified", items[0].PubDate)
	w.Write([]byte(xml.Header))
	w.Write(buf)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"

	"github.com/hashicorp/terraform/plugin"
	"github.com/mitchellh/cli"
	"github.com/mitchellh/panicwrap"
	"github.com/mitchellh/prefixedio"
)

func main() {
	os.Exit(realMain())
}

func realMain() int {
	var wrapConfig panicwrap.WrapConfig

	if !panicwrap.Wrapped(&wrapConfig) {
		// Determine where logs should go in general (requested by the user)
		logWriter, err := logOutput()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Couldn't setup log output: %s", err)
			return 1
		}
		if logWriter == nil {
			logWriter = ioutil.Discard
		}

		// We always send logs to a temporary file that we use in case
		// there is a panic. Otherwise, we delete it.
		logTempFile, err := ioutil.TempFile("", "terraform-log")
		if err != nil {
			fmt.Fprintf(os.Stderr, "Couldn't setup logging tempfile: %s", err)
			return 1
		}
		defer os.Remove(logTempFile.Name())
		defer logTempFile.Close()

		// Tell the logger to log to this file
		os.Setenv(EnvLog, "")
		os.Setenv(EnvLogFile, "")

		// Setup the prefixed readers that send data properly to
		// stdout/stderr.
		outR, outW := io.Pipe()
		go copyOutput(outR)

		// Create the configuration for panicwrap and wrap our executable
		wrapConfig.Handler = panicHandler(logTempFile)
		wrapConfig.Writer = io.MultiWriter(logTempFile, logWriter)
		wrapConfig.Stdout = outW
		exitStatus, err := panicwrap.Wrap(&wrapConfig)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Couldn't start Terraform: %s", err)
			return 1
		}

		// If >= 0, we're the parent, so just exit
		if exitStatus >= 0 {
			return exitStatus
		}

		// We're the child, so just close the tempfile we made in order to
		// save file handles since the tempfile is only used by the parent.
		logTempFile.Close()
	}

	// Call the real main
	return wrappedMain()
}

func wrappedMain() int {
	log.SetOutput(os.Stderr)

	// Load the configuration
	config := BuiltinConfig
	if err := config.Discover(); err != nil {
		fmt.Fprintf(os.Stderr, "Error discovering plugins: %s", err)
		return 1
	}

	// Make sure we clean up any managed plugins at the end of this
	defer plugin.CleanupClients()

	// Initialize the TFConfig settings for the commands...
	ContextOpts.Providers = config.ProviderFactories()
	ContextOpts.Provisioners = config.ProvisionerFactories()

	// Get the command line args. We shortcut "--version" and "-v" to
	// just show the version.
	args := os.Args[1:]
	for _, arg := range args {
		if arg == "-v" || arg == "-version" || arg == "--version" {
			newArgs := make([]string, len(args)+1)
			newArgs[0] = "version"
			copy(newArgs[1:], args)
			args = newArgs
			break
		}
	}

	cli := &cli.CLI{
		Args:       args,
		Commands:   Commands,
		HelpFunc:   cli.BasicHelpFunc("terraform"),
		HelpWriter: os.Stdout,
	}

	// Load the configuration file if we have one, that can be used to
	// define extra providers and provisioners.
	clicfgFile, err := cliConfigFile()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error loading CLI configuration: \n\n%s\n", err)
		return 1
	}

	if clicfgFile != "" {
		usrcfg, err := LoadConfig(clicfgFile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error loading CLI configuration: \n\n%s\n", err)
			return 1
		}

		config = *config.Merge(usrcfg)
	}

	// Initialize the TFConfig settings for the commands...
	ContextOpts.Providers = config.ProviderFactories()
	ContextOpts.Provisioners = config.ProvisionerFactories()

	exitCode, err := cli.Run()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err.Error())
		return 1
	}

	return exitCode
}

func cliConfigFile() (string, error) {
	mustExist := true
	configFilePath := os.Getenv("TERRAFORM_CONFIG")
	if configFilePath == "" {
		var err error
		configFilePath, err = configFile()
		mustExist = false

		if err != nil {
			log.Printf(
				"[ERROR] Error detecting default CLI config file path: %s",
				err)
		}
	}

	log.Printf("[DEBUG] Attempting to open CLI config file: %s", configFilePath)
	f, err := os.Open(configFilePath)
	if err == nil {
		f.Close()
		return configFilePath, nil
	}

	if mustExist || !os.IsNotExist(err) {
		return "", err
	}

	log.Println("[DEBUG] File doesn't exist, but doesn't need to. Ignoring.")
	return "", nil
}

// copyOutput uses output prefixes to determine whether data on stdout
// should go to stdout or stderr. This is due to panicwrap using stderr
// as the log and error channel.
func copyOutput(r io.Reader) {
	pr, err := prefixedio.NewReader(r)
	if err != nil {
		panic(err)
	}

	stderrR, err := pr.Prefix(ErrorPrefix)
	if err != nil {
		panic(err)
	}
	stdoutR, err := pr.Prefix(OutputPrefix)
	if err != nil {
		panic(err)
	}
	defaultR, err := pr.Prefix("")
	if err != nil {
		panic(err)
	}

	go io.Copy(os.Stderr, stderrR)
	go io.Copy(os.Stdout, stdoutR)
	go io.Copy(os.Stdout, defaultR)
}
<commit_msg>Fix issue where output could be truncated<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sync"

	"github.com/hashicorp/terraform/plugin"
	"github.com/mitchellh/cli"
	"github.com/mitchellh/panicwrap"
	"github.com/mitchellh/prefixedio"
)

func main() {
	os.Exit(realMain())
}

func realMain() int {
	var wrapConfig panicwrap.WrapConfig

	if !panicwrap.Wrapped(&wrapConfig) {
		// Determine where logs should go in general (requested by the user)
		logWriter, err := logOutput()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Couldn't setup log output: %s", err)
			return 1
		}
		if logWriter == nil {
			logWriter = ioutil.Discard
		}

		// We always send logs to a temporary file that we use in case
		// there is a panic. Otherwise, we delete it.
		logTempFile, err := ioutil.TempFile("", "terraform-log")
		if err != nil {
			fmt.Fprintf(os.Stderr, "Couldn't setup logging tempfile: %s", err)
			return 1
		}
		defer os.Remove(logTempFile.Name())
		defer logTempFile.Close()

		// Tell the logger to log to this file
		os.Setenv(EnvLog, "")
		os.Setenv(EnvLogFile, "")

		// Setup the prefixed readers that send data properly to
		// stdout/stderr.
		doneCh := make(chan struct{})
		outR, outW := io.Pipe()
		go copyOutput(outR, doneCh)

		// Create the configuration for panicwrap and wrap our executable
		wrapConfig.Handler = panicHandler(logTempFile)
		wrapConfig.Writer = io.MultiWriter(logTempFile, logWriter)
		wrapConfig.Stdout = outW
		exitStatus, err := panicwrap.Wrap(&wrapConfig)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Couldn't start Terraform: %s", err)
			return 1
		}

		// If >= 0, we're the parent, so just exit
		if exitStatus >= 0 {
			// Close the stdout writer so that our copy process can finish
			outW.Close()

			// Wait for the output copying to finish
			<-doneCh

			return exitStatus
		}

		// We're the child, so just close the tempfile we made in order to
		// save file handles since the tempfile is only used by the parent.
		logTempFile.Close()
	}

	// Call the real main
	return wrappedMain()
}

func wrappedMain() int {
	log.SetOutput(os.Stderr)

	// Load the configuration
	config := BuiltinConfig
	if err := config.Discover(); err != nil {
		fmt.Fprintf(os.Stderr, "Error discovering plugins: %s", err)
		return 1
	}

	// Make sure we clean up any managed plugins at the end of this
	defer plugin.CleanupClients()

	// Initialize the TFConfig settings for the commands...
	ContextOpts.Providers = config.ProviderFactories()
	ContextOpts.Provisioners = config.ProvisionerFactories()

	// Get the command line args. We shortcut "--version" and "-v" to
	// just show the version.
	args := os.Args[1:]
	for _, arg := range args {
		if arg == "-v" || arg == "-version" || arg == "--version" {
			newArgs := make([]string, len(args)+1)
			newArgs[0] = "version"
			copy(newArgs[1:], args)
			args = newArgs
			break
		}
	}

	cli := &cli.CLI{
		Args:       args,
		Commands:   Commands,
		HelpFunc:   cli.BasicHelpFunc("terraform"),
		HelpWriter: os.Stdout,
	}

	// Load the configuration file if we have one, that can be used to
	// define extra providers and provisioners.
	clicfgFile, err := cliConfigFile()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error loading CLI configuration: \n\n%s\n", err)
		return 1
	}

	if clicfgFile != "" {
		usrcfg, err := LoadConfig(clicfgFile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error loading CLI configuration: \n\n%s\n", err)
			return 1
		}

		config = *config.Merge(usrcfg)
	}

	// Initialize the TFConfig settings for the commands...
	ContextOpts.Providers = config.ProviderFactories()
	ContextOpts.Provisioners = config.ProvisionerFactories()

	exitCode, err := cli.Run()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err.Error())
		return 1
	}

	return exitCode
}

func cliConfigFile() (string, error) {
	mustExist := true
	configFilePath := os.Getenv("TERRAFORM_CONFIG")
	if configFilePath == "" {
		var err error
		configFilePath, err = configFile()
		mustExist = false

		if err != nil {
			log.Printf(
				"[ERROR] Error detecting default CLI config file path: %s",
				err)
		}
	}

	log.Printf("[DEBUG] Attempting to open CLI config file: %s", configFilePath)
	f, err := os.Open(configFilePath)
	if err == nil {
		f.Close()
		return configFilePath, nil
	}

	if mustExist || !os.IsNotExist(err) {
		return "", err
	}

	log.Println("[DEBUG] File doesn't exist, but doesn't need to. Ignoring.")
	return "", nil
}

// copyOutput uses output prefixes to determine whether data on stdout
// should go to stdout or stderr. This is due to panicwrap using stderr
// as the log and error channel.
func copyOutput(r io.Reader, doneCh chan<- struct{}) {
	defer close(doneCh)

	pr, err := prefixedio.NewReader(r)
	if err != nil {
		panic(err)
	}

	stderrR, err := pr.Prefix(ErrorPrefix)
	if err != nil {
		panic(err)
	}
	stdoutR, err := pr.Prefix(OutputPrefix)
	if err != nil {
		panic(err)
	}
	defaultR, err := pr.Prefix("")
	if err != nil {
		panic(err)
	}

	var wg sync.WaitGroup
	wg.Add(3)
	go func() {
		defer wg.Done()
		io.Copy(os.Stderr, stderrR)
	}()
	go func() {
		defer wg.Done()
		io.Copy(os.Stdout, stdoutR)
	}()
	go func() {
		defer wg.Done()
		io.Copy(os.Stdout, defaultR)
	}()

	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/jessevdk/go-flags"

	"github.com/feliixx/mgodatagen/datagen"
)

const version = "0.7.4"

func main() {
	var options datagen.Options
	p := flags.NewParser(&options, flags.Default&^flags.HelpFlag)
	_, err := p.Parse()
	if err != nil {
		fmt.Println("try mgodatagen --help for more informations")
		os.Exit(1)
	}
	if options.Help {
		p.WriteHelp(os.Stdout)
		os.Exit(0)
	}

	if options.Version {
		fmt.Printf("mgodatagen version %s\n", version)
		os.Exit(0)
	}

	err = datagen.Generate(&options, os.Stdout)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>version 0.7.5<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/jessevdk/go-flags"

	"github.com/feliixx/mgodatagen/datagen"
)

const version = "0.7.5"

func main() {
	var options datagen.Options
	p := flags.NewParser(&options, flags.Default&^flags.HelpFlag)
	_, err := p.Parse()
	if err != nil {
		fmt.Println("try mgodatagen --help for more informations")
		os.Exit(1)
	}
	if options.Help {
		p.WriteHelp(os.Stdout)
		os.Exit(0)
	}

	if options.Version {
		fmt.Printf("mgodatagen version %s\n", version)
		os.Exit(0)
	}

	err = datagen.Generate(&options, os.Stdout)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"os"
	"time"

	"github.com/freeusd/solebtc/Godeps/_workspace/src/github.com/gin-gonic/gin"
	"github.com/freeusd/solebtc/Godeps/_workspace/src/github.com/robfig/cron"
	"github.com/freeusd/solebtc/controllers/v1"
	"github.com/freeusd/solebtc/errors"
	"github.com/freeusd/solebtc/middlewares"
	"github.com/freeusd/solebtc/models"
	"github.com/freeusd/solebtc/services/cache"
	"github.com/freeusd/solebtc/services/cache/memory"
	"github.com/freeusd/solebtc/services/hub"
	"github.com/freeusd/solebtc/services/hub/list"
	"github.com/freeusd/solebtc/services/mail"
	"github.com/freeusd/solebtc/services/mail/mandrill"
	"github.com/freeusd/solebtc/services/storage"
	"github.com/freeusd/solebtc/services/storage/mysql"
	"github.com/freeusd/solebtc/utils"
)

var (
	logWriter   io.Writer = os.Stdout
	panicWriter io.Writer = os.Stderr
	mailer      mail.Mailer
	store       storage.Storage
	memoryCache cache.Cache
	connsHub    hub.Hub
)

func init() {
	// ORDER MATTERs
	initConfig()
	initMailer()
	initStorage()
	initCache()
	initHub()
	initCronjob()
}

func main() {
	gin.SetMode(ginEnvMode())
	router := gin.New()

	// middlewares
	recovery := gin.RecoveryWithWriter(panicWriter)
	logger := middlewares.LoggerWithWriter(logWriter)
	cors := middlewares.CORS()
	errorWriter := middlewares.ErrorWriter()
	authRequired := middlewares.AuthRequired(store.GetAuthToken, config.AuthToken.Lifetime)

	// globally use middlewares
	router.Use(recovery, logger, cors, errorWriter)

	// version 1 api endpoints
	v1Endpoints := router.Group("/v1")

	// user endpoints
	v1UserEndpoints := v1Endpoints.Group("/users")
	v1UserEndpoints.GET("", authRequired, v1.UserInfo(store.GetUserByID))
	v1UserEndpoints.POST("", v1.Signup(store.CreateUser, store.GetUserByID))
	v1UserEndpoints.PUT("/:id/status", v1.VerifyEmail(store.GetSessionByToken, store.GetUserByID, store.UpdateUserStatus))
	v1UserEndpoints.GET("/referees", authRequired, v1.RefereeList(store.GetRefereesSince, store.GetRefereesUntil))

	// auth token endpoints
	v1AuthTokenEndpoints := v1Endpoints.Group("/auth_tokens")
	v1AuthTokenEndpoints.POST("", v1.Login(store.GetUserByEmail, store.CreateAuthToken))
	v1AuthTokenEndpoints.DELETE("", authRequired, v1.Logout(store.DeleteAuthToken))

	// session endpoints
	v1SessionEndpoints := v1Endpoints.Group("/sessions")
	emailVerificationTemplate := template.Must(template.ParseFiles(config.Template.EmailVerificationTemplate))
	v1SessionEndpoints.POST("", authRequired,
		v1.RequestVerifyEmail(store.GetUserByID, store.UpsertSession, mailer.SendEmail, emailVerificationTemplate),
	)

	// income endpoints
	v1IncomeEndpoints := v1Endpoints.Group("/incomes", authRequired)
	v1IncomeEndpoints.POST("/rewards",
		v1.GetReward(store.GetUserByID,
			memoryCache.GetLatestTotalReward,
			memoryCache.GetLatestConfig,
			memoryCache.GetRewardRatesByType,
			createRewardIncome,
			memoryCache.InsertIncome,
			connsHub.Broadcast),
	)
	v1IncomeEndpoints.GET("/rewards", v1.RewardList(store.GetRewardIncomesSince, store.GetRewardIncomesUntil))
	v1IncomeEndpoints.GET("/rewards/referees/:referee_id", v1.RefereeRewardList(store.GetUserByID, store.GetRewardIncomesSince, store.GetRewardIncomesUntil))

	// websocket endpoint
	v1Endpoints.GET("/websocket",
		v1.Websocket(
			connsHub.Len,
			memoryCache.GetLatestConfig,
			memoryCache.GetLatestIncomes,
			connsHub.Broadcast,
			hub.WrapPutWebsocketConn(connsHub.PutConn)),
	)

	fmt.Fprintf(logWriter, "SoleBTC is running on %s\n", config.HTTP.Port)
	panicIfErrored(router.Run(config.HTTP.Port))
}

func createRewardIncome(income models.Income, now time.Time) *errors.Error {
	if err := store.CreateRewardIncome(income, now); err != nil {
		return err
	}

	totalReward := income.Income
	if income.RefererID > 0 {
		totalReward += income.RefererIncome
	}
	memoryCache.IncrementTotalReward(now, totalReward)

	return nil
}

func initMailer() {
	// mailer
	mailer = mandrill.New(config.Mandrill.Key, config.Mandrill.FromEmail, config.Mandrill.FromName)
}

func initStorage() {
	// storage service
	s, err := mysql.New(config.DB.DataSourceName)
	panicIfErrored(err)
	s.SetMaxOpenConns(config.DB.MaxOpenConns)
	s.SetMaxIdleConns(config.DB.MaxIdleConns)
	store = s
}

func initCache() {
	memoryCache = memory.New(config.Cache.NumCachedIncomes)

	// init config in cache
	config, err := store.GetLatestConfig()
	panicIfErrored(err)
	memoryCache.SetLatestConfig(config)

	// init rates in cache
	lessRates, err := store.GetRewardRatesByType(models.RewardRateTypeLess)
	panicIfErrored(err)
	memoryCache.SetRewardRates(models.RewardRateTypeLess, lessRates)

	moreRates, err := store.GetRewardRatesByType(models.RewardRateTypeMore)
	panicIfErrored(err)
	memoryCache.SetRewardRates(models.RewardRateTypeMore, moreRates)

	// update bitcoin price on start
	updateBitcoinPrice()
}

func initHub() {
	connsHub = list.New()
}

func initCronjob() {
	c := cron.New()
	panicIfErrored(c.AddFunc("@every 1m", syncCache))
	panicIfErrored(c.AddFunc("@every 1m", updateBitcoinPrice))
	panicIfErrored(c.AddFunc("@daily", createWithdrawal))
	c.Start()
}

// update bitcoin price in cache
func updateBitcoinPrice() {
	defer func() {
		if err := recover(); err != nil {
			fmt.Fprintf(logWriter, "Update bitcoin price panic: %v\n", err)
		}
	}()

	// get bitcoin price from blockchain.info
	p, err := utils.BitcoinPrice()
	if err != nil {
		fmt.Fprintf(logWriter, "Fetch bitcoin price error: %v\n", err)
		return
	}

	// update bitcoin price in database
	if err := store.UpdateLatestBitcoinPrice(p); err != nil {
		fmt.Fprintf(logWriter, "Update bitcoin price in database error: %v\n", err)
		return
	}

	// update bitcoin price in cache
	memoryCache.UpdateBitcoinPrice(p)

	// broadcast bitcoin price to all users
	msg, _ := json.Marshal(models.WebsocketMessage{
		BitcoinPrice: utils.HumanReadableUSD(p),
	})
	connsHub.Broadcast(msg)

	fmt.Fprintf(logWriter, "Successfully update bitcoin price to %v\n", p)
}

// automatically create withdrawal
func createWithdrawal() {
	users, err := store.GetWithdrawableUsers()
	if err != nil {
		fmt.Fprintf(panicWriter, "Get withdrawable users error: %v\n", err)
		return
	}

	f := func(users []models.User, handler func(err error, u models.User)) {
		for i := range users {
			handler(store.CreateWithdrawal(models.Withdrawal{
				UserID:         users[i].ID,
				Amount:         users[i].Balance,
				BitcoinAddress: users[i].BitcoinAddress,
			}), users[i])
		}
	}

	// create withdrawal, move errored ones into retry queue
	retryUsers := []models.User{}
	f(users, func(err error, u models.User) {
		if err != nil {
			retryUsers = append(retryUsers, u)
		}
	})

	// retry with error output
	errored := false
	f(retryUsers, func(err error, u models.User) {
		if err != nil {
			fmt.Fprintf(panicWriter, "Create withdrawal for user %v error: %v\n", u, err)
			errored = true
		}
	})

	if !errored {
		fmt.Fprintf(logWriter, "Create withdrawals successfully...\n")
	}
}

// sync cache with storage
func syncCache() {
	// update config in cache
	config, err := store.GetLatestConfig()
	if err != nil {
		fmt.Fprintf(panicWriter, "Update latest config error: %v", err)
		return
	}
	memoryCache.SetLatestConfig(config)

	// update rates in cache
	lessRates, err := store.GetRewardRatesByType(models.RewardRateTypeLess)
	if err != nil {
		fmt.Fprintf(panicWriter, "Update less rate error: %v", err)
		return
	}

	moreRates, err := store.GetRewardRatesByType(models.RewardRateTypeMore)
	if err != nil {
		fmt.Fprintf(panicWriter, "Update more rate error: %v", err)
		return
	}

	memoryCache.SetRewardRates(models.RewardRateTypeMore, moreRates)
	memoryCache.SetRewardRates(models.RewardRateTypeLess, lessRates)

	fmt.Fprintf(logWriter, "Successfully sync cache\n")
}

// fail fast on initialization
func panicIfErrored(err error) {
	if err != nil {
		// Tricky:
		// pass a nil *errors.Error into this function
		// err is not nil
		// see discussion here:
		// https://github.com/go-playground/validator/issues/134
		// or
		// http://stackoverflow.com/questions/29138591/hiding-nil-values-understanding-why-golang-fails-here/29138676#29138676
		if e, ok := err.(*errors.Error); ok {
			if e != nil {
				panic(e.ErrStringForLogging)
			}
		} else {
			panic(err)
		}
	}
}
<commit_msg>InitHub should be run before InitCache<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"os"
	"time"

	"github.com/freeusd/solebtc/Godeps/_workspace/src/github.com/gin-gonic/gin"
	"github.com/freeusd/solebtc/Godeps/_workspace/src/github.com/robfig/cron"
	"github.com/freeusd/solebtc/controllers/v1"
	"github.com/freeusd/solebtc/errors"
	"github.com/freeusd/solebtc/middlewares"
	"github.com/freeusd/solebtc/models"
	"github.com/freeusd/solebtc/services/cache"
	"github.com/freeusd/solebtc/services/cache/memory"
	"github.com/freeusd/solebtc/services/hub"
	"github.com/freeusd/solebtc/services/hub/list"
	"github.com/freeusd/solebtc/services/mail"
	"github.com/freeusd/solebtc/services/mail/mandrill"
	"github.com/freeusd/solebtc/services/storage"
	"github.com/freeusd/solebtc/services/storage/mysql"
	"github.com/freeusd/solebtc/utils"
)

var (
	logWriter   io.Writer = os.Stdout
	panicWriter io.Writer = os.Stderr
	mailer      mail.Mailer
	store       storage.Storage
	memoryCache cache.Cache
	connsHub    hub.Hub
)

func init() {
	// ORDER MATTERs
	initConfig()
	initHub()
	initMailer()
	initStorage()
	initCache()
	initCronjob()
}

func main() {
	gin.SetMode(ginEnvMode())
	router := gin.New()

	// middlewares
	recovery := gin.RecoveryWithWriter(panicWriter)
	logger := middlewares.LoggerWithWriter(logWriter)
	cors := middlewares.CORS()
	errorWriter := middlewares.ErrorWriter()
	authRequired := middlewares.AuthRequired(store.GetAuthToken, config.AuthToken.Lifetime)

	// globally use middlewares
	router.Use(recovery, logger, cors, errorWriter)

	// version 1 api endpoints
	v1Endpoints := router.Group("/v1")

	// user endpoints
	v1UserEndpoints := v1Endpoints.Group("/users")
	v1UserEndpoints.GET("", authRequired, v1.UserInfo(store.GetUserByID))
	v1UserEndpoints.POST("", v1.Signup(store.CreateUser, store.GetUserByID))
	v1UserEndpoints.PUT("/:id/status", v1.VerifyEmail(store.GetSessionByToken, store.GetUserByID, store.UpdateUserStatus))
	v1UserEndpoints.GET("/referees", authRequired, v1.RefereeList(store.GetRefereesSince, store.GetRefereesUntil))

	// auth token endpoints
	v1AuthTokenEndpoints := v1Endpoints.Group("/auth_tokens")
	v1AuthTokenEndpoints.POST("", v1.Login(store.GetUserByEmail, store.CreateAuthToken))
	v1AuthTokenEndpoints.DELETE("", authRequired, v1.Logout(store.DeleteAuthToken))

	// session endpoints
	v1SessionEndpoints := v1Endpoints.Group("/sessions")
	emailVerificationTemplate := template.Must(template.ParseFiles(config.Template.EmailVerificationTemplate))
	v1SessionEndpoints.POST("", authRequired,
		v1.RequestVerifyEmail(store.GetUserByID, store.UpsertSession, mailer.SendEmail, emailVerificationTemplate),
	)

	// income endpoints
	v1IncomeEndpoints := v1Endpoints.Group("/incomes", authRequired)
	v1IncomeEndpoints.POST("/rewards",
		v1.GetReward(store.GetUserByID,
			memoryCache.GetLatestTotalReward,
			memoryCache.GetLatestConfig,
			memoryCache.GetRewardRatesByType,
			createRewardIncome,
			memoryCache.InsertIncome,
			connsHub.Broadcast),
	)
	v1IncomeEndpoints.GET("/rewards", v1.RewardList(store.GetRewardIncomesSince, store.GetRewardIncomesUntil))
	v1IncomeEndpoints.GET("/rewards/referees/:referee_id", v1.RefereeRewardList(store.GetUserByID, store.GetRewardIncomesSince, store.GetRewardIncomesUntil))

	// websocket endpoint
	v1Endpoints.GET("/websocket",
		v1.Websocket(
			connsHub.Len,
			memoryCache.GetLatestConfig,
			memoryCache.GetLatestIncomes,
			connsHub.Broadcast,
			hub.WrapPutWebsocketConn(connsHub.PutConn)),
	)

	fmt.Fprintf(logWriter, "SoleBTC is running on %s\n", config.HTTP.Port)
	panicIfErrored(router.Run(config.HTTP.Port))
}

func createRewardIncome(income models.Income, now time.Time) *errors.Error {
	if err := store.CreateRewardIncome(income, now); err != nil {
		return err
	}

	totalReward := income.Income
	if income.RefererID > 0 {
		totalReward += income.RefererIncome
	}
	memoryCache.IncrementTotalReward(now, totalReward)

	return nil
}

func initMailer() {
	// mailer
	mailer = mandrill.New(config.Mandrill.Key, config.Mandrill.FromEmail, config.Mandrill.FromName)
}

func initStorage() {
	// storage service
	s, err := mysql.New(config.DB.DataSourceName)
	panicIfErrored(err)
	s.SetMaxOpenConns(config.DB.MaxOpenConns)
	s.SetMaxIdleConns(config.DB.MaxIdleConns)
	store = s
}

func initCache() {
	memoryCache = memory.New(config.Cache.NumCachedIncomes)

	// init config in cache
	config, err := store.GetLatestConfig()
	panicIfErrored(err)
	memoryCache.SetLatestConfig(config)

	// init rates in cache
	lessRates, err := store.GetRewardRatesByType(models.RewardRateTypeLess)
	panicIfErrored(err)
	memoryCache.SetRewardRates(models.RewardRateTypeLess, lessRates)

	moreRates, err := store.GetRewardRatesByType(models.RewardRateTypeMore)
	panicIfErrored(err)
	memoryCache.SetRewardRates(models.RewardRateTypeMore, moreRates)

	// update bitcoin price on start
	updateBitcoinPrice()
}

func initHub() {
	connsHub = list.New()
}

func initCronjob() {
	c := cron.New()
	panicIfErrored(c.AddFunc("@every 1m", syncCache))
	panicIfErrored(c.AddFunc("@every 1m", updateBitcoinPrice))
	panicIfErrored(c.AddFunc("@daily", createWithdrawal))
	c.Start()
}

// update bitcoin price in cache
func updateBitcoinPrice() {
	defer func() {
		if err := recover(); err != nil {
			fmt.Fprintf(logWriter, "Update bitcoin price panic: %v\n", err)
		}
	}()

	// get bitcoin price from blockchain.info
	p, err := utils.BitcoinPrice()
	if err != nil {
		fmt.Fprintf(logWriter, "Fetch bitcoin price error: %v\n", err)
		return
	}

	// update bitcoin price in database
	if err := store.UpdateLatestBitcoinPrice(p); err != nil {
		fmt.Fprintf(logWriter, "Update bitcoin price in database error: %v\n", err)
		return
	}

	// update bitcoin price in cache
	memoryCache.UpdateBitcoinPrice(p)

	// broadcast bitcoin price to all users
	msg, _ := json.Marshal(models.WebsocketMessage{
		BitcoinPrice: utils.HumanReadableUSD(p),
	})
	connsHub.Broadcast(msg)

	fmt.Fprintf(logWriter, "Successfully update bitcoin price to %v\n", p)
}

// automatically create withdrawal
func createWithdrawal() {
	users, err := store.GetWithdrawableUsers()
	if err != nil {
		fmt.Fprintf(panicWriter, "Get withdrawable users error: %v\n", err)
		return
	}

	f := func(users []models.User, handler func(err error, u models.User)) {
		for i := range users {
			handler(store.CreateWithdrawal(models.Withdrawal{
				UserID:         users[i].ID,
				Amount:         users[i].Balance,
				BitcoinAddress: users[i].BitcoinAddress,
			}), users[i])
		}
	}

	// create withdrawal, move errored ones into retry queue
	retryUsers := []models.User{}
	f(users, func(err error, u models.User) {
		if err != nil {
			retryUsers = append(retryUsers, u)
		}
	})

	// retry with error output
	errored := false
	f(retryUsers, func(err error, u models.User) {
		if err != nil {
			fmt.Fprintf(panicWriter, "Create withdrawal for user %v error: %v\n", u, err)
			errored = true
		}
	})

	if !errored {
		fmt.Fprintf(logWriter, "Create withdrawals successfully...\n")
	}
}

// sync cache with storage
func syncCache() {
	// update config in cache
	config, err := store.GetLatestConfig()
	if err != nil {
		fmt.Fprintf(panicWriter, "Update latest config error: %v", err)
		return
	}
	memoryCache.SetLatestConfig(config)

	// update rates in cache
	lessRates, err := store.GetRewardRatesByType(models.RewardRateTypeLess)
	if err != nil {
		fmt.Fprintf(panicWriter, "Update less rate error: %v", err)
		return
	}

	moreRates, err := store.GetRewardRatesByType(models.RewardRateTypeMore)
	if err != nil {
		fmt.Fprintf(panicWriter, "Update more rate error: %v", err)
		return
	}

	memoryCache.SetRewardRates(models.RewardRateTypeMore, moreRates)
	memoryCache.SetRewardRates(models.RewardRateTypeLess, lessRates)

	fmt.Fprintf(logWriter, "Successfully sync cache\n")
}

// fail fast on initialization
func panicIfErrored(err error) {
	if err != nil {
		// Tricky:
		// pass a nil *errors.Error into this function
		// err is not nil
		// see discussion here:
		// https://github.com/go-playground/validator/issues/134
		// or
		// http://stackoverflow.com/questions/29138591/hiding-nil-values-understanding-why-golang-fails-here/29138676#29138676
		if e, ok := err.(*errors.Error); ok {
			if e != nil {
				panic(e.ErrStringForLogging)
			}
		} else {
			panic(err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"net"
	"os"
	"os/signal"
	"unsafe"

	bpf "github.com/kinvolk/gobpf-elf-loader/bpf"
)

type EventType uint32

const (
	_ EventType = iota
	EventConnect
	EventAccept
	EventClose
)

func (e EventType) String() string {
	switch e {
	case EventConnect:
		return "connect"
	case EventAccept:
		return "accept"
	case EventClose:
		return "close"
	default:
		return "unknown"
	}
}

type tcpEventV4 struct {
	// Timestamp must be the first field, the sorting depends on it
	Timestamp uint64

	Cpu   uint64
	Type  uint32
	Pid   uint32
	Comm  [16]byte
	SAddr uint32
	DAddr uint32
	SPort uint16
	DPort uint16
	NetNS uint32
}

type tcpEventV6 struct {
	// Timestamp must be the first field, the sorting depends on it
	Timestamp uint64

	Cpu    uint64
	Type   uint32
	Pid    uint32
	Comm   [16]byte
	SAddrH uint64
	SAddrL uint64
	DAddrH uint64
	DAddrL uint64
	SPort  uint16
	DPort  uint16
	NetNS  uint32
}

var byteOrder binary.ByteOrder

// In lack of binary.HostEndian ...
func init() {
	var i int32 = 0x01020304
	u := unsafe.Pointer(&i)
	pb := (*byte)(u)
	b := *pb
	if b == 0x04 {
		byteOrder = binary.LittleEndian
	} else {
		byteOrder = binary.BigEndian
	}
}

var lastTimestampV4 uint64
var lastTimestampV6 uint64

func tcpEventCbV4(event tcpEventV4) {
	timestamp := uint64(event.Timestamp)
	cpu := event.Cpu
	typ := EventType(event.Type)
	pid := event.Pid & 0xffffffff
	comm := string(event.Comm[:bytes.IndexByte(event.Comm[:], 0)])

	saddrbuf := make([]byte, 4)
	daddrbuf := make([]byte, 4)

	binary.LittleEndian.PutUint32(saddrbuf, uint32(event.SAddr))
	binary.LittleEndian.PutUint32(daddrbuf, uint32(event.DAddr))

	sIP := net.IPv4(saddrbuf[0], saddrbuf[1], saddrbuf[2], saddrbuf[3])
	dIP := net.IPv4(daddrbuf[0], daddrbuf[1], daddrbuf[2], daddrbuf[3])

	sport := event.SPort
	dport := event.DPort
	netns := event.NetNS

	fmt.Printf("%v cpu#%d %s %v %q %v:%v %v:%v %v\n", timestamp, cpu, typ, pid, comm, sIP, sport, dIP, dport, netns)

	if lastTimestampV4 > timestamp {
		fmt.Printf("ERROR: late event!\n")
		os.Exit(1)
	}

	lastTimestampV4 = timestamp
}

func tcpEventCbV6(event tcpEventV6) {
	timestamp := uint64(event.Timestamp)
	cpu := event.Cpu
	typ := EventType(event.Type)
	pid := event.Pid & 0xffffffff

	saddrbuf := make([]byte, 16)
	daddrbuf := make([]byte, 16)

	binary.LittleEndian.PutUint64(saddrbuf, event.SAddrH)
	binary.LittleEndian.PutUint64(saddrbuf[4:], event.SAddrL)
	binary.LittleEndian.PutUint64(daddrbuf, event.DAddrH)
	binary.LittleEndian.PutUint64(daddrbuf[4:], event.DAddrL)

	sIP := net.IP(saddrbuf)
	dIP := net.IP(daddrbuf)

	sport := event.SPort
	dport := event.DPort
	netns := event.NetNS

	fmt.Printf("%v cpu#%d %s %v %v:%v %v:%v %v\n", timestamp, cpu, typ, pid, sIP, sport, dIP, dport, netns)

	if lastTimestampV6 > timestamp {
		fmt.Printf("ERROR: late event!\n")
		os.Exit(1)
	}

	lastTimestampV6 = timestamp
}

func main() {
	if len(os.Args) != 2 {
		fmt.Fprintf(os.Stderr, "Usage: %s ${GOPATH}/src/github.com/kinvolk/tcptracer-bpf/ebpf/${DISTRO}/x86_64/$(uname -r)/ebpf.o\n", os.Args[0])
		os.Exit(1)
	}
	fileName := os.Args[1]
	b := bpf.NewBpfPerfEvent(fileName)
	if b == nil {
		fmt.Fprintf(os.Stderr, "System doesn't support BPF\n")
		os.Exit(1)
	}

	err := b.Load()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}

	fmt.Printf("Ready.\n")

	channelV4 := make(chan []byte)
	channelV6 := make(chan []byte)

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt, os.Kill)

	go func() {
		var event tcpEventV4
		for {
			data := <-channelV4
			err := binary.Read(bytes.NewBuffer(data), byteOrder, &event)
			if err != nil {
				fmt.Printf("failed to decode received data: %s\n", err)
				continue
			}
			tcpEventCbV4(event)
		}
	}()

	go func() {
		var event tcpEventV6
		for {
			data := <-channelV6
			err := binary.Read(bytes.NewBuffer(data), byteOrder, &event)
			if err != nil {
				fmt.Printf("failed to decode received data: %s\n", err)
				continue
			}
			tcpEventCbV6(event)
		}
	}()

	b.PollStart("tcp_event_v4", channelV4)
	b.PollStart("tcp_event_v6", channelV6)
	<-sig
	b.PollStop("tcp_event_v4")
	b.PollStop("tcp_event_v6")
}
<commit_msg>fix map names and event structures<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"net"
	"os"
	"os/signal"
	"unsafe"

	bpf "github.com/kinvolk/gobpf-elf-loader/bpf"
)

type EventType uint32

const (
	_ EventType = iota
	EventConnect
	EventAccept
	EventClose
)

func (e EventType) String() string {
	switch e {
	case EventConnect:
		return "connect"
	case EventAccept:
		return "accept"
	case EventClose:
		return "close"
	default:
		return "unknown"
	}
}

type tcpEventV4 struct {
	// Timestamp must be the first field, the sorting depends on it
	Timestamp uint64

	Cpu   uint64
	Type  uint32
	Pid   uint32
	Comm  [16]byte
	SAddr uint32
	DAddr uint32
	SPort uint16
	DPort uint16
	NetNS uint32
}

type tcpEventV6 struct {
	// Timestamp must be the first field, the sorting depends on it
	Timestamp uint64

	Cpu    uint64
	Type   uint32
	Pid    uint32
	Comm   [16]byte
	SAddrH uint64
	SAddrL uint64
	DAddrH uint64
	DAddrL uint64
	SPort  uint16
	DPort  uint16
	NetNS  uint32
}

var byteOrder binary.ByteOrder

// In lack of binary.HostEndian ...
func init() {
	var i int32 = 0x01020304
	u := unsafe.Pointer(&i)
	pb := (*byte)(u)
	b := *pb
	if b == 0x04 {
		byteOrder = binary.LittleEndian
	} else {
		byteOrder = binary.BigEndian
	}
}

var lastTimestampV4 uint64
var lastTimestampV6 uint64

func tcpEventCbV4(event tcpEventV4) {
	timestamp := uint64(event.Timestamp)
	cpu := event.Cpu
	typ := EventType(event.Type)
	pid := event.Pid & 0xffffffff
	comm := string(event.Comm[:bytes.IndexByte(event.Comm[:], 0)])

	saddrbuf := make([]byte, 4)
	daddrbuf := make([]byte, 4)

	binary.LittleEndian.PutUint32(saddrbuf, uint32(event.SAddr))
	binary.LittleEndian.PutUint32(daddrbuf, uint32(event.DAddr))

	sIP := net.IPv4(saddrbuf[0], saddrbuf[1], saddrbuf[2], saddrbuf[3])
	dIP := net.IPv4(daddrbuf[0], daddrbuf[1], daddrbuf[2], daddrbuf[3])

	sport := event.SPort
	dport := event.DPort
	netns := event.NetNS

	fmt.Printf("%v cpu#%d %s %v %q %v:%v %v:%v %v\n", timestamp, cpu, typ, pid, comm, sIP, sport, dIP, dport, netns)

	if lastTimestampV4 > timestamp {
		fmt.Printf("ERROR: late event!\n")
		os.Exit(1)
	}

	lastTimestampV4 = timestamp
}

func tcpEventCbV6(event tcpEventV6) {
	timestamp := uint64(event.Timestamp)
	cpu := event.Cpu
	typ := EventType(event.Type)
	pid := event.Pid & 0xffffffff

	saddrbuf := make([]byte, 16)
	daddrbuf := make([]byte, 16)

	binary.LittleEndian.PutUint64(saddrbuf, event.SAddrH)
	binary.LittleEndian.PutUint64(saddrbuf[4:], event.SAddrL)
	binary.LittleEndian.PutUint64(daddrbuf, event.DAddrH)
	binary.LittleEndian.PutUint64(daddrbuf[4:], event.DAddrL)

	sIP := net.IP(saddrbuf)
	dIP := net.IP(daddrbuf)

	sport := event.SPort
	dport := event.DPort
	netns := event.NetNS

	fmt.Printf("%v cpu#%d %s %v %v:%v %v:%v %v\n", timestamp, cpu, typ, pid, sIP, sport, dIP, dport, netns)

	if lastTimestampV6 > timestamp {
		fmt.Printf("ERROR: late event!\n")
		os.Exit(1)
	}

	lastTimestampV6 = timestamp
}

func main() {
	if len(os.Args) != 2 {
		fmt.Fprintf(os.Stderr, "Usage: %s ${GOPATH}/src/github.com/kinvolk/tcptracer-bpf/ebpf/${DISTRO}/x86_64/$(uname -r)/ebpf.o\n", os.Args[0])
		os.Exit(1)
	}
	fileName := os.Args[1]
	b := bpf.NewBpfPerfEvent(fileName)
	if b == nil {
		fmt.Fprintf(os.Stderr, "System doesn't support BPF\n")
		os.Exit(1)
	}

	err := b.Load()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}

	fmt.Printf("Ready.\n")

	channelV4 := make(chan []byte)
	channelV6 := make(chan []byte)

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt, os.Kill)

	go func() {
		var event tcpEventV4
		for {
			data := <-channelV4
			err := binary.Read(bytes.NewBuffer(data), byteOrder, &event)
			if err != nil {
				fmt.Printf("failed to decode received data: %s\n", err)
				continue
			}
			tcpEventCbV4(event)
		}
	}()

	go func() {
		var event tcpEventV6
		for {
			data := <-channelV6
			err := binary.Read(bytes.NewBuffer(data), byteOrder, &event)
			if err != nil {
				fmt.Printf("failed to decode received data: %s\n", err)
				continue
			}
			tcpEventCbV6(event)
		}
	}()

	b.PollStart("tcp_event_ipv4", channelV4)
	b.PollStart("tcp_event_ipv6", channelV6)
	<-sig
	b.PollStop("tcp_event_ipv4")
	b.PollStop("tcp_event_ipv6")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"golang.org/x/net/context"

	"github.com/Shopify/sarama"
	cluster "github.com/bsm/sarama-cluster"
	"github.com/codegangsta/cli"
	"github.com/mathpl/go-tsdmetrics"
	"github.com/rcrowley/go-metrics"
)

var version = "0.4"

func generateConsumerLag(r tsdmetrics.TaggedRegistry) {
	valsSent := make(map[string]int64, 0)
	valsCommitted := make(map[string]int64, 0)
	valsHWM := make(map[string]int64, 0)

	fn := func(n string, tm tsdmetrics.TaggedMetric) {
		switch n {
		case "kafka_httpcat.consumer.sent":
			if m, ok := tm.GetMetric().(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsSent[tm.GetTags()["partition"]] = m.Value()
			}
		case "kafka_httpcat.consumer.committed":
			if m, ok := tm.GetMetric().(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsCommitted[tm.GetTags()["partition"]] = m.Value()
			}
		case "kafka_httpcat.consumer.high_water_mark":
			if m, ok := tm.GetMetric().(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsHWM[tm.GetTags()["partition"]] = m.Value()
			}
		}
	}

	r.Each(fn)

	for partition, hwmOffset := range valsHWM {
		if sentOffset, ok := valsSent[partition]; ok {
			i := r.GetOrRegister("kafka_httpcat.consumer.sent.offset_lag", tsdmetrics.Tags{"partition": partition}, metrics.NewGauge())
			if m, ok := i.(metrics.Gauge); ok {
				offsetLag := hwmOffset - sentOffset
				m.Update(offsetLag)
			} else {
				log.Print("Unexpected metric type")
			}
		}

		if committedOffset, ok := valsCommitted[partition]; ok {
			i := r.GetOrRegister("kafka_httpcat.consumer.committed.offset_lag", tsdmetrics.Tags{"partition": partition}, metrics.NewGauge())
			if m, ok := i.(metrics.Gauge); ok {
				offsetLag := hwmOffset - committedOffset
				m.Update(offsetLag)
			} else {
				log.Print("Unexpected metric type")
			}
		}
	}
}

func commaDelimitedToStringList(s string) []string {
	list := strings.Split(s, ",")
	cleanList := make([]string, 0)
	for _, v := range list {
		c := strings.TrimSpace(v)
		if c != "" {
			cleanList = append(cleanList, c)
		}
	}
	return cleanList
}

func stringListToHeaderMap(l []string) (map[string][]string, error) {
	headers := make(map[string][]string, len(l))
	for _, h := range l {
		idx := strings.Index(h, ":")
		if idx == -1 {
			return nil, fmt.Errorf("Unable to parse header %s", h)
		}
		headers[h[0:idx]] = append(headers[h[0:idx]], strings.TrimSpace(h[idx+1:]))
	}

	return headers, nil
}

func commaDelimitedToIntList(s string) ([]int, error) {
	list := strings.Split(s, ",")
	intList := make([]int, len(list))
	for i, v := range list {
		var err error
		intList[i], err = strconv.Atoi(strings.TrimSpace(v))
		if err != nil {
			return nil, err
		}
	}
	return intList, nil
}

func main() {
	app := cli.NewApp()
	app.Name = "kafka_httpcat"
	app.Usage = "Forward kafka data to http endpoint"
	app.Version = version

	host, err := os.Hostname()
	if err != nil {
		log.Printf("Unable to get hostname: %s", err)
		os.Exit(1)
	}

	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:   "verbosity",
			Value:  2,
			Usage:  "verbosity (0-5)",
			EnvVar: "VERBOSITY",
		},
		cli.StringFlag{
			Name:   "target-host-list, t",
			Usage:  "Comma delimited target hosts",
			EnvVar: "TARGET_HOST_LIST",
		},
		cli.StringFlag{
			Name:   "target-path, p",
			Usage:  "HTTP path",
			EnvVar: "TARGET_PATH",
		},
		cli.StringFlag{
			Name:   "headers, H",
			Usage:  "Comma delimited headers",
			EnvVar: "HTTP_HEADERS",
		},
		cli.StringFlag{
			Name:   "method, m",
			Value:  "POST",
			Usage:  "HTTP method",
			EnvVar: "HTTP_METHOD",
		},
		cli.StringFlag{
			Name:   "expected-statuses, e",
			Value:  "200",
			Usage:  "Comma delimited list of expected HTTP status",
			EnvVar: "HTTP_EXPECTED_STATUSES",
		},
		cli.StringFlag{
			Name:   "kafka-broker-list, b",
			Usage:  "Comma delimited kafka broker list.",
			EnvVar: "KAFKA_BROKER_LIST",
		},
		cli.StringFlag{
			Name:   "kafka-topic, T",
			Usage:  "Kafka topic.",
			EnvVar: "KAFKA_TOPIC",
		},
		cli.StringFlag{
			Name:   "kafka-consumer-id, i",
			Value:  fmt.Sprintf("%s-%d", host, os.Getpid()),
			Usage:  "Kafka consumer id.",
			EnvVar: "KAFKA_CONSUMER_ID",
		},
		cli.StringFlag{
			Name:   "kafka-consumer-group, g",
			Usage:  "Kafka consumer group.",
			EnvVar: "KAFKA_CONSUMER_GROUP",
		},
		cli.StringFlag{
			Name:   "kafka-start-offset, o",
			Value:  "newest",
			Usage:  "Kafka offset to start with (newest or oldest)",
			EnvVar: "KAFKA_START_OFFSET",
		},
		cli.IntFlag{
			Name:   "kafka-commit-batch, c",
			Value:  1000,
			Usage:  "Commit consumed messages every X messages.",
			EnvVar: "KAFKA_COMMIT_BATCH",
		},
		cli.StringFlag{
			Name:   "metrics-report-url, r",
			Usage:  "Where to send OpenTSDB metrics.",
			EnvVar: "METRICS_REPORT_URL",
		},
		cli.StringFlag{
			Name:   "metrics-tags",
			Usage:  "Comma delimited list of default tags",
			EnvVar: "METRICS_TAGS",
		},
	}

	app.Action = func(c *cli.Context) error {
		expectedStatuses, err := commaDelimitedToIntList(c.String("expected-statuses"))
		if err != nil {
			return fmt.Errorf("Expected http status must be an integer: %s", err)
		}

		httpHeaderList := commaDelimitedToStringList(c.String("headers"))
		httpHeaders, err := stringListToHeaderMap(httpHeaderList)
		if err != nil {
			return fmt.Errorf("Unable to parse headers: %s", err)
		}

		targetHosts := commaDelimitedToStringList(c.String("target-host-list"))
		defaultTags, err := tsdmetrics.TagsFromString(c.String("metrics-tags"))
		if err != nil {
			return err
		}
		defaultTags = defaultTags.AddTags(tsdmetrics.Tags{"topic": c.String("kafka-topic"), "consumergroup": c.String("kafka-consumer-group")})

		rootRegistry := tsdmetrics.NewSegmentedTaggedRegistry("", defaultTags, nil)
		tsdmetrics.RegisterTaggedRuntimeMemStats(rootRegistry)
		metricsRegistry := tsdmetrics.NewSegmentedTaggedRegistry("kafka_httpcat", nil, rootRegistry)
		metricsTsdb := tsdmetrics.TaggedOpenTSDB{Addr: c.String("metrics-report-url"), Registry: rootRegistry, FlushInterval: 15 * time.Second, DurationUnit: time.Millisecond, Format: tsdmetrics.Json}

		log.Printf("Connecting to: %s", c.String("kafka-broker-list"))

		// Init config
		config := cluster.NewConfig()
		config.Consumer.Return.Errors = true
		config.Group.Return.Notifications = true

		switch c.String("kafka-start-offset") {
		case "oldest":
			config.Consumer.Offsets.Initial = sarama.OffsetOldest
		case "newest":
			config.Consumer.Offsets.Initial = sarama.OffsetNewest
		default:
			return fmt.Errorf("offset should be `oldest` or `newest`")
		}

		brokerList := strings.Split(c.String("kafka-broker-list"), ",")
		topicList := strings.Split(c.String("kafka-topic"), ",")

		// Init consumer, consume errors & messages
		consumer, err := cluster.NewConsumer(brokerList, c.String("kafka-consumer-group"), topicList, config)
		if err != nil {
			log.Fatal(err)
		}

		go func() {
			for err := range consumer.Errors() {
				fmt.Printf("Error: %s\n", err.Error())
			}
		}()

		go func() {
			for note := range consumer.Notifications() {
				fmt.Printf("Rebalanced: %+v\n", note)
			}
		}()

		batchSize := int64(c.Int("kafka-commit-batch"))

		go func() {
			for {
				hwms := consumer.HighWaterMarks()
				for _, hwm := range hwms {
					for partition, offset := range hwm {
						tags := tsdmetrics.Tags{"partition": fmt.Sprintf("%d", partition)}
						m := metricsRegistry.GetOrRegister("consumer.high_water_mark", tags, metrics.NewGauge())
						m.(metrics.Gauge).Update(offset)
					}
				}
				time.Sleep(5 * time.Second)
			}
		}()

		go func() {
			sender := NewHTTPSender(targetHosts, c.String("target-path"), c.String("method"), httpHeaders, expectedStatuses)
			for msg := range consumer.Messages() {
				if err := sender.RRSend(msg.Value); err != nil {
					log.Printf("Error send data: %s\n", err)
				}

				tags := tsdmetrics.Tags{"partition": fmt.Sprintf("%d", msg.Partition)}
				s := metricsRegistry.GetOrRegister("consumer.sent", tags, metrics.NewGauge())
				s.(metrics.Gauge).Update(msg.Offset)

				if msg.Offset%batchSize == 0 {
					consumer.MarkOffset(msg, "")
					c := metricsRegistry.GetOrRegister("consumer.committed", tags, metrics.NewGauge())
					c.(metrics.Gauge).Update(msg.Offset)
					log.Printf("Commited offset partition:%d offset:%d\n", msg.Partition, msg.Offset)
				}
			}
		}()

		collectFn := tsdmetrics.RuntimeCaptureFn
		collectFn = append(collectFn, generateConsumerLag)

		go metricsTsdb.RunWithPreprocessing(context.Background(), collectFn)

		wait := make(chan os.Signal, 1)
		signal.Notify(wait, syscall.SIGINT, syscall.SIGTERM)

		<-wait
		consumer.CommitOffsets()

		if err := consumer.Close(); err != nil {
			fmt.Printf("Failed to close consumer: %s\n", err)
		}

		return nil
	}

	err = app.Run(os.Args)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>fix hwm calculation<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"golang.org/x/net/context"

	"github.com/Shopify/sarama"
	cluster "github.com/bsm/sarama-cluster"
	"github.com/codegangsta/cli"
	"github.com/mathpl/go-tsdmetrics"
	"github.com/rcrowley/go-metrics"
)

var version = "0.4"

func generateConsumerLag(r tsdmetrics.TaggedRegistry) {
	valsSent := make(map[string]int64, 0)
	valsCommitted := make(map[string]int64, 0)
	valsHWM := make(map[string]int64, 0)

	fn := func(n string, tm tsdmetrics.TaggedMetric) {
		switch n {
		case "kafka_httpcat.consumer.sent":
			if m, ok := tm.GetMetric().(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsSent[tm.GetTags()["partition"]] = m.Value()
			}
		case "kafka_httpcat.consumer.committed":
			if m, ok := tm.GetMetric().(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsCommitted[tm.GetTags()["partition"]] = m.Value()
			}
		case "kafka_httpcat.consumer.high_water_mark":
			if m, ok := tm.GetMetric().(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsHWM[tm.GetTags()["partition"]] = m.Value()
			}
		}
	}

	r.Each(fn)

	for partition, hwmOffset := range valsHWM {
		if sentOffset, ok := valsSent[partition]; ok {
			i := r.GetOrRegister("kafka_httpcat.consumer.sent.offset_lag", tsdmetrics.Tags{"partition": partition}, metrics.NewGauge())
			if m, ok := i.(metrics.Gauge); ok {
				offsetLag := hwmOffset - sentOffset
				m.Update(offsetLag)
			} else {
				log.Print("Unexpected metric type")
			}
		}

		if committedOffset, ok := valsCommitted[partition]; ok {
			i := r.GetOrRegister("kafka_httpcat.consumer.committed.offset_lag", tsdmetrics.Tags{"partition": partition}, metrics.NewGauge())
			if m, ok := i.(metrics.Gauge); ok {
				offsetLag := hwmOffset - committedOffset
				m.Update(offsetLag)
			} else {
				log.Print("Unexpected metric type")
			}
		}
	}
}

func commaDelimitedToStringList(s string) []string {
	list := strings.Split(s, ",")
	cleanList := make([]string, 0)
	for _, v := range list {
		c := strings.TrimSpace(v)
		if c != "" {
			cleanList = append(cleanList, c)
		}
	}
	return cleanList
}

func stringListToHeaderMap(l []string) (map[string][]string, error) {
	headers := make(map[string][]string, len(l))
	for _, h := range l {
		idx := strings.Index(h, ":")
		if idx == -1 {
			return nil, fmt.Errorf("Unable to parse header %s", h)
		}
		headers[h[0:idx]] = append(headers[h[0:idx]], strings.TrimSpace(h[idx+1:]))
	}

	return headers, nil
}

func commaDelimitedToIntList(s string) ([]int, error) {
	list := strings.Split(s, ",")
	intList := make([]int, len(list))
	for i, v := range list {
		var err error
		intList[i], err = strconv.Atoi(strings.TrimSpace(v))
		if err != nil {
			return nil, err
		}
	}
	return intList, nil
}

func main() {
	app := cli.NewApp()
	app.Name = "kafka_httpcat"
	app.Usage = "Forward kafka data to http endpoint"
	app.Version = version

	host, err := os.Hostname()
	if err != nil {
		log.Printf("Unable to get hostname: %s", err)
		os.Exit(1)
	}

	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:   "verbosity",
			Value:  2,
			Usage:  "verbosity (0-5)",
			EnvVar: "VERBOSITY",
		},
		cli.StringFlag{
			Name:   "target-host-list, t",
			Usage:  "Comma delimited target hosts",
			EnvVar: "TARGET_HOST_LIST",
		},
		cli.StringFlag{
			Name:   "target-path, p",
			Usage:  "HTTP path",
			EnvVar: "TARGET_PATH",
		},
		cli.StringFlag{
			Name:   "headers, H",
			Usage:  "Comma delimited headers",
			EnvVar: "HTTP_HEADERS",
		},
		cli.StringFlag{
			Name:   "method, m",
			Value:  "POST",
			Usage:  "HTTP method",
			EnvVar: "HTTP_METHOD",
		},
		cli.StringFlag{
			Name:   "expected-statuses, e",
			Value:  "200",
			Usage:  "Comma delimited list of expected HTTP status",
			EnvVar: "HTTP_EXPECTED_STATUSES",
		},
		cli.StringFlag{
			Name:   "kafka-broker-list, b",
			Usage:  "Comma delimited kafka broker list.",
			EnvVar: "KAFKA_BROKER_LIST",
		},
		cli.StringFlag{
			Name:   "kafka-topic, T",
			Usage:  "Kafka topic.",
			EnvVar: "KAFKA_TOPIC",
		},
		cli.StringFlag{
			Name:   "kafka-consumer-id, i",
			Value:  fmt.Sprintf("%s-%d", host, os.Getpid()),
			Usage:  "Kafka consumer id.",
			EnvVar: "KAFKA_CONSUMER_ID",
		},
		cli.StringFlag{
			Name:   "kafka-consumer-group, g",
			Usage:  "Kafka consumer group.",
			EnvVar: "KAFKA_CONSUMER_GROUP",
		},
		cli.StringFlag{
			Name:   "kafka-start-offset, o",
			Value:  "newest",
			Usage:  "Kafka offset to start with (newest or oldest)",
			EnvVar: "KAFKA_START_OFFSET",
		},
		cli.IntFlag{
			Name:   "kafka-commit-batch, c",
			Value:  1000,
			Usage:  "Commit consumed messages every X messages.",
			EnvVar: "KAFKA_COMMIT_BATCH",
		},
		cli.StringFlag{
			Name:   "metrics-report-url, r",
			Usage:  "Where to send OpenTSDB metrics.",
			EnvVar: "METRICS_REPORT_URL",
		},
		cli.StringFlag{
			Name:   "metrics-tags",
			Usage:  "Comma delimited list of default tags",
			EnvVar: "METRICS_TAGS",
		},
	}

	app.Action = func(c *cli.Context) error {
		expectedStatuses, err := commaDelimitedToIntList(c.String("expected-statuses"))
		if err != nil {
			return fmt.Errorf("Expected http status must be an integer: %s", err)
		}

		httpHeaderList := commaDelimitedToStringList(c.String("headers"))
		httpHeaders, err := stringListToHeaderMap(httpHeaderList)
		if err != nil {
			return fmt.Errorf("Unable to parse headers: %s", err)
		}

		targetHosts := commaDelimitedToStringList(c.String("target-host-list"))
		defaultTags, err := tsdmetrics.TagsFromString(c.String("metrics-tags"))
		if err != nil {
			return err
		}
		defaultTags = defaultTags.AddTags(tsdmetrics.Tags{"topic": c.String("kafka-topic"), "consumergroup": c.String("kafka-consumer-group")})

		rootRegistry := tsdmetrics.NewSegmentedTaggedRegistry("", defaultTags, nil)
		tsdmetrics.RegisterTaggedRuntimeMemStats(rootRegistry)
		metricsRegistry := tsdmetrics.NewSegmentedTaggedRegistry("kafka_httpcat", nil, rootRegistry)
		metricsTsdb := tsdmetrics.TaggedOpenTSDB{Addr: c.String("metrics-report-url"), Registry: rootRegistry, FlushInterval: 15 * time.Second, DurationUnit: time.Millisecond, Format: tsdmetrics.Json}

		log.Printf("Connecting to: %s", c.String("kafka-broker-list"))

		// Init config
		config := cluster.NewConfig()
		config.Consumer.Return.Errors = true
		config.Group.Return.Notifications = true

		switch c.String("kafka-start-offset") {
		case "oldest":
			config.Consumer.Offsets.Initial = sarama.OffsetOldest
		case "newest":
			config.Consumer.Offsets.Initial = sarama.OffsetNewest
		default:
			return fmt.Errorf("offset should be `oldest` or `newest`")
		}

		brokerList := strings.Split(c.String("kafka-broker-list"), ",")
		topicList := strings.Split(c.String("kafka-topic"), ",")

		// Init consumer, consume errors & messages
		consumer, err := cluster.NewConsumer(brokerList, c.String("kafka-consumer-group"), topicList, config)
		if err != nil {
			log.Fatal(err)
		}

		wait := make(chan os.Signal, 1)
		signal.Notify(wait, syscall.SIGINT, syscall.SIGTERM)

		go func() {
			for err := range consumer.Errors() {
				fmt.Printf("Error: %s\n", err.Error())

				// Abort all
				wait <- syscall.SIGTERM
			}
		}()

		go func() {
			for note := range consumer.Notifications() {
				fmt.Printf("Rebalanced: %+v\n", note)
			}
		}()

		batchSize := int64(c.Int("kafka-commit-batch"))

		hwmUpdate := func(r tsdmetrics.TaggedRegistry) {
			hwms := consumer.HighWaterMarks()
			for _, hwm := range hwms {
				for partition, offset := range hwm {
					tags := tsdmetrics.Tags{"partition": fmt.Sprintf("%d", partition)}
					m := metricsRegistry.GetOrRegister("consumer.high_water_mark", tags, metrics.NewGauge())
					m.(metrics.Gauge).Update(offset)
				}
			}
		}

		go func() {
			sender := NewHTTPSender(targetHosts, c.String("target-path"), c.String("method"), httpHeaders, expectedStatuses)
			for msg := range consumer.Messages() {
				if err := sender.RRSend(msg.Value); err != nil {
					log.Printf("Error send data: %s\n", err)
				}

				tags := tsdmetrics.Tags{"partition": fmt.Sprintf("%d", msg.Partition)}
				s := metricsRegistry.GetOrRegister("consumer.sent", tags, metrics.NewGauge())
				s.(metrics.Gauge).Update(msg.Offset)

				if msg.Offset%batchSize == 0 {
					consumer.MarkOffset(msg, "")
					c := metricsRegistry.GetOrRegister("consumer.committed", tags, metrics.NewGauge())
					c.(metrics.Gauge).Update(msg.Offset)
					log.Printf("Commited offset partition:%d offset:%d\n", msg.Partition, msg.Offset)
				}
			}
		}()

		collectFn := tsdmetrics.RuntimeCaptureFn
		collectFn = append(collectFn, hwmUpdate)
		collectFn = append(collectFn, generateConsumerLag)

		go metricsTsdb.RunWithPreprocessing(context.Background(), collectFn)

		<-wait
		consumer.CommitOffsets()

		if err := consumer.Close(); err != nil {
			fmt.Printf("Failed to close consumer: %s\n", err)
		}

		return nil
	}

	err = app.Run(os.Args)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"crypto/tls"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

var (
	tmpl *template.Template
)

func main() {
	port := os.Getenv("PORT")
	if port == "" {
		log.Fatal("$PORT must be set")
	}

	// disable HTTP/2 client
	tr := http.DefaultTransport.(*http.Transport)
	tr.TLSClientConfig = &tls.Config{
		ClientSessionCache: tls.NewLRUClientSessionCache(0),
	}
	tr.MaxIdleConns = 0
	tr.MaxIdleConnsPerHost = int(^uint(0) >> 1) // unlimited
	tr.IdleConnTimeout = 300 * time.Second
	tr.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}

	tmpl = template.Must(template.ParseFiles("app.gohtml"))

	http.Handle("/", &httputil.ReverseProxy{Director: director, ModifyResponse: modifyResponse})
	http.Handle("/md/", http.NotFoundHandler())
	http.Handle("/assets/", addHeaders(http.HandlerFunc(assetsHandler)))
	http.Handle("/assets/md/", http.StripPrefix("/assets/md", addHeaders(http.FileServer(http.Dir("static")))))

	srv := &http.Server{Addr: ":" + port}

	shutdown := make(chan os.Signal, 1)
	signal.Notify(shutdown, os.Interrupt, syscall.SIGTERM)
	idleConnsClosed := make(chan struct{})
	go func() {
		<-shutdown

		// We received an interrupt/termination signal, shut down.
		if err := srv.Shutdown(context.Background()); err != nil {
			// Error from closing listeners, or context timeout:
			log.Printf("HTTP server Shutdown: %v", err)
		}
		close(idleConnsClosed)
	}()

	if err := srv.ListenAndServe(); err != http.ErrServerClosed {
		// Error starting or closing listener:
		log.Fatalf("HTTP server ListenAndServe: %v", err)
	}

	<-idleConnsClosed
}

func getOrigin(req *http.Request) string {
	if strings.HasPrefix(req.Host, "canary") {
		return "canary.discordapp.com"
	} else if strings.HasPrefix(req.Host, "ptb") {
		return "ptb.discordapp.com"
	} else {
		return "discordapp.com"
	}
}

func director(req *http.Request) {
	req.URL.Scheme = "https"
	req.URL.Host = getOrigin(req)
	req.Host = req.URL.Host

	if !strings.HasPrefix(req.URL.Path, "/assets/") {
		// read uncompressed response
		delete(req.Header, "Accept-Encoding")
	}

	// remove Cloudflare headers (Cloudflare rejects requests with Cf-Connecting-Ip)
	for k := range req.Header {
		if strings.HasPrefix(k, "Cf-") {
			delete(req.Header, k)
		}
	}

	if _, ok := req.Header["User-Agent"]; !ok {
		// explicitly disable User-Agent so it's not set to default value
		req.Header.Set("User-Agent", "")
	}
}

func modifyResponse(res *http.Response) error {
	// remove __cfduid cookie to let Cloudflare cache
	delete(res.Header, "Set-Cookie")

	if res.StatusCode >= 500 {
		return nil
	}

	// hide from search engines
	res.Header.Set("X-Robots-Tag", "noindex, nofollow, noarchive, nocache, noimageindex, noodp")

	if strings.HasPrefix(res.Request.URL.Path, "/assets/") {
		return nil
	}

	// prevent caching HTML (assets might not load while offline)
	if cc := res.Header.Get("Cache-Control"); !strings.Contains(cc, "no-cache") &&
		!strings.Contains(cc, "no-store") &&
		!strings.Contains(cc, "max-age") {
		res.Header.Add("Cache-Control", "max-age=0")
	}

	if !strings.HasPrefix(res.Header.Get("Content-Type"), "text/html") {
		return nil
	}

	// inject links and scripts
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}
	res.Body.Close()
	s := string(b)
	s = strings.ReplaceAll(s, " integrity=", " crossorigin integrity=")
	if iMeta := strings.Index(s, "<meta "); iMeta == -1 {
		log.Print("modifyResponse: missing <meta> tag")
	} else if iHead := strings.Index(s[iMeta:], ""); iHead == -1 {
		log.Print("modifyResponse: missing  tag")
	} else if iScript := strings.Index(s[iMeta+iHead:], "